2023-10-09T04:19:06.691 INFO:root:teuthology version: 0.0.1.dev194+g54e62bc 2023-10-09T04:19:06.691 DEBUG:teuthology.run:Teuthology command: teuthology --name teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi --owner scheduled_teuthology@teuthology --verbose --archive /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417477 --description upgrade:quincy-x/stress-split/{0-distro/ubuntu_20.04 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} -- /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417477/orig.config.yaml 2023-10-09T04:19:06.748 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-09T04:19:06.833 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417477 branch: reef description: upgrade:quincy-x/stress-split/{0-distro/ubuntu_20.04 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' first_in_suite: false job_id: '7417477' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: ubuntu os_version: '20.04' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 kernel: hwe: true workunit: branch: reef sha1: a340fb5463aa3715e93e7472d5f60dfdcf184ab3 owner: scheduled_teuthology@teuthology priority: 100 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 5305 sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:quincy-x suite_branch: reef suite_path: /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: a340fb5463aa3715e93e7472d5f60dfdcf184ab3 targets: smithi133.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM= smithi194.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM= tasks: - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 54e62bcbac4e53d9685e08328b790d3b20d71cae timestamp: 2023-10-07_14:23:11 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2830 2023-10-09T04:19:06.834 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa; will attempt to use it 2023-10-09T04:19:06.834 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks 2023-10-09T04:19:06.834 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-10-09T04:19:06.836 INFO:teuthology.task.internal:Checking packages... 2023-10-09T04:19:06.858 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'default' and ceph hash 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0' 2023-10-09T04:19:06.859 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T04:19:06.859 INFO:teuthology.packaging:ref: None 2023-10-09T04:19:06.859 INFO:teuthology.packaging:tag: None 2023-10-09T04:19:06.859 INFO:teuthology.packaging:branch: reef 2023-10-09T04:19:06.859 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T04:19:06.859 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=reef 2023-10-09T04:19:07.172 INFO:teuthology.task.internal:Found packages for ceph version 18.2.0-610-g397de1b0-1focal 2023-10-09T04:19:07.173 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-10-09T04:19:07.178 INFO:teuthology.task.internal:no buildpackages task found 2023-10-09T04:19:07.178 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-10-09T04:19:07.193 INFO:teuthology.task.internal:Saving configuration 2023-10-09T04:19:07.214 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-10-09T04:19:07.222 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T04:19:07.250 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi133.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417477', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 04:12:58.831775', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGETOy3xONfbx7kUFCNiUtRLNfkcrcEsWzPdq6Np/6Bw'} 2023-10-09T04:19:07.270 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi194.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417477', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 04:12:58.830015', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGETOy3xONfbx7kUFCNiUtRLNfkcrcEsWzPdq6Np/6Bw'} 2023-10-09T04:19:07.271 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-10-09T04:19:07.278 INFO:teuthology.task.internal:roles: ubuntu@smithi133.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-10-09T04:19:07.279 INFO:teuthology.task.internal:roles: ubuntu@smithi194.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-10-09T04:19:07.279 INFO:teuthology.run_tasks:Running task console_log... 2023-10-09T04:19:07.355 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f3e5769a1f0>, signals=[15]) 2023-10-09T04:19:07.356 INFO:teuthology.run_tasks:Running task internal.connect... 2023-10-09T04:19:07.361 INFO:teuthology.task.internal:Opening connections... 2023-10-09T04:19:07.361 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi133.front.sepia.ceph.com 2023-10-09T04:19:07.364 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi133.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T04:19:07.413 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi194.front.sepia.ceph.com 2023-10-09T04:19:07.413 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi194.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T04:19:07.457 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-10-09T04:19:07.466 DEBUG:teuthology.orchestra.run.smithi133:> uname -m 2023-10-09T04:19:07.493 INFO:teuthology.orchestra.run.smithi133.stdout:x86_64 2023-10-09T04:19:07.494 DEBUG:teuthology.orchestra.run.smithi133:> cat /etc/os-release 2023-10-09T04:19:07.543 INFO:teuthology.orchestra.run.smithi133.stdout:NAME="Ubuntu" 2023-10-09T04:19:07.544 INFO:teuthology.orchestra.run.smithi133.stdout:VERSION="20.04.4 LTS (Focal Fossa)" 2023-10-09T04:19:07.544 INFO:teuthology.orchestra.run.smithi133.stdout:ID=ubuntu 2023-10-09T04:19:07.544 INFO:teuthology.orchestra.run.smithi133.stdout:ID_LIKE=debian 2023-10-09T04:19:07.544 INFO:teuthology.orchestra.run.smithi133.stdout:PRETTY_NAME="Ubuntu 20.04.4 LTS" 2023-10-09T04:19:07.544 INFO:teuthology.orchestra.run.smithi133.stdout:VERSION_ID="20.04" 2023-10-09T04:19:07.544 INFO:teuthology.orchestra.run.smithi133.stdout:HOME_URL="https://www.ubuntu.com/" 2023-10-09T04:19:07.544 INFO:teuthology.orchestra.run.smithi133.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2023-10-09T04:19:07.544 INFO:teuthology.orchestra.run.smithi133.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2023-10-09T04:19:07.545 INFO:teuthology.orchestra.run.smithi133.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2023-10-09T04:19:07.545 INFO:teuthology.orchestra.run.smithi133.stdout:VERSION_CODENAME=focal 2023-10-09T04:19:07.545 INFO:teuthology.orchestra.run.smithi133.stdout:UBUNTU_CODENAME=focal 2023-10-09T04:19:07.545 INFO:teuthology.lock.ops:Updating smithi133.front.sepia.ceph.com on lock server 2023-10-09T04:19:07.569 DEBUG:teuthology.orchestra.run.smithi194:> uname -m 2023-10-09T04:19:07.577 INFO:teuthology.orchestra.run.smithi194.stdout:x86_64 2023-10-09T04:19:07.577 DEBUG:teuthology.orchestra.run.smithi194:> cat /etc/os-release 2023-10-09T04:19:07.625 INFO:teuthology.orchestra.run.smithi194.stdout:NAME="Ubuntu" 2023-10-09T04:19:07.625 INFO:teuthology.orchestra.run.smithi194.stdout:VERSION="20.04.4 LTS (Focal Fossa)" 2023-10-09T04:19:07.625 INFO:teuthology.orchestra.run.smithi194.stdout:ID=ubuntu 2023-10-09T04:19:07.625 INFO:teuthology.orchestra.run.smithi194.stdout:ID_LIKE=debian 2023-10-09T04:19:07.625 INFO:teuthology.orchestra.run.smithi194.stdout:PRETTY_NAME="Ubuntu 20.04.4 LTS" 2023-10-09T04:19:07.625 INFO:teuthology.orchestra.run.smithi194.stdout:VERSION_ID="20.04" 2023-10-09T04:19:07.625 INFO:teuthology.orchestra.run.smithi194.stdout:HOME_URL="https://www.ubuntu.com/" 2023-10-09T04:19:07.625 INFO:teuthology.orchestra.run.smithi194.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2023-10-09T04:19:07.625 INFO:teuthology.orchestra.run.smithi194.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2023-10-09T04:19:07.626 INFO:teuthology.orchestra.run.smithi194.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2023-10-09T04:19:07.626 INFO:teuthology.orchestra.run.smithi194.stdout:VERSION_CODENAME=focal 2023-10-09T04:19:07.626 INFO:teuthology.orchestra.run.smithi194.stdout:UBUNTU_CODENAME=focal 2023-10-09T04:19:07.626 INFO:teuthology.lock.ops:Updating smithi194.front.sepia.ceph.com on lock server 2023-10-09T04:19:07.644 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-10-09T04:19:07.656 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-10-09T04:19:07.664 INFO:teuthology.task.internal:Checking for old test directory... 2023-10-09T04:19:07.664 DEBUG:teuthology.orchestra.run.smithi133:> test '!' -e /home/ubuntu/cephtest 2023-10-09T04:19:07.666 DEBUG:teuthology.orchestra.run.smithi194:> test '!' -e /home/ubuntu/cephtest 2023-10-09T04:19:07.672 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-10-09T04:19:07.681 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-10-09T04:19:07.681 DEBUG:teuthology.orchestra.run.smithi133:> test -z $(ls -A /var/lib/ceph) 2023-10-09T04:19:07.684 DEBUG:teuthology.orchestra.run.smithi194:> test -z $(ls -A /var/lib/ceph) 2023-10-09T04:19:07.901 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-10-09T04:19:07.955 INFO:teuthology.run_tasks:Running task kernel... 2023-10-09T04:19:07.970 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-10-09T04:19:07.970 INFO:teuthology.task.kernel:normalize config orig: {'hwe': True} 2023-10-09T04:19:07.970 DEBUG:teuthology.task.kernel:normalized overrides {'mon.a': {'hwe': True}, 'mon.c': {'hwe': True}, 'mgr.y': {'hwe': True}, 'osd.0': {'hwe': True}, 'osd.1': {'hwe': True}, 'osd.2': {'hwe': True}, 'osd.3': {'hwe': True}, 'client.0': {'hwe': True}, 'node-exporter.a': {'hwe': True}, 'alertmanager.a': {'hwe': True}, 'mon.b': {'hwe': True}, 'mgr.x': {'hwe': True}, 'osd.4': {'hwe': True}, 'osd.5': {'hwe': True}, 'osd.6': {'hwe': True}, 'osd.7': {'hwe': True}, 'client.1': {'hwe': True}, 'prometheus.a': {'hwe': True}, 'grafana.a': {'hwe': True}, 'node-exporter.b': {'hwe': True}} 2023-10-09T04:19:07.970 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro', 'hwe': True}, 'mon.b': {'kdb': True, 'sha1': 'distro', 'hwe': True}}, timeout 300 2023-10-09T04:19:07.971 DEBUG:teuthology.orchestra.run.smithi133:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-09T04:19:07.971 DEBUG:teuthology.orchestra.run.smithi194:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-09T04:19:07.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:19:07.977 DEBUG:teuthology.orchestra.run.smithi133:> uname -r 2023-10-09T04:19:07.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:19:07.978 DEBUG:teuthology.orchestra.run.smithi194:> uname -r 2023-10-09T04:19:07.984 INFO:teuthology.orchestra.run.smithi133.stdout:5.4.0-124-generic 2023-10-09T04:19:07.984 INFO:teuthology.task.kernel:Running kernel on smithi133: 5.4.0-124-generic 2023-10-09T04:19:07.984 DEBUG:teuthology.orchestra.run.smithi133:> sudo apt-get clean 2023-10-09T04:19:07.985 INFO:teuthology.orchestra.run.smithi194.stdout:5.4.0-124-generic 2023-10-09T04:19:07.985 INFO:teuthology.task.kernel:Running kernel on smithi194: 5.4.0-124-generic 2023-10-09T04:19:07.986 DEBUG:teuthology.orchestra.run.smithi194:> sudo apt-get clean 2023-10-09T04:19:08.082 DEBUG:teuthology.orchestra.run.smithi194:> sudo apt-get update 2023-10-09T04:19:08.097 DEBUG:teuthology.orchestra.run.smithi133:> sudo apt-get update 2023-10-09T04:19:08.288 INFO:teuthology.orchestra.run.smithi133.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-09T04:19:08.292 INFO:teuthology.orchestra.run.smithi133.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal-updates InRelease [114 kB] 2023-10-09T04:19:08.300 INFO:teuthology.orchestra.run.smithi194.stdout:Get:1 http://security.ubuntu.com/ubuntu focal-security InRelease [114 kB] 2023-10-09T04:19:08.353 INFO:teuthology.orchestra.run.smithi133.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal-backports InRelease [108 kB] 2023-10-09T04:19:08.376 INFO:teuthology.orchestra.run.smithi133.stdout:Get:4 http://security.ubuntu.com/ubuntu focal-security InRelease [114 kB] 2023-10-09T04:19:08.400 INFO:teuthology.orchestra.run.smithi194.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-09T04:19:08.458 INFO:teuthology.orchestra.run.smithi194.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease [114 kB] 2023-10-09T04:19:08.673 INFO:teuthology.orchestra.run.smithi133.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages [2,890 kB] 2023-10-09T04:19:08.799 INFO:teuthology.orchestra.run.smithi194.stdout:Get:4 http://security.ubuntu.com/ubuntu focal-security/main i386 Packages [660 kB] 2023-10-09T04:19:08.800 INFO:teuthology.orchestra.run.smithi194.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal-backports InRelease [108 kB] 2023-10-09T04:19:08.841 INFO:teuthology.orchestra.run.smithi133.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main i386 Packages [893 kB] 2023-10-09T04:19:08.856 INFO:teuthology.orchestra.run.smithi133.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main Translation-en [473 kB] 2023-10-09T04:19:08.860 INFO:teuthology.orchestra.run.smithi133.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 c-n-f Metadata [17.1 kB] 2023-10-09T04:19:08.861 INFO:teuthology.orchestra.run.smithi133.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/restricted i386 Packages [34.6 kB] 2023-10-09T04:19:08.861 INFO:teuthology.orchestra.run.smithi133.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 Packages [2,383 kB] 2023-10-09T04:19:08.881 INFO:teuthology.orchestra.run.smithi133.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/restricted Translation-en [334 kB] 2023-10-09T04:19:08.884 INFO:teuthology.orchestra.run.smithi133.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 c-n-f Metadata [572 B] 2023-10-09T04:19:08.884 INFO:teuthology.orchestra.run.smithi133.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal-updates/universe i386 Packages [749 kB] 2023-10-09T04:19:08.891 INFO:teuthology.orchestra.run.smithi133.stdout:Get:14 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 Packages [1,121 kB] 2023-10-09T04:19:08.899 INFO:teuthology.orchestra.run.smithi133.stdout:Get:15 http://archive.ubuntu.com/ubuntu focal-updates/universe Translation-en [268 kB] 2023-10-09T04:19:08.901 INFO:teuthology.orchestra.run.smithi133.stdout:Get:16 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 c-n-f Metadata [25.6 kB] 2023-10-09T04:19:08.901 INFO:teuthology.orchestra.run.smithi133.stdout:Get:17 http://archive.ubuntu.com/ubuntu focal-updates/multiverse i386 Packages [8,436 B] 2023-10-09T04:19:08.901 INFO:teuthology.orchestra.run.smithi133.stdout:Get:18 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 Packages [25.8 kB] 2023-10-09T04:19:08.902 INFO:teuthology.orchestra.run.smithi133.stdout:Get:19 http://archive.ubuntu.com/ubuntu focal-updates/multiverse Translation-en [7,484 B] 2023-10-09T04:19:08.909 INFO:teuthology.orchestra.run.smithi133.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 c-n-f Metadata [620 B] 2023-10-09T04:19:08.909 INFO:teuthology.orchestra.run.smithi133.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 Packages [45.7 kB] 2023-10-09T04:19:08.909 INFO:teuthology.orchestra.run.smithi133.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal-backports/main i386 Packages [36.1 kB] 2023-10-09T04:19:08.924 INFO:teuthology.orchestra.run.smithi133.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 c-n-f Metadata [1,420 B] 2023-10-09T04:19:08.925 INFO:teuthology.orchestra.run.smithi133.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 Packages [25.0 kB] 2023-10-09T04:19:08.925 INFO:teuthology.orchestra.run.smithi133.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal-backports/universe i386 Packages [13.8 kB] 2023-10-09T04:19:08.925 INFO:teuthology.orchestra.run.smithi133.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 c-n-f Metadata [880 B] 2023-10-09T04:19:08.987 INFO:teuthology.orchestra.run.smithi194.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages [2,890 kB] 2023-10-09T04:19:08.996 INFO:teuthology.orchestra.run.smithi194.stdout:Get:7 http://security.ubuntu.com/ubuntu focal-security/main amd64 Packages [2,500 kB] 2023-10-09T04:19:09.024 INFO:teuthology.orchestra.run.smithi133.stdout:Get:27 http://security.ubuntu.com/ubuntu focal-security/main i386 Packages [660 kB] 2023-10-09T04:19:09.103 INFO:teuthology.orchestra.run.smithi194.stdout:Get:8 http://security.ubuntu.com/ubuntu focal-security/main Translation-en [390 kB] 2023-10-09T04:19:09.117 INFO:teuthology.orchestra.run.smithi194.stdout:Get:9 http://security.ubuntu.com/ubuntu focal-security/main amd64 c-n-f Metadata [13.2 kB] 2023-10-09T04:19:09.117 INFO:teuthology.orchestra.run.smithi194.stdout:Get:10 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 Packages [2,267 kB] 2023-10-09T04:19:09.235 INFO:teuthology.orchestra.run.smithi194.stdout:Get:11 http://security.ubuntu.com/ubuntu focal-security/restricted i386 Packages [33.3 kB] 2023-10-09T04:19:09.288 INFO:teuthology.orchestra.run.smithi194.stdout:Get:12 http://security.ubuntu.com/ubuntu focal-security/restricted Translation-en [317 kB] 2023-10-09T04:19:09.295 INFO:teuthology.orchestra.run.smithi194.stdout:Get:13 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 c-n-f Metadata [572 B] 2023-10-09T04:19:09.295 INFO:teuthology.orchestra.run.smithi194.stdout:Get:14 http://security.ubuntu.com/ubuntu focal-security/universe i386 Packages [616 kB] 2023-10-09T04:19:09.304 INFO:teuthology.orchestra.run.smithi194.stdout:Get:15 http://security.ubuntu.com/ubuntu focal-security/universe amd64 Packages [890 kB] 2023-10-09T04:19:09.309 INFO:teuthology.orchestra.run.smithi194.stdout:Get:16 http://security.ubuntu.com/ubuntu focal-security/universe Translation-en [186 kB] 2023-10-09T04:19:09.311 INFO:teuthology.orchestra.run.smithi194.stdout:Get:17 http://security.ubuntu.com/ubuntu focal-security/universe amd64 c-n-f Metadata [19.1 kB] 2023-10-09T04:19:09.311 INFO:teuthology.orchestra.run.smithi194.stdout:Get:18 http://security.ubuntu.com/ubuntu focal-security/multiverse i386 Packages [7,196 B] 2023-10-09T04:19:09.311 INFO:teuthology.orchestra.run.smithi194.stdout:Get:19 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 Packages [23.6 kB] 2023-10-09T04:19:09.331 INFO:teuthology.orchestra.run.smithi194.stdout:Get:20 http://security.ubuntu.com/ubuntu focal-security/multiverse Translation-en [5,504 B] 2023-10-09T04:19:09.331 INFO:teuthology.orchestra.run.smithi194.stdout:Get:21 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 c-n-f Metadata [548 B] 2023-10-09T04:19:09.351 INFO:teuthology.orchestra.run.smithi133.stdout:Get:28 http://security.ubuntu.com/ubuntu focal-security/main amd64 Packages [2,500 kB] 2023-10-09T04:19:09.451 INFO:teuthology.orchestra.run.smithi194.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal-updates/main i386 Packages [893 kB] 2023-10-09T04:19:09.469 INFO:teuthology.orchestra.run.smithi194.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal-updates/main Translation-en [473 kB] 2023-10-09T04:19:09.478 INFO:teuthology.orchestra.run.smithi194.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 c-n-f Metadata [17.1 kB] 2023-10-09T04:19:09.479 INFO:teuthology.orchestra.run.smithi194.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal-updates/restricted i386 Packages [34.6 kB] 2023-10-09T04:19:09.479 INFO:teuthology.orchestra.run.smithi194.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 Packages [2,383 kB] 2023-10-09T04:19:09.540 INFO:teuthology.orchestra.run.smithi133.stdout:Get:29 http://security.ubuntu.com/ubuntu focal-security/main Translation-en [390 kB] 2023-10-09T04:19:09.553 INFO:teuthology.orchestra.run.smithi133.stdout:Get:30 http://security.ubuntu.com/ubuntu focal-security/main amd64 c-n-f Metadata [13.2 kB] 2023-10-09T04:19:09.554 INFO:teuthology.orchestra.run.smithi194.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal-updates/restricted Translation-en [334 kB] 2023-10-09T04:19:09.554 INFO:teuthology.orchestra.run.smithi133.stdout:Get:31 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 Packages [2,267 kB] 2023-10-09T04:19:09.560 INFO:teuthology.orchestra.run.smithi194.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 c-n-f Metadata [572 B] 2023-10-09T04:19:09.622 INFO:teuthology.orchestra.run.smithi133.stdout:Get:32 http://security.ubuntu.com/ubuntu focal-security/restricted i386 Packages [33.3 kB] 2023-10-09T04:19:09.622 INFO:teuthology.orchestra.run.smithi133.stdout:Get:33 http://security.ubuntu.com/ubuntu focal-security/restricted Translation-en [317 kB] 2023-10-09T04:19:09.628 INFO:teuthology.orchestra.run.smithi133.stdout:Get:34 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 c-n-f Metadata [572 B] 2023-10-09T04:19:09.629 INFO:teuthology.orchestra.run.smithi133.stdout:Get:35 http://security.ubuntu.com/ubuntu focal-security/universe i386 Packages [616 kB] 2023-10-09T04:19:09.639 INFO:teuthology.orchestra.run.smithi133.stdout:Get:36 http://security.ubuntu.com/ubuntu focal-security/universe amd64 Packages [890 kB] 2023-10-09T04:19:09.724 INFO:teuthology.orchestra.run.smithi194.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal-updates/universe i386 Packages [749 kB] 2023-10-09T04:19:09.840 INFO:teuthology.orchestra.run.smithi133.stdout:Get:37 http://security.ubuntu.com/ubuntu focal-security/universe Translation-en [186 kB] 2023-10-09T04:19:10.256 INFO:teuthology.orchestra.run.smithi133.stdout:Get:38 http://security.ubuntu.com/ubuntu focal-security/universe amd64 c-n-f Metadata [19.1 kB] 2023-10-09T04:19:10.262 INFO:teuthology.orchestra.run.smithi133.stdout:Get:39 http://security.ubuntu.com/ubuntu focal-security/multiverse i386 Packages [7,196 B] 2023-10-09T04:19:10.264 INFO:teuthology.orchestra.run.smithi133.stdout:Get:40 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 Packages [23.6 kB] 2023-10-09T04:19:10.272 INFO:teuthology.orchestra.run.smithi133.stdout:Get:41 http://security.ubuntu.com/ubuntu focal-security/multiverse Translation-en [5,504 B] 2023-10-09T04:19:10.274 INFO:teuthology.orchestra.run.smithi133.stdout:Get:42 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 c-n-f Metadata [548 B] 2023-10-09T04:19:10.309 INFO:teuthology.orchestra.run.smithi194.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 Packages [1,121 kB] 2023-10-09T04:19:10.418 INFO:teuthology.orchestra.run.smithi194.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal-updates/universe Translation-en [268 kB] 2023-10-09T04:19:10.438 INFO:teuthology.orchestra.run.smithi194.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 c-n-f Metadata [25.6 kB] 2023-10-09T04:19:10.440 INFO:teuthology.orchestra.run.smithi194.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 Packages [25.8 kB] 2023-10-09T04:19:10.442 INFO:teuthology.orchestra.run.smithi194.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal-updates/multiverse i386 Packages [8,436 B] 2023-10-09T04:19:10.443 INFO:teuthology.orchestra.run.smithi194.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal-updates/multiverse Translation-en [7,484 B] 2023-10-09T04:19:10.443 INFO:teuthology.orchestra.run.smithi194.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 c-n-f Metadata [620 B] 2023-10-09T04:19:10.443 INFO:teuthology.orchestra.run.smithi194.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 Packages [45.7 kB] 2023-10-09T04:19:10.446 INFO:teuthology.orchestra.run.smithi194.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal-backports/main i386 Packages [36.1 kB] 2023-10-09T04:19:10.449 INFO:teuthology.orchestra.run.smithi194.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 c-n-f Metadata [1,420 B] 2023-10-09T04:19:10.613 INFO:teuthology.orchestra.run.smithi194.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 Packages [25.0 kB] 2023-10-09T04:19:10.796 INFO:teuthology.orchestra.run.smithi194.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal-backports/universe i386 Packages [13.8 kB] 2023-10-09T04:19:10.818 INFO:teuthology.orchestra.run.smithi194.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 c-n-f Metadata [880 B] 2023-10-09T04:19:12.869 INFO:teuthology.orchestra.run.smithi133.stdout:Fetched 17.6 MB in 3s (6,733 kB/s) 2023-10-09T04:19:12.904 INFO:teuthology.orchestra.run.smithi194.stdout:Fetched 17.6 MB in 3s (6,174 kB/s) 2023-10-09T04:19:14.295 INFO:teuthology.orchestra.run.smithi194.stdout:Reading package lists... 2023-10-09T04:19:14.296 INFO:teuthology.orchestra.run.smithi133.stdout:Reading package lists... 2023-10-09T04:19:14.321 DEBUG:teuthology.orchestra.run.smithi194:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-10-09T04:19:14.322 DEBUG:teuthology.orchestra.run.smithi133:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-10-09T04:19:14.391 INFO:teuthology.orchestra.run.smithi194.stdout:Reading package lists... 2023-10-09T04:19:14.424 INFO:teuthology.orchestra.run.smithi133.stdout:Reading package lists... 2023-10-09T04:19:14.571 INFO:teuthology.orchestra.run.smithi194.stdout:Building dependency tree... 2023-10-09T04:19:14.572 INFO:teuthology.orchestra.run.smithi194.stdout:Reading state information... 2023-10-09T04:19:14.625 INFO:teuthology.orchestra.run.smithi133.stdout:Building dependency tree... 2023-10-09T04:19:14.625 INFO:teuthology.orchestra.run.smithi133.stdout:Reading state information... 2023-10-09T04:19:14.740 INFO:teuthology.orchestra.run.smithi194.stdout:The following additional packages will be installed: 2023-10-09T04:19:14.741 INFO:teuthology.orchestra.run.smithi194.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-10-09T04:19:14.741 INFO:teuthology.orchestra.run.smithi194.stdout: libusbmuxd6 linux-image-5.15.0-86-generic linux-modules-5.15.0-86-generic 2023-10-09T04:19:14.741 INFO:teuthology.orchestra.run.smithi194.stdout: linux-modules-extra-5.15.0-86-generic thermald upower usbmuxd 2023-10-09T04:19:14.742 INFO:teuthology.orchestra.run.smithi194.stdout:Suggested packages: 2023-10-09T04:19:14.742 INFO:teuthology.orchestra.run.smithi194.stdout: libusbmuxd-tools fdutils linux-doc | linux-hwe-5.15-source-5.15.0 2023-10-09T04:19:14.742 INFO:teuthology.orchestra.run.smithi194.stdout: linux-hwe-5.15-tools linux-headers-5.15.0-86-generic 2023-10-09T04:19:14.795 INFO:teuthology.orchestra.run.smithi194.stdout:The following NEW packages will be installed: 2023-10-09T04:19:14.795 INFO:teuthology.orchestra.run.smithi194.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-10-09T04:19:14.795 INFO:teuthology.orchestra.run.smithi194.stdout: libusbmuxd6 linux-image-5.15.0-86-generic linux-image-generic-hwe-20.04 2023-10-09T04:19:14.796 INFO:teuthology.orchestra.run.smithi194.stdout: linux-modules-5.15.0-86-generic linux-modules-extra-5.15.0-86-generic 2023-10-09T04:19:14.796 INFO:teuthology.orchestra.run.smithi194.stdout: thermald upower usbmuxd 2023-10-09T04:19:14.802 INFO:teuthology.orchestra.run.smithi133.stdout:The following additional packages will be installed: 2023-10-09T04:19:14.803 INFO:teuthology.orchestra.run.smithi133.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-10-09T04:19:14.803 INFO:teuthology.orchestra.run.smithi133.stdout: libusbmuxd6 linux-image-5.15.0-86-generic linux-modules-5.15.0-86-generic 2023-10-09T04:19:14.803 INFO:teuthology.orchestra.run.smithi133.stdout: linux-modules-extra-5.15.0-86-generic thermald upower usbmuxd 2023-10-09T04:19:14.804 INFO:teuthology.orchestra.run.smithi133.stdout:Suggested packages: 2023-10-09T04:19:14.805 INFO:teuthology.orchestra.run.smithi133.stdout: libusbmuxd-tools fdutils linux-doc | linux-hwe-5.15-source-5.15.0 2023-10-09T04:19:14.805 INFO:teuthology.orchestra.run.smithi133.stdout: linux-hwe-5.15-tools linux-headers-5.15.0-86-generic 2023-10-09T04:19:14.855 INFO:teuthology.orchestra.run.smithi133.stdout:The following NEW packages will be installed: 2023-10-09T04:19:14.856 INFO:teuthology.orchestra.run.smithi133.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-10-09T04:19:14.856 INFO:teuthology.orchestra.run.smithi133.stdout: libusbmuxd6 linux-image-5.15.0-86-generic linux-image-generic-hwe-20.04 2023-10-09T04:19:14.856 INFO:teuthology.orchestra.run.smithi133.stdout: linux-modules-5.15.0-86-generic linux-modules-extra-5.15.0-86-generic 2023-10-09T04:19:14.856 INFO:teuthology.orchestra.run.smithi133.stdout: thermald upower usbmuxd 2023-10-09T04:19:14.858 INFO:teuthology.orchestra.run.smithi194.stdout:0 upgraded, 13 newly installed, 0 to remove and 274 not upgraded. 2023-10-09T04:19:14.858 INFO:teuthology.orchestra.run.smithi194.stdout:Need to get 90.9 MB of archives. 2023-10-09T04:19:14.858 INFO:teuthology.orchestra.run.smithi194.stdout:After this operation, 497 MB of additional disk space will be used. 2023-10-09T04:19:14.858 INFO:teuthology.orchestra.run.smithi194.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 libdbus-glib-1-2 amd64 0.110-5fakssync1 [59.1 kB] 2023-10-09T04:19:14.915 INFO:teuthology.orchestra.run.smithi133.stdout:0 upgraded, 13 newly installed, 0 to remove and 274 not upgraded. 2023-10-09T04:19:14.915 INFO:teuthology.orchestra.run.smithi133.stdout:Need to get 90.9 MB of archives. 2023-10-09T04:19:14.915 INFO:teuthology.orchestra.run.smithi133.stdout:After this operation, 497 MB of additional disk space will be used. 2023-10-09T04:19:14.915 INFO:teuthology.orchestra.run.smithi133.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 libdbus-glib-1-2 amd64 0.110-5fakssync1 [59.1 kB] 2023-10-09T04:19:14.924 INFO:teuthology.orchestra.run.smithi194.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 libplist3 amd64 2.1.0-4build2 [31.6 kB] 2023-10-09T04:19:14.931 INFO:teuthology.orchestra.run.smithi194.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 libusbmuxd6 amd64 2.0.1-2 [19.1 kB] 2023-10-09T04:19:14.936 INFO:teuthology.orchestra.run.smithi194.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 libimobiledevice6 amd64 1.2.1~git20191129.9f79242-1build1 [65.2 kB] 2023-10-09T04:19:14.953 INFO:teuthology.orchestra.run.smithi194.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 libupower-glib3 amd64 0.99.11-1build2 [43.2 kB] 2023-10-09T04:19:14.962 INFO:teuthology.orchestra.run.smithi194.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-5.15.0-86-generic amd64 5.15.0-86.96~20.04.1 [20.9 MB] 2023-10-09T04:19:14.981 INFO:teuthology.orchestra.run.smithi133.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 libplist3 amd64 2.1.0-4build2 [31.6 kB] 2023-10-09T04:19:14.988 INFO:teuthology.orchestra.run.smithi133.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 libusbmuxd6 amd64 2.0.1-2 [19.1 kB] 2023-10-09T04:19:14.993 INFO:teuthology.orchestra.run.smithi133.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 libimobiledevice6 amd64 1.2.1~git20191129.9f79242-1build1 [65.2 kB] 2023-10-09T04:19:15.011 INFO:teuthology.orchestra.run.smithi133.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 libupower-glib3 amd64 0.99.11-1build2 [43.2 kB] 2023-10-09T04:19:15.019 INFO:teuthology.orchestra.run.smithi133.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-5.15.0-86-generic amd64 5.15.0-86.96~20.04.1 [20.9 MB] 2023-10-09T04:19:15.433 INFO:teuthology.orchestra.run.smithi194.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-5.15.0-86-generic amd64 5.15.0-86.96~20.04.1 [11.4 MB] 2023-10-09T04:19:15.701 INFO:teuthology.orchestra.run.smithi194.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-extra-5.15.0-86-generic amd64 5.15.0-86.96~20.04.1 [58.0 MB] 2023-10-09T04:19:15.718 INFO:teuthology.orchestra.run.smithi133.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-5.15.0-86-generic amd64 5.15.0-86.96~20.04.1 [11.4 MB] 2023-10-09T04:19:15.902 INFO:teuthology.orchestra.run.smithi133.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-extra-5.15.0-86-generic amd64 5.15.0-86.96~20.04.1 [58.0 MB] 2023-10-09T04:19:17.347 INFO:teuthology.orchestra.run.smithi194.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-generic-hwe-20.04 amd64 5.15.0.86.96~20.04.44 [2,648 B] 2023-10-09T04:19:17.348 INFO:teuthology.orchestra.run.smithi194.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libevdev2 amd64 1.9.0+dfsg-1ubuntu0.2 [31.6 kB] 2023-10-09T04:19:17.348 INFO:teuthology.orchestra.run.smithi194.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 thermald amd64 1.9.1-1ubuntu0.6 [233 kB] 2023-10-09T04:19:17.352 INFO:teuthology.orchestra.run.smithi194.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 upower amd64 0.99.11-1build2 [104 kB] 2023-10-09T04:19:17.353 INFO:teuthology.orchestra.run.smithi194.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 usbmuxd amd64 1.1.1~git20191130.9af2b12-1 [38.4 kB] 2023-10-09T04:19:17.425 INFO:teuthology.orchestra.run.smithi133.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-generic-hwe-20.04 amd64 5.15.0.86.96~20.04.44 [2,648 B] 2023-10-09T04:19:17.426 INFO:teuthology.orchestra.run.smithi133.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libevdev2 amd64 1.9.0+dfsg-1ubuntu0.2 [31.6 kB] 2023-10-09T04:19:17.427 INFO:teuthology.orchestra.run.smithi133.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 thermald amd64 1.9.1-1ubuntu0.6 [233 kB] 2023-10-09T04:19:17.432 INFO:teuthology.orchestra.run.smithi133.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 upower amd64 0.99.11-1build2 [104 kB] 2023-10-09T04:19:17.435 INFO:teuthology.orchestra.run.smithi133.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 usbmuxd amd64 1.1.1~git20191130.9af2b12-1 [38.4 kB] 2023-10-09T04:19:18.035 INFO:teuthology.orchestra.run.smithi194.stdout:Fetched 90.9 MB in 3s (35.6 MB/s) 2023-10-09T04:19:18.113 INFO:teuthology.orchestra.run.smithi133.stdout:Fetched 90.9 MB in 3s (35.3 MB/s) 2023-10-09T04:19:18.244 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package libdbus-glib-1-2:amd64. 2023-10-09T04:19:18.311 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package libdbus-glib-1-2:amd64. 2023-10-09T04:19:19.967 INFO:teuthology.orchestra.run.smithi194.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 104373 files and directories currently installed.) 2023-10-09T04:19:19.972 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../00-libdbus-glib-1-2_0.110-5fakssync1_amd64.deb ... 2023-10-09T04:19:20.074 INFO:teuthology.orchestra.run.smithi133.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 104373 files and directories currently installed.) 2023-10-09T04:19:20.079 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../00-libdbus-glib-1-2_0.110-5fakssync1_amd64.deb ... 2023-10-09T04:19:20.183 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-10-09T04:19:20.200 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-10-09T04:19:20.555 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package libplist3:amd64. 2023-10-09T04:19:20.561 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package libplist3:amd64. 2023-10-09T04:19:20.568 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../01-libplist3_2.1.0-4build2_amd64.deb ... 2023-10-09T04:19:20.575 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../01-libplist3_2.1.0-4build2_amd64.deb ... 2023-10-09T04:19:20.605 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking libplist3:amd64 (2.1.0-4build2) ... 2023-10-09T04:19:20.661 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking libplist3:amd64 (2.1.0-4build2) ... 2023-10-09T04:19:20.925 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package libusbmuxd6:amd64. 2023-10-09T04:19:20.938 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../02-libusbmuxd6_2.0.1-2_amd64.deb ... 2023-10-09T04:19:20.983 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking libusbmuxd6:amd64 (2.0.1-2) ... 2023-10-09T04:19:21.014 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package libusbmuxd6:amd64. 2023-10-09T04:19:21.028 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../02-libusbmuxd6_2.0.1-2_amd64.deb ... 2023-10-09T04:19:21.064 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking libusbmuxd6:amd64 (2.0.1-2) ... 2023-10-09T04:19:21.336 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package libimobiledevice6:amd64. 2023-10-09T04:19:21.350 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../03-libimobiledevice6_1.2.1~git20191129.9f79242-1build1_amd64.deb ... 2023-10-09T04:19:21.394 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-10-09T04:19:21.450 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package libimobiledevice6:amd64. 2023-10-09T04:19:21.464 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../03-libimobiledevice6_1.2.1~git20191129.9f79242-1build1_amd64.deb ... 2023-10-09T04:19:21.508 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-10-09T04:19:21.731 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package libupower-glib3:amd64. 2023-10-09T04:19:21.745 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../04-libupower-glib3_0.99.11-1build2_amd64.deb ... 2023-10-09T04:19:21.789 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking libupower-glib3:amd64 (0.99.11-1build2) ... 2023-10-09T04:19:21.903 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package libupower-glib3:amd64. 2023-10-09T04:19:21.918 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../04-libupower-glib3_0.99.11-1build2_amd64.deb ... 2023-10-09T04:19:21.953 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking libupower-glib3:amd64 (0.99.11-1build2) ... 2023-10-09T04:19:22.108 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package linux-modules-5.15.0-86-generic. 2023-10-09T04:19:22.122 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../05-linux-modules-5.15.0-86-generic_5.15.0-86.96~20.04.1_amd64.deb ... 2023-10-09T04:19:22.158 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking linux-modules-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T04:19:22.298 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package linux-modules-5.15.0-86-generic. 2023-10-09T04:19:22.312 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../05-linux-modules-5.15.0-86-generic_5.15.0-86.96~20.04.1_amd64.deb ... 2023-10-09T04:19:22.378 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking linux-modules-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T04:19:24.600 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package linux-image-5.15.0-86-generic. 2023-10-09T04:19:24.613 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../06-linux-image-5.15.0-86-generic_5.15.0-86.96~20.04.1_amd64.deb ... 2023-10-09T04:19:24.697 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking linux-image-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T04:19:24.981 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package linux-image-5.15.0-86-generic. 2023-10-09T04:19:24.995 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../06-linux-image-5.15.0-86-generic_5.15.0-86.96~20.04.1_amd64.deb ... 2023-10-09T04:19:25.101 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking linux-image-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T04:19:25.228 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-86-generic. 2023-10-09T04:19:25.242 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../07-linux-modules-extra-5.15.0-86-generic_5.15.0-86.96~20.04.1_amd64.deb ... 2023-10-09T04:19:25.286 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking linux-modules-extra-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T04:19:25.618 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-86-generic. 2023-10-09T04:19:25.631 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../07-linux-modules-extra-5.15.0-86-generic_5.15.0-86.96~20.04.1_amd64.deb ... 2023-10-09T04:19:25.692 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking linux-modules-extra-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T04:19:32.112 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package linux-image-generic-hwe-20.04. 2023-10-09T04:19:32.128 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../08-linux-image-generic-hwe-20.04_5.15.0.86.96~20.04.44_amd64.deb ... 2023-10-09T04:19:32.170 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking linux-image-generic-hwe-20.04 (5.15.0.86.96~20.04.44) ... 2023-10-09T04:19:32.506 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package libevdev2:amd64. 2023-10-09T04:19:32.522 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../09-libevdev2_1.9.0+dfsg-1ubuntu0.2_amd64.deb ... 2023-10-09T04:19:32.564 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-10-09T04:19:32.658 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package linux-image-generic-hwe-20.04. 2023-10-09T04:19:32.676 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../08-linux-image-generic-hwe-20.04_5.15.0.86.96~20.04.44_amd64.deb ... 2023-10-09T04:19:32.716 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking linux-image-generic-hwe-20.04 (5.15.0.86.96~20.04.44) ... 2023-10-09T04:19:32.950 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package thermald. 2023-10-09T04:19:32.966 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../10-thermald_1.9.1-1ubuntu0.6_amd64.deb ... 2023-10-09T04:19:33.008 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking thermald (1.9.1-1ubuntu0.6) ... 2023-10-09T04:19:33.085 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package libevdev2:amd64. 2023-10-09T04:19:33.100 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../09-libevdev2_1.9.0+dfsg-1ubuntu0.2_amd64.deb ... 2023-10-09T04:19:33.143 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-10-09T04:19:33.555 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package upower. 2023-10-09T04:19:33.563 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package thermald. 2023-10-09T04:19:33.568 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../11-upower_0.99.11-1build2_amd64.deb ... 2023-10-09T04:19:33.577 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../10-thermald_1.9.1-1ubuntu0.6_amd64.deb ... 2023-10-09T04:19:33.621 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking thermald (1.9.1-1ubuntu0.6) ... 2023-10-09T04:19:33.622 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking upower (0.99.11-1build2) ... 2023-10-09T04:19:34.018 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package usbmuxd. 2023-10-09T04:19:34.031 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../12-usbmuxd_1.1.1~git20191130.9af2b12-1_amd64.deb ... 2023-10-09T04:19:34.076 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-10-09T04:19:34.193 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package upower. 2023-10-09T04:19:34.208 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../11-upower_0.99.11-1build2_amd64.deb ... 2023-10-09T04:19:34.283 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking upower (0.99.11-1build2) ... 2023-10-09T04:19:34.364 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up libplist3:amd64 (2.1.0-4build2) ... 2023-10-09T04:19:34.488 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-10-09T04:19:34.623 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up libusbmuxd6:amd64 (2.0.1-2) ... 2023-10-09T04:19:34.697 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package usbmuxd. 2023-10-09T04:19:34.712 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../12-usbmuxd_1.1.1~git20191130.9af2b12-1_amd64.deb ... 2023-10-09T04:19:34.746 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-10-09T04:19:34.766 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up libupower-glib3:amd64 (0.99.11-1build2) ... 2023-10-09T04:19:34.892 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-10-09T04:19:35.018 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-10-09T04:19:35.097 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up libplist3:amd64 (2.1.0-4build2) ... 2023-10-09T04:19:35.144 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up upower (0.99.11-1build2) ... 2023-10-09T04:19:35.218 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-10-09T04:19:35.360 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up libusbmuxd6:amd64 (2.0.1-2) ... 2023-10-09T04:19:35.486 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up libupower-glib3:amd64 (0.99.11-1build2) ... 2023-10-09T04:19:35.629 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-10-09T04:19:35.664 INFO:teuthology.orchestra.run.smithi133.stdout:upower.service is a disabled or a static unit, not starting it. 2023-10-09T04:19:35.715 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-10-09T04:19:35.755 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-10-09T04:19:35.906 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up upower (0.99.11-1build2) ... 2023-10-09T04:19:35.934 INFO:teuthology.orchestra.run.smithi133.stdout:Warning: The home dir /var/lib/usbmux you specified can't be accessed: No such file or directory 2023-10-09T04:19:35.953 INFO:teuthology.orchestra.run.smithi133.stdout:Adding system user `usbmux' (UID 114) ... 2023-10-09T04:19:35.953 INFO:teuthology.orchestra.run.smithi133.stdout:Adding new user `usbmux' (UID 114) with group `plugdev' ... 2023-10-09T04:19:36.448 INFO:teuthology.orchestra.run.smithi133.stdout:Not creating home directory `/var/lib/usbmux'. 2023-10-09T04:19:36.456 INFO:teuthology.orchestra.run.smithi194.stdout:upower.service is a disabled or a static unit, not starting it. 2023-10-09T04:19:36.510 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-10-09T04:19:36.667 INFO:teuthology.orchestra.run.smithi133.stdout:usbmuxd.service is a disabled or a static unit, not starting it. 2023-10-09T04:19:36.711 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up thermald (1.9.1-1ubuntu0.6) ... 2023-10-09T04:19:36.742 INFO:teuthology.orchestra.run.smithi194.stdout:Warning: The home dir /var/lib/usbmux you specified can't be accessed: No such file or directory 2023-10-09T04:19:36.749 INFO:teuthology.orchestra.run.smithi194.stdout:Adding system user `usbmux' (UID 114) ... 2023-10-09T04:19:36.749 INFO:teuthology.orchestra.run.smithi194.stdout:Adding new user `usbmux' (UID 114) with group `plugdev' ... 2023-10-09T04:19:37.048 INFO:teuthology.orchestra.run.smithi133.stdout:Created symlink /etc/systemd/system/dbus-org.freedesktop.thermald.service -> /lib/systemd/system/thermald.service. 2023-10-09T04:19:37.049 INFO:teuthology.orchestra.run.smithi133.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/thermald.service -> /lib/systemd/system/thermald.service. 2023-10-09T04:19:37.285 INFO:teuthology.orchestra.run.smithi194.stdout:Not creating home directory `/var/lib/usbmux'. 2023-10-09T04:19:37.498 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up linux-image-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T04:19:37.504 INFO:teuthology.orchestra.run.smithi194.stdout:usbmuxd.service is a disabled or a static unit, not starting it. 2023-10-09T04:19:37.573 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up thermald (1.9.1-1ubuntu0.6) ... 2023-10-09T04:19:37.884 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/dbus-org.freedesktop.thermald.service -> /lib/systemd/system/thermald.service. 2023-10-09T04:19:37.884 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/thermald.service -> /lib/systemd/system/thermald.service. 2023-10-09T04:19:38.419 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up linux-image-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T04:19:38.851 INFO:teuthology.orchestra.run.smithi133.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-86-generic 2023-10-09T04:19:38.852 INFO:teuthology.orchestra.run.smithi133.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-86-generic 2023-10-09T04:19:39.010 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up linux-modules-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T04:19:39.746 INFO:teuthology.orchestra.run.smithi194.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-86-generic 2023-10-09T04:19:39.746 INFO:teuthology.orchestra.run.smithi194.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-86-generic 2023-10-09T04:19:39.921 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up linux-modules-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T04:19:40.419 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up linux-modules-extra-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T04:19:41.314 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up linux-modules-extra-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T04:19:41.804 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up linux-image-generic-hwe-20.04 (5.15.0.86.96~20.04.44) ... 2023-10-09T04:19:41.929 INFO:teuthology.orchestra.run.smithi133.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-10-09T04:19:42.226 INFO:teuthology.orchestra.run.smithi133.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-10-09T04:19:42.724 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up linux-image-generic-hwe-20.04 (5.15.0.86.96~20.04.44) ... 2023-10-09T04:19:42.849 INFO:teuthology.orchestra.run.smithi194.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-10-09T04:19:43.138 INFO:teuthology.orchestra.run.smithi194.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-10-09T04:19:43.713 INFO:teuthology.orchestra.run.smithi133.stdout:Processing triggers for dbus (1.12.16-2ubuntu2.2) ... 2023-10-09T04:19:43.822 INFO:teuthology.orchestra.run.smithi133.stdout:Processing triggers for linux-image-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T04:19:43.920 INFO:teuthology.orchestra.run.smithi133.stdout:/etc/kernel/postinst.d/initramfs-tools: 2023-10-09T04:19:43.920 INFO:teuthology.orchestra.run.smithi133.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-86-generic 2023-10-09T04:19:45.460 INFO:teuthology.orchestra.run.smithi194.stdout:Processing triggers for dbus (1.12.16-2ubuntu2.2) ... 2023-10-09T04:19:45.568 INFO:teuthology.orchestra.run.smithi194.stdout:Processing triggers for linux-image-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T04:19:45.655 INFO:teuthology.orchestra.run.smithi194.stdout:/etc/kernel/postinst.d/initramfs-tools: 2023-10-09T04:19:45.656 INFO:teuthology.orchestra.run.smithi194.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-86-generic 2023-10-09T04:20:09.224 INFO:teuthology.orchestra.run.smithi133.stdout:/etc/kernel/postinst.d/zz-update-grub: 2023-10-09T04:20:09.224 INFO:teuthology.orchestra.run.smithi133.stdout:Sourcing file `/etc/default/grub' 2023-10-09T04:20:09.233 INFO:teuthology.orchestra.run.smithi133.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-10-09T04:20:09.249 INFO:teuthology.orchestra.run.smithi133.stdout:Generating grub configuration file ... 2023-10-09T04:20:09.618 INFO:teuthology.orchestra.run.smithi133.stdout:Found linux image: /boot/vmlinuz-5.15.0-86-generic 2023-10-09T04:20:09.638 INFO:teuthology.orchestra.run.smithi133.stdout:Found initrd image: /boot/initrd.img-5.15.0-86-generic 2023-10-09T04:20:10.058 INFO:teuthology.orchestra.run.smithi133.stdout:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-10-09T04:20:10.070 INFO:teuthology.orchestra.run.smithi133.stdout:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-10-09T04:20:11.235 INFO:teuthology.orchestra.run.smithi194.stdout:/etc/kernel/postinst.d/zz-update-grub: 2023-10-09T04:20:11.235 INFO:teuthology.orchestra.run.smithi194.stdout:Sourcing file `/etc/default/grub' 2023-10-09T04:20:11.254 INFO:teuthology.orchestra.run.smithi194.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-10-09T04:20:11.284 INFO:teuthology.orchestra.run.smithi194.stdout:Generating grub configuration file ... 2023-10-09T04:20:11.628 INFO:teuthology.orchestra.run.smithi194.stdout:Found linux image: /boot/vmlinuz-5.15.0-86-generic 2023-10-09T04:20:11.662 INFO:teuthology.orchestra.run.smithi194.stdout:Found initrd image: /boot/initrd.img-5.15.0-86-generic 2023-10-09T04:20:12.072 INFO:teuthology.orchestra.run.smithi194.stdout:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-10-09T04:20:12.083 INFO:teuthology.orchestra.run.smithi194.stdout:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-10-09T04:20:12.127 INFO:teuthology.orchestra.run.smithi133.stdout:done 2023-10-09T04:20:12.539 DEBUG:teuthology.orchestra.run.smithi133:> dpkg -s linux-image-generic-hwe-20.04 2023-10-09T04:20:12.583 INFO:teuthology.orchestra.run.smithi133.stdout:Package: linux-image-generic-hwe-20.04 2023-10-09T04:20:12.584 INFO:teuthology.orchestra.run.smithi133.stdout:Status: install ok installed 2023-10-09T04:20:12.584 INFO:teuthology.orchestra.run.smithi133.stdout:Priority: optional 2023-10-09T04:20:12.584 INFO:teuthology.orchestra.run.smithi133.stdout:Section: kernel 2023-10-09T04:20:12.584 INFO:teuthology.orchestra.run.smithi133.stdout:Installed-Size: 20 2023-10-09T04:20:12.584 INFO:teuthology.orchestra.run.smithi133.stdout:Maintainer: Ubuntu Kernel Team 2023-10-09T04:20:12.584 INFO:teuthology.orchestra.run.smithi133.stdout:Architecture: amd64 2023-10-09T04:20:12.584 INFO:teuthology.orchestra.run.smithi133.stdout:Source: linux-meta-hwe-5.15 2023-10-09T04:20:12.584 INFO:teuthology.orchestra.run.smithi133.stdout:Version: 5.15.0.86.96~20.04.44 2023-10-09T04:20:12.584 INFO:teuthology.orchestra.run.smithi133.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-86), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-10-09T04:20:12.585 INFO:teuthology.orchestra.run.smithi133.stdout:Depends: linux-image-5.15.0-86-generic, linux-modules-extra-5.15.0-86-generic, linux-firmware, intel-microcode, amd64-microcode 2023-10-09T04:20:12.585 INFO:teuthology.orchestra.run.smithi133.stdout:Recommends: thermald 2023-10-09T04:20:12.585 INFO:teuthology.orchestra.run.smithi133.stdout:Description: Generic Linux kernel image 2023-10-09T04:20:12.585 INFO:teuthology.orchestra.run.smithi133.stdout: This package will always depend on the latest generic kernel image 2023-10-09T04:20:12.585 INFO:teuthology.orchestra.run.smithi133.stdout: available. 2023-10-09T04:20:12.585 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.4.0-124-generic Expected: 5.15.0-86-generic 2023-10-09T04:20:12.585 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-10-09T04:20:12.585 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-10-09T04:20:12.585 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2023-10-09T04:20:12.586 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi133.front.sepia.ceph.com, path=None, version=distro) 2023-10-09T04:20:12.586 DEBUG:teuthology.orchestra.run.smithi133:> sudo apt-get clean 2023-10-09T04:20:12.678 DEBUG:teuthology.orchestra.run.smithi133:> sudo apt-get update 2023-10-09T04:20:12.956 INFO:teuthology.orchestra.run.smithi133.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-09T04:20:12.957 INFO:teuthology.orchestra.run.smithi133.stdout:Hit:2 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-10-09T04:20:13.026 INFO:teuthology.orchestra.run.smithi133.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-10-09T04:20:13.115 INFO:teuthology.orchestra.run.smithi133.stdout:Hit:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-10-09T04:20:14.134 INFO:teuthology.orchestra.run.smithi194.stdout:done 2023-10-09T04:20:14.574 DEBUG:teuthology.orchestra.run.smithi194:> dpkg -s linux-image-generic-hwe-20.04 2023-10-09T04:20:14.624 INFO:teuthology.orchestra.run.smithi194.stdout:Package: linux-image-generic-hwe-20.04 2023-10-09T04:20:14.624 INFO:teuthology.orchestra.run.smithi194.stdout:Status: install ok installed 2023-10-09T04:20:14.624 INFO:teuthology.orchestra.run.smithi194.stdout:Priority: optional 2023-10-09T04:20:14.624 INFO:teuthology.orchestra.run.smithi194.stdout:Section: kernel 2023-10-09T04:20:14.625 INFO:teuthology.orchestra.run.smithi194.stdout:Installed-Size: 20 2023-10-09T04:20:14.625 INFO:teuthology.orchestra.run.smithi194.stdout:Maintainer: Ubuntu Kernel Team 2023-10-09T04:20:14.625 INFO:teuthology.orchestra.run.smithi194.stdout:Architecture: amd64 2023-10-09T04:20:14.625 INFO:teuthology.orchestra.run.smithi194.stdout:Source: linux-meta-hwe-5.15 2023-10-09T04:20:14.625 INFO:teuthology.orchestra.run.smithi194.stdout:Version: 5.15.0.86.96~20.04.44 2023-10-09T04:20:14.625 INFO:teuthology.orchestra.run.smithi194.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-86), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-10-09T04:20:14.625 INFO:teuthology.orchestra.run.smithi194.stdout:Depends: linux-image-5.15.0-86-generic, linux-modules-extra-5.15.0-86-generic, linux-firmware, intel-microcode, amd64-microcode 2023-10-09T04:20:14.625 INFO:teuthology.orchestra.run.smithi194.stdout:Recommends: thermald 2023-10-09T04:20:14.625 INFO:teuthology.orchestra.run.smithi194.stdout:Description: Generic Linux kernel image 2023-10-09T04:20:14.625 INFO:teuthology.orchestra.run.smithi194.stdout: This package will always depend on the latest generic kernel image 2023-10-09T04:20:14.625 INFO:teuthology.orchestra.run.smithi194.stdout: available. 2023-10-09T04:20:14.626 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.4.0-124-generic Expected: 5.15.0-86-generic 2023-10-09T04:20:14.626 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-10-09T04:20:14.626 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-10-09T04:20:14.626 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2023-10-09T04:20:14.626 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi194.front.sepia.ceph.com, path=None, version=distro) 2023-10-09T04:20:14.626 DEBUG:teuthology.orchestra.run.smithi194:> sudo apt-get clean 2023-10-09T04:20:14.691 INFO:teuthology.orchestra.run.smithi133.stdout:Reading package lists... 2023-10-09T04:20:14.716 DEBUG:teuthology.orchestra.run.smithi133:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-10-09T04:20:14.729 DEBUG:teuthology.orchestra.run.smithi194:> sudo apt-get update 2023-10-09T04:20:14.781 INFO:teuthology.orchestra.run.smithi133.stdout:Reading package lists... 2023-10-09T04:20:14.857 INFO:teuthology.orchestra.run.smithi194.stdout:Hit:1 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-10-09T04:20:14.857 INFO:teuthology.orchestra.run.smithi194.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-09T04:20:14.870 INFO:teuthology.orchestra.run.smithi194.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-10-09T04:20:14.890 INFO:teuthology.orchestra.run.smithi194.stdout:Hit:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-10-09T04:20:14.960 INFO:teuthology.orchestra.run.smithi133.stdout:Building dependency tree... 2023-10-09T04:20:14.960 INFO:teuthology.orchestra.run.smithi133.stdout:Reading state information... 2023-10-09T04:20:15.173 INFO:teuthology.orchestra.run.smithi133.stdout:linux-image-generic-hwe-20.04 is already the newest version (5.15.0.86.96~20.04.44). 2023-10-09T04:20:15.173 INFO:teuthology.orchestra.run.smithi133.stdout:0 upgraded, 0 newly installed, 0 to remove and 274 not upgraded. 2023-10-09T04:20:15.175 DEBUG:teuthology.orchestra.run.smithi133:> dpkg -s linux-image-generic-hwe-20.04 2023-10-09T04:20:15.192 INFO:teuthology.orchestra.run.smithi133.stdout:Package: linux-image-generic-hwe-20.04 2023-10-09T04:20:15.192 INFO:teuthology.orchestra.run.smithi133.stdout:Status: install ok installed 2023-10-09T04:20:15.192 INFO:teuthology.orchestra.run.smithi133.stdout:Priority: optional 2023-10-09T04:20:15.192 INFO:teuthology.orchestra.run.smithi133.stdout:Section: kernel 2023-10-09T04:20:15.192 INFO:teuthology.orchestra.run.smithi133.stdout:Installed-Size: 20 2023-10-09T04:20:15.193 INFO:teuthology.orchestra.run.smithi133.stdout:Maintainer: Ubuntu Kernel Team 2023-10-09T04:20:15.193 INFO:teuthology.orchestra.run.smithi133.stdout:Architecture: amd64 2023-10-09T04:20:15.193 INFO:teuthology.orchestra.run.smithi133.stdout:Source: linux-meta-hwe-5.15 2023-10-09T04:20:15.193 INFO:teuthology.orchestra.run.smithi133.stdout:Version: 5.15.0.86.96~20.04.44 2023-10-09T04:20:15.193 INFO:teuthology.orchestra.run.smithi133.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-86), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-10-09T04:20:15.193 INFO:teuthology.orchestra.run.smithi133.stdout:Depends: linux-image-5.15.0-86-generic, linux-modules-extra-5.15.0-86-generic, linux-firmware, intel-microcode, amd64-microcode 2023-10-09T04:20:15.193 INFO:teuthology.orchestra.run.smithi133.stdout:Recommends: thermald 2023-10-09T04:20:15.193 INFO:teuthology.orchestra.run.smithi133.stdout:Description: Generic Linux kernel image 2023-10-09T04:20:15.193 INFO:teuthology.orchestra.run.smithi133.stdout: This package will always depend on the latest generic kernel image 2023-10-09T04:20:15.193 INFO:teuthology.orchestra.run.smithi133.stdout: available. 2023-10-09T04:20:15.194 DEBUG:teuthology.orchestra.run.smithi133:> mktemp 2023-10-09T04:20:15.198 INFO:teuthology.orchestra.run.smithi133.stdout:/tmp/tmp.6DCpKOyWwP 2023-10-09T04:20:15.198 DEBUG:teuthology.orchestra.run.smithi133:> sudo cp /boot/grub/grub.cfg /tmp/tmp.6DCpKOyWwP 2023-10-09T04:20:15.208 DEBUG:teuthology.orchestra.run.smithi133:> sudo chmod 0666 /tmp/tmp.6DCpKOyWwP 2023-10-09T04:20:15.240 DEBUG:teuthology.orchestra.remote:smithi133:/tmp/tmp.6DCpKOyWwP is 9KB 2023-10-09T04:20:15.256 DEBUG:teuthology.orchestra.run.smithi133:> rm -fr /tmp/tmp.6DCpKOyWwP 2023-10-09T04:20:15.263 DEBUG:teuthology.orchestra.run.smithi133:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2023-10-09T04:20:15.321 DEBUG:teuthology.orchestra.run.smithi133:> set -ex 2023-10-09T04:20:15.321 DEBUG:teuthology.orchestra.run.smithi133:> sudo dd of=/etc/grub.d/01_ceph_kernel 2023-10-09T04:20:15.321 DEBUG:teuthology.orchestra.run.smithi133:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2023-10-09T04:20:15.355 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-86-generic 2023-10-09T04:20:15.356 DEBUG:teuthology.orchestra.run.smithi133:> sudo update-grub 2023-10-09T04:20:16.643 INFO:teuthology.orchestra.run.smithi133.stderr:Sourcing file `/etc/default/grub' 2023-10-09T04:20:16.648 INFO:teuthology.orchestra.run.smithi133.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-10-09T04:20:16.652 INFO:teuthology.orchestra.run.smithi194.stdout:Reading package lists... 2023-10-09T04:20:16.657 INFO:teuthology.orchestra.run.smithi133.stderr:Generating grub configuration file ... 2023-10-09T04:20:16.677 DEBUG:teuthology.orchestra.run.smithi194:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-10-09T04:20:16.743 INFO:teuthology.orchestra.run.smithi194.stdout:Reading package lists... 2023-10-09T04:20:16.944 INFO:teuthology.orchestra.run.smithi194.stdout:Building dependency tree... 2023-10-09T04:20:16.944 INFO:teuthology.orchestra.run.smithi194.stdout:Reading state information... 2023-10-09T04:20:16.956 INFO:teuthology.orchestra.run.smithi133.stderr:Found linux image: /boot/vmlinuz-5.15.0-86-generic 2023-10-09T04:20:16.968 INFO:teuthology.orchestra.run.smithi133.stderr:Found initrd image: /boot/initrd.img-5.15.0-86-generic 2023-10-09T04:20:17.162 INFO:teuthology.orchestra.run.smithi194.stdout:linux-image-generic-hwe-20.04 is already the newest version (5.15.0.86.96~20.04.44). 2023-10-09T04:20:17.162 INFO:teuthology.orchestra.run.smithi194.stdout:0 upgraded, 0 newly installed, 0 to remove and 274 not upgraded. 2023-10-09T04:20:17.164 DEBUG:teuthology.orchestra.run.smithi194:> dpkg -s linux-image-generic-hwe-20.04 2023-10-09T04:20:17.186 INFO:teuthology.orchestra.run.smithi194.stdout:Package: linux-image-generic-hwe-20.04 2023-10-09T04:20:17.187 INFO:teuthology.orchestra.run.smithi194.stdout:Status: install ok installed 2023-10-09T04:20:17.187 INFO:teuthology.orchestra.run.smithi194.stdout:Priority: optional 2023-10-09T04:20:17.187 INFO:teuthology.orchestra.run.smithi194.stdout:Section: kernel 2023-10-09T04:20:17.187 INFO:teuthology.orchestra.run.smithi194.stdout:Installed-Size: 20 2023-10-09T04:20:17.187 INFO:teuthology.orchestra.run.smithi194.stdout:Maintainer: Ubuntu Kernel Team 2023-10-09T04:20:17.187 INFO:teuthology.orchestra.run.smithi194.stdout:Architecture: amd64 2023-10-09T04:20:17.187 INFO:teuthology.orchestra.run.smithi194.stdout:Source: linux-meta-hwe-5.15 2023-10-09T04:20:17.187 INFO:teuthology.orchestra.run.smithi194.stdout:Version: 5.15.0.86.96~20.04.44 2023-10-09T04:20:17.187 INFO:teuthology.orchestra.run.smithi194.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-86), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-10-09T04:20:17.187 INFO:teuthology.orchestra.run.smithi194.stdout:Depends: linux-image-5.15.0-86-generic, linux-modules-extra-5.15.0-86-generic, linux-firmware, intel-microcode, amd64-microcode 2023-10-09T04:20:17.187 INFO:teuthology.orchestra.run.smithi194.stdout:Recommends: thermald 2023-10-09T04:20:17.188 INFO:teuthology.orchestra.run.smithi194.stdout:Description: Generic Linux kernel image 2023-10-09T04:20:17.188 INFO:teuthology.orchestra.run.smithi194.stdout: This package will always depend on the latest generic kernel image 2023-10-09T04:20:17.188 INFO:teuthology.orchestra.run.smithi194.stdout: available. 2023-10-09T04:20:17.188 DEBUG:teuthology.orchestra.run.smithi194:> mktemp 2023-10-09T04:20:17.192 INFO:teuthology.orchestra.run.smithi194.stdout:/tmp/tmp.ymvet4T5pK 2023-10-09T04:20:17.192 DEBUG:teuthology.orchestra.run.smithi194:> sudo cp /boot/grub/grub.cfg /tmp/tmp.ymvet4T5pK 2023-10-09T04:20:17.251 DEBUG:teuthology.orchestra.run.smithi194:> sudo chmod 0666 /tmp/tmp.ymvet4T5pK 2023-10-09T04:20:17.291 DEBUG:teuthology.orchestra.remote:smithi194:/tmp/tmp.ymvet4T5pK is 9KB 2023-10-09T04:20:17.304 DEBUG:teuthology.orchestra.run.smithi194:> rm -fr /tmp/tmp.ymvet4T5pK 2023-10-09T04:20:17.311 DEBUG:teuthology.orchestra.run.smithi194:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2023-10-09T04:20:17.359 INFO:teuthology.orchestra.run.smithi133.stderr:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-10-09T04:20:17.369 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2023-10-09T04:20:17.370 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd of=/etc/grub.d/01_ceph_kernel 2023-10-09T04:20:17.370 DEBUG:teuthology.orchestra.run.smithi194:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2023-10-09T04:20:17.370 INFO:teuthology.orchestra.run.smithi133.stderr:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-10-09T04:20:17.401 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-86-generic 2023-10-09T04:20:17.401 DEBUG:teuthology.orchestra.run.smithi194:> sudo update-grub 2023-10-09T04:20:18.356 INFO:teuthology.orchestra.run.smithi133.stderr:done 2023-10-09T04:20:18.357 DEBUG:teuthology.orchestra.run.smithi133:> sudo shutdown -r now 2023-10-09T04:20:18.757 INFO:teuthology.orchestra.run.smithi194.stderr:Sourcing file `/etc/default/grub' 2023-10-09T04:20:18.760 INFO:teuthology.orchestra.run.smithi194.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-10-09T04:20:18.775 INFO:teuthology.orchestra.run.smithi194.stderr:Generating grub configuration file ... 2023-10-09T04:20:19.081 INFO:teuthology.orchestra.run.smithi194.stderr:Found linux image: /boot/vmlinuz-5.15.0-86-generic 2023-10-09T04:20:19.093 INFO:teuthology.orchestra.run.smithi194.stderr:Found initrd image: /boot/initrd.img-5.15.0-86-generic 2023-10-09T04:20:19.549 INFO:teuthology.orchestra.run.smithi194.stderr:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-10-09T04:20:19.561 INFO:teuthology.orchestra.run.smithi194.stderr:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-10-09T04:20:20.552 INFO:teuthology.orchestra.run.smithi194.stderr:done 2023-10-09T04:20:20.553 DEBUG:teuthology.orchestra.run.smithi194:> sudo shutdown -r now 2023-10-09T04:20:48.382 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2023-10-09T04:20:48.383 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi133.front.sepia.ceph.com' 2023-10-09T04:20:48.383 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi133.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T04:20:50.560 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2023-10-09T04:20:50.561 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi194.front.sepia.ceph.com' 2023-10-09T04:20:50.561 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi194.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T04:21:48.416 DEBUG:teuthology.orchestra.remote:timed out 2023-10-09T04:21:50.566 DEBUG:teuthology.orchestra.remote:timed out 2023-10-09T04:21:57.422 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi133.front.sepia.ceph.com' 2023-10-09T04:21:57.423 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi133.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T04:21:57.425 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.133 2023-10-09T04:21:59.569 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi194.front.sepia.ceph.com' 2023-10-09T04:21:59.569 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi194.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T04:22:00.263 DEBUG:teuthology.orchestra.run.smithi194:> true 2023-10-09T04:22:01.051 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi194.front.sepia.ceph.com' 2023-10-09T04:22:01.065 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.15.0-86-generic"... 2023-10-09T04:22:01.065 DEBUG:teuthology.orchestra.run.smithi194:> uname -r 2023-10-09T04:22:01.098 INFO:teuthology.orchestra.run.smithi194.stdout:5.15.0-86-generic 2023-10-09T04:22:02.934 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-86-generic vs 5.15.0-86-generic 2023-10-09T04:22:02.934 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-10-09T04:22:02.934 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2023-10-09T04:22:03.936 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-10-09T04:22:03.936 DEBUG:teuthology.orchestra.run.smithi194:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-09T04:22:04.050 INFO:teuthology.orchestra.run.smithi194.stdout:ttyS1 2023-10-09T04:22:04.075 DEBUG:teuthology.parallel:result is None 2023-10-09T04:22:09.426 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi133.front.sepia.ceph.com' 2023-10-09T04:22:09.427 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi133.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T04:22:09.653 DEBUG:teuthology.orchestra.run.smithi133:> true 2023-10-09T04:22:10.176 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi133.front.sepia.ceph.com' 2023-10-09T04:22:10.176 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.15.0-86-generic"... 2023-10-09T04:22:10.176 DEBUG:teuthology.orchestra.run.smithi133:> uname -r 2023-10-09T04:22:10.226 INFO:teuthology.orchestra.run.smithi133.stdout:5.15.0-86-generic 2023-10-09T04:22:10.226 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-86-generic vs 5.15.0-86-generic 2023-10-09T04:22:10.226 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-10-09T04:22:10.226 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2023-10-09T04:22:11.228 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-10-09T04:22:11.228 DEBUG:teuthology.orchestra.run.smithi133:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-09T04:22:11.603 INFO:teuthology.orchestra.run.smithi133.stdout:ttyS1 2023-10-09T04:22:11.642 DEBUG:teuthology.parallel:result is None 2023-10-09T04:22:11.643 INFO:teuthology.run_tasks:Running task internal.base... 2023-10-09T04:22:11.764 INFO:teuthology.task.internal:Creating test directory... 2023-10-09T04:22:11.764 DEBUG:teuthology.orchestra.run.smithi133:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-09T04:22:11.767 DEBUG:teuthology.orchestra.run.smithi194:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-09T04:22:11.775 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-10-09T04:22:11.868 INFO:teuthology.run_tasks:Running task internal.archive... 2023-10-09T04:22:11.938 INFO:teuthology.task.internal:Creating archive directory... 2023-10-09T04:22:11.938 DEBUG:teuthology.orchestra.run.smithi133:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-09T04:22:11.941 DEBUG:teuthology.orchestra.run.smithi194:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-09T04:22:12.043 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-10-09T04:22:12.055 INFO:teuthology.task.internal:Enabling coredump saving... 2023-10-09T04:22:12.055 DEBUG:teuthology.orchestra.run.smithi133:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-09T04:22:12.057 DEBUG:teuthology.orchestra.run.smithi194:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-09T04:22:12.073 INFO:teuthology.orchestra.run.smithi133.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T04:22:12.076 INFO:teuthology.orchestra.run.smithi194.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T04:22:12.079 INFO:teuthology.orchestra.run.smithi133.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T04:22:12.083 INFO:teuthology.orchestra.run.smithi194.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T04:22:12.085 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-10-09T04:22:12.093 INFO:teuthology.task.internal:Configuring sudo... 2023-10-09T04:22:12.093 DEBUG:teuthology.orchestra.run.smithi133:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-09T04:22:12.126 DEBUG:teuthology.orchestra.run.smithi194:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-09T04:22:12.142 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-10-09T04:22:12.152 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-10-09T04:22:12.153 DEBUG:teuthology.orchestra.run.smithi133:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-09T04:22:12.181 DEBUG:teuthology.orchestra.run.smithi194:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-09T04:22:12.190 DEBUG:teuthology.orchestra.run.smithi133:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T04:22:12.235 DEBUG:teuthology.orchestra.run.smithi133:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T04:22:12.282 DEBUG:teuthology.orchestra.run.smithi133:> set -ex 2023-10-09T04:22:12.283 DEBUG:teuthology.orchestra.run.smithi133:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-09T04:22:12.350 DEBUG:teuthology.orchestra.run.smithi194:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T04:22:12.357 DEBUG:teuthology.orchestra.run.smithi194:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T04:22:12.406 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2023-10-09T04:22:12.406 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-09T04:22:12.480 DEBUG:teuthology.orchestra.run.smithi133:> sudo service rsyslog restart 2023-10-09T04:22:12.483 DEBUG:teuthology.orchestra.run.smithi194:> sudo service rsyslog restart 2023-10-09T04:22:12.568 INFO:teuthology.run_tasks:Running task internal.timer... 2023-10-09T04:22:12.575 INFO:teuthology.task.internal:Starting timer... 2023-10-09T04:22:12.575 INFO:teuthology.run_tasks:Running task pcp... 2023-10-09T04:22:12.587 INFO:teuthology.run_tasks:Running task selinux... 2023-10-09T04:22:12.598 DEBUG:teuthology.task.selinux:Excluding smithi133: OS 'ubuntu' does not support SELinux 2023-10-09T04:22:12.598 DEBUG:teuthology.task.selinux:Excluding smithi194: OS 'ubuntu' does not support SELinux 2023-10-09T04:22:12.598 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-10-09T04:22:12.598 DEBUG:teuthology.task.selinux:Existing SELinux modes: {} 2023-10-09T04:22:12.598 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-10-09T04:22:12.598 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-10-09T04:22:12.609 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2023-10-09T04:22:12.615 INFO:teuthology.repo_utils:Fetching main from origin 2023-10-09T04:22:12.759 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-10-09T04:22:12.774 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-10-09T04:22:12.775 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi133.front.sepia.ceph.com,smithi194.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-10-09T04:27:50.134 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi133.front.sepia.ceph.com'), Remote(name='ubuntu@smithi194.front.sepia.ceph.com')] 2023-10-09T04:27:50.135 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi133.front.sepia.ceph.com' 2023-10-09T04:27:50.136 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi133.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T04:27:50.218 DEBUG:teuthology.orchestra.run.smithi133:> true 2023-10-09T04:27:50.336 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi133.front.sepia.ceph.com' 2023-10-09T04:27:50.337 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi194.front.sepia.ceph.com' 2023-10-09T04:27:50.337 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi194.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T04:27:50.426 DEBUG:teuthology.orchestra.run.smithi194:> true 2023-10-09T04:27:50.544 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi194.front.sepia.ceph.com' 2023-10-09T04:27:50.545 INFO:teuthology.run_tasks:Running task clock... 2023-10-09T04:27:50.555 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-10-09T04:27:50.555 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-09T04:27:50.556 DEBUG:teuthology.orchestra.run.smithi133:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T04:27:50.558 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-09T04:27:50.558 DEBUG:teuthology.orchestra.run.smithi194:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T04:27:50.590 INFO:teuthology.orchestra.run.smithi133.stdout: 9 Oct 04:27:50 ntpd[22714]: ntpd 4.2.8p12@1.3728-o (1): Starting 2023-10-09T04:27:50.590 INFO:teuthology.orchestra.run.smithi133.stdout: 9 Oct 04:27:50 ntpd[22714]: Command line: ntpd -gq 2023-10-09T04:27:50.591 INFO:teuthology.orchestra.run.smithi133.stdout: 9 Oct 04:27:50 ntpd[22714]: proto: precision = 0.046 usec (-24) 2023-10-09T04:27:50.591 INFO:teuthology.orchestra.run.smithi133.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-10-09T04:27:50.591 INFO:teuthology.orchestra.run.smithi133.stdout: 9 Oct 04:27:50 ntpd[22714]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-10-09T04:27:50.592 INFO:teuthology.orchestra.run.smithi133.stderr:restrict ::: KOD does nothing without LIMITED. 2023-10-09T04:27:50.593 INFO:teuthology.orchestra.run.smithi133.stdout: 9 Oct 04:27:50 ntpd[22714]: restrict ::: KOD does nothing without LIMITED. 2023-10-09T04:27:50.593 INFO:teuthology.orchestra.run.smithi133.stdout: 9 Oct 04:27:50 ntpd[22714]: Listen and drop on 0 v6wildcard [::]:123 2023-10-09T04:27:50.593 INFO:teuthology.orchestra.run.smithi133.stdout: 9 Oct 04:27:50 ntpd[22714]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2023-10-09T04:27:50.593 INFO:teuthology.orchestra.run.smithi133.stdout: 9 Oct 04:27:50 ntpd[22714]: Listen normally on 2 lo 127.0.0.1:123 2023-10-09T04:27:50.593 INFO:teuthology.orchestra.run.smithi133.stdout: 9 Oct 04:27:50 ntpd[22714]: Listen normally on 3 enp3s0f1 172.21.15.133:123 2023-10-09T04:27:50.593 INFO:teuthology.orchestra.run.smithi133.stdout: 9 Oct 04:27:50 ntpd[22714]: Listen normally on 4 lo [::1]:123 2023-10-09T04:27:50.593 INFO:teuthology.orchestra.run.smithi133.stdout: 9 Oct 04:27:50 ntpd[22714]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe88:8087%5]:123 2023-10-09T04:27:50.593 INFO:teuthology.orchestra.run.smithi133.stdout: 9 Oct 04:27:50 ntpd[22714]: Listening on routing socket on fd #22 for interface updates 2023-10-09T04:27:50.616 INFO:teuthology.orchestra.run.smithi194.stdout: 9 Oct 04:27:50 ntpd[22720]: ntpd 4.2.8p12@1.3728-o (1): Starting 2023-10-09T04:27:50.616 INFO:teuthology.orchestra.run.smithi194.stdout: 9 Oct 04:27:50 ntpd[22720]: Command line: ntpd -gq 2023-10-09T04:27:50.616 INFO:teuthology.orchestra.run.smithi194.stdout: 9 Oct 04:27:50 ntpd[22720]: proto: precision = 0.088 usec (-23) 2023-10-09T04:27:50.617 INFO:teuthology.orchestra.run.smithi194.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-10-09T04:27:50.617 INFO:teuthology.orchestra.run.smithi194.stdout: 9 Oct 04:27:50 ntpd[22720]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-10-09T04:27:50.617 INFO:teuthology.orchestra.run.smithi194.stderr:restrict ::: KOD does nothing without LIMITED. 2023-10-09T04:27:50.618 INFO:teuthology.orchestra.run.smithi194.stdout: 9 Oct 04:27:50 ntpd[22720]: restrict ::: KOD does nothing without LIMITED. 2023-10-09T04:27:50.619 INFO:teuthology.orchestra.run.smithi194.stdout: 9 Oct 04:27:50 ntpd[22720]: Listen and drop on 0 v6wildcard [::]:123 2023-10-09T04:27:50.619 INFO:teuthology.orchestra.run.smithi194.stdout: 9 Oct 04:27:50 ntpd[22720]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2023-10-09T04:27:50.619 INFO:teuthology.orchestra.run.smithi194.stdout: 9 Oct 04:27:50 ntpd[22720]: Listen normally on 2 lo 127.0.0.1:123 2023-10-09T04:27:50.619 INFO:teuthology.orchestra.run.smithi194.stdout: 9 Oct 04:27:50 ntpd[22720]: Listen normally on 3 enp3s0f1 172.21.15.194:123 2023-10-09T04:27:50.619 INFO:teuthology.orchestra.run.smithi194.stdout: 9 Oct 04:27:50 ntpd[22720]: Listen normally on 4 lo [::1]:123 2023-10-09T04:27:50.619 INFO:teuthology.orchestra.run.smithi194.stdout: 9 Oct 04:27:50 ntpd[22720]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe88:7ea5%5]:123 2023-10-09T04:27:50.619 INFO:teuthology.orchestra.run.smithi194.stdout: 9 Oct 04:27:50 ntpd[22720]: Listening on routing socket on fd #22 for interface updates 2023-10-09T04:27:51.617 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:51 ntpd[22720]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:51.617 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:51 ntpd[22720]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:51.617 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:51 ntpd[22720]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:51.617 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:51 ntpd[22720]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:51.617 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:51 ntpd[22720]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:51.617 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:51 ntpd[22720]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:51.617 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:51 ntpd[22720]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T04:27:51.617 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:51 ntpd[22720]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T04:27:53.591 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:53 ntpd[22714]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:53.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:53 ntpd[22714]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:53.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:53 ntpd[22714]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:53.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:53 ntpd[22714]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:53.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:53 ntpd[22714]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:53.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:53 ntpd[22714]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:53.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:53 ntpd[22714]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T04:27:53.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:53 ntpd[22714]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T04:27:53.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:53 ntpd[22714]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:53.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:53 ntpd[22714]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:53.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:53 ntpd[22714]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T04:27:53.593 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:53 ntpd[22714]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T04:27:53.616 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:53 ntpd[22720]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:53.617 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:53 ntpd[22720]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:53.617 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:53 ntpd[22720]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:53.617 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:53 ntpd[22720]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:53.617 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:53 ntpd[22720]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T04:27:53.617 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:53 ntpd[22720]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T04:27:54.591 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:54 ntpd[22714]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:54.591 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:54 ntpd[22714]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:54.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:54 ntpd[22714]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:54.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:54 ntpd[22714]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:54.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:54 ntpd[22714]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:54.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:54 ntpd[22714]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:54.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:54 ntpd[22714]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T04:27:54.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:54 ntpd[22714]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T04:27:54.616 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:54 ntpd[22720]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:54.616 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:54 ntpd[22720]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:54.617 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:54 ntpd[22720]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:54.617 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:54 ntpd[22720]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:54.617 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:54 ntpd[22720]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T04:27:54.617 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:54 ntpd[22720]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T04:27:55.591 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:55 ntpd[22714]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:55.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:55 ntpd[22714]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:55.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:55 ntpd[22714]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:55.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:55 ntpd[22714]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:55.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:55 ntpd[22714]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:55.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:55 ntpd[22714]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:55.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:55 ntpd[22714]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T04:27:55.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:55 ntpd[22714]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T04:27:55.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:55 ntpd[22714]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:55.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:55 ntpd[22714]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:55.593 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:55 ntpd[22714]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T04:27:55.593 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:55 ntpd[22714]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T04:27:55.617 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:55 ntpd[22720]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:55.617 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:55 ntpd[22720]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:55.617 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:55 ntpd[22720]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:55.617 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:55 ntpd[22720]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:55.617 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:55 ntpd[22720]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:55.617 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:55 ntpd[22720]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:55.617 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:55 ntpd[22720]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T04:27:55.618 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:55 ntpd[22720]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T04:27:55.618 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:55 ntpd[22720]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:55.618 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:55 ntpd[22720]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:55.618 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:55 ntpd[22720]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T04:27:55.618 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:55 ntpd[22720]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T04:27:56.591 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:56 ntpd[22714]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:56.591 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:56 ntpd[22714]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:56.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:56 ntpd[22714]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:56.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:56 ntpd[22714]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:56.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:56 ntpd[22714]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T04:27:56.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:56 ntpd[22714]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T04:27:56.618 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:56 ntpd[22720]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:56.618 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:56 ntpd[22720]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:56.618 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:56 ntpd[22720]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:56.618 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:56 ntpd[22720]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:56.619 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:56 ntpd[22720]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T04:27:56.619 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:56 ntpd[22720]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T04:27:57.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:57 ntpd[22714]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:57.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:57 ntpd[22714]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:57.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:57 ntpd[22714]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:57.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:57 ntpd[22714]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:57.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:57 ntpd[22714]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:57.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:57 ntpd[22714]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:57.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:57 ntpd[22714]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T04:27:57.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:57 ntpd[22714]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T04:27:57.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:57 ntpd[22714]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:57.593 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:57 ntpd[22714]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:57.593 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:57 ntpd[22714]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T04:27:57.593 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:57 ntpd[22714]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T04:27:57.617 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:57 ntpd[22720]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:57.617 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:57 ntpd[22720]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:57.617 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:57 ntpd[22720]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:57.617 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:57 ntpd[22720]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:57.618 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:57 ntpd[22720]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:57.618 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:57 ntpd[22720]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:57.618 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:57 ntpd[22720]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T04:27:57.618 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:57 ntpd[22720]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T04:27:57.618 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:57 ntpd[22720]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:57.618 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:57 ntpd[22720]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:57.618 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:57 ntpd[22720]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T04:27:57.618 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:57 ntpd[22720]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T04:27:57.618 INFO:teuthology.orchestra.run.smithi194.stdout: 9 Oct 04:27:57 ntpd[22720]: ntpd: time slew -0.000156 s 2023-10-09T04:27:57.619 INFO:teuthology.orchestra.run.smithi194.stdout:ntpd: time slew -0.000156s 2023-10-09T04:27:57.619 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:57 ntpd[22720]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2023-10-09T04:27:57.619 INFO:teuthology.orchestra.run.smithi194.stderr: 9 Oct 04:27:57 ntpd[22720]: can't open /var/log/ntpstats/loopstats.20231009: Permission denied 2023-10-09T04:27:57.689 INFO:teuthology.orchestra.run.smithi194.stdout: remote refid st t when poll reach delay offset jitter 2023-10-09T04:27:57.689 INFO:teuthology.orchestra.run.smithi194.stdout:============================================================================== 2023-10-09T04:27:57.689 INFO:teuthology.orchestra.run.smithi194.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T04:27:57.689 INFO:teuthology.orchestra.run.smithi194.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T04:27:57.689 INFO:teuthology.orchestra.run.smithi194.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T04:27:57.689 INFO:teuthology.orchestra.run.smithi194.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T04:27:58.591 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:58 ntpd[22714]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:58.591 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:58 ntpd[22714]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:58.591 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:58 ntpd[22714]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:58.591 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:58 ntpd[22714]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:58.591 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:58 ntpd[22714]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T04:27:58.591 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:58 ntpd[22714]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T04:27:59.591 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:59 ntpd[22714]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:59.591 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:59 ntpd[22714]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:59.591 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:59 ntpd[22714]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:59.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:59 ntpd[22714]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:59.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:59 ntpd[22714]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T04:27:59.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:59 ntpd[22714]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T04:27:59.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:59 ntpd[22714]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T04:27:59.592 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:59 ntpd[22714]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T04:27:59.592 INFO:teuthology.orchestra.run.smithi133.stdout: 9 Oct 04:27:59 ntpd[22714]: ntpd: time slew +0.000968 s 2023-10-09T04:27:59.592 INFO:teuthology.orchestra.run.smithi133.stdout:ntpd: time slew +0.000968s 2023-10-09T04:27:59.593 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:59 ntpd[22714]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2023-10-09T04:27:59.593 INFO:teuthology.orchestra.run.smithi133.stderr: 9 Oct 04:27:59 ntpd[22714]: can't open /var/log/ntpstats/loopstats.20231009: Permission denied 2023-10-09T04:27:59.655 INFO:teuthology.orchestra.run.smithi133.stdout: remote refid st t when poll reach delay offset jitter 2023-10-09T04:27:59.655 INFO:teuthology.orchestra.run.smithi133.stdout:============================================================================== 2023-10-09T04:27:59.655 INFO:teuthology.orchestra.run.smithi133.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T04:27:59.655 INFO:teuthology.orchestra.run.smithi133.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T04:27:59.656 INFO:teuthology.orchestra.run.smithi133.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T04:27:59.656 INFO:teuthology.orchestra.run.smithi133.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T04:27:59.656 INFO:teuthology.run_tasks:Running task install... 2023-10-09T04:27:59.665 DEBUG:teuthology.task.install:project ceph 2023-10-09T04:27:59.665 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}} 2023-10-09T04:27:59.665 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'} 2023-10-09T04:27:59.665 INFO:teuthology.task.install:Using flavor: default 2023-10-09T04:27:59.670 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-10-09T04:27:59.671 INFO:teuthology.task.install:extra packages: [] 2023-10-09T04:27:59.671 DEBUG:teuthology.orchestra.run.smithi133:> sudo apt-key list | grep Ceph 2023-10-09T04:27:59.671 DEBUG:teuthology.orchestra.run.smithi194:> sudo apt-key list | grep Ceph 2023-10-09T04:27:59.778 INFO:teuthology.orchestra.run.smithi194.stderr:Warning: apt-key output should not be parsed (stdout is not a terminal) 2023-10-09T04:27:59.795 INFO:teuthology.orchestra.run.smithi133.stderr:Warning: apt-key output should not be parsed (stdout is not a terminal) 2023-10-09T04:27:59.835 INFO:teuthology.orchestra.run.smithi194.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2023-10-09T04:27:59.835 INFO:teuthology.orchestra.run.smithi194.stdout:uid [ unknown] Ceph.com (release key) 2023-10-09T04:27:59.836 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2023-10-09T04:27:59.836 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T04:27:59.837 INFO:teuthology.packaging:ref: None 2023-10-09T04:27:59.837 INFO:teuthology.packaging:tag: None 2023-10-09T04:27:59.837 INFO:teuthology.packaging:branch: quincy 2023-10-09T04:27:59.837 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T04:27:59.837 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=quincy 2023-10-09T04:27:59.852 INFO:teuthology.orchestra.run.smithi133.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2023-10-09T04:27:59.852 INFO:teuthology.orchestra.run.smithi133.stdout:uid [ unknown] Ceph.com (release key) 2023-10-09T04:27:59.853 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2023-10-09T04:27:59.853 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T04:27:59.853 INFO:teuthology.packaging:ref: None 2023-10-09T04:27:59.853 INFO:teuthology.packaging:tag: None 2023-10-09T04:27:59.853 INFO:teuthology.packaging:branch: quincy 2023-10-09T04:27:59.853 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T04:27:59.853 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=quincy 2023-10-09T04:28:00.118 INFO:teuthology.task.install.deb:Pulling from https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default/ 2023-10-09T04:28:00.119 INFO:teuthology.task.install.deb:Package version is 17.2.6-1303-gb2903cb0-1focal 2023-10-09T04:28:00.154 INFO:teuthology.task.install.deb:Pulling from https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default/ 2023-10-09T04:28:00.155 INFO:teuthology.task.install.deb:Package version is 17.2.6-1303-gb2903cb0-1focal 2023-10-09T04:28:00.279 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2023-10-09T04:28:00.280 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2023-10-09T04:28:00.294 DEBUG:teuthology.orchestra.run.smithi194:> sudo apt-get update 2023-10-09T04:28:00.424 DEBUG:teuthology.orchestra.run.smithi133:> set -ex 2023-10-09T04:28:00.425 DEBUG:teuthology.orchestra.run.smithi133:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2023-10-09T04:28:00.441 DEBUG:teuthology.orchestra.run.smithi133:> sudo apt-get update 2023-10-09T04:28:00.535 INFO:teuthology.orchestra.run.smithi194.stdout:Ign:1 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal InRelease 2023-10-09T04:28:00.556 INFO:teuthology.orchestra.run.smithi194.stdout:Get:2 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal Release [7,662 B] 2023-10-09T04:28:00.580 INFO:teuthology.orchestra.run.smithi194.stdout:Ign:3 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal Release.gpg 2023-10-09T04:28:00.602 INFO:teuthology.orchestra.run.smithi194.stdout:Get:4 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 Packages [17.4 kB] 2023-10-09T04:28:00.643 INFO:teuthology.orchestra.run.smithi194.stdout:Hit:5 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-09T04:28:00.643 INFO:teuthology.orchestra.run.smithi194.stdout:Hit:6 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-10-09T04:28:00.643 INFO:teuthology.orchestra.run.smithi194.stdout:Get:7 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main i386 Packages [3,367 B] 2023-10-09T04:28:00.697 INFO:teuthology.orchestra.run.smithi133.stdout:Hit:1 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-10-09T04:28:00.713 INFO:teuthology.orchestra.run.smithi194.stdout:Hit:8 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-10-09T04:28:00.794 INFO:teuthology.orchestra.run.smithi194.stdout:Hit:9 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-10-09T04:28:00.889 INFO:teuthology.orchestra.run.smithi133.stdout:Ign:2 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal InRelease 2023-10-09T04:28:00.956 INFO:teuthology.orchestra.run.smithi133.stdout:Get:3 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal Release [7,662 B] 2023-10-09T04:28:01.006 INFO:teuthology.orchestra.run.smithi133.stdout:Ign:4 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal Release.gpg 2023-10-09T04:28:01.046 INFO:teuthology.orchestra.run.smithi133.stdout:Get:5 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main i386 Packages [3,367 B] 2023-10-09T04:28:01.071 INFO:teuthology.orchestra.run.smithi133.stdout:Hit:6 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-09T04:28:01.071 INFO:teuthology.orchestra.run.smithi133.stdout:Hit:7 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-10-09T04:28:01.087 INFO:teuthology.orchestra.run.smithi133.stdout:Hit:8 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-10-09T04:28:01.107 INFO:teuthology.orchestra.run.smithi133.stdout:Get:9 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 Packages [17.4 kB] 2023-10-09T04:28:01.280 INFO:teuthology.orchestra.run.smithi194.stdout:Fetched 28.4 kB in 1s (36.6 kB/s) 2023-10-09T04:28:01.566 INFO:teuthology.orchestra.run.smithi133.stdout:Fetched 28.4 kB in 1s (30.9 kB/s) 2023-10-09T04:28:02.335 INFO:teuthology.orchestra.run.smithi194.stdout:Reading package lists... 2023-10-09T04:28:02.361 DEBUG:teuthology.orchestra.run.smithi194:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=17.2.6-1303-gb2903cb0-1focal cephadm=17.2.6-1303-gb2903cb0-1focal ceph-mds=17.2.6-1303-gb2903cb0-1focal ceph-mgr=17.2.6-1303-gb2903cb0-1focal ceph-common=17.2.6-1303-gb2903cb0-1focal ceph-fuse=17.2.6-1303-gb2903cb0-1focal ceph-test=17.2.6-1303-gb2903cb0-1focal radosgw=17.2.6-1303-gb2903cb0-1focal python3-rados=17.2.6-1303-gb2903cb0-1focal python3-rgw=17.2.6-1303-gb2903cb0-1focal python3-cephfs=17.2.6-1303-gb2903cb0-1focal python3-rbd=17.2.6-1303-gb2903cb0-1focal libcephfs2=17.2.6-1303-gb2903cb0-1focal libcephfs-dev=17.2.6-1303-gb2903cb0-1focal librados2=17.2.6-1303-gb2903cb0-1focal librbd1=17.2.6-1303-gb2903cb0-1focal rbd-fuse=17.2.6-1303-gb2903cb0-1focal 2023-10-09T04:28:02.438 INFO:teuthology.orchestra.run.smithi194.stdout:Reading package lists... 2023-10-09T04:28:02.616 INFO:teuthology.orchestra.run.smithi133.stdout:Reading package lists... 2023-10-09T04:28:02.641 INFO:teuthology.orchestra.run.smithi194.stdout:Building dependency tree... 2023-10-09T04:28:02.641 INFO:teuthology.orchestra.run.smithi194.stdout:Reading state information... 2023-10-09T04:28:02.644 DEBUG:teuthology.orchestra.run.smithi133:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=17.2.6-1303-gb2903cb0-1focal cephadm=17.2.6-1303-gb2903cb0-1focal ceph-mds=17.2.6-1303-gb2903cb0-1focal ceph-mgr=17.2.6-1303-gb2903cb0-1focal ceph-common=17.2.6-1303-gb2903cb0-1focal ceph-fuse=17.2.6-1303-gb2903cb0-1focal ceph-test=17.2.6-1303-gb2903cb0-1focal radosgw=17.2.6-1303-gb2903cb0-1focal python3-rados=17.2.6-1303-gb2903cb0-1focal python3-rgw=17.2.6-1303-gb2903cb0-1focal python3-cephfs=17.2.6-1303-gb2903cb0-1focal python3-rbd=17.2.6-1303-gb2903cb0-1focal libcephfs2=17.2.6-1303-gb2903cb0-1focal libcephfs-dev=17.2.6-1303-gb2903cb0-1focal librados2=17.2.6-1303-gb2903cb0-1focal librbd1=17.2.6-1303-gb2903cb0-1focal rbd-fuse=17.2.6-1303-gb2903cb0-1focal 2023-10-09T04:28:02.716 INFO:teuthology.orchestra.run.smithi133.stdout:Reading package lists... 2023-10-09T04:28:02.820 INFO:teuthology.orchestra.run.smithi194.stdout:The following packages were automatically installed and are no longer required: 2023-10-09T04:28:02.822 INFO:teuthology.orchestra.run.smithi194.stdout: libboost-iostreams1.71.0 libboost-thread1.71.0 2023-10-09T04:28:02.822 INFO:teuthology.orchestra.run.smithi194.stdout:Use 'sudo apt autoremove' to remove them. 2023-10-09T04:28:02.822 INFO:teuthology.orchestra.run.smithi194.stdout:The following additional packages will be installed: 2023-10-09T04:28:02.822 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2023-10-09T04:28:02.822 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd ceph-volume 2023-10-09T04:28:02.823 INFO:teuthology.orchestra.run.smithi194.stdout: fonts-mathjax jq libdouble-conversion3 libjq1 libjs-mathjax libjs-sphinxdoc 2023-10-09T04:28:02.823 INFO:teuthology.orchestra.run.smithi194.stdout: libjs-underscore liblbfgsb0 liblttng-ust0 liblua5.3-0 liboath0 libonig5 2023-10-09T04:28:02.823 INFO:teuthology.orchestra.run.smithi194.stdout: libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 librabbitmq4 2023-10-09T04:28:02.824 INFO:teuthology.orchestra.run.smithi194.stdout: libradosstriper1 librdkafka1 librgw2 libsqlite3-mod-ceph libthrift-0.13.0 2023-10-09T04:28:02.824 INFO:teuthology.orchestra.run.smithi194.stdout: nvme-cli python-natsort-doc python-pastedeploy-tpl python3-atomicwrites 2023-10-09T04:28:02.824 INFO:teuthology.orchestra.run.smithi194.stdout: python3-attr python3-bcrypt python3-blinker python3-bs4 python3-cachetools 2023-10-09T04:28:02.824 INFO:teuthology.orchestra.run.smithi194.stdout: python3-ceph-argparse python3-ceph-common python3-cherrypy3 2023-10-09T04:28:02.824 INFO:teuthology.orchestra.run.smithi194.stdout: python3-cryptography python3-decorator python3-google-auth python3-html5lib 2023-10-09T04:28:02.824 INFO:teuthology.orchestra.run.smithi194.stdout: python3-jinja2 python3-joblib python3-jwt python3-kubernetes 2023-10-09T04:28:02.824 INFO:teuthology.orchestra.run.smithi194.stdout: python3-logutils python3-lxml python3-mako python3-markupsafe 2023-10-09T04:28:02.824 INFO:teuthology.orchestra.run.smithi194.stdout: python3-natsort python3-oauthlib python3-openssl python3-packaging 2023-10-09T04:28:02.824 INFO:teuthology.orchestra.run.smithi194.stdout: python3-paste python3-pastedeploy python3-pastescript python3-pecan 2023-10-09T04:28:02.824 INFO:teuthology.orchestra.run.smithi194.stdout: python3-pluggy python3-prettytable python3-psutil python3-py python3-pyasn1 2023-10-09T04:28:02.824 INFO:teuthology.orchestra.run.smithi194.stdout: python3-pyasn1-modules python3-pyinotify python3-pytest python3-repoze.lru 2023-10-09T04:28:02.824 INFO:teuthology.orchestra.run.smithi194.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-scipy 2023-10-09T04:28:02.825 INFO:teuthology.orchestra.run.smithi194.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2023-10-09T04:28:02.825 INFO:teuthology.orchestra.run.smithi194.stdout: python3-sklearn python3-sklearn-lib python3-soupsieve python3-tempita 2023-10-09T04:28:02.825 INFO:teuthology.orchestra.run.smithi194.stdout: python3-waitress python3-wcwidth python3-webencodings python3-webob 2023-10-09T04:28:02.825 INFO:teuthology.orchestra.run.smithi194.stdout: python3-websocket python3-webtest python3-werkzeug qttranslations5-l10n 2023-10-09T04:28:02.825 INFO:teuthology.orchestra.run.smithi194.stdout: socat xmlstarlet 2023-10-09T04:28:02.825 INFO:teuthology.orchestra.run.smithi194.stdout:Suggested packages: 2023-10-09T04:28:02.825 INFO:teuthology.orchestra.run.smithi194.stdout: python3-influxdb fonts-mathjax-extras fonts-stix libjs-mathjax-doc 2023-10-09T04:28:02.825 INFO:teuthology.orchestra.run.smithi194.stdout: python-attr-doc python-blinker-doc python-cryptography-doc 2023-10-09T04:28:02.826 INFO:teuthology.orchestra.run.smithi194.stdout: python3-cryptography-vectors python3-genshi python-jinja2-doc python3-crypto 2023-10-09T04:28:02.826 INFO:teuthology.orchestra.run.smithi194.stdout: python3-lxml-dbg python-lxml-doc python3-beaker python-mako-doc 2023-10-09T04:28:02.826 INFO:teuthology.orchestra.run.smithi194.stdout: python-openssl-doc python3-openssl-dbg httpd-wsgi libapache2-mod-python 2023-10-09T04:28:02.826 INFO:teuthology.orchestra.run.smithi194.stdout: libapache2-mod-scgi libjs-mochikit python-pecan-doc python-psutil-doc 2023-10-09T04:28:02.826 INFO:teuthology.orchestra.run.smithi194.stdout: subversion python-pyinotify-doc python-scipy-doc python3-dap 2023-10-09T04:28:02.826 INFO:teuthology.orchestra.run.smithi194.stdout: python-sklearn-doc ipython3 python-waitress-doc python-webob-doc 2023-10-09T04:28:02.826 INFO:teuthology.orchestra.run.smithi194.stdout: python-webtest-doc python-werkzeug-doc python3-termcolor python3-watchdog 2023-10-09T04:28:02.826 INFO:teuthology.orchestra.run.smithi194.stdout:Recommended packages: 2023-10-09T04:28:02.826 INFO:teuthology.orchestra.run.smithi194.stdout: btrfs-tools 2023-10-09T04:28:02.898 INFO:teuthology.orchestra.run.smithi133.stdout:Building dependency tree... 2023-10-09T04:28:02.899 INFO:teuthology.orchestra.run.smithi133.stdout:Reading state information... 2023-10-09T04:28:02.944 INFO:teuthology.orchestra.run.smithi194.stdout:The following NEW packages will be installed: 2023-10-09T04:28:02.944 INFO:teuthology.orchestra.run.smithi194.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2023-10-09T04:28:02.944 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2023-10-09T04:28:02.945 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm 2023-10-09T04:28:02.945 INFO:teuthology.orchestra.run.smithi194.stdout: fonts-mathjax jq libcephfs-dev libcephfs2 libdouble-conversion3 libjq1 2023-10-09T04:28:02.946 INFO:teuthology.orchestra.run.smithi194.stdout: libjs-mathjax libjs-sphinxdoc libjs-underscore liblbfgsb0 liblttng-ust0 2023-10-09T04:28:02.946 INFO:teuthology.orchestra.run.smithi194.stdout: liblua5.3-0 liboath0 libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 2023-10-09T04:28:02.946 INFO:teuthology.orchestra.run.smithi194.stdout: libqt5network5 librabbitmq4 libradosstriper1 librdkafka1 librgw2 2023-10-09T04:28:02.946 INFO:teuthology.orchestra.run.smithi194.stdout: libsqlite3-mod-ceph libthrift-0.13.0 nvme-cli python-natsort-doc 2023-10-09T04:28:02.946 INFO:teuthology.orchestra.run.smithi194.stdout: python-pastedeploy-tpl python3-atomicwrites python3-attr python3-bcrypt 2023-10-09T04:28:02.946 INFO:teuthology.orchestra.run.smithi194.stdout: python3-blinker python3-bs4 python3-cachetools python3-ceph-argparse 2023-10-09T04:28:02.946 INFO:teuthology.orchestra.run.smithi194.stdout: python3-ceph-common python3-cephfs python3-cherrypy3 python3-cryptography 2023-10-09T04:28:02.946 INFO:teuthology.orchestra.run.smithi194.stdout: python3-decorator python3-google-auth python3-html5lib python3-jinja2 2023-10-09T04:28:02.946 INFO:teuthology.orchestra.run.smithi194.stdout: python3-joblib python3-jwt python3-kubernetes python3-logutils python3-lxml 2023-10-09T04:28:02.946 INFO:teuthology.orchestra.run.smithi194.stdout: python3-mako python3-markupsafe python3-natsort python3-oauthlib 2023-10-09T04:28:02.946 INFO:teuthology.orchestra.run.smithi194.stdout: python3-openssl python3-packaging python3-paste python3-pastedeploy 2023-10-09T04:28:02.947 INFO:teuthology.orchestra.run.smithi194.stdout: python3-pastescript python3-pecan python3-pluggy python3-prettytable 2023-10-09T04:28:02.947 INFO:teuthology.orchestra.run.smithi194.stdout: python3-psutil python3-py python3-pyasn1 python3-pyasn1-modules 2023-10-09T04:28:02.947 INFO:teuthology.orchestra.run.smithi194.stdout: python3-pyinotify python3-pytest python3-rados python3-rbd 2023-10-09T04:28:02.947 INFO:teuthology.orchestra.run.smithi194.stdout: python3-repoze.lru python3-requests-oauthlib python3-rgw python3-routes 2023-10-09T04:28:02.947 INFO:teuthology.orchestra.run.smithi194.stdout: python3-rsa python3-scipy python3-simplegeneric python3-simplejson 2023-10-09T04:28:02.947 INFO:teuthology.orchestra.run.smithi194.stdout: python3-singledispatch python3-sklearn python3-sklearn-lib python3-soupsieve 2023-10-09T04:28:02.947 INFO:teuthology.orchestra.run.smithi194.stdout: python3-tempita python3-waitress python3-wcwidth python3-webencodings 2023-10-09T04:28:02.947 INFO:teuthology.orchestra.run.smithi194.stdout: python3-webob python3-websocket python3-webtest python3-werkzeug 2023-10-09T04:28:02.947 INFO:teuthology.orchestra.run.smithi194.stdout: qttranslations5-l10n radosgw rbd-fuse socat xmlstarlet 2023-10-09T04:28:02.948 INFO:teuthology.orchestra.run.smithi194.stdout:The following packages will be upgraded: 2023-10-09T04:28:02.948 INFO:teuthology.orchestra.run.smithi194.stdout: librados2 librbd1 2023-10-09T04:28:03.047 INFO:teuthology.orchestra.run.smithi194.stdout:2 upgraded, 108 newly installed, 0 to remove and 256 not upgraded. 2023-10-09T04:28:03.047 INFO:teuthology.orchestra.run.smithi194.stdout:Need to get 209 MB of archives. 2023-10-09T04:28:03.047 INFO:teuthology.orchestra.run.smithi194.stdout:After this operation, 980 MB of additional disk space will be used. 2023-10-09T04:28:03.047 INFO:teuthology.orchestra.run.smithi194.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 liblttng-ust0 amd64 2.11.0-1 [160 kB] 2023-10-09T04:28:03.070 INFO:teuthology.orchestra.run.smithi133.stdout:The following packages were automatically installed and are no longer required: 2023-10-09T04:28:03.071 INFO:teuthology.orchestra.run.smithi133.stdout: libboost-iostreams1.71.0 libboost-thread1.71.0 2023-10-09T04:28:03.071 INFO:teuthology.orchestra.run.smithi133.stdout:Use 'sudo apt autoremove' to remove them. 2023-10-09T04:28:03.071 INFO:teuthology.orchestra.run.smithi133.stdout:The following additional packages will be installed: 2023-10-09T04:28:03.071 INFO:teuthology.orchestra.run.smithi133.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2023-10-09T04:28:03.072 INFO:teuthology.orchestra.run.smithi133.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd ceph-volume 2023-10-09T04:28:03.073 INFO:teuthology.orchestra.run.smithi133.stdout: fonts-mathjax jq libdouble-conversion3 libjq1 libjs-mathjax libjs-sphinxdoc 2023-10-09T04:28:03.073 INFO:teuthology.orchestra.run.smithi133.stdout: libjs-underscore liblbfgsb0 liblttng-ust0 liblua5.3-0 liboath0 libonig5 2023-10-09T04:28:03.073 INFO:teuthology.orchestra.run.smithi133.stdout: libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 librabbitmq4 2023-10-09T04:28:03.073 INFO:teuthology.orchestra.run.smithi133.stdout: libradosstriper1 librdkafka1 librgw2 libsqlite3-mod-ceph libthrift-0.13.0 2023-10-09T04:28:03.073 INFO:teuthology.orchestra.run.smithi133.stdout: nvme-cli python-natsort-doc python-pastedeploy-tpl python3-atomicwrites 2023-10-09T04:28:03.073 INFO:teuthology.orchestra.run.smithi133.stdout: python3-attr python3-bcrypt python3-blinker python3-bs4 python3-cachetools 2023-10-09T04:28:03.073 INFO:teuthology.orchestra.run.smithi133.stdout: python3-ceph-argparse python3-ceph-common python3-cherrypy3 2023-10-09T04:28:03.073 INFO:teuthology.orchestra.run.smithi133.stdout: python3-cryptography python3-decorator python3-google-auth python3-html5lib 2023-10-09T04:28:03.073 INFO:teuthology.orchestra.run.smithi133.stdout: python3-jinja2 python3-joblib python3-jwt python3-kubernetes 2023-10-09T04:28:03.073 INFO:teuthology.orchestra.run.smithi133.stdout: python3-logutils python3-lxml python3-mako python3-markupsafe 2023-10-09T04:28:03.074 INFO:teuthology.orchestra.run.smithi133.stdout: python3-natsort python3-oauthlib python3-openssl python3-packaging 2023-10-09T04:28:03.074 INFO:teuthology.orchestra.run.smithi133.stdout: python3-paste python3-pastedeploy python3-pastescript python3-pecan 2023-10-09T04:28:03.074 INFO:teuthology.orchestra.run.smithi133.stdout: python3-pluggy python3-prettytable python3-psutil python3-py python3-pyasn1 2023-10-09T04:28:03.074 INFO:teuthology.orchestra.run.smithi133.stdout: python3-pyasn1-modules python3-pyinotify python3-pytest python3-repoze.lru 2023-10-09T04:28:03.074 INFO:teuthology.orchestra.run.smithi133.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-scipy 2023-10-09T04:28:03.074 INFO:teuthology.orchestra.run.smithi133.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2023-10-09T04:28:03.074 INFO:teuthology.orchestra.run.smithi133.stdout: python3-sklearn python3-sklearn-lib python3-soupsieve python3-tempita 2023-10-09T04:28:03.074 INFO:teuthology.orchestra.run.smithi133.stdout: python3-waitress python3-wcwidth python3-webencodings python3-webob 2023-10-09T04:28:03.074 INFO:teuthology.orchestra.run.smithi133.stdout: python3-websocket python3-webtest python3-werkzeug qttranslations5-l10n 2023-10-09T04:28:03.074 INFO:teuthology.orchestra.run.smithi133.stdout: socat xmlstarlet 2023-10-09T04:28:03.075 INFO:teuthology.orchestra.run.smithi133.stdout:Suggested packages: 2023-10-09T04:28:03.075 INFO:teuthology.orchestra.run.smithi133.stdout: python3-influxdb fonts-mathjax-extras fonts-stix libjs-mathjax-doc 2023-10-09T04:28:03.075 INFO:teuthology.orchestra.run.smithi133.stdout: python-attr-doc python-blinker-doc python-cryptography-doc 2023-10-09T04:28:03.075 INFO:teuthology.orchestra.run.smithi133.stdout: python3-cryptography-vectors python3-genshi python-jinja2-doc python3-crypto 2023-10-09T04:28:03.075 INFO:teuthology.orchestra.run.smithi133.stdout: python3-lxml-dbg python-lxml-doc python3-beaker python-mako-doc 2023-10-09T04:28:03.075 INFO:teuthology.orchestra.run.smithi133.stdout: python-openssl-doc python3-openssl-dbg httpd-wsgi libapache2-mod-python 2023-10-09T04:28:03.075 INFO:teuthology.orchestra.run.smithi133.stdout: libapache2-mod-scgi libjs-mochikit python-pecan-doc python-psutil-doc 2023-10-09T04:28:03.076 INFO:teuthology.orchestra.run.smithi133.stdout: subversion python-pyinotify-doc python-scipy-doc python3-dap 2023-10-09T04:28:03.076 INFO:teuthology.orchestra.run.smithi133.stdout: python-sklearn-doc ipython3 python-waitress-doc python-webob-doc 2023-10-09T04:28:03.076 INFO:teuthology.orchestra.run.smithi133.stdout: python-webtest-doc python-werkzeug-doc python3-termcolor python3-watchdog 2023-10-09T04:28:03.076 INFO:teuthology.orchestra.run.smithi133.stdout:Recommended packages: 2023-10-09T04:28:03.076 INFO:teuthology.orchestra.run.smithi133.stdout: btrfs-tools 2023-10-09T04:28:03.094 INFO:teuthology.orchestra.run.smithi194.stdout:Get:2 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 librbd1 amd64 17.2.6-1303-gb2903cb0-1focal [3,503 kB] 2023-10-09T04:28:03.193 INFO:teuthology.orchestra.run.smithi133.stdout:The following NEW packages will be installed: 2023-10-09T04:28:03.193 INFO:teuthology.orchestra.run.smithi133.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2023-10-09T04:28:03.193 INFO:teuthology.orchestra.run.smithi133.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2023-10-09T04:28:03.193 INFO:teuthology.orchestra.run.smithi133.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm 2023-10-09T04:28:03.193 INFO:teuthology.orchestra.run.smithi133.stdout: fonts-mathjax jq libcephfs-dev libcephfs2 libdouble-conversion3 libjq1 2023-10-09T04:28:03.193 INFO:teuthology.orchestra.run.smithi133.stdout: libjs-mathjax libjs-sphinxdoc libjs-underscore liblbfgsb0 liblttng-ust0 2023-10-09T04:28:03.193 INFO:teuthology.orchestra.run.smithi133.stdout: liblua5.3-0 liboath0 libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 2023-10-09T04:28:03.193 INFO:teuthology.orchestra.run.smithi133.stdout: libqt5network5 librabbitmq4 libradosstriper1 librdkafka1 librgw2 2023-10-09T04:28:03.194 INFO:teuthology.orchestra.run.smithi133.stdout: libsqlite3-mod-ceph libthrift-0.13.0 nvme-cli python-natsort-doc 2023-10-09T04:28:03.194 INFO:teuthology.orchestra.run.smithi133.stdout: python-pastedeploy-tpl python3-atomicwrites python3-attr python3-bcrypt 2023-10-09T04:28:03.194 INFO:teuthology.orchestra.run.smithi133.stdout: python3-blinker python3-bs4 python3-cachetools python3-ceph-argparse 2023-10-09T04:28:03.194 INFO:teuthology.orchestra.run.smithi133.stdout: python3-ceph-common python3-cephfs python3-cherrypy3 python3-cryptography 2023-10-09T04:28:03.194 INFO:teuthology.orchestra.run.smithi133.stdout: python3-decorator python3-google-auth python3-html5lib python3-jinja2 2023-10-09T04:28:03.194 INFO:teuthology.orchestra.run.smithi133.stdout: python3-joblib python3-jwt python3-kubernetes python3-logutils python3-lxml 2023-10-09T04:28:03.195 INFO:teuthology.orchestra.run.smithi133.stdout: python3-mako python3-markupsafe python3-natsort python3-oauthlib 2023-10-09T04:28:03.195 INFO:teuthology.orchestra.run.smithi133.stdout: python3-openssl python3-packaging python3-paste python3-pastedeploy 2023-10-09T04:28:03.195 INFO:teuthology.orchestra.run.smithi133.stdout: python3-pastescript python3-pecan python3-pluggy python3-prettytable 2023-10-09T04:28:03.195 INFO:teuthology.orchestra.run.smithi133.stdout: python3-psutil python3-py python3-pyasn1 python3-pyasn1-modules 2023-10-09T04:28:03.195 INFO:teuthology.orchestra.run.smithi133.stdout: python3-pyinotify python3-pytest python3-rados python3-rbd 2023-10-09T04:28:03.195 INFO:teuthology.orchestra.run.smithi133.stdout: python3-repoze.lru python3-requests-oauthlib python3-rgw python3-routes 2023-10-09T04:28:03.195 INFO:teuthology.orchestra.run.smithi133.stdout: python3-rsa python3-scipy python3-simplegeneric python3-simplejson 2023-10-09T04:28:03.195 INFO:teuthology.orchestra.run.smithi133.stdout: python3-singledispatch python3-sklearn python3-sklearn-lib python3-soupsieve 2023-10-09T04:28:03.195 INFO:teuthology.orchestra.run.smithi133.stdout: python3-tempita python3-waitress python3-wcwidth python3-webencodings 2023-10-09T04:28:03.195 INFO:teuthology.orchestra.run.smithi133.stdout: python3-webob python3-websocket python3-webtest python3-werkzeug 2023-10-09T04:28:03.195 INFO:teuthology.orchestra.run.smithi133.stdout: qttranslations5-l10n radosgw rbd-fuse socat xmlstarlet 2023-10-09T04:28:03.196 INFO:teuthology.orchestra.run.smithi133.stdout:The following packages will be upgraded: 2023-10-09T04:28:03.196 INFO:teuthology.orchestra.run.smithi133.stdout: librados2 librbd1 2023-10-09T04:28:03.291 INFO:teuthology.orchestra.run.smithi194.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-prettytable all 0.7.2-5 [20.1 kB] 2023-10-09T04:28:03.297 INFO:teuthology.orchestra.run.smithi194.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 liblua5.3-0 amd64 5.3.3-1.1ubuntu2 [116 kB] 2023-10-09T04:28:03.299 INFO:teuthology.orchestra.run.smithi194.stdout:Get:5 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 librados2 amd64 17.2.6-1303-gb2903cb0-1focal [3,552 kB] 2023-10-09T04:28:03.343 INFO:teuthology.orchestra.run.smithi194.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal/main amd64 librabbitmq4 amd64 0.10.0-1 [35.8 kB] 2023-10-09T04:28:03.355 INFO:teuthology.orchestra.run.smithi194.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal/universe amd64 librdkafka1 amd64 1.2.1-1ubuntu1 [410 kB] 2023-10-09T04:28:03.388 INFO:teuthology.orchestra.run.smithi133.stdout:2 upgraded, 108 newly installed, 0 to remove and 256 not upgraded. 2023-10-09T04:28:03.389 INFO:teuthology.orchestra.run.smithi133.stdout:Need to get 209 MB of archives. 2023-10-09T04:28:03.389 INFO:teuthology.orchestra.run.smithi133.stdout:After this operation, 980 MB of additional disk space will be used. 2023-10-09T04:28:03.389 INFO:teuthology.orchestra.run.smithi133.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 liblttng-ust0 amd64 2.11.0-1 [160 kB] 2023-10-09T04:28:03.407 INFO:teuthology.orchestra.run.smithi194.stdout:Get:8 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 libcephfs2 amd64 17.2.6-1303-gb2903cb0-1focal [761 kB] 2023-10-09T04:28:03.431 INFO:teuthology.orchestra.run.smithi194.stdout:Get:9 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 python3-rados amd64 17.2.6-1303-gb2903cb0-1focal [414 kB] 2023-10-09T04:28:03.445 INFO:teuthology.orchestra.run.smithi194.stdout:Get:10 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-argparse all 17.2.6-1303-gb2903cb0-1focal [22.6 kB] 2023-10-09T04:28:03.446 INFO:teuthology.orchestra.run.smithi194.stdout:Get:11 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 python3-cephfs amd64 17.2.6-1303-gb2903cb0-1focal [200 kB] 2023-10-09T04:28:03.453 INFO:teuthology.orchestra.run.smithi194.stdout:Get:12 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-common all 17.2.6-1303-gb2903cb0-1focal [47.2 kB] 2023-10-09T04:28:03.454 INFO:teuthology.orchestra.run.smithi194.stdout:Get:13 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 python3-rbd amd64 17.2.6-1303-gb2903cb0-1focal [387 kB] 2023-10-09T04:28:03.467 INFO:teuthology.orchestra.run.smithi194.stdout:Get:14 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 librgw2 amd64 17.2.6-1303-gb2903cb0-1focal [5,573 kB] 2023-10-09T04:28:03.480 INFO:teuthology.orchestra.run.smithi194.stdout:Get:15 http://archive.ubuntu.com/ubuntu focal/universe amd64 libdouble-conversion3 amd64 3.1.5-4ubuntu1 [37.9 kB] 2023-10-09T04:28:03.480 INFO:teuthology.orchestra.run.smithi194.stdout:Get:16 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libpcre2-16-0 amd64 10.34-7ubuntu0.1 [181 kB] 2023-10-09T04:28:03.484 INFO:teuthology.orchestra.run.smithi194.stdout:Get:17 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 libqt5core5a amd64 5.12.8+dfsg-0ubuntu2.1 [2,006 kB] 2023-10-09T04:28:03.650 INFO:teuthology.orchestra.run.smithi194.stdout:Get:18 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 python3-rgw amd64 17.2.6-1303-gb2903cb0-1focal [116 kB] 2023-10-09T04:28:03.654 INFO:teuthology.orchestra.run.smithi194.stdout:Get:19 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 libradosstriper1 amd64 17.2.6-1303-gb2903cb0-1focal [480 kB] 2023-10-09T04:28:03.669 INFO:teuthology.orchestra.run.smithi194.stdout:Get:20 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 ceph-common amd64 17.2.6-1303-gb2903cb0-1focal [27.7 MB] 2023-10-09T04:28:03.697 INFO:teuthology.orchestra.run.smithi194.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 libqt5dbus5 amd64 5.12.8+dfsg-0ubuntu2.1 [208 kB] 2023-10-09T04:28:03.701 INFO:teuthology.orchestra.run.smithi194.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 libqt5network5 amd64 5.12.8+dfsg-0ubuntu2.1 [673 kB] 2023-10-09T04:28:03.718 INFO:teuthology.orchestra.run.smithi194.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal/universe amd64 libthrift-0.13.0 amd64 0.13.0-2build2 [249 kB] 2023-10-09T04:28:03.720 INFO:teuthology.orchestra.run.smithi194.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal/main amd64 liboath0 amd64 2.6.1-1.3 [44.7 kB] 2023-10-09T04:28:03.720 INFO:teuthology.orchestra.run.smithi194.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-cryptography amd64 2.8-3ubuntu0.1 [211 kB] 2023-10-09T04:28:03.723 INFO:teuthology.orchestra.run.smithi194.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-openssl all 19.0.0-1build1 [43.3 kB] 2023-10-09T04:28:03.723 INFO:teuthology.orchestra.run.smithi194.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bcrypt amd64 3.1.7-2ubuntu1 [30.4 kB] 2023-10-09T04:28:03.724 INFO:teuthology.orchestra.run.smithi194.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cherrypy3 all 8.9.1-6 [160 kB] 2023-10-09T04:28:03.725 INFO:teuthology.orchestra.run.smithi194.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-natsort all 7.0.1-1 [29.2 kB] 2023-10-09T04:28:03.735 INFO:teuthology.orchestra.run.smithi194.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-jwt all 1.7.1-2ubuntu2.1 [18.0 kB] 2023-10-09T04:28:03.754 INFO:teuthology.orchestra.run.smithi194.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-logutils all 0.3.3-7 [17.3 kB] 2023-10-09T04:28:03.755 INFO:teuthology.orchestra.run.smithi194.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-markupsafe amd64 1.1.0-1build2 [13.9 kB] 2023-10-09T04:28:03.756 INFO:teuthology.orchestra.run.smithi194.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-mako all 1.1.0+ds1-1ubuntu2.1 [59.2 kB] 2023-10-09T04:28:03.758 INFO:teuthology.orchestra.run.smithi194.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2023-10-09T04:28:03.759 INFO:teuthology.orchestra.run.smithi194.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-singledispatch all 3.4.0.3-2build1 [7,128 B] 2023-10-09T04:28:03.759 INFO:teuthology.orchestra.run.smithi194.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webob all 1:1.8.5-2 [84.4 kB] 2023-10-09T04:28:03.761 INFO:teuthology.orchestra.run.smithi194.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-soupsieve all 1.9.5+dfsg-1 [29.1 kB] 2023-10-09T04:28:03.762 INFO:teuthology.orchestra.run.smithi194.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bs4 all 4.8.2-1 [83.0 kB] 2023-10-09T04:28:03.764 INFO:teuthology.orchestra.run.smithi194.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-waitress all 1.4.1-1ubuntu0.1 [81.4 kB] 2023-10-09T04:28:03.786 INFO:teuthology.orchestra.run.smithi133.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-prettytable all 0.7.2-5 [20.1 kB] 2023-10-09T04:28:03.788 INFO:teuthology.orchestra.run.smithi194.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-tempita all 0.5.2-6 [14.8 kB] 2023-10-09T04:28:03.788 INFO:teuthology.orchestra.run.smithi194.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-paste all 3.4.0+dfsg1-1 [456 kB] 2023-10-09T04:28:03.793 INFO:teuthology.orchestra.run.smithi133.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 liblua5.3-0 amd64 5.3.3-1.1ubuntu2 [116 kB] 2023-10-09T04:28:03.798 INFO:teuthology.orchestra.run.smithi194.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal/main amd64 python-pastedeploy-tpl all 2.1.0-1 [5,016 B] 2023-10-09T04:28:03.799 INFO:teuthology.orchestra.run.smithi194.stdout:Get:43 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastedeploy all 2.1.0-1 [26.5 kB] 2023-10-09T04:28:03.799 INFO:teuthology.orchestra.run.smithi194.stdout:Get:44 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webtest all 2.0.32-2 [28.2 kB] 2023-10-09T04:28:03.800 INFO:teuthology.orchestra.run.smithi194.stdout:Get:45 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pecan all 1.3.3-3ubuntu1 [85.8 kB] 2023-10-09T04:28:03.801 INFO:teuthology.orchestra.run.smithi194.stdout:Get:46 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-werkzeug all 0.16.1+dfsg1-2ubuntu0.1 [183 kB] 2023-10-09T04:28:03.804 INFO:teuthology.orchestra.run.smithi194.stdout:Get:47 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-jinja2 all 2.10.1-2 [95.5 kB] 2023-10-09T04:28:03.815 INFO:teuthology.orchestra.run.smithi194.stdout:Get:48 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2023-10-09T04:28:03.816 INFO:teuthology.orchestra.run.smithi194.stdout:Get:49 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libjs-underscore all 1.9.1~dfsg-1ubuntu0.20.04.1 [99.5 kB] 2023-10-09T04:28:03.839 INFO:teuthology.orchestra.run.smithi133.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 librabbitmq4 amd64 0.10.0-1 [35.8 kB] 2023-10-09T04:28:03.840 INFO:teuthology.orchestra.run.smithi194.stdout:Get:50 http://archive.ubuntu.com/ubuntu focal/main amd64 libjs-sphinxdoc all 1.8.5-7ubuntu3 [97.1 kB] 2023-10-09T04:28:03.842 INFO:teuthology.orchestra.run.smithi194.stdout:Get:51 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-routes all 2.4.1-2 [88.1 kB] 2023-10-09T04:28:03.844 INFO:teuthology.orchestra.run.smithi194.stdout:Get:52 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-decorator all 4.4.2-0ubuntu1 [10.3 kB] 2023-10-09T04:28:03.845 INFO:teuthology.orchestra.run.smithi194.stdout:Get:53 http://archive.ubuntu.com/ubuntu focal/universe amd64 liblbfgsb0 amd64 3.0+dfsg.3-7build1 [29.5 kB] 2023-10-09T04:28:03.846 INFO:teuthology.orchestra.run.smithi194.stdout:Get:54 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-scipy amd64 1.3.3-3build1 [11.1 MB] 2023-10-09T04:28:03.851 INFO:teuthology.orchestra.run.smithi133.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/universe amd64 librdkafka1 amd64 1.2.1-1ubuntu1 [410 kB] 2023-10-09T04:28:03.979 INFO:teuthology.orchestra.run.smithi133.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal/universe amd64 libdouble-conversion3 amd64 3.1.5-4ubuntu1 [37.9 kB] 2023-10-09T04:28:03.984 INFO:teuthology.orchestra.run.smithi133.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libpcre2-16-0 amd64 10.34-7ubuntu0.1 [181 kB] 2023-10-09T04:28:04.014 INFO:teuthology.orchestra.run.smithi133.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 libqt5core5a amd64 5.12.8+dfsg-0ubuntu2.1 [2,006 kB] 2023-10-09T04:28:04.121 INFO:teuthology.orchestra.run.smithi194.stdout:Get:55 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn-lib amd64 0.22.2.post1+dfsg-5 [2,358 kB] 2023-10-09T04:28:04.149 INFO:teuthology.orchestra.run.smithi194.stdout:Get:56 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-joblib all 0.14.0-3 [192 kB] 2023-10-09T04:28:04.153 INFO:teuthology.orchestra.run.smithi194.stdout:Get:57 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn all 0.22.2.post1+dfsg-5 [1,773 kB] 2023-10-09T04:28:04.154 INFO:teuthology.orchestra.run.smithi133.stdout:Get:9 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 librbd1 amd64 17.2.6-1303-gb2903cb0-1focal [3,503 kB] 2023-10-09T04:28:04.156 INFO:teuthology.orchestra.run.smithi133.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 libqt5dbus5 amd64 5.12.8+dfsg-0ubuntu2.1 [208 kB] 2023-10-09T04:28:04.166 INFO:teuthology.orchestra.run.smithi133.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 libqt5network5 amd64 5.12.8+dfsg-0ubuntu2.1 [673 kB] 2023-10-09T04:28:04.189 INFO:teuthology.orchestra.run.smithi133.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/universe amd64 libthrift-0.13.0 amd64 0.13.0-2build2 [249 kB] 2023-10-09T04:28:04.255 INFO:teuthology.orchestra.run.smithi194.stdout:Get:58 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cachetools all 4.0.0-1 [9,820 B] 2023-10-09T04:28:04.255 INFO:teuthology.orchestra.run.smithi194.stdout:Get:59 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1 all 0.4.2-3build1 [46.7 kB] 2023-10-09T04:28:04.256 INFO:teuthology.orchestra.run.smithi194.stdout:Get:60 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1-modules all 0.2.1-0.2build1 [32.9 kB] 2023-10-09T04:28:04.256 INFO:teuthology.orchestra.run.smithi194.stdout:Get:61 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-rsa all 4.0-3 [24.7 kB] 2023-10-09T04:28:04.257 INFO:teuthology.orchestra.run.smithi194.stdout:Get:62 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-google-auth all 1.5.1-2 [35.6 kB] 2023-10-09T04:28:04.258 INFO:teuthology.orchestra.run.smithi194.stdout:Get:63 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-blinker all 1.4+dfsg1-0.3ubuntu1 [13.2 kB] 2023-10-09T04:28:04.258 INFO:teuthology.orchestra.run.smithi194.stdout:Get:64 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-oauthlib all 3.1.0-1ubuntu2 [84.8 kB] 2023-10-09T04:28:04.261 INFO:teuthology.orchestra.run.smithi194.stdout:Get:65 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-requests-oauthlib all 1.0.0-1.1build1 [16.8 kB] 2023-10-09T04:28:04.262 INFO:teuthology.orchestra.run.smithi194.stdout:Get:66 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-websocket all 0.53.0-2ubuntu1 [32.3 kB] 2023-10-09T04:28:04.285 INFO:teuthology.orchestra.run.smithi194.stdout:Get:67 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-kubernetes all 7.0.0~a1-2 [271 kB] 2023-10-09T04:28:04.290 INFO:teuthology.orchestra.run.smithi194.stdout:Get:68 http://archive.ubuntu.com/ubuntu focal/universe amd64 libonig5 amd64 6.9.4-1 [142 kB] 2023-10-09T04:28:04.293 INFO:teuthology.orchestra.run.smithi194.stdout:Get:69 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 libjq1 amd64 1.6-1ubuntu0.20.04.1 [121 kB] 2023-10-09T04:28:04.316 INFO:teuthology.orchestra.run.smithi194.stdout:Get:70 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 jq amd64 1.6-1ubuntu0.20.04.1 [50.2 kB] 2023-10-09T04:28:04.317 INFO:teuthology.orchestra.run.smithi194.stdout:Get:71 http://archive.ubuntu.com/ubuntu focal/main amd64 socat amd64 1.7.3.3-2 [323 kB] 2023-10-09T04:28:04.322 INFO:teuthology.orchestra.run.smithi194.stdout:Get:72 http://archive.ubuntu.com/ubuntu focal/universe amd64 xmlstarlet amd64 1.6.1-2build1 [267 kB] 2023-10-09T04:28:04.325 INFO:teuthology.orchestra.run.smithi194.stdout:Get:73 http://archive.ubuntu.com/ubuntu focal/universe amd64 fonts-mathjax all 2.7.4+dfsg-1 [2,208 kB] 2023-10-09T04:28:04.355 INFO:teuthology.orchestra.run.smithi194.stdout:Get:74 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 nvme-cli amd64 1.9-1ubuntu0.1 [283 kB] 2023-10-09T04:28:04.360 INFO:teuthology.orchestra.run.smithi133.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 liboath0 amd64 2.6.1-1.3 [44.7 kB] 2023-10-09T04:28:04.370 INFO:teuthology.orchestra.run.smithi194.stdout:Get:75 http://archive.ubuntu.com/ubuntu focal/universe amd64 libjs-mathjax all 2.7.4+dfsg-1 [5,654 kB] 2023-10-09T04:28:04.528 INFO:teuthology.orchestra.run.smithi194.stdout:Get:76 http://archive.ubuntu.com/ubuntu focal/universe amd64 python-natsort-doc all 7.0.1-1 [103 kB] 2023-10-09T04:28:04.539 INFO:teuthology.orchestra.run.smithi194.stdout:Get:77 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-atomicwrites all 1.1.5-2build1 [7,016 B] 2023-10-09T04:28:04.540 INFO:teuthology.orchestra.run.smithi194.stdout:Get:78 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-attr all 19.3.0-2 [33.9 kB] 2023-10-09T04:28:04.541 INFO:teuthology.orchestra.run.smithi194.stdout:Get:79 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webencodings all 0.5.1-1ubuntu1 [11.0 kB] 2023-10-09T04:28:04.542 INFO:teuthology.orchestra.run.smithi194.stdout:Get:80 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-html5lib all 1.0.1-2 [84.3 kB] 2023-10-09T04:28:04.543 INFO:teuthology.orchestra.run.smithi194.stdout:Get:81 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-lxml amd64 4.5.0-1ubuntu0.5 [1,384 kB] 2023-10-09T04:28:04.561 INFO:teuthology.orchestra.run.smithi194.stdout:Get:82 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-packaging all 20.3-1 [26.8 kB] 2023-10-09T04:28:04.561 INFO:teuthology.orchestra.run.smithi194.stdout:Get:83 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2023-10-09T04:28:04.562 INFO:teuthology.orchestra.run.smithi194.stdout:Get:84 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pluggy all 0.13.0-2 [18.4 kB] 2023-10-09T04:28:04.569 INFO:teuthology.orchestra.run.smithi194.stdout:Get:85 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-psutil amd64 5.5.1-1ubuntu4 [141 kB] 2023-10-09T04:28:04.606 INFO:teuthology.orchestra.run.smithi194.stdout:Get:86 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 python3-py all 1.8.1-1ubuntu0.1 [65.6 kB] 2023-10-09T04:28:04.608 INFO:teuthology.orchestra.run.smithi133.stdout:Get:14 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-cryptography amd64 2.8-3ubuntu0.1 [211 kB] 2023-10-09T04:28:04.656 INFO:teuthology.orchestra.run.smithi194.stdout:Get:87 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyinotify all 0.9.6-1.2ubuntu1 [24.8 kB] 2023-10-09T04:28:04.657 INFO:teuthology.orchestra.run.smithi194.stdout:Get:88 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-wcwidth all 0.1.8+dfsg1-3 [17.4 kB] 2023-10-09T04:28:04.657 INFO:teuthology.orchestra.run.smithi194.stdout:Get:89 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pytest all 4.6.9-1 [178 kB] 2023-10-09T04:28:04.733 INFO:teuthology.orchestra.run.smithi194.stdout:Get:90 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplejson amd64 3.16.0-2ubuntu2 [50.8 kB] 2023-10-09T04:28:04.733 INFO:teuthology.orchestra.run.smithi194.stdout:Get:91 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 ceph-base amd64 17.2.6-1303-gb2903cb0-1focal [6,449 kB] 2023-10-09T04:28:04.733 INFO:teuthology.orchestra.run.smithi194.stdout:Get:92 http://archive.ubuntu.com/ubuntu focal/universe amd64 qttranslations5-l10n all 5.12.8-0ubuntu1 [1,486 kB] 2023-10-09T04:28:04.804 INFO:teuthology.orchestra.run.smithi133.stdout:Get:15 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-openssl all 19.0.0-1build1 [43.3 kB] 2023-10-09T04:28:04.824 INFO:teuthology.orchestra.run.smithi194.stdout:Get:93 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-modules-core all 17.2.6-1303-gb2903cb0-1focal [221 kB] 2023-10-09T04:28:04.824 INFO:teuthology.orchestra.run.smithi133.stdout:Get:16 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bcrypt amd64 3.1.7-2ubuntu1 [30.4 kB] 2023-10-09T04:28:04.829 INFO:teuthology.orchestra.run.smithi194.stdout:Get:94 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 libsqlite3-mod-ceph amd64 17.2.6-1303-gb2903cb0-1focal [152 kB] 2023-10-09T04:28:04.835 INFO:teuthology.orchestra.run.smithi194.stdout:Get:95 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr amd64 17.2.6-1303-gb2903cb0-1focal [1,591 kB] 2023-10-09T04:28:04.838 INFO:teuthology.orchestra.run.smithi133.stdout:Get:17 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cherrypy3 all 8.9.1-6 [160 kB] 2023-10-09T04:28:04.887 INFO:teuthology.orchestra.run.smithi194.stdout:Get:96 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 ceph-mon amd64 17.2.6-1303-gb2903cb0-1focal [7,846 kB] 2023-10-09T04:28:04.891 INFO:teuthology.orchestra.run.smithi133.stdout:Get:18 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-natsort all 7.0.1-1 [29.2 kB] 2023-10-09T04:28:04.898 INFO:teuthology.orchestra.run.smithi133.stdout:Get:19 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-jwt all 1.7.1-2ubuntu2.1 [18.0 kB] 2023-10-09T04:28:04.901 INFO:teuthology.orchestra.run.smithi133.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-logutils all 0.3.3-7 [17.3 kB] 2023-10-09T04:28:04.905 INFO:teuthology.orchestra.run.smithi133.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-markupsafe amd64 1.1.0-1build2 [13.9 kB] 2023-10-09T04:28:04.908 INFO:teuthology.orchestra.run.smithi133.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-mako all 1.1.0+ds1-1ubuntu2.1 [59.2 kB] 2023-10-09T04:28:04.920 INFO:teuthology.orchestra.run.smithi133.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2023-10-09T04:28:05.086 INFO:teuthology.orchestra.run.smithi133.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-singledispatch all 3.4.0.3-2build1 [7,128 B] 2023-10-09T04:28:05.146 INFO:teuthology.orchestra.run.smithi194.stdout:Get:97 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 ceph-osd amd64 17.2.6-1303-gb2903cb0-1focal [28.7 MB] 2023-10-09T04:28:05.169 INFO:teuthology.orchestra.run.smithi133.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webob all 1:1.8.5-2 [84.4 kB] 2023-10-09T04:28:05.356 INFO:teuthology.orchestra.run.smithi133.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-soupsieve all 1.9.5+dfsg-1 [29.1 kB] 2023-10-09T04:28:05.391 INFO:teuthology.orchestra.run.smithi133.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bs4 all 4.8.2-1 [83.0 kB] 2023-10-09T04:28:05.455 INFO:teuthology.orchestra.run.smithi133.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-waitress all 1.4.1-1ubuntu0.1 [81.4 kB] 2023-10-09T04:28:05.496 INFO:teuthology.orchestra.run.smithi133.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-tempita all 0.5.2-6 [14.8 kB] 2023-10-09T04:28:05.502 INFO:teuthology.orchestra.run.smithi133.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-paste all 3.4.0+dfsg1-1 [456 kB] 2023-10-09T04:28:05.613 INFO:teuthology.orchestra.run.smithi133.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal/main amd64 python-pastedeploy-tpl all 2.1.0-1 [5,016 B] 2023-10-09T04:28:05.613 INFO:teuthology.orchestra.run.smithi133.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastedeploy all 2.1.0-1 [26.5 kB] 2023-10-09T04:28:05.617 INFO:teuthology.orchestra.run.smithi133.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webtest all 2.0.32-2 [28.2 kB] 2023-10-09T04:28:05.621 INFO:teuthology.orchestra.run.smithi133.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pecan all 1.3.3-3ubuntu1 [85.8 kB] 2023-10-09T04:28:05.797 INFO:teuthology.orchestra.run.smithi133.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-werkzeug all 0.16.1+dfsg1-2ubuntu0.1 [183 kB] 2023-10-09T04:28:06.194 INFO:teuthology.orchestra.run.smithi194.stdout:Get:98 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 ceph amd64 17.2.6-1303-gb2903cb0-1focal [3,932 B] 2023-10-09T04:28:06.194 INFO:teuthology.orchestra.run.smithi194.stdout:Get:99 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 ceph-fuse amd64 17.2.6-1303-gb2903cb0-1focal [870 kB] 2023-10-09T04:28:06.201 INFO:teuthology.orchestra.run.smithi194.stdout:Get:100 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 ceph-mds amd64 17.2.6-1303-gb2903cb0-1focal [2,405 kB] 2023-10-09T04:28:06.211 INFO:teuthology.orchestra.run.smithi133.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-jinja2 all 2.10.1-2 [95.5 kB] 2023-10-09T04:28:06.218 INFO:teuthology.orchestra.run.smithi194.stdout:Get:101 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 cephadm amd64 17.2.6-1303-gb2903cb0-1focal [79.1 kB] 2023-10-09T04:28:06.219 INFO:teuthology.orchestra.run.smithi194.stdout:Get:102 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-cephadm all 17.2.6-1303-gb2903cb0-1focal [117 kB] 2023-10-09T04:28:06.220 INFO:teuthology.orchestra.run.smithi194.stdout:Get:103 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-dashboard all 17.2.6-1303-gb2903cb0-1focal [1,760 kB] 2023-10-09T04:28:06.243 INFO:teuthology.orchestra.run.smithi133.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2023-10-09T04:28:06.247 INFO:teuthology.orchestra.run.smithi133.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libjs-underscore all 1.9.1~dfsg-1ubuntu0.20.04.1 [99.5 kB] 2023-10-09T04:28:06.264 INFO:teuthology.orchestra.run.smithi194.stdout:Get:104 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-diskprediction-local all 17.2.6-1303-gb2903cb0-1focal [7,887 kB] 2023-10-09T04:28:06.296 INFO:teuthology.orchestra.run.smithi133.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal/main amd64 libjs-sphinxdoc all 1.8.5-7ubuntu3 [97.1 kB] 2023-10-09T04:28:06.328 INFO:teuthology.orchestra.run.smithi133.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-routes all 2.4.1-2 [88.1 kB] 2023-10-09T04:28:06.356 INFO:teuthology.orchestra.run.smithi133.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-decorator all 4.4.2-0ubuntu1 [10.3 kB] 2023-10-09T04:28:06.358 INFO:teuthology.orchestra.run.smithi133.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal/universe amd64 liblbfgsb0 amd64 3.0+dfsg.3-7build1 [29.5 kB] 2023-10-09T04:28:06.366 INFO:teuthology.orchestra.run.smithi133.stdout:Get:43 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-scipy amd64 1.3.3-3build1 [11.1 MB] 2023-10-09T04:28:06.526 INFO:teuthology.orchestra.run.smithi194.stdout:Get:105 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-k8sevents all 17.2.6-1303-gb2903cb0-1focal [4,048 B] 2023-10-09T04:28:06.526 INFO:teuthology.orchestra.run.smithi194.stdout:Get:106 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 ceph-test amd64 17.2.6-1303-gb2903cb0-1focal [56.3 MB] 2023-10-09T04:28:06.845 INFO:teuthology.orchestra.run.smithi133.stdout:Get:44 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn-lib amd64 0.22.2.post1+dfsg-5 [2,358 kB] 2023-10-09T04:28:06.880 INFO:teuthology.orchestra.run.smithi133.stdout:Get:45 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-joblib all 0.14.0-3 [192 kB] 2023-10-09T04:28:07.046 INFO:teuthology.orchestra.run.smithi133.stdout:Get:46 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn all 0.22.2.post1+dfsg-5 [1,773 kB] 2023-10-09T04:28:07.734 INFO:teuthology.orchestra.run.smithi133.stdout:Get:47 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cachetools all 4.0.0-1 [9,820 B] 2023-10-09T04:28:07.734 INFO:teuthology.orchestra.run.smithi133.stdout:Get:48 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1 all 0.4.2-3build1 [46.7 kB] 2023-10-09T04:28:07.736 INFO:teuthology.orchestra.run.smithi133.stdout:Get:49 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1-modules all 0.2.1-0.2build1 [32.9 kB] 2023-10-09T04:28:07.737 INFO:teuthology.orchestra.run.smithi133.stdout:Get:50 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-rsa all 4.0-3 [24.7 kB] 2023-10-09T04:28:07.738 INFO:teuthology.orchestra.run.smithi133.stdout:Get:51 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-google-auth all 1.5.1-2 [35.6 kB] 2023-10-09T04:28:07.739 INFO:teuthology.orchestra.run.smithi133.stdout:Get:52 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-blinker all 1.4+dfsg1-0.3ubuntu1 [13.2 kB] 2023-10-09T04:28:07.740 INFO:teuthology.orchestra.run.smithi133.stdout:Get:53 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-oauthlib all 3.1.0-1ubuntu2 [84.8 kB] 2023-10-09T04:28:07.742 INFO:teuthology.orchestra.run.smithi133.stdout:Get:54 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-requests-oauthlib all 1.0.0-1.1build1 [16.8 kB] 2023-10-09T04:28:07.743 INFO:teuthology.orchestra.run.smithi133.stdout:Get:55 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-websocket all 0.53.0-2ubuntu1 [32.3 kB] 2023-10-09T04:28:07.815 INFO:teuthology.orchestra.run.smithi133.stdout:Get:56 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-kubernetes all 7.0.0~a1-2 [271 kB] 2023-10-09T04:28:07.988 INFO:teuthology.orchestra.run.smithi133.stdout:Get:57 http://archive.ubuntu.com/ubuntu focal/universe amd64 libonig5 amd64 6.9.4-1 [142 kB] 2023-10-09T04:28:08.367 INFO:teuthology.orchestra.run.smithi133.stdout:Get:58 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 libjq1 amd64 1.6-1ubuntu0.20.04.1 [121 kB] 2023-10-09T04:28:08.419 INFO:teuthology.orchestra.run.smithi133.stdout:Get:59 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 jq amd64 1.6-1ubuntu0.20.04.1 [50.2 kB] 2023-10-09T04:28:08.455 INFO:teuthology.orchestra.run.smithi133.stdout:Get:60 http://archive.ubuntu.com/ubuntu focal/main amd64 socat amd64 1.7.3.3-2 [323 kB] 2023-10-09T04:28:08.555 INFO:teuthology.orchestra.run.smithi133.stdout:Get:61 http://archive.ubuntu.com/ubuntu focal/universe amd64 xmlstarlet amd64 1.6.1-2build1 [267 kB] 2023-10-09T04:28:08.603 INFO:teuthology.orchestra.run.smithi133.stdout:Get:62 http://archive.ubuntu.com/ubuntu focal/universe amd64 fonts-mathjax all 2.7.4+dfsg-1 [2,208 kB] 2023-10-09T04:28:08.637 INFO:teuthology.orchestra.run.smithi194.stdout:Get:107 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 ceph-volume all 17.2.6-1303-gb2903cb0-1focal [229 kB] 2023-10-09T04:28:08.638 INFO:teuthology.orchestra.run.smithi194.stdout:Get:108 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 libcephfs-dev amd64 17.2.6-1303-gb2903cb0-1focal [21.9 kB] 2023-10-09T04:28:08.638 INFO:teuthology.orchestra.run.smithi194.stdout:Get:109 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 radosgw amd64 17.2.6-1303-gb2903cb0-1focal [17.0 MB] 2023-10-09T04:28:08.757 INFO:teuthology.orchestra.run.smithi133.stdout:Get:63 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 nvme-cli amd64 1.9-1ubuntu0.1 [283 kB] 2023-10-09T04:28:08.767 INFO:teuthology.orchestra.run.smithi133.stdout:Get:64 http://archive.ubuntu.com/ubuntu focal/universe amd64 libjs-mathjax all 2.7.4+dfsg-1 [5,654 kB] 2023-10-09T04:28:08.916 INFO:teuthology.orchestra.run.smithi133.stdout:Get:65 http://archive.ubuntu.com/ubuntu focal/universe amd64 python-natsort-doc all 7.0.1-1 [103 kB] 2023-10-09T04:28:08.917 INFO:teuthology.orchestra.run.smithi133.stdout:Get:66 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-atomicwrites all 1.1.5-2build1 [7,016 B] 2023-10-09T04:28:08.918 INFO:teuthology.orchestra.run.smithi133.stdout:Get:67 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-attr all 19.3.0-2 [33.9 kB] 2023-10-09T04:28:09.082 INFO:teuthology.orchestra.run.smithi133.stdout:Get:68 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webencodings all 0.5.1-1ubuntu1 [11.0 kB] 2023-10-09T04:28:09.166 INFO:teuthology.orchestra.run.smithi133.stdout:Get:69 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-html5lib all 1.0.1-2 [84.3 kB] 2023-10-09T04:28:09.186 INFO:teuthology.orchestra.run.smithi194.stdout:Get:110 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 rbd-fuse amd64 17.2.6-1303-gb2903cb0-1focal [79.5 kB] 2023-10-09T04:28:09.348 INFO:teuthology.orchestra.run.smithi133.stdout:Get:70 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-lxml amd64 4.5.0-1ubuntu0.5 [1,384 kB] 2023-10-09T04:28:09.680 INFO:teuthology.orchestra.run.smithi133.stdout:Get:71 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-packaging all 20.3-1 [26.8 kB] 2023-10-09T04:28:09.681 INFO:teuthology.orchestra.run.smithi133.stdout:Get:72 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2023-10-09T04:28:09.686 INFO:teuthology.orchestra.run.smithi133.stdout:Get:73 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pluggy all 0.13.0-2 [18.4 kB] 2023-10-09T04:28:09.686 INFO:teuthology.orchestra.run.smithi133.stdout:Get:74 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-psutil amd64 5.5.1-1ubuntu4 [141 kB] 2023-10-09T04:28:09.697 INFO:teuthology.orchestra.run.smithi133.stdout:Get:75 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 python3-py all 1.8.1-1ubuntu0.1 [65.6 kB] 2023-10-09T04:28:09.701 INFO:teuthology.orchestra.run.smithi133.stdout:Get:76 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyinotify all 0.9.6-1.2ubuntu1 [24.8 kB] 2023-10-09T04:28:09.702 INFO:teuthology.orchestra.run.smithi133.stdout:Get:77 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-wcwidth all 0.1.8+dfsg1-3 [17.4 kB] 2023-10-09T04:28:09.704 INFO:teuthology.orchestra.run.smithi133.stdout:Get:78 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pytest all 4.6.9-1 [178 kB] 2023-10-09T04:28:09.837 INFO:teuthology.orchestra.run.smithi194.stdout:Fetched 209 MB in 6s (33.6 MB/s) 2023-10-09T04:28:09.879 INFO:teuthology.orchestra.run.smithi133.stdout:Get:79 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplejson amd64 3.16.0-2ubuntu2 [50.8 kB] 2023-10-09T04:28:10.024 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package liblttng-ust0:amd64. 2023-10-09T04:28:10.071 INFO:teuthology.orchestra.run.smithi194.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 112643 files and directories currently installed.) 2023-10-09T04:28:10.075 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../000-liblttng-ust0_2.11.0-1_amd64.deb ... 2023-10-09T04:28:10.136 INFO:teuthology.orchestra.run.smithi133.stdout:Get:80 http://archive.ubuntu.com/ubuntu focal/universe amd64 qttranslations5-l10n all 5.12.8-0ubuntu1 [1,486 kB] 2023-10-09T04:28:10.156 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking liblttng-ust0:amd64 (2.11.0-1) ... 2023-10-09T04:28:10.493 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../001-librbd1_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:10.602 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking librbd1 (17.2.6-1303-gb2903cb0-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-10-09T04:28:11.277 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../002-librados2_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:11.345 INFO:teuthology.orchestra.run.smithi133.stdout:Get:81 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 librados2 amd64 17.2.6-1303-gb2903cb0-1focal [3,552 kB] 2023-10-09T04:28:11.406 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking librados2 (17.2.6-1303-gb2903cb0-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-10-09T04:28:12.076 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package libcephfs2. 2023-10-09T04:28:12.089 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../003-libcephfs2_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:12.134 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking libcephfs2 (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:12.445 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-rados. 2023-10-09T04:28:12.458 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../004-python3-rados_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:12.487 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-rados (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:12.754 INFO:teuthology.orchestra.run.smithi133.stdout:Get:82 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 libcephfs2 amd64 17.2.6-1303-gb2903cb0-1focal [761 kB] 2023-10-09T04:28:12.840 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-ceph-argparse. 2023-10-09T04:28:12.853 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../005-python3-ceph-argparse_17.2.6-1303-gb2903cb0-1focal_all.deb ... 2023-10-09T04:28:12.897 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-ceph-argparse (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:12.933 INFO:teuthology.orchestra.run.smithi133.stdout:Get:83 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 python3-rados amd64 17.2.6-1303-gb2903cb0-1focal [414 kB] 2023-10-09T04:28:13.035 INFO:teuthology.orchestra.run.smithi133.stdout:Get:84 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-argparse all 17.2.6-1303-gb2903cb0-1focal [22.6 kB] 2023-10-09T04:28:13.037 INFO:teuthology.orchestra.run.smithi133.stdout:Get:85 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 python3-cephfs amd64 17.2.6-1303-gb2903cb0-1focal [200 kB] 2023-10-09T04:28:13.075 INFO:teuthology.orchestra.run.smithi133.stdout:Get:86 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-common all 17.2.6-1303-gb2903cb0-1focal [47.2 kB] 2023-10-09T04:28:13.086 INFO:teuthology.orchestra.run.smithi133.stdout:Get:87 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 python3-rbd amd64 17.2.6-1303-gb2903cb0-1focal [387 kB] 2023-10-09T04:28:13.134 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-cephfs. 2023-10-09T04:28:13.147 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../006-python3-cephfs_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:13.158 INFO:teuthology.orchestra.run.smithi133.stdout:Get:88 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 librgw2 amd64 17.2.6-1303-gb2903cb0-1focal [5,573 kB] 2023-10-09T04:28:13.193 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-cephfs (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:13.504 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-ceph-common. 2023-10-09T04:28:13.518 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../007-python3-ceph-common_17.2.6-1303-gb2903cb0-1focal_all.deb ... 2023-10-09T04:28:13.554 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-ceph-common (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:13.840 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-prettytable. 2023-10-09T04:28:13.855 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../008-python3-prettytable_0.7.2-5_all.deb ... 2023-10-09T04:28:13.899 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-prettytable (0.7.2-5) ... 2023-10-09T04:28:14.124 INFO:teuthology.orchestra.run.smithi133.stdout:Get:89 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 python3-rgw amd64 17.2.6-1303-gb2903cb0-1focal [116 kB] 2023-10-09T04:28:14.140 INFO:teuthology.orchestra.run.smithi133.stdout:Get:90 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 libradosstriper1 amd64 17.2.6-1303-gb2903cb0-1focal [480 kB] 2023-10-09T04:28:14.156 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-rbd. 2023-10-09T04:28:14.171 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../009-python3-rbd_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:14.196 INFO:teuthology.orchestra.run.smithi133.stdout:Get:91 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 ceph-common amd64 17.2.6-1303-gb2903cb0-1focal [27.7 MB] 2023-10-09T04:28:14.214 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-rbd (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:14.567 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package liblua5.3-0:amd64. 2023-10-09T04:28:14.581 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../010-liblua5.3-0_5.3.3-1.1ubuntu2_amd64.deb ... 2023-10-09T04:28:14.625 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-10-09T04:28:14.937 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package librabbitmq4:amd64. 2023-10-09T04:28:14.950 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../011-librabbitmq4_0.10.0-1_amd64.deb ... 2023-10-09T04:28:14.995 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking librabbitmq4:amd64 (0.10.0-1) ... 2023-10-09T04:28:15.340 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package librdkafka1:amd64. 2023-10-09T04:28:15.355 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../012-librdkafka1_1.2.1-1ubuntu1_amd64.deb ... 2023-10-09T04:28:15.398 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-10-09T04:28:15.776 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2023-10-09T04:28:15.789 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../013-libdouble-conversion3_3.1.5-4ubuntu1_amd64.deb ... 2023-10-09T04:28:15.834 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking libdouble-conversion3:amd64 (3.1.5-4ubuntu1) ... 2023-10-09T04:28:16.137 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2023-10-09T04:28:16.150 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../014-libpcre2-16-0_10.34-7ubuntu0.1_amd64.deb ... 2023-10-09T04:28:16.187 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking libpcre2-16-0:amd64 (10.34-7ubuntu0.1) ... 2023-10-09T04:28:16.657 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package libqt5core5a:amd64. 2023-10-09T04:28:16.672 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../015-libqt5core5a_5.12.8+dfsg-0ubuntu2.1_amd64.deb ... 2023-10-09T04:28:16.725 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking libqt5core5a:amd64 (5.12.8+dfsg-0ubuntu2.1) ... 2023-10-09T04:28:16.748 INFO:teuthology.orchestra.run.smithi133.stdout:Get:92 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 ceph-base amd64 17.2.6-1303-gb2903cb0-1focal [6,449 kB] 2023-10-09T04:28:17.072 INFO:teuthology.orchestra.run.smithi133.stdout:Get:93 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-modules-core all 17.2.6-1303-gb2903cb0-1focal [221 kB] 2023-10-09T04:28:17.074 INFO:teuthology.orchestra.run.smithi133.stdout:Get:94 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 libsqlite3-mod-ceph amd64 17.2.6-1303-gb2903cb0-1focal [152 kB] 2023-10-09T04:28:17.079 INFO:teuthology.orchestra.run.smithi133.stdout:Get:95 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr amd64 17.2.6-1303-gb2903cb0-1focal [1,591 kB] 2023-10-09T04:28:17.179 INFO:teuthology.orchestra.run.smithi133.stdout:Get:96 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 ceph-mon amd64 17.2.6-1303-gb2903cb0-1focal [7,846 kB] 2023-10-09T04:28:17.244 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2023-10-09T04:28:17.257 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../016-libqt5dbus5_5.12.8+dfsg-0ubuntu2.1_amd64.deb ... 2023-10-09T04:28:17.302 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking libqt5dbus5:amd64 (5.12.8+dfsg-0ubuntu2.1) ... 2023-10-09T04:28:17.573 INFO:teuthology.orchestra.run.smithi133.stdout:Get:97 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 ceph-osd amd64 17.2.6-1303-gb2903cb0-1focal [28.7 MB] 2023-10-09T04:28:17.622 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package libqt5network5:amd64. 2023-10-09T04:28:17.636 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../017-libqt5network5_5.12.8+dfsg-0ubuntu2.1_amd64.deb ... 2023-10-09T04:28:17.713 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking libqt5network5:amd64 (5.12.8+dfsg-0ubuntu2.1) ... 2023-10-09T04:28:18.099 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package libthrift-0.13.0. 2023-10-09T04:28:18.113 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../018-libthrift-0.13.0_0.13.0-2build2_amd64.deb ... 2023-10-09T04:28:18.157 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking libthrift-0.13.0 (0.13.0-2build2) ... 2023-10-09T04:28:18.477 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package librgw2. 2023-10-09T04:28:18.490 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../019-librgw2_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:18.535 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking librgw2 (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:18.842 INFO:teuthology.orchestra.run.smithi133.stdout:Get:98 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 ceph amd64 17.2.6-1303-gb2903cb0-1focal [3,932 B] 2023-10-09T04:28:18.916 INFO:teuthology.orchestra.run.smithi133.stdout:Get:99 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 ceph-fuse amd64 17.2.6-1303-gb2903cb0-1focal [870 kB] 2023-10-09T04:28:18.923 INFO:teuthology.orchestra.run.smithi133.stdout:Get:100 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 ceph-mds amd64 17.2.6-1303-gb2903cb0-1focal [2,405 kB] 2023-10-09T04:28:18.952 INFO:teuthology.orchestra.run.smithi133.stdout:Get:101 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 cephadm amd64 17.2.6-1303-gb2903cb0-1focal [79.1 kB] 2023-10-09T04:28:18.954 INFO:teuthology.orchestra.run.smithi133.stdout:Get:102 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-cephadm all 17.2.6-1303-gb2903cb0-1focal [117 kB] 2023-10-09T04:28:18.956 INFO:teuthology.orchestra.run.smithi133.stdout:Get:103 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-dashboard all 17.2.6-1303-gb2903cb0-1focal [1,760 kB] 2023-10-09T04:28:19.020 INFO:teuthology.orchestra.run.smithi133.stdout:Get:104 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-diskprediction-local all 17.2.6-1303-gb2903cb0-1focal [7,887 kB] 2023-10-09T04:28:19.274 INFO:teuthology.orchestra.run.smithi133.stdout:Get:105 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-k8sevents all 17.2.6-1303-gb2903cb0-1focal [4,048 B] 2023-10-09T04:28:19.275 INFO:teuthology.orchestra.run.smithi133.stdout:Get:106 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 ceph-test amd64 17.2.6-1303-gb2903cb0-1focal [56.3 MB] 2023-10-09T04:28:19.313 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-rgw. 2023-10-09T04:28:19.326 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../020-python3-rgw_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:19.371 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-rgw (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:19.750 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package liboath0. 2023-10-09T04:28:19.763 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../021-liboath0_2.6.1-1.3_amd64.deb ... 2023-10-09T04:28:19.808 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking liboath0 (2.6.1-1.3) ... 2023-10-09T04:28:20.111 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package libradosstriper1. 2023-10-09T04:28:20.124 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../022-libradosstriper1_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:20.169 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking libradosstriper1 (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:20.631 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package ceph-common. 2023-10-09T04:28:20.646 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../023-ceph-common_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:20.689 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking ceph-common (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:21.395 INFO:teuthology.orchestra.run.smithi133.stdout:Get:107 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 ceph-volume all 17.2.6-1303-gb2903cb0-1focal [229 kB] 2023-10-09T04:28:21.397 INFO:teuthology.orchestra.run.smithi133.stdout:Get:108 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 libcephfs-dev amd64 17.2.6-1303-gb2903cb0-1focal [21.9 kB] 2023-10-09T04:28:21.397 INFO:teuthology.orchestra.run.smithi133.stdout:Get:109 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 radosgw amd64 17.2.6-1303-gb2903cb0-1focal [17.0 MB] 2023-10-09T04:28:21.937 INFO:teuthology.orchestra.run.smithi133.stdout:Get:110 https://1.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/ubuntu/focal/flavors/default focal/main amd64 rbd-fuse amd64 17.2.6-1303-gb2903cb0-1focal [79.5 kB] 2023-10-09T04:28:22.591 INFO:teuthology.orchestra.run.smithi133.stdout:Fetched 209 MB in 19s (11.2 MB/s) 2023-10-09T04:28:22.762 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package liblttng-ust0:amd64. 2023-10-09T04:28:22.808 INFO:teuthology.orchestra.run.smithi133.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 112643 files and directories currently installed.) 2023-10-09T04:28:22.811 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../000-liblttng-ust0_2.11.0-1_amd64.deb ... 2023-10-09T04:28:22.893 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking liblttng-ust0:amd64 (2.11.0-1) ... 2023-10-09T04:28:23.237 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../001-librbd1_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:23.361 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package ceph-base. 2023-10-09T04:28:23.375 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../024-ceph-base_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:23.380 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking librbd1 (17.2.6-1303-gb2903cb0-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-10-09T04:28:23.429 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking ceph-base (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:24.059 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../002-librados2_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:24.185 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking librados2 (17.2.6-1303-gb2903cb0-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-10-09T04:28:24.390 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-cryptography. 2023-10-09T04:28:24.403 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../025-python3-cryptography_2.8-3ubuntu0.1_amd64.deb ... 2023-10-09T04:28:24.448 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-cryptography (2.8-3ubuntu0.1) ... 2023-10-09T04:28:24.827 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-openssl. 2023-10-09T04:28:24.842 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../026-python3-openssl_19.0.0-1build1_all.deb ... 2023-10-09T04:28:24.864 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package libcephfs2. 2023-10-09T04:28:24.878 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../003-libcephfs2_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:24.885 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-openssl (19.0.0-1build1) ... 2023-10-09T04:28:24.923 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking libcephfs2 (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:25.138 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2023-10-09T04:28:25.151 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../027-ceph-mgr-modules-core_17.2.6-1303-gb2903cb0-1focal_all.deb ... 2023-10-09T04:28:25.187 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking ceph-mgr-modules-core (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:25.243 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-rados. 2023-10-09T04:28:25.256 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../004-python3-rados_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:25.301 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-rados (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:25.584 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-bcrypt. 2023-10-09T04:28:25.598 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../028-python3-bcrypt_3.1.7-2ubuntu1_amd64.deb ... 2023-10-09T04:28:25.642 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-bcrypt (3.1.7-2ubuntu1) ... 2023-10-09T04:28:25.684 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-ceph-argparse. 2023-10-09T04:28:25.698 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../005-python3-ceph-argparse_17.2.6-1303-gb2903cb0-1focal_all.deb ... 2023-10-09T04:28:25.734 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-ceph-argparse (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:25.953 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-cherrypy3. 2023-10-09T04:28:25.966 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../029-python3-cherrypy3_8.9.1-6_all.deb ... 2023-10-09T04:28:25.987 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-cephfs. 2023-10-09T04:28:26.000 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../006-python3-cephfs_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:26.003 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-cherrypy3 (8.9.1-6) ... 2023-10-09T04:28:26.045 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-cephfs (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:26.340 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-natsort. 2023-10-09T04:28:26.353 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../030-python3-natsort_7.0.1-1_all.deb ... 2023-10-09T04:28:26.381 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-ceph-common. 2023-10-09T04:28:26.395 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../007-python3-ceph-common_17.2.6-1303-gb2903cb0-1focal_all.deb ... 2023-10-09T04:28:26.398 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-natsort (7.0.1-1) ... 2023-10-09T04:28:26.439 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-ceph-common (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:26.709 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-jwt. 2023-10-09T04:28:26.724 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../031-python3-jwt_1.7.1-2ubuntu2.1_all.deb ... 2023-10-09T04:28:26.751 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-prettytable. 2023-10-09T04:28:26.765 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../008-python3-prettytable_0.7.2-5_all.deb ... 2023-10-09T04:28:26.767 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-jwt (1.7.1-2ubuntu2.1) ... 2023-10-09T04:28:26.810 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-prettytable (0.7.2-5) ... 2023-10-09T04:28:27.079 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-logutils. 2023-10-09T04:28:27.087 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../032-python3-logutils_0.3.3-7_all.deb ... 2023-10-09T04:28:27.104 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-rbd. 2023-10-09T04:28:27.117 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../009-python3-rbd_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:27.129 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-logutils (0.3.3-7) ... 2023-10-09T04:28:27.162 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-rbd (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:27.398 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-markupsafe. 2023-10-09T04:28:27.406 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../033-python3-markupsafe_1.1.0-1build2_amd64.deb ... 2023-10-09T04:28:27.448 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-markupsafe (1.1.0-1build2) ... 2023-10-09T04:28:27.557 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package liblua5.3-0:amd64. 2023-10-09T04:28:27.570 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../010-liblua5.3-0_5.3.3-1.1ubuntu2_amd64.deb ... 2023-10-09T04:28:27.623 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-10-09T04:28:27.751 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-mako. 2023-10-09T04:28:27.766 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../034-python3-mako_1.1.0+ds1-1ubuntu2.1_all.deb ... 2023-10-09T04:28:27.809 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-10-09T04:28:27.994 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package librabbitmq4:amd64. 2023-10-09T04:28:28.007 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../011-librabbitmq4_0.10.0-1_amd64.deb ... 2023-10-09T04:28:28.052 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking librabbitmq4:amd64 (0.10.0-1) ... 2023-10-09T04:28:28.131 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-simplegeneric. 2023-10-09T04:28:28.146 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../035-python3-simplegeneric_0.8.1-3_all.deb ... 2023-10-09T04:28:28.189 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2023-10-09T04:28:28.438 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package librdkafka1:amd64. 2023-10-09T04:28:28.451 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../012-librdkafka1_1.2.1-1ubuntu1_amd64.deb ... 2023-10-09T04:28:28.492 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-singledispatch. 2023-10-09T04:28:28.504 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-10-09T04:28:28.505 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../036-python3-singledispatch_3.4.0.3-2build1_all.deb ... 2023-10-09T04:28:28.550 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-singledispatch (3.4.0.3-2build1) ... 2023-10-09T04:28:28.844 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-webob. 2023-10-09T04:28:28.857 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../037-python3-webob_1%3a1.8.5-2_all.deb ... 2023-10-09T04:28:28.894 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-webob (1:1.8.5-2) ... 2023-10-09T04:28:28.949 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2023-10-09T04:28:28.962 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../013-libdouble-conversion3_3.1.5-4ubuntu1_amd64.deb ... 2023-10-09T04:28:29.007 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking libdouble-conversion3:amd64 (3.1.5-4ubuntu1) ... 2023-10-09T04:28:29.272 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-soupsieve. 2023-10-09T04:28:29.285 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../038-python3-soupsieve_1.9.5+dfsg-1_all.deb ... 2023-10-09T04:28:29.330 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-soupsieve (1.9.5+dfsg-1) ... 2023-10-09T04:28:29.343 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2023-10-09T04:28:29.358 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../014-libpcre2-16-0_10.34-7ubuntu0.1_amd64.deb ... 2023-10-09T04:28:29.427 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking libpcre2-16-0:amd64 (10.34-7ubuntu0.1) ... 2023-10-09T04:28:29.633 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-bs4. 2023-10-09T04:28:29.647 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../039-python3-bs4_4.8.2-1_all.deb ... 2023-10-09T04:28:29.707 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-bs4 (4.8.2-1) ... 2023-10-09T04:28:29.980 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package libqt5core5a:amd64. 2023-10-09T04:28:29.993 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../015-libqt5core5a_5.12.8+dfsg-0ubuntu2.1_amd64.deb ... 2023-10-09T04:28:30.057 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking libqt5core5a:amd64 (5.12.8+dfsg-0ubuntu2.1) ... 2023-10-09T04:28:30.069 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-waitress. 2023-10-09T04:28:30.082 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../040-python3-waitress_1.4.1-1ubuntu0.1_all.deb ... 2023-10-09T04:28:30.130 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-waitress (1.4.1-1ubuntu0.1) ... 2023-10-09T04:28:30.463 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-tempita. 2023-10-09T04:28:30.478 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../041-python3-tempita_0.5.2-6_all.deb ... 2023-10-09T04:28:30.521 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-tempita (0.5.2-6) ... 2023-10-09T04:28:30.558 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2023-10-09T04:28:30.574 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../016-libqt5dbus5_5.12.8+dfsg-0ubuntu2.1_amd64.deb ... 2023-10-09T04:28:30.616 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking libqt5dbus5:amd64 (5.12.8+dfsg-0ubuntu2.1) ... 2023-10-09T04:28:30.807 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-paste. 2023-10-09T04:28:30.822 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../042-python3-paste_3.4.0+dfsg1-1_all.deb ... 2023-10-09T04:28:30.866 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-paste (3.4.0+dfsg1-1) ... 2023-10-09T04:28:30.995 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package libqt5network5:amd64. 2023-10-09T04:28:31.008 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../017-libqt5network5_5.12.8+dfsg-0ubuntu2.1_amd64.deb ... 2023-10-09T04:28:31.054 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking libqt5network5:amd64 (5.12.8+dfsg-0ubuntu2.1) ... 2023-10-09T04:28:31.177 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2023-10-09T04:28:31.191 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../043-python-pastedeploy-tpl_2.1.0-1_all.deb ... 2023-10-09T04:28:31.235 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python-pastedeploy-tpl (2.1.0-1) ... 2023-10-09T04:28:31.424 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package libthrift-0.13.0. 2023-10-09T04:28:31.438 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../018-libthrift-0.13.0_0.13.0-2build2_amd64.deb ... 2023-10-09T04:28:31.482 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking libthrift-0.13.0 (0.13.0-2build2) ... 2023-10-09T04:28:31.546 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-pastedeploy. 2023-10-09T04:28:31.562 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../044-python3-pastedeploy_2.1.0-1_all.deb ... 2023-10-09T04:28:31.605 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-pastedeploy (2.1.0-1) ... 2023-10-09T04:28:31.835 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package librgw2. 2023-10-09T04:28:31.848 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../019-librgw2_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:31.893 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking librgw2 (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:31.916 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-webtest. 2023-10-09T04:28:31.929 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../045-python3-webtest_2.0.32-2_all.deb ... 2023-10-09T04:28:31.965 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-webtest (2.0.32-2) ... 2023-10-09T04:28:32.260 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-pecan. 2023-10-09T04:28:32.275 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../046-python3-pecan_1.3.3-3ubuntu1_all.deb ... 2023-10-09T04:28:32.318 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-pecan (1.3.3-3ubuntu1) ... 2023-10-09T04:28:32.656 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-werkzeug. 2023-10-09T04:28:32.670 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../047-python3-werkzeug_0.16.1+dfsg1-2ubuntu0.1_all.deb ... 2023-10-09T04:28:32.671 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-rgw. 2023-10-09T04:28:32.684 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../020-python3-rgw_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:32.714 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-10-09T04:28:32.745 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-rgw (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:33.084 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2023-10-09T04:28:33.097 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../048-libsqlite3-mod-ceph_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:33.142 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking libsqlite3-mod-ceph (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:33.478 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package ceph-mgr. 2023-10-09T04:28:33.491 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../049-ceph-mgr_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:33.536 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking ceph-mgr (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:34.031 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package ceph-mon. 2023-10-09T04:28:34.033 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package liboath0. 2023-10-09T04:28:34.044 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../050-ceph-mon_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:34.046 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../021-liboath0_2.6.1-1.3_amd64.deb ... 2023-10-09T04:28:34.091 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking liboath0 (2.6.1-1.3) ... 2023-10-09T04:28:34.314 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking ceph-mon (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:34.402 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package libradosstriper1. 2023-10-09T04:28:34.417 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../022-libradosstriper1_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:34.452 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking libradosstriper1 (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:34.897 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package ceph-common. 2023-10-09T04:28:34.911 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../023-ceph-common_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:34.955 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking ceph-common (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:35.751 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package ceph-osd. 2023-10-09T04:28:35.765 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../051-ceph-osd_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:35.809 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking ceph-osd (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:37.644 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package ceph-base. 2023-10-09T04:28:37.658 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../024-ceph-base_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:37.713 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking ceph-base (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:38.587 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package ceph. 2023-10-09T04:28:38.603 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../052-ceph_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:38.654 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking ceph (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:38.708 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-cryptography. 2023-10-09T04:28:38.721 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../025-python3-cryptography_2.8-3ubuntu0.1_amd64.deb ... 2023-10-09T04:28:38.765 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-cryptography (2.8-3ubuntu0.1) ... 2023-10-09T04:28:38.965 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package ceph-fuse. 2023-10-09T04:28:38.977 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../053-ceph-fuse_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:39.023 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking ceph-fuse (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:39.410 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-openssl. 2023-10-09T04:28:39.425 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../026-python3-openssl_19.0.0-1build1_all.deb ... 2023-10-09T04:28:39.426 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package ceph-mds. 2023-10-09T04:28:39.441 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../054-ceph-mds_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:39.477 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-openssl (19.0.0-1build1) ... 2023-10-09T04:28:39.483 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking ceph-mds (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:39.755 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2023-10-09T04:28:39.768 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../027-ceph-mgr-modules-core_17.2.6-1303-gb2903cb0-1focal_all.deb ... 2023-10-09T04:28:39.813 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking ceph-mgr-modules-core (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:40.020 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package cephadm. 2023-10-09T04:28:40.033 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../055-cephadm_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:40.078 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking cephadm (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:40.134 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-bcrypt. 2023-10-09T04:28:40.150 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../028-python3-bcrypt_3.1.7-2ubuntu1_amd64.deb ... 2023-10-09T04:28:40.192 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-bcrypt (3.1.7-2ubuntu1) ... 2023-10-09T04:28:40.389 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-jinja2. 2023-10-09T04:28:40.403 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../056-python3-jinja2_2.10.1-2_all.deb ... 2023-10-09T04:28:40.495 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-cherrypy3. 2023-10-09T04:28:40.508 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../029-python3-cherrypy3_8.9.1-6_all.deb ... 2023-10-09T04:28:40.514 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-jinja2 (2.10.1-2) ... 2023-10-09T04:28:40.553 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-cherrypy3 (8.9.1-6) ... 2023-10-09T04:28:40.850 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2023-10-09T04:28:40.863 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../057-ceph-mgr-cephadm_17.2.6-1303-gb2903cb0-1focal_all.deb ... 2023-10-09T04:28:40.908 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking ceph-mgr-cephadm (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:40.928 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-natsort. 2023-10-09T04:28:40.943 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../030-python3-natsort_7.0.1-1_all.deb ... 2023-10-09T04:28:40.981 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-natsort (7.0.1-1) ... 2023-10-09T04:28:41.261 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-repoze.lru. 2023-10-09T04:28:41.275 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../058-python3-repoze.lru_0.7-2_all.deb ... 2023-10-09T04:28:41.284 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-jwt. 2023-10-09T04:28:41.298 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../031-python3-jwt_1.7.1-2ubuntu2.1_all.deb ... 2023-10-09T04:28:41.319 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2023-10-09T04:28:41.342 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-jwt (1.7.1-2ubuntu2.1) ... 2023-10-09T04:28:41.572 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package libjs-underscore. 2023-10-09T04:28:41.585 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../059-libjs-underscore_1.9.1~dfsg-1ubuntu0.20.04.1_all.deb ... 2023-10-09T04:28:41.630 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-10-09T04:28:41.646 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-logutils. 2023-10-09T04:28:41.660 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../032-python3-logutils_0.3.3-7_all.deb ... 2023-10-09T04:28:41.704 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-logutils (0.3.3-7) ... 2023-10-09T04:28:41.916 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package libjs-sphinxdoc. 2023-10-09T04:28:41.930 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../060-libjs-sphinxdoc_1.8.5-7ubuntu3_all.deb ... 2023-10-09T04:28:41.948 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-markupsafe. 2023-10-09T04:28:41.961 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../033-python3-markupsafe_1.1.0-1build2_amd64.deb ... 2023-10-09T04:28:41.973 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-10-09T04:28:41.998 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-markupsafe (1.1.0-1build2) ... 2023-10-09T04:28:42.259 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-routes. 2023-10-09T04:28:42.274 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../061-python3-routes_2.4.1-2_all.deb ... 2023-10-09T04:28:42.301 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-mako. 2023-10-09T04:28:42.315 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../034-python3-mako_1.1.0+ds1-1ubuntu2.1_all.deb ... 2023-10-09T04:28:42.317 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-routes (2.4.1-2) ... 2023-10-09T04:28:42.359 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-10-09T04:28:42.646 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2023-10-09T04:28:42.661 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../062-ceph-mgr-dashboard_17.2.6-1303-gb2903cb0-1focal_all.deb ... 2023-10-09T04:28:42.680 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-simplegeneric. 2023-10-09T04:28:42.693 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../035-python3-simplegeneric_0.8.1-3_all.deb ... 2023-10-09T04:28:42.704 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking ceph-mgr-dashboard (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:42.738 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2023-10-09T04:28:43.032 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-singledispatch. 2023-10-09T04:28:43.045 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../036-python3-singledispatch_3.4.0.3-2build1_all.deb ... 2023-10-09T04:28:43.107 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-singledispatch (3.4.0.3-2build1) ... 2023-10-09T04:28:43.191 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-decorator. 2023-10-09T04:28:43.206 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../063-python3-decorator_4.4.2-0ubuntu1_all.deb ... 2023-10-09T04:28:43.248 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-decorator (4.4.2-0ubuntu1) ... 2023-10-09T04:28:43.401 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-webob. 2023-10-09T04:28:43.415 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../037-python3-webob_1%3a1.8.5-2_all.deb ... 2023-10-09T04:28:43.459 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-webob (1:1.8.5-2) ... 2023-10-09T04:28:43.551 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package liblbfgsb0:amd64. 2023-10-09T04:28:43.564 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../064-liblbfgsb0_3.0+dfsg.3-7build1_amd64.deb ... 2023-10-09T04:28:43.609 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-10-09T04:28:43.788 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-soupsieve. 2023-10-09T04:28:43.802 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../038-python3-soupsieve_1.9.5+dfsg-1_all.deb ... 2023-10-09T04:28:43.846 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-soupsieve (1.9.5+dfsg-1) ... 2023-10-09T04:28:43.912 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-scipy. 2023-10-09T04:28:43.927 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../065-python3-scipy_1.3.3-3build1_amd64.deb ... 2023-10-09T04:28:43.970 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-scipy (1.3.3-3build1) ... 2023-10-09T04:28:44.132 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-bs4. 2023-10-09T04:28:44.146 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../039-python3-bs4_4.8.2-1_all.deb ... 2023-10-09T04:28:44.190 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-bs4 (4.8.2-1) ... 2023-10-09T04:28:44.546 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-waitress. 2023-10-09T04:28:44.556 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../040-python3-waitress_1.4.1-1ubuntu0.1_all.deb ... 2023-10-09T04:28:44.604 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-waitress (1.4.1-1ubuntu0.1) ... 2023-10-09T04:28:44.954 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-tempita. 2023-10-09T04:28:44.968 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../041-python3-tempita_0.5.2-6_all.deb ... 2023-10-09T04:28:45.013 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-tempita (0.5.2-6) ... 2023-10-09T04:28:45.262 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-sklearn-lib. 2023-10-09T04:28:45.278 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../066-python3-sklearn-lib_0.22.2.post1+dfsg-5_amd64.deb ... 2023-10-09T04:28:45.307 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-paste. 2023-10-09T04:28:45.320 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-10-09T04:28:45.320 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../042-python3-paste_3.4.0+dfsg1-1_all.deb ... 2023-10-09T04:28:45.365 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-paste (3.4.0+dfsg1-1) ... 2023-10-09T04:28:45.668 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2023-10-09T04:28:45.683 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../043-python-pastedeploy-tpl_2.1.0-1_all.deb ... 2023-10-09T04:28:45.726 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python-pastedeploy-tpl (2.1.0-1) ... 2023-10-09T04:28:45.840 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-joblib. 2023-10-09T04:28:45.853 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../067-python3-joblib_0.14.0-3_all.deb ... 2023-10-09T04:28:45.898 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-joblib (0.14.0-3) ... 2023-10-09T04:28:46.029 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-pastedeploy. 2023-10-09T04:28:46.043 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../044-python3-pastedeploy_2.1.0-1_all.deb ... 2023-10-09T04:28:46.087 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-pastedeploy (2.1.0-1) ... 2023-10-09T04:28:46.251 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-sklearn. 2023-10-09T04:28:46.266 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../068-python3-sklearn_0.22.2.post1+dfsg-5_all.deb ... 2023-10-09T04:28:46.309 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-10-09T04:28:46.391 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-webtest. 2023-10-09T04:28:46.404 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../045-python3-webtest_2.0.32-2_all.deb ... 2023-10-09T04:28:46.449 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-webtest (2.0.32-2) ... 2023-10-09T04:28:46.760 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-pecan. 2023-10-09T04:28:46.773 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../046-python3-pecan_1.3.3-3ubuntu1_all.deb ... 2023-10-09T04:28:46.810 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-pecan (1.3.3-3ubuntu1) ... 2023-10-09T04:28:47.017 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2023-10-09T04:28:47.025 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../069-ceph-mgr-diskprediction-local_17.2.6-1303-gb2903cb0-1focal_all.deb ... 2023-10-09T04:28:47.067 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking ceph-mgr-diskprediction-local (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:47.156 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-werkzeug. 2023-10-09T04:28:47.165 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../047-python3-werkzeug_0.16.1+dfsg1-2ubuntu0.1_all.deb ... 2023-10-09T04:28:47.206 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-10-09T04:28:47.551 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2023-10-09T04:28:47.565 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../048-libsqlite3-mod-ceph_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:47.609 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking libsqlite3-mod-ceph (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:47.945 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package ceph-mgr. 2023-10-09T04:28:47.959 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../049-ceph-mgr_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:48.003 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking ceph-mgr (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:48.186 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-cachetools. 2023-10-09T04:28:48.202 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../070-python3-cachetools_4.0.0-1_all.deb ... 2023-10-09T04:28:48.244 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-cachetools (4.0.0-1) ... 2023-10-09T04:28:48.498 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package ceph-mon. 2023-10-09T04:28:48.514 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../050-ceph-mon_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:48.556 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking ceph-mon (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:48.564 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-pyasn1. 2023-10-09T04:28:48.579 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../071-python3-pyasn1_0.4.2-3build1_all.deb ... 2023-10-09T04:28:48.622 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-pyasn1 (0.4.2-3build1) ... 2023-10-09T04:28:48.958 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-pyasn1-modules. 2023-10-09T04:28:48.972 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../072-python3-pyasn1-modules_0.2.1-0.2build1_all.deb ... 2023-10-09T04:28:49.016 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-10-09T04:28:49.319 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-rsa. 2023-10-09T04:28:49.335 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../073-python3-rsa_4.0-3_all.deb ... 2023-10-09T04:28:49.377 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-rsa (4.0-3) ... 2023-10-09T04:28:49.672 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-google-auth. 2023-10-09T04:28:49.685 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package ceph-osd. 2023-10-09T04:28:49.687 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../074-python3-google-auth_1.5.1-2_all.deb ... 2023-10-09T04:28:49.698 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../051-ceph-osd_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:49.730 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-google-auth (1.5.1-2) ... 2023-10-09T04:28:49.743 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking ceph-osd (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:50.083 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-blinker. 2023-10-09T04:28:50.099 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../075-python3-blinker_1.4+dfsg1-0.3ubuntu1_all.deb ... 2023-10-09T04:28:50.141 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-10-09T04:28:50.444 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-oauthlib. 2023-10-09T04:28:50.458 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../076-python3-oauthlib_3.1.0-1ubuntu2_all.deb ... 2023-10-09T04:28:50.502 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-oauthlib (3.1.0-1ubuntu2) ... 2023-10-09T04:28:50.830 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-requests-oauthlib. 2023-10-09T04:28:50.844 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../077-python3-requests-oauthlib_1.0.0-1.1build1_all.deb ... 2023-10-09T04:28:50.888 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-10-09T04:28:51.199 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-websocket. 2023-10-09T04:28:51.215 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../078-python3-websocket_0.53.0-2ubuntu1_all.deb ... 2023-10-09T04:28:51.257 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-websocket (0.53.0-2ubuntu1) ... 2023-10-09T04:28:51.560 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-kubernetes. 2023-10-09T04:28:51.574 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../079-python3-kubernetes_7.0.0~a1-2_all.deb ... 2023-10-09T04:28:51.618 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-kubernetes (7.0.0~a1-2) ... 2023-10-09T04:28:52.157 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2023-10-09T04:28:52.170 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../080-ceph-mgr-k8sevents_17.2.6-1303-gb2903cb0-1focal_all.deb ... 2023-10-09T04:28:52.240 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking ceph-mgr-k8sevents (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:52.513 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package ceph. 2023-10-09T04:28:52.527 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../052-ceph_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:52.571 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking ceph (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:52.644 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package libonig5:amd64. 2023-10-09T04:28:52.658 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../081-libonig5_6.9.4-1_amd64.deb ... 2023-10-09T04:28:52.702 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking libonig5:amd64 (6.9.4-1) ... 2023-10-09T04:28:52.849 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package ceph-fuse. 2023-10-09T04:28:52.862 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../053-ceph-fuse_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:52.899 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking ceph-fuse (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:53.064 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package libjq1:amd64. 2023-10-09T04:28:53.078 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../082-libjq1_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-10-09T04:28:53.122 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-10-09T04:28:53.302 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package ceph-mds. 2023-10-09T04:28:53.315 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../054-ceph-mds_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:53.360 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking ceph-mds (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:53.383 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package jq. 2023-10-09T04:28:53.399 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../083-jq_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-10-09T04:28:53.441 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking jq (1.6-1ubuntu0.20.04.1) ... 2023-10-09T04:28:53.677 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package socat. 2023-10-09T04:28:53.690 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../084-socat_1.7.3.3-2_amd64.deb ... 2023-10-09T04:28:53.735 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking socat (1.7.3.3-2) ... 2023-10-09T04:28:53.906 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package cephadm. 2023-10-09T04:28:53.918 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../055-cephadm_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:53.955 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking cephadm (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:54.061 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package xmlstarlet. 2023-10-09T04:28:54.074 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../085-xmlstarlet_1.6.1-2build1_amd64.deb ... 2023-10-09T04:28:54.119 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking xmlstarlet (1.6.1-2build1) ... 2023-10-09T04:28:54.259 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-jinja2. 2023-10-09T04:28:54.272 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../056-python3-jinja2_2.10.1-2_all.deb ... 2023-10-09T04:28:54.317 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-jinja2 (2.10.1-2) ... 2023-10-09T04:28:54.464 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package ceph-test. 2023-10-09T04:28:54.474 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../086-ceph-test_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:28:54.514 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking ceph-test (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:54.652 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2023-10-09T04:28:54.666 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../057-ceph-mgr-cephadm_17.2.6-1303-gb2903cb0-1focal_all.deb ... 2023-10-09T04:28:54.709 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking ceph-mgr-cephadm (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:55.105 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-repoze.lru. 2023-10-09T04:28:55.118 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../058-python3-repoze.lru_0.7-2_all.deb ... 2023-10-09T04:28:55.163 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2023-10-09T04:28:55.415 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package libjs-underscore. 2023-10-09T04:28:55.430 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../059-libjs-underscore_1.9.1~dfsg-1ubuntu0.20.04.1_all.deb ... 2023-10-09T04:28:55.473 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-10-09T04:28:55.751 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package libjs-sphinxdoc. 2023-10-09T04:28:55.764 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../060-libjs-sphinxdoc_1.8.5-7ubuntu3_all.deb ... 2023-10-09T04:28:55.809 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-10-09T04:28:56.087 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-routes. 2023-10-09T04:28:56.102 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../061-python3-routes_2.4.1-2_all.deb ... 2023-10-09T04:28:56.136 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-routes (2.4.1-2) ... 2023-10-09T04:28:56.490 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2023-10-09T04:28:56.504 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../062-ceph-mgr-dashboard_17.2.6-1303-gb2903cb0-1focal_all.deb ... 2023-10-09T04:28:56.548 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking ceph-mgr-dashboard (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:57.043 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-decorator. 2023-10-09T04:28:57.058 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../063-python3-decorator_4.4.2-0ubuntu1_all.deb ... 2023-10-09T04:28:57.101 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-decorator (4.4.2-0ubuntu1) ... 2023-10-09T04:28:57.404 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package liblbfgsb0:amd64. 2023-10-09T04:28:57.417 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../064-liblbfgsb0_3.0+dfsg.3-7build1_amd64.deb ... 2023-10-09T04:28:57.470 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-10-09T04:28:57.790 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-scipy. 2023-10-09T04:28:57.803 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../065-python3-scipy_1.3.3-3build1_amd64.deb ... 2023-10-09T04:28:57.847 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-scipy (1.3.3-3build1) ... 2023-10-09T04:28:59.156 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-sklearn-lib. 2023-10-09T04:28:59.164 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../066-python3-sklearn-lib_0.22.2.post1+dfsg-5_amd64.deb ... 2023-10-09T04:28:59.206 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-10-09T04:28:59.509 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package ceph-volume. 2023-10-09T04:28:59.523 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../087-ceph-volume_17.2.6-1303-gb2903cb0-1focal_all.deb ... 2023-10-09T04:28:59.567 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking ceph-volume (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:28:59.760 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-joblib. 2023-10-09T04:28:59.773 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../067-python3-joblib_0.14.0-3_all.deb ... 2023-10-09T04:28:59.818 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-joblib (0.14.0-3) ... 2023-10-09T04:28:59.908 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package fonts-mathjax. 2023-10-09T04:28:59.917 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../088-fonts-mathjax_2.7.4+dfsg-1_all.deb ... 2023-10-09T04:28:59.963 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking fonts-mathjax (2.7.4+dfsg-1) ... 2023-10-09T04:29:00.223 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-sklearn. 2023-10-09T04:29:00.237 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../068-python3-sklearn_0.22.2.post1+dfsg-5_all.deb ... 2023-10-09T04:29:00.281 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-10-09T04:29:00.500 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package libcephfs-dev. 2023-10-09T04:29:00.509 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../089-libcephfs-dev_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:29:00.550 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking libcephfs-dev (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:00.953 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package nvme-cli. 2023-10-09T04:29:00.962 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../090-nvme-cli_1.9-1ubuntu0.1_amd64.deb ... 2023-10-09T04:29:00.964 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2023-10-09T04:29:00.979 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../069-ceph-mgr-diskprediction-local_17.2.6-1303-gb2903cb0-1focal_all.deb ... 2023-10-09T04:29:01.019 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking nvme-cli (1.9-1ubuntu0.1) ... 2023-10-09T04:29:01.030 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking ceph-mgr-diskprediction-local (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:01.345 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package libjs-mathjax. 2023-10-09T04:29:01.358 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../091-libjs-mathjax_2.7.4+dfsg-1_all.deb ... 2023-10-09T04:29:01.403 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking libjs-mathjax (2.7.4+dfsg-1) ... 2023-10-09T04:29:02.125 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-cachetools. 2023-10-09T04:29:02.139 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../070-python3-cachetools_4.0.0-1_all.deb ... 2023-10-09T04:29:02.183 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-cachetools (4.0.0-1) ... 2023-10-09T04:29:02.486 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-pyasn1. 2023-10-09T04:29:02.499 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../071-python3-pyasn1_0.4.2-3build1_all.deb ... 2023-10-09T04:29:02.544 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-pyasn1 (0.4.2-3build1) ... 2023-10-09T04:29:02.880 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-pyasn1-modules. 2023-10-09T04:29:02.895 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../072-python3-pyasn1-modules_0.2.1-0.2build1_all.deb ... 2023-10-09T04:29:02.938 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-10-09T04:29:03.021 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python-natsort-doc. 2023-10-09T04:29:03.036 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../092-python-natsort-doc_7.0.1-1_all.deb ... 2023-10-09T04:29:03.079 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python-natsort-doc (7.0.1-1) ... 2023-10-09T04:29:03.258 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-rsa. 2023-10-09T04:29:03.274 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../073-python3-rsa_4.0-3_all.deb ... 2023-10-09T04:29:03.316 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-rsa (4.0-3) ... 2023-10-09T04:29:03.423 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-atomicwrites. 2023-10-09T04:29:03.438 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../093-python3-atomicwrites_1.1.5-2build1_all.deb ... 2023-10-09T04:29:03.481 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-atomicwrites (1.1.5-2build1) ... 2023-10-09T04:29:03.628 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-google-auth. 2023-10-09T04:29:03.643 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../074-python3-google-auth_1.5.1-2_all.deb ... 2023-10-09T04:29:03.686 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-google-auth (1.5.1-2) ... 2023-10-09T04:29:03.818 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-attr. 2023-10-09T04:29:03.832 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../094-python3-attr_19.3.0-2_all.deb ... 2023-10-09T04:29:03.875 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-attr (19.3.0-2) ... 2023-10-09T04:29:04.013 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-blinker. 2023-10-09T04:29:04.022 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../075-python3-blinker_1.4+dfsg1-0.3ubuntu1_all.deb ... 2023-10-09T04:29:04.063 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-10-09T04:29:04.228 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-webencodings. 2023-10-09T04:29:04.242 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../095-python3-webencodings_0.5.1-1ubuntu1_all.deb ... 2023-10-09T04:29:04.286 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-webencodings (0.5.1-1ubuntu1) ... 2023-10-09T04:29:04.324 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-oauthlib. 2023-10-09T04:29:04.333 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../076-python3-oauthlib_3.1.0-1ubuntu2_all.deb ... 2023-10-09T04:29:04.366 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-oauthlib (3.1.0-1ubuntu2) ... 2023-10-09T04:29:04.589 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-html5lib. 2023-10-09T04:29:04.603 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../096-python3-html5lib_1.0.1-2_all.deb ... 2023-10-09T04:29:04.645 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-requests-oauthlib. 2023-10-09T04:29:04.647 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-html5lib (1.0.1-2) ... 2023-10-09T04:29:04.654 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../077-python3-requests-oauthlib_1.0.0-1.1build1_all.deb ... 2023-10-09T04:29:04.686 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-10-09T04:29:04.964 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-websocket. 2023-10-09T04:29:04.977 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../078-python3-websocket_0.53.0-2ubuntu1_all.deb ... 2023-10-09T04:29:04.992 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-lxml:amd64. 2023-10-09T04:29:05.006 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../097-python3-lxml_4.5.0-1ubuntu0.5_amd64.deb ... 2023-10-09T04:29:05.022 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-websocket (0.53.0-2ubuntu1) ... 2023-10-09T04:29:05.058 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-10-09T04:29:05.341 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-kubernetes. 2023-10-09T04:29:05.356 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../079-python3-kubernetes_7.0.0~a1-2_all.deb ... 2023-10-09T04:29:05.400 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-kubernetes (7.0.0~a1-2) ... 2023-10-09T04:29:05.528 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-packaging. 2023-10-09T04:29:05.543 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../098-python3-packaging_20.3-1_all.deb ... 2023-10-09T04:29:05.586 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-packaging (20.3-1) ... 2023-10-09T04:29:05.872 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-pastescript. 2023-10-09T04:29:05.887 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../099-python3-pastescript_2.0.2-4_all.deb ... 2023-10-09T04:29:05.930 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2023-10-09T04:29:05.954 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2023-10-09T04:29:05.970 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../080-ceph-mgr-k8sevents_17.2.6-1303-gb2903cb0-1focal_all.deb ... 2023-10-09T04:29:06.012 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking ceph-mgr-k8sevents (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:06.431 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package libonig5:amd64. 2023-10-09T04:29:06.445 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../081-libonig5_6.9.4-1_amd64.deb ... 2023-10-09T04:29:06.481 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking libonig5:amd64 (6.9.4-1) ... 2023-10-09T04:29:06.568 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-pluggy. 2023-10-09T04:29:06.582 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../100-python3-pluggy_0.13.0-2_all.deb ... 2023-10-09T04:29:06.626 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-pluggy (0.13.0-2) ... 2023-10-09T04:29:06.842 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package libjq1:amd64. 2023-10-09T04:29:06.856 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../082-libjq1_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-10-09T04:29:06.900 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-10-09T04:29:06.920 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-psutil. 2023-10-09T04:29:06.935 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../101-python3-psutil_5.5.1-1ubuntu4_amd64.deb ... 2023-10-09T04:29:06.978 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-psutil (5.5.1-1ubuntu4) ... 2023-10-09T04:29:07.195 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package jq. 2023-10-09T04:29:07.210 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../083-jq_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-10-09T04:29:07.253 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking jq (1.6-1ubuntu0.20.04.1) ... 2023-10-09T04:29:07.306 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-py. 2023-10-09T04:29:07.320 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../102-python3-py_1.8.1-1ubuntu0.1_all.deb ... 2023-10-09T04:29:07.364 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-py (1.8.1-1ubuntu0.1) ... 2023-10-09T04:29:07.506 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package socat. 2023-10-09T04:29:07.520 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../084-socat_1.7.3.3-2_amd64.deb ... 2023-10-09T04:29:07.564 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking socat (1.7.3.3-2) ... 2023-10-09T04:29:07.717 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-pyinotify. 2023-10-09T04:29:07.731 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../103-python3-pyinotify_0.9.6-1.2ubuntu1_all.deb ... 2023-10-09T04:29:07.775 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-10-09T04:29:07.934 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package xmlstarlet. 2023-10-09T04:29:07.950 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../085-xmlstarlet_1.6.1-2build1_amd64.deb ... 2023-10-09T04:29:08.000 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking xmlstarlet (1.6.1-2build1) ... 2023-10-09T04:29:08.089 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-wcwidth. 2023-10-09T04:29:08.101 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../104-python3-wcwidth_0.1.8+dfsg1-3_all.deb ... 2023-10-09T04:29:08.145 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-wcwidth (0.1.8+dfsg1-3) ... 2023-10-09T04:29:08.355 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package ceph-test. 2023-10-09T04:29:08.371 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../086-ceph-test_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:29:08.413 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking ceph-test (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:08.448 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-pytest. 2023-10-09T04:29:08.462 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../105-python3-pytest_4.6.9-1_all.deb ... 2023-10-09T04:29:08.506 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-pytest (4.6.9-1) ... 2023-10-09T04:29:08.851 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package python3-simplejson. 2023-10-09T04:29:08.867 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../106-python3-simplejson_3.16.0-2ubuntu2_amd64.deb ... 2023-10-09T04:29:08.909 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking python3-simplejson (3.16.0-2ubuntu2) ... 2023-10-09T04:29:09.182 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package qttranslations5-l10n. 2023-10-09T04:29:09.196 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../107-qttranslations5-l10n_5.12.8-0ubuntu1_all.deb ... 2023-10-09T04:29:09.240 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking qttranslations5-l10n (5.12.8-0ubuntu1) ... 2023-10-09T04:29:09.994 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package radosgw. 2023-10-09T04:29:10.010 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../108-radosgw_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:29:10.052 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking radosgw (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:11.788 INFO:teuthology.orchestra.run.smithi194.stdout:Selecting previously unselected package rbd-fuse. 2023-10-09T04:29:11.802 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing to unpack .../109-rbd-fuse_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:29:11.846 INFO:teuthology.orchestra.run.smithi194.stdout:Unpacking rbd-fuse (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:12.151 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-attr (19.3.0-2) ... 2023-10-09T04:29:12.443 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up librabbitmq4:amd64 (0.10.0-1) ... 2023-10-09T04:29:12.569 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up libdouble-conversion3:amd64 (3.1.5-4ubuntu1) ... 2023-10-09T04:29:12.695 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up nvme-cli (1.9-1ubuntu0.1) ... 2023-10-09T04:29:12.888 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up cephadm (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:13.411 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package ceph-volume. 2023-10-09T04:29:13.426 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../087-ceph-volume_17.2.6-1303-gb2903cb0-1focal_all.deb ... 2023-10-09T04:29:13.468 INFO:teuthology.orchestra.run.smithi194.stdout:Adding system user cephadm....done 2023-10-09T04:29:13.469 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking ceph-volume (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:13.623 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up fonts-mathjax (2.7.4+dfsg-1) ... 2023-10-09T04:29:13.740 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-waitress (1.4.1-1ubuntu0.1) ... 2023-10-09T04:29:13.801 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package fonts-mathjax. 2023-10-09T04:29:13.816 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../088-fonts-mathjax_2.7.4+dfsg-1_all.deb ... 2023-10-09T04:29:13.859 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking fonts-mathjax (2.7.4+dfsg-1) ... 2023-10-09T04:29:14.085 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-repoze.lru (0.7-2) ... 2023-10-09T04:29:14.367 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up libjs-mathjax (2.7.4+dfsg-1) ... 2023-10-09T04:29:14.394 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package libcephfs-dev. 2023-10-09T04:29:14.410 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../089-libcephfs-dev_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:29:14.452 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking libcephfs-dev (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:14.485 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up liboath0 (2.6.1-1.3) ... 2023-10-09T04:29:14.613 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-py (1.8.1-1ubuntu0.1) ... 2023-10-09T04:29:14.872 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package nvme-cli. 2023-10-09T04:29:14.887 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../090-nvme-cli_1.9-1ubuntu0.1_amd64.deb ... 2023-10-09T04:29:14.946 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking nvme-cli (1.9-1ubuntu0.1) ... 2023-10-09T04:29:14.947 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-joblib (0.14.0-3) ... 2023-10-09T04:29:15.277 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package libjs-mathjax. 2023-10-09T04:29:15.290 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../091-libjs-mathjax_2.7.4+dfsg-1_all.deb ... 2023-10-09T04:29:15.335 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking libjs-mathjax (2.7.4+dfsg-1) ... 2023-10-09T04:29:15.349 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-jwt (1.7.1-2ubuntu2.1) ... 2023-10-09T04:29:15.651 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-cachetools (4.0.0-1) ... 2023-10-09T04:29:15.935 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-bcrypt (3.1.7-2ubuntu1) ... 2023-10-09T04:29:16.219 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-10-09T04:29:16.512 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-ceph-argparse (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:16.813 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-10-09T04:29:16.939 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-markupsafe (1.1.0-1build2) ... 2023-10-09T04:29:16.987 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python-natsort-doc. 2023-10-09T04:29:17.002 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../092-python-natsort-doc_7.0.1-1_all.deb ... 2023-10-09T04:29:17.036 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python-natsort-doc (7.0.1-1) ... 2023-10-09T04:29:17.223 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-webencodings (0.5.1-1ubuntu1) ... 2023-10-09T04:29:17.373 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-atomicwrites. 2023-10-09T04:29:17.387 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../093-python3-atomicwrites_1.1.5-2build1_all.deb ... 2023-10-09T04:29:17.430 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-atomicwrites (1.1.5-2build1) ... 2023-10-09T04:29:17.508 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up libpcre2-16-0:amd64 (10.34-7ubuntu0.1) ... 2023-10-09T04:29:17.633 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-psutil (5.5.1-1ubuntu4) ... 2023-10-09T04:29:17.733 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-attr. 2023-10-09T04:29:17.747 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../094-python3-attr_19.3.0-2_all.deb ... 2023-10-09T04:29:17.783 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-attr (19.3.0-2) ... 2023-10-09T04:29:18.010 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-natsort (7.0.1-1) ... 2023-10-09T04:29:18.119 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-webencodings. 2023-10-09T04:29:18.133 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../095-python3-webencodings_0.5.1-1ubuntu1_all.deb ... 2023-10-09T04:29:18.177 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-webencodings (0.5.1-1ubuntu1) ... 2023-10-09T04:29:18.302 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-atomicwrites (1.1.5-2build1) ... 2023-10-09T04:29:18.464 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-html5lib. 2023-10-09T04:29:18.478 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../096-python3-html5lib_1.0.1-2_all.deb ... 2023-10-09T04:29:18.522 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-html5lib (1.0.1-2) ... 2023-10-09T04:29:18.578 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-simplejson (3.16.0-2ubuntu2) ... 2023-10-09T04:29:18.875 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-lxml:amd64. 2023-10-09T04:29:18.890 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../097-python3-lxml_4.5.0-1ubuntu0.5_amd64.deb ... 2023-10-09T04:29:18.913 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-decorator (4.4.2-0ubuntu1) ... 2023-10-09T04:29:18.942 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-10-09T04:29:19.197 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-jinja2 (2.10.1-2) ... 2023-10-09T04:29:19.403 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-packaging. 2023-10-09T04:29:19.419 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../098-python3-packaging_20.3-1_all.deb ... 2023-10-09T04:29:19.461 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-packaging (20.3-1) ... 2023-10-09T04:29:19.531 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-tempita (0.5.2-6) ... 2023-10-09T04:29:19.764 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-pastescript. 2023-10-09T04:29:19.778 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../099-python3-pastescript_2.0.2-4_all.deb ... 2023-10-09T04:29:19.807 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-packaging (20.3-1) ... 2023-10-09T04:29:19.823 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2023-10-09T04:29:20.101 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python-pastedeploy-tpl (2.1.0-1) ... 2023-10-09T04:29:20.144 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-pluggy. 2023-10-09T04:29:20.157 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../100-python3-pluggy_0.13.0-2_all.deb ... 2023-10-09T04:29:20.201 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-pluggy (0.13.0-2) ... 2023-10-09T04:29:20.227 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up qttranslations5-l10n (5.12.8-0ubuntu1) ... 2023-10-09T04:29:20.352 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-wcwidth (0.1.8+dfsg1-3) ... 2023-10-09T04:29:20.637 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-paste (3.4.0+dfsg1-1) ... 2023-10-09T04:29:20.662 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-psutil. 2023-10-09T04:29:20.679 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../101-python3-psutil_5.5.1-1ubuntu4_amd64.deb ... 2023-10-09T04:29:20.720 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-psutil (5.5.1-1ubuntu4) ... 2023-10-09T04:29:21.030 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-10-09T04:29:21.040 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-py. 2023-10-09T04:29:21.056 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../102-python3-py_1.8.1-1ubuntu0.1_all.deb ... 2023-10-09T04:29:21.099 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-py (1.8.1-1ubuntu0.1) ... 2023-10-09T04:29:21.383 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-cryptography (2.8-3ubuntu0.1) ... 2023-10-09T04:29:21.421 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-pyinotify. 2023-10-09T04:29:21.436 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../103-python3-pyinotify_0.9.6-1.2ubuntu1_all.deb ... 2023-10-09T04:29:21.479 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-10-09T04:29:21.773 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-wcwidth. 2023-10-09T04:29:21.788 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../104-python3-wcwidth_0.1.8+dfsg1-3_all.deb ... 2023-10-09T04:29:21.819 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up socat (1.7.3.3-2) ... 2023-10-09T04:29:21.831 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-wcwidth (0.1.8+dfsg1-3) ... 2023-10-09T04:29:21.945 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-ceph-common (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:22.143 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-pytest. 2023-10-09T04:29:22.157 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../105-python3-pytest_4.6.9-1_all.deb ... 2023-10-09T04:29:22.201 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-pytest (4.6.9-1) ... 2023-10-09T04:29:22.271 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up libqt5core5a:amd64 (5.12.8+dfsg-0ubuntu2.1) ... 2023-10-09T04:29:22.405 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-html5lib (1.0.1-2) ... 2023-10-09T04:29:22.546 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package python3-simplejson. 2023-10-09T04:29:22.563 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../106-python3-simplejson_3.16.0-2ubuntu2_amd64.deb ... 2023-10-09T04:29:22.596 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking python3-simplejson (3.16.0-2ubuntu2) ... 2023-10-09T04:29:22.740 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-10-09T04:29:22.857 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package qttranslations5-l10n. 2023-10-09T04:29:22.869 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up xmlstarlet (1.6.1-2build1) ... 2023-10-09T04:29:22.871 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../107-qttranslations5-l10n_5.12.8-0ubuntu1_all.deb ... 2023-10-09T04:29:22.915 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking qttranslations5-l10n (5.12.8-0ubuntu1) ... 2023-10-09T04:29:23.012 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-pluggy (0.13.0-2) ... 2023-10-09T04:29:23.296 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-10-09T04:29:23.614 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up libqt5dbus5:amd64 (5.12.8+dfsg-0ubuntu2.1) ... 2023-10-09T04:29:23.731 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-10-09T04:29:23.744 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package radosgw. 2023-10-09T04:29:23.760 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../108-radosgw_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:29:23.802 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking radosgw (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:23.857 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-pyasn1 (0.4.2-3build1) ... 2023-10-09T04:29:24.192 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-singledispatch (3.4.0.3-2build1) ... 2023-10-09T04:29:24.476 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-logutils (0.3.3-7) ... 2023-10-09T04:29:24.771 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up liblttng-ust0:amd64 (2.11.0-1) ... 2023-10-09T04:29:24.905 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2023-10-09T04:29:25.206 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-prettytable (0.7.2-5) ... 2023-10-09T04:29:25.491 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-soupsieve (1.9.5+dfsg-1) ... 2023-10-09T04:29:25.550 INFO:teuthology.orchestra.run.smithi133.stdout:Selecting previously unselected package rbd-fuse. 2023-10-09T04:29:25.565 INFO:teuthology.orchestra.run.smithi133.stdout:Preparing to unpack .../109-rbd-fuse_17.2.6-1303-gb2903cb0-1focal_amd64.deb ... 2023-10-09T04:29:25.619 INFO:teuthology.orchestra.run.smithi133.stdout:Unpacking rbd-fuse (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:25.786 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-websocket (0.53.0-2ubuntu1) ... 2023-10-09T04:29:25.954 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-attr (19.3.0-2) ... 2023-10-09T04:29:26.161 INFO:teuthology.orchestra.run.smithi194.stdout:update-alternatives: using /usr/bin/python3-wsdump to provide /usr/bin/wsdump (wsdump) in auto mode 2023-10-09T04:29:26.249 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up librabbitmq4:amd64 (0.10.0-1) ... 2023-10-09T04:29:26.375 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up libdouble-conversion3:amd64 (3.1.5-4ubuntu1) ... 2023-10-09T04:29:26.394 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up libonig5:amd64 (6.9.4-1) ... 2023-10-09T04:29:26.501 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up nvme-cli (1.9-1ubuntu0.1) ... 2023-10-09T04:29:26.520 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-10-09T04:29:26.696 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up cephadm (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:26.829 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-cherrypy3 (8.9.1-6) ... 2023-10-09T04:29:27.189 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-10-09T04:29:27.315 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-10-09T04:29:27.332 INFO:teuthology.orchestra.run.smithi133.stdout:Adding system user cephadm....done 2023-10-09T04:29:27.441 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-webob (1:1.8.5-2) ... 2023-10-09T04:29:27.493 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up fonts-mathjax (2.7.4+dfsg-1) ... 2023-10-09T04:29:27.603 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-waitress (1.4.1-1ubuntu0.1) ... 2023-10-09T04:29:27.759 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-10-09T04:29:27.946 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-repoze.lru (0.7-2) ... 2023-10-09T04:29:28.052 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up librados2 (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:28.169 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-scipy (1.3.3-3build1) ... 2023-10-09T04:29:28.247 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up libjs-mathjax (2.7.4+dfsg-1) ... 2023-10-09T04:29:28.373 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up liboath0 (2.6.1-1.3) ... 2023-10-09T04:29:28.491 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-py (1.8.1-1ubuntu0.1) ... 2023-10-09T04:29:28.792 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-joblib (0.14.0-3) ... 2023-10-09T04:29:29.194 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-jwt (1.7.1-2ubuntu2.1) ... 2023-10-09T04:29:29.487 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-cachetools (4.0.0-1) ... 2023-10-09T04:29:29.641 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up libsqlite3-mod-ceph (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:29.772 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-bcrypt (3.1.7-2ubuntu1) ... 2023-10-09T04:29:29.775 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-10-09T04:29:29.893 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-pytest (4.6.9-1) ... 2023-10-09T04:29:30.048 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-10-09T04:29:30.261 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-openssl (19.0.0-1build1) ... 2023-10-09T04:29:30.340 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-ceph-argparse (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:30.562 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-pastedeploy (2.1.0-1) ... 2023-10-09T04:29:30.625 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-10-09T04:29:30.751 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-markupsafe (1.1.0-1build2) ... 2023-10-09T04:29:30.880 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up libqt5network5:amd64 (5.12.8+dfsg-0ubuntu2.1) ... 2023-10-09T04:29:31.005 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-bs4 (4.8.2-1) ... 2023-10-09T04:29:31.060 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-webencodings (0.5.1-1ubuntu1) ... 2023-10-09T04:29:31.320 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up libpcre2-16-0:amd64 (10.34-7ubuntu0.1) ... 2023-10-09T04:29:31.331 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up libcephfs2 (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:31.446 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-psutil (5.5.1-1ubuntu4) ... 2023-10-09T04:29:31.457 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-oauthlib (3.1.0-1ubuntu2) ... 2023-10-09T04:29:31.776 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up libradosstriper1 (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:31.831 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-natsort (7.0.1-1) ... 2023-10-09T04:29:31.902 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-10-09T04:29:32.125 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-atomicwrites (1.1.5-2build1) ... 2023-10-09T04:29:32.228 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-10-09T04:29:32.402 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-simplejson (3.16.0-2ubuntu2) ... 2023-10-09T04:29:32.728 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-decorator (4.4.2-0ubuntu1) ... 2023-10-09T04:29:33.013 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-jinja2 (2.10.1-2) ... 2023-10-09T04:29:33.266 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up librbd1 (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:33.347 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-tempita (0.5.2-6) ... 2023-10-09T04:29:33.392 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-10-09T04:29:33.518 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python-natsort-doc (7.0.1-1) ... 2023-10-09T04:29:33.640 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-packaging (20.3-1) ... 2023-10-09T04:29:33.644 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up jq (1.6-1ubuntu0.20.04.1) ... 2023-10-09T04:29:33.770 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-rsa (4.0-3) ... 2023-10-09T04:29:33.924 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python-pastedeploy-tpl (2.1.0-1) ... 2023-10-09T04:29:34.053 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up qttranslations5-l10n (5.12.8-0ubuntu1) ... 2023-10-09T04:29:34.071 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up libthrift-0.13.0 (0.13.0-2build2) ... 2023-10-09T04:29:34.179 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-wcwidth (0.1.8+dfsg1-3) ... 2023-10-09T04:29:34.197 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-webtest (2.0.32-2) ... 2023-10-09T04:29:34.464 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-paste (3.4.0+dfsg1-1) ... 2023-10-09T04:29:34.506 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up ceph-mgr-modules-core (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:34.632 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up ceph-fuse (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:34.857 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-10-09T04:29:34.859 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-10-09T04:29:34.859 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-10-09T04:29:35.217 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-cryptography (2.8-3ubuntu0.1) ... 2023-10-09T04:29:35.350 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up libcephfs-dev (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:35.593 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up socat (1.7.3.3-2) ... 2023-10-09T04:29:35.595 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-10-09T04:29:35.721 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-ceph-common (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:35.897 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-pastescript (2.0.2-4) ... 2023-10-09T04:29:36.040 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up libqt5core5a:amd64 (5.12.8+dfsg-0ubuntu2.1) ... 2023-10-09T04:29:36.174 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-html5lib (1.0.1-2) ... 2023-10-09T04:29:36.374 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-pecan (1.3.3-3ubuntu1) ... 2023-10-09T04:29:36.525 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-10-09T04:29:36.651 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up xmlstarlet (1.6.1-2build1) ... 2023-10-09T04:29:36.735 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-rados (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:36.769 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-pluggy (0.13.0-2) ... 2023-10-09T04:29:36.861 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up librgw2 (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:36.987 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-rbd (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:37.045 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-10-09T04:29:37.363 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up libqt5dbus5:amd64 (5.12.8+dfsg-0ubuntu2.1) ... 2023-10-09T04:29:37.479 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up rbd-fuse (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:37.489 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-10-09T04:29:37.598 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-pyasn1 (0.4.2-3build1) ... 2023-10-09T04:29:37.605 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-routes (2.4.1-2) ... 2023-10-09T04:29:37.898 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-google-auth (1.5.1-2) ... 2023-10-09T04:29:37.941 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-singledispatch (3.4.0.3-2build1) ... 2023-10-09T04:29:38.199 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-rgw (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:38.217 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-logutils (0.3.3-7) ... 2023-10-09T04:29:38.344 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-cephfs (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:38.470 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up python3-kubernetes (7.0.0~a1-2) ... 2023-10-09T04:29:38.535 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up liblttng-ust0:amd64 (2.11.0-1) ... 2023-10-09T04:29:38.669 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2023-10-09T04:29:38.970 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-prettytable (0.7.2-5) ... 2023-10-09T04:29:39.263 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-soupsieve (1.9.5+dfsg-1) ... 2023-10-09T04:29:39.398 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up ceph-common (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:39.564 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-websocket (0.53.0-2ubuntu1) ... 2023-10-09T04:29:39.946 INFO:teuthology.orchestra.run.smithi194.stdout:Adding group ceph....done 2023-10-09T04:29:39.959 INFO:teuthology.orchestra.run.smithi133.stdout:update-alternatives: using /usr/bin/python3-wsdump to provide /usr/bin/wsdump (wsdump) in auto mode 2023-10-09T04:29:40.191 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up libonig5:amd64 (6.9.4-1) ... 2023-10-09T04:29:40.317 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-10-09T04:29:40.322 INFO:teuthology.orchestra.run.smithi194.stdout:Adding system user ceph....done 2023-10-09T04:29:40.514 INFO:teuthology.orchestra.run.smithi194.stdout:Setting system user ceph properties....done 2023-10-09T04:29:40.539 INFO:teuthology.orchestra.run.smithi194.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2023-10-09T04:29:40.643 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-cherrypy3 (8.9.1-6) ... 2023-10-09T04:29:40.685 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2023-10-09T04:29:40.995 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-10-09T04:29:41.006 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2023-10-09T04:29:41.120 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-10-09T04:29:41.247 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-webob (1:1.8.5-2) ... 2023-10-09T04:29:41.494 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up ceph-test (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:41.556 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-10-09T04:29:41.622 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up radosgw (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:41.832 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up librados2 (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:41.889 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-10-09T04:29:41.889 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-10-09T04:29:41.958 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-scipy (1.3.3-3build1) ... 2023-10-09T04:29:42.571 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up ceph-base (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:42.963 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2023-10-09T04:29:43.423 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up ceph-mds (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:43.455 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up libsqlite3-mod-ceph (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:43.573 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-10-09T04:29:43.649 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-10-09T04:29:43.650 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-10-09T04:29:43.698 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-pytest (4.6.9-1) ... 2023-10-09T04:29:44.075 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-openssl (19.0.0-1build1) ... 2023-10-09T04:29:44.100 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up ceph-mgr (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:44.330 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-10-09T04:29:44.330 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-10-09T04:29:44.376 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-pastedeploy (2.1.0-1) ... 2023-10-09T04:29:44.652 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up libqt5network5:amd64 (5.12.8+dfsg-0ubuntu2.1) ... 2023-10-09T04:29:44.786 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-bs4 (4.8.2-1) ... 2023-10-09T04:29:44.794 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up ceph-osd (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:45.113 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up libcephfs2 (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:45.238 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-oauthlib (3.1.0-1ubuntu2) ... 2023-10-09T04:29:45.291 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-10-09T04:29:45.292 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-10-09T04:29:45.548 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up libradosstriper1 (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:45.674 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-10-09T04:29:45.763 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up ceph-mgr-k8sevents (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:45.889 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up ceph-mgr-diskprediction-local (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:46.009 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-10-09T04:29:46.057 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up ceph-mon (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:46.283 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-10-09T04:29:46.283 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-10-09T04:29:46.750 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up ceph-mgr-cephadm (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:46.884 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up ceph (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:47.018 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up ceph-mgr-dashboard (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:47.122 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up librbd1 (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:47.161 INFO:teuthology.orchestra.run.smithi194.stdout:Setting up ceph-volume (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:47.240 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-10-09T04:29:47.288 INFO:teuthology.orchestra.run.smithi194.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-10-09T04:29:47.366 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python-natsort-doc (7.0.1-1) ... 2023-10-09T04:29:47.431 INFO:teuthology.orchestra.run.smithi194.stdout:Processing triggers for systemd (245.4-4ubuntu3.17) ... 2023-10-09T04:29:47.492 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up jq (1.6-1ubuntu0.20.04.1) ... 2023-10-09T04:29:47.617 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-rsa (4.0-3) ... 2023-10-09T04:29:47.750 INFO:teuthology.orchestra.run.smithi194.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-10-09T04:29:47.902 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up libthrift-0.13.0 (0.13.0-2build2) ... 2023-10-09T04:29:48.011 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-webtest (2.0.32-2) ... 2023-10-09T04:29:48.312 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up ceph-mgr-modules-core (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:48.438 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up ceph-fuse (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:48.651 INFO:teuthology.orchestra.run.smithi133.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-10-09T04:29:48.651 INFO:teuthology.orchestra.run.smithi133.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-10-09T04:29:49.114 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up libcephfs-dev (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:49.326 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-10-09T04:29:49.636 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-pastescript (2.0.2-4) ... 2023-10-09T04:29:49.929 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-pecan (1.3.3-3ubuntu1) ... 2023-10-09T04:29:50.298 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-rados (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:50.424 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up librgw2 (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:50.550 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-rbd (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:50.670 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up rbd-fuse (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:50.794 INFO:teuthology.orchestra.run.smithi194.stdout:Processing triggers for fontconfig (2.13.1-2ubuntu3) ... 2023-10-09T04:29:50.796 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-routes (2.4.1-2) ... 2023-10-09T04:29:51.081 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-google-auth (1.5.1-2) ... 2023-10-09T04:29:51.382 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-rgw (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:51.384 INFO:teuthology.orchestra.run.smithi194.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2023-10-09T04:29:51.388 DEBUG:teuthology.parallel:result is None 2023-10-09T04:29:51.508 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-cephfs (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:51.634 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up python3-kubernetes (7.0.0~a1-2) ... 2023-10-09T04:29:52.571 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up ceph-common (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:53.107 INFO:teuthology.orchestra.run.smithi133.stdout:Adding group ceph....done 2023-10-09T04:29:53.552 INFO:teuthology.orchestra.run.smithi133.stdout:Adding system user ceph....done 2023-10-09T04:29:53.737 INFO:teuthology.orchestra.run.smithi133.stdout:Setting system user ceph properties....done 2023-10-09T04:29:53.751 INFO:teuthology.orchestra.run.smithi133.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2023-10-09T04:29:53.895 INFO:teuthology.orchestra.run.smithi133.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2023-10-09T04:29:54.206 INFO:teuthology.orchestra.run.smithi133.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2023-10-09T04:29:54.682 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up ceph-test (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:54.808 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up radosgw (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:55.079 INFO:teuthology.orchestra.run.smithi133.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-10-09T04:29:55.079 INFO:teuthology.orchestra.run.smithi133.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-10-09T04:29:55.768 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up ceph-base (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:56.158 INFO:teuthology.orchestra.run.smithi133.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2023-10-09T04:29:56.629 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up ceph-mds (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:56.845 INFO:teuthology.orchestra.run.smithi133.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-10-09T04:29:56.845 INFO:teuthology.orchestra.run.smithi133.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-10-09T04:29:57.297 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up ceph-mgr (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:57.510 INFO:teuthology.orchestra.run.smithi133.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-10-09T04:29:57.510 INFO:teuthology.orchestra.run.smithi133.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-10-09T04:29:57.958 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up ceph-osd (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:58.449 INFO:teuthology.orchestra.run.smithi133.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-10-09T04:29:58.449 INFO:teuthology.orchestra.run.smithi133.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-10-09T04:29:58.918 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up ceph-mgr-k8sevents (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:59.052 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up ceph-mgr-diskprediction-local (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:59.186 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up ceph-mon (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:59.402 INFO:teuthology.orchestra.run.smithi133.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-10-09T04:29:59.403 INFO:teuthology.orchestra.run.smithi133.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-10-09T04:29:59.854 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up ceph-mgr-cephadm (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:29:59.997 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up ceph (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:30:00.123 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up ceph-mgr-dashboard (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:30:00.257 INFO:teuthology.orchestra.run.smithi133.stdout:Setting up ceph-volume (17.2.6-1303-gb2903cb0-1focal) ... 2023-10-09T04:30:00.384 INFO:teuthology.orchestra.run.smithi133.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-10-09T04:30:00.535 INFO:teuthology.orchestra.run.smithi133.stdout:Processing triggers for systemd (245.4-4ubuntu3.17) ... 2023-10-09T04:30:00.820 INFO:teuthology.orchestra.run.smithi133.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-10-09T04:30:03.848 INFO:teuthology.orchestra.run.smithi133.stdout:Processing triggers for fontconfig (2.13.1-2ubuntu3) ... 2023-10-09T04:30:04.424 INFO:teuthology.orchestra.run.smithi133.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2023-10-09T04:30:04.427 DEBUG:teuthology.parallel:result is None 2023-10-09T04:30:04.427 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T04:30:04.427 INFO:teuthology.packaging:ref: None 2023-10-09T04:30:04.427 INFO:teuthology.packaging:tag: None 2023-10-09T04:30:04.427 INFO:teuthology.packaging:branch: quincy 2023-10-09T04:30:04.427 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T04:30:04.428 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=quincy 2023-10-09T04:30:04.692 DEBUG:teuthology.orchestra.run.smithi133:> dpkg-query -W -f '${Version}' ceph 2023-10-09T04:30:04.715 INFO:teuthology.orchestra.run.smithi133.stdout:17.2.6-1303-gb2903cb0-1focal 2023-10-09T04:30:04.715 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1303-gb2903cb0-1focal 2023-10-09T04:30:04.715 INFO:teuthology.task.install:The correct ceph version 17.2.6-1303-gb2903cb0-1focal is installed. 2023-10-09T04:30:04.716 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T04:30:04.716 INFO:teuthology.packaging:ref: None 2023-10-09T04:30:04.716 INFO:teuthology.packaging:tag: None 2023-10-09T04:30:04.716 INFO:teuthology.packaging:branch: quincy 2023-10-09T04:30:04.716 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T04:30:04.716 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=quincy 2023-10-09T04:30:04.963 DEBUG:teuthology.orchestra.run.smithi194:> dpkg-query -W -f '${Version}' ceph 2023-10-09T04:30:04.988 INFO:teuthology.orchestra.run.smithi194.stdout:17.2.6-1303-gb2903cb0-1focal 2023-10-09T04:30:04.988 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1303-gb2903cb0-1focal 2023-10-09T04:30:04.988 INFO:teuthology.task.install:The correct ceph version 17.2.6-1303-gb2903cb0-1focal is installed. 2023-10-09T04:30:04.988 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-10-09T04:30:04.989 DEBUG:teuthology.orchestra.run.smithi133:> set -ex 2023-10-09T04:30:04.989 DEBUG:teuthology.orchestra.run.smithi133:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-10-09T04:30:05.004 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2023-10-09T04:30:05.004 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-10-09T04:30:05.045 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-10-09T04:30:05.045 DEBUG:teuthology.orchestra.run.smithi133:> set -ex 2023-10-09T04:30:05.046 DEBUG:teuthology.orchestra.run.smithi133:> sudo dd of=/usr/bin/daemon-helper 2023-10-09T04:30:05.061 DEBUG:teuthology.orchestra.run.smithi133:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-10-09T04:30:05.120 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2023-10-09T04:30:05.121 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd of=/usr/bin/daemon-helper 2023-10-09T04:30:05.136 DEBUG:teuthology.orchestra.run.smithi194:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-10-09T04:30:05.190 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-10-09T04:30:05.191 DEBUG:teuthology.orchestra.run.smithi133:> set -ex 2023-10-09T04:30:05.191 DEBUG:teuthology.orchestra.run.smithi133:> sudo dd of=/usr/bin/adjust-ulimits 2023-10-09T04:30:05.204 DEBUG:teuthology.orchestra.run.smithi133:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-10-09T04:30:05.260 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2023-10-09T04:30:05.260 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd of=/usr/bin/adjust-ulimits 2023-10-09T04:30:05.276 DEBUG:teuthology.orchestra.run.smithi194:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-10-09T04:30:05.331 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-10-09T04:30:05.331 DEBUG:teuthology.orchestra.run.smithi133:> set -ex 2023-10-09T04:30:05.331 DEBUG:teuthology.orchestra.run.smithi133:> sudo dd of=/usr/bin/stdin-killer 2023-10-09T04:30:05.347 DEBUG:teuthology.orchestra.run.smithi133:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-10-09T04:30:05.403 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2023-10-09T04:30:05.403 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd of=/usr/bin/stdin-killer 2023-10-09T04:30:05.418 DEBUG:teuthology.orchestra.run.smithi194:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-10-09T04:30:05.477 INFO:teuthology.run_tasks:Running task cephadm... 2023-10-09T04:30:05.543 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'} 2023-10-09T04:30:05.543 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2023-10-09T04:30:05.544 INFO:tasks.cephadm:Cluster fsid is 855b5206-665c-11ee-8db5-212e2dc638e7 2023-10-09T04:30:05.544 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-10-09T04:30:05.544 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.133', 'mon.c': '[v2:172.21.15.133:3301,v1:172.21.15.133:6790]', 'mon.b': '172.21.15.194'} 2023-10-09T04:30:05.544 INFO:tasks.cephadm:First mon is mon.a on smithi133 2023-10-09T04:30:05.544 INFO:tasks.cephadm:First mgr is y 2023-10-09T04:30:05.545 INFO:tasks.cephadm:Normalizing hostnames... 2023-10-09T04:30:05.545 DEBUG:teuthology.orchestra.run.smithi133:> sudo hostname $(hostname -s) 2023-10-09T04:30:05.561 DEBUG:teuthology.orchestra.run.smithi194:> sudo hostname $(hostname -s) 2023-10-09T04:30:05.577 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2023-10-09T04:30:05.578 DEBUG:teuthology.orchestra.run.smithi133:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-09T04:30:05.748 INFO:teuthology.orchestra.run.smithi133.stdout:-rw-rw-r-- 1 ubuntu ubuntu 366887 Oct 9 04:30 /home/ubuntu/cephtest/cephadm 2023-10-09T04:30:05.748 DEBUG:teuthology.orchestra.run.smithi194:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-09T04:30:05.908 INFO:teuthology.orchestra.run.smithi194.stdout:-rw-rw-r-- 1 ubuntu ubuntu 366887 Oct 9 04:30 /home/ubuntu/cephtest/cephadm 2023-10-09T04:30:05.908 DEBUG:teuthology.orchestra.run.smithi133:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-09T04:30:05.918 DEBUG:teuthology.orchestra.run.smithi194:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-09T04:30:05.936 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2023-10-09T04:30:05.937 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-10-09T04:30:05.963 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-10-09T04:30:06.217 INFO:teuthology.orchestra.run.smithi133.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-10-09T04:30:06.225 INFO:teuthology.orchestra.run.smithi194.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-10-09T04:30:49.299 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T04:30:49.299 INFO:teuthology.orchestra.run.smithi133.stdout: "ceph_version": "ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)", 2023-10-09T04:30:49.299 INFO:teuthology.orchestra.run.smithi133.stdout: "image_id": "5517396c50d4b07181c79c353f571b455dbdd80614d068cc9a6e42eb27b7bccf", 2023-10-09T04:30:49.299 INFO:teuthology.orchestra.run.smithi133.stdout: "repo_digests": [ 2023-10-09T04:30:49.300 INFO:teuthology.orchestra.run.smithi133.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:67105d45415a7f6878d82dd27e27caaa40ebf0ca6488030995660f1cad5fce9d" 2023-10-09T04:30:49.300 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T04:30:49.300 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T04:30:54.978 INFO:teuthology.orchestra.run.smithi194.stdout:{ 2023-10-09T04:30:54.978 INFO:teuthology.orchestra.run.smithi194.stdout: "ceph_version": "ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)", 2023-10-09T04:30:54.978 INFO:teuthology.orchestra.run.smithi194.stdout: "image_id": "5517396c50d4b07181c79c353f571b455dbdd80614d068cc9a6e42eb27b7bccf", 2023-10-09T04:30:54.978 INFO:teuthology.orchestra.run.smithi194.stdout: "repo_digests": [ 2023-10-09T04:30:54.979 INFO:teuthology.orchestra.run.smithi194.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:67105d45415a7f6878d82dd27e27caaa40ebf0ca6488030995660f1cad5fce9d" 2023-10-09T04:30:54.979 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T04:30:54.979 INFO:teuthology.orchestra.run.smithi194.stdout:} 2023-10-09T04:30:55.002 DEBUG:teuthology.orchestra.run.smithi133:> sudo mkdir -p /etc/ceph 2023-10-09T04:30:55.018 DEBUG:teuthology.orchestra.run.smithi194:> sudo mkdir -p /etc/ceph 2023-10-09T04:30:55.034 DEBUG:teuthology.orchestra.run.smithi133:> sudo chmod 777 /etc/ceph 2023-10-09T04:30:55.079 DEBUG:teuthology.orchestra.run.smithi194:> sudo chmod 777 /etc/ceph 2023-10-09T04:30:55.094 INFO:tasks.cephadm:Writing seed config... 2023-10-09T04:30:55.095 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2023-10-09T04:30:55.095 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2023-10-09T04:30:55.095 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-10-09T04:30:55.095 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-10-09T04:30:55.095 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-10-09T04:30:55.095 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2023-10-09T04:30:55.095 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-10-09T04:30:55.095 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-10-09T04:30:55.095 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-10-09T04:30:55.096 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-10-09T04:30:55.096 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-10-09T04:30:55.096 DEBUG:teuthology.orchestra.run.smithi133:> set -ex 2023-10-09T04:30:55.096 DEBUG:teuthology.orchestra.run.smithi133:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-10-09T04:30:55.128 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 855b5206-665c-11ee-8db5-212e2dc638e7 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-10-09T04:30:55.128 DEBUG:teuthology.orchestra.run.smithi133:mon.a> sudo journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mon.a.service 2023-10-09T04:30:55.171 DEBUG:teuthology.orchestra.run.smithi133:mgr.y> sudo journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mgr.y.service 2023-10-09T04:30:55.214 INFO:tasks.cephadm:Bootstrapping... 2023-10-09T04:30:55.215 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.133 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-10-09T04:30:55.326 INFO:journalctl@ceph.mon.a.smithi133.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:30:55.419 INFO:teuthology.orchestra.run.smithi133.stdout:-------------------------------------------------------------------------------- 2023-10-09T04:30:55.420 INFO:teuthology.orchestra.run.smithi133.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', '855b5206-665c-11ee-8db5-212e2dc638e7', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.133', '--skip-admin-label'] 2023-10-09T04:30:55.420 INFO:teuthology.orchestra.run.smithi133.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2023-10-09T04:30:55.420 INFO:teuthology.orchestra.run.smithi133.stdout:Verifying podman|docker is present... 2023-10-09T04:30:55.420 INFO:teuthology.orchestra.run.smithi133.stdout:Verifying lvm2 is present... 2023-10-09T04:30:55.421 INFO:teuthology.orchestra.run.smithi133.stdout:Verifying time synchronization is in place... 2023-10-09T04:30:55.428 INFO:teuthology.orchestra.run.smithi133.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-10-09T04:30:55.428 INFO:teuthology.orchestra.run.smithi133.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-10-09T04:30:55.435 INFO:teuthology.orchestra.run.smithi133.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-10-09T04:30:55.435 INFO:teuthology.orchestra.run.smithi133.stdout:systemctl: stdout inactive 2023-10-09T04:30:55.440 INFO:teuthology.orchestra.run.smithi133.stdout:Non-zero exit code 1 from systemctl is-enabled chronyd.service 2023-10-09T04:30:55.440 INFO:teuthology.orchestra.run.smithi133.stdout:systemctl: stderr Failed to get unit file state for chronyd.service: No such file or directory 2023-10-09T04:30:55.447 INFO:teuthology.orchestra.run.smithi133.stdout:Non-zero exit code 3 from systemctl is-active chronyd.service 2023-10-09T04:30:55.447 INFO:teuthology.orchestra.run.smithi133.stdout:systemctl: stdout inactive 2023-10-09T04:30:55.452 INFO:teuthology.orchestra.run.smithi133.stdout:Non-zero exit code 1 from systemctl is-enabled systemd-timesyncd.service 2023-10-09T04:30:55.452 INFO:teuthology.orchestra.run.smithi133.stdout:systemctl: stdout masked 2023-10-09T04:30:55.458 INFO:teuthology.orchestra.run.smithi133.stdout:Non-zero exit code 3 from systemctl is-active systemd-timesyncd.service 2023-10-09T04:30:55.459 INFO:teuthology.orchestra.run.smithi133.stdout:systemctl: stdout inactive 2023-10-09T04:30:55.464 INFO:teuthology.orchestra.run.smithi133.stdout:Non-zero exit code 1 from systemctl is-enabled ntpd.service 2023-10-09T04:30:55.464 INFO:teuthology.orchestra.run.smithi133.stdout:systemctl: stderr Failed to get unit file state for ntpd.service: No such file or directory 2023-10-09T04:30:55.471 INFO:teuthology.orchestra.run.smithi133.stdout:Non-zero exit code 3 from systemctl is-active ntpd.service 2023-10-09T04:30:55.471 INFO:teuthology.orchestra.run.smithi133.stdout:systemctl: stdout inactive 2023-10-09T04:30:55.476 INFO:teuthology.orchestra.run.smithi133.stdout:systemctl: stdout enabled 2023-10-09T04:30:55.482 INFO:teuthology.orchestra.run.smithi133.stdout:systemctl: stdout active 2023-10-09T04:30:55.482 INFO:teuthology.orchestra.run.smithi133.stdout:Unit ntp.service is enabled and running 2023-10-09T04:30:55.482 INFO:teuthology.orchestra.run.smithi133.stdout:Repeating the final host check... 2023-10-09T04:30:55.482 INFO:teuthology.orchestra.run.smithi133.stdout:docker (/usr/bin/docker) is present 2023-10-09T04:30:55.482 INFO:teuthology.orchestra.run.smithi133.stdout:systemctl is present 2023-10-09T04:30:55.482 INFO:teuthology.orchestra.run.smithi133.stdout:lvcreate is present 2023-10-09T04:30:55.489 INFO:teuthology.orchestra.run.smithi133.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-10-09T04:30:55.489 INFO:teuthology.orchestra.run.smithi133.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-10-09T04:30:55.495 INFO:teuthology.orchestra.run.smithi133.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-10-09T04:30:55.495 INFO:teuthology.orchestra.run.smithi133.stdout:systemctl: stdout inactive 2023-10-09T04:30:55.500 INFO:teuthology.orchestra.run.smithi133.stdout:Non-zero exit code 1 from systemctl is-enabled chronyd.service 2023-10-09T04:30:55.500 INFO:teuthology.orchestra.run.smithi133.stdout:systemctl: stderr Failed to get unit file state for chronyd.service: No such file or directory 2023-10-09T04:30:55.507 INFO:teuthology.orchestra.run.smithi133.stdout:Non-zero exit code 3 from systemctl is-active chronyd.service 2023-10-09T04:30:55.507 INFO:teuthology.orchestra.run.smithi133.stdout:systemctl: stdout inactive 2023-10-09T04:30:55.513 INFO:teuthology.orchestra.run.smithi133.stdout:Non-zero exit code 1 from systemctl is-enabled systemd-timesyncd.service 2023-10-09T04:30:55.513 INFO:teuthology.orchestra.run.smithi133.stdout:systemctl: stdout masked 2023-10-09T04:30:55.519 INFO:teuthology.orchestra.run.smithi133.stdout:Non-zero exit code 3 from systemctl is-active systemd-timesyncd.service 2023-10-09T04:30:55.519 INFO:teuthology.orchestra.run.smithi133.stdout:systemctl: stdout inactive 2023-10-09T04:30:55.525 INFO:teuthology.orchestra.run.smithi133.stdout:Non-zero exit code 1 from systemctl is-enabled ntpd.service 2023-10-09T04:30:55.525 INFO:teuthology.orchestra.run.smithi133.stdout:systemctl: stderr Failed to get unit file state for ntpd.service: No such file or directory 2023-10-09T04:30:55.531 INFO:teuthology.orchestra.run.smithi133.stdout:Non-zero exit code 3 from systemctl is-active ntpd.service 2023-10-09T04:30:55.531 INFO:teuthology.orchestra.run.smithi133.stdout:systemctl: stdout inactive 2023-10-09T04:30:55.538 INFO:teuthology.orchestra.run.smithi133.stdout:systemctl: stdout enabled 2023-10-09T04:30:55.543 INFO:teuthology.orchestra.run.smithi133.stdout:systemctl: stdout active 2023-10-09T04:30:55.544 INFO:teuthology.orchestra.run.smithi133.stdout:Unit ntp.service is enabled and running 2023-10-09T04:30:55.544 INFO:teuthology.orchestra.run.smithi133.stdout:Host looks OK 2023-10-09T04:30:55.544 INFO:teuthology.orchestra.run.smithi133.stdout:Cluster fsid: 855b5206-665c-11ee-8db5-212e2dc638e7 2023-10-09T04:30:55.544 INFO:teuthology.orchestra.run.smithi133.stdout:Acquiring lock 139718287646000 on /run/cephadm/855b5206-665c-11ee-8db5-212e2dc638e7.lock 2023-10-09T04:30:55.544 INFO:teuthology.orchestra.run.smithi133.stdout:Lock 139718287646000 acquired on /run/cephadm/855b5206-665c-11ee-8db5-212e2dc638e7.lock 2023-10-09T04:30:55.544 INFO:teuthology.orchestra.run.smithi133.stdout:Verifying IP 172.21.15.133 port 3300 ... 2023-10-09T04:30:55.545 INFO:teuthology.orchestra.run.smithi133.stdout:Verifying IP 172.21.15.133 port 6789 ... 2023-10-09T04:30:55.545 INFO:teuthology.orchestra.run.smithi133.stdout:Base mon IP(s) is [172.21.15.133:3300, 172.21.15.133:6789], mon addrv is [v2:172.21.15.133:3300,v1:172.21.15.133:6789] 2023-10-09T04:30:55.550 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 2023-10-09T04:30:55.550 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.133 metric 100 2023-10-09T04:30:55.550 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/sbin/ip: stdout 172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown 2023-10-09T04:30:55.551 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.133 2023-10-09T04:30:55.551 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/sbin/ip: stdout 172.21.15.254 dev enp3s0f1 proto dhcp scope link src 172.21.15.133 metric 100 2023-10-09T04:30:55.555 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2023-10-09T04:30:55.555 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2023-10-09T04:30:55.555 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 expires 1745sec pref medium 2023-10-09T04:30:55.558 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-10-09T04:30:55.558 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/sbin/ip: stdout inet6 ::1/128 scope host 2023-10-09T04:30:55.559 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-10-09T04:30:55.559 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2023-10-09T04:30:55.559 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/sbin/ip: stdout inet6 fe80::ec4:7aff:fe88:8087/64 scope link 2023-10-09T04:30:55.559 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-10-09T04:30:55.559 INFO:teuthology.orchestra.run.smithi133.stdout:Mon IP `172.21.15.133` is in CIDR network `172.21.0.0/20` 2023-10-09T04:30:55.560 INFO:teuthology.orchestra.run.smithi133.stdout:Mon IP `172.21.15.133` is in CIDR network `172.21.0.0/20` 2023-10-09T04:30:55.560 INFO:teuthology.orchestra.run.smithi133.stdout:Mon IP `172.21.15.133` is in CIDR network `172.21.15.254/32` 2023-10-09T04:30:55.560 INFO:teuthology.orchestra.run.smithi133.stdout:Mon IP `172.21.15.133` is in CIDR network `172.21.15.254/32` 2023-10-09T04:30:55.560 INFO:teuthology.orchestra.run.smithi133.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20', '172.21.15.254/32', '172.21.15.254/32'] 2023-10-09T04:30:55.560 INFO:teuthology.orchestra.run.smithi133.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-10-09T04:30:55.561 INFO:teuthology.orchestra.run.smithi133.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-10-09T04:30:55.565 INFO:journalctl@ceph.mgr.y.smithi133.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:30:55.867 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/docker: stdout quincy: Pulling from ceph-ci/ceph 2023-10-09T04:30:55.867 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/docker: stdout Digest: sha256:67105d45415a7f6878d82dd27e27caaa40ebf0ca6488030995660f1cad5fce9d 2023-10-09T04:30:55.867 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/docker: stdout Status: Image is up to date for quay.ceph.io/ceph-ci/ceph:quincy 2023-10-09T04:30:55.868 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/docker: stdout quay.ceph.io/ceph-ci/ceph:quincy 2023-10-09T04:30:57.477 INFO:teuthology.orchestra.run.smithi133.stdout:ceph: stdout ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable) 2023-10-09T04:30:57.477 INFO:teuthology.orchestra.run.smithi133.stdout:Ceph version: ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable) 2023-10-09T04:30:57.477 INFO:teuthology.orchestra.run.smithi133.stdout:Extracting ceph user uid/gid from container image... 2023-10-09T04:30:58.980 INFO:teuthology.orchestra.run.smithi133.stdout:stat: stdout 167 167 2023-10-09T04:30:58.980 INFO:teuthology.orchestra.run.smithi133.stdout:Creating initial keys... 2023-10-09T04:31:00.359 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-authtool: stdout AQADgiNluRLZKxAA66FXuBUBGmpfpGwapQMcCw== 2023-10-09T04:31:01.781 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-authtool: stdout AQAFgiNlGHEeDBAAiYXkJDN/07uaW2rkVvUQYg== 2023-10-09T04:31:03.196 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-authtool: stdout AQAGgiNlIRwfJBAAuToIrG4+wtr4eoGAGcY+TQ== 2023-10-09T04:31:03.197 INFO:teuthology.orchestra.run.smithi133.stdout:Creating initial monmap... 2023-10-09T04:31:04.683 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-09T04:31:04.683 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2023-10-09T04:31:04.683 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 855b5206-665c-11ee-8db5-212e2dc638e7 2023-10-09T04:31:04.683 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-09T04:31:04.683 INFO:teuthology.orchestra.run.smithi133.stdout:monmaptool for a [v2:172.21.15.133:3300,v1:172.21.15.133:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-09T04:31:04.683 INFO:teuthology.orchestra.run.smithi133.stdout:setting min_mon_release = octopus 2023-10-09T04:31:04.683 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/monmaptool: set fsid to 855b5206-665c-11ee-8db5-212e2dc638e7 2023-10-09T04:31:04.683 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-09T04:31:04.683 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T04:31:04.684 INFO:teuthology.orchestra.run.smithi133.stdout:Creating mon... 2023-10-09T04:31:06.131 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T04:31:06.131 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 1 imported monmap: 2023-10-09T04:31:06.131 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr epoch 0 2023-10-09T04:31:06.131 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr fsid 855b5206-665c-11ee-8db5-212e2dc638e7 2023-10-09T04:31:06.131 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr last_changed 2023-10-09T04:31:04.046038+0000 2023-10-09T04:31:06.131 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr created 2023-10-09T04:31:04.046038+0000 2023-10-09T04:31:06.131 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr min_mon_release 15 (octopus) 2023-10-09T04:31:06.131 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr election_strategy: 1 2023-10-09T04:31:06.131 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr 0: [v2:172.21.15.133:3300/0,v1:172.21.15.133:6789/0] mon.a 2023-10-09T04:31:06.131 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr 2023-10-09T04:31:06.131 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 0 /usr/bin/ceph-mon: set fsid to 855b5206-665c-11ee-8db5-212e2dc638e7 2023-10-09T04:31:06.132 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: RocksDB version: 6.15.5 2023-10-09T04:31:06.132 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr 2023-10-09T04:31:06.132 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T04:31:06.132 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Compile date Oct 8 2023 2023-10-09T04:31:06.132 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: DB SUMMARY 2023-10-09T04:31:06.132 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr 2023-10-09T04:31:06.132 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: DB Session ID: 84Z56SQU60G1H6353436 2023-10-09T04:31:06.132 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr 2023-10-09T04:31:06.132 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 0, files: 2023-10-09T04:31:06.132 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr 2023-10-09T04:31:06.132 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 2023-10-09T04:31:06.132 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr 2023-10-09T04:31:06.133 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.error_if_exists: 0 2023-10-09T04:31:06.133 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.create_if_missing: 1 2023-10-09T04:31:06.133 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T04:31:06.133 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T04:31:06.133 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.env: 0x563f0a811a80 2023-10-09T04:31:06.133 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.fs: Posix File System 2023-10-09T04:31:06.133 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.info_log: 0x563f0d2c4ea0 2023-10-09T04:31:06.133 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T04:31:06.133 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.statistics: (nil) 2023-10-09T04:31:06.133 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.use_fsync: 0 2023-10-09T04:31:06.133 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T04:31:06.133 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T04:31:06.134 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T04:31:06.134 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T04:31:06.134 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T04:31:06.134 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T04:31:06.134 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T04:31:06.134 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T04:31:06.134 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T04:31:06.134 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T04:31:06.134 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T04:31:06.134 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.db_log_dir: 2023-10-09T04:31:06.134 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-10-09T04:31:06.134 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T04:31:06.135 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T04:31:06.135 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T04:31:06.135 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T04:31:06.135 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T04:31:06.135 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T04:31:06.135 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T04:31:06.135 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T04:31:06.135 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.write_buffer_manager: 0x563f0dd9a060 2023-10-09T04:31:06.135 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T04:31:06.135 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T04:31:06.135 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T04:31:06.137 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T04:31:06.137 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T04:31:06.137 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T04:31:06.137 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T04:31:06.137 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T04:31:06.137 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T04:31:06.137 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.unordered_write: 0 2023-10-09T04:31:06.138 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T04:31:06.138 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T04:31:06.138 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T04:31:06.138 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T04:31:06.138 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.row_cache: None 2023-10-09T04:31:06.138 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.wal_filter: None 2023-10-09T04:31:06.138 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T04:31:06.138 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T04:31:06.138 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.preserve_deletes: 0 2023-10-09T04:31:06.138 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.two_write_queues: 0 2023-10-09T04:31:06.138 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T04:31:06.139 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.atomic_flush: 0 2023-10-09T04:31:06.139 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T04:31:06.139 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T04:31:06.139 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T04:31:06.139 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T04:31:06.139 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T04:31:06.139 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.best_efforts_recovery: 0 2023-10-09T04:31:06.139 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T04:31:06.139 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T04:31:06.139 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.allow_data_in_errors: 0 2023-10-09T04:31:06.139 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.db_host_id: __hostname__ 2023-10-09T04:31:06.139 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T04:31:06.140 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T04:31:06.140 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T04:31:06.140 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T04:31:06.140 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T04:31:06.140 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T04:31:06.140 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T04:31:06.140 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T04:31:06.140 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T04:31:06.140 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T04:31:06.140 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T04:31:06.140 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.max_open_files: -1 2023-10-09T04:31:06.140 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T04:31:06.140 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T04:31:06.141 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T04:31:06.141 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T04:31:06.141 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T04:31:06.141 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Compression algorithms supported: 2023-10-09T04:31:06.141 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T04:31:06.141 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: kZSTD supported: 0 2023-10-09T04:31:06.141 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: kXpressCompression supported: 0 2023-10-09T04:31:06.141 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T04:31:06.141 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: kZlibCompression supported: 1 2023-10-09T04:31:06.141 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T04:31:06.141 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T04:31:06.141 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T04:31:06.142 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T04:31:06.142 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.465+0000 7fc43e10bb80 4 rocksdb: [db/db_impl/db_impl_open.cc:281] Creating manifest 1 2023-10-09T04:31:06.142 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr 2023-10-09T04:31:06.142 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 2023-10-09T04:31:06.142 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr 2023-10-09T04:31:06.142 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-10-09T04:31:06.142 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr 2023-10-09T04:31:06.142 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T04:31:06.142 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.merge_operator: 2023-10-09T04:31:06.142 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.compaction_filter: None 2023-10-09T04:31:06.143 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T04:31:06.143 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.sst_partitioner_factory: None 2023-10-09T04:31:06.143 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T04:31:06.143 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T04:31:06.143 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x563f0d078ff8) 2023-10-09T04:31:06.143 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr cache_index_and_filter_blocks: 1 2023-10-09T04:31:06.143 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T04:31:06.143 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T04:31:06.143 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr pin_top_level_index_and_filter: 1 2023-10-09T04:31:06.143 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr index_type: 0 2023-10-09T04:31:06.143 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr data_block_index_type: 0 2023-10-09T04:31:06.143 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr index_shortening: 1 2023-10-09T04:31:06.144 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr data_block_hash_table_util_ratio: 0.750000 2023-10-09T04:31:06.144 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr hash_index_allow_collision: 1 2023-10-09T04:31:06.144 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr checksum: 1 2023-10-09T04:31:06.144 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr no_block_cache: 0 2023-10-09T04:31:06.144 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr block_cache: 0x563f0d0ab090 2023-10-09T04:31:06.144 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr block_cache_name: BinnedLRUCache 2023-10-09T04:31:06.144 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr block_cache_options: 2023-10-09T04:31:06.144 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr capacity : 536870912 2023-10-09T04:31:06.144 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr num_shard_bits : 4 2023-10-09T04:31:06.144 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr strict_capacity_limit : 0 2023-10-09T04:31:06.144 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr high_pri_pool_ratio: 0.000 2023-10-09T04:31:06.144 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr block_cache_compressed: (nil) 2023-10-09T04:31:06.145 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr persistent_cache: (nil) 2023-10-09T04:31:06.145 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr block_size: 4096 2023-10-09T04:31:06.145 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr block_size_deviation: 10 2023-10-09T04:31:06.145 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr block_restart_interval: 16 2023-10-09T04:31:06.145 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr index_block_restart_interval: 1 2023-10-09T04:31:06.145 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr metadata_block_size: 4096 2023-10-09T04:31:06.145 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr partition_filters: 0 2023-10-09T04:31:06.145 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr use_delta_encoding: 1 2023-10-09T04:31:06.145 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T04:31:06.145 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr whole_key_filtering: 1 2023-10-09T04:31:06.145 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr verify_compression: 0 2023-10-09T04:31:06.145 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr read_amp_bytes_per_bit: 0 2023-10-09T04:31:06.146 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr format_version: 4 2023-10-09T04:31:06.146 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr enable_index_compression: 1 2023-10-09T04:31:06.146 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr block_align: 0 2023-10-09T04:31:06.146 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr 2023-10-09T04:31:06.146 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T04:31:06.146 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T04:31:06.146 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.compression: NoCompression 2023-10-09T04:31:06.146 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T04:31:06.146 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T04:31:06.146 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T04:31:06.146 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.num_levels: 7 2023-10-09T04:31:06.147 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T04:31:06.147 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T04:31:06.147 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T04:31:06.147 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T04:31:06.147 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T04:31:06.147 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T04:31:06.147 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T04:31:06.147 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:31:06.147 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T04:31:06.147 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T04:31:06.147 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T04:31:06.147 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T04:31:06.148 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T04:31:06.148 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T04:31:06.148 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:31:06.148 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T04:31:06.148 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T04:31:06.148 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T04:31:06.148 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T04:31:06.148 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T04:31:06.148 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T04:31:06.148 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T04:31:06.148 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T04:31:06.148 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T04:31:06.149 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T04:31:06.149 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T04:31:06.149 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T04:31:06.149 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T04:31:06.149 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T04:31:06.149 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T04:31:06.149 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T04:31:06.149 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T04:31:06.149 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T04:31:06.149 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T04:31:06.149 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.arena_block_size: 4194304 2023-10-09T04:31:06.149 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T04:31:06.150 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T04:31:06.150 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T04:31:06.150 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T04:31:06.150 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T04:31:06.150 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T04:31:06.150 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T04:31:06.150 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T04:31:06.150 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T04:31:06.150 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T04:31:06.150 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T04:31:06.150 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T04:31:06.150 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T04:31:06.151 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T04:31:06.151 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.table_properties_collectors: 2023-10-09T04:31:06.151 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T04:31:06.151 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T04:31:06.151 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T04:31:06.151 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T04:31:06.151 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T04:31:06.151 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.bloom_locality: 0 2023-10-09T04:31:06.151 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T04:31:06.151 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T04:31:06.151 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T04:31:06.151 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.force_consistency_checks: 1 2023-10-09T04:31:06.151 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T04:31:06.152 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.ttl: 2592000 2023-10-09T04:31:06.152 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T04:31:06.152 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.enable_blob_files: false 2023-10-09T04:31:06.152 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.min_blob_size: 0 2023-10-09T04:31:06.152 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.blob_file_size: 268435456 2023-10-09T04:31:06.152 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T04:31:06.152 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T04:31:06.152 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T04:31:06.152 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T04:31:06.152 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr 2023-10-09T04:31:06.152 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-10-09T04:31:06.152 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr 2023-10-09T04:31:06.153 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.469+0000 7fc43e10bb80 4 rocksdb: [db/version_set.cc:4082] Creating manifest 3 2023-10-09T04:31:06.153 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr 2023-10-09T04:31:06.153 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.473+0000 7fc43e10bb80 4 rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x563f0d03f180 2023-10-09T04:31:06.153 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.473+0000 7fc43e10bb80 4 rocksdb: DB pointer 0x563f0d0f2000 2023-10-09T04:31:06.153 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.473+0000 7fc42fd58700 4 rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-10-09T04:31:06.153 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.473+0000 7fc42fd58700 4 rocksdb: [db/db_impl/db_impl.cc:903] 2023-10-09T04:31:06.153 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr ** DB Stats ** 2023-10-09T04:31:06.153 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:31:06.153 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T04:31:06.153 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T04:31:06.153 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:31:06.153 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T04:31:06.154 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T04:31:06.154 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:31:06.154 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr 2023-10-09T04:31:06.154 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2023-10-09T04:31:06.154 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:31:06.154 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:31:06.154 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T04:31:06.154 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T04:31:06.154 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr 2023-10-09T04:31:06.154 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2023-10-09T04:31:06.154 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:31:06.155 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:31:06.155 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:31:06.155 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T04:31:06.155 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T04:31:06.155 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr AddFile(Total Files): cumulative 0, interval 0 2023-10-09T04:31:06.155 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T04:31:06.155 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr AddFile(Keys): cumulative 0, interval 0 2023-10-09T04:31:06.155 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:31:06.155 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:31:06.155 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T04:31:06.155 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr 2023-10-09T04:31:06.156 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr ** File Read Latency Histogram By Level [default] ** 2023-10-09T04:31:06.156 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr 2023-10-09T04:31:06.156 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2023-10-09T04:31:06.156 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:31:06.156 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:31:06.156 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T04:31:06.156 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T04:31:06.156 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr 2023-10-09T04:31:06.156 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2023-10-09T04:31:06.156 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:31:06.156 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:31:06.156 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:31:06.156 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T04:31:06.157 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T04:31:06.157 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr AddFile(Total Files): cumulative 0, interval 0 2023-10-09T04:31:06.157 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T04:31:06.157 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr AddFile(Keys): cumulative 0, interval 0 2023-10-09T04:31:06.157 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:31:06.157 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:31:06.157 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T04:31:06.157 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr 2023-10-09T04:31:06.157 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr ** File Read Latency Histogram By Level [default] ** 2023-10-09T04:31:06.157 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr 2023-10-09T04:31:06.157 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.477+0000 7fc43e10bb80 4 rocksdb: [db/db_impl/db_impl.cc:446] Shutdown: canceling all background work 2023-10-09T04:31:06.157 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.477+0000 7fc43e10bb80 4 rocksdb: [db/db_impl/db_impl.cc:625] Shutdown complete 2023-10-09T04:31:06.157 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph-mon: stderr debug 2023-10-09T04:31:05.477+0000 7fc43e10bb80 0 /usr/bin/ceph-mon: created monfs at /var/lib/ceph/mon/ceph-a for mon.a 2023-10-09T04:31:06.158 INFO:teuthology.orchestra.run.smithi133.stdout:create mon.a on 2023-10-09T04:31:06.351 INFO:teuthology.orchestra.run.smithi133.stdout:systemctl: stderr Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-10-09T04:31:06.547 INFO:teuthology.orchestra.run.smithi133.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /etc/systemd/system/ceph.target. 2023-10-09T04:31:06.762 INFO:teuthology.orchestra.run.smithi133.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-855b5206-665c-11ee-8db5-212e2dc638e7.target -> /etc/systemd/system/ceph-855b5206-665c-11ee-8db5-212e2dc638e7.target. 2023-10-09T04:31:06.762 INFO:teuthology.orchestra.run.smithi133.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-855b5206-665c-11ee-8db5-212e2dc638e7.target -> /etc/systemd/system/ceph-855b5206-665c-11ee-8db5-212e2dc638e7.target. 2023-10-09T04:31:06.990 INFO:teuthology.orchestra.run.smithi133.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mon.a 2023-10-09T04:31:06.990 INFO:teuthology.orchestra.run.smithi133.stdout:systemctl: stderr Failed to reset failed state of unit ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mon.a.service: Unit ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mon.a.service not loaded. 2023-10-09T04:31:07.188 INFO:teuthology.orchestra.run.smithi133.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-855b5206-665c-11ee-8db5-212e2dc638e7.target.wants/ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mon.a.service -> /etc/systemd/system/ceph-855b5206-665c-11ee-8db5-212e2dc638e7@.service. 2023-10-09T04:31:07.202 INFO:teuthology.orchestra.run.smithi133.stdout:firewalld does not appear to be present 2023-10-09T04:31:07.202 INFO:teuthology.orchestra.run.smithi133.stdout:Not possible to enable service . firewalld.service is not available 2023-10-09T04:31:07.203 INFO:teuthology.orchestra.run.smithi133.stdout:Waiting for mon to start... 2023-10-09T04:31:07.203 INFO:teuthology.orchestra.run.smithi133.stdout:Waiting for mon... 2023-10-09T04:31:08.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:08 smithi133 bash[27045]: cluster 2023-10-09T04:31:08.618292+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T04:31:09.448 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout cluster: 2023-10-09T04:31:09.448 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout id: 855b5206-665c-11ee-8db5-212e2dc638e7 2023-10-09T04:31:09.448 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2023-10-09T04:31:09.448 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout 2023-10-09T04:31:09.449 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout services: 2023-10-09T04:31:09.449 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.226503s) 2023-10-09T04:31:09.449 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2023-10-09T04:31:09.449 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2023-10-09T04:31:09.449 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout 2023-10-09T04:31:09.450 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout data: 2023-10-09T04:31:09.450 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2023-10-09T04:31:09.450 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2023-10-09T04:31:09.450 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2023-10-09T04:31:09.450 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout pgs: 2023-10-09T04:31:09.450 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout 2023-10-09T04:31:09.451 INFO:teuthology.orchestra.run.smithi133.stdout:mon is available 2023-10-09T04:31:09.451 INFO:teuthology.orchestra.run.smithi133.stdout:Assimilating anything we can from ceph.conf... 2023-10-09T04:31:09.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:09 smithi133 bash[27045]: cluster 2023-10-09T04:31:08.630650+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T04:31:09.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:09 smithi133 bash[27045]: cluster 2023-10-09T04:31:08.632832+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.133:3300/0,v1:172.21.15.133:6789/0]} removed_ranks: {} 2023-10-09T04:31:09.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:09 smithi133 bash[27045]: cluster 2023-10-09T04:31:08.635678+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap 2023-10-09T04:31:09.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:09 smithi133 bash[27045]: cluster 2023-10-09T04:31:08.638492+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-10-09T04:31:09.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:09 smithi133 bash[27045]: cluster 2023-10-09T04:31:08.639058+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e1: no daemons active 2023-10-09T04:31:09.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:09 smithi133 bash[27045]: audit 2023-10-09T04:31:08.857084+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.133:0/1329399921' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-10-09T04:31:11.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:10 smithi133 bash[27045]: audit 2023-10-09T04:31:10.549653+0000 mon.a (mon.0) 8 : audit [INF] from='client.? 172.21.15.133:0/2558380474' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-09T04:31:11.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:10 smithi133 bash[27045]: audit 2023-10-09T04:31:10.556095+0000 mon.a (mon.0) 9 : audit [INF] from='client.? 172.21.15.133:0/2558380474' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-10-09T04:31:11.227 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout 2023-10-09T04:31:11.227 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout [global] 2023-10-09T04:31:11.227 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout fsid = 855b5206-665c-11ee-8db5-212e2dc638e7 2023-10-09T04:31:11.228 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.133:3300,v1:172.21.15.133:6789] 2023-10-09T04:31:11.228 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-10-09T04:31:11.228 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-10-09T04:31:11.228 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-10-09T04:31:11.228 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-10-09T04:31:11.228 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout 2023-10-09T04:31:11.229 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout [mgr] 2023-10-09T04:31:11.229 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-10-09T04:31:11.229 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout 2023-10-09T04:31:11.229 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout [osd] 2023-10-09T04:31:11.229 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-10-09T04:31:11.229 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-10-09T04:31:11.230 INFO:teuthology.orchestra.run.smithi133.stdout:Generating new minimal ceph.conf... 2023-10-09T04:31:12.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:12 smithi133 bash[27045]: audit 2023-10-09T04:31:12.321118+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.133:0/1007476998' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:31:12.897 INFO:teuthology.orchestra.run.smithi133.stdout:Restarting the monitor... 2023-10-09T04:31:13.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:12 smithi133 systemd[1]: Stopping Ceph mon.a for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T04:31:13.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:13 smithi133 bash[27045]: debug 2023-10-09T04:31:13.041+0000 7f11333e6700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T04:31:13.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:13 smithi133 bash[27045]: debug 2023-10-09T04:31:13.041+0000 7f11333e6700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-10-09T04:31:13.962 INFO:teuthology.orchestra.run.smithi133.stdout:Setting mon public_network to 172.21.15.254/32,172.21.0.0/20 2023-10-09T04:31:14.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:13 smithi133 bash[27452]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T04:31:14.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:13 smithi133 bash[27518]: Error response from daemon: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T04:31:14.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:13 smithi133 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mon.a.service: Succeeded. 2023-10-09T04:31:14.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:13 smithi133 systemd[1]: Stopped Ceph mon.a for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:31:14.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:13 smithi133 systemd[1]: Started Ceph mon.a for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:31:15.505 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.197+0000 7fa591bdbb80 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T04:31:15.505 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.197+0000 7fa591bdbb80 0 ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable), process ceph-mon, pid 7 2023-10-09T04:31:15.505 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.197+0000 7fa591bdbb80 0 pidfile_write: ignore empty --pid-file 2023-10-09T04:31:15.505 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 0 load: jerasure load: lrc 2023-10-09T04:31:15.505 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: RocksDB version: 6.15.5 2023-10-09T04:31:15.505 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T04:31:15.505 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Compile date Oct 8 2023 2023-10-09T04:31:15.505 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: DB SUMMARY 2023-10-09T04:31:15.506 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: DB Session ID: HHB42WZRUT6DQZE1G21P 2023-10-09T04:31:15.506 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: CURRENT file: CURRENT 2023-10-09T04:31:15.506 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: IDENTITY file: IDENTITY 2023-10-09T04:31:15.506 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2023-10-09T04:31:15.506 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2023-10-09T04:31:15.506 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83645 ; 2023-10-09T04:31:15.506 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.error_if_exists: 0 2023-10-09T04:31:15.506 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.create_if_missing: 0 2023-10-09T04:31:15.506 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T04:31:15.506 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T04:31:15.506 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.env: 0x559774fcea80 2023-10-09T04:31:15.506 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.fs: Posix File System 2023-10-09T04:31:15.507 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.info_log: 0x5597763ada60 2023-10-09T04:31:15.507 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T04:31:15.507 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.statistics: (nil) 2023-10-09T04:31:15.507 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.use_fsync: 0 2023-10-09T04:31:15.507 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T04:31:15.507 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T04:31:15.507 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T04:31:15.508 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T04:31:15.508 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T04:31:15.508 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T04:31:15.508 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T04:31:15.508 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T04:31:15.508 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T04:31:15.508 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T04:31:15.508 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T04:31:15.508 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.db_log_dir: 2023-10-09T04:31:15.508 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-10-09T04:31:15.508 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T04:31:15.509 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T04:31:15.509 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T04:31:15.509 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T04:31:15.509 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T04:31:15.509 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T04:31:15.509 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T04:31:15.509 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T04:31:15.509 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.write_buffer_manager: 0x5597770a2930 2023-10-09T04:31:15.509 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T04:31:15.509 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T04:31:15.509 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T04:31:15.509 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T04:31:15.510 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T04:31:15.510 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T04:31:15.510 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T04:31:15.510 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T04:31:15.510 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T04:31:15.510 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.unordered_write: 0 2023-10-09T04:31:15.510 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T04:31:15.510 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T04:31:15.510 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T04:31:15.510 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T04:31:15.510 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.row_cache: None 2023-10-09T04:31:15.511 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.wal_filter: None 2023-10-09T04:31:15.511 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T04:31:15.511 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T04:31:15.511 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.preserve_deletes: 0 2023-10-09T04:31:15.511 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.two_write_queues: 0 2023-10-09T04:31:15.511 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T04:31:15.511 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.atomic_flush: 0 2023-10-09T04:31:15.511 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T04:31:15.511 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T04:31:15.511 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T04:31:15.511 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T04:31:15.511 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T04:31:15.511 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.best_efforts_recovery: 0 2023-10-09T04:31:15.512 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T04:31:15.512 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T04:31:15.512 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.allow_data_in_errors: 0 2023-10-09T04:31:15.512 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.db_host_id: __hostname__ 2023-10-09T04:31:15.512 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T04:31:15.512 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T04:31:15.512 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T04:31:15.512 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T04:31:15.512 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T04:31:15.512 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T04:31:15.512 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T04:31:15.512 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T04:31:15.513 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T04:31:15.513 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T04:31:15.513 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T04:31:15.513 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.max_open_files: -1 2023-10-09T04:31:15.513 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T04:31:15.513 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T04:31:15.513 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T04:31:15.513 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T04:31:15.513 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T04:31:15.513 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Compression algorithms supported: 2023-10-09T04:31:15.513 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T04:31:15.513 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: kZSTD supported: 0 2023-10-09T04:31:15.513 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: kXpressCompression supported: 0 2023-10-09T04:31:15.514 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T04:31:15.514 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: kZlibCompression supported: 1 2023-10-09T04:31:15.514 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T04:31:15.514 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T04:31:15.514 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T04:31:15.514 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T04:31:15.514 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2023-10-09T04:31:15.514 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-10-09T04:31:15.514 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T04:31:15.514 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.merge_operator: 2023-10-09T04:31:15.515 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.compaction_filter: None 2023-10-09T04:31:15.516 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T04:31:15.516 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.sst_partitioner_factory: None 2023-10-09T04:31:15.516 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T04:31:15.516 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T04:31:15.516 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x559776380fb8) 2023-10-09T04:31:15.516 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: cache_index_and_filter_blocks: 1 2023-10-09T04:31:15.516 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T04:31:15.516 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T04:31:15.516 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: pin_top_level_index_and_filter: 1 2023-10-09T04:31:15.516 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: index_type: 0 2023-10-09T04:31:15.516 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: data_block_index_type: 0 2023-10-09T04:31:15.517 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: index_shortening: 1 2023-10-09T04:31:15.517 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: data_block_hash_table_util_ratio: 0.750000 2023-10-09T04:31:15.517 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: hash_index_allow_collision: 1 2023-10-09T04:31:15.517 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: checksum: 1 2023-10-09T04:31:15.517 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: no_block_cache: 0 2023-10-09T04:31:15.517 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: block_cache: 0x5597763b3090 2023-10-09T04:31:15.517 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: block_cache_name: BinnedLRUCache 2023-10-09T04:31:15.517 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: block_cache_options: 2023-10-09T04:31:15.517 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: capacity : 536870912 2023-10-09T04:31:15.517 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: num_shard_bits : 4 2023-10-09T04:31:15.517 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: strict_capacity_limit : 0 2023-10-09T04:31:15.517 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: high_pri_pool_ratio: 0.000 2023-10-09T04:31:15.518 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: block_cache_compressed: (nil) 2023-10-09T04:31:15.518 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: persistent_cache: (nil) 2023-10-09T04:31:15.518 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: block_size: 4096 2023-10-09T04:31:15.518 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: block_size_deviation: 10 2023-10-09T04:31:15.518 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: block_restart_interval: 16 2023-10-09T04:31:15.518 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: index_block_restart_interval: 1 2023-10-09T04:31:15.518 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: metadata_block_size: 4096 2023-10-09T04:31:15.518 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: partition_filters: 0 2023-10-09T04:31:15.518 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: use_delta_encoding: 1 2023-10-09T04:31:15.518 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T04:31:15.518 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: whole_key_filtering: 1 2023-10-09T04:31:15.518 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: verify_compression: 0 2023-10-09T04:31:15.519 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: read_amp_bytes_per_bit: 0 2023-10-09T04:31:15.519 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: format_version: 4 2023-10-09T04:31:15.519 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: enable_index_compression: 1 2023-10-09T04:31:15.519 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: block_align: 0 2023-10-09T04:31:15.519 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T04:31:15.519 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T04:31:15.519 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.compression: NoCompression 2023-10-09T04:31:15.519 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T04:31:15.519 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T04:31:15.519 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T04:31:15.519 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.num_levels: 7 2023-10-09T04:31:15.519 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T04:31:15.520 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T04:31:15.520 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T04:31:15.520 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T04:31:15.520 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T04:31:15.520 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T04:31:15.520 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T04:31:15.520 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:31:15.520 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T04:31:15.520 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T04:31:15.520 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T04:31:15.520 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T04:31:15.520 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T04:31:15.520 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T04:31:15.521 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:31:15.521 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T04:31:15.521 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T04:31:15.521 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T04:31:15.521 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T04:31:15.521 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T04:31:15.521 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T04:31:15.521 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T04:31:15.521 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T04:31:15.521 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T04:31:15.521 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T04:31:15.521 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T04:31:15.522 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T04:31:15.522 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T04:31:15.522 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T04:31:15.522 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T04:31:15.522 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T04:31:15.522 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T04:31:15.522 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T04:31:15.522 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T04:31:15.522 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.arena_block_size: 4194304 2023-10-09T04:31:15.522 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T04:31:15.522 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T04:31:15.522 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T04:31:15.523 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T04:31:15.523 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T04:31:15.523 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T04:31:15.523 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T04:31:15.523 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T04:31:15.523 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T04:31:15.523 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T04:31:15.523 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T04:31:15.523 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T04:31:15.523 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T04:31:15.523 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T04:31:15.523 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.table_properties_collectors: 2023-10-09T04:31:15.524 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T04:31:15.524 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T04:31:15.524 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T04:31:15.524 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T04:31:15.524 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T04:31:15.524 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.bloom_locality: 0 2023-10-09T04:31:15.524 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T04:31:15.524 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T04:31:15.524 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T04:31:15.524 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.force_consistency_checks: 1 2023-10-09T04:31:15.524 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T04:31:15.524 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.ttl: 2592000 2023-10-09T04:31:15.525 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T04:31:15.525 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.enable_blob_files: false 2023-10-09T04:31:15.525 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.min_blob_size: 0 2023-10-09T04:31:15.525 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.blob_file_size: 268435456 2023-10-09T04:31:15.525 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T04:31:15.525 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T04:31:15.525 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.201+0000 7fa591bdbb80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T04:31:15.525 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.205+0000 7fa591bdbb80 4 rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T04:31:15.525 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.205+0000 7fa591bdbb80 4 rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2023-10-09T04:31:15.525 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.205+0000 7fa591bdbb80 4 rocksdb: [db/version_set.cc:4082] Creating manifest 13 2023-10-09T04:31:15.525 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.205+0000 7fa591bdbb80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696825875207210, "job": 1, "event": "recovery_started", "wal_files": [10]} 2023-10-09T04:31:15.525 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.205+0000 7fa591bdbb80 4 rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2023-10-09T04:31:15.526 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.205+0000 7fa591bdbb80 3 rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-09T04:31:15.526 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.205+0000 7fa591bdbb80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696825875208832, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80192, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78460, "index_size": 220, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10136, "raw_average_key_size": 47, "raw_value_size": 72668, "raw_average_value_size": 341, "num_data_blocks": 10, "num_entries": 213, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696825875, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "f3f60ce5-6dcc-4737-bd6d-6fc3a9754473", "db_session_id": "HHB42WZRUT6DQZE1G21P"}} 2023-10-09T04:31:15.526 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.205+0000 7fa591bdbb80 4 rocksdb: [db/version_set.cc:4082] Creating manifest 15 2023-10-09T04:31:15.526 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.209+0000 7fa591bdbb80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696825875209481, "job": 1, "event": "recovery_finished"} 2023-10-09T04:31:15.526 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.209+0000 7fa591bdbb80 4 rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T04:31:15.526 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.209+0000 7fa591bdbb80 4 rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x559776347180 2023-10-09T04:31:15.526 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.209+0000 7fa591bdbb80 4 rocksdb: DB pointer 0x55977641c000 2023-10-09T04:31:15.526 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.209+0000 7fa581e2a700 4 rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-10-09T04:31:15.526 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.209+0000 7fa581e2a700 4 rocksdb: [db/db_impl/db_impl.cc:903] 2023-10-09T04:31:15.526 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: ** DB Stats ** 2023-10-09T04:31:15.526 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:31:15.526 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T04:31:15.527 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T04:31:15.527 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:31:15.527 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T04:31:15.527 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T04:31:15.527 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:31:15.527 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: ** Compaction Stats [default] ** 2023-10-09T04:31:15.527 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:31:15.527 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:31:15.527 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: L0 2/0 80.07 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 70.9 0.00 0.00 1 0.001 0 0 2023-10-09T04:31:15.527 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: Sum 2/0 80.07 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 70.9 0.00 0.00 1 0.001 0 0 2023-10-09T04:31:15.527 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 70.9 0.00 0.00 1 0.001 0 0 2023-10-09T04:31:15.527 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: ** Compaction Stats [default] ** 2023-10-09T04:31:15.528 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:31:15.528 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:31:15.528 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 70.9 0.00 0.00 1 0.001 0 0 2023-10-09T04:31:15.528 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:31:15.528 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T04:31:15.528 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T04:31:15.528 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T04:31:15.528 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T04:31:15.528 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T04:31:15.528 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: Cumulative compaction: 0.00 GB write, 12.80 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:31:15.528 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: Interval compaction: 0.00 GB write, 12.80 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:31:15.528 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T04:31:15.529 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T04:31:15.529 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: ** Compaction Stats [default] ** 2023-10-09T04:31:15.529 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:31:15.529 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:31:15.529 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: L0 2/0 80.07 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 70.9 0.00 0.00 1 0.001 0 0 2023-10-09T04:31:15.529 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: Sum 2/0 80.07 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 70.9 0.00 0.00 1 0.001 0 0 2023-10-09T04:31:15.529 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T04:31:15.529 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: ** Compaction Stats [default] ** 2023-10-09T04:31:15.529 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:31:15.529 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:31:15.529 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 70.9 0.00 0.00 1 0.001 0 0 2023-10-09T04:31:15.529 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:31:15.529 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T04:31:15.530 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T04:31:15.530 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T04:31:15.530 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T04:31:15.530 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T04:31:15.530 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: Cumulative compaction: 0.00 GB write, 12.66 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:31:15.530 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:31:15.530 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T04:31:15.530 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T04:31:15.530 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.209+0000 7fa591bdbb80 0 starting mon.a rank 0 at public addrs [v2:172.21.15.133:3300/0,v1:172.21.15.133:6789/0] at bind addrs [v2:172.21.15.133:3300/0,v1:172.21.15.133:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 855b5206-665c-11ee-8db5-212e2dc638e7 2023-10-09T04:31:15.530 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.213+0000 7fa591bdbb80 1 mon.a@-1(???) e1 preinit fsid 855b5206-665c-11ee-8db5-212e2dc638e7 2023-10-09T04:31:15.531 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.213+0000 7fa591bdbb80 0 mon.a@-1(???).mds e1 new map 2023-10-09T04:31:15.531 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.213+0000 7fa591bdbb80 0 mon.a@-1(???).mds e1 print_map 2023-10-09T04:31:15.531 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: e1 2023-10-09T04:31:15.531 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T04:31:15.531 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T04:31:15.531 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: legacy client fscid: -1 2023-10-09T04:31:15.531 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: 2023-10-09T04:31:15.531 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: No filesystems configured 2023-10-09T04:31:15.531 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.213+0000 7fa591bdbb80 0 mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-09T04:31:15.531 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.213+0000 7fa591bdbb80 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T04:31:15.531 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.213+0000 7fa591bdbb80 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T04:31:15.531 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.213+0000 7fa591bdbb80 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T04:31:15.532 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: debug 2023-10-09T04:31:15.213+0000 7fa591bdbb80 1 mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-10-09T04:31:15.532 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: cluster 2023-10-09T04:31:15.218426+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T04:31:15.532 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: cluster 2023-10-09T04:31:15.218503+0000 mon.a (mon.0) 2 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.133:3300/0,v1:172.21.15.133:6789/0]} removed_ranks: {} 2023-10-09T04:31:15.532 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: cluster 2023-10-09T04:31:15.238303+0000 mon.a (mon.0) 3 : cluster [DBG] fsmap 2023-10-09T04:31:15.532 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: cluster 2023-10-09T04:31:15.238364+0000 mon.a (mon.0) 4 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-10-09T04:31:15.532 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:15 smithi133 bash[27558]: cluster 2023-10-09T04:31:15.239106+0000 mon.a (mon.0) 5 : cluster [DBG] mgrmap e1: no daemons active 2023-10-09T04:31:16.173 INFO:teuthology.orchestra.run.smithi133.stdout:Wrote config to /etc/ceph/ceph.conf 2023-10-09T04:31:16.174 INFO:teuthology.orchestra.run.smithi133.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-10-09T04:31:16.174 INFO:teuthology.orchestra.run.smithi133.stdout:Creating mgr... 2023-10-09T04:31:16.174 INFO:teuthology.orchestra.run.smithi133.stdout:Verifying port 9283 ... 2023-10-09T04:31:16.417 INFO:teuthology.orchestra.run.smithi133.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mgr.y 2023-10-09T04:31:16.417 INFO:teuthology.orchestra.run.smithi133.stdout:systemctl: stderr Failed to reset failed state of unit ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mgr.y.service: Unit ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mgr.y.service not loaded. 2023-10-09T04:31:16.626 INFO:teuthology.orchestra.run.smithi133.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-855b5206-665c-11ee-8db5-212e2dc638e7.target.wants/ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mgr.y.service -> /etc/systemd/system/ceph-855b5206-665c-11ee-8db5-212e2dc638e7@.service. 2023-10-09T04:31:16.637 INFO:teuthology.orchestra.run.smithi133.stdout:firewalld does not appear to be present 2023-10-09T04:31:16.637 INFO:teuthology.orchestra.run.smithi133.stdout:Not possible to enable service . firewalld.service is not available 2023-10-09T04:31:16.637 INFO:teuthology.orchestra.run.smithi133.stdout:firewalld does not appear to be present 2023-10-09T04:31:16.637 INFO:teuthology.orchestra.run.smithi133.stdout:Not possible to open ports <[9283]>. firewalld.service is not available 2023-10-09T04:31:16.638 INFO:teuthology.orchestra.run.smithi133.stdout:Waiting for mgr to start... 2023-10-09T04:31:16.638 INFO:teuthology.orchestra.run.smithi133.stdout:Waiting for mgr... 2023-10-09T04:31:16.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:16 smithi133 bash[27558]: audit 2023-10-09T04:31:15.471245+0000 mon.a (mon.0) 6 : audit [INF] from='client.? 172.21.15.133:0/1559429860' entity='client.admin' 2023-10-09T04:31:18.500 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:18 smithi133 bash[27558]: audit 2023-10-09T04:31:18.239918+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.133:0/540096873' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T04:31:18.500 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:18 smithi133 bash[27821]: debug 2023-10-09T04:31:18.113+0000 7f9c4b94d1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:31:18.982 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout 2023-10-09T04:31:18.982 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout { 2023-10-09T04:31:18.982 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "fsid": "855b5206-665c-11ee-8db5-212e2dc638e7", 2023-10-09T04:31:18.982 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "health": { 2023-10-09T04:31:18.982 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-09T04:31:18.982 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-09T04:31:18.982 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-09T04:31:18.982 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:31:18.982 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-09T04:31:18.983 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-09T04:31:18.983 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout 0 2023-10-09T04:31:18.983 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout ], 2023-10-09T04:31:18.983 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-09T04:31:18.983 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "a" 2023-10-09T04:31:18.983 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout ], 2023-10-09T04:31:18.983 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "quorum_age": 3, 2023-10-09T04:31:18.983 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-09T04:31:18.983 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:31:18.983 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-10-09T04:31:18.983 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-09T04:31:18.983 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:31:18.984 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-09T04:31:18.984 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:31:18.984 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-09T04:31:18.984 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-09T04:31:18.984 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-09T04:31:18.984 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-09T04:31:18.984 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-09T04:31:18.984 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-09T04:31:18.987 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:31:18.987 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-09T04:31:18.987 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-09T04:31:18.987 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-09T04:31:18.987 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-09T04:31:18.987 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-09T04:31:18.988 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-09T04:31:18.988 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-09T04:31:18.988 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-09T04:31:18.988 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-09T04:31:18.988 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:31:18.988 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-09T04:31:18.988 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:31:18.988 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-09T04:31:18.988 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-09T04:31:18.988 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:31:18.988 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-09T04:31:18.988 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "available": false, 2023-10-09T04:31:18.989 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-09T04:31:18.989 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-09T04:31:18.989 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-09T04:31:18.989 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-09T04:31:18.989 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "restful" 2023-10-09T04:31:18.989 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout ], 2023-10-09T04:31:18.989 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T04:31:18.989 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:31:18.989 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-09T04:31:18.989 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:31:18.989 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "modified": "2023-10-09T04:31:08.634399+0000", 2023-10-09T04:31:18.989 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T04:31:18.989 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:31:18.990 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-09T04:31:18.990 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout } 2023-10-09T04:31:18.990 INFO:teuthology.orchestra.run.smithi133.stdout:mgr not available, waiting (1/15)... 2023-10-09T04:31:19.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:19 smithi133 bash[27821]: debug 2023-10-09T04:31:19.057+0000 7f9c4b94d1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:31:19.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:19 smithi133 bash[27821]: debug 2023-10-09T04:31:19.165+0000 7f9c4b94d1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:31:19.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:19 smithi133 bash[27821]: debug 2023-10-09T04:31:19.273+0000 7f9c4b94d1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:31:19.930 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:19 smithi133 bash[27821]: debug 2023-10-09T04:31:19.561+0000 7f9c4b94d1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:31:20.293 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:19 smithi133 bash[27821]: debug 2023-10-09T04:31:19.925+0000 7f9c4b94d1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:31:20.293 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:20 smithi133 bash[27821]: debug 2023-10-09T04:31:20.021+0000 7f9c4b94d1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:31:20.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:20 smithi133 bash[27821]: debug 2023-10-09T04:31:20.289+0000 7f9c4b94d1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:31:21.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:21 smithi133 bash[27821]: debug 2023-10-09T04:31:21.281+0000 7f9c4b94d1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:31:21.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:21 smithi133 bash[27821]: debug 2023-10-09T04:31:21.389+0000 7f9c4b94d1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:31:21.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:21 smithi133 bash[27821]: debug 2023-10-09T04:31:21.505+0000 7f9c4b94d1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:31:22.049 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:21 smithi133 bash[27821]: debug 2023-10-09T04:31:21.977+0000 7f9c4b94d1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:31:22.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:22 smithi133 bash[27558]: audit 2023-10-09T04:31:22.159967+0000 mon.a (mon.0) 8 : audit [DBG] from='client.? 172.21.15.133:0/3968251143' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T04:31:22.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:22 smithi133 bash[27821]: debug 2023-10-09T04:31:22.197+0000 7f9c4b94d1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:31:22.798 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:22 smithi133 bash[27821]: debug 2023-10-09T04:31:22.585+0000 7f9c4b94d1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:31:22.815 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout 2023-10-09T04:31:22.815 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout { 2023-10-09T04:31:22.815 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "fsid": "855b5206-665c-11ee-8db5-212e2dc638e7", 2023-10-09T04:31:22.815 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "health": { 2023-10-09T04:31:22.815 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-09T04:31:22.815 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-09T04:31:22.815 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-09T04:31:22.815 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:31:22.815 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-09T04:31:22.815 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-09T04:31:22.815 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout 0 2023-10-09T04:31:22.816 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout ], 2023-10-09T04:31:22.816 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-09T04:31:22.816 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "a" 2023-10-09T04:31:22.816 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout ], 2023-10-09T04:31:22.816 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "quorum_age": 6, 2023-10-09T04:31:22.816 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-09T04:31:22.816 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:31:22.816 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-10-09T04:31:22.816 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-09T04:31:22.816 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:31:22.816 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-09T04:31:22.816 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:31:22.817 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-09T04:31:22.817 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-09T04:31:22.817 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-09T04:31:22.817 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-09T04:31:22.817 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-09T04:31:22.817 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-09T04:31:22.817 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:31:22.817 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-09T04:31:22.817 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-09T04:31:22.817 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-09T04:31:22.817 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-09T04:31:22.817 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-09T04:31:22.817 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-09T04:31:22.818 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-09T04:31:22.818 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-09T04:31:22.818 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-09T04:31:22.818 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:31:22.818 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-09T04:31:22.818 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:31:22.818 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-09T04:31:22.818 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-09T04:31:22.818 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:31:22.818 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-09T04:31:22.818 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "available": false, 2023-10-09T04:31:22.818 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-09T04:31:22.819 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-09T04:31:22.819 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-09T04:31:22.819 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-09T04:31:22.819 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "restful" 2023-10-09T04:31:22.819 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout ], 2023-10-09T04:31:22.819 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T04:31:22.819 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:31:22.819 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-09T04:31:22.819 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:31:22.819 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "modified": "2023-10-09T04:31:08.634399+0000", 2023-10-09T04:31:22.819 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T04:31:22.819 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:31:22.819 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-09T04:31:22.820 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout } 2023-10-09T04:31:22.820 INFO:teuthology.orchestra.run.smithi133.stdout:mgr not available, waiting (2/15)... 2023-10-09T04:31:23.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:23 smithi133 bash[27821]: debug 2023-10-09T04:31:23.069+0000 7f9c4b94d1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:31:23.884 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:23 smithi133 bash[27821]: debug 2023-10-09T04:31:23.613+0000 7f9c4b94d1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:31:23.884 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:23 smithi133 bash[27821]: debug 2023-10-09T04:31:23.717+0000 7f9c4b94d1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:31:24.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:23 smithi133 bash[27821]: debug 2023-10-09T04:31:23.881+0000 7f9c4b94d1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:31:24.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:24 smithi133 bash[27821]: debug 2023-10-09T04:31:24.177+0000 7f9c4b94d1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:31:24.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:24 smithi133 bash[27821]: debug 2023-10-09T04:31:24.313+0000 7f9c4b94d1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:31:24.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:24 smithi133 bash[27821]: debug 2023-10-09T04:31:24.425+0000 7f9c4b94d1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:31:24.842 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:24 smithi133 bash[27821]: debug 2023-10-09T04:31:24.545+0000 7f9c4b94d1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:31:24.843 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:24 smithi133 bash[27821]: debug 2023-10-09T04:31:24.729+0000 7f9c4b94d1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:31:25.298 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:24 smithi133 bash[27821]: debug 2023-10-09T04:31:24.853+0000 7f9c4b94d1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:31:26.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:26 smithi133 bash[27821]: debug 2023-10-09T04:31:26.097+0000 7f9c4b94d1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:31:26.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:26 smithi133 bash[27558]: audit 2023-10-09T04:31:26.109665+0000 mon.a (mon.0) 9 : audit [DBG] from='client.? 172.21.15.133:0/1919457651' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T04:31:26.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:26 smithi133 bash[27558]: cluster 2023-10-09T04:31:26.119530+0000 mon.a (mon.0) 10 : cluster [INF] Activating manager daemon y 2023-10-09T04:31:26.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:26 smithi133 bash[27558]: cluster 2023-10-09T04:31:26.124904+0000 mon.a (mon.0) 11 : cluster [DBG] mgrmap e2: y(active, starting, since 0.00560988s) 2023-10-09T04:31:26.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:26 smithi133 bash[27558]: audit 2023-10-09T04:31:26.127122+0000 mon.a (mon.0) 12 : audit [DBG] from='mgr.14100 172.21.15.133:0/2566873516' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:31:26.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:26 smithi133 bash[27558]: audit 2023-10-09T04:31:26.127470+0000 mon.a (mon.0) 13 : audit [DBG] from='mgr.14100 172.21.15.133:0/2566873516' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:31:26.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:26 smithi133 bash[27558]: audit 2023-10-09T04:31:26.127799+0000 mon.a (mon.0) 14 : audit [DBG] from='mgr.14100 172.21.15.133:0/2566873516' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:31:26.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:26 smithi133 bash[27558]: audit 2023-10-09T04:31:26.128085+0000 mon.a (mon.0) 15 : audit [DBG] from='mgr.14100 172.21.15.133:0/2566873516' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:31:26.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:26 smithi133 bash[27558]: audit 2023-10-09T04:31:26.128375+0000 mon.a (mon.0) 16 : audit [DBG] from='mgr.14100 172.21.15.133:0/2566873516' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:31:26.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:26 smithi133 bash[27558]: cluster 2023-10-09T04:31:26.161584+0000 mon.a (mon.0) 17 : cluster [INF] Manager daemon y is now available 2023-10-09T04:31:26.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:26 smithi133 bash[27558]: audit 2023-10-09T04:31:26.179852+0000 mon.a (mon.0) 18 : audit [INF] from='mgr.14100 172.21.15.133:0/2566873516' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:31:26.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:26 smithi133 bash[27558]: audit 2023-10-09T04:31:26.184488+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.14100 172.21.15.133:0/2566873516' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:31:26.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:26 smithi133 bash[27558]: audit 2023-10-09T04:31:26.188327+0000 mon.a (mon.0) 20 : audit [INF] from='mgr.14100 172.21.15.133:0/2566873516' entity='mgr.y' 2023-10-09T04:31:26.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:26 smithi133 bash[27558]: audit 2023-10-09T04:31:26.190813+0000 mon.a (mon.0) 21 : audit [INF] from='mgr.14100 172.21.15.133:0/2566873516' entity='mgr.y' 2023-10-09T04:31:26.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:26 smithi133 bash[27558]: audit 2023-10-09T04:31:26.193699+0000 mon.a (mon.0) 22 : audit [INF] from='mgr.14100 172.21.15.133:0/2566873516' entity='mgr.y' 2023-10-09T04:31:26.736 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout 2023-10-09T04:31:26.737 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout { 2023-10-09T04:31:26.737 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "fsid": "855b5206-665c-11ee-8db5-212e2dc638e7", 2023-10-09T04:31:26.737 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "health": { 2023-10-09T04:31:26.737 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-09T04:31:26.737 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-09T04:31:26.737 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-09T04:31:26.737 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:31:26.737 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-09T04:31:26.738 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-09T04:31:26.738 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout 0 2023-10-09T04:31:26.738 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout ], 2023-10-09T04:31:26.738 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-09T04:31:26.738 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "a" 2023-10-09T04:31:26.738 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout ], 2023-10-09T04:31:26.738 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "quorum_age": 10, 2023-10-09T04:31:26.738 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-09T04:31:26.738 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:31:26.738 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-10-09T04:31:26.739 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-09T04:31:26.739 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:31:26.739 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-09T04:31:26.739 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:31:26.739 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-09T04:31:26.739 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-09T04:31:26.739 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-09T04:31:26.739 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-09T04:31:26.739 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-09T04:31:26.739 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-09T04:31:26.742 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:31:26.743 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-09T04:31:26.743 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-09T04:31:26.743 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-09T04:31:26.743 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-09T04:31:26.743 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-09T04:31:26.743 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-09T04:31:26.743 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-09T04:31:26.743 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-09T04:31:26.743 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-09T04:31:26.744 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:31:26.744 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-09T04:31:26.744 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:31:26.744 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-09T04:31:26.744 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-09T04:31:26.744 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:31:26.744 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-09T04:31:26.744 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "available": false, 2023-10-09T04:31:26.744 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-09T04:31:26.744 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-09T04:31:26.744 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-09T04:31:26.744 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-09T04:31:26.745 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "restful" 2023-10-09T04:31:26.745 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout ], 2023-10-09T04:31:26.745 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T04:31:26.745 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:31:26.745 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-09T04:31:26.745 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:31:26.745 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "modified": "2023-10-09T04:31:08.634399+0000", 2023-10-09T04:31:26.745 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T04:31:26.745 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:31:26.745 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-09T04:31:26.745 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout } 2023-10-09T04:31:26.745 INFO:teuthology.orchestra.run.smithi133.stdout:mgr not available, waiting (3/15)... 2023-10-09T04:31:28.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:28 smithi133 bash[27558]: cluster 2023-10-09T04:31:27.131405+0000 mon.a (mon.0) 23 : cluster [DBG] mgrmap e3: y(active, since 1.01211s) 2023-10-09T04:31:30.392 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:30 smithi133 bash[27558]: cluster 2023-10-09T04:31:29.132766+0000 mon.a (mon.0) 24 : cluster [DBG] mgrmap e4: y(active, since 3s) 2023-10-09T04:31:31.019 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout 2023-10-09T04:31:31.019 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout { 2023-10-09T04:31:31.019 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "fsid": "855b5206-665c-11ee-8db5-212e2dc638e7", 2023-10-09T04:31:31.019 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "health": { 2023-10-09T04:31:31.019 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-09T04:31:31.020 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-09T04:31:31.020 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-09T04:31:31.020 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:31:31.020 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-09T04:31:31.020 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-09T04:31:31.020 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout 0 2023-10-09T04:31:31.020 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout ], 2023-10-09T04:31:31.020 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-09T04:31:31.020 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "a" 2023-10-09T04:31:31.020 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout ], 2023-10-09T04:31:31.020 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "quorum_age": 15, 2023-10-09T04:31:31.020 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-09T04:31:31.021 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:31:31.021 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-10-09T04:31:31.021 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-09T04:31:31.021 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:31:31.021 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-09T04:31:31.021 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:31:31.021 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-09T04:31:31.021 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-09T04:31:31.021 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-09T04:31:31.021 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-09T04:31:31.021 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-09T04:31:31.021 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-09T04:31:31.022 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:31:31.022 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-09T04:31:31.022 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-09T04:31:31.022 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-09T04:31:31.022 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-09T04:31:31.022 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-09T04:31:31.022 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-09T04:31:31.022 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-09T04:31:31.022 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-09T04:31:31.022 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-09T04:31:31.022 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:31:31.022 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-09T04:31:31.023 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:31:31.023 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-09T04:31:31.023 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-09T04:31:31.023 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:31:31.023 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-09T04:31:31.023 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "available": true, 2023-10-09T04:31:31.023 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-09T04:31:31.023 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-09T04:31:31.023 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-09T04:31:31.023 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-09T04:31:31.023 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "restful" 2023-10-09T04:31:31.023 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout ], 2023-10-09T04:31:31.024 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T04:31:31.024 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:31:31.024 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-09T04:31:31.024 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:31:31.024 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "modified": "2023-10-09T04:31:08.634399+0000", 2023-10-09T04:31:31.024 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T04:31:31.024 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:31:31.024 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-09T04:31:31.024 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout } 2023-10-09T04:31:31.024 INFO:teuthology.orchestra.run.smithi133.stdout:mgr is available 2023-10-09T04:31:31.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:31 smithi133 bash[27558]: audit 2023-10-09T04:31:30.310947+0000 mon.a (mon.0) 25 : audit [DBG] from='client.? 172.21.15.133:0/2215966270' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T04:31:32.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:32 smithi133 bash[27558]: audit 2023-10-09T04:31:32.405387+0000 mon.a (mon.0) 26 : audit [INF] from='client.? 172.21.15.133:0/3495799249' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-09T04:31:33.045 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout 2023-10-09T04:31:33.045 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout [global] 2023-10-09T04:31:33.045 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout fsid = 855b5206-665c-11ee-8db5-212e2dc638e7 2023-10-09T04:31:33.045 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-10-09T04:31:33.045 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-10-09T04:31:33.045 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-10-09T04:31:33.046 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-10-09T04:31:33.046 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout 2023-10-09T04:31:33.046 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout [mgr] 2023-10-09T04:31:33.046 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-10-09T04:31:33.046 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout 2023-10-09T04:31:33.046 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout [osd] 2023-10-09T04:31:33.046 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-10-09T04:31:33.046 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-10-09T04:31:33.046 INFO:teuthology.orchestra.run.smithi133.stdout:Enabling cephadm module... 2023-10-09T04:31:35.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:34 smithi133 bash[27558]: audit 2023-10-09T04:31:34.537441+0000 mon.a (mon.0) 27 : audit [INF] from='client.? 172.21.15.133:0/3098782708' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-10-09T04:31:35.048 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:34 smithi133 bash[27821]: ignoring --setuser ceph since I am not root 2023-10-09T04:31:35.049 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:34 smithi133 bash[27821]: ignoring --setgroup ceph since I am not root 2023-10-09T04:31:35.049 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:34 smithi133 bash[27821]: debug 2023-10-09T04:31:34.793+0000 7f32f57d81c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:31:35.863 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:35 smithi133 bash[27558]: audit 2023-10-09T04:31:34.592505+0000 mon.a (mon.0) 28 : audit [INF] from='client.? 172.21.15.133:0/3098782708' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-10-09T04:31:35.863 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:35 smithi133 bash[27558]: cluster 2023-10-09T04:31:34.592562+0000 mon.a (mon.0) 29 : cluster [DBG] mgrmap e5: y(active, since 8s) 2023-10-09T04:31:35.863 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:35 smithi133 bash[27821]: debug 2023-10-09T04:31:35.685+0000 7f32f57d81c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:31:35.863 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:35 smithi133 bash[27821]: debug 2023-10-09T04:31:35.781+0000 7f32f57d81c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:31:36.151 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:35 smithi133 bash[27821]: debug 2023-10-09T04:31:35.885+0000 7f32f57d81c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:31:36.504 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:36 smithi133 bash[27821]: debug 2023-10-09T04:31:36.149+0000 7f32f57d81c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:31:36.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:36 smithi133 bash[27558]: audit 2023-10-09T04:31:36.601912+0000 mon.a (mon.0) 30 : audit [DBG] from='client.? 172.21.15.133:0/98692648' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-09T04:31:36.798 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:36 smithi133 bash[27821]: debug 2023-10-09T04:31:36.501+0000 7f32f57d81c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:31:36.799 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:36 smithi133 bash[27821]: debug 2023-10-09T04:31:36.597+0000 7f32f57d81c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:31:37.295 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout { 2023-10-09T04:31:37.296 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "epoch": 5, 2023-10-09T04:31:37.296 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "available": true, 2023-10-09T04:31:37.296 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-10-09T04:31:37.296 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-10-09T04:31:37.296 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout } 2023-10-09T04:31:37.296 INFO:teuthology.orchestra.run.smithi133.stdout:Waiting for the mgr to restart... 2023-10-09T04:31:37.296 INFO:teuthology.orchestra.run.smithi133.stdout:Waiting for mgr epoch 5... 2023-10-09T04:31:37.298 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:36 smithi133 bash[27821]: debug 2023-10-09T04:31:36.869+0000 7f32f57d81c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:31:38.139 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:37 smithi133 bash[27821]: debug 2023-10-09T04:31:37.861+0000 7f32f57d81c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:31:38.139 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:37 smithi133 bash[27821]: debug 2023-10-09T04:31:37.973+0000 7f32f57d81c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:31:38.140 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:38 smithi133 bash[27821]: debug 2023-10-09T04:31:38.085+0000 7f32f57d81c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:31:39.048 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:38 smithi133 bash[27821]: debug 2023-10-09T04:31:38.561+0000 7f32f57d81c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:31:39.048 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:38 smithi133 bash[27821]: debug 2023-10-09T04:31:38.781+0000 7f32f57d81c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:31:39.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:39 smithi133 bash[27821]: debug 2023-10-09T04:31:39.149+0000 7f32f57d81c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:31:40.048 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:39 smithi133 bash[27821]: debug 2023-10-09T04:31:39.637+0000 7f32f57d81c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:31:40.458 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:40 smithi133 bash[27821]: debug 2023-10-09T04:31:40.185+0000 7f32f57d81c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:31:40.459 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:40 smithi133 bash[27821]: debug 2023-10-09T04:31:40.297+0000 7f32f57d81c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:31:40.751 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:40 smithi133 bash[27821]: debug 2023-10-09T04:31:40.457+0000 7f32f57d81c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:31:41.048 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:40 smithi133 bash[27821]: debug 2023-10-09T04:31:40.749+0000 7f32f57d81c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:31:41.048 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:40 smithi133 bash[27821]: debug 2023-10-09T04:31:40.885+0000 7f32f57d81c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:31:41.048 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:41 smithi133 bash[27821]: debug 2023-10-09T04:31:40.997+0000 7f32f57d81c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:31:41.422 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:41 smithi133 bash[27821]: debug 2023-10-09T04:31:41.121+0000 7f32f57d81c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:31:41.422 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:41 smithi133 bash[27821]: debug 2023-10-09T04:31:41.301+0000 7f32f57d81c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:31:41.798 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:41 smithi133 bash[27821]: debug 2023-10-09T04:31:41.421+0000 7f32f57d81c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:31:43.048 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:42 smithi133 bash[27821]: debug 2023-10-09T04:31:42.617+0000 7f32f57d81c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:31:43.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:42 smithi133 bash[27558]: cluster 2023-10-09T04:31:42.639522+0000 mon.a (mon.0) 31 : cluster [INF] Active manager daemon y restarted 2023-10-09T04:31:43.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:42 smithi133 bash[27558]: cluster 2023-10-09T04:31:42.640228+0000 mon.a (mon.0) 32 : cluster [INF] Activating manager daemon y 2023-10-09T04:31:43.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:42 smithi133 bash[27558]: cluster 2023-10-09T04:31:42.647241+0000 mon.a (mon.0) 33 : cluster [DBG] osdmap e2: 0 total, 0 up, 0 in 2023-10-09T04:31:43.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:42 smithi133 bash[27558]: cluster 2023-10-09T04:31:42.647472+0000 mon.a (mon.0) 34 : cluster [DBG] mgrmap e6: y(active, starting, since 0.00752425s) 2023-10-09T04:31:43.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:42 smithi133 bash[27558]: audit 2023-10-09T04:31:42.648921+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.14120 172.21.15.133:0/577502051' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:31:43.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:42 smithi133 bash[27558]: audit 2023-10-09T04:31:42.650299+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.14120 172.21.15.133:0/577502051' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:31:43.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:42 smithi133 bash[27558]: audit 2023-10-09T04:31:42.651422+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.14120 172.21.15.133:0/577502051' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:31:43.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:42 smithi133 bash[27558]: audit 2023-10-09T04:31:42.651636+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.14120 172.21.15.133:0/577502051' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:31:43.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:42 smithi133 bash[27558]: audit 2023-10-09T04:31:42.651847+0000 mon.a (mon.0) 39 : audit [DBG] from='mgr.14120 172.21.15.133:0/577502051' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:31:43.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:42 smithi133 bash[27558]: cluster 2023-10-09T04:31:42.677763+0000 mon.a (mon.0) 40 : cluster [INF] Manager daemon y is now available 2023-10-09T04:31:44.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:43 smithi133 bash[27558]: cephadm 2023-10-09T04:31:42.696016+0000 mgr.y (mgr.14120) 1 : cephadm [INF] Found migration_current of "None". Setting to last migration. 2023-10-09T04:31:44.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:43 smithi133 bash[27558]: audit 2023-10-09T04:31:42.702869+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.14120 172.21.15.133:0/577502051' entity='mgr.y' 2023-10-09T04:31:44.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:43 smithi133 bash[27558]: audit 2023-10-09T04:31:42.709523+0000 mon.a (mon.0) 42 : audit [INF] from='mgr.14120 172.21.15.133:0/577502051' entity='mgr.y' 2023-10-09T04:31:44.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:43 smithi133 bash[27558]: audit 2023-10-09T04:31:42.713958+0000 mon.a (mon.0) 43 : audit [DBG] from='mgr.14120 172.21.15.133:0/577502051' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:31:44.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:43 smithi133 bash[27558]: audit 2023-10-09T04:31:42.716373+0000 mon.a (mon.0) 44 : audit [DBG] from='mgr.14120 172.21.15.133:0/577502051' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:31:44.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:43 smithi133 bash[27558]: audit 2023-10-09T04:31:42.722344+0000 mon.a (mon.0) 45 : audit [DBG] from='mgr.14120 172.21.15.133:0/577502051' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:31:44.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:43 smithi133 bash[27558]: audit 2023-10-09T04:31:42.737378+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.14120 172.21.15.133:0/577502051' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:31:44.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:43 smithi133 bash[27558]: audit 2023-10-09T04:31:42.741267+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.14120 172.21.15.133:0/577502051' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:31:44.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:43 smithi133 bash[27558]: cluster 2023-10-09T04:31:43.652340+0000 mon.a (mon.0) 48 : cluster [DBG] mgrmap e7: y(active, since 1.01238s) 2023-10-09T04:31:44.342 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout { 2023-10-09T04:31:44.342 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2023-10-09T04:31:44.343 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "initialized": true 2023-10-09T04:31:44.343 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout } 2023-10-09T04:31:44.343 INFO:teuthology.orchestra.run.smithi133.stdout:mgr epoch 5 is available 2023-10-09T04:31:44.343 INFO:teuthology.orchestra.run.smithi133.stdout:Setting orchestrator backend to cephadm... 2023-10-09T04:31:45.065 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:44 smithi133 bash[27558]: audit 2023-10-09T04:31:43.650279+0000 mgr.y (mgr.14120) 2 : audit [DBG] from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-09T04:31:45.066 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:44 smithi133 bash[27558]: audit 2023-10-09T04:31:43.662918+0000 mgr.y (mgr.14120) 3 : audit [DBG] from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-09T04:31:45.066 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:44 smithi133 bash[27558]: cephadm 2023-10-09T04:31:43.667732+0000 mgr.y (mgr.14120) 4 : cephadm [INF] [09/Oct/2023:04:31:43] ENGINE Bus STARTING 2023-10-09T04:31:45.066 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:44 smithi133 bash[27558]: cephadm 2023-10-09T04:31:43.791323+0000 mgr.y (mgr.14120) 5 : cephadm [INF] [09/Oct/2023:04:31:43] ENGINE Serving on https://172.21.15.133:7150 2023-10-09T04:31:45.066 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:44 smithi133 bash[27558]: cephadm 2023-10-09T04:31:43.791632+0000 mgr.y (mgr.14120) 6 : cephadm [INF] [09/Oct/2023:04:31:43] ENGINE Bus STARTED 2023-10-09T04:31:45.066 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:44 smithi133 bash[27558]: audit 2023-10-09T04:31:43.800916+0000 mon.a (mon.0) 49 : audit [INF] from='mgr.14120 172.21.15.133:0/577502051' entity='mgr.y' 2023-10-09T04:31:45.066 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:44 smithi133 bash[27558]: audit 2023-10-09T04:31:43.843456+0000 mon.a (mon.0) 50 : audit [DBG] from='mgr.14120 172.21.15.133:0/577502051' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:31:46.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:45 smithi133 bash[27558]: cluster 2023-10-09T04:31:44.807606+0000 mon.a (mon.0) 51 : cluster [DBG] mgrmap e8: y(active, since 2s) 2023-10-09T04:31:46.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:45 smithi133 bash[27558]: audit 2023-10-09T04:31:45.743003+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.14120 172.21.15.133:0/577502051' entity='mgr.y' 2023-10-09T04:31:46.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:45 smithi133 bash[27558]: audit 2023-10-09T04:31:45.765955+0000 mon.a (mon.0) 53 : audit [DBG] from='mgr.14120 172.21.15.133:0/577502051' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:31:47.113 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:46 smithi133 bash[27558]: audit 2023-10-09T04:31:45.733716+0000 mgr.y (mgr.14120) 7 : audit [DBG] from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:31:48.377 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout value unchanged 2023-10-09T04:31:48.378 INFO:teuthology.orchestra.run.smithi133.stdout:Generating ssh key... 2023-10-09T04:31:49.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:48 smithi133 bash[27558]: audit 2023-10-09T04:31:47.748490+0000 mgr.y (mgr.14120) 8 : audit [DBG] from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:31:50.278 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:49 smithi133 bash[27821]: Generating public/private rsa key pair. 2023-10-09T04:31:50.278 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:49 smithi133 bash[27821]: Your identification has been saved in /tmp/tmp4jto0pwr/key. 2023-10-09T04:31:50.279 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:49 smithi133 bash[27821]: Your public key has been saved in /tmp/tmp4jto0pwr/key.pub. 2023-10-09T04:31:50.279 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:49 smithi133 bash[27821]: The key fingerprint is: 2023-10-09T04:31:50.279 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:49 smithi133 bash[27821]: SHA256:GsRof5jNtKDRiSqR1Q6GJX+gq+azh2PqaACB3dlwESk ceph-855b5206-665c-11ee-8db5-212e2dc638e7 2023-10-09T04:31:50.279 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:49 smithi133 bash[27821]: The key's randomart image is: 2023-10-09T04:31:50.279 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:49 smithi133 bash[27821]: +---[RSA 3072]----+ 2023-10-09T04:31:50.279 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:49 smithi133 bash[27821]: |o++o.+++ | 2023-10-09T04:31:50.279 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:49 smithi133 bash[27821]: |+*+.E*o. | 2023-10-09T04:31:50.279 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:49 smithi133 bash[27821]: |+o.o*.* . | 2023-10-09T04:31:50.279 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:49 smithi133 bash[27821]: |.o +.= B . | 2023-10-09T04:31:50.279 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:49 smithi133 bash[27821]: |+ . . = S | 2023-10-09T04:31:50.279 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:49 smithi133 bash[27821]: |o. + | 2023-10-09T04:31:50.280 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:49 smithi133 bash[27821]: |o.. . | 2023-10-09T04:31:50.280 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:49 smithi133 bash[27821]: |+* . | 2023-10-09T04:31:50.280 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:49 smithi133 bash[27821]: |B+= | 2023-10-09T04:31:50.280 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:31:49 smithi133 bash[27821]: +----[SHA256]-----+ 2023-10-09T04:31:51.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:50 smithi133 bash[27558]: audit 2023-10-09T04:31:49.755492+0000 mgr.y (mgr.14120) 9 : audit [DBG] from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:31:51.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:50 smithi133 bash[27558]: cephadm 2023-10-09T04:31:49.755900+0000 mgr.y (mgr.14120) 10 : cephadm [INF] Generating ssh key... 2023-10-09T04:31:51.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:50 smithi133 bash[27558]: audit 2023-10-09T04:31:49.998329+0000 mon.a (mon.0) 54 : audit [INF] from='mgr.14120 172.21.15.133:0/577502051' entity='mgr.y' 2023-10-09T04:31:51.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:50 smithi133 bash[27558]: audit 2023-10-09T04:31:50.003367+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.14120 172.21.15.133:0/577502051' entity='mgr.y' 2023-10-09T04:31:52.746 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDIQrEyiuSaKmYcvT73+p42a3K6RIkGJM4yGIDn8J0HE7T9lH6JvTdLeHgzDyhaeC9TdhWLTwXcP70dZbkmXjorABwiA/fZYpkIerBKW0Wgs2LGl3u6Br/BO3ASYMP0UJA3VaXn9IZL62IOBlUdqtz+rfwfHDa+hc8dnTsgBlrozggxD3pg/utGpf59n1QnVIoet2xN3dPqc0Dwgh0IZM3tedXi25NrEqWtxYx7JHxAa3B9LM5xXdyF/U6V/B4hk89Vje3Y6MRC8Znf3Gw2CxBH2qjThkN6b6n0WX/9AeEj4sRX9DKKrcY84CAUagBmvLSfYy6vbzsW/iYyUOFLKA36aiYoZ3wNo3hvmbB0srlTM22V1EjUzT54ktGtZ59T2nuTNwkCRu8GoJKexA/SEs4bP8QBHd0eOx0fDy2BMOFBF5yz6/kSK6E650dUR8iRyV3B3ZXiZsNVsSfk8SJ1RWVWbF5ADkkuHKljAg/Zx6pk7VvzCLjfUxrvXugByzOpzv8= ceph-855b5206-665c-11ee-8db5-212e2dc638e7 2023-10-09T04:31:52.747 INFO:teuthology.orchestra.run.smithi133.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-10-09T04:31:52.747 INFO:teuthology.orchestra.run.smithi133.stdout:Adding key to root@localhost authorized_keys... 2023-10-09T04:31:52.747 INFO:teuthology.orchestra.run.smithi133.stdout:Adding host smithi133... 2023-10-09T04:31:53.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:52 smithi133 bash[27558]: audit 2023-10-09T04:31:52.090216+0000 mgr.y (mgr.14120) 11 : audit [DBG] from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:31:54.960 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:54 smithi133 bash[27558]: audit 2023-10-09T04:31:54.105672+0000 mgr.y (mgr.14120) 12 : audit [DBG] from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi133", "addr": "172.21.15.133", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:31:56.377 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:56 smithi133 bash[27558]: cephadm 2023-10-09T04:31:54.871271+0000 mgr.y (mgr.14120) 13 : cephadm [INF] Deploying cephadm binary to smithi133 2023-10-09T04:31:57.809 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout Added host 'smithi133' with addr '172.21.15.133' 2023-10-09T04:31:57.809 INFO:teuthology.orchestra.run.smithi133.stdout:Deploying unmanaged mon service... 2023-10-09T04:31:58.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:57 smithi133 bash[27558]: audit 2023-10-09T04:31:56.770702+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.14120 172.21.15.133:0/577502051' entity='mgr.y' 2023-10-09T04:31:58.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:57 smithi133 bash[27558]: cephadm 2023-10-09T04:31:56.782210+0000 mgr.y (mgr.14120) 14 : cephadm [INF] Added host smithi133 2023-10-09T04:31:58.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:31:57 smithi133 bash[27558]: audit 2023-10-09T04:31:56.787697+0000 mon.a (mon.0) 57 : audit [DBG] from='mgr.14120 172.21.15.133:0/577502051' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:32:00.530 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2023-10-09T04:32:00.531 INFO:teuthology.orchestra.run.smithi133.stdout:Deploying unmanaged mgr service... 2023-10-09T04:32:01.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:00 smithi133 bash[27558]: audit 2023-10-09T04:31:59.746481+0000 mgr.y (mgr.14120) 15 : audit [DBG] from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:32:01.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:00 smithi133 bash[27558]: cephadm 2023-10-09T04:31:59.749647+0000 mgr.y (mgr.14120) 16 : cephadm [INF] Saving service mon spec with placement count:5 2023-10-09T04:32:01.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:00 smithi133 bash[27558]: audit 2023-10-09T04:31:59.754067+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.14120 172.21.15.133:0/577502051' entity='mgr.y' 2023-10-09T04:32:01.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:00 smithi133 bash[27558]: audit 2023-10-09T04:32:00.115818+0000 mon.a (mon.0) 59 : audit [INF] from='mgr.14120 172.21.15.133:0/577502051' entity='mgr.y' 2023-10-09T04:32:01.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:00 smithi133 bash[27558]: audit 2023-10-09T04:32:00.393394+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.14120 172.21.15.133:0/577502051' entity='mgr.y' 2023-10-09T04:32:02.729 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2023-10-09T04:32:03.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:03 smithi133 bash[27558]: audit 2023-10-09T04:32:02.035605+0000 mgr.y (mgr.14120) 17 : audit [DBG] from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:32:03.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:03 smithi133 bash[27558]: cephadm 2023-10-09T04:32:02.036646+0000 mgr.y (mgr.14120) 18 : cephadm [INF] Saving service mgr spec with placement count:2 2023-10-09T04:32:03.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:03 smithi133 bash[27558]: audit 2023-10-09T04:32:02.041092+0000 mon.a (mon.0) 61 : audit [INF] from='mgr.14120 172.21.15.133:0/577502051' entity='mgr.y' 2023-10-09T04:32:04.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:04 smithi133 bash[27558]: cluster 2023-10-09T04:32:02.653251+0000 mgr.y (mgr.14120) 19 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:32:05.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:05 smithi133 bash[27558]: audit 2023-10-09T04:32:04.160317+0000 mon.a (mon.0) 62 : audit [INF] from='client.? 172.21.15.133:0/4017762652' entity='client.admin' 2023-10-09T04:32:06.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:06 smithi133 bash[27558]: cluster 2023-10-09T04:32:04.653672+0000 mgr.y (mgr.14120) 20 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:32:06.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:06 smithi133 bash[27558]: audit 2023-10-09T04:32:05.728538+0000 mon.a (mon.0) 63 : audit [INF] from='mgr.14120 172.21.15.133:0/577502051' entity='mgr.y' 2023-10-09T04:32:07.107 INFO:teuthology.orchestra.run.smithi133.stdout:Enabling the dashboard module... 2023-10-09T04:32:07.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:07 smithi133 bash[27558]: audit 2023-10-09T04:32:06.405548+0000 mon.a (mon.0) 64 : audit [INF] from='client.? 172.21.15.133:0/3646461145' entity='client.admin' 2023-10-09T04:32:07.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:07 smithi133 bash[27558]: audit 2023-10-09T04:32:06.469273+0000 mon.a (mon.0) 65 : audit [INF] from='mgr.14120 172.21.15.133:0/577502051' entity='mgr.y' 2023-10-09T04:32:08.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:08 smithi133 bash[27558]: cluster 2023-10-09T04:32:06.654046+0000 mgr.y (mgr.14120) 21 : cluster [DBG] pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:32:09.733 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:09 smithi133 bash[27558]: audit 2023-10-09T04:32:08.651656+0000 mon.a (mon.0) 66 : audit [INF] from='client.? 172.21.15.133:0/214719733' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-10-09T04:32:09.733 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:32:09 smithi133 bash[27821]: ignoring --setuser ceph since I am not root 2023-10-09T04:32:09.733 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:32:09 smithi133 bash[27821]: ignoring --setgroup ceph since I am not root 2023-10-09T04:32:09.733 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:32:09 smithi133 bash[27821]: debug 2023-10-09T04:32:09.689+0000 7f3d45dc11c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:32:10.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:10 smithi133 bash[27558]: audit 2023-10-09T04:32:09.480907+0000 mon.a (mon.0) 67 : audit [INF] from='client.? 172.21.15.133:0/214719733' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-10-09T04:32:10.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:10 smithi133 bash[27558]: cluster 2023-10-09T04:32:09.481045+0000 mon.a (mon.0) 68 : cluster [DBG] mgrmap e9: y(active, since 26s) 2023-10-09T04:32:10.968 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:32:10 smithi133 bash[27821]: debug 2023-10-09T04:32:10.560+0000 7f3d45dc11c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:32:10.968 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:32:10 smithi133 bash[27821]: debug 2023-10-09T04:32:10.660+0000 7f3d45dc11c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:32:10.968 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:32:10 smithi133 bash[27821]: debug 2023-10-09T04:32:10.764+0000 7f3d45dc11c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:32:11.298 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:32:11 smithi133 bash[27821]: debug 2023-10-09T04:32:11.028+0000 7f3d45dc11c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:32:11.727 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:32:11 smithi133 bash[27821]: debug 2023-10-09T04:32:11.408+0000 7f3d45dc11c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:32:11.727 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:32:11 smithi133 bash[27821]: debug 2023-10-09T04:32:11.504+0000 7f3d45dc11c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:32:12.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:11 smithi133 bash[27558]: audit 2023-10-09T04:32:11.675487+0000 mon.a (mon.0) 69 : audit [DBG] from='client.? 172.21.15.133:0/4218043550' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-09T04:32:12.048 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:32:11 smithi133 bash[27821]: debug 2023-10-09T04:32:11.796+0000 7f3d45dc11c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:32:12.467 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout { 2023-10-09T04:32:12.467 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "epoch": 9, 2023-10-09T04:32:12.467 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "available": true, 2023-10-09T04:32:12.468 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-10-09T04:32:12.468 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-10-09T04:32:12.468 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout } 2023-10-09T04:32:12.468 INFO:teuthology.orchestra.run.smithi133.stdout:Waiting for the mgr to restart... 2023-10-09T04:32:12.468 INFO:teuthology.orchestra.run.smithi133.stdout:Waiting for mgr epoch 9... 2023-10-09T04:32:13.298 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:32:12 smithi133 bash[27821]: debug 2023-10-09T04:32:12.832+0000 7f3d45dc11c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:32:13.298 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:32:12 smithi133 bash[27821]: debug 2023-10-09T04:32:12.940+0000 7f3d45dc11c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:32:13.298 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:32:13 smithi133 bash[27821]: debug 2023-10-09T04:32:13.060+0000 7f3d45dc11c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:32:13.798 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:32:13 smithi133 bash[27821]: debug 2023-10-09T04:32:13.520+0000 7f3d45dc11c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:32:13.798 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:32:13 smithi133 bash[27821]: debug 2023-10-09T04:32:13.736+0000 7f3d45dc11c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:32:14.457 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:32:14 smithi133 bash[27821]: debug 2023-10-09T04:32:14.116+0000 7f3d45dc11c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:32:14.798 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:32:14 smithi133 bash[27821]: debug 2023-10-09T04:32:14.600+0000 7f3d45dc11c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:32:15.415 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:32:15 smithi133 bash[27821]: debug 2023-10-09T04:32:15.144+0000 7f3d45dc11c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:32:15.415 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:32:15 smithi133 bash[27821]: debug 2023-10-09T04:32:15.252+0000 7f3d45dc11c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:32:15.415 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:32:15 smithi133 bash[27821]: debug 2023-10-09T04:32:15.412+0000 7f3d45dc11c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:32:15.964 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:32:15 smithi133 bash[27821]: debug 2023-10-09T04:32:15.708+0000 7f3d45dc11c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:32:15.964 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:32:15 smithi133 bash[27821]: debug 2023-10-09T04:32:15.844+0000 7f3d45dc11c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:32:15.964 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:32:15 smithi133 bash[27821]: debug 2023-10-09T04:32:15.960+0000 7f3d45dc11c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:32:16.268 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:32:16 smithi133 bash[27821]: debug 2023-10-09T04:32:16.084+0000 7f3d45dc11c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:32:16.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:32:16 smithi133 bash[27821]: debug 2023-10-09T04:32:16.264+0000 7f3d45dc11c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:32:16.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:32:16 smithi133 bash[27821]: debug 2023-10-09T04:32:16.384+0000 7f3d45dc11c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:32:18.048 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:32:17 smithi133 bash[27821]: debug 2023-10-09T04:32:17.588+0000 7f3d45dc11c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:32:18.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:17 smithi133 bash[27558]: cluster 2023-10-09T04:32:17.611575+0000 mon.a (mon.0) 70 : cluster [INF] Active manager daemon y restarted 2023-10-09T04:32:18.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:17 smithi133 bash[27558]: cluster 2023-10-09T04:32:17.612042+0000 mon.a (mon.0) 71 : cluster [INF] Activating manager daemon y 2023-10-09T04:32:18.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:17 smithi133 bash[27558]: cluster 2023-10-09T04:32:17.617901+0000 mon.a (mon.0) 72 : cluster [DBG] osdmap e3: 0 total, 0 up, 0 in 2023-10-09T04:32:18.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:17 smithi133 bash[27558]: cluster 2023-10-09T04:32:17.618127+0000 mon.a (mon.0) 73 : cluster [DBG] mgrmap e10: y(active, starting, since 0.00627402s) 2023-10-09T04:32:18.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:17 smithi133 bash[27558]: audit 2023-10-09T04:32:17.619531+0000 mon.a (mon.0) 74 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:32:18.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:17 smithi133 bash[27558]: audit 2023-10-09T04:32:17.621121+0000 mon.a (mon.0) 75 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:32:18.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:17 smithi133 bash[27558]: audit 2023-10-09T04:32:17.622059+0000 mon.a (mon.0) 76 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:32:18.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:17 smithi133 bash[27558]: audit 2023-10-09T04:32:17.622278+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:32:18.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:17 smithi133 bash[27558]: audit 2023-10-09T04:32:17.622454+0000 mon.a (mon.0) 78 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:32:18.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:17 smithi133 bash[27558]: cluster 2023-10-09T04:32:17.649854+0000 mon.a (mon.0) 79 : cluster [INF] Manager daemon y is now available 2023-10-09T04:32:19.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:18 smithi133 bash[27558]: audit 2023-10-09T04:32:17.679387+0000 mon.a (mon.0) 80 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:32:19.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:18 smithi133 bash[27558]: audit 2023-10-09T04:32:17.681904+0000 mon.a (mon.0) 81 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:32:19.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:18 smithi133 bash[27558]: audit 2023-10-09T04:32:17.709953+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:32:19.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:18 smithi133 bash[27558]: audit 2023-10-09T04:32:17.714196+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:32:19.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:18 smithi133 bash[27558]: cephadm 2023-10-09T04:32:18.200310+0000 mgr.y (mgr.14152) 1 : cephadm [INF] [09/Oct/2023:04:32:18] ENGINE Bus STARTING 2023-10-09T04:32:19.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:18 smithi133 bash[27558]: cephadm 2023-10-09T04:32:18.322315+0000 mgr.y (mgr.14152) 2 : cephadm [INF] [09/Oct/2023:04:32:18] ENGINE Serving on https://172.21.15.133:7150 2023-10-09T04:32:19.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:18 smithi133 bash[27558]: cephadm 2023-10-09T04:32:18.322614+0000 mgr.y (mgr.14152) 3 : cephadm [INF] [09/Oct/2023:04:32:18] ENGINE Bus STARTED 2023-10-09T04:32:19.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:18 smithi133 bash[27558]: audit 2023-10-09T04:32:18.329689+0000 mon.a (mon.0) 84 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:32:19.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:18 smithi133 bash[27558]: cluster 2023-10-09T04:32:18.624350+0000 mon.a (mon.0) 85 : cluster [DBG] mgrmap e11: y(active, since 1.01249s) 2023-10-09T04:32:19.304 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout { 2023-10-09T04:32:19.304 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2023-10-09T04:32:19.304 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout "initialized": true 2023-10-09T04:32:19.304 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout } 2023-10-09T04:32:19.304 INFO:teuthology.orchestra.run.smithi133.stdout:mgr epoch 9 is available 2023-10-09T04:32:19.305 INFO:teuthology.orchestra.run.smithi133.stdout:Generating a dashboard self-signed certificate... 2023-10-09T04:32:20.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:19 smithi133 bash[27558]: audit 2023-10-09T04:32:18.623050+0000 mgr.y (mgr.14152) 4 : audit [DBG] from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-09T04:32:20.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:19 smithi133 bash[27558]: audit 2023-10-09T04:32:18.638058+0000 mgr.y (mgr.14152) 5 : audit [DBG] from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-09T04:32:21.007 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:20 smithi133 bash[27558]: cluster 2023-10-09T04:32:19.672260+0000 mon.a (mon.0) 86 : cluster [DBG] mgrmap e12: y(active, since 2s) 2023-10-09T04:32:21.449 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2023-10-09T04:32:21.449 INFO:teuthology.orchestra.run.smithi133.stdout:Creating initial admin user... 2023-10-09T04:32:22.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:21 smithi133 bash[27558]: audit 2023-10-09T04:32:20.693637+0000 mgr.y (mgr.14152) 6 : audit [DBG] from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:32:22.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:21 smithi133 bash[27558]: audit 2023-10-09T04:32:20.749076+0000 mon.a (mon.0) 87 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:32:22.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:21 smithi133 bash[27558]: audit 2023-10-09T04:32:20.754500+0000 mon.a (mon.0) 88 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:32:23.978 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$uygt3PpkfeMP9iNYIX2h.O315K3OqX3zn6Z0hPlTT7rX3FsnknIkW", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1696825943, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-10-09T04:32:23.978 INFO:teuthology.orchestra.run.smithi133.stdout:Fetching dashboard port number... 2023-10-09T04:32:24.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:24 smithi133 bash[27558]: audit 2023-10-09T04:32:22.950180+0000 mgr.y (mgr.14152) 7 : audit [DBG] from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:32:24.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:24 smithi133 bash[27558]: audit 2023-10-09T04:32:23.192263+0000 mon.a (mon.0) 89 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:32:25.463 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:25 smithi133 bash[27558]: cluster 2023-10-09T04:32:24.196391+0000 mon.a (mon.0) 90 : cluster [DBG] mgrmap e13: y(active, since 6s) 2023-10-09T04:32:25.463 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:25 smithi133 bash[27558]: audit 2023-10-09T04:32:24.795297+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:32:26.144 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stdout 8443 2023-10-09T04:32:26.145 INFO:teuthology.orchestra.run.smithi133.stdout:firewalld does not appear to be present 2023-10-09T04:32:26.145 INFO:teuthology.orchestra.run.smithi133.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2023-10-09T04:32:26.147 INFO:teuthology.orchestra.run.smithi133.stdout:Ceph Dashboard is now available at: 2023-10-09T04:32:26.148 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T04:32:26.148 INFO:teuthology.orchestra.run.smithi133.stdout: URL: https://smithi133.front.sepia.ceph.com:8443/ 2023-10-09T04:32:26.148 INFO:teuthology.orchestra.run.smithi133.stdout: User: admin 2023-10-09T04:32:26.148 INFO:teuthology.orchestra.run.smithi133.stdout: Password: jh5vly7x87 2023-10-09T04:32:26.148 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T04:32:26.148 INFO:teuthology.orchestra.run.smithi133.stdout:Saving cluster configuration to /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config directory 2023-10-09T04:32:26.149 INFO:teuthology.orchestra.run.smithi133.stdout:Enabling autotune for osd_memory_target 2023-10-09T04:32:26.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:26 smithi133 bash[27558]: audit 2023-10-09T04:32:25.428670+0000 mon.a (mon.0) 92 : audit [DBG] from='client.? 172.21.15.133:0/718340437' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-10-09T04:32:26.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:26 smithi133 bash[27558]: audit 2023-10-09T04:32:25.443022+0000 mon.a (mon.0) 93 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:32:30.537 INFO:teuthology.orchestra.run.smithi133.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2023-10-09T04:32:30.537 INFO:teuthology.orchestra.run.smithi133.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-10-09T04:32:30.537 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T04:32:30.537 INFO:teuthology.orchestra.run.smithi133.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-10-09T04:32:30.537 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T04:32:30.538 INFO:teuthology.orchestra.run.smithi133.stdout:Or, if you are only running a single cluster on this host: 2023-10-09T04:32:30.538 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T04:32:30.538 INFO:teuthology.orchestra.run.smithi133.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2023-10-09T04:32:30.538 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T04:32:30.538 INFO:teuthology.orchestra.run.smithi133.stdout:Please consider enabling telemetry to help improve Ceph: 2023-10-09T04:32:30.538 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T04:32:30.538 INFO:teuthology.orchestra.run.smithi133.stdout: ceph telemetry on 2023-10-09T04:32:30.538 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T04:32:30.538 INFO:teuthology.orchestra.run.smithi133.stdout:For more information see: 2023-10-09T04:32:30.538 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T04:32:30.539 INFO:teuthology.orchestra.run.smithi133.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2023-10-09T04:32:30.539 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T04:32:30.539 INFO:teuthology.orchestra.run.smithi133.stdout:Bootstrap complete. 2023-10-09T04:32:30.573 INFO:tasks.cephadm:Fetching config... 2023-10-09T04:32:30.573 DEBUG:teuthology.orchestra.run.smithi133:> set -ex 2023-10-09T04:32:30.574 DEBUG:teuthology.orchestra.run.smithi133:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-10-09T04:32:30.580 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-10-09T04:32:30.580 DEBUG:teuthology.orchestra.run.smithi133:> set -ex 2023-10-09T04:32:30.581 DEBUG:teuthology.orchestra.run.smithi133:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-10-09T04:32:30.627 INFO:tasks.cephadm:Fetching mon keyring... 2023-10-09T04:32:30.627 DEBUG:teuthology.orchestra.run.smithi133:> set -ex 2023-10-09T04:32:30.627 DEBUG:teuthology.orchestra.run.smithi133:> sudo dd if=/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/keyring of=/dev/stdout 2023-10-09T04:32:30.678 INFO:tasks.cephadm:Fetching pub ssh key... 2023-10-09T04:32:30.679 DEBUG:teuthology.orchestra.run.smithi133:> set -ex 2023-10-09T04:32:30.679 DEBUG:teuthology.orchestra.run.smithi133:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-10-09T04:32:30.726 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-10-09T04:32:30.727 DEBUG:teuthology.orchestra.run.smithi133:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDIQrEyiuSaKmYcvT73+p42a3K6RIkGJM4yGIDn8J0HE7T9lH6JvTdLeHgzDyhaeC9TdhWLTwXcP70dZbkmXjorABwiA/fZYpkIerBKW0Wgs2LGl3u6Br/BO3ASYMP0UJA3VaXn9IZL62IOBlUdqtz+rfwfHDa+hc8dnTsgBlrozggxD3pg/utGpf59n1QnVIoet2xN3dPqc0Dwgh0IZM3tedXi25NrEqWtxYx7JHxAa3B9LM5xXdyF/U6V/B4hk89Vje3Y6MRC8Znf3Gw2CxBH2qjThkN6b6n0WX/9AeEj4sRX9DKKrcY84CAUagBmvLSfYy6vbzsW/iYyUOFLKA36aiYoZ3wNo3hvmbB0srlTM22V1EjUzT54ktGtZ59T2nuTNwkCRu8GoJKexA/SEs4bP8QBHd0eOx0fDy2BMOFBF5yz6/kSK6E650dUR8iRyV3B3ZXiZsNVsSfk8SJ1RWVWbF5ADkkuHKljAg/Zx6pk7VvzCLjfUxrvXugByzOpzv8= ceph-855b5206-665c-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-09T04:32:30.790 INFO:teuthology.orchestra.run.smithi133.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDIQrEyiuSaKmYcvT73+p42a3K6RIkGJM4yGIDn8J0HE7T9lH6JvTdLeHgzDyhaeC9TdhWLTwXcP70dZbkmXjorABwiA/fZYpkIerBKW0Wgs2LGl3u6Br/BO3ASYMP0UJA3VaXn9IZL62IOBlUdqtz+rfwfHDa+hc8dnTsgBlrozggxD3pg/utGpf59n1QnVIoet2xN3dPqc0Dwgh0IZM3tedXi25NrEqWtxYx7JHxAa3B9LM5xXdyF/U6V/B4hk89Vje3Y6MRC8Znf3Gw2CxBH2qjThkN6b6n0WX/9AeEj4sRX9DKKrcY84CAUagBmvLSfYy6vbzsW/iYyUOFLKA36aiYoZ3wNo3hvmbB0srlTM22V1EjUzT54ktGtZ59T2nuTNwkCRu8GoJKexA/SEs4bP8QBHd0eOx0fDy2BMOFBF5yz6/kSK6E650dUR8iRyV3B3ZXiZsNVsSfk8SJ1RWVWbF5ADkkuHKljAg/Zx6pk7VvzCLjfUxrvXugByzOpzv8= ceph-855b5206-665c-11ee-8db5-212e2dc638e7 2023-10-09T04:32:30.801 DEBUG:teuthology.orchestra.run.smithi194:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDIQrEyiuSaKmYcvT73+p42a3K6RIkGJM4yGIDn8J0HE7T9lH6JvTdLeHgzDyhaeC9TdhWLTwXcP70dZbkmXjorABwiA/fZYpkIerBKW0Wgs2LGl3u6Br/BO3ASYMP0UJA3VaXn9IZL62IOBlUdqtz+rfwfHDa+hc8dnTsgBlrozggxD3pg/utGpf59n1QnVIoet2xN3dPqc0Dwgh0IZM3tedXi25NrEqWtxYx7JHxAa3B9LM5xXdyF/U6V/B4hk89Vje3Y6MRC8Znf3Gw2CxBH2qjThkN6b6n0WX/9AeEj4sRX9DKKrcY84CAUagBmvLSfYy6vbzsW/iYyUOFLKA36aiYoZ3wNo3hvmbB0srlTM22V1EjUzT54ktGtZ59T2nuTNwkCRu8GoJKexA/SEs4bP8QBHd0eOx0fDy2BMOFBF5yz6/kSK6E650dUR8iRyV3B3ZXiZsNVsSfk8SJ1RWVWbF5ADkkuHKljAg/Zx6pk7VvzCLjfUxrvXugByzOpzv8= ceph-855b5206-665c-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-09T04:32:30.824 INFO:teuthology.orchestra.run.smithi194.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDIQrEyiuSaKmYcvT73+p42a3K6RIkGJM4yGIDn8J0HE7T9lH6JvTdLeHgzDyhaeC9TdhWLTwXcP70dZbkmXjorABwiA/fZYpkIerBKW0Wgs2LGl3u6Br/BO3ASYMP0UJA3VaXn9IZL62IOBlUdqtz+rfwfHDa+hc8dnTsgBlrozggxD3pg/utGpf59n1QnVIoet2xN3dPqc0Dwgh0IZM3tedXi25NrEqWtxYx7JHxAa3B9LM5xXdyF/U6V/B4hk89Vje3Y6MRC8Znf3Gw2CxBH2qjThkN6b6n0WX/9AeEj4sRX9DKKrcY84CAUagBmvLSfYy6vbzsW/iYyUOFLKA36aiYoZ3wNo3hvmbB0srlTM22V1EjUzT54ktGtZ59T2nuTNwkCRu8GoJKexA/SEs4bP8QBHd0eOx0fDy2BMOFBF5yz6/kSK6E650dUR8iRyV3B3ZXiZsNVsSfk8SJ1RWVWbF5ADkkuHKljAg/Zx6pk7VvzCLjfUxrvXugByzOpzv8= ceph-855b5206-665c-11ee-8db5-212e2dc638e7 2023-10-09T04:32:30.835 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-10-09T04:32:31.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:30 smithi133 bash[27558]: audit 2023-10-09T04:32:29.829171+0000 mon.a (mon.0) 94 : audit [INF] from='client.? 172.21.15.133:0/755260518' entity='client.admin' 2023-10-09T04:32:37.064 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:39.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:38 smithi133 bash[27558]: cluster 2023-10-09T04:32:37.623754+0000 mgr.y (mgr.14152) 8 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:32:39.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:38 smithi133 bash[27558]: audit 2023-10-09T04:32:37.743044+0000 mon.a (mon.0) 95 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:32:39.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:38 smithi133 bash[27558]: audit 2023-10-09T04:32:37.749716+0000 mon.a (mon.0) 96 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:32:39.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:38 smithi133 bash[27558]: audit 2023-10-09T04:32:37.750825+0000 mon.a (mon.0) 97 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi133", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:32:39.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:38 smithi133 bash[27558]: audit 2023-10-09T04:32:37.757914+0000 mon.a (mon.0) 98 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:32:39.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:38 smithi133 bash[27558]: audit 2023-10-09T04:32:37.791425+0000 mon.a (mon.0) 99 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:32:39.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:38 smithi133 bash[27558]: audit 2023-10-09T04:32:37.801690+0000 mon.a (mon.0) 100 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:32:41.071 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-10-09T04:32:41.071 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-10-09T04:32:41.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:41 smithi133 bash[27558]: cluster 2023-10-09T04:32:39.624200+0000 mgr.y (mgr.14152) 9 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:32:41.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:41 smithi133 bash[27558]: audit 2023-10-09T04:32:40.332157+0000 mon.a (mon.0) 101 : audit [INF] from='client.? 172.21.15.133:0/1464293625' entity='client.admin' 2023-10-09T04:32:41.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:41 smithi133 bash[27558]: audit 2023-10-09T04:32:40.356531+0000 mon.a (mon.0) 102 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:32:41.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:41 smithi133 bash[27558]: audit 2023-10-09T04:32:40.365402+0000 mon.a (mon.0) 103 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:32:43.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:43 smithi133 bash[27558]: cluster 2023-10-09T04:32:41.624610+0000 mgr.y (mgr.14152) 10 : cluster [DBG] pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:32:45.793 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:45.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:45 smithi133 bash[27558]: cluster 2023-10-09T04:32:43.625000+0000 mgr.y (mgr.14152) 11 : cluster [DBG] pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:32:47.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:47 smithi133 bash[27558]: cluster 2023-10-09T04:32:45.625424+0000 mgr.y (mgr.14152) 12 : cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:32:49.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:49 smithi133 bash[27558]: cluster 2023-10-09T04:32:47.625790+0000 mgr.y (mgr.14152) 13 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:32:49.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:49 smithi133 bash[27558]: audit 2023-10-09T04:32:48.880174+0000 mon.a (mon.0) 104 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:32:49.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:49 smithi133 bash[27558]: audit 2023-10-09T04:32:48.886228+0000 mon.a (mon.0) 105 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:32:49.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:49 smithi133 bash[27558]: audit 2023-10-09T04:32:48.888676+0000 mon.a (mon.0) 106 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:32:49.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:49 smithi133 bash[27558]: audit 2023-10-09T04:32:48.889785+0000 mon.a (mon.0) 107 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:32:49.621 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi194 2023-10-09T04:32:49.622 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2023-10-09T04:32:49.622 DEBUG:teuthology.orchestra.run.smithi194:> dd of=/etc/ceph/ceph.conf 2023-10-09T04:32:49.630 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2023-10-09T04:32:49.630 DEBUG:teuthology.orchestra.run.smithi194:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:32:49.680 INFO:tasks.cephadm:Adding host smithi194 to orchestrator... 2023-10-09T04:32:49.680 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph orch host add smithi194 2023-10-09T04:32:50.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:50 smithi133 bash[27558]: audit 2023-10-09T04:32:48.874779+0000 mgr.y (mgr.14152) 14 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:32:50.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:50 smithi133 bash[27558]: cephadm 2023-10-09T04:32:48.891243+0000 mgr.y (mgr.14152) 15 : cephadm [INF] Updating smithi133:/etc/ceph/ceph.conf 2023-10-09T04:32:50.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:50 smithi133 bash[27558]: cephadm 2023-10-09T04:32:49.055646+0000 mgr.y (mgr.14152) 16 : cephadm [INF] Updating smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:32:50.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:50 smithi133 bash[27558]: cephadm 2023-10-09T04:32:49.220816+0000 mgr.y (mgr.14152) 17 : cephadm [INF] Updating smithi133:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:32:50.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:50 smithi133 bash[27558]: cephadm 2023-10-09T04:32:49.385145+0000 mgr.y (mgr.14152) 18 : cephadm [INF] Updating smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:32:50.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:50 smithi133 bash[27558]: audit 2023-10-09T04:32:49.556486+0000 mon.a (mon.0) 108 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:32:50.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:50 smithi133 bash[27558]: audit 2023-10-09T04:32:49.562039+0000 mon.a (mon.0) 109 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:32:50.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:50 smithi133 bash[27558]: audit 2023-10-09T04:32:49.566125+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:32:51.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:51 smithi133 bash[27558]: cluster 2023-10-09T04:32:49.626165+0000 mgr.y (mgr.14152) 19 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:32:53.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:53 smithi133 bash[27558]: cluster 2023-10-09T04:32:51.626529+0000 mgr.y (mgr.14152) 20 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:32:54.406 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:55.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:55 smithi133 bash[27558]: cluster 2023-10-09T04:32:53.626924+0000 mgr.y (mgr.14152) 21 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:32:57.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:57 smithi133 bash[27558]: cluster 2023-10-09T04:32:55.627389+0000 mgr.y (mgr.14152) 22 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:32:58.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:58 smithi133 bash[27558]: audit 2023-10-09T04:32:57.343250+0000 mgr.y (mgr.14152) 23 : audit [DBG] from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi194", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:32:59.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:59 smithi133 bash[27558]: cluster 2023-10-09T04:32:57.627830+0000 mgr.y (mgr.14152) 24 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:32:59.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:32:59 smithi133 bash[27558]: cephadm 2023-10-09T04:32:58.222616+0000 mgr.y (mgr.14152) 25 : cephadm [INF] Deploying cephadm binary to smithi194 2023-10-09T04:33:00.392 INFO:teuthology.orchestra.run.smithi133.stdout:Added host 'smithi194' with addr '172.21.15.194' 2023-10-09T04:33:01.174 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph orch host ls --format=json 2023-10-09T04:33:01.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:01 smithi133 bash[27558]: cluster 2023-10-09T04:32:59.628236+0000 mgr.y (mgr.14152) 26 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:01.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:01 smithi133 bash[27558]: audit 2023-10-09T04:33:00.390514+0000 mon.a (mon.0) 111 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:01.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:01 smithi133 bash[27558]: cephadm 2023-10-09T04:33:00.391322+0000 mgr.y (mgr.14152) 27 : cephadm [INF] Added host smithi194 2023-10-09T04:33:01.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:01 smithi133 bash[27558]: audit 2023-10-09T04:33:00.419703+0000 mon.a (mon.0) 112 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:33:01.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:01 smithi133 bash[27558]: audit 2023-10-09T04:33:00.743712+0000 mon.a (mon.0) 113 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:03.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:03 smithi133 bash[27558]: cluster 2023-10-09T04:33:01.628648+0000 mgr.y (mgr.14152) 28 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:03.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:03 smithi133 bash[27558]: audit 2023-10-09T04:33:02.067789+0000 mon.a (mon.0) 114 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:03.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:03 smithi133 bash[27558]: audit 2023-10-09T04:33:02.805696+0000 mon.a (mon.0) 115 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:05.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:05 smithi133 bash[27558]: cluster 2023-10-09T04:33:03.629070+0000 mgr.y (mgr.14152) 29 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:05.899 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:07.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:07 smithi133 bash[27558]: cluster 2023-10-09T04:33:05.629517+0000 mgr.y (mgr.14152) 30 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:08.981 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T04:33:08.982 INFO:teuthology.orchestra.run.smithi133.stdout:[{"addr": "172.21.15.133", "hostname": "smithi133", "labels": [], "status": ""}, {"addr": "172.21.15.194", "hostname": "smithi194", "labels": [], "status": ""}] 2023-10-09T04:33:09.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:09 smithi133 bash[27558]: cluster 2023-10-09T04:33:07.629938+0000 mgr.y (mgr.14152) 31 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:09.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:09 smithi133 bash[27558]: audit 2023-10-09T04:33:08.941908+0000 mon.a (mon.0) 116 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:09.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:09 smithi133 bash[27558]: audit 2023-10-09T04:33:08.947538+0000 mon.a (mon.0) 117 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:09.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:09 smithi133 bash[27558]: audit 2023-10-09T04:33:08.953797+0000 mon.a (mon.0) 118 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:09.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:09 smithi133 bash[27558]: audit 2023-10-09T04:33:08.959946+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:09.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:09 smithi133 bash[27558]: audit 2023-10-09T04:33:08.961005+0000 mon.a (mon.0) 120 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi194", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:33:09.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:09 smithi133 bash[27558]: audit 2023-10-09T04:33:08.962498+0000 mon.a (mon.0) 121 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:09.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:09 smithi133 bash[27558]: audit 2023-10-09T04:33:08.963427+0000 mon.a (mon.0) 122 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:33:09.801 INFO:tasks.cephadm:Setting crush tunables to default 2023-10-09T04:33:09.802 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd crush tunables default 2023-10-09T04:33:10.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:10 smithi133 bash[27558]: cephadm 2023-10-09T04:33:08.964871+0000 mgr.y (mgr.14152) 32 : cephadm [INF] Updating smithi194:/etc/ceph/ceph.conf 2023-10-09T04:33:10.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:10 smithi133 bash[27558]: audit 2023-10-09T04:33:08.980637+0000 mgr.y (mgr.14152) 33 : audit [DBG] from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:33:10.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:10 smithi133 bash[27558]: cephadm 2023-10-09T04:33:09.129254+0000 mgr.y (mgr.14152) 34 : cephadm [INF] Updating smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:33:10.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:10 smithi133 bash[27558]: cephadm 2023-10-09T04:33:09.294022+0000 mgr.y (mgr.14152) 35 : cephadm [INF] Updating smithi194:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:33:10.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:10 smithi133 bash[27558]: cephadm 2023-10-09T04:33:09.458454+0000 mgr.y (mgr.14152) 36 : cephadm [INF] Updating smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:33:10.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:10 smithi133 bash[27558]: audit 2023-10-09T04:33:09.629052+0000 mon.a (mon.0) 123 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:10.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:10 smithi133 bash[27558]: audit 2023-10-09T04:33:09.633723+0000 mon.a (mon.0) 124 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:10.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:10 smithi133 bash[27558]: audit 2023-10-09T04:33:09.637041+0000 mon.a (mon.0) 125 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:11.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:11 smithi133 bash[27558]: cluster 2023-10-09T04:33:09.630356+0000 mgr.y (mgr.14152) 37 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:13.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:13 smithi133 bash[27558]: cluster 2023-10-09T04:33:11.630792+0000 mgr.y (mgr.14152) 38 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:14.528 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:15.547 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:15 smithi133 bash[27558]: cluster 2023-10-09T04:33:13.631254+0000 mgr.y (mgr.14152) 39 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:17.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:17 smithi133 bash[27558]: cluster 2023-10-09T04:33:15.631691+0000 mgr.y (mgr.14152) 40 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:18.277 INFO:teuthology.orchestra.run.smithi133.stderr:adjusted tunables profile to default 2023-10-09T04:33:18.546 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:18 smithi133 bash[27558]: audit 2023-10-09T04:33:17.479329+0000 mon.a (mon.0) 126 : audit [INF] from='client.? 172.21.15.133:0/2780383392' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-10-09T04:33:19.025 INFO:tasks.cephadm:Adding mon.a on smithi133 2023-10-09T04:33:19.026 INFO:tasks.cephadm:Adding mon.c on smithi133 2023-10-09T04:33:19.026 INFO:tasks.cephadm:Adding mon.b on smithi194 2023-10-09T04:33:19.026 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph orch apply mon '3;smithi133:172.21.15.133=a;smithi133:[v2:172.21.15.133:3301,v1:172.21.15.133:6790]=c;smithi194:172.21.15.194=b' 2023-10-09T04:33:19.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:19 smithi133 bash[27558]: cluster 2023-10-09T04:33:17.632067+0000 mgr.y (mgr.14152) 41 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:19.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:19 smithi133 bash[27558]: audit 2023-10-09T04:33:18.271187+0000 mon.a (mon.0) 127 : audit [INF] from='client.? 172.21.15.133:0/2780383392' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-10-09T04:33:19.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:19 smithi133 bash[27558]: cluster 2023-10-09T04:33:18.271246+0000 mon.a (mon.0) 128 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T04:33:20.259 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:33:21.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:21 smithi133 bash[27558]: cluster 2023-10-09T04:33:19.632435+0000 mgr.y (mgr.14152) 42 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:23.253 INFO:teuthology.orchestra.run.smithi194.stdout:Scheduled mon update... 2023-10-09T04:33:23.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:23 smithi133 bash[27558]: cluster 2023-10-09T04:33:21.632826+0000 mgr.y (mgr.14152) 43 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:23.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:23 smithi133 bash[27558]: audit 2023-10-09T04:33:23.251864+0000 mon.a (mon.0) 129 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:24.041 DEBUG:teuthology.orchestra.run.smithi133:mon.c> sudo journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mon.c.service 2023-10-09T04:33:24.043 DEBUG:teuthology.orchestra.run.smithi194:mon.b> sudo journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mon.b.service 2023-10-09T04:33:24.047 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-09T04:33:24.047 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-09T04:33:24.295 INFO:journalctl@ceph.mon.c.smithi133.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:33:24.477 INFO:journalctl@ceph.mon.b.smithi194.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:33:24.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:24 smithi133 bash[27558]: audit 2023-10-09T04:33:23.240923+0000 mgr.y (mgr.14152) 44 : audit [DBG] from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi133:172.21.15.133=a;smithi133:[v2:172.21.15.133:3301,v1:172.21.15.133:6790]=c;smithi194:172.21.15.194=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:33:24.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:24 smithi133 bash[27558]: cephadm 2023-10-09T04:33:23.246853+0000 mgr.y (mgr.14152) 45 : cephadm [INF] Saving service mon spec with placement smithi133:172.21.15.133=a;smithi133:[v2:172.21.15.133:3301,v1:172.21.15.133:6790]=c;smithi194:172.21.15.194=b;count:3 2023-10-09T04:33:24.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:24 smithi133 bash[27558]: audit 2023-10-09T04:33:23.301984+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:33:24.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:24 smithi133 bash[27558]: audit 2023-10-09T04:33:23.305842+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:24.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:24 smithi133 bash[27558]: audit 2023-10-09T04:33:23.307541+0000 mon.a (mon.0) 132 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:33:24.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:24 smithi133 bash[27558]: audit 2023-10-09T04:33:23.314614+0000 mon.a (mon.0) 133 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:24.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:24 smithi133 bash[27558]: audit 2023-10-09T04:33:23.317219+0000 mon.a (mon.0) 134 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:33:24.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:24 smithi133 bash[27558]: audit 2023-10-09T04:33:23.318398+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:24.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:24 smithi133 bash[27558]: cephadm 2023-10-09T04:33:23.319872+0000 mgr.y (mgr.14152) 46 : cephadm [INF] Deploying daemon mon.b on smithi194 2023-10-09T04:33:25.737 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:33:25.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:25 smithi133 bash[27558]: cluster 2023-10-09T04:33:23.633226+0000 mgr.y (mgr.14152) 47 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:27.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:27 smithi133 bash[27558]: cluster 2023-10-09T04:33:25.633657+0000 mgr.y (mgr.14152) 48 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:28.377 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:28 smithi194 systemd[1]: Started Ceph mon.b for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:33:29.696 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:29 smithi133 bash[27558]: cluster 2023-10-09T04:33:27.634076+0000 mgr.y (mgr.14152) 49 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:29.696 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:29 smithi133 bash[27558]: audit 2023-10-09T04:33:28.342362+0000 mon.a (mon.0) 136 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:29.696 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:29 smithi133 bash[27558]: audit 2023-10-09T04:33:28.347349+0000 mon.a (mon.0) 137 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:29.696 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:29 smithi133 bash[27558]: audit 2023-10-09T04:33:28.351524+0000 mon.a (mon.0) 138 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:29.696 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:29 smithi133 bash[27558]: audit 2023-10-09T04:33:28.352356+0000 mon.a (mon.0) 139 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:33:29.696 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:29 smithi133 bash[27558]: audit 2023-10-09T04:33:28.353886+0000 mon.a (mon.0) 140 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:29.696 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:29 smithi133 bash[27558]: cephadm 2023-10-09T04:33:28.355571+0000 mgr.y (mgr.14152) 50 : cephadm [INF] Deploying daemon mon.c on smithi133 2023-10-09T04:33:29.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.580+0000 7f7a56cbab80 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T04:33:29.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.580+0000 7f7a56cbab80 0 ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable), process ceph-mon, pid 7 2023-10-09T04:33:29.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.580+0000 7f7a56cbab80 0 pidfile_write: ignore empty --pid-file 2023-10-09T04:33:29.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 0 load: jerasure load: lrc 2023-10-09T04:33:29.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: RocksDB version: 6.15.5 2023-10-09T04:33:29.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T04:33:29.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Compile date Oct 8 2023 2023-10-09T04:33:29.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: DB SUMMARY 2023-10-09T04:33:29.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: DB Session ID: O7BV1OG982FNAVYU8NN4 2023-10-09T04:33:29.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: CURRENT file: CURRENT 2023-10-09T04:33:29.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: IDENTITY file: IDENTITY 2023-10-09T04:33:29.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2023-10-09T04:33:29.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2023-10-09T04:33:29.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2023-10-09T04:33:29.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.error_if_exists: 0 2023-10-09T04:33:29.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.create_if_missing: 0 2023-10-09T04:33:29.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T04:33:29.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T04:33:29.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.env: 0x560a69df9a80 2023-10-09T04:33:29.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.fs: Posix File System 2023-10-09T04:33:29.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.info_log: 0x560a6a8a5a60 2023-10-09T04:33:29.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T04:33:29.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.statistics: (nil) 2023-10-09T04:33:29.889 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.use_fsync: 0 2023-10-09T04:33:29.889 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T04:33:29.889 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T04:33:29.889 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T04:33:29.890 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T04:33:29.890 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T04:33:29.890 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T04:33:29.890 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T04:33:29.890 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T04:33:29.890 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T04:33:29.890 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T04:33:29.890 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T04:33:29.891 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.db_log_dir: 2023-10-09T04:33:29.891 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2023-10-09T04:33:29.891 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T04:33:29.891 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T04:33:29.891 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T04:33:29.891 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T04:33:29.891 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T04:33:29.891 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T04:33:29.891 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T04:33:29.891 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T04:33:29.891 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.write_buffer_manager: 0x560a6b59a930 2023-10-09T04:33:29.891 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T04:33:29.892 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T04:33:29.892 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T04:33:29.892 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T04:33:29.892 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T04:33:29.892 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T04:33:29.892 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T04:33:29.892 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T04:33:29.892 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T04:33:29.892 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.unordered_write: 0 2023-10-09T04:33:29.892 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T04:33:29.892 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T04:33:29.892 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T04:33:29.892 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T04:33:29.893 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.row_cache: None 2023-10-09T04:33:29.893 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.wal_filter: None 2023-10-09T04:33:29.893 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T04:33:29.893 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T04:33:29.893 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.preserve_deletes: 0 2023-10-09T04:33:29.893 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.two_write_queues: 0 2023-10-09T04:33:29.893 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T04:33:29.893 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.atomic_flush: 0 2023-10-09T04:33:29.893 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T04:33:29.893 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T04:33:29.893 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T04:33:29.893 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T04:33:29.894 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T04:33:29.894 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.best_efforts_recovery: 0 2023-10-09T04:33:29.894 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T04:33:29.894 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T04:33:29.894 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.allow_data_in_errors: 0 2023-10-09T04:33:29.894 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.db_host_id: __hostname__ 2023-10-09T04:33:29.894 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T04:33:29.894 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T04:33:29.894 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T04:33:29.894 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T04:33:29.894 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T04:33:29.894 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T04:33:29.895 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T04:33:29.895 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T04:33:29.895 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T04:33:29.895 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T04:33:29.895 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T04:33:29.895 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.max_open_files: -1 2023-10-09T04:33:29.895 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T04:33:29.895 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T04:33:29.895 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T04:33:29.895 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T04:33:29.895 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T04:33:29.895 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Compression algorithms supported: 2023-10-09T04:33:29.896 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T04:33:29.896 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: kZSTD supported: 0 2023-10-09T04:33:29.896 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: kXpressCompression supported: 0 2023-10-09T04:33:29.896 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T04:33:29.896 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: kZlibCompression supported: 1 2023-10-09T04:33:29.896 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T04:33:29.896 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T04:33:29.896 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T04:33:29.896 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T04:33:29.896 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2023-10-09T04:33:29.896 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.584+0000 7f7a56cbab80 4 rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-10-09T04:33:29.896 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T04:33:29.896 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.merge_operator: 2023-10-09T04:33:29.897 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.compaction_filter: None 2023-10-09T04:33:29.897 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T04:33:29.897 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.sst_partitioner_factory: None 2023-10-09T04:33:29.897 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T04:33:29.897 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T04:33:29.897 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x560a6a878fb8) 2023-10-09T04:33:29.897 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cache_index_and_filter_blocks: 1 2023-10-09T04:33:29.897 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T04:33:29.897 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T04:33:29.897 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: pin_top_level_index_and_filter: 1 2023-10-09T04:33:29.897 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: index_type: 0 2023-10-09T04:33:29.897 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: data_block_index_type: 0 2023-10-09T04:33:29.898 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: index_shortening: 1 2023-10-09T04:33:29.898 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: data_block_hash_table_util_ratio: 0.750000 2023-10-09T04:33:29.898 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: hash_index_allow_collision: 1 2023-10-09T04:33:29.898 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: checksum: 1 2023-10-09T04:33:29.898 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: no_block_cache: 0 2023-10-09T04:33:29.898 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: block_cache: 0x560a6a8ab090 2023-10-09T04:33:29.898 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: block_cache_name: BinnedLRUCache 2023-10-09T04:33:29.898 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: block_cache_options: 2023-10-09T04:33:29.898 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: capacity : 536870912 2023-10-09T04:33:29.898 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: num_shard_bits : 4 2023-10-09T04:33:29.898 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: strict_capacity_limit : 0 2023-10-09T04:33:29.898 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: high_pri_pool_ratio: 0.000 2023-10-09T04:33:29.899 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: block_cache_compressed: (nil) 2023-10-09T04:33:29.899 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: persistent_cache: (nil) 2023-10-09T04:33:29.899 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: block_size: 4096 2023-10-09T04:33:29.899 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: block_size_deviation: 10 2023-10-09T04:33:29.899 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: block_restart_interval: 16 2023-10-09T04:33:29.899 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: index_block_restart_interval: 1 2023-10-09T04:33:29.899 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: metadata_block_size: 4096 2023-10-09T04:33:29.899 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: partition_filters: 0 2023-10-09T04:33:29.899 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: use_delta_encoding: 1 2023-10-09T04:33:29.899 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T04:33:29.899 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: whole_key_filtering: 1 2023-10-09T04:33:29.899 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: verify_compression: 0 2023-10-09T04:33:29.899 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: read_amp_bytes_per_bit: 0 2023-10-09T04:33:29.900 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: format_version: 4 2023-10-09T04:33:29.900 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: enable_index_compression: 1 2023-10-09T04:33:29.900 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: block_align: 0 2023-10-09T04:33:29.900 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T04:33:29.900 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T04:33:29.900 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.compression: NoCompression 2023-10-09T04:33:29.900 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T04:33:29.900 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T04:33:29.900 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T04:33:29.900 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.num_levels: 7 2023-10-09T04:33:29.900 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T04:33:29.900 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T04:33:29.901 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T04:33:29.901 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T04:33:29.901 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T04:33:29.901 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T04:33:29.901 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T04:33:29.901 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:33:29.901 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T04:33:29.901 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T04:33:29.901 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T04:33:29.901 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T04:33:29.902 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T04:33:29.902 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T04:33:29.902 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:33:29.902 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T04:33:29.902 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T04:33:29.902 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T04:33:29.902 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T04:33:29.902 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T04:33:29.902 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T04:33:29.902 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T04:33:29.902 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T04:33:29.902 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T04:33:29.903 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T04:33:29.903 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T04:33:29.903 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T04:33:29.903 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T04:33:29.903 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T04:33:29.903 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T04:33:29.903 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T04:33:29.903 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T04:33:29.903 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T04:33:29.903 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T04:33:29.903 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.arena_block_size: 4194304 2023-10-09T04:33:29.903 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T04:33:29.904 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T04:33:29.904 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T04:33:29.904 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T04:33:29.904 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T04:33:29.904 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T04:33:29.904 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T04:33:29.904 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T04:33:29.904 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T04:33:29.904 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T04:33:29.904 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T04:33:29.904 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T04:33:29.904 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T04:33:29.905 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T04:33:29.905 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.table_properties_collectors: 2023-10-09T04:33:29.905 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T04:33:29.905 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T04:33:29.906 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T04:33:29.906 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T04:33:29.906 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T04:33:29.906 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.bloom_locality: 0 2023-10-09T04:33:29.906 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T04:33:29.906 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T04:33:29.907 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T04:33:29.907 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.force_consistency_checks: 1 2023-10-09T04:33:29.907 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T04:33:29.907 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.ttl: 2592000 2023-10-09T04:33:29.907 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T04:33:29.907 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.enable_blob_files: false 2023-10-09T04:33:29.909 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.min_blob_size: 0 2023-10-09T04:33:29.909 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.blob_file_size: 268435456 2023-10-09T04:33:29.910 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T04:33:29.910 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T04:33:29.910 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T04:33:29.910 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T04:33:29.910 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-10-09T04:33:29.910 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: [db/version_set.cc:4082] Creating manifest 7 2023-10-09T04:33:29.910 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696826009591755, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-10-09T04:33:29.910 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2023-10-09T04:33:29.910 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 3 rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-09T04:33:29.910 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696826009592113, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696826009, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "9f5b740a-006c-4b3c-8322-326a0f3d621f", "db_session_id": "O7BV1OG982FNAVYU8NN4"}} 2023-10-09T04:33:29.910 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: [db/version_set.cc:4082] Creating manifest 9 2023-10-09T04:33:29.911 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696826009592476, "job": 1, "event": "recovery_finished"} 2023-10-09T04:33:29.911 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T04:33:29.911 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x560a6a83f180 2023-10-09T04:33:29.911 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 4 rocksdb: DB pointer 0x560a6a914000 2023-10-09T04:33:29.911 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a46f09700 4 rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-10-09T04:33:29.911 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a46f09700 4 rocksdb: [db/db_impl/db_impl.cc:903] 2023-10-09T04:33:29.911 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: ** DB Stats ** 2023-10-09T04:33:29.911 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:33:29.911 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T04:33:29.912 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T04:33:29.912 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:33:29.912 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T04:33:29.912 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T04:33:29.912 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:33:29.912 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: ** Compaction Stats [default] ** 2023-10-09T04:33:29.912 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:33:29.912 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:33:29.912 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.5 0.00 0.00 1 0.000 0 0 2023-10-09T04:33:29.912 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.5 0.00 0.00 1 0.000 0 0 2023-10-09T04:33:29.912 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.5 0.00 0.00 1 0.000 0 0 2023-10-09T04:33:29.913 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: ** Compaction Stats [default] ** 2023-10-09T04:33:29.913 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:33:29.913 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:33:29.913 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.5 0.00 0.00 1 0.000 0 0 2023-10-09T04:33:29.913 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:33:29.913 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T04:33:29.913 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T04:33:29.913 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T04:33:29.913 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T04:33:29.913 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T04:33:29.913 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: Cumulative compaction: 0.00 GB write, 0.47 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:33:29.913 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: Interval compaction: 0.00 GB write, 0.47 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:33:29.913 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T04:33:29.914 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T04:33:29.914 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: ** Compaction Stats [default] ** 2023-10-09T04:33:29.914 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:33:29.914 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:33:29.914 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.5 0.00 0.00 1 0.000 0 0 2023-10-09T04:33:29.914 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.5 0.00 0.00 1 0.000 0 0 2023-10-09T04:33:29.914 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T04:33:29.914 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: ** Compaction Stats [default] ** 2023-10-09T04:33:29.914 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:33:29.914 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:33:29.914 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.5 0.00 0.00 1 0.000 0 0 2023-10-09T04:33:29.915 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:33:29.915 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T04:33:29.915 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T04:33:29.915 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T04:33:29.915 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T04:33:29.915 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T04:33:29.915 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: Cumulative compaction: 0.00 GB write, 0.46 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:33:29.915 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:33:29.915 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T04:33:29.915 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T04:33:29.915 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 0 mon.b does not exist in monmap, will attempt to join an existing cluster 2023-10-09T04:33:29.915 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 0 using public_addr v2:172.21.15.194:0/0 -> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] 2023-10-09T04:33:29.915 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.588+0000 7f7a56cbab80 0 starting mon.b rank -1 at public addrs [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] at bind addrs [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 855b5206-665c-11ee-8db5-212e2dc638e7 2023-10-09T04:33:29.916 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.592+0000 7f7a56cbab80 1 mon.b@-1(???) e0 preinit fsid 855b5206-665c-11ee-8db5-212e2dc638e7 2023-10-09T04:33:29.916 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.612+0000 7f7a49f0f700 0 mon.b@-1(synchronizing).mds e1 new map 2023-10-09T04:33:29.916 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.612+0000 7f7a49f0f700 0 mon.b@-1(synchronizing).mds e1 print_map 2023-10-09T04:33:29.916 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: e1 2023-10-09T04:33:29.916 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T04:33:29.916 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T04:33:29.916 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: legacy client fscid: -1 2023-10-09T04:33:29.916 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: 2023-10-09T04:33:29.916 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: No filesystems configured 2023-10-09T04:33:29.916 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.612+0000 7f7a49f0f700 1 mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-10-09T04:33:29.916 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.612+0000 7f7a49f0f700 1 mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-10-09T04:33:29.916 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.612+0000 7f7a49f0f700 1 mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-10-09T04:33:29.917 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.612+0000 7f7a49f0f700 1 mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-10-09T04:33:29.917 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.612+0000 7f7a49f0f700 1 mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-10-09T04:33:29.917 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.612+0000 7f7a49f0f700 1 mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-10-09T04:33:29.917 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.612+0000 7f7a49f0f700 0 mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-09T04:33:29.917 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.612+0000 7f7a49f0f700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T04:33:29.917 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.612+0000 7f7a49f0f700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T04:33:29.917 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.612+0000 7f7a49f0f700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T04:33:29.917 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cluster 2023-10-09T04:32:45.625424+0000 mgr.y (mgr.14152) 12 : cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:29.917 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cluster 2023-10-09T04:32:47.625790+0000 mgr.y (mgr.14152) 13 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:29.917 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:32:48.880174+0000 mon.a (mon.0) 104 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:29.917 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:32:48.886228+0000 mon.a (mon.0) 105 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:33:29.917 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:32:48.888676+0000 mon.a (mon.0) 106 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:29.918 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:32:48.889785+0000 mon.a (mon.0) 107 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:33:29.918 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:32:48.874779+0000 mgr.y (mgr.14152) 14 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:33:29.918 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cephadm 2023-10-09T04:32:48.891243+0000 mgr.y (mgr.14152) 15 : cephadm [INF] Updating smithi133:/etc/ceph/ceph.conf 2023-10-09T04:33:29.918 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cephadm 2023-10-09T04:32:49.055646+0000 mgr.y (mgr.14152) 16 : cephadm [INF] Updating smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:33:29.918 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cephadm 2023-10-09T04:32:49.220816+0000 mgr.y (mgr.14152) 17 : cephadm [INF] Updating smithi133:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:33:29.918 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cephadm 2023-10-09T04:32:49.385145+0000 mgr.y (mgr.14152) 18 : cephadm [INF] Updating smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:33:29.918 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:32:49.556486+0000 mon.a (mon.0) 108 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:29.918 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:32:49.562039+0000 mon.a (mon.0) 109 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:29.918 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:32:49.566125+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:29.918 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cluster 2023-10-09T04:32:49.626165+0000 mgr.y (mgr.14152) 19 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:29.918 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cluster 2023-10-09T04:32:51.626529+0000 mgr.y (mgr.14152) 20 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:29.919 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cluster 2023-10-09T04:32:53.626924+0000 mgr.y (mgr.14152) 21 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:29.919 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cluster 2023-10-09T04:32:55.627389+0000 mgr.y (mgr.14152) 22 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:29.919 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:32:57.343250+0000 mgr.y (mgr.14152) 23 : audit [DBG] from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi194", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:33:29.919 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cluster 2023-10-09T04:32:57.627830+0000 mgr.y (mgr.14152) 24 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:29.919 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cephadm 2023-10-09T04:32:58.222616+0000 mgr.y (mgr.14152) 25 : cephadm [INF] Deploying cephadm binary to smithi194 2023-10-09T04:33:29.919 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cluster 2023-10-09T04:32:59.628236+0000 mgr.y (mgr.14152) 26 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:29.919 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:33:00.390514+0000 mon.a (mon.0) 111 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:29.919 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cephadm 2023-10-09T04:33:00.391322+0000 mgr.y (mgr.14152) 27 : cephadm [INF] Added host smithi194 2023-10-09T04:33:29.919 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:33:00.419703+0000 mon.a (mon.0) 112 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:33:29.919 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:33:00.743712+0000 mon.a (mon.0) 113 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:29.919 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cluster 2023-10-09T04:33:01.628648+0000 mgr.y (mgr.14152) 28 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:29.919 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:33:02.067789+0000 mon.a (mon.0) 114 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:29.919 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:33:02.805696+0000 mon.a (mon.0) 115 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:29.920 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cluster 2023-10-09T04:33:03.629070+0000 mgr.y (mgr.14152) 29 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:29.920 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cluster 2023-10-09T04:33:05.629517+0000 mgr.y (mgr.14152) 30 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:29.920 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cluster 2023-10-09T04:33:07.629938+0000 mgr.y (mgr.14152) 31 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:29.920 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:33:08.941908+0000 mon.a (mon.0) 116 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:29.920 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:33:08.947538+0000 mon.a (mon.0) 117 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:29.920 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:33:08.953797+0000 mon.a (mon.0) 118 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:29.920 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:33:08.959946+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:29.920 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:33:08.961005+0000 mon.a (mon.0) 120 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi194", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:33:29.920 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:33:08.962498+0000 mon.a (mon.0) 121 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:29.920 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:33:08.963427+0000 mon.a (mon.0) 122 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:33:29.921 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cephadm 2023-10-09T04:33:08.964871+0000 mgr.y (mgr.14152) 32 : cephadm [INF] Updating smithi194:/etc/ceph/ceph.conf 2023-10-09T04:33:29.921 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:33:08.980637+0000 mgr.y (mgr.14152) 33 : audit [DBG] from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:33:29.921 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cephadm 2023-10-09T04:33:09.129254+0000 mgr.y (mgr.14152) 34 : cephadm [INF] Updating smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:33:29.921 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cephadm 2023-10-09T04:33:09.294022+0000 mgr.y (mgr.14152) 35 : cephadm [INF] Updating smithi194:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:33:29.921 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cephadm 2023-10-09T04:33:09.458454+0000 mgr.y (mgr.14152) 36 : cephadm [INF] Updating smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:33:29.921 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:33:09.629052+0000 mon.a (mon.0) 123 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:29.921 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:33:09.633723+0000 mon.a (mon.0) 124 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:29.921 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:33:09.637041+0000 mon.a (mon.0) 125 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:29.921 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cluster 2023-10-09T04:33:09.630356+0000 mgr.y (mgr.14152) 37 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:29.921 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cluster 2023-10-09T04:33:11.630792+0000 mgr.y (mgr.14152) 38 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:29.922 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cluster 2023-10-09T04:33:13.631254+0000 mgr.y (mgr.14152) 39 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:29.922 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cluster 2023-10-09T04:33:15.631691+0000 mgr.y (mgr.14152) 40 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:29.922 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:33:17.479329+0000 mon.a (mon.0) 126 : audit [INF] from='client.? 172.21.15.133:0/2780383392' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-10-09T04:33:29.922 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cluster 2023-10-09T04:33:17.632067+0000 mgr.y (mgr.14152) 41 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:29.922 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:33:18.271187+0000 mon.a (mon.0) 127 : audit [INF] from='client.? 172.21.15.133:0/2780383392' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-10-09T04:33:29.923 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cluster 2023-10-09T04:33:18.271246+0000 mon.a (mon.0) 128 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T04:33:29.923 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cluster 2023-10-09T04:33:19.632435+0000 mgr.y (mgr.14152) 42 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:29.923 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cluster 2023-10-09T04:33:21.632826+0000 mgr.y (mgr.14152) 43 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:29.923 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:33:23.251864+0000 mon.a (mon.0) 129 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:29.923 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:33:23.240923+0000 mgr.y (mgr.14152) 44 : audit [DBG] from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi133:172.21.15.133=a;smithi133:[v2:172.21.15.133:3301,v1:172.21.15.133:6790]=c;smithi194:172.21.15.194=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:33:29.923 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cephadm 2023-10-09T04:33:23.246853+0000 mgr.y (mgr.14152) 45 : cephadm [INF] Saving service mon spec with placement smithi133:172.21.15.133=a;smithi133:[v2:172.21.15.133:3301,v1:172.21.15.133:6790]=c;smithi194:172.21.15.194=b;count:3 2023-10-09T04:33:29.923 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:33:23.301984+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:33:29.923 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:33:23.305842+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:29.923 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:33:23.307541+0000 mon.a (mon.0) 132 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:33:29.923 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:33:23.314614+0000 mon.a (mon.0) 133 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:29.924 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:33:23.317219+0000 mon.a (mon.0) 134 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:33:29.924 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:33:23.318398+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:29.924 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cephadm 2023-10-09T04:33:23.319872+0000 mgr.y (mgr.14152) 46 : cephadm [INF] Deploying daemon mon.b on smithi194 2023-10-09T04:33:29.924 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cluster 2023-10-09T04:33:23.633226+0000 mgr.y (mgr.14152) 47 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:29.924 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cluster 2023-10-09T04:33:25.633657+0000 mgr.y (mgr.14152) 48 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:29.924 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cluster 2023-10-09T04:33:27.634076+0000 mgr.y (mgr.14152) 49 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:29.924 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:33:28.342362+0000 mon.a (mon.0) 136 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:29.924 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:33:28.347349+0000 mon.a (mon.0) 137 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:29.924 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:33:28.351524+0000 mon.a (mon.0) 138 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:29.924 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:33:28.352356+0000 mon.a (mon.0) 139 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:33:29.924 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: audit 2023-10-09T04:33:28.353886+0000 mon.a (mon.0) 140 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:29.924 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: cephadm 2023-10-09T04:33:28.355571+0000 mgr.y (mgr.14152) 50 : cephadm [INF] Deploying daemon mon.c on smithi133 2023-10-09T04:33:29.925 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:29 smithi194 bash[31648]: debug 2023-10-09T04:33:29.616+0000 7f7a49f0f700 1 mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-10-09T04:33:33.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:33 smithi133 bash[41291]: debug 2023-10-09T04:33:33.703+0000 7f5ee1550700 1 mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-10-09T04:33:34.697 INFO:teuthology.orchestra.run.smithi194.stdout: 2023-10-09T04:33:34.697 INFO:teuthology.orchestra.run.smithi194.stdout:{"epoch":2,"fsid":"855b5206-665c-11ee-8db5-212e2dc638e7","modified":"2023-10-09T04:33:29.623164Z","created":"2023-10-09T04:31:04.046038Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:3300","nonce":0},{"type":"v1","addr":"172.21.15.133:6789","nonce":0}]},"addr":"172.21.15.133:6789/0","public_addr":"172.21.15.133:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:3300","nonce":0},{"type":"v1","addr":"172.21.15.194:6789","nonce":0}]},"addr":"172.21.15.194:6789/0","public_addr":"172.21.15.194:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-10-09T04:33:34.703 INFO:teuthology.orchestra.run.smithi194.stderr:dumped monmap epoch 2 2023-10-09T04:33:35.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:34 smithi133 bash[27558]: audit 2023-10-09T04:33:29.623791+0000 mon.a (mon.0) 141 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:33:35.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:34 smithi133 bash[27558]: audit 2023-10-09T04:33:29.627884+0000 mon.a (mon.0) 142 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:33:35.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:34 smithi133 bash[27558]: cluster 2023-10-09T04:33:29.628070+0000 mon.a (mon.0) 143 : cluster [INF] mon.a calling monitor election 2023-10-09T04:33:35.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:34 smithi133 bash[27558]: cluster 2023-10-09T04:33:29.634396+0000 mgr.y (mgr.14152) 51 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:35.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:34 smithi133 bash[27558]: audit 2023-10-09T04:33:29.646161+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:33:35.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:34 smithi133 bash[27558]: audit 2023-10-09T04:33:29.646347+0000 mon.a (mon.0) 145 : audit [DBG] from='client.? 172.21.15.194:0/3086274444' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:33:35.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:34 smithi133 bash[27558]: audit 2023-10-09T04:33:30.623998+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:33:35.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:34 smithi133 bash[27558]: audit 2023-10-09T04:33:31.624094+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:33:35.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:34 smithi133 bash[27558]: cluster 2023-10-09T04:33:31.625414+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-10-09T04:33:35.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:34 smithi133 bash[27558]: cluster 2023-10-09T04:33:31.634832+0000 mgr.y (mgr.14152) 52 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:35.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:34 smithi133 bash[27558]: audit 2023-10-09T04:33:32.624275+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:33:35.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:34 smithi133 bash[27558]: audit 2023-10-09T04:33:33.624226+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:33:35.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:34 smithi133 bash[27558]: cluster 2023-10-09T04:33:33.635235+0000 mgr.y (mgr.14152) 53 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:35.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:34 smithi133 bash[27558]: audit 2023-10-09T04:33:33.712584+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:33:35.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:34 smithi133 bash[27558]: audit 2023-10-09T04:33:34.624549+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:33:35.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:34 smithi133 bash[27558]: cluster 2023-10-09T04:33:34.668158+0000 mon.a (mon.0) 152 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T04:33:35.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:34 smithi133 bash[27558]: cluster 2023-10-09T04:33:34.674063+0000 mon.a (mon.0) 153 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.133:3300/0,v1:172.21.15.133:6789/0],b=[v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0]} removed_ranks: {} 2023-10-09T04:33:35.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:34 smithi133 bash[27558]: cluster 2023-10-09T04:33:34.694189+0000 mon.a (mon.0) 154 : cluster [DBG] fsmap 2023-10-09T04:33:35.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:34 smithi133 bash[27558]: cluster 2023-10-09T04:33:34.694248+0000 mon.a (mon.0) 155 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T04:33:35.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:34 smithi133 bash[27558]: cluster 2023-10-09T04:33:34.694515+0000 mon.a (mon.0) 156 : cluster [DBG] mgrmap e13: y(active, since 77s) 2023-10-09T04:33:35.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:34 smithi133 bash[27558]: cluster 2023-10-09T04:33:34.701883+0000 mon.a (mon.0) 157 : cluster [INF] overall HEALTH_OK 2023-10-09T04:33:35.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:34 smithi133 bash[27558]: audit 2023-10-09T04:33:34.706952+0000 mon.a (mon.0) 158 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:35.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:34 smithi133 bash[27558]: audit 2023-10-09T04:33:34.711678+0000 mon.a (mon.0) 159 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:35.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:34 smithi133 bash[27558]: audit 2023-10-09T04:33:34.713428+0000 mon.a (mon.0) 160 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:33:35.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:34 smithi133 bash[27558]: audit 2023-10-09T04:33:34.741624+0000 mon.a (mon.0) 161 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:35.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:34 smithi194 bash[31648]: audit 2023-10-09T04:33:29.623791+0000 mon.a (mon.0) 141 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:33:35.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:34 smithi194 bash[31648]: audit 2023-10-09T04:33:29.627884+0000 mon.a (mon.0) 142 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:33:35.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:34 smithi194 bash[31648]: cluster 2023-10-09T04:33:29.628070+0000 mon.a (mon.0) 143 : cluster [INF] mon.a calling monitor election 2023-10-09T04:33:35.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:34 smithi194 bash[31648]: cluster 2023-10-09T04:33:29.634396+0000 mgr.y (mgr.14152) 51 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:35.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:34 smithi194 bash[31648]: audit 2023-10-09T04:33:29.646161+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:33:35.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:34 smithi194 bash[31648]: audit 2023-10-09T04:33:29.646347+0000 mon.a (mon.0) 145 : audit [DBG] from='client.? 172.21.15.194:0/3086274444' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:33:35.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:34 smithi194 bash[31648]: audit 2023-10-09T04:33:30.623998+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:33:35.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:34 smithi194 bash[31648]: audit 2023-10-09T04:33:31.624094+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:33:35.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:34 smithi194 bash[31648]: cluster 2023-10-09T04:33:31.625414+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-10-09T04:33:35.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:34 smithi194 bash[31648]: cluster 2023-10-09T04:33:31.634832+0000 mgr.y (mgr.14152) 52 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:35.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:34 smithi194 bash[31648]: audit 2023-10-09T04:33:32.624275+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:33:35.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:34 smithi194 bash[31648]: audit 2023-10-09T04:33:33.624226+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:33:35.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:34 smithi194 bash[31648]: cluster 2023-10-09T04:33:33.635235+0000 mgr.y (mgr.14152) 53 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:35.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:34 smithi194 bash[31648]: audit 2023-10-09T04:33:33.712584+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:33:35.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:34 smithi194 bash[31648]: audit 2023-10-09T04:33:34.624549+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:33:35.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:34 smithi194 bash[31648]: cluster 2023-10-09T04:33:34.668158+0000 mon.a (mon.0) 152 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T04:33:35.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:34 smithi194 bash[31648]: cluster 2023-10-09T04:33:34.674063+0000 mon.a (mon.0) 153 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.133:3300/0,v1:172.21.15.133:6789/0],b=[v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0]} removed_ranks: {} 2023-10-09T04:33:35.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:34 smithi194 bash[31648]: cluster 2023-10-09T04:33:34.694189+0000 mon.a (mon.0) 154 : cluster [DBG] fsmap 2023-10-09T04:33:35.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:34 smithi194 bash[31648]: cluster 2023-10-09T04:33:34.694248+0000 mon.a (mon.0) 155 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T04:33:35.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:34 smithi194 bash[31648]: cluster 2023-10-09T04:33:34.694515+0000 mon.a (mon.0) 156 : cluster [DBG] mgrmap e13: y(active, since 77s) 2023-10-09T04:33:35.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:34 smithi194 bash[31648]: cluster 2023-10-09T04:33:34.701883+0000 mon.a (mon.0) 157 : cluster [INF] overall HEALTH_OK 2023-10-09T04:33:35.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:34 smithi194 bash[31648]: audit 2023-10-09T04:33:34.706952+0000 mon.a (mon.0) 158 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:35.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:34 smithi194 bash[31648]: audit 2023-10-09T04:33:34.711678+0000 mon.a (mon.0) 159 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:35.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:34 smithi194 bash[31648]: audit 2023-10-09T04:33:34.713428+0000 mon.a (mon.0) 160 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:33:35.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:34 smithi194 bash[31648]: audit 2023-10-09T04:33:34.741624+0000 mon.a (mon.0) 161 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:36.438 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-09T04:33:36.439 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-09T04:33:40.717 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:33:41.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[27558]: cluster 2023-10-09T04:33:35.635592+0000 mgr.y (mgr.14152) 54 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:41.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[27558]: audit 2023-10-09T04:33:35.720297+0000 mon.a (mon.0) 166 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:33:41.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[27558]: cluster 2023-10-09T04:33:35.720712+0000 mon.a (mon.0) 167 : cluster [INF] mon.a calling monitor election 2023-10-09T04:33:41.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[27558]: cluster 2023-10-09T04:33:35.724451+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-10-09T04:33:41.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[27558]: audit 2023-10-09T04:33:35.742829+0000 mon.a (mon.0) 168 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:33:41.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[27558]: audit 2023-10-09T04:33:35.743220+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:33:41.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[27558]: audit 2023-10-09T04:33:36.713860+0000 mon.a (mon.0) 170 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:33:41.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[27558]: cluster 2023-10-09T04:33:37.636045+0000 mgr.y (mgr.14152) 55 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:41.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[27558]: audit 2023-10-09T04:33:37.714209+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:33:41.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[27558]: cluster 2023-10-09T04:33:37.716163+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-09T04:33:41.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[27558]: audit 2023-10-09T04:33:38.713884+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:33:41.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[27558]: cluster 2023-10-09T04:33:39.636349+0000 mgr.y (mgr.14152) 56 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:41.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[27558]: audit 2023-10-09T04:33:39.714238+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:33:41.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[27558]: audit 2023-10-09T04:33:40.714321+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:33:41.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[27558]: cluster 2023-10-09T04:33:40.726345+0000 mon.a (mon.0) 175 : cluster [INF] mon.a calling monitor election 2023-10-09T04:33:41.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[27558]: cluster 2023-10-09T04:33:40.763821+0000 mon.a (mon.0) 176 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:33:41.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[27558]: cluster 2023-10-09T04:33:40.771438+0000 mon.a (mon.0) 177 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.133:3300/0,v1:172.21.15.133:6789/0],b=[v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0],c=[v2:172.21.15.133:3301/0,v1:172.21.15.133:6790/0]} removed_ranks: {} 2023-10-09T04:33:41.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[27558]: cluster 2023-10-09T04:33:40.789863+0000 mon.a (mon.0) 178 : cluster [DBG] fsmap 2023-10-09T04:33:41.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[27558]: cluster 2023-10-09T04:33:40.789989+0000 mon.a (mon.0) 179 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T04:33:41.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[27558]: cluster 2023-10-09T04:33:40.790328+0000 mon.a (mon.0) 180 : cluster [DBG] mgrmap e13: y(active, since 83s) 2023-10-09T04:33:41.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[27558]: cluster 2023-10-09T04:33:40.797975+0000 mon.a (mon.0) 181 : cluster [INF] overall HEALTH_OK 2023-10-09T04:33:41.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[27558]: audit 2023-10-09T04:33:40.800886+0000 mon.a (mon.0) 182 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:41.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[27558]: audit 2023-10-09T04:33:40.804618+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:41.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[27558]: audit 2023-10-09T04:33:40.808383+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:41.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[27558]: audit 2023-10-09T04:33:40.811764+0000 mon.a (mon.0) 185 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:41.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[27558]: audit 2023-10-09T04:33:40.814968+0000 mon.a (mon.0) 186 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:41.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: audit 2023-10-09T04:33:29.623791+0000 mon.a (mon.0) 141 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:33:41.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: audit 2023-10-09T04:33:29.627884+0000 mon.a (mon.0) 142 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:33:41.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: cluster 2023-10-09T04:33:29.628070+0000 mon.a (mon.0) 143 : cluster [INF] mon.a calling monitor election 2023-10-09T04:33:41.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: cluster 2023-10-09T04:33:29.634396+0000 mgr.y (mgr.14152) 51 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:41.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: audit 2023-10-09T04:33:29.646161+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:33:41.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: audit 2023-10-09T04:33:29.646347+0000 mon.a (mon.0) 145 : audit [DBG] from='client.? 172.21.15.194:0/3086274444' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:33:41.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: audit 2023-10-09T04:33:30.623998+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:33:41.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: audit 2023-10-09T04:33:31.624094+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:33:41.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: cluster 2023-10-09T04:33:31.625414+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-10-09T04:33:41.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: cluster 2023-10-09T04:33:31.634832+0000 mgr.y (mgr.14152) 52 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:41.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: audit 2023-10-09T04:33:32.624275+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:33:41.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: audit 2023-10-09T04:33:33.624226+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:33:41.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: cluster 2023-10-09T04:33:33.635235+0000 mgr.y (mgr.14152) 53 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:41.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: audit 2023-10-09T04:33:33.712584+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:33:41.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: audit 2023-10-09T04:33:34.624549+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:33:41.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: cluster 2023-10-09T04:33:34.668158+0000 mon.a (mon.0) 152 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T04:33:41.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: cluster 2023-10-09T04:33:34.674063+0000 mon.a (mon.0) 153 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.133:3300/0,v1:172.21.15.133:6789/0],b=[v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0]} removed_ranks: {} 2023-10-09T04:33:41.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: cluster 2023-10-09T04:33:34.694189+0000 mon.a (mon.0) 154 : cluster [DBG] fsmap 2023-10-09T04:33:41.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: cluster 2023-10-09T04:33:34.694248+0000 mon.a (mon.0) 155 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T04:33:41.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: cluster 2023-10-09T04:33:34.694515+0000 mon.a (mon.0) 156 : cluster [DBG] mgrmap e13: y(active, since 77s) 2023-10-09T04:33:41.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: cluster 2023-10-09T04:33:34.701883+0000 mon.a (mon.0) 157 : cluster [INF] overall HEALTH_OK 2023-10-09T04:33:41.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: audit 2023-10-09T04:33:34.706952+0000 mon.a (mon.0) 158 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:41.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: audit 2023-10-09T04:33:34.711678+0000 mon.a (mon.0) 159 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:41.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: audit 2023-10-09T04:33:34.713428+0000 mon.a (mon.0) 160 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:33:41.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: audit 2023-10-09T04:33:34.741624+0000 mon.a (mon.0) 161 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:41.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: cluster 2023-10-09T04:33:35.635592+0000 mgr.y (mgr.14152) 54 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:41.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: audit 2023-10-09T04:33:35.720297+0000 mon.a (mon.0) 166 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:33:41.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: cluster 2023-10-09T04:33:35.720712+0000 mon.a (mon.0) 167 : cluster [INF] mon.a calling monitor election 2023-10-09T04:33:41.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: cluster 2023-10-09T04:33:35.724451+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-10-09T04:33:41.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: audit 2023-10-09T04:33:35.742829+0000 mon.a (mon.0) 168 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:33:41.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: audit 2023-10-09T04:33:35.743220+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:33:41.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: audit 2023-10-09T04:33:36.713860+0000 mon.a (mon.0) 170 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:33:41.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: cluster 2023-10-09T04:33:37.636045+0000 mgr.y (mgr.14152) 55 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:41.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: audit 2023-10-09T04:33:37.714209+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:33:41.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: cluster 2023-10-09T04:33:37.716163+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-09T04:33:41.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: audit 2023-10-09T04:33:38.713884+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:33:41.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: cluster 2023-10-09T04:33:39.636349+0000 mgr.y (mgr.14152) 56 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:41.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: audit 2023-10-09T04:33:39.714238+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:33:41.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: audit 2023-10-09T04:33:40.714321+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:33:41.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: cluster 2023-10-09T04:33:40.726345+0000 mon.a (mon.0) 175 : cluster [INF] mon.a calling monitor election 2023-10-09T04:33:41.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: cluster 2023-10-09T04:33:40.763821+0000 mon.a (mon.0) 176 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:33:41.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: cluster 2023-10-09T04:33:40.771438+0000 mon.a (mon.0) 177 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.133:3300/0,v1:172.21.15.133:6789/0],b=[v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0],c=[v2:172.21.15.133:3301/0,v1:172.21.15.133:6790/0]} removed_ranks: {} 2023-10-09T04:33:41.056 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: cluster 2023-10-09T04:33:40.789863+0000 mon.a (mon.0) 178 : cluster [DBG] fsmap 2023-10-09T04:33:41.056 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: cluster 2023-10-09T04:33:40.789989+0000 mon.a (mon.0) 179 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T04:33:41.056 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: cluster 2023-10-09T04:33:40.790328+0000 mon.a (mon.0) 180 : cluster [DBG] mgrmap e13: y(active, since 83s) 2023-10-09T04:33:41.056 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: cluster 2023-10-09T04:33:40.797975+0000 mon.a (mon.0) 181 : cluster [INF] overall HEALTH_OK 2023-10-09T04:33:41.056 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: audit 2023-10-09T04:33:40.800886+0000 mon.a (mon.0) 182 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:41.056 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: audit 2023-10-09T04:33:40.804618+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:41.056 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: audit 2023-10-09T04:33:40.808383+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:41.056 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: audit 2023-10-09T04:33:40.811764+0000 mon.a (mon.0) 185 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:41.056 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:40 smithi133 bash[41291]: audit 2023-10-09T04:33:40.814968+0000 mon.a (mon.0) 186 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:41.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:40 smithi194 bash[31648]: cluster 2023-10-09T04:33:35.635592+0000 mgr.y (mgr.14152) 54 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:41.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:40 smithi194 bash[31648]: audit 2023-10-09T04:33:35.720297+0000 mon.a (mon.0) 166 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:33:41.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:40 smithi194 bash[31648]: cluster 2023-10-09T04:33:35.720712+0000 mon.a (mon.0) 167 : cluster [INF] mon.a calling monitor election 2023-10-09T04:33:41.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:40 smithi194 bash[31648]: cluster 2023-10-09T04:33:35.724451+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-10-09T04:33:41.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:40 smithi194 bash[31648]: audit 2023-10-09T04:33:35.742829+0000 mon.a (mon.0) 168 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:33:41.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:40 smithi194 bash[31648]: audit 2023-10-09T04:33:35.743220+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:33:41.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:40 smithi194 bash[31648]: audit 2023-10-09T04:33:36.713860+0000 mon.a (mon.0) 170 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:33:41.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:40 smithi194 bash[31648]: cluster 2023-10-09T04:33:37.636045+0000 mgr.y (mgr.14152) 55 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:41.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:40 smithi194 bash[31648]: audit 2023-10-09T04:33:37.714209+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:33:41.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:40 smithi194 bash[31648]: cluster 2023-10-09T04:33:37.716163+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-09T04:33:41.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:40 smithi194 bash[31648]: audit 2023-10-09T04:33:38.713884+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:33:41.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:40 smithi194 bash[31648]: cluster 2023-10-09T04:33:39.636349+0000 mgr.y (mgr.14152) 56 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:41.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:40 smithi194 bash[31648]: audit 2023-10-09T04:33:39.714238+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:33:41.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:40 smithi194 bash[31648]: audit 2023-10-09T04:33:40.714321+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:33:41.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:40 smithi194 bash[31648]: cluster 2023-10-09T04:33:40.726345+0000 mon.a (mon.0) 175 : cluster [INF] mon.a calling monitor election 2023-10-09T04:33:41.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:40 smithi194 bash[31648]: cluster 2023-10-09T04:33:40.763821+0000 mon.a (mon.0) 176 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:33:41.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:40 smithi194 bash[31648]: cluster 2023-10-09T04:33:40.771438+0000 mon.a (mon.0) 177 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.133:3300/0,v1:172.21.15.133:6789/0],b=[v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0],c=[v2:172.21.15.133:3301/0,v1:172.21.15.133:6790/0]} removed_ranks: {} 2023-10-09T04:33:41.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:40 smithi194 bash[31648]: cluster 2023-10-09T04:33:40.789863+0000 mon.a (mon.0) 178 : cluster [DBG] fsmap 2023-10-09T04:33:41.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:40 smithi194 bash[31648]: cluster 2023-10-09T04:33:40.789989+0000 mon.a (mon.0) 179 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T04:33:41.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:40 smithi194 bash[31648]: cluster 2023-10-09T04:33:40.790328+0000 mon.a (mon.0) 180 : cluster [DBG] mgrmap e13: y(active, since 83s) 2023-10-09T04:33:41.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:40 smithi194 bash[31648]: cluster 2023-10-09T04:33:40.797975+0000 mon.a (mon.0) 181 : cluster [INF] overall HEALTH_OK 2023-10-09T04:33:41.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:40 smithi194 bash[31648]: audit 2023-10-09T04:33:40.800886+0000 mon.a (mon.0) 182 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:41.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:40 smithi194 bash[31648]: audit 2023-10-09T04:33:40.804618+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:41.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:40 smithi194 bash[31648]: audit 2023-10-09T04:33:40.808383+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:41.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:40 smithi194 bash[31648]: audit 2023-10-09T04:33:40.811764+0000 mon.a (mon.0) 185 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:41.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:40 smithi194 bash[31648]: audit 2023-10-09T04:33:40.814968+0000 mon.a (mon.0) 186 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:41.973 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:41 smithi194 bash[31648]: audit 2023-10-09T04:33:41.193418+0000 mon.a (mon.0) 187 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:41.973 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:41 smithi194 bash[31648]: audit 2023-10-09T04:33:41.195241+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:33:41.973 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:41 smithi194 bash[31648]: cephadm 2023-10-09T04:33:41.197497+0000 mgr.y (mgr.14152) 57 : cephadm [INF] Updating smithi133:/etc/ceph/ceph.conf 2023-10-09T04:33:41.973 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:41 smithi194 bash[31648]: cephadm 2023-10-09T04:33:41.198345+0000 mgr.y (mgr.14152) 58 : cephadm [INF] Updating smithi194:/etc/ceph/ceph.conf 2023-10-09T04:33:41.973 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:41 smithi194 bash[31648]: cephadm 2023-10-09T04:33:41.363530+0000 mgr.y (mgr.14152) 59 : cephadm [INF] Updating smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:33:41.974 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:41 smithi194 bash[31648]: cephadm 2023-10-09T04:33:41.364007+0000 mgr.y (mgr.14152) 60 : cephadm [INF] Updating smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:33:41.974 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:41 smithi194 bash[31648]: audit 2023-10-09T04:33:41.536693+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:41.974 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:41 smithi194 bash[31648]: audit 2023-10-09T04:33:41.540333+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:41.974 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:41 smithi194 bash[31648]: audit 2023-10-09T04:33:41.544318+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:41.974 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:41 smithi194 bash[31648]: audit 2023-10-09T04:33:41.547537+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:41.974 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:41 smithi194 bash[31648]: audit 2023-10-09T04:33:41.551712+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:41.974 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:41 smithi194 bash[31648]: audit 2023-10-09T04:33:41.619292+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:33:41.974 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:41 smithi194 bash[31648]: audit 2023-10-09T04:33:41.620959+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:33:41.974 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:41 smithi194 bash[31648]: audit 2023-10-09T04:33:41.622460+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:41.974 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:41 smithi194 bash[31648]: audit 2023-10-09T04:33:41.714487+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:33:42.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:41 smithi133 bash[27558]: audit 2023-10-09T04:33:41.193418+0000 mon.a (mon.0) 187 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:42.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:41 smithi133 bash[27558]: audit 2023-10-09T04:33:41.195241+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:33:42.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:41 smithi133 bash[27558]: cephadm 2023-10-09T04:33:41.197497+0000 mgr.y (mgr.14152) 57 : cephadm [INF] Updating smithi133:/etc/ceph/ceph.conf 2023-10-09T04:33:42.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:41 smithi133 bash[27558]: cephadm 2023-10-09T04:33:41.198345+0000 mgr.y (mgr.14152) 58 : cephadm [INF] Updating smithi194:/etc/ceph/ceph.conf 2023-10-09T04:33:42.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:41 smithi133 bash[27558]: cephadm 2023-10-09T04:33:41.363530+0000 mgr.y (mgr.14152) 59 : cephadm [INF] Updating smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:33:42.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:41 smithi133 bash[27558]: cephadm 2023-10-09T04:33:41.364007+0000 mgr.y (mgr.14152) 60 : cephadm [INF] Updating smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:33:42.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:41 smithi133 bash[27558]: audit 2023-10-09T04:33:41.536693+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:42.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:41 smithi133 bash[27558]: audit 2023-10-09T04:33:41.540333+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:42.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:41 smithi133 bash[27558]: audit 2023-10-09T04:33:41.544318+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:42.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:41 smithi133 bash[27558]: audit 2023-10-09T04:33:41.547537+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:42.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:41 smithi133 bash[27558]: audit 2023-10-09T04:33:41.551712+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:42.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:41 smithi133 bash[27558]: audit 2023-10-09T04:33:41.619292+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:33:42.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:41 smithi133 bash[27558]: audit 2023-10-09T04:33:41.620959+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:33:42.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:41 smithi133 bash[27558]: audit 2023-10-09T04:33:41.622460+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:42.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:41 smithi133 bash[27558]: audit 2023-10-09T04:33:41.714487+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:33:42.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:41 smithi133 bash[41291]: audit 2023-10-09T04:33:41.193418+0000 mon.a (mon.0) 187 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:42.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:41 smithi133 bash[41291]: audit 2023-10-09T04:33:41.195241+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:33:42.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:41 smithi133 bash[41291]: cephadm 2023-10-09T04:33:41.197497+0000 mgr.y (mgr.14152) 57 : cephadm [INF] Updating smithi133:/etc/ceph/ceph.conf 2023-10-09T04:33:42.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:41 smithi133 bash[41291]: cephadm 2023-10-09T04:33:41.198345+0000 mgr.y (mgr.14152) 58 : cephadm [INF] Updating smithi194:/etc/ceph/ceph.conf 2023-10-09T04:33:42.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:41 smithi133 bash[41291]: cephadm 2023-10-09T04:33:41.363530+0000 mgr.y (mgr.14152) 59 : cephadm [INF] Updating smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:33:42.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:41 smithi133 bash[41291]: cephadm 2023-10-09T04:33:41.364007+0000 mgr.y (mgr.14152) 60 : cephadm [INF] Updating smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:33:42.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:41 smithi133 bash[41291]: audit 2023-10-09T04:33:41.536693+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:42.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:41 smithi133 bash[41291]: audit 2023-10-09T04:33:41.540333+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:42.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:41 smithi133 bash[41291]: audit 2023-10-09T04:33:41.544318+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:42.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:41 smithi133 bash[41291]: audit 2023-10-09T04:33:41.547537+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:42.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:41 smithi133 bash[41291]: audit 2023-10-09T04:33:41.551712+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:42.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:41 smithi133 bash[41291]: audit 2023-10-09T04:33:41.619292+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:33:42.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:41 smithi133 bash[41291]: audit 2023-10-09T04:33:41.620959+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:33:42.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:41 smithi133 bash[41291]: audit 2023-10-09T04:33:41.622460+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:42.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:41 smithi133 bash[41291]: audit 2023-10-09T04:33:41.714487+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:33:43.748 INFO:teuthology.orchestra.run.smithi194.stdout: 2023-10-09T04:33:43.748 INFO:teuthology.orchestra.run.smithi194.stdout:{"epoch":3,"fsid":"855b5206-665c-11ee-8db5-212e2dc638e7","modified":"2023-10-09T04:33:35.714126Z","created":"2023-10-09T04:31:04.046038Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:3300","nonce":0},{"type":"v1","addr":"172.21.15.133:6789","nonce":0}]},"addr":"172.21.15.133:6789/0","public_addr":"172.21.15.133:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:3300","nonce":0},{"type":"v1","addr":"172.21.15.194:6789","nonce":0}]},"addr":"172.21.15.194:6789/0","public_addr":"172.21.15.194:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:3301","nonce":0},{"type":"v1","addr":"172.21.15.133:6790","nonce":0}]},"addr":"172.21.15.133:6790/0","public_addr":"172.21.15.133:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2023-10-09T04:33:43.753 INFO:teuthology.orchestra.run.smithi194.stderr:dumped monmap epoch 3 2023-10-09T04:33:43.832 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:43 smithi194 bash[31648]: cephadm 2023-10-09T04:33:41.618698+0000 mgr.y (mgr.14152) 61 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-10-09T04:33:43.833 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:43 smithi194 bash[31648]: cephadm 2023-10-09T04:33:41.624201+0000 mgr.y (mgr.14152) 62 : cephadm [INF] Reconfiguring daemon mon.a on smithi133 2023-10-09T04:33:43.833 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:43 smithi194 bash[31648]: cluster 2023-10-09T04:33:41.636726+0000 mgr.y (mgr.14152) 63 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:43.880 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:43 smithi133 bash[41291]: cephadm 2023-10-09T04:33:41.618698+0000 mgr.y (mgr.14152) 61 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-10-09T04:33:43.880 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:43 smithi133 bash[41291]: cephadm 2023-10-09T04:33:41.624201+0000 mgr.y (mgr.14152) 62 : cephadm [INF] Reconfiguring daemon mon.a on smithi133 2023-10-09T04:33:43.881 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:43 smithi133 bash[41291]: cluster 2023-10-09T04:33:41.636726+0000 mgr.y (mgr.14152) 63 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:43.881 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:43 smithi133 bash[27558]: cephadm 2023-10-09T04:33:41.618698+0000 mgr.y (mgr.14152) 61 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-10-09T04:33:43.881 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:43 smithi133 bash[27558]: cephadm 2023-10-09T04:33:41.624201+0000 mgr.y (mgr.14152) 62 : cephadm [INF] Reconfiguring daemon mon.a on smithi133 2023-10-09T04:33:43.881 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:43 smithi133 bash[27558]: cluster 2023-10-09T04:33:41.636726+0000 mgr.y (mgr.14152) 63 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:44.424 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-10-09T04:33:44.425 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph config generate-minimal-conf 2023-10-09T04:33:44.734 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:44 smithi133 bash[27558]: audit 2023-10-09T04:33:43.591749+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:44.734 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:44 smithi133 bash[27558]: audit 2023-10-09T04:33:43.598017+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:44.734 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:44 smithi133 bash[27558]: cluster 2023-10-09T04:33:43.637134+0000 mgr.y (mgr.14152) 64 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:44.734 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:44 smithi133 bash[27558]: cephadm 2023-10-09T04:33:43.643391+0000 mgr.y (mgr.14152) 65 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-10-09T04:33:44.734 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:44 smithi133 bash[27558]: audit 2023-10-09T04:33:43.643976+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:33:44.734 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:44 smithi133 bash[27558]: audit 2023-10-09T04:33:43.645626+0000 mon.a (mon.0) 201 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:33:44.735 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:44 smithi133 bash[27558]: audit 2023-10-09T04:33:43.647364+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:44.735 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:44 smithi133 bash[27558]: cephadm 2023-10-09T04:33:43.649249+0000 mgr.y (mgr.14152) 66 : cephadm [INF] Reconfiguring daemon mon.c on smithi133 2023-10-09T04:33:44.735 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:44 smithi133 bash[27558]: audit 2023-10-09T04:33:43.746989+0000 mon.a (mon.0) 203 : audit [DBG] from='client.? 172.21.15.194:0/3631475153' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:33:44.735 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:44 smithi133 bash[41291]: audit 2023-10-09T04:33:43.591749+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:44.735 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:44 smithi133 bash[41291]: audit 2023-10-09T04:33:43.598017+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:44.735 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:44 smithi133 bash[41291]: cluster 2023-10-09T04:33:43.637134+0000 mgr.y (mgr.14152) 64 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:44.735 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:44 smithi133 bash[41291]: cephadm 2023-10-09T04:33:43.643391+0000 mgr.y (mgr.14152) 65 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-10-09T04:33:44.735 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:44 smithi133 bash[41291]: audit 2023-10-09T04:33:43.643976+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:33:44.735 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:44 smithi133 bash[41291]: audit 2023-10-09T04:33:43.645626+0000 mon.a (mon.0) 201 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:33:44.736 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:44 smithi133 bash[41291]: audit 2023-10-09T04:33:43.647364+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:44.736 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:44 smithi133 bash[41291]: cephadm 2023-10-09T04:33:43.649249+0000 mgr.y (mgr.14152) 66 : cephadm [INF] Reconfiguring daemon mon.c on smithi133 2023-10-09T04:33:44.736 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:44 smithi133 bash[41291]: audit 2023-10-09T04:33:43.746989+0000 mon.a (mon.0) 203 : audit [DBG] from='client.? 172.21.15.194:0/3631475153' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:33:44.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:44 smithi194 bash[31648]: audit 2023-10-09T04:33:43.591749+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:44.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:44 smithi194 bash[31648]: audit 2023-10-09T04:33:43.598017+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:44.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:44 smithi194 bash[31648]: cluster 2023-10-09T04:33:43.637134+0000 mgr.y (mgr.14152) 64 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:44.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:44 smithi194 bash[31648]: cephadm 2023-10-09T04:33:43.643391+0000 mgr.y (mgr.14152) 65 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-10-09T04:33:44.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:44 smithi194 bash[31648]: audit 2023-10-09T04:33:43.643976+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:33:44.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:44 smithi194 bash[31648]: audit 2023-10-09T04:33:43.645626+0000 mon.a (mon.0) 201 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:33:44.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:44 smithi194 bash[31648]: audit 2023-10-09T04:33:43.647364+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:44.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:44 smithi194 bash[31648]: cephadm 2023-10-09T04:33:43.649249+0000 mgr.y (mgr.14152) 66 : cephadm [INF] Reconfiguring daemon mon.c on smithi133 2023-10-09T04:33:44.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:44 smithi194 bash[31648]: audit 2023-10-09T04:33:43.746989+0000 mon.a (mon.0) 203 : audit [DBG] from='client.? 172.21.15.194:0/3631475153' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:33:46.779 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:46 smithi194 bash[31648]: audit 2023-10-09T04:33:45.527133+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:46.779 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:46 smithi194 bash[31648]: audit 2023-10-09T04:33:45.535042+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:46.779 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:46 smithi194 bash[31648]: audit 2023-10-09T04:33:45.569164+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:33:46.780 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:46 smithi194 bash[31648]: audit 2023-10-09T04:33:45.571230+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:33:46.780 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:46 smithi194 bash[31648]: audit 2023-10-09T04:33:45.573098+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:46.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:46 smithi133 bash[41291]: audit 2023-10-09T04:33:45.527133+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:46.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:46 smithi133 bash[41291]: audit 2023-10-09T04:33:45.535042+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:46.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:46 smithi133 bash[41291]: audit 2023-10-09T04:33:45.569164+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:33:46.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:46 smithi133 bash[41291]: audit 2023-10-09T04:33:45.571230+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:33:46.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:46 smithi133 bash[41291]: audit 2023-10-09T04:33:45.573098+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:46.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:46 smithi133 bash[27558]: audit 2023-10-09T04:33:45.527133+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:46.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:46 smithi133 bash[27558]: audit 2023-10-09T04:33:45.535042+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:46.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:46 smithi133 bash[27558]: audit 2023-10-09T04:33:45.569164+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:33:46.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:46 smithi133 bash[27558]: audit 2023-10-09T04:33:45.571230+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:33:46.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:46 smithi133 bash[27558]: audit 2023-10-09T04:33:45.573098+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:47.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:47 smithi133 bash[27558]: cephadm 2023-10-09T04:33:45.568453+0000 mgr.y (mgr.14152) 67 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-10-09T04:33:47.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:47 smithi133 bash[27558]: cephadm 2023-10-09T04:33:45.574845+0000 mgr.y (mgr.14152) 68 : cephadm [INF] Reconfiguring daemon mon.b on smithi194 2023-10-09T04:33:47.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:47 smithi133 bash[27558]: cluster 2023-10-09T04:33:45.637584+0000 mgr.y (mgr.14152) 69 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:47.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:47 smithi133 bash[27558]: audit 2023-10-09T04:33:47.387309+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:47.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:47 smithi133 bash[27558]: audit 2023-10-09T04:33:47.391858+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:47.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:47 smithi133 bash[27558]: audit 2023-10-09T04:33:47.395642+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:33:47.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:47 smithi133 bash[27558]: audit 2023-10-09T04:33:47.398257+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:47.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:47 smithi133 bash[27558]: audit 2023-10-09T04:33:47.399619+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:33:47.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:47 smithi133 bash[27558]: audit 2023-10-09T04:33:47.406229+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:47.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:47 smithi133 bash[41291]: cephadm 2023-10-09T04:33:45.568453+0000 mgr.y (mgr.14152) 67 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-10-09T04:33:47.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:47 smithi133 bash[41291]: cephadm 2023-10-09T04:33:45.574845+0000 mgr.y (mgr.14152) 68 : cephadm [INF] Reconfiguring daemon mon.b on smithi194 2023-10-09T04:33:47.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:47 smithi133 bash[41291]: cluster 2023-10-09T04:33:45.637584+0000 mgr.y (mgr.14152) 69 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:47.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:47 smithi133 bash[41291]: audit 2023-10-09T04:33:47.387309+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:47.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:47 smithi133 bash[41291]: audit 2023-10-09T04:33:47.391858+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:47.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:47 smithi133 bash[41291]: audit 2023-10-09T04:33:47.395642+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:33:47.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:47 smithi133 bash[41291]: audit 2023-10-09T04:33:47.398257+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:47.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:47 smithi133 bash[41291]: audit 2023-10-09T04:33:47.399619+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:33:47.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:47 smithi133 bash[41291]: audit 2023-10-09T04:33:47.406229+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:47.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:47 smithi194 bash[31648]: cephadm 2023-10-09T04:33:45.568453+0000 mgr.y (mgr.14152) 67 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-10-09T04:33:47.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:47 smithi194 bash[31648]: cephadm 2023-10-09T04:33:45.574845+0000 mgr.y (mgr.14152) 68 : cephadm [INF] Reconfiguring daemon mon.b on smithi194 2023-10-09T04:33:47.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:47 smithi194 bash[31648]: cluster 2023-10-09T04:33:45.637584+0000 mgr.y (mgr.14152) 69 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:47.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:47 smithi194 bash[31648]: audit 2023-10-09T04:33:47.387309+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:47.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:47 smithi194 bash[31648]: audit 2023-10-09T04:33:47.391858+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:47.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:47 smithi194 bash[31648]: audit 2023-10-09T04:33:47.395642+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:33:47.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:47 smithi194 bash[31648]: audit 2023-10-09T04:33:47.398257+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:47.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:47 smithi194 bash[31648]: audit 2023-10-09T04:33:47.399619+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:33:47.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:47 smithi194 bash[31648]: audit 2023-10-09T04:33:47.406229+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:33:49.146 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:49.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:49 smithi133 bash[27558]: cluster 2023-10-09T04:33:47.637940+0000 mgr.y (mgr.14152) 70 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:49.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:49 smithi133 bash[41291]: cluster 2023-10-09T04:33:47.637940+0000 mgr.y (mgr.14152) 70 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:49.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:49 smithi194 bash[31648]: cluster 2023-10-09T04:33:47.637940+0000 mgr.y (mgr.14152) 70 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:51.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:51 smithi133 bash[27558]: cluster 2023-10-09T04:33:49.638203+0000 mgr.y (mgr.14152) 71 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:51.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:51 smithi133 bash[41291]: cluster 2023-10-09T04:33:49.638203+0000 mgr.y (mgr.14152) 71 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:51.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:51 smithi194 bash[31648]: cluster 2023-10-09T04:33:49.638203+0000 mgr.y (mgr.14152) 71 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:52.245 INFO:teuthology.orchestra.run.smithi133.stdout:# minimal ceph.conf for 855b5206-665c-11ee-8db5-212e2dc638e7 2023-10-09T04:33:52.245 INFO:teuthology.orchestra.run.smithi133.stdout:[global] 2023-10-09T04:33:52.245 INFO:teuthology.orchestra.run.smithi133.stdout: fsid = 855b5206-665c-11ee-8db5-212e2dc638e7 2023-10-09T04:33:52.245 INFO:teuthology.orchestra.run.smithi133.stdout: mon_host = [v2:172.21.15.133:3300/0,v1:172.21.15.133:6789/0] [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] [v2:172.21.15.133:3301/0,v1:172.21.15.133:6790/0] 2023-10-09T04:33:52.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:52 smithi133 bash[27558]: audit 2023-10-09T04:33:52.244069+0000 mon.a (mon.0) 215 : audit [DBG] from='client.? 172.21.15.133:0/1677981507' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:52.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:52 smithi133 bash[41291]: audit 2023-10-09T04:33:52.244069+0000 mon.a (mon.0) 215 : audit [DBG] from='client.? 172.21.15.133:0/1677981507' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:52.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:52 smithi194 bash[31648]: audit 2023-10-09T04:33:52.244069+0000 mon.a (mon.0) 215 : audit [DBG] from='client.? 172.21.15.133:0/1677981507' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:53.052 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-10-09T04:33:53.053 DEBUG:teuthology.orchestra.run.smithi133:> set -ex 2023-10-09T04:33:53.053 DEBUG:teuthology.orchestra.run.smithi133:> sudo dd of=/etc/ceph/ceph.conf 2023-10-09T04:33:53.068 DEBUG:teuthology.orchestra.run.smithi133:> set -ex 2023-10-09T04:33:53.068 DEBUG:teuthology.orchestra.run.smithi133:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:33:53.129 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2023-10-09T04:33:53.129 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd of=/etc/ceph/ceph.conf 2023-10-09T04:33:53.146 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2023-10-09T04:33:53.146 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:33:53.204 INFO:tasks.cephadm:Adding mgr.y on smithi133 2023-10-09T04:33:53.204 INFO:tasks.cephadm:Adding mgr.x on smithi194 2023-10-09T04:33:53.205 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph orch apply mgr '2;smithi133=y;smithi194=x' 2023-10-09T04:33:53.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:53 smithi194 bash[31648]: cluster 2023-10-09T04:33:51.638568+0000 mgr.y (mgr.14152) 72 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:54.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:53 smithi133 bash[27558]: cluster 2023-10-09T04:33:51.638568+0000 mgr.y (mgr.14152) 72 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:54.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:53 smithi133 bash[41291]: cluster 2023-10-09T04:33:51.638568+0000 mgr.y (mgr.14152) 72 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:55.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:55 smithi194 bash[31648]: cluster 2023-10-09T04:33:53.639000+0000 mgr.y (mgr.14152) 73 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:56.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:55 smithi133 bash[27558]: cluster 2023-10-09T04:33:53.639000+0000 mgr.y (mgr.14152) 73 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:56.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:55 smithi133 bash[41291]: cluster 2023-10-09T04:33:53.639000+0000 mgr.y (mgr.14152) 73 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:56.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:56 smithi194 bash[31648]: cluster 2023-10-09T04:33:55.639418+0000 mgr.y (mgr.14152) 74 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:56.963 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:33:57.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:56 smithi133 bash[27558]: cluster 2023-10-09T04:33:55.639418+0000 mgr.y (mgr.14152) 74 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:57.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:56 smithi133 bash[41291]: cluster 2023-10-09T04:33:55.639418+0000 mgr.y (mgr.14152) 74 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:59.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:33:58 smithi133 bash[27558]: cluster 2023-10-09T04:33:57.639845+0000 mgr.y (mgr.14152) 75 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:59.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:33:58 smithi133 bash[41291]: cluster 2023-10-09T04:33:57.639845+0000 mgr.y (mgr.14152) 75 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:59.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:33:58 smithi194 bash[31648]: cluster 2023-10-09T04:33:57.639845+0000 mgr.y (mgr.14152) 75 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:33:59.813 INFO:teuthology.orchestra.run.smithi194.stdout:Scheduled mgr update... 2023-10-09T04:34:00.837 DEBUG:teuthology.orchestra.run.smithi194:mgr.x> sudo journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mgr.x.service 2023-10-09T04:34:00.840 INFO:tasks.cephadm:Deploying OSDs... 2023-10-09T04:34:00.840 DEBUG:teuthology.orchestra.run.smithi133:> set -ex 2023-10-09T04:34:00.840 DEBUG:teuthology.orchestra.run.smithi133:> dd if=/scratch_devs of=/dev/stdout 2023-10-09T04:34:00.847 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-09T04:34:00.848 DEBUG:teuthology.orchestra.run.smithi133:> stat /dev/vg_nvme/lv_1 2023-10-09T04:34:00.897 INFO:teuthology.orchestra.run.smithi133.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-09T04:34:00.897 INFO:teuthology.orchestra.run.smithi133.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T04:34:00.897 INFO:teuthology.orchestra.run.smithi133.stdout:Device: 5h/5d Inode: 491 Links: 1 2023-10-09T04:34:00.897 INFO:teuthology.orchestra.run.smithi133.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T04:34:00.897 INFO:teuthology.orchestra.run.smithi133.stdout:Access: 2023-10-09 04:26:20.731030837 +0000 2023-10-09T04:34:00.897 INFO:teuthology.orchestra.run.smithi133.stdout:Modify: 2023-10-09 04:26:20.727030928 +0000 2023-10-09T04:34:00.897 INFO:teuthology.orchestra.run.smithi133.stdout:Change: 2023-10-09 04:26:20.727030928 +0000 2023-10-09T04:34:00.897 INFO:teuthology.orchestra.run.smithi133.stdout: Birth: - 2023-10-09T04:34:00.898 DEBUG:teuthology.orchestra.run.smithi133:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-09T04:34:00.956 INFO:teuthology.orchestra.run.smithi133.stderr:1+0 records in 2023-10-09T04:34:00.956 INFO:teuthology.orchestra.run.smithi133.stderr:1+0 records out 2023-10-09T04:34:00.956 INFO:teuthology.orchestra.run.smithi133.stderr:512 bytes copied, 0.000411169 s, 1.2 MB/s 2023-10-09T04:34:00.957 DEBUG:teuthology.orchestra.run.smithi133:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-09T04:34:01.014 DEBUG:teuthology.orchestra.run.smithi133:> stat /dev/vg_nvme/lv_2 2023-10-09T04:34:01.068 INFO:teuthology.orchestra.run.smithi133.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-09T04:34:01.068 INFO:teuthology.orchestra.run.smithi133.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T04:34:01.068 INFO:teuthology.orchestra.run.smithi133.stdout:Device: 5h/5d Inode: 498 Links: 1 2023-10-09T04:34:01.068 INFO:teuthology.orchestra.run.smithi133.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T04:34:01.068 INFO:teuthology.orchestra.run.smithi133.stdout:Access: 2023-10-09 04:26:21.611010543 +0000 2023-10-09T04:34:01.068 INFO:teuthology.orchestra.run.smithi133.stdout:Modify: 2023-10-09 04:26:21.279018199 +0000 2023-10-09T04:34:01.069 INFO:teuthology.orchestra.run.smithi133.stdout:Change: 2023-10-09 04:26:21.279018199 +0000 2023-10-09T04:34:01.069 INFO:teuthology.orchestra.run.smithi133.stdout: Birth: - 2023-10-09T04:34:01.069 DEBUG:teuthology.orchestra.run.smithi133:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-09T04:34:01.120 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:00 smithi133 bash[27558]: cluster 2023-10-09T04:33:59.640249+0000 mgr.y (mgr.14152) 76 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:01.120 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:00 smithi133 bash[27558]: audit 2023-10-09T04:33:59.802302+0000 mgr.y (mgr.14152) 77 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi133=y;smithi194=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:34:01.121 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:00 smithi133 bash[27558]: cephadm 2023-10-09T04:33:59.805615+0000 mgr.y (mgr.14152) 78 : cephadm [INF] Saving service mgr spec with placement smithi133=y;smithi194=x;count:2 2023-10-09T04:34:01.121 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:00 smithi133 bash[27558]: audit 2023-10-09T04:33:59.811967+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:01.121 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:00 smithi133 bash[27558]: audit 2023-10-09T04:33:59.818764+0000 mon.a (mon.0) 217 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:34:01.121 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:00 smithi133 bash[27558]: audit 2023-10-09T04:33:59.821655+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:01.121 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:00 smithi133 bash[27558]: audit 2023-10-09T04:33:59.823074+0000 mon.a (mon.0) 219 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:34:01.121 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:00 smithi133 bash[27558]: audit 2023-10-09T04:33:59.830774+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:01.121 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:00 smithi133 bash[27558]: audit 2023-10-09T04:33:59.832903+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:34:01.121 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:00 smithi133 bash[27558]: audit 2023-10-09T04:33:59.836890+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T04:34:01.121 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:00 smithi133 bash[27558]: audit 2023-10-09T04:33:59.837981+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:34:01.121 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:00 smithi133 bash[27558]: audit 2023-10-09T04:33:59.839217+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:01.121 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:00 smithi133 bash[27558]: cephadm 2023-10-09T04:33:59.840514+0000 mgr.y (mgr.14152) 79 : cephadm [INF] Deploying daemon mgr.x on smithi194 2023-10-09T04:34:01.122 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:00 smithi133 bash[41291]: cluster 2023-10-09T04:33:59.640249+0000 mgr.y (mgr.14152) 76 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:01.122 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:00 smithi133 bash[41291]: audit 2023-10-09T04:33:59.802302+0000 mgr.y (mgr.14152) 77 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi133=y;smithi194=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:34:01.122 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:00 smithi133 bash[41291]: cephadm 2023-10-09T04:33:59.805615+0000 mgr.y (mgr.14152) 78 : cephadm [INF] Saving service mgr spec with placement smithi133=y;smithi194=x;count:2 2023-10-09T04:34:01.122 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:00 smithi133 bash[41291]: audit 2023-10-09T04:33:59.811967+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:01.122 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:00 smithi133 bash[41291]: audit 2023-10-09T04:33:59.818764+0000 mon.a (mon.0) 217 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:34:01.122 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:00 smithi133 bash[41291]: audit 2023-10-09T04:33:59.821655+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:01.122 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:00 smithi133 bash[41291]: audit 2023-10-09T04:33:59.823074+0000 mon.a (mon.0) 219 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:34:01.122 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:00 smithi133 bash[41291]: audit 2023-10-09T04:33:59.830774+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:01.122 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:00 smithi133 bash[41291]: audit 2023-10-09T04:33:59.832903+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:34:01.122 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:00 smithi133 bash[41291]: audit 2023-10-09T04:33:59.836890+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T04:34:01.123 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:00 smithi133 bash[41291]: audit 2023-10-09T04:33:59.837981+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:34:01.123 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:00 smithi133 bash[41291]: audit 2023-10-09T04:33:59.839217+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:01.123 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:00 smithi133 bash[41291]: cephadm 2023-10-09T04:33:59.840514+0000 mgr.y (mgr.14152) 79 : cephadm [INF] Deploying daemon mgr.x on smithi194 2023-10-09T04:34:01.123 INFO:teuthology.orchestra.run.smithi133.stderr:1+0 records in 2023-10-09T04:34:01.124 INFO:teuthology.orchestra.run.smithi133.stderr:1+0 records out 2023-10-09T04:34:01.124 INFO:teuthology.orchestra.run.smithi133.stderr:512 bytes copied, 0.000364715 s, 1.4 MB/s 2023-10-09T04:34:01.124 DEBUG:teuthology.orchestra.run.smithi133:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-09T04:34:01.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:00 smithi194 bash[31648]: cluster 2023-10-09T04:33:59.640249+0000 mgr.y (mgr.14152) 76 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:01.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:00 smithi194 bash[31648]: audit 2023-10-09T04:33:59.802302+0000 mgr.y (mgr.14152) 77 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi133=y;smithi194=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:34:01.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:00 smithi194 bash[31648]: cephadm 2023-10-09T04:33:59.805615+0000 mgr.y (mgr.14152) 78 : cephadm [INF] Saving service mgr spec with placement smithi133=y;smithi194=x;count:2 2023-10-09T04:34:01.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:00 smithi194 bash[31648]: audit 2023-10-09T04:33:59.811967+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:01.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:00 smithi194 bash[31648]: audit 2023-10-09T04:33:59.818764+0000 mon.a (mon.0) 217 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:34:01.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:00 smithi194 bash[31648]: audit 2023-10-09T04:33:59.821655+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:01.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:00 smithi194 bash[31648]: audit 2023-10-09T04:33:59.823074+0000 mon.a (mon.0) 219 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:34:01.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:00 smithi194 bash[31648]: audit 2023-10-09T04:33:59.830774+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:01.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:00 smithi194 bash[31648]: audit 2023-10-09T04:33:59.832903+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:34:01.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:00 smithi194 bash[31648]: audit 2023-10-09T04:33:59.836890+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T04:34:01.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:00 smithi194 bash[31648]: audit 2023-10-09T04:33:59.837981+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:34:01.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:00 smithi194 bash[31648]: audit 2023-10-09T04:33:59.839217+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:01.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:00 smithi194 bash[31648]: cephadm 2023-10-09T04:33:59.840514+0000 mgr.y (mgr.14152) 79 : cephadm [INF] Deploying daemon mgr.x on smithi194 2023-10-09T04:34:01.174 DEBUG:teuthology.orchestra.run.smithi133:> stat /dev/vg_nvme/lv_3 2023-10-09T04:34:01.225 INFO:teuthology.orchestra.run.smithi133.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-09T04:34:01.225 INFO:teuthology.orchestra.run.smithi133.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T04:34:01.225 INFO:teuthology.orchestra.run.smithi133.stdout:Device: 5h/5d Inode: 504 Links: 1 2023-10-09T04:34:01.225 INFO:teuthology.orchestra.run.smithi133.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T04:34:01.225 INFO:teuthology.orchestra.run.smithi133.stdout:Access: 2023-10-09 04:26:22.154997997 +0000 2023-10-09T04:34:01.225 INFO:teuthology.orchestra.run.smithi133.stdout:Modify: 2023-10-09 04:26:21.835005377 +0000 2023-10-09T04:34:01.225 INFO:teuthology.orchestra.run.smithi133.stdout:Change: 2023-10-09 04:26:21.835005377 +0000 2023-10-09T04:34:01.225 INFO:teuthology.orchestra.run.smithi133.stdout: Birth: - 2023-10-09T04:34:01.226 DEBUG:teuthology.orchestra.run.smithi133:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-09T04:34:01.283 INFO:teuthology.orchestra.run.smithi133.stderr:1+0 records in 2023-10-09T04:34:01.283 INFO:teuthology.orchestra.run.smithi133.stderr:1+0 records out 2023-10-09T04:34:01.283 INFO:teuthology.orchestra.run.smithi133.stderr:512 bytes copied, 0.000401552 s, 1.3 MB/s 2023-10-09T04:34:01.284 DEBUG:teuthology.orchestra.run.smithi133:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-09T04:34:01.334 DEBUG:teuthology.orchestra.run.smithi133:> stat /dev/vg_nvme/lv_4 2023-10-09T04:34:01.384 INFO:teuthology.orchestra.run.smithi133.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-09T04:34:01.385 INFO:teuthology.orchestra.run.smithi133.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T04:34:01.385 INFO:teuthology.orchestra.run.smithi133.stdout:Device: 5h/5d Inode: 508 Links: 1 2023-10-09T04:34:01.385 INFO:teuthology.orchestra.run.smithi133.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T04:34:01.385 INFO:teuthology.orchestra.run.smithi133.stdout:Access: 2023-10-09 04:26:22.358993293 +0000 2023-10-09T04:34:01.385 INFO:teuthology.orchestra.run.smithi133.stdout:Modify: 2023-10-09 04:26:22.354993385 +0000 2023-10-09T04:34:01.386 INFO:teuthology.orchestra.run.smithi133.stdout:Change: 2023-10-09 04:26:22.354993385 +0000 2023-10-09T04:34:01.386 INFO:teuthology.orchestra.run.smithi133.stdout: Birth: - 2023-10-09T04:34:01.386 DEBUG:teuthology.orchestra.run.smithi133:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-09T04:34:01.414 INFO:journalctl@ceph.mgr.x.smithi194.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:34:01.440 INFO:teuthology.orchestra.run.smithi133.stderr:1+0 records in 2023-10-09T04:34:01.440 INFO:teuthology.orchestra.run.smithi133.stderr:1+0 records out 2023-10-09T04:34:01.440 INFO:teuthology.orchestra.run.smithi133.stderr:512 bytes copied, 0.000417492 s, 1.2 MB/s 2023-10-09T04:34:01.441 DEBUG:teuthology.orchestra.run.smithi133:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-09T04:34:01.494 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2023-10-09T04:34:01.494 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/scratch_devs of=/dev/stdout 2023-10-09T04:34:01.501 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-09T04:34:01.501 DEBUG:teuthology.orchestra.run.smithi194:> stat /dev/vg_nvme/lv_1 2023-10-09T04:34:01.549 INFO:teuthology.orchestra.run.smithi194.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-09T04:34:01.549 INFO:teuthology.orchestra.run.smithi194.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T04:34:01.549 INFO:teuthology.orchestra.run.smithi194.stdout:Device: 5h/5d Inode: 489 Links: 1 2023-10-09T04:34:01.549 INFO:teuthology.orchestra.run.smithi194.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T04:34:01.549 INFO:teuthology.orchestra.run.smithi194.stdout:Access: 2023-10-09 04:26:21.073341856 +0000 2023-10-09T04:34:01.549 INFO:teuthology.orchestra.run.smithi194.stdout:Modify: 2023-10-09 04:26:20.749344064 +0000 2023-10-09T04:34:01.549 INFO:teuthology.orchestra.run.smithi194.stdout:Change: 2023-10-09 04:26:20.749344064 +0000 2023-10-09T04:34:01.549 INFO:teuthology.orchestra.run.smithi194.stdout: Birth: - 2023-10-09T04:34:01.550 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-09T04:34:01.602 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records in 2023-10-09T04:34:01.602 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records out 2023-10-09T04:34:01.602 INFO:teuthology.orchestra.run.smithi194.stderr:512 bytes copied, 0.000308449 s, 1.7 MB/s 2023-10-09T04:34:01.603 DEBUG:teuthology.orchestra.run.smithi194:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-09T04:34:01.655 DEBUG:teuthology.orchestra.run.smithi194:> stat /dev/vg_nvme/lv_2 2023-10-09T04:34:01.702 INFO:teuthology.orchestra.run.smithi194.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-09T04:34:01.702 INFO:teuthology.orchestra.run.smithi194.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T04:34:01.702 INFO:teuthology.orchestra.run.smithi194.stdout:Device: 5h/5d Inode: 498 Links: 1 2023-10-09T04:34:01.702 INFO:teuthology.orchestra.run.smithi194.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T04:34:01.703 INFO:teuthology.orchestra.run.smithi194.stdout:Access: 2023-10-09 04:26:21.589348016 +0000 2023-10-09T04:34:01.703 INFO:teuthology.orchestra.run.smithi194.stdout:Modify: 2023-10-09 04:26:21.285344387 +0000 2023-10-09T04:34:01.703 INFO:teuthology.orchestra.run.smithi194.stdout:Change: 2023-10-09 04:26:21.285344387 +0000 2023-10-09T04:34:01.703 INFO:teuthology.orchestra.run.smithi194.stdout: Birth: - 2023-10-09T04:34:01.703 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-09T04:34:01.758 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records in 2023-10-09T04:34:01.758 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records out 2023-10-09T04:34:01.758 INFO:teuthology.orchestra.run.smithi194.stderr:512 bytes copied, 0.000252381 s, 2.0 MB/s 2023-10-09T04:34:01.759 DEBUG:teuthology.orchestra.run.smithi194:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-09T04:34:01.810 DEBUG:teuthology.orchestra.run.smithi194:> stat /dev/vg_nvme/lv_3 2023-10-09T04:34:01.861 INFO:teuthology.orchestra.run.smithi194.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-09T04:34:01.861 INFO:teuthology.orchestra.run.smithi194.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T04:34:01.861 INFO:teuthology.orchestra.run.smithi194.stdout:Device: 5h/5d Inode: 503 Links: 1 2023-10-09T04:34:01.862 INFO:teuthology.orchestra.run.smithi194.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T04:34:01.862 INFO:teuthology.orchestra.run.smithi194.stdout:Access: 2023-10-09 04:26:22.137354289 +0000 2023-10-09T04:34:01.862 INFO:teuthology.orchestra.run.smithi194.stdout:Modify: 2023-10-09 04:26:21.801350547 +0000 2023-10-09T04:34:01.862 INFO:teuthology.orchestra.run.smithi194.stdout:Change: 2023-10-09 04:26:21.801350547 +0000 2023-10-09T04:34:01.862 INFO:teuthology.orchestra.run.smithi194.stdout: Birth: - 2023-10-09T04:34:01.862 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-09T04:34:01.916 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records in 2023-10-09T04:34:01.917 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records out 2023-10-09T04:34:01.917 INFO:teuthology.orchestra.run.smithi194.stderr:512 bytes copied, 0.000401175 s, 1.3 MB/s 2023-10-09T04:34:01.918 DEBUG:teuthology.orchestra.run.smithi194:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-09T04:34:01.970 DEBUG:teuthology.orchestra.run.smithi194:> stat /dev/vg_nvme/lv_4 2023-10-09T04:34:02.021 INFO:teuthology.orchestra.run.smithi194.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-09T04:34:02.021 INFO:teuthology.orchestra.run.smithi194.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T04:34:02.021 INFO:teuthology.orchestra.run.smithi194.stdout:Device: 5h/5d Inode: 509 Links: 1 2023-10-09T04:34:02.021 INFO:teuthology.orchestra.run.smithi194.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T04:34:02.021 INFO:teuthology.orchestra.run.smithi194.stdout:Access: 2023-10-09 04:26:22.725360133 +0000 2023-10-09T04:34:02.021 INFO:teuthology.orchestra.run.smithi194.stdout:Modify: 2023-10-09 04:26:22.401356912 +0000 2023-10-09T04:34:02.021 INFO:teuthology.orchestra.run.smithi194.stdout:Change: 2023-10-09 04:26:22.401356912 +0000 2023-10-09T04:34:02.021 INFO:teuthology.orchestra.run.smithi194.stdout: Birth: - 2023-10-09T04:34:02.022 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-09T04:34:02.077 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records in 2023-10-09T04:34:02.077 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records out 2023-10-09T04:34:02.078 INFO:teuthology.orchestra.run.smithi194.stderr:512 bytes copied, 0.000336811 s, 1.5 MB/s 2023-10-09T04:34:02.078 DEBUG:teuthology.orchestra.run.smithi194:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-09T04:34:02.134 INFO:tasks.cephadm:Deploying osd.0 on smithi133 with /dev/vg_nvme/lv_4... 2023-10-09T04:34:02.134 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-09T04:34:02.586 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:34:02 smithi194 systemd[1]: Started Ceph mgr.x for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:34:03.744 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:34:03 smithi194 bash[32713]: debug 2023-10-09T04:34:03.575+0000 7f6ba0361700 1 -- 172.21.15.194:0/3885394147 <== mon.1 v2:172.21.15.194:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5611808889c0 con 0x56118098ec00 2023-10-09T04:34:03.744 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:03 smithi194 bash[31648]: cluster 2023-10-09T04:34:01.640690+0000 mgr.y (mgr.14152) 80 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:03.744 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:03 smithi194 bash[31648]: audit 2023-10-09T04:34:02.561254+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:03.745 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:03 smithi194 bash[31648]: audit 2023-10-09T04:34:02.566812+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:03.745 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:03 smithi194 bash[31648]: audit 2023-10-09T04:34:02.617309+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:03.745 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:03 smithi194 bash[31648]: audit 2023-10-09T04:34:02.623200+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:03.745 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:03 smithi194 bash[31648]: audit 2023-10-09T04:34:02.689985+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:34:04.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:03 smithi133 bash[27558]: cluster 2023-10-09T04:34:01.640690+0000 mgr.y (mgr.14152) 80 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:04.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:03 smithi133 bash[27558]: audit 2023-10-09T04:34:02.561254+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:04.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:03 smithi133 bash[27558]: audit 2023-10-09T04:34:02.566812+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:04.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:03 smithi133 bash[27558]: audit 2023-10-09T04:34:02.617309+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:04.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:03 smithi133 bash[27558]: audit 2023-10-09T04:34:02.623200+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:04.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:03 smithi133 bash[27558]: audit 2023-10-09T04:34:02.689985+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:34:04.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:03 smithi133 bash[41291]: cluster 2023-10-09T04:34:01.640690+0000 mgr.y (mgr.14152) 80 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:04.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:03 smithi133 bash[41291]: audit 2023-10-09T04:34:02.561254+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:04.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:03 smithi133 bash[41291]: audit 2023-10-09T04:34:02.566812+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:04.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:03 smithi133 bash[41291]: audit 2023-10-09T04:34:02.617309+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:04.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:03 smithi133 bash[41291]: audit 2023-10-09T04:34:02.623200+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:04.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:03 smithi133 bash[41291]: audit 2023-10-09T04:34:02.689985+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:34:04.136 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:34:03 smithi194 bash[32713]: debug 2023-10-09T04:34:03.739+0000 7f6ba8a6f1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:34:04.886 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:34:04 smithi194 bash[32713]: debug 2023-10-09T04:34:04.627+0000 7f6ba8a6f1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:34:04.886 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:34:04 smithi194 bash[32713]: debug 2023-10-09T04:34:04.727+0000 7f6ba8a6f1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:34:04.886 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:34:04 smithi194 bash[32713]: debug 2023-10-09T04:34:04.827+0000 7f6ba8a6f1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:34:05.386 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:34:05 smithi194 bash[32713]: debug 2023-10-09T04:34:05.087+0000 7f6ba8a6f1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:34:05.803 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:05 smithi194 bash[31648]: cluster 2023-10-09T04:34:03.641122+0000 mgr.y (mgr.14152) 81 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:05.804 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:34:05 smithi194 bash[32713]: debug 2023-10-09T04:34:05.431+0000 7f6ba8a6f1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:34:05.804 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:34:05 smithi194 bash[32713]: debug 2023-10-09T04:34:05.527+0000 7f6ba8a6f1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:34:06.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:05 smithi133 bash[27558]: cluster 2023-10-09T04:34:03.641122+0000 mgr.y (mgr.14152) 81 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:06.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:05 smithi133 bash[41291]: cluster 2023-10-09T04:34:03.641122+0000 mgr.y (mgr.14152) 81 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:06.136 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:34:05 smithi194 bash[32713]: debug 2023-10-09T04:34:05.799+0000 7f6ba8a6f1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:34:06.867 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:07.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:06 smithi194 bash[31648]: cluster 2023-10-09T04:34:05.641559+0000 mgr.y (mgr.14152) 82 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:07.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:06 smithi194 bash[31648]: audit 2023-10-09T04:34:05.808065+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:07.136 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:34:06 smithi194 bash[32713]: debug 2023-10-09T04:34:06.787+0000 7f6ba8a6f1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:34:07.136 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:34:06 smithi194 bash[32713]: debug 2023-10-09T04:34:06.903+0000 7f6ba8a6f1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:34:07.136 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:34:07 smithi194 bash[32713]: debug 2023-10-09T04:34:07.015+0000 7f6ba8a6f1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:34:07.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:06 smithi133 bash[27558]: cluster 2023-10-09T04:34:05.641559+0000 mgr.y (mgr.14152) 82 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:07.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:06 smithi133 bash[27558]: audit 2023-10-09T04:34:05.808065+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:07.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:06 smithi133 bash[41291]: cluster 2023-10-09T04:34:05.641559+0000 mgr.y (mgr.14152) 82 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:07.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:06 smithi133 bash[41291]: audit 2023-10-09T04:34:05.808065+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:07.886 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:34:07 smithi194 bash[32713]: debug 2023-10-09T04:34:07.479+0000 7f6ba8a6f1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:34:07.886 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:34:07 smithi194 bash[32713]: debug 2023-10-09T04:34:07.711+0000 7f6ba8a6f1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:34:08.386 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:34:08 smithi194 bash[32713]: debug 2023-10-09T04:34:08.095+0000 7f6ba8a6f1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:34:08.750 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:08 smithi133 bash[27558]: cluster 2023-10-09T04:34:07.641926+0000 mgr.y (mgr.14152) 83 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:08.751 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:08 smithi133 bash[27558]: audit 2023-10-09T04:34:07.727433+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:08.886 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:34:08 smithi194 bash[32713]: debug 2023-10-09T04:34:08.615+0000 7f6ba8a6f1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:34:08.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:08 smithi194 bash[31648]: cluster 2023-10-09T04:34:07.641926+0000 mgr.y (mgr.14152) 83 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:08.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:08 smithi194 bash[31648]: audit 2023-10-09T04:34:07.727433+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:08.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:08 smithi194 bash[31648]: audit 2023-10-09T04:34:07.732588+0000 mon.a (mon.0) 232 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:08.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:08 smithi194 bash[31648]: audit 2023-10-09T04:34:08.149694+0000 mon.a (mon.0) 233 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:08.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:08 smithi194 bash[31648]: audit 2023-10-09T04:34:08.150915+0000 mon.a (mon.0) 234 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:34:08.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:08 smithi194 bash[31648]: audit 2023-10-09T04:34:08.157122+0000 mon.a (mon.0) 235 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:08.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:08 smithi194 bash[31648]: cephadm 2023-10-09T04:34:08.222854+0000 mgr.y (mgr.14152) 84 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-10-09T04:34:08.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:08 smithi194 bash[31648]: audit 2023-10-09T04:34:08.223463+0000 mon.a (mon.0) 236 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:34:08.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:08 smithi194 bash[31648]: audit 2023-10-09T04:34:08.225193+0000 mon.a (mon.0) 237 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:34:08.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:08 smithi194 bash[31648]: audit 2023-10-09T04:34:08.226571+0000 mon.a (mon.0) 238 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:08.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:08 smithi194 bash[31648]: cephadm 2023-10-09T04:34:08.228496+0000 mgr.y (mgr.14152) 85 : cephadm [INF] Reconfiguring daemon mgr.y on smithi133 2023-10-09T04:34:09.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:08 smithi133 bash[27558]: audit 2023-10-09T04:34:07.732588+0000 mon.a (mon.0) 232 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:09.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:08 smithi133 bash[27558]: audit 2023-10-09T04:34:08.149694+0000 mon.a (mon.0) 233 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:09.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:08 smithi133 bash[27558]: audit 2023-10-09T04:34:08.150915+0000 mon.a (mon.0) 234 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:34:09.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:08 smithi133 bash[27558]: audit 2023-10-09T04:34:08.157122+0000 mon.a (mon.0) 235 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:09.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:08 smithi133 bash[27558]: cephadm 2023-10-09T04:34:08.222854+0000 mgr.y (mgr.14152) 84 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-10-09T04:34:09.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:08 smithi133 bash[27558]: audit 2023-10-09T04:34:08.223463+0000 mon.a (mon.0) 236 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:34:09.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:08 smithi133 bash[27558]: audit 2023-10-09T04:34:08.225193+0000 mon.a (mon.0) 237 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:34:09.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:08 smithi133 bash[27558]: audit 2023-10-09T04:34:08.226571+0000 mon.a (mon.0) 238 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:09.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:08 smithi133 bash[27558]: cephadm 2023-10-09T04:34:08.228496+0000 mgr.y (mgr.14152) 85 : cephadm [INF] Reconfiguring daemon mgr.y on smithi133 2023-10-09T04:34:09.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:08 smithi133 bash[41291]: cluster 2023-10-09T04:34:07.641926+0000 mgr.y (mgr.14152) 83 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:09.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:08 smithi133 bash[41291]: audit 2023-10-09T04:34:07.727433+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:09.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:08 smithi133 bash[41291]: audit 2023-10-09T04:34:07.732588+0000 mon.a (mon.0) 232 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:09.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:08 smithi133 bash[41291]: audit 2023-10-09T04:34:08.149694+0000 mon.a (mon.0) 233 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:09.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:08 smithi133 bash[41291]: audit 2023-10-09T04:34:08.150915+0000 mon.a (mon.0) 234 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:34:09.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:08 smithi133 bash[41291]: audit 2023-10-09T04:34:08.157122+0000 mon.a (mon.0) 235 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:09.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:08 smithi133 bash[41291]: cephadm 2023-10-09T04:34:08.222854+0000 mgr.y (mgr.14152) 84 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-10-09T04:34:09.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:08 smithi133 bash[41291]: audit 2023-10-09T04:34:08.223463+0000 mon.a (mon.0) 236 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:34:09.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:08 smithi133 bash[41291]: audit 2023-10-09T04:34:08.225193+0000 mon.a (mon.0) 237 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:34:09.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:08 smithi133 bash[41291]: audit 2023-10-09T04:34:08.226571+0000 mon.a (mon.0) 238 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:09.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:08 smithi133 bash[41291]: cephadm 2023-10-09T04:34:08.228496+0000 mgr.y (mgr.14152) 85 : cephadm [INF] Reconfiguring daemon mgr.y on smithi133 2023-10-09T04:34:09.438 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:34:09 smithi194 bash[32713]: debug 2023-10-09T04:34:09.163+0000 7f6ba8a6f1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:34:09.439 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:34:09 smithi194 bash[32713]: debug 2023-10-09T04:34:09.271+0000 7f6ba8a6f1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:34:09.738 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:34:09 smithi194 bash[32713]: debug 2023-10-09T04:34:09.431+0000 7f6ba8a6f1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:34:10.110 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:34:09 smithi194 bash[32713]: debug 2023-10-09T04:34:09.731+0000 7f6ba8a6f1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:34:10.110 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:34:09 smithi194 bash[32713]: debug 2023-10-09T04:34:09.875+0000 7f6ba8a6f1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:34:10.110 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:34:09 smithi194 bash[32713]: debug 2023-10-09T04:34:09.983+0000 7f6ba8a6f1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:34:10.386 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:34:10 smithi194 bash[32713]: debug 2023-10-09T04:34:10.103+0000 7f6ba8a6f1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:34:10.386 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:34:10 smithi194 bash[32713]: debug 2023-10-09T04:34:10.291+0000 7f6ba8a6f1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:34:10.886 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:34:10 smithi194 bash[32713]: debug 2023-10-09T04:34:10.411+0000 7f6ba8a6f1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:34:11.095 INFO:teuthology.orchestra.run.smithi133.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-09T04:34:11.095 INFO:teuthology.orchestra.run.smithi133.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-09T04:34:11.095 INFO:teuthology.orchestra.run.smithi133.stderr: stderr: 10+0 records in 2023-10-09T04:34:11.096 INFO:teuthology.orchestra.run.smithi133.stderr:10+0 records out 2023-10-09T04:34:11.096 INFO:teuthology.orchestra.run.smithi133.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0158235 s, 663 MB/s 2023-10-09T04:34:11.096 INFO:teuthology.orchestra.run.smithi133.stderr:--> Zapping successful for: 2023-10-09T04:34:11.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:11 smithi133 bash[27558]: cluster 2023-10-09T04:34:09.642311+0000 mgr.y (mgr.14152) 86 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:11.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:11 smithi133 bash[27558]: audit 2023-10-09T04:34:11.025952+0000 mon.a (mon.0) 239 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:11.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:11 smithi133 bash[27558]: audit 2023-10-09T04:34:11.034426+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:11.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:11 smithi133 bash[27558]: audit 2023-10-09T04:34:11.049804+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:34:11.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:11 smithi133 bash[27558]: audit 2023-10-09T04:34:11.052214+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:11.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:11 smithi133 bash[27558]: audit 2023-10-09T04:34:11.053583+0000 mon.a (mon.0) 243 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:34:11.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:11 smithi133 bash[27558]: audit 2023-10-09T04:34:11.062768+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:11.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:11 smithi133 bash[41291]: cluster 2023-10-09T04:34:09.642311+0000 mgr.y (mgr.14152) 86 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:11.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:11 smithi133 bash[41291]: audit 2023-10-09T04:34:11.025952+0000 mon.a (mon.0) 239 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:11.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:11 smithi133 bash[41291]: audit 2023-10-09T04:34:11.034426+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:11.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:11 smithi133 bash[41291]: audit 2023-10-09T04:34:11.049804+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:34:11.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:11 smithi133 bash[41291]: audit 2023-10-09T04:34:11.052214+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:11.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:11 smithi133 bash[41291]: audit 2023-10-09T04:34:11.053583+0000 mon.a (mon.0) 243 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:34:11.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:11 smithi133 bash[41291]: audit 2023-10-09T04:34:11.062768+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:11.632 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:11 smithi194 bash[31648]: cluster 2023-10-09T04:34:09.642311+0000 mgr.y (mgr.14152) 86 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:11.632 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:11 smithi194 bash[31648]: audit 2023-10-09T04:34:11.025952+0000 mon.a (mon.0) 239 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:11.633 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:11 smithi194 bash[31648]: audit 2023-10-09T04:34:11.034426+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:11.633 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:11 smithi194 bash[31648]: audit 2023-10-09T04:34:11.049804+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:34:11.633 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:11 smithi194 bash[31648]: audit 2023-10-09T04:34:11.052214+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:11.633 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:11 smithi194 bash[31648]: audit 2023-10-09T04:34:11.053583+0000 mon.a (mon.0) 243 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:34:11.633 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:11 smithi194 bash[31648]: audit 2023-10-09T04:34:11.062768+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:11.771 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi133:vg_nvme/lv_4 2023-10-09T04:34:11.886 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:34:11 smithi194 bash[32713]: debug 2023-10-09T04:34:11.627+0000 7f6ba8a6f1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:34:12.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:12 smithi133 bash[27558]: cluster 2023-10-09T04:34:11.654046+0000 mon.a (mon.0) 245 : cluster [DBG] Standby manager daemon x started 2023-10-09T04:34:12.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:12 smithi133 bash[27558]: audit 2023-10-09T04:34:11.655641+0000 mon.b (mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.194:0/3353069958' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:34:12.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:12 smithi133 bash[27558]: audit 2023-10-09T04:34:11.656457+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.194:0/3353069958' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:34:12.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:12 smithi133 bash[27558]: audit 2023-10-09T04:34:11.658389+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.194:0/3353069958' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:34:12.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:12 smithi133 bash[27558]: audit 2023-10-09T04:34:11.658973+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.194:0/3353069958' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:34:12.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:12 smithi133 bash[41291]: cluster 2023-10-09T04:34:11.654046+0000 mon.a (mon.0) 245 : cluster [DBG] Standby manager daemon x started 2023-10-09T04:34:12.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:12 smithi133 bash[41291]: audit 2023-10-09T04:34:11.655641+0000 mon.b (mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.194:0/3353069958' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:34:12.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:12 smithi133 bash[41291]: audit 2023-10-09T04:34:11.656457+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.194:0/3353069958' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:34:12.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:12 smithi133 bash[41291]: audit 2023-10-09T04:34:11.658389+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.194:0/3353069958' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:34:12.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:12 smithi133 bash[41291]: audit 2023-10-09T04:34:11.658973+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.194:0/3353069958' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:34:12.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:12 smithi194 bash[31648]: cluster 2023-10-09T04:34:11.654046+0000 mon.a (mon.0) 245 : cluster [DBG] Standby manager daemon x started 2023-10-09T04:34:12.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:12 smithi194 bash[31648]: audit 2023-10-09T04:34:11.655641+0000 mon.b (mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.194:0/3353069958' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:34:12.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:12 smithi194 bash[31648]: audit 2023-10-09T04:34:11.656457+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.194:0/3353069958' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:34:12.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:12 smithi194 bash[31648]: audit 2023-10-09T04:34:11.658389+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.194:0/3353069958' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:34:12.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:12 smithi194 bash[31648]: audit 2023-10-09T04:34:11.658973+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.194:0/3353069958' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:34:13.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:13 smithi133 bash[27558]: cluster 2023-10-09T04:34:11.642708+0000 mgr.y (mgr.14152) 87 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:13.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:13 smithi133 bash[27558]: cluster 2023-10-09T04:34:12.277370+0000 mon.a (mon.0) 246 : cluster [DBG] mgrmap e14: y(active, since 114s), standbys: x 2023-10-09T04:34:13.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:13 smithi133 bash[27558]: audit 2023-10-09T04:34:12.277523+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:34:13.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:13 smithi133 bash[41291]: cluster 2023-10-09T04:34:11.642708+0000 mgr.y (mgr.14152) 87 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:13.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:13 smithi133 bash[41291]: cluster 2023-10-09T04:34:12.277370+0000 mon.a (mon.0) 246 : cluster [DBG] mgrmap e14: y(active, since 114s), standbys: x 2023-10-09T04:34:13.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:13 smithi133 bash[41291]: audit 2023-10-09T04:34:12.277523+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:34:13.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:13 smithi194 bash[31648]: cluster 2023-10-09T04:34:11.642708+0000 mgr.y (mgr.14152) 87 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:13.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:13 smithi194 bash[31648]: cluster 2023-10-09T04:34:12.277370+0000 mon.a (mon.0) 246 : cluster [DBG] mgrmap e14: y(active, since 114s), standbys: x 2023-10-09T04:34:13.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:13 smithi194 bash[31648]: audit 2023-10-09T04:34:12.277523+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:34:15.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:15 smithi133 bash[27558]: cluster 2023-10-09T04:34:13.643126+0000 mgr.y (mgr.14152) 88 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:15.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:15 smithi133 bash[41291]: cluster 2023-10-09T04:34:13.643126+0000 mgr.y (mgr.14152) 88 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:15.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:15 smithi194 bash[31648]: cluster 2023-10-09T04:34:13.643126+0000 mgr.y (mgr.14152) 88 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:16.501 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:17.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:17 smithi133 bash[27558]: cluster 2023-10-09T04:34:15.643655+0000 mgr.y (mgr.14152) 89 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:17.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:17 smithi133 bash[41291]: cluster 2023-10-09T04:34:15.643655+0000 mgr.y (mgr.14152) 89 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:17.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:17 smithi194 bash[31648]: cluster 2023-10-09T04:34:15.643655+0000 mgr.y (mgr.14152) 89 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:19.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:19 smithi133 bash[27558]: cluster 2023-10-09T04:34:17.644110+0000 mgr.y (mgr.14152) 90 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:19.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:19 smithi133 bash[41291]: cluster 2023-10-09T04:34:17.644110+0000 mgr.y (mgr.14152) 90 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:19.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:19 smithi194 bash[31648]: cluster 2023-10-09T04:34:17.644110+0000 mgr.y (mgr.14152) 90 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:20.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:20 smithi133 bash[27558]: audit 2023-10-09T04:34:19.566155+0000 mgr.y (mgr.14152) 91 : audit [DBG] from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi133:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:34:20.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:20 smithi133 bash[27558]: audit 2023-10-09T04:34:19.573110+0000 mon.a (mon.0) 248 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:34:20.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:20 smithi133 bash[27558]: audit 2023-10-09T04:34:19.577585+0000 mon.a (mon.0) 249 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:34:20.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:20 smithi133 bash[27558]: audit 2023-10-09T04:34:19.579124+0000 mon.a (mon.0) 250 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:20.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:20 smithi133 bash[41291]: audit 2023-10-09T04:34:19.566155+0000 mgr.y (mgr.14152) 91 : audit [DBG] from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi133:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:34:20.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:20 smithi133 bash[41291]: audit 2023-10-09T04:34:19.573110+0000 mon.a (mon.0) 248 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:34:20.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:20 smithi133 bash[41291]: audit 2023-10-09T04:34:19.577585+0000 mon.a (mon.0) 249 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:34:20.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:20 smithi133 bash[41291]: audit 2023-10-09T04:34:19.579124+0000 mon.a (mon.0) 250 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:20.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:20 smithi194 bash[31648]: audit 2023-10-09T04:34:19.566155+0000 mgr.y (mgr.14152) 91 : audit [DBG] from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi133:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:34:20.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:20 smithi194 bash[31648]: audit 2023-10-09T04:34:19.573110+0000 mon.a (mon.0) 248 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:34:20.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:20 smithi194 bash[31648]: audit 2023-10-09T04:34:19.577585+0000 mon.a (mon.0) 249 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:34:20.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:20 smithi194 bash[31648]: audit 2023-10-09T04:34:19.579124+0000 mon.a (mon.0) 250 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:21.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:21 smithi133 bash[27558]: cluster 2023-10-09T04:34:19.644532+0000 mgr.y (mgr.14152) 92 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:21.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:21 smithi133 bash[41291]: cluster 2023-10-09T04:34:19.644532+0000 mgr.y (mgr.14152) 92 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:21.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:21 smithi194 bash[31648]: cluster 2023-10-09T04:34:19.644532+0000 mgr.y (mgr.14152) 92 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:23.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:23 smithi133 bash[27558]: cluster 2023-10-09T04:34:21.644987+0000 mgr.y (mgr.14152) 93 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:23.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:23 smithi133 bash[41291]: cluster 2023-10-09T04:34:21.644987+0000 mgr.y (mgr.14152) 93 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:23.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:23 smithi194 bash[31648]: cluster 2023-10-09T04:34:21.644987+0000 mgr.y (mgr.14152) 93 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:25.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:25 smithi133 bash[27558]: cluster 2023-10-09T04:34:23.645466+0000 mgr.y (mgr.14152) 94 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:25.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:25 smithi133 bash[41291]: cluster 2023-10-09T04:34:23.645466+0000 mgr.y (mgr.14152) 94 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:25.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:25 smithi194 bash[31648]: cluster 2023-10-09T04:34:23.645466+0000 mgr.y (mgr.14152) 94 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:27.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:27 smithi194 bash[31648]: cluster 2023-10-09T04:34:25.645896+0000 mgr.y (mgr.14152) 95 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:27.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:27 smithi133 bash[27558]: cluster 2023-10-09T04:34:25.645896+0000 mgr.y (mgr.14152) 95 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:27.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:27 smithi133 bash[41291]: cluster 2023-10-09T04:34:25.645896+0000 mgr.y (mgr.14152) 95 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:28.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:28 smithi194 bash[31648]: audit 2023-10-09T04:34:27.815028+0000 mon.a (mon.0) 251 : audit [INF] from='client.? 172.21.15.133:0/884465557' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "32633ec2-9b8f-42d8-bdfb-5c9668a26071"}]: dispatch 2023-10-09T04:34:28.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:28 smithi194 bash[31648]: audit 2023-10-09T04:34:27.821098+0000 mon.a (mon.0) 252 : audit [INF] from='client.? 172.21.15.133:0/884465557' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "32633ec2-9b8f-42d8-bdfb-5c9668a26071"}]': finished 2023-10-09T04:34:28.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:28 smithi194 bash[31648]: cluster 2023-10-09T04:34:27.821204+0000 mon.a (mon.0) 253 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-10-09T04:34:28.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:28 smithi194 bash[31648]: audit 2023-10-09T04:34:27.821423+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:34:28.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:28 smithi133 bash[27558]: audit 2023-10-09T04:34:27.815028+0000 mon.a (mon.0) 251 : audit [INF] from='client.? 172.21.15.133:0/884465557' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "32633ec2-9b8f-42d8-bdfb-5c9668a26071"}]: dispatch 2023-10-09T04:34:28.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:28 smithi133 bash[27558]: audit 2023-10-09T04:34:27.821098+0000 mon.a (mon.0) 252 : audit [INF] from='client.? 172.21.15.133:0/884465557' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "32633ec2-9b8f-42d8-bdfb-5c9668a26071"}]': finished 2023-10-09T04:34:28.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:28 smithi133 bash[27558]: cluster 2023-10-09T04:34:27.821204+0000 mon.a (mon.0) 253 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-10-09T04:34:28.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:28 smithi133 bash[27558]: audit 2023-10-09T04:34:27.821423+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:34:28.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:28 smithi133 bash[41291]: audit 2023-10-09T04:34:27.815028+0000 mon.a (mon.0) 251 : audit [INF] from='client.? 172.21.15.133:0/884465557' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "32633ec2-9b8f-42d8-bdfb-5c9668a26071"}]: dispatch 2023-10-09T04:34:28.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:28 smithi133 bash[41291]: audit 2023-10-09T04:34:27.821098+0000 mon.a (mon.0) 252 : audit [INF] from='client.? 172.21.15.133:0/884465557' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "32633ec2-9b8f-42d8-bdfb-5c9668a26071"}]': finished 2023-10-09T04:34:28.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:28 smithi133 bash[41291]: cluster 2023-10-09T04:34:27.821204+0000 mon.a (mon.0) 253 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-10-09T04:34:28.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:28 smithi133 bash[41291]: audit 2023-10-09T04:34:27.821423+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:34:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:29 smithi194 bash[31648]: cluster 2023-10-09T04:34:27.646327+0000 mgr.y (mgr.14152) 96 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:29 smithi194 bash[31648]: audit 2023-10-09T04:34:28.587922+0000 mon.a (mon.0) 255 : audit [DBG] from='client.? 172.21.15.133:0/2446026630' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:34:29.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:29 smithi133 bash[27558]: cluster 2023-10-09T04:34:27.646327+0000 mgr.y (mgr.14152) 96 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:29.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:29 smithi133 bash[27558]: audit 2023-10-09T04:34:28.587922+0000 mon.a (mon.0) 255 : audit [DBG] from='client.? 172.21.15.133:0/2446026630' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:34:29.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:29 smithi133 bash[41291]: cluster 2023-10-09T04:34:27.646327+0000 mgr.y (mgr.14152) 96 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:29.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:29 smithi133 bash[41291]: audit 2023-10-09T04:34:28.587922+0000 mon.a (mon.0) 255 : audit [DBG] from='client.? 172.21.15.133:0/2446026630' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:34:31.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:31 smithi133 bash[27558]: cluster 2023-10-09T04:34:29.646698+0000 mgr.y (mgr.14152) 97 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:31.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:31 smithi133 bash[41291]: cluster 2023-10-09T04:34:29.646698+0000 mgr.y (mgr.14152) 97 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:31.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:31 smithi194 bash[31648]: cluster 2023-10-09T04:34:29.646698+0000 mgr.y (mgr.14152) 97 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:33.640 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:33 smithi194 bash[31648]: cluster 2023-10-09T04:34:31.646975+0000 mgr.y (mgr.14152) 98 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:33.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:33 smithi133 bash[27558]: cluster 2023-10-09T04:34:31.646975+0000 mgr.y (mgr.14152) 98 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:33.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:33 smithi133 bash[41291]: cluster 2023-10-09T04:34:31.646975+0000 mgr.y (mgr.14152) 98 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:35.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:35 smithi194 bash[31648]: cluster 2023-10-09T04:34:33.647408+0000 mgr.y (mgr.14152) 99 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:35.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:35 smithi133 bash[27558]: cluster 2023-10-09T04:34:33.647408+0000 mgr.y (mgr.14152) 99 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:35.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:35 smithi133 bash[41291]: cluster 2023-10-09T04:34:33.647408+0000 mgr.y (mgr.14152) 99 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:37.595 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:37 smithi133 bash[27558]: cluster 2023-10-09T04:34:35.647848+0000 mgr.y (mgr.14152) 100 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:37.595 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:37 smithi133 bash[41291]: cluster 2023-10-09T04:34:35.647848+0000 mgr.y (mgr.14152) 100 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:37.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:37 smithi194 bash[31648]: cluster 2023-10-09T04:34:35.647848+0000 mgr.y (mgr.14152) 100 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:39.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:39 smithi194 bash[31648]: cluster 2023-10-09T04:34:37.648288+0000 mgr.y (mgr.14152) 101 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:39.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:39 smithi133 bash[27558]: cluster 2023-10-09T04:34:37.648288+0000 mgr.y (mgr.14152) 101 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:39.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:39 smithi133 bash[41291]: cluster 2023-10-09T04:34:37.648288+0000 mgr.y (mgr.14152) 101 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:41.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:41 smithi194 bash[31648]: cluster 2023-10-09T04:34:39.648659+0000 mgr.y (mgr.14152) 102 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:41.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:41 smithi194 bash[31648]: audit 2023-10-09T04:34:40.664127+0000 mon.a (mon.0) 256 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:34:41.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:41 smithi194 bash[31648]: audit 2023-10-09T04:34:40.665719+0000 mon.a (mon.0) 257 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:41.667 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:41 smithi133 bash[27558]: cluster 2023-10-09T04:34:39.648659+0000 mgr.y (mgr.14152) 102 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:41.668 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:41 smithi133 bash[27558]: audit 2023-10-09T04:34:40.664127+0000 mon.a (mon.0) 256 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:34:41.668 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:41 smithi133 bash[27558]: audit 2023-10-09T04:34:40.665719+0000 mon.a (mon.0) 257 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:41.668 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:41 smithi133 bash[41291]: cluster 2023-10-09T04:34:39.648659+0000 mgr.y (mgr.14152) 102 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:41.668 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:41 smithi133 bash[41291]: audit 2023-10-09T04:34:40.664127+0000 mon.a (mon.0) 256 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:34:41.668 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:41 smithi133 bash[41291]: audit 2023-10-09T04:34:40.665719+0000 mon.a (mon.0) 257 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:42.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:42 smithi133 bash[27558]: cephadm 2023-10-09T04:34:40.667017+0000 mgr.y (mgr.14152) 103 : cephadm [INF] Deploying daemon osd.0 on smithi133 2023-10-09T04:34:42.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:42 smithi133 bash[41291]: cephadm 2023-10-09T04:34:40.667017+0000 mgr.y (mgr.14152) 103 : cephadm [INF] Deploying daemon osd.0 on smithi133 2023-10-09T04:34:42.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:42 smithi194 bash[31648]: cephadm 2023-10-09T04:34:40.667017+0000 mgr.y (mgr.14152) 103 : cephadm [INF] Deploying daemon osd.0 on smithi133 2023-10-09T04:34:43.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:43 smithi194 bash[31648]: cluster 2023-10-09T04:34:41.649105+0000 mgr.y (mgr.14152) 104 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:43.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:43 smithi133 bash[27558]: cluster 2023-10-09T04:34:41.649105+0000 mgr.y (mgr.14152) 104 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:43.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:43 smithi133 bash[41291]: cluster 2023-10-09T04:34:41.649105+0000 mgr.y (mgr.14152) 104 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:45.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:45 smithi133 bash[41291]: cluster 2023-10-09T04:34:43.649535+0000 mgr.y (mgr.14152) 105 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:45.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:45 smithi133 bash[41291]: audit 2023-10-09T04:34:45.184223+0000 mon.a (mon.0) 258 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:45.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:45 smithi133 bash[41291]: audit 2023-10-09T04:34:45.184555+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:34:45.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:45 smithi133 bash[41291]: audit 2023-10-09T04:34:45.191900+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:45.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:45 smithi133 bash[27558]: cluster 2023-10-09T04:34:43.649535+0000 mgr.y (mgr.14152) 105 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:45.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:45 smithi133 bash[27558]: audit 2023-10-09T04:34:45.184223+0000 mon.a (mon.0) 258 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:45.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:45 smithi133 bash[27558]: audit 2023-10-09T04:34:45.184555+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:34:45.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:45 smithi133 bash[27558]: audit 2023-10-09T04:34:45.191900+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:45.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:45 smithi194 bash[31648]: cluster 2023-10-09T04:34:43.649535+0000 mgr.y (mgr.14152) 105 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:45.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:45 smithi194 bash[31648]: audit 2023-10-09T04:34:45.184223+0000 mon.a (mon.0) 258 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:45.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:45 smithi194 bash[31648]: audit 2023-10-09T04:34:45.184555+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:34:45.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:45 smithi194 bash[31648]: audit 2023-10-09T04:34:45.191900+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:47.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:47 smithi194 bash[31648]: cluster 2023-10-09T04:34:45.649920+0000 mgr.y (mgr.14152) 106 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:47.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:47 smithi133 bash[27558]: cluster 2023-10-09T04:34:45.649920+0000 mgr.y (mgr.14152) 106 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:47.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:47 smithi133 bash[41291]: cluster 2023-10-09T04:34:45.649920+0000 mgr.y (mgr.14152) 106 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:49.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:49 smithi194 bash[31648]: cluster 2023-10-09T04:34:47.650288+0000 mgr.y (mgr.14152) 107 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:49.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:49 smithi133 bash[27558]: cluster 2023-10-09T04:34:47.650288+0000 mgr.y (mgr.14152) 107 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:49.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:49 smithi133 bash[41291]: cluster 2023-10-09T04:34:47.650288+0000 mgr.y (mgr.14152) 107 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:51.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:51 smithi194 bash[31648]: cluster 2023-10-09T04:34:49.650736+0000 mgr.y (mgr.14152) 108 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:51.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:51 smithi194 bash[31648]: audit 2023-10-09T04:34:51.323701+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:51.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:51 smithi194 bash[31648]: audit 2023-10-09T04:34:51.329031+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:51.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:51 smithi133 bash[27558]: cluster 2023-10-09T04:34:49.650736+0000 mgr.y (mgr.14152) 108 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:51.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:51 smithi133 bash[27558]: audit 2023-10-09T04:34:51.323701+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:51.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:51 smithi133 bash[27558]: audit 2023-10-09T04:34:51.329031+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:51.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:51 smithi133 bash[41291]: cluster 2023-10-09T04:34:49.650736+0000 mgr.y (mgr.14152) 108 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:51.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:51 smithi133 bash[41291]: audit 2023-10-09T04:34:51.323701+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:51.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:51 smithi133 bash[41291]: audit 2023-10-09T04:34:51.329031+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:52.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:52 smithi133 bash[27558]: audit 2023-10-09T04:34:51.700829+0000 mon.a (mon.0) 263 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:52.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:52 smithi133 bash[27558]: audit 2023-10-09T04:34:51.702261+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:34:52.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:52 smithi133 bash[27558]: audit 2023-10-09T04:34:51.710562+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:52.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:52 smithi133 bash[27558]: audit 2023-10-09T04:34:52.049176+0000 mon.c (mon.2) 2 : audit [INF] from='osd.0 [v2:172.21.15.133:6802/1933306305,v1:172.21.15.133:6803/1933306305]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T04:34:52.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:52 smithi133 bash[27558]: audit 2023-10-09T04:34:52.049875+0000 mon.a (mon.0) 266 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T04:34:52.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:52 smithi133 bash[41291]: audit 2023-10-09T04:34:51.700829+0000 mon.a (mon.0) 263 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:52.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:52 smithi133 bash[41291]: audit 2023-10-09T04:34:51.702261+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:34:52.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:52 smithi133 bash[41291]: audit 2023-10-09T04:34:51.710562+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:52.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:52 smithi133 bash[41291]: audit 2023-10-09T04:34:52.049176+0000 mon.c (mon.2) 2 : audit [INF] from='osd.0 [v2:172.21.15.133:6802/1933306305,v1:172.21.15.133:6803/1933306305]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T04:34:52.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:52 smithi133 bash[41291]: audit 2023-10-09T04:34:52.049875+0000 mon.a (mon.0) 266 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T04:34:52.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:52 smithi194 bash[31648]: audit 2023-10-09T04:34:51.700829+0000 mon.a (mon.0) 263 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:52.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:52 smithi194 bash[31648]: audit 2023-10-09T04:34:51.702261+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:34:52.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:52 smithi194 bash[31648]: audit 2023-10-09T04:34:51.710562+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:52.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:52 smithi194 bash[31648]: audit 2023-10-09T04:34:52.049176+0000 mon.c (mon.2) 2 : audit [INF] from='osd.0 [v2:172.21.15.133:6802/1933306305,v1:172.21.15.133:6803/1933306305]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T04:34:52.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:52 smithi194 bash[31648]: audit 2023-10-09T04:34:52.049875+0000 mon.a (mon.0) 266 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T04:34:53.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:53 smithi194 bash[31648]: cluster 2023-10-09T04:34:51.651072+0000 mgr.y (mgr.14152) 109 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:53.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:53 smithi194 bash[31648]: audit 2023-10-09T04:34:52.711873+0000 mon.a (mon.0) 267 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T04:34:53.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:53 smithi194 bash[31648]: cluster 2023-10-09T04:34:52.711951+0000 mon.a (mon.0) 268 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-10-09T04:34:53.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:53 smithi194 bash[31648]: audit 2023-10-09T04:34:52.712193+0000 mon.a (mon.0) 269 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:34:53.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:53 smithi194 bash[31648]: audit 2023-10-09T04:34:52.715126+0000 mon.c (mon.2) 3 : audit [INF] from='osd.0 [v2:172.21.15.133:6802/1933306305,v1:172.21.15.133:6803/1933306305]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:34:53.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:53 smithi194 bash[31648]: audit 2023-10-09T04:34:52.715863+0000 mon.a (mon.0) 270 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:34:53.702 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:53 smithi133 bash[27558]: cluster 2023-10-09T04:34:51.651072+0000 mgr.y (mgr.14152) 109 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:53.702 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:53 smithi133 bash[27558]: audit 2023-10-09T04:34:52.711873+0000 mon.a (mon.0) 267 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T04:34:53.702 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:53 smithi133 bash[27558]: cluster 2023-10-09T04:34:52.711951+0000 mon.a (mon.0) 268 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-10-09T04:34:53.702 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:53 smithi133 bash[27558]: audit 2023-10-09T04:34:52.712193+0000 mon.a (mon.0) 269 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:34:53.702 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:53 smithi133 bash[27558]: audit 2023-10-09T04:34:52.715126+0000 mon.c (mon.2) 3 : audit [INF] from='osd.0 [v2:172.21.15.133:6802/1933306305,v1:172.21.15.133:6803/1933306305]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:34:53.702 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:53 smithi133 bash[27558]: audit 2023-10-09T04:34:52.715863+0000 mon.a (mon.0) 270 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:34:53.702 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:53 smithi133 bash[41291]: cluster 2023-10-09T04:34:51.651072+0000 mgr.y (mgr.14152) 109 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:53.702 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:53 smithi133 bash[41291]: audit 2023-10-09T04:34:52.711873+0000 mon.a (mon.0) 267 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T04:34:53.703 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:53 smithi133 bash[41291]: cluster 2023-10-09T04:34:52.711951+0000 mon.a (mon.0) 268 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-10-09T04:34:53.703 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:53 smithi133 bash[41291]: audit 2023-10-09T04:34:52.712193+0000 mon.a (mon.0) 269 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:34:53.703 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:53 smithi133 bash[41291]: audit 2023-10-09T04:34:52.715126+0000 mon.c (mon.2) 3 : audit [INF] from='osd.0 [v2:172.21.15.133:6802/1933306305,v1:172.21.15.133:6803/1933306305]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:34:53.703 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:53 smithi133 bash[41291]: audit 2023-10-09T04:34:52.715863+0000 mon.a (mon.0) 270 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:34:54.886 INFO:teuthology.orchestra.run.smithi133.stdout:Created osd(s) 0 on host 'smithi133' 2023-10-09T04:34:55.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:54 smithi133 bash[27558]: cluster 2023-10-09T04:34:53.651470+0000 mgr.y (mgr.14152) 110 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:55.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:54 smithi133 bash[27558]: audit 2023-10-09T04:34:53.715948+0000 mon.a (mon.0) 271 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]': finished 2023-10-09T04:34:55.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:54 smithi133 bash[27558]: cluster 2023-10-09T04:34:53.716077+0000 mon.a (mon.0) 272 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-10-09T04:34:55.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:54 smithi133 bash[27558]: audit 2023-10-09T04:34:53.716244+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:34:55.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:54 smithi133 bash[27558]: audit 2023-10-09T04:34:53.723088+0000 mon.a (mon.0) 274 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:34:55.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:54 smithi133 bash[41291]: cluster 2023-10-09T04:34:53.651470+0000 mgr.y (mgr.14152) 110 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:55.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:54 smithi133 bash[41291]: audit 2023-10-09T04:34:53.715948+0000 mon.a (mon.0) 271 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]': finished 2023-10-09T04:34:55.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:54 smithi133 bash[41291]: cluster 2023-10-09T04:34:53.716077+0000 mon.a (mon.0) 272 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-10-09T04:34:55.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:54 smithi133 bash[41291]: audit 2023-10-09T04:34:53.716244+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:34:55.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:54 smithi133 bash[41291]: audit 2023-10-09T04:34:53.723088+0000 mon.a (mon.0) 274 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:34:55.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:54 smithi194 bash[31648]: cluster 2023-10-09T04:34:53.651470+0000 mgr.y (mgr.14152) 110 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:34:55.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:54 smithi194 bash[31648]: audit 2023-10-09T04:34:53.715948+0000 mon.a (mon.0) 271 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]': finished 2023-10-09T04:34:55.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:54 smithi194 bash[31648]: cluster 2023-10-09T04:34:53.716077+0000 mon.a (mon.0) 272 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-10-09T04:34:55.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:54 smithi194 bash[31648]: audit 2023-10-09T04:34:53.716244+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:34:55.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:54 smithi194 bash[31648]: audit 2023-10-09T04:34:53.723088+0000 mon.a (mon.0) 274 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:34:55.647 DEBUG:teuthology.orchestra.run.smithi133:osd.0> sudo journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@osd.0.service 2023-10-09T04:34:55.650 INFO:tasks.cephadm:Deploying osd.1 on smithi133 with /dev/vg_nvme/lv_3... 2023-10-09T04:34:55.650 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-09T04:34:55.923 INFO:journalctl@ceph.osd.0.smithi133.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:34:56.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:55 smithi133 bash[41291]: cluster 2023-10-09T04:34:53.028502+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:34:56.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:55 smithi133 bash[41291]: cluster 2023-10-09T04:34:53.028744+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:34:56.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:55 smithi133 bash[41291]: cluster 2023-10-09T04:34:54.719679+0000 mon.a (mon.0) 275 : cluster [INF] osd.0 [v2:172.21.15.133:6802/1933306305,v1:172.21.15.133:6803/1933306305] boot 2023-10-09T04:34:56.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:55 smithi133 bash[41291]: cluster 2023-10-09T04:34:54.719796+0000 mon.a (mon.0) 276 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-10-09T04:34:56.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:55 smithi133 bash[41291]: audit 2023-10-09T04:34:54.721379+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:34:56.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:55 smithi133 bash[41291]: audit 2023-10-09T04:34:54.853295+0000 mon.a (mon.0) 278 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:34:56.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:55 smithi133 bash[41291]: audit 2023-10-09T04:34:54.858328+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:56.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:55 smithi133 bash[41291]: audit 2023-10-09T04:34:54.864568+0000 mon.a (mon.0) 280 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:56.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:55 smithi133 bash[27558]: cluster 2023-10-09T04:34:53.028502+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:34:56.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:55 smithi133 bash[27558]: cluster 2023-10-09T04:34:53.028744+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:34:56.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:55 smithi133 bash[27558]: cluster 2023-10-09T04:34:54.719679+0000 mon.a (mon.0) 275 : cluster [INF] osd.0 [v2:172.21.15.133:6802/1933306305,v1:172.21.15.133:6803/1933306305] boot 2023-10-09T04:34:56.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:55 smithi133 bash[27558]: cluster 2023-10-09T04:34:54.719796+0000 mon.a (mon.0) 276 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-10-09T04:34:56.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:55 smithi133 bash[27558]: audit 2023-10-09T04:34:54.721379+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:34:56.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:55 smithi133 bash[27558]: audit 2023-10-09T04:34:54.853295+0000 mon.a (mon.0) 278 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:34:56.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:55 smithi133 bash[27558]: audit 2023-10-09T04:34:54.858328+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:56.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:55 smithi133 bash[27558]: audit 2023-10-09T04:34:54.864568+0000 mon.a (mon.0) 280 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:56.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:55 smithi194 bash[31648]: cluster 2023-10-09T04:34:53.028502+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:34:56.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:55 smithi194 bash[31648]: cluster 2023-10-09T04:34:53.028744+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:34:56.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:55 smithi194 bash[31648]: cluster 2023-10-09T04:34:54.719679+0000 mon.a (mon.0) 275 : cluster [INF] osd.0 [v2:172.21.15.133:6802/1933306305,v1:172.21.15.133:6803/1933306305] boot 2023-10-09T04:34:56.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:55 smithi194 bash[31648]: cluster 2023-10-09T04:34:54.719796+0000 mon.a (mon.0) 276 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-10-09T04:34:56.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:55 smithi194 bash[31648]: audit 2023-10-09T04:34:54.721379+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:34:56.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:55 smithi194 bash[31648]: audit 2023-10-09T04:34:54.853295+0000 mon.a (mon.0) 278 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:34:56.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:55 smithi194 bash[31648]: audit 2023-10-09T04:34:54.858328+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:56.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:55 smithi194 bash[31648]: audit 2023-10-09T04:34:54.864568+0000 mon.a (mon.0) 280 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:34:57.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:56 smithi194 bash[31648]: cluster 2023-10-09T04:34:55.652038+0000 mgr.y (mgr.14152) 111 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:34:57.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:56 smithi194 bash[31648]: cluster 2023-10-09T04:34:55.866157+0000 mon.a (mon.0) 281 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-10-09T04:34:57.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:56 smithi133 bash[27558]: cluster 2023-10-09T04:34:55.652038+0000 mgr.y (mgr.14152) 111 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:34:57.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:56 smithi133 bash[27558]: cluster 2023-10-09T04:34:55.866157+0000 mon.a (mon.0) 281 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-10-09T04:34:57.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:56 smithi133 bash[41291]: cluster 2023-10-09T04:34:55.652038+0000 mgr.y (mgr.14152) 111 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:34:57.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:56 smithi133 bash[41291]: cluster 2023-10-09T04:34:55.866157+0000 mon.a (mon.0) 281 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-10-09T04:34:59.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:34:58 smithi194 bash[31648]: cluster 2023-10-09T04:34:57.652490+0000 mgr.y (mgr.14152) 112 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:34:59.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:34:58 smithi133 bash[27558]: cluster 2023-10-09T04:34:57.652490+0000 mgr.y (mgr.14152) 112 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:34:59.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:34:58 smithi133 bash[41291]: cluster 2023-10-09T04:34:57.652490+0000 mgr.y (mgr.14152) 112 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:34:59.629 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:01.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:01 smithi133 bash[27558]: cluster 2023-10-09T04:34:59.652918+0000 mgr.y (mgr.14152) 113 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:01.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:01 smithi133 bash[41291]: cluster 2023-10-09T04:34:59.652918+0000 mgr.y (mgr.14152) 113 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:01.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:01 smithi194 bash[31648]: cluster 2023-10-09T04:34:59.652918+0000 mgr.y (mgr.14152) 113 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:03.530 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:03 smithi133 bash[27558]: cluster 2023-10-09T04:35:01.653467+0000 mgr.y (mgr.14152) 114 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:03.531 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:03 smithi133 bash[41291]: cluster 2023-10-09T04:35:01.653467+0000 mgr.y (mgr.14152) 114 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:03.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:03 smithi194 bash[31648]: cluster 2023-10-09T04:35:01.653467+0000 mgr.y (mgr.14152) 114 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:04.260 INFO:teuthology.orchestra.run.smithi133.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-09T04:35:04.303 INFO:teuthology.orchestra.run.smithi133.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-09T04:35:04.304 INFO:teuthology.orchestra.run.smithi133.stderr: stderr: 10+0 records in 2023-10-09T04:35:04.304 INFO:teuthology.orchestra.run.smithi133.stderr:10+0 records out 2023-10-09T04:35:04.304 INFO:teuthology.orchestra.run.smithi133.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.014447 s, 726 MB/s 2023-10-09T04:35:04.304 INFO:teuthology.orchestra.run.smithi133.stderr:--> Zapping successful for: 2023-10-09T04:35:05.141 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi133:vg_nvme/lv_3 2023-10-09T04:35:05.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:05 smithi133 bash[27558]: cluster 2023-10-09T04:35:03.653827+0000 mgr.y (mgr.14152) 115 : cluster [DBG] pgmap v82: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:05.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:05 smithi133 bash[41291]: cluster 2023-10-09T04:35:03.653827+0000 mgr.y (mgr.14152) 115 : cluster [DBG] pgmap v82: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:05.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:05 smithi194 bash[31648]: cluster 2023-10-09T04:35:03.653827+0000 mgr.y (mgr.14152) 115 : cluster [DBG] pgmap v82: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:07.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:06 smithi133 bash[27558]: cluster 2023-10-09T04:35:05.654301+0000 mgr.y (mgr.14152) 116 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:07.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:06 smithi133 bash[27558]: cephadm 2023-10-09T04:35:05.747553+0000 mgr.y (mgr.14152) 117 : cephadm [INF] Detected new or changed devices on smithi133 2023-10-09T04:35:07.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:06 smithi133 bash[27558]: audit 2023-10-09T04:35:05.755869+0000 mon.a (mon.0) 282 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:07.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:06 smithi133 bash[27558]: audit 2023-10-09T04:35:05.762710+0000 mon.a (mon.0) 283 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:07.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:06 smithi133 bash[27558]: audit 2023-10-09T04:35:05.764619+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:35:07.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:06 smithi133 bash[27558]: cephadm 2023-10-09T04:35:05.765651+0000 mgr.y (mgr.14152) 118 : cephadm [INF] Adjusting osd_memory_target on smithi133 to 16241M 2023-10-09T04:35:07.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:06 smithi133 bash[27558]: audit 2023-10-09T04:35:05.773697+0000 mon.a (mon.0) 285 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:07.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:06 smithi133 bash[27558]: audit 2023-10-09T04:35:05.775785+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:35:07.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:06 smithi133 bash[27558]: audit 2023-10-09T04:35:05.777502+0000 mon.a (mon.0) 287 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:35:07.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:06 smithi133 bash[27558]: audit 2023-10-09T04:35:05.787133+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:07.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:06 smithi133 bash[41291]: cluster 2023-10-09T04:35:05.654301+0000 mgr.y (mgr.14152) 116 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:07.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:06 smithi133 bash[41291]: cephadm 2023-10-09T04:35:05.747553+0000 mgr.y (mgr.14152) 117 : cephadm [INF] Detected new or changed devices on smithi133 2023-10-09T04:35:07.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:06 smithi133 bash[41291]: audit 2023-10-09T04:35:05.755869+0000 mon.a (mon.0) 282 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:07.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:06 smithi133 bash[41291]: audit 2023-10-09T04:35:05.762710+0000 mon.a (mon.0) 283 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:07.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:06 smithi133 bash[41291]: audit 2023-10-09T04:35:05.764619+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:35:07.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:06 smithi133 bash[41291]: cephadm 2023-10-09T04:35:05.765651+0000 mgr.y (mgr.14152) 118 : cephadm [INF] Adjusting osd_memory_target on smithi133 to 16241M 2023-10-09T04:35:07.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:06 smithi133 bash[41291]: audit 2023-10-09T04:35:05.773697+0000 mon.a (mon.0) 285 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:07.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:06 smithi133 bash[41291]: audit 2023-10-09T04:35:05.775785+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:35:07.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:06 smithi133 bash[41291]: audit 2023-10-09T04:35:05.777502+0000 mon.a (mon.0) 287 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:35:07.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:06 smithi133 bash[41291]: audit 2023-10-09T04:35:05.787133+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:07.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:06 smithi194 bash[31648]: cluster 2023-10-09T04:35:05.654301+0000 mgr.y (mgr.14152) 116 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:07.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:06 smithi194 bash[31648]: cephadm 2023-10-09T04:35:05.747553+0000 mgr.y (mgr.14152) 117 : cephadm [INF] Detected new or changed devices on smithi133 2023-10-09T04:35:07.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:06 smithi194 bash[31648]: audit 2023-10-09T04:35:05.755869+0000 mon.a (mon.0) 282 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:07.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:06 smithi194 bash[31648]: audit 2023-10-09T04:35:05.762710+0000 mon.a (mon.0) 283 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:07.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:06 smithi194 bash[31648]: audit 2023-10-09T04:35:05.764619+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:35:07.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:06 smithi194 bash[31648]: cephadm 2023-10-09T04:35:05.765651+0000 mgr.y (mgr.14152) 118 : cephadm [INF] Adjusting osd_memory_target on smithi133 to 16241M 2023-10-09T04:35:07.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:06 smithi194 bash[31648]: audit 2023-10-09T04:35:05.773697+0000 mon.a (mon.0) 285 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:07.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:06 smithi194 bash[31648]: audit 2023-10-09T04:35:05.775785+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:35:07.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:06 smithi194 bash[31648]: audit 2023-10-09T04:35:05.777502+0000 mon.a (mon.0) 287 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:35:07.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:06 smithi194 bash[31648]: audit 2023-10-09T04:35:05.787133+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:09.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:08 smithi133 bash[27558]: cluster 2023-10-09T04:35:07.654768+0000 mgr.y (mgr.14152) 119 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:09.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:08 smithi133 bash[41291]: cluster 2023-10-09T04:35:07.654768+0000 mgr.y (mgr.14152) 119 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:09.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:08 smithi194 bash[31648]: cluster 2023-10-09T04:35:07.654768+0000 mgr.y (mgr.14152) 119 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:09.892 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:11.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:11 smithi133 bash[27558]: cluster 2023-10-09T04:35:09.655240+0000 mgr.y (mgr.14152) 120 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:11.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:11 smithi133 bash[41291]: cluster 2023-10-09T04:35:09.655240+0000 mgr.y (mgr.14152) 120 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:11.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:11 smithi194 bash[31648]: cluster 2023-10-09T04:35:09.655240+0000 mgr.y (mgr.14152) 120 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:13.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:13 smithi133 bash[27558]: cluster 2023-10-09T04:35:11.655629+0000 mgr.y (mgr.14152) 121 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:13.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:13 smithi133 bash[27558]: audit 2023-10-09T04:35:13.168232+0000 mon.a (mon.0) 289 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:35:13.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:13 smithi133 bash[27558]: audit 2023-10-09T04:35:13.172934+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:35:13.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:13 smithi133 bash[27558]: audit 2023-10-09T04:35:13.174059+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:35:13.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:13 smithi133 bash[41291]: cluster 2023-10-09T04:35:11.655629+0000 mgr.y (mgr.14152) 121 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:13.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:13 smithi133 bash[41291]: audit 2023-10-09T04:35:13.168232+0000 mon.a (mon.0) 289 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:35:13.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:13 smithi133 bash[41291]: audit 2023-10-09T04:35:13.172934+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:35:13.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:13 smithi133 bash[41291]: audit 2023-10-09T04:35:13.174059+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:35:13.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:13 smithi194 bash[31648]: cluster 2023-10-09T04:35:11.655629+0000 mgr.y (mgr.14152) 121 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:13.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:13 smithi194 bash[31648]: audit 2023-10-09T04:35:13.168232+0000 mon.a (mon.0) 289 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:35:13.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:13 smithi194 bash[31648]: audit 2023-10-09T04:35:13.172934+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:35:13.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:13 smithi194 bash[31648]: audit 2023-10-09T04:35:13.174059+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:35:14.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:14 smithi133 bash[41291]: audit 2023-10-09T04:35:13.161520+0000 mgr.y (mgr.14152) 122 : audit [DBG] from='client.24149 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi133:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:35:14.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:14 smithi133 bash[27558]: audit 2023-10-09T04:35:13.161520+0000 mgr.y (mgr.14152) 122 : audit [DBG] from='client.24149 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi133:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:35:14.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:14 smithi194 bash[31648]: audit 2023-10-09T04:35:13.161520+0000 mgr.y (mgr.14152) 122 : audit [DBG] from='client.24149 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi133:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:35:15.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:15 smithi133 bash[27558]: cluster 2023-10-09T04:35:13.656157+0000 mgr.y (mgr.14152) 123 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:15.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:15 smithi133 bash[41291]: cluster 2023-10-09T04:35:13.656157+0000 mgr.y (mgr.14152) 123 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:15.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:15 smithi194 bash[31648]: cluster 2023-10-09T04:35:13.656157+0000 mgr.y (mgr.14152) 123 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:17.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:17 smithi133 bash[27558]: cluster 2023-10-09T04:35:15.656633+0000 mgr.y (mgr.14152) 124 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:17.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:17 smithi133 bash[41291]: cluster 2023-10-09T04:35:15.656633+0000 mgr.y (mgr.14152) 124 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:17.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:17 smithi194 bash[31648]: cluster 2023-10-09T04:35:15.656633+0000 mgr.y (mgr.14152) 124 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:19.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:19 smithi194 bash[31648]: cluster 2023-10-09T04:35:17.657064+0000 mgr.y (mgr.14152) 125 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:19.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:19 smithi133 bash[27558]: cluster 2023-10-09T04:35:17.657064+0000 mgr.y (mgr.14152) 125 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:19.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:19 smithi133 bash[41291]: cluster 2023-10-09T04:35:17.657064+0000 mgr.y (mgr.14152) 125 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:21.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:21 smithi194 bash[31648]: cluster 2023-10-09T04:35:19.657547+0000 mgr.y (mgr.14152) 126 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:21.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:21 smithi133 bash[27558]: cluster 2023-10-09T04:35:19.657547+0000 mgr.y (mgr.14152) 126 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:21.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:21 smithi133 bash[41291]: cluster 2023-10-09T04:35:19.657547+0000 mgr.y (mgr.14152) 126 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:22.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:22 smithi194 bash[31648]: audit 2023-10-09T04:35:21.711247+0000 mon.c (mon.2) 4 : audit [INF] from='client.? 172.21.15.133:0/552813669' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7f2a961d-a790-4337-91c7-1e45f900bbd4"}]: dispatch 2023-10-09T04:35:22.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:22 smithi194 bash[31648]: audit 2023-10-09T04:35:21.711846+0000 mon.a (mon.0) 292 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7f2a961d-a790-4337-91c7-1e45f900bbd4"}]: dispatch 2023-10-09T04:35:22.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:22 smithi194 bash[31648]: audit 2023-10-09T04:35:21.725626+0000 mon.a (mon.0) 293 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7f2a961d-a790-4337-91c7-1e45f900bbd4"}]': finished 2023-10-09T04:35:22.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:22 smithi194 bash[31648]: cluster 2023-10-09T04:35:21.725720+0000 mon.a (mon.0) 294 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-10-09T04:35:22.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:22 smithi194 bash[31648]: audit 2023-10-09T04:35:21.725985+0000 mon.a (mon.0) 295 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:35:22.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:22 smithi133 bash[27558]: audit 2023-10-09T04:35:21.711247+0000 mon.c (mon.2) 4 : audit [INF] from='client.? 172.21.15.133:0/552813669' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7f2a961d-a790-4337-91c7-1e45f900bbd4"}]: dispatch 2023-10-09T04:35:22.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:22 smithi133 bash[27558]: audit 2023-10-09T04:35:21.711846+0000 mon.a (mon.0) 292 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7f2a961d-a790-4337-91c7-1e45f900bbd4"}]: dispatch 2023-10-09T04:35:22.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:22 smithi133 bash[27558]: audit 2023-10-09T04:35:21.725626+0000 mon.a (mon.0) 293 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7f2a961d-a790-4337-91c7-1e45f900bbd4"}]': finished 2023-10-09T04:35:22.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:22 smithi133 bash[27558]: cluster 2023-10-09T04:35:21.725720+0000 mon.a (mon.0) 294 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-10-09T04:35:22.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:22 smithi133 bash[27558]: audit 2023-10-09T04:35:21.725985+0000 mon.a (mon.0) 295 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:35:22.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:22 smithi133 bash[41291]: audit 2023-10-09T04:35:21.711247+0000 mon.c (mon.2) 4 : audit [INF] from='client.? 172.21.15.133:0/552813669' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7f2a961d-a790-4337-91c7-1e45f900bbd4"}]: dispatch 2023-10-09T04:35:22.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:22 smithi133 bash[41291]: audit 2023-10-09T04:35:21.711846+0000 mon.a (mon.0) 292 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7f2a961d-a790-4337-91c7-1e45f900bbd4"}]: dispatch 2023-10-09T04:35:22.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:22 smithi133 bash[41291]: audit 2023-10-09T04:35:21.725626+0000 mon.a (mon.0) 293 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7f2a961d-a790-4337-91c7-1e45f900bbd4"}]': finished 2023-10-09T04:35:22.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:22 smithi133 bash[41291]: cluster 2023-10-09T04:35:21.725720+0000 mon.a (mon.0) 294 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-10-09T04:35:22.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:22 smithi133 bash[41291]: audit 2023-10-09T04:35:21.725985+0000 mon.a (mon.0) 295 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:35:23.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:23 smithi194 bash[31648]: cluster 2023-10-09T04:35:21.657875+0000 mgr.y (mgr.14152) 127 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:23.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:23 smithi194 bash[31648]: audit 2023-10-09T04:35:22.448651+0000 mon.a (mon.0) 296 : audit [DBG] from='client.? 172.21.15.133:0/2884731942' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:35:23.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:23 smithi133 bash[27558]: cluster 2023-10-09T04:35:21.657875+0000 mgr.y (mgr.14152) 127 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:23.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:23 smithi133 bash[27558]: audit 2023-10-09T04:35:22.448651+0000 mon.a (mon.0) 296 : audit [DBG] from='client.? 172.21.15.133:0/2884731942' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:35:23.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:23 smithi133 bash[41291]: cluster 2023-10-09T04:35:21.657875+0000 mgr.y (mgr.14152) 127 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:23.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:23 smithi133 bash[41291]: audit 2023-10-09T04:35:22.448651+0000 mon.a (mon.0) 296 : audit [DBG] from='client.? 172.21.15.133:0/2884731942' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:35:25.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:25 smithi133 bash[27558]: cluster 2023-10-09T04:35:23.658293+0000 mgr.y (mgr.14152) 128 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:25.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:25 smithi133 bash[41291]: cluster 2023-10-09T04:35:23.658293+0000 mgr.y (mgr.14152) 128 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:25.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:25 smithi194 bash[31648]: cluster 2023-10-09T04:35:23.658293+0000 mgr.y (mgr.14152) 128 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:27.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:27 smithi194 bash[31648]: cluster 2023-10-09T04:35:25.658791+0000 mgr.y (mgr.14152) 129 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:27.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:27 smithi133 bash[27558]: cluster 2023-10-09T04:35:25.658791+0000 mgr.y (mgr.14152) 129 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:27.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:27 smithi133 bash[41291]: cluster 2023-10-09T04:35:25.658791+0000 mgr.y (mgr.14152) 129 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:29 smithi194 bash[31648]: cluster 2023-10-09T04:35:27.659247+0000 mgr.y (mgr.14152) 130 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:29.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:29 smithi133 bash[27558]: cluster 2023-10-09T04:35:27.659247+0000 mgr.y (mgr.14152) 130 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:29.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:29 smithi133 bash[41291]: cluster 2023-10-09T04:35:27.659247+0000 mgr.y (mgr.14152) 130 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:31.588 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:31 smithi133 bash[27558]: cluster 2023-10-09T04:35:29.659763+0000 mgr.y (mgr.14152) 131 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:31.589 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:31 smithi133 bash[41291]: cluster 2023-10-09T04:35:29.659763+0000 mgr.y (mgr.14152) 131 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:31.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:31 smithi194 bash[31648]: cluster 2023-10-09T04:35:29.659763+0000 mgr.y (mgr.14152) 131 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:33.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:33 smithi133 bash[41291]: cluster 2023-10-09T04:35:31.660246+0000 mgr.y (mgr.14152) 132 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:33.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:33 smithi133 bash[27558]: cluster 2023-10-09T04:35:31.660246+0000 mgr.y (mgr.14152) 132 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:33.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:33 smithi194 bash[31648]: cluster 2023-10-09T04:35:31.660246+0000 mgr.y (mgr.14152) 132 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:35.613 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:35 smithi133 bash[27558]: cluster 2023-10-09T04:35:33.660739+0000 mgr.y (mgr.14152) 133 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:35.613 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:35 smithi133 bash[27558]: audit 2023-10-09T04:35:34.524496+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:35:35.613 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:35 smithi133 bash[27558]: audit 2023-10-09T04:35:34.526000+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:35:35.613 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:35 smithi133 bash[27558]: cephadm 2023-10-09T04:35:34.527164+0000 mgr.y (mgr.14152) 134 : cephadm [INF] Deploying daemon osd.1 on smithi133 2023-10-09T04:35:35.613 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:35 smithi133 bash[41291]: cluster 2023-10-09T04:35:33.660739+0000 mgr.y (mgr.14152) 133 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:35.614 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:35 smithi133 bash[41291]: audit 2023-10-09T04:35:34.524496+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:35:35.614 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:35 smithi133 bash[41291]: audit 2023-10-09T04:35:34.526000+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:35:35.614 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:35 smithi133 bash[41291]: cephadm 2023-10-09T04:35:34.527164+0000 mgr.y (mgr.14152) 134 : cephadm [INF] Deploying daemon osd.1 on smithi133 2023-10-09T04:35:35.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:35 smithi194 bash[31648]: cluster 2023-10-09T04:35:33.660739+0000 mgr.y (mgr.14152) 133 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:35.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:35 smithi194 bash[31648]: audit 2023-10-09T04:35:34.524496+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:35:35.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:35 smithi194 bash[31648]: audit 2023-10-09T04:35:34.526000+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:35:35.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:35 smithi194 bash[31648]: cephadm 2023-10-09T04:35:34.527164+0000 mgr.y (mgr.14152) 134 : cephadm [INF] Deploying daemon osd.1 on smithi133 2023-10-09T04:35:37.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:37 smithi194 bash[31648]: cluster 2023-10-09T04:35:35.661201+0000 mgr.y (mgr.14152) 135 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:37.737 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:37 smithi133 bash[27558]: cluster 2023-10-09T04:35:35.661201+0000 mgr.y (mgr.14152) 135 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:37.737 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:37 smithi133 bash[41291]: cluster 2023-10-09T04:35:35.661201+0000 mgr.y (mgr.14152) 135 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:39.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:39 smithi194 bash[31648]: cluster 2023-10-09T04:35:37.661627+0000 mgr.y (mgr.14152) 136 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:39.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:39 smithi194 bash[31648]: audit 2023-10-09T04:35:39.015226+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:39.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:39 smithi194 bash[31648]: audit 2023-10-09T04:35:39.022328+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:39.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:39 smithi194 bash[31648]: audit 2023-10-09T04:35:39.050623+0000 mon.a (mon.0) 301 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:35:39.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:39 smithi133 bash[27558]: cluster 2023-10-09T04:35:37.661627+0000 mgr.y (mgr.14152) 136 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:39.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:39 smithi133 bash[27558]: audit 2023-10-09T04:35:39.015226+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:39.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:39 smithi133 bash[27558]: audit 2023-10-09T04:35:39.022328+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:39.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:39 smithi133 bash[27558]: audit 2023-10-09T04:35:39.050623+0000 mon.a (mon.0) 301 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:35:39.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:39 smithi133 bash[41291]: cluster 2023-10-09T04:35:37.661627+0000 mgr.y (mgr.14152) 136 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:39.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:39 smithi133 bash[41291]: audit 2023-10-09T04:35:39.015226+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:39.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:39 smithi133 bash[41291]: audit 2023-10-09T04:35:39.022328+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:39.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:39 smithi133 bash[41291]: audit 2023-10-09T04:35:39.050623+0000 mon.a (mon.0) 301 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:35:41.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:41 smithi194 bash[31648]: cluster 2023-10-09T04:35:39.662044+0000 mgr.y (mgr.14152) 137 : cluster [DBG] pgmap v101: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:41.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:41 smithi133 bash[41291]: cluster 2023-10-09T04:35:39.662044+0000 mgr.y (mgr.14152) 137 : cluster [DBG] pgmap v101: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:41.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:41 smithi133 bash[27558]: cluster 2023-10-09T04:35:39.662044+0000 mgr.y (mgr.14152) 137 : cluster [DBG] pgmap v101: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:43.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:43 smithi194 bash[31648]: cluster 2023-10-09T04:35:41.662528+0000 mgr.y (mgr.14152) 138 : cluster [DBG] pgmap v102: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:43.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:43 smithi133 bash[27558]: cluster 2023-10-09T04:35:41.662528+0000 mgr.y (mgr.14152) 138 : cluster [DBG] pgmap v102: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:43.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:43 smithi133 bash[41291]: cluster 2023-10-09T04:35:41.662528+0000 mgr.y (mgr.14152) 138 : cluster [DBG] pgmap v102: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:45.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:45 smithi133 bash[41291]: cluster 2023-10-09T04:35:43.662953+0000 mgr.y (mgr.14152) 139 : cluster [DBG] pgmap v103: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:45.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:45 smithi133 bash[41291]: audit 2023-10-09T04:35:45.295385+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:45.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:45 smithi133 bash[41291]: audit 2023-10-09T04:35:45.300548+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:45.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:45 smithi133 bash[41291]: audit 2023-10-09T04:35:45.301585+0000 mon.a (mon.0) 304 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:35:45.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:45 smithi133 bash[41291]: audit 2023-10-09T04:35:45.302655+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:35:45.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:45 smithi133 bash[41291]: audit 2023-10-09T04:35:45.308599+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:45.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:45 smithi133 bash[27558]: cluster 2023-10-09T04:35:43.662953+0000 mgr.y (mgr.14152) 139 : cluster [DBG] pgmap v103: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:45.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:45 smithi133 bash[27558]: audit 2023-10-09T04:35:45.295385+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:45.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:45 smithi133 bash[27558]: audit 2023-10-09T04:35:45.300548+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:45.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:45 smithi133 bash[27558]: audit 2023-10-09T04:35:45.301585+0000 mon.a (mon.0) 304 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:35:45.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:45 smithi133 bash[27558]: audit 2023-10-09T04:35:45.302655+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:35:45.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:45 smithi133 bash[27558]: audit 2023-10-09T04:35:45.308599+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:45.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:45 smithi194 bash[31648]: cluster 2023-10-09T04:35:43.662953+0000 mgr.y (mgr.14152) 139 : cluster [DBG] pgmap v103: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:45.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:45 smithi194 bash[31648]: audit 2023-10-09T04:35:45.295385+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:45.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:45 smithi194 bash[31648]: audit 2023-10-09T04:35:45.300548+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:45.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:45 smithi194 bash[31648]: audit 2023-10-09T04:35:45.301585+0000 mon.a (mon.0) 304 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:35:45.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:45 smithi194 bash[31648]: audit 2023-10-09T04:35:45.302655+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:35:45.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:45 smithi194 bash[31648]: audit 2023-10-09T04:35:45.308599+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:46.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:46 smithi194 bash[31648]: audit 2023-10-09T04:35:45.986787+0000 mon.c (mon.2) 5 : audit [INF] from='osd.1 [v2:172.21.15.133:6810/2965860635,v1:172.21.15.133:6811/2965860635]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T04:35:46.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:46 smithi194 bash[31648]: audit 2023-10-09T04:35:45.987437+0000 mon.a (mon.0) 307 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T04:35:46.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:46 smithi133 bash[27558]: audit 2023-10-09T04:35:45.986787+0000 mon.c (mon.2) 5 : audit [INF] from='osd.1 [v2:172.21.15.133:6810/2965860635,v1:172.21.15.133:6811/2965860635]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T04:35:46.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:46 smithi133 bash[27558]: audit 2023-10-09T04:35:45.987437+0000 mon.a (mon.0) 307 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T04:35:46.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:46 smithi133 bash[41291]: audit 2023-10-09T04:35:45.986787+0000 mon.c (mon.2) 5 : audit [INF] from='osd.1 [v2:172.21.15.133:6810/2965860635,v1:172.21.15.133:6811/2965860635]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T04:35:46.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:46 smithi133 bash[41291]: audit 2023-10-09T04:35:45.987437+0000 mon.a (mon.0) 307 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T04:35:47.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:47 smithi194 bash[31648]: cluster 2023-10-09T04:35:45.663442+0000 mgr.y (mgr.14152) 140 : cluster [DBG] pgmap v104: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:47.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:47 smithi194 bash[31648]: audit 2023-10-09T04:35:46.377490+0000 mon.a (mon.0) 308 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T04:35:47.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:47 smithi194 bash[31648]: cluster 2023-10-09T04:35:46.377594+0000 mon.a (mon.0) 309 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-10-09T04:35:47.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:47 smithi194 bash[31648]: audit 2023-10-09T04:35:46.377825+0000 mon.a (mon.0) 310 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:35:47.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:47 smithi194 bash[31648]: audit 2023-10-09T04:35:46.378397+0000 mon.c (mon.2) 6 : audit [INF] from='osd.1 [v2:172.21.15.133:6810/2965860635,v1:172.21.15.133:6811/2965860635]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:35:47.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:47 smithi194 bash[31648]: audit 2023-10-09T04:35:46.378838+0000 mon.a (mon.0) 311 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:35:47.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:47 smithi133 bash[27558]: cluster 2023-10-09T04:35:45.663442+0000 mgr.y (mgr.14152) 140 : cluster [DBG] pgmap v104: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:47.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:47 smithi133 bash[27558]: audit 2023-10-09T04:35:46.377490+0000 mon.a (mon.0) 308 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T04:35:47.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:47 smithi133 bash[27558]: cluster 2023-10-09T04:35:46.377594+0000 mon.a (mon.0) 309 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-10-09T04:35:47.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:47 smithi133 bash[27558]: audit 2023-10-09T04:35:46.377825+0000 mon.a (mon.0) 310 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:35:47.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:47 smithi133 bash[27558]: audit 2023-10-09T04:35:46.378397+0000 mon.c (mon.2) 6 : audit [INF] from='osd.1 [v2:172.21.15.133:6810/2965860635,v1:172.21.15.133:6811/2965860635]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:35:47.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:47 smithi133 bash[27558]: audit 2023-10-09T04:35:46.378838+0000 mon.a (mon.0) 311 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:35:47.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:47 smithi133 bash[41291]: cluster 2023-10-09T04:35:45.663442+0000 mgr.y (mgr.14152) 140 : cluster [DBG] pgmap v104: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:47.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:47 smithi133 bash[41291]: audit 2023-10-09T04:35:46.377490+0000 mon.a (mon.0) 308 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T04:35:47.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:47 smithi133 bash[41291]: cluster 2023-10-09T04:35:46.377594+0000 mon.a (mon.0) 309 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-10-09T04:35:47.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:47 smithi133 bash[41291]: audit 2023-10-09T04:35:46.377825+0000 mon.a (mon.0) 310 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:35:47.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:47 smithi133 bash[41291]: audit 2023-10-09T04:35:46.378397+0000 mon.c (mon.2) 6 : audit [INF] from='osd.1 [v2:172.21.15.133:6810/2965860635,v1:172.21.15.133:6811/2965860635]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:35:47.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:47 smithi133 bash[41291]: audit 2023-10-09T04:35:46.378838+0000 mon.a (mon.0) 311 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:35:48.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:48 smithi133 bash[27558]: audit 2023-10-09T04:35:47.379242+0000 mon.a (mon.0) 312 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]': finished 2023-10-09T04:35:48.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:48 smithi133 bash[27558]: cluster 2023-10-09T04:35:47.379367+0000 mon.a (mon.0) 313 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-10-09T04:35:48.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:48 smithi133 bash[27558]: audit 2023-10-09T04:35:47.379793+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:35:48.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:48 smithi133 bash[27558]: audit 2023-10-09T04:35:47.387440+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:35:48.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:48 smithi133 bash[41291]: audit 2023-10-09T04:35:47.379242+0000 mon.a (mon.0) 312 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]': finished 2023-10-09T04:35:48.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:48 smithi133 bash[41291]: cluster 2023-10-09T04:35:47.379367+0000 mon.a (mon.0) 313 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-10-09T04:35:48.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:48 smithi133 bash[41291]: audit 2023-10-09T04:35:47.379793+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:35:48.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:48 smithi133 bash[41291]: audit 2023-10-09T04:35:47.387440+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:35:48.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:48 smithi194 bash[31648]: audit 2023-10-09T04:35:47.379242+0000 mon.a (mon.0) 312 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]': finished 2023-10-09T04:35:48.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:48 smithi194 bash[31648]: cluster 2023-10-09T04:35:47.379367+0000 mon.a (mon.0) 313 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-10-09T04:35:48.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:48 smithi194 bash[31648]: audit 2023-10-09T04:35:47.379793+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:35:48.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:48 smithi194 bash[31648]: audit 2023-10-09T04:35:47.387440+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:35:49.032 INFO:teuthology.orchestra.run.smithi133.stdout:Created osd(s) 1 on host 'smithi133' 2023-10-09T04:35:49.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:49 smithi133 bash[27558]: cluster 2023-10-09T04:35:47.001656+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:35:49.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:49 smithi133 bash[27558]: cluster 2023-10-09T04:35:47.001793+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:35:49.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:49 smithi133 bash[27558]: cluster 2023-10-09T04:35:47.664021+0000 mgr.y (mgr.14152) 141 : cluster [DBG] pgmap v107: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:49.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:49 smithi133 bash[27558]: audit 2023-10-09T04:35:48.390242+0000 mon.a (mon.0) 316 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:35:49.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:49 smithi133 bash[27558]: cluster 2023-10-09T04:35:48.394284+0000 mon.a (mon.0) 317 : cluster [INF] osd.1 [v2:172.21.15.133:6810/2965860635,v1:172.21.15.133:6811/2965860635] boot 2023-10-09T04:35:49.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:49 smithi133 bash[27558]: cluster 2023-10-09T04:35:48.394344+0000 mon.a (mon.0) 318 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-10-09T04:35:49.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:49 smithi133 bash[27558]: audit 2023-10-09T04:35:48.394482+0000 mon.a (mon.0) 319 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:35:49.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:49 smithi133 bash[27558]: audit 2023-10-09T04:35:48.996395+0000 mon.a (mon.0) 320 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:49.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:49 smithi133 bash[27558]: audit 2023-10-09T04:35:49.004367+0000 mon.a (mon.0) 321 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:49.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:49 smithi133 bash[27558]: audit 2023-10-09T04:35:49.011467+0000 mon.a (mon.0) 322 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:35:49.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:49 smithi133 bash[41291]: cluster 2023-10-09T04:35:47.001656+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:35:49.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:49 smithi133 bash[41291]: cluster 2023-10-09T04:35:47.001793+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:35:49.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:49 smithi133 bash[41291]: cluster 2023-10-09T04:35:47.664021+0000 mgr.y (mgr.14152) 141 : cluster [DBG] pgmap v107: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:49.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:49 smithi133 bash[41291]: audit 2023-10-09T04:35:48.390242+0000 mon.a (mon.0) 316 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:35:49.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:49 smithi133 bash[41291]: cluster 2023-10-09T04:35:48.394284+0000 mon.a (mon.0) 317 : cluster [INF] osd.1 [v2:172.21.15.133:6810/2965860635,v1:172.21.15.133:6811/2965860635] boot 2023-10-09T04:35:49.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:49 smithi133 bash[41291]: cluster 2023-10-09T04:35:48.394344+0000 mon.a (mon.0) 318 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-10-09T04:35:49.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:49 smithi133 bash[41291]: audit 2023-10-09T04:35:48.394482+0000 mon.a (mon.0) 319 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:35:49.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:49 smithi133 bash[41291]: audit 2023-10-09T04:35:48.996395+0000 mon.a (mon.0) 320 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:49.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:49 smithi133 bash[41291]: audit 2023-10-09T04:35:49.004367+0000 mon.a (mon.0) 321 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:49.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:49 smithi133 bash[41291]: audit 2023-10-09T04:35:49.011467+0000 mon.a (mon.0) 322 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:35:49.737 DEBUG:teuthology.orchestra.run.smithi133:osd.1> sudo journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@osd.1.service 2023-10-09T04:35:49.739 INFO:tasks.cephadm:Deploying osd.2 on smithi133 with /dev/vg_nvme/lv_2... 2023-10-09T04:35:49.739 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-10-09T04:35:49.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:49 smithi194 bash[31648]: cluster 2023-10-09T04:35:47.001656+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:35:49.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:49 smithi194 bash[31648]: cluster 2023-10-09T04:35:47.001793+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:35:49.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:49 smithi194 bash[31648]: cluster 2023-10-09T04:35:47.664021+0000 mgr.y (mgr.14152) 141 : cluster [DBG] pgmap v107: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:35:49.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:49 smithi194 bash[31648]: audit 2023-10-09T04:35:48.390242+0000 mon.a (mon.0) 316 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:35:49.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:49 smithi194 bash[31648]: cluster 2023-10-09T04:35:48.394284+0000 mon.a (mon.0) 317 : cluster [INF] osd.1 [v2:172.21.15.133:6810/2965860635,v1:172.21.15.133:6811/2965860635] boot 2023-10-09T04:35:49.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:49 smithi194 bash[31648]: cluster 2023-10-09T04:35:48.394344+0000 mon.a (mon.0) 318 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-10-09T04:35:49.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:49 smithi194 bash[31648]: audit 2023-10-09T04:35:48.394482+0000 mon.a (mon.0) 319 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:35:49.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:49 smithi194 bash[31648]: audit 2023-10-09T04:35:48.996395+0000 mon.a (mon.0) 320 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:49.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:49 smithi194 bash[31648]: audit 2023-10-09T04:35:49.004367+0000 mon.a (mon.0) 321 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:35:49.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:49 smithi194 bash[31648]: audit 2023-10-09T04:35:49.011467+0000 mon.a (mon.0) 322 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:35:50.071 INFO:journalctl@ceph.osd.1.smithi133.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:35:51.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:50 smithi133 bash[27558]: cluster 2023-10-09T04:35:49.664578+0000 mgr.y (mgr.14152) 142 : cluster [DBG] pgmap v109: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:35:51.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:50 smithi133 bash[27558]: cluster 2023-10-09T04:35:49.999322+0000 mon.a (mon.0) 323 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-10-09T04:35:51.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:51 smithi133 bash[41291]: cluster 2023-10-09T04:35:49.664578+0000 mgr.y (mgr.14152) 142 : cluster [DBG] pgmap v109: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:35:51.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:51 smithi133 bash[41291]: cluster 2023-10-09T04:35:49.999322+0000 mon.a (mon.0) 323 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-10-09T04:35:51.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:51 smithi194 bash[31648]: cluster 2023-10-09T04:35:49.664578+0000 mgr.y (mgr.14152) 142 : cluster [DBG] pgmap v109: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:35:51.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:51 smithi194 bash[31648]: cluster 2023-10-09T04:35:49.999322+0000 mon.a (mon.0) 323 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-10-09T04:35:53.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:53 smithi133 bash[27558]: cluster 2023-10-09T04:35:51.665009+0000 mgr.y (mgr.14152) 143 : cluster [DBG] pgmap v111: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:35:53.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:53 smithi133 bash[41291]: cluster 2023-10-09T04:35:51.665009+0000 mgr.y (mgr.14152) 143 : cluster [DBG] pgmap v111: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:35:53.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:53 smithi194 bash[31648]: cluster 2023-10-09T04:35:51.665009+0000 mgr.y (mgr.14152) 143 : cluster [DBG] pgmap v111: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:35:53.781 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:55.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:55 smithi133 bash[41291]: cluster 2023-10-09T04:35:53.665570+0000 mgr.y (mgr.14152) 144 : cluster [DBG] pgmap v112: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:35:55.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:55 smithi133 bash[27558]: cluster 2023-10-09T04:35:53.665570+0000 mgr.y (mgr.14152) 144 : cluster [DBG] pgmap v112: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:35:55.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:55 smithi194 bash[31648]: cluster 2023-10-09T04:35:53.665570+0000 mgr.y (mgr.14152) 144 : cluster [DBG] pgmap v112: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:35:57.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:57 smithi133 bash[41291]: cluster 2023-10-09T04:35:55.666001+0000 mgr.y (mgr.14152) 145 : cluster [DBG] pgmap v113: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:35:57.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:57 smithi133 bash[27558]: cluster 2023-10-09T04:35:55.666001+0000 mgr.y (mgr.14152) 145 : cluster [DBG] pgmap v113: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:35:57.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:57 smithi194 bash[31648]: cluster 2023-10-09T04:35:55.666001+0000 mgr.y (mgr.14152) 145 : cluster [DBG] pgmap v113: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:35:58.491 INFO:teuthology.orchestra.run.smithi133.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-10-09T04:35:58.491 INFO:teuthology.orchestra.run.smithi133.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-10-09T04:35:58.491 INFO:teuthology.orchestra.run.smithi133.stderr: stderr: 10+0 records in 2023-10-09T04:35:58.492 INFO:teuthology.orchestra.run.smithi133.stderr:10+0 records out 2023-10-09T04:35:58.492 INFO:teuthology.orchestra.run.smithi133.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0186214 s, 563 MB/s 2023-10-09T04:35:58.492 INFO:teuthology.orchestra.run.smithi133.stderr:--> Zapping successful for: 2023-10-09T04:35:59.439 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:35:59 smithi133 bash[27558]: cluster 2023-10-09T04:35:57.666484+0000 mgr.y (mgr.14152) 146 : cluster [DBG] pgmap v114: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:35:59.439 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:35:59 smithi133 bash[41291]: cluster 2023-10-09T04:35:57.666484+0000 mgr.y (mgr.14152) 146 : cluster [DBG] pgmap v114: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:35:59.440 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi133:vg_nvme/lv_2 2023-10-09T04:35:59.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:35:59 smithi194 bash[31648]: cluster 2023-10-09T04:35:57.666484+0000 mgr.y (mgr.14152) 146 : cluster [DBG] pgmap v114: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:01.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:00 smithi133 bash[27558]: cluster 2023-10-09T04:35:59.666951+0000 mgr.y (mgr.14152) 147 : cluster [DBG] pgmap v115: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:01.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:00 smithi133 bash[27558]: cephadm 2023-10-09T04:35:59.955525+0000 mgr.y (mgr.14152) 148 : cephadm [INF] Detected new or changed devices on smithi133 2023-10-09T04:36:01.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:00 smithi133 bash[27558]: audit 2023-10-09T04:35:59.962689+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:01.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:00 smithi133 bash[27558]: audit 2023-10-09T04:35:59.970751+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:01.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:00 smithi133 bash[27558]: audit 2023-10-09T04:35:59.972497+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:36:01.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:00 smithi133 bash[27558]: audit 2023-10-09T04:35:59.973599+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:36:01.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:00 smithi133 bash[27558]: cephadm 2023-10-09T04:35:59.974373+0000 mgr.y (mgr.14152) 149 : cephadm [INF] Adjusting osd_memory_target on smithi133 to 8120M 2023-10-09T04:36:01.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:00 smithi133 bash[27558]: audit 2023-10-09T04:35:59.979374+0000 mon.a (mon.0) 328 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:01.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:00 smithi133 bash[27558]: audit 2023-10-09T04:35:59.980925+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:36:01.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:00 smithi133 bash[27558]: audit 2023-10-09T04:35:59.982316+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:36:01.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:00 smithi133 bash[27558]: audit 2023-10-09T04:35:59.988776+0000 mon.a (mon.0) 331 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:01.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:00 smithi133 bash[41291]: cluster 2023-10-09T04:35:59.666951+0000 mgr.y (mgr.14152) 147 : cluster [DBG] pgmap v115: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:01.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:00 smithi133 bash[41291]: cephadm 2023-10-09T04:35:59.955525+0000 mgr.y (mgr.14152) 148 : cephadm [INF] Detected new or changed devices on smithi133 2023-10-09T04:36:01.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:00 smithi133 bash[41291]: audit 2023-10-09T04:35:59.962689+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:01.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:00 smithi133 bash[41291]: audit 2023-10-09T04:35:59.970751+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:01.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:00 smithi133 bash[41291]: audit 2023-10-09T04:35:59.972497+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:36:01.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:00 smithi133 bash[41291]: audit 2023-10-09T04:35:59.973599+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:36:01.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:00 smithi133 bash[41291]: cephadm 2023-10-09T04:35:59.974373+0000 mgr.y (mgr.14152) 149 : cephadm [INF] Adjusting osd_memory_target on smithi133 to 8120M 2023-10-09T04:36:01.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:00 smithi133 bash[41291]: audit 2023-10-09T04:35:59.979374+0000 mon.a (mon.0) 328 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:01.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:00 smithi133 bash[41291]: audit 2023-10-09T04:35:59.980925+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:36:01.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:00 smithi133 bash[41291]: audit 2023-10-09T04:35:59.982316+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:36:01.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:00 smithi133 bash[41291]: audit 2023-10-09T04:35:59.988776+0000 mon.a (mon.0) 331 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:01.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:00 smithi194 bash[31648]: cluster 2023-10-09T04:35:59.666951+0000 mgr.y (mgr.14152) 147 : cluster [DBG] pgmap v115: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:01.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:00 smithi194 bash[31648]: cephadm 2023-10-09T04:35:59.955525+0000 mgr.y (mgr.14152) 148 : cephadm [INF] Detected new or changed devices on smithi133 2023-10-09T04:36:01.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:00 smithi194 bash[31648]: audit 2023-10-09T04:35:59.962689+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:01.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:00 smithi194 bash[31648]: audit 2023-10-09T04:35:59.970751+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:01.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:00 smithi194 bash[31648]: audit 2023-10-09T04:35:59.972497+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:36:01.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:00 smithi194 bash[31648]: audit 2023-10-09T04:35:59.973599+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:36:01.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:00 smithi194 bash[31648]: cephadm 2023-10-09T04:35:59.974373+0000 mgr.y (mgr.14152) 149 : cephadm [INF] Adjusting osd_memory_target on smithi133 to 8120M 2023-10-09T04:36:01.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:00 smithi194 bash[31648]: audit 2023-10-09T04:35:59.979374+0000 mon.a (mon.0) 328 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:01.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:00 smithi194 bash[31648]: audit 2023-10-09T04:35:59.980925+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:36:01.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:00 smithi194 bash[31648]: audit 2023-10-09T04:35:59.982316+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:36:01.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:00 smithi194 bash[31648]: audit 2023-10-09T04:35:59.988776+0000 mon.a (mon.0) 331 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:03.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:02 smithi133 bash[27558]: cluster 2023-10-09T04:36:01.667388+0000 mgr.y (mgr.14152) 150 : cluster [DBG] pgmap v116: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:03.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:02 smithi133 bash[41291]: cluster 2023-10-09T04:36:01.667388+0000 mgr.y (mgr.14152) 150 : cluster [DBG] pgmap v116: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:03.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:02 smithi194 bash[31648]: cluster 2023-10-09T04:36:01.667388+0000 mgr.y (mgr.14152) 150 : cluster [DBG] pgmap v116: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:04.172 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:36:05.245 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:04 smithi133 bash[27558]: cluster 2023-10-09T04:36:03.667776+0000 mgr.y (mgr.14152) 151 : cluster [DBG] pgmap v117: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:05.245 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:04 smithi133 bash[41291]: cluster 2023-10-09T04:36:03.667776+0000 mgr.y (mgr.14152) 151 : cluster [DBG] pgmap v117: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:05.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:04 smithi194 bash[31648]: cluster 2023-10-09T04:36:03.667776+0000 mgr.y (mgr.14152) 151 : cluster [DBG] pgmap v117: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:07.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:07 smithi133 bash[27558]: cluster 2023-10-09T04:36:05.668252+0000 mgr.y (mgr.14152) 152 : cluster [DBG] pgmap v118: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:07.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:07 smithi133 bash[27558]: audit 2023-10-09T04:36:07.199223+0000 mon.a (mon.0) 332 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:36:07.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:07 smithi133 bash[27558]: audit 2023-10-09T04:36:07.204407+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:36:07.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:07 smithi133 bash[27558]: audit 2023-10-09T04:36:07.205535+0000 mon.a (mon.0) 334 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:36:07.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:07 smithi133 bash[41291]: cluster 2023-10-09T04:36:05.668252+0000 mgr.y (mgr.14152) 152 : cluster [DBG] pgmap v118: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:07.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:07 smithi133 bash[41291]: audit 2023-10-09T04:36:07.199223+0000 mon.a (mon.0) 332 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:36:07.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:07 smithi133 bash[41291]: audit 2023-10-09T04:36:07.204407+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:36:07.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:07 smithi133 bash[41291]: audit 2023-10-09T04:36:07.205535+0000 mon.a (mon.0) 334 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:36:07.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:07 smithi194 bash[31648]: cluster 2023-10-09T04:36:05.668252+0000 mgr.y (mgr.14152) 152 : cluster [DBG] pgmap v118: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:07.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:07 smithi194 bash[31648]: audit 2023-10-09T04:36:07.199223+0000 mon.a (mon.0) 332 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:36:07.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:07 smithi194 bash[31648]: audit 2023-10-09T04:36:07.204407+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:36:07.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:07 smithi194 bash[31648]: audit 2023-10-09T04:36:07.205535+0000 mon.a (mon.0) 334 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:36:08.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:08 smithi194 bash[31648]: audit 2023-10-09T04:36:07.193577+0000 mgr.y (mgr.14152) 153 : audit [DBG] from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi133:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:36:08.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:08 smithi133 bash[41291]: audit 2023-10-09T04:36:07.193577+0000 mgr.y (mgr.14152) 153 : audit [DBG] from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi133:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:36:08.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:08 smithi133 bash[27558]: audit 2023-10-09T04:36:07.193577+0000 mgr.y (mgr.14152) 153 : audit [DBG] from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi133:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:36:09.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:09 smithi194 bash[31648]: cluster 2023-10-09T04:36:07.668730+0000 mgr.y (mgr.14152) 154 : cluster [DBG] pgmap v119: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:09.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:09 smithi133 bash[27558]: cluster 2023-10-09T04:36:07.668730+0000 mgr.y (mgr.14152) 154 : cluster [DBG] pgmap v119: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:09.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:09 smithi133 bash[41291]: cluster 2023-10-09T04:36:07.668730+0000 mgr.y (mgr.14152) 154 : cluster [DBG] pgmap v119: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:11.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:11 smithi194 bash[31648]: cluster 2023-10-09T04:36:09.669223+0000 mgr.y (mgr.14152) 155 : cluster [DBG] pgmap v120: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:11.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:11 smithi133 bash[27558]: cluster 2023-10-09T04:36:09.669223+0000 mgr.y (mgr.14152) 155 : cluster [DBG] pgmap v120: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:11.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:11 smithi133 bash[41291]: cluster 2023-10-09T04:36:09.669223+0000 mgr.y (mgr.14152) 155 : cluster [DBG] pgmap v120: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:13.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:13 smithi133 bash[27558]: cluster 2023-10-09T04:36:11.669601+0000 mgr.y (mgr.14152) 156 : cluster [DBG] pgmap v121: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:13.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:13 smithi133 bash[41291]: cluster 2023-10-09T04:36:11.669601+0000 mgr.y (mgr.14152) 156 : cluster [DBG] pgmap v121: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:13.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:13 smithi194 bash[31648]: cluster 2023-10-09T04:36:11.669601+0000 mgr.y (mgr.14152) 156 : cluster [DBG] pgmap v121: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:15.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:15 smithi194 bash[31648]: cluster 2023-10-09T04:36:13.670096+0000 mgr.y (mgr.14152) 157 : cluster [DBG] pgmap v122: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:15.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:15 smithi133 bash[27558]: cluster 2023-10-09T04:36:13.670096+0000 mgr.y (mgr.14152) 157 : cluster [DBG] pgmap v122: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:15.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:15 smithi133 bash[41291]: cluster 2023-10-09T04:36:13.670096+0000 mgr.y (mgr.14152) 157 : cluster [DBG] pgmap v122: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:16.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:16 smithi194 bash[31648]: audit 2023-10-09T04:36:15.680953+0000 mon.a (mon.0) 335 : audit [INF] from='client.? 172.21.15.133:0/427125638' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "08751330-a327-4bf8-9c4d-9ff986b1c67f"}]: dispatch 2023-10-09T04:36:16.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:16 smithi194 bash[31648]: audit 2023-10-09T04:36:15.695282+0000 mon.a (mon.0) 336 : audit [INF] from='client.? 172.21.15.133:0/427125638' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "08751330-a327-4bf8-9c4d-9ff986b1c67f"}]': finished 2023-10-09T04:36:16.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:16 smithi194 bash[31648]: cluster 2023-10-09T04:36:15.695360+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-10-09T04:36:16.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:16 smithi194 bash[31648]: audit 2023-10-09T04:36:15.695589+0000 mon.a (mon.0) 338 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:36:16.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:16 smithi133 bash[41291]: audit 2023-10-09T04:36:15.680953+0000 mon.a (mon.0) 335 : audit [INF] from='client.? 172.21.15.133:0/427125638' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "08751330-a327-4bf8-9c4d-9ff986b1c67f"}]: dispatch 2023-10-09T04:36:16.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:16 smithi133 bash[41291]: audit 2023-10-09T04:36:15.695282+0000 mon.a (mon.0) 336 : audit [INF] from='client.? 172.21.15.133:0/427125638' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "08751330-a327-4bf8-9c4d-9ff986b1c67f"}]': finished 2023-10-09T04:36:16.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:16 smithi133 bash[41291]: cluster 2023-10-09T04:36:15.695360+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-10-09T04:36:16.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:16 smithi133 bash[41291]: audit 2023-10-09T04:36:15.695589+0000 mon.a (mon.0) 338 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:36:16.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:16 smithi133 bash[27558]: audit 2023-10-09T04:36:15.680953+0000 mon.a (mon.0) 335 : audit [INF] from='client.? 172.21.15.133:0/427125638' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "08751330-a327-4bf8-9c4d-9ff986b1c67f"}]: dispatch 2023-10-09T04:36:16.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:16 smithi133 bash[27558]: audit 2023-10-09T04:36:15.695282+0000 mon.a (mon.0) 336 : audit [INF] from='client.? 172.21.15.133:0/427125638' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "08751330-a327-4bf8-9c4d-9ff986b1c67f"}]': finished 2023-10-09T04:36:16.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:16 smithi133 bash[27558]: cluster 2023-10-09T04:36:15.695360+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-10-09T04:36:16.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:16 smithi133 bash[27558]: audit 2023-10-09T04:36:15.695589+0000 mon.a (mon.0) 338 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:36:17.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:17 smithi194 bash[31648]: cluster 2023-10-09T04:36:15.670603+0000 mgr.y (mgr.14152) 158 : cluster [DBG] pgmap v123: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:17.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:17 smithi194 bash[31648]: audit 2023-10-09T04:36:16.459091+0000 mon.c (mon.2) 7 : audit [DBG] from='client.? 172.21.15.133:0/2317607235' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:36:17.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:17 smithi133 bash[27558]: cluster 2023-10-09T04:36:15.670603+0000 mgr.y (mgr.14152) 158 : cluster [DBG] pgmap v123: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:17.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:17 smithi133 bash[27558]: audit 2023-10-09T04:36:16.459091+0000 mon.c (mon.2) 7 : audit [DBG] from='client.? 172.21.15.133:0/2317607235' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:36:17.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:17 smithi133 bash[41291]: cluster 2023-10-09T04:36:15.670603+0000 mgr.y (mgr.14152) 158 : cluster [DBG] pgmap v123: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:17.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:17 smithi133 bash[41291]: audit 2023-10-09T04:36:16.459091+0000 mon.c (mon.2) 7 : audit [DBG] from='client.? 172.21.15.133:0/2317607235' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:36:19.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:19 smithi133 bash[27558]: cluster 2023-10-09T04:36:17.671062+0000 mgr.y (mgr.14152) 159 : cluster [DBG] pgmap v125: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:19.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:19 smithi133 bash[41291]: cluster 2023-10-09T04:36:17.671062+0000 mgr.y (mgr.14152) 159 : cluster [DBG] pgmap v125: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:19.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:19 smithi194 bash[31648]: cluster 2023-10-09T04:36:17.671062+0000 mgr.y (mgr.14152) 159 : cluster [DBG] pgmap v125: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:21.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:21 smithi194 bash[31648]: cluster 2023-10-09T04:36:19.671501+0000 mgr.y (mgr.14152) 160 : cluster [DBG] pgmap v126: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:21.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:21 smithi133 bash[41291]: cluster 2023-10-09T04:36:19.671501+0000 mgr.y (mgr.14152) 160 : cluster [DBG] pgmap v126: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:21.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:21 smithi133 bash[27558]: cluster 2023-10-09T04:36:19.671501+0000 mgr.y (mgr.14152) 160 : cluster [DBG] pgmap v126: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:23.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:23 smithi194 bash[31648]: cluster 2023-10-09T04:36:21.671992+0000 mgr.y (mgr.14152) 161 : cluster [DBG] pgmap v127: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:23.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:23 smithi133 bash[27558]: cluster 2023-10-09T04:36:21.671992+0000 mgr.y (mgr.14152) 161 : cluster [DBG] pgmap v127: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:23.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:23 smithi133 bash[41291]: cluster 2023-10-09T04:36:21.671992+0000 mgr.y (mgr.14152) 161 : cluster [DBG] pgmap v127: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:25.612 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:25 smithi133 bash[27558]: cluster 2023-10-09T04:36:23.672475+0000 mgr.y (mgr.14152) 162 : cluster [DBG] pgmap v128: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:25.613 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:25 smithi133 bash[41291]: cluster 2023-10-09T04:36:23.672475+0000 mgr.y (mgr.14152) 162 : cluster [DBG] pgmap v128: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:25.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:25 smithi194 bash[31648]: cluster 2023-10-09T04:36:23.672475+0000 mgr.y (mgr.14152) 162 : cluster [DBG] pgmap v128: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:27.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:27 smithi133 bash[41291]: cluster 2023-10-09T04:36:25.672974+0000 mgr.y (mgr.14152) 163 : cluster [DBG] pgmap v129: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:27.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:27 smithi133 bash[27558]: cluster 2023-10-09T04:36:25.672974+0000 mgr.y (mgr.14152) 163 : cluster [DBG] pgmap v129: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:27.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:27 smithi194 bash[31648]: cluster 2023-10-09T04:36:25.672974+0000 mgr.y (mgr.14152) 163 : cluster [DBG] pgmap v129: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:29 smithi194 bash[31648]: cluster 2023-10-09T04:36:27.673499+0000 mgr.y (mgr.14152) 164 : cluster [DBG] pgmap v130: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:29 smithi194 bash[31648]: audit 2023-10-09T04:36:28.533674+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T04:36:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:29 smithi194 bash[31648]: audit 2023-10-09T04:36:28.534886+0000 mon.a (mon.0) 340 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:36:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:29 smithi194 bash[31648]: cephadm 2023-10-09T04:36:28.535857+0000 mgr.y (mgr.14152) 165 : cephadm [INF] Deploying daemon osd.2 on smithi133 2023-10-09T04:36:29.678 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:29 smithi133 bash[27558]: cluster 2023-10-09T04:36:27.673499+0000 mgr.y (mgr.14152) 164 : cluster [DBG] pgmap v130: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:29.679 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:29 smithi133 bash[27558]: audit 2023-10-09T04:36:28.533674+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T04:36:29.679 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:29 smithi133 bash[27558]: audit 2023-10-09T04:36:28.534886+0000 mon.a (mon.0) 340 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:36:29.679 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:29 smithi133 bash[27558]: cephadm 2023-10-09T04:36:28.535857+0000 mgr.y (mgr.14152) 165 : cephadm [INF] Deploying daemon osd.2 on smithi133 2023-10-09T04:36:29.679 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:29 smithi133 bash[41291]: cluster 2023-10-09T04:36:27.673499+0000 mgr.y (mgr.14152) 164 : cluster [DBG] pgmap v130: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:29.679 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:29 smithi133 bash[41291]: audit 2023-10-09T04:36:28.533674+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T04:36:29.679 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:29 smithi133 bash[41291]: audit 2023-10-09T04:36:28.534886+0000 mon.a (mon.0) 340 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:36:29.679 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:29 smithi133 bash[41291]: cephadm 2023-10-09T04:36:28.535857+0000 mgr.y (mgr.14152) 165 : cephadm [INF] Deploying daemon osd.2 on smithi133 2023-10-09T04:36:31.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:31 smithi194 bash[31648]: cluster 2023-10-09T04:36:29.673877+0000 mgr.y (mgr.14152) 166 : cluster [DBG] pgmap v131: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:31.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:31 smithi133 bash[27558]: cluster 2023-10-09T04:36:29.673877+0000 mgr.y (mgr.14152) 166 : cluster [DBG] pgmap v131: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:31.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:31 smithi133 bash[41291]: cluster 2023-10-09T04:36:29.673877+0000 mgr.y (mgr.14152) 166 : cluster [DBG] pgmap v131: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:33.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:33 smithi133 bash[41291]: cluster 2023-10-09T04:36:31.674371+0000 mgr.y (mgr.14152) 167 : cluster [DBG] pgmap v132: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:33.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:33 smithi133 bash[41291]: audit 2023-10-09T04:36:33.198454+0000 mon.a (mon.0) 341 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:33.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:33 smithi133 bash[41291]: audit 2023-10-09T04:36:33.205179+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:33.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:33 smithi133 bash[41291]: audit 2023-10-09T04:36:33.207411+0000 mon.a (mon.0) 343 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:36:33.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:33 smithi133 bash[27558]: cluster 2023-10-09T04:36:31.674371+0000 mgr.y (mgr.14152) 167 : cluster [DBG] pgmap v132: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:33.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:33 smithi133 bash[27558]: audit 2023-10-09T04:36:33.198454+0000 mon.a (mon.0) 341 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:33.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:33 smithi133 bash[27558]: audit 2023-10-09T04:36:33.205179+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:33.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:33 smithi133 bash[27558]: audit 2023-10-09T04:36:33.207411+0000 mon.a (mon.0) 343 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:36:33.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:33 smithi194 bash[31648]: cluster 2023-10-09T04:36:31.674371+0000 mgr.y (mgr.14152) 167 : cluster [DBG] pgmap v132: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:33.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:33 smithi194 bash[31648]: audit 2023-10-09T04:36:33.198454+0000 mon.a (mon.0) 341 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:33.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:33 smithi194 bash[31648]: audit 2023-10-09T04:36:33.205179+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:33.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:33 smithi194 bash[31648]: audit 2023-10-09T04:36:33.207411+0000 mon.a (mon.0) 343 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:36:35.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:35 smithi194 bash[31648]: cluster 2023-10-09T04:36:33.674904+0000 mgr.y (mgr.14152) 168 : cluster [DBG] pgmap v133: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:35.640 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:35 smithi133 bash[27558]: cluster 2023-10-09T04:36:33.674904+0000 mgr.y (mgr.14152) 168 : cluster [DBG] pgmap v133: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:35.640 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:35 smithi133 bash[41291]: cluster 2023-10-09T04:36:33.674904+0000 mgr.y (mgr.14152) 168 : cluster [DBG] pgmap v133: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:37.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:37 smithi133 bash[27558]: cluster 2023-10-09T04:36:35.675418+0000 mgr.y (mgr.14152) 169 : cluster [DBG] pgmap v134: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:37.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:37 smithi133 bash[41291]: cluster 2023-10-09T04:36:35.675418+0000 mgr.y (mgr.14152) 169 : cluster [DBG] pgmap v134: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:37.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:37 smithi194 bash[31648]: cluster 2023-10-09T04:36:35.675418+0000 mgr.y (mgr.14152) 169 : cluster [DBG] pgmap v134: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:39.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:39 smithi133 bash[41291]: cluster 2023-10-09T04:36:37.675895+0000 mgr.y (mgr.14152) 170 : cluster [DBG] pgmap v135: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:39.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:39 smithi133 bash[27558]: cluster 2023-10-09T04:36:37.675895+0000 mgr.y (mgr.14152) 170 : cluster [DBG] pgmap v135: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:39.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:39 smithi194 bash[31648]: cluster 2023-10-09T04:36:37.675895+0000 mgr.y (mgr.14152) 170 : cluster [DBG] pgmap v135: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:40.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:40 smithi194 bash[31648]: audit 2023-10-09T04:36:39.600554+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:40.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:40 smithi194 bash[31648]: audit 2023-10-09T04:36:39.608973+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:40.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:40 smithi194 bash[31648]: cluster 2023-10-09T04:36:39.676387+0000 mgr.y (mgr.14152) 171 : cluster [DBG] pgmap v136: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:40.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:40 smithi194 bash[31648]: audit 2023-10-09T04:36:40.027263+0000 mon.a (mon.0) 346 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:36:40.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:40 smithi194 bash[31648]: audit 2023-10-09T04:36:40.028910+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:36:40.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:40 smithi194 bash[31648]: audit 2023-10-09T04:36:40.038975+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:40.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:40 smithi194 bash[31648]: audit 2023-10-09T04:36:40.267235+0000 mon.c (mon.2) 8 : audit [INF] from='osd.2 [v2:172.21.15.133:6818/1465201170,v1:172.21.15.133:6819/1465201170]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:36:40.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:40 smithi194 bash[31648]: audit 2023-10-09T04:36:40.267837+0000 mon.a (mon.0) 349 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:36:41.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:40 smithi133 bash[41291]: audit 2023-10-09T04:36:39.600554+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:41.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:40 smithi133 bash[41291]: audit 2023-10-09T04:36:39.608973+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:41.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:40 smithi133 bash[41291]: cluster 2023-10-09T04:36:39.676387+0000 mgr.y (mgr.14152) 171 : cluster [DBG] pgmap v136: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:41.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:40 smithi133 bash[41291]: audit 2023-10-09T04:36:40.027263+0000 mon.a (mon.0) 346 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:36:41.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:40 smithi133 bash[41291]: audit 2023-10-09T04:36:40.028910+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:36:41.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:40 smithi133 bash[41291]: audit 2023-10-09T04:36:40.038975+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:41.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:40 smithi133 bash[41291]: audit 2023-10-09T04:36:40.267235+0000 mon.c (mon.2) 8 : audit [INF] from='osd.2 [v2:172.21.15.133:6818/1465201170,v1:172.21.15.133:6819/1465201170]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:36:41.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:40 smithi133 bash[41291]: audit 2023-10-09T04:36:40.267837+0000 mon.a (mon.0) 349 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:36:41.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:40 smithi133 bash[27558]: audit 2023-10-09T04:36:39.600554+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:41.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:40 smithi133 bash[27558]: audit 2023-10-09T04:36:39.608973+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:41.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:40 smithi133 bash[27558]: cluster 2023-10-09T04:36:39.676387+0000 mgr.y (mgr.14152) 171 : cluster [DBG] pgmap v136: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:41.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:40 smithi133 bash[27558]: audit 2023-10-09T04:36:40.027263+0000 mon.a (mon.0) 346 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:36:41.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:40 smithi133 bash[27558]: audit 2023-10-09T04:36:40.028910+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:36:41.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:40 smithi133 bash[27558]: audit 2023-10-09T04:36:40.038975+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:41.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:40 smithi133 bash[27558]: audit 2023-10-09T04:36:40.267235+0000 mon.c (mon.2) 8 : audit [INF] from='osd.2 [v2:172.21.15.133:6818/1465201170,v1:172.21.15.133:6819/1465201170]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:36:41.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:40 smithi133 bash[27558]: audit 2023-10-09T04:36:40.267837+0000 mon.a (mon.0) 349 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:36:42.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:42 smithi194 bash[31648]: audit 2023-10-09T04:36:41.047042+0000 mon.a (mon.0) 350 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T04:36:42.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:42 smithi194 bash[31648]: cluster 2023-10-09T04:36:41.047147+0000 mon.a (mon.0) 351 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-10-09T04:36:42.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:42 smithi194 bash[31648]: audit 2023-10-09T04:36:41.047389+0000 mon.a (mon.0) 352 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:36:42.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:42 smithi194 bash[31648]: audit 2023-10-09T04:36:41.048102+0000 mon.c (mon.2) 9 : audit [INF] from='osd.2 [v2:172.21.15.133:6818/1465201170,v1:172.21.15.133:6819/1465201170]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:36:42.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:42 smithi194 bash[31648]: audit 2023-10-09T04:36:41.048761+0000 mon.a (mon.0) 353 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:36:42.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:42 smithi133 bash[27558]: audit 2023-10-09T04:36:41.047042+0000 mon.a (mon.0) 350 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T04:36:42.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:42 smithi133 bash[27558]: cluster 2023-10-09T04:36:41.047147+0000 mon.a (mon.0) 351 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-10-09T04:36:42.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:42 smithi133 bash[27558]: audit 2023-10-09T04:36:41.047389+0000 mon.a (mon.0) 352 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:36:42.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:42 smithi133 bash[27558]: audit 2023-10-09T04:36:41.048102+0000 mon.c (mon.2) 9 : audit [INF] from='osd.2 [v2:172.21.15.133:6818/1465201170,v1:172.21.15.133:6819/1465201170]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:36:42.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:42 smithi133 bash[27558]: audit 2023-10-09T04:36:41.048761+0000 mon.a (mon.0) 353 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:36:42.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:42 smithi133 bash[41291]: audit 2023-10-09T04:36:41.047042+0000 mon.a (mon.0) 350 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T04:36:42.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:42 smithi133 bash[41291]: cluster 2023-10-09T04:36:41.047147+0000 mon.a (mon.0) 351 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-10-09T04:36:42.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:42 smithi133 bash[41291]: audit 2023-10-09T04:36:41.047389+0000 mon.a (mon.0) 352 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:36:42.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:42 smithi133 bash[41291]: audit 2023-10-09T04:36:41.048102+0000 mon.c (mon.2) 9 : audit [INF] from='osd.2 [v2:172.21.15.133:6818/1465201170,v1:172.21.15.133:6819/1465201170]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:36:42.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:42 smithi133 bash[41291]: audit 2023-10-09T04:36:41.048761+0000 mon.a (mon.0) 353 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:36:43.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:43 smithi194 bash[31648]: cluster 2023-10-09T04:36:41.676888+0000 mgr.y (mgr.14152) 172 : cluster [DBG] pgmap v138: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:43.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:43 smithi194 bash[31648]: audit 2023-10-09T04:36:42.045106+0000 mon.a (mon.0) 354 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]': finished 2023-10-09T04:36:43.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:43 smithi194 bash[31648]: cluster 2023-10-09T04:36:42.045216+0000 mon.a (mon.0) 355 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-10-09T04:36:43.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:43 smithi194 bash[31648]: audit 2023-10-09T04:36:42.045564+0000 mon.a (mon.0) 356 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:36:43.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:43 smithi194 bash[31648]: audit 2023-10-09T04:36:42.054857+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:36:43.422 INFO:teuthology.orchestra.run.smithi133.stdout:Created osd(s) 2 on host 'smithi133' 2023-10-09T04:36:43.477 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:43 smithi133 bash[41291]: cluster 2023-10-09T04:36:41.676888+0000 mgr.y (mgr.14152) 172 : cluster [DBG] pgmap v138: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:43.477 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:43 smithi133 bash[41291]: audit 2023-10-09T04:36:42.045106+0000 mon.a (mon.0) 354 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]': finished 2023-10-09T04:36:43.478 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:43 smithi133 bash[41291]: cluster 2023-10-09T04:36:42.045216+0000 mon.a (mon.0) 355 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-10-09T04:36:43.478 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:43 smithi133 bash[41291]: audit 2023-10-09T04:36:42.045564+0000 mon.a (mon.0) 356 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:36:43.478 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:43 smithi133 bash[41291]: audit 2023-10-09T04:36:42.054857+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:36:43.478 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:43 smithi133 bash[27558]: cluster 2023-10-09T04:36:41.676888+0000 mgr.y (mgr.14152) 172 : cluster [DBG] pgmap v138: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:43.478 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:43 smithi133 bash[27558]: audit 2023-10-09T04:36:42.045106+0000 mon.a (mon.0) 354 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]': finished 2023-10-09T04:36:43.478 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:43 smithi133 bash[27558]: cluster 2023-10-09T04:36:42.045216+0000 mon.a (mon.0) 355 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-10-09T04:36:43.478 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:43 smithi133 bash[27558]: audit 2023-10-09T04:36:42.045564+0000 mon.a (mon.0) 356 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:36:43.478 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:43 smithi133 bash[27558]: audit 2023-10-09T04:36:42.054857+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:36:44.143 DEBUG:teuthology.orchestra.run.smithi133:osd.2> sudo journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@osd.2.service 2023-10-09T04:36:44.145 INFO:tasks.cephadm:Deploying osd.3 on smithi133 with /dev/vg_nvme/lv_1... 2023-10-09T04:36:44.145 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-10-09T04:36:44.403 INFO:journalctl@ceph.osd.2.smithi133.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:36:44.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:44 smithi133 bash[27558]: cluster 2023-10-09T04:36:41.220361+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:36:44.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:44 smithi133 bash[27558]: cluster 2023-10-09T04:36:41.220540+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:36:44.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:44 smithi133 bash[27558]: audit 2023-10-09T04:36:43.055324+0000 mon.a (mon.0) 358 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:36:44.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:44 smithi133 bash[27558]: cluster 2023-10-09T04:36:43.060350+0000 mon.a (mon.0) 359 : cluster [INF] osd.2 [v2:172.21.15.133:6818/1465201170,v1:172.21.15.133:6819/1465201170] boot 2023-10-09T04:36:44.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:44 smithi133 bash[27558]: cluster 2023-10-09T04:36:43.060403+0000 mon.a (mon.0) 360 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-10-09T04:36:44.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:44 smithi133 bash[27558]: audit 2023-10-09T04:36:43.060586+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:36:44.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:44 smithi133 bash[27558]: audit 2023-10-09T04:36:43.377722+0000 mon.a (mon.0) 362 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:44.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:44 smithi133 bash[27558]: audit 2023-10-09T04:36:43.378188+0000 mon.a (mon.0) 363 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:36:44.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:44 smithi133 bash[27558]: audit 2023-10-09T04:36:43.384470+0000 mon.a (mon.0) 364 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:44.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:44 smithi133 bash[41291]: cluster 2023-10-09T04:36:41.220361+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:36:44.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:44 smithi133 bash[41291]: cluster 2023-10-09T04:36:41.220540+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:36:44.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:44 smithi133 bash[41291]: audit 2023-10-09T04:36:43.055324+0000 mon.a (mon.0) 358 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:36:44.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:44 smithi133 bash[41291]: cluster 2023-10-09T04:36:43.060350+0000 mon.a (mon.0) 359 : cluster [INF] osd.2 [v2:172.21.15.133:6818/1465201170,v1:172.21.15.133:6819/1465201170] boot 2023-10-09T04:36:44.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:44 smithi133 bash[41291]: cluster 2023-10-09T04:36:43.060403+0000 mon.a (mon.0) 360 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-10-09T04:36:44.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:44 smithi133 bash[41291]: audit 2023-10-09T04:36:43.060586+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:36:44.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:44 smithi133 bash[41291]: audit 2023-10-09T04:36:43.377722+0000 mon.a (mon.0) 362 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:44.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:44 smithi133 bash[41291]: audit 2023-10-09T04:36:43.378188+0000 mon.a (mon.0) 363 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:36:44.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:44 smithi133 bash[41291]: audit 2023-10-09T04:36:43.384470+0000 mon.a (mon.0) 364 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:44.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:44 smithi194 bash[31648]: cluster 2023-10-09T04:36:41.220361+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:36:44.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:44 smithi194 bash[31648]: cluster 2023-10-09T04:36:41.220540+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:36:44.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:44 smithi194 bash[31648]: audit 2023-10-09T04:36:43.055324+0000 mon.a (mon.0) 358 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:36:44.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:44 smithi194 bash[31648]: cluster 2023-10-09T04:36:43.060350+0000 mon.a (mon.0) 359 : cluster [INF] osd.2 [v2:172.21.15.133:6818/1465201170,v1:172.21.15.133:6819/1465201170] boot 2023-10-09T04:36:44.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:44 smithi194 bash[31648]: cluster 2023-10-09T04:36:43.060403+0000 mon.a (mon.0) 360 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-10-09T04:36:44.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:44 smithi194 bash[31648]: audit 2023-10-09T04:36:43.060586+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:36:44.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:44 smithi194 bash[31648]: audit 2023-10-09T04:36:43.377722+0000 mon.a (mon.0) 362 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:44.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:44 smithi194 bash[31648]: audit 2023-10-09T04:36:43.378188+0000 mon.a (mon.0) 363 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:36:44.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:44 smithi194 bash[31648]: audit 2023-10-09T04:36:43.384470+0000 mon.a (mon.0) 364 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:45.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:45 smithi133 bash[41291]: cluster 2023-10-09T04:36:43.677305+0000 mgr.y (mgr.14152) 173 : cluster [DBG] pgmap v141: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:45.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:45 smithi133 bash[41291]: audit 2023-10-09T04:36:43.822297+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T04:36:45.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:45 smithi133 bash[41291]: audit 2023-10-09T04:36:44.272411+0000 mon.a (mon.0) 366 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T04:36:45.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:45 smithi133 bash[41291]: cluster 2023-10-09T04:36:44.272485+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-10-09T04:36:45.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:45 smithi133 bash[41291]: audit 2023-10-09T04:36:44.273399+0000 mon.a (mon.0) 368 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:36:45.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:45 smithi133 bash[27558]: cluster 2023-10-09T04:36:43.677305+0000 mgr.y (mgr.14152) 173 : cluster [DBG] pgmap v141: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:45.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:45 smithi133 bash[27558]: audit 2023-10-09T04:36:43.822297+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T04:36:45.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:45 smithi133 bash[27558]: audit 2023-10-09T04:36:44.272411+0000 mon.a (mon.0) 366 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T04:36:45.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:45 smithi133 bash[27558]: cluster 2023-10-09T04:36:44.272485+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-10-09T04:36:45.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:45 smithi133 bash[27558]: audit 2023-10-09T04:36:44.273399+0000 mon.a (mon.0) 368 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:36:45.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:45 smithi194 bash[31648]: cluster 2023-10-09T04:36:43.677305+0000 mgr.y (mgr.14152) 173 : cluster [DBG] pgmap v141: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:36:45.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:45 smithi194 bash[31648]: audit 2023-10-09T04:36:43.822297+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T04:36:45.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:45 smithi194 bash[31648]: audit 2023-10-09T04:36:44.272411+0000 mon.a (mon.0) 366 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T04:36:45.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:45 smithi194 bash[31648]: cluster 2023-10-09T04:36:44.272485+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-10-09T04:36:45.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:45 smithi194 bash[31648]: audit 2023-10-09T04:36:44.273399+0000 mon.a (mon.0) 368 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:36:46.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:46 smithi133 bash[27558]: audit 2023-10-09T04:36:45.275528+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:36:46.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:46 smithi133 bash[27558]: cluster 2023-10-09T04:36:45.275639+0000 mon.a (mon.0) 370 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-10-09T04:36:46.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:46 smithi133 bash[41291]: audit 2023-10-09T04:36:45.275528+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:36:46.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:46 smithi133 bash[41291]: cluster 2023-10-09T04:36:45.275639+0000 mon.a (mon.0) 370 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-10-09T04:36:46.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:46 smithi194 bash[31648]: audit 2023-10-09T04:36:45.275528+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:36:46.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:46 smithi194 bash[31648]: cluster 2023-10-09T04:36:45.275639+0000 mon.a (mon.0) 370 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-10-09T04:36:47.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:47 smithi194 bash[31648]: cluster 2023-10-09T04:36:45.677873+0000 mgr.y (mgr.14152) 174 : cluster [DBG] pgmap v144: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:36:47.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:47 smithi194 bash[31648]: cluster 2023-10-09T04:36:46.319178+0000 mon.a (mon.0) 371 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-10-09T04:36:47.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:47 smithi194 bash[31648]: audit 2023-10-09T04:36:47.310108+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:47.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:47 smithi133 bash[27558]: cluster 2023-10-09T04:36:45.677873+0000 mgr.y (mgr.14152) 174 : cluster [DBG] pgmap v144: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:36:47.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:47 smithi133 bash[27558]: cluster 2023-10-09T04:36:46.319178+0000 mon.a (mon.0) 371 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-10-09T04:36:47.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:47 smithi133 bash[27558]: audit 2023-10-09T04:36:47.310108+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:47.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:47 smithi133 bash[41291]: cluster 2023-10-09T04:36:45.677873+0000 mgr.y (mgr.14152) 174 : cluster [DBG] pgmap v144: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:36:47.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:47 smithi133 bash[41291]: cluster 2023-10-09T04:36:46.319178+0000 mon.a (mon.0) 371 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-10-09T04:36:47.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:47 smithi133 bash[41291]: audit 2023-10-09T04:36:47.310108+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:48.167 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:36:50.074 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:49 smithi133 bash[41291]: cluster 2023-10-09T04:36:47.678336+0000 mgr.y (mgr.14152) 175 : cluster [DBG] pgmap v146: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:36:50.074 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:49 smithi133 bash[41291]: cluster 2023-10-09T04:36:48.333245+0000 mon.a (mon.0) 373 : cluster [DBG] mgrmap e15: y(active, since 4m), standbys: x 2023-10-09T04:36:50.074 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:49 smithi133 bash[41291]: audit 2023-10-09T04:36:49.072171+0000 mon.a (mon.0) 374 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T04:36:50.074 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:49 smithi133 bash[27558]: cluster 2023-10-09T04:36:47.678336+0000 mgr.y (mgr.14152) 175 : cluster [DBG] pgmap v146: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:36:50.075 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:49 smithi133 bash[27558]: cluster 2023-10-09T04:36:48.333245+0000 mon.a (mon.0) 373 : cluster [DBG] mgrmap e15: y(active, since 4m), standbys: x 2023-10-09T04:36:50.075 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:49 smithi133 bash[27558]: audit 2023-10-09T04:36:49.072171+0000 mon.a (mon.0) 374 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T04:36:50.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:49 smithi194 bash[31648]: cluster 2023-10-09T04:36:47.678336+0000 mgr.y (mgr.14152) 175 : cluster [DBG] pgmap v146: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:36:50.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:49 smithi194 bash[31648]: cluster 2023-10-09T04:36:48.333245+0000 mon.a (mon.0) 373 : cluster [DBG] mgrmap e15: y(active, since 4m), standbys: x 2023-10-09T04:36:50.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:49 smithi194 bash[31648]: audit 2023-10-09T04:36:49.072171+0000 mon.a (mon.0) 374 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T04:36:52.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:51 smithi133 bash[27558]: cluster 2023-10-09T04:36:49.678715+0000 mgr.y (mgr.14152) 176 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:36:52.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:51 smithi133 bash[27558]: audit 2023-10-09T04:36:49.968140+0000 mon.a (mon.0) 375 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T04:36:52.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:51 smithi133 bash[27558]: audit 2023-10-09T04:36:49.969078+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:36:52.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:51 smithi133 bash[27558]: audit 2023-10-09T04:36:49.969402+0000 mon.a (mon.0) 377 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:36:52.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:51 smithi133 bash[27558]: audit 2023-10-09T04:36:49.969538+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:36:52.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:51 smithi133 bash[27558]: audit 2023-10-09T04:36:49.975689+0000 mon.a (mon.0) 379 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:36:52.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:51 smithi133 bash[27558]: audit 2023-10-09T04:36:49.975850+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:36:52.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:51 smithi133 bash[27558]: audit 2023-10-09T04:36:49.976016+0000 mon.a (mon.0) 381 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:36:52.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:51 smithi133 bash[27558]: audit 2023-10-09T04:36:49.976330+0000 mon.b (mon.1) 7 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T04:36:52.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:51 smithi133 bash[27558]: audit 2023-10-09T04:36:50.857551+0000 mon.b (mon.1) 8 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T04:36:52.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:51 smithi133 bash[27558]: audit 2023-10-09T04:36:50.886122+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:52.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:51 smithi133 bash[27558]: audit 2023-10-09T04:36:50.888670+0000 mon.a (mon.0) 383 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:36:52.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:51 smithi133 bash[27558]: audit 2023-10-09T04:36:50.888924+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:36:52.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:51 smithi133 bash[27558]: audit 2023-10-09T04:36:50.889181+0000 mon.a (mon.0) 385 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:36:52.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:51 smithi133 bash[41291]: cluster 2023-10-09T04:36:49.678715+0000 mgr.y (mgr.14152) 176 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:36:52.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:51 smithi133 bash[41291]: audit 2023-10-09T04:36:49.968140+0000 mon.a (mon.0) 375 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T04:36:52.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:51 smithi133 bash[41291]: audit 2023-10-09T04:36:49.969078+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:36:52.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:51 smithi133 bash[41291]: audit 2023-10-09T04:36:49.969402+0000 mon.a (mon.0) 377 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:36:52.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:51 smithi133 bash[41291]: audit 2023-10-09T04:36:49.969538+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:36:52.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:51 smithi133 bash[41291]: audit 2023-10-09T04:36:49.975689+0000 mon.a (mon.0) 379 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:36:52.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:51 smithi133 bash[41291]: audit 2023-10-09T04:36:49.975850+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:36:52.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:51 smithi133 bash[41291]: audit 2023-10-09T04:36:49.976016+0000 mon.a (mon.0) 381 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:36:52.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:51 smithi133 bash[41291]: audit 2023-10-09T04:36:49.976330+0000 mon.b (mon.1) 7 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T04:36:52.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:51 smithi133 bash[41291]: audit 2023-10-09T04:36:50.857551+0000 mon.b (mon.1) 8 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T04:36:52.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:51 smithi133 bash[41291]: audit 2023-10-09T04:36:50.886122+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:52.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:51 smithi133 bash[41291]: audit 2023-10-09T04:36:50.888670+0000 mon.a (mon.0) 383 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:36:52.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:51 smithi133 bash[41291]: audit 2023-10-09T04:36:50.888924+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:36:52.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:51 smithi133 bash[41291]: audit 2023-10-09T04:36:50.889181+0000 mon.a (mon.0) 385 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:36:52.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:51 smithi194 bash[31648]: cluster 2023-10-09T04:36:49.678715+0000 mgr.y (mgr.14152) 176 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:36:52.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:51 smithi194 bash[31648]: audit 2023-10-09T04:36:49.968140+0000 mon.a (mon.0) 375 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T04:36:52.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:51 smithi194 bash[31648]: audit 2023-10-09T04:36:49.969078+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:36:52.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:51 smithi194 bash[31648]: audit 2023-10-09T04:36:49.969402+0000 mon.a (mon.0) 377 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:36:52.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:51 smithi194 bash[31648]: audit 2023-10-09T04:36:49.969538+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:36:52.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:51 smithi194 bash[31648]: audit 2023-10-09T04:36:49.975689+0000 mon.a (mon.0) 379 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:36:52.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:51 smithi194 bash[31648]: audit 2023-10-09T04:36:49.975850+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:36:52.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:51 smithi194 bash[31648]: audit 2023-10-09T04:36:49.976016+0000 mon.a (mon.0) 381 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:36:52.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:51 smithi194 bash[31648]: audit 2023-10-09T04:36:49.976330+0000 mon.b (mon.1) 7 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T04:36:52.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:51 smithi194 bash[31648]: audit 2023-10-09T04:36:50.857551+0000 mon.b (mon.1) 8 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T04:36:52.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:51 smithi194 bash[31648]: audit 2023-10-09T04:36:50.886122+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:52.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:51 smithi194 bash[31648]: audit 2023-10-09T04:36:50.888670+0000 mon.a (mon.0) 383 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:36:52.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:51 smithi194 bash[31648]: audit 2023-10-09T04:36:50.888924+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:36:52.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:51 smithi194 bash[31648]: audit 2023-10-09T04:36:50.889181+0000 mon.a (mon.0) 385 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:36:53.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:52 smithi133 bash[41291]: audit 2023-10-09T04:36:50.888452+0000 mon.c (mon.2) 10 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T04:36:53.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:52 smithi133 bash[41291]: cluster 2023-10-09T04:36:51.679194+0000 mgr.y (mgr.14152) 177 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:36:53.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:52 smithi133 bash[41291]: audit 2023-10-09T04:36:51.788452+0000 mon.c (mon.2) 11 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T04:36:53.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:52 smithi133 bash[27558]: audit 2023-10-09T04:36:50.888452+0000 mon.c (mon.2) 10 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T04:36:53.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:52 smithi133 bash[27558]: cluster 2023-10-09T04:36:51.679194+0000 mgr.y (mgr.14152) 177 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:36:53.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:52 smithi133 bash[27558]: audit 2023-10-09T04:36:51.788452+0000 mon.c (mon.2) 11 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T04:36:53.064 INFO:teuthology.orchestra.run.smithi133.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-10-09T04:36:53.065 INFO:teuthology.orchestra.run.smithi133.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-10-09T04:36:53.065 INFO:teuthology.orchestra.run.smithi133.stderr: stderr: 10+0 records in 2023-10-09T04:36:53.065 INFO:teuthology.orchestra.run.smithi133.stderr:10+0 records out 2023-10-09T04:36:53.065 INFO:teuthology.orchestra.run.smithi133.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0167628 s, 626 MB/s 2023-10-09T04:36:53.065 INFO:teuthology.orchestra.run.smithi133.stderr: stderr: 2023-10-09T04:36:53.065 INFO:teuthology.orchestra.run.smithi133.stderr:--> Zapping successful for: 2023-10-09T04:36:53.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:52 smithi194 bash[31648]: audit 2023-10-09T04:36:50.888452+0000 mon.c (mon.2) 10 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T04:36:53.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:52 smithi194 bash[31648]: cluster 2023-10-09T04:36:51.679194+0000 mgr.y (mgr.14152) 177 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:36:53.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:52 smithi194 bash[31648]: audit 2023-10-09T04:36:51.788452+0000 mon.c (mon.2) 11 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T04:36:53.954 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi133:vg_nvme/lv_1 2023-10-09T04:36:55.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:55 smithi133 bash[41291]: cluster 2023-10-09T04:36:53.679537+0000 mgr.y (mgr.14152) 178 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:36:55.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:55 smithi133 bash[41291]: cephadm 2023-10-09T04:36:54.523975+0000 mgr.y (mgr.14152) 179 : cephadm [INF] Detected new or changed devices on smithi133 2023-10-09T04:36:55.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:55 smithi133 bash[41291]: audit 2023-10-09T04:36:54.534405+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:55.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:55 smithi133 bash[41291]: audit 2023-10-09T04:36:54.541935+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:55.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:55 smithi133 bash[41291]: audit 2023-10-09T04:36:54.543606+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:36:55.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:55 smithi133 bash[41291]: audit 2023-10-09T04:36:54.544958+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:36:55.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:55 smithi133 bash[41291]: audit 2023-10-09T04:36:54.546266+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:36:55.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:55 smithi133 bash[41291]: cephadm 2023-10-09T04:36:54.547250+0000 mgr.y (mgr.14152) 180 : cephadm [INF] Adjusting osd_memory_target on smithi133 to 5413M 2023-10-09T04:36:55.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:55 smithi133 bash[41291]: audit 2023-10-09T04:36:54.556325+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:55.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:55 smithi133 bash[41291]: audit 2023-10-09T04:36:54.558205+0000 mon.a (mon.0) 392 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:36:55.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:55 smithi133 bash[41291]: audit 2023-10-09T04:36:54.559479+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:36:55.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:55 smithi133 bash[41291]: audit 2023-10-09T04:36:54.569014+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:55.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:55 smithi133 bash[27558]: cluster 2023-10-09T04:36:53.679537+0000 mgr.y (mgr.14152) 178 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:36:55.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:55 smithi133 bash[27558]: cephadm 2023-10-09T04:36:54.523975+0000 mgr.y (mgr.14152) 179 : cephadm [INF] Detected new or changed devices on smithi133 2023-10-09T04:36:55.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:55 smithi133 bash[27558]: audit 2023-10-09T04:36:54.534405+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:55.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:55 smithi133 bash[27558]: audit 2023-10-09T04:36:54.541935+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:55.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:55 smithi133 bash[27558]: audit 2023-10-09T04:36:54.543606+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:36:55.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:55 smithi133 bash[27558]: audit 2023-10-09T04:36:54.544958+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:36:55.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:55 smithi133 bash[27558]: audit 2023-10-09T04:36:54.546266+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:36:55.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:55 smithi133 bash[27558]: cephadm 2023-10-09T04:36:54.547250+0000 mgr.y (mgr.14152) 180 : cephadm [INF] Adjusting osd_memory_target on smithi133 to 5413M 2023-10-09T04:36:55.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:55 smithi133 bash[27558]: audit 2023-10-09T04:36:54.556325+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:55.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:55 smithi133 bash[27558]: audit 2023-10-09T04:36:54.558205+0000 mon.a (mon.0) 392 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:36:55.803 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:55 smithi133 bash[27558]: audit 2023-10-09T04:36:54.559479+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:36:55.803 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:55 smithi133 bash[27558]: audit 2023-10-09T04:36:54.569014+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:55.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:55 smithi194 bash[31648]: cluster 2023-10-09T04:36:53.679537+0000 mgr.y (mgr.14152) 178 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:36:55.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:55 smithi194 bash[31648]: cephadm 2023-10-09T04:36:54.523975+0000 mgr.y (mgr.14152) 179 : cephadm [INF] Detected new or changed devices on smithi133 2023-10-09T04:36:55.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:55 smithi194 bash[31648]: audit 2023-10-09T04:36:54.534405+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:55.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:55 smithi194 bash[31648]: audit 2023-10-09T04:36:54.541935+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:55.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:55 smithi194 bash[31648]: audit 2023-10-09T04:36:54.543606+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:36:55.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:55 smithi194 bash[31648]: audit 2023-10-09T04:36:54.544958+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:36:55.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:55 smithi194 bash[31648]: audit 2023-10-09T04:36:54.546266+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:36:55.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:55 smithi194 bash[31648]: cephadm 2023-10-09T04:36:54.547250+0000 mgr.y (mgr.14152) 180 : cephadm [INF] Adjusting osd_memory_target on smithi133 to 5413M 2023-10-09T04:36:55.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:55 smithi194 bash[31648]: audit 2023-10-09T04:36:54.556325+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:55.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:55 smithi194 bash[31648]: audit 2023-10-09T04:36:54.558205+0000 mon.a (mon.0) 392 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:36:55.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:55 smithi194 bash[31648]: audit 2023-10-09T04:36:54.559479+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:36:55.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:55 smithi194 bash[31648]: audit 2023-10-09T04:36:54.569014+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:36:57.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:57 smithi133 bash[27558]: cluster 2023-10-09T04:36:55.680078+0000 mgr.y (mgr.14152) 181 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2023-10-09T04:36:57.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:57 smithi133 bash[41291]: cluster 2023-10-09T04:36:55.680078+0000 mgr.y (mgr.14152) 181 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2023-10-09T04:36:57.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:57 smithi194 bash[31648]: cluster 2023-10-09T04:36:55.680078+0000 mgr.y (mgr.14152) 181 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2023-10-09T04:36:58.707 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:36:59.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:36:59 smithi133 bash[41291]: cluster 2023-10-09T04:36:57.680666+0000 mgr.y (mgr.14152) 182 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2023-10-09T04:36:59.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:36:59 smithi133 bash[27558]: cluster 2023-10-09T04:36:57.680666+0000 mgr.y (mgr.14152) 182 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2023-10-09T04:36:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:36:59 smithi194 bash[31648]: cluster 2023-10-09T04:36:57.680666+0000 mgr.y (mgr.14152) 182 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2023-10-09T04:37:01.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:01 smithi133 bash[27558]: cluster 2023-10-09T04:36:59.681214+0000 mgr.y (mgr.14152) 183 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-10-09T04:37:01.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:01 smithi133 bash[41291]: cluster 2023-10-09T04:36:59.681214+0000 mgr.y (mgr.14152) 183 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-10-09T04:37:01.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:01 smithi194 bash[31648]: cluster 2023-10-09T04:36:59.681214+0000 mgr.y (mgr.14152) 183 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-10-09T04:37:02.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:02 smithi194 bash[31648]: audit 2023-10-09T04:37:02.034926+0000 mon.a (mon.0) 395 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:37:02.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:02 smithi194 bash[31648]: audit 2023-10-09T04:37:02.040089+0000 mon.a (mon.0) 396 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:37:02.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:02 smithi194 bash[31648]: audit 2023-10-09T04:37:02.041242+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:37:03.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:02 smithi133 bash[27558]: audit 2023-10-09T04:37:02.034926+0000 mon.a (mon.0) 395 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:37:03.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:02 smithi133 bash[27558]: audit 2023-10-09T04:37:02.040089+0000 mon.a (mon.0) 396 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:37:03.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:02 smithi133 bash[27558]: audit 2023-10-09T04:37:02.041242+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:37:03.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:02 smithi133 bash[41291]: audit 2023-10-09T04:37:02.034926+0000 mon.a (mon.0) 395 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:37:03.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:02 smithi133 bash[41291]: audit 2023-10-09T04:37:02.040089+0000 mon.a (mon.0) 396 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:37:03.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:02 smithi133 bash[41291]: audit 2023-10-09T04:37:02.041242+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:37:03.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:03 smithi194 bash[31648]: cluster 2023-10-09T04:37:01.681782+0000 mgr.y (mgr.14152) 184 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T04:37:03.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:03 smithi194 bash[31648]: audit 2023-10-09T04:37:02.028909+0000 mgr.y (mgr.14152) 185 : audit [DBG] from='client.24212 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi133:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:37:04.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:03 smithi133 bash[27558]: cluster 2023-10-09T04:37:01.681782+0000 mgr.y (mgr.14152) 184 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T04:37:04.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:03 smithi133 bash[27558]: audit 2023-10-09T04:37:02.028909+0000 mgr.y (mgr.14152) 185 : audit [DBG] from='client.24212 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi133:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:37:04.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:03 smithi133 bash[41291]: cluster 2023-10-09T04:37:01.681782+0000 mgr.y (mgr.14152) 184 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T04:37:04.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:03 smithi133 bash[41291]: audit 2023-10-09T04:37:02.028909+0000 mgr.y (mgr.14152) 185 : audit [DBG] from='client.24212 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi133:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:37:05.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:05 smithi194 bash[31648]: cluster 2023-10-09T04:37:03.682391+0000 mgr.y (mgr.14152) 186 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T04:37:06.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:05 smithi133 bash[27558]: cluster 2023-10-09T04:37:03.682391+0000 mgr.y (mgr.14152) 186 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T04:37:06.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:05 smithi133 bash[41291]: cluster 2023-10-09T04:37:03.682391+0000 mgr.y (mgr.14152) 186 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T04:37:07.876 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:07 smithi133 bash[27558]: cluster 2023-10-09T04:37:05.682974+0000 mgr.y (mgr.14152) 187 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T04:37:07.876 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:07 smithi133 bash[41291]: cluster 2023-10-09T04:37:05.682974+0000 mgr.y (mgr.14152) 187 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T04:37:07.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:07 smithi194 bash[31648]: cluster 2023-10-09T04:37:05.682974+0000 mgr.y (mgr.14152) 187 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T04:37:09.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:09 smithi133 bash[27558]: cluster 2023-10-09T04:37:07.683507+0000 mgr.y (mgr.14152) 188 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:09.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:09 smithi133 bash[41291]: cluster 2023-10-09T04:37:07.683507+0000 mgr.y (mgr.14152) 188 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:09.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:09 smithi194 bash[31648]: cluster 2023-10-09T04:37:07.683507+0000 mgr.y (mgr.14152) 188 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:11.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:11 smithi194 bash[31648]: cluster 2023-10-09T04:37:09.684036+0000 mgr.y (mgr.14152) 189 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:11.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:11 smithi194 bash[31648]: audit 2023-10-09T04:37:10.430321+0000 mon.a (mon.0) 398 : audit [INF] from='client.? 172.21.15.133:0/4169938713' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "56bbf807-c8f7-4226-b46e-802b539274ca"}]: dispatch 2023-10-09T04:37:11.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:11 smithi194 bash[31648]: audit 2023-10-09T04:37:10.437633+0000 mon.a (mon.0) 399 : audit [INF] from='client.? 172.21.15.133:0/4169938713' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "56bbf807-c8f7-4226-b46e-802b539274ca"}]': finished 2023-10-09T04:37:11.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:11 smithi194 bash[31648]: cluster 2023-10-09T04:37:10.437770+0000 mon.a (mon.0) 400 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-10-09T04:37:11.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:11 smithi194 bash[31648]: audit 2023-10-09T04:37:10.437990+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:37:11.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:11 smithi194 bash[31648]: audit 2023-10-09T04:37:11.203819+0000 mon.a (mon.0) 402 : audit [DBG] from='client.? 172.21.15.133:0/3588526447' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:37:11.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:11 smithi133 bash[27558]: cluster 2023-10-09T04:37:09.684036+0000 mgr.y (mgr.14152) 189 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:11.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:11 smithi133 bash[27558]: audit 2023-10-09T04:37:10.430321+0000 mon.a (mon.0) 398 : audit [INF] from='client.? 172.21.15.133:0/4169938713' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "56bbf807-c8f7-4226-b46e-802b539274ca"}]: dispatch 2023-10-09T04:37:11.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:11 smithi133 bash[27558]: audit 2023-10-09T04:37:10.437633+0000 mon.a (mon.0) 399 : audit [INF] from='client.? 172.21.15.133:0/4169938713' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "56bbf807-c8f7-4226-b46e-802b539274ca"}]': finished 2023-10-09T04:37:11.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:11 smithi133 bash[27558]: cluster 2023-10-09T04:37:10.437770+0000 mon.a (mon.0) 400 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-10-09T04:37:11.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:11 smithi133 bash[27558]: audit 2023-10-09T04:37:10.437990+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:37:11.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:11 smithi133 bash[27558]: audit 2023-10-09T04:37:11.203819+0000 mon.a (mon.0) 402 : audit [DBG] from='client.? 172.21.15.133:0/3588526447' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:37:11.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:11 smithi133 bash[41291]: cluster 2023-10-09T04:37:09.684036+0000 mgr.y (mgr.14152) 189 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:11.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:11 smithi133 bash[41291]: audit 2023-10-09T04:37:10.430321+0000 mon.a (mon.0) 398 : audit [INF] from='client.? 172.21.15.133:0/4169938713' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "56bbf807-c8f7-4226-b46e-802b539274ca"}]: dispatch 2023-10-09T04:37:11.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:11 smithi133 bash[41291]: audit 2023-10-09T04:37:10.437633+0000 mon.a (mon.0) 399 : audit [INF] from='client.? 172.21.15.133:0/4169938713' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "56bbf807-c8f7-4226-b46e-802b539274ca"}]': finished 2023-10-09T04:37:11.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:11 smithi133 bash[41291]: cluster 2023-10-09T04:37:10.437770+0000 mon.a (mon.0) 400 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-10-09T04:37:11.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:11 smithi133 bash[41291]: audit 2023-10-09T04:37:10.437990+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:37:11.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:11 smithi133 bash[41291]: audit 2023-10-09T04:37:11.203819+0000 mon.a (mon.0) 402 : audit [DBG] from='client.? 172.21.15.133:0/3588526447' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:37:13.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:13 smithi194 bash[31648]: cluster 2023-10-09T04:37:11.684616+0000 mgr.y (mgr.14152) 190 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:13.757 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:13 smithi133 bash[41291]: cluster 2023-10-09T04:37:11.684616+0000 mgr.y (mgr.14152) 190 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:13.758 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:13 smithi133 bash[27558]: cluster 2023-10-09T04:37:11.684616+0000 mgr.y (mgr.14152) 190 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:15.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:15 smithi194 bash[31648]: cluster 2023-10-09T04:37:13.685156+0000 mgr.y (mgr.14152) 191 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:15.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:15 smithi133 bash[41291]: cluster 2023-10-09T04:37:13.685156+0000 mgr.y (mgr.14152) 191 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:15.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:15 smithi133 bash[27558]: cluster 2023-10-09T04:37:13.685156+0000 mgr.y (mgr.14152) 191 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:17.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:17 smithi194 bash[31648]: cluster 2023-10-09T04:37:15.685748+0000 mgr.y (mgr.14152) 192 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:17.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:17 smithi133 bash[27558]: cluster 2023-10-09T04:37:15.685748+0000 mgr.y (mgr.14152) 192 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:17.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:17 smithi133 bash[41291]: cluster 2023-10-09T04:37:15.685748+0000 mgr.y (mgr.14152) 192 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:19.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:19 smithi194 bash[31648]: cluster 2023-10-09T04:37:17.686195+0000 mgr.y (mgr.14152) 193 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:19.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:19 smithi133 bash[41291]: cluster 2023-10-09T04:37:17.686195+0000 mgr.y (mgr.14152) 193 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:19.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:19 smithi133 bash[27558]: cluster 2023-10-09T04:37:17.686195+0000 mgr.y (mgr.14152) 193 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:21.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:21 smithi133 bash[41291]: cluster 2023-10-09T04:37:19.686602+0000 mgr.y (mgr.14152) 194 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:21.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:21 smithi133 bash[27558]: cluster 2023-10-09T04:37:19.686602+0000 mgr.y (mgr.14152) 194 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:21.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:21 smithi194 bash[31648]: cluster 2023-10-09T04:37:19.686602+0000 mgr.y (mgr.14152) 194 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:23.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:23 smithi194 bash[31648]: cluster 2023-10-09T04:37:21.687100+0000 mgr.y (mgr.14152) 195 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:23.680 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:23 smithi133 bash[41291]: cluster 2023-10-09T04:37:21.687100+0000 mgr.y (mgr.14152) 195 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:23.681 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:23 smithi133 bash[27558]: cluster 2023-10-09T04:37:21.687100+0000 mgr.y (mgr.14152) 195 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:24.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:24 smithi194 bash[31648]: audit 2023-10-09T04:37:23.400932+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T04:37:24.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:24 smithi194 bash[31648]: audit 2023-10-09T04:37:23.402657+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:37:24.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:24 smithi194 bash[31648]: cephadm 2023-10-09T04:37:23.403979+0000 mgr.y (mgr.14152) 196 : cephadm [INF] Deploying daemon osd.3 on smithi133 2023-10-09T04:37:24.679 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:24 smithi133 bash[41291]: audit 2023-10-09T04:37:23.400932+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T04:37:24.679 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:24 smithi133 bash[41291]: audit 2023-10-09T04:37:23.402657+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:37:24.679 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:24 smithi133 bash[41291]: cephadm 2023-10-09T04:37:23.403979+0000 mgr.y (mgr.14152) 196 : cephadm [INF] Deploying daemon osd.3 on smithi133 2023-10-09T04:37:24.679 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:24 smithi133 bash[27558]: audit 2023-10-09T04:37:23.400932+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T04:37:24.679 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:24 smithi133 bash[27558]: audit 2023-10-09T04:37:23.402657+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:37:24.679 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:24 smithi133 bash[27558]: cephadm 2023-10-09T04:37:23.403979+0000 mgr.y (mgr.14152) 196 : cephadm [INF] Deploying daemon osd.3 on smithi133 2023-10-09T04:37:25.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:25 smithi194 bash[31648]: cluster 2023-10-09T04:37:23.687549+0000 mgr.y (mgr.14152) 197 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:25.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:25 smithi133 bash[27558]: cluster 2023-10-09T04:37:23.687549+0000 mgr.y (mgr.14152) 197 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:25.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:25 smithi133 bash[41291]: cluster 2023-10-09T04:37:23.687549+0000 mgr.y (mgr.14152) 197 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:27.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:27 smithi194 bash[31648]: cluster 2023-10-09T04:37:25.688009+0000 mgr.y (mgr.14152) 198 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:27.674 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:27 smithi133 bash[41291]: cluster 2023-10-09T04:37:25.688009+0000 mgr.y (mgr.14152) 198 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:27.674 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:27 smithi133 bash[27558]: cluster 2023-10-09T04:37:25.688009+0000 mgr.y (mgr.14152) 198 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:29.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:29 smithi133 bash[41291]: cluster 2023-10-09T04:37:27.688529+0000 mgr.y (mgr.14152) 199 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:29.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:29 smithi133 bash[41291]: audit 2023-10-09T04:37:28.173872+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:29.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:29 smithi133 bash[41291]: audit 2023-10-09T04:37:28.186588+0000 mon.a (mon.0) 406 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:29.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:29 smithi133 bash[41291]: audit 2023-10-09T04:37:28.187096+0000 mon.a (mon.0) 407 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:37:29.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:29 smithi133 bash[27558]: cluster 2023-10-09T04:37:27.688529+0000 mgr.y (mgr.14152) 199 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:29.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:29 smithi133 bash[27558]: audit 2023-10-09T04:37:28.173872+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:29.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:29 smithi133 bash[27558]: audit 2023-10-09T04:37:28.186588+0000 mon.a (mon.0) 406 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:29.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:29 smithi133 bash[27558]: audit 2023-10-09T04:37:28.187096+0000 mon.a (mon.0) 407 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:37:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:29 smithi194 bash[31648]: cluster 2023-10-09T04:37:27.688529+0000 mgr.y (mgr.14152) 199 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:29 smithi194 bash[31648]: audit 2023-10-09T04:37:28.173872+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:29 smithi194 bash[31648]: audit 2023-10-09T04:37:28.186588+0000 mon.a (mon.0) 406 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:29 smithi194 bash[31648]: audit 2023-10-09T04:37:28.187096+0000 mon.a (mon.0) 407 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:37:31.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:31 smithi194 bash[31648]: cluster 2023-10-09T04:37:29.689037+0000 mgr.y (mgr.14152) 200 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:31.640 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:31 smithi133 bash[41291]: cluster 2023-10-09T04:37:29.689037+0000 mgr.y (mgr.14152) 200 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:31.640 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:31 smithi133 bash[27558]: cluster 2023-10-09T04:37:29.689037+0000 mgr.y (mgr.14152) 200 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:33.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:33 smithi194 bash[31648]: cluster 2023-10-09T04:37:31.689479+0000 mgr.y (mgr.14152) 201 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:33.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:33 smithi133 bash[27558]: cluster 2023-10-09T04:37:31.689479+0000 mgr.y (mgr.14152) 201 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:33.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:33 smithi133 bash[41291]: cluster 2023-10-09T04:37:31.689479+0000 mgr.y (mgr.14152) 201 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:35.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:35 smithi133 bash[27558]: cluster 2023-10-09T04:37:33.689947+0000 mgr.y (mgr.14152) 202 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:35.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:35 smithi133 bash[27558]: audit 2023-10-09T04:37:34.680650+0000 mon.a (mon.0) 408 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:35.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:35 smithi133 bash[27558]: audit 2023-10-09T04:37:34.689703+0000 mon.a (mon.0) 409 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:35.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:35 smithi133 bash[27558]: audit 2023-10-09T04:37:34.691150+0000 mon.a (mon.0) 410 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:37:35.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:35 smithi133 bash[27558]: audit 2023-10-09T04:37:34.692302+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:37:35.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:35 smithi133 bash[27558]: audit 2023-10-09T04:37:34.700174+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:35.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:35 smithi133 bash[27558]: audit 2023-10-09T04:37:35.270288+0000 mon.c (mon.2) 12 : audit [INF] from='osd.3 [v2:172.21.15.133:6826/1093481727,v1:172.21.15.133:6827/1093481727]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T04:37:35.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:35 smithi133 bash[27558]: audit 2023-10-09T04:37:35.270923+0000 mon.a (mon.0) 413 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T04:37:35.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:35 smithi133 bash[41291]: cluster 2023-10-09T04:37:33.689947+0000 mgr.y (mgr.14152) 202 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:35.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:35 smithi133 bash[41291]: audit 2023-10-09T04:37:34.680650+0000 mon.a (mon.0) 408 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:35.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:35 smithi133 bash[41291]: audit 2023-10-09T04:37:34.689703+0000 mon.a (mon.0) 409 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:35.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:35 smithi133 bash[41291]: audit 2023-10-09T04:37:34.691150+0000 mon.a (mon.0) 410 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:37:35.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:35 smithi133 bash[41291]: audit 2023-10-09T04:37:34.692302+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:37:35.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:35 smithi133 bash[41291]: audit 2023-10-09T04:37:34.700174+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:35.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:35 smithi133 bash[41291]: audit 2023-10-09T04:37:35.270288+0000 mon.c (mon.2) 12 : audit [INF] from='osd.3 [v2:172.21.15.133:6826/1093481727,v1:172.21.15.133:6827/1093481727]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T04:37:35.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:35 smithi133 bash[41291]: audit 2023-10-09T04:37:35.270923+0000 mon.a (mon.0) 413 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T04:37:35.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:35 smithi194 bash[31648]: cluster 2023-10-09T04:37:33.689947+0000 mgr.y (mgr.14152) 202 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:35.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:35 smithi194 bash[31648]: audit 2023-10-09T04:37:34.680650+0000 mon.a (mon.0) 408 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:35.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:35 smithi194 bash[31648]: audit 2023-10-09T04:37:34.689703+0000 mon.a (mon.0) 409 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:35.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:35 smithi194 bash[31648]: audit 2023-10-09T04:37:34.691150+0000 mon.a (mon.0) 410 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:37:35.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:35 smithi194 bash[31648]: audit 2023-10-09T04:37:34.692302+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:37:35.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:35 smithi194 bash[31648]: audit 2023-10-09T04:37:34.700174+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:35.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:35 smithi194 bash[31648]: audit 2023-10-09T04:37:35.270288+0000 mon.c (mon.2) 12 : audit [INF] from='osd.3 [v2:172.21.15.133:6826/1093481727,v1:172.21.15.133:6827/1093481727]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T04:37:35.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:35 smithi194 bash[31648]: audit 2023-10-09T04:37:35.270923+0000 mon.a (mon.0) 413 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T04:37:37.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:36 smithi133 bash[27558]: cluster 2023-10-09T04:37:35.690493+0000 mgr.y (mgr.14152) 203 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:37.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:36 smithi133 bash[27558]: audit 2023-10-09T04:37:35.708258+0000 mon.a (mon.0) 414 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T04:37:37.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:36 smithi133 bash[27558]: cluster 2023-10-09T04:37:35.708356+0000 mon.a (mon.0) 415 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-10-09T04:37:37.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:36 smithi133 bash[27558]: audit 2023-10-09T04:37:35.708527+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:37:37.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:36 smithi133 bash[27558]: audit 2023-10-09T04:37:35.709180+0000 mon.c (mon.2) 13 : audit [INF] from='osd.3 [v2:172.21.15.133:6826/1093481727,v1:172.21.15.133:6827/1093481727]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:37:37.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:36 smithi133 bash[27558]: audit 2023-10-09T04:37:35.709608+0000 mon.a (mon.0) 417 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:37:37.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:36 smithi133 bash[41291]: cluster 2023-10-09T04:37:35.690493+0000 mgr.y (mgr.14152) 203 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:37.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:36 smithi133 bash[41291]: audit 2023-10-09T04:37:35.708258+0000 mon.a (mon.0) 414 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T04:37:37.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:36 smithi133 bash[41291]: cluster 2023-10-09T04:37:35.708356+0000 mon.a (mon.0) 415 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-10-09T04:37:37.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:36 smithi133 bash[41291]: audit 2023-10-09T04:37:35.708527+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:37:37.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:36 smithi133 bash[41291]: audit 2023-10-09T04:37:35.709180+0000 mon.c (mon.2) 13 : audit [INF] from='osd.3 [v2:172.21.15.133:6826/1093481727,v1:172.21.15.133:6827/1093481727]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:37:37.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:36 smithi133 bash[41291]: audit 2023-10-09T04:37:35.709608+0000 mon.a (mon.0) 417 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:37:37.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:36 smithi194 bash[31648]: cluster 2023-10-09T04:37:35.690493+0000 mgr.y (mgr.14152) 203 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:37.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:36 smithi194 bash[31648]: audit 2023-10-09T04:37:35.708258+0000 mon.a (mon.0) 414 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T04:37:37.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:36 smithi194 bash[31648]: cluster 2023-10-09T04:37:35.708356+0000 mon.a (mon.0) 415 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-10-09T04:37:37.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:36 smithi194 bash[31648]: audit 2023-10-09T04:37:35.708527+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:37:37.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:36 smithi194 bash[31648]: audit 2023-10-09T04:37:35.709180+0000 mon.c (mon.2) 13 : audit [INF] from='osd.3 [v2:172.21.15.133:6826/1093481727,v1:172.21.15.133:6827/1093481727]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:37:37.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:36 smithi194 bash[31648]: audit 2023-10-09T04:37:35.709608+0000 mon.a (mon.0) 417 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:37:38.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:37 smithi133 bash[41291]: audit 2023-10-09T04:37:36.710256+0000 mon.a (mon.0) 418 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]': finished 2023-10-09T04:37:38.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:37 smithi133 bash[41291]: cluster 2023-10-09T04:37:36.710422+0000 mon.a (mon.0) 419 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-10-09T04:37:38.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:37 smithi133 bash[41291]: audit 2023-10-09T04:37:36.711145+0000 mon.a (mon.0) 420 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:37:38.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:37 smithi133 bash[41291]: audit 2023-10-09T04:37:36.717144+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:37:38.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:37 smithi133 bash[27558]: audit 2023-10-09T04:37:36.710256+0000 mon.a (mon.0) 418 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]': finished 2023-10-09T04:37:38.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:37 smithi133 bash[27558]: cluster 2023-10-09T04:37:36.710422+0000 mon.a (mon.0) 419 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-10-09T04:37:38.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:37 smithi133 bash[27558]: audit 2023-10-09T04:37:36.711145+0000 mon.a (mon.0) 420 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:37:38.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:37 smithi133 bash[27558]: audit 2023-10-09T04:37:36.717144+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:37:38.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:37 smithi194 bash[31648]: audit 2023-10-09T04:37:36.710256+0000 mon.a (mon.0) 418 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]': finished 2023-10-09T04:37:38.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:37 smithi194 bash[31648]: cluster 2023-10-09T04:37:36.710422+0000 mon.a (mon.0) 419 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-10-09T04:37:38.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:37 smithi194 bash[31648]: audit 2023-10-09T04:37:36.711145+0000 mon.a (mon.0) 420 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:37:38.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:37 smithi194 bash[31648]: audit 2023-10-09T04:37:36.717144+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:37:38.449 INFO:teuthology.orchestra.run.smithi133.stdout:Created osd(s) 3 on host 'smithi133' 2023-10-09T04:37:38.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:38 smithi133 bash[27558]: cluster 2023-10-09T04:37:36.237200+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:37:38.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:38 smithi133 bash[27558]: cluster 2023-10-09T04:37:36.237427+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:37:38.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:38 smithi133 bash[27558]: cluster 2023-10-09T04:37:37.691039+0000 mgr.y (mgr.14152) 204 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:38.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:38 smithi133 bash[27558]: cluster 2023-10-09T04:37:37.720051+0000 mon.a (mon.0) 422 : cluster [INF] osd.3 [v2:172.21.15.133:6826/1093481727,v1:172.21.15.133:6827/1093481727] boot 2023-10-09T04:37:38.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:38 smithi133 bash[27558]: cluster 2023-10-09T04:37:37.720157+0000 mon.a (mon.0) 423 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-10-09T04:37:38.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:38 smithi133 bash[27558]: audit 2023-10-09T04:37:37.720794+0000 mon.a (mon.0) 424 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:37:38.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:38 smithi133 bash[27558]: audit 2023-10-09T04:37:38.421516+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:38.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:38 smithi133 bash[27558]: audit 2023-10-09T04:37:38.428301+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:38.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:38 smithi133 bash[27558]: audit 2023-10-09T04:37:38.441447+0000 mon.a (mon.0) 427 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:37:38.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:38 smithi133 bash[41291]: cluster 2023-10-09T04:37:36.237200+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:37:38.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:38 smithi133 bash[41291]: cluster 2023-10-09T04:37:36.237427+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:37:38.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:38 smithi133 bash[41291]: cluster 2023-10-09T04:37:37.691039+0000 mgr.y (mgr.14152) 204 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:38.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:38 smithi133 bash[41291]: cluster 2023-10-09T04:37:37.720051+0000 mon.a (mon.0) 422 : cluster [INF] osd.3 [v2:172.21.15.133:6826/1093481727,v1:172.21.15.133:6827/1093481727] boot 2023-10-09T04:37:38.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:38 smithi133 bash[41291]: cluster 2023-10-09T04:37:37.720157+0000 mon.a (mon.0) 423 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-10-09T04:37:38.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:38 smithi133 bash[41291]: audit 2023-10-09T04:37:37.720794+0000 mon.a (mon.0) 424 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:37:38.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:38 smithi133 bash[41291]: audit 2023-10-09T04:37:38.421516+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:38.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:38 smithi133 bash[41291]: audit 2023-10-09T04:37:38.428301+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:38.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:38 smithi133 bash[41291]: audit 2023-10-09T04:37:38.441447+0000 mon.a (mon.0) 427 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:37:39.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:38 smithi194 bash[31648]: cluster 2023-10-09T04:37:36.237200+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:37:39.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:38 smithi194 bash[31648]: cluster 2023-10-09T04:37:36.237427+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:37:39.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:38 smithi194 bash[31648]: cluster 2023-10-09T04:37:37.691039+0000 mgr.y (mgr.14152) 204 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:37:39.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:38 smithi194 bash[31648]: cluster 2023-10-09T04:37:37.720051+0000 mon.a (mon.0) 422 : cluster [INF] osd.3 [v2:172.21.15.133:6826/1093481727,v1:172.21.15.133:6827/1093481727] boot 2023-10-09T04:37:39.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:38 smithi194 bash[31648]: cluster 2023-10-09T04:37:37.720157+0000 mon.a (mon.0) 423 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-10-09T04:37:39.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:38 smithi194 bash[31648]: audit 2023-10-09T04:37:37.720794+0000 mon.a (mon.0) 424 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:37:39.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:38 smithi194 bash[31648]: audit 2023-10-09T04:37:38.421516+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:39.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:38 smithi194 bash[31648]: audit 2023-10-09T04:37:38.428301+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:39.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:38 smithi194 bash[31648]: audit 2023-10-09T04:37:38.441447+0000 mon.a (mon.0) 427 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:37:39.164 DEBUG:teuthology.orchestra.run.smithi133:osd.3> sudo journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@osd.3.service 2023-10-09T04:37:39.166 INFO:tasks.cephadm:Deploying osd.4 on smithi194 with /dev/vg_nvme/lv_4... 2023-10-09T04:37:39.166 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-09T04:37:39.379 INFO:journalctl@ceph.osd.3.smithi133.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:37:40.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:39 smithi133 bash[27558]: cluster 2023-10-09T04:37:38.734361+0000 mon.a (mon.0) 428 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-10-09T04:37:40.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:39 smithi133 bash[41291]: cluster 2023-10-09T04:37:38.734361+0000 mon.a (mon.0) 428 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-10-09T04:37:40.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:39 smithi194 bash[31648]: cluster 2023-10-09T04:37:38.734361+0000 mon.a (mon.0) 428 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-10-09T04:37:41.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:41 smithi194 bash[31648]: cluster 2023-10-09T04:37:39.691600+0000 mgr.y (mgr.14152) 205 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:37:41.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:41 smithi133 bash[27558]: cluster 2023-10-09T04:37:39.691600+0000 mgr.y (mgr.14152) 205 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:37:41.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:41 smithi133 bash[41291]: cluster 2023-10-09T04:37:39.691600+0000 mgr.y (mgr.14152) 205 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:37:43.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:43 smithi133 bash[27558]: cluster 2023-10-09T04:37:41.692072+0000 mgr.y (mgr.14152) 206 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:37:43.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:43 smithi133 bash[41291]: cluster 2023-10-09T04:37:41.692072+0000 mgr.y (mgr.14152) 206 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:37:43.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:43 smithi194 bash[31648]: cluster 2023-10-09T04:37:41.692072+0000 mgr.y (mgr.14152) 206 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:37:43.886 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:37:45.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:45 smithi133 bash[27558]: cluster 2023-10-09T04:37:43.692588+0000 mgr.y (mgr.14152) 207 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:37:45.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:45 smithi133 bash[41291]: cluster 2023-10-09T04:37:43.692588+0000 mgr.y (mgr.14152) 207 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:37:45.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:45 smithi194 bash[31648]: cluster 2023-10-09T04:37:43.692588+0000 mgr.y (mgr.14152) 207 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:37:47.314 INFO:teuthology.orchestra.run.smithi194.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-09T04:37:47.314 INFO:teuthology.orchestra.run.smithi194.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-09T04:37:47.315 INFO:teuthology.orchestra.run.smithi194.stderr: stderr: 10+0 records in 2023-10-09T04:37:47.315 INFO:teuthology.orchestra.run.smithi194.stderr:10+0 records out 2023-10-09T04:37:47.315 INFO:teuthology.orchestra.run.smithi194.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0252491 s, 415 MB/s 2023-10-09T04:37:47.315 INFO:teuthology.orchestra.run.smithi194.stderr:--> Zapping successful for: 2023-10-09T04:37:47.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:47 smithi194 bash[31648]: cluster 2023-10-09T04:37:45.693218+0000 mgr.y (mgr.14152) 208 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:37:47.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:47 smithi133 bash[27558]: cluster 2023-10-09T04:37:45.693218+0000 mgr.y (mgr.14152) 208 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:37:47.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:47 smithi133 bash[41291]: cluster 2023-10-09T04:37:45.693218+0000 mgr.y (mgr.14152) 208 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:37:48.038 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi194:vg_nvme/lv_4 2023-10-09T04:37:49.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:49 smithi194 bash[31648]: cluster 2023-10-09T04:37:47.693725+0000 mgr.y (mgr.14152) 209 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:37:49.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:49 smithi194 bash[31648]: audit 2023-10-09T04:37:48.745670+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:49.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:49 smithi194 bash[31648]: audit 2023-10-09T04:37:48.754393+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:49.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:49 smithi194 bash[31648]: audit 2023-10-09T04:37:48.756190+0000 mon.a (mon.0) 431 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:37:49.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:49 smithi194 bash[31648]: audit 2023-10-09T04:37:48.757684+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:37:49.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:49 smithi194 bash[31648]: audit 2023-10-09T04:37:48.759000+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:37:49.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:49 smithi194 bash[31648]: audit 2023-10-09T04:37:48.760713+0000 mon.a (mon.0) 434 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:37:49.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:49 smithi194 bash[31648]: audit 2023-10-09T04:37:48.768647+0000 mon.a (mon.0) 435 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:49.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:49 smithi194 bash[31648]: audit 2023-10-09T04:37:48.770459+0000 mon.a (mon.0) 436 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:37:49.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:49 smithi194 bash[31648]: audit 2023-10-09T04:37:48.771492+0000 mon.a (mon.0) 437 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:37:49.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:49 smithi194 bash[31648]: audit 2023-10-09T04:37:48.777915+0000 mon.a (mon.0) 438 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:49.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:49 smithi133 bash[27558]: cluster 2023-10-09T04:37:47.693725+0000 mgr.y (mgr.14152) 209 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:37:49.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:49 smithi133 bash[27558]: audit 2023-10-09T04:37:48.745670+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:49.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:49 smithi133 bash[27558]: audit 2023-10-09T04:37:48.754393+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:49.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:49 smithi133 bash[27558]: audit 2023-10-09T04:37:48.756190+0000 mon.a (mon.0) 431 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:37:49.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:49 smithi133 bash[27558]: audit 2023-10-09T04:37:48.757684+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:37:49.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:49 smithi133 bash[27558]: audit 2023-10-09T04:37:48.759000+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:37:49.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:49 smithi133 bash[27558]: audit 2023-10-09T04:37:48.760713+0000 mon.a (mon.0) 434 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:37:49.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:49 smithi133 bash[27558]: audit 2023-10-09T04:37:48.768647+0000 mon.a (mon.0) 435 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:49.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:49 smithi133 bash[27558]: audit 2023-10-09T04:37:48.770459+0000 mon.a (mon.0) 436 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:37:49.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:49 smithi133 bash[27558]: audit 2023-10-09T04:37:48.771492+0000 mon.a (mon.0) 437 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:37:49.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:49 smithi133 bash[27558]: audit 2023-10-09T04:37:48.777915+0000 mon.a (mon.0) 438 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:49.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:49 smithi133 bash[41291]: cluster 2023-10-09T04:37:47.693725+0000 mgr.y (mgr.14152) 209 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:37:49.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:49 smithi133 bash[41291]: audit 2023-10-09T04:37:48.745670+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:49.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:49 smithi133 bash[41291]: audit 2023-10-09T04:37:48.754393+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:49.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:49 smithi133 bash[41291]: audit 2023-10-09T04:37:48.756190+0000 mon.a (mon.0) 431 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:37:49.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:49 smithi133 bash[41291]: audit 2023-10-09T04:37:48.757684+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:37:49.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:49 smithi133 bash[41291]: audit 2023-10-09T04:37:48.759000+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:37:49.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:49 smithi133 bash[41291]: audit 2023-10-09T04:37:48.760713+0000 mon.a (mon.0) 434 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:37:49.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:49 smithi133 bash[41291]: audit 2023-10-09T04:37:48.768647+0000 mon.a (mon.0) 435 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:49.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:49 smithi133 bash[41291]: audit 2023-10-09T04:37:48.770459+0000 mon.a (mon.0) 436 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:37:49.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:49 smithi133 bash[41291]: audit 2023-10-09T04:37:48.771492+0000 mon.a (mon.0) 437 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:37:49.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:49 smithi133 bash[41291]: audit 2023-10-09T04:37:48.777915+0000 mon.a (mon.0) 438 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:37:50.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:50 smithi194 bash[31648]: cephadm 2023-10-09T04:37:48.736084+0000 mgr.y (mgr.14152) 210 : cephadm [INF] Detected new or changed devices on smithi133 2023-10-09T04:37:50.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:50 smithi194 bash[31648]: cephadm 2023-10-09T04:37:48.762587+0000 mgr.y (mgr.14152) 211 : cephadm [INF] Adjusting osd_memory_target on smithi133 to 4060M 2023-10-09T04:37:50.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:50 smithi133 bash[27558]: cephadm 2023-10-09T04:37:48.736084+0000 mgr.y (mgr.14152) 210 : cephadm [INF] Detected new or changed devices on smithi133 2023-10-09T04:37:50.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:50 smithi133 bash[27558]: cephadm 2023-10-09T04:37:48.762587+0000 mgr.y (mgr.14152) 211 : cephadm [INF] Adjusting osd_memory_target on smithi133 to 4060M 2023-10-09T04:37:50.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:50 smithi133 bash[41291]: cephadm 2023-10-09T04:37:48.736084+0000 mgr.y (mgr.14152) 210 : cephadm [INF] Detected new or changed devices on smithi133 2023-10-09T04:37:50.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:50 smithi133 bash[41291]: cephadm 2023-10-09T04:37:48.762587+0000 mgr.y (mgr.14152) 211 : cephadm [INF] Adjusting osd_memory_target on smithi133 to 4060M 2023-10-09T04:37:51.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:51 smithi194 bash[31648]: cluster 2023-10-09T04:37:49.694234+0000 mgr.y (mgr.14152) 212 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:37:51.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:51 smithi133 bash[27558]: cluster 2023-10-09T04:37:49.694234+0000 mgr.y (mgr.14152) 212 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:37:51.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:51 smithi133 bash[41291]: cluster 2023-10-09T04:37:49.694234+0000 mgr.y (mgr.14152) 212 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:37:52.759 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:37:53.606 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:53 smithi194 bash[31648]: cluster 2023-10-09T04:37:51.694762+0000 mgr.y (mgr.14152) 213 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:37:53.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:53 smithi133 bash[27558]: cluster 2023-10-09T04:37:51.694762+0000 mgr.y (mgr.14152) 213 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:37:53.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:53 smithi133 bash[41291]: cluster 2023-10-09T04:37:51.694762+0000 mgr.y (mgr.14152) 213 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:37:55.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:55 smithi194 bash[31648]: cluster 2023-10-09T04:37:53.695280+0000 mgr.y (mgr.14152) 214 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:37:55.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:55 smithi133 bash[41291]: cluster 2023-10-09T04:37:53.695280+0000 mgr.y (mgr.14152) 214 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:37:55.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:55 smithi133 bash[27558]: cluster 2023-10-09T04:37:53.695280+0000 mgr.y (mgr.14152) 214 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:37:56.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:56 smithi194 bash[31648]: audit 2023-10-09T04:37:55.705372+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:37:56.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:56 smithi194 bash[31648]: audit 2023-10-09T04:37:55.711610+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:37:56.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:56 smithi194 bash[31648]: audit 2023-10-09T04:37:55.712915+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:37:56.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:56 smithi133 bash[27558]: audit 2023-10-09T04:37:55.705372+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:37:56.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:56 smithi133 bash[27558]: audit 2023-10-09T04:37:55.711610+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:37:56.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:56 smithi133 bash[27558]: audit 2023-10-09T04:37:55.712915+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:37:56.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:56 smithi133 bash[41291]: audit 2023-10-09T04:37:55.705372+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:37:56.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:56 smithi133 bash[41291]: audit 2023-10-09T04:37:55.711610+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:37:56.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:56 smithi133 bash[41291]: audit 2023-10-09T04:37:55.712915+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:37:57.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:57 smithi194 bash[31648]: cluster 2023-10-09T04:37:55.695820+0000 mgr.y (mgr.14152) 215 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:37:57.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:57 smithi194 bash[31648]: audit 2023-10-09T04:37:55.697856+0000 mgr.y (mgr.14152) 216 : audit [DBG] from='client.24226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi194:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:37:57.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:57 smithi133 bash[27558]: cluster 2023-10-09T04:37:55.695820+0000 mgr.y (mgr.14152) 215 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:37:57.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:57 smithi133 bash[27558]: audit 2023-10-09T04:37:55.697856+0000 mgr.y (mgr.14152) 216 : audit [DBG] from='client.24226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi194:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:37:57.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:57 smithi133 bash[41291]: cluster 2023-10-09T04:37:55.695820+0000 mgr.y (mgr.14152) 215 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:37:57.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:57 smithi133 bash[41291]: audit 2023-10-09T04:37:55.697856+0000 mgr.y (mgr.14152) 216 : audit [DBG] from='client.24226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi194:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:37:59.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:37:59 smithi194 bash[31648]: cluster 2023-10-09T04:37:57.696361+0000 mgr.y (mgr.14152) 217 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:37:59.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:37:59 smithi133 bash[27558]: cluster 2023-10-09T04:37:57.696361+0000 mgr.y (mgr.14152) 217 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:37:59.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:37:59 smithi133 bash[41291]: cluster 2023-10-09T04:37:57.696361+0000 mgr.y (mgr.14152) 217 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:01.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:01 smithi194 bash[31648]: cluster 2023-10-09T04:37:59.696924+0000 mgr.y (mgr.14152) 218 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:01.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:01 smithi133 bash[27558]: cluster 2023-10-09T04:37:59.696924+0000 mgr.y (mgr.14152) 218 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:01.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:01 smithi133 bash[41291]: cluster 2023-10-09T04:37:59.696924+0000 mgr.y (mgr.14152) 218 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:03.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:03 smithi194 bash[31648]: cluster 2023-10-09T04:38:01.697582+0000 mgr.y (mgr.14152) 219 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:03.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:03 smithi133 bash[27558]: cluster 2023-10-09T04:38:01.697582+0000 mgr.y (mgr.14152) 219 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:03.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:03 smithi133 bash[41291]: cluster 2023-10-09T04:38:01.697582+0000 mgr.y (mgr.14152) 219 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:04.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:04 smithi133 bash[27558]: audit 2023-10-09T04:38:03.790115+0000 mon.b (mon.1) 9 : audit [INF] from='client.? 172.21.15.194:0/4080094031' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "509ec8cc-5042-428e-9a69-8e602b5afce8"}]: dispatch 2023-10-09T04:38:04.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:04 smithi133 bash[27558]: audit 2023-10-09T04:38:03.790118+0000 mon.a (mon.0) 442 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "509ec8cc-5042-428e-9a69-8e602b5afce8"}]: dispatch 2023-10-09T04:38:04.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:04 smithi133 bash[27558]: audit 2023-10-09T04:38:03.801572+0000 mon.a (mon.0) 443 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "509ec8cc-5042-428e-9a69-8e602b5afce8"}]': finished 2023-10-09T04:38:04.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:04 smithi133 bash[27558]: cluster 2023-10-09T04:38:03.801688+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-10-09T04:38:04.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:04 smithi133 bash[27558]: audit 2023-10-09T04:38:03.801822+0000 mon.a (mon.0) 445 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:38:04.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:04 smithi133 bash[41291]: audit 2023-10-09T04:38:03.790115+0000 mon.b (mon.1) 9 : audit [INF] from='client.? 172.21.15.194:0/4080094031' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "509ec8cc-5042-428e-9a69-8e602b5afce8"}]: dispatch 2023-10-09T04:38:04.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:04 smithi133 bash[41291]: audit 2023-10-09T04:38:03.790118+0000 mon.a (mon.0) 442 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "509ec8cc-5042-428e-9a69-8e602b5afce8"}]: dispatch 2023-10-09T04:38:04.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:04 smithi133 bash[41291]: audit 2023-10-09T04:38:03.801572+0000 mon.a (mon.0) 443 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "509ec8cc-5042-428e-9a69-8e602b5afce8"}]': finished 2023-10-09T04:38:04.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:04 smithi133 bash[41291]: cluster 2023-10-09T04:38:03.801688+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-10-09T04:38:04.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:04 smithi133 bash[41291]: audit 2023-10-09T04:38:03.801822+0000 mon.a (mon.0) 445 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:38:04.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:04 smithi194 bash[31648]: audit 2023-10-09T04:38:03.790115+0000 mon.b (mon.1) 9 : audit [INF] from='client.? 172.21.15.194:0/4080094031' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "509ec8cc-5042-428e-9a69-8e602b5afce8"}]: dispatch 2023-10-09T04:38:04.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:04 smithi194 bash[31648]: audit 2023-10-09T04:38:03.790118+0000 mon.a (mon.0) 442 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "509ec8cc-5042-428e-9a69-8e602b5afce8"}]: dispatch 2023-10-09T04:38:04.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:04 smithi194 bash[31648]: audit 2023-10-09T04:38:03.801572+0000 mon.a (mon.0) 443 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "509ec8cc-5042-428e-9a69-8e602b5afce8"}]': finished 2023-10-09T04:38:04.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:04 smithi194 bash[31648]: cluster 2023-10-09T04:38:03.801688+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-10-09T04:38:04.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:04 smithi194 bash[31648]: audit 2023-10-09T04:38:03.801822+0000 mon.a (mon.0) 445 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:38:05.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:05 smithi133 bash[41291]: cluster 2023-10-09T04:38:03.698239+0000 mgr.y (mgr.14152) 220 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:05.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:05 smithi133 bash[41291]: audit 2023-10-09T04:38:04.531100+0000 mon.b (mon.1) 10 : audit [DBG] from='client.? 172.21.15.194:0/4095233848' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:38:05.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:05 smithi133 bash[27558]: cluster 2023-10-09T04:38:03.698239+0000 mgr.y (mgr.14152) 220 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:05.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:05 smithi133 bash[27558]: audit 2023-10-09T04:38:04.531100+0000 mon.b (mon.1) 10 : audit [DBG] from='client.? 172.21.15.194:0/4095233848' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:38:05.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:05 smithi194 bash[31648]: cluster 2023-10-09T04:38:03.698239+0000 mgr.y (mgr.14152) 220 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:05.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:05 smithi194 bash[31648]: audit 2023-10-09T04:38:04.531100+0000 mon.b (mon.1) 10 : audit [DBG] from='client.? 172.21.15.194:0/4095233848' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:38:07.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:07 smithi194 bash[31648]: cluster 2023-10-09T04:38:05.698853+0000 mgr.y (mgr.14152) 221 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:07.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:07 smithi133 bash[27558]: cluster 2023-10-09T04:38:05.698853+0000 mgr.y (mgr.14152) 221 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:07.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:07 smithi133 bash[41291]: cluster 2023-10-09T04:38:05.698853+0000 mgr.y (mgr.14152) 221 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:09.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:09 smithi133 bash[27558]: cluster 2023-10-09T04:38:07.699373+0000 mgr.y (mgr.14152) 222 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:09.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:09 smithi133 bash[41291]: cluster 2023-10-09T04:38:07.699373+0000 mgr.y (mgr.14152) 222 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:09.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:09 smithi194 bash[31648]: cluster 2023-10-09T04:38:07.699373+0000 mgr.y (mgr.14152) 222 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:11.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:11 smithi133 bash[27558]: cluster 2023-10-09T04:38:09.699906+0000 mgr.y (mgr.14152) 223 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:11.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:11 smithi133 bash[41291]: cluster 2023-10-09T04:38:09.699906+0000 mgr.y (mgr.14152) 223 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:11.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:11 smithi194 bash[31648]: cluster 2023-10-09T04:38:09.699906+0000 mgr.y (mgr.14152) 223 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:13.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:13 smithi194 bash[31648]: cluster 2023-10-09T04:38:11.700446+0000 mgr.y (mgr.14152) 224 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:13.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:13 smithi133 bash[27558]: cluster 2023-10-09T04:38:11.700446+0000 mgr.y (mgr.14152) 224 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:13.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:13 smithi133 bash[41291]: cluster 2023-10-09T04:38:11.700446+0000 mgr.y (mgr.14152) 224 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:15.685 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:15 smithi194 bash[31648]: cluster 2023-10-09T04:38:13.700961+0000 mgr.y (mgr.14152) 225 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:15.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:15 smithi133 bash[27558]: cluster 2023-10-09T04:38:13.700961+0000 mgr.y (mgr.14152) 225 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:15.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:15 smithi133 bash[41291]: cluster 2023-10-09T04:38:13.700961+0000 mgr.y (mgr.14152) 225 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:16.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:16 smithi133 bash[27558]: audit 2023-10-09T04:38:16.261595+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T04:38:16.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:16 smithi133 bash[27558]: audit 2023-10-09T04:38:16.263352+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:38:16.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:16 smithi133 bash[41291]: audit 2023-10-09T04:38:16.261595+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T04:38:16.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:16 smithi133 bash[41291]: audit 2023-10-09T04:38:16.263352+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:38:16.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:16 smithi194 bash[31648]: audit 2023-10-09T04:38:16.261595+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T04:38:16.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:16 smithi194 bash[31648]: audit 2023-10-09T04:38:16.263352+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:38:17.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:17 smithi194 bash[31648]: cluster 2023-10-09T04:38:15.701550+0000 mgr.y (mgr.14152) 226 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:17.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:17 smithi194 bash[31648]: cephadm 2023-10-09T04:38:16.264663+0000 mgr.y (mgr.14152) 227 : cephadm [INF] Deploying daemon osd.4 on smithi194 2023-10-09T04:38:17.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:17 smithi133 bash[27558]: cluster 2023-10-09T04:38:15.701550+0000 mgr.y (mgr.14152) 226 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:17.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:17 smithi133 bash[27558]: cephadm 2023-10-09T04:38:16.264663+0000 mgr.y (mgr.14152) 227 : cephadm [INF] Deploying daemon osd.4 on smithi194 2023-10-09T04:38:17.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:17 smithi133 bash[41291]: cluster 2023-10-09T04:38:15.701550+0000 mgr.y (mgr.14152) 226 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:17.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:17 smithi133 bash[41291]: cephadm 2023-10-09T04:38:16.264663+0000 mgr.y (mgr.14152) 227 : cephadm [INF] Deploying daemon osd.4 on smithi194 2023-10-09T04:38:19.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:19 smithi133 bash[27558]: cluster 2023-10-09T04:38:17.702200+0000 mgr.y (mgr.14152) 228 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:19.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:19 smithi133 bash[41291]: cluster 2023-10-09T04:38:17.702200+0000 mgr.y (mgr.14152) 228 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:19.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:19 smithi194 bash[31648]: cluster 2023-10-09T04:38:17.702200+0000 mgr.y (mgr.14152) 228 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:21.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:21 smithi133 bash[27558]: cluster 2023-10-09T04:38:19.702785+0000 mgr.y (mgr.14152) 229 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:21.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:21 smithi133 bash[27558]: audit 2023-10-09T04:38:20.675273+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:21.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:21 smithi133 bash[27558]: audit 2023-10-09T04:38:20.681664+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:21.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:21 smithi133 bash[27558]: audit 2023-10-09T04:38:20.695035+0000 mon.a (mon.0) 450 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:38:21.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:21 smithi133 bash[41291]: cluster 2023-10-09T04:38:19.702785+0000 mgr.y (mgr.14152) 229 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:21.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:21 smithi133 bash[41291]: audit 2023-10-09T04:38:20.675273+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:21.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:21 smithi133 bash[41291]: audit 2023-10-09T04:38:20.681664+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:21.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:21 smithi133 bash[41291]: audit 2023-10-09T04:38:20.695035+0000 mon.a (mon.0) 450 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:38:21.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:21 smithi194 bash[31648]: cluster 2023-10-09T04:38:19.702785+0000 mgr.y (mgr.14152) 229 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:21.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:21 smithi194 bash[31648]: audit 2023-10-09T04:38:20.675273+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:21.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:21 smithi194 bash[31648]: audit 2023-10-09T04:38:20.681664+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:21.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:21 smithi194 bash[31648]: audit 2023-10-09T04:38:20.695035+0000 mon.a (mon.0) 450 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:38:23.691 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:23 smithi194 bash[31648]: cluster 2023-10-09T04:38:21.703301+0000 mgr.y (mgr.14152) 230 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:23.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:23 smithi133 bash[27558]: cluster 2023-10-09T04:38:21.703301+0000 mgr.y (mgr.14152) 230 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:23.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:23 smithi133 bash[41291]: cluster 2023-10-09T04:38:21.703301+0000 mgr.y (mgr.14152) 230 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:25.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:25 smithi133 bash[27558]: cluster 2023-10-09T04:38:23.703825+0000 mgr.y (mgr.14152) 231 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:25.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:25 smithi133 bash[41291]: cluster 2023-10-09T04:38:23.703825+0000 mgr.y (mgr.14152) 231 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:25.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:25 smithi194 bash[31648]: cluster 2023-10-09T04:38:23.703825+0000 mgr.y (mgr.14152) 231 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:27.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:26 smithi194 bash[31648]: cluster 2023-10-09T04:38:25.704389+0000 mgr.y (mgr.14152) 232 : cluster [DBG] pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:27.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:26 smithi194 bash[31648]: audit 2023-10-09T04:38:25.841832+0000 mon.a (mon.0) 451 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:27.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:26 smithi194 bash[31648]: audit 2023-10-09T04:38:25.849990+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:27.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:26 smithi194 bash[31648]: audit 2023-10-09T04:38:26.220749+0000 mon.a (mon.0) 453 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:38:27.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:26 smithi194 bash[31648]: audit 2023-10-09T04:38:26.222498+0000 mon.a (mon.0) 454 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:38:27.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:26 smithi194 bash[31648]: audit 2023-10-09T04:38:26.233278+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:27.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:26 smithi133 bash[27558]: cluster 2023-10-09T04:38:25.704389+0000 mgr.y (mgr.14152) 232 : cluster [DBG] pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:27.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:26 smithi133 bash[27558]: audit 2023-10-09T04:38:25.841832+0000 mon.a (mon.0) 451 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:27.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:26 smithi133 bash[27558]: audit 2023-10-09T04:38:25.849990+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:27.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:26 smithi133 bash[27558]: audit 2023-10-09T04:38:26.220749+0000 mon.a (mon.0) 453 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:38:27.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:26 smithi133 bash[27558]: audit 2023-10-09T04:38:26.222498+0000 mon.a (mon.0) 454 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:38:27.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:26 smithi133 bash[27558]: audit 2023-10-09T04:38:26.233278+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:27.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:26 smithi133 bash[41291]: cluster 2023-10-09T04:38:25.704389+0000 mgr.y (mgr.14152) 232 : cluster [DBG] pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:27.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:26 smithi133 bash[41291]: audit 2023-10-09T04:38:25.841832+0000 mon.a (mon.0) 451 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:27.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:26 smithi133 bash[41291]: audit 2023-10-09T04:38:25.849990+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:27.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:26 smithi133 bash[41291]: audit 2023-10-09T04:38:26.220749+0000 mon.a (mon.0) 453 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:38:27.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:26 smithi133 bash[41291]: audit 2023-10-09T04:38:26.222498+0000 mon.a (mon.0) 454 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:38:27.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:26 smithi133 bash[41291]: audit 2023-10-09T04:38:26.233278+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:28.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:27 smithi194 bash[31648]: audit 2023-10-09T04:38:27.272961+0000 mon.a (mon.0) 456 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T04:38:28.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:27 smithi194 bash[31648]: audit 2023-10-09T04:38:27.273142+0000 mon.b (mon.1) 11 : audit [INF] from='osd.4 [v2:172.21.15.194:6800/1874338403,v1:172.21.15.194:6801/1874338403]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T04:38:28.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:27 smithi133 bash[27558]: audit 2023-10-09T04:38:27.272961+0000 mon.a (mon.0) 456 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T04:38:28.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:27 smithi133 bash[27558]: audit 2023-10-09T04:38:27.273142+0000 mon.b (mon.1) 11 : audit [INF] from='osd.4 [v2:172.21.15.194:6800/1874338403,v1:172.21.15.194:6801/1874338403]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T04:38:28.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:27 smithi133 bash[41291]: audit 2023-10-09T04:38:27.272961+0000 mon.a (mon.0) 456 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T04:38:28.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:27 smithi133 bash[41291]: audit 2023-10-09T04:38:27.273142+0000 mon.b (mon.1) 11 : audit [INF] from='osd.4 [v2:172.21.15.194:6800/1874338403,v1:172.21.15.194:6801/1874338403]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T04:38:29.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:28 smithi194 bash[31648]: cluster 2023-10-09T04:38:27.704955+0000 mgr.y (mgr.14152) 233 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:29.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:28 smithi194 bash[31648]: audit 2023-10-09T04:38:27.858489+0000 mon.a (mon.0) 457 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T04:38:29.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:28 smithi194 bash[31648]: cluster 2023-10-09T04:38:27.858577+0000 mon.a (mon.0) 458 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-10-09T04:38:29.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:28 smithi194 bash[31648]: audit 2023-10-09T04:38:27.858822+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:38:29.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:28 smithi194 bash[31648]: audit 2023-10-09T04:38:27.860229+0000 mon.a (mon.0) 460 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2023-10-09T04:38:29.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:28 smithi194 bash[31648]: audit 2023-10-09T04:38:27.860716+0000 mon.b (mon.1) 12 : audit [INF] from='osd.4 [v2:172.21.15.194:6800/1874338403,v1:172.21.15.194:6801/1874338403]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2023-10-09T04:38:29.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:28 smithi133 bash[27558]: cluster 2023-10-09T04:38:27.704955+0000 mgr.y (mgr.14152) 233 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:29.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:28 smithi133 bash[27558]: audit 2023-10-09T04:38:27.858489+0000 mon.a (mon.0) 457 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T04:38:29.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:28 smithi133 bash[27558]: cluster 2023-10-09T04:38:27.858577+0000 mon.a (mon.0) 458 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-10-09T04:38:29.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:28 smithi133 bash[27558]: audit 2023-10-09T04:38:27.858822+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:38:29.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:28 smithi133 bash[27558]: audit 2023-10-09T04:38:27.860229+0000 mon.a (mon.0) 460 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2023-10-09T04:38:29.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:28 smithi133 bash[27558]: audit 2023-10-09T04:38:27.860716+0000 mon.b (mon.1) 12 : audit [INF] from='osd.4 [v2:172.21.15.194:6800/1874338403,v1:172.21.15.194:6801/1874338403]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2023-10-09T04:38:29.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:28 smithi133 bash[41291]: cluster 2023-10-09T04:38:27.704955+0000 mgr.y (mgr.14152) 233 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:29.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:28 smithi133 bash[41291]: audit 2023-10-09T04:38:27.858489+0000 mon.a (mon.0) 457 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T04:38:29.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:28 smithi133 bash[41291]: cluster 2023-10-09T04:38:27.858577+0000 mon.a (mon.0) 458 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-10-09T04:38:29.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:28 smithi133 bash[41291]: audit 2023-10-09T04:38:27.858822+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:38:29.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:28 smithi133 bash[41291]: audit 2023-10-09T04:38:27.860229+0000 mon.a (mon.0) 460 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2023-10-09T04:38:29.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:28 smithi133 bash[41291]: audit 2023-10-09T04:38:27.860716+0000 mon.b (mon.1) 12 : audit [INF] from='osd.4 [v2:172.21.15.194:6800/1874338403,v1:172.21.15.194:6801/1874338403]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2023-10-09T04:38:29.425 INFO:teuthology.orchestra.run.smithi194.stdout:Created osd(s) 4 on host 'smithi194' 2023-10-09T04:38:30.126 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:29 smithi194 bash[31648]: audit 2023-10-09T04:38:28.857622+0000 mon.a (mon.0) 461 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]': finished 2023-10-09T04:38:30.126 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:29 smithi194 bash[31648]: cluster 2023-10-09T04:38:28.857760+0000 mon.a (mon.0) 462 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-10-09T04:38:30.126 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:29 smithi194 bash[31648]: audit 2023-10-09T04:38:28.858399+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:38:30.127 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:29 smithi194 bash[31648]: audit 2023-10-09T04:38:28.865092+0000 mon.a (mon.0) 464 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:38:30.127 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:29 smithi194 bash[31648]: audit 2023-10-09T04:38:29.387388+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:30.127 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:29 smithi194 bash[31648]: audit 2023-10-09T04:38:29.392479+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:30.127 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:29 smithi194 bash[31648]: audit 2023-10-09T04:38:29.423687+0000 mon.a (mon.0) 467 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:38:30.128 DEBUG:teuthology.orchestra.run.smithi194:osd.4> sudo journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@osd.4.service 2023-10-09T04:38:30.132 INFO:tasks.cephadm:Deploying osd.5 on smithi194 with /dev/vg_nvme/lv_3... 2023-10-09T04:38:30.132 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-09T04:38:30.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:29 smithi133 bash[27558]: audit 2023-10-09T04:38:28.857622+0000 mon.a (mon.0) 461 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]': finished 2023-10-09T04:38:30.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:29 smithi133 bash[27558]: cluster 2023-10-09T04:38:28.857760+0000 mon.a (mon.0) 462 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-10-09T04:38:30.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:29 smithi133 bash[27558]: audit 2023-10-09T04:38:28.858399+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:38:30.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:29 smithi133 bash[27558]: audit 2023-10-09T04:38:28.865092+0000 mon.a (mon.0) 464 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:38:30.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:29 smithi133 bash[27558]: audit 2023-10-09T04:38:29.387388+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:30.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:29 smithi133 bash[27558]: audit 2023-10-09T04:38:29.392479+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:30.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:29 smithi133 bash[27558]: audit 2023-10-09T04:38:29.423687+0000 mon.a (mon.0) 467 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:38:30.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:29 smithi133 bash[41291]: audit 2023-10-09T04:38:28.857622+0000 mon.a (mon.0) 461 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]': finished 2023-10-09T04:38:30.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:29 smithi133 bash[41291]: cluster 2023-10-09T04:38:28.857760+0000 mon.a (mon.0) 462 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-10-09T04:38:30.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:29 smithi133 bash[41291]: audit 2023-10-09T04:38:28.858399+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:38:30.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:29 smithi133 bash[41291]: audit 2023-10-09T04:38:28.865092+0000 mon.a (mon.0) 464 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:38:30.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:29 smithi133 bash[41291]: audit 2023-10-09T04:38:29.387388+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:30.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:29 smithi133 bash[41291]: audit 2023-10-09T04:38:29.392479+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:30.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:29 smithi133 bash[41291]: audit 2023-10-09T04:38:29.423687+0000 mon.a (mon.0) 467 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:38:30.370 INFO:journalctl@ceph.osd.4.smithi194.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:38:31.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:30 smithi194 bash[31648]: cluster 2023-10-09T04:38:28.300905+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:38:31.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:30 smithi194 bash[31648]: cluster 2023-10-09T04:38:28.301112+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:38:31.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:30 smithi194 bash[31648]: cluster 2023-10-09T04:38:29.705575+0000 mgr.y (mgr.14152) 234 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:31.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:30 smithi194 bash[31648]: cluster 2023-10-09T04:38:29.869248+0000 mon.a (mon.0) 468 : cluster [INF] osd.4 [v2:172.21.15.194:6800/1874338403,v1:172.21.15.194:6801/1874338403] boot 2023-10-09T04:38:31.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:30 smithi194 bash[31648]: cluster 2023-10-09T04:38:29.869501+0000 mon.a (mon.0) 469 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-10-09T04:38:31.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:30 smithi194 bash[31648]: audit 2023-10-09T04:38:29.870361+0000 mon.a (mon.0) 470 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:38:31.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:30 smithi194 bash[31648]: cluster 2023-10-09T04:38:30.341736+0000 mon.a (mon.0) 471 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-10-09T04:38:31.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:30 smithi133 bash[41291]: cluster 2023-10-09T04:38:28.300905+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:38:31.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:30 smithi133 bash[41291]: cluster 2023-10-09T04:38:28.301112+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:38:31.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:30 smithi133 bash[41291]: cluster 2023-10-09T04:38:29.705575+0000 mgr.y (mgr.14152) 234 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:31.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:30 smithi133 bash[41291]: cluster 2023-10-09T04:38:29.869248+0000 mon.a (mon.0) 468 : cluster [INF] osd.4 [v2:172.21.15.194:6800/1874338403,v1:172.21.15.194:6801/1874338403] boot 2023-10-09T04:38:31.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:30 smithi133 bash[41291]: cluster 2023-10-09T04:38:29.869501+0000 mon.a (mon.0) 469 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-10-09T04:38:31.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:30 smithi133 bash[41291]: audit 2023-10-09T04:38:29.870361+0000 mon.a (mon.0) 470 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:38:31.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:30 smithi133 bash[41291]: cluster 2023-10-09T04:38:30.341736+0000 mon.a (mon.0) 471 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-10-09T04:38:31.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:30 smithi133 bash[27558]: cluster 2023-10-09T04:38:28.300905+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:38:31.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:30 smithi133 bash[27558]: cluster 2023-10-09T04:38:28.301112+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:38:31.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:30 smithi133 bash[27558]: cluster 2023-10-09T04:38:29.705575+0000 mgr.y (mgr.14152) 234 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:38:31.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:30 smithi133 bash[27558]: cluster 2023-10-09T04:38:29.869248+0000 mon.a (mon.0) 468 : cluster [INF] osd.4 [v2:172.21.15.194:6800/1874338403,v1:172.21.15.194:6801/1874338403] boot 2023-10-09T04:38:31.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:30 smithi133 bash[27558]: cluster 2023-10-09T04:38:29.869501+0000 mon.a (mon.0) 469 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-10-09T04:38:31.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:30 smithi133 bash[27558]: audit 2023-10-09T04:38:29.870361+0000 mon.a (mon.0) 470 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:38:31.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:30 smithi133 bash[27558]: cluster 2023-10-09T04:38:30.341736+0000 mon.a (mon.0) 471 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-10-09T04:38:32.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:32 smithi194 bash[31648]: cluster 2023-10-09T04:38:31.357549+0000 mon.a (mon.0) 472 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-10-09T04:38:32.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:32 smithi133 bash[27558]: cluster 2023-10-09T04:38:31.357549+0000 mon.a (mon.0) 472 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-10-09T04:38:32.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:32 smithi133 bash[41291]: cluster 2023-10-09T04:38:31.357549+0000 mon.a (mon.0) 472 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-10-09T04:38:33.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:33 smithi194 bash[31648]: cluster 2023-10-09T04:38:31.706150+0000 mgr.y (mgr.14152) 235 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:38:33.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:33 smithi194 bash[31648]: cluster 2023-10-09T04:38:32.362837+0000 mon.a (mon.0) 473 : cluster [DBG] osdmap e33: 5 total, 5 up, 5 in 2023-10-09T04:38:33.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:33 smithi133 bash[27558]: cluster 2023-10-09T04:38:31.706150+0000 mgr.y (mgr.14152) 235 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:38:33.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:33 smithi133 bash[27558]: cluster 2023-10-09T04:38:32.362837+0000 mon.a (mon.0) 473 : cluster [DBG] osdmap e33: 5 total, 5 up, 5 in 2023-10-09T04:38:33.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:33 smithi133 bash[41291]: cluster 2023-10-09T04:38:31.706150+0000 mgr.y (mgr.14152) 235 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:38:33.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:33 smithi133 bash[41291]: cluster 2023-10-09T04:38:32.362837+0000 mon.a (mon.0) 473 : cluster [DBG] osdmap e33: 5 total, 5 up, 5 in 2023-10-09T04:38:34.190 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:38:34.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:34 smithi194 bash[31648]: cluster 2023-10-09T04:38:33.367555+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e34: 5 total, 5 up, 5 in 2023-10-09T04:38:34.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:34 smithi133 bash[27558]: cluster 2023-10-09T04:38:33.367555+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e34: 5 total, 5 up, 5 in 2023-10-09T04:38:34.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:34 smithi133 bash[41291]: cluster 2023-10-09T04:38:33.367555+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e34: 5 total, 5 up, 5 in 2023-10-09T04:38:35.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:35 smithi194 bash[31648]: cluster 2023-10-09T04:38:33.706702+0000 mgr.y (mgr.14152) 236 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:38:35.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:35 smithi133 bash[27558]: cluster 2023-10-09T04:38:33.706702+0000 mgr.y (mgr.14152) 236 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:38:35.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:35 smithi133 bash[41291]: cluster 2023-10-09T04:38:33.706702+0000 mgr.y (mgr.14152) 236 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:38:37.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:37 smithi133 bash[27558]: cluster 2023-10-09T04:38:35.707309+0000 mgr.y (mgr.14152) 237 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 107 KiB/s, 0 objects/s recovering 2023-10-09T04:38:37.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:37 smithi133 bash[41291]: cluster 2023-10-09T04:38:35.707309+0000 mgr.y (mgr.14152) 237 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 107 KiB/s, 0 objects/s recovering 2023-10-09T04:38:37.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:37 smithi194 bash[31648]: cluster 2023-10-09T04:38:35.707309+0000 mgr.y (mgr.14152) 237 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 107 KiB/s, 0 objects/s recovering 2023-10-09T04:38:39.086 INFO:teuthology.orchestra.run.smithi194.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-09T04:38:39.087 INFO:teuthology.orchestra.run.smithi194.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-09T04:38:39.087 INFO:teuthology.orchestra.run.smithi194.stderr: stderr: 10+0 records in 2023-10-09T04:38:39.087 INFO:teuthology.orchestra.run.smithi194.stderr:10+0 records out 2023-10-09T04:38:39.087 INFO:teuthology.orchestra.run.smithi194.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0173317 s, 605 MB/s 2023-10-09T04:38:39.087 INFO:teuthology.orchestra.run.smithi194.stderr:--> Zapping successful for: 2023-10-09T04:38:39.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:39 smithi194 bash[31648]: cluster 2023-10-09T04:38:37.707857+0000 mgr.y (mgr.14152) 238 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 91 KiB/s, 0 objects/s recovering 2023-10-09T04:38:39.794 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi194:vg_nvme/lv_3 2023-10-09T04:38:39.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:39 smithi133 bash[27558]: cluster 2023-10-09T04:38:37.707857+0000 mgr.y (mgr.14152) 238 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 91 KiB/s, 0 objects/s recovering 2023-10-09T04:38:39.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:39 smithi133 bash[41291]: cluster 2023-10-09T04:38:37.707857+0000 mgr.y (mgr.14152) 238 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 91 KiB/s, 0 objects/s recovering 2023-10-09T04:38:41.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:41 smithi133 bash[27558]: cluster 2023-10-09T04:38:39.708451+0000 mgr.y (mgr.14152) 239 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T04:38:41.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:41 smithi133 bash[27558]: audit 2023-10-09T04:38:40.630274+0000 mon.a (mon.0) 475 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:41.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:41 smithi133 bash[27558]: audit 2023-10-09T04:38:40.636142+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:41.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:41 smithi133 bash[27558]: audit 2023-10-09T04:38:40.637620+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:38:41.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:41 smithi133 bash[27558]: audit 2023-10-09T04:38:40.646361+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:41.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:41 smithi133 bash[27558]: audit 2023-10-09T04:38:40.647998+0000 mon.a (mon.0) 479 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:38:41.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:41 smithi133 bash[27558]: audit 2023-10-09T04:38:40.649693+0000 mon.a (mon.0) 480 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:38:41.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:41 smithi133 bash[27558]: audit 2023-10-09T04:38:40.658517+0000 mon.a (mon.0) 481 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:41.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:41 smithi133 bash[41291]: cluster 2023-10-09T04:38:39.708451+0000 mgr.y (mgr.14152) 239 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T04:38:41.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:41 smithi133 bash[41291]: audit 2023-10-09T04:38:40.630274+0000 mon.a (mon.0) 475 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:41.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:41 smithi133 bash[41291]: audit 2023-10-09T04:38:40.636142+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:41.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:41 smithi133 bash[41291]: audit 2023-10-09T04:38:40.637620+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:38:41.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:41 smithi133 bash[41291]: audit 2023-10-09T04:38:40.646361+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:41.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:41 smithi133 bash[41291]: audit 2023-10-09T04:38:40.647998+0000 mon.a (mon.0) 479 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:38:41.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:41 smithi133 bash[41291]: audit 2023-10-09T04:38:40.649693+0000 mon.a (mon.0) 480 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:38:41.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:41 smithi133 bash[41291]: audit 2023-10-09T04:38:40.658517+0000 mon.a (mon.0) 481 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:41.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:41 smithi194 bash[31648]: cluster 2023-10-09T04:38:39.708451+0000 mgr.y (mgr.14152) 239 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T04:38:41.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:41 smithi194 bash[31648]: audit 2023-10-09T04:38:40.630274+0000 mon.a (mon.0) 475 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:41.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:41 smithi194 bash[31648]: audit 2023-10-09T04:38:40.636142+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:41.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:41 smithi194 bash[31648]: audit 2023-10-09T04:38:40.637620+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:38:41.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:41 smithi194 bash[31648]: audit 2023-10-09T04:38:40.646361+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:41.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:41 smithi194 bash[31648]: audit 2023-10-09T04:38:40.647998+0000 mon.a (mon.0) 479 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:38:41.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:41 smithi194 bash[31648]: audit 2023-10-09T04:38:40.649693+0000 mon.a (mon.0) 480 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:38:41.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:41 smithi194 bash[31648]: audit 2023-10-09T04:38:40.658517+0000 mon.a (mon.0) 481 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:38:42.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:42 smithi133 bash[27558]: cephadm 2023-10-09T04:38:40.622290+0000 mgr.y (mgr.14152) 240 : cephadm [INF] Detected new or changed devices on smithi194 2023-10-09T04:38:42.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:42 smithi133 bash[27558]: cephadm 2023-10-09T04:38:40.638783+0000 mgr.y (mgr.14152) 241 : cephadm [INF] Adjusting osd_memory_target on smithi194 to 17265M 2023-10-09T04:38:42.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:42 smithi133 bash[41291]: cephadm 2023-10-09T04:38:40.622290+0000 mgr.y (mgr.14152) 240 : cephadm [INF] Detected new or changed devices on smithi194 2023-10-09T04:38:42.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:42 smithi133 bash[41291]: cephadm 2023-10-09T04:38:40.638783+0000 mgr.y (mgr.14152) 241 : cephadm [INF] Adjusting osd_memory_target on smithi194 to 17265M 2023-10-09T04:38:42.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:42 smithi194 bash[31648]: cephadm 2023-10-09T04:38:40.622290+0000 mgr.y (mgr.14152) 240 : cephadm [INF] Detected new or changed devices on smithi194 2023-10-09T04:38:42.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:42 smithi194 bash[31648]: cephadm 2023-10-09T04:38:40.638783+0000 mgr.y (mgr.14152) 241 : cephadm [INF] Adjusting osd_memory_target on smithi194 to 17265M 2023-10-09T04:38:43.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:43 smithi133 bash[27558]: cluster 2023-10-09T04:38:41.709009+0000 mgr.y (mgr.14152) 242 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 62 KiB/s, 0 objects/s recovering 2023-10-09T04:38:43.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:43 smithi133 bash[41291]: cluster 2023-10-09T04:38:41.709009+0000 mgr.y (mgr.14152) 242 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 62 KiB/s, 0 objects/s recovering 2023-10-09T04:38:43.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:43 smithi194 bash[31648]: cluster 2023-10-09T04:38:41.709009+0000 mgr.y (mgr.14152) 242 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 62 KiB/s, 0 objects/s recovering 2023-10-09T04:38:44.516 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:38:45.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:45 smithi194 bash[31648]: cluster 2023-10-09T04:38:43.709573+0000 mgr.y (mgr.14152) 243 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 56 KiB/s, 0 objects/s recovering 2023-10-09T04:38:45.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:45 smithi133 bash[27558]: cluster 2023-10-09T04:38:43.709573+0000 mgr.y (mgr.14152) 243 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 56 KiB/s, 0 objects/s recovering 2023-10-09T04:38:45.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:45 smithi133 bash[41291]: cluster 2023-10-09T04:38:43.709573+0000 mgr.y (mgr.14152) 243 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 56 KiB/s, 0 objects/s recovering 2023-10-09T04:38:47.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:47 smithi133 bash[27558]: cluster 2023-10-09T04:38:45.710216+0000 mgr.y (mgr.14152) 244 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-10-09T04:38:47.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:47 smithi133 bash[41291]: cluster 2023-10-09T04:38:45.710216+0000 mgr.y (mgr.14152) 244 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-10-09T04:38:47.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:47 smithi194 bash[31648]: cluster 2023-10-09T04:38:45.710216+0000 mgr.y (mgr.14152) 244 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-10-09T04:38:48.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:48 smithi133 bash[27558]: audit 2023-10-09T04:38:47.421289+0000 mgr.y (mgr.14152) 245 : audit [DBG] from='client.24253 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi194:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:38:48.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:48 smithi133 bash[27558]: audit 2023-10-09T04:38:47.426491+0000 mon.a (mon.0) 482 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:38:48.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:48 smithi133 bash[27558]: audit 2023-10-09T04:38:47.429982+0000 mon.a (mon.0) 483 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:38:48.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:48 smithi133 bash[27558]: audit 2023-10-09T04:38:47.431015+0000 mon.a (mon.0) 484 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:38:48.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:48 smithi133 bash[41291]: audit 2023-10-09T04:38:47.421289+0000 mgr.y (mgr.14152) 245 : audit [DBG] from='client.24253 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi194:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:38:48.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:48 smithi133 bash[41291]: audit 2023-10-09T04:38:47.426491+0000 mon.a (mon.0) 482 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:38:48.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:48 smithi133 bash[41291]: audit 2023-10-09T04:38:47.429982+0000 mon.a (mon.0) 483 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:38:48.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:48 smithi133 bash[41291]: audit 2023-10-09T04:38:47.431015+0000 mon.a (mon.0) 484 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:38:48.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:48 smithi194 bash[31648]: audit 2023-10-09T04:38:47.421289+0000 mgr.y (mgr.14152) 245 : audit [DBG] from='client.24253 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi194:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:38:48.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:48 smithi194 bash[31648]: audit 2023-10-09T04:38:47.426491+0000 mon.a (mon.0) 482 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:38:48.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:48 smithi194 bash[31648]: audit 2023-10-09T04:38:47.429982+0000 mon.a (mon.0) 483 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:38:48.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:48 smithi194 bash[31648]: audit 2023-10-09T04:38:47.431015+0000 mon.a (mon.0) 484 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:38:49.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:49 smithi133 bash[27558]: cluster 2023-10-09T04:38:47.710778+0000 mgr.y (mgr.14152) 246 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:38:49.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:49 smithi133 bash[41291]: cluster 2023-10-09T04:38:47.710778+0000 mgr.y (mgr.14152) 246 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:38:49.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:49 smithi194 bash[31648]: cluster 2023-10-09T04:38:47.710778+0000 mgr.y (mgr.14152) 246 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:38:51.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:51 smithi133 bash[27558]: cluster 2023-10-09T04:38:49.711382+0000 mgr.y (mgr.14152) 247 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:38:51.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:51 smithi133 bash[41291]: cluster 2023-10-09T04:38:49.711382+0000 mgr.y (mgr.14152) 247 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:38:51.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:51 smithi194 bash[31648]: cluster 2023-10-09T04:38:49.711382+0000 mgr.y (mgr.14152) 247 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:38:53.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:53 smithi133 bash[27558]: cluster 2023-10-09T04:38:51.712001+0000 mgr.y (mgr.14152) 248 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:38:53.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:53 smithi133 bash[41291]: cluster 2023-10-09T04:38:51.712001+0000 mgr.y (mgr.14152) 248 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:38:53.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:53 smithi194 bash[31648]: cluster 2023-10-09T04:38:51.712001+0000 mgr.y (mgr.14152) 248 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:38:55.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:55 smithi133 bash[27558]: cluster 2023-10-09T04:38:53.712581+0000 mgr.y (mgr.14152) 249 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:38:55.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:55 smithi133 bash[41291]: cluster 2023-10-09T04:38:53.712581+0000 mgr.y (mgr.14152) 249 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:38:55.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:55 smithi194 bash[31648]: cluster 2023-10-09T04:38:53.712581+0000 mgr.y (mgr.14152) 249 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:38:56.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:56 smithi133 bash[41291]: audit 2023-10-09T04:38:55.741456+0000 mon.a (mon.0) 485 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bbdc1e67-fd5b-49d0-9e42-cf151d167943"}]: dispatch 2023-10-09T04:38:56.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:56 smithi133 bash[41291]: audit 2023-10-09T04:38:55.741581+0000 mon.b (mon.1) 13 : audit [INF] from='client.? 172.21.15.194:0/2208446650' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bbdc1e67-fd5b-49d0-9e42-cf151d167943"}]: dispatch 2023-10-09T04:38:56.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:56 smithi133 bash[41291]: audit 2023-10-09T04:38:55.750984+0000 mon.a (mon.0) 486 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bbdc1e67-fd5b-49d0-9e42-cf151d167943"}]': finished 2023-10-09T04:38:56.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:56 smithi133 bash[41291]: cluster 2023-10-09T04:38:55.751101+0000 mon.a (mon.0) 487 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-10-09T04:38:56.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:56 smithi133 bash[41291]: audit 2023-10-09T04:38:55.751291+0000 mon.a (mon.0) 488 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:38:56.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:56 smithi133 bash[27558]: audit 2023-10-09T04:38:55.741456+0000 mon.a (mon.0) 485 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bbdc1e67-fd5b-49d0-9e42-cf151d167943"}]: dispatch 2023-10-09T04:38:56.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:56 smithi133 bash[27558]: audit 2023-10-09T04:38:55.741581+0000 mon.b (mon.1) 13 : audit [INF] from='client.? 172.21.15.194:0/2208446650' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bbdc1e67-fd5b-49d0-9e42-cf151d167943"}]: dispatch 2023-10-09T04:38:56.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:56 smithi133 bash[27558]: audit 2023-10-09T04:38:55.750984+0000 mon.a (mon.0) 486 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bbdc1e67-fd5b-49d0-9e42-cf151d167943"}]': finished 2023-10-09T04:38:56.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:56 smithi133 bash[27558]: cluster 2023-10-09T04:38:55.751101+0000 mon.a (mon.0) 487 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-10-09T04:38:56.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:56 smithi133 bash[27558]: audit 2023-10-09T04:38:55.751291+0000 mon.a (mon.0) 488 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:38:56.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:56 smithi194 bash[31648]: audit 2023-10-09T04:38:55.741456+0000 mon.a (mon.0) 485 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bbdc1e67-fd5b-49d0-9e42-cf151d167943"}]: dispatch 2023-10-09T04:38:56.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:56 smithi194 bash[31648]: audit 2023-10-09T04:38:55.741581+0000 mon.b (mon.1) 13 : audit [INF] from='client.? 172.21.15.194:0/2208446650' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bbdc1e67-fd5b-49d0-9e42-cf151d167943"}]: dispatch 2023-10-09T04:38:56.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:56 smithi194 bash[31648]: audit 2023-10-09T04:38:55.750984+0000 mon.a (mon.0) 486 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bbdc1e67-fd5b-49d0-9e42-cf151d167943"}]': finished 2023-10-09T04:38:56.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:56 smithi194 bash[31648]: cluster 2023-10-09T04:38:55.751101+0000 mon.a (mon.0) 487 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-10-09T04:38:56.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:56 smithi194 bash[31648]: audit 2023-10-09T04:38:55.751291+0000 mon.a (mon.0) 488 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:38:57.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:57 smithi133 bash[27558]: cluster 2023-10-09T04:38:55.713075+0000 mgr.y (mgr.14152) 250 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:38:57.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:57 smithi133 bash[27558]: audit 2023-10-09T04:38:56.489798+0000 mon.b (mon.1) 14 : audit [DBG] from='client.? 172.21.15.194:0/3612221821' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:38:57.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:57 smithi133 bash[41291]: cluster 2023-10-09T04:38:55.713075+0000 mgr.y (mgr.14152) 250 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:38:57.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:57 smithi133 bash[41291]: audit 2023-10-09T04:38:56.489798+0000 mon.b (mon.1) 14 : audit [DBG] from='client.? 172.21.15.194:0/3612221821' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:38:57.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:57 smithi194 bash[31648]: cluster 2023-10-09T04:38:55.713075+0000 mgr.y (mgr.14152) 250 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:38:57.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:57 smithi194 bash[31648]: audit 2023-10-09T04:38:56.489798+0000 mon.b (mon.1) 14 : audit [DBG] from='client.? 172.21.15.194:0/3612221821' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:38:59.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:38:59 smithi133 bash[27558]: cluster 2023-10-09T04:38:57.713697+0000 mgr.y (mgr.14152) 251 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:38:59.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:38:59 smithi133 bash[41291]: cluster 2023-10-09T04:38:57.713697+0000 mgr.y (mgr.14152) 251 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:38:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:38:59 smithi194 bash[31648]: cluster 2023-10-09T04:38:57.713697+0000 mgr.y (mgr.14152) 251 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:01.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:01 smithi133 bash[27558]: cluster 2023-10-09T04:38:59.714367+0000 mgr.y (mgr.14152) 252 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:01.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:01 smithi133 bash[41291]: cluster 2023-10-09T04:38:59.714367+0000 mgr.y (mgr.14152) 252 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:01.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:01 smithi194 bash[31648]: cluster 2023-10-09T04:38:59.714367+0000 mgr.y (mgr.14152) 252 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:03.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:03 smithi133 bash[27558]: cluster 2023-10-09T04:39:01.715056+0000 mgr.y (mgr.14152) 253 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:03.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:03 smithi133 bash[41291]: cluster 2023-10-09T04:39:01.715056+0000 mgr.y (mgr.14152) 253 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:03.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:03 smithi194 bash[31648]: cluster 2023-10-09T04:39:01.715056+0000 mgr.y (mgr.14152) 253 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:05.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:05 smithi133 bash[27558]: cluster 2023-10-09T04:39:03.715619+0000 mgr.y (mgr.14152) 254 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:05.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:05 smithi133 bash[41291]: cluster 2023-10-09T04:39:03.715619+0000 mgr.y (mgr.14152) 254 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:05.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:05 smithi194 bash[31648]: cluster 2023-10-09T04:39:03.715619+0000 mgr.y (mgr.14152) 254 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:07.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:07 smithi133 bash[27558]: cluster 2023-10-09T04:39:05.716189+0000 mgr.y (mgr.14152) 255 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:07.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:07 smithi133 bash[41291]: cluster 2023-10-09T04:39:05.716189+0000 mgr.y (mgr.14152) 255 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:07.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:07 smithi194 bash[31648]: cluster 2023-10-09T04:39:05.716189+0000 mgr.y (mgr.14152) 255 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:08.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:08 smithi133 bash[27558]: audit 2023-10-09T04:39:08.176965+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T04:39:08.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:08 smithi133 bash[27558]: audit 2023-10-09T04:39:08.178840+0000 mon.a (mon.0) 490 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:39:08.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:08 smithi133 bash[41291]: audit 2023-10-09T04:39:08.176965+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T04:39:08.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:08 smithi133 bash[41291]: audit 2023-10-09T04:39:08.178840+0000 mon.a (mon.0) 490 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:39:08.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:08 smithi194 bash[31648]: audit 2023-10-09T04:39:08.176965+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T04:39:08.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:08 smithi194 bash[31648]: audit 2023-10-09T04:39:08.178840+0000 mon.a (mon.0) 490 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:39:09.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:09 smithi133 bash[27558]: cluster 2023-10-09T04:39:07.716770+0000 mgr.y (mgr.14152) 256 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:09.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:09 smithi133 bash[27558]: cephadm 2023-10-09T04:39:08.180183+0000 mgr.y (mgr.14152) 257 : cephadm [INF] Deploying daemon osd.5 on smithi194 2023-10-09T04:39:09.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:09 smithi133 bash[41291]: cluster 2023-10-09T04:39:07.716770+0000 mgr.y (mgr.14152) 256 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:09.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:09 smithi133 bash[41291]: cephadm 2023-10-09T04:39:08.180183+0000 mgr.y (mgr.14152) 257 : cephadm [INF] Deploying daemon osd.5 on smithi194 2023-10-09T04:39:09.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:09 smithi194 bash[31648]: cluster 2023-10-09T04:39:07.716770+0000 mgr.y (mgr.14152) 256 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:09.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:09 smithi194 bash[31648]: cephadm 2023-10-09T04:39:08.180183+0000 mgr.y (mgr.14152) 257 : cephadm [INF] Deploying daemon osd.5 on smithi194 2023-10-09T04:39:11.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:11 smithi133 bash[27558]: cluster 2023-10-09T04:39:09.717467+0000 mgr.y (mgr.14152) 258 : cluster [DBG] pgmap v231: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:11.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:11 smithi133 bash[41291]: cluster 2023-10-09T04:39:09.717467+0000 mgr.y (mgr.14152) 258 : cluster [DBG] pgmap v231: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:11.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:11 smithi194 bash[31648]: cluster 2023-10-09T04:39:09.717467+0000 mgr.y (mgr.14152) 258 : cluster [DBG] pgmap v231: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:13.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:13 smithi194 bash[31648]: cluster 2023-10-09T04:39:11.717976+0000 mgr.y (mgr.14152) 259 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:13.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:13 smithi194 bash[31648]: audit 2023-10-09T04:39:12.474122+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:13.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:13 smithi194 bash[31648]: audit 2023-10-09T04:39:12.480050+0000 mon.a (mon.0) 492 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:13.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:13 smithi194 bash[31648]: audit 2023-10-09T04:39:12.480414+0000 mon.a (mon.0) 493 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:39:13.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:13 smithi133 bash[41291]: cluster 2023-10-09T04:39:11.717976+0000 mgr.y (mgr.14152) 259 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:13.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:13 smithi133 bash[41291]: audit 2023-10-09T04:39:12.474122+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:13.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:13 smithi133 bash[41291]: audit 2023-10-09T04:39:12.480050+0000 mon.a (mon.0) 492 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:13.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:13 smithi133 bash[41291]: audit 2023-10-09T04:39:12.480414+0000 mon.a (mon.0) 493 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:39:13.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:13 smithi133 bash[27558]: cluster 2023-10-09T04:39:11.717976+0000 mgr.y (mgr.14152) 259 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:13.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:13 smithi133 bash[27558]: audit 2023-10-09T04:39:12.474122+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:13.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:13 smithi133 bash[27558]: audit 2023-10-09T04:39:12.480050+0000 mon.a (mon.0) 492 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:13.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:13 smithi133 bash[27558]: audit 2023-10-09T04:39:12.480414+0000 mon.a (mon.0) 493 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:39:15.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:15 smithi133 bash[27558]: cluster 2023-10-09T04:39:13.718506+0000 mgr.y (mgr.14152) 260 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:15.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:15 smithi133 bash[41291]: cluster 2023-10-09T04:39:13.718506+0000 mgr.y (mgr.14152) 260 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:15.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:15 smithi194 bash[31648]: cluster 2023-10-09T04:39:13.718506+0000 mgr.y (mgr.14152) 260 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:17.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:17 smithi194 bash[31648]: cluster 2023-10-09T04:39:15.719100+0000 mgr.y (mgr.14152) 261 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:17.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:17 smithi133 bash[27558]: cluster 2023-10-09T04:39:15.719100+0000 mgr.y (mgr.14152) 261 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:17.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:17 smithi133 bash[41291]: cluster 2023-10-09T04:39:15.719100+0000 mgr.y (mgr.14152) 261 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:18.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:18 smithi194 bash[31648]: audit 2023-10-09T04:39:17.671742+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:18.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:18 smithi194 bash[31648]: audit 2023-10-09T04:39:17.681444+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:18.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:18 smithi194 bash[31648]: audit 2023-10-09T04:39:17.682784+0000 mon.a (mon.0) 496 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:39:18.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:18 smithi194 bash[31648]: audit 2023-10-09T04:39:17.684011+0000 mon.a (mon.0) 497 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:39:18.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:18 smithi194 bash[31648]: audit 2023-10-09T04:39:17.691491+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:18.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:18 smithi194 bash[31648]: cluster 2023-10-09T04:39:17.719473+0000 mgr.y (mgr.14152) 262 : cluster [DBG] pgmap v235: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:19.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:18 smithi133 bash[27558]: audit 2023-10-09T04:39:17.671742+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:19.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:18 smithi133 bash[27558]: audit 2023-10-09T04:39:17.681444+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:19.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:18 smithi133 bash[27558]: audit 2023-10-09T04:39:17.682784+0000 mon.a (mon.0) 496 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:39:19.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:18 smithi133 bash[27558]: audit 2023-10-09T04:39:17.684011+0000 mon.a (mon.0) 497 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:39:19.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:18 smithi133 bash[27558]: audit 2023-10-09T04:39:17.691491+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:19.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:18 smithi133 bash[27558]: cluster 2023-10-09T04:39:17.719473+0000 mgr.y (mgr.14152) 262 : cluster [DBG] pgmap v235: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:19.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:18 smithi133 bash[41291]: audit 2023-10-09T04:39:17.671742+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:19.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:18 smithi133 bash[41291]: audit 2023-10-09T04:39:17.681444+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:19.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:18 smithi133 bash[41291]: audit 2023-10-09T04:39:17.682784+0000 mon.a (mon.0) 496 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:39:19.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:18 smithi133 bash[41291]: audit 2023-10-09T04:39:17.684011+0000 mon.a (mon.0) 497 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:39:19.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:18 smithi133 bash[41291]: audit 2023-10-09T04:39:17.691491+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:19.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:18 smithi133 bash[41291]: cluster 2023-10-09T04:39:17.719473+0000 mgr.y (mgr.14152) 262 : cluster [DBG] pgmap v235: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:20.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:19 smithi133 bash[27558]: audit 2023-10-09T04:39:19.105280+0000 mon.a (mon.0) 499 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T04:39:20.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:19 smithi133 bash[27558]: audit 2023-10-09T04:39:19.105717+0000 mon.b (mon.1) 15 : audit [INF] from='osd.5 [v2:172.21.15.194:6808/3989986662,v1:172.21.15.194:6809/3989986662]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T04:39:20.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:19 smithi133 bash[41291]: audit 2023-10-09T04:39:19.105280+0000 mon.a (mon.0) 499 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T04:39:20.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:19 smithi133 bash[41291]: audit 2023-10-09T04:39:19.105717+0000 mon.b (mon.1) 15 : audit [INF] from='osd.5 [v2:172.21.15.194:6808/3989986662,v1:172.21.15.194:6809/3989986662]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T04:39:20.126 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:19 smithi194 bash[31648]: audit 2023-10-09T04:39:19.105280+0000 mon.a (mon.0) 499 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T04:39:20.126 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:19 smithi194 bash[31648]: audit 2023-10-09T04:39:19.105717+0000 mon.b (mon.1) 15 : audit [INF] from='osd.5 [v2:172.21.15.194:6808/3989986662,v1:172.21.15.194:6809/3989986662]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T04:39:20.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:20 smithi194 bash[31648]: audit 2023-10-09T04:39:19.702346+0000 mon.a (mon.0) 500 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T04:39:20.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:20 smithi194 bash[31648]: cluster 2023-10-09T04:39:19.702406+0000 mon.a (mon.0) 501 : cluster [DBG] osdmap e36: 6 total, 5 up, 6 in 2023-10-09T04:39:20.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:20 smithi194 bash[31648]: audit 2023-10-09T04:39:19.702535+0000 mon.a (mon.0) 502 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:39:20.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:20 smithi194 bash[31648]: audit 2023-10-09T04:39:19.704407+0000 mon.a (mon.0) 503 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2023-10-09T04:39:20.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:20 smithi194 bash[31648]: audit 2023-10-09T04:39:19.704827+0000 mon.b (mon.1) 16 : audit [INF] from='osd.5 [v2:172.21.15.194:6808/3989986662,v1:172.21.15.194:6809/3989986662]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2023-10-09T04:39:20.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:20 smithi194 bash[31648]: cluster 2023-10-09T04:39:19.719926+0000 mgr.y (mgr.14152) 263 : cluster [DBG] pgmap v237: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:21.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:20 smithi133 bash[41291]: audit 2023-10-09T04:39:19.702346+0000 mon.a (mon.0) 500 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T04:39:21.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:20 smithi133 bash[41291]: cluster 2023-10-09T04:39:19.702406+0000 mon.a (mon.0) 501 : cluster [DBG] osdmap e36: 6 total, 5 up, 6 in 2023-10-09T04:39:21.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:20 smithi133 bash[41291]: audit 2023-10-09T04:39:19.702535+0000 mon.a (mon.0) 502 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:39:21.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:20 smithi133 bash[41291]: audit 2023-10-09T04:39:19.704407+0000 mon.a (mon.0) 503 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2023-10-09T04:39:21.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:20 smithi133 bash[41291]: audit 2023-10-09T04:39:19.704827+0000 mon.b (mon.1) 16 : audit [INF] from='osd.5 [v2:172.21.15.194:6808/3989986662,v1:172.21.15.194:6809/3989986662]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2023-10-09T04:39:21.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:20 smithi133 bash[41291]: cluster 2023-10-09T04:39:19.719926+0000 mgr.y (mgr.14152) 263 : cluster [DBG] pgmap v237: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:21.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:20 smithi133 bash[27558]: audit 2023-10-09T04:39:19.702346+0000 mon.a (mon.0) 500 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T04:39:21.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:20 smithi133 bash[27558]: cluster 2023-10-09T04:39:19.702406+0000 mon.a (mon.0) 501 : cluster [DBG] osdmap e36: 6 total, 5 up, 6 in 2023-10-09T04:39:21.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:20 smithi133 bash[27558]: audit 2023-10-09T04:39:19.702535+0000 mon.a (mon.0) 502 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:39:21.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:20 smithi133 bash[27558]: audit 2023-10-09T04:39:19.704407+0000 mon.a (mon.0) 503 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2023-10-09T04:39:21.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:20 smithi133 bash[27558]: audit 2023-10-09T04:39:19.704827+0000 mon.b (mon.1) 16 : audit [INF] from='osd.5 [v2:172.21.15.194:6808/3989986662,v1:172.21.15.194:6809/3989986662]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2023-10-09T04:39:21.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:20 smithi133 bash[27558]: cluster 2023-10-09T04:39:19.719926+0000 mgr.y (mgr.14152) 263 : cluster [DBG] pgmap v237: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:21.347 INFO:teuthology.orchestra.run.smithi194.stdout:Created osd(s) 5 on host 'smithi194' 2023-10-09T04:39:21.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:21 smithi194 bash[31648]: audit 2023-10-09T04:39:20.711499+0000 mon.a (mon.0) 504 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]': finished 2023-10-09T04:39:21.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:21 smithi194 bash[31648]: cluster 2023-10-09T04:39:20.711613+0000 mon.a (mon.0) 505 : cluster [DBG] osdmap e37: 6 total, 5 up, 6 in 2023-10-09T04:39:21.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:21 smithi194 bash[31648]: audit 2023-10-09T04:39:20.711831+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:39:21.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:21 smithi194 bash[31648]: audit 2023-10-09T04:39:20.717959+0000 mon.a (mon.0) 507 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:39:21.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:21 smithi194 bash[31648]: audit 2023-10-09T04:39:21.323034+0000 mon.a (mon.0) 508 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:21.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:21 smithi194 bash[31648]: audit 2023-10-09T04:39:21.328970+0000 mon.a (mon.0) 509 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:21.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:21 smithi194 bash[31648]: audit 2023-10-09T04:39:21.338613+0000 mon.a (mon.0) 510 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:39:22.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:21 smithi133 bash[41291]: audit 2023-10-09T04:39:20.711499+0000 mon.a (mon.0) 504 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]': finished 2023-10-09T04:39:22.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:21 smithi133 bash[41291]: cluster 2023-10-09T04:39:20.711613+0000 mon.a (mon.0) 505 : cluster [DBG] osdmap e37: 6 total, 5 up, 6 in 2023-10-09T04:39:22.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:21 smithi133 bash[41291]: audit 2023-10-09T04:39:20.711831+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:39:22.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:21 smithi133 bash[41291]: audit 2023-10-09T04:39:20.717959+0000 mon.a (mon.0) 507 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:39:22.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:21 smithi133 bash[41291]: audit 2023-10-09T04:39:21.323034+0000 mon.a (mon.0) 508 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:22.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:21 smithi133 bash[41291]: audit 2023-10-09T04:39:21.328970+0000 mon.a (mon.0) 509 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:22.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:21 smithi133 bash[41291]: audit 2023-10-09T04:39:21.338613+0000 mon.a (mon.0) 510 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:39:22.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:21 smithi133 bash[27558]: audit 2023-10-09T04:39:20.711499+0000 mon.a (mon.0) 504 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]': finished 2023-10-09T04:39:22.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:21 smithi133 bash[27558]: cluster 2023-10-09T04:39:20.711613+0000 mon.a (mon.0) 505 : cluster [DBG] osdmap e37: 6 total, 5 up, 6 in 2023-10-09T04:39:22.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:21 smithi133 bash[27558]: audit 2023-10-09T04:39:20.711831+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:39:22.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:21 smithi133 bash[27558]: audit 2023-10-09T04:39:20.717959+0000 mon.a (mon.0) 507 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:39:22.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:21 smithi133 bash[27558]: audit 2023-10-09T04:39:21.323034+0000 mon.a (mon.0) 508 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:22.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:21 smithi133 bash[27558]: audit 2023-10-09T04:39:21.328970+0000 mon.a (mon.0) 509 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:22.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:21 smithi133 bash[27558]: audit 2023-10-09T04:39:21.338613+0000 mon.a (mon.0) 510 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:39:22.081 DEBUG:teuthology.orchestra.run.smithi194:osd.5> sudo journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@osd.5.service 2023-10-09T04:39:22.084 INFO:tasks.cephadm:Deploying osd.6 on smithi194 with /dev/vg_nvme/lv_2... 2023-10-09T04:39:22.084 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-10-09T04:39:22.323 INFO:journalctl@ceph.osd.5.smithi194.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:39:23.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:22 smithi133 bash[27558]: cluster 2023-10-09T04:39:20.099532+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:39:23.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:22 smithi133 bash[27558]: cluster 2023-10-09T04:39:20.099741+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:39:23.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:22 smithi133 bash[27558]: audit 2023-10-09T04:39:21.717506+0000 mon.a (mon.0) 511 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:39:23.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:22 smithi133 bash[27558]: cluster 2023-10-09T04:39:21.720294+0000 mgr.y (mgr.14152) 264 : cluster [DBG] pgmap v240: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:23.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:22 smithi133 bash[27558]: cluster 2023-10-09T04:39:21.720516+0000 mon.a (mon.0) 512 : cluster [INF] osd.5 [v2:172.21.15.194:6808/3989986662,v1:172.21.15.194:6809/3989986662] boot 2023-10-09T04:39:23.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:22 smithi133 bash[27558]: cluster 2023-10-09T04:39:21.720564+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-10-09T04:39:23.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:22 smithi133 bash[27558]: audit 2023-10-09T04:39:21.720782+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:39:23.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:22 smithi133 bash[41291]: cluster 2023-10-09T04:39:20.099532+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:39:23.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:22 smithi133 bash[41291]: cluster 2023-10-09T04:39:20.099741+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:39:23.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:22 smithi133 bash[41291]: audit 2023-10-09T04:39:21.717506+0000 mon.a (mon.0) 511 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:39:23.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:22 smithi133 bash[41291]: cluster 2023-10-09T04:39:21.720294+0000 mgr.y (mgr.14152) 264 : cluster [DBG] pgmap v240: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:23.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:22 smithi133 bash[41291]: cluster 2023-10-09T04:39:21.720516+0000 mon.a (mon.0) 512 : cluster [INF] osd.5 [v2:172.21.15.194:6808/3989986662,v1:172.21.15.194:6809/3989986662] boot 2023-10-09T04:39:23.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:22 smithi133 bash[41291]: cluster 2023-10-09T04:39:21.720564+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-10-09T04:39:23.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:22 smithi133 bash[41291]: audit 2023-10-09T04:39:21.720782+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:39:23.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:22 smithi194 bash[31648]: cluster 2023-10-09T04:39:20.099532+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:39:23.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:22 smithi194 bash[31648]: cluster 2023-10-09T04:39:20.099741+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:39:23.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:22 smithi194 bash[31648]: audit 2023-10-09T04:39:21.717506+0000 mon.a (mon.0) 511 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:39:23.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:22 smithi194 bash[31648]: cluster 2023-10-09T04:39:21.720294+0000 mgr.y (mgr.14152) 264 : cluster [DBG] pgmap v240: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:39:23.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:22 smithi194 bash[31648]: cluster 2023-10-09T04:39:21.720516+0000 mon.a (mon.0) 512 : cluster [INF] osd.5 [v2:172.21.15.194:6808/3989986662,v1:172.21.15.194:6809/3989986662] boot 2023-10-09T04:39:23.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:22 smithi194 bash[31648]: cluster 2023-10-09T04:39:21.720564+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-10-09T04:39:23.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:22 smithi194 bash[31648]: audit 2023-10-09T04:39:21.720782+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:39:24.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:23 smithi133 bash[27558]: cluster 2023-10-09T04:39:22.737737+0000 mon.a (mon.0) 515 : cluster [DBG] osdmap e39: 6 total, 6 up, 6 in 2023-10-09T04:39:24.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:23 smithi133 bash[41291]: cluster 2023-10-09T04:39:22.737737+0000 mon.a (mon.0) 515 : cluster [DBG] osdmap e39: 6 total, 6 up, 6 in 2023-10-09T04:39:24.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:23 smithi194 bash[31648]: cluster 2023-10-09T04:39:22.737737+0000 mon.a (mon.0) 515 : cluster [DBG] osdmap e39: 6 total, 6 up, 6 in 2023-10-09T04:39:25.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:24 smithi133 bash[27558]: cluster 2023-10-09T04:39:23.720855+0000 mgr.y (mgr.14152) 265 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:25.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:24 smithi133 bash[27558]: cluster 2023-10-09T04:39:23.745268+0000 mon.a (mon.0) 516 : cluster [DBG] osdmap e40: 6 total, 6 up, 6 in 2023-10-09T04:39:25.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:24 smithi133 bash[41291]: cluster 2023-10-09T04:39:23.720855+0000 mgr.y (mgr.14152) 265 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:25.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:24 smithi133 bash[41291]: cluster 2023-10-09T04:39:23.745268+0000 mon.a (mon.0) 516 : cluster [DBG] osdmap e40: 6 total, 6 up, 6 in 2023-10-09T04:39:25.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:24 smithi194 bash[31648]: cluster 2023-10-09T04:39:23.720855+0000 mgr.y (mgr.14152) 265 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:25.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:24 smithi194 bash[31648]: cluster 2023-10-09T04:39:23.745268+0000 mon.a (mon.0) 516 : cluster [DBG] osdmap e40: 6 total, 6 up, 6 in 2023-10-09T04:39:26.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:25 smithi133 bash[27558]: cluster 2023-10-09T04:39:24.749881+0000 mon.a (mon.0) 517 : cluster [DBG] osdmap e41: 6 total, 6 up, 6 in 2023-10-09T04:39:26.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:25 smithi133 bash[41291]: cluster 2023-10-09T04:39:24.749881+0000 mon.a (mon.0) 517 : cluster [DBG] osdmap e41: 6 total, 6 up, 6 in 2023-10-09T04:39:26.110 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:39:26.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:25 smithi194 bash[31648]: cluster 2023-10-09T04:39:24.749881+0000 mon.a (mon.0) 517 : cluster [DBG] osdmap e41: 6 total, 6 up, 6 in 2023-10-09T04:39:27.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:26 smithi133 bash[27558]: cluster 2023-10-09T04:39:25.721388+0000 mgr.y (mgr.14152) 266 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:27.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:26 smithi133 bash[27558]: cluster 2023-10-09T04:39:25.754004+0000 mon.a (mon.0) 518 : cluster [DBG] osdmap e42: 6 total, 6 up, 6 in 2023-10-09T04:39:27.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:26 smithi133 bash[41291]: cluster 2023-10-09T04:39:25.721388+0000 mgr.y (mgr.14152) 266 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:27.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:26 smithi133 bash[41291]: cluster 2023-10-09T04:39:25.754004+0000 mon.a (mon.0) 518 : cluster [DBG] osdmap e42: 6 total, 6 up, 6 in 2023-10-09T04:39:27.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:26 smithi194 bash[31648]: cluster 2023-10-09T04:39:25.721388+0000 mgr.y (mgr.14152) 266 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:27.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:26 smithi194 bash[31648]: cluster 2023-10-09T04:39:25.754004+0000 mon.a (mon.0) 518 : cluster [DBG] osdmap e42: 6 total, 6 up, 6 in 2023-10-09T04:39:29.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:28 smithi133 bash[27558]: cluster 2023-10-09T04:39:27.721907+0000 mgr.y (mgr.14152) 267 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:29.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:28 smithi133 bash[41291]: cluster 2023-10-09T04:39:27.721907+0000 mgr.y (mgr.14152) 267 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:29.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:28 smithi194 bash[31648]: cluster 2023-10-09T04:39:27.721907+0000 mgr.y (mgr.14152) 267 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:30.427 INFO:teuthology.orchestra.run.smithi194.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-10-09T04:39:30.427 INFO:teuthology.orchestra.run.smithi194.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-10-09T04:39:30.428 INFO:teuthology.orchestra.run.smithi194.stderr: stderr: 10+0 records in 2023-10-09T04:39:30.428 INFO:teuthology.orchestra.run.smithi194.stderr:10+0 records out 2023-10-09T04:39:30.428 INFO:teuthology.orchestra.run.smithi194.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0173258 s, 605 MB/s 2023-10-09T04:39:30.428 INFO:teuthology.orchestra.run.smithi194.stderr:--> Zapping successful for: 2023-10-09T04:39:31.183 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi194:vg_nvme/lv_2 2023-10-09T04:39:31.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:31 smithi194 bash[31648]: cluster 2023-10-09T04:39:29.722454+0000 mgr.y (mgr.14152) 268 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:31.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:31 smithi133 bash[27558]: cluster 2023-10-09T04:39:29.722454+0000 mgr.y (mgr.14152) 268 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:31.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:31 smithi133 bash[41291]: cluster 2023-10-09T04:39:29.722454+0000 mgr.y (mgr.14152) 268 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:33.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:32 smithi133 bash[27558]: cluster 2023-10-09T04:39:31.723001+0000 mgr.y (mgr.14152) 269 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:33.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:32 smithi133 bash[27558]: cephadm 2023-10-09T04:39:31.942285+0000 mgr.y (mgr.14152) 270 : cephadm [INF] Detected new or changed devices on smithi194 2023-10-09T04:39:33.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:32 smithi133 bash[27558]: audit 2023-10-09T04:39:31.951043+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:33.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:32 smithi133 bash[27558]: audit 2023-10-09T04:39:31.959345+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:33.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:32 smithi133 bash[27558]: audit 2023-10-09T04:39:31.960928+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:39:33.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:32 smithi133 bash[27558]: audit 2023-10-09T04:39:31.962282+0000 mon.a (mon.0) 522 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:39:33.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:32 smithi133 bash[27558]: cephadm 2023-10-09T04:39:31.963286+0000 mgr.y (mgr.14152) 271 : cephadm [INF] Adjusting osd_memory_target on smithi194 to 8632M 2023-10-09T04:39:33.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:32 smithi133 bash[27558]: audit 2023-10-09T04:39:31.970947+0000 mon.a (mon.0) 523 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:33.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:32 smithi133 bash[27558]: audit 2023-10-09T04:39:31.972865+0000 mon.a (mon.0) 524 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:39:33.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:32 smithi133 bash[27558]: audit 2023-10-09T04:39:31.974433+0000 mon.a (mon.0) 525 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:39:33.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:32 smithi133 bash[27558]: audit 2023-10-09T04:39:31.984313+0000 mon.a (mon.0) 526 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:33.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:32 smithi133 bash[41291]: cluster 2023-10-09T04:39:31.723001+0000 mgr.y (mgr.14152) 269 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:33.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:32 smithi133 bash[41291]: cephadm 2023-10-09T04:39:31.942285+0000 mgr.y (mgr.14152) 270 : cephadm [INF] Detected new or changed devices on smithi194 2023-10-09T04:39:33.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:32 smithi133 bash[41291]: audit 2023-10-09T04:39:31.951043+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:33.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:32 smithi133 bash[41291]: audit 2023-10-09T04:39:31.959345+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:33.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:32 smithi133 bash[41291]: audit 2023-10-09T04:39:31.960928+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:39:33.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:32 smithi133 bash[41291]: audit 2023-10-09T04:39:31.962282+0000 mon.a (mon.0) 522 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:39:33.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:32 smithi133 bash[41291]: cephadm 2023-10-09T04:39:31.963286+0000 mgr.y (mgr.14152) 271 : cephadm [INF] Adjusting osd_memory_target on smithi194 to 8632M 2023-10-09T04:39:33.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:32 smithi133 bash[41291]: audit 2023-10-09T04:39:31.970947+0000 mon.a (mon.0) 523 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:33.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:32 smithi133 bash[41291]: audit 2023-10-09T04:39:31.972865+0000 mon.a (mon.0) 524 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:39:33.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:32 smithi133 bash[41291]: audit 2023-10-09T04:39:31.974433+0000 mon.a (mon.0) 525 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:39:33.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:32 smithi133 bash[41291]: audit 2023-10-09T04:39:31.984313+0000 mon.a (mon.0) 526 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:33.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:32 smithi194 bash[31648]: cluster 2023-10-09T04:39:31.723001+0000 mgr.y (mgr.14152) 269 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:33.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:32 smithi194 bash[31648]: cephadm 2023-10-09T04:39:31.942285+0000 mgr.y (mgr.14152) 270 : cephadm [INF] Detected new or changed devices on smithi194 2023-10-09T04:39:33.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:32 smithi194 bash[31648]: audit 2023-10-09T04:39:31.951043+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:33.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:32 smithi194 bash[31648]: audit 2023-10-09T04:39:31.959345+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:33.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:32 smithi194 bash[31648]: audit 2023-10-09T04:39:31.960928+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:39:33.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:32 smithi194 bash[31648]: audit 2023-10-09T04:39:31.962282+0000 mon.a (mon.0) 522 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:39:33.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:32 smithi194 bash[31648]: cephadm 2023-10-09T04:39:31.963286+0000 mgr.y (mgr.14152) 271 : cephadm [INF] Adjusting osd_memory_target on smithi194 to 8632M 2023-10-09T04:39:33.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:32 smithi194 bash[31648]: audit 2023-10-09T04:39:31.970947+0000 mon.a (mon.0) 523 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:33.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:32 smithi194 bash[31648]: audit 2023-10-09T04:39:31.972865+0000 mon.a (mon.0) 524 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:39:33.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:32 smithi194 bash[31648]: audit 2023-10-09T04:39:31.974433+0000 mon.a (mon.0) 525 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:39:33.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:32 smithi194 bash[31648]: audit 2023-10-09T04:39:31.984313+0000 mon.a (mon.0) 526 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:39:35.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:34 smithi133 bash[27558]: cluster 2023-10-09T04:39:33.723533+0000 mgr.y (mgr.14152) 272 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:35.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:34 smithi133 bash[41291]: cluster 2023-10-09T04:39:33.723533+0000 mgr.y (mgr.14152) 272 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:35.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:34 smithi194 bash[31648]: cluster 2023-10-09T04:39:33.723533+0000 mgr.y (mgr.14152) 272 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:35.920 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:39:37.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:37 smithi194 bash[31648]: cluster 2023-10-09T04:39:35.724073+0000 mgr.y (mgr.14152) 273 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:37.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:37 smithi133 bash[41291]: cluster 2023-10-09T04:39:35.724073+0000 mgr.y (mgr.14152) 273 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:37.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:37 smithi133 bash[27558]: cluster 2023-10-09T04:39:35.724073+0000 mgr.y (mgr.14152) 273 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:39.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:39 smithi194 bash[31648]: cluster 2023-10-09T04:39:37.724617+0000 mgr.y (mgr.14152) 274 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:39.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:39 smithi194 bash[31648]: audit 2023-10-09T04:39:38.874848+0000 mon.a (mon.0) 527 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:39:39.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:39 smithi194 bash[31648]: audit 2023-10-09T04:39:38.880756+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:39:39.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:39 smithi194 bash[31648]: audit 2023-10-09T04:39:38.882062+0000 mon.a (mon.0) 529 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:39:39.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:39 smithi133 bash[27558]: cluster 2023-10-09T04:39:37.724617+0000 mgr.y (mgr.14152) 274 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:39.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:39 smithi133 bash[27558]: audit 2023-10-09T04:39:38.874848+0000 mon.a (mon.0) 527 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:39:39.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:39 smithi133 bash[27558]: audit 2023-10-09T04:39:38.880756+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:39:39.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:39 smithi133 bash[27558]: audit 2023-10-09T04:39:38.882062+0000 mon.a (mon.0) 529 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:39:39.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:39 smithi133 bash[41291]: cluster 2023-10-09T04:39:37.724617+0000 mgr.y (mgr.14152) 274 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:39.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:39 smithi133 bash[41291]: audit 2023-10-09T04:39:38.874848+0000 mon.a (mon.0) 527 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:39:39.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:39 smithi133 bash[41291]: audit 2023-10-09T04:39:38.880756+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:39:39.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:39 smithi133 bash[41291]: audit 2023-10-09T04:39:38.882062+0000 mon.a (mon.0) 529 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:39:40.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:40 smithi194 bash[31648]: audit 2023-10-09T04:39:38.868495+0000 mgr.y (mgr.14152) 275 : audit [DBG] from='client.24280 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi194:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:39:40.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:40 smithi133 bash[27558]: audit 2023-10-09T04:39:38.868495+0000 mgr.y (mgr.14152) 275 : audit [DBG] from='client.24280 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi194:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:39:40.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:40 smithi133 bash[41291]: audit 2023-10-09T04:39:38.868495+0000 mgr.y (mgr.14152) 275 : audit [DBG] from='client.24280 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi194:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:39:41.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:41 smithi194 bash[31648]: cluster 2023-10-09T04:39:39.725250+0000 mgr.y (mgr.14152) 276 : cluster [DBG] pgmap v253: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:41.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:41 smithi133 bash[27558]: cluster 2023-10-09T04:39:39.725250+0000 mgr.y (mgr.14152) 276 : cluster [DBG] pgmap v253: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:41.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:41 smithi133 bash[41291]: cluster 2023-10-09T04:39:39.725250+0000 mgr.y (mgr.14152) 276 : cluster [DBG] pgmap v253: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:43.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:43 smithi194 bash[31648]: cluster 2023-10-09T04:39:41.725837+0000 mgr.y (mgr.14152) 277 : cluster [DBG] pgmap v254: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:43.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:43 smithi133 bash[27558]: cluster 2023-10-09T04:39:41.725837+0000 mgr.y (mgr.14152) 277 : cluster [DBG] pgmap v254: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:43.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:43 smithi133 bash[41291]: cluster 2023-10-09T04:39:41.725837+0000 mgr.y (mgr.14152) 277 : cluster [DBG] pgmap v254: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:45.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:45 smithi133 bash[27558]: cluster 2023-10-09T04:39:43.726416+0000 mgr.y (mgr.14152) 278 : cluster [DBG] pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:45.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:45 smithi133 bash[41291]: cluster 2023-10-09T04:39:43.726416+0000 mgr.y (mgr.14152) 278 : cluster [DBG] pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:45.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:45 smithi194 bash[31648]: cluster 2023-10-09T04:39:43.726416+0000 mgr.y (mgr.14152) 278 : cluster [DBG] pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:47.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:47 smithi133 bash[27558]: cluster 2023-10-09T04:39:45.726967+0000 mgr.y (mgr.14152) 279 : cluster [DBG] pgmap v256: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:47.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:47 smithi133 bash[27558]: audit 2023-10-09T04:39:47.256429+0000 mon.a (mon.0) 530 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7b69d014-f4c0-4e73-a14e-7d51b7ee3ea2"}]: dispatch 2023-10-09T04:39:47.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:47 smithi133 bash[27558]: audit 2023-10-09T04:39:47.256912+0000 mon.b (mon.1) 17 : audit [INF] from='client.? 172.21.15.194:0/837549863' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7b69d014-f4c0-4e73-a14e-7d51b7ee3ea2"}]: dispatch 2023-10-09T04:39:47.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:47 smithi133 bash[27558]: audit 2023-10-09T04:39:47.264439+0000 mon.a (mon.0) 531 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7b69d014-f4c0-4e73-a14e-7d51b7ee3ea2"}]': finished 2023-10-09T04:39:47.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:47 smithi133 bash[27558]: cluster 2023-10-09T04:39:47.264539+0000 mon.a (mon.0) 532 : cluster [DBG] osdmap e43: 7 total, 6 up, 7 in 2023-10-09T04:39:47.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:47 smithi133 bash[27558]: audit 2023-10-09T04:39:47.264780+0000 mon.a (mon.0) 533 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:39:47.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:47 smithi133 bash[41291]: cluster 2023-10-09T04:39:45.726967+0000 mgr.y (mgr.14152) 279 : cluster [DBG] pgmap v256: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:47.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:47 smithi133 bash[41291]: audit 2023-10-09T04:39:47.256429+0000 mon.a (mon.0) 530 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7b69d014-f4c0-4e73-a14e-7d51b7ee3ea2"}]: dispatch 2023-10-09T04:39:47.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:47 smithi133 bash[41291]: audit 2023-10-09T04:39:47.256912+0000 mon.b (mon.1) 17 : audit [INF] from='client.? 172.21.15.194:0/837549863' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7b69d014-f4c0-4e73-a14e-7d51b7ee3ea2"}]: dispatch 2023-10-09T04:39:47.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:47 smithi133 bash[41291]: audit 2023-10-09T04:39:47.264439+0000 mon.a (mon.0) 531 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7b69d014-f4c0-4e73-a14e-7d51b7ee3ea2"}]': finished 2023-10-09T04:39:47.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:47 smithi133 bash[41291]: cluster 2023-10-09T04:39:47.264539+0000 mon.a (mon.0) 532 : cluster [DBG] osdmap e43: 7 total, 6 up, 7 in 2023-10-09T04:39:47.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:47 smithi133 bash[41291]: audit 2023-10-09T04:39:47.264780+0000 mon.a (mon.0) 533 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:39:47.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:47 smithi194 bash[31648]: cluster 2023-10-09T04:39:45.726967+0000 mgr.y (mgr.14152) 279 : cluster [DBG] pgmap v256: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:47.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:47 smithi194 bash[31648]: audit 2023-10-09T04:39:47.256429+0000 mon.a (mon.0) 530 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7b69d014-f4c0-4e73-a14e-7d51b7ee3ea2"}]: dispatch 2023-10-09T04:39:47.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:47 smithi194 bash[31648]: audit 2023-10-09T04:39:47.256912+0000 mon.b (mon.1) 17 : audit [INF] from='client.? 172.21.15.194:0/837549863' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7b69d014-f4c0-4e73-a14e-7d51b7ee3ea2"}]: dispatch 2023-10-09T04:39:47.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:47 smithi194 bash[31648]: audit 2023-10-09T04:39:47.264439+0000 mon.a (mon.0) 531 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7b69d014-f4c0-4e73-a14e-7d51b7ee3ea2"}]': finished 2023-10-09T04:39:47.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:47 smithi194 bash[31648]: cluster 2023-10-09T04:39:47.264539+0000 mon.a (mon.0) 532 : cluster [DBG] osdmap e43: 7 total, 6 up, 7 in 2023-10-09T04:39:47.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:47 smithi194 bash[31648]: audit 2023-10-09T04:39:47.264780+0000 mon.a (mon.0) 533 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:39:48.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:48 smithi133 bash[41291]: audit 2023-10-09T04:39:47.995962+0000 mon.b (mon.1) 18 : audit [DBG] from='client.? 172.21.15.194:0/3153153053' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:39:48.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:48 smithi133 bash[27558]: audit 2023-10-09T04:39:47.995962+0000 mon.b (mon.1) 18 : audit [DBG] from='client.? 172.21.15.194:0/3153153053' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:39:48.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:48 smithi194 bash[31648]: audit 2023-10-09T04:39:47.995962+0000 mon.b (mon.1) 18 : audit [DBG] from='client.? 172.21.15.194:0/3153153053' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:39:49.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:49 smithi133 bash[27558]: cluster 2023-10-09T04:39:47.727457+0000 mgr.y (mgr.14152) 280 : cluster [DBG] pgmap v258: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:49.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:49 smithi133 bash[41291]: cluster 2023-10-09T04:39:47.727457+0000 mgr.y (mgr.14152) 280 : cluster [DBG] pgmap v258: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:49.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:49 smithi194 bash[31648]: cluster 2023-10-09T04:39:47.727457+0000 mgr.y (mgr.14152) 280 : cluster [DBG] pgmap v258: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:51.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:51 smithi133 bash[27558]: cluster 2023-10-09T04:39:49.728005+0000 mgr.y (mgr.14152) 281 : cluster [DBG] pgmap v259: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:51.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:51 smithi133 bash[41291]: cluster 2023-10-09T04:39:49.728005+0000 mgr.y (mgr.14152) 281 : cluster [DBG] pgmap v259: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:51.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:51 smithi194 bash[31648]: cluster 2023-10-09T04:39:49.728005+0000 mgr.y (mgr.14152) 281 : cluster [DBG] pgmap v259: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:53.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:53 smithi133 bash[27558]: cluster 2023-10-09T04:39:51.728566+0000 mgr.y (mgr.14152) 282 : cluster [DBG] pgmap v260: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:53.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:53 smithi133 bash[41291]: cluster 2023-10-09T04:39:51.728566+0000 mgr.y (mgr.14152) 282 : cluster [DBG] pgmap v260: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:53.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:53 smithi194 bash[31648]: cluster 2023-10-09T04:39:51.728566+0000 mgr.y (mgr.14152) 282 : cluster [DBG] pgmap v260: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:55.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:55 smithi133 bash[27558]: cluster 2023-10-09T04:39:53.729100+0000 mgr.y (mgr.14152) 283 : cluster [DBG] pgmap v261: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:55.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:55 smithi133 bash[41291]: cluster 2023-10-09T04:39:53.729100+0000 mgr.y (mgr.14152) 283 : cluster [DBG] pgmap v261: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:55.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:55 smithi194 bash[31648]: cluster 2023-10-09T04:39:53.729100+0000 mgr.y (mgr.14152) 283 : cluster [DBG] pgmap v261: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:57.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:57 smithi194 bash[31648]: cluster 2023-10-09T04:39:55.729689+0000 mgr.y (mgr.14152) 284 : cluster [DBG] pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:57.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:57 smithi133 bash[27558]: cluster 2023-10-09T04:39:55.729689+0000 mgr.y (mgr.14152) 284 : cluster [DBG] pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:57.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:57 smithi133 bash[41291]: cluster 2023-10-09T04:39:55.729689+0000 mgr.y (mgr.14152) 284 : cluster [DBG] pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:59.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:39:59 smithi194 bash[31648]: cluster 2023-10-09T04:39:57.730271+0000 mgr.y (mgr.14152) 285 : cluster [DBG] pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:59.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:39:59 smithi133 bash[27558]: cluster 2023-10-09T04:39:57.730271+0000 mgr.y (mgr.14152) 285 : cluster [DBG] pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:39:59.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:39:59 smithi133 bash[41291]: cluster 2023-10-09T04:39:57.730271+0000 mgr.y (mgr.14152) 285 : cluster [DBG] pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:40:00.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:00 smithi194 bash[31648]: audit 2023-10-09T04:39:59.955884+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T04:40:00.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:00 smithi194 bash[31648]: audit 2023-10-09T04:39:59.957628+0000 mon.a (mon.0) 535 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:40:00.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:00 smithi194 bash[31648]: cluster 2023-10-09T04:40:00.000092+0000 mon.a (mon.0) 536 : cluster [INF] overall HEALTH_OK 2023-10-09T04:40:00.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:00 smithi133 bash[27558]: audit 2023-10-09T04:39:59.955884+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T04:40:00.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:00 smithi133 bash[27558]: audit 2023-10-09T04:39:59.957628+0000 mon.a (mon.0) 535 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:40:00.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:00 smithi133 bash[27558]: cluster 2023-10-09T04:40:00.000092+0000 mon.a (mon.0) 536 : cluster [INF] overall HEALTH_OK 2023-10-09T04:40:00.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:00 smithi133 bash[41291]: audit 2023-10-09T04:39:59.955884+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T04:40:00.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:00 smithi133 bash[41291]: audit 2023-10-09T04:39:59.957628+0000 mon.a (mon.0) 535 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:40:00.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:00 smithi133 bash[41291]: cluster 2023-10-09T04:40:00.000092+0000 mon.a (mon.0) 536 : cluster [INF] overall HEALTH_OK 2023-10-09T04:40:01.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:01 smithi194 bash[31648]: cluster 2023-10-09T04:39:59.730842+0000 mgr.y (mgr.14152) 286 : cluster [DBG] pgmap v264: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:40:01.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:01 smithi194 bash[31648]: cephadm 2023-10-09T04:39:59.958964+0000 mgr.y (mgr.14152) 287 : cephadm [INF] Deploying daemon osd.6 on smithi194 2023-10-09T04:40:01.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:01 smithi133 bash[41291]: cluster 2023-10-09T04:39:59.730842+0000 mgr.y (mgr.14152) 286 : cluster [DBG] pgmap v264: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:40:01.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:01 smithi133 bash[41291]: cephadm 2023-10-09T04:39:59.958964+0000 mgr.y (mgr.14152) 287 : cephadm [INF] Deploying daemon osd.6 on smithi194 2023-10-09T04:40:01.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:01 smithi133 bash[27558]: cluster 2023-10-09T04:39:59.730842+0000 mgr.y (mgr.14152) 286 : cluster [DBG] pgmap v264: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:40:01.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:01 smithi133 bash[27558]: cephadm 2023-10-09T04:39:59.958964+0000 mgr.y (mgr.14152) 287 : cephadm [INF] Deploying daemon osd.6 on smithi194 2023-10-09T04:40:03.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:03 smithi194 bash[31648]: cluster 2023-10-09T04:40:01.731345+0000 mgr.y (mgr.14152) 288 : cluster [DBG] pgmap v265: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:40:03.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:03 smithi133 bash[27558]: cluster 2023-10-09T04:40:01.731345+0000 mgr.y (mgr.14152) 288 : cluster [DBG] pgmap v265: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:40:03.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:03 smithi133 bash[41291]: cluster 2023-10-09T04:40:01.731345+0000 mgr.y (mgr.14152) 288 : cluster [DBG] pgmap v265: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:40:05.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:05 smithi194 bash[31648]: cluster 2023-10-09T04:40:03.731876+0000 mgr.y (mgr.14152) 289 : cluster [DBG] pgmap v266: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:40:05.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:05 smithi194 bash[31648]: audit 2023-10-09T04:40:04.440174+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:05.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:05 smithi194 bash[31648]: audit 2023-10-09T04:40:04.451260+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:05.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:05 smithi194 bash[31648]: audit 2023-10-09T04:40:04.459108+0000 mon.a (mon.0) 539 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:40:05.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:05 smithi133 bash[27558]: cluster 2023-10-09T04:40:03.731876+0000 mgr.y (mgr.14152) 289 : cluster [DBG] pgmap v266: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:40:05.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:05 smithi133 bash[27558]: audit 2023-10-09T04:40:04.440174+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:05.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:05 smithi133 bash[27558]: audit 2023-10-09T04:40:04.451260+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:05.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:05 smithi133 bash[27558]: audit 2023-10-09T04:40:04.459108+0000 mon.a (mon.0) 539 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:40:05.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:05 smithi133 bash[41291]: cluster 2023-10-09T04:40:03.731876+0000 mgr.y (mgr.14152) 289 : cluster [DBG] pgmap v266: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:40:05.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:05 smithi133 bash[41291]: audit 2023-10-09T04:40:04.440174+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:05.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:05 smithi133 bash[41291]: audit 2023-10-09T04:40:04.451260+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:05.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:05 smithi133 bash[41291]: audit 2023-10-09T04:40:04.459108+0000 mon.a (mon.0) 539 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:40:07.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:07 smithi194 bash[31648]: cluster 2023-10-09T04:40:05.732443+0000 mgr.y (mgr.14152) 290 : cluster [DBG] pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:40:07.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:07 smithi133 bash[27558]: cluster 2023-10-09T04:40:05.732443+0000 mgr.y (mgr.14152) 290 : cluster [DBG] pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:40:07.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:07 smithi133 bash[41291]: cluster 2023-10-09T04:40:05.732443+0000 mgr.y (mgr.14152) 290 : cluster [DBG] pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:40:09.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:09 smithi194 bash[31648]: cluster 2023-10-09T04:40:07.733051+0000 mgr.y (mgr.14152) 291 : cluster [DBG] pgmap v268: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:40:09.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:09 smithi133 bash[27558]: cluster 2023-10-09T04:40:07.733051+0000 mgr.y (mgr.14152) 291 : cluster [DBG] pgmap v268: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:40:09.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:09 smithi133 bash[41291]: cluster 2023-10-09T04:40:07.733051+0000 mgr.y (mgr.14152) 291 : cluster [DBG] pgmap v268: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:40:10.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:10 smithi194 bash[31648]: audit 2023-10-09T04:40:09.706846+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:10.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:10 smithi194 bash[31648]: audit 2023-10-09T04:40:09.714745+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:10.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:10 smithi194 bash[31648]: cluster 2023-10-09T04:40:09.733647+0000 mgr.y (mgr.14152) 292 : cluster [DBG] pgmap v269: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:40:10.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:10 smithi194 bash[31648]: audit 2023-10-09T04:40:10.134714+0000 mon.a (mon.0) 542 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:40:10.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:10 smithi194 bash[31648]: audit 2023-10-09T04:40:10.136548+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:40:10.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:10 smithi194 bash[31648]: audit 2023-10-09T04:40:10.146254+0000 mon.a (mon.0) 544 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:11.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:10 smithi133 bash[27558]: audit 2023-10-09T04:40:09.706846+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:11.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:10 smithi133 bash[27558]: audit 2023-10-09T04:40:09.714745+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:11.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:10 smithi133 bash[27558]: cluster 2023-10-09T04:40:09.733647+0000 mgr.y (mgr.14152) 292 : cluster [DBG] pgmap v269: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:40:11.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:10 smithi133 bash[27558]: audit 2023-10-09T04:40:10.134714+0000 mon.a (mon.0) 542 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:40:11.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:10 smithi133 bash[27558]: audit 2023-10-09T04:40:10.136548+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:40:11.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:10 smithi133 bash[27558]: audit 2023-10-09T04:40:10.146254+0000 mon.a (mon.0) 544 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:11.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:10 smithi133 bash[41291]: audit 2023-10-09T04:40:09.706846+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:11.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:10 smithi133 bash[41291]: audit 2023-10-09T04:40:09.714745+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:11.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:10 smithi133 bash[41291]: cluster 2023-10-09T04:40:09.733647+0000 mgr.y (mgr.14152) 292 : cluster [DBG] pgmap v269: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:40:11.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:10 smithi133 bash[41291]: audit 2023-10-09T04:40:10.134714+0000 mon.a (mon.0) 542 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:40:11.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:10 smithi133 bash[41291]: audit 2023-10-09T04:40:10.136548+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:40:11.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:10 smithi133 bash[41291]: audit 2023-10-09T04:40:10.146254+0000 mon.a (mon.0) 544 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:11.993 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:11 smithi194 bash[31648]: audit 2023-10-09T04:40:11.099100+0000 mon.a (mon.0) 545 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T04:40:11.993 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:11 smithi194 bash[31648]: audit 2023-10-09T04:40:11.099592+0000 mon.b (mon.1) 19 : audit [INF] from='osd.6 [v2:172.21.15.194:6816/2607555245,v1:172.21.15.194:6817/2607555245]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T04:40:12.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:11 smithi133 bash[27558]: audit 2023-10-09T04:40:11.099100+0000 mon.a (mon.0) 545 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T04:40:12.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:11 smithi133 bash[27558]: audit 2023-10-09T04:40:11.099592+0000 mon.b (mon.1) 19 : audit [INF] from='osd.6 [v2:172.21.15.194:6816/2607555245,v1:172.21.15.194:6817/2607555245]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T04:40:12.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:11 smithi133 bash[41291]: audit 2023-10-09T04:40:11.099100+0000 mon.a (mon.0) 545 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T04:40:12.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:11 smithi133 bash[41291]: audit 2023-10-09T04:40:11.099592+0000 mon.b (mon.1) 19 : audit [INF] from='osd.6 [v2:172.21.15.194:6816/2607555245,v1:172.21.15.194:6817/2607555245]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T04:40:12.761 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:12 smithi194 bash[31648]: audit 2023-10-09T04:40:11.731902+0000 mon.a (mon.0) 546 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T04:40:12.761 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:12 smithi194 bash[31648]: cluster 2023-10-09T04:40:11.732011+0000 mon.a (mon.0) 547 : cluster [DBG] osdmap e44: 7 total, 6 up, 7 in 2023-10-09T04:40:12.762 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:12 smithi194 bash[31648]: audit 2023-10-09T04:40:11.732207+0000 mon.a (mon.0) 548 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:40:12.762 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:12 smithi194 bash[31648]: cluster 2023-10-09T04:40:11.734019+0000 mgr.y (mgr.14152) 293 : cluster [DBG] pgmap v271: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:40:12.762 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:12 smithi194 bash[31648]: audit 2023-10-09T04:40:11.734127+0000 mon.a (mon.0) 549 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2023-10-09T04:40:12.762 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:12 smithi194 bash[31648]: audit 2023-10-09T04:40:11.734740+0000 mon.b (mon.1) 20 : audit [INF] from='osd.6 [v2:172.21.15.194:6816/2607555245,v1:172.21.15.194:6817/2607555245]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2023-10-09T04:40:13.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:12 smithi133 bash[27558]: audit 2023-10-09T04:40:11.731902+0000 mon.a (mon.0) 546 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T04:40:13.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:12 smithi133 bash[27558]: cluster 2023-10-09T04:40:11.732011+0000 mon.a (mon.0) 547 : cluster [DBG] osdmap e44: 7 total, 6 up, 7 in 2023-10-09T04:40:13.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:12 smithi133 bash[27558]: audit 2023-10-09T04:40:11.732207+0000 mon.a (mon.0) 548 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:40:13.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:12 smithi133 bash[27558]: cluster 2023-10-09T04:40:11.734019+0000 mgr.y (mgr.14152) 293 : cluster [DBG] pgmap v271: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:40:13.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:12 smithi133 bash[27558]: audit 2023-10-09T04:40:11.734127+0000 mon.a (mon.0) 549 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2023-10-09T04:40:13.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:12 smithi133 bash[27558]: audit 2023-10-09T04:40:11.734740+0000 mon.b (mon.1) 20 : audit [INF] from='osd.6 [v2:172.21.15.194:6816/2607555245,v1:172.21.15.194:6817/2607555245]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2023-10-09T04:40:13.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:12 smithi133 bash[41291]: audit 2023-10-09T04:40:11.731902+0000 mon.a (mon.0) 546 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T04:40:13.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:12 smithi133 bash[41291]: cluster 2023-10-09T04:40:11.732011+0000 mon.a (mon.0) 547 : cluster [DBG] osdmap e44: 7 total, 6 up, 7 in 2023-10-09T04:40:13.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:12 smithi133 bash[41291]: audit 2023-10-09T04:40:11.732207+0000 mon.a (mon.0) 548 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:40:13.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:12 smithi133 bash[41291]: cluster 2023-10-09T04:40:11.734019+0000 mgr.y (mgr.14152) 293 : cluster [DBG] pgmap v271: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:40:13.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:12 smithi133 bash[41291]: audit 2023-10-09T04:40:11.734127+0000 mon.a (mon.0) 549 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2023-10-09T04:40:13.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:12 smithi133 bash[41291]: audit 2023-10-09T04:40:11.734740+0000 mon.b (mon.1) 20 : audit [INF] from='osd.6 [v2:172.21.15.194:6816/2607555245,v1:172.21.15.194:6817/2607555245]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2023-10-09T04:40:13.240 INFO:teuthology.orchestra.run.smithi194.stdout:Created osd(s) 6 on host 'smithi194' 2023-10-09T04:40:13.923 DEBUG:teuthology.orchestra.run.smithi194:osd.6> sudo journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@osd.6.service 2023-10-09T04:40:13.925 INFO:tasks.cephadm:Deploying osd.7 on smithi194 with /dev/vg_nvme/lv_1... 2023-10-09T04:40:13.925 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-10-09T04:40:14.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:13 smithi133 bash[27558]: audit 2023-10-09T04:40:12.741259+0000 mon.a (mon.0) 550 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]': finished 2023-10-09T04:40:14.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:13 smithi133 bash[27558]: cluster 2023-10-09T04:40:12.741445+0000 mon.a (mon.0) 551 : cluster [DBG] osdmap e45: 7 total, 6 up, 7 in 2023-10-09T04:40:14.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:13 smithi133 bash[27558]: audit 2023-10-09T04:40:12.741799+0000 mon.a (mon.0) 552 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:40:14.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:13 smithi133 bash[27558]: audit 2023-10-09T04:40:12.746513+0000 mon.a (mon.0) 553 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:40:14.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:13 smithi133 bash[27558]: audit 2023-10-09T04:40:13.220150+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:14.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:13 smithi133 bash[27558]: audit 2023-10-09T04:40:13.227970+0000 mon.a (mon.0) 555 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:14.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:13 smithi133 bash[27558]: audit 2023-10-09T04:40:13.237306+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:40:14.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:13 smithi133 bash[41291]: audit 2023-10-09T04:40:12.741259+0000 mon.a (mon.0) 550 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]': finished 2023-10-09T04:40:14.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:13 smithi133 bash[41291]: cluster 2023-10-09T04:40:12.741445+0000 mon.a (mon.0) 551 : cluster [DBG] osdmap e45: 7 total, 6 up, 7 in 2023-10-09T04:40:14.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:13 smithi133 bash[41291]: audit 2023-10-09T04:40:12.741799+0000 mon.a (mon.0) 552 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:40:14.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:13 smithi133 bash[41291]: audit 2023-10-09T04:40:12.746513+0000 mon.a (mon.0) 553 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:40:14.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:13 smithi133 bash[41291]: audit 2023-10-09T04:40:13.220150+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:14.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:13 smithi133 bash[41291]: audit 2023-10-09T04:40:13.227970+0000 mon.a (mon.0) 555 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:14.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:13 smithi133 bash[41291]: audit 2023-10-09T04:40:13.237306+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:40:14.070 INFO:journalctl@ceph.osd.6.smithi194.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:40:14.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:13 smithi194 bash[31648]: audit 2023-10-09T04:40:12.741259+0000 mon.a (mon.0) 550 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]': finished 2023-10-09T04:40:14.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:13 smithi194 bash[31648]: cluster 2023-10-09T04:40:12.741445+0000 mon.a (mon.0) 551 : cluster [DBG] osdmap e45: 7 total, 6 up, 7 in 2023-10-09T04:40:14.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:13 smithi194 bash[31648]: audit 2023-10-09T04:40:12.741799+0000 mon.a (mon.0) 552 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:40:14.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:13 smithi194 bash[31648]: audit 2023-10-09T04:40:12.746513+0000 mon.a (mon.0) 553 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:40:14.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:13 smithi194 bash[31648]: audit 2023-10-09T04:40:13.220150+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:14.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:13 smithi194 bash[31648]: audit 2023-10-09T04:40:13.227970+0000 mon.a (mon.0) 555 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:14.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:13 smithi194 bash[31648]: audit 2023-10-09T04:40:13.237306+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:40:15.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:14 smithi133 bash[41291]: cluster 2023-10-09T04:40:12.106384+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:40:15.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:14 smithi133 bash[41291]: cluster 2023-10-09T04:40:12.106674+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:40:15.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:14 smithi133 bash[41291]: cluster 2023-10-09T04:40:13.734607+0000 mgr.y (mgr.14152) 294 : cluster [DBG] pgmap v273: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:40:15.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:14 smithi133 bash[41291]: cluster 2023-10-09T04:40:13.744962+0000 mon.a (mon.0) 557 : cluster [INF] osd.6 [v2:172.21.15.194:6816/2607555245,v1:172.21.15.194:6817/2607555245] boot 2023-10-09T04:40:15.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:14 smithi133 bash[41291]: cluster 2023-10-09T04:40:13.745042+0000 mon.a (mon.0) 558 : cluster [DBG] osdmap e46: 7 total, 7 up, 7 in 2023-10-09T04:40:15.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:14 smithi133 bash[41291]: audit 2023-10-09T04:40:13.745709+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:40:15.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:14 smithi133 bash[27558]: cluster 2023-10-09T04:40:12.106384+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:40:15.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:14 smithi133 bash[27558]: cluster 2023-10-09T04:40:12.106674+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:40:15.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:14 smithi133 bash[27558]: cluster 2023-10-09T04:40:13.734607+0000 mgr.y (mgr.14152) 294 : cluster [DBG] pgmap v273: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:40:15.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:14 smithi133 bash[27558]: cluster 2023-10-09T04:40:13.744962+0000 mon.a (mon.0) 557 : cluster [INF] osd.6 [v2:172.21.15.194:6816/2607555245,v1:172.21.15.194:6817/2607555245] boot 2023-10-09T04:40:15.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:14 smithi133 bash[27558]: cluster 2023-10-09T04:40:13.745042+0000 mon.a (mon.0) 558 : cluster [DBG] osdmap e46: 7 total, 7 up, 7 in 2023-10-09T04:40:15.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:14 smithi133 bash[27558]: audit 2023-10-09T04:40:13.745709+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:40:15.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:14 smithi194 bash[31648]: cluster 2023-10-09T04:40:12.106384+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:40:15.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:14 smithi194 bash[31648]: cluster 2023-10-09T04:40:12.106674+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:40:15.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:14 smithi194 bash[31648]: cluster 2023-10-09T04:40:13.734607+0000 mgr.y (mgr.14152) 294 : cluster [DBG] pgmap v273: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:40:15.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:14 smithi194 bash[31648]: cluster 2023-10-09T04:40:13.744962+0000 mon.a (mon.0) 557 : cluster [INF] osd.6 [v2:172.21.15.194:6816/2607555245,v1:172.21.15.194:6817/2607555245] boot 2023-10-09T04:40:15.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:14 smithi194 bash[31648]: cluster 2023-10-09T04:40:13.745042+0000 mon.a (mon.0) 558 : cluster [DBG] osdmap e46: 7 total, 7 up, 7 in 2023-10-09T04:40:15.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:14 smithi194 bash[31648]: audit 2023-10-09T04:40:13.745709+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:40:16.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:15 smithi133 bash[27558]: cluster 2023-10-09T04:40:14.758535+0000 mon.a (mon.0) 560 : cluster [DBG] osdmap e47: 7 total, 7 up, 7 in 2023-10-09T04:40:16.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:15 smithi133 bash[41291]: cluster 2023-10-09T04:40:14.758535+0000 mon.a (mon.0) 560 : cluster [DBG] osdmap e47: 7 total, 7 up, 7 in 2023-10-09T04:40:16.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:15 smithi194 bash[31648]: cluster 2023-10-09T04:40:14.758535+0000 mon.a (mon.0) 560 : cluster [DBG] osdmap e47: 7 total, 7 up, 7 in 2023-10-09T04:40:17.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:16 smithi133 bash[27558]: cluster 2023-10-09T04:40:15.735182+0000 mgr.y (mgr.14152) 295 : cluster [DBG] pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:17.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:16 smithi133 bash[27558]: cluster 2023-10-09T04:40:15.766575+0000 mon.a (mon.0) 561 : cluster [DBG] osdmap e48: 7 total, 7 up, 7 in 2023-10-09T04:40:17.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:16 smithi133 bash[41291]: cluster 2023-10-09T04:40:15.735182+0000 mgr.y (mgr.14152) 295 : cluster [DBG] pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:17.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:16 smithi133 bash[41291]: cluster 2023-10-09T04:40:15.766575+0000 mon.a (mon.0) 561 : cluster [DBG] osdmap e48: 7 total, 7 up, 7 in 2023-10-09T04:40:17.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:16 smithi194 bash[31648]: cluster 2023-10-09T04:40:15.735182+0000 mgr.y (mgr.14152) 295 : cluster [DBG] pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:17.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:16 smithi194 bash[31648]: cluster 2023-10-09T04:40:15.766575+0000 mon.a (mon.0) 561 : cluster [DBG] osdmap e48: 7 total, 7 up, 7 in 2023-10-09T04:40:18.016 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:40:18.041 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:17 smithi194 bash[31648]: cluster 2023-10-09T04:40:16.771917+0000 mon.a (mon.0) 562 : cluster [DBG] osdmap e49: 7 total, 7 up, 7 in 2023-10-09T04:40:18.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:17 smithi133 bash[27558]: cluster 2023-10-09T04:40:16.771917+0000 mon.a (mon.0) 562 : cluster [DBG] osdmap e49: 7 total, 7 up, 7 in 2023-10-09T04:40:18.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:17 smithi133 bash[41291]: cluster 2023-10-09T04:40:16.771917+0000 mon.a (mon.0) 562 : cluster [DBG] osdmap e49: 7 total, 7 up, 7 in 2023-10-09T04:40:19.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:18 smithi133 bash[27558]: cluster 2023-10-09T04:40:17.735794+0000 mgr.y (mgr.14152) 296 : cluster [DBG] pgmap v279: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:19.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:18 smithi133 bash[27558]: cluster 2023-10-09T04:40:17.780448+0000 mon.a (mon.0) 563 : cluster [DBG] osdmap e50: 7 total, 7 up, 7 in 2023-10-09T04:40:19.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:18 smithi133 bash[41291]: cluster 2023-10-09T04:40:17.735794+0000 mgr.y (mgr.14152) 296 : cluster [DBG] pgmap v279: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:19.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:18 smithi133 bash[41291]: cluster 2023-10-09T04:40:17.780448+0000 mon.a (mon.0) 563 : cluster [DBG] osdmap e50: 7 total, 7 up, 7 in 2023-10-09T04:40:19.099 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:18 smithi194 bash[31648]: cluster 2023-10-09T04:40:17.735794+0000 mgr.y (mgr.14152) 296 : cluster [DBG] pgmap v279: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:19.100 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:18 smithi194 bash[31648]: cluster 2023-10-09T04:40:17.780448+0000 mon.a (mon.0) 563 : cluster [DBG] osdmap e50: 7 total, 7 up, 7 in 2023-10-09T04:40:21.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:21 smithi194 bash[31648]: cluster 2023-10-09T04:40:19.736417+0000 mgr.y (mgr.14152) 297 : cluster [DBG] pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 116 KiB/s, 0 objects/s recovering 2023-10-09T04:40:21.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:21 smithi133 bash[27558]: cluster 2023-10-09T04:40:19.736417+0000 mgr.y (mgr.14152) 297 : cluster [DBG] pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 116 KiB/s, 0 objects/s recovering 2023-10-09T04:40:21.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:21 smithi133 bash[41291]: cluster 2023-10-09T04:40:19.736417+0000 mgr.y (mgr.14152) 297 : cluster [DBG] pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 116 KiB/s, 0 objects/s recovering 2023-10-09T04:40:22.410 INFO:teuthology.orchestra.run.smithi194.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-10-09T04:40:22.411 INFO:teuthology.orchestra.run.smithi194.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-10-09T04:40:22.411 INFO:teuthology.orchestra.run.smithi194.stderr: stderr: 10+0 records in 2023-10-09T04:40:22.411 INFO:teuthology.orchestra.run.smithi194.stderr:10+0 records out 2023-10-09T04:40:22.411 INFO:teuthology.orchestra.run.smithi194.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0171352 s, 612 MB/s 2023-10-09T04:40:22.411 INFO:teuthology.orchestra.run.smithi194.stderr:--> Zapping successful for: 2023-10-09T04:40:23.170 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi194:vg_nvme/lv_1 2023-10-09T04:40:23.481 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:23 smithi194 bash[31648]: cluster 2023-10-09T04:40:21.737088+0000 mgr.y (mgr.14152) 298 : cluster [DBG] pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-09T04:40:23.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:23 smithi133 bash[27558]: cluster 2023-10-09T04:40:21.737088+0000 mgr.y (mgr.14152) 298 : cluster [DBG] pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-09T04:40:23.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:23 smithi133 bash[41291]: cluster 2023-10-09T04:40:21.737088+0000 mgr.y (mgr.14152) 298 : cluster [DBG] pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-09T04:40:25.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:24 smithi133 bash[27558]: cluster 2023-10-09T04:40:23.737728+0000 mgr.y (mgr.14152) 299 : cluster [DBG] pgmap v283: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T04:40:25.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:24 smithi133 bash[27558]: cephadm 2023-10-09T04:40:23.935214+0000 mgr.y (mgr.14152) 300 : cephadm [INF] Detected new or changed devices on smithi194 2023-10-09T04:40:25.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:24 smithi133 bash[27558]: audit 2023-10-09T04:40:23.944103+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:25.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:24 smithi133 bash[27558]: audit 2023-10-09T04:40:23.952996+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:25.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:24 smithi133 bash[27558]: audit 2023-10-09T04:40:23.954542+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:40:25.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:24 smithi133 bash[27558]: audit 2023-10-09T04:40:23.955787+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:40:25.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:24 smithi133 bash[27558]: audit 2023-10-09T04:40:23.956945+0000 mon.a (mon.0) 568 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:40:25.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:24 smithi133 bash[27558]: cephadm 2023-10-09T04:40:23.957927+0000 mgr.y (mgr.14152) 301 : cephadm [INF] Adjusting osd_memory_target on smithi194 to 5755M 2023-10-09T04:40:25.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:24 smithi133 bash[27558]: audit 2023-10-09T04:40:23.964495+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:25.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:24 smithi133 bash[27558]: audit 2023-10-09T04:40:23.966264+0000 mon.a (mon.0) 570 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:40:25.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:24 smithi133 bash[27558]: audit 2023-10-09T04:40:23.967441+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:40:25.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:24 smithi133 bash[27558]: audit 2023-10-09T04:40:23.976243+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:25.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:24 smithi133 bash[41291]: cluster 2023-10-09T04:40:23.737728+0000 mgr.y (mgr.14152) 299 : cluster [DBG] pgmap v283: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T04:40:25.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:24 smithi133 bash[41291]: cephadm 2023-10-09T04:40:23.935214+0000 mgr.y (mgr.14152) 300 : cephadm [INF] Detected new or changed devices on smithi194 2023-10-09T04:40:25.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:24 smithi133 bash[41291]: audit 2023-10-09T04:40:23.944103+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:25.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:24 smithi133 bash[41291]: audit 2023-10-09T04:40:23.952996+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:25.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:24 smithi133 bash[41291]: audit 2023-10-09T04:40:23.954542+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:40:25.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:24 smithi133 bash[41291]: audit 2023-10-09T04:40:23.955787+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:40:25.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:24 smithi133 bash[41291]: audit 2023-10-09T04:40:23.956945+0000 mon.a (mon.0) 568 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:40:25.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:24 smithi133 bash[41291]: cephadm 2023-10-09T04:40:23.957927+0000 mgr.y (mgr.14152) 301 : cephadm [INF] Adjusting osd_memory_target on smithi194 to 5755M 2023-10-09T04:40:25.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:24 smithi133 bash[41291]: audit 2023-10-09T04:40:23.964495+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:25.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:24 smithi133 bash[41291]: audit 2023-10-09T04:40:23.966264+0000 mon.a (mon.0) 570 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:40:25.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:24 smithi133 bash[41291]: audit 2023-10-09T04:40:23.967441+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:40:25.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:24 smithi133 bash[41291]: audit 2023-10-09T04:40:23.976243+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:25.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:24 smithi194 bash[31648]: cluster 2023-10-09T04:40:23.737728+0000 mgr.y (mgr.14152) 299 : cluster [DBG] pgmap v283: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T04:40:25.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:24 smithi194 bash[31648]: cephadm 2023-10-09T04:40:23.935214+0000 mgr.y (mgr.14152) 300 : cephadm [INF] Detected new or changed devices on smithi194 2023-10-09T04:40:25.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:24 smithi194 bash[31648]: audit 2023-10-09T04:40:23.944103+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:25.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:24 smithi194 bash[31648]: audit 2023-10-09T04:40:23.952996+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:25.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:24 smithi194 bash[31648]: audit 2023-10-09T04:40:23.954542+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:40:25.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:24 smithi194 bash[31648]: audit 2023-10-09T04:40:23.955787+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:40:25.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:24 smithi194 bash[31648]: audit 2023-10-09T04:40:23.956945+0000 mon.a (mon.0) 568 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:40:25.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:24 smithi194 bash[31648]: cephadm 2023-10-09T04:40:23.957927+0000 mgr.y (mgr.14152) 301 : cephadm [INF] Adjusting osd_memory_target on smithi194 to 5755M 2023-10-09T04:40:25.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:24 smithi194 bash[31648]: audit 2023-10-09T04:40:23.964495+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:25.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:24 smithi194 bash[31648]: audit 2023-10-09T04:40:23.966264+0000 mon.a (mon.0) 570 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:40:25.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:24 smithi194 bash[31648]: audit 2023-10-09T04:40:23.967441+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:40:25.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:24 smithi194 bash[31648]: audit 2023-10-09T04:40:23.976243+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:27.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:27 smithi194 bash[31648]: cluster 2023-10-09T04:40:25.738282+0000 mgr.y (mgr.14152) 302 : cluster [DBG] pgmap v284: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 64 KiB/s, 0 objects/s recovering 2023-10-09T04:40:27.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:27 smithi133 bash[27558]: cluster 2023-10-09T04:40:25.738282+0000 mgr.y (mgr.14152) 302 : cluster [DBG] pgmap v284: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 64 KiB/s, 0 objects/s recovering 2023-10-09T04:40:27.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:27 smithi133 bash[41291]: cluster 2023-10-09T04:40:25.738282+0000 mgr.y (mgr.14152) 302 : cluster [DBG] pgmap v284: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 64 KiB/s, 0 objects/s recovering 2023-10-09T04:40:27.908 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:40:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:29 smithi194 bash[31648]: cluster 2023-10-09T04:40:27.738763+0000 mgr.y (mgr.14152) 303 : cluster [DBG] pgmap v285: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T04:40:29.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:29 smithi133 bash[27558]: cluster 2023-10-09T04:40:27.738763+0000 mgr.y (mgr.14152) 303 : cluster [DBG] pgmap v285: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T04:40:29.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:29 smithi133 bash[41291]: cluster 2023-10-09T04:40:27.738763+0000 mgr.y (mgr.14152) 303 : cluster [DBG] pgmap v285: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T04:40:31.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:31 smithi133 bash[27558]: cluster 2023-10-09T04:40:29.739329+0000 mgr.y (mgr.14152) 304 : cluster [DBG] pgmap v286: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-10-09T04:40:31.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:31 smithi133 bash[27558]: audit 2023-10-09T04:40:30.959198+0000 mon.a (mon.0) 573 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:40:31.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:31 smithi133 bash[27558]: audit 2023-10-09T04:40:30.965193+0000 mon.a (mon.0) 574 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:40:31.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:31 smithi133 bash[27558]: audit 2023-10-09T04:40:30.966774+0000 mon.a (mon.0) 575 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:40:31.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:31 smithi133 bash[41291]: cluster 2023-10-09T04:40:29.739329+0000 mgr.y (mgr.14152) 304 : cluster [DBG] pgmap v286: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-10-09T04:40:31.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:31 smithi133 bash[41291]: audit 2023-10-09T04:40:30.959198+0000 mon.a (mon.0) 573 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:40:31.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:31 smithi133 bash[41291]: audit 2023-10-09T04:40:30.965193+0000 mon.a (mon.0) 574 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:40:31.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:31 smithi133 bash[41291]: audit 2023-10-09T04:40:30.966774+0000 mon.a (mon.0) 575 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:40:31.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:31 smithi194 bash[31648]: cluster 2023-10-09T04:40:29.739329+0000 mgr.y (mgr.14152) 304 : cluster [DBG] pgmap v286: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-10-09T04:40:31.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:31 smithi194 bash[31648]: audit 2023-10-09T04:40:30.959198+0000 mon.a (mon.0) 573 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:40:31.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:31 smithi194 bash[31648]: audit 2023-10-09T04:40:30.965193+0000 mon.a (mon.0) 574 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:40:31.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:31 smithi194 bash[31648]: audit 2023-10-09T04:40:30.966774+0000 mon.a (mon.0) 575 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:40:32.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:32 smithi133 bash[27558]: audit 2023-10-09T04:40:30.953159+0000 mgr.y (mgr.14152) 305 : audit [DBG] from='client.24307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi194:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:40:32.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:32 smithi133 bash[41291]: audit 2023-10-09T04:40:30.953159+0000 mgr.y (mgr.14152) 305 : audit [DBG] from='client.24307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi194:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:40:32.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:32 smithi194 bash[31648]: audit 2023-10-09T04:40:30.953159+0000 mgr.y (mgr.14152) 305 : audit [DBG] from='client.24307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi194:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:40:33.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:33 smithi133 bash[27558]: cluster 2023-10-09T04:40:31.739930+0000 mgr.y (mgr.14152) 306 : cluster [DBG] pgmap v287: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:33.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:33 smithi133 bash[41291]: cluster 2023-10-09T04:40:31.739930+0000 mgr.y (mgr.14152) 306 : cluster [DBG] pgmap v287: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:33.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:33 smithi194 bash[31648]: cluster 2023-10-09T04:40:31.739930+0000 mgr.y (mgr.14152) 306 : cluster [DBG] pgmap v287: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:35.764 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:35 smithi194 bash[31648]: cluster 2023-10-09T04:40:33.740470+0000 mgr.y (mgr.14152) 307 : cluster [DBG] pgmap v288: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:35.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:35 smithi133 bash[27558]: cluster 2023-10-09T04:40:33.740470+0000 mgr.y (mgr.14152) 307 : cluster [DBG] pgmap v288: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:35.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:35 smithi133 bash[41291]: cluster 2023-10-09T04:40:33.740470+0000 mgr.y (mgr.14152) 307 : cluster [DBG] pgmap v288: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:37.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:37 smithi133 bash[27558]: cluster 2023-10-09T04:40:35.741047+0000 mgr.y (mgr.14152) 308 : cluster [DBG] pgmap v289: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:37.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:37 smithi133 bash[41291]: cluster 2023-10-09T04:40:35.741047+0000 mgr.y (mgr.14152) 308 : cluster [DBG] pgmap v289: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:37.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:37 smithi194 bash[31648]: cluster 2023-10-09T04:40:35.741047+0000 mgr.y (mgr.14152) 308 : cluster [DBG] pgmap v289: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:39.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:39 smithi133 bash[27558]: cluster 2023-10-09T04:40:37.741572+0000 mgr.y (mgr.14152) 309 : cluster [DBG] pgmap v290: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:39.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:39 smithi133 bash[27558]: audit 2023-10-09T04:40:39.314652+0000 mon.a (mon.0) 576 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "02506268-102c-4654-9b5c-68d36c28f7f7"}]: dispatch 2023-10-09T04:40:39.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:39 smithi133 bash[27558]: audit 2023-10-09T04:40:39.315194+0000 mon.b (mon.1) 21 : audit [INF] from='client.? 172.21.15.194:0/4076959799' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "02506268-102c-4654-9b5c-68d36c28f7f7"}]: dispatch 2023-10-09T04:40:39.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:39 smithi133 bash[27558]: audit 2023-10-09T04:40:39.329627+0000 mon.a (mon.0) 577 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "02506268-102c-4654-9b5c-68d36c28f7f7"}]': finished 2023-10-09T04:40:39.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:39 smithi133 bash[27558]: cluster 2023-10-09T04:40:39.329758+0000 mon.a (mon.0) 578 : cluster [DBG] osdmap e51: 8 total, 7 up, 8 in 2023-10-09T04:40:39.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:39 smithi133 bash[27558]: audit 2023-10-09T04:40:39.329975+0000 mon.a (mon.0) 579 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:40:39.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:39 smithi133 bash[41291]: cluster 2023-10-09T04:40:37.741572+0000 mgr.y (mgr.14152) 309 : cluster [DBG] pgmap v290: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:39.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:39 smithi133 bash[41291]: audit 2023-10-09T04:40:39.314652+0000 mon.a (mon.0) 576 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "02506268-102c-4654-9b5c-68d36c28f7f7"}]: dispatch 2023-10-09T04:40:39.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:39 smithi133 bash[41291]: audit 2023-10-09T04:40:39.315194+0000 mon.b (mon.1) 21 : audit [INF] from='client.? 172.21.15.194:0/4076959799' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "02506268-102c-4654-9b5c-68d36c28f7f7"}]: dispatch 2023-10-09T04:40:39.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:39 smithi133 bash[41291]: audit 2023-10-09T04:40:39.329627+0000 mon.a (mon.0) 577 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "02506268-102c-4654-9b5c-68d36c28f7f7"}]': finished 2023-10-09T04:40:39.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:39 smithi133 bash[41291]: cluster 2023-10-09T04:40:39.329758+0000 mon.a (mon.0) 578 : cluster [DBG] osdmap e51: 8 total, 7 up, 8 in 2023-10-09T04:40:39.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:39 smithi133 bash[41291]: audit 2023-10-09T04:40:39.329975+0000 mon.a (mon.0) 579 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:40:39.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:39 smithi194 bash[31648]: cluster 2023-10-09T04:40:37.741572+0000 mgr.y (mgr.14152) 309 : cluster [DBG] pgmap v290: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:39.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:39 smithi194 bash[31648]: audit 2023-10-09T04:40:39.314652+0000 mon.a (mon.0) 576 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "02506268-102c-4654-9b5c-68d36c28f7f7"}]: dispatch 2023-10-09T04:40:39.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:39 smithi194 bash[31648]: audit 2023-10-09T04:40:39.315194+0000 mon.b (mon.1) 21 : audit [INF] from='client.? 172.21.15.194:0/4076959799' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "02506268-102c-4654-9b5c-68d36c28f7f7"}]: dispatch 2023-10-09T04:40:39.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:39 smithi194 bash[31648]: audit 2023-10-09T04:40:39.329627+0000 mon.a (mon.0) 577 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "02506268-102c-4654-9b5c-68d36c28f7f7"}]': finished 2023-10-09T04:40:39.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:39 smithi194 bash[31648]: cluster 2023-10-09T04:40:39.329758+0000 mon.a (mon.0) 578 : cluster [DBG] osdmap e51: 8 total, 7 up, 8 in 2023-10-09T04:40:39.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:39 smithi194 bash[31648]: audit 2023-10-09T04:40:39.329975+0000 mon.a (mon.0) 579 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:40:40.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:40 smithi133 bash[27558]: audit 2023-10-09T04:40:40.080152+0000 mon.b (mon.1) 22 : audit [DBG] from='client.? 172.21.15.194:0/2071768403' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:40:40.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:40 smithi133 bash[41291]: audit 2023-10-09T04:40:40.080152+0000 mon.b (mon.1) 22 : audit [DBG] from='client.? 172.21.15.194:0/2071768403' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:40:40.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:40 smithi194 bash[31648]: audit 2023-10-09T04:40:40.080152+0000 mon.b (mon.1) 22 : audit [DBG] from='client.? 172.21.15.194:0/2071768403' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:40:41.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:41 smithi133 bash[27558]: cluster 2023-10-09T04:40:39.742292+0000 mgr.y (mgr.14152) 310 : cluster [DBG] pgmap v292: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:41.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:41 smithi133 bash[41291]: cluster 2023-10-09T04:40:39.742292+0000 mgr.y (mgr.14152) 310 : cluster [DBG] pgmap v292: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:41.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:41 smithi194 bash[31648]: cluster 2023-10-09T04:40:39.742292+0000 mgr.y (mgr.14152) 310 : cluster [DBG] pgmap v292: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:43.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:43 smithi133 bash[27558]: cluster 2023-10-09T04:40:41.742976+0000 mgr.y (mgr.14152) 311 : cluster [DBG] pgmap v293: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:43.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:43 smithi133 bash[41291]: cluster 2023-10-09T04:40:41.742976+0000 mgr.y (mgr.14152) 311 : cluster [DBG] pgmap v293: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:43.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:43 smithi194 bash[31648]: cluster 2023-10-09T04:40:41.742976+0000 mgr.y (mgr.14152) 311 : cluster [DBG] pgmap v293: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:45.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:45 smithi133 bash[27558]: cluster 2023-10-09T04:40:43.743577+0000 mgr.y (mgr.14152) 312 : cluster [DBG] pgmap v294: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:45.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:45 smithi133 bash[41291]: cluster 2023-10-09T04:40:43.743577+0000 mgr.y (mgr.14152) 312 : cluster [DBG] pgmap v294: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:45.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:45 smithi194 bash[31648]: cluster 2023-10-09T04:40:43.743577+0000 mgr.y (mgr.14152) 312 : cluster [DBG] pgmap v294: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:47.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:47 smithi133 bash[27558]: cluster 2023-10-09T04:40:45.744168+0000 mgr.y (mgr.14152) 313 : cluster [DBG] pgmap v295: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:47.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:47 smithi133 bash[41291]: cluster 2023-10-09T04:40:45.744168+0000 mgr.y (mgr.14152) 313 : cluster [DBG] pgmap v295: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:47.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:47 smithi194 bash[31648]: cluster 2023-10-09T04:40:45.744168+0000 mgr.y (mgr.14152) 313 : cluster [DBG] pgmap v295: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:49.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:49 smithi194 bash[31648]: cluster 2023-10-09T04:40:47.744711+0000 mgr.y (mgr.14152) 314 : cluster [DBG] pgmap v296: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:49.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:49 smithi133 bash[27558]: cluster 2023-10-09T04:40:47.744711+0000 mgr.y (mgr.14152) 314 : cluster [DBG] pgmap v296: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:49.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:49 smithi133 bash[41291]: cluster 2023-10-09T04:40:47.744711+0000 mgr.y (mgr.14152) 314 : cluster [DBG] pgmap v296: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:51.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:51 smithi194 bash[31648]: cluster 2023-10-09T04:40:49.745297+0000 mgr.y (mgr.14152) 315 : cluster [DBG] pgmap v297: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:51.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:51 smithi133 bash[27558]: cluster 2023-10-09T04:40:49.745297+0000 mgr.y (mgr.14152) 315 : cluster [DBG] pgmap v297: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:51.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:51 smithi133 bash[41291]: cluster 2023-10-09T04:40:49.745297+0000 mgr.y (mgr.14152) 315 : cluster [DBG] pgmap v297: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:52.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:52 smithi194 bash[31648]: audit 2023-10-09T04:40:51.974855+0000 mon.a (mon.0) 580 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T04:40:52.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:52 smithi194 bash[31648]: audit 2023-10-09T04:40:51.976471+0000 mon.a (mon.0) 581 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:40:52.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:52 smithi133 bash[27558]: audit 2023-10-09T04:40:51.974855+0000 mon.a (mon.0) 580 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T04:40:52.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:52 smithi133 bash[27558]: audit 2023-10-09T04:40:51.976471+0000 mon.a (mon.0) 581 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:40:52.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:52 smithi133 bash[41291]: audit 2023-10-09T04:40:51.974855+0000 mon.a (mon.0) 580 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T04:40:52.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:52 smithi133 bash[41291]: audit 2023-10-09T04:40:51.976471+0000 mon.a (mon.0) 581 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:40:53.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:53 smithi194 bash[31648]: cluster 2023-10-09T04:40:51.745863+0000 mgr.y (mgr.14152) 316 : cluster [DBG] pgmap v298: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:53.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:53 smithi194 bash[31648]: cephadm 2023-10-09T04:40:51.977589+0000 mgr.y (mgr.14152) 317 : cephadm [INF] Deploying daemon osd.7 on smithi194 2023-10-09T04:40:53.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:53 smithi133 bash[27558]: cluster 2023-10-09T04:40:51.745863+0000 mgr.y (mgr.14152) 316 : cluster [DBG] pgmap v298: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:53.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:53 smithi133 bash[27558]: cephadm 2023-10-09T04:40:51.977589+0000 mgr.y (mgr.14152) 317 : cephadm [INF] Deploying daemon osd.7 on smithi194 2023-10-09T04:40:53.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:53 smithi133 bash[41291]: cluster 2023-10-09T04:40:51.745863+0000 mgr.y (mgr.14152) 316 : cluster [DBG] pgmap v298: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:53.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:53 smithi133 bash[41291]: cephadm 2023-10-09T04:40:51.977589+0000 mgr.y (mgr.14152) 317 : cephadm [INF] Deploying daemon osd.7 on smithi194 2023-10-09T04:40:55.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:55 smithi133 bash[27558]: cluster 2023-10-09T04:40:53.746418+0000 mgr.y (mgr.14152) 318 : cluster [DBG] pgmap v299: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:55.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:55 smithi133 bash[41291]: cluster 2023-10-09T04:40:53.746418+0000 mgr.y (mgr.14152) 318 : cluster [DBG] pgmap v299: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:55.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:55 smithi194 bash[31648]: cluster 2023-10-09T04:40:53.746418+0000 mgr.y (mgr.14152) 318 : cluster [DBG] pgmap v299: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:57.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:57 smithi194 bash[31648]: cluster 2023-10-09T04:40:55.747009+0000 mgr.y (mgr.14152) 319 : cluster [DBG] pgmap v300: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:57.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:57 smithi194 bash[31648]: audit 2023-10-09T04:40:56.467043+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:57.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:57 smithi194 bash[31648]: audit 2023-10-09T04:40:56.472828+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:57.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:57 smithi194 bash[31648]: audit 2023-10-09T04:40:56.498300+0000 mon.a (mon.0) 584 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:40:57.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:57 smithi133 bash[27558]: cluster 2023-10-09T04:40:55.747009+0000 mgr.y (mgr.14152) 319 : cluster [DBG] pgmap v300: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:57.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:57 smithi133 bash[27558]: audit 2023-10-09T04:40:56.467043+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:57.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:57 smithi133 bash[27558]: audit 2023-10-09T04:40:56.472828+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:57.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:57 smithi133 bash[27558]: audit 2023-10-09T04:40:56.498300+0000 mon.a (mon.0) 584 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:40:57.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:57 smithi133 bash[41291]: cluster 2023-10-09T04:40:55.747009+0000 mgr.y (mgr.14152) 319 : cluster [DBG] pgmap v300: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:57.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:57 smithi133 bash[41291]: audit 2023-10-09T04:40:56.467043+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:57.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:57 smithi133 bash[41291]: audit 2023-10-09T04:40:56.472828+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:40:57.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:57 smithi133 bash[41291]: audit 2023-10-09T04:40:56.498300+0000 mon.a (mon.0) 584 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:40:59.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:40:59 smithi133 bash[27558]: cluster 2023-10-09T04:40:57.747551+0000 mgr.y (mgr.14152) 320 : cluster [DBG] pgmap v301: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:59.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:40:59 smithi133 bash[41291]: cluster 2023-10-09T04:40:57.747551+0000 mgr.y (mgr.14152) 320 : cluster [DBG] pgmap v301: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:40:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:40:59 smithi194 bash[31648]: cluster 2023-10-09T04:40:57.747551+0000 mgr.y (mgr.14152) 320 : cluster [DBG] pgmap v301: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:41:01.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:01 smithi194 bash[31648]: cluster 2023-10-09T04:40:59.748149+0000 mgr.y (mgr.14152) 321 : cluster [DBG] pgmap v302: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:41:01.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:01 smithi133 bash[27558]: cluster 2023-10-09T04:40:59.748149+0000 mgr.y (mgr.14152) 321 : cluster [DBG] pgmap v302: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:41:01.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:01 smithi133 bash[41291]: cluster 2023-10-09T04:40:59.748149+0000 mgr.y (mgr.14152) 321 : cluster [DBG] pgmap v302: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:41:03.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:02 smithi133 bash[27558]: cluster 2023-10-09T04:41:01.748782+0000 mgr.y (mgr.14152) 322 : cluster [DBG] pgmap v303: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:41:03.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:02 smithi133 bash[27558]: audit 2023-10-09T04:41:01.752193+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:03.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:02 smithi133 bash[27558]: audit 2023-10-09T04:41:01.761036+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:03.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:02 smithi133 bash[27558]: audit 2023-10-09T04:41:01.762542+0000 mon.a (mon.0) 587 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:41:03.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:02 smithi133 bash[27558]: audit 2023-10-09T04:41:01.763872+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:41:03.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:02 smithi133 bash[27558]: audit 2023-10-09T04:41:01.770680+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:03.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:02 smithi133 bash[41291]: cluster 2023-10-09T04:41:01.748782+0000 mgr.y (mgr.14152) 322 : cluster [DBG] pgmap v303: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:41:03.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:02 smithi133 bash[41291]: audit 2023-10-09T04:41:01.752193+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:03.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:02 smithi133 bash[41291]: audit 2023-10-09T04:41:01.761036+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:03.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:02 smithi133 bash[41291]: audit 2023-10-09T04:41:01.762542+0000 mon.a (mon.0) 587 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:41:03.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:02 smithi133 bash[41291]: audit 2023-10-09T04:41:01.763872+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:41:03.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:02 smithi133 bash[41291]: audit 2023-10-09T04:41:01.770680+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:03.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:02 smithi194 bash[31648]: cluster 2023-10-09T04:41:01.748782+0000 mgr.y (mgr.14152) 322 : cluster [DBG] pgmap v303: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:41:03.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:02 smithi194 bash[31648]: audit 2023-10-09T04:41:01.752193+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:03.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:02 smithi194 bash[31648]: audit 2023-10-09T04:41:01.761036+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:03.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:02 smithi194 bash[31648]: audit 2023-10-09T04:41:01.762542+0000 mon.a (mon.0) 587 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:41:03.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:02 smithi194 bash[31648]: audit 2023-10-09T04:41:01.763872+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:41:03.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:02 smithi194 bash[31648]: audit 2023-10-09T04:41:01.770680+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:04.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:03 smithi133 bash[27558]: audit 2023-10-09T04:41:03.230436+0000 mon.a (mon.0) 590 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T04:41:04.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:03 smithi133 bash[27558]: audit 2023-10-09T04:41:03.230875+0000 mon.b (mon.1) 23 : audit [INF] from='osd.7 [v2:172.21.15.194:6824/1051622545,v1:172.21.15.194:6825/1051622545]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T04:41:04.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:03 smithi133 bash[41291]: audit 2023-10-09T04:41:03.230436+0000 mon.a (mon.0) 590 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T04:41:04.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:03 smithi133 bash[41291]: audit 2023-10-09T04:41:03.230875+0000 mon.b (mon.1) 23 : audit [INF] from='osd.7 [v2:172.21.15.194:6824/1051622545,v1:172.21.15.194:6825/1051622545]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T04:41:04.132 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:03 smithi194 bash[31648]: audit 2023-10-09T04:41:03.230436+0000 mon.a (mon.0) 590 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T04:41:04.132 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:03 smithi194 bash[31648]: audit 2023-10-09T04:41:03.230875+0000 mon.b (mon.1) 23 : audit [INF] from='osd.7 [v2:172.21.15.194:6824/1051622545,v1:172.21.15.194:6825/1051622545]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T04:41:04.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:04 smithi194 bash[31648]: cluster 2023-10-09T04:41:03.749311+0000 mgr.y (mgr.14152) 323 : cluster [DBG] pgmap v304: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:41:04.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:04 smithi194 bash[31648]: audit 2023-10-09T04:41:03.766834+0000 mon.a (mon.0) 591 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T04:41:04.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:04 smithi194 bash[31648]: cluster 2023-10-09T04:41:03.766936+0000 mon.a (mon.0) 592 : cluster [DBG] osdmap e52: 8 total, 7 up, 8 in 2023-10-09T04:41:04.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:04 smithi194 bash[31648]: audit 2023-10-09T04:41:03.767130+0000 mon.a (mon.0) 593 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:41:04.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:04 smithi194 bash[31648]: audit 2023-10-09T04:41:03.768744+0000 mon.a (mon.0) 594 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2023-10-09T04:41:04.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:04 smithi194 bash[31648]: audit 2023-10-09T04:41:03.769601+0000 mon.b (mon.1) 24 : audit [INF] from='osd.7 [v2:172.21.15.194:6824/1051622545,v1:172.21.15.194:6825/1051622545]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2023-10-09T04:41:05.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:04 smithi133 bash[27558]: cluster 2023-10-09T04:41:03.749311+0000 mgr.y (mgr.14152) 323 : cluster [DBG] pgmap v304: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:41:05.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:04 smithi133 bash[27558]: audit 2023-10-09T04:41:03.766834+0000 mon.a (mon.0) 591 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T04:41:05.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:04 smithi133 bash[27558]: cluster 2023-10-09T04:41:03.766936+0000 mon.a (mon.0) 592 : cluster [DBG] osdmap e52: 8 total, 7 up, 8 in 2023-10-09T04:41:05.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:04 smithi133 bash[27558]: audit 2023-10-09T04:41:03.767130+0000 mon.a (mon.0) 593 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:41:05.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:04 smithi133 bash[27558]: audit 2023-10-09T04:41:03.768744+0000 mon.a (mon.0) 594 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2023-10-09T04:41:05.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:04 smithi133 bash[27558]: audit 2023-10-09T04:41:03.769601+0000 mon.b (mon.1) 24 : audit [INF] from='osd.7 [v2:172.21.15.194:6824/1051622545,v1:172.21.15.194:6825/1051622545]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2023-10-09T04:41:05.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:04 smithi133 bash[41291]: cluster 2023-10-09T04:41:03.749311+0000 mgr.y (mgr.14152) 323 : cluster [DBG] pgmap v304: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:41:05.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:04 smithi133 bash[41291]: audit 2023-10-09T04:41:03.766834+0000 mon.a (mon.0) 591 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T04:41:05.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:04 smithi133 bash[41291]: cluster 2023-10-09T04:41:03.766936+0000 mon.a (mon.0) 592 : cluster [DBG] osdmap e52: 8 total, 7 up, 8 in 2023-10-09T04:41:05.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:04 smithi133 bash[41291]: audit 2023-10-09T04:41:03.767130+0000 mon.a (mon.0) 593 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:41:05.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:04 smithi133 bash[41291]: audit 2023-10-09T04:41:03.768744+0000 mon.a (mon.0) 594 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2023-10-09T04:41:05.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:04 smithi133 bash[41291]: audit 2023-10-09T04:41:03.769601+0000 mon.b (mon.1) 24 : audit [INF] from='osd.7 [v2:172.21.15.194:6824/1051622545,v1:172.21.15.194:6825/1051622545]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2023-10-09T04:41:05.387 INFO:teuthology.orchestra.run.smithi194.stdout:Created osd(s) 7 on host 'smithi194' 2023-10-09T04:41:05.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:05 smithi194 bash[31648]: audit 2023-10-09T04:41:04.771553+0000 mon.a (mon.0) 595 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]': finished 2023-10-09T04:41:05.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:05 smithi194 bash[31648]: cluster 2023-10-09T04:41:04.771761+0000 mon.a (mon.0) 596 : cluster [DBG] osdmap e53: 8 total, 7 up, 8 in 2023-10-09T04:41:05.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:05 smithi194 bash[31648]: audit 2023-10-09T04:41:04.772432+0000 mon.a (mon.0) 597 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:41:05.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:05 smithi194 bash[31648]: audit 2023-10-09T04:41:04.782342+0000 mon.a (mon.0) 598 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:41:05.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:05 smithi194 bash[31648]: audit 2023-10-09T04:41:05.350017+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:05.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:05 smithi194 bash[31648]: audit 2023-10-09T04:41:05.359883+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:05.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:05 smithi194 bash[31648]: audit 2023-10-09T04:41:05.364278+0000 mon.a (mon.0) 601 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:41:05.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:05 smithi194 bash[31648]: cluster 2023-10-09T04:41:05.390036+0000 mon.a (mon.0) 602 : cluster [INF] osd.7 [v2:172.21.15.194:6824/1051622545,v1:172.21.15.194:6825/1051622545] boot 2023-10-09T04:41:05.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:05 smithi194 bash[31648]: cluster 2023-10-09T04:41:05.390139+0000 mon.a (mon.0) 603 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-10-09T04:41:05.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:05 smithi194 bash[31648]: audit 2023-10-09T04:41:05.390748+0000 mon.a (mon.0) 604 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:41:06.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:05 smithi133 bash[27558]: audit 2023-10-09T04:41:04.771553+0000 mon.a (mon.0) 595 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]': finished 2023-10-09T04:41:06.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:05 smithi133 bash[27558]: cluster 2023-10-09T04:41:04.771761+0000 mon.a (mon.0) 596 : cluster [DBG] osdmap e53: 8 total, 7 up, 8 in 2023-10-09T04:41:06.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:05 smithi133 bash[27558]: audit 2023-10-09T04:41:04.772432+0000 mon.a (mon.0) 597 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:41:06.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:05 smithi133 bash[27558]: audit 2023-10-09T04:41:04.782342+0000 mon.a (mon.0) 598 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:41:06.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:05 smithi133 bash[27558]: audit 2023-10-09T04:41:05.350017+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:06.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:05 smithi133 bash[27558]: audit 2023-10-09T04:41:05.359883+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:06.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:05 smithi133 bash[27558]: audit 2023-10-09T04:41:05.364278+0000 mon.a (mon.0) 601 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:41:06.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:05 smithi133 bash[27558]: cluster 2023-10-09T04:41:05.390036+0000 mon.a (mon.0) 602 : cluster [INF] osd.7 [v2:172.21.15.194:6824/1051622545,v1:172.21.15.194:6825/1051622545] boot 2023-10-09T04:41:06.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:05 smithi133 bash[27558]: cluster 2023-10-09T04:41:05.390139+0000 mon.a (mon.0) 603 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-10-09T04:41:06.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:05 smithi133 bash[27558]: audit 2023-10-09T04:41:05.390748+0000 mon.a (mon.0) 604 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:41:06.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:05 smithi133 bash[41291]: audit 2023-10-09T04:41:04.771553+0000 mon.a (mon.0) 595 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]': finished 2023-10-09T04:41:06.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:05 smithi133 bash[41291]: cluster 2023-10-09T04:41:04.771761+0000 mon.a (mon.0) 596 : cluster [DBG] osdmap e53: 8 total, 7 up, 8 in 2023-10-09T04:41:06.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:05 smithi133 bash[41291]: audit 2023-10-09T04:41:04.772432+0000 mon.a (mon.0) 597 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:41:06.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:05 smithi133 bash[41291]: audit 2023-10-09T04:41:04.782342+0000 mon.a (mon.0) 598 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:41:06.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:05 smithi133 bash[41291]: audit 2023-10-09T04:41:05.350017+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:06.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:05 smithi133 bash[41291]: audit 2023-10-09T04:41:05.359883+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:06.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:05 smithi133 bash[41291]: audit 2023-10-09T04:41:05.364278+0000 mon.a (mon.0) 601 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:41:06.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:05 smithi133 bash[41291]: cluster 2023-10-09T04:41:05.390036+0000 mon.a (mon.0) 602 : cluster [INF] osd.7 [v2:172.21.15.194:6824/1051622545,v1:172.21.15.194:6825/1051622545] boot 2023-10-09T04:41:06.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:05 smithi133 bash[41291]: cluster 2023-10-09T04:41:05.390139+0000 mon.a (mon.0) 603 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-10-09T04:41:06.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:05 smithi133 bash[41291]: audit 2023-10-09T04:41:05.390748+0000 mon.a (mon.0) 604 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:41:06.149 DEBUG:teuthology.orchestra.run.smithi194:osd.7> sudo journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@osd.7.service 2023-10-09T04:41:06.151 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-10-09T04:41:06.151 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd stat -f json 2023-10-09T04:41:06.309 INFO:journalctl@ceph.osd.7.smithi194.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:41:07.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:06 smithi133 bash[27558]: cluster 2023-10-09T04:41:04.212364+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:41:07.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:06 smithi133 bash[27558]: cluster 2023-10-09T04:41:04.212540+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:41:07.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:06 smithi133 bash[27558]: cluster 2023-10-09T04:41:05.749832+0000 mgr.y (mgr.14152) 324 : cluster [DBG] pgmap v308: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:41:07.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:06 smithi133 bash[27558]: cluster 2023-10-09T04:41:06.401610+0000 mon.a (mon.0) 605 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-10-09T04:41:07.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:06 smithi133 bash[41291]: cluster 2023-10-09T04:41:04.212364+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:41:07.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:06 smithi133 bash[41291]: cluster 2023-10-09T04:41:04.212540+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:41:07.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:06 smithi133 bash[41291]: cluster 2023-10-09T04:41:05.749832+0000 mgr.y (mgr.14152) 324 : cluster [DBG] pgmap v308: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:41:07.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:06 smithi133 bash[41291]: cluster 2023-10-09T04:41:06.401610+0000 mon.a (mon.0) 605 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-10-09T04:41:07.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:06 smithi194 bash[31648]: cluster 2023-10-09T04:41:04.212364+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:41:07.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:06 smithi194 bash[31648]: cluster 2023-10-09T04:41:04.212540+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:41:07.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:06 smithi194 bash[31648]: cluster 2023-10-09T04:41:05.749832+0000 mgr.y (mgr.14152) 324 : cluster [DBG] pgmap v308: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:41:07.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:06 smithi194 bash[31648]: cluster 2023-10-09T04:41:06.401610+0000 mon.a (mon.0) 605 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-10-09T04:41:08.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:08 smithi133 bash[27558]: cluster 2023-10-09T04:41:07.409840+0000 mon.a (mon.0) 606 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-10-09T04:41:08.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:08 smithi133 bash[41291]: cluster 2023-10-09T04:41:07.409840+0000 mon.a (mon.0) 606 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-10-09T04:41:08.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:08 smithi194 bash[31648]: cluster 2023-10-09T04:41:07.409840+0000 mon.a (mon.0) 606 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-10-09T04:41:09.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:09 smithi133 bash[27558]: cluster 2023-10-09T04:41:07.750348+0000 mgr.y (mgr.14152) 325 : cluster [DBG] pgmap v311: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:09.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:09 smithi133 bash[27558]: cluster 2023-10-09T04:41:08.423146+0000 mon.a (mon.0) 607 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-10-09T04:41:09.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:09 smithi133 bash[41291]: cluster 2023-10-09T04:41:07.750348+0000 mgr.y (mgr.14152) 325 : cluster [DBG] pgmap v311: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:09.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:09 smithi133 bash[41291]: cluster 2023-10-09T04:41:08.423146+0000 mon.a (mon.0) 607 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-10-09T04:41:09.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:09 smithi194 bash[31648]: cluster 2023-10-09T04:41:07.750348+0000 mgr.y (mgr.14152) 325 : cluster [DBG] pgmap v311: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:09.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:09 smithi194 bash[31648]: cluster 2023-10-09T04:41:08.423146+0000 mon.a (mon.0) 607 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-10-09T04:41:10.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:10 smithi133 bash[27558]: cluster 2023-10-09T04:41:09.424749+0000 mon.a (mon.0) 608 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-10-09T04:41:10.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:10 smithi133 bash[41291]: cluster 2023-10-09T04:41:09.424749+0000 mon.a (mon.0) 608 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-10-09T04:41:10.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:10 smithi194 bash[31648]: cluster 2023-10-09T04:41:09.424749+0000 mon.a (mon.0) 608 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-10-09T04:41:10.913 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:41:11.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:11 smithi133 bash[27558]: cluster 2023-10-09T04:41:09.750902+0000 mgr.y (mgr.14152) 326 : cluster [DBG] pgmap v314: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:11.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:11 smithi133 bash[41291]: cluster 2023-10-09T04:41:09.750902+0000 mgr.y (mgr.14152) 326 : cluster [DBG] pgmap v314: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:11.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:11 smithi194 bash[31648]: cluster 2023-10-09T04:41:09.750902+0000 mgr.y (mgr.14152) 326 : cluster [DBG] pgmap v314: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:13.740 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:13 smithi133 bash[27558]: cluster 2023-10-09T04:41:11.751386+0000 mgr.y (mgr.14152) 327 : cluster [DBG] pgmap v315: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 108 KiB/s, 0 objects/s recovering 2023-10-09T04:41:13.741 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:13 smithi133 bash[41291]: cluster 2023-10-09T04:41:11.751386+0000 mgr.y (mgr.14152) 327 : cluster [DBG] pgmap v315: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 108 KiB/s, 0 objects/s recovering 2023-10-09T04:41:13.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:13 smithi194 bash[31648]: cluster 2023-10-09T04:41:11.751386+0000 mgr.y (mgr.14152) 327 : cluster [DBG] pgmap v315: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 108 KiB/s, 0 objects/s recovering 2023-10-09T04:41:14.133 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T04:41:14.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:14 smithi194 bash[31648]: audit 2023-10-09T04:41:14.130797+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.133:0/1566019680' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T04:41:14.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:14 smithi133 bash[27558]: audit 2023-10-09T04:41:14.130797+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.133:0/1566019680' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T04:41:14.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:14 smithi133 bash[41291]: audit 2023-10-09T04:41:14.130797+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.133:0/1566019680' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T04:41:14.994 INFO:teuthology.orchestra.run.smithi133.stdout:{"epoch":58,"num_osds":8,"num_up_osds":8,"osd_up_since":1696826465,"num_in_osds":8,"osd_in_since":1696826439,"num_remapped_pgs":0} 2023-10-09T04:41:14.994 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T04:41:15.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:15 smithi133 bash[27558]: cluster 2023-10-09T04:41:13.751809+0000 mgr.y (mgr.14152) 328 : cluster [DBG] pgmap v316: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 91 KiB/s, 0 objects/s recovering 2023-10-09T04:41:15.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:15 smithi133 bash[27558]: audit 2023-10-09T04:41:15.116704+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:15.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:15 smithi133 bash[27558]: audit 2023-10-09T04:41:15.123083+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:15.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:15 smithi133 bash[27558]: audit 2023-10-09T04:41:15.124407+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:41:15.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:15 smithi133 bash[27558]: audit 2023-10-09T04:41:15.125448+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:41:15.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:15 smithi133 bash[27558]: audit 2023-10-09T04:41:15.126349+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:41:15.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:15 smithi133 bash[27558]: audit 2023-10-09T04:41:15.127199+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:41:15.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:15 smithi133 bash[27558]: audit 2023-10-09T04:41:15.133441+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:15.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:15 smithi133 bash[27558]: audit 2023-10-09T04:41:15.134807+0000 mon.a (mon.0) 616 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:41:15.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:15 smithi133 bash[27558]: audit 2023-10-09T04:41:15.135789+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:41:15.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:15 smithi133 bash[27558]: audit 2023-10-09T04:41:15.141803+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:15.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:15 smithi133 bash[41291]: cluster 2023-10-09T04:41:13.751809+0000 mgr.y (mgr.14152) 328 : cluster [DBG] pgmap v316: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 91 KiB/s, 0 objects/s recovering 2023-10-09T04:41:15.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:15 smithi133 bash[41291]: audit 2023-10-09T04:41:15.116704+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:15.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:15 smithi133 bash[41291]: audit 2023-10-09T04:41:15.123083+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:15.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:15 smithi133 bash[41291]: audit 2023-10-09T04:41:15.124407+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:41:15.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:15 smithi133 bash[41291]: audit 2023-10-09T04:41:15.125448+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:41:15.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:15 smithi133 bash[41291]: audit 2023-10-09T04:41:15.126349+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:41:15.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:15 smithi133 bash[41291]: audit 2023-10-09T04:41:15.127199+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:41:15.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:15 smithi133 bash[41291]: audit 2023-10-09T04:41:15.133441+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:15.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:15 smithi133 bash[41291]: audit 2023-10-09T04:41:15.134807+0000 mon.a (mon.0) 616 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:41:15.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:15 smithi133 bash[41291]: audit 2023-10-09T04:41:15.135789+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:41:15.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:15 smithi133 bash[41291]: audit 2023-10-09T04:41:15.141803+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:15.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:15 smithi194 bash[31648]: cluster 2023-10-09T04:41:13.751809+0000 mgr.y (mgr.14152) 328 : cluster [DBG] pgmap v316: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 91 KiB/s, 0 objects/s recovering 2023-10-09T04:41:15.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:15 smithi194 bash[31648]: audit 2023-10-09T04:41:15.116704+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:15.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:15 smithi194 bash[31648]: audit 2023-10-09T04:41:15.123083+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:15.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:15 smithi194 bash[31648]: audit 2023-10-09T04:41:15.124407+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:41:15.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:15 smithi194 bash[31648]: audit 2023-10-09T04:41:15.125448+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:41:15.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:15 smithi194 bash[31648]: audit 2023-10-09T04:41:15.126349+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:41:15.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:15 smithi194 bash[31648]: audit 2023-10-09T04:41:15.127199+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:41:15.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:15 smithi194 bash[31648]: audit 2023-10-09T04:41:15.133441+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:15.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:15 smithi194 bash[31648]: audit 2023-10-09T04:41:15.134807+0000 mon.a (mon.0) 616 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:41:15.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:15 smithi194 bash[31648]: audit 2023-10-09T04:41:15.135789+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:41:15.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:15 smithi194 bash[31648]: audit 2023-10-09T04:41:15.141803+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:16.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:16 smithi133 bash[27558]: cephadm 2023-10-09T04:41:15.107863+0000 mgr.y (mgr.14152) 329 : cephadm [INF] Detected new or changed devices on smithi194 2023-10-09T04:41:16.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:16 smithi133 bash[27558]: cephadm 2023-10-09T04:41:15.127912+0000 mgr.y (mgr.14152) 330 : cephadm [INF] Adjusting osd_memory_target on smithi194 to 4316M 2023-10-09T04:41:16.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:16 smithi133 bash[41291]: cephadm 2023-10-09T04:41:15.107863+0000 mgr.y (mgr.14152) 329 : cephadm [INF] Detected new or changed devices on smithi194 2023-10-09T04:41:16.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:16 smithi133 bash[41291]: cephadm 2023-10-09T04:41:15.127912+0000 mgr.y (mgr.14152) 330 : cephadm [INF] Adjusting osd_memory_target on smithi194 to 4316M 2023-10-09T04:41:16.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:16 smithi194 bash[31648]: cephadm 2023-10-09T04:41:15.107863+0000 mgr.y (mgr.14152) 329 : cephadm [INF] Detected new or changed devices on smithi194 2023-10-09T04:41:16.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:16 smithi194 bash[31648]: cephadm 2023-10-09T04:41:15.127912+0000 mgr.y (mgr.14152) 330 : cephadm [INF] Adjusting osd_memory_target on smithi194 to 4316M 2023-10-09T04:41:17.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:17 smithi133 bash[27558]: cluster 2023-10-09T04:41:15.752426+0000 mgr.y (mgr.14152) 331 : cluster [DBG] pgmap v317: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T04:41:17.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:17 smithi133 bash[41291]: cluster 2023-10-09T04:41:15.752426+0000 mgr.y (mgr.14152) 331 : cluster [DBG] pgmap v317: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T04:41:17.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:17 smithi194 bash[31648]: cluster 2023-10-09T04:41:15.752426+0000 mgr.y (mgr.14152) 331 : cluster [DBG] pgmap v317: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T04:41:19.754 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:41:19.784 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:19 smithi133 bash[27558]: cluster 2023-10-09T04:41:17.752872+0000 mgr.y (mgr.14152) 332 : cluster [DBG] pgmap v318: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s, 0 objects/s recovering 2023-10-09T04:41:19.784 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:19 smithi133 bash[41291]: cluster 2023-10-09T04:41:17.752872+0000 mgr.y (mgr.14152) 332 : cluster [DBG] pgmap v318: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s, 0 objects/s recovering 2023-10-09T04:41:19.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:19 smithi194 bash[31648]: cluster 2023-10-09T04:41:17.752872+0000 mgr.y (mgr.14152) 332 : cluster [DBG] pgmap v318: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s, 0 objects/s recovering 2023-10-09T04:41:21.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:21 smithi133 bash[27558]: cluster 2023-10-09T04:41:19.753417+0000 mgr.y (mgr.14152) 333 : cluster [DBG] pgmap v319: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 56 KiB/s, 0 objects/s recovering 2023-10-09T04:41:21.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:21 smithi133 bash[41291]: cluster 2023-10-09T04:41:19.753417+0000 mgr.y (mgr.14152) 333 : cluster [DBG] pgmap v319: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 56 KiB/s, 0 objects/s recovering 2023-10-09T04:41:21.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:21 smithi194 bash[31648]: cluster 2023-10-09T04:41:19.753417+0000 mgr.y (mgr.14152) 333 : cluster [DBG] pgmap v319: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 56 KiB/s, 0 objects/s recovering 2023-10-09T04:41:22.881 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T04:41:22.881 INFO:teuthology.orchestra.run.smithi133.stdout:{"epoch":58,"fsid":"855b5206-665c-11ee-8db5-212e2dc638e7","created":"2023-10-09T04:31:08.633675+0000","modified":"2023-10-09T04:41:09.415091+0000","last_up_change":"2023-10-09T04:41:05.383200+0000","last_in_change":"2023-10-09T04:40:39.315748+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T04:36:44.260202+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"32633ec2-9b8f-42d8-bdfb-5c9668a26071","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6802","nonce":1933306305},{"type":"v1","addr":"172.21.15.133:6803","nonce":1933306305}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6804","nonce":1933306305},{"type":"v1","addr":"172.21.15.133:6805","nonce":1933306305}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6808","nonce":1933306305},{"type":"v1","addr":"172.21.15.133:6809","nonce":1933306305}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6806","nonce":1933306305},{"type":"v1","addr":"172.21.15.133:6807","nonce":1933306305}]},"public_addr":"172.21.15.133:6803/1933306305","cluster_addr":"172.21.15.133:6805/1933306305","heartbeat_back_addr":"172.21.15.133:6809/1933306305","heartbeat_front_addr":"172.21.15.133:6807/1933306305","state":["exists","up"]},{"osd":1,"uuid":"7f2a961d-a790-4337-91c7-1e45f900bbd4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6810","nonce":2965860635},{"type":"v1","addr":"172.21.15.133:6811","nonce":2965860635}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6812","nonce":2965860635},{"type":"v1","addr":"172.21.15.133:6813","nonce":2965860635}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6816","nonce":2965860635},{"type":"v1","addr":"172.21.15.133:6817","nonce":2965860635}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6814","nonce":2965860635},{"type":"v1","addr":"172.21.15.133:6815","nonce":2965860635}]},"public_addr":"172.21.15.133:6811/2965860635","cluster_addr":"172.21.15.133:6813/2965860635","heartbeat_back_addr":"172.21.15.133:6817/2965860635","heartbeat_front_addr":"172.21.15.133:6815/2965860635","state":["exists","up"]},{"osd":2,"uuid":"08751330-a327-4bf8-9c4d-9ff986b1c67f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6818","nonce":1465201170},{"type":"v1","addr":"172.21.15.133:6819","nonce":1465201170}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6820","nonce":1465201170},{"type":"v1","addr":"172.21.15.133:6821","nonce":1465201170}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6824","nonce":1465201170},{"type":"v1","addr":"172.21.15.133:6825","nonce":1465201170}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6822","nonce":1465201170},{"type":"v1","addr":"172.21.15.133:6823","nonce":1465201170}]},"public_addr":"172.21.15.133:6819/1465201170","cluster_addr":"172.21.15.133:6821/1465201170","heartbeat_back_addr":"172.21.15.133:6825/1465201170","heartbeat_front_addr":"172.21.15.133:6823/1465201170","state":["exists","up"]},{"osd":3,"uuid":"56bbf807-c8f7-4226-b46e-802b539274ca","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6826","nonce":1093481727},{"type":"v1","addr":"172.21.15.133:6827","nonce":1093481727}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6828","nonce":1093481727},{"type":"v1","addr":"172.21.15.133:6829","nonce":1093481727}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6832","nonce":1093481727},{"type":"v1","addr":"172.21.15.133:6833","nonce":1093481727}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6830","nonce":1093481727},{"type":"v1","addr":"172.21.15.133:6831","nonce":1093481727}]},"public_addr":"172.21.15.133:6827/1093481727","cluster_addr":"172.21.15.133:6829/1093481727","heartbeat_back_addr":"172.21.15.133:6833/1093481727","heartbeat_front_addr":"172.21.15.133:6831/1093481727","state":["exists","up"]},{"osd":4,"uuid":"509ec8cc-5042-428e-9a69-8e602b5afce8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6801","nonce":1874338403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6803","nonce":1874338403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6807","nonce":1874338403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6805","nonce":1874338403}]},"public_addr":"172.21.15.194:6801/1874338403","cluster_addr":"172.21.15.194:6803/1874338403","heartbeat_back_addr":"172.21.15.194:6807/1874338403","heartbeat_front_addr":"172.21.15.194:6805/1874338403","state":["exists","up"]},{"osd":5,"uuid":"bbdc1e67-fd5b-49d0-9e42-cf151d167943","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6809","nonce":3989986662}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6811","nonce":3989986662}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6815","nonce":3989986662}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6813","nonce":3989986662}]},"public_addr":"172.21.15.194:6809/3989986662","cluster_addr":"172.21.15.194:6811/3989986662","heartbeat_back_addr":"172.21.15.194:6815/3989986662","heartbeat_front_addr":"172.21.15.194:6813/3989986662","state":["exists","up"]},{"osd":6,"uuid":"7b69d014-f4c0-4e73-a14e-7d51b7ee3ea2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6817","nonce":2607555245}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6819","nonce":2607555245}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6823","nonce":2607555245}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6820","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6821","nonce":2607555245}]},"public_addr":"172.21.15.194:6817/2607555245","cluster_addr":"172.21.15.194:6819/2607555245","heartbeat_back_addr":"172.21.15.194:6823/2607555245","heartbeat_front_addr":"172.21.15.194:6821/2607555245","state":["exists","up"]},{"osd":7,"uuid":"02506268-102c-4654-9b5c-68d36c28f7f7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6824","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6825","nonce":1051622545}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6827","nonce":1051622545}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6831","nonce":1051622545}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6828","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6829","nonce":1051622545}]},"public_addr":"172.21.15.194:6825/1051622545","cluster_addr":"172.21.15.194:6827/1051622545","heartbeat_back_addr":"172.21.15.194:6831/1051622545","heartbeat_front_addr":"172.21.15.194:6829/1051622545","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:34:53.028753+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:35:47.001796+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:36:41.220545+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:37:36.237434+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:38:28.301119+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:39:20.099747+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:40:12.106681+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:41:04.212546+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.133:0/4290202542":"2023-10-10T04:32:17.611609+0000","172.21.15.133:6800/788799879":"2023-10-10T04:32:17.611609+0000","172.21.15.133:6801/788799879":"2023-10-10T04:32:17.611609+0000","172.21.15.133:0/1411744140":"2023-10-10T04:31:42.639571+0000","172.21.15.133:0/4173745360":"2023-10-10T04:31:42.639571+0000","172.21.15.133:0/1793292086":"2023-10-10T04:32:17.611609+0000","172.21.15.133:0/740467257":"2023-10-10T04:31:42.639571+0000","172.21.15.133:6800/1761810618":"2023-10-10T04:31:42.639571+0000","172.21.15.133:0/3619387632":"2023-10-10T04:32:17.611609+0000","172.21.15.133:6801/1761810618":"2023-10-10T04:31:42.639571+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T04:41:23.632 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-10-09T04:36:44.260202+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2023-10-09T04:41:23.632 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool get .mgr pg_num 2023-10-09T04:41:23.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:23 smithi133 bash[27558]: cluster 2023-10-09T04:41:21.753977+0000 mgr.y (mgr.14152) 334 : cluster [DBG] pgmap v320: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-10-09T04:41:23.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:23 smithi133 bash[27558]: audit 2023-10-09T04:41:22.876942+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.133:0/1671291067' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:41:23.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:23 smithi133 bash[41291]: cluster 2023-10-09T04:41:21.753977+0000 mgr.y (mgr.14152) 334 : cluster [DBG] pgmap v320: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-10-09T04:41:23.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:23 smithi133 bash[41291]: audit 2023-10-09T04:41:22.876942+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.133:0/1671291067' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:41:23.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:23 smithi194 bash[31648]: cluster 2023-10-09T04:41:21.753977+0000 mgr.y (mgr.14152) 334 : cluster [DBG] pgmap v320: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-10-09T04:41:23.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:23 smithi194 bash[31648]: audit 2023-10-09T04:41:22.876942+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.133:0/1671291067' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:41:25.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:25 smithi133 bash[27558]: cluster 2023-10-09T04:41:23.754531+0000 mgr.y (mgr.14152) 335 : cluster [DBG] pgmap v321: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:25.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:25 smithi133 bash[41291]: cluster 2023-10-09T04:41:23.754531+0000 mgr.y (mgr.14152) 335 : cluster [DBG] pgmap v321: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:25.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:25 smithi194 bash[31648]: cluster 2023-10-09T04:41:23.754531+0000 mgr.y (mgr.14152) 335 : cluster [DBG] pgmap v321: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:27.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:27 smithi133 bash[27558]: cluster 2023-10-09T04:41:25.755165+0000 mgr.y (mgr.14152) 336 : cluster [DBG] pgmap v322: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:27.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:27 smithi133 bash[41291]: cluster 2023-10-09T04:41:25.755165+0000 mgr.y (mgr.14152) 336 : cluster [DBG] pgmap v322: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:27.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:27 smithi194 bash[31648]: cluster 2023-10-09T04:41:25.755165+0000 mgr.y (mgr.14152) 336 : cluster [DBG] pgmap v322: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:28.391 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:41:29.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:29 smithi133 bash[27558]: cluster 2023-10-09T04:41:27.755740+0000 mgr.y (mgr.14152) 337 : cluster [DBG] pgmap v323: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:29.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:29 smithi133 bash[41291]: cluster 2023-10-09T04:41:27.755740+0000 mgr.y (mgr.14152) 337 : cluster [DBG] pgmap v323: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:29.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:29 smithi194 bash[31648]: cluster 2023-10-09T04:41:27.755740+0000 mgr.y (mgr.14152) 337 : cluster [DBG] pgmap v323: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:31.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:31 smithi133 bash[41291]: cluster 2023-10-09T04:41:29.756323+0000 mgr.y (mgr.14152) 338 : cluster [DBG] pgmap v324: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:31.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:31 smithi133 bash[27558]: cluster 2023-10-09T04:41:29.756323+0000 mgr.y (mgr.14152) 338 : cluster [DBG] pgmap v324: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:31.836 INFO:teuthology.orchestra.run.smithi133.stdout:pg_num: 1 2023-10-09T04:41:31.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:31 smithi194 bash[31648]: cluster 2023-10-09T04:41:29.756323+0000 mgr.y (mgr.14152) 338 : cluster [DBG] pgmap v324: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:32.670 INFO:tasks.cephadm:Adding prometheus.a on smithi194 2023-10-09T04:41:32.671 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph orch apply prometheus '1;smithi194=a' 2023-10-09T04:41:32.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:32 smithi133 bash[27558]: audit 2023-10-09T04:41:31.833628+0000 mon.a (mon.0) 619 : audit [DBG] from='client.? 172.21.15.133:0/631458502' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-10-09T04:41:32.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:32 smithi133 bash[41291]: audit 2023-10-09T04:41:31.833628+0000 mon.a (mon.0) 619 : audit [DBG] from='client.? 172.21.15.133:0/631458502' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-10-09T04:41:32.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:32 smithi194 bash[31648]: audit 2023-10-09T04:41:31.833628+0000 mon.a (mon.0) 619 : audit [DBG] from='client.? 172.21.15.133:0/631458502' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-10-09T04:41:33.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:33 smithi133 bash[27558]: cluster 2023-10-09T04:41:31.756869+0000 mgr.y (mgr.14152) 339 : cluster [DBG] pgmap v325: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:33.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:33 smithi133 bash[41291]: cluster 2023-10-09T04:41:31.756869+0000 mgr.y (mgr.14152) 339 : cluster [DBG] pgmap v325: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:33.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:33 smithi194 bash[31648]: cluster 2023-10-09T04:41:31.756869+0000 mgr.y (mgr.14152) 339 : cluster [DBG] pgmap v325: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:35.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:35 smithi133 bash[27558]: cluster 2023-10-09T04:41:33.757461+0000 mgr.y (mgr.14152) 340 : cluster [DBG] pgmap v326: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:35.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:35 smithi133 bash[41291]: cluster 2023-10-09T04:41:33.757461+0000 mgr.y (mgr.14152) 340 : cluster [DBG] pgmap v326: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:35.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:35 smithi194 bash[31648]: cluster 2023-10-09T04:41:33.757461+0000 mgr.y (mgr.14152) 340 : cluster [DBG] pgmap v326: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:37.421 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:41:37.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:37 smithi133 bash[27558]: cluster 2023-10-09T04:41:35.758058+0000 mgr.y (mgr.14152) 341 : cluster [DBG] pgmap v327: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:37.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:37 smithi133 bash[41291]: cluster 2023-10-09T04:41:35.758058+0000 mgr.y (mgr.14152) 341 : cluster [DBG] pgmap v327: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:37.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:37 smithi194 bash[31648]: cluster 2023-10-09T04:41:35.758058+0000 mgr.y (mgr.14152) 341 : cluster [DBG] pgmap v327: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:39.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:39 smithi133 bash[27558]: cluster 2023-10-09T04:41:37.758602+0000 mgr.y (mgr.14152) 342 : cluster [DBG] pgmap v328: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:39.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:39 smithi133 bash[41291]: cluster 2023-10-09T04:41:37.758602+0000 mgr.y (mgr.14152) 342 : cluster [DBG] pgmap v328: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:39.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:39 smithi194 bash[31648]: cluster 2023-10-09T04:41:37.758602+0000 mgr.y (mgr.14152) 342 : cluster [DBG] pgmap v328: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:40.624 INFO:teuthology.orchestra.run.smithi194.stdout:Scheduled prometheus update... 2023-10-09T04:41:41.427 DEBUG:teuthology.orchestra.run.smithi194:prometheus.a> sudo journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@prometheus.a.service 2023-10-09T04:41:41.430 INFO:tasks.cephadm:Adding node-exporter.a on smithi133 2023-10-09T04:41:41.430 INFO:tasks.cephadm:Adding node-exporter.b on smithi194 2023-10-09T04:41:41.431 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph orch apply node-exporter '2;smithi133=a;smithi194=b' 2023-10-09T04:41:41.512 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:41:41.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:41 smithi194 bash[31648]: cluster 2023-10-09T04:41:39.759194+0000 mgr.y (mgr.14152) 343 : cluster [DBG] pgmap v329: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:41.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:41 smithi194 bash[31648]: audit 2023-10-09T04:41:40.609173+0000 mgr.y (mgr.14152) 344 : audit [DBG] from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi194=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:41:41.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:41 smithi194 bash[31648]: cephadm 2023-10-09T04:41:40.612690+0000 mgr.y (mgr.14152) 345 : cephadm [INF] Saving service prometheus spec with placement smithi194=a;count:1 2023-10-09T04:41:41.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:41 smithi194 bash[31648]: audit 2023-10-09T04:41:40.619297+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:41.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:41 smithi194 bash[31648]: audit 2023-10-09T04:41:40.643167+0000 mon.a (mon.0) 621 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:41:41.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:41 smithi194 bash[31648]: audit 2023-10-09T04:41:41.017908+0000 mon.a (mon.0) 622 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:41:41.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:41 smithi194 bash[31648]: audit 2023-10-09T04:41:41.019040+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:41:41.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:41 smithi194 bash[31648]: audit 2023-10-09T04:41:41.026283+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:42.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:41 smithi133 bash[27558]: cluster 2023-10-09T04:41:39.759194+0000 mgr.y (mgr.14152) 343 : cluster [DBG] pgmap v329: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:42.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:41 smithi133 bash[27558]: audit 2023-10-09T04:41:40.609173+0000 mgr.y (mgr.14152) 344 : audit [DBG] from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi194=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:41:42.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:41 smithi133 bash[27558]: cephadm 2023-10-09T04:41:40.612690+0000 mgr.y (mgr.14152) 345 : cephadm [INF] Saving service prometheus spec with placement smithi194=a;count:1 2023-10-09T04:41:42.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:41 smithi133 bash[27558]: audit 2023-10-09T04:41:40.619297+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:42.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:41 smithi133 bash[27558]: audit 2023-10-09T04:41:40.643167+0000 mon.a (mon.0) 621 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:41:42.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:41 smithi133 bash[27558]: audit 2023-10-09T04:41:41.017908+0000 mon.a (mon.0) 622 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:41:42.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:41 smithi133 bash[27558]: audit 2023-10-09T04:41:41.019040+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:41:42.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:41 smithi133 bash[27558]: audit 2023-10-09T04:41:41.026283+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:42.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:41 smithi133 bash[41291]: cluster 2023-10-09T04:41:39.759194+0000 mgr.y (mgr.14152) 343 : cluster [DBG] pgmap v329: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:42.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:41 smithi133 bash[41291]: audit 2023-10-09T04:41:40.609173+0000 mgr.y (mgr.14152) 344 : audit [DBG] from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi194=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:41:42.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:41 smithi133 bash[41291]: cephadm 2023-10-09T04:41:40.612690+0000 mgr.y (mgr.14152) 345 : cephadm [INF] Saving service prometheus spec with placement smithi194=a;count:1 2023-10-09T04:41:42.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:41 smithi133 bash[41291]: audit 2023-10-09T04:41:40.619297+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:42.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:41 smithi133 bash[41291]: audit 2023-10-09T04:41:40.643167+0000 mon.a (mon.0) 621 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:41:42.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:41 smithi133 bash[41291]: audit 2023-10-09T04:41:41.017908+0000 mon.a (mon.0) 622 : audit [DBG] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:41:42.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:41 smithi133 bash[41291]: audit 2023-10-09T04:41:41.019040+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:41:42.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:41 smithi133 bash[41291]: audit 2023-10-09T04:41:41.026283+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:42.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:42 smithi194 bash[31648]: cephadm 2023-10-09T04:41:41.046613+0000 mgr.y (mgr.14152) 346 : cephadm [INF] Deploying daemon prometheus.a on smithi194 2023-10-09T04:41:43.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:42 smithi133 bash[27558]: cephadm 2023-10-09T04:41:41.046613+0000 mgr.y (mgr.14152) 346 : cephadm [INF] Deploying daemon prometheus.a on smithi194 2023-10-09T04:41:43.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:42 smithi133 bash[41291]: cephadm 2023-10-09T04:41:41.046613+0000 mgr.y (mgr.14152) 346 : cephadm [INF] Deploying daemon prometheus.a on smithi194 2023-10-09T04:41:43.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:43 smithi194 bash[31648]: cluster 2023-10-09T04:41:41.759872+0000 mgr.y (mgr.14152) 347 : cluster [DBG] pgmap v330: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:44.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:43 smithi133 bash[27558]: cluster 2023-10-09T04:41:41.759872+0000 mgr.y (mgr.14152) 347 : cluster [DBG] pgmap v330: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:44.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:43 smithi133 bash[41291]: cluster 2023-10-09T04:41:41.759872+0000 mgr.y (mgr.14152) 347 : cluster [DBG] pgmap v330: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:45.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:45 smithi194 bash[31648]: cluster 2023-10-09T04:41:43.760470+0000 mgr.y (mgr.14152) 348 : cluster [DBG] pgmap v331: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:46.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:45 smithi133 bash[27558]: cluster 2023-10-09T04:41:43.760470+0000 mgr.y (mgr.14152) 348 : cluster [DBG] pgmap v331: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:46.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:45 smithi133 bash[41291]: cluster 2023-10-09T04:41:43.760470+0000 mgr.y (mgr.14152) 348 : cluster [DBG] pgmap v331: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:46.219 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:41:48.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:47 smithi133 bash[27558]: cluster 2023-10-09T04:41:45.761058+0000 mgr.y (mgr.14152) 349 : cluster [DBG] pgmap v332: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:48.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:47 smithi133 bash[41291]: cluster 2023-10-09T04:41:45.761058+0000 mgr.y (mgr.14152) 349 : cluster [DBG] pgmap v332: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:48.181 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:47 smithi194 bash[31648]: cluster 2023-10-09T04:41:45.761058+0000 mgr.y (mgr.14152) 349 : cluster [DBG] pgmap v332: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:50.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:49 smithi133 bash[27558]: cluster 2023-10-09T04:41:47.761659+0000 mgr.y (mgr.14152) 350 : cluster [DBG] pgmap v333: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:50.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:49 smithi133 bash[41291]: cluster 2023-10-09T04:41:47.761659+0000 mgr.y (mgr.14152) 350 : cluster [DBG] pgmap v333: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:50.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:49 smithi194 bash[31648]: cluster 2023-10-09T04:41:47.761659+0000 mgr.y (mgr.14152) 350 : cluster [DBG] pgmap v333: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:52.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:51 smithi133 bash[27558]: cluster 2023-10-09T04:41:49.762173+0000 mgr.y (mgr.14152) 351 : cluster [DBG] pgmap v334: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:52.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:51 smithi133 bash[41291]: cluster 2023-10-09T04:41:49.762173+0000 mgr.y (mgr.14152) 351 : cluster [DBG] pgmap v334: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:52.108 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:51 smithi194 bash[31648]: cluster 2023-10-09T04:41:49.762173+0000 mgr.y (mgr.14152) 351 : cluster [DBG] pgmap v334: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:53.301 INFO:teuthology.orchestra.run.smithi194.stdout:Scheduled node-exporter update... 2023-10-09T04:41:54.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:53 smithi133 bash[27558]: cluster 2023-10-09T04:41:51.762781+0000 mgr.y (mgr.14152) 352 : cluster [DBG] pgmap v335: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:54.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:53 smithi133 bash[27558]: audit 2023-10-09T04:41:53.298346+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:54.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:53 smithi133 bash[41291]: cluster 2023-10-09T04:41:51.762781+0000 mgr.y (mgr.14152) 352 : cluster [DBG] pgmap v335: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:54.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:53 smithi133 bash[41291]: audit 2023-10-09T04:41:53.298346+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:54.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:53 smithi194 bash[31648]: cluster 2023-10-09T04:41:51.762781+0000 mgr.y (mgr.14152) 352 : cluster [DBG] pgmap v335: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:54.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:53 smithi194 bash[31648]: audit 2023-10-09T04:41:53.298346+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:54.461 DEBUG:teuthology.orchestra.run.smithi133:node-exporter.a> sudo journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@node-exporter.a.service 2023-10-09T04:41:54.464 DEBUG:teuthology.orchestra.run.smithi194:node-exporter.b> sudo journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@node-exporter.b.service 2023-10-09T04:41:54.507 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:41:54.509 INFO:tasks.cephadm:Adding alertmanager.a on smithi133 2023-10-09T04:41:54.509 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph orch apply alertmanager '1;smithi133=a' 2023-10-09T04:41:54.604 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:41:54.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:54 smithi133 bash[27558]: audit 2023-10-09T04:41:53.288370+0000 mgr.y (mgr.14152) 353 : audit [DBG] from='client.24352 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi133=a;smithi194=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:41:54.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:54 smithi133 bash[27558]: cephadm 2023-10-09T04:41:53.291914+0000 mgr.y (mgr.14152) 354 : cephadm [INF] Saving service node-exporter spec with placement smithi133=a;smithi194=b;count:2 2023-10-09T04:41:54.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:54 smithi133 bash[41291]: audit 2023-10-09T04:41:53.288370+0000 mgr.y (mgr.14152) 353 : audit [DBG] from='client.24352 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi133=a;smithi194=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:41:54.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:54 smithi133 bash[41291]: cephadm 2023-10-09T04:41:53.291914+0000 mgr.y (mgr.14152) 354 : cephadm [INF] Saving service node-exporter spec with placement smithi133=a;smithi194=b;count:2 2023-10-09T04:41:54.841 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:54 smithi194 bash[31648]: audit 2023-10-09T04:41:53.288370+0000 mgr.y (mgr.14152) 353 : audit [DBG] from='client.24352 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi133=a;smithi194=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:41:54.841 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:54 smithi194 bash[31648]: cephadm 2023-10-09T04:41:53.291914+0000 mgr.y (mgr.14152) 354 : cephadm [INF] Saving service node-exporter spec with placement smithi133=a;smithi194=b;count:2 2023-10-09T04:41:56.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:55 smithi133 bash[27558]: cluster 2023-10-09T04:41:53.763378+0000 mgr.y (mgr.14152) 355 : cluster [DBG] pgmap v336: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:56.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:55 smithi133 bash[41291]: cluster 2023-10-09T04:41:53.763378+0000 mgr.y (mgr.14152) 355 : cluster [DBG] pgmap v336: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:56.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:55 smithi194 bash[31648]: cluster 2023-10-09T04:41:53.763378+0000 mgr.y (mgr.14152) 355 : cluster [DBG] pgmap v336: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:56.964 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:41:56 smithi194 bash[65289]: ts=2023-10-09T04:41:56.925Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-10-09T04:41:56.965 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:56 smithi194 bash[31648]: cluster 2023-10-09T04:41:55.763950+0000 mgr.y (mgr.14152) 356 : cluster [DBG] pgmap v337: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:56.965 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:56 smithi194 bash[31648]: audit 2023-10-09T04:41:55.883322+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:56.965 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:56 smithi194 bash[31648]: audit 2023-10-09T04:41:55.889685+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:56.965 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:56 smithi194 bash[31648]: audit 2023-10-09T04:41:55.912562+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:56.965 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:56 smithi194 bash[31648]: audit 2023-10-09T04:41:55.918041+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T04:41:57.136 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:41:56 smithi133 bash[27821]: ignoring --setuser ceph since I am not root 2023-10-09T04:41:57.136 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:41:56 smithi133 bash[27821]: ignoring --setgroup ceph since I am not root 2023-10-09T04:41:57.136 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:41:56 smithi133 bash[27821]: debug 2023-10-09T04:41:56.972+0000 7fe23f2fc700 1 -- 172.21.15.133:0/2258351616 <== mon.2 v2:172.21.15.133:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x564ab46389c0 con 0x564ab53be800 2023-10-09T04:41:57.137 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:56 smithi133 bash[27558]: cluster 2023-10-09T04:41:55.763950+0000 mgr.y (mgr.14152) 356 : cluster [DBG] pgmap v337: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:57.137 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:56 smithi133 bash[27558]: audit 2023-10-09T04:41:55.883322+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:57.137 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:56 smithi133 bash[27558]: audit 2023-10-09T04:41:55.889685+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:57.137 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:56 smithi133 bash[27558]: audit 2023-10-09T04:41:55.912562+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:57.137 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:56 smithi133 bash[27558]: audit 2023-10-09T04:41:55.918041+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T04:41:57.137 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:56 smithi133 bash[41291]: cluster 2023-10-09T04:41:55.763950+0000 mgr.y (mgr.14152) 356 : cluster [DBG] pgmap v337: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:41:57.137 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:56 smithi133 bash[41291]: audit 2023-10-09T04:41:55.883322+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:57.137 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:56 smithi133 bash[41291]: audit 2023-10-09T04:41:55.889685+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:57.138 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:56 smithi133 bash[41291]: audit 2023-10-09T04:41:55.912562+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' 2023-10-09T04:41:57.138 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:56 smithi133 bash[41291]: audit 2023-10-09T04:41:55.918041+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T04:41:57.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:41:56 smithi194 bash[32713]: ignoring --setuser ceph since I am not root 2023-10-09T04:41:57.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:41:56 smithi194 bash[32713]: ignoring --setgroup ceph since I am not root 2023-10-09T04:41:57.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:41:56 smithi194 bash[32713]: debug 2023-10-09T04:41:56.969+0000 7f45e1c56700 1 -- 172.21.15.194:0/3466391917 <== mon.1 v2:172.21.15.194:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x561ba9ea89c0 con 0x561baac2ec00 2023-10-09T04:41:57.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:41:57 smithi194 bash[32713]: debug 2023-10-09T04:41:57.121+0000 7f45e9f461c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:41:58.181 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:41:58 smithi194 bash[32713]: debug 2023-10-09T04:41:58.069+0000 7f45e9f461c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:41:58.182 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:57 smithi194 bash[31648]: audit 2023-10-09T04:41:56.920296+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T04:41:58.182 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:41:57 smithi194 bash[31648]: cluster 2023-10-09T04:41:56.920398+0000 mon.a (mon.0) 631 : cluster [DBG] mgrmap e16: y(active, since 9m), standbys: x 2023-10-09T04:41:58.259 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:41:57 smithi133 bash[27821]: debug 2023-10-09T04:41:57.128+0000 7fe2475ec1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:41:58.259 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:41:58 smithi133 bash[27821]: debug 2023-10-09T04:41:58.052+0000 7fe2475ec1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:41:58.259 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:41:58 smithi133 bash[27821]: debug 2023-10-09T04:41:58.152+0000 7fe2475ec1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:41:58.260 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:57 smithi133 bash[27558]: audit 2023-10-09T04:41:56.920296+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T04:41:58.260 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:41:57 smithi133 bash[27558]: cluster 2023-10-09T04:41:56.920398+0000 mon.a (mon.0) 631 : cluster [DBG] mgrmap e16: y(active, since 9m), standbys: x 2023-10-09T04:41:58.260 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:57 smithi133 bash[41291]: audit 2023-10-09T04:41:56.920296+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14152 172.21.15.133:0/1657608018' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T04:41:58.260 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:41:57 smithi133 bash[41291]: cluster 2023-10-09T04:41:56.920398+0000 mon.a (mon.0) 631 : cluster [DBG] mgrmap e16: y(active, since 9m), standbys: x 2023-10-09T04:41:58.524 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:41:58 smithi133 bash[27821]: debug 2023-10-09T04:41:58.252+0000 7fe2475ec1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:41:58.578 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:41:58 smithi194 bash[32713]: debug 2023-10-09T04:41:58.177+0000 7f45e9f461c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:41:58.578 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:41:58 smithi194 bash[32713]: debug 2023-10-09T04:41:58.285+0000 7f45e9f461c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:41:58.800 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:41:58 smithi133 bash[27821]: debug 2023-10-09T04:41:58.516+0000 7fe2475ec1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:41:58.886 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:41:58 smithi194 bash[32713]: debug 2023-10-09T04:41:58.573+0000 7f45e9f461c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:41:59.241 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:41:58 smithi133 bash[27821]: debug 2023-10-09T04:41:58.864+0000 7fe2475ec1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:41:59.241 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:41:58 smithi133 bash[27821]: debug 2023-10-09T04:41:58.960+0000 7fe2475ec1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:41:59.292 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:41:59.321 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:41:58 smithi194 bash[32713]: debug 2023-10-09T04:41:58.953+0000 7f45e9f461c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:41:59.322 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:41:59 smithi194 bash[32713]: debug 2023-10-09T04:41:59.057+0000 7f45e9f461c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:41:59.550 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:41:59 smithi133 bash[27821]: debug 2023-10-09T04:41:59.236+0000 7fe2475ec1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:41:59.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:41:59 smithi194 bash[32713]: debug 2023-10-09T04:41:59.353+0000 7f45e9f461c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:42:00.550 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:42:00 smithi133 bash[27821]: debug 2023-10-09T04:42:00.223+0000 7fe2475ec1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:42:00.550 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:42:00 smithi133 bash[27821]: debug 2023-10-09T04:42:00.339+0000 7fe2475ec1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:42:00.551 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:42:00 smithi133 bash[27821]: debug 2023-10-09T04:42:00.455+0000 7fe2475ec1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:42:00.592 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:42:00 smithi194 bash[32713]: debug 2023-10-09T04:42:00.357+0000 7f45e9f461c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:42:00.592 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:42:00 smithi194 bash[32713]: debug 2023-10-09T04:42:00.465+0000 7f45e9f461c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:42:00.886 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:42:00 smithi194 bash[32713]: debug 2023-10-09T04:42:00.589+0000 7f45e9f461c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:42:01.300 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:42:00 smithi133 bash[27821]: debug 2023-10-09T04:42:00.931+0000 7fe2475ec1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:42:01.300 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:42:01 smithi133 bash[27821]: debug 2023-10-09T04:42:01.151+0000 7fe2475ec1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:42:01.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:42:01 smithi194 bash[32713]: debug 2023-10-09T04:42:01.057+0000 7f45e9f461c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:42:01.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:42:01 smithi194 bash[32713]: debug 2023-10-09T04:42:01.273+0000 7f45e9f461c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:42:01.800 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:42:01 smithi133 bash[27821]: debug 2023-10-09T04:42:01.519+0000 7fe2475ec1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:42:02.130 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:42:01 smithi194 bash[32713]: debug 2023-10-09T04:42:01.637+0000 7f45e9f461c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:42:02.300 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:42:02 smithi133 bash[27821]: debug 2023-10-09T04:42:02.007+0000 7fe2475ec1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:42:02.386 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:42:02 smithi194 bash[32713]: debug 2023-10-09T04:42:02.125+0000 7f45e9f461c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:42:02.838 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:42:02 smithi133 bash[27821]: debug 2023-10-09T04:42:02.559+0000 7fe2475ec1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:42:02.839 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:42:02 smithi133 bash[27821]: debug 2023-10-09T04:42:02.671+0000 7fe2475ec1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:42:02.955 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:42:02 smithi194 bash[32713]: debug 2023-10-09T04:42:02.689+0000 7f45e9f461c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:42:02.955 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:42:02 smithi194 bash[32713]: debug 2023-10-09T04:42:02.789+0000 7f45e9f461c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:42:03.138 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:42:02 smithi133 bash[27821]: debug 2023-10-09T04:42:02.831+0000 7fe2475ec1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:42:03.253 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:42:02 smithi194 bash[32713]: debug 2023-10-09T04:42:02.949+0000 7f45e9f461c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:42:03.396 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:42:03 smithi133 bash[27821]: debug 2023-10-09T04:42:03.131+0000 7fe2475ec1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:42:03.396 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:42:03 smithi133 bash[27821]: debug 2023-10-09T04:42:03.275+0000 7fe2475ec1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:42:03.624 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:42:03 smithi194 bash[32713]: debug 2023-10-09T04:42:03.249+0000 7f45e9f461c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:42:03.624 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:42:03 smithi194 bash[32713]: debug 2023-10-09T04:42:03.389+0000 7f45e9f461c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:42:03.624 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:42:03 smithi194 bash[32713]: debug 2023-10-09T04:42:03.497+0000 7f45e9f461c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:42:03.712 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:42:03 smithi133 bash[27821]: debug 2023-10-09T04:42:03.391+0000 7fe2475ec1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:42:03.712 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:42:03 smithi133 bash[27821]: debug 2023-10-09T04:42:03.515+0000 7fe2475ec1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:42:03.887 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:42:03 smithi194 bash[32713]: debug 2023-10-09T04:42:03.621+0000 7f45e9f461c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:42:03.887 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:42:03 smithi194 bash[32713]: debug 2023-10-09T04:42:03.801+0000 7f45e9f461c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:42:04.050 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:42:03 smithi133 bash[27821]: debug 2023-10-09T04:42:03.703+0000 7fe2475ec1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:42:04.051 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:42:03 smithi133 bash[27821]: debug 2023-10-09T04:42:03.831+0000 7fe2475ec1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:42:04.386 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:42:03 smithi194 bash[32713]: debug 2023-10-09T04:42:03.925+0000 7f45e9f461c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:42:05.300 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[27821]: debug 2023-10-09T04:42:05.039+0000 7fe2475ec1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:42:05.300 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[27821]: [09/Oct/2023:04:42:05] ENGINE Bus STARTING 2023-10-09T04:42:05.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[27558]: cluster 2023-10-09T04:42:05.063431+0000 mon.a (mon.0) 632 : cluster [INF] Active manager daemon y restarted 2023-10-09T04:42:05.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[27558]: cluster 2023-10-09T04:42:05.063939+0000 mon.a (mon.0) 633 : cluster [INF] Activating manager daemon y 2023-10-09T04:42:05.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[27558]: cluster 2023-10-09T04:42:05.071547+0000 mon.a (mon.0) 634 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-10-09T04:42:05.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[27558]: cluster 2023-10-09T04:42:05.071840+0000 mon.a (mon.0) 635 : cluster [DBG] mgrmap e17: y(active, starting, since 0.00808349s), standbys: x 2023-10-09T04:42:05.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[27558]: audit 2023-10-09T04:42:05.077129+0000 mon.a (mon.0) 636 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:42:05.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[27558]: audit 2023-10-09T04:42:05.077446+0000 mon.a (mon.0) 637 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:42:05.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[27558]: audit 2023-10-09T04:42:05.077712+0000 mon.a (mon.0) 638 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:42:05.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[27558]: audit 2023-10-09T04:42:05.079043+0000 mon.a (mon.0) 639 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:42:05.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[27558]: audit 2023-10-09T04:42:05.079237+0000 mon.a (mon.0) 640 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:42:05.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[27558]: audit 2023-10-09T04:42:05.079451+0000 mon.a (mon.0) 641 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:42:05.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[27558]: audit 2023-10-09T04:42:05.079660+0000 mon.a (mon.0) 642 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:42:05.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[27558]: audit 2023-10-09T04:42:05.080033+0000 mon.a (mon.0) 643 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:42:05.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[27558]: audit 2023-10-09T04:42:05.080390+0000 mon.a (mon.0) 644 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:42:05.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[27558]: audit 2023-10-09T04:42:05.080804+0000 mon.a (mon.0) 645 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:42:05.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[27558]: audit 2023-10-09T04:42:05.081187+0000 mon.a (mon.0) 646 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:42:05.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[27558]: audit 2023-10-09T04:42:05.081410+0000 mon.a (mon.0) 647 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:42:05.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[27558]: audit 2023-10-09T04:42:05.081711+0000 mon.a (mon.0) 648 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:42:05.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[27558]: audit 2023-10-09T04:42:05.081988+0000 mon.a (mon.0) 649 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:42:05.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[27558]: audit 2023-10-09T04:42:05.082197+0000 mon.a (mon.0) 650 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:42:05.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[27558]: audit 2023-10-09T04:42:05.082682+0000 mon.a (mon.0) 651 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:42:05.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[27558]: cluster 2023-10-09T04:42:05.113767+0000 mon.a (mon.0) 652 : cluster [INF] Manager daemon y is now available 2023-10-09T04:42:05.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[41291]: cluster 2023-10-09T04:42:05.063431+0000 mon.a (mon.0) 632 : cluster [INF] Active manager daemon y restarted 2023-10-09T04:42:05.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[41291]: cluster 2023-10-09T04:42:05.063939+0000 mon.a (mon.0) 633 : cluster [INF] Activating manager daemon y 2023-10-09T04:42:05.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[41291]: cluster 2023-10-09T04:42:05.071547+0000 mon.a (mon.0) 634 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-10-09T04:42:05.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[41291]: cluster 2023-10-09T04:42:05.071840+0000 mon.a (mon.0) 635 : cluster [DBG] mgrmap e17: y(active, starting, since 0.00808349s), standbys: x 2023-10-09T04:42:05.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[41291]: audit 2023-10-09T04:42:05.077129+0000 mon.a (mon.0) 636 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:42:05.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[41291]: audit 2023-10-09T04:42:05.077446+0000 mon.a (mon.0) 637 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:42:05.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[41291]: audit 2023-10-09T04:42:05.077712+0000 mon.a (mon.0) 638 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:42:05.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[41291]: audit 2023-10-09T04:42:05.079043+0000 mon.a (mon.0) 639 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:42:05.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[41291]: audit 2023-10-09T04:42:05.079237+0000 mon.a (mon.0) 640 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:42:05.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[41291]: audit 2023-10-09T04:42:05.079451+0000 mon.a (mon.0) 641 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:42:05.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[41291]: audit 2023-10-09T04:42:05.079660+0000 mon.a (mon.0) 642 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:42:05.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[41291]: audit 2023-10-09T04:42:05.080033+0000 mon.a (mon.0) 643 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:42:05.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[41291]: audit 2023-10-09T04:42:05.080390+0000 mon.a (mon.0) 644 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:42:05.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[41291]: audit 2023-10-09T04:42:05.080804+0000 mon.a (mon.0) 645 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:42:05.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[41291]: audit 2023-10-09T04:42:05.081187+0000 mon.a (mon.0) 646 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:42:05.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[41291]: audit 2023-10-09T04:42:05.081410+0000 mon.a (mon.0) 647 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:42:05.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[41291]: audit 2023-10-09T04:42:05.081711+0000 mon.a (mon.0) 648 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:42:05.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[41291]: audit 2023-10-09T04:42:05.081988+0000 mon.a (mon.0) 649 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:42:05.305 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[41291]: audit 2023-10-09T04:42:05.082197+0000 mon.a (mon.0) 650 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:42:05.305 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[41291]: audit 2023-10-09T04:42:05.082682+0000 mon.a (mon.0) 651 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:42:05.305 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[41291]: cluster 2023-10-09T04:42:05.113767+0000 mon.a (mon.0) 652 : cluster [INF] Manager daemon y is now available 2023-10-09T04:42:05.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:42:05 smithi194 bash[32713]: debug 2023-10-09T04:42:05.120+0000 7f45e9f461c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:42:05.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:42:05 smithi194 bash[32713]: [09/Oct/2023:04:42:05] ENGINE Bus STARTING 2023-10-09T04:42:05.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:42:05 smithi194 bash[32713]: CherryPy Checker: 2023-10-09T04:42:05.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:42:05 smithi194 bash[32713]: The Application mounted at '' has an empty config. 2023-10-09T04:42:05.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:42:05 smithi194 bash[32713]: [09/Oct/2023:04:42:05] ENGINE Serving on http://:::9283 2023-10-09T04:42:05.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:42:05 smithi194 bash[32713]: [09/Oct/2023:04:42:05] ENGINE Bus STARTED 2023-10-09T04:42:05.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:05 smithi194 bash[31648]: cluster 2023-10-09T04:42:05.063431+0000 mon.a (mon.0) 632 : cluster [INF] Active manager daemon y restarted 2023-10-09T04:42:05.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:05 smithi194 bash[31648]: cluster 2023-10-09T04:42:05.063939+0000 mon.a (mon.0) 633 : cluster [INF] Activating manager daemon y 2023-10-09T04:42:05.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:05 smithi194 bash[31648]: cluster 2023-10-09T04:42:05.071547+0000 mon.a (mon.0) 634 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-10-09T04:42:05.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:05 smithi194 bash[31648]: cluster 2023-10-09T04:42:05.071840+0000 mon.a (mon.0) 635 : cluster [DBG] mgrmap e17: y(active, starting, since 0.00808349s), standbys: x 2023-10-09T04:42:05.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:05 smithi194 bash[31648]: audit 2023-10-09T04:42:05.077129+0000 mon.a (mon.0) 636 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:42:05.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:05 smithi194 bash[31648]: audit 2023-10-09T04:42:05.077446+0000 mon.a (mon.0) 637 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:42:05.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:05 smithi194 bash[31648]: audit 2023-10-09T04:42:05.077712+0000 mon.a (mon.0) 638 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:42:05.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:05 smithi194 bash[31648]: audit 2023-10-09T04:42:05.079043+0000 mon.a (mon.0) 639 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:42:05.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:05 smithi194 bash[31648]: audit 2023-10-09T04:42:05.079237+0000 mon.a (mon.0) 640 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:42:05.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:05 smithi194 bash[31648]: audit 2023-10-09T04:42:05.079451+0000 mon.a (mon.0) 641 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:42:05.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:05 smithi194 bash[31648]: audit 2023-10-09T04:42:05.079660+0000 mon.a (mon.0) 642 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:42:05.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:05 smithi194 bash[31648]: audit 2023-10-09T04:42:05.080033+0000 mon.a (mon.0) 643 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:42:05.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:05 smithi194 bash[31648]: audit 2023-10-09T04:42:05.080390+0000 mon.a (mon.0) 644 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:42:05.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:05 smithi194 bash[31648]: audit 2023-10-09T04:42:05.080804+0000 mon.a (mon.0) 645 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:42:05.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:05 smithi194 bash[31648]: audit 2023-10-09T04:42:05.081187+0000 mon.a (mon.0) 646 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:42:05.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:05 smithi194 bash[31648]: audit 2023-10-09T04:42:05.081410+0000 mon.a (mon.0) 647 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:42:05.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:05 smithi194 bash[31648]: audit 2023-10-09T04:42:05.081711+0000 mon.a (mon.0) 648 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:42:05.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:05 smithi194 bash[31648]: audit 2023-10-09T04:42:05.081988+0000 mon.a (mon.0) 649 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:42:05.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:05 smithi194 bash[31648]: audit 2023-10-09T04:42:05.082197+0000 mon.a (mon.0) 650 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:42:05.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:05 smithi194 bash[31648]: audit 2023-10-09T04:42:05.082682+0000 mon.a (mon.0) 651 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:42:05.390 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:05 smithi194 bash[31648]: cluster 2023-10-09T04:42:05.113767+0000 mon.a (mon.0) 652 : cluster [INF] Manager daemon y is now available 2023-10-09T04:42:05.800 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[27821]: CherryPy Checker: 2023-10-09T04:42:05.800 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[27821]: The Application mounted at '' has an empty config. 2023-10-09T04:42:05.801 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[27821]: [09/Oct/2023:04:42:05] ENGINE Serving on http://:::9283 2023-10-09T04:42:05.801 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:42:05 smithi133 bash[27821]: [09/Oct/2023:04:42:05] ENGINE Bus STARTED 2023-10-09T04:42:06.096 INFO:teuthology.orchestra.run.smithi194.stdout:Scheduled alertmanager update... 2023-10-09T04:42:06.386 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:42:06 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:42:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:42:06.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:06 smithi194 bash[31648]: audit 2023-10-09T04:42:05.143906+0000 mon.a (mon.0) 653 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:42:06.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:06 smithi194 bash[31648]: cluster 2023-10-09T04:42:05.144996+0000 mon.a (mon.0) 654 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T04:42:06.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:06 smithi194 bash[31648]: cluster 2023-10-09T04:42:05.145381+0000 mon.a (mon.0) 655 : cluster [DBG] Standby manager daemon x started 2023-10-09T04:42:06.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:06 smithi194 bash[31648]: audit 2023-10-09T04:42:05.149073+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.194:0/594042489' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:42:06.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:06 smithi194 bash[31648]: audit 2023-10-09T04:42:05.150084+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.194:0/594042489' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:42:06.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:06 smithi194 bash[31648]: audit 2023-10-09T04:42:05.150622+0000 mon.a (mon.0) 656 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:42:06.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:06 smithi194 bash[31648]: audit 2023-10-09T04:42:05.153894+0000 mon.b (mon.1) 27 : audit [DBG] from='mgr.? 172.21.15.194:0/594042489' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:42:06.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:06 smithi194 bash[31648]: audit 2023-10-09T04:42:05.155354+0000 mon.b (mon.1) 28 : audit [DBG] from='mgr.? 172.21.15.194:0/594042489' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:42:06.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:06 smithi194 bash[31648]: audit 2023-10-09T04:42:05.170362+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:06.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:06 smithi194 bash[31648]: audit 2023-10-09T04:42:05.180524+0000 mon.a (mon.0) 658 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:06.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:06 smithi194 bash[31648]: audit 2023-10-09T04:42:05.193170+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:42:06.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:06 smithi194 bash[31648]: audit 2023-10-09T04:42:05.201424+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:42:06.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:06 smithi194 bash[31648]: cephadm 2023-10-09T04:42:05.768471+0000 mgr.y (mgr.14466) 1 : cephadm [INF] [09/Oct/2023:04:42:05] ENGINE Bus STARTING 2023-10-09T04:42:06.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:06 smithi194 bash[31648]: audit 2023-10-09T04:42:05.860542+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:06.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:06 smithi194 bash[31648]: audit 2023-10-09T04:42:05.866562+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:06.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:06 smithi194 bash[31648]: cephadm 2023-10-09T04:42:05.887388+0000 mgr.y (mgr.14466) 2 : cephadm [INF] [09/Oct/2023:04:42:05] ENGINE Serving on https://172.21.15.133:7150 2023-10-09T04:42:06.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:06 smithi194 bash[31648]: cephadm 2023-10-09T04:42:05.887688+0000 mgr.y (mgr.14466) 3 : cephadm [INF] [09/Oct/2023:04:42:05] ENGINE Bus STARTED 2023-10-09T04:42:06.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:06 smithi194 bash[31648]: audit 2023-10-09T04:42:05.896715+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:06.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:06 smithi194 bash[31648]: cluster 2023-10-09T04:42:06.079799+0000 mon.a (mon.0) 664 : cluster [DBG] mgrmap e18: y(active, since 1.01603s), standbys: x 2023-10-09T04:42:06.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:06 smithi194 bash[31648]: audit 2023-10-09T04:42:06.092898+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:06.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[27558]: audit 2023-10-09T04:42:05.143906+0000 mon.a (mon.0) 653 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:42:06.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[27558]: cluster 2023-10-09T04:42:05.144996+0000 mon.a (mon.0) 654 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T04:42:06.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[27558]: cluster 2023-10-09T04:42:05.145381+0000 mon.a (mon.0) 655 : cluster [DBG] Standby manager daemon x started 2023-10-09T04:42:06.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[27558]: audit 2023-10-09T04:42:05.149073+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.194:0/594042489' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:42:06.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[27558]: audit 2023-10-09T04:42:05.150084+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.194:0/594042489' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:42:06.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[27558]: audit 2023-10-09T04:42:05.150622+0000 mon.a (mon.0) 656 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:42:06.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[27558]: audit 2023-10-09T04:42:05.153894+0000 mon.b (mon.1) 27 : audit [DBG] from='mgr.? 172.21.15.194:0/594042489' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:42:06.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[27558]: audit 2023-10-09T04:42:05.155354+0000 mon.b (mon.1) 28 : audit [DBG] from='mgr.? 172.21.15.194:0/594042489' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:42:06.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[27558]: audit 2023-10-09T04:42:05.170362+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:06.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[27558]: audit 2023-10-09T04:42:05.180524+0000 mon.a (mon.0) 658 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:06.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[27558]: audit 2023-10-09T04:42:05.193170+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:42:06.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[27558]: audit 2023-10-09T04:42:05.201424+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:42:06.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[27558]: cephadm 2023-10-09T04:42:05.768471+0000 mgr.y (mgr.14466) 1 : cephadm [INF] [09/Oct/2023:04:42:05] ENGINE Bus STARTING 2023-10-09T04:42:06.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[27558]: audit 2023-10-09T04:42:05.860542+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:06.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[27558]: audit 2023-10-09T04:42:05.866562+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:06.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[27558]: cephadm 2023-10-09T04:42:05.887388+0000 mgr.y (mgr.14466) 2 : cephadm [INF] [09/Oct/2023:04:42:05] ENGINE Serving on https://172.21.15.133:7150 2023-10-09T04:42:06.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[27558]: cephadm 2023-10-09T04:42:05.887688+0000 mgr.y (mgr.14466) 3 : cephadm [INF] [09/Oct/2023:04:42:05] ENGINE Bus STARTED 2023-10-09T04:42:06.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[27558]: audit 2023-10-09T04:42:05.896715+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:06.553 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[27558]: cluster 2023-10-09T04:42:06.079799+0000 mon.a (mon.0) 664 : cluster [DBG] mgrmap e18: y(active, since 1.01603s), standbys: x 2023-10-09T04:42:06.553 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[27558]: audit 2023-10-09T04:42:06.092898+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:06.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[41291]: audit 2023-10-09T04:42:05.143906+0000 mon.a (mon.0) 653 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:42:06.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[41291]: cluster 2023-10-09T04:42:05.144996+0000 mon.a (mon.0) 654 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T04:42:06.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[41291]: cluster 2023-10-09T04:42:05.145381+0000 mon.a (mon.0) 655 : cluster [DBG] Standby manager daemon x started 2023-10-09T04:42:06.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[41291]: audit 2023-10-09T04:42:05.149073+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.194:0/594042489' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:42:06.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[41291]: audit 2023-10-09T04:42:05.150084+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.194:0/594042489' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:42:06.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[41291]: audit 2023-10-09T04:42:05.150622+0000 mon.a (mon.0) 656 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:42:06.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[41291]: audit 2023-10-09T04:42:05.153894+0000 mon.b (mon.1) 27 : audit [DBG] from='mgr.? 172.21.15.194:0/594042489' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:42:06.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[41291]: audit 2023-10-09T04:42:05.155354+0000 mon.b (mon.1) 28 : audit [DBG] from='mgr.? 172.21.15.194:0/594042489' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:42:06.554 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[41291]: audit 2023-10-09T04:42:05.170362+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:06.554 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[41291]: audit 2023-10-09T04:42:05.180524+0000 mon.a (mon.0) 658 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:06.554 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[41291]: audit 2023-10-09T04:42:05.193170+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:42:06.554 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[41291]: audit 2023-10-09T04:42:05.201424+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:42:06.554 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[41291]: cephadm 2023-10-09T04:42:05.768471+0000 mgr.y (mgr.14466) 1 : cephadm [INF] [09/Oct/2023:04:42:05] ENGINE Bus STARTING 2023-10-09T04:42:06.554 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[41291]: audit 2023-10-09T04:42:05.860542+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:06.554 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[41291]: audit 2023-10-09T04:42:05.866562+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:06.554 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[41291]: cephadm 2023-10-09T04:42:05.887388+0000 mgr.y (mgr.14466) 2 : cephadm [INF] [09/Oct/2023:04:42:05] ENGINE Serving on https://172.21.15.133:7150 2023-10-09T04:42:06.554 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[41291]: cephadm 2023-10-09T04:42:05.887688+0000 mgr.y (mgr.14466) 3 : cephadm [INF] [09/Oct/2023:04:42:05] ENGINE Bus STARTED 2023-10-09T04:42:06.554 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[41291]: audit 2023-10-09T04:42:05.896715+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:06.554 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[41291]: cluster 2023-10-09T04:42:06.079799+0000 mon.a (mon.0) 664 : cluster [DBG] mgrmap e18: y(active, since 1.01603s), standbys: x 2023-10-09T04:42:06.554 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:06 smithi133 bash[41291]: audit 2023-10-09T04:42:06.092898+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:06.916 DEBUG:teuthology.orchestra.run.smithi133:alertmanager.a> sudo journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@alertmanager.a.service 2023-10-09T04:42:06.918 INFO:tasks.cephadm:Adding grafana.a on smithi194 2023-10-09T04:42:06.918 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph orch apply grafana '1;smithi194=a' 2023-10-09T04:42:07.087 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:42:07.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:07 smithi133 bash[27558]: audit 2023-10-09T04:42:06.086260+0000 mgr.y (mgr.14466) 4 : audit [DBG] from='client.24374 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi133=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:42:07.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:07 smithi133 bash[27558]: cephadm 2023-10-09T04:42:06.088843+0000 mgr.y (mgr.14466) 5 : cephadm [INF] Saving service alertmanager spec with placement smithi133=a;count:1 2023-10-09T04:42:07.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:07 smithi133 bash[27558]: cluster 2023-10-09T04:42:06.089011+0000 mgr.y (mgr.14466) 6 : cluster [DBG] pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:07.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:07 smithi133 bash[41291]: audit 2023-10-09T04:42:06.086260+0000 mgr.y (mgr.14466) 4 : audit [DBG] from='client.24374 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi133=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:42:07.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:07 smithi133 bash[41291]: cephadm 2023-10-09T04:42:06.088843+0000 mgr.y (mgr.14466) 5 : cephadm [INF] Saving service alertmanager spec with placement smithi133=a;count:1 2023-10-09T04:42:07.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:07 smithi133 bash[41291]: cluster 2023-10-09T04:42:06.089011+0000 mgr.y (mgr.14466) 6 : cluster [DBG] pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:07.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:07 smithi194 bash[31648]: audit 2023-10-09T04:42:06.086260+0000 mgr.y (mgr.14466) 4 : audit [DBG] from='client.24374 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi133=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:42:07.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:07 smithi194 bash[31648]: cephadm 2023-10-09T04:42:06.088843+0000 mgr.y (mgr.14466) 5 : cephadm [INF] Saving service alertmanager spec with placement smithi133=a;count:1 2023-10-09T04:42:07.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:07 smithi194 bash[31648]: cluster 2023-10-09T04:42:06.089011+0000 mgr.y (mgr.14466) 6 : cluster [DBG] pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:08.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:08 smithi194 bash[31648]: cluster 2023-10-09T04:42:07.081009+0000 mgr.y (mgr.14466) 7 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:08.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:08 smithi194 bash[31648]: cluster 2023-10-09T04:42:08.088765+0000 mon.a (mon.0) 666 : cluster [DBG] mgrmap e19: y(active, since 3s), standbys: x 2023-10-09T04:42:08.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:08 smithi133 bash[41291]: cluster 2023-10-09T04:42:07.081009+0000 mgr.y (mgr.14466) 7 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:08.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:08 smithi133 bash[41291]: cluster 2023-10-09T04:42:08.088765+0000 mon.a (mon.0) 666 : cluster [DBG] mgrmap e19: y(active, since 3s), standbys: x 2023-10-09T04:42:08.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:08 smithi133 bash[27558]: cluster 2023-10-09T04:42:07.081009+0000 mgr.y (mgr.14466) 7 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:08.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:08 smithi133 bash[27558]: cluster 2023-10-09T04:42:08.088765+0000 mon.a (mon.0) 666 : cluster [DBG] mgrmap e19: y(active, since 3s), standbys: x 2023-10-09T04:42:10.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:10 smithi194 bash[31648]: cluster 2023-10-09T04:42:09.081674+0000 mgr.y (mgr.14466) 8 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:10.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:10 smithi133 bash[27558]: cluster 2023-10-09T04:42:09.081674+0000 mgr.y (mgr.14466) 8 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:10.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:10 smithi133 bash[41291]: cluster 2023-10-09T04:42:09.081674+0000 mgr.y (mgr.14466) 8 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:11.444 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:42:12.547 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:12 smithi194 bash[31648]: cluster 2023-10-09T04:42:11.082305+0000 mgr.y (mgr.14466) 9 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:12.547 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:12 smithi194 bash[31648]: audit 2023-10-09T04:42:11.180902+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:12.547 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:12 smithi194 bash[31648]: audit 2023-10-09T04:42:11.187881+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:12.547 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:12 smithi194 bash[31648]: audit 2023-10-09T04:42:11.211241+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:12.547 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:12 smithi194 bash[31648]: audit 2023-10-09T04:42:11.218379+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:12.548 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:12 smithi194 bash[31648]: audit 2023-10-09T04:42:11.933681+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:12.548 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:12 smithi194 bash[31648]: audit 2023-10-09T04:42:11.940105+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:12.548 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:12 smithi194 bash[31648]: audit 2023-10-09T04:42:11.941848+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:42:12.548 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:12 smithi194 bash[31648]: audit 2023-10-09T04:42:11.942931+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:42:12.548 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:12 smithi194 bash[31648]: audit 2023-10-09T04:42:11.943790+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:42:12.548 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:12 smithi194 bash[31648]: audit 2023-10-09T04:42:11.944649+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:42:12.548 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:12 smithi194 bash[31648]: cephadm 2023-10-09T04:42:11.945340+0000 mgr.y (mgr.14466) 10 : cephadm [INF] Adjusting osd_memory_target on smithi194 to 4060M 2023-10-09T04:42:12.548 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:12 smithi194 bash[31648]: audit 2023-10-09T04:42:11.950701+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:12.548 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:12 smithi194 bash[31648]: audit 2023-10-09T04:42:11.958665+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:12.548 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:12 smithi194 bash[31648]: audit 2023-10-09T04:42:11.965760+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:12.548 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:12 smithi194 bash[31648]: audit 2023-10-09T04:42:11.967769+0000 mon.a (mon.0) 680 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:42:12.548 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:12 smithi194 bash[31648]: audit 2023-10-09T04:42:11.968874+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:42:12.549 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:12 smithi194 bash[31648]: cephadm 2023-10-09T04:42:11.970515+0000 mgr.y (mgr.14466) 11 : cephadm [INF] Updating smithi133:/etc/ceph/ceph.conf 2023-10-09T04:42:12.549 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:12 smithi194 bash[31648]: cephadm 2023-10-09T04:42:11.970823+0000 mgr.y (mgr.14466) 12 : cephadm [INF] Updating smithi194:/etc/ceph/ceph.conf 2023-10-09T04:42:12.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[41291]: cluster 2023-10-09T04:42:11.082305+0000 mgr.y (mgr.14466) 9 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:12.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[41291]: audit 2023-10-09T04:42:11.180902+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:12.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[41291]: audit 2023-10-09T04:42:11.187881+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:12.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[41291]: audit 2023-10-09T04:42:11.211241+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:12.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[41291]: audit 2023-10-09T04:42:11.218379+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:12.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[41291]: audit 2023-10-09T04:42:11.933681+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:12.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[41291]: audit 2023-10-09T04:42:11.940105+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:12.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[41291]: audit 2023-10-09T04:42:11.941848+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:42:12.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[41291]: audit 2023-10-09T04:42:11.942931+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:42:12.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[41291]: audit 2023-10-09T04:42:11.943790+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:42:12.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[41291]: audit 2023-10-09T04:42:11.944649+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:42:12.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[41291]: cephadm 2023-10-09T04:42:11.945340+0000 mgr.y (mgr.14466) 10 : cephadm [INF] Adjusting osd_memory_target on smithi194 to 4060M 2023-10-09T04:42:12.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[41291]: audit 2023-10-09T04:42:11.950701+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:12.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[41291]: audit 2023-10-09T04:42:11.958665+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:12.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[41291]: audit 2023-10-09T04:42:11.965760+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:12.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[41291]: audit 2023-10-09T04:42:11.967769+0000 mon.a (mon.0) 680 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:42:12.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[41291]: audit 2023-10-09T04:42:11.968874+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:42:12.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[41291]: cephadm 2023-10-09T04:42:11.970515+0000 mgr.y (mgr.14466) 11 : cephadm [INF] Updating smithi133:/etc/ceph/ceph.conf 2023-10-09T04:42:12.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[41291]: cephadm 2023-10-09T04:42:11.970823+0000 mgr.y (mgr.14466) 12 : cephadm [INF] Updating smithi194:/etc/ceph/ceph.conf 2023-10-09T04:42:12.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[27558]: cluster 2023-10-09T04:42:11.082305+0000 mgr.y (mgr.14466) 9 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:12.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[27558]: audit 2023-10-09T04:42:11.180902+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:12.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[27558]: audit 2023-10-09T04:42:11.187881+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:12.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[27558]: audit 2023-10-09T04:42:11.211241+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:12.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[27558]: audit 2023-10-09T04:42:11.218379+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:12.553 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[27558]: audit 2023-10-09T04:42:11.933681+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:12.553 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[27558]: audit 2023-10-09T04:42:11.940105+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:12.553 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[27558]: audit 2023-10-09T04:42:11.941848+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:42:12.553 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[27558]: audit 2023-10-09T04:42:11.942931+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:42:12.553 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[27558]: audit 2023-10-09T04:42:11.943790+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:42:12.553 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[27558]: audit 2023-10-09T04:42:11.944649+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:42:12.553 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[27558]: cephadm 2023-10-09T04:42:11.945340+0000 mgr.y (mgr.14466) 10 : cephadm [INF] Adjusting osd_memory_target on smithi194 to 4060M 2023-10-09T04:42:12.553 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[27558]: audit 2023-10-09T04:42:11.950701+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:12.553 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[27558]: audit 2023-10-09T04:42:11.958665+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:12.553 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[27558]: audit 2023-10-09T04:42:11.965760+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:12.553 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[27558]: audit 2023-10-09T04:42:11.967769+0000 mon.a (mon.0) 680 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:42:12.553 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[27558]: audit 2023-10-09T04:42:11.968874+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:42:12.554 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[27558]: cephadm 2023-10-09T04:42:11.970515+0000 mgr.y (mgr.14466) 11 : cephadm [INF] Updating smithi133:/etc/ceph/ceph.conf 2023-10-09T04:42:12.554 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:12 smithi133 bash[27558]: cephadm 2023-10-09T04:42:11.970823+0000 mgr.y (mgr.14466) 12 : cephadm [INF] Updating smithi194:/etc/ceph/ceph.conf 2023-10-09T04:42:13.801 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:13 smithi133 systemd[1]: Started Ceph node-exporter.a for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:42:13.801 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:13 smithi133 bash[72837]: Unable to find image 'quay.io/prometheus/node-exporter:v1.5.0' locally 2023-10-09T04:42:13.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:13 smithi133 bash[27558]: cephadm 2023-10-09T04:42:12.135663+0000 mgr.y (mgr.14466) 13 : cephadm [INF] Updating smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:42:13.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:13 smithi133 bash[27558]: cephadm 2023-10-09T04:42:12.136588+0000 mgr.y (mgr.14466) 14 : cephadm [INF] Updating smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:42:13.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:13 smithi133 bash[27558]: cephadm 2023-10-09T04:42:12.300555+0000 mgr.y (mgr.14466) 15 : cephadm [INF] Updating smithi133:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:42:13.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:13 smithi133 bash[27558]: cephadm 2023-10-09T04:42:12.301664+0000 mgr.y (mgr.14466) 16 : cephadm [INF] Updating smithi194:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:42:13.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:13 smithi133 bash[27558]: cephadm 2023-10-09T04:42:12.465957+0000 mgr.y (mgr.14466) 17 : cephadm [INF] Updating smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:42:13.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:13 smithi133 bash[27558]: cephadm 2023-10-09T04:42:12.466377+0000 mgr.y (mgr.14466) 18 : cephadm [INF] Updating smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:42:13.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:13 smithi133 bash[27558]: audit 2023-10-09T04:42:12.640696+0000 mon.a (mon.0) 682 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:13.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:13 smithi133 bash[27558]: audit 2023-10-09T04:42:12.644928+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:13.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:13 smithi133 bash[27558]: audit 2023-10-09T04:42:12.650249+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:13.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:13 smithi133 bash[27558]: audit 2023-10-09T04:42:12.653729+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:13.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:13 smithi133 bash[27558]: audit 2023-10-09T04:42:12.658652+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:13.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:13 smithi133 bash[27558]: cephadm 2023-10-09T04:42:12.662712+0000 mgr.y (mgr.14466) 19 : cephadm [INF] Deploying daemon node-exporter.a on smithi133 2023-10-09T04:42:13.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:13 smithi133 bash[27558]: audit 2023-10-09T04:42:13.437118+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:13.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:13 smithi133 bash[27558]: audit 2023-10-09T04:42:13.442615+0000 mon.a (mon.0) 688 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:13.803 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:13 smithi133 bash[27558]: audit 2023-10-09T04:42:13.483552+0000 mon.a (mon.0) 689 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:13.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:13 smithi133 bash[41291]: cephadm 2023-10-09T04:42:12.135663+0000 mgr.y (mgr.14466) 13 : cephadm [INF] Updating smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:42:13.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:13 smithi133 bash[41291]: cephadm 2023-10-09T04:42:12.136588+0000 mgr.y (mgr.14466) 14 : cephadm [INF] Updating smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:42:13.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:13 smithi133 bash[41291]: cephadm 2023-10-09T04:42:12.300555+0000 mgr.y (mgr.14466) 15 : cephadm [INF] Updating smithi133:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:42:13.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:13 smithi133 bash[41291]: cephadm 2023-10-09T04:42:12.301664+0000 mgr.y (mgr.14466) 16 : cephadm [INF] Updating smithi194:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:42:13.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:13 smithi133 bash[41291]: cephadm 2023-10-09T04:42:12.465957+0000 mgr.y (mgr.14466) 17 : cephadm [INF] Updating smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:42:13.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:13 smithi133 bash[41291]: cephadm 2023-10-09T04:42:12.466377+0000 mgr.y (mgr.14466) 18 : cephadm [INF] Updating smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:42:13.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:13 smithi133 bash[41291]: audit 2023-10-09T04:42:12.640696+0000 mon.a (mon.0) 682 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:13.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:13 smithi133 bash[41291]: audit 2023-10-09T04:42:12.644928+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:13.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:13 smithi133 bash[41291]: audit 2023-10-09T04:42:12.650249+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:13.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:13 smithi133 bash[41291]: audit 2023-10-09T04:42:12.653729+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:13.804 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:13 smithi133 bash[41291]: audit 2023-10-09T04:42:12.658652+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:13.804 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:13 smithi133 bash[41291]: cephadm 2023-10-09T04:42:12.662712+0000 mgr.y (mgr.14466) 19 : cephadm [INF] Deploying daemon node-exporter.a on smithi133 2023-10-09T04:42:13.804 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:13 smithi133 bash[41291]: audit 2023-10-09T04:42:13.437118+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:13.804 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:13 smithi133 bash[41291]: audit 2023-10-09T04:42:13.442615+0000 mon.a (mon.0) 688 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:13.804 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:13 smithi133 bash[41291]: audit 2023-10-09T04:42:13.483552+0000 mon.a (mon.0) 689 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:13.901 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:13 smithi194 bash[31648]: cephadm 2023-10-09T04:42:12.135663+0000 mgr.y (mgr.14466) 13 : cephadm [INF] Updating smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:42:13.901 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:13 smithi194 bash[31648]: cephadm 2023-10-09T04:42:12.136588+0000 mgr.y (mgr.14466) 14 : cephadm [INF] Updating smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:42:13.901 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:13 smithi194 bash[31648]: cephadm 2023-10-09T04:42:12.300555+0000 mgr.y (mgr.14466) 15 : cephadm [INF] Updating smithi133:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:42:13.901 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:13 smithi194 bash[31648]: cephadm 2023-10-09T04:42:12.301664+0000 mgr.y (mgr.14466) 16 : cephadm [INF] Updating smithi194:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:42:13.901 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:13 smithi194 bash[31648]: cephadm 2023-10-09T04:42:12.465957+0000 mgr.y (mgr.14466) 17 : cephadm [INF] Updating smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:42:13.901 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:13 smithi194 bash[31648]: cephadm 2023-10-09T04:42:12.466377+0000 mgr.y (mgr.14466) 18 : cephadm [INF] Updating smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:42:13.901 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:13 smithi194 bash[31648]: audit 2023-10-09T04:42:12.640696+0000 mon.a (mon.0) 682 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:13.901 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:13 smithi194 bash[31648]: audit 2023-10-09T04:42:12.644928+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:13.901 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:13 smithi194 bash[31648]: audit 2023-10-09T04:42:12.650249+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:13.902 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:13 smithi194 bash[31648]: audit 2023-10-09T04:42:12.653729+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:13.902 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:13 smithi194 bash[31648]: audit 2023-10-09T04:42:12.658652+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:13.902 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:13 smithi194 bash[31648]: cephadm 2023-10-09T04:42:12.662712+0000 mgr.y (mgr.14466) 19 : cephadm [INF] Deploying daemon node-exporter.a on smithi133 2023-10-09T04:42:13.902 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:13 smithi194 bash[31648]: audit 2023-10-09T04:42:13.437118+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:13.902 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:13 smithi194 bash[31648]: audit 2023-10-09T04:42:13.442615+0000 mon.a (mon.0) 688 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:13.902 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:13 smithi194 bash[31648]: audit 2023-10-09T04:42:13.483552+0000 mon.a (mon.0) 689 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:14.254 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:14 smithi194 systemd[1]: Started Ceph node-exporter.b for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:42:14.313 INFO:teuthology.orchestra.run.smithi194.stdout:Scheduled grafana update... 2023-10-09T04:42:14.319 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:14 smithi133 bash[72837]: v1.5.0: Pulling from prometheus/node-exporter 2023-10-09T04:42:14.319 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:14 smithi133 bash[72837]: 22b70bddd3ac: Pulling fs layer 2023-10-09T04:42:14.320 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:14 smithi133 bash[72837]: 5c12815fee55: Pulling fs layer 2023-10-09T04:42:14.513 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:14 smithi194 bash[66301]: Unable to find image 'quay.io/prometheus/node-exporter:v1.5.0' locally 2023-10-09T04:42:14.634 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:14 smithi133 bash[72837]: c0e87333d380: Pulling fs layer 2023-10-09T04:42:14.634 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:14 smithi133 bash[72837]: 5c12815fee55: Verifying Checksum 2023-10-09T04:42:14.635 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:14 smithi133 bash[72837]: 5c12815fee55: Download complete 2023-10-09T04:42:14.635 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:14 smithi133 bash[72837]: 22b70bddd3ac: Verifying Checksum 2023-10-09T04:42:14.635 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:14 smithi133 bash[72837]: 22b70bddd3ac: Download complete 2023-10-09T04:42:14.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:14 smithi194 bash[31648]: cluster 2023-10-09T04:42:13.082937+0000 mgr.y (mgr.14466) 20 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:14.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:14 smithi194 bash[31648]: cephadm 2023-10-09T04:42:13.484692+0000 mgr.y (mgr.14466) 21 : cephadm [INF] Deploying daemon node-exporter.b on smithi194 2023-10-09T04:42:14.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:14 smithi194 bash[31648]: audit 2023-10-09T04:42:14.244985+0000 mon.a (mon.0) 690 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:14.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:14 smithi194 bash[31648]: audit 2023-10-09T04:42:14.252546+0000 mon.a (mon.0) 691 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:14.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:14 smithi194 bash[31648]: audit 2023-10-09T04:42:14.310314+0000 mon.a (mon.0) 692 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:14.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:14 smithi194 bash[31648]: audit 2023-10-09T04:42:14.314234+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:14.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:14 smithi194 bash[31648]: audit 2023-10-09T04:42:14.319378+0000 mon.a (mon.0) 694 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:14.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:14 smithi194 bash[31648]: audit 2023-10-09T04:42:14.329831+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:15.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:14 smithi133 bash[27558]: cluster 2023-10-09T04:42:13.082937+0000 mgr.y (mgr.14466) 20 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:15.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:14 smithi133 bash[27558]: cephadm 2023-10-09T04:42:13.484692+0000 mgr.y (mgr.14466) 21 : cephadm [INF] Deploying daemon node-exporter.b on smithi194 2023-10-09T04:42:15.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:14 smithi133 bash[27558]: audit 2023-10-09T04:42:14.244985+0000 mon.a (mon.0) 690 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:15.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:14 smithi133 bash[27558]: audit 2023-10-09T04:42:14.252546+0000 mon.a (mon.0) 691 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:15.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:14 smithi133 bash[27558]: audit 2023-10-09T04:42:14.310314+0000 mon.a (mon.0) 692 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:15.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:14 smithi133 bash[27558]: audit 2023-10-09T04:42:14.314234+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:15.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:14 smithi133 bash[27558]: audit 2023-10-09T04:42:14.319378+0000 mon.a (mon.0) 694 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:15.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:14 smithi133 bash[27558]: audit 2023-10-09T04:42:14.329831+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:15.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:14 smithi133 bash[41291]: cluster 2023-10-09T04:42:13.082937+0000 mgr.y (mgr.14466) 20 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:15.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:14 smithi133 bash[41291]: cephadm 2023-10-09T04:42:13.484692+0000 mgr.y (mgr.14466) 21 : cephadm [INF] Deploying daemon node-exporter.b on smithi194 2023-10-09T04:42:15.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:14 smithi133 bash[41291]: audit 2023-10-09T04:42:14.244985+0000 mon.a (mon.0) 690 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:15.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:14 smithi133 bash[41291]: audit 2023-10-09T04:42:14.252546+0000 mon.a (mon.0) 691 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:15.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:14 smithi133 bash[41291]: audit 2023-10-09T04:42:14.310314+0000 mon.a (mon.0) 692 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:15.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:14 smithi133 bash[41291]: audit 2023-10-09T04:42:14.314234+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:15.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:14 smithi133 bash[41291]: audit 2023-10-09T04:42:14.319378+0000 mon.a (mon.0) 694 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:15.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:14 smithi133 bash[41291]: audit 2023-10-09T04:42:14.329831+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:15.054 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:14 smithi133 bash[72837]: c0e87333d380: Download complete 2023-10-09T04:42:15.054 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:14 smithi133 bash[72837]: 22b70bddd3ac: Pull complete 2023-10-09T04:42:15.140 DEBUG:teuthology.orchestra.run.smithi194:grafana.a> sudo journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@grafana.a.service 2023-10-09T04:42:15.142 INFO:tasks.cephadm:Setting up client nodes... 2023-10-09T04:42:15.142 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-09T04:42:15.376 INFO:journalctl@ceph.grafana.a.smithi194.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:42:15.452 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:15 smithi194 bash[66301]: v1.5.0: Pulling from prometheus/node-exporter 2023-10-09T04:42:15.550 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:15 smithi133 bash[72837]: 5c12815fee55: Pull complete 2023-10-09T04:42:15.750 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:15 smithi194 bash[31648]: audit 2023-10-09T04:42:14.299661+0000 mgr.y (mgr.14466) 22 : audit [DBG] from='client.24394 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi194=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:42:15.751 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:15 smithi194 bash[31648]: cephadm 2023-10-09T04:42:14.303566+0000 mgr.y (mgr.14466) 23 : cephadm [INF] Saving service grafana spec with placement smithi194=a;count:1 2023-10-09T04:42:15.751 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:15 smithi194 bash[31648]: cephadm 2023-10-09T04:42:14.346180+0000 mgr.y (mgr.14466) 24 : cephadm [INF] Deploying daemon alertmanager.a on smithi133 2023-10-09T04:42:15.751 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:15 smithi194 bash[31648]: audit 2023-10-09T04:42:15.177414+0000 mon.a (mon.0) 696 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:15.751 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:15 smithi194 bash[66301]: 22b70bddd3ac: Pulling fs layer 2023-10-09T04:42:15.751 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:15 smithi194 bash[66301]: 5c12815fee55: Pulling fs layer 2023-10-09T04:42:15.751 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:15 smithi194 bash[66301]: c0e87333d380: Pulling fs layer 2023-10-09T04:42:15.751 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:15 smithi194 bash[66301]: 5c12815fee55: Verifying Checksum 2023-10-09T04:42:15.751 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:15 smithi194 bash[66301]: 5c12815fee55: Download complete 2023-10-09T04:42:15.751 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:15 smithi194 bash[66301]: 22b70bddd3ac: Verifying Checksum 2023-10-09T04:42:15.752 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:15 smithi194 bash[66301]: 22b70bddd3ac: Download complete 2023-10-09T04:42:15.752 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:15 smithi194 bash[66301]: c0e87333d380: Verifying Checksum 2023-10-09T04:42:15.752 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:15 smithi194 bash[66301]: c0e87333d380: Download complete 2023-10-09T04:42:16.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:15 smithi133 bash[27558]: audit 2023-10-09T04:42:14.299661+0000 mgr.y (mgr.14466) 22 : audit [DBG] from='client.24394 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi194=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:42:16.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:15 smithi133 bash[27558]: cephadm 2023-10-09T04:42:14.303566+0000 mgr.y (mgr.14466) 23 : cephadm [INF] Saving service grafana spec with placement smithi194=a;count:1 2023-10-09T04:42:16.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:15 smithi133 bash[27558]: cephadm 2023-10-09T04:42:14.346180+0000 mgr.y (mgr.14466) 24 : cephadm [INF] Deploying daemon alertmanager.a on smithi133 2023-10-09T04:42:16.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:15 smithi133 bash[27558]: audit 2023-10-09T04:42:15.177414+0000 mon.a (mon.0) 696 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:16.051 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:15 smithi133 bash[72837]: c0e87333d380: Pull complete 2023-10-09T04:42:16.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:15 smithi133 bash[41291]: audit 2023-10-09T04:42:14.299661+0000 mgr.y (mgr.14466) 22 : audit [DBG] from='client.24394 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi194=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:42:16.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:15 smithi133 bash[41291]: cephadm 2023-10-09T04:42:14.303566+0000 mgr.y (mgr.14466) 23 : cephadm [INF] Saving service grafana spec with placement smithi194=a;count:1 2023-10-09T04:42:16.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:15 smithi133 bash[41291]: cephadm 2023-10-09T04:42:14.346180+0000 mgr.y (mgr.14466) 24 : cephadm [INF] Deploying daemon alertmanager.a on smithi133 2023-10-09T04:42:16.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:15 smithi133 bash[41291]: audit 2023-10-09T04:42:15.177414+0000 mon.a (mon.0) 696 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:16.136 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:15 smithi194 bash[66301]: 22b70bddd3ac: Pull complete 2023-10-09T04:42:16.550 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:16 smithi133 bash[72837]: Digest: sha256:39c642b2b337e38c18e80266fb14383754178202f40103646337722a594d984c 2023-10-09T04:42:16.550 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:16 smithi133 bash[72837]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.5.0 2023-10-09T04:42:16.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:42:16 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:42:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:42:16.637 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:16 smithi194 bash[66301]: 5c12815fee55: Pull complete 2023-10-09T04:42:16.923 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:16 smithi194 bash[66301]: c0e87333d380: Pull complete 2023-10-09T04:42:16.924 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:16 smithi194 bash[66301]: Digest: sha256:39c642b2b337e38c18e80266fb14383754178202f40103646337722a594d984c 2023-10-09T04:42:16.924 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:16 smithi194 bash[66301]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.5.0 2023-10-09T04:42:16.924 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:16 smithi194 bash[31648]: cluster 2023-10-09T04:42:15.083299+0000 mgr.y (mgr.14466) 25 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:17.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:16 smithi133 bash[41291]: cluster 2023-10-09T04:42:15.083299+0000 mgr.y (mgr.14466) 25 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:17.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:16 smithi133 bash[27558]: cluster 2023-10-09T04:42:15.083299+0000 mgr.y (mgr.14466) 25 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:18.387 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.180Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-10-09T04:42:18.388 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.180Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-10-09T04:42:18.388 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.181Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-10-09T04:42:18.388 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.181Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T04:42:18.388 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.181Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-10-09T04:42:18.388 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.181Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-10-09T04:42:18.388 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-10-09T04:42:18.388 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=arp 2023-10-09T04:42:18.388 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=bcache 2023-10-09T04:42:18.388 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=bonding 2023-10-09T04:42:18.388 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=btrfs 2023-10-09T04:42:18.389 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=conntrack 2023-10-09T04:42:18.389 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=cpu 2023-10-09T04:42:18.389 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-10-09T04:42:18.389 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=diskstats 2023-10-09T04:42:18.389 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=dmi 2023-10-09T04:42:18.389 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=edac 2023-10-09T04:42:18.389 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=entropy 2023-10-09T04:42:18.389 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-10-09T04:42:18.389 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=filefd 2023-10-09T04:42:18.389 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=filesystem 2023-10-09T04:42:18.389 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=hwmon 2023-10-09T04:42:18.390 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=infiniband 2023-10-09T04:42:18.390 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=ipvs 2023-10-09T04:42:18.390 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=loadavg 2023-10-09T04:42:18.390 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=mdadm 2023-10-09T04:42:18.390 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=meminfo 2023-10-09T04:42:18.390 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=netclass 2023-10-09T04:42:18.390 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=netdev 2023-10-09T04:42:18.390 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=netstat 2023-10-09T04:42:18.390 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=nfs 2023-10-09T04:42:18.390 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=nfsd 2023-10-09T04:42:18.390 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=nvme 2023-10-09T04:42:18.391 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=os 2023-10-09T04:42:18.391 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-10-09T04:42:18.391 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=pressure 2023-10-09T04:42:18.391 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=rapl 2023-10-09T04:42:18.391 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=schedstat 2023-10-09T04:42:18.391 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=selinux 2023-10-09T04:42:18.391 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=sockstat 2023-10-09T04:42:18.391 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=softnet 2023-10-09T04:42:18.391 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=stat 2023-10-09T04:42:18.391 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=tapestats 2023-10-09T04:42:18.391 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=textfile 2023-10-09T04:42:18.392 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.182Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-10-09T04:42:18.392 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.183Z caller=node_exporter.go:117 level=info collector=time 2023-10-09T04:42:18.392 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.183Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-10-09T04:42:18.392 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.183Z caller=node_exporter.go:117 level=info collector=uname 2023-10-09T04:42:18.392 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.183Z caller=node_exporter.go:117 level=info collector=vmstat 2023-10-09T04:42:18.392 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.183Z caller=node_exporter.go:117 level=info collector=xfs 2023-10-09T04:42:18.392 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.183Z caller=node_exporter.go:117 level=info collector=zfs 2023-10-09T04:42:18.392 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.183Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-10-09T04:42:18.392 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[66301]: ts=2023-10-09T04:42:18.183Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-10-09T04:42:18.676 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.510Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-10-09T04:42:18.676 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.510Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-10-09T04:42:18.676 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.511Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-10-09T04:42:18.677 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.511Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-10-09T04:42:18.677 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.511Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-10-09T04:42:18.677 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.511Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T04:42:18.677 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-10-09T04:42:18.677 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=arp 2023-10-09T04:42:18.677 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=bcache 2023-10-09T04:42:18.677 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=bonding 2023-10-09T04:42:18.677 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=btrfs 2023-10-09T04:42:18.677 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=conntrack 2023-10-09T04:42:18.677 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=cpu 2023-10-09T04:42:18.677 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-10-09T04:42:18.678 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=diskstats 2023-10-09T04:42:18.678 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=dmi 2023-10-09T04:42:18.678 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=edac 2023-10-09T04:42:18.678 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=entropy 2023-10-09T04:42:18.678 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-10-09T04:42:18.678 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=filefd 2023-10-09T04:42:18.678 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=filesystem 2023-10-09T04:42:18.678 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=hwmon 2023-10-09T04:42:18.678 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=infiniband 2023-10-09T04:42:18.678 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=ipvs 2023-10-09T04:42:18.678 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=loadavg 2023-10-09T04:42:18.679 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=mdadm 2023-10-09T04:42:18.679 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=meminfo 2023-10-09T04:42:18.679 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=netclass 2023-10-09T04:42:18.679 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=netdev 2023-10-09T04:42:18.679 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=netstat 2023-10-09T04:42:18.679 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=nfs 2023-10-09T04:42:18.679 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=nfsd 2023-10-09T04:42:18.679 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=nvme 2023-10-09T04:42:18.679 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=os 2023-10-09T04:42:18.679 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-10-09T04:42:18.679 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=pressure 2023-10-09T04:42:18.679 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=rapl 2023-10-09T04:42:18.679 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=schedstat 2023-10-09T04:42:18.680 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=selinux 2023-10-09T04:42:18.680 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=sockstat 2023-10-09T04:42:18.680 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=softnet 2023-10-09T04:42:18.680 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=stat 2023-10-09T04:42:18.680 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=tapestats 2023-10-09T04:42:18.680 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=textfile 2023-10-09T04:42:18.680 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-10-09T04:42:18.680 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=time 2023-10-09T04:42:18.680 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-10-09T04:42:18.680 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=uname 2023-10-09T04:42:18.680 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=vmstat 2023-10-09T04:42:18.680 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=xfs 2023-10-09T04:42:18.680 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.512Z caller=node_exporter.go:117 level=info collector=zfs 2023-10-09T04:42:18.681 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.513Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-10-09T04:42:18.681 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[72837]: ts=2023-10-09T04:42:18.513Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-10-09T04:42:18.681 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[27558]: cluster 2023-10-09T04:42:17.083961+0000 mgr.y (mgr.14466) 26 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:19.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:18 smithi133 bash[41291]: cluster 2023-10-09T04:42:17.083961+0000 mgr.y (mgr.14466) 26 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:19.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:18 smithi194 bash[31648]: cluster 2023-10-09T04:42:17.083961+0000 mgr.y (mgr.14466) 26 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:20.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:20 smithi133 bash[27558]: cluster 2023-10-09T04:42:19.084451+0000 mgr.y (mgr.14466) 27 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:20.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:20 smithi133 bash[27558]: audit 2023-10-09T04:42:20.181071+0000 mon.a (mon.0) 697 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:20.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:20 smithi133 bash[41291]: cluster 2023-10-09T04:42:19.084451+0000 mgr.y (mgr.14466) 27 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:20.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:20 smithi133 bash[41291]: audit 2023-10-09T04:42:20.181071+0000 mon.a (mon.0) 697 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:20.895 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:42:21.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:20 smithi194 bash[31648]: cluster 2023-10-09T04:42:19.084451+0000 mgr.y (mgr.14466) 27 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:21.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:20 smithi194 bash[31648]: audit 2023-10-09T04:42:20.181071+0000 mon.a (mon.0) 697 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:23.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:22 smithi133 bash[27558]: cluster 2023-10-09T04:42:21.084970+0000 mgr.y (mgr.14466) 28 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:23.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:22 smithi133 bash[41291]: cluster 2023-10-09T04:42:21.084970+0000 mgr.y (mgr.14466) 28 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:23.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:22 smithi194 bash[31648]: cluster 2023-10-09T04:42:21.084970+0000 mgr.y (mgr.14466) 28 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:24.962 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:24 smithi133 bash[27558]: cluster 2023-10-09T04:42:23.085569+0000 mgr.y (mgr.14466) 29 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:24.962 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:24 smithi133 bash[41291]: cluster 2023-10-09T04:42:23.085569+0000 mgr.y (mgr.14466) 29 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:25.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:24 smithi194 bash[31648]: cluster 2023-10-09T04:42:23.085569+0000 mgr.y (mgr.14466) 29 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:26.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:42:26 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:42:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:42:26.769 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:26 smithi133 bash[41291]: cluster 2023-10-09T04:42:25.085893+0000 mgr.y (mgr.14466) 30 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:26.769 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:42:26 smithi133 systemd[1]: Started Ceph alertmanager.a for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:42:26.769 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:26 smithi133 bash[27558]: cluster 2023-10-09T04:42:25.085893+0000 mgr.y (mgr.14466) 30 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:26.798 INFO:teuthology.orchestra.run.smithi133.stdout:[client.0] 2023-10-09T04:42:26.799 INFO:teuthology.orchestra.run.smithi133.stdout: key = AQCyhCNlt2T6LhAAajAeIelEV5Zo9XDydKWcMQ== 2023-10-09T04:42:27.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:26 smithi194 bash[31648]: cluster 2023-10-09T04:42:25.085893+0000 mgr.y (mgr.14466) 30 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:28.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:27 smithi133 bash[27558]: audit 2023-10-09T04:42:26.740392+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:28.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:27 smithi133 bash[27558]: audit 2023-10-09T04:42:26.747666+0000 mon.a (mon.0) 699 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:28.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:27 smithi133 bash[27558]: audit 2023-10-09T04:42:26.787978+0000 mon.a (mon.0) 700 : audit [INF] from='client.? 172.21.15.133:0/2849628552' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T04:42:28.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:27 smithi133 bash[27558]: audit 2023-10-09T04:42:26.795971+0000 mon.a (mon.0) 701 : audit [INF] from='client.? 172.21.15.133:0/2849628552' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T04:42:28.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:27 smithi133 bash[27558]: audit 2023-10-09T04:42:26.802594+0000 mon.a (mon.0) 702 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:28.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:27 smithi133 bash[27558]: audit 2023-10-09T04:42:26.810694+0000 mon.a (mon.0) 703 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:28.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:27 smithi133 bash[27558]: audit 2023-10-09T04:42:26.876471+0000 mon.a (mon.0) 704 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:42:28.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:27 smithi133 bash[41291]: audit 2023-10-09T04:42:26.740392+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:28.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:27 smithi133 bash[41291]: audit 2023-10-09T04:42:26.747666+0000 mon.a (mon.0) 699 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:28.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:27 smithi133 bash[41291]: audit 2023-10-09T04:42:26.787978+0000 mon.a (mon.0) 700 : audit [INF] from='client.? 172.21.15.133:0/2849628552' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T04:42:28.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:27 smithi133 bash[41291]: audit 2023-10-09T04:42:26.795971+0000 mon.a (mon.0) 701 : audit [INF] from='client.? 172.21.15.133:0/2849628552' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T04:42:28.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:27 smithi133 bash[41291]: audit 2023-10-09T04:42:26.802594+0000 mon.a (mon.0) 702 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:28.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:27 smithi133 bash[41291]: audit 2023-10-09T04:42:26.810694+0000 mon.a (mon.0) 703 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:28.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:27 smithi133 bash[41291]: audit 2023-10-09T04:42:26.876471+0000 mon.a (mon.0) 704 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:42:28.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:27 smithi194 bash[31648]: audit 2023-10-09T04:42:26.740392+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:28.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:27 smithi194 bash[31648]: audit 2023-10-09T04:42:26.747666+0000 mon.a (mon.0) 699 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:28.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:27 smithi194 bash[31648]: audit 2023-10-09T04:42:26.787978+0000 mon.a (mon.0) 700 : audit [INF] from='client.? 172.21.15.133:0/2849628552' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T04:42:28.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:27 smithi194 bash[31648]: audit 2023-10-09T04:42:26.795971+0000 mon.a (mon.0) 701 : audit [INF] from='client.? 172.21.15.133:0/2849628552' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T04:42:28.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:27 smithi194 bash[31648]: audit 2023-10-09T04:42:26.802594+0000 mon.a (mon.0) 702 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:28.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:27 smithi194 bash[31648]: audit 2023-10-09T04:42:26.810694+0000 mon.a (mon.0) 703 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:28.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:27 smithi194 bash[31648]: audit 2023-10-09T04:42:26.876471+0000 mon.a (mon.0) 704 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:42:28.693 DEBUG:teuthology.orchestra.run.smithi133:> set -ex 2023-10-09T04:42:28.693 DEBUG:teuthology.orchestra.run.smithi133:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-10-09T04:42:28.693 DEBUG:teuthology.orchestra.run.smithi133:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-10-09T04:42:28.712 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-09T04:42:28.957 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:28 smithi133 bash[27558]: cluster 2023-10-09T04:42:27.086382+0000 mgr.y (mgr.14466) 31 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:28.958 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:28 smithi133 bash[41291]: cluster 2023-10-09T04:42:27.086382+0000 mgr.y (mgr.14466) 31 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:29.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:28 smithi194 bash[31648]: cluster 2023-10-09T04:42:27.086382+0000 mgr.y (mgr.14466) 31 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:29.300 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:42:28 smithi133 bash[73393]: ts=2023-10-09T04:42:28.951Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-10-09T04:42:29.301 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:42:28 smithi133 bash[73393]: ts=2023-10-09T04:42:28.952Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-10-09T04:42:29.301 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:42:28 smithi133 bash[73393]: ts=2023-10-09T04:42:28.953Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.133 port=9094 2023-10-09T04:42:29.301 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:42:28 smithi133 bash[73393]: ts=2023-10-09T04:42:28.955Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T04:42:29.301 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:42:28 smithi133 bash[73393]: ts=2023-10-09T04:42:28.993Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T04:42:29.301 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:42:28 smithi133 bash[73393]: ts=2023-10-09T04:42:28.994Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T04:42:29.301 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:42:29 smithi133 bash[73393]: ts=2023-10-09T04:42:28.999Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-10-09T04:42:29.301 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:42:29 smithi133 bash[73393]: ts=2023-10-09T04:42:28.999Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-10-09T04:42:31.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:30 smithi194 bash[31648]: cluster 2023-10-09T04:42:29.086967+0000 mgr.y (mgr.14466) 32 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:31.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:30 smithi194 bash[31648]: audit 2023-10-09T04:42:30.186247+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:31.300 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:42:30 smithi133 bash[73393]: ts=2023-10-09T04:42:30.955Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000249255s 2023-10-09T04:42:31.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:30 smithi133 bash[27558]: cluster 2023-10-09T04:42:29.086967+0000 mgr.y (mgr.14466) 32 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:31.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:30 smithi133 bash[27558]: audit 2023-10-09T04:42:30.186247+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:31.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:30 smithi133 bash[41291]: cluster 2023-10-09T04:42:29.086967+0000 mgr.y (mgr.14466) 32 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:31.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:30 smithi133 bash[41291]: audit 2023-10-09T04:42:30.186247+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:32.685 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:42:33.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:32 smithi194 bash[31648]: cluster 2023-10-09T04:42:31.087690+0000 mgr.y (mgr.14466) 33 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:33.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:32 smithi194 bash[31648]: audit 2023-10-09T04:42:32.618477+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:33.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:32 smithi194 bash[31648]: audit 2023-10-09T04:42:32.625741+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:33.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:32 smithi133 bash[27558]: cluster 2023-10-09T04:42:31.087690+0000 mgr.y (mgr.14466) 33 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:33.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:32 smithi133 bash[27558]: audit 2023-10-09T04:42:32.618477+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:33.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:32 smithi133 bash[27558]: audit 2023-10-09T04:42:32.625741+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:33.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:32 smithi133 bash[41291]: cluster 2023-10-09T04:42:31.087690+0000 mgr.y (mgr.14466) 33 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:33.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:32 smithi133 bash[41291]: audit 2023-10-09T04:42:32.618477+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:33.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:32 smithi133 bash[41291]: audit 2023-10-09T04:42:32.625741+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:34.050 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:42:33 smithi133 bash[27821]: [09/Oct/2023:04:42:33] ENGINE Bus STOPPING 2023-10-09T04:42:34.051 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:42:33 smithi133 bash[27821]: [09/Oct/2023:04:42:33] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T04:42:34.051 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:42:33 smithi133 bash[27821]: [09/Oct/2023:04:42:33] ENGINE Bus STOPPED 2023-10-09T04:42:34.051 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:42:33 smithi133 bash[27821]: [09/Oct/2023:04:42:33] ENGINE Bus STARTING 2023-10-09T04:42:34.051 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:42:33 smithi133 bash[27821]: [09/Oct/2023:04:42:33] ENGINE Serving on http://:::9283 2023-10-09T04:42:34.051 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:42:33 smithi133 bash[27821]: [09/Oct/2023:04:42:33] ENGINE Bus STARTED 2023-10-09T04:42:34.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:34 smithi194 bash[31648]: cluster 2023-10-09T04:42:33.088264+0000 mgr.y (mgr.14466) 34 : cluster [DBG] pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:34.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:34 smithi194 bash[31648]: audit 2023-10-09T04:42:33.629289+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:34.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:34 smithi194 bash[31648]: audit 2023-10-09T04:42:33.637147+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:34.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:34 smithi194 bash[31648]: audit 2023-10-09T04:42:33.638740+0000 mon.a (mon.0) 710 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:42:34.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:34 smithi194 bash[31648]: audit 2023-10-09T04:42:33.640008+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:42:34.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:34 smithi194 bash[31648]: audit 2023-10-09T04:42:33.646925+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:34.889 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:34 smithi194 bash[31648]: audit 2023-10-09T04:42:33.713113+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:34.889 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:34 smithi194 bash[31648]: audit 2023-10-09T04:42:33.717896+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:34.890 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:34 smithi194 bash[31648]: audit 2023-10-09T04:42:33.722557+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T04:42:34.890 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:34 smithi194 bash[31648]: audit 2023-10-09T04:42:33.723390+0000 mgr.y (mgr.14466) 35 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T04:42:34.890 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:34 smithi194 bash[31648]: audit 2023-10-09T04:42:33.730995+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:34.890 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:34 smithi194 bash[31648]: cephadm 2023-10-09T04:42:33.752057+0000 mgr.y (mgr.14466) 36 : cephadm [INF] Deploying daemon grafana.a on smithi194 2023-10-09T04:42:35.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:34 smithi133 bash[27558]: cluster 2023-10-09T04:42:33.088264+0000 mgr.y (mgr.14466) 34 : cluster [DBG] pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:35.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:34 smithi133 bash[27558]: audit 2023-10-09T04:42:33.629289+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:35.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:34 smithi133 bash[27558]: audit 2023-10-09T04:42:33.637147+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:35.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:34 smithi133 bash[27558]: audit 2023-10-09T04:42:33.638740+0000 mon.a (mon.0) 710 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:42:35.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:34 smithi133 bash[27558]: audit 2023-10-09T04:42:33.640008+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:42:35.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:34 smithi133 bash[27558]: audit 2023-10-09T04:42:33.646925+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:35.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:34 smithi133 bash[27558]: audit 2023-10-09T04:42:33.713113+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:35.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:34 smithi133 bash[27558]: audit 2023-10-09T04:42:33.717896+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:35.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:34 smithi133 bash[27558]: audit 2023-10-09T04:42:33.722557+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T04:42:35.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:34 smithi133 bash[27558]: audit 2023-10-09T04:42:33.723390+0000 mgr.y (mgr.14466) 35 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T04:42:35.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:34 smithi133 bash[27558]: audit 2023-10-09T04:42:33.730995+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:35.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:34 smithi133 bash[27558]: cephadm 2023-10-09T04:42:33.752057+0000 mgr.y (mgr.14466) 36 : cephadm [INF] Deploying daemon grafana.a on smithi194 2023-10-09T04:42:35.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:34 smithi133 bash[41291]: cluster 2023-10-09T04:42:33.088264+0000 mgr.y (mgr.14466) 34 : cluster [DBG] pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:35.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:34 smithi133 bash[41291]: audit 2023-10-09T04:42:33.629289+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:35.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:34 smithi133 bash[41291]: audit 2023-10-09T04:42:33.637147+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:35.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:34 smithi133 bash[41291]: audit 2023-10-09T04:42:33.638740+0000 mon.a (mon.0) 710 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:42:35.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:34 smithi133 bash[41291]: audit 2023-10-09T04:42:33.640008+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:42:35.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:34 smithi133 bash[41291]: audit 2023-10-09T04:42:33.646925+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:35.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:34 smithi133 bash[41291]: audit 2023-10-09T04:42:33.713113+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:35.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:34 smithi133 bash[41291]: audit 2023-10-09T04:42:33.717896+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:35.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:34 smithi133 bash[41291]: audit 2023-10-09T04:42:33.722557+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T04:42:35.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:34 smithi133 bash[41291]: audit 2023-10-09T04:42:33.723390+0000 mgr.y (mgr.14466) 35 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T04:42:35.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:34 smithi133 bash[41291]: audit 2023-10-09T04:42:33.730995+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:35.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:34 smithi133 bash[41291]: cephadm 2023-10-09T04:42:33.752057+0000 mgr.y (mgr.14466) 36 : cephadm [INF] Deploying daemon grafana.a on smithi194 2023-10-09T04:42:36.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:35 smithi133 bash[27558]: audit 2023-10-09T04:42:35.182050+0000 mon.a (mon.0) 717 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:36.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:35 smithi133 bash[41291]: audit 2023-10-09T04:42:35.182050+0000 mon.a (mon.0) 717 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:36.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:35 smithi194 bash[31648]: audit 2023-10-09T04:42:35.182050+0000 mon.a (mon.0) 717 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:36.433 INFO:teuthology.orchestra.run.smithi194.stdout:[client.1] 2023-10-09T04:42:36.433 INFO:teuthology.orchestra.run.smithi194.stdout: key = AQC8hCNlTYMlGRAAlGEG0baN7xgWJsdl3nOZDw== 2023-10-09T04:42:36.636 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:42:36 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:42:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:42:37.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:36 smithi133 bash[41291]: cluster 2023-10-09T04:42:35.088850+0000 mgr.y (mgr.14466) 37 : cluster [DBG] pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:37.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:36 smithi133 bash[41291]: audit 2023-10-09T04:42:36.421554+0000 mon.a (mon.0) 718 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T04:42:37.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:36 smithi133 bash[41291]: audit 2023-10-09T04:42:36.422159+0000 mon.b (mon.1) 29 : audit [INF] from='client.? 172.21.15.194:0/2708928061' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T04:42:37.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:36 smithi133 bash[41291]: audit 2023-10-09T04:42:36.429919+0000 mon.a (mon.0) 719 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T04:42:37.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:36 smithi133 bash[27558]: cluster 2023-10-09T04:42:35.088850+0000 mgr.y (mgr.14466) 37 : cluster [DBG] pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:37.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:36 smithi133 bash[27558]: audit 2023-10-09T04:42:36.421554+0000 mon.a (mon.0) 718 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T04:42:37.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:36 smithi133 bash[27558]: audit 2023-10-09T04:42:36.422159+0000 mon.b (mon.1) 29 : audit [INF] from='client.? 172.21.15.194:0/2708928061' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T04:42:37.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:36 smithi133 bash[27558]: audit 2023-10-09T04:42:36.429919+0000 mon.a (mon.0) 719 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T04:42:37.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:36 smithi194 bash[31648]: cluster 2023-10-09T04:42:35.088850+0000 mgr.y (mgr.14466) 37 : cluster [DBG] pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:37.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:36 smithi194 bash[31648]: audit 2023-10-09T04:42:36.421554+0000 mon.a (mon.0) 718 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T04:42:37.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:36 smithi194 bash[31648]: audit 2023-10-09T04:42:36.422159+0000 mon.b (mon.1) 29 : audit [INF] from='client.? 172.21.15.194:0/2708928061' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T04:42:37.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:36 smithi194 bash[31648]: audit 2023-10-09T04:42:36.429919+0000 mon.a (mon.0) 719 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T04:42:38.360 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2023-10-09T04:42:38.360 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-10-09T04:42:38.360 DEBUG:teuthology.orchestra.run.smithi194:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-10-09T04:42:38.378 INFO:tasks.cephadm:Waiting for OSDs to come up 2023-10-09T04:42:38.378 DEBUG:teuthology.orchestra.run.smithi133:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-10-09T04:42:38.669 INFO:teuthology.misc.health.smithi133.stdout: 2023-10-09T04:42:38.669 INFO:teuthology.misc.health.smithi133.stdout:{"epoch":59,"fsid":"855b5206-665c-11ee-8db5-212e2dc638e7","created":"2023-10-09T04:31:08.633675+0000","modified":"2023-10-09T04:42:05.063548+0000","last_up_change":"2023-10-09T04:41:05.383200+0000","last_in_change":"2023-10-09T04:40:39.315748+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T04:36:44.260202+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"32633ec2-9b8f-42d8-bdfb-5c9668a26071","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6802","nonce":1933306305},{"type":"v1","addr":"172.21.15.133:6803","nonce":1933306305}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6804","nonce":1933306305},{"type":"v1","addr":"172.21.15.133:6805","nonce":1933306305}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6808","nonce":1933306305},{"type":"v1","addr":"172.21.15.133:6809","nonce":1933306305}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6806","nonce":1933306305},{"type":"v1","addr":"172.21.15.133:6807","nonce":1933306305}]},"public_addr":"172.21.15.133:6803/1933306305","cluster_addr":"172.21.15.133:6805/1933306305","heartbeat_back_addr":"172.21.15.133:6809/1933306305","heartbeat_front_addr":"172.21.15.133:6807/1933306305","state":["exists","up"]},{"osd":1,"uuid":"7f2a961d-a790-4337-91c7-1e45f900bbd4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6810","nonce":2965860635},{"type":"v1","addr":"172.21.15.133:6811","nonce":2965860635}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6812","nonce":2965860635},{"type":"v1","addr":"172.21.15.133:6813","nonce":2965860635}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6816","nonce":2965860635},{"type":"v1","addr":"172.21.15.133:6817","nonce":2965860635}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6814","nonce":2965860635},{"type":"v1","addr":"172.21.15.133:6815","nonce":2965860635}]},"public_addr":"172.21.15.133:6811/2965860635","cluster_addr":"172.21.15.133:6813/2965860635","heartbeat_back_addr":"172.21.15.133:6817/2965860635","heartbeat_front_addr":"172.21.15.133:6815/2965860635","state":["exists","up"]},{"osd":2,"uuid":"08751330-a327-4bf8-9c4d-9ff986b1c67f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6818","nonce":1465201170},{"type":"v1","addr":"172.21.15.133:6819","nonce":1465201170}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6820","nonce":1465201170},{"type":"v1","addr":"172.21.15.133:6821","nonce":1465201170}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6824","nonce":1465201170},{"type":"v1","addr":"172.21.15.133:6825","nonce":1465201170}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6822","nonce":1465201170},{"type":"v1","addr":"172.21.15.133:6823","nonce":1465201170}]},"public_addr":"172.21.15.133:6819/1465201170","cluster_addr":"172.21.15.133:6821/1465201170","heartbeat_back_addr":"172.21.15.133:6825/1465201170","heartbeat_front_addr":"172.21.15.133:6823/1465201170","state":["exists","up"]},{"osd":3,"uuid":"56bbf807-c8f7-4226-b46e-802b539274ca","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6826","nonce":1093481727},{"type":"v1","addr":"172.21.15.133:6827","nonce":1093481727}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6828","nonce":1093481727},{"type":"v1","addr":"172.21.15.133:6829","nonce":1093481727}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6832","nonce":1093481727},{"type":"v1","addr":"172.21.15.133:6833","nonce":1093481727}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6830","nonce":1093481727},{"type":"v1","addr":"172.21.15.133:6831","nonce":1093481727}]},"public_addr":"172.21.15.133:6827/1093481727","cluster_addr":"172.21.15.133:6829/1093481727","heartbeat_back_addr":"172.21.15.133:6833/1093481727","heartbeat_front_addr":"172.21.15.133:6831/1093481727","state":["exists","up"]},{"osd":4,"uuid":"509ec8cc-5042-428e-9a69-8e602b5afce8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6801","nonce":1874338403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6803","nonce":1874338403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6807","nonce":1874338403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6805","nonce":1874338403}]},"public_addr":"172.21.15.194:6801/1874338403","cluster_addr":"172.21.15.194:6803/1874338403","heartbeat_back_addr":"172.21.15.194:6807/1874338403","heartbeat_front_addr":"172.21.15.194:6805/1874338403","state":["exists","up"]},{"osd":5,"uuid":"bbdc1e67-fd5b-49d0-9e42-cf151d167943","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6809","nonce":3989986662}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6811","nonce":3989986662}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6815","nonce":3989986662}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6813","nonce":3989986662}]},"public_addr":"172.21.15.194:6809/3989986662","cluster_addr":"172.21.15.194:6811/3989986662","heartbeat_back_addr":"172.21.15.194:6815/3989986662","heartbeat_front_addr":"172.21.15.194:6813/3989986662","state":["exists","up"]},{"osd":6,"uuid":"7b69d014-f4c0-4e73-a14e-7d51b7ee3ea2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6817","nonce":2607555245}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6819","nonce":2607555245}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6823","nonce":2607555245}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6820","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6821","nonce":2607555245}]},"public_addr":"172.21.15.194:6817/2607555245","cluster_addr":"172.21.15.194:6819/2607555245","heartbeat_back_addr":"172.21.15.194:6823/2607555245","heartbeat_front_addr":"172.21.15.194:6821/2607555245","state":["exists","up"]},{"osd":7,"uuid":"02506268-102c-4654-9b5c-68d36c28f7f7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6824","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6825","nonce":1051622545}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6827","nonce":1051622545}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6831","nonce":1051622545}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6828","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6829","nonce":1051622545}]},"public_addr":"172.21.15.194:6825/1051622545","cluster_addr":"172.21.15.194:6827/1051622545","heartbeat_back_addr":"172.21.15.194:6831/1051622545","heartbeat_front_addr":"172.21.15.194:6829/1051622545","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:34:53.028753+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:35:47.001796+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:36:41.220545+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:37:36.237434+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:38:28.301119+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:39:20.099747+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:40:12.106681+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:41:04.212546+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.133:0/1309073801":"2023-10-10T04:42:05.063462+0000","172.21.15.133:6801/768614786":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/4290202542":"2023-10-10T04:32:17.611609+0000","172.21.15.133:6800/788799879":"2023-10-10T04:32:17.611609+0000","172.21.15.133:6800/768614786":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/320185713":"2023-10-10T04:42:05.063462+0000","172.21.15.133:6801/788799879":"2023-10-10T04:32:17.611609+0000","172.21.15.133:0/1411744140":"2023-10-10T04:31:42.639571+0000","172.21.15.133:0/55818837":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/4173745360":"2023-10-10T04:31:42.639571+0000","172.21.15.133:0/1793292086":"2023-10-10T04:32:17.611609+0000","172.21.15.133:0/3636510019":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/740467257":"2023-10-10T04:31:42.639571+0000","172.21.15.133:6800/1761810618":"2023-10-10T04:31:42.639571+0000","172.21.15.133:0/3619387632":"2023-10-10T04:32:17.611609+0000","172.21.15.133:6801/1761810618":"2023-10-10T04:31:42.639571+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T04:42:38.684 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-10-09T04:42:38.684 INFO:tasks.cephadm:Creating RBD pool 2023-10-09T04:42:38.684 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- sudo ceph --cluster ceph osd pool create rbd 8 2023-10-09T04:42:38.961 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:38 smithi133 bash[41291]: cluster 2023-10-09T04:42:37.089507+0000 mgr.y (mgr.14466) 38 : cluster [DBG] pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:38.962 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:38 smithi133 bash[27558]: cluster 2023-10-09T04:42:37.089507+0000 mgr.y (mgr.14466) 38 : cluster [DBG] pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:39.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:38 smithi194 bash[31648]: cluster 2023-10-09T04:42:37.089507+0000 mgr.y (mgr.14466) 38 : cluster [DBG] pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:39.300 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:42:38 smithi133 bash[73393]: ts=2023-10-09T04:42:38.958Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003340678s 2023-10-09T04:42:40.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:39 smithi133 bash[41291]: audit 2023-10-09T04:42:38.664810+0000 mon.a (mon.0) 720 : audit [DBG] from='client.? 172.21.15.133:0/3128917609' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:42:40.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:39 smithi133 bash[27558]: audit 2023-10-09T04:42:38.664810+0000 mon.a (mon.0) 720 : audit [DBG] from='client.? 172.21.15.133:0/3128917609' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:42:40.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:39 smithi194 bash[31648]: audit 2023-10-09T04:42:38.664810+0000 mon.a (mon.0) 720 : audit [DBG] from='client.? 172.21.15.133:0/3128917609' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:42:41.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:40 smithi133 bash[27558]: cluster 2023-10-09T04:42:39.090040+0000 mgr.y (mgr.14466) 39 : cluster [DBG] pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:41.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:40 smithi133 bash[41291]: cluster 2023-10-09T04:42:39.090040+0000 mgr.y (mgr.14466) 39 : cluster [DBG] pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:41.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:40 smithi194 bash[31648]: cluster 2023-10-09T04:42:39.090040+0000 mgr.y (mgr.14466) 39 : cluster [DBG] pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:43.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:42 smithi133 bash[41291]: cluster 2023-10-09T04:42:41.090694+0000 mgr.y (mgr.14466) 40 : cluster [DBG] pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:43.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:42 smithi133 bash[27558]: cluster 2023-10-09T04:42:41.090694+0000 mgr.y (mgr.14466) 40 : cluster [DBG] pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:43.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:42 smithi194 bash[31648]: cluster 2023-10-09T04:42:41.090694+0000 mgr.y (mgr.14466) 40 : cluster [DBG] pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:43.446 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:42:44.842 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:44 smithi133 bash[27558]: cluster 2023-10-09T04:42:43.091353+0000 mgr.y (mgr.14466) 41 : cluster [DBG] pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:44.843 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:44 smithi133 bash[41291]: cluster 2023-10-09T04:42:43.091353+0000 mgr.y (mgr.14466) 41 : cluster [DBG] pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:45.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:44 smithi194 bash[31648]: cluster 2023-10-09T04:42:43.091353+0000 mgr.y (mgr.14466) 41 : cluster [DBG] pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:46.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:42:46 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:42:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:42:47.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:46 smithi133 bash[41291]: cluster 2023-10-09T04:42:45.091872+0000 mgr.y (mgr.14466) 42 : cluster [DBG] pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:47.073 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:46 smithi133 bash[27558]: cluster 2023-10-09T04:42:45.091872+0000 mgr.y (mgr.14466) 42 : cluster [DBG] pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:47.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:46 smithi194 bash[31648]: cluster 2023-10-09T04:42:45.091872+0000 mgr.y (mgr.14466) 42 : cluster [DBG] pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:47.691 INFO:teuthology.orchestra.run.smithi133.stderr:pool 'rbd' created 2023-10-09T04:42:47.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:47 smithi194 bash[31648]: audit 2023-10-09T04:42:47.074310+0000 mon.c (mon.2) 16 : audit [INF] from='client.? 172.21.15.133:0/1864938413' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T04:42:47.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:47 smithi194 bash[31648]: audit 2023-10-09T04:42:47.074833+0000 mon.a (mon.0) 721 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T04:42:47.972 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:47 smithi133 bash[27558]: audit 2023-10-09T04:42:47.074310+0000 mon.c (mon.2) 16 : audit [INF] from='client.? 172.21.15.133:0/1864938413' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T04:42:47.973 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:47 smithi133 bash[27558]: audit 2023-10-09T04:42:47.074833+0000 mon.a (mon.0) 721 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T04:42:47.973 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:47 smithi133 bash[41291]: audit 2023-10-09T04:42:47.074310+0000 mon.c (mon.2) 16 : audit [INF] from='client.? 172.21.15.133:0/1864938413' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T04:42:47.973 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:47 smithi133 bash[41291]: audit 2023-10-09T04:42:47.074833+0000 mon.a (mon.0) 721 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T04:42:48.515 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-10-09T04:42:48.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:48 smithi133 bash[27558]: cluster 2023-10-09T04:42:47.092458+0000 mgr.y (mgr.14466) 43 : cluster [DBG] pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:48.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:48 smithi133 bash[27558]: audit 2023-10-09T04:42:47.680703+0000 mon.a (mon.0) 722 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T04:42:48.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:48 smithi133 bash[27558]: cluster 2023-10-09T04:42:47.680822+0000 mon.a (mon.0) 723 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-10-09T04:42:48.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:48 smithi133 bash[41291]: cluster 2023-10-09T04:42:47.092458+0000 mgr.y (mgr.14466) 43 : cluster [DBG] pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:48.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:48 smithi133 bash[41291]: audit 2023-10-09T04:42:47.680703+0000 mon.a (mon.0) 722 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T04:42:48.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:48 smithi133 bash[41291]: cluster 2023-10-09T04:42:47.680822+0000 mon.a (mon.0) 723 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-10-09T04:42:49.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:48 smithi194 bash[31648]: cluster 2023-10-09T04:42:47.092458+0000 mgr.y (mgr.14466) 43 : cluster [DBG] pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:49.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:48 smithi194 bash[31648]: audit 2023-10-09T04:42:47.680703+0000 mon.a (mon.0) 722 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T04:42:49.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:48 smithi194 bash[31648]: cluster 2023-10-09T04:42:47.680822+0000 mon.a (mon.0) 723 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-10-09T04:42:50.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:49 smithi133 bash[27558]: cluster 2023-10-09T04:42:48.689827+0000 mon.a (mon.0) 724 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-10-09T04:42:50.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:49 smithi133 bash[41291]: cluster 2023-10-09T04:42:48.689827+0000 mon.a (mon.0) 724 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-10-09T04:42:50.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:49 smithi194 bash[31648]: cluster 2023-10-09T04:42:48.689827+0000 mon.a (mon.0) 724 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-10-09T04:42:51.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:50 smithi133 bash[41291]: cluster 2023-10-09T04:42:49.093127+0000 mgr.y (mgr.14466) 44 : cluster [DBG] pgmap v27: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:51.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:50 smithi133 bash[41291]: cluster 2023-10-09T04:42:49.688003+0000 mon.a (mon.0) 725 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T04:42:51.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:50 smithi133 bash[41291]: cluster 2023-10-09T04:42:49.700425+0000 mon.a (mon.0) 726 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-10-09T04:42:51.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:50 smithi133 bash[41291]: audit 2023-10-09T04:42:50.190242+0000 mon.a (mon.0) 727 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:51.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:50 smithi133 bash[41291]: audit 2023-10-09T04:42:50.191496+0000 mon.a (mon.0) 728 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:51.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:50 smithi133 bash[27558]: cluster 2023-10-09T04:42:49.093127+0000 mgr.y (mgr.14466) 44 : cluster [DBG] pgmap v27: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:51.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:50 smithi133 bash[27558]: cluster 2023-10-09T04:42:49.688003+0000 mon.a (mon.0) 725 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T04:42:51.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:50 smithi133 bash[27558]: cluster 2023-10-09T04:42:49.700425+0000 mon.a (mon.0) 726 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-10-09T04:42:51.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:50 smithi133 bash[27558]: audit 2023-10-09T04:42:50.190242+0000 mon.a (mon.0) 727 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:51.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:50 smithi133 bash[27558]: audit 2023-10-09T04:42:50.191496+0000 mon.a (mon.0) 728 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:51.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:50 smithi194 bash[31648]: cluster 2023-10-09T04:42:49.093127+0000 mgr.y (mgr.14466) 44 : cluster [DBG] pgmap v27: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:51.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:50 smithi194 bash[31648]: cluster 2023-10-09T04:42:49.688003+0000 mon.a (mon.0) 725 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T04:42:51.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:50 smithi194 bash[31648]: cluster 2023-10-09T04:42:49.700425+0000 mon.a (mon.0) 726 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-10-09T04:42:51.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:50 smithi194 bash[31648]: audit 2023-10-09T04:42:50.190242+0000 mon.a (mon.0) 727 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:51.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:50 smithi194 bash[31648]: audit 2023-10-09T04:42:50.191496+0000 mon.a (mon.0) 728 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:53.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:52 smithi133 bash[27558]: cluster 2023-10-09T04:42:51.093757+0000 mgr.y (mgr.14466) 45 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:53.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:52 smithi133 bash[41291]: cluster 2023-10-09T04:42:51.093757+0000 mgr.y (mgr.14466) 45 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:53.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:52 smithi194 bash[31648]: cluster 2023-10-09T04:42:51.093757+0000 mgr.y (mgr.14466) 45 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:53.284 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:42:54.962 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:54 smithi194 bash[31648]: cluster 2023-10-09T04:42:53.094459+0000 mgr.y (mgr.14466) 46 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:55.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:54 smithi133 bash[41291]: cluster 2023-10-09T04:42:53.094459+0000 mgr.y (mgr.14466) 46 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:55.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:54 smithi133 bash[27558]: cluster 2023-10-09T04:42:53.094459+0000 mgr.y (mgr.14466) 46 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:56.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:42:56 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:42:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:42:56.721 INFO:teuthology.orchestra.run.smithi133.stderr:enabled application 'rbd' on pool 'rbd' 2023-10-09T04:42:57.004 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:56 smithi133 bash[41291]: cluster 2023-10-09T04:42:55.095194+0000 mgr.y (mgr.14466) 47 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:57.005 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:56 smithi133 bash[41291]: audit 2023-10-09T04:42:56.638955+0000 mon.c (mon.2) 17 : audit [INF] from='client.? 172.21.15.133:0/797675164' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:42:57.005 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:56 smithi133 bash[41291]: audit 2023-10-09T04:42:56.639533+0000 mon.a (mon.0) 729 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:42:57.005 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:56 smithi133 bash[27558]: cluster 2023-10-09T04:42:55.095194+0000 mgr.y (mgr.14466) 47 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:57.005 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:56 smithi133 bash[27558]: audit 2023-10-09T04:42:56.638955+0000 mon.c (mon.2) 17 : audit [INF] from='client.? 172.21.15.133:0/797675164' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:42:57.005 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:56 smithi133 bash[27558]: audit 2023-10-09T04:42:56.639533+0000 mon.a (mon.0) 729 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:42:57.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:56 smithi194 bash[31648]: cluster 2023-10-09T04:42:55.095194+0000 mgr.y (mgr.14466) 47 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:57.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:56 smithi194 bash[31648]: audit 2023-10-09T04:42:56.638955+0000 mon.c (mon.2) 17 : audit [INF] from='client.? 172.21.15.133:0/797675164' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:42:57.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:56 smithi194 bash[31648]: audit 2023-10-09T04:42:56.639533+0000 mon.a (mon.0) 729 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:42:57.630 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T04:42:57.631 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T04:42:57.631 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T04:42:58.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:57 smithi133 bash[27558]: audit 2023-10-09T04:42:56.710869+0000 mon.a (mon.0) 730 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:42:58.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:57 smithi133 bash[27558]: cluster 2023-10-09T04:42:56.711057+0000 mon.a (mon.0) 731 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-10-09T04:42:58.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:57 smithi133 bash[41291]: audit 2023-10-09T04:42:56.710869+0000 mon.a (mon.0) 730 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:42:58.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:57 smithi133 bash[41291]: cluster 2023-10-09T04:42:56.711057+0000 mon.a (mon.0) 731 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-10-09T04:42:58.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:57 smithi194 bash[31648]: audit 2023-10-09T04:42:56.710869+0000 mon.a (mon.0) 730 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:42:58.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:57 smithi194 bash[31648]: cluster 2023-10-09T04:42:56.711057+0000 mon.a (mon.0) 731 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-10-09T04:42:58.137 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:57 smithi194 systemd[1]: Started Ceph grafana.a for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:42:59.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:58 smithi133 bash[41291]: cluster 2023-10-09T04:42:57.095818+0000 mgr.y (mgr.14466) 48 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:59.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:58 smithi133 bash[41291]: cluster 2023-10-09T04:42:57.714788+0000 mon.a (mon.0) 732 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T04:42:59.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:58 smithi133 bash[41291]: cluster 2023-10-09T04:42:57.714839+0000 mon.a (mon.0) 733 : cluster [INF] Cluster is now healthy 2023-10-09T04:42:59.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:58 smithi133 bash[41291]: audit 2023-10-09T04:42:57.975727+0000 mon.a (mon.0) 734 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:59.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:58 smithi133 bash[41291]: audit 2023-10-09T04:42:57.983765+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:59.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:58 smithi133 bash[41291]: audit 2023-10-09T04:42:58.042203+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:59.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:58 smithi133 bash[41291]: audit 2023-10-09T04:42:58.047553+0000 mon.a (mon.0) 737 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:59.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:42:58 smithi133 bash[41291]: audit 2023-10-09T04:42:58.113093+0000 mon.a (mon.0) 738 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:42:59.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:58 smithi133 bash[27558]: cluster 2023-10-09T04:42:57.095818+0000 mgr.y (mgr.14466) 48 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:59.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:58 smithi133 bash[27558]: cluster 2023-10-09T04:42:57.714788+0000 mon.a (mon.0) 732 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T04:42:59.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:58 smithi133 bash[27558]: cluster 2023-10-09T04:42:57.714839+0000 mon.a (mon.0) 733 : cluster [INF] Cluster is now healthy 2023-10-09T04:42:59.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:58 smithi133 bash[27558]: audit 2023-10-09T04:42:57.975727+0000 mon.a (mon.0) 734 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:59.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:58 smithi133 bash[27558]: audit 2023-10-09T04:42:57.983765+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:59.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:58 smithi133 bash[27558]: audit 2023-10-09T04:42:58.042203+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:59.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:58 smithi133 bash[27558]: audit 2023-10-09T04:42:58.047553+0000 mon.a (mon.0) 737 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:59.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:42:58 smithi133 bash[27558]: audit 2023-10-09T04:42:58.113093+0000 mon.a (mon.0) 738 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:42:59.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:58 smithi194 bash[31648]: cluster 2023-10-09T04:42:57.095818+0000 mgr.y (mgr.14466) 48 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:42:59.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:58 smithi194 bash[31648]: cluster 2023-10-09T04:42:57.714788+0000 mon.a (mon.0) 732 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T04:42:59.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:58 smithi194 bash[31648]: cluster 2023-10-09T04:42:57.714839+0000 mon.a (mon.0) 733 : cluster [INF] Cluster is now healthy 2023-10-09T04:42:59.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:58 smithi194 bash[31648]: audit 2023-10-09T04:42:57.975727+0000 mon.a (mon.0) 734 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:59.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:58 smithi194 bash[31648]: audit 2023-10-09T04:42:57.983765+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:59.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:58 smithi194 bash[31648]: audit 2023-10-09T04:42:58.042203+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:59.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:58 smithi194 bash[31648]: audit 2023-10-09T04:42:58.047553+0000 mon.a (mon.0) 737 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:42:59.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:42:58 smithi194 bash[31648]: audit 2023-10-09T04:42:58.113093+0000 mon.a (mon.0) 738 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:42:59.395 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=settings t=2023-10-09T04:42:59.141089976Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2023-10-09T04:42:59.395 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=settings t=2023-10-09T04:42:59.141295752Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2023-10-09T04:42:59.395 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=settings t=2023-10-09T04:42:59.141318957Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2023-10-09T04:42:59.396 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=settings t=2023-10-09T04:42:59.141327582Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2023-10-09T04:42:59.396 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=settings t=2023-10-09T04:42:59.141338273Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2023-10-09T04:42:59.396 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=settings t=2023-10-09T04:42:59.141350562Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2023-10-09T04:42:59.396 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=settings t=2023-10-09T04:42:59.141361312Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-10-09T04:42:59.396 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=settings t=2023-10-09T04:42:59.141368849Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-10-09T04:42:59.396 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=settings t=2023-10-09T04:42:59.141376793Z level=info msg="Path Home" path=/usr/share/grafana 2023-10-09T04:42:59.396 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=settings t=2023-10-09T04:42:59.141384604Z level=info msg="Path Data" path=/var/lib/grafana 2023-10-09T04:42:59.396 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=settings t=2023-10-09T04:42:59.141393716Z level=info msg="Path Logs" path=/var/log/grafana 2023-10-09T04:42:59.396 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=settings t=2023-10-09T04:42:59.141412422Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2023-10-09T04:42:59.396 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=settings t=2023-10-09T04:42:59.141419957Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2023-10-09T04:42:59.396 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=settings t=2023-10-09T04:42:59.141431231Z level=info msg="App mode production" 2023-10-09T04:42:59.397 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=sqlstore t=2023-10-09T04:42:59.141496194Z level=info msg="Connecting to DB" dbtype=sqlite3 2023-10-09T04:42:59.397 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=sqlstore t=2023-10-09T04:42:59.141534379Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-10-09T04:42:59.397 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:42:59.142585022Z level=info msg="Starting DB migrations" 2023-10-09T04:42:59.397 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:42:59.142703302Z level=info msg="Executing migration" id="create migration_log table" 2023-10-09T04:42:59.397 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:42:59.175802501Z level=info msg="Executing migration" id="create user table" 2023-10-09T04:42:59.397 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:42:59.200977842Z level=info msg="Executing migration" id="add unique index user.login" 2023-10-09T04:42:59.397 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:42:59.226219391Z level=info msg="Executing migration" id="add unique index user.email" 2023-10-09T04:42:59.397 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:42:59.25129052Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2023-10-09T04:42:59.397 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:42:59.276636163Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2023-10-09T04:42:59.397 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:42:59.318561131Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2023-10-09T04:42:59.397 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:42:59.343591271Z level=info msg="Executing migration" id="create user table v2" 2023-10-09T04:42:59.398 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:42:59.368871287Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2023-10-09T04:42:59.398 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:42:59.394033807Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2023-10-09T04:42:59.673 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:42:59.419666001Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2023-10-09T04:42:59.673 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:42:59.444775116Z level=info msg="Executing migration" id="Drop old table user_v1" 2023-10-09T04:42:59.673 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:42:59.48673661Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2023-10-09T04:42:59.673 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:42:59.511980709Z level=info msg="Executing migration" id="Update user table charset" 2023-10-09T04:42:59.673 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:42:59.537200462Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2023-10-09T04:42:59.673 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:42:59.562447087Z level=info msg="Executing migration" id="Add missing user data" 2023-10-09T04:42:59.673 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:42:59.587754478Z level=info msg="Executing migration" id="Add is_disabled column to user" 2023-10-09T04:42:59.673 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:42:59.629609252Z level=info msg="Executing migration" id="Add index user.login/user.email" 2023-10-09T04:42:59.674 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:42:59.671623294Z level=info msg="Executing migration" id="Add is_service_account column to user" 2023-10-09T04:42:59.925 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:42:59.713771067Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2023-10-09T04:42:59.925 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:42:59.755851641Z level=info msg="Executing migration" id="create temp user table v1-7" 2023-10-09T04:42:59.925 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:42:59.781401531Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2023-10-09T04:42:59.926 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:42:59.806640913Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2023-10-09T04:42:59.926 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:42:59.831793074Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2023-10-09T04:42:59.926 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:42:59.857005389Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2023-10-09T04:42:59.926 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:42:59.899015383Z level=info msg="Executing migration" id="Update temp_user table charset" 2023-10-09T04:43:00.178 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:42:59.924186043Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2023-10-09T04:43:00.178 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:42:59.966024307Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2023-10-09T04:43:00.178 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:42:59 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:42:59.991460535Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2023-10-09T04:43:00.178 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.016620889Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2023-10-09T04:43:00.178 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.041901834Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2023-10-09T04:43:00.178 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.083886787Z level=info msg="Executing migration" id="create temp_user v2" 2023-10-09T04:43:00.178 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.109119953Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2023-10-09T04:43:00.179 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.151006501Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2023-10-09T04:43:00.179 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.176453336Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2023-10-09T04:43:00.455 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.218132248Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2023-10-09T04:43:00.455 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.243482451Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2023-10-09T04:43:00.455 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.268883845Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2023-10-09T04:43:00.455 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.294145823Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2023-10-09T04:43:00.456 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.335941034Z level=info msg="Executing migration" id="create star table" 2023-10-09T04:43:00.456 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.361399739Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2023-10-09T04:43:00.456 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.386882045Z level=info msg="Executing migration" id="create org table v1" 2023-10-09T04:43:00.456 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.42024053Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2023-10-09T04:43:00.726 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.453964454Z level=info msg="Executing migration" id="create org_user table v1" 2023-10-09T04:43:00.726 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.470849416Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2023-10-09T04:43:00.726 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.496109907Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2023-10-09T04:43:00.726 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.521319263Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2023-10-09T04:43:00.726 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.546526813Z level=info msg="Executing migration" id="Update org table charset" 2023-10-09T04:43:00.726 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.571656081Z level=info msg="Executing migration" id="Update org_user table charset" 2023-10-09T04:43:00.726 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.596850971Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2023-10-09T04:43:00.726 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.622200335Z level=info msg="Executing migration" id="create dashboard table" 2023-10-09T04:43:00.726 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.647398904Z level=info msg="Executing migration" id="add index dashboard.account_id" 2023-10-09T04:43:00.727 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.689371125Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2023-10-09T04:43:00.985 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[31648]: cluster 2023-10-09T04:42:59.096356+0000 mgr.y (mgr.14466) 49 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:00.985 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[31648]: audit 2023-10-09T04:43:00.204717+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:00.985 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.739905497Z level=info msg="Executing migration" id="create dashboard_tag table" 2023-10-09T04:43:00.985 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.782021535Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2023-10-09T04:43:00.985 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.807168165Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2023-10-09T04:43:00.985 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.849100027Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2023-10-09T04:43:00.985 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.890999895Z level=info msg="Executing migration" id="create dashboard v2" 2023-10-09T04:43:00.986 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.916414417Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2023-10-09T04:43:00.986 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.958315153Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2023-10-09T04:43:00.986 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:00 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:00.983577731Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2023-10-09T04:43:01.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:00 smithi133 bash[27558]: cluster 2023-10-09T04:42:59.096356+0000 mgr.y (mgr.14466) 49 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:01.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:00 smithi133 bash[27558]: audit 2023-10-09T04:43:00.204717+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:01.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:00 smithi133 bash[41291]: cluster 2023-10-09T04:42:59.096356+0000 mgr.y (mgr.14466) 49 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:01.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:00 smithi133 bash[41291]: audit 2023-10-09T04:43:00.204717+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:01.253 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:01 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:01.008760866Z level=info msg="Executing migration" id="drop table dashboard_v1" 2023-10-09T04:43:01.254 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:01 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:01.033991654Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2023-10-09T04:43:01.254 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:01 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:01.075896619Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2023-10-09T04:43:01.254 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:01 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:01.143035353Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2023-10-09T04:43:01.254 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:01 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:01.185084135Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2023-10-09T04:43:01.254 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:01 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:01.210251079Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2023-10-09T04:43:01.255 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:01 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:01.25233288Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2023-10-09T04:43:01.522 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:01 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:01.277514433Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2023-10-09T04:43:01.522 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:01 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:01.319474411Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2023-10-09T04:43:01.523 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:01 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:01.361461308Z level=info msg="Executing migration" id="Update dashboard table charset" 2023-10-09T04:43:01.523 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:01 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:01.403519922Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2023-10-09T04:43:01.523 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:01 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:01.428653294Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2023-10-09T04:43:01.523 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:01 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:01.470736511Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2023-10-09T04:43:01.523 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:01 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:01.495925221Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2023-10-09T04:43:01.523 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:01 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:01.52119098Z level=info msg="Executing migration" id="Add column uid in dashboard" 2023-10-09T04:43:01.775 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:01 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:01.5463593Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2023-10-09T04:43:01.775 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:01 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:01.571602503Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2023-10-09T04:43:01.775 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:01 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:01.596959999Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2023-10-09T04:43:01.775 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:01 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:01.638773726Z level=info msg="Executing migration" id="Update dashboard title length" 2023-10-09T04:43:01.775 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:01 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:01.680749913Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2023-10-09T04:43:01.775 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:01 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:01.722755499Z level=info msg="Executing migration" id="create dashboard_provisioning" 2023-10-09T04:43:01.775 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:01 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:01.773350506Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2023-10-09T04:43:02.043 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:01 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:01.798474221Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2023-10-09T04:43:02.044 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:01 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:01.823737115Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2023-10-09T04:43:02.044 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:01 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:01.848864475Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2023-10-09T04:43:02.044 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:01 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:01.890981384Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2023-10-09T04:43:02.044 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:01 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:01.916055829Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2023-10-09T04:43:02.044 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:01 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:01.958178223Z level=info msg="Executing migration" id="Add check_sum column" 2023-10-09T04:43:02.044 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:01 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:01.991698188Z level=info msg="Executing migration" id="Add index for dashboard_title" 2023-10-09T04:43:02.044 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:02.016895721Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2023-10-09T04:43:02.044 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:02.042094969Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2023-10-09T04:43:02.321 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:02.084014789Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2023-10-09T04:43:02.321 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:02.109370507Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2023-10-09T04:43:02.321 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:02.134695107Z level=info msg="Executing migration" id="create data_source table" 2023-10-09T04:43:02.321 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:02.159867423Z level=info msg="Executing migration" id="add index data_source.account_id" 2023-10-09T04:43:02.321 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:02.185037101Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2023-10-09T04:43:02.321 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:02.210342504Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2023-10-09T04:43:02.321 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:02.252200917Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2023-10-09T04:43:02.321 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:02.277358244Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2023-10-09T04:43:02.321 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:02.319359571Z level=info msg="Executing migration" id="create data_source table v2" 2023-10-09T04:43:02.405 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:02.581 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:02.344424807Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2023-10-09T04:43:02.582 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:02.369876566Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2023-10-09T04:43:02.582 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:02.39510013Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2023-10-09T04:43:02.582 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:02.420235429Z level=info msg="Executing migration" id="Add column with_credentials" 2023-10-09T04:43:02.582 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:02.462283803Z level=info msg="Executing migration" id="Add secure json data column" 2023-10-09T04:43:02.582 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:02.504416829Z level=info msg="Executing migration" id="Update data_source table charset" 2023-10-09T04:43:02.582 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:02.529632088Z level=info msg="Executing migration" id="Update initial version to 1" 2023-10-09T04:43:02.582 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:02.554941228Z level=info msg="Executing migration" id="Add read_only data column" 2023-10-09T04:43:02.582 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:02.580153877Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2023-10-09T04:43:02.868 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[31648]: cluster 2023-10-09T04:43:01.097043+0000 mgr.y (mgr.14466) 50 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:02.868 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:02.605356534Z level=info msg="Executing migration" id="Update json_data with nulls" 2023-10-09T04:43:02.869 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:02.647279799Z level=info msg="Executing migration" id="Add uid column" 2023-10-09T04:43:02.869 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:02.689174668Z level=info msg="Executing migration" id="Update uid value" 2023-10-09T04:43:02.869 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:02.714520108Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2023-10-09T04:43:02.869 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:02.756779708Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2023-10-09T04:43:02.869 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:02.782029274Z level=info msg="Executing migration" id="create api_key table" 2023-10-09T04:43:02.869 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:02.823715237Z level=info msg="Executing migration" id="add index api_key.account_id" 2023-10-09T04:43:02.870 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:02.866071629Z level=info msg="Executing migration" id="add index api_key.key" 2023-10-09T04:43:03.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:02 smithi133 bash[27558]: cluster 2023-10-09T04:43:01.097043+0000 mgr.y (mgr.14466) 50 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:03.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:02 smithi133 bash[41291]: cluster 2023-10-09T04:43:01.097043+0000 mgr.y (mgr.14466) 50 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:03.138 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:02.908060926Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2023-10-09T04:43:03.138 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:02.933402977Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2023-10-09T04:43:03.138 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:02 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:02.975246731Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2023-10-09T04:43:03.138 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:03 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:03.017220479Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2023-10-09T04:43:03.138 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:03 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:03.059228639Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2023-10-09T04:43:03.139 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:03 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:03.084672486Z level=info msg="Executing migration" id="create api_key table v2" 2023-10-09T04:43:03.139 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:03 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:03.109817668Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2023-10-09T04:43:03.139 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:03 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:03.13498122Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2023-10-09T04:43:03.386 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:03 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:03.16022011Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2023-10-09T04:43:03.387 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:03 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:03.185482187Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2023-10-09T04:43:03.387 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:03 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:03.210863747Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2023-10-09T04:43:03.387 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:03 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:03.235972008Z level=info msg="Executing migration" id="Update api_key table charset" 2023-10-09T04:43:03.387 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:03 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:03.277893682Z level=info msg="Executing migration" id="Add expires to api_key table" 2023-10-09T04:43:03.387 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:03 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:03.31992014Z level=info msg="Executing migration" id="Add service account foreign key" 2023-10-09T04:43:03.387 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:03 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:03.345168278Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2023-10-09T04:43:03.649 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:03 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:03.387287465Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2023-10-09T04:43:03.649 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:03 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:03.429441979Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2023-10-09T04:43:03.649 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:03 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:03.471317466Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2023-10-09T04:43:03.649 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:03 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:03.513234301Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2023-10-09T04:43:03.649 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:03 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:03.538595554Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2023-10-09T04:43:03.649 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:03 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:03.56385393Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2023-10-09T04:43:03.650 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:03 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:03.605971263Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2023-10-09T04:43:03.901 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:03 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:03.647996394Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2023-10-09T04:43:03.902 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:03 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:03.698628166Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2023-10-09T04:43:03.902 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:03 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:03.740574366Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2023-10-09T04:43:03.902 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:03 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:03.765912947Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2023-10-09T04:43:03.902 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:03 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:03.808034602Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2023-10-09T04:43:03.902 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:03 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:03.83301732Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2023-10-09T04:43:03.902 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:03 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:03.875098756Z level=info msg="Executing migration" id="create quota table v1" 2023-10-09T04:43:04.171 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:03 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:03.900482206Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2023-10-09T04:43:04.171 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:03 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:03.925704275Z level=info msg="Executing migration" id="Update quota table charset" 2023-10-09T04:43:04.171 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:03 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:03.967507103Z level=info msg="Executing migration" id="create plugin_setting table" 2023-10-09T04:43:04.171 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:03 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:03.992878044Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2023-10-09T04:43:04.171 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:04.034804887Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2023-10-09T04:43:04.171 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:04.076887027Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2023-10-09T04:43:04.171 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:04.102051221Z level=info msg="Executing migration" id="create session table" 2023-10-09T04:43:04.171 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:04.14401208Z level=info msg="Executing migration" id="Drop old table playlist table" 2023-10-09T04:43:04.171 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:04.16928318Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2023-10-09T04:43:04.473 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[31648]: cluster 2023-10-09T04:43:03.097624+0000 mgr.y (mgr.14466) 51 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:04.473 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[31648]: audit 2023-10-09T04:43:03.257945+0000 mon.a (mon.0) 740 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:04.473 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[31648]: audit 2023-10-09T04:43:03.265091+0000 mon.a (mon.0) 741 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:04.473 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[31648]: audit 2023-10-09T04:43:04.114869+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:04.473 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[31648]: audit 2023-10-09T04:43:04.123013+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:04.473 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[31648]: audit 2023-10-09T04:43:04.124484+0000 mon.a (mon.0) 744 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:43:04.473 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[31648]: audit 2023-10-09T04:43:04.125726+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:43:04.474 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[31648]: audit 2023-10-09T04:43:04.134348+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:04.474 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:04.194576687Z level=info msg="Executing migration" id="create playlist table v2" 2023-10-09T04:43:04.474 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:04.219698336Z level=info msg="Executing migration" id="create playlist item table v2" 2023-10-09T04:43:04.474 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:04.244971637Z level=info msg="Executing migration" id="Update playlist table charset" 2023-10-09T04:43:04.474 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:04.287051361Z level=info msg="Executing migration" id="Update playlist_item table charset" 2023-10-09T04:43:04.474 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:04.312203046Z level=info msg="Executing migration" id="drop preferences table v2" 2023-10-09T04:43:04.474 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:04.33737615Z level=info msg="Executing migration" id="drop preferences table v3" 2023-10-09T04:43:04.475 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:04.362790528Z level=info msg="Executing migration" id="create preferences table v3" 2023-10-09T04:43:04.475 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:04.387988484Z level=info msg="Executing migration" id="Update preferences table charset" 2023-10-09T04:43:04.475 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:04.413221307Z level=info msg="Executing migration" id="Add column team_id in preferences" 2023-10-09T04:43:04.475 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:04.471717053Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2023-10-09T04:43:04.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:04 smithi133 bash[41291]: cluster 2023-10-09T04:43:03.097624+0000 mgr.y (mgr.14466) 51 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:04.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:04 smithi133 bash[41291]: audit 2023-10-09T04:43:03.257945+0000 mon.a (mon.0) 740 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:04.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:04 smithi133 bash[41291]: audit 2023-10-09T04:43:03.265091+0000 mon.a (mon.0) 741 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:04.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:04 smithi133 bash[41291]: audit 2023-10-09T04:43:04.114869+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:04.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:04 smithi133 bash[41291]: audit 2023-10-09T04:43:04.123013+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:04.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:04 smithi133 bash[41291]: audit 2023-10-09T04:43:04.124484+0000 mon.a (mon.0) 744 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:43:04.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:04 smithi133 bash[41291]: audit 2023-10-09T04:43:04.125726+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:43:04.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:04 smithi133 bash[41291]: audit 2023-10-09T04:43:04.134348+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:04.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:04 smithi133 bash[27558]: cluster 2023-10-09T04:43:03.097624+0000 mgr.y (mgr.14466) 51 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:04.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:04 smithi133 bash[27558]: audit 2023-10-09T04:43:03.257945+0000 mon.a (mon.0) 740 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:04.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:04 smithi133 bash[27558]: audit 2023-10-09T04:43:03.265091+0000 mon.a (mon.0) 741 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:04.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:04 smithi133 bash[27558]: audit 2023-10-09T04:43:04.114869+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:04.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:04 smithi133 bash[27558]: audit 2023-10-09T04:43:04.123013+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:04.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:04 smithi133 bash[27558]: audit 2023-10-09T04:43:04.124484+0000 mon.a (mon.0) 744 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:43:04.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:04 smithi133 bash[27558]: audit 2023-10-09T04:43:04.125726+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:43:04.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:04 smithi133 bash[27558]: audit 2023-10-09T04:43:04.134348+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:04.742 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:04.513719259Z level=info msg="Executing migration" id="Add column week_start in preferences" 2023-10-09T04:43:04.742 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:04.538992307Z level=info msg="Executing migration" id="Add column preferences.json_data" 2023-10-09T04:43:04.742 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:04.564197897Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2023-10-09T04:43:04.742 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:04.589421729Z level=info msg="Executing migration" id="Add preferences index org_id" 2023-10-09T04:43:04.743 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:04.631303305Z level=info msg="Executing migration" id="Add preferences index user_id" 2023-10-09T04:43:04.743 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:04.656946228Z level=info msg="Executing migration" id="create alert table v1" 2023-10-09T04:43:04.743 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:04.698832365Z level=info msg="Executing migration" id="add index alert org_id & id " 2023-10-09T04:43:04.743 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:04.74091805Z level=info msg="Executing migration" id="add index alert state" 2023-10-09T04:43:04.996 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:04.766202996Z level=info msg="Executing migration" id="add index alert dashboard_id" 2023-10-09T04:43:04.996 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:04.792729623Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2023-10-09T04:43:04.996 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:04.818050097Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2023-10-09T04:43:04.996 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:04.859905897Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2023-10-09T04:43:04.996 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:04.902622644Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2023-10-09T04:43:04.997 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:04.927784088Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2023-10-09T04:43:04.997 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:04 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:04.99475613Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2023-10-09T04:43:05.269 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.02004549Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2023-10-09T04:43:05.270 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.061981575Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2023-10-09T04:43:05.270 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.087150698Z level=info msg="Executing migration" id="create alert_notification table v1" 2023-10-09T04:43:05.270 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.112544891Z level=info msg="Executing migration" id="Add column is_default" 2023-10-09T04:43:05.270 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.137687101Z level=info msg="Executing migration" id="Add column frequency" 2023-10-09T04:43:05.270 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.162930141Z level=info msg="Executing migration" id="Add column send_reminder" 2023-10-09T04:43:05.270 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.204813595Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2023-10-09T04:43:05.270 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.230169295Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2023-10-09T04:43:05.525 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[31648]: cephadm 2023-10-09T04:43:04.204478+0000 mgr.y (mgr.14466) 52 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T04:43:05.526 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[31648]: cephadm 2023-10-09T04:43:04.209043+0000 mgr.y (mgr.14466) 53 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi133 2023-10-09T04:43:05.526 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[31648]: audit 2023-10-09T04:43:05.191146+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:05.526 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[31648]: audit 2023-10-09T04:43:05.192337+0000 mon.a (mon.0) 748 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:05.526 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.27177905Z level=info msg="Executing migration" id="Update alert table charset" 2023-10-09T04:43:05.526 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.305786081Z level=info msg="Executing migration" id="Update alert_notification table charset" 2023-10-09T04:43:05.526 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.330983008Z level=info msg="Executing migration" id="create notification_journal table v1" 2023-10-09T04:43:05.526 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.356252169Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2023-10-09T04:43:05.527 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.381423389Z level=info msg="Executing migration" id="drop alert_notification_journal" 2023-10-09T04:43:05.527 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.40671307Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2023-10-09T04:43:05.527 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.448804827Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2023-10-09T04:43:05.527 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.473922065Z level=info msg="Executing migration" id="Add for to alert table" 2023-10-09T04:43:05.527 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.499220403Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2023-10-09T04:43:05.527 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.524490702Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2023-10-09T04:43:05.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:05 smithi133 bash[41291]: cephadm 2023-10-09T04:43:04.204478+0000 mgr.y (mgr.14466) 52 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T04:43:05.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:05 smithi133 bash[41291]: cephadm 2023-10-09T04:43:04.209043+0000 mgr.y (mgr.14466) 53 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi133 2023-10-09T04:43:05.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:05 smithi133 bash[41291]: audit 2023-10-09T04:43:05.191146+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:05.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:05 smithi133 bash[41291]: audit 2023-10-09T04:43:05.192337+0000 mon.a (mon.0) 748 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:05.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:05 smithi133 bash[27558]: cephadm 2023-10-09T04:43:04.204478+0000 mgr.y (mgr.14466) 52 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T04:43:05.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:05 smithi133 bash[27558]: cephadm 2023-10-09T04:43:04.209043+0000 mgr.y (mgr.14466) 53 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi133 2023-10-09T04:43:05.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:05 smithi133 bash[27558]: audit 2023-10-09T04:43:05.191146+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:05.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:05 smithi133 bash[27558]: audit 2023-10-09T04:43:05.192337+0000 mon.a (mon.0) 748 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:05.778 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.549796432Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2023-10-09T04:43:05.778 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.591803532Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2023-10-09T04:43:05.778 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.616987316Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2023-10-09T04:43:05.778 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.65893348Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2023-10-09T04:43:05.778 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.700970493Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2023-10-09T04:43:05.778 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.726187952Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2023-10-09T04:43:05.778 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.751469444Z level=info msg="Executing migration" id="Drop old annotation table v4" 2023-10-09T04:43:06.030 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.776707487Z level=info msg="Executing migration" id="create annotation table v5" 2023-10-09T04:43:06.030 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.818643645Z level=info msg="Executing migration" id="add index annotation 0 v3" 2023-10-09T04:43:06.030 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.84383085Z level=info msg="Executing migration" id="add index annotation 1 v3" 2023-10-09T04:43:06.030 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.885668847Z level=info msg="Executing migration" id="add index annotation 2 v3" 2023-10-09T04:43:06.030 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.911036211Z level=info msg="Executing migration" id="add index annotation 3 v3" 2023-10-09T04:43:06.030 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.952985829Z level=info msg="Executing migration" id="add index annotation 4 v3" 2023-10-09T04:43:06.031 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:05 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:05.978303904Z level=info msg="Executing migration" id="Update annotation table charset" 2023-10-09T04:43:06.031 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:06 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:06.003348827Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2023-10-09T04:43:06.031 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:06 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:06.028636533Z level=info msg="Executing migration" id="Drop category_id index" 2023-10-09T04:43:06.282 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:06 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:06.070727191Z level=info msg="Executing migration" id="Add column tags to annotation table" 2023-10-09T04:43:06.282 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:06 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:06.11269434Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2023-10-09T04:43:06.282 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:06 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:06.138002763Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2023-10-09T04:43:06.282 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:06 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:06.179872473Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2023-10-09T04:43:06.282 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:06 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:06.205097582Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2023-10-09T04:43:06.282 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:06 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:06.255251731Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2023-10-09T04:43:06.282 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:06 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:06.28060218Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2023-10-09T04:43:06.559 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:06 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:06.322554982Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2023-10-09T04:43:06.559 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:06 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:06.34777259Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2023-10-09T04:43:06.559 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:06 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:06.389778036Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2023-10-09T04:43:06.560 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:06 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:06.423647276Z level=info msg="Executing migration" id="Add created time to annotation table" 2023-10-09T04:43:06.560 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:06 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:06.465632356Z level=info msg="Executing migration" id="Add updated time to annotation table" 2023-10-09T04:43:06.560 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:06 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:06.490893194Z level=info msg="Executing migration" id="Add index for created in annotation table" 2023-10-09T04:43:06.560 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:06 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:06.516097321Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2023-10-09T04:43:06.560 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:06 smithi194 bash[31648]: cluster 2023-10-09T04:43:05.098143+0000 mgr.y (mgr.14466) 54 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:06.560 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:43:06 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:43:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:43:06.594 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T04:43:06.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:06 smithi133 bash[27558]: cluster 2023-10-09T04:43:05.098143+0000 mgr.y (mgr.14466) 54 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:06.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:06 smithi133 bash[41291]: cluster 2023-10-09T04:43:05.098143+0000 mgr.y (mgr.14466) 54 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:06.820 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:06 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:06.558219938Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2023-10-09T04:43:06.820 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:06 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:06.583355479Z level=info msg="Executing migration" id="Add epoch_end column" 2023-10-09T04:43:06.820 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:06 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:06.608528537Z level=info msg="Executing migration" id="Add index for epoch_end" 2023-10-09T04:43:06.820 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:06 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:06.650411344Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2023-10-09T04:43:06.820 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:06 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:06.675887805Z level=info msg="Executing migration" id="Move region to single row" 2023-10-09T04:43:06.820 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:06 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:06.701091074Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2023-10-09T04:43:06.820 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:06 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:06.743043312Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2023-10-09T04:43:06.820 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:06 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:06.76833095Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2023-10-09T04:43:06.821 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:06 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:06.818570039Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2023-10-09T04:43:07.072 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:06 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:06.860450521Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2023-10-09T04:43:07.072 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:06 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:06.910687748Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2023-10-09T04:43:07.072 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:06 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:06.952677168Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2023-10-09T04:43:07.072 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:06 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:06.978139901Z level=info msg="Executing migration" id="create test_data table" 2023-10-09T04:43:07.072 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.003456309Z level=info msg="Executing migration" id="create dashboard_version table v1" 2023-10-09T04:43:07.072 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.028600125Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2023-10-09T04:43:07.342 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.070752111Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2023-10-09T04:43:07.342 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.095941791Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2023-10-09T04:43:07.342 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.121156598Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2023-10-09T04:43:07.342 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.163072989Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2023-10-09T04:43:07.342 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.188384643Z level=info msg="Executing migration" id="create team table" 2023-10-09T04:43:07.342 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.231133349Z level=info msg="Executing migration" id="add index team.org_id" 2023-10-09T04:43:07.342 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.256343699Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2023-10-09T04:43:07.342 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.281513764Z level=info msg="Executing migration" id="create team member table" 2023-10-09T04:43:07.343 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.306759471Z level=info msg="Executing migration" id="add index team_member.org_id" 2023-10-09T04:43:07.602 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[31648]: audit 2023-10-09T04:43:06.583655+0000 mon.a (mon.0) 749 : audit [DBG] from='client.? 172.21.15.133:0/1467461648' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T04:43:07.603 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.340641538Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2023-10-09T04:43:07.603 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.365812345Z level=info msg="Executing migration" id="add index team_member.team_id" 2023-10-09T04:43:07.603 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.391065629Z level=info msg="Executing migration" id="Add column email to team table" 2023-10-09T04:43:07.603 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.416234227Z level=info msg="Executing migration" id="Add column external to team_member table" 2023-10-09T04:43:07.603 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.458214987Z level=info msg="Executing migration" id="Add column permission to team_member table" 2023-10-09T04:43:07.603 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.483521814Z level=info msg="Executing migration" id="create dashboard acl table" 2023-10-09T04:43:07.603 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.508775283Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2023-10-09T04:43:07.603 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.533990459Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2023-10-09T04:43:07.603 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.576002644Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2023-10-09T04:43:07.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:07 smithi133 bash[27558]: audit 2023-10-09T04:43:06.583655+0000 mon.a (mon.0) 749 : audit [DBG] from='client.? 172.21.15.133:0/1467461648' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T04:43:07.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:07 smithi133 bash[41291]: audit 2023-10-09T04:43:06.583655+0000 mon.a (mon.0) 749 : audit [DBG] from='client.? 172.21.15.133:0/1467461648' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T04:43:07.854 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.601258053Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2023-10-09T04:43:07.855 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.64323857Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2023-10-09T04:43:07.855 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.668545254Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2023-10-09T04:43:07.855 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.71049611Z level=info msg="Executing migration" id="add index dashboard_permission" 2023-10-09T04:43:07.855 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.73573332Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2023-10-09T04:43:07.855 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.760964449Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2023-10-09T04:43:07.855 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.802905907Z level=info msg="Executing migration" id="create tag table" 2023-10-09T04:43:07.855 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.828206054Z level=info msg="Executing migration" id="add index tag.key_value" 2023-10-09T04:43:07.855 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.853367358Z level=info msg="Executing migration" id="create login attempt table" 2023-10-09T04:43:08.110 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.895356824Z level=info msg="Executing migration" id="add index login_attempt.username" 2023-10-09T04:43:08.111 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.920562863Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2023-10-09T04:43:08.111 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.962543888Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2023-10-09T04:43:08.111 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:07 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:07.996640712Z level=info msg="Executing migration" id="create login_attempt v2" 2023-10-09T04:43:08.111 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:08.024262812Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2023-10-09T04:43:08.111 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:08.066227713Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2023-10-09T04:43:08.111 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:08.109418721Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2023-10-09T04:43:08.357 INFO:teuthology.orchestra.run.smithi133.stdout:{"epoch":19,"active_gid":14466,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6800","nonce":275314455},{"type":"v1","addr":"172.21.15.133:6801","nonce":275314455}]},"active_addr":"172.21.15.133:6801/275314455","active_change":"2023-10-09T04:42:05.063746+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24358,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.133:8443/","prometheus":"http://172.21.15.133:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":59,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.133:0","nonce":2159982120}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.133:0","nonce":1787141186}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.133:0","nonce":1575170036}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.133:0","nonce":2419271305}]}]} 2023-10-09T04:43:08.362 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T04:43:08.362 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T04:43:08.362 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T04:43:08.385 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:08.134707455Z level=info msg="Executing migration" id="create user auth table" 2023-10-09T04:43:08.385 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:08.161629594Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2023-10-09T04:43:08.386 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:08.203607334Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2023-10-09T04:43:08.386 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:08.228773768Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2023-10-09T04:43:08.386 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:08.24746601Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2023-10-09T04:43:08.386 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:08.272760892Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2023-10-09T04:43:08.386 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:08.297973944Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2023-10-09T04:43:08.386 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:08.333182212Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2023-10-09T04:43:08.386 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:08.358419058Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2023-10-09T04:43:08.386 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:08.3842704Z level=info msg="Executing migration" id="create server_lock table" 2023-10-09T04:43:08.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[31648]: cluster 2023-10-09T04:43:07.098958+0000 mgr.y (mgr.14466) 55 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:08.637 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:08.409522759Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2023-10-09T04:43:08.637 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:08.434765184Z level=info msg="Executing migration" id="create user auth token table" 2023-10-09T04:43:08.637 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:08.478048883Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2023-10-09T04:43:08.637 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:08.503224081Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2023-10-09T04:43:08.638 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:08.530317428Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2023-10-09T04:43:08.638 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:08.555682338Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2023-10-09T04:43:08.638 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:08.591334997Z level=info msg="Executing migration" id="create cache_data table" 2023-10-09T04:43:08.638 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:08.616645361Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2023-10-09T04:43:08.680 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:08 smithi133 bash[27558]: cluster 2023-10-09T04:43:07.098958+0000 mgr.y (mgr.14466) 55 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:08.708 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:08 smithi133 bash[41291]: cluster 2023-10-09T04:43:07.098958+0000 mgr.y (mgr.14466) 55 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:08.914 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:08.659112127Z level=info msg="Executing migration" id="create short_url table v1" 2023-10-09T04:43:08.914 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:08.685759902Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2023-10-09T04:43:08.914 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:08.71101121Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2023-10-09T04:43:08.914 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:08.753477556Z level=info msg="Executing migration" id="delete alert_definition table" 2023-10-09T04:43:08.914 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:08.820523931Z level=info msg="Executing migration" id="recreate alert_definition table" 2023-10-09T04:43:08.914 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:08.845820666Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2023-10-09T04:43:08.915 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:08.871047169Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2023-10-09T04:43:08.915 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:08.912980569Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2023-10-09T04:43:09.166 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:08.954822616Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2023-10-09T04:43:09.167 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:08 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:08.980302523Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2023-10-09T04:43:09.167 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.00547725Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2023-10-09T04:43:09.167 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.030746584Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2023-10-09T04:43:09.167 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.072631116Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2023-10-09T04:43:09.167 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.097890091Z level=info msg="Executing migration" id="drop alert_definition table" 2023-10-09T04:43:09.167 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.12296333Z level=info msg="Executing migration" id="delete alert_definition_version table" 2023-10-09T04:43:09.167 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.165100783Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2023-10-09T04:43:09.427 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.190375648Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2023-10-09T04:43:09.427 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.215603277Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2023-10-09T04:43:09.427 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.240824986Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2023-10-09T04:43:09.427 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.274622624Z level=info msg="Executing migration" id="drop alert_definition_version table" 2023-10-09T04:43:09.427 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.299912446Z level=info msg="Executing migration" id="create alert_instance table" 2023-10-09T04:43:09.428 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.325137185Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2023-10-09T04:43:09.428 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.350340624Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2023-10-09T04:43:09.428 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.375578656Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2023-10-09T04:43:09.428 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.400771736Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2023-10-09T04:43:09.428 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.426055324Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2023-10-09T04:43:09.713 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.467989008Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2023-10-09T04:43:09.713 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.518189821Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2023-10-09T04:43:09.713 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.551831947Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2023-10-09T04:43:09.713 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.577080669Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2023-10-09T04:43:09.713 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.618972377Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2023-10-09T04:43:09.713 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.644295294Z level=info msg="Executing migration" id="create alert_rule table" 2023-10-09T04:43:09.713 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.669548019Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2023-10-09T04:43:09.714 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.711390992Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2023-10-09T04:43:09.964 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.753468776Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2023-10-09T04:43:09.965 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.79525868Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2023-10-09T04:43:09.965 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.820614886Z level=info msg="Executing migration" id="add column for to alert_rule" 2023-10-09T04:43:09.965 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.854138841Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2023-10-09T04:43:09.965 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.896110054Z level=info msg="Executing migration" id="add column labels to alert_rule" 2023-10-09T04:43:09.965 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.921311813Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2023-10-09T04:43:10.050 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:43:09 smithi133 systemd[1]: Stopping Ceph alertmanager.a for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T04:43:10.050 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:43:09 smithi133 bash[73393]: ts=2023-10-09T04:43:09.926Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-10-09T04:43:10.242 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.963317352Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2023-10-09T04:43:10.242 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:09 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:09.988543102Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2023-10-09T04:43:10.242 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:10.013853649Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2023-10-09T04:43:10.242 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:10.05579416Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2023-10-09T04:43:10.242 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:10.081061875Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2023-10-09T04:43:10.242 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:10.106243281Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2023-10-09T04:43:10.242 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:10.131385266Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2023-10-09T04:43:10.242 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:10.173425054Z level=info msg="Executing migration" id="create alert_rule_version table" 2023-10-09T04:43:10.243 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:10.198706985Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2023-10-09T04:43:10.503 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:10.240641159Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2023-10-09T04:43:10.503 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:10.265836279Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2023-10-09T04:43:10.503 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:10.291210641Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2023-10-09T04:43:10.503 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:10.316508922Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2023-10-09T04:43:10.503 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:10.341764267Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2023-10-09T04:43:10.503 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:10.38374511Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2023-10-09T04:43:10.503 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:10.409013118Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2023-10-09T04:43:10.503 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:10.450811861Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2023-10-09T04:43:10.504 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:10.476364677Z level=info msg="Executing migration" id=create_alert_configuration_table 2023-10-09T04:43:10.504 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[31648]: cluster 2023-10-09T04:43:09.099497+0000 mgr.y (mgr.14466) 56 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:10.504 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[31648]: audit 2023-10-09T04:43:10.212841+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:10.745 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:10 smithi133 bash[27558]: cluster 2023-10-09T04:43:09.099497+0000 mgr.y (mgr.14466) 56 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:10.745 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:10 smithi133 bash[27558]: audit 2023-10-09T04:43:10.212841+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:10.745 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:43:10 smithi133 bash[75020]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T04:43:10.745 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:43:10 smithi133 bash[75083]: Error response from daemon: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T04:43:10.745 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:43:10 smithi133 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T04:43:10.745 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:43:10 smithi133 systemd[1]: Stopped Ceph alertmanager.a for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:43:10.746 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:43:10 smithi133 systemd[1]: Started Ceph alertmanager.a for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:43:10.746 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:10 smithi133 bash[41291]: cluster 2023-10-09T04:43:09.099497+0000 mgr.y (mgr.14466) 56 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:10.746 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:10 smithi133 bash[41291]: audit 2023-10-09T04:43:10.212841+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:10.755 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:10.501561437Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2023-10-09T04:43:10.755 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:10.526863661Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2023-10-09T04:43:10.755 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:10.551970313Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2023-10-09T04:43:10.755 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:10.577297483Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2023-10-09T04:43:10.755 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:10.619044658Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2023-10-09T04:43:10.755 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:10.6610714Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2023-10-09T04:43:10.755 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:10.686543084Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2023-10-09T04:43:10.755 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:10.7117481Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2023-10-09T04:43:10.755 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:10.753642499Z level=info msg="Executing migration" id="create provenance_type table" 2023-10-09T04:43:11.099 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:10.778932409Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2023-10-09T04:43:11.099 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:10.821041992Z level=info msg="Executing migration" id="create alert_image table" 2023-10-09T04:43:11.099 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:10.84617405Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2023-10-09T04:43:11.099 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:10.888029362Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2023-10-09T04:43:11.099 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:10.913268646Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2023-10-09T04:43:11.100 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:10 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:10.963501352Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2023-10-09T04:43:11.283 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:11 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:11.097982192Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2023-10-09T04:43:11.284 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:11 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:11.098232423Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2023-10-09T04:43:11.284 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:11 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:11.183915284Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2023-10-09T04:43:11.284 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:11 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:11.238279521Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2023-10-09T04:43:11.387 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:11 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:11.282580688Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2023-10-09T04:43:11.387 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:11 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:11.2830212Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2023-10-09T04:43:11.387 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:11 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:11.283337073Z level=info msg="alerts found to migrate" alerts=0 2023-10-09T04:43:11.720 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:11 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:11.422099053Z level=info msg="Executing migration" id="create library_element table v1" 2023-10-09T04:43:12.037 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:11 smithi194 bash[31648]: audit 2023-10-09T04:43:10.718591+0000 mon.a (mon.0) 751 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:12.037 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:11 smithi194 bash[31648]: audit 2023-10-09T04:43:10.724198+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:12.038 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:11 smithi194 bash[31648]: cephadm 2023-10-09T04:43:10.742592+0000 mgr.y (mgr.14466) 57 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T04:43:12.038 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:11 smithi194 bash[31648]: cephadm 2023-10-09T04:43:10.754620+0000 mgr.y (mgr.14466) 58 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi194 2023-10-09T04:43:12.038 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:11 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:11.867921022Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2023-10-09T04:43:12.038 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:12 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:12.036262968Z level=info msg="Executing migration" id="create library_element_connection table v1" 2023-10-09T04:43:12.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:11 smithi133 bash[41291]: audit 2023-10-09T04:43:10.718591+0000 mon.a (mon.0) 751 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:12.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:11 smithi133 bash[41291]: audit 2023-10-09T04:43:10.724198+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:12.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:11 smithi133 bash[41291]: cephadm 2023-10-09T04:43:10.742592+0000 mgr.y (mgr.14466) 57 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T04:43:12.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:11 smithi133 bash[41291]: cephadm 2023-10-09T04:43:10.754620+0000 mgr.y (mgr.14466) 58 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi194 2023-10-09T04:43:12.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:11 smithi133 bash[27558]: audit 2023-10-09T04:43:10.718591+0000 mon.a (mon.0) 751 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:12.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:11 smithi133 bash[27558]: audit 2023-10-09T04:43:10.724198+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:12.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:11 smithi133 bash[27558]: cephadm 2023-10-09T04:43:10.742592+0000 mgr.y (mgr.14466) 57 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T04:43:12.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:11 smithi133 bash[27558]: cephadm 2023-10-09T04:43:10.754620+0000 mgr.y (mgr.14466) 58 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi194 2023-10-09T04:43:12.051 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:43:11 smithi133 bash[75109]: ts=2023-10-09T04:43:11.886Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-10-09T04:43:12.051 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:43:11 smithi133 bash[75109]: ts=2023-10-09T04:43:11.886Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-10-09T04:43:12.052 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:43:11 smithi133 bash[75109]: ts=2023-10-09T04:43:11.888Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.133 port=9094 2023-10-09T04:43:12.052 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:43:11 smithi133 bash[75109]: ts=2023-10-09T04:43:11.889Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T04:43:12.052 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:43:11 smithi133 bash[75109]: ts=2023-10-09T04:43:11.933Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T04:43:12.052 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:43:11 smithi133 bash[75109]: ts=2023-10-09T04:43:11.934Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T04:43:12.052 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:43:11 smithi133 bash[75109]: ts=2023-10-09T04:43:11.939Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-10-09T04:43:12.052 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:43:11 smithi133 bash[75109]: ts=2023-10-09T04:43:11.939Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-10-09T04:43:12.637 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:12 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:12.344068208Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2023-10-09T04:43:12.637 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:12 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:12.507400343Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2023-10-09T04:43:13.024 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:12 smithi194 bash[31648]: cluster 2023-10-09T04:43:11.100131+0000 mgr.y (mgr.14466) 59 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:13.025 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:12 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:12.695988664Z level=info msg="Executing migration" id="increase max description length to 2048" 2023-10-09T04:43:13.025 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:12 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:12.755031823Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2023-10-09T04:43:13.025 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:12 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:12.850911153Z level=info msg="Executing migration" id="create data_keys table" 2023-10-09T04:43:13.025 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:12 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:12.934195816Z level=info msg="Executing migration" id="create secrets table" 2023-10-09T04:43:13.025 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:13 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:13.023351464Z level=info msg="Executing migration" id="rename data_keys name column to id" 2023-10-09T04:43:13.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:12 smithi133 bash[41291]: cluster 2023-10-09T04:43:11.100131+0000 mgr.y (mgr.14466) 59 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:13.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:12 smithi133 bash[27558]: cluster 2023-10-09T04:43:11.100131+0000 mgr.y (mgr.14466) 59 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:13.276 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:13 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:13.115378343Z level=info msg="Executing migration" id="add name column into data_keys" 2023-10-09T04:43:13.277 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:13 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:13.165875129Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2023-10-09T04:43:13.277 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:13 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:13.21664298Z level=info msg="Executing migration" id="rename data_keys name column to label" 2023-10-09T04:43:13.622 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:13 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:13.275343347Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2023-10-09T04:43:13.622 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:13 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:13.334261546Z level=info msg="Executing migration" id="create kv_store table v1" 2023-10-09T04:43:13.622 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:13 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:13.35941439Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2023-10-09T04:43:13.622 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:13 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:13.42514917Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2023-10-09T04:43:13.622 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:13 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:13.520806575Z level=info msg="Executing migration" id="create permission table" 2023-10-09T04:43:13.887 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:13 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:13.620663385Z level=info msg="Executing migration" id="add unique index permission.role_id" 2023-10-09T04:43:13.887 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:13 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:13.813418156Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2023-10-09T04:43:14.203 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:14.232 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:43:13 smithi133 bash[75109]: ts=2023-10-09T04:43:13.890Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000335355s 2023-10-09T04:43:14.387 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:14 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:14.132514655Z level=info msg="Executing migration" id="create role table" 2023-10-09T04:43:14.387 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:14 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:14.254257531Z level=info msg="Executing migration" id="add column display_name" 2023-10-09T04:43:14.387 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:14 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:14.329707629Z level=info msg="Executing migration" id="add column group_name" 2023-10-09T04:43:14.732 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:14 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:14.405725548Z level=info msg="Executing migration" id="add index role.org_id" 2023-10-09T04:43:14.732 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:14 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:14.51323036Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2023-10-09T04:43:14.844 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:14 smithi194 bash[31648]: cluster 2023-10-09T04:43:13.100710+0000 mgr.y (mgr.14466) 60 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:14.986 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:14 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:14.843715532Z level=info msg="Executing migration" id="add index role_org_id_uid" 2023-10-09T04:43:15.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:14 smithi133 bash[41291]: cluster 2023-10-09T04:43:13.100710+0000 mgr.y (mgr.14466) 60 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:15.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:14 smithi133 bash[27558]: cluster 2023-10-09T04:43:13.100710+0000 mgr.y (mgr.14466) 60 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:15.349 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:14 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:14.985443702Z level=info msg="Executing migration" id="create team role table" 2023-10-09T04:43:15.349 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:15 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:15.34778075Z level=info msg="Executing migration" id="add index team_role.org_id" 2023-10-09T04:43:15.612 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:15 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:15.543513432Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2023-10-09T04:43:15.612 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:15 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:15.610583679Z level=info msg="Executing migration" id="add index team_role.team_id" 2023-10-09T04:43:15.887 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:15 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:15.6861264Z level=info msg="Executing migration" id="create user role table" 2023-10-09T04:43:15.887 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:15 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:15.794969712Z level=info msg="Executing migration" id="add index user_role.org_id" 2023-10-09T04:43:16.183 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:15 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:15.895126873Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2023-10-09T04:43:16.183 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:15 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:15.98301595Z level=info msg="Executing migration" id="add index user_role.user_id" 2023-10-09T04:43:16.183 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:16 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:16.04757726Z level=info msg="Executing migration" id="create builtin role table" 2023-10-09T04:43:16.183 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:16 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:16.096995203Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2023-10-09T04:43:16.183 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:16 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:16.138957714Z level=info msg="Executing migration" id="add index builtin_role.name" 2023-10-09T04:43:16.455 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:16 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:16.180917652Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2023-10-09T04:43:16.455 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:16 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:16.256793131Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2023-10-09T04:43:16.455 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:16 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:16.282121039Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2023-10-09T04:43:16.455 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:16 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:16.349676273Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2023-10-09T04:43:16.455 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:16 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:16.453749681Z level=info msg="Executing migration" id="add unique index role.uid" 2023-10-09T04:43:16.455 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:43:16 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:43:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:43:16.736 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:16 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:16.578624624Z level=info msg="Executing migration" id="create seed assignment table" 2023-10-09T04:43:17.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:16 smithi133 bash[41291]: cluster 2023-10-09T04:43:15.101243+0000 mgr.y (mgr.14466) 61 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:17.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:16 smithi133 bash[27558]: cluster 2023-10-09T04:43:15.101243+0000 mgr.y (mgr.14466) 61 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:17.114 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:16 smithi194 bash[31648]: cluster 2023-10-09T04:43:15.101243+0000 mgr.y (mgr.14466) 61 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:17.114 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:16 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:16.809758013Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2023-10-09T04:43:17.387 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:17 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:17.112838555Z level=info msg="Executing migration" id="add column hidden to role table" 2023-10-09T04:43:17.387 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:17 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:17.268911964Z level=info msg="Executing migration" id="create query_history table v1" 2023-10-09T04:43:17.387 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:17 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:17.352876737Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2023-10-09T04:43:17.702 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:17 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:17.445541798Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2023-10-09T04:43:17.702 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:17 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:17.487497106Z level=info msg="Executing migration" id="teams permissions migration" 2023-10-09T04:43:17.702 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:17 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:17.529344781Z level=info msg="Executing migration" id="dashboard permissions" 2023-10-09T04:43:17.702 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:17 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:17.571617694Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2023-10-09T04:43:17.702 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:17 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:17.638843313Z level=info msg="Executing migration" id="drop managed folder create actions" 2023-10-09T04:43:17.702 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:17 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:17.664310924Z level=info msg="Executing migration" id="alerting notification permissions" 2023-10-09T04:43:17.703 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:17 smithi194 systemd[1]: Stopping Ceph prometheus.a for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T04:43:17.703 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:17 smithi194 bash[65289]: ts=2023-10-09T04:43:17.635Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T04:43:17.703 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:17 smithi194 bash[65289]: ts=2023-10-09T04:43:17.635Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-10-09T04:43:17.703 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:17 smithi194 bash[65289]: ts=2023-10-09T04:43:17.635Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-10-09T04:43:17.703 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:17 smithi194 bash[65289]: ts=2023-10-09T04:43:17.635Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T04:43:17.703 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:17 smithi194 bash[65289]: ts=2023-10-09T04:43:17.635Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T04:43:17.703 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:17 smithi194 bash[65289]: ts=2023-10-09T04:43:17.635Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-10-09T04:43:17.703 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:17 smithi194 bash[65289]: ts=2023-10-09T04:43:17.635Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-10-09T04:43:17.704 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:17 smithi194 bash[65289]: ts=2023-10-09T04:43:17.635Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-10-09T04:43:17.704 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:17 smithi194 bash[65289]: ts=2023-10-09T04:43:17.635Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-10-09T04:43:17.704 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:17 smithi194 bash[65289]: ts=2023-10-09T04:43:17.636Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T04:43:17.704 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:17 smithi194 bash[65289]: ts=2023-10-09T04:43:17.636Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-10-09T04:43:17.704 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:17 smithi194 bash[65289]: ts=2023-10-09T04:43:17.636Z caller=main.go:1147 level=info msg="See you next time!" 2023-10-09T04:43:17.796 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T04:43:17.796 INFO:teuthology.orchestra.run.smithi133.stdout:{"epoch":63,"fsid":"855b5206-665c-11ee-8db5-212e2dc638e7","created":"2023-10-09T04:31:08.633675+0000","modified":"2023-10-09T04:42:56.703971+0000","last_up_change":"2023-10-09T04:41:05.383200+0000","last_in_change":"2023-10-09T04:40:39.315748+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T04:36:44.260202+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T04:42:47.518485+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"32633ec2-9b8f-42d8-bdfb-5c9668a26071","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6802","nonce":1933306305},{"type":"v1","addr":"172.21.15.133:6803","nonce":1933306305}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6804","nonce":1933306305},{"type":"v1","addr":"172.21.15.133:6805","nonce":1933306305}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6808","nonce":1933306305},{"type":"v1","addr":"172.21.15.133:6809","nonce":1933306305}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6806","nonce":1933306305},{"type":"v1","addr":"172.21.15.133:6807","nonce":1933306305}]},"public_addr":"172.21.15.133:6803/1933306305","cluster_addr":"172.21.15.133:6805/1933306305","heartbeat_back_addr":"172.21.15.133:6809/1933306305","heartbeat_front_addr":"172.21.15.133:6807/1933306305","state":["exists","up"]},{"osd":1,"uuid":"7f2a961d-a790-4337-91c7-1e45f900bbd4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6810","nonce":2965860635},{"type":"v1","addr":"172.21.15.133:6811","nonce":2965860635}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6812","nonce":2965860635},{"type":"v1","addr":"172.21.15.133:6813","nonce":2965860635}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6816","nonce":2965860635},{"type":"v1","addr":"172.21.15.133:6817","nonce":2965860635}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6814","nonce":2965860635},{"type":"v1","addr":"172.21.15.133:6815","nonce":2965860635}]},"public_addr":"172.21.15.133:6811/2965860635","cluster_addr":"172.21.15.133:6813/2965860635","heartbeat_back_addr":"172.21.15.133:6817/2965860635","heartbeat_front_addr":"172.21.15.133:6815/2965860635","state":["exists","up"]},{"osd":2,"uuid":"08751330-a327-4bf8-9c4d-9ff986b1c67f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6818","nonce":1465201170},{"type":"v1","addr":"172.21.15.133:6819","nonce":1465201170}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6820","nonce":1465201170},{"type":"v1","addr":"172.21.15.133:6821","nonce":1465201170}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6824","nonce":1465201170},{"type":"v1","addr":"172.21.15.133:6825","nonce":1465201170}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6822","nonce":1465201170},{"type":"v1","addr":"172.21.15.133:6823","nonce":1465201170}]},"public_addr":"172.21.15.133:6819/1465201170","cluster_addr":"172.21.15.133:6821/1465201170","heartbeat_back_addr":"172.21.15.133:6825/1465201170","heartbeat_front_addr":"172.21.15.133:6823/1465201170","state":["exists","up"]},{"osd":3,"uuid":"56bbf807-c8f7-4226-b46e-802b539274ca","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6826","nonce":1093481727},{"type":"v1","addr":"172.21.15.133:6827","nonce":1093481727}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6828","nonce":1093481727},{"type":"v1","addr":"172.21.15.133:6829","nonce":1093481727}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6832","nonce":1093481727},{"type":"v1","addr":"172.21.15.133:6833","nonce":1093481727}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6830","nonce":1093481727},{"type":"v1","addr":"172.21.15.133:6831","nonce":1093481727}]},"public_addr":"172.21.15.133:6827/1093481727","cluster_addr":"172.21.15.133:6829/1093481727","heartbeat_back_addr":"172.21.15.133:6833/1093481727","heartbeat_front_addr":"172.21.15.133:6831/1093481727","state":["exists","up"]},{"osd":4,"uuid":"509ec8cc-5042-428e-9a69-8e602b5afce8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6801","nonce":1874338403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6803","nonce":1874338403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6807","nonce":1874338403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6805","nonce":1874338403}]},"public_addr":"172.21.15.194:6801/1874338403","cluster_addr":"172.21.15.194:6803/1874338403","heartbeat_back_addr":"172.21.15.194:6807/1874338403","heartbeat_front_addr":"172.21.15.194:6805/1874338403","state":["exists","up"]},{"osd":5,"uuid":"bbdc1e67-fd5b-49d0-9e42-cf151d167943","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6809","nonce":3989986662}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6811","nonce":3989986662}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6815","nonce":3989986662}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6813","nonce":3989986662}]},"public_addr":"172.21.15.194:6809/3989986662","cluster_addr":"172.21.15.194:6811/3989986662","heartbeat_back_addr":"172.21.15.194:6815/3989986662","heartbeat_front_addr":"172.21.15.194:6813/3989986662","state":["exists","up"]},{"osd":6,"uuid":"7b69d014-f4c0-4e73-a14e-7d51b7ee3ea2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6817","nonce":2607555245}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6819","nonce":2607555245}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6823","nonce":2607555245}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6820","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6821","nonce":2607555245}]},"public_addr":"172.21.15.194:6817/2607555245","cluster_addr":"172.21.15.194:6819/2607555245","heartbeat_back_addr":"172.21.15.194:6823/2607555245","heartbeat_front_addr":"172.21.15.194:6821/2607555245","state":["exists","up"]},{"osd":7,"uuid":"02506268-102c-4654-9b5c-68d36c28f7f7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6824","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6825","nonce":1051622545}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6827","nonce":1051622545}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6831","nonce":1051622545}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6828","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6829","nonce":1051622545}]},"public_addr":"172.21.15.194:6825/1051622545","cluster_addr":"172.21.15.194:6827/1051622545","heartbeat_back_addr":"172.21.15.194:6831/1051622545","heartbeat_front_addr":"172.21.15.194:6829/1051622545","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:34:53.028753+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:35:47.001796+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:36:41.220545+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:37:36.237434+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:38:28.301119+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:39:20.099747+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:40:12.106681+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:41:04.212546+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.133:0/1309073801":"2023-10-10T04:42:05.063462+0000","172.21.15.133:6801/768614786":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/4290202542":"2023-10-10T04:32:17.611609+0000","172.21.15.133:6800/788799879":"2023-10-10T04:32:17.611609+0000","172.21.15.133:6800/768614786":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/320185713":"2023-10-10T04:42:05.063462+0000","172.21.15.133:6801/788799879":"2023-10-10T04:32:17.611609+0000","172.21.15.133:0/1411744140":"2023-10-10T04:31:42.639571+0000","172.21.15.133:0/55818837":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/4173745360":"2023-10-10T04:31:42.639571+0000","172.21.15.133:0/1793292086":"2023-10-10T04:32:17.611609+0000","172.21.15.133:0/3636510019":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/740467257":"2023-10-10T04:31:42.639571+0000","172.21.15.133:6800/1761810618":"2023-10-10T04:31:42.639571+0000","172.21.15.133:0/3619387632":"2023-10-10T04:32:17.611609+0000","172.21.15.133:6801/1761810618":"2023-10-10T04:31:42.639571+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T04:43:17.978 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:17 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:17.706822518Z level=info msg="Executing migration" id="create query_history_star table v1" 2023-10-09T04:43:17.979 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:17 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:17.799129008Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2023-10-09T04:43:17.979 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:17 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:17.902906515Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2023-10-09T04:43:18.310 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:18 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:18.0635285Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2023-10-09T04:43:18.535 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T04:43:18.535 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T04:43:18.580 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:18 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:18.308034411Z level=info msg="Executing migration" id="create correlation table v1" 2023-10-09T04:43:18.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:18 smithi133 bash[27558]: cluster 2023-10-09T04:43:17.101852+0000 mgr.y (mgr.14466) 62 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:18.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:18 smithi133 bash[27558]: audit 2023-10-09T04:43:17.791878+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.133:0/1776316794' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:43:18.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:18 smithi133 bash[41291]: cluster 2023-10-09T04:43:17.101852+0000 mgr.y (mgr.14466) 62 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:18.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:18 smithi133 bash[41291]: audit 2023-10-09T04:43:17.791878+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.133:0/1776316794' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:43:18.841 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:18 smithi194 bash[67777]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T04:43:18.841 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:18 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:18.578836238Z level=info msg="Executing migration" id="add index correlations.uid" 2023-10-09T04:43:18.841 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:18 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:18.710198013Z level=info msg="Executing migration" id="add index correlations.source_uid" 2023-10-09T04:43:18.841 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:18 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:18.787075007Z level=info msg="Executing migration" id="add correlation config column" 2023-10-09T04:43:18.842 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:18 smithi194 bash[31648]: cluster 2023-10-09T04:43:17.101852+0000 mgr.y (mgr.14466) 62 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:18.842 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:18 smithi194 bash[31648]: audit 2023-10-09T04:43:17.791878+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.133:0/1776316794' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:43:19.137 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:18 smithi194 bash[67840]: Error response from daemon: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T04:43:19.137 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:18 smithi194 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T04:43:19.137 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:18 smithi194 systemd[1]: Stopped Ceph prometheus.a for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:43:19.137 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:18 smithi194 systemd[1]: Started Ceph prometheus.a for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:43:19.137 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:18 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:18.854713309Z level=info msg="Executing migration" id="create entity_events table" 2023-10-09T04:43:19.137 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:18 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:18.8980077Z level=info msg="Executing migration" id="create dashboard public config v1" 2023-10-09T04:43:19.138 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:19 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:19.040490507Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2023-10-09T04:43:19.138 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:19 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:19.040745134Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2023-10-09T04:43:19.267 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:43:18 smithi133 bash[27821]: [09/Oct/2023:04:43:18] ENGINE Bus STOPPING 2023-10-09T04:43:19.267 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:43:18 smithi133 bash[27821]: [09/Oct/2023:04:43:18] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T04:43:19.267 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:43:18 smithi133 bash[27821]: [09/Oct/2023:04:43:18] ENGINE Bus STOPPED 2023-10-09T04:43:19.267 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:43:18 smithi133 bash[27821]: [09/Oct/2023:04:43:18] ENGINE Bus STARTING 2023-10-09T04:43:19.268 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27821]: [09/Oct/2023:04:43:19] ENGINE Serving on http://:::9283 2023-10-09T04:43:19.268 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27821]: [09/Oct/2023:04:43:19] ENGINE Bus STARTED 2023-10-09T04:43:19.268 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27821]: [09/Oct/2023:04:43:19] ENGINE Bus STOPPING 2023-10-09T04:43:19.268 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27821]: [09/Oct/2023:04:43:19] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T04:43:19.268 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27821]: [09/Oct/2023:04:43:19] ENGINE Bus STOPPED 2023-10-09T04:43:19.268 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27821]: [09/Oct/2023:04:43:19] ENGINE Bus STARTING 2023-10-09T04:43:19.268 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27821]: [09/Oct/2023:04:43:19] ENGINE Serving on http://:::9283 2023-10-09T04:43:19.268 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27821]: [09/Oct/2023:04:43:19] ENGINE Bus STARTED 2023-10-09T04:43:19.268 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27821]: [09/Oct/2023:04:43:19] ENGINE Bus STOPPING 2023-10-09T04:43:19.268 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27821]: [09/Oct/2023:04:43:19] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T04:43:19.269 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27821]: [09/Oct/2023:04:43:19] ENGINE Bus STOPPED 2023-10-09T04:43:19.269 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27821]: [09/Oct/2023:04:43:19] ENGINE Bus STARTING 2023-10-09T04:43:19.269 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27821]: [09/Oct/2023:04:43:19] ENGINE Serving on http://:::9283 2023-10-09T04:43:19.269 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27821]: [09/Oct/2023:04:43:19] ENGINE Bus STARTED 2023-10-09T04:43:19.637 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:19 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:19.338403754Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-10-09T04:43:19.637 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:19 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:19.33868948Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-10-09T04:43:19.637 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:19 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:19.490569127Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2023-10-09T04:43:20.076 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:19 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:19.819510675Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2023-10-09T04:43:20.076 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:19 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:19.967838491Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2023-10-09T04:43:20.076 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:19 smithi194 bash[31648]: audit 2023-10-09T04:43:18.880577+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:20.076 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:19 smithi194 bash[31648]: audit 2023-10-09T04:43:18.887658+0000 mon.a (mon.0) 754 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:20.076 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:19 smithi194 bash[31648]: audit 2023-10-09T04:43:18.894775+0000 mon.a (mon.0) 755 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:43:20.076 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:19 smithi194 bash[31648]: audit 2023-10-09T04:43:18.895439+0000 mgr.y (mgr.14466) 63 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:43:20.077 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:19 smithi194 bash[31648]: audit 2023-10-09T04:43:18.897229+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi133.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T04:43:20.077 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:19 smithi194 bash[31648]: audit 2023-10-09T04:43:18.897729+0000 mgr.y (mgr.14466) 64 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi133.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T04:43:20.077 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:19 smithi194 bash[31648]: audit 2023-10-09T04:43:18.905037+0000 mon.a (mon.0) 757 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:20.077 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:19 smithi194 bash[31648]: audit 2023-10-09T04:43:18.917126+0000 mon.a (mon.0) 758 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:43:20.077 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:19 smithi194 bash[31648]: audit 2023-10-09T04:43:18.917779+0000 mgr.y (mgr.14466) 65 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:43:20.077 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:19 smithi194 bash[31648]: audit 2023-10-09T04:43:18.924570+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi194.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T04:43:20.077 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:19 smithi194 bash[31648]: audit 2023-10-09T04:43:18.925218+0000 mgr.y (mgr.14466) 66 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi194.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T04:43:20.077 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:19 smithi194 bash[31648]: audit 2023-10-09T04:43:18.934507+0000 mon.a (mon.0) 760 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:20.077 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:19 smithi194 bash[31648]: audit 2023-10-09T04:43:18.945506+0000 mon.a (mon.0) 761 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:43:20.077 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:19 smithi194 bash[31648]: audit 2023-10-09T04:43:18.946265+0000 mgr.y (mgr.14466) 67 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:43:20.077 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:19 smithi194 bash[31648]: audit 2023-10-09T04:43:18.953672+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi194.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T04:43:20.078 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:19 smithi194 bash[31648]: audit 2023-10-09T04:43:18.954225+0000 mgr.y (mgr.14466) 68 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi194.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T04:43:20.078 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:19 smithi194 bash[31648]: audit 2023-10-09T04:43:18.961232+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:20.078 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:19 smithi194 bash[31648]: audit 2023-10-09T04:43:18.969217+0000 mon.a (mon.0) 764 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:43:20.078 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:19 smithi194 bash[31648]: audit 2023-10-09T04:43:19.300972+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:20.078 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:19 smithi194 bash[31648]: audit 2023-10-09T04:43:19.311751+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:20.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27558]: audit 2023-10-09T04:43:18.880577+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:20.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27558]: audit 2023-10-09T04:43:18.887658+0000 mon.a (mon.0) 754 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:20.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27558]: audit 2023-10-09T04:43:18.894775+0000 mon.a (mon.0) 755 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:43:20.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27558]: audit 2023-10-09T04:43:18.895439+0000 mgr.y (mgr.14466) 63 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:43:20.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27558]: audit 2023-10-09T04:43:18.897229+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi133.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T04:43:20.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27558]: audit 2023-10-09T04:43:18.897729+0000 mgr.y (mgr.14466) 64 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi133.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T04:43:20.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27558]: audit 2023-10-09T04:43:18.905037+0000 mon.a (mon.0) 757 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:20.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27558]: audit 2023-10-09T04:43:18.917126+0000 mon.a (mon.0) 758 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:43:20.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27558]: audit 2023-10-09T04:43:18.917779+0000 mgr.y (mgr.14466) 65 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:43:20.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27558]: audit 2023-10-09T04:43:18.924570+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi194.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T04:43:20.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27558]: audit 2023-10-09T04:43:18.925218+0000 mgr.y (mgr.14466) 66 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi194.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T04:43:20.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27558]: audit 2023-10-09T04:43:18.934507+0000 mon.a (mon.0) 760 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:20.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27558]: audit 2023-10-09T04:43:18.945506+0000 mon.a (mon.0) 761 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:43:20.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27558]: audit 2023-10-09T04:43:18.946265+0000 mgr.y (mgr.14466) 67 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:43:20.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27558]: audit 2023-10-09T04:43:18.953672+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi194.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T04:43:20.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27558]: audit 2023-10-09T04:43:18.954225+0000 mgr.y (mgr.14466) 68 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi194.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T04:43:20.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27558]: audit 2023-10-09T04:43:18.961232+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:20.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27558]: audit 2023-10-09T04:43:18.969217+0000 mon.a (mon.0) 764 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:43:20.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27558]: audit 2023-10-09T04:43:19.300972+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:20.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[27558]: audit 2023-10-09T04:43:19.311751+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:20.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[41291]: audit 2023-10-09T04:43:18.880577+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:20.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[41291]: audit 2023-10-09T04:43:18.887658+0000 mon.a (mon.0) 754 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:20.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[41291]: audit 2023-10-09T04:43:18.894775+0000 mon.a (mon.0) 755 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:43:20.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[41291]: audit 2023-10-09T04:43:18.895439+0000 mgr.y (mgr.14466) 63 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:43:20.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[41291]: audit 2023-10-09T04:43:18.897229+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi133.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T04:43:20.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[41291]: audit 2023-10-09T04:43:18.897729+0000 mgr.y (mgr.14466) 64 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi133.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T04:43:20.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[41291]: audit 2023-10-09T04:43:18.905037+0000 mon.a (mon.0) 757 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:20.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[41291]: audit 2023-10-09T04:43:18.917126+0000 mon.a (mon.0) 758 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:43:20.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[41291]: audit 2023-10-09T04:43:18.917779+0000 mgr.y (mgr.14466) 65 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:43:20.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[41291]: audit 2023-10-09T04:43:18.924570+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi194.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T04:43:20.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[41291]: audit 2023-10-09T04:43:18.925218+0000 mgr.y (mgr.14466) 66 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi194.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T04:43:20.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[41291]: audit 2023-10-09T04:43:18.934507+0000 mon.a (mon.0) 760 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:20.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[41291]: audit 2023-10-09T04:43:18.945506+0000 mon.a (mon.0) 761 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:43:20.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[41291]: audit 2023-10-09T04:43:18.946265+0000 mgr.y (mgr.14466) 67 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:43:20.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[41291]: audit 2023-10-09T04:43:18.953672+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi194.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T04:43:20.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[41291]: audit 2023-10-09T04:43:18.954225+0000 mgr.y (mgr.14466) 68 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi194.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T04:43:20.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[41291]: audit 2023-10-09T04:43:18.961232+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:20.305 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[41291]: audit 2023-10-09T04:43:18.969217+0000 mon.a (mon.0) 764 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:43:20.305 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[41291]: audit 2023-10-09T04:43:19.300972+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:20.305 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:19 smithi133 bash[41291]: audit 2023-10-09T04:43:19.311751+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:20.386 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:20.074773533Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-10-09T04:43:20.387 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:20.174649594Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2023-10-09T04:43:20.387 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:20.294133575Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-10-09T04:43:20.697 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:20.397443328Z level=info msg="Executing migration" id="Drop public config table" 2023-10-09T04:43:20.697 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:20.490747177Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2023-10-09T04:43:20.697 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:20.576870249Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2023-10-09T04:43:20.697 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:20.602082036Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-10-09T04:43:20.698 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:20.644280757Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2023-10-09T04:43:20.698 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:20.695711233Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2023-10-09T04:43:20.951 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:20.763354468Z level=info msg="Executing migration" id="add annotations_enabled column" 2023-10-09T04:43:20.951 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:20.788570261Z level=info msg="Executing migration" id="add time_selection_enabled column" 2023-10-09T04:43:20.951 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:20.81377868Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2023-10-09T04:43:20.951 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:20.840299917Z level=info msg="Executing migration" id="add share column" 2023-10-09T04:43:20.951 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:20.882290137Z level=info msg="Executing migration" id="create default alerting folders" 2023-10-09T04:43:20.951 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:20.907597548Z level=info msg="Executing migration" id="create file table" 2023-10-09T04:43:20.951 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:20.94955784Z level=info msg="Executing migration" id="file table idx: path natural pk" 2023-10-09T04:43:20.952 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67868]: ts=2023-10-09T04:43:20.701Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-10-09T04:43:20.952 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67868]: ts=2023-10-09T04:43:20.701Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-10-09T04:43:20.952 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67868]: ts=2023-10-09T04:43:20.701Z caller=main.go:570 level=info host_details="(Linux 5.15.0-86-generic #96~20.04.1-Ubuntu SMP Thu Sep 21 13:23:37 UTC 2023 x86_64 smithi194 (none))" 2023-10-09T04:43:20.952 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67868]: ts=2023-10-09T04:43:20.702Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T04:43:20.952 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67868]: ts=2023-10-09T04:43:20.702Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T04:43:20.952 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67868]: ts=2023-10-09T04:43:20.704Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T04:43:20.952 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67868]: ts=2023-10-09T04:43:20.704Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-10-09T04:43:20.952 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67868]: ts=2023-10-09T04:43:20.705Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-10-09T04:43:20.952 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67868]: ts=2023-10-09T04:43:20.705Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-10-09T04:43:20.952 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67868]: ts=2023-10-09T04:43:20.708Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T04:43:20.952 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67868]: ts=2023-10-09T04:43:20.708Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=3.174µs 2023-10-09T04:43:20.953 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67868]: ts=2023-10-09T04:43:20.708Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T04:43:20.953 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67868]: ts=2023-10-09T04:43:20.709Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2023-10-09T04:43:20.953 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67868]: ts=2023-10-09T04:43:20.709Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2023-10-09T04:43:20.953 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67868]: ts=2023-10-09T04:43:20.709Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=40.915µs wal_replay_duration=843.936µs wbl_replay_duration=115ns total_replay_duration=934.142µs 2023-10-09T04:43:20.953 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67868]: ts=2023-10-09T04:43:20.710Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T04:43:20.953 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67868]: ts=2023-10-09T04:43:20.710Z caller=main.go:1029 level=info msg="TSDB started" 2023-10-09T04:43:20.953 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67868]: ts=2023-10-09T04:43:20.710Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T04:43:20.953 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67868]: ts=2023-10-09T04:43:20.733Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=22.863634ms db_storage=918ns remote_storage=1.374µs web_handler=447ns query_engine=579ns scrape=300.385µs scrape_sd=31.323µs notify=21.609µs notify_sd=6.683µs rules=22.2303ms tracing=7.107µs 2023-10-09T04:43:20.953 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67868]: ts=2023-10-09T04:43:20.733Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-10-09T04:43:20.953 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67868]: ts=2023-10-09T04:43:20.733Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-10-09T04:43:21.202 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[31648]: cluster 2023-10-09T04:43:19.102296+0000 mgr.y (mgr.14466) 69 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:21.202 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[31648]: audit 2023-10-09T04:43:20.184918+0000 mon.a (mon.0) 767 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:21.202 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:20 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:20.993154412Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2023-10-09T04:43:21.202 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:21.018449735Z level=info msg="Executing migration" id="create file_meta table" 2023-10-09T04:43:21.202 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:21.043694161Z level=info msg="Executing migration" id="file table idx: path key" 2023-10-09T04:43:21.202 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:21.085617778Z level=info msg="Executing migration" id="set path collation in file table" 2023-10-09T04:43:21.202 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:21.129693245Z level=info msg="Executing migration" id="managed permissions migration" 2023-10-09T04:43:21.202 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:21.17285992Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2023-10-09T04:43:21.203 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:21.200599478Z level=info msg="Executing migration" id="RBAC action name migrator" 2023-10-09T04:43:21.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:20 smithi133 bash[41291]: cluster 2023-10-09T04:43:19.102296+0000 mgr.y (mgr.14466) 69 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:21.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:20 smithi133 bash[41291]: audit 2023-10-09T04:43:20.184918+0000 mon.a (mon.0) 767 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:21.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:20 smithi133 bash[27558]: cluster 2023-10-09T04:43:19.102296+0000 mgr.y (mgr.14466) 69 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:21.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:20 smithi133 bash[27558]: audit 2023-10-09T04:43:20.184918+0000 mon.a (mon.0) 767 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:21.466 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:21.228255078Z level=info msg="Executing migration" id="Add UID column to playlist" 2023-10-09T04:43:21.466 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:21.253408484Z level=info msg="Executing migration" id="Update uid column values in playlist" 2023-10-09T04:43:21.466 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:21.295325549Z level=info msg="Executing migration" id="Add index for uid in playlist" 2023-10-09T04:43:21.466 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:21.320672935Z level=info msg="Executing migration" id="update group index for alert rules" 2023-10-09T04:43:21.466 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:21.362574248Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2023-10-09T04:43:21.466 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:21.389143936Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2023-10-09T04:43:21.466 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:21.414340745Z level=info msg="Executing migration" id="add action column to seed_assignment" 2023-10-09T04:43:21.467 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:21.439622403Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2023-10-09T04:43:21.721 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:21.464783037Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2023-10-09T04:43:21.721 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:21.490072964Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2023-10-09T04:43:21.721 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:21.541576049Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2023-10-09T04:43:21.721 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:21.566796155Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2023-10-09T04:43:21.721 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:21.592094785Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2023-10-09T04:43:21.721 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:21.642616814Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2023-10-09T04:43:21.722 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:21.667778771Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2023-10-09T04:43:21.722 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:21.694382251Z level=info msg="Executing migration" id="create folder table" 2023-10-09T04:43:21.722 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:21.719581427Z level=info msg="Executing migration" id="Add index for parent_uid" 2023-10-09T04:43:21.981 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:21.744922759Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2023-10-09T04:43:21.981 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:21.770116399Z level=info msg="Executing migration" id="Update folder title length" 2023-10-09T04:43:21.981 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:21.795201241Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2023-10-09T04:43:21.981 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=migrator t=2023-10-09T04:43:21.838618783Z level=info msg="migrations completed" performed=485 skipped=0 duration=22.695937112s 2023-10-09T04:43:21.981 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=sqlstore t=2023-10-09T04:43:21.839956845Z level=info msg="Created default organization" 2023-10-09T04:43:21.982 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=plugin.loader t=2023-10-09T04:43:21.950054732Z level=info msg="Plugin registered" pluginID=input 2023-10-09T04:43:21.982 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=plugin.loader t=2023-10-09T04:43:21.979671978Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2023-10-09T04:43:22.300 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:43:21 smithi133 bash[75109]: ts=2023-10-09T04:43:21.891Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.001951748s 2023-10-09T04:43:22.387 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=plugin.loader t=2023-10-09T04:43:21.979702603Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2023-10-09T04:43:22.387 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=secrets t=2023-10-09T04:43:21.979744161Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2023-10-09T04:43:22.387 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=query_data t=2023-10-09T04:43:21.981766761Z level=info msg="Query Service initialization" 2023-10-09T04:43:22.387 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:21 smithi194 bash[67167]: logger=live.push_http t=2023-10-09T04:43:21.986141601Z level=info msg="Live Push Gateway initialization" 2023-10-09T04:43:23.208 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:22 smithi194 bash[31648]: cluster 2023-10-09T04:43:21.102928+0000 mgr.y (mgr.14466) 70 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:23.208 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:22 smithi194 bash[67167]: logger=infra.usagestats.collector t=2023-10-09T04:43:22.927814289Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2023-10-09T04:43:23.208 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:22 smithi194 bash[67167]: logger=provisioning.datasources t=2023-10-09T04:43:22.92849634Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2023-10-09T04:43:23.208 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:22 smithi194 bash[67167]: logger=provisioning.datasources t=2023-10-09T04:43:22.962263641Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2023-10-09T04:43:23.209 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:23 smithi194 bash[67167]: logger=provisioning.datasources t=2023-10-09T04:43:23.034728097Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2023-10-09T04:43:23.209 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:23 smithi194 bash[67167]: logger=provisioning.alerting t=2023-10-09T04:43:23.083313876Z level=info msg="starting to provision alerting" 2023-10-09T04:43:23.209 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:23 smithi194 bash[67167]: logger=provisioning.alerting t=2023-10-09T04:43:23.083357889Z level=info msg="finished to provision alerting" 2023-10-09T04:43:23.209 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:23 smithi194 bash[67167]: logger=grafanaStorageLogger t=2023-10-09T04:43:23.083662481Z level=info msg="storage starting" 2023-10-09T04:43:23.209 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:23 smithi194 bash[67167]: logger=ngalert.state.manager t=2023-10-09T04:43:23.083687233Z level=info msg="Warming state cache for startup" 2023-10-09T04:43:23.209 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:23 smithi194 bash[67167]: logger=http.server t=2023-10-09T04:43:23.088987182Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2023-10-09T04:43:23.209 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:23 smithi194 bash[67167]: logger=sqlstore.transactions t=2023-10-09T04:43:23.09407557Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-10-09T04:43:23.209 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:23 smithi194 bash[67167]: logger=ngalert.state.manager t=2023-10-09T04:43:23.127563153Z level=info msg="State cache has been initialized" states=0 duration=43.8758ms 2023-10-09T04:43:23.209 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:23 smithi194 bash[67167]: logger=ngalert.multiorg.alertmanager t=2023-10-09T04:43:23.127623212Z level=info msg="starting MultiOrg Alertmanager" 2023-10-09T04:43:23.209 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:23 smithi194 bash[67167]: logger=ticker t=2023-10-09T04:43:23.127661855Z level=info msg=starting first_tick=2023-10-09T04:43:30Z 2023-10-09T04:43:23.210 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:43:23 smithi194 bash[67167]: logger=sqlstore.transactions t=2023-10-09T04:43:23.206727263Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-10-09T04:43:23.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:22 smithi133 bash[41291]: cluster 2023-10-09T04:43:21.102928+0000 mgr.y (mgr.14466) 70 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:23.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:22 smithi133 bash[27558]: cluster 2023-10-09T04:43:21.102928+0000 mgr.y (mgr.14466) 70 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:23.310 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:25.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:24 smithi133 bash[27558]: cluster 2023-10-09T04:43:23.103497+0000 mgr.y (mgr.14466) 71 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:25.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:24 smithi133 bash[27558]: audit 2023-10-09T04:43:24.210560+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:25.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:24 smithi133 bash[27558]: audit 2023-10-09T04:43:24.218465+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:25.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:24 smithi133 bash[41291]: cluster 2023-10-09T04:43:23.103497+0000 mgr.y (mgr.14466) 71 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:25.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:24 smithi133 bash[41291]: audit 2023-10-09T04:43:24.210560+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:25.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:24 smithi133 bash[41291]: audit 2023-10-09T04:43:24.218465+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:25.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:24 smithi194 bash[31648]: cluster 2023-10-09T04:43:23.103497+0000 mgr.y (mgr.14466) 71 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:25.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:24 smithi194 bash[31648]: audit 2023-10-09T04:43:24.210560+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:25.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:24 smithi194 bash[31648]: audit 2023-10-09T04:43:24.218465+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:26.500 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T04:43:26.500 INFO:teuthology.orchestra.run.smithi133.stdout:{"epoch":63,"fsid":"855b5206-665c-11ee-8db5-212e2dc638e7","created":"2023-10-09T04:31:08.633675+0000","modified":"2023-10-09T04:42:56.703971+0000","last_up_change":"2023-10-09T04:41:05.383200+0000","last_in_change":"2023-10-09T04:40:39.315748+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T04:36:44.260202+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T04:42:47.518485+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"32633ec2-9b8f-42d8-bdfb-5c9668a26071","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6802","nonce":1933306305},{"type":"v1","addr":"172.21.15.133:6803","nonce":1933306305}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6804","nonce":1933306305},{"type":"v1","addr":"172.21.15.133:6805","nonce":1933306305}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6808","nonce":1933306305},{"type":"v1","addr":"172.21.15.133:6809","nonce":1933306305}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6806","nonce":1933306305},{"type":"v1","addr":"172.21.15.133:6807","nonce":1933306305}]},"public_addr":"172.21.15.133:6803/1933306305","cluster_addr":"172.21.15.133:6805/1933306305","heartbeat_back_addr":"172.21.15.133:6809/1933306305","heartbeat_front_addr":"172.21.15.133:6807/1933306305","state":["exists","up"]},{"osd":1,"uuid":"7f2a961d-a790-4337-91c7-1e45f900bbd4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6810","nonce":2965860635},{"type":"v1","addr":"172.21.15.133:6811","nonce":2965860635}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6812","nonce":2965860635},{"type":"v1","addr":"172.21.15.133:6813","nonce":2965860635}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6816","nonce":2965860635},{"type":"v1","addr":"172.21.15.133:6817","nonce":2965860635}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6814","nonce":2965860635},{"type":"v1","addr":"172.21.15.133:6815","nonce":2965860635}]},"public_addr":"172.21.15.133:6811/2965860635","cluster_addr":"172.21.15.133:6813/2965860635","heartbeat_back_addr":"172.21.15.133:6817/2965860635","heartbeat_front_addr":"172.21.15.133:6815/2965860635","state":["exists","up"]},{"osd":2,"uuid":"08751330-a327-4bf8-9c4d-9ff986b1c67f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6818","nonce":1465201170},{"type":"v1","addr":"172.21.15.133:6819","nonce":1465201170}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6820","nonce":1465201170},{"type":"v1","addr":"172.21.15.133:6821","nonce":1465201170}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6824","nonce":1465201170},{"type":"v1","addr":"172.21.15.133:6825","nonce":1465201170}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6822","nonce":1465201170},{"type":"v1","addr":"172.21.15.133:6823","nonce":1465201170}]},"public_addr":"172.21.15.133:6819/1465201170","cluster_addr":"172.21.15.133:6821/1465201170","heartbeat_back_addr":"172.21.15.133:6825/1465201170","heartbeat_front_addr":"172.21.15.133:6823/1465201170","state":["exists","up"]},{"osd":3,"uuid":"56bbf807-c8f7-4226-b46e-802b539274ca","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6826","nonce":1093481727},{"type":"v1","addr":"172.21.15.133:6827","nonce":1093481727}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6828","nonce":1093481727},{"type":"v1","addr":"172.21.15.133:6829","nonce":1093481727}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6832","nonce":1093481727},{"type":"v1","addr":"172.21.15.133:6833","nonce":1093481727}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6830","nonce":1093481727},{"type":"v1","addr":"172.21.15.133:6831","nonce":1093481727}]},"public_addr":"172.21.15.133:6827/1093481727","cluster_addr":"172.21.15.133:6829/1093481727","heartbeat_back_addr":"172.21.15.133:6833/1093481727","heartbeat_front_addr":"172.21.15.133:6831/1093481727","state":["exists","up"]},{"osd":4,"uuid":"509ec8cc-5042-428e-9a69-8e602b5afce8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6801","nonce":1874338403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6803","nonce":1874338403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6807","nonce":1874338403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6805","nonce":1874338403}]},"public_addr":"172.21.15.194:6801/1874338403","cluster_addr":"172.21.15.194:6803/1874338403","heartbeat_back_addr":"172.21.15.194:6807/1874338403","heartbeat_front_addr":"172.21.15.194:6805/1874338403","state":["exists","up"]},{"osd":5,"uuid":"bbdc1e67-fd5b-49d0-9e42-cf151d167943","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6809","nonce":3989986662}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6811","nonce":3989986662}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6815","nonce":3989986662}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6813","nonce":3989986662}]},"public_addr":"172.21.15.194:6809/3989986662","cluster_addr":"172.21.15.194:6811/3989986662","heartbeat_back_addr":"172.21.15.194:6815/3989986662","heartbeat_front_addr":"172.21.15.194:6813/3989986662","state":["exists","up"]},{"osd":6,"uuid":"7b69d014-f4c0-4e73-a14e-7d51b7ee3ea2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6817","nonce":2607555245}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6819","nonce":2607555245}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6823","nonce":2607555245}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6820","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6821","nonce":2607555245}]},"public_addr":"172.21.15.194:6817/2607555245","cluster_addr":"172.21.15.194:6819/2607555245","heartbeat_back_addr":"172.21.15.194:6823/2607555245","heartbeat_front_addr":"172.21.15.194:6821/2607555245","state":["exists","up"]},{"osd":7,"uuid":"02506268-102c-4654-9b5c-68d36c28f7f7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6824","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6825","nonce":1051622545}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6827","nonce":1051622545}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6831","nonce":1051622545}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6828","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6829","nonce":1051622545}]},"public_addr":"172.21.15.194:6825/1051622545","cluster_addr":"172.21.15.194:6827/1051622545","heartbeat_back_addr":"172.21.15.194:6831/1051622545","heartbeat_front_addr":"172.21.15.194:6829/1051622545","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:34:53.028753+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:35:47.001796+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:36:41.220545+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:37:36.237434+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:38:28.301119+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:39:20.099747+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:40:12.106681+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:41:04.212546+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.133:0/1309073801":"2023-10-10T04:42:05.063462+0000","172.21.15.133:6801/768614786":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/4290202542":"2023-10-10T04:32:17.611609+0000","172.21.15.133:6800/788799879":"2023-10-10T04:32:17.611609+0000","172.21.15.133:6800/768614786":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/320185713":"2023-10-10T04:42:05.063462+0000","172.21.15.133:6801/788799879":"2023-10-10T04:32:17.611609+0000","172.21.15.133:0/1411744140":"2023-10-10T04:31:42.639571+0000","172.21.15.133:0/55818837":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/4173745360":"2023-10-10T04:31:42.639571+0000","172.21.15.133:0/1793292086":"2023-10-10T04:32:17.611609+0000","172.21.15.133:0/3636510019":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/740467257":"2023-10-10T04:31:42.639571+0000","172.21.15.133:6800/1761810618":"2023-10-10T04:31:42.639571+0000","172.21.15.133:0/3619387632":"2023-10-10T04:32:17.611609+0000","172.21.15.133:6801/1761810618":"2023-10-10T04:31:42.639571+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T04:43:26.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:43:26 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:43:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:43:27.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:26 smithi133 bash[27558]: cluster 2023-10-09T04:43:25.104052+0000 mgr.y (mgr.14466) 72 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:27.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:26 smithi133 bash[27558]: audit 2023-10-09T04:43:26.356724+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:27.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:26 smithi133 bash[27558]: audit 2023-10-09T04:43:26.366977+0000 mon.a (mon.0) 771 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:27.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:26 smithi133 bash[27558]: audit 2023-10-09T04:43:26.495768+0000 mon.a (mon.0) 772 : audit [DBG] from='client.? 172.21.15.133:0/1024062751' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:43:27.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:26 smithi133 bash[27558]: audit 2023-10-09T04:43:26.798626+0000 mon.a (mon.0) 773 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:43:27.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:26 smithi133 bash[27558]: audit 2023-10-09T04:43:26.799879+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:43:27.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:26 smithi133 bash[27558]: audit 2023-10-09T04:43:26.808288+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:27.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:26 smithi133 bash[27558]: audit 2023-10-09T04:43:26.881284+0000 mon.a (mon.0) 776 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:43:27.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:26 smithi133 bash[27558]: audit 2023-10-09T04:43:26.883710+0000 mon.a (mon.0) 777 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:43:27.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:26 smithi133 bash[27558]: audit 2023-10-09T04:43:26.884603+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:43:27.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:26 smithi133 bash[27558]: audit 2023-10-09T04:43:26.893979+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:27.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:26 smithi133 bash[41291]: cluster 2023-10-09T04:43:25.104052+0000 mgr.y (mgr.14466) 72 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:27.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:26 smithi133 bash[41291]: audit 2023-10-09T04:43:26.356724+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:27.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:26 smithi133 bash[41291]: audit 2023-10-09T04:43:26.366977+0000 mon.a (mon.0) 771 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:27.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:26 smithi133 bash[41291]: audit 2023-10-09T04:43:26.495768+0000 mon.a (mon.0) 772 : audit [DBG] from='client.? 172.21.15.133:0/1024062751' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:43:27.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:26 smithi133 bash[41291]: audit 2023-10-09T04:43:26.798626+0000 mon.a (mon.0) 773 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:43:27.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:26 smithi133 bash[41291]: audit 2023-10-09T04:43:26.799879+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:43:27.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:26 smithi133 bash[41291]: audit 2023-10-09T04:43:26.808288+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:27.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:26 smithi133 bash[41291]: audit 2023-10-09T04:43:26.881284+0000 mon.a (mon.0) 776 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:43:27.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:26 smithi133 bash[41291]: audit 2023-10-09T04:43:26.883710+0000 mon.a (mon.0) 777 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:43:27.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:26 smithi133 bash[41291]: audit 2023-10-09T04:43:26.884603+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:43:27.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:26 smithi133 bash[41291]: audit 2023-10-09T04:43:26.893979+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:27.351 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T04:43:27.352 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T04:43:27.352 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T04:43:27.353 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T04:43:27.353 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T04:43:27.353 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T04:43:27.354 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T04:43:27.354 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T04:43:27.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:26 smithi194 bash[31648]: cluster 2023-10-09T04:43:25.104052+0000 mgr.y (mgr.14466) 72 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:27.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:26 smithi194 bash[31648]: audit 2023-10-09T04:43:26.356724+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:27.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:26 smithi194 bash[31648]: audit 2023-10-09T04:43:26.366977+0000 mon.a (mon.0) 771 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:27.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:26 smithi194 bash[31648]: audit 2023-10-09T04:43:26.495768+0000 mon.a (mon.0) 772 : audit [DBG] from='client.? 172.21.15.133:0/1024062751' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:43:27.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:26 smithi194 bash[31648]: audit 2023-10-09T04:43:26.798626+0000 mon.a (mon.0) 773 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:43:27.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:26 smithi194 bash[31648]: audit 2023-10-09T04:43:26.799879+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:43:27.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:26 smithi194 bash[31648]: audit 2023-10-09T04:43:26.808288+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:27.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:26 smithi194 bash[31648]: audit 2023-10-09T04:43:26.881284+0000 mon.a (mon.0) 776 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:43:27.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:26 smithi194 bash[31648]: audit 2023-10-09T04:43:26.883710+0000 mon.a (mon.0) 777 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:43:27.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:26 smithi194 bash[31648]: audit 2023-10-09T04:43:26.884603+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:43:27.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:26 smithi194 bash[31648]: audit 2023-10-09T04:43:26.893979+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:43:29.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:28 smithi133 bash[27558]: cluster 2023-10-09T04:43:27.104718+0000 mgr.y (mgr.14466) 73 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:29.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:28 smithi133 bash[41291]: cluster 2023-10-09T04:43:27.104718+0000 mgr.y (mgr.14466) 73 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:29.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:28 smithi194 bash[31648]: cluster 2023-10-09T04:43:27.104718+0000 mgr.y (mgr.14466) 73 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:30.800 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:43:30 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:43:30] "GET /metrics HTTP/1.1" 200 203314 "" "Prometheus/2.43.0" 2023-10-09T04:43:31.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:30 smithi133 bash[27558]: cluster 2023-10-09T04:43:29.105309+0000 mgr.y (mgr.14466) 74 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:31.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:30 smithi133 bash[41291]: cluster 2023-10-09T04:43:29.105309+0000 mgr.y (mgr.14466) 74 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:31.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:30 smithi194 bash[31648]: cluster 2023-10-09T04:43:29.105309+0000 mgr.y (mgr.14466) 74 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:32.264 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:32.264 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:32.267 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:32.267 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:32.268 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:32.268 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:32.268 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:32.269 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:33.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:32 smithi133 bash[27558]: cluster 2023-10-09T04:43:31.105961+0000 mgr.y (mgr.14466) 75 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:33.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:32 smithi133 bash[41291]: cluster 2023-10-09T04:43:31.105961+0000 mgr.y (mgr.14466) 75 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:33.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:32 smithi194 bash[31648]: cluster 2023-10-09T04:43:31.105961+0000 mgr.y (mgr.14466) 75 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:35.255 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:34 smithi133 bash[27558]: cluster 2023-10-09T04:43:33.106540+0000 mgr.y (mgr.14466) 76 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:35.255 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:34 smithi133 bash[41291]: cluster 2023-10-09T04:43:33.106540+0000 mgr.y (mgr.14466) 76 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:35.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:34 smithi194 bash[31648]: cluster 2023-10-09T04:43:33.106540+0000 mgr.y (mgr.14466) 76 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:36.251 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:36 smithi133 bash[41291]: audit 2023-10-09T04:43:35.185108+0000 mon.a (mon.0) 780 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:36.251 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:35 smithi133 bash[27558]: audit 2023-10-09T04:43:35.185108+0000 mon.a (mon.0) 780 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:36.353 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:35 smithi194 bash[31648]: audit 2023-10-09T04:43:35.185108+0000 mon.a (mon.0) 780 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:36.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:43:36 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:43:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:43:37.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:37 smithi133 bash[41291]: cluster 2023-10-09T04:43:35.106995+0000 mgr.y (mgr.14466) 77 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:37.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:37 smithi133 bash[27558]: cluster 2023-10-09T04:43:35.106995+0000 mgr.y (mgr.14466) 77 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:37.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:37 smithi194 bash[31648]: cluster 2023-10-09T04:43:35.106995+0000 mgr.y (mgr.14466) 77 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:39.215 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:39 smithi133 bash[41291]: cluster 2023-10-09T04:43:37.107580+0000 mgr.y (mgr.14466) 78 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:39.215 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:39 smithi133 bash[27558]: cluster 2023-10-09T04:43:37.107580+0000 mgr.y (mgr.14466) 78 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:39.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:39 smithi194 bash[31648]: cluster 2023-10-09T04:43:37.107580+0000 mgr.y (mgr.14466) 78 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:40.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:40 smithi133 bash[41291]: cluster 2023-10-09T04:43:39.107984+0000 mgr.y (mgr.14466) 79 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:40.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:40 smithi133 bash[27558]: cluster 2023-10-09T04:43:39.107984+0000 mgr.y (mgr.14466) 79 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:40.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:40 smithi194 bash[31648]: cluster 2023-10-09T04:43:39.107984+0000 mgr.y (mgr.14466) 79 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:40.735 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:43:40 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:43:40] "GET /metrics HTTP/1.1" 200 203300 "" "Prometheus/2.43.0" 2023-10-09T04:43:42.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:42 smithi133 bash[27558]: cluster 2023-10-09T04:43:41.108622+0000 mgr.y (mgr.14466) 80 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:42.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:42 smithi133 bash[41291]: cluster 2023-10-09T04:43:41.108622+0000 mgr.y (mgr.14466) 80 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:42.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:42 smithi194 bash[31648]: cluster 2023-10-09T04:43:41.108622+0000 mgr.y (mgr.14466) 80 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:44.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:44 smithi133 bash[41291]: cluster 2023-10-09T04:43:43.109299+0000 mgr.y (mgr.14466) 81 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:44.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:44 smithi133 bash[27558]: cluster 2023-10-09T04:43:43.109299+0000 mgr.y (mgr.14466) 81 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:44.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:44 smithi194 bash[31648]: cluster 2023-10-09T04:43:43.109299+0000 mgr.y (mgr.14466) 81 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:46.131 INFO:teuthology.orchestra.run.smithi133.stdout:231928234017 2023-10-09T04:43:46.131 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T04:43:46.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:46 smithi194 bash[31648]: cluster 2023-10-09T04:43:45.109870+0000 mgr.y (mgr.14466) 82 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:46.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:43:46 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:43:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:43:46.831 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:46 smithi133 bash[41291]: cluster 2023-10-09T04:43:45.109870+0000 mgr.y (mgr.14466) 82 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:46.831 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:46 smithi133 bash[27558]: cluster 2023-10-09T04:43:45.109870+0000 mgr.y (mgr.14466) 82 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:48.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:48 smithi133 bash[27558]: cluster 2023-10-09T04:43:47.110458+0000 mgr.y (mgr.14466) 83 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:48.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:48 smithi133 bash[41291]: cluster 2023-10-09T04:43:47.110458+0000 mgr.y (mgr.14466) 83 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:48.721 INFO:teuthology.orchestra.run.smithi133.stdout:197568495660 2023-10-09T04:43:48.721 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T04:43:48.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:48 smithi194 bash[31648]: cluster 2023-10-09T04:43:47.110458+0000 mgr.y (mgr.14466) 83 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:49.294 INFO:teuthology.orchestra.run.smithi133.stdout:107374182475 2023-10-09T04:43:49.305 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T04:43:50.617 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:43:50 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:43:50] "GET /metrics HTTP/1.1" 200 203297 "" "Prometheus/2.43.0" 2023-10-09T04:43:50.618 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:50 smithi133 bash[27558]: cluster 2023-10-09T04:43:49.110969+0000 mgr.y (mgr.14466) 84 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:50.618 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:50 smithi133 bash[27558]: audit 2023-10-09T04:43:50.184762+0000 mon.a (mon.0) 781 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:50.618 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:50 smithi133 bash[41291]: cluster 2023-10-09T04:43:49.110969+0000 mgr.y (mgr.14466) 84 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:50.618 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:50 smithi133 bash[41291]: audit 2023-10-09T04:43:50.184762+0000 mon.a (mon.0) 781 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:50.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:50 smithi194 bash[31648]: cluster 2023-10-09T04:43:49.110969+0000 mgr.y (mgr.14466) 84 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:50.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:50 smithi194 bash[31648]: audit 2023-10-09T04:43:50.184762+0000 mon.a (mon.0) 781 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:51.356 INFO:teuthology.orchestra.run.smithi133.stdout:128849018946 2023-10-09T04:43:51.357 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T04:43:51.402 INFO:teuthology.orchestra.run.smithi133.stdout:77309411415 2023-10-09T04:43:51.402 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T04:43:51.948 INFO:teuthology.orchestra.run.smithi133.stdout:55834574945 2023-10-09T04:43:51.949 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T04:43:52.701 INFO:teuthology.orchestra.run.smithi133.stdout:34359738475 2023-10-09T04:43:52.701 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T04:43:52.777 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:52 smithi133 bash[41291]: cluster 2023-10-09T04:43:51.111313+0000 mgr.y (mgr.14466) 85 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:52.777 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:52 smithi133 bash[27558]: cluster 2023-10-09T04:43:51.111313+0000 mgr.y (mgr.14466) 85 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:52.778 INFO:teuthology.orchestra.run.smithi133.stdout:163208757302 2023-10-09T04:43:52.779 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T04:43:52.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:52 smithi194 bash[31648]: cluster 2023-10-09T04:43:51.111313+0000 mgr.y (mgr.14466) 85 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:54.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:54 smithi133 bash[41291]: cluster 2023-10-09T04:43:53.111861+0000 mgr.y (mgr.14466) 86 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:54.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:54 smithi133 bash[27558]: cluster 2023-10-09T04:43:53.111861+0000 mgr.y (mgr.14466) 86 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:54.889 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:54 smithi194 bash[31648]: cluster 2023-10-09T04:43:53.111861+0000 mgr.y (mgr.14466) 86 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:55.814 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:55.815 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:55.815 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:55.815 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:55.816 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:55.817 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:56.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:56 smithi194 bash[31648]: cluster 2023-10-09T04:43:55.112493+0000 mgr.y (mgr.14466) 87 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:56.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:43:56 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:43:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:43:56.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:56 smithi133 bash[27558]: cluster 2023-10-09T04:43:55.112493+0000 mgr.y (mgr.14466) 87 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:56.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:56 smithi133 bash[41291]: cluster 2023-10-09T04:43:55.112493+0000 mgr.y (mgr.14466) 87 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:56.822 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:56.822 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:58.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:43:58 smithi133 bash[41291]: cluster 2023-10-09T04:43:57.112963+0000 mgr.y (mgr.14466) 88 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:58.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:43:58 smithi133 bash[27558]: cluster 2023-10-09T04:43:57.112963+0000 mgr.y (mgr.14466) 88 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:43:58.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:43:58 smithi194 bash[31648]: cluster 2023-10-09T04:43:57.112963+0000 mgr.y (mgr.14466) 88 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:00.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:00 smithi133 bash[27558]: cluster 2023-10-09T04:43:59.113548+0000 mgr.y (mgr.14466) 89 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:00.800 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:44:00 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:44:00] "GET /metrics HTTP/1.1" 200 203297 "" "Prometheus/2.43.0" 2023-10-09T04:44:00.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:00 smithi133 bash[41291]: cluster 2023-10-09T04:43:59.113548+0000 mgr.y (mgr.14466) 89 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:00.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:00 smithi194 bash[31648]: cluster 2023-10-09T04:43:59.113548+0000 mgr.y (mgr.14466) 89 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:02.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:02 smithi133 bash[27558]: cluster 2023-10-09T04:44:01.114155+0000 mgr.y (mgr.14466) 90 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:02.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:02 smithi133 bash[41291]: cluster 2023-10-09T04:44:01.114155+0000 mgr.y (mgr.14466) 90 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:02.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:02 smithi194 bash[31648]: cluster 2023-10-09T04:44:01.114155+0000 mgr.y (mgr.14466) 90 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:04.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:04 smithi133 bash[41291]: cluster 2023-10-09T04:44:03.114697+0000 mgr.y (mgr.14466) 91 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:04.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:04 smithi133 bash[27558]: cluster 2023-10-09T04:44:03.114697+0000 mgr.y (mgr.14466) 91 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:04.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:04 smithi194 bash[31648]: cluster 2023-10-09T04:44:03.114697+0000 mgr.y (mgr.14466) 91 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:05.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:05 smithi133 bash[27558]: audit 2023-10-09T04:44:05.187332+0000 mon.a (mon.0) 782 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:05.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:05 smithi133 bash[41291]: audit 2023-10-09T04:44:05.187332+0000 mon.a (mon.0) 782 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:05.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:05 smithi194 bash[31648]: audit 2023-10-09T04:44:05.187332+0000 mon.a (mon.0) 782 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:06.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:06 smithi194 bash[31648]: cluster 2023-10-09T04:44:05.115085+0000 mgr.y (mgr.14466) 92 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:06.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:44:06 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:44:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:44:06.863 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:06 smithi133 bash[27558]: cluster 2023-10-09T04:44:05.115085+0000 mgr.y (mgr.14466) 92 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:06.863 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:06 smithi133 bash[41291]: cluster 2023-10-09T04:44:05.115085+0000 mgr.y (mgr.14466) 92 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:07.856 INFO:teuthology.orchestra.run.smithi133.stdout:107374182479 2023-10-09T04:44:08.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:08 smithi133 bash[41291]: cluster 2023-10-09T04:44:07.115578+0000 mgr.y (mgr.14466) 93 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:08.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:08 smithi133 bash[41291]: audit 2023-10-09T04:44:07.853958+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.133:0/492516316' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T04:44:08.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:08 smithi133 bash[27558]: cluster 2023-10-09T04:44:07.115578+0000 mgr.y (mgr.14466) 93 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:08.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:08 smithi133 bash[27558]: audit 2023-10-09T04:44:07.853958+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.133:0/492516316' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T04:44:08.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:08 smithi194 bash[31648]: cluster 2023-10-09T04:44:07.115578+0000 mgr.y (mgr.14466) 93 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:08.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:08 smithi194 bash[31648]: audit 2023-10-09T04:44:07.853958+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.133:0/492516316' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T04:44:09.247 INFO:teuthology.orchestra.run.smithi133.stdout:55834574949 2023-10-09T04:44:09.365 INFO:teuthology.orchestra.run.smithi133.stdout:77309411419 2023-10-09T04:44:09.601 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:09 smithi133 bash[27558]: audit 2023-10-09T04:44:09.244640+0000 mon.a (mon.0) 783 : audit [DBG] from='client.? 172.21.15.133:0/4161588259' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T04:44:09.602 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:09 smithi133 bash[27558]: audit 2023-10-09T04:44:09.362227+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.133:0/2913017154' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T04:44:09.602 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:09 smithi133 bash[41291]: audit 2023-10-09T04:44:09.244640+0000 mon.a (mon.0) 783 : audit [DBG] from='client.? 172.21.15.133:0/4161588259' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T04:44:09.602 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:09 smithi133 bash[41291]: audit 2023-10-09T04:44:09.362227+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.133:0/2913017154' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T04:44:09.722 INFO:teuthology.orchestra.run.smithi133.stdout:128849018950 2023-10-09T04:44:09.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:09 smithi194 bash[31648]: audit 2023-10-09T04:44:09.244640+0000 mon.a (mon.0) 783 : audit [DBG] from='client.? 172.21.15.133:0/4161588259' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T04:44:09.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:09 smithi194 bash[31648]: audit 2023-10-09T04:44:09.362227+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.133:0/2913017154' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T04:44:10.478 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:44:10 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:44:10] "GET /metrics HTTP/1.1" 200 203296 "" "Prometheus/2.43.0" 2023-10-09T04:44:10.479 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182475 got 107374182479 for osd.3 2023-10-09T04:44:10.479 DEBUG:teuthology.parallel:result is None 2023-10-09T04:44:10.740 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:10 smithi133 bash[41291]: cluster 2023-10-09T04:44:09.115898+0000 mgr.y (mgr.14466) 94 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:11.260 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:10 smithi133 bash[41291]: audit 2023-10-09T04:44:09.720096+0000 mon.a (mon.0) 784 : audit [DBG] from='client.? 172.21.15.133:0/1331134739' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T04:44:11.260 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:10 smithi133 bash[27558]: cluster 2023-10-09T04:44:09.115898+0000 mgr.y (mgr.14466) 94 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:11.260 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:10 smithi133 bash[27558]: audit 2023-10-09T04:44:09.720096+0000 mon.a (mon.0) 784 : audit [DBG] from='client.? 172.21.15.133:0/1331134739' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T04:44:11.261 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:10 smithi194 bash[31648]: cluster 2023-10-09T04:44:09.115898+0000 mgr.y (mgr.14466) 94 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:11.261 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:10 smithi194 bash[31648]: audit 2023-10-09T04:44:09.720096+0000 mon.a (mon.0) 784 : audit [DBG] from='client.? 172.21.15.133:0/1331134739' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T04:44:12.072 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574945 got 55834574949 for osd.1 2023-10-09T04:44:12.090 DEBUG:teuthology.parallel:result is None 2023-10-09T04:44:12.506 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411415 got 77309411419 for osd.2 2023-10-09T04:44:12.506 DEBUG:teuthology.parallel:result is None 2023-10-09T04:44:12.512 INFO:teuthology.orchestra.run.smithi133.stdout:197568495665 2023-10-09T04:44:12.643 INFO:teuthology.orchestra.run.smithi133.stdout:163208757307 2023-10-09T04:44:12.669 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018946 got 128849018950 for osd.4 2023-10-09T04:44:12.669 DEBUG:teuthology.parallel:result is None 2023-10-09T04:44:12.765 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:12 smithi133 bash[27558]: cluster 2023-10-09T04:44:11.116490+0000 mgr.y (mgr.14466) 95 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:12.765 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:12 smithi133 bash[27558]: audit 2023-10-09T04:44:12.509650+0000 mon.a (mon.0) 785 : audit [DBG] from='client.? 172.21.15.133:0/2870578894' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T04:44:12.765 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:12 smithi133 bash[41291]: cluster 2023-10-09T04:44:11.116490+0000 mgr.y (mgr.14466) 95 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:12.765 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:12 smithi133 bash[41291]: audit 2023-10-09T04:44:12.509650+0000 mon.a (mon.0) 785 : audit [DBG] from='client.? 172.21.15.133:0/2870578894' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T04:44:12.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:12 smithi194 bash[31648]: cluster 2023-10-09T04:44:11.116490+0000 mgr.y (mgr.14466) 95 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:12.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:12 smithi194 bash[31648]: audit 2023-10-09T04:44:12.509650+0000 mon.a (mon.0) 785 : audit [DBG] from='client.? 172.21.15.133:0/2870578894' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T04:44:13.085 INFO:teuthology.orchestra.run.smithi133.stdout:34359738480 2023-10-09T04:44:13.151 INFO:teuthology.orchestra.run.smithi133.stdout:231928234023 2023-10-09T04:44:13.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:13 smithi133 bash[27558]: audit 2023-10-09T04:44:12.640586+0000 mon.a (mon.0) 786 : audit [DBG] from='client.? 172.21.15.133:0/3902941915' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T04:44:13.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:13 smithi133 bash[27558]: audit 2023-10-09T04:44:13.082213+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.133:0/1318918845' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T04:44:13.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:13 smithi133 bash[27558]: audit 2023-10-09T04:44:13.149069+0000 mon.a (mon.0) 787 : audit [DBG] from='client.? 172.21.15.133:0/4194025741' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T04:44:13.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:13 smithi133 bash[41291]: audit 2023-10-09T04:44:12.640586+0000 mon.a (mon.0) 786 : audit [DBG] from='client.? 172.21.15.133:0/3902941915' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T04:44:13.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:13 smithi133 bash[41291]: audit 2023-10-09T04:44:13.082213+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.133:0/1318918845' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T04:44:13.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:13 smithi133 bash[41291]: audit 2023-10-09T04:44:13.149069+0000 mon.a (mon.0) 787 : audit [DBG] from='client.? 172.21.15.133:0/4194025741' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T04:44:13.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:13 smithi194 bash[31648]: audit 2023-10-09T04:44:12.640586+0000 mon.a (mon.0) 786 : audit [DBG] from='client.? 172.21.15.133:0/3902941915' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T04:44:13.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:13 smithi194 bash[31648]: audit 2023-10-09T04:44:13.082213+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.133:0/1318918845' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T04:44:13.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:13 smithi194 bash[31648]: audit 2023-10-09T04:44:13.149069+0000 mon.a (mon.0) 787 : audit [DBG] from='client.? 172.21.15.133:0/4194025741' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T04:44:14.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:14 smithi133 bash[27558]: cluster 2023-10-09T04:44:13.116840+0000 mgr.y (mgr.14466) 96 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:14.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:14 smithi133 bash[41291]: cluster 2023-10-09T04:44:13.116840+0000 mgr.y (mgr.14466) 96 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:14.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:14 smithi194 bash[31648]: cluster 2023-10-09T04:44:13.116840+0000 mgr.y (mgr.14466) 96 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:15.139 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757302 got 163208757307 for osd.5 2023-10-09T04:44:15.139 DEBUG:teuthology.parallel:result is None 2023-10-09T04:44:15.304 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495660 got 197568495665 for osd.6 2023-10-09T04:44:15.304 DEBUG:teuthology.parallel:result is None 2023-10-09T04:44:15.501 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738475 got 34359738480 for osd.0 2023-10-09T04:44:15.502 DEBUG:teuthology.parallel:result is None 2023-10-09T04:44:15.585 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234017 got 231928234023 for osd.7 2023-10-09T04:44:15.585 DEBUG:teuthology.parallel:result is None 2023-10-09T04:44:15.586 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T04:44:15.586 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:44:16.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:16 smithi194 bash[31648]: cluster 2023-10-09T04:44:15.117275+0000 mgr.y (mgr.14466) 97 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:16.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:44:16 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:44:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:44:16.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:16 smithi133 bash[41291]: cluster 2023-10-09T04:44:15.117275+0000 mgr.y (mgr.14466) 97 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:16.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:16 smithi133 bash[27558]: cluster 2023-10-09T04:44:15.117275+0000 mgr.y (mgr.14466) 97 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:18.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:18 smithi133 bash[27558]: cluster 2023-10-09T04:44:17.117920+0000 mgr.y (mgr.14466) 98 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:18.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:18 smithi133 bash[41291]: cluster 2023-10-09T04:44:17.117920+0000 mgr.y (mgr.14466) 98 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:18.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:18 smithi194 bash[31648]: cluster 2023-10-09T04:44:17.117920+0000 mgr.y (mgr.14466) 98 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:20.354 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:44:20.800 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:44:20 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:44:20] "GET /metrics HTTP/1.1" 200 203310 "" "Prometheus/2.43.0" 2023-10-09T04:44:20.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:20 smithi133 bash[41291]: cluster 2023-10-09T04:44:19.118498+0000 mgr.y (mgr.14466) 99 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:20.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:20 smithi133 bash[41291]: audit 2023-10-09T04:44:20.188585+0000 mon.a (mon.0) 788 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:20.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:20 smithi133 bash[27558]: cluster 2023-10-09T04:44:19.118498+0000 mgr.y (mgr.14466) 99 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:20.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:20 smithi133 bash[27558]: audit 2023-10-09T04:44:20.188585+0000 mon.a (mon.0) 788 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:20.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:20 smithi194 bash[31648]: cluster 2023-10-09T04:44:19.118498+0000 mgr.y (mgr.14466) 99 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:20.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:20 smithi194 bash[31648]: audit 2023-10-09T04:44:20.188585+0000 mon.a (mon.0) 788 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:22.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:22 smithi133 bash[27558]: cluster 2023-10-09T04:44:21.119226+0000 mgr.y (mgr.14466) 100 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:22.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:22 smithi133 bash[41291]: cluster 2023-10-09T04:44:21.119226+0000 mgr.y (mgr.14466) 100 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:22.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:22 smithi194 bash[31648]: cluster 2023-10-09T04:44:21.119226+0000 mgr.y (mgr.14466) 100 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:23.842 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T04:44:23.846 INFO:teuthology.orchestra.run.smithi133.stderr:dumped all 2023-10-09T04:44:24.701 INFO:teuthology.orchestra.run.smithi133.stdout:{"pg_ready":true,"pg_map":{"version":76,"stamp":"2023-10-09T04:44:23.119510+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2384960,"kb_used_data":6656,"kb_used_omap":0,"kb_used_meta":2378240,"kb_avail":747576256,"statfs":{"total":767960285184,"available":765518086144,"internally_reserved":0,"allocated":6815744,"data_stored":3875675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2435317760},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003379"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:42:49.727613+0000","last_change":"2023-10-09T04:42:48.699613+0000","last_active":"2023-10-09T04:42:49.727613+0000","last_peered":"2023-10-09T04:42:49.727613+0000","last_clean":"2023-10-09T04:42:49.727613+0000","last_became_active":"2023-10-09T04:42:48.699095+0000","last_became_peered":"2023-10-09T04:42:48.699095+0000","last_unstale":"2023-10-09T04:42:49.727613+0000","last_undegraded":"2023-10-09T04:42:49.727613+0000","last_fullsized":"2023-10-09T04:42:49.727613+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:57.871245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:42:49.702307+0000","last_change":"2023-10-09T04:42:48.849748+0000","last_active":"2023-10-09T04:42:49.702307+0000","last_peered":"2023-10-09T04:42:49.702307+0000","last_clean":"2023-10-09T04:42:49.702307+0000","last_became_active":"2023-10-09T04:42:48.848834+0000","last_became_peered":"2023-10-09T04:42:48.848834+0000","last_unstale":"2023-10-09T04:42:49.702307+0000","last_undegraded":"2023-10-09T04:42:49.702307+0000","last_fullsized":"2023-10-09T04:42:49.702307+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:20.475654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:42:49.705049+0000","last_change":"2023-10-09T04:42:48.851447+0000","last_active":"2023-10-09T04:42:49.705049+0000","last_peered":"2023-10-09T04:42:49.705049+0000","last_clean":"2023-10-09T04:42:49.705049+0000","last_became_active":"2023-10-09T04:42:48.850687+0000","last_became_peered":"2023-10-09T04:42:48.850687+0000","last_unstale":"2023-10-09T04:42:49.705049+0000","last_undegraded":"2023-10-09T04:42:49.705049+0000","last_fullsized":"2023-10-09T04:42:49.705049+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:32:22.356405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:42:49.705080+0000","last_change":"2023-10-09T04:42:48.697850+0000","last_active":"2023-10-09T04:42:49.705080+0000","last_peered":"2023-10-09T04:42:49.705080+0000","last_clean":"2023-10-09T04:42:49.705080+0000","last_became_active":"2023-10-09T04:42:48.697520+0000","last_became_peered":"2023-10-09T04:42:48.697520+0000","last_unstale":"2023-10-09T04:42:49.705080+0000","last_undegraded":"2023-10-09T04:42:49.705080+0000","last_fullsized":"2023-10-09T04:42:49.705080+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:04.728265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":267,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:42:49.704995+0000","last_change":"2023-10-09T04:41:09.432397+0000","last_active":"2023-10-09T04:42:49.704995+0000","last_peered":"2023-10-09T04:42:49.704995+0000","last_clean":"2023-10-09T04:42:49.704995+0000","last_became_active":"2023-10-09T04:41:09.431675+0000","last_became_peered":"2023-10-09T04:41:09.431675+0000","last_unstale":"2023-10-09T04:42:49.704995+0000","last_undegraded":"2023-10-09T04:42:49.704995+0000","last_fullsized":"2023-10-09T04:42:49.704995+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_clean_scrub_stamp":"2023-10-09T04:36:44.261013+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:23.657333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:42:49.767070+0000","last_change":"2023-10-09T04:42:48.698570+0000","last_active":"2023-10-09T04:42:49.767070+0000","last_peered":"2023-10-09T04:42:49.767070+0000","last_clean":"2023-10-09T04:42:49.767070+0000","last_became_active":"2023-10-09T04:42:48.698098+0000","last_became_peered":"2023-10-09T04:42:48.698098+0000","last_unstale":"2023-10-09T04:42:49.767070+0000","last_undegraded":"2023-10-09T04:42:49.767070+0000","last_fullsized":"2023-10-09T04:42:49.767070+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:35.340926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:42:49.765494+0000","last_change":"2023-10-09T04:42:48.728368+0000","last_active":"2023-10-09T04:42:49.765494+0000","last_peered":"2023-10-09T04:42:49.765494+0000","last_clean":"2023-10-09T04:42:49.765494+0000","last_became_active":"2023-10-09T04:42:48.727757+0000","last_became_peered":"2023-10-09T04:42:48.727757+0000","last_unstale":"2023-10-09T04:42:49.765494+0000","last_undegraded":"2023-10-09T04:42:49.765494+0000","last_fullsized":"2023-10-09T04:42:49.765494+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:21:02.404428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:42:49.767080+0000","last_change":"2023-10-09T04:42:48.698230+0000","last_active":"2023-10-09T04:42:49.767080+0000","last_peered":"2023-10-09T04:42:49.767080+0000","last_clean":"2023-10-09T04:42:49.767080+0000","last_became_active":"2023-10-09T04:42:48.697694+0000","last_became_peered":"2023-10-09T04:42:48.697694+0000","last_unstale":"2023-10-09T04:42:49.767080+0000","last_undegraded":"2023-10-09T04:42:49.767080+0000","last_fullsized":"2023-10-09T04:42:49.767080+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:46:21.257527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:42:49.702308+0000","last_change":"2023-10-09T04:42:48.696990+0000","last_active":"2023-10-09T04:42:49.702308+0000","last_peered":"2023-10-09T04:42:49.702308+0000","last_clean":"2023-10-09T04:42:49.702308+0000","last_became_active":"2023-10-09T04:42:48.696505+0000","last_became_peered":"2023-10-09T04:42:48.696505+0000","last_unstale":"2023-10-09T04:42:49.702308+0000","last_undegraded":"2023-10-09T04:42:49.702308+0000","last_fullsized":"2023-10-09T04:42:49.702308+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:55.113687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234025,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298476,"kb_used_data":1188,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446676,"statfs":{"total":95995035648,"available":95689396224,"internally_reserved":0,"allocated":1216512,"data_stored":852958,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:44:06 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":0.93799999999999994,"15min":0.95299999999999996},"min":{"1min":0.55700000000000005,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.3300000000000001,"5min":1.458,"15min":1.458},"last":0.82999999999999996},{"interface":"front","average":{"1min":0.90300000000000002,"5min":0.94099999999999995,"15min":0.96099999999999997},"min":{"1min":0.65500000000000003,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.145,"5min":1.5580000000000001,"15min":1.5580000000000001},"last":1.0129999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:44:06 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":1.0049999999999999,"15min":1.0269999999999999},"min":{"1min":0.68100000000000005,"5min":0.622,"15min":0.622},"max":{"1min":1.3240000000000001,"5min":1.423,"15min":1.423},"last":1.3340000000000001},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.91900000000000004,"15min":0.93300000000000005},"min":{"1min":0.59399999999999997,"5min":0.435,"15min":0.435},"max":{"1min":1.208,"5min":1.9950000000000001,"15min":1.9950000000000001},"last":0.89600000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:44:06 2023","interfaces":[{"interface":"back","average":{"1min":0.86099999999999999,"5min":0.98499999999999999,"15min":1.022},"min":{"1min":0.56299999999999994,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.1539999999999999,"5min":2.0819999999999999,"15min":2.0819999999999999},"last":0.78700000000000003},{"interface":"front","average":{"1min":0.94099999999999995,"5min":0.96399999999999997,"15min":0.98899999999999999},"min":{"1min":0.59199999999999997,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":1.3169999999999999,"5min":1.4730000000000001,"15min":1.4730000000000001},"last":1.2829999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:44:06 2023","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":0.86799999999999999,"15min":0.874},"min":{"1min":0.59499999999999997,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.038,"5min":1.4159999999999999,"15min":1.4159999999999999},"last":1.107},{"interface":"front","average":{"1min":1.0760000000000001,"5min":1.0549999999999999,"15min":1.071},"min":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.4039999999999999,"5min":1.5129999999999999,"15min":1.5129999999999999},"last":1.165}]},{"osd":4,"last update":"Mon Oct 9 04:44:06 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":0.84799999999999998,"15min":0.81999999999999995},"min":{"1min":0.54200000000000004,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.2969999999999999,"5min":1.2969999999999999,"15min":1.2969999999999999},"last":1.1359999999999999},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.83099999999999996,"15min":0.80400000000000005},"min":{"1min":0.54100000000000004,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.169,"5min":1.1890000000000001,"15min":1.1890000000000001},"last":0.96499999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:44:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":0.99399999999999999,"15min":0.998},"min":{"1min":0.55700000000000005,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.419,"5min":1.827,"15min":1.827},"last":1.236},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.0289999999999999,"15min":1.0449999999999999},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.349,"5min":1.9179999999999999,"15min":1.9179999999999999},"last":1.1899999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:44:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.0329999999999999,"15min":1.0389999999999999},"min":{"1min":0.58099999999999996,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.454,"5min":1.454,"15min":1.454},"last":1.073},{"interface":"front","average":{"1min":1.0209999999999999,"5min":0.95499999999999996,"15min":0.94699999999999995},"min":{"1min":0.68600000000000005,"5min":0.505,"15min":0.505},"max":{"1min":1.3089999999999999,"5min":1.3089999999999999,"15min":1.3089999999999999},"last":1.206}]}]},{"osd":6,"up_from":46,"seq":197568495667,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298480,"kb_used_data":1192,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446672,"statfs":{"total":95995035648,"available":95689392128,"internally_reserved":0,"allocated":1220608,"data_stored":853281,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:44:19 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":0.89500000000000002,"15min":0.94799999999999995},"min":{"1min":0.47599999999999998,"5min":0.437,"15min":0.437},"max":{"1min":1.202,"5min":1.7669999999999999,"15min":1.7669999999999999},"last":0.77200000000000002},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.90800000000000003,"15min":0.96599999999999997},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.1379999999999999,"5min":1.3149999999999999,"15min":1.3149999999999999},"last":1.1160000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:44:19 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":0.88300000000000001,"15min":0.90300000000000002},"min":{"1min":0.56200000000000006,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":1.1619999999999999,"5min":1.2949999999999999,"15min":1.2949999999999999},"last":0.92700000000000005},{"interface":"front","average":{"1min":0.91600000000000004,"5min":0.90200000000000002,"15min":0.91800000000000004},"min":{"1min":0.71599999999999997,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.4059999999999999,"5min":1.4850000000000001,"15min":1.4850000000000001},"last":0.96799999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:44:19 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":0.95099999999999996,"15min":0.95299999999999996},"min":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"max":{"1min":3.238,"5min":3.238,"15min":3.238},"last":0.83399999999999996},{"interface":"front","average":{"1min":0.88500000000000001,"5min":0.90400000000000003,"15min":0.91800000000000004},"min":{"1min":0.64600000000000002,"5min":0.625,"15min":0.625},"max":{"1min":1.101,"5min":1.238,"15min":1.238},"last":1.071}]},{"osd":3,"last update":"Mon Oct 9 04:44:19 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":0.95999999999999996,"15min":0.98199999999999998},"min":{"1min":0.63100000000000001,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.1950000000000001,"5min":1.8879999999999999,"15min":1.8879999999999999},"last":1.0269999999999999},{"interface":"front","average":{"1min":1.0149999999999999,"5min":0.95899999999999996,"15min":0.96899999999999997},"min":{"1min":0.55500000000000005,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.3460000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":1.1539999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:44:19 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.84599999999999997,"15min":0.80400000000000005},"min":{"1min":0.56599999999999995,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.302,"5min":1.302,"15min":1.302},"last":0.67100000000000004},{"interface":"front","average":{"1min":1.004,"5min":0.91700000000000004,"15min":0.90700000000000003},"min":{"1min":0.74299999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.151,"5min":1.4319999999999999,"15min":1.4319999999999999},"last":0.86499999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:44:19 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":0.92300000000000004,"15min":0.89400000000000002},"min":{"1min":0.69199999999999995,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.3100000000000001,"5min":1.397,"15min":1.397},"last":1.1870000000000001},{"interface":"front","average":{"1min":1.075,"5min":0.97399999999999998,"15min":0.96399999999999997},"min":{"1min":0.73499999999999999,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.2709999999999999,"5min":1.6000000000000001,"15min":1.6000000000000001},"last":1.22}]},{"osd":7,"last update":"Mon Oct 9 04:44:08 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":0.96499999999999997,"15min":0.96099999999999997},"min":{"1min":0.73999999999999999,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.452,"5min":1.452,"15min":1.452},"last":1.2629999999999999},{"interface":"front","average":{"1min":1.2250000000000001,"5min":1.1120000000000001,"15min":1.1000000000000001},"min":{"1min":0.94799999999999995,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.5049999999999999,"5min":1.5660000000000001,"15min":1.5660000000000001},"last":1.3129999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574952,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:44:05 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.69299999999999995,"15min":0.60199999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.84099999999999997,"5min":1.179,"15min":1.179},"last":0.92400000000000004},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.75700000000000001,"15min":0.65000000000000002},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.069,"5min":1.2170000000000001,"15min":1.2170000000000001},"last":0.72799999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:43:49 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.76900000000000002,"15min":0.68300000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.95499999999999996,"5min":1.484,"15min":1.484},"last":0.84599999999999997},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.76000000000000001,"15min":0.68000000000000005},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.282,"5min":1.371,"15min":1.371},"last":0.63400000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:43:46 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":0.81200000000000006,"15min":0.80900000000000005},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":1.1100000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.79400000000000004},{"interface":"front","average":{"1min":0.73099999999999998,"5min":0.80800000000000005,"15min":0.78200000000000003},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":1.133,"5min":1.133,"15min":1.133},"last":0.68400000000000005}]},{"osd":4,"last update":"Mon Oct 9 04:43:40 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":0.998,"15min":0.97399999999999998},"min":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.2490000000000001,"5min":1.607,"15min":1.607},"last":1.0369999999999999},{"interface":"front","average":{"1min":0.98799999999999999,"5min":1.026,"15min":0.98599999999999999},"min":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"max":{"1min":1.395,"5min":1.4239999999999999,"15min":1.4239999999999999},"last":0.97999999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:43:26 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":1.0049999999999999,"15min":1.0049999999999999},"min":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"max":{"1min":1.55,"5min":1.55,"15min":1.55},"last":1.0580000000000001},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.0229999999999999,"15min":1.006},"min":{"1min":0.78600000000000003,"5min":0.69099999999999995,"15min":0.69099999999999995},"max":{"1min":1.4590000000000001,"5min":1.5449999999999999,"15min":1.5449999999999999},"last":0.88900000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:43:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.0649999999999999,"15min":1.0649999999999999},"min":{"1min":0.69499999999999995,"5min":0.57599999999999996,"15min":0.57599999999999996},"max":{"1min":1.2150000000000001,"5min":1.639,"15min":1.639},"last":1.018},{"interface":"front","average":{"1min":1.069,"5min":1.091,"15min":1.0920000000000001},"min":{"1min":0.69299999999999995,"5min":0.65400000000000003,"15min":0.65400000000000003},"max":{"1min":1.331,"5min":1.552,"15min":1.552},"last":1.1040000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:44:07 2023","interfaces":[{"interface":"back","average":{"1min":1.101,"5min":1.135,"15min":1.153},"min":{"1min":0.80900000000000005,"5min":0.78600000000000003,"15min":0.78600000000000003},"max":{"1min":1.3839999999999999,"5min":1.758,"15min":1.758},"last":1.085},{"interface":"front","average":{"1min":1.0600000000000001,"5min":1.147,"15min":1.1739999999999999},"min":{"1min":0.63900000000000001,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.3280000000000001,"5min":1.6899999999999999,"15min":1.6899999999999999},"last":1.1539999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738482,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298484,"kb_used_data":1196,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446668,"statfs":{"total":95995035648,"available":95689388032,"internally_reserved":0,"allocated":1224704,"data_stored":853546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:44:05 2023","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":0.72699999999999998,"15min":0.59899999999999998},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.26900000000000002},"max":{"1min":1.3240000000000001,"5min":1.3240000000000001,"15min":1.3240000000000001},"last":0.76700000000000002},{"interface":"front","average":{"1min":0.71199999999999997,"5min":0.745,"15min":0.63900000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.032,"5min":1.103,"15min":1.103},"last":0.71699999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:43:53 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.78500000000000003,"15min":0.67400000000000004},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.378},"max":{"1min":1.319,"5min":1.319,"15min":1.319},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.79800000000000004,"15min":0.69399999999999995},"min":{"1min":0.48999999999999999,"5min":0.40000000000000002,"15min":0.34499999999999997},"max":{"1min":1.274,"5min":1.2789999999999999,"15min":1.2789999999999999},"last":0.66800000000000004}]},{"osd":3,"last update":"Mon Oct 9 04:43:47 2023","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":0.85599999999999998,"15min":0.83999999999999997},"min":{"1min":0.58699999999999997,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":1.2210000000000001,"5min":1.4299999999999999,"15min":1.4299999999999999},"last":0.84899999999999998},{"interface":"front","average":{"1min":0.75600000000000001,"5min":0.79300000000000004,"15min":0.80000000000000004},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.96099999999999997,"5min":1.2190000000000001,"15min":1.264},"last":0.49299999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:43:44 2023","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":0.95199999999999996,"15min":0.93400000000000005},"min":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"max":{"1min":1.2050000000000001,"5min":1.256,"15min":1.256},"last":0.90400000000000003},{"interface":"front","average":{"1min":1.0189999999999999,"5min":0.98699999999999999,"15min":0.95199999999999996},"min":{"1min":0.72499999999999998,"5min":0.67900000000000005,"15min":0.67900000000000005},"max":{"1min":1.3149999999999999,"5min":1.3400000000000001,"15min":1.3400000000000001},"last":0.75700000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:43:28 2023","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":1.05,"15min":1.0409999999999999},"min":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"max":{"1min":1.276,"5min":1.448,"15min":1.448},"last":1.0609999999999999},{"interface":"front","average":{"1min":0.98099999999999998,"5min":1.0069999999999999,"15min":1.01},"min":{"1min":0.72199999999999998,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.206,"5min":1.3879999999999999,"15min":1.3879999999999999},"last":0.72899999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:43:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.081,"15min":1.0940000000000001},"min":{"1min":0.82999999999999996,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":1.5760000000000001,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":0.80600000000000005},{"interface":"front","average":{"1min":1.081,"5min":1.1359999999999999,"15min":1.1539999999999999},"min":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"max":{"1min":1.385,"5min":1.522,"15min":1.522},"last":0.92000000000000004}]},{"osd":7,"last update":"Mon Oct 9 04:44:11 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":1.101,"15min":1.131},"min":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"max":{"1min":1.2829999999999999,"5min":1.5189999999999999,"15min":1.5189999999999999},"last":0.88100000000000001},{"interface":"front","average":{"1min":1.1639999999999999,"5min":1.161,"15min":1.177},"min":{"1min":0.68200000000000005,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":2.097,"5min":2.097,"15min":2.097},"last":0.99399999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411422,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:43:57 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.71399999999999997,"15min":0.66700000000000004},"min":{"1min":0.45100000000000001,"5min":0.437,"15min":0.437},"max":{"1min":1.1850000000000001,"5min":1.1850000000000001,"15min":1.1850000000000001},"last":0.62},{"interface":"front","average":{"1min":0.69999999999999996,"5min":0.76800000000000002,"15min":0.69399999999999995},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.93999999999999995,"5min":1.2989999999999999,"15min":1.2989999999999999},"last":0.92100000000000004}]},{"osd":1,"last update":"Mon Oct 9 04:43:57 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":0.77500000000000002,"15min":0.69399999999999995},"min":{"1min":0.497,"5min":0.497,"15min":0.42999999999999999},"max":{"1min":1.165,"5min":1.272,"15min":1.272},"last":0.83199999999999996},{"interface":"front","average":{"1min":0.751,"5min":0.82199999999999995,"15min":0.71599999999999997},"min":{"1min":0.48299999999999998,"5min":0.46300000000000002,"15min":0.441},"max":{"1min":1.042,"5min":1.395,"15min":1.395},"last":0.79200000000000004}]},{"osd":3,"last update":"Mon Oct 9 04:43:48 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.85199999999999998,"15min":0.80800000000000005},"min":{"1min":0.52600000000000002,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.4259999999999999,"5min":1.4259999999999999,"15min":1.4259999999999999},"last":0.753},{"interface":"front","average":{"1min":0.83099999999999996,"5min":0.88500000000000001,"15min":0.85199999999999998},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.2729999999999999,"5min":1.306,"15min":1.306},"last":1.032}]},{"osd":4,"last update":"Mon Oct 9 04:43:41 2023","interfaces":[{"interface":"back","average":{"1min":0.98399999999999999,"5min":1.01,"15min":0.96099999999999997},"min":{"1min":0.71299999999999997,"5min":0.69299999999999995,"15min":0.69299999999999995},"max":{"1min":1.2909999999999999,"5min":1.72,"15min":1.72},"last":0.89300000000000002},{"interface":"front","average":{"1min":0.995,"5min":1.0329999999999999,"15min":1.01},"min":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"max":{"1min":1.3160000000000001,"5min":1.4790000000000001,"15min":1.4790000000000001},"last":1.107}]},{"osd":5,"last update":"Mon Oct 9 04:43:32 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":1.0720000000000001,"15min":1.0720000000000001},"min":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"max":{"1min":1.423,"5min":1.6180000000000001,"15min":1.6180000000000001},"last":1.254},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.0720000000000001,"15min":1.0600000000000001},"min":{"1min":0.621,"5min":0.621,"15min":0.621},"max":{"1min":1.347,"5min":1.6659999999999999,"15min":1.6659999999999999},"last":0.95299999999999996}]},{"osd":6,"last update":"Mon Oct 9 04:44:19 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.109,"15min":1.155},"min":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"max":{"1min":1.367,"5min":1.706,"15min":1.706},"last":1.2969999999999999},{"interface":"front","average":{"1min":0.93000000000000005,"5min":1.014,"15min":1.0580000000000001},"min":{"1min":0.64200000000000002,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.228,"5min":1.4330000000000001,"15min":1.4330000000000001},"last":0.98199999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:44:10 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":1.194,"15min":1.236},"min":{"1min":0.77100000000000002,"5min":0.622,"15min":0.622},"max":{"1min":1.5840000000000001,"5min":1.7729999999999999,"15min":1.7729999999999999},"last":1.012},{"interface":"front","average":{"1min":1.127,"5min":1.2070000000000001,"15min":1.238},"min":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"max":{"1min":1.454,"5min":1.454,"15min":1.454},"last":1.2010000000000001}]}]},{"osd":3,"up_from":25,"seq":107374182482,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:43:49 2023","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":0.68999999999999995,"15min":0.69799999999999995},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":1.091,"5min":1.091,"15min":1.091},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.76600000000000001,"15min":0.73499999999999999},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.0669999999999999,"5min":1.266,"15min":1.266},"last":0.73899999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:43:49 2023","interfaces":[{"interface":"back","average":{"1min":0.71999999999999997,"5min":0.76700000000000002,"15min":0.75700000000000001},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":1.1699999999999999,"5min":1.1819999999999999,"15min":1.1819999999999999},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.75600000000000001,"15min":0.76100000000000001},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.94299999999999995,"5min":1.7250000000000001,"15min":1.7250000000000001},"last":0.88700000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:43:49 2023","interfaces":[{"interface":"back","average":{"1min":0.74099999999999999,"5min":0.86699999999999999,"15min":0.86099999999999999},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.98499999999999999,"5min":1.8320000000000001,"15min":1.8320000000000001},"last":0.80500000000000005},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.79600000000000004,"15min":0.81399999999999995},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.173,"5min":1.1779999999999999,"15min":1.218},"last":0.54200000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:43:45 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.97399999999999998,"15min":0.94999999999999996},"min":{"1min":0.751,"5min":0.66000000000000003,"15min":0.66000000000000003},"max":{"1min":1.202,"5min":1.5009999999999999,"15min":1.5009999999999999},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.96399999999999997,"5min":1.014,"15min":0.95299999999999996},"min":{"1min":0.67400000000000004,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.391,"5min":2.298,"15min":2.298},"last":1.3759999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:43:27 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.0269999999999999,"15min":1.0249999999999999},"min":{"1min":0.68799999999999994,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.296,"5min":1.603,"15min":1.603},"last":0.83099999999999996},{"interface":"front","average":{"1min":0.97599999999999998,"5min":1.0600000000000001,"15min":1.0649999999999999},"min":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"max":{"1min":1.2649999999999999,"5min":1.9930000000000001,"15min":1.9930000000000001},"last":1.4630000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:44:19 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":1.073,"15min":1.089},"min":{"1min":0.72899999999999998,"5min":0.71599999999999997,"15min":0.71599999999999997},"max":{"1min":1.621,"5min":2.1030000000000002,"15min":2.1030000000000002},"last":1.419},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.0660000000000001,"15min":1.0820000000000001},"min":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":1.29,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":0.85999999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:44:10 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.1319999999999999,"15min":1.1739999999999999},"min":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"max":{"1min":1.3300000000000001,"5min":1.367,"15min":1.367},"last":1.008},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.1830000000000001,"15min":1.22},"min":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"max":{"1min":1.534,"5min":1.534,"15min":1.534},"last":1.3169999999999999}]}]},{"osd":4,"up_from":30,"seq":128849018953,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:43:41 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":0.88600000000000001,"15min":0.85599999999999998},"min":{"1min":0.54500000000000004,"5min":0.505,"15min":0.505},"max":{"1min":1.0940000000000001,"5min":1.294,"15min":1.294},"last":1.2110000000000001},{"interface":"front","average":{"1min":0.89500000000000002,"5min":0.89900000000000002,"15min":0.89700000000000002},"min":{"1min":0.57899999999999996,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":1.1160000000000001,"5min":1.8180000000000001,"15min":1.8180000000000001},"last":1.0169999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:43:41 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":0.89500000000000002,"15min":0.85299999999999998},"min":{"1min":0.57999999999999996,"5min":0.504,"15min":0.504},"max":{"1min":1.278,"5min":1.278,"15min":1.278},"last":1.4239999999999999},{"interface":"front","average":{"1min":0.96699999999999997,"5min":0.92500000000000004,"15min":0.89600000000000002},"min":{"1min":0.66900000000000004,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.4099999999999999,"5min":1.9039999999999999,"15min":1.9039999999999999},"last":0.84399999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:43:41 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.94299999999999995,"15min":0.91300000000000003},"min":{"1min":0.60499999999999998,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.548,"5min":1.724,"15min":1.724},"last":1.119},{"interface":"front","average":{"1min":0.94599999999999995,"5min":0.90000000000000002,"15min":0.89200000000000002},"min":{"1min":0.73999999999999999,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.23,"5min":1.25,"15min":1.25},"last":1.048}]},{"osd":3,"last update":"Mon Oct 9 04:43:41 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":0.91900000000000004,"15min":0.91000000000000003},"min":{"1min":0.66600000000000004,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.319,"5min":1.319,"15min":1.319},"last":0.80400000000000005},{"interface":"front","average":{"1min":0.998,"5min":0.94299999999999995,"15min":0.91400000000000003},"min":{"1min":0.72899999999999998,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.377,"5min":1.377,"15min":1.377},"last":1.375}]},{"osd":5,"last update":"Mon Oct 9 04:43:29 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":0.80500000000000005,"15min":0.76200000000000001},"min":{"1min":0.47499999999999998,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":1.288,"5min":1.3819999999999999,"15min":1.3819999999999999},"last":1.1799999999999999},{"interface":"front","average":{"1min":0.97599999999999998,"5min":0.83399999999999996,"15min":0.77000000000000002},"min":{"1min":0.60099999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.6539999999999999,"5min":1.6539999999999999,"15min":1.6539999999999999},"last":0.91200000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:44:19 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":0.879,"15min":0.79500000000000004},"min":{"1min":0.70599999999999996,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.327,"5min":1.474,"15min":1.474},"last":1.3169999999999999},{"interface":"front","average":{"1min":1.1319999999999999,"5min":0.999,"15min":0.96699999999999997},"min":{"1min":0.81599999999999995,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.4279999999999999,"5min":1.9850000000000001,"15min":1.9850000000000001},"last":1.2609999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:44:11 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":0.999,"15min":0.97799999999999998},"min":{"1min":0.54100000000000004,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.268,"5min":1.752,"15min":1.752},"last":1.155},{"interface":"front","average":{"1min":1.1890000000000001,"5min":1.0720000000000001,"15min":1.034},"min":{"1min":0.71699999999999997,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.4390000000000001,"5min":1.6180000000000001,"15min":1.6180000000000001},"last":1.478}]}]},{"osd":5,"up_from":38,"seq":163208757309,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:43:29 2023","interfaces":[{"interface":"back","average":{"1min":0.85999999999999999,"5min":0.88,"15min":0.872},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":1.161,"5min":1.3720000000000001,"15min":1.3720000000000001},"last":1.0620000000000001},{"interface":"front","average":{"1min":0.871,"5min":0.92600000000000005,"15min":0.93000000000000005},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.1879999999999999,"5min":1.7869999999999999,"15min":1.7869999999999999},"last":1.248}]},{"osd":1,"last update":"Mon Oct 9 04:43:29 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.90900000000000003,"15min":0.94799999999999995},"min":{"1min":0.64200000000000002,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.0940000000000001,"5min":2.0510000000000002,"15min":2.0510000000000002},"last":1.024},{"interface":"front","average":{"1min":0.88800000000000001,"5min":0.89700000000000002,"15min":0.88500000000000001},"min":{"1min":0.68700000000000006,"5min":0.63500000000000001,"15min":0.63500000000000001},"max":{"1min":1.0649999999999999,"5min":1.2889999999999999,"15min":1.2889999999999999},"last":0.75900000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:43:29 2023","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":0.88200000000000001,"15min":0.85699999999999998},"min":{"1min":0.54400000000000004,"5min":0.505,"15min":0.505},"max":{"1min":1.024,"5min":1.204,"15min":1.204},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.91200000000000003,"5min":0.95699999999999996,"15min":0.95799999999999996},"min":{"1min":0.67000000000000004,"5min":0.60599999999999998,"15min":0.60599999999999998},"max":{"1min":1.1499999999999999,"5min":1.885,"15min":1.885},"last":1.179}]},{"osd":3,"last update":"Mon Oct 9 04:43:29 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.93600000000000005,"15min":0.92100000000000004},"min":{"1min":0.746,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.2110000000000001,"5min":1.321,"15min":1.321},"last":0.92600000000000005},{"interface":"front","average":{"1min":0.996,"5min":1.004,"15min":0.996},"min":{"1min":0.76100000000000001,"5min":0.61099999999999999,"15min":0.61099999999999999},"max":{"1min":1.212,"5min":1.9710000000000001,"15min":1.9710000000000001},"last":0.79900000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:43:29 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":0.79500000000000004,"15min":0.76100000000000001},"min":{"1min":0.5,"5min":0.5,"15min":0.5},"max":{"1min":0.96799999999999997,"5min":1.3759999999999999,"15min":1.3759999999999999},"last":0.97399999999999998},{"interface":"front","average":{"1min":0.86199999999999999,"5min":0.88400000000000001,"15min":0.82699999999999996},"min":{"1min":0.60899999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.1779999999999999,"5min":1.5269999999999999,"15min":1.5269999999999999},"last":0.83199999999999996}]},{"osd":6,"last update":"Mon Oct 9 04:43:21 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":0.92200000000000004,"15min":0.89900000000000002},"min":{"1min":0.69599999999999995,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.4670000000000001,"5min":1.4670000000000001,"15min":1.4670000000000001},"last":1.2250000000000001},{"interface":"front","average":{"1min":0.86599999999999999,"5min":1.0109999999999999,"15min":1.0349999999999999},"min":{"1min":0.57799999999999996,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.272,"5min":2.169,"15min":2.169},"last":1.304}]},{"osd":7,"last update":"Mon Oct 9 04:44:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.0549999999999999,"15min":1.0389999999999999},"min":{"1min":0.625,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.3759999999999999,"5min":1.5049999999999999,"15min":1.5049999999999999},"last":1.121},{"interface":"front","average":{"1min":1.091,"5min":0.98599999999999999,"15min":0.95799999999999996},"min":{"1min":0.80200000000000005,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.4710000000000001,"5min":1.4710000000000001,"15min":1.4710000000000001},"last":1.095}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:44:24.704 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:44:24.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:24 smithi194 bash[31648]: cluster 2023-10-09T04:44:23.119791+0000 mgr.y (mgr.14466) 101 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:24.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:24 smithi194 bash[31648]: audit 2023-10-09T04:44:23.829644+0000 mgr.y (mgr.14466) 102 : audit [DBG] from='client.24551 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:44:25.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:24 smithi133 bash[27558]: cluster 2023-10-09T04:44:23.119791+0000 mgr.y (mgr.14466) 101 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:25.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:24 smithi133 bash[27558]: audit 2023-10-09T04:44:23.829644+0000 mgr.y (mgr.14466) 102 : audit [DBG] from='client.24551 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:44:25.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:24 smithi133 bash[41291]: cluster 2023-10-09T04:44:23.119791+0000 mgr.y (mgr.14466) 101 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:25.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:24 smithi133 bash[41291]: audit 2023-10-09T04:44:23.829644+0000 mgr.y (mgr.14466) 102 : audit [DBG] from='client.24551 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:44:26.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:26 smithi194 bash[31648]: cluster 2023-10-09T04:44:25.120412+0000 mgr.y (mgr.14466) 103 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:26.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:44:26 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:44:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:44:27.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:26 smithi133 bash[27558]: cluster 2023-10-09T04:44:25.120412+0000 mgr.y (mgr.14466) 103 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:27.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:26 smithi133 bash[41291]: cluster 2023-10-09T04:44:25.120412+0000 mgr.y (mgr.14466) 103 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:27.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:27 smithi194 bash[31648]: audit 2023-10-09T04:44:26.966665+0000 mon.a (mon.0) 789 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:44:27.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:27 smithi194 bash[31648]: audit 2023-10-09T04:44:27.495151+0000 mon.a (mon.0) 790 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:44:27.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:27 smithi194 bash[31648]: audit 2023-10-09T04:44:27.497072+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:44:27.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:27 smithi194 bash[31648]: audit 2023-10-09T04:44:27.505843+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:28.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:27 smithi133 bash[41291]: audit 2023-10-09T04:44:26.966665+0000 mon.a (mon.0) 789 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:44:28.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:27 smithi133 bash[41291]: audit 2023-10-09T04:44:27.495151+0000 mon.a (mon.0) 790 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:44:28.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:27 smithi133 bash[41291]: audit 2023-10-09T04:44:27.497072+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:44:28.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:27 smithi133 bash[41291]: audit 2023-10-09T04:44:27.505843+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:28.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:27 smithi133 bash[27558]: audit 2023-10-09T04:44:26.966665+0000 mon.a (mon.0) 789 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:44:28.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:27 smithi133 bash[27558]: audit 2023-10-09T04:44:27.495151+0000 mon.a (mon.0) 790 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:44:28.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:27 smithi133 bash[27558]: audit 2023-10-09T04:44:27.497072+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:44:28.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:27 smithi133 bash[27558]: audit 2023-10-09T04:44:27.505843+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:28.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:28 smithi194 bash[31648]: cluster 2023-10-09T04:44:27.121046+0000 mgr.y (mgr.14466) 104 : cluster [DBG] pgmap v78: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:29.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:28 smithi133 bash[41291]: cluster 2023-10-09T04:44:27.121046+0000 mgr.y (mgr.14466) 104 : cluster [DBG] pgmap v78: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:29.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:28 smithi133 bash[27558]: cluster 2023-10-09T04:44:27.121046+0000 mgr.y (mgr.14466) 104 : cluster [DBG] pgmap v78: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:29.475 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:44:30.645 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:44:30 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:44:30] "GET /metrics HTTP/1.1" 200 203310 "" "Prometheus/2.43.0" 2023-10-09T04:44:30.646 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:30 smithi133 bash[41291]: cluster 2023-10-09T04:44:29.121599+0000 mgr.y (mgr.14466) 105 : cluster [DBG] pgmap v79: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:30.646 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:30 smithi133 bash[27558]: cluster 2023-10-09T04:44:29.121599+0000 mgr.y (mgr.14466) 105 : cluster [DBG] pgmap v79: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:30.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:30 smithi194 bash[31648]: cluster 2023-10-09T04:44:29.121599+0000 mgr.y (mgr.14466) 105 : cluster [DBG] pgmap v79: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:32.798 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T04:44:32.799 INFO:teuthology.orchestra.run.smithi133.stderr:dumped all 2023-10-09T04:44:32.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:32 smithi133 bash[41291]: cluster 2023-10-09T04:44:31.122281+0000 mgr.y (mgr.14466) 106 : cluster [DBG] pgmap v80: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:32.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:32 smithi133 bash[27558]: cluster 2023-10-09T04:44:31.122281+0000 mgr.y (mgr.14466) 106 : cluster [DBG] pgmap v80: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:32.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:32 smithi194 bash[31648]: cluster 2023-10-09T04:44:31.122281+0000 mgr.y (mgr.14466) 106 : cluster [DBG] pgmap v80: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:33.753 INFO:teuthology.orchestra.run.smithi133.stdout:{"pg_ready":true,"pg_map":{"version":80,"stamp":"2023-10-09T04:44:31.121913+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2384960,"kb_used_data":6656,"kb_used_omap":0,"kb_used_meta":2378240,"kb_avail":747576256,"statfs":{"total":767960285184,"available":765518086144,"internally_reserved":0,"allocated":6815744,"data_stored":3875675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2435317760},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003708"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:42:49.727613+0000","last_change":"2023-10-09T04:42:48.699613+0000","last_active":"2023-10-09T04:42:49.727613+0000","last_peered":"2023-10-09T04:42:49.727613+0000","last_clean":"2023-10-09T04:42:49.727613+0000","last_became_active":"2023-10-09T04:42:48.699095+0000","last_became_peered":"2023-10-09T04:42:48.699095+0000","last_unstale":"2023-10-09T04:42:49.727613+0000","last_undegraded":"2023-10-09T04:42:49.727613+0000","last_fullsized":"2023-10-09T04:42:49.727613+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:57.871245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:42:49.702307+0000","last_change":"2023-10-09T04:42:48.849748+0000","last_active":"2023-10-09T04:42:49.702307+0000","last_peered":"2023-10-09T04:42:49.702307+0000","last_clean":"2023-10-09T04:42:49.702307+0000","last_became_active":"2023-10-09T04:42:48.848834+0000","last_became_peered":"2023-10-09T04:42:48.848834+0000","last_unstale":"2023-10-09T04:42:49.702307+0000","last_undegraded":"2023-10-09T04:42:49.702307+0000","last_fullsized":"2023-10-09T04:42:49.702307+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:20.475654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:42:49.705049+0000","last_change":"2023-10-09T04:42:48.851447+0000","last_active":"2023-10-09T04:42:49.705049+0000","last_peered":"2023-10-09T04:42:49.705049+0000","last_clean":"2023-10-09T04:42:49.705049+0000","last_became_active":"2023-10-09T04:42:48.850687+0000","last_became_peered":"2023-10-09T04:42:48.850687+0000","last_unstale":"2023-10-09T04:42:49.705049+0000","last_undegraded":"2023-10-09T04:42:49.705049+0000","last_fullsized":"2023-10-09T04:42:49.705049+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:32:22.356405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:42:49.705080+0000","last_change":"2023-10-09T04:42:48.697850+0000","last_active":"2023-10-09T04:42:49.705080+0000","last_peered":"2023-10-09T04:42:49.705080+0000","last_clean":"2023-10-09T04:42:49.705080+0000","last_became_active":"2023-10-09T04:42:48.697520+0000","last_became_peered":"2023-10-09T04:42:48.697520+0000","last_unstale":"2023-10-09T04:42:49.705080+0000","last_undegraded":"2023-10-09T04:42:49.705080+0000","last_fullsized":"2023-10-09T04:42:49.705080+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:04.728265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":267,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:42:49.704995+0000","last_change":"2023-10-09T04:41:09.432397+0000","last_active":"2023-10-09T04:42:49.704995+0000","last_peered":"2023-10-09T04:42:49.704995+0000","last_clean":"2023-10-09T04:42:49.704995+0000","last_became_active":"2023-10-09T04:41:09.431675+0000","last_became_peered":"2023-10-09T04:41:09.431675+0000","last_unstale":"2023-10-09T04:42:49.704995+0000","last_undegraded":"2023-10-09T04:42:49.704995+0000","last_fullsized":"2023-10-09T04:42:49.704995+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_clean_scrub_stamp":"2023-10-09T04:36:44.261013+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:23.657333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:42:49.767070+0000","last_change":"2023-10-09T04:42:48.698570+0000","last_active":"2023-10-09T04:42:49.767070+0000","last_peered":"2023-10-09T04:42:49.767070+0000","last_clean":"2023-10-09T04:42:49.767070+0000","last_became_active":"2023-10-09T04:42:48.698098+0000","last_became_peered":"2023-10-09T04:42:48.698098+0000","last_unstale":"2023-10-09T04:42:49.767070+0000","last_undegraded":"2023-10-09T04:42:49.767070+0000","last_fullsized":"2023-10-09T04:42:49.767070+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:35.340926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:42:49.765494+0000","last_change":"2023-10-09T04:42:48.728368+0000","last_active":"2023-10-09T04:42:49.765494+0000","last_peered":"2023-10-09T04:42:49.765494+0000","last_clean":"2023-10-09T04:42:49.765494+0000","last_became_active":"2023-10-09T04:42:48.727757+0000","last_became_peered":"2023-10-09T04:42:48.727757+0000","last_unstale":"2023-10-09T04:42:49.765494+0000","last_undegraded":"2023-10-09T04:42:49.765494+0000","last_fullsized":"2023-10-09T04:42:49.765494+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:21:02.404428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:42:49.767080+0000","last_change":"2023-10-09T04:42:48.698230+0000","last_active":"2023-10-09T04:42:49.767080+0000","last_peered":"2023-10-09T04:42:49.767080+0000","last_clean":"2023-10-09T04:42:49.767080+0000","last_became_active":"2023-10-09T04:42:48.697694+0000","last_became_peered":"2023-10-09T04:42:48.697694+0000","last_unstale":"2023-10-09T04:42:49.767080+0000","last_undegraded":"2023-10-09T04:42:49.767080+0000","last_fullsized":"2023-10-09T04:42:49.767080+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:46:21.257527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:42:49.702308+0000","last_change":"2023-10-09T04:42:48.696990+0000","last_active":"2023-10-09T04:42:49.702308+0000","last_peered":"2023-10-09T04:42:49.702308+0000","last_clean":"2023-10-09T04:42:49.702308+0000","last_became_active":"2023-10-09T04:42:48.696505+0000","last_became_peered":"2023-10-09T04:42:48.696505+0000","last_unstale":"2023-10-09T04:42:49.702308+0000","last_undegraded":"2023-10-09T04:42:49.702308+0000","last_fullsized":"2023-10-09T04:42:49.702308+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:55.113687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234027,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298476,"kb_used_data":1188,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446676,"statfs":{"total":95995035648,"available":95689396224,"internally_reserved":0,"allocated":1216512,"data_stored":852958,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:44:06 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":0.93799999999999994,"15min":0.95299999999999996},"min":{"1min":0.55700000000000005,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.3300000000000001,"5min":1.458,"15min":1.458},"last":1.127},{"interface":"front","average":{"1min":0.90300000000000002,"5min":0.94099999999999995,"15min":0.96099999999999997},"min":{"1min":0.65500000000000003,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.145,"5min":1.5580000000000001,"15min":1.5580000000000001},"last":0.84199999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:44:06 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":1.0049999999999999,"15min":1.0269999999999999},"min":{"1min":0.68100000000000005,"5min":0.622,"15min":0.622},"max":{"1min":1.3240000000000001,"5min":1.423,"15min":1.423},"last":1.212},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.91900000000000004,"15min":0.93300000000000005},"min":{"1min":0.59399999999999997,"5min":0.435,"15min":0.435},"max":{"1min":1.208,"5min":1.9950000000000001,"15min":1.9950000000000001},"last":0.879}]},{"osd":2,"last update":"Mon Oct 9 04:44:06 2023","interfaces":[{"interface":"back","average":{"1min":0.86099999999999999,"5min":0.98499999999999999,"15min":1.022},"min":{"1min":0.56299999999999994,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.1539999999999999,"5min":2.0819999999999999,"15min":2.0819999999999999},"last":0.996},{"interface":"front","average":{"1min":0.94099999999999995,"5min":0.96399999999999997,"15min":0.98899999999999999},"min":{"1min":0.59199999999999997,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":1.3169999999999999,"5min":1.4730000000000001,"15min":1.4730000000000001},"last":1.0209999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:44:06 2023","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":0.86799999999999999,"15min":0.874},"min":{"1min":0.59499999999999997,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.038,"5min":1.4159999999999999,"15min":1.4159999999999999},"last":1.1830000000000001},{"interface":"front","average":{"1min":1.0760000000000001,"5min":1.0549999999999999,"15min":1.071},"min":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.4039999999999999,"5min":1.5129999999999999,"15min":1.5129999999999999},"last":1.262}]},{"osd":4,"last update":"Mon Oct 9 04:44:06 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":0.84799999999999998,"15min":0.81999999999999995},"min":{"1min":0.54200000000000004,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.2969999999999999,"5min":1.2969999999999999,"15min":1.2969999999999999},"last":1.153},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.83099999999999996,"15min":0.80400000000000005},"min":{"1min":0.54100000000000004,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.169,"5min":1.1890000000000001,"15min":1.1890000000000001},"last":0.92300000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:44:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":0.99399999999999999,"15min":0.998},"min":{"1min":0.55700000000000005,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.419,"5min":1.827,"15min":1.827},"last":0.96799999999999997},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.0289999999999999,"15min":1.0449999999999999},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.349,"5min":1.9179999999999999,"15min":1.9179999999999999},"last":1.2370000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:44:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.0329999999999999,"15min":1.0389999999999999},"min":{"1min":0.58099999999999996,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.454,"5min":1.454,"15min":1.454},"last":1.0509999999999999},{"interface":"front","average":{"1min":1.0209999999999999,"5min":0.95499999999999996,"15min":0.94699999999999995},"min":{"1min":0.68600000000000005,"5min":0.505,"15min":0.505},"max":{"1min":1.3089999999999999,"5min":1.3089999999999999,"15min":1.3089999999999999},"last":1.075}]}]},{"osd":6,"up_from":46,"seq":197568495669,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298480,"kb_used_data":1192,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446672,"statfs":{"total":95995035648,"available":95689392128,"internally_reserved":0,"allocated":1220608,"data_stored":853281,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:44:19 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":0.89500000000000002,"15min":0.94799999999999995},"min":{"1min":0.47599999999999998,"5min":0.437,"15min":0.437},"max":{"1min":1.202,"5min":1.7669999999999999,"15min":1.7669999999999999},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.90800000000000003,"15min":0.96599999999999997},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.1379999999999999,"5min":1.3149999999999999,"15min":1.3149999999999999},"last":1.0649999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:44:19 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":0.88300000000000001,"15min":0.90300000000000002},"min":{"1min":0.56200000000000006,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":1.1619999999999999,"5min":1.2949999999999999,"15min":1.2949999999999999},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.91600000000000004,"5min":0.90200000000000002,"15min":0.91800000000000004},"min":{"1min":0.71599999999999997,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.4059999999999999,"5min":1.4850000000000001,"15min":1.4850000000000001},"last":0.68700000000000006}]},{"osd":2,"last update":"Mon Oct 9 04:44:19 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":0.95099999999999996,"15min":0.95299999999999996},"min":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"max":{"1min":3.238,"5min":3.238,"15min":3.238},"last":0.92100000000000004},{"interface":"front","average":{"1min":0.88500000000000001,"5min":0.90400000000000003,"15min":0.91800000000000004},"min":{"1min":0.64600000000000002,"5min":0.625,"15min":0.625},"max":{"1min":1.101,"5min":1.238,"15min":1.238},"last":0.874}]},{"osd":3,"last update":"Mon Oct 9 04:44:19 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":0.95999999999999996,"15min":0.98199999999999998},"min":{"1min":0.63100000000000001,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.1950000000000001,"5min":1.8879999999999999,"15min":1.8879999999999999},"last":1.155},{"interface":"front","average":{"1min":1.0149999999999999,"5min":0.95899999999999996,"15min":0.96899999999999997},"min":{"1min":0.55500000000000005,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.3460000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.84199999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:44:19 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.84599999999999997,"15min":0.80400000000000005},"min":{"1min":0.56599999999999995,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.302,"5min":1.302,"15min":1.302},"last":0.79900000000000004},{"interface":"front","average":{"1min":1.004,"5min":0.91700000000000004,"15min":0.90700000000000003},"min":{"1min":0.74299999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.151,"5min":1.4319999999999999,"15min":1.4319999999999999},"last":1.214}]},{"osd":5,"last update":"Mon Oct 9 04:44:19 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":0.92300000000000004,"15min":0.89400000000000002},"min":{"1min":0.69199999999999995,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.3100000000000001,"5min":1.397,"15min":1.397},"last":1.1779999999999999},{"interface":"front","average":{"1min":1.075,"5min":0.97399999999999998,"15min":0.96399999999999997},"min":{"1min":0.73499999999999999,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.2709999999999999,"5min":1.6000000000000001,"15min":1.6000000000000001},"last":1.2609999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:44:08 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":0.96499999999999997,"15min":0.96099999999999997},"min":{"1min":0.73999999999999999,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.452,"5min":1.452,"15min":1.452},"last":0.998},{"interface":"front","average":{"1min":1.2250000000000001,"5min":1.1120000000000001,"15min":1.1000000000000001},"min":{"1min":0.94799999999999995,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.5049999999999999,"5min":1.5660000000000001,"15min":1.5660000000000001},"last":1.1080000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574954,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:44:05 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.69299999999999995,"15min":0.60199999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.84099999999999997,"5min":1.179,"15min":1.179},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.75700000000000001,"15min":0.65000000000000002},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.069,"5min":1.2170000000000001,"15min":1.2170000000000001},"last":0.84799999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:43:49 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.76900000000000002,"15min":0.68300000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.95499999999999996,"5min":1.484,"15min":1.484},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.76000000000000001,"15min":0.68000000000000005},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.282,"5min":1.371,"15min":1.371},"last":0.96899999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:43:46 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":0.81200000000000006,"15min":0.80900000000000005},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":1.1100000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":1.222},{"interface":"front","average":{"1min":0.73099999999999998,"5min":0.80800000000000005,"15min":0.78200000000000003},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":1.133,"5min":1.133,"15min":1.133},"last":0.92300000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:43:40 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":0.998,"15min":0.97399999999999998},"min":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.2490000000000001,"5min":1.607,"15min":1.607},"last":1.0289999999999999},{"interface":"front","average":{"1min":0.98799999999999999,"5min":1.026,"15min":0.98599999999999999},"min":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"max":{"1min":1.395,"5min":1.4239999999999999,"15min":1.4239999999999999},"last":1.452}]},{"osd":5,"last update":"Mon Oct 9 04:44:28 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":0.98499999999999999,"15min":0.999},"min":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"max":{"1min":1.4019999999999999,"5min":1.55,"15min":1.55},"last":1.385},{"interface":"front","average":{"1min":1,"5min":1.024,"15min":1.006},"min":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":1.3140000000000001,"5min":1.5449999999999999,"15min":1.5449999999999999},"last":1.149}]},{"osd":6,"last update":"Mon Oct 9 04:44:25 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.0549999999999999,"15min":1.0620000000000001},"min":{"1min":0.66100000000000003,"5min":0.57599999999999996,"15min":0.57599999999999996},"max":{"1min":1.448,"5min":1.639,"15min":1.639},"last":1.048},{"interface":"front","average":{"1min":0.96599999999999997,"5min":1.0660000000000001,"15min":1.0840000000000001},"min":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.4710000000000001,"5min":1.552,"15min":1.552},"last":1.099}]},{"osd":7,"last update":"Mon Oct 9 04:44:07 2023","interfaces":[{"interface":"back","average":{"1min":1.101,"5min":1.135,"15min":1.153},"min":{"1min":0.80900000000000005,"5min":0.78600000000000003,"15min":0.78600000000000003},"max":{"1min":1.3839999999999999,"5min":1.758,"15min":1.758},"last":1.2769999999999999},{"interface":"front","average":{"1min":1.0600000000000001,"5min":1.147,"15min":1.1739999999999999},"min":{"1min":0.63900000000000001,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.3280000000000001,"5min":1.6899999999999999,"15min":1.6899999999999999},"last":1.1779999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738484,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298484,"kb_used_data":1196,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446668,"statfs":{"total":95995035648,"available":95689388032,"internally_reserved":0,"allocated":1224704,"data_stored":853546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:44:05 2023","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":0.72699999999999998,"15min":0.59899999999999998},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.26900000000000002},"max":{"1min":1.3240000000000001,"5min":1.3240000000000001,"15min":1.3240000000000001},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.71199999999999997,"5min":0.745,"15min":0.63900000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.032,"5min":1.103,"15min":1.103},"last":0.78600000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:43:53 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.78500000000000003,"15min":0.67400000000000004},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.378},"max":{"1min":1.319,"5min":1.319,"15min":1.319},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.79800000000000004,"15min":0.69399999999999995},"min":{"1min":0.48999999999999999,"5min":0.40000000000000002,"15min":0.34499999999999997},"max":{"1min":1.274,"5min":1.2789999999999999,"15min":1.2789999999999999},"last":0.54700000000000004}]},{"osd":3,"last update":"Mon Oct 9 04:43:47 2023","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":0.85599999999999998,"15min":0.83999999999999997},"min":{"1min":0.58699999999999997,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":1.2210000000000001,"5min":1.4299999999999999,"15min":1.4299999999999999},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.75600000000000001,"5min":0.79300000000000004,"15min":0.80000000000000004},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.96099999999999997,"5min":1.2190000000000001,"15min":1.264},"last":0.75600000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:43:44 2023","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":0.95199999999999996,"15min":0.93400000000000005},"min":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"max":{"1min":1.2050000000000001,"5min":1.256,"15min":1.256},"last":1.0309999999999999},{"interface":"front","average":{"1min":1.0189999999999999,"5min":0.98699999999999999,"15min":0.95199999999999996},"min":{"1min":0.72499999999999998,"5min":0.67900000000000005,"15min":0.67900000000000005},"max":{"1min":1.3149999999999999,"5min":1.3400000000000001,"15min":1.3400000000000001},"last":0.873}]},{"osd":5,"last update":"Mon Oct 9 04:44:29 2023","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":1.0649999999999999,"15min":1.046},"min":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"max":{"1min":2.0219999999999998,"5min":2.0219999999999998,"15min":2.0219999999999998},"last":1.0489999999999999},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.98999999999999999,"15min":1.004},"min":{"1min":0.61699999999999999,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.623,"5min":1.623,"15min":1.623},"last":0.89100000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:44:22 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":1.0509999999999999,"15min":1.0840000000000001},"min":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"max":{"1min":1.4350000000000001,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":0.93600000000000005},{"interface":"front","average":{"1min":1.111,"5min":1.125,"15min":1.151},"min":{"1min":0.66300000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"max":{"1min":1.7330000000000001,"5min":1.7330000000000001,"15min":1.7330000000000001},"last":0.85999999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:44:11 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":1.101,"15min":1.131},"min":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"max":{"1min":1.2829999999999999,"5min":1.5189999999999999,"15min":1.5189999999999999},"last":1.0900000000000001},{"interface":"front","average":{"1min":1.1639999999999999,"5min":1.161,"15min":1.177},"min":{"1min":0.68200000000000005,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":2.097,"5min":2.097,"15min":2.097},"last":0.90400000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411424,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:43:57 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.71399999999999997,"15min":0.66700000000000004},"min":{"1min":0.45100000000000001,"5min":0.437,"15min":0.437},"max":{"1min":1.1850000000000001,"5min":1.1850000000000001,"15min":1.1850000000000001},"last":0.76600000000000001},{"interface":"front","average":{"1min":0.69999999999999996,"5min":0.76800000000000002,"15min":0.69399999999999995},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.93999999999999995,"5min":1.2989999999999999,"15min":1.2989999999999999},"last":1.056}]},{"osd":1,"last update":"Mon Oct 9 04:43:57 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":0.77500000000000002,"15min":0.69399999999999995},"min":{"1min":0.497,"5min":0.497,"15min":0.42999999999999999},"max":{"1min":1.165,"5min":1.272,"15min":1.272},"last":0.91200000000000003},{"interface":"front","average":{"1min":0.751,"5min":0.82199999999999995,"15min":0.71599999999999997},"min":{"1min":0.48299999999999998,"5min":0.46300000000000002,"15min":0.441},"max":{"1min":1.042,"5min":1.395,"15min":1.395},"last":0.87}]},{"osd":3,"last update":"Mon Oct 9 04:43:48 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.85199999999999998,"15min":0.80800000000000005},"min":{"1min":0.52600000000000002,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.4259999999999999,"5min":1.4259999999999999,"15min":1.4259999999999999},"last":1.1140000000000001},{"interface":"front","average":{"1min":0.83099999999999996,"5min":0.88500000000000001,"15min":0.85199999999999998},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.2729999999999999,"5min":1.306,"15min":1.306},"last":0.96799999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:43:41 2023","interfaces":[{"interface":"back","average":{"1min":0.98399999999999999,"5min":1.01,"15min":0.96099999999999997},"min":{"1min":0.71299999999999997,"5min":0.69299999999999995,"15min":0.69299999999999995},"max":{"1min":1.2909999999999999,"5min":1.72,"15min":1.72},"last":1.24},{"interface":"front","average":{"1min":0.995,"5min":1.0329999999999999,"15min":1.01},"min":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"max":{"1min":1.3160000000000001,"5min":1.4790000000000001,"15min":1.4790000000000001},"last":1.264}]},{"osd":5,"last update":"Mon Oct 9 04:43:32 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":1.0720000000000001,"15min":1.0720000000000001},"min":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"max":{"1min":1.423,"5min":1.6180000000000001,"15min":1.6180000000000001},"last":1.419},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.0720000000000001,"15min":1.0600000000000001},"min":{"1min":0.621,"5min":0.621,"15min":0.621},"max":{"1min":1.347,"5min":1.6659999999999999,"15min":1.6659999999999999},"last":1.335}]},{"osd":6,"last update":"Mon Oct 9 04:44:19 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.109,"15min":1.155},"min":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"max":{"1min":1.367,"5min":1.706,"15min":1.706},"last":1.365},{"interface":"front","average":{"1min":0.93000000000000005,"5min":1.014,"15min":1.0580000000000001},"min":{"1min":0.64200000000000002,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.228,"5min":1.4330000000000001,"15min":1.4330000000000001},"last":1.3069999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:44:10 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":1.194,"15min":1.236},"min":{"1min":0.77100000000000002,"5min":0.622,"15min":0.622},"max":{"1min":1.5840000000000001,"5min":1.7729999999999999,"15min":1.7729999999999999},"last":1.3959999999999999},{"interface":"front","average":{"1min":1.127,"5min":1.2070000000000001,"15min":1.238},"min":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"max":{"1min":1.454,"5min":1.454,"15min":1.454},"last":1.552}]}]},{"osd":3,"up_from":25,"seq":107374182484,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:43:49 2023","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":0.68999999999999995,"15min":0.69799999999999995},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":1.091,"5min":1.091,"15min":1.091},"last":0.627},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.76600000000000001,"15min":0.73499999999999999},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.0669999999999999,"5min":1.266,"15min":1.266},"last":0.58499999999999996}]},{"osd":1,"last update":"Mon Oct 9 04:43:49 2023","interfaces":[{"interface":"back","average":{"1min":0.71999999999999997,"5min":0.76700000000000002,"15min":0.75700000000000001},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":1.1699999999999999,"5min":1.1819999999999999,"15min":1.1819999999999999},"last":0.876},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.75600000000000001,"15min":0.76100000000000001},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.94299999999999995,"5min":1.7250000000000001,"15min":1.7250000000000001},"last":0.51700000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:43:49 2023","interfaces":[{"interface":"back","average":{"1min":0.74099999999999999,"5min":0.86699999999999999,"15min":0.86099999999999999},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.98499999999999999,"5min":1.8320000000000001,"15min":1.8320000000000001},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.79600000000000004,"15min":0.81399999999999995},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.173,"5min":1.1779999999999999,"15min":1.218},"last":0.72599999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:43:45 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.97399999999999998,"15min":0.94999999999999996},"min":{"1min":0.751,"5min":0.66000000000000003,"15min":0.66000000000000003},"max":{"1min":1.202,"5min":1.5009999999999999,"15min":1.5009999999999999},"last":0.93700000000000006},{"interface":"front","average":{"1min":0.96399999999999997,"5min":1.014,"15min":0.95299999999999996},"min":{"1min":0.67400000000000004,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.391,"5min":2.298,"15min":2.298},"last":0.99199999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:43:27 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.0269999999999999,"15min":1.0249999999999999},"min":{"1min":0.68799999999999994,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.296,"5min":1.603,"15min":1.603},"last":0.80800000000000005},{"interface":"front","average":{"1min":0.97599999999999998,"5min":1.0600000000000001,"15min":1.0649999999999999},"min":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"max":{"1min":1.2649999999999999,"5min":1.9930000000000001,"15min":1.9930000000000001},"last":0.83799999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:44:19 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":1.073,"15min":1.089},"min":{"1min":0.72899999999999998,"5min":0.71599999999999997,"15min":0.71599999999999997},"max":{"1min":1.621,"5min":2.1030000000000002,"15min":2.1030000000000002},"last":0.94899999999999995},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.0660000000000001,"15min":1.0820000000000001},"min":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":1.29,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":0.79000000000000004}]},{"osd":7,"last update":"Mon Oct 9 04:44:10 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.1319999999999999,"15min":1.1739999999999999},"min":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"max":{"1min":1.3300000000000001,"5min":1.367,"15min":1.367},"last":0.97299999999999998},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.1830000000000001,"15min":1.22},"min":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"max":{"1min":1.534,"5min":1.534,"15min":1.534},"last":0.90500000000000003}]}]},{"osd":4,"up_from":30,"seq":128849018955,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:43:41 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":0.88600000000000001,"15min":0.85599999999999998},"min":{"1min":0.54500000000000004,"5min":0.505,"15min":0.505},"max":{"1min":1.0940000000000001,"5min":1.294,"15min":1.294},"last":0.749},{"interface":"front","average":{"1min":0.89500000000000002,"5min":0.89900000000000002,"15min":0.89700000000000002},"min":{"1min":0.57899999999999996,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":1.1160000000000001,"5min":1.8180000000000001,"15min":1.8180000000000001},"last":1.194}]},{"osd":1,"last update":"Mon Oct 9 04:43:41 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":0.89500000000000002,"15min":0.85299999999999998},"min":{"1min":0.57999999999999996,"5min":0.504,"15min":0.504},"max":{"1min":1.278,"5min":1.278,"15min":1.278},"last":1.077},{"interface":"front","average":{"1min":0.96699999999999997,"5min":0.92500000000000004,"15min":0.89600000000000002},"min":{"1min":0.66900000000000004,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.4099999999999999,"5min":1.9039999999999999,"15min":1.9039999999999999},"last":1.304}]},{"osd":2,"last update":"Mon Oct 9 04:43:41 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.94299999999999995,"15min":0.91300000000000003},"min":{"1min":0.60499999999999998,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.548,"5min":1.724,"15min":1.724},"last":0.98999999999999999},{"interface":"front","average":{"1min":0.94599999999999995,"5min":0.90000000000000002,"15min":0.89200000000000002},"min":{"1min":0.73999999999999999,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.23,"5min":1.25,"15min":1.25},"last":0.79600000000000004}]},{"osd":3,"last update":"Mon Oct 9 04:43:41 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":0.91900000000000004,"15min":0.91000000000000003},"min":{"1min":0.66600000000000004,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.319,"5min":1.319,"15min":1.319},"last":1.135},{"interface":"front","average":{"1min":0.998,"5min":0.94299999999999995,"15min":0.91400000000000003},"min":{"1min":0.72899999999999998,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.377,"5min":1.377,"15min":1.377},"last":0.96299999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:43:29 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":0.80500000000000005,"15min":0.76200000000000001},"min":{"1min":0.47499999999999998,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":1.288,"5min":1.3819999999999999,"15min":1.3819999999999999},"last":0.83299999999999996},{"interface":"front","average":{"1min":0.97599999999999998,"5min":0.83399999999999996,"15min":0.77000000000000002},"min":{"1min":0.60099999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.6539999999999999,"5min":1.6539999999999999,"15min":1.6539999999999999},"last":1.0369999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:44:19 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":0.879,"15min":0.79500000000000004},"min":{"1min":0.70599999999999996,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.327,"5min":1.474,"15min":1.474},"last":0.86699999999999999},{"interface":"front","average":{"1min":1.1319999999999999,"5min":0.999,"15min":0.96699999999999997},"min":{"1min":0.81599999999999995,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.4279999999999999,"5min":1.9850000000000001,"15min":1.9850000000000001},"last":1.3680000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:44:11 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":0.999,"15min":0.97799999999999998},"min":{"1min":0.54100000000000004,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.268,"5min":1.752,"15min":1.752},"last":0.89700000000000002},{"interface":"front","average":{"1min":1.1890000000000001,"5min":1.0720000000000001,"15min":1.034},"min":{"1min":0.71699999999999997,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.4390000000000001,"5min":1.6180000000000001,"15min":1.6180000000000001},"last":1.163}]}]},{"osd":5,"up_from":38,"seq":163208757311,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":0.88500000000000001,"15min":0.873},"min":{"1min":0.54900000000000004,"5min":0.499,"15min":0.499},"max":{"1min":1.296,"5min":1.3720000000000001,"15min":1.3720000000000001},"last":0.97199999999999998},{"interface":"front","average":{"1min":0.93799999999999994,"5min":0.92700000000000005,"15min":0.93000000000000005},"min":{"1min":0.64800000000000002,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.276,"5min":1.7869999999999999,"15min":1.7869999999999999},"last":0.90100000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":0.90200000000000002,"15min":0.94599999999999995},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.234,"5min":2.0510000000000002,"15min":2.0510000000000002},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.94399999999999995,"5min":0.91000000000000003,"15min":0.88900000000000001},"min":{"1min":0.66000000000000003,"5min":0.63500000000000001,"15min":0.63500000000000001},"max":{"1min":1.5529999999999999,"5min":1.5529999999999999,"15min":1.5529999999999999},"last":0.76000000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":0.876,"15min":0.85499999999999998},"min":{"1min":0.57699999999999996,"5min":0.505,"15min":0.505},"max":{"1min":0.98699999999999999,"5min":1.204,"15min":1.204},"last":0.71299999999999997},{"interface":"front","average":{"1min":1.0329999999999999,"5min":0.97199999999999998,"15min":0.96299999999999997},"min":{"1min":0.67500000000000004,"5min":0.60599999999999998,"15min":0.60599999999999998},"max":{"1min":1.2230000000000001,"5min":1.885,"15min":1.885},"last":1.0309999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.93200000000000005,"15min":0.92000000000000004},"min":{"1min":0.70599999999999996,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.0660000000000001,"5min":1.321,"15min":1.321},"last":1.0649999999999999},{"interface":"front","average":{"1min":1.0149999999999999,"5min":1.008,"15min":0.997},"min":{"1min":0.76300000000000001,"5min":0.61099999999999999,"15min":0.61099999999999999},"max":{"1min":1.2629999999999999,"5min":1.9710000000000001,"15min":1.9710000000000001},"last":1.1020000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":0.83899999999999997,"15min":0.77600000000000002},"min":{"1min":0.78500000000000003,"5min":0.5,"15min":0.5},"max":{"1min":1.1870000000000001,"5min":1.3759999999999999,"15min":1.3759999999999999},"last":0.93100000000000005},{"interface":"front","average":{"1min":1.054,"5min":0.93500000000000005,"15min":0.84399999999999997},"min":{"1min":0.626,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.339,"5min":1.5269999999999999,"15min":1.5269999999999999},"last":1.143}]},{"osd":6,"last update":"Mon Oct 9 04:44:21 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":0.95499999999999996,"15min":0.91000000000000003},"min":{"1min":0.71099999999999997,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.256,"5min":1.4670000000000001,"15min":1.4670000000000001},"last":1.331},{"interface":"front","average":{"1min":1.1279999999999999,"5min":1.0269999999999999,"15min":1.0409999999999999},"min":{"1min":0.69399999999999995,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.3129999999999999,"5min":2.169,"15min":2.169},"last":1.2}]},{"osd":7,"last update":"Mon Oct 9 04:44:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.0549999999999999,"15min":1.0389999999999999},"min":{"1min":0.625,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.3759999999999999,"5min":1.5049999999999999,"15min":1.5049999999999999},"last":1.266},{"interface":"front","average":{"1min":1.091,"5min":0.98599999999999999,"15min":0.95799999999999996},"min":{"1min":0.80200000000000005,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.4710000000000001,"5min":1.4710000000000001,"15min":1.4710000000000001},"last":1.2949999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:44:33.756 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T04:44:33.756 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T04:44:33.756 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T04:44:33.756 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T04:44:33.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:33 smithi194 bash[31648]: audit 2023-10-09T04:44:32.783595+0000 mgr.y (mgr.14466) 107 : audit [DBG] from='client.14652 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:44:34.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:33 smithi133 bash[41291]: audit 2023-10-09T04:44:32.783595+0000 mgr.y (mgr.14466) 107 : audit [DBG] from='client.14652 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:44:34.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:33 smithi133 bash[27558]: audit 2023-10-09T04:44:32.783595+0000 mgr.y (mgr.14466) 107 : audit [DBG] from='client.14652 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:44:34.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:34 smithi194 bash[31648]: cluster 2023-10-09T04:44:33.122881+0000 mgr.y (mgr.14466) 108 : cluster [DBG] pgmap v81: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:35.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:34 smithi133 bash[27558]: cluster 2023-10-09T04:44:33.122881+0000 mgr.y (mgr.14466) 108 : cluster [DBG] pgmap v81: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:35.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:34 smithi133 bash[41291]: cluster 2023-10-09T04:44:33.122881+0000 mgr.y (mgr.14466) 108 : cluster [DBG] pgmap v81: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:35.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:35 smithi194 bash[31648]: audit 2023-10-09T04:44:35.189040+0000 mon.a (mon.0) 793 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:36.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:35 smithi133 bash[27558]: audit 2023-10-09T04:44:35.189040+0000 mon.a (mon.0) 793 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:36.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:35 smithi133 bash[41291]: audit 2023-10-09T04:44:35.189040+0000 mon.a (mon.0) 793 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:36.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:36 smithi194 bash[31648]: cluster 2023-10-09T04:44:35.123419+0000 mgr.y (mgr.14466) 109 : cluster [DBG] pgmap v82: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:36.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:44:36 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:44:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:44:37.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:36 smithi133 bash[41291]: cluster 2023-10-09T04:44:35.123419+0000 mgr.y (mgr.14466) 109 : cluster [DBG] pgmap v82: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:37.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:36 smithi133 bash[27558]: cluster 2023-10-09T04:44:35.123419+0000 mgr.y (mgr.14466) 109 : cluster [DBG] pgmap v82: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:38.529 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:44:38.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:38 smithi194 bash[31648]: cluster 2023-10-09T04:44:37.124133+0000 mgr.y (mgr.14466) 110 : cluster [DBG] pgmap v83: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:39.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:38 smithi133 bash[41291]: cluster 2023-10-09T04:44:37.124133+0000 mgr.y (mgr.14466) 110 : cluster [DBG] pgmap v83: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:39.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:38 smithi133 bash[27558]: cluster 2023-10-09T04:44:37.124133+0000 mgr.y (mgr.14466) 110 : cluster [DBG] pgmap v83: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:40.608 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:44:40 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:44:40] "GET /metrics HTTP/1.1" 200 203312 "" "Prometheus/2.43.0" 2023-10-09T04:44:40.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:40 smithi194 bash[31648]: cluster 2023-10-09T04:44:39.124775+0000 mgr.y (mgr.14466) 111 : cluster [DBG] pgmap v84: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:41.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:40 smithi133 bash[27558]: cluster 2023-10-09T04:44:39.124775+0000 mgr.y (mgr.14466) 111 : cluster [DBG] pgmap v84: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:41.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:40 smithi133 bash[41291]: cluster 2023-10-09T04:44:39.124775+0000 mgr.y (mgr.14466) 111 : cluster [DBG] pgmap v84: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:42.154 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T04:44:42.154 INFO:teuthology.orchestra.run.smithi133.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T04:44:42.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:42 smithi133 bash[41291]: cluster 2023-10-09T04:44:41.125455+0000 mgr.y (mgr.14466) 112 : cluster [DBG] pgmap v85: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:42.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:42 smithi133 bash[41291]: audit 2023-10-09T04:44:42.148239+0000 mon.a (mon.0) 794 : audit [DBG] from='client.? 172.21.15.133:0/595375092' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T04:44:42.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:42 smithi133 bash[27558]: cluster 2023-10-09T04:44:41.125455+0000 mgr.y (mgr.14466) 112 : cluster [DBG] pgmap v85: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:42.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:42 smithi133 bash[27558]: audit 2023-10-09T04:44:42.148239+0000 mon.a (mon.0) 794 : audit [DBG] from='client.? 172.21.15.133:0/595375092' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T04:44:42.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:42 smithi194 bash[31648]: cluster 2023-10-09T04:44:41.125455+0000 mgr.y (mgr.14466) 112 : cluster [DBG] pgmap v85: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:42.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:42 smithi194 bash[31648]: audit 2023-10-09T04:44:42.148239+0000 mon.a (mon.0) 794 : audit [DBG] from='client.? 172.21.15.133:0/595375092' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T04:44:43.041 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T04:44:43.041 INFO:tasks.cephadm:Setup complete, yielding 2023-10-09T04:44:43.041 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-10-09T04:44:43.097 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi133.front.sepia.ceph.com 2023-10-09T04:44:43.097 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- bash -c 'ceph fs volume create foo' 2023-10-09T04:44:44.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:44 smithi194 bash[31648]: cluster 2023-10-09T04:44:43.126132+0000 mgr.y (mgr.14466) 113 : cluster [DBG] pgmap v86: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:45.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:44 smithi133 bash[27558]: cluster 2023-10-09T04:44:43.126132+0000 mgr.y (mgr.14466) 113 : cluster [DBG] pgmap v86: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:45.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:44 smithi133 bash[41291]: cluster 2023-10-09T04:44:43.126132+0000 mgr.y (mgr.14466) 113 : cluster [DBG] pgmap v86: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:46.621 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:44:46 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:44:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:44:46.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:46 smithi194 bash[31648]: cluster 2023-10-09T04:44:45.126861+0000 mgr.y (mgr.14466) 114 : cluster [DBG] pgmap v87: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:47.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:46 smithi133 bash[27558]: cluster 2023-10-09T04:44:45.126861+0000 mgr.y (mgr.14466) 114 : cluster [DBG] pgmap v87: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:47.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:46 smithi133 bash[41291]: cluster 2023-10-09T04:44:45.126861+0000 mgr.y (mgr.14466) 114 : cluster [DBG] pgmap v87: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:47.868 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:44:48.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:48 smithi194 bash[31648]: cluster 2023-10-09T04:44:47.127579+0000 mgr.y (mgr.14466) 115 : cluster [DBG] pgmap v88: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:48.894 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:48 smithi133 bash[41291]: cluster 2023-10-09T04:44:47.127579+0000 mgr.y (mgr.14466) 115 : cluster [DBG] pgmap v88: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:48.894 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:48 smithi133 bash[27558]: cluster 2023-10-09T04:44:47.127579+0000 mgr.y (mgr.14466) 115 : cluster [DBG] pgmap v88: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:50.628 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:44:50 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:44:50] "GET /metrics HTTP/1.1" 200 203310 "" "Prometheus/2.43.0" 2023-10-09T04:44:50.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:50 smithi194 bash[31648]: cluster 2023-10-09T04:44:49.127999+0000 mgr.y (mgr.14466) 116 : cluster [DBG] pgmap v89: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:50.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:50 smithi194 bash[31648]: audit 2023-10-09T04:44:50.189539+0000 mon.a (mon.0) 795 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:51.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:50 smithi133 bash[41291]: cluster 2023-10-09T04:44:49.127999+0000 mgr.y (mgr.14466) 116 : cluster [DBG] pgmap v89: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:51.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:50 smithi133 bash[41291]: audit 2023-10-09T04:44:50.189539+0000 mon.a (mon.0) 795 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:51.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:50 smithi133 bash[27558]: cluster 2023-10-09T04:44:49.127999+0000 mgr.y (mgr.14466) 116 : cluster [DBG] pgmap v89: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:51.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:50 smithi133 bash[27558]: audit 2023-10-09T04:44:50.189539+0000 mon.a (mon.0) 795 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:52.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:51 smithi133 bash[27558]: audit 2023-10-09T04:44:51.250994+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T04:44:52.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:51 smithi133 bash[41291]: audit 2023-10-09T04:44:51.250994+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T04:44:52.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:51 smithi194 bash[31648]: audit 2023-10-09T04:44:51.250994+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T04:44:53.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:52 smithi133 bash[27558]: cluster 2023-10-09T04:44:51.128570+0000 mgr.y (mgr.14466) 117 : cluster [DBG] pgmap v90: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:53.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:52 smithi133 bash[27558]: audit 2023-10-09T04:44:51.250022+0000 mgr.y (mgr.14466) 118 : audit [DBG] from='client.24569 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:44:53.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:52 smithi133 bash[27558]: audit 2023-10-09T04:44:51.746707+0000 mon.a (mon.0) 797 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T04:44:53.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:52 smithi133 bash[27558]: cluster 2023-10-09T04:44:51.746811+0000 mon.a (mon.0) 798 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-10-09T04:44:53.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:52 smithi133 bash[27558]: audit 2023-10-09T04:44:51.756983+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T04:44:53.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:52 smithi133 bash[27558]: debug 2023-10-09T04:44:52.760+0000 7fa584e30700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T04:44:53.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:52 smithi133 bash[41291]: cluster 2023-10-09T04:44:51.128570+0000 mgr.y (mgr.14466) 117 : cluster [DBG] pgmap v90: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:53.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:52 smithi133 bash[41291]: audit 2023-10-09T04:44:51.250022+0000 mgr.y (mgr.14466) 118 : audit [DBG] from='client.24569 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:44:53.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:52 smithi133 bash[41291]: audit 2023-10-09T04:44:51.746707+0000 mon.a (mon.0) 797 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T04:44:53.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:52 smithi133 bash[41291]: cluster 2023-10-09T04:44:51.746811+0000 mon.a (mon.0) 798 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-10-09T04:44:53.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:52 smithi133 bash[41291]: audit 2023-10-09T04:44:51.756983+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T04:44:53.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:52 smithi194 bash[31648]: cluster 2023-10-09T04:44:51.128570+0000 mgr.y (mgr.14466) 117 : cluster [DBG] pgmap v90: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:53.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:52 smithi194 bash[31648]: audit 2023-10-09T04:44:51.250022+0000 mgr.y (mgr.14466) 118 : audit [DBG] from='client.24569 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:44:53.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:52 smithi194 bash[31648]: audit 2023-10-09T04:44:51.746707+0000 mon.a (mon.0) 797 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T04:44:53.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:52 smithi194 bash[31648]: cluster 2023-10-09T04:44:51.746811+0000 mon.a (mon.0) 798 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-10-09T04:44:53.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:52 smithi194 bash[31648]: audit 2023-10-09T04:44:51.756983+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T04:44:54.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[41291]: audit 2023-10-09T04:44:52.762626+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T04:44:54.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[41291]: cluster 2023-10-09T04:44:52.762751+0000 mon.a (mon.0) 801 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-10-09T04:44:54.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[41291]: audit 2023-10-09T04:44:52.763695+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T04:44:54.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[41291]: cluster 2023-10-09T04:44:52.764974+0000 mon.a (mon.0) 803 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T04:44:54.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[41291]: cluster 2023-10-09T04:44:52.765018+0000 mon.a (mon.0) 804 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T04:44:54.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[41291]: cluster 2023-10-09T04:44:52.776581+0000 mon.a (mon.0) 805 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-10-09T04:44:54.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[41291]: audit 2023-10-09T04:44:52.777144+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T04:44:54.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[41291]: cluster 2023-10-09T04:44:52.777238+0000 mon.a (mon.0) 807 : cluster [DBG] fsmap foo:0 2023-10-09T04:44:54.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[41291]: cephadm 2023-10-09T04:44:52.780694+0000 mgr.y (mgr.14466) 119 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-10-09T04:44:54.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[41291]: audit 2023-10-09T04:44:52.787991+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:54.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[41291]: audit 2023-10-09T04:44:52.808686+0000 mon.a (mon.0) 809 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:44:54.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[41291]: audit 2023-10-09T04:44:52.811035+0000 mon.a (mon.0) 810 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:44:54.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[41291]: audit 2023-10-09T04:44:52.812394+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:44:54.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[41291]: audit 2023-10-09T04:44:52.820911+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:54.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[41291]: audit 2023-10-09T04:44:52.825575+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi133.tsuabh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:44:54.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[41291]: audit 2023-10-09T04:44:52.829982+0000 mon.a (mon.0) 814 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi133.tsuabh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T04:44:54.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[41291]: audit 2023-10-09T04:44:52.831169+0000 mon.a (mon.0) 815 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:44:54.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[41291]: cephadm 2023-10-09T04:44:52.832585+0000 mgr.y (mgr.14466) 120 : cephadm [INF] Deploying daemon mds.foo.smithi133.tsuabh on smithi133 2023-10-09T04:44:54.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[27558]: audit 2023-10-09T04:44:52.762626+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T04:44:54.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[27558]: cluster 2023-10-09T04:44:52.762751+0000 mon.a (mon.0) 801 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-10-09T04:44:54.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[27558]: audit 2023-10-09T04:44:52.763695+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T04:44:54.054 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[27558]: cluster 2023-10-09T04:44:52.764974+0000 mon.a (mon.0) 803 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T04:44:54.054 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[27558]: cluster 2023-10-09T04:44:52.765018+0000 mon.a (mon.0) 804 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T04:44:54.054 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[27558]: cluster 2023-10-09T04:44:52.776581+0000 mon.a (mon.0) 805 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-10-09T04:44:54.054 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[27558]: audit 2023-10-09T04:44:52.777144+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T04:44:54.054 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[27558]: cluster 2023-10-09T04:44:52.777238+0000 mon.a (mon.0) 807 : cluster [DBG] fsmap foo:0 2023-10-09T04:44:54.054 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[27558]: cephadm 2023-10-09T04:44:52.780694+0000 mgr.y (mgr.14466) 119 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-10-09T04:44:54.054 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[27558]: audit 2023-10-09T04:44:52.787991+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:54.054 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[27558]: audit 2023-10-09T04:44:52.808686+0000 mon.a (mon.0) 809 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:44:54.054 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[27558]: audit 2023-10-09T04:44:52.811035+0000 mon.a (mon.0) 810 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:44:54.054 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[27558]: audit 2023-10-09T04:44:52.812394+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:44:54.054 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[27558]: audit 2023-10-09T04:44:52.820911+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:54.055 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[27558]: audit 2023-10-09T04:44:52.825575+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi133.tsuabh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:44:54.055 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[27558]: audit 2023-10-09T04:44:52.829982+0000 mon.a (mon.0) 814 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi133.tsuabh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T04:44:54.055 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[27558]: audit 2023-10-09T04:44:52.831169+0000 mon.a (mon.0) 815 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:44:54.055 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:53 smithi133 bash[27558]: cephadm 2023-10-09T04:44:52.832585+0000 mgr.y (mgr.14466) 120 : cephadm [INF] Deploying daemon mds.foo.smithi133.tsuabh on smithi133 2023-10-09T04:44:54.115 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-10-09T04:44:54.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:53 smithi194 bash[31648]: audit 2023-10-09T04:44:52.762626+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T04:44:54.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:53 smithi194 bash[31648]: cluster 2023-10-09T04:44:52.762751+0000 mon.a (mon.0) 801 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-10-09T04:44:54.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:53 smithi194 bash[31648]: audit 2023-10-09T04:44:52.763695+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T04:44:54.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:53 smithi194 bash[31648]: cluster 2023-10-09T04:44:52.764974+0000 mon.a (mon.0) 803 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T04:44:54.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:53 smithi194 bash[31648]: cluster 2023-10-09T04:44:52.765018+0000 mon.a (mon.0) 804 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T04:44:54.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:53 smithi194 bash[31648]: cluster 2023-10-09T04:44:52.776581+0000 mon.a (mon.0) 805 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-10-09T04:44:54.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:53 smithi194 bash[31648]: audit 2023-10-09T04:44:52.777144+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T04:44:54.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:53 smithi194 bash[31648]: cluster 2023-10-09T04:44:52.777238+0000 mon.a (mon.0) 807 : cluster [DBG] fsmap foo:0 2023-10-09T04:44:54.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:53 smithi194 bash[31648]: cephadm 2023-10-09T04:44:52.780694+0000 mgr.y (mgr.14466) 119 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-10-09T04:44:54.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:53 smithi194 bash[31648]: audit 2023-10-09T04:44:52.787991+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:54.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:53 smithi194 bash[31648]: audit 2023-10-09T04:44:52.808686+0000 mon.a (mon.0) 809 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:44:54.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:53 smithi194 bash[31648]: audit 2023-10-09T04:44:52.811035+0000 mon.a (mon.0) 810 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:44:54.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:53 smithi194 bash[31648]: audit 2023-10-09T04:44:52.812394+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:44:54.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:53 smithi194 bash[31648]: audit 2023-10-09T04:44:52.820911+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:54.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:53 smithi194 bash[31648]: audit 2023-10-09T04:44:52.825575+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi133.tsuabh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:44:54.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:53 smithi194 bash[31648]: audit 2023-10-09T04:44:52.829982+0000 mon.a (mon.0) 814 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi133.tsuabh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T04:44:54.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:53 smithi194 bash[31648]: audit 2023-10-09T04:44:52.831169+0000 mon.a (mon.0) 815 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:44:54.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:53 smithi194 bash[31648]: cephadm 2023-10-09T04:44:52.832585+0000 mgr.y (mgr.14466) 120 : cephadm [INF] Deploying daemon mds.foo.smithi133.tsuabh on smithi133 2023-10-09T04:44:55.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:54 smithi133 bash[27558]: cluster 2023-10-09T04:44:53.129183+0000 mgr.y (mgr.14466) 121 : cluster [DBG] pgmap v94: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:55.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:54 smithi133 bash[27558]: cluster 2023-10-09T04:44:53.776980+0000 mon.a (mon.0) 816 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-10-09T04:44:55.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:54 smithi133 bash[41291]: cluster 2023-10-09T04:44:53.129183+0000 mgr.y (mgr.14466) 121 : cluster [DBG] pgmap v94: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:55.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:54 smithi133 bash[41291]: cluster 2023-10-09T04:44:53.776980+0000 mon.a (mon.0) 816 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-10-09T04:44:55.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:54 smithi194 bash[31648]: cluster 2023-10-09T04:44:53.129183+0000 mgr.y (mgr.14466) 121 : cluster [DBG] pgmap v94: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:55.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:54 smithi194 bash[31648]: cluster 2023-10-09T04:44:53.776980+0000 mon.a (mon.0) 816 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-10-09T04:44:56.031 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:55 smithi133 bash[41291]: cluster 2023-10-09T04:44:54.784941+0000 mon.a (mon.0) 817 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-10-09T04:44:56.032 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:55 smithi133 bash[27558]: cluster 2023-10-09T04:44:54.784941+0000 mon.a (mon.0) 817 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-10-09T04:44:56.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:55 smithi194 bash[31648]: cluster 2023-10-09T04:44:54.784941+0000 mon.a (mon.0) 817 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-10-09T04:44:56.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:44:56 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:44:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:44:57.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:56 smithi133 bash[27558]: cluster 2023-10-09T04:44:55.129752+0000 mgr.y (mgr.14466) 122 : cluster [DBG] pgmap v97: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:57.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:56 smithi133 bash[27558]: audit 2023-10-09T04:44:55.991077+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:57.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:56 smithi133 bash[27558]: audit 2023-10-09T04:44:55.998432+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:57.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:56 smithi133 bash[27558]: audit 2023-10-09T04:44:56.008333+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:57.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:56 smithi133 bash[27558]: audit 2023-10-09T04:44:56.009291+0000 mon.a (mon.0) 821 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi194.yxnfqn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:44:57.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:56 smithi133 bash[27558]: audit 2023-10-09T04:44:56.013451+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi194.yxnfqn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T04:44:57.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:56 smithi133 bash[27558]: audit 2023-10-09T04:44:56.014694+0000 mon.a (mon.0) 823 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:44:57.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:56 smithi133 bash[41291]: cluster 2023-10-09T04:44:55.129752+0000 mgr.y (mgr.14466) 122 : cluster [DBG] pgmap v97: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:57.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:56 smithi133 bash[41291]: audit 2023-10-09T04:44:55.991077+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:57.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:56 smithi133 bash[41291]: audit 2023-10-09T04:44:55.998432+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:57.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:56 smithi133 bash[41291]: audit 2023-10-09T04:44:56.008333+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:57.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:56 smithi133 bash[41291]: audit 2023-10-09T04:44:56.009291+0000 mon.a (mon.0) 821 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi194.yxnfqn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:44:57.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:56 smithi133 bash[41291]: audit 2023-10-09T04:44:56.013451+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi194.yxnfqn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T04:44:57.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:56 smithi133 bash[41291]: audit 2023-10-09T04:44:56.014694+0000 mon.a (mon.0) 823 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:44:57.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:56 smithi194 bash[31648]: cluster 2023-10-09T04:44:55.129752+0000 mgr.y (mgr.14466) 122 : cluster [DBG] pgmap v97: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:57.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:56 smithi194 bash[31648]: audit 2023-10-09T04:44:55.991077+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:57.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:56 smithi194 bash[31648]: audit 2023-10-09T04:44:55.998432+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:57.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:56 smithi194 bash[31648]: audit 2023-10-09T04:44:56.008333+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:57.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:56 smithi194 bash[31648]: audit 2023-10-09T04:44:56.009291+0000 mon.a (mon.0) 821 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi194.yxnfqn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:44:57.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:56 smithi194 bash[31648]: audit 2023-10-09T04:44:56.013451+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi194.yxnfqn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T04:44:57.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:56 smithi194 bash[31648]: audit 2023-10-09T04:44:56.014694+0000 mon.a (mon.0) 823 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:44:58.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:58 smithi194 bash[31648]: cephadm 2023-10-09T04:44:56.016111+0000 mgr.y (mgr.14466) 123 : cephadm [INF] Deploying daemon mds.foo.smithi194.yxnfqn on smithi194 2023-10-09T04:44:58.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:57 smithi133 bash[27558]: cephadm 2023-10-09T04:44:56.016111+0000 mgr.y (mgr.14466) 123 : cephadm [INF] Deploying daemon mds.foo.smithi194.yxnfqn on smithi194 2023-10-09T04:44:58.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:58 smithi133 bash[41291]: cephadm 2023-10-09T04:44:56.016111+0000 mgr.y (mgr.14466) 123 : cephadm [INF] Deploying daemon mds.foo.smithi194.yxnfqn on smithi194 2023-10-09T04:44:59.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[27558]: cluster 2023-10-09T04:44:57.130611+0000 mgr.y (mgr.14466) 124 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:59.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[27558]: cluster 2023-10-09T04:44:58.003586+0000 mon.a (mon.0) 824 : cluster [DBG] mds.? [v2:172.21.15.133:6834/4227843513,v1:172.21.15.133:6835/4227843513] up:boot 2023-10-09T04:44:59.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[27558]: cluster 2023-10-09T04:44:58.003726+0000 mon.a (mon.0) 825 : cluster [INF] daemon mds.foo.smithi133.tsuabh assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T04:44:59.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[27558]: cluster 2023-10-09T04:44:58.003869+0000 mon.a (mon.0) 826 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T04:44:59.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[27558]: cluster 2023-10-09T04:44:58.003945+0000 mon.a (mon.0) 827 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T04:44:59.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[27558]: cluster 2023-10-09T04:44:58.003990+0000 mon.a (mon.0) 828 : cluster [INF] Cluster is now healthy 2023-10-09T04:44:59.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[27558]: cluster 2023-10-09T04:44:58.004312+0000 mon.a (mon.0) 829 : cluster [DBG] fsmap foo:0 1 up:standby 2023-10-09T04:44:59.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[27558]: cluster 2023-10-09T04:44:58.009489+0000 mon.a (mon.0) 830 : cluster [DBG] fsmap foo:1 {0=foo.smithi133.tsuabh=up:creating} 2023-10-09T04:44:59.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[27558]: audit 2023-10-09T04:44:58.009849+0000 mon.a (mon.0) 831 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi133.tsuabh"}]: dispatch 2023-10-09T04:44:59.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[27558]: cluster 2023-10-09T04:44:58.026209+0000 mon.a (mon.0) 832 : cluster [INF] daemon mds.foo.smithi133.tsuabh is now active in filesystem foo as rank 0 2023-10-09T04:44:59.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[27558]: audit 2023-10-09T04:44:58.788477+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:59.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[27558]: audit 2023-10-09T04:44:58.795517+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:59.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[27558]: audit 2023-10-09T04:44:58.843443+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:59.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[27558]: audit 2023-10-09T04:44:58.850369+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:59.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[27558]: audit 2023-10-09T04:44:58.855003+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:59.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[27558]: audit 2023-10-09T04:44:58.921252+0000 mon.a (mon.0) 838 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:44:59.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[41291]: cluster 2023-10-09T04:44:57.130611+0000 mgr.y (mgr.14466) 124 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:59.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[41291]: cluster 2023-10-09T04:44:58.003586+0000 mon.a (mon.0) 824 : cluster [DBG] mds.? [v2:172.21.15.133:6834/4227843513,v1:172.21.15.133:6835/4227843513] up:boot 2023-10-09T04:44:59.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[41291]: cluster 2023-10-09T04:44:58.003726+0000 mon.a (mon.0) 825 : cluster [INF] daemon mds.foo.smithi133.tsuabh assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T04:44:59.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[41291]: cluster 2023-10-09T04:44:58.003869+0000 mon.a (mon.0) 826 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T04:44:59.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[41291]: cluster 2023-10-09T04:44:58.003945+0000 mon.a (mon.0) 827 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T04:44:59.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[41291]: cluster 2023-10-09T04:44:58.003990+0000 mon.a (mon.0) 828 : cluster [INF] Cluster is now healthy 2023-10-09T04:44:59.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[41291]: cluster 2023-10-09T04:44:58.004312+0000 mon.a (mon.0) 829 : cluster [DBG] fsmap foo:0 1 up:standby 2023-10-09T04:44:59.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[41291]: cluster 2023-10-09T04:44:58.009489+0000 mon.a (mon.0) 830 : cluster [DBG] fsmap foo:1 {0=foo.smithi133.tsuabh=up:creating} 2023-10-09T04:44:59.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[41291]: audit 2023-10-09T04:44:58.009849+0000 mon.a (mon.0) 831 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi133.tsuabh"}]: dispatch 2023-10-09T04:44:59.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[41291]: cluster 2023-10-09T04:44:58.026209+0000 mon.a (mon.0) 832 : cluster [INF] daemon mds.foo.smithi133.tsuabh is now active in filesystem foo as rank 0 2023-10-09T04:44:59.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[41291]: audit 2023-10-09T04:44:58.788477+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:59.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[41291]: audit 2023-10-09T04:44:58.795517+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:59.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[41291]: audit 2023-10-09T04:44:58.843443+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:59.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[41291]: audit 2023-10-09T04:44:58.850369+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:59.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[41291]: audit 2023-10-09T04:44:58.855003+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:59.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:44:59 smithi133 bash[41291]: audit 2023-10-09T04:44:58.921252+0000 mon.a (mon.0) 838 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:44:59.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:59 smithi194 bash[31648]: cluster 2023-10-09T04:44:57.130611+0000 mgr.y (mgr.14466) 124 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:44:59.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:59 smithi194 bash[31648]: cluster 2023-10-09T04:44:58.003586+0000 mon.a (mon.0) 824 : cluster [DBG] mds.? [v2:172.21.15.133:6834/4227843513,v1:172.21.15.133:6835/4227843513] up:boot 2023-10-09T04:44:59.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:59 smithi194 bash[31648]: cluster 2023-10-09T04:44:58.003726+0000 mon.a (mon.0) 825 : cluster [INF] daemon mds.foo.smithi133.tsuabh assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T04:44:59.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:59 smithi194 bash[31648]: cluster 2023-10-09T04:44:58.003869+0000 mon.a (mon.0) 826 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T04:44:59.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:59 smithi194 bash[31648]: cluster 2023-10-09T04:44:58.003945+0000 mon.a (mon.0) 827 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T04:44:59.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:59 smithi194 bash[31648]: cluster 2023-10-09T04:44:58.003990+0000 mon.a (mon.0) 828 : cluster [INF] Cluster is now healthy 2023-10-09T04:44:59.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:59 smithi194 bash[31648]: cluster 2023-10-09T04:44:58.004312+0000 mon.a (mon.0) 829 : cluster [DBG] fsmap foo:0 1 up:standby 2023-10-09T04:44:59.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:59 smithi194 bash[31648]: cluster 2023-10-09T04:44:58.009489+0000 mon.a (mon.0) 830 : cluster [DBG] fsmap foo:1 {0=foo.smithi133.tsuabh=up:creating} 2023-10-09T04:44:59.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:59 smithi194 bash[31648]: audit 2023-10-09T04:44:58.009849+0000 mon.a (mon.0) 831 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi133.tsuabh"}]: dispatch 2023-10-09T04:44:59.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:59 smithi194 bash[31648]: cluster 2023-10-09T04:44:58.026209+0000 mon.a (mon.0) 832 : cluster [INF] daemon mds.foo.smithi133.tsuabh is now active in filesystem foo as rank 0 2023-10-09T04:44:59.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:59 smithi194 bash[31648]: audit 2023-10-09T04:44:58.788477+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:59.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:59 smithi194 bash[31648]: audit 2023-10-09T04:44:58.795517+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:59.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:59 smithi194 bash[31648]: audit 2023-10-09T04:44:58.843443+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:59.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:59 smithi194 bash[31648]: audit 2023-10-09T04:44:58.850369+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:59.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:59 smithi194 bash[31648]: audit 2023-10-09T04:44:58.855003+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:44:59.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:44:59 smithi194 bash[31648]: audit 2023-10-09T04:44:58.921252+0000 mon.a (mon.0) 838 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:44:59.885 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:00.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:00 smithi194 bash[31648]: cluster 2023-10-09T04:44:59.012460+0000 mon.a (mon.0) 839 : cluster [DBG] mds.? [v2:172.21.15.133:6834/4227843513,v1:172.21.15.133:6835/4227843513] up:active 2023-10-09T04:45:00.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:00 smithi194 bash[31648]: cluster 2023-10-09T04:44:59.012530+0000 mon.a (mon.0) 840 : cluster [DBG] fsmap foo:1 {0=foo.smithi133.tsuabh=up:active} 2023-10-09T04:45:00.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:00 smithi133 bash[41291]: cluster 2023-10-09T04:44:59.012460+0000 mon.a (mon.0) 839 : cluster [DBG] mds.? [v2:172.21.15.133:6834/4227843513,v1:172.21.15.133:6835/4227843513] up:active 2023-10-09T04:45:00.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:00 smithi133 bash[41291]: cluster 2023-10-09T04:44:59.012530+0000 mon.a (mon.0) 840 : cluster [DBG] fsmap foo:1 {0=foo.smithi133.tsuabh=up:active} 2023-10-09T04:45:00.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:00 smithi133 bash[27558]: cluster 2023-10-09T04:44:59.012460+0000 mon.a (mon.0) 839 : cluster [DBG] mds.? [v2:172.21.15.133:6834/4227843513,v1:172.21.15.133:6835/4227843513] up:active 2023-10-09T04:45:00.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:00 smithi133 bash[27558]: cluster 2023-10-09T04:44:59.012530+0000 mon.a (mon.0) 840 : cluster [DBG] fsmap foo:1 {0=foo.smithi133.tsuabh=up:active} 2023-10-09T04:45:00.800 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:45:00 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:45:00] "GET /metrics HTTP/1.1" 200 203310 "" "Prometheus/2.43.0" 2023-10-09T04:45:01.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:01 smithi133 bash[27558]: cluster 2023-10-09T04:44:59.131075+0000 mgr.y (mgr.14466) 125 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:01.441 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:01 smithi133 bash[27558]: cluster 2023-10-09T04:45:00.021561+0000 mon.a (mon.0) 841 : cluster [DBG] mds.? [v2:172.21.15.194:6832/1679470116,v1:172.21.15.194:6833/1679470116] up:boot 2023-10-09T04:45:01.442 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:01 smithi133 bash[27558]: cluster 2023-10-09T04:45:00.021615+0000 mon.a (mon.0) 842 : cluster [INF] Dropping low affinity active daemon mds.foo.smithi133.tsuabh in favor of higher affinity standby. 2023-10-09T04:45:01.442 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:01 smithi133 bash[27558]: cluster 2023-10-09T04:45:00.021641+0000 mon.a (mon.0) 843 : cluster [WRN] Replacing daemon mds.foo.smithi133.tsuabh as rank 0 with standby daemon mds.foo.smithi194.yxnfqn 2023-10-09T04:45:01.442 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:01 smithi133 bash[27558]: cluster 2023-10-09T04:45:00.021956+0000 mon.a (mon.0) 844 : cluster [WRN] Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2023-10-09T04:45:01.442 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:01 smithi133 bash[27558]: cluster 2023-10-09T04:45:00.021977+0000 mon.a (mon.0) 845 : cluster [WRN] Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2023-10-09T04:45:01.442 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:01 smithi133 bash[27558]: cluster 2023-10-09T04:45:00.022209+0000 mon.a (mon.0) 846 : cluster [DBG] fsmap foo:1 {0=foo.smithi133.tsuabh=up:active} 1 up:standby 2023-10-09T04:45:01.442 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:01 smithi133 bash[27558]: audit 2023-10-09T04:45:00.022385+0000 mon.a (mon.0) 847 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi194.yxnfqn"}]: dispatch 2023-10-09T04:45:01.442 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:01 smithi133 bash[27558]: cluster 2023-10-09T04:45:00.029698+0000 mon.a (mon.0) 848 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-10-09T04:45:01.442 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:01 smithi133 bash[27558]: cluster 2023-10-09T04:45:00.029976+0000 mon.a (mon.0) 849 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi194.yxnfqn=up:replay} 2023-10-09T04:45:01.442 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:01 smithi133 bash[27558]: audit 2023-10-09T04:45:00.241873+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:45:01.443 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:01 smithi133 bash[41291]: cluster 2023-10-09T04:44:59.131075+0000 mgr.y (mgr.14466) 125 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:01.443 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:01 smithi133 bash[41291]: cluster 2023-10-09T04:45:00.021561+0000 mon.a (mon.0) 841 : cluster [DBG] mds.? [v2:172.21.15.194:6832/1679470116,v1:172.21.15.194:6833/1679470116] up:boot 2023-10-09T04:45:01.443 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:01 smithi133 bash[41291]: cluster 2023-10-09T04:45:00.021615+0000 mon.a (mon.0) 842 : cluster [INF] Dropping low affinity active daemon mds.foo.smithi133.tsuabh in favor of higher affinity standby. 2023-10-09T04:45:01.443 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:01 smithi133 bash[41291]: cluster 2023-10-09T04:45:00.021641+0000 mon.a (mon.0) 843 : cluster [WRN] Replacing daemon mds.foo.smithi133.tsuabh as rank 0 with standby daemon mds.foo.smithi194.yxnfqn 2023-10-09T04:45:01.443 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:01 smithi133 bash[41291]: cluster 2023-10-09T04:45:00.021956+0000 mon.a (mon.0) 844 : cluster [WRN] Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2023-10-09T04:45:01.443 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:01 smithi133 bash[41291]: cluster 2023-10-09T04:45:00.021977+0000 mon.a (mon.0) 845 : cluster [WRN] Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2023-10-09T04:45:01.443 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:01 smithi133 bash[41291]: cluster 2023-10-09T04:45:00.022209+0000 mon.a (mon.0) 846 : cluster [DBG] fsmap foo:1 {0=foo.smithi133.tsuabh=up:active} 1 up:standby 2023-10-09T04:45:01.443 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:01 smithi133 bash[41291]: audit 2023-10-09T04:45:00.022385+0000 mon.a (mon.0) 847 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi194.yxnfqn"}]: dispatch 2023-10-09T04:45:01.443 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:01 smithi133 bash[41291]: cluster 2023-10-09T04:45:00.029698+0000 mon.a (mon.0) 848 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-10-09T04:45:01.443 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:01 smithi133 bash[41291]: cluster 2023-10-09T04:45:00.029976+0000 mon.a (mon.0) 849 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi194.yxnfqn=up:replay} 2023-10-09T04:45:01.443 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:01 smithi133 bash[41291]: audit 2023-10-09T04:45:00.241873+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:45:01.444 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:01 smithi194 bash[31648]: cluster 2023-10-09T04:44:59.131075+0000 mgr.y (mgr.14466) 125 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:01.444 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:01 smithi194 bash[31648]: cluster 2023-10-09T04:45:00.021561+0000 mon.a (mon.0) 841 : cluster [DBG] mds.? [v2:172.21.15.194:6832/1679470116,v1:172.21.15.194:6833/1679470116] up:boot 2023-10-09T04:45:01.444 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:01 smithi194 bash[31648]: cluster 2023-10-09T04:45:00.021615+0000 mon.a (mon.0) 842 : cluster [INF] Dropping low affinity active daemon mds.foo.smithi133.tsuabh in favor of higher affinity standby. 2023-10-09T04:45:01.444 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:01 smithi194 bash[31648]: cluster 2023-10-09T04:45:00.021641+0000 mon.a (mon.0) 843 : cluster [WRN] Replacing daemon mds.foo.smithi133.tsuabh as rank 0 with standby daemon mds.foo.smithi194.yxnfqn 2023-10-09T04:45:01.444 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:01 smithi194 bash[31648]: cluster 2023-10-09T04:45:00.021956+0000 mon.a (mon.0) 844 : cluster [WRN] Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2023-10-09T04:45:01.444 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:01 smithi194 bash[31648]: cluster 2023-10-09T04:45:00.021977+0000 mon.a (mon.0) 845 : cluster [WRN] Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2023-10-09T04:45:01.444 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:01 smithi194 bash[31648]: cluster 2023-10-09T04:45:00.022209+0000 mon.a (mon.0) 846 : cluster [DBG] fsmap foo:1 {0=foo.smithi133.tsuabh=up:active} 1 up:standby 2023-10-09T04:45:01.444 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:01 smithi194 bash[31648]: audit 2023-10-09T04:45:00.022385+0000 mon.a (mon.0) 847 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi194.yxnfqn"}]: dispatch 2023-10-09T04:45:01.445 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:01 smithi194 bash[31648]: cluster 2023-10-09T04:45:00.029698+0000 mon.a (mon.0) 848 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-10-09T04:45:01.445 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:01 smithi194 bash[31648]: cluster 2023-10-09T04:45:00.029976+0000 mon.a (mon.0) 849 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi194.yxnfqn=up:replay} 2023-10-09T04:45:01.445 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:01 smithi194 bash[31648]: audit 2023-10-09T04:45:00.241873+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:45:02.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:02 smithi133 bash[27558]: cluster 2023-10-09T04:45:01.024076+0000 mon.a (mon.0) 851 : cluster [INF] Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2023-10-09T04:45:02.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:02 smithi133 bash[27558]: cluster 2023-10-09T04:45:01.028653+0000 mon.a (mon.0) 852 : cluster [DBG] mds.? [v2:172.21.15.194:6832/1679470116,v1:172.21.15.194:6833/1679470116] up:reconnect 2023-10-09T04:45:02.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:02 smithi133 bash[27558]: cluster 2023-10-09T04:45:01.028713+0000 mon.a (mon.0) 853 : cluster [DBG] mds.? [v2:172.21.15.133:6834/3356898887,v1:172.21.15.133:6835/3356898887] up:boot 2023-10-09T04:45:02.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:02 smithi133 bash[27558]: cluster 2023-10-09T04:45:01.028751+0000 mon.a (mon.0) 854 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi194.yxnfqn=up:reconnect} 1 up:standby 2023-10-09T04:45:02.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:02 smithi133 bash[27558]: audit 2023-10-09T04:45:01.028967+0000 mon.a (mon.0) 855 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi133.tsuabh"}]: dispatch 2023-10-09T04:45:02.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:02 smithi133 bash[41291]: cluster 2023-10-09T04:45:01.024076+0000 mon.a (mon.0) 851 : cluster [INF] Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2023-10-09T04:45:02.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:02 smithi133 bash[41291]: cluster 2023-10-09T04:45:01.028653+0000 mon.a (mon.0) 852 : cluster [DBG] mds.? [v2:172.21.15.194:6832/1679470116,v1:172.21.15.194:6833/1679470116] up:reconnect 2023-10-09T04:45:02.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:02 smithi133 bash[41291]: cluster 2023-10-09T04:45:01.028713+0000 mon.a (mon.0) 853 : cluster [DBG] mds.? [v2:172.21.15.133:6834/3356898887,v1:172.21.15.133:6835/3356898887] up:boot 2023-10-09T04:45:02.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:02 smithi133 bash[41291]: cluster 2023-10-09T04:45:01.028751+0000 mon.a (mon.0) 854 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi194.yxnfqn=up:reconnect} 1 up:standby 2023-10-09T04:45:02.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:02 smithi133 bash[41291]: audit 2023-10-09T04:45:01.028967+0000 mon.a (mon.0) 855 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi133.tsuabh"}]: dispatch 2023-10-09T04:45:02.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:02 smithi194 bash[31648]: cluster 2023-10-09T04:45:01.024076+0000 mon.a (mon.0) 851 : cluster [INF] Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2023-10-09T04:45:02.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:02 smithi194 bash[31648]: cluster 2023-10-09T04:45:01.028653+0000 mon.a (mon.0) 852 : cluster [DBG] mds.? [v2:172.21.15.194:6832/1679470116,v1:172.21.15.194:6833/1679470116] up:reconnect 2023-10-09T04:45:02.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:02 smithi194 bash[31648]: cluster 2023-10-09T04:45:01.028713+0000 mon.a (mon.0) 853 : cluster [DBG] mds.? [v2:172.21.15.133:6834/3356898887,v1:172.21.15.133:6835/3356898887] up:boot 2023-10-09T04:45:02.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:02 smithi194 bash[31648]: cluster 2023-10-09T04:45:01.028751+0000 mon.a (mon.0) 854 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi194.yxnfqn=up:reconnect} 1 up:standby 2023-10-09T04:45:02.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:02 smithi194 bash[31648]: audit 2023-10-09T04:45:01.028967+0000 mon.a (mon.0) 855 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi133.tsuabh"}]: dispatch 2023-10-09T04:45:03.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:03 smithi133 bash[41291]: cluster 2023-10-09T04:45:01.132126+0000 mgr.y (mgr.14466) 126 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.8 KiB/s wr, 7 op/s 2023-10-09T04:45:03.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:03 smithi133 bash[41291]: cluster 2023-10-09T04:45:02.035307+0000 mon.a (mon.0) 856 : cluster [DBG] mds.? [v2:172.21.15.194:6832/1679470116,v1:172.21.15.194:6833/1679470116] up:rejoin 2023-10-09T04:45:03.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:03 smithi133 bash[41291]: cluster 2023-10-09T04:45:02.035395+0000 mon.a (mon.0) 857 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi194.yxnfqn=up:rejoin} 1 up:standby 2023-10-09T04:45:03.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:03 smithi133 bash[41291]: cluster 2023-10-09T04:45:02.039225+0000 mon.a (mon.0) 858 : cluster [INF] daemon mds.foo.smithi194.yxnfqn is now active in filesystem foo as rank 0 2023-10-09T04:45:03.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:03 smithi133 bash[27558]: cluster 2023-10-09T04:45:01.132126+0000 mgr.y (mgr.14466) 126 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.8 KiB/s wr, 7 op/s 2023-10-09T04:45:03.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:03 smithi133 bash[27558]: cluster 2023-10-09T04:45:02.035307+0000 mon.a (mon.0) 856 : cluster [DBG] mds.? [v2:172.21.15.194:6832/1679470116,v1:172.21.15.194:6833/1679470116] up:rejoin 2023-10-09T04:45:03.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:03 smithi133 bash[27558]: cluster 2023-10-09T04:45:02.035395+0000 mon.a (mon.0) 857 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi194.yxnfqn=up:rejoin} 1 up:standby 2023-10-09T04:45:03.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:03 smithi133 bash[27558]: cluster 2023-10-09T04:45:02.039225+0000 mon.a (mon.0) 858 : cluster [INF] daemon mds.foo.smithi194.yxnfqn is now active in filesystem foo as rank 0 2023-10-09T04:45:03.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:03 smithi194 bash[31648]: cluster 2023-10-09T04:45:01.132126+0000 mgr.y (mgr.14466) 126 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.8 KiB/s wr, 7 op/s 2023-10-09T04:45:03.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:03 smithi194 bash[31648]: cluster 2023-10-09T04:45:02.035307+0000 mon.a (mon.0) 856 : cluster [DBG] mds.? [v2:172.21.15.194:6832/1679470116,v1:172.21.15.194:6833/1679470116] up:rejoin 2023-10-09T04:45:03.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:03 smithi194 bash[31648]: cluster 2023-10-09T04:45:02.035395+0000 mon.a (mon.0) 857 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi194.yxnfqn=up:rejoin} 1 up:standby 2023-10-09T04:45:03.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:03 smithi194 bash[31648]: cluster 2023-10-09T04:45:02.039225+0000 mon.a (mon.0) 858 : cluster [INF] daemon mds.foo.smithi194.yxnfqn is now active in filesystem foo as rank 0 2023-10-09T04:45:04.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:04 smithi133 bash[41291]: cluster 2023-10-09T04:45:03.032709+0000 mon.a (mon.0) 859 : cluster [INF] Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2023-10-09T04:45:04.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:04 smithi133 bash[41291]: cluster 2023-10-09T04:45:03.032742+0000 mon.a (mon.0) 860 : cluster [INF] Cluster is now healthy 2023-10-09T04:45:04.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:04 smithi133 bash[41291]: cluster 2023-10-09T04:45:03.037433+0000 mon.a (mon.0) 861 : cluster [DBG] mds.? [v2:172.21.15.194:6832/1679470116,v1:172.21.15.194:6833/1679470116] up:active 2023-10-09T04:45:04.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:04 smithi133 bash[41291]: cluster 2023-10-09T04:45:03.037533+0000 mon.a (mon.0) 862 : cluster [DBG] fsmap foo:1 {0=foo.smithi194.yxnfqn=up:active} 1 up:standby 2023-10-09T04:45:04.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:04 smithi133 bash[41291]: cluster 2023-10-09T04:45:03.132609+0000 mgr.y (mgr.14466) 127 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.6 KiB/s wr, 6 op/s 2023-10-09T04:45:04.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:04 smithi133 bash[41291]: audit 2023-10-09T04:45:03.814146+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:45:04.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:04 smithi133 bash[41291]: audit 2023-10-09T04:45:03.820254+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:45:04.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:04 smithi133 bash[27558]: cluster 2023-10-09T04:45:03.032709+0000 mon.a (mon.0) 859 : cluster [INF] Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2023-10-09T04:45:04.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:04 smithi133 bash[27558]: cluster 2023-10-09T04:45:03.032742+0000 mon.a (mon.0) 860 : cluster [INF] Cluster is now healthy 2023-10-09T04:45:04.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:04 smithi133 bash[27558]: cluster 2023-10-09T04:45:03.037433+0000 mon.a (mon.0) 861 : cluster [DBG] mds.? [v2:172.21.15.194:6832/1679470116,v1:172.21.15.194:6833/1679470116] up:active 2023-10-09T04:45:04.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:04 smithi133 bash[27558]: cluster 2023-10-09T04:45:03.037533+0000 mon.a (mon.0) 862 : cluster [DBG] fsmap foo:1 {0=foo.smithi194.yxnfqn=up:active} 1 up:standby 2023-10-09T04:45:04.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:04 smithi133 bash[27558]: cluster 2023-10-09T04:45:03.132609+0000 mgr.y (mgr.14466) 127 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.6 KiB/s wr, 6 op/s 2023-10-09T04:45:04.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:04 smithi133 bash[27558]: audit 2023-10-09T04:45:03.814146+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:45:04.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:04 smithi133 bash[27558]: audit 2023-10-09T04:45:03.820254+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:45:04.361 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-10-09T04:45:04.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:04 smithi194 bash[31648]: cluster 2023-10-09T04:45:03.032709+0000 mon.a (mon.0) 859 : cluster [INF] Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2023-10-09T04:45:04.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:04 smithi194 bash[31648]: cluster 2023-10-09T04:45:03.032742+0000 mon.a (mon.0) 860 : cluster [INF] Cluster is now healthy 2023-10-09T04:45:04.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:04 smithi194 bash[31648]: cluster 2023-10-09T04:45:03.037433+0000 mon.a (mon.0) 861 : cluster [DBG] mds.? [v2:172.21.15.194:6832/1679470116,v1:172.21.15.194:6833/1679470116] up:active 2023-10-09T04:45:04.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:04 smithi194 bash[31648]: cluster 2023-10-09T04:45:03.037533+0000 mon.a (mon.0) 862 : cluster [DBG] fsmap foo:1 {0=foo.smithi194.yxnfqn=up:active} 1 up:standby 2023-10-09T04:45:04.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:04 smithi194 bash[31648]: cluster 2023-10-09T04:45:03.132609+0000 mgr.y (mgr.14466) 127 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.6 KiB/s wr, 6 op/s 2023-10-09T04:45:04.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:04 smithi194 bash[31648]: audit 2023-10-09T04:45:03.814146+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:45:04.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:04 smithi194 bash[31648]: audit 2023-10-09T04:45:03.820254+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:45:06.353 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:06 smithi194 bash[31648]: audit 2023-10-09T04:45:05.076829+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:45:06.354 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:06 smithi194 bash[31648]: audit 2023-10-09T04:45:05.087798+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:45:06.354 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:06 smithi194 bash[31648]: audit 2023-10-09T04:45:05.089399+0000 mon.a (mon.0) 867 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:45:06.354 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:06 smithi194 bash[31648]: audit 2023-10-09T04:45:05.090931+0000 mon.a (mon.0) 868 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:45:06.354 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:06 smithi194 bash[31648]: audit 2023-10-09T04:45:05.098443+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:45:06.354 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:06 smithi194 bash[31648]: audit 2023-10-09T04:45:05.130077+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T04:45:06.354 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:06 smithi194 bash[31648]: audit 2023-10-09T04:45:05.130659+0000 mon.a (mon.0) 871 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2023-10-09T04:45:06.354 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:06 smithi194 bash[31648]: audit 2023-10-09T04:45:05.130931+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-10-09T04:45:06.355 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:06 smithi194 bash[31648]: cluster 2023-10-09T04:45:05.133100+0000 mgr.y (mgr.14466) 128 : cluster [DBG] pgmap v103: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.3 KiB/s wr, 5 op/s 2023-10-09T04:45:06.355 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:06 smithi194 bash[31648]: audit 2023-10-09T04:45:05.174129+0000 mon.a (mon.0) 873 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:45:06.355 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:06 smithi194 bash[31648]: audit 2023-10-09T04:45:05.177508+0000 mon.a (mon.0) 874 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:45:06.355 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:06 smithi194 bash[31648]: audit 2023-10-09T04:45:05.178962+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:45:06.355 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:06 smithi194 bash[31648]: audit 2023-10-09T04:45:05.187902+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:45:06.355 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:06 smithi194 bash[31648]: audit 2023-10-09T04:45:05.189304+0000 mon.a (mon.0) 877 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:06.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:06 smithi133 bash[41291]: audit 2023-10-09T04:45:05.076829+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:45:06.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:06 smithi133 bash[41291]: audit 2023-10-09T04:45:05.087798+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:45:06.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:06 smithi133 bash[41291]: audit 2023-10-09T04:45:05.089399+0000 mon.a (mon.0) 867 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:45:06.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:06 smithi133 bash[41291]: audit 2023-10-09T04:45:05.090931+0000 mon.a (mon.0) 868 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:45:06.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:06 smithi133 bash[41291]: audit 2023-10-09T04:45:05.098443+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:45:06.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:06 smithi133 bash[41291]: audit 2023-10-09T04:45:05.130077+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T04:45:06.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:06 smithi133 bash[41291]: audit 2023-10-09T04:45:05.130659+0000 mon.a (mon.0) 871 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2023-10-09T04:45:06.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:06 smithi133 bash[41291]: audit 2023-10-09T04:45:05.130931+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-10-09T04:45:06.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:06 smithi133 bash[41291]: cluster 2023-10-09T04:45:05.133100+0000 mgr.y (mgr.14466) 128 : cluster [DBG] pgmap v103: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.3 KiB/s wr, 5 op/s 2023-10-09T04:45:06.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:06 smithi133 bash[41291]: audit 2023-10-09T04:45:05.174129+0000 mon.a (mon.0) 873 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:45:06.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:06 smithi133 bash[41291]: audit 2023-10-09T04:45:05.177508+0000 mon.a (mon.0) 874 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:45:06.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:06 smithi133 bash[41291]: audit 2023-10-09T04:45:05.178962+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:45:06.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:06 smithi133 bash[41291]: audit 2023-10-09T04:45:05.187902+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:45:06.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:06 smithi133 bash[41291]: audit 2023-10-09T04:45:05.189304+0000 mon.a (mon.0) 877 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:06.554 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:06 smithi133 bash[27558]: audit 2023-10-09T04:45:05.076829+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:45:06.554 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:06 smithi133 bash[27558]: audit 2023-10-09T04:45:05.087798+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:45:06.554 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:06 smithi133 bash[27558]: audit 2023-10-09T04:45:05.089399+0000 mon.a (mon.0) 867 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:45:06.554 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:06 smithi133 bash[27558]: audit 2023-10-09T04:45:05.090931+0000 mon.a (mon.0) 868 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:45:06.554 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:06 smithi133 bash[27558]: audit 2023-10-09T04:45:05.098443+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:45:06.555 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:06 smithi133 bash[27558]: audit 2023-10-09T04:45:05.130077+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T04:45:06.555 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:06 smithi133 bash[27558]: audit 2023-10-09T04:45:05.130659+0000 mon.a (mon.0) 871 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2023-10-09T04:45:06.555 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:06 smithi133 bash[27558]: audit 2023-10-09T04:45:05.130931+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-10-09T04:45:06.555 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:06 smithi133 bash[27558]: cluster 2023-10-09T04:45:05.133100+0000 mgr.y (mgr.14466) 128 : cluster [DBG] pgmap v103: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.3 KiB/s wr, 5 op/s 2023-10-09T04:45:06.556 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:06 smithi133 bash[27558]: audit 2023-10-09T04:45:05.174129+0000 mon.a (mon.0) 873 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:45:06.556 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:06 smithi133 bash[27558]: audit 2023-10-09T04:45:05.177508+0000 mon.a (mon.0) 874 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:45:06.556 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:06 smithi133 bash[27558]: audit 2023-10-09T04:45:05.178962+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:45:06.556 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:06 smithi133 bash[27558]: audit 2023-10-09T04:45:05.187902+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:45:06.557 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:06 smithi133 bash[27558]: audit 2023-10-09T04:45:05.189304+0000 mon.a (mon.0) 877 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:06.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:45:06 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:45:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:45:07.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:07 smithi194 bash[31648]: audit 2023-10-09T04:45:06.113154+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-09T04:45:07.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:07 smithi194 bash[31648]: audit 2023-10-09T04:45:06.113295+0000 mon.a (mon.0) 879 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2023-10-09T04:45:07.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:07 smithi194 bash[31648]: audit 2023-10-09T04:45:06.113467+0000 mon.a (mon.0) 880 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-10-09T04:45:07.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:07 smithi194 bash[31648]: cluster 2023-10-09T04:45:06.113551+0000 mon.a (mon.0) 881 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-10-09T04:45:07.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:07 smithi133 bash[27558]: audit 2023-10-09T04:45:06.113154+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-09T04:45:07.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:07 smithi133 bash[27558]: audit 2023-10-09T04:45:06.113295+0000 mon.a (mon.0) 879 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2023-10-09T04:45:07.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:07 smithi133 bash[27558]: audit 2023-10-09T04:45:06.113467+0000 mon.a (mon.0) 880 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-10-09T04:45:07.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:07 smithi133 bash[27558]: cluster 2023-10-09T04:45:06.113551+0000 mon.a (mon.0) 881 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-10-09T04:45:07.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:07 smithi133 bash[41291]: audit 2023-10-09T04:45:06.113154+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-09T04:45:07.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:07 smithi133 bash[41291]: audit 2023-10-09T04:45:06.113295+0000 mon.a (mon.0) 879 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2023-10-09T04:45:07.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:07 smithi133 bash[41291]: audit 2023-10-09T04:45:06.113467+0000 mon.a (mon.0) 880 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-10-09T04:45:07.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:07 smithi133 bash[41291]: cluster 2023-10-09T04:45:06.113551+0000 mon.a (mon.0) 881 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-10-09T04:45:08.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:08 smithi194 bash[31648]: cluster 2023-10-09T04:45:07.113764+0000 mon.a (mon.0) 882 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-10-09T04:45:08.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:08 smithi194 bash[31648]: cluster 2023-10-09T04:45:07.133561+0000 mgr.y (mgr.14466) 129 : cluster [DBG] pgmap v106: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 1.8 KiB/s wr, 12 op/s 2023-10-09T04:45:08.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:08 smithi133 bash[41291]: cluster 2023-10-09T04:45:07.113764+0000 mon.a (mon.0) 882 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-10-09T04:45:08.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:08 smithi133 bash[41291]: cluster 2023-10-09T04:45:07.133561+0000 mgr.y (mgr.14466) 129 : cluster [DBG] pgmap v106: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 1.8 KiB/s wr, 12 op/s 2023-10-09T04:45:08.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:08 smithi133 bash[27558]: cluster 2023-10-09T04:45:07.113764+0000 mon.a (mon.0) 882 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-10-09T04:45:08.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:08 smithi133 bash[27558]: cluster 2023-10-09T04:45:07.133561+0000 mgr.y (mgr.14466) 129 : cluster [DBG] pgmap v106: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 1.8 KiB/s wr, 12 op/s 2023-10-09T04:45:09.137 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:10.348 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:10 smithi133 bash[41291]: cluster 2023-10-09T04:45:09.133998+0000 mgr.y (mgr.14466) 130 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 0 B/s wr, 4 op/s 2023-10-09T04:45:10.348 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:10 smithi133 bash[27558]: cluster 2023-10-09T04:45:09.133998+0000 mgr.y (mgr.14466) 130 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 0 B/s wr, 4 op/s 2023-10-09T04:45:10.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:10 smithi194 bash[31648]: cluster 2023-10-09T04:45:09.133998+0000 mgr.y (mgr.14466) 130 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 0 B/s wr, 4 op/s 2023-10-09T04:45:10.800 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:45:10 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:45:10] "GET /metrics HTTP/1.1" 200 238890 "" "Prometheus/2.43.0" 2023-10-09T04:45:11.554 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:11 smithi133 bash[41291]: audit 2023-10-09T04:45:10.254289+0000 mon.a (mon.0) 883 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:45:11.554 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:11 smithi133 bash[27558]: audit 2023-10-09T04:45:10.254289+0000 mon.a (mon.0) 883 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:45:11.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:11 smithi194 bash[31648]: audit 2023-10-09T04:45:10.254289+0000 mon.a (mon.0) 883 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:45:12.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:12 smithi133 bash[27558]: cluster 2023-10-09T04:45:11.134992+0000 mgr.y (mgr.14466) 131 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 0 B/s wr, 4 op/s 2023-10-09T04:45:12.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:12 smithi133 bash[41291]: cluster 2023-10-09T04:45:11.134992+0000 mgr.y (mgr.14466) 131 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 0 B/s wr, 4 op/s 2023-10-09T04:45:12.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:12 smithi194 bash[31648]: cluster 2023-10-09T04:45:11.134992+0000 mgr.y (mgr.14466) 131 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 0 B/s wr, 4 op/s 2023-10-09T04:45:13.311 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-10-09T04:45:13.320 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T04:45:13.320 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T04:45:13.320 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T04:45:14.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:14 smithi133 bash[27558]: cluster 2023-10-09T04:45:13.135622+0000 mgr.y (mgr.14466) 132 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 0 B/s wr, 4 op/s 2023-10-09T04:45:14.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:14 smithi133 bash[41291]: cluster 2023-10-09T04:45:13.135622+0000 mgr.y (mgr.14466) 132 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 0 B/s wr, 4 op/s 2023-10-09T04:45:14.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:14 smithi194 bash[31648]: cluster 2023-10-09T04:45:13.135622+0000 mgr.y (mgr.14466) 132 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 0 B/s wr, 4 op/s 2023-10-09T04:45:16.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:16 smithi194 bash[31648]: cluster 2023-10-09T04:45:15.136248+0000 mgr.y (mgr.14466) 133 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:16.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:45:16 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:45:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:45:16.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:16 smithi133 bash[41291]: cluster 2023-10-09T04:45:15.136248+0000 mgr.y (mgr.14466) 133 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:16.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:16 smithi133 bash[27558]: cluster 2023-10-09T04:45:15.136248+0000 mgr.y (mgr.14466) 133 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:18.102 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:18.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:18 smithi133 bash[27558]: cluster 2023-10-09T04:45:17.137355+0000 mgr.y (mgr.14466) 134 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:18.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:18 smithi133 bash[41291]: cluster 2023-10-09T04:45:17.137355+0000 mgr.y (mgr.14466) 134 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:18.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:18 smithi194 bash[31648]: cluster 2023-10-09T04:45:17.137355+0000 mgr.y (mgr.14466) 134 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:20.800 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:45:20 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:45:20] "GET /metrics HTTP/1.1" 200 238955 "" "Prometheus/2.43.0" 2023-10-09T04:45:20.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:20 smithi133 bash[27558]: cluster 2023-10-09T04:45:19.137966+0000 mgr.y (mgr.14466) 135 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:20.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:20 smithi133 bash[27558]: audit 2023-10-09T04:45:20.189683+0000 mon.a (mon.0) 884 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:20.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:20 smithi133 bash[41291]: cluster 2023-10-09T04:45:19.137966+0000 mgr.y (mgr.14466) 135 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:20.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:20 smithi133 bash[41291]: audit 2023-10-09T04:45:20.189683+0000 mon.a (mon.0) 884 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:20.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:20 smithi194 bash[31648]: cluster 2023-10-09T04:45:19.137966+0000 mgr.y (mgr.14466) 135 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:20.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:20 smithi194 bash[31648]: audit 2023-10-09T04:45:20.189683+0000 mon.a (mon.0) 884 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:21.373 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T04:45:21.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:21 smithi133 bash[27558]: audit 2023-10-09T04:45:21.360929+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.133:0/494285214' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T04:45:21.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:21 smithi133 bash[41291]: audit 2023-10-09T04:45:21.360929+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.133:0/494285214' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T04:45:21.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:21 smithi194 bash[31648]: audit 2023-10-09T04:45:21.360929+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.133:0/494285214' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T04:45:22.189 INFO:teuthology.orchestra.run.smithi133.stdout:{"epoch":19,"active_gid":14466,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6800","nonce":275314455},{"type":"v1","addr":"172.21.15.133:6801","nonce":275314455}]},"active_addr":"172.21.15.133:6801/275314455","active_change":"2023-10-09T04:42:05.063746+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24358,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.133:8443/","prometheus":"http://172.21.15.133:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":59,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.133:0","nonce":2159982120}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.133:0","nonce":1787141186}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.133:0","nonce":1575170036}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.133:0","nonce":2419271305}]}]} 2023-10-09T04:45:22.195 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T04:45:22.195 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T04:45:22.195 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T04:45:22.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:22 smithi133 bash[27558]: cluster 2023-10-09T04:45:21.139014+0000 mgr.y (mgr.14466) 136 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:22.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:22 smithi133 bash[41291]: cluster 2023-10-09T04:45:21.139014+0000 mgr.y (mgr.14466) 136 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:22.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:22 smithi194 bash[31648]: cluster 2023-10-09T04:45:21.139014+0000 mgr.y (mgr.14466) 136 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:24.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:24 smithi133 bash[41291]: cluster 2023-10-09T04:45:23.139598+0000 mgr.y (mgr.14466) 137 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:24.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:24 smithi133 bash[27558]: cluster 2023-10-09T04:45:23.139598+0000 mgr.y (mgr.14466) 137 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:24.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:24 smithi194 bash[31648]: cluster 2023-10-09T04:45:23.139598+0000 mgr.y (mgr.14466) 137 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:26.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:26 smithi194 bash[31648]: cluster 2023-10-09T04:45:25.140253+0000 mgr.y (mgr.14466) 138 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:26.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:45:26 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:45:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:45:26.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:26 smithi133 bash[41291]: cluster 2023-10-09T04:45:25.140253+0000 mgr.y (mgr.14466) 138 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:26.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:26 smithi133 bash[27558]: cluster 2023-10-09T04:45:25.140253+0000 mgr.y (mgr.14466) 138 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:26.978 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:28.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:28 smithi133 bash[41291]: cluster 2023-10-09T04:45:27.140960+0000 mgr.y (mgr.14466) 139 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:28.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:28 smithi133 bash[27558]: cluster 2023-10-09T04:45:27.140960+0000 mgr.y (mgr.14466) 139 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:28.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:28 smithi194 bash[31648]: cluster 2023-10-09T04:45:27.140960+0000 mgr.y (mgr.14466) 139 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:30.357 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T04:45:30.358 INFO:teuthology.orchestra.run.smithi133.stdout:{"epoch":71,"fsid":"855b5206-665c-11ee-8db5-212e2dc638e7","created":"2023-10-09T04:31:08.633675+0000","modified":"2023-10-09T04:45:07.107197+0000","last_up_change":"2023-10-09T04:41:05.383200+0000","last_in_change":"2023-10-09T04:40:39.315748+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T04:36:44.260202+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T04:42:47.518485+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T04:44:51.685755+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T04:44:52.162649+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"32633ec2-9b8f-42d8-bdfb-5c9668a26071","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6802","nonce":1933306305},{"type":"v1","addr":"172.21.15.133:6803","nonce":1933306305}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6804","nonce":1933306305},{"type":"v1","addr":"172.21.15.133:6805","nonce":1933306305}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6808","nonce":1933306305},{"type":"v1","addr":"172.21.15.133:6809","nonce":1933306305}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6806","nonce":1933306305},{"type":"v1","addr":"172.21.15.133:6807","nonce":1933306305}]},"public_addr":"172.21.15.133:6803/1933306305","cluster_addr":"172.21.15.133:6805/1933306305","heartbeat_back_addr":"172.21.15.133:6809/1933306305","heartbeat_front_addr":"172.21.15.133:6807/1933306305","state":["exists","up"]},{"osd":1,"uuid":"7f2a961d-a790-4337-91c7-1e45f900bbd4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6810","nonce":2965860635},{"type":"v1","addr":"172.21.15.133:6811","nonce":2965860635}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6812","nonce":2965860635},{"type":"v1","addr":"172.21.15.133:6813","nonce":2965860635}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6816","nonce":2965860635},{"type":"v1","addr":"172.21.15.133:6817","nonce":2965860635}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6814","nonce":2965860635},{"type":"v1","addr":"172.21.15.133:6815","nonce":2965860635}]},"public_addr":"172.21.15.133:6811/2965860635","cluster_addr":"172.21.15.133:6813/2965860635","heartbeat_back_addr":"172.21.15.133:6817/2965860635","heartbeat_front_addr":"172.21.15.133:6815/2965860635","state":["exists","up"]},{"osd":2,"uuid":"08751330-a327-4bf8-9c4d-9ff986b1c67f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":70,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6818","nonce":1465201170},{"type":"v1","addr":"172.21.15.133:6819","nonce":1465201170}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6820","nonce":1465201170},{"type":"v1","addr":"172.21.15.133:6821","nonce":1465201170}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6824","nonce":1465201170},{"type":"v1","addr":"172.21.15.133:6825","nonce":1465201170}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6822","nonce":1465201170},{"type":"v1","addr":"172.21.15.133:6823","nonce":1465201170}]},"public_addr":"172.21.15.133:6819/1465201170","cluster_addr":"172.21.15.133:6821/1465201170","heartbeat_back_addr":"172.21.15.133:6825/1465201170","heartbeat_front_addr":"172.21.15.133:6823/1465201170","state":["exists","up"]},{"osd":3,"uuid":"56bbf807-c8f7-4226-b46e-802b539274ca","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6826","nonce":1093481727},{"type":"v1","addr":"172.21.15.133:6827","nonce":1093481727}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6828","nonce":1093481727},{"type":"v1","addr":"172.21.15.133:6829","nonce":1093481727}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6832","nonce":1093481727},{"type":"v1","addr":"172.21.15.133:6833","nonce":1093481727}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6830","nonce":1093481727},{"type":"v1","addr":"172.21.15.133:6831","nonce":1093481727}]},"public_addr":"172.21.15.133:6827/1093481727","cluster_addr":"172.21.15.133:6829/1093481727","heartbeat_back_addr":"172.21.15.133:6833/1093481727","heartbeat_front_addr":"172.21.15.133:6831/1093481727","state":["exists","up"]},{"osd":4,"uuid":"509ec8cc-5042-428e-9a69-8e602b5afce8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6801","nonce":1874338403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6803","nonce":1874338403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6807","nonce":1874338403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6805","nonce":1874338403}]},"public_addr":"172.21.15.194:6801/1874338403","cluster_addr":"172.21.15.194:6803/1874338403","heartbeat_back_addr":"172.21.15.194:6807/1874338403","heartbeat_front_addr":"172.21.15.194:6805/1874338403","state":["exists","up"]},{"osd":5,"uuid":"bbdc1e67-fd5b-49d0-9e42-cf151d167943","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":70,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6809","nonce":3989986662}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6811","nonce":3989986662}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6815","nonce":3989986662}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6813","nonce":3989986662}]},"public_addr":"172.21.15.194:6809/3989986662","cluster_addr":"172.21.15.194:6811/3989986662","heartbeat_back_addr":"172.21.15.194:6815/3989986662","heartbeat_front_addr":"172.21.15.194:6813/3989986662","state":["exists","up"]},{"osd":6,"uuid":"7b69d014-f4c0-4e73-a14e-7d51b7ee3ea2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":70,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6817","nonce":2607555245}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6819","nonce":2607555245}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6823","nonce":2607555245}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6820","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6821","nonce":2607555245}]},"public_addr":"172.21.15.194:6817/2607555245","cluster_addr":"172.21.15.194:6819/2607555245","heartbeat_back_addr":"172.21.15.194:6823/2607555245","heartbeat_front_addr":"172.21.15.194:6821/2607555245","state":["exists","up"]},{"osd":7,"uuid":"02506268-102c-4654-9b5c-68d36c28f7f7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6824","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6825","nonce":1051622545}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6827","nonce":1051622545}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6831","nonce":1051622545}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6828","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6829","nonce":1051622545}]},"public_addr":"172.21.15.194:6825/1051622545","cluster_addr":"172.21.15.194:6827/1051622545","heartbeat_back_addr":"172.21.15.194:6831/1051622545","heartbeat_front_addr":"172.21.15.194:6829/1051622545","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:34:53.028753+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:35:47.001796+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:36:41.220545+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:37:36.237434+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:38:28.301119+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:39:20.099747+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:40:12.106681+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:41:04.212546+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.7","mappings":[{"from":1,"to":2}]},{"pgid":"4.a","mappings":[{"from":1,"to":2}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.133:6801/1761810618":"2023-10-10T04:31:42.639571+0000","172.21.15.133:0/3619387632":"2023-10-10T04:32:17.611609+0000","172.21.15.133:6800/1761810618":"2023-10-10T04:31:42.639571+0000","172.21.15.133:6835/4227843513":"2023-10-10T04:45:00.021661+0000","172.21.15.133:0/740467257":"2023-10-10T04:31:42.639571+0000","172.21.15.133:0/3636510019":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/1793292086":"2023-10-10T04:32:17.611609+0000","172.21.15.133:0/4173745360":"2023-10-10T04:31:42.639571+0000","172.21.15.133:0/55818837":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/1411744140":"2023-10-10T04:31:42.639571+0000","172.21.15.133:0/320185713":"2023-10-10T04:42:05.063462+0000","172.21.15.133:6800/768614786":"2023-10-10T04:42:05.063462+0000","172.21.15.133:6801/788799879":"2023-10-10T04:32:17.611609+0000","172.21.15.133:6800/788799879":"2023-10-10T04:32:17.611609+0000","172.21.15.133:0/4290202542":"2023-10-10T04:32:17.611609+0000","172.21.15.133:6834/4227843513":"2023-10-10T04:45:00.021661+0000","172.21.15.133:6801/768614786":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/1309073801":"2023-10-10T04:42:05.063462+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T04:45:30.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:30 smithi133 bash[27558]: cluster 2023-10-09T04:45:29.141607+0000 mgr.y (mgr.14466) 140 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:30.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:30 smithi133 bash[27558]: audit 2023-10-09T04:45:30.353063+0000 mon.a (mon.0) 885 : audit [DBG] from='client.? 172.21.15.133:0/2942370435' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:45:30.801 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:45:30 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:45:30] "GET /metrics HTTP/1.1" 200 238955 "" "Prometheus/2.43.0" 2023-10-09T04:45:30.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:30 smithi133 bash[41291]: cluster 2023-10-09T04:45:29.141607+0000 mgr.y (mgr.14466) 140 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:30.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:30 smithi133 bash[41291]: audit 2023-10-09T04:45:30.353063+0000 mon.a (mon.0) 885 : audit [DBG] from='client.? 172.21.15.133:0/2942370435' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:45:30.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:30 smithi194 bash[31648]: cluster 2023-10-09T04:45:29.141607+0000 mgr.y (mgr.14466) 140 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:30.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:30 smithi194 bash[31648]: audit 2023-10-09T04:45:30.353063+0000 mon.a (mon.0) 885 : audit [DBG] from='client.? 172.21.15.133:0/2942370435' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:45:31.118 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T04:45:31.119 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T04:45:32.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:32 smithi133 bash[41291]: cluster 2023-10-09T04:45:31.142734+0000 mgr.y (mgr.14466) 141 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:32.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:32 smithi133 bash[27558]: cluster 2023-10-09T04:45:31.142734+0000 mgr.y (mgr.14466) 141 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:32.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:32 smithi194 bash[31648]: cluster 2023-10-09T04:45:31.142734+0000 mgr.y (mgr.14466) 141 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:34.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:34 smithi133 bash[41291]: cluster 2023-10-09T04:45:33.143373+0000 mgr.y (mgr.14466) 142 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:34.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:34 smithi133 bash[27558]: cluster 2023-10-09T04:45:33.143373+0000 mgr.y (mgr.14466) 142 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:34.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:34 smithi194 bash[31648]: cluster 2023-10-09T04:45:33.143373+0000 mgr.y (mgr.14466) 142 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:35.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:35 smithi133 bash[27558]: audit 2023-10-09T04:45:35.190979+0000 mon.a (mon.0) 886 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:35.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:35 smithi133 bash[41291]: audit 2023-10-09T04:45:35.190979+0000 mon.a (mon.0) 886 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:35.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:35 smithi194 bash[31648]: audit 2023-10-09T04:45:35.190979+0000 mon.a (mon.0) 886 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:35.892 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:36.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:36 smithi194 bash[31648]: cluster 2023-10-09T04:45:35.143987+0000 mgr.y (mgr.14466) 143 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:36.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:45:36 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:45:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:45:36.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:36 smithi133 bash[41291]: cluster 2023-10-09T04:45:35.143987+0000 mgr.y (mgr.14466) 143 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:36.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:36 smithi133 bash[27558]: cluster 2023-10-09T04:45:35.143987+0000 mgr.y (mgr.14466) 143 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:38.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:38 smithi133 bash[41291]: cluster 2023-10-09T04:45:37.145058+0000 mgr.y (mgr.14466) 144 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:38.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:38 smithi133 bash[27558]: cluster 2023-10-09T04:45:37.145058+0000 mgr.y (mgr.14466) 144 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:38.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:38 smithi194 bash[31648]: cluster 2023-10-09T04:45:37.145058+0000 mgr.y (mgr.14466) 144 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:39.270 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T04:45:39.270 INFO:teuthology.orchestra.run.smithi133.stdout:{"epoch":71,"fsid":"855b5206-665c-11ee-8db5-212e2dc638e7","created":"2023-10-09T04:31:08.633675+0000","modified":"2023-10-09T04:45:07.107197+0000","last_up_change":"2023-10-09T04:41:05.383200+0000","last_in_change":"2023-10-09T04:40:39.315748+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T04:36:44.260202+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T04:42:47.518485+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T04:44:51.685755+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T04:44:52.162649+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"32633ec2-9b8f-42d8-bdfb-5c9668a26071","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6802","nonce":1933306305},{"type":"v1","addr":"172.21.15.133:6803","nonce":1933306305}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6804","nonce":1933306305},{"type":"v1","addr":"172.21.15.133:6805","nonce":1933306305}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6808","nonce":1933306305},{"type":"v1","addr":"172.21.15.133:6809","nonce":1933306305}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6806","nonce":1933306305},{"type":"v1","addr":"172.21.15.133:6807","nonce":1933306305}]},"public_addr":"172.21.15.133:6803/1933306305","cluster_addr":"172.21.15.133:6805/1933306305","heartbeat_back_addr":"172.21.15.133:6809/1933306305","heartbeat_front_addr":"172.21.15.133:6807/1933306305","state":["exists","up"]},{"osd":1,"uuid":"7f2a961d-a790-4337-91c7-1e45f900bbd4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6810","nonce":2965860635},{"type":"v1","addr":"172.21.15.133:6811","nonce":2965860635}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6812","nonce":2965860635},{"type":"v1","addr":"172.21.15.133:6813","nonce":2965860635}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6816","nonce":2965860635},{"type":"v1","addr":"172.21.15.133:6817","nonce":2965860635}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6814","nonce":2965860635},{"type":"v1","addr":"172.21.15.133:6815","nonce":2965860635}]},"public_addr":"172.21.15.133:6811/2965860635","cluster_addr":"172.21.15.133:6813/2965860635","heartbeat_back_addr":"172.21.15.133:6817/2965860635","heartbeat_front_addr":"172.21.15.133:6815/2965860635","state":["exists","up"]},{"osd":2,"uuid":"08751330-a327-4bf8-9c4d-9ff986b1c67f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":70,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6818","nonce":1465201170},{"type":"v1","addr":"172.21.15.133:6819","nonce":1465201170}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6820","nonce":1465201170},{"type":"v1","addr":"172.21.15.133:6821","nonce":1465201170}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6824","nonce":1465201170},{"type":"v1","addr":"172.21.15.133:6825","nonce":1465201170}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6822","nonce":1465201170},{"type":"v1","addr":"172.21.15.133:6823","nonce":1465201170}]},"public_addr":"172.21.15.133:6819/1465201170","cluster_addr":"172.21.15.133:6821/1465201170","heartbeat_back_addr":"172.21.15.133:6825/1465201170","heartbeat_front_addr":"172.21.15.133:6823/1465201170","state":["exists","up"]},{"osd":3,"uuid":"56bbf807-c8f7-4226-b46e-802b539274ca","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6826","nonce":1093481727},{"type":"v1","addr":"172.21.15.133:6827","nonce":1093481727}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6828","nonce":1093481727},{"type":"v1","addr":"172.21.15.133:6829","nonce":1093481727}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6832","nonce":1093481727},{"type":"v1","addr":"172.21.15.133:6833","nonce":1093481727}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6830","nonce":1093481727},{"type":"v1","addr":"172.21.15.133:6831","nonce":1093481727}]},"public_addr":"172.21.15.133:6827/1093481727","cluster_addr":"172.21.15.133:6829/1093481727","heartbeat_back_addr":"172.21.15.133:6833/1093481727","heartbeat_front_addr":"172.21.15.133:6831/1093481727","state":["exists","up"]},{"osd":4,"uuid":"509ec8cc-5042-428e-9a69-8e602b5afce8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6801","nonce":1874338403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6803","nonce":1874338403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6807","nonce":1874338403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6805","nonce":1874338403}]},"public_addr":"172.21.15.194:6801/1874338403","cluster_addr":"172.21.15.194:6803/1874338403","heartbeat_back_addr":"172.21.15.194:6807/1874338403","heartbeat_front_addr":"172.21.15.194:6805/1874338403","state":["exists","up"]},{"osd":5,"uuid":"bbdc1e67-fd5b-49d0-9e42-cf151d167943","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":70,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6809","nonce":3989986662}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6811","nonce":3989986662}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6815","nonce":3989986662}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6813","nonce":3989986662}]},"public_addr":"172.21.15.194:6809/3989986662","cluster_addr":"172.21.15.194:6811/3989986662","heartbeat_back_addr":"172.21.15.194:6815/3989986662","heartbeat_front_addr":"172.21.15.194:6813/3989986662","state":["exists","up"]},{"osd":6,"uuid":"7b69d014-f4c0-4e73-a14e-7d51b7ee3ea2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":70,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6817","nonce":2607555245}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6819","nonce":2607555245}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6823","nonce":2607555245}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6820","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6821","nonce":2607555245}]},"public_addr":"172.21.15.194:6817/2607555245","cluster_addr":"172.21.15.194:6819/2607555245","heartbeat_back_addr":"172.21.15.194:6823/2607555245","heartbeat_front_addr":"172.21.15.194:6821/2607555245","state":["exists","up"]},{"osd":7,"uuid":"02506268-102c-4654-9b5c-68d36c28f7f7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6824","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6825","nonce":1051622545}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6827","nonce":1051622545}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6831","nonce":1051622545}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6828","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6829","nonce":1051622545}]},"public_addr":"172.21.15.194:6825/1051622545","cluster_addr":"172.21.15.194:6827/1051622545","heartbeat_back_addr":"172.21.15.194:6831/1051622545","heartbeat_front_addr":"172.21.15.194:6829/1051622545","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:34:53.028753+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:35:47.001796+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:36:41.220545+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:37:36.237434+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:38:28.301119+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:39:20.099747+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:40:12.106681+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:41:04.212546+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.7","mappings":[{"from":1,"to":2}]},{"pgid":"4.a","mappings":[{"from":1,"to":2}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.133:6801/1761810618":"2023-10-10T04:31:42.639571+0000","172.21.15.133:0/3619387632":"2023-10-10T04:32:17.611609+0000","172.21.15.133:6800/1761810618":"2023-10-10T04:31:42.639571+0000","172.21.15.133:6835/4227843513":"2023-10-10T04:45:00.021661+0000","172.21.15.133:0/740467257":"2023-10-10T04:31:42.639571+0000","172.21.15.133:0/3636510019":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/1793292086":"2023-10-10T04:32:17.611609+0000","172.21.15.133:0/4173745360":"2023-10-10T04:31:42.639571+0000","172.21.15.133:0/55818837":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/1411744140":"2023-10-10T04:31:42.639571+0000","172.21.15.133:0/320185713":"2023-10-10T04:42:05.063462+0000","172.21.15.133:6800/768614786":"2023-10-10T04:42:05.063462+0000","172.21.15.133:6801/788799879":"2023-10-10T04:32:17.611609+0000","172.21.15.133:6800/788799879":"2023-10-10T04:32:17.611609+0000","172.21.15.133:0/4290202542":"2023-10-10T04:32:17.611609+0000","172.21.15.133:6834/4227843513":"2023-10-10T04:45:00.021661+0000","172.21.15.133:6801/768614786":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/1309073801":"2023-10-10T04:42:05.063462+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T04:45:39.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:39 smithi133 bash[41291]: audit 2023-10-09T04:45:39.265108+0000 mon.a (mon.0) 887 : audit [DBG] from='client.? 172.21.15.133:0/4000902237' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:45:39.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:39 smithi133 bash[27558]: audit 2023-10-09T04:45:39.265108+0000 mon.a (mon.0) 887 : audit [DBG] from='client.? 172.21.15.133:0/4000902237' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:45:39.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:39 smithi194 bash[31648]: audit 2023-10-09T04:45:39.265108+0000 mon.a (mon.0) 887 : audit [DBG] from='client.? 172.21.15.133:0/4000902237' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:45:40.116 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T04:45:40.117 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T04:45:40.117 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T04:45:40.118 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T04:45:40.118 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T04:45:40.119 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T04:45:40.120 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T04:45:40.120 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T04:45:40.550 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:45:40 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:45:40] "GET /metrics HTTP/1.1" 200 238969 "" "Prometheus/2.43.0" 2023-10-09T04:45:40.890 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:40 smithi194 bash[31648]: cluster 2023-10-09T04:45:39.145636+0000 mgr.y (mgr.14466) 145 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:41.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:40 smithi133 bash[27558]: cluster 2023-10-09T04:45:39.145636+0000 mgr.y (mgr.14466) 145 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:41.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:40 smithi133 bash[41291]: cluster 2023-10-09T04:45:39.145636+0000 mgr.y (mgr.14466) 145 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:42.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:42 smithi194 bash[31648]: cluster 2023-10-09T04:45:41.146540+0000 mgr.y (mgr.14466) 146 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:43.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:42 smithi133 bash[27558]: cluster 2023-10-09T04:45:41.146540+0000 mgr.y (mgr.14466) 146 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:43.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:42 smithi133 bash[41291]: cluster 2023-10-09T04:45:41.146540+0000 mgr.y (mgr.14466) 146 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:44.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:44 smithi194 bash[31648]: cluster 2023-10-09T04:45:43.147244+0000 mgr.y (mgr.14466) 147 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:45.015 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:45.016 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:45.016 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:45.017 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:45.018 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:45.018 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:45.023 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:45.023 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:45.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:44 smithi133 bash[41291]: cluster 2023-10-09T04:45:43.147244+0000 mgr.y (mgr.14466) 147 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:45.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:44 smithi133 bash[27558]: cluster 2023-10-09T04:45:43.147244+0000 mgr.y (mgr.14466) 147 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:46.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:46 smithi194 bash[31648]: cluster 2023-10-09T04:45:45.147868+0000 mgr.y (mgr.14466) 148 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:46.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:45:46 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:45:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:45:47.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:46 smithi133 bash[27558]: cluster 2023-10-09T04:45:45.147868+0000 mgr.y (mgr.14466) 148 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:47.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:46 smithi133 bash[41291]: cluster 2023-10-09T04:45:45.147868+0000 mgr.y (mgr.14466) 148 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:48.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:48 smithi194 bash[31648]: cluster 2023-10-09T04:45:47.148930+0000 mgr.y (mgr.14466) 149 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:49.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:48 smithi133 bash[27558]: cluster 2023-10-09T04:45:47.148930+0000 mgr.y (mgr.14466) 149 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:49.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:48 smithi133 bash[41291]: cluster 2023-10-09T04:45:47.148930+0000 mgr.y (mgr.14466) 149 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:50.800 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:45:50 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:45:50] "GET /metrics HTTP/1.1" 200 238941 "" "Prometheus/2.43.0" 2023-10-09T04:45:50.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:50 smithi133 bash[27558]: cluster 2023-10-09T04:45:49.149377+0000 mgr.y (mgr.14466) 150 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:50.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:50 smithi133 bash[27558]: audit 2023-10-09T04:45:50.191557+0000 mon.a (mon.0) 888 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:50.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:50 smithi133 bash[41291]: cluster 2023-10-09T04:45:49.149377+0000 mgr.y (mgr.14466) 150 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:50.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:50 smithi133 bash[41291]: audit 2023-10-09T04:45:50.191557+0000 mon.a (mon.0) 888 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:50.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:50 smithi194 bash[31648]: cluster 2023-10-09T04:45:49.149377+0000 mgr.y (mgr.14466) 150 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:50.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:50 smithi194 bash[31648]: audit 2023-10-09T04:45:50.191557+0000 mon.a (mon.0) 888 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:52.854 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:52 smithi133 bash[41291]: cluster 2023-10-09T04:45:51.150254+0000 mgr.y (mgr.14466) 151 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:52.854 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:52 smithi133 bash[27558]: cluster 2023-10-09T04:45:51.150254+0000 mgr.y (mgr.14466) 151 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:52.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:52 smithi194 bash[31648]: cluster 2023-10-09T04:45:51.150254+0000 mgr.y (mgr.14466) 151 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:54.846 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:54 smithi133 bash[27558]: cluster 2023-10-09T04:45:53.150835+0000 mgr.y (mgr.14466) 152 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:54.847 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:54 smithi133 bash[41291]: cluster 2023-10-09T04:45:53.150835+0000 mgr.y (mgr.14466) 152 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:54.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:54 smithi194 bash[31648]: cluster 2023-10-09T04:45:53.150835+0000 mgr.y (mgr.14466) 152 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:56.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:56 smithi194 bash[31648]: cluster 2023-10-09T04:45:55.151493+0000 mgr.y (mgr.14466) 153 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:56.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:45:56 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:45:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:45:56.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:56 smithi133 bash[41291]: cluster 2023-10-09T04:45:55.151493+0000 mgr.y (mgr.14466) 153 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:56.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:56 smithi133 bash[27558]: cluster 2023-10-09T04:45:55.151493+0000 mgr.y (mgr.14466) 153 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:58.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:45:58 smithi194 bash[31648]: cluster 2023-10-09T04:45:57.152287+0000 mgr.y (mgr.14466) 154 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:59.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:45:58 smithi133 bash[27558]: cluster 2023-10-09T04:45:57.152287+0000 mgr.y (mgr.14466) 154 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:59.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:45:58 smithi133 bash[41291]: cluster 2023-10-09T04:45:57.152287+0000 mgr.y (mgr.14466) 154 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:45:59.361 INFO:teuthology.orchestra.run.smithi133.stdout:197568495686 2023-10-09T04:45:59.361 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T04:46:00.593 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:46:00 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:46:00] "GET /metrics HTTP/1.1" 200 238941 "" "Prometheus/2.43.0" 2023-10-09T04:46:00.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:00 smithi194 bash[31648]: cluster 2023-10-09T04:45:59.152673+0000 mgr.y (mgr.14466) 155 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:01.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:00 smithi133 bash[27558]: cluster 2023-10-09T04:45:59.152673+0000 mgr.y (mgr.14466) 155 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:01.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:00 smithi133 bash[41291]: cluster 2023-10-09T04:45:59.152673+0000 mgr.y (mgr.14466) 155 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:01.241 INFO:teuthology.orchestra.run.smithi133.stdout:231928234044 2023-10-09T04:46:01.242 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T04:46:02.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:02 smithi194 bash[31648]: cluster 2023-10-09T04:46:01.153547+0000 mgr.y (mgr.14466) 156 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:03.047 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:02 smithi133 bash[41291]: cluster 2023-10-09T04:46:01.153547+0000 mgr.y (mgr.14466) 156 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:03.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:02 smithi133 bash[27558]: cluster 2023-10-09T04:46:01.153547+0000 mgr.y (mgr.14466) 156 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:04.255 INFO:teuthology.orchestra.run.smithi133.stdout:77309411442 2023-10-09T04:46:04.255 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T04:46:04.811 INFO:teuthology.orchestra.run.smithi133.stdout:34359738503 2023-10-09T04:46:04.811 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T04:46:04.877 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:04 smithi133 bash[27558]: cluster 2023-10-09T04:46:03.154000+0000 mgr.y (mgr.14466) 157 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:04.878 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:04 smithi133 bash[41291]: cluster 2023-10-09T04:46:03.154000+0000 mgr.y (mgr.14466) 157 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:04.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:04 smithi194 bash[31648]: cluster 2023-10-09T04:46:03.154000+0000 mgr.y (mgr.14466) 157 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:05.607 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:05 smithi133 bash[41291]: audit 2023-10-09T04:46:05.190377+0000 mon.a (mon.0) 889 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:05.607 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:05 smithi133 bash[41291]: audit 2023-10-09T04:46:05.328670+0000 mon.a (mon.0) 890 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:46:05.607 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:05 smithi133 bash[27558]: audit 2023-10-09T04:46:05.190377+0000 mon.a (mon.0) 889 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:05.608 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:05 smithi133 bash[27558]: audit 2023-10-09T04:46:05.328670+0000 mon.a (mon.0) 890 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:46:05.653 INFO:teuthology.orchestra.run.smithi133.stdout:163208757330 2023-10-09T04:46:05.653 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T04:46:05.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:05 smithi194 bash[31648]: audit 2023-10-09T04:46:05.190377+0000 mon.a (mon.0) 889 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:05.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:05 smithi194 bash[31648]: audit 2023-10-09T04:46:05.328670+0000 mon.a (mon.0) 890 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:46:06.596 INFO:teuthology.orchestra.run.smithi133.stdout:55834574973 2023-10-09T04:46:06.596 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T04:46:06.609 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:46:06 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:46:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:46:06.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:06 smithi194 bash[31648]: cluster 2023-10-09T04:46:05.154741+0000 mgr.y (mgr.14466) 158 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:06.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:06 smithi194 bash[31648]: audit 2023-10-09T04:46:05.749963+0000 mon.a (mon.0) 891 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:46:06.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:06 smithi194 bash[31648]: audit 2023-10-09T04:46:05.750808+0000 mon.a (mon.0) 892 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:46:06.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:06 smithi194 bash[31648]: audit 2023-10-09T04:46:05.755934+0000 mon.a (mon.0) 893 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:46:06.913 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:06 smithi133 bash[27558]: cluster 2023-10-09T04:46:05.154741+0000 mgr.y (mgr.14466) 158 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:06.913 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:06 smithi133 bash[27558]: audit 2023-10-09T04:46:05.749963+0000 mon.a (mon.0) 891 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:46:06.914 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:06 smithi133 bash[27558]: audit 2023-10-09T04:46:05.750808+0000 mon.a (mon.0) 892 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:46:06.914 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:06 smithi133 bash[27558]: audit 2023-10-09T04:46:05.755934+0000 mon.a (mon.0) 893 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:46:06.914 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:06 smithi133 bash[41291]: cluster 2023-10-09T04:46:05.154741+0000 mgr.y (mgr.14466) 158 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:06.914 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:06 smithi133 bash[41291]: audit 2023-10-09T04:46:05.749963+0000 mon.a (mon.0) 891 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:46:06.914 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:06 smithi133 bash[41291]: audit 2023-10-09T04:46:05.750808+0000 mon.a (mon.0) 892 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:46:06.914 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:06 smithi133 bash[41291]: audit 2023-10-09T04:46:05.755934+0000 mon.a (mon.0) 893 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:46:06.915 INFO:teuthology.orchestra.run.smithi133.stdout:107374182503 2023-10-09T04:46:06.915 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T04:46:06.924 INFO:teuthology.orchestra.run.smithi133.stdout:128849018974 2023-10-09T04:46:06.925 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T04:46:08.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:08 smithi194 bash[31648]: cluster 2023-10-09T04:46:07.155372+0000 mgr.y (mgr.14466) 159 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:09.024 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:46:09.025 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:46:09.029 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:46:09.029 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:46:09.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:08 smithi133 bash[41291]: cluster 2023-10-09T04:46:07.155372+0000 mgr.y (mgr.14466) 159 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:09.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:08 smithi133 bash[27558]: cluster 2023-10-09T04:46:07.155372+0000 mgr.y (mgr.14466) 159 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:10.036 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:46:10.639 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:10 smithi133 bash[27558]: cluster 2023-10-09T04:46:09.155896+0000 mgr.y (mgr.14466) 160 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:10.639 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:46:10 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:46:10] "GET /metrics HTTP/1.1" 200 238941 "" "Prometheus/2.43.0" 2023-10-09T04:46:10.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:10 smithi194 bash[31648]: cluster 2023-10-09T04:46:09.155896+0000 mgr.y (mgr.14466) 160 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:11.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:10 smithi133 bash[41291]: cluster 2023-10-09T04:46:09.155896+0000 mgr.y (mgr.14466) 160 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:11.052 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:46:11.053 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:46:11.054 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:46:12.882 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:12 smithi133 bash[27558]: cluster 2023-10-09T04:46:11.156514+0000 mgr.y (mgr.14466) 161 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:12.883 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:12 smithi133 bash[41291]: cluster 2023-10-09T04:46:11.156514+0000 mgr.y (mgr.14466) 161 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:12.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:12 smithi194 bash[31648]: cluster 2023-10-09T04:46:11.156514+0000 mgr.y (mgr.14466) 161 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:14.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:14 smithi133 bash[27558]: cluster 2023-10-09T04:46:13.157136+0000 mgr.y (mgr.14466) 162 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:14.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:14 smithi133 bash[41291]: cluster 2023-10-09T04:46:13.157136+0000 mgr.y (mgr.14466) 162 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:14.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:14 smithi194 bash[31648]: cluster 2023-10-09T04:46:13.157136+0000 mgr.y (mgr.14466) 162 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:16.628 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:46:16 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:46:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:46:16.727 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:16 smithi133 bash[41291]: cluster 2023-10-09T04:46:15.157622+0000 mgr.y (mgr.14466) 163 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:16.727 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:16 smithi133 bash[27558]: cluster 2023-10-09T04:46:15.157622+0000 mgr.y (mgr.14466) 163 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:16.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:16 smithi194 bash[31648]: cluster 2023-10-09T04:46:15.157622+0000 mgr.y (mgr.14466) 163 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:18.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:18 smithi194 bash[31648]: cluster 2023-10-09T04:46:17.158513+0000 mgr.y (mgr.14466) 164 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:18.907 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:18 smithi133 bash[27558]: cluster 2023-10-09T04:46:17.158513+0000 mgr.y (mgr.14466) 164 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:18.907 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:18 smithi133 bash[41291]: cluster 2023-10-09T04:46:17.158513+0000 mgr.y (mgr.14466) 164 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:20.550 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:46:20 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:46:20] "GET /metrics HTTP/1.1" 200 238933 "" "Prometheus/2.43.0" 2023-10-09T04:46:20.898 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:20 smithi133 bash[27558]: cluster 2023-10-09T04:46:19.159008+0000 mgr.y (mgr.14466) 165 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:20.898 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:20 smithi133 bash[27558]: audit 2023-10-09T04:46:20.191563+0000 mon.a (mon.0) 894 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:20.898 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:20 smithi133 bash[41291]: cluster 2023-10-09T04:46:19.159008+0000 mgr.y (mgr.14466) 165 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:20.898 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:20 smithi133 bash[41291]: audit 2023-10-09T04:46:20.191563+0000 mon.a (mon.0) 894 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:21.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:20 smithi194 bash[31648]: cluster 2023-10-09T04:46:19.159008+0000 mgr.y (mgr.14466) 165 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:21.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:20 smithi194 bash[31648]: audit 2023-10-09T04:46:20.191563+0000 mon.a (mon.0) 894 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:22.302 INFO:teuthology.orchestra.run.smithi133.stdout:231928234050 2023-10-09T04:46:22.725 INFO:teuthology.orchestra.run.smithi133.stdout:77309411447 2023-10-09T04:46:23.025 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:22 smithi133 bash[27558]: cluster 2023-10-09T04:46:21.159550+0000 mgr.y (mgr.14466) 166 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:23.025 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:22 smithi133 bash[27558]: audit 2023-10-09T04:46:22.299043+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.133:0/2563973943' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T04:46:23.026 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:22 smithi133 bash[41291]: cluster 2023-10-09T04:46:21.159550+0000 mgr.y (mgr.14466) 166 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:23.026 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:22 smithi133 bash[41291]: audit 2023-10-09T04:46:22.299043+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.133:0/2563973943' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T04:46:23.135 INFO:teuthology.orchestra.run.smithi133.stdout:107374182507 2023-10-09T04:46:23.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:22 smithi194 bash[31648]: cluster 2023-10-09T04:46:21.159550+0000 mgr.y (mgr.14466) 166 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:23.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:22 smithi194 bash[31648]: audit 2023-10-09T04:46:22.299043+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.133:0/2563973943' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T04:46:23.522 INFO:teuthology.orchestra.run.smithi133.stdout:34359738507 2023-10-09T04:46:23.522 INFO:teuthology.orchestra.run.smithi133.stdout:197568495692 2023-10-09T04:46:23.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:23 smithi133 bash[27558]: audit 2023-10-09T04:46:22.722871+0000 mon.a (mon.0) 895 : audit [DBG] from='client.? 172.21.15.133:0/2187717954' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T04:46:23.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:23 smithi133 bash[27558]: audit 2023-10-09T04:46:23.132803+0000 mon.a (mon.0) 896 : audit [DBG] from='client.? 172.21.15.133:0/948219495' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T04:46:23.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:23 smithi133 bash[27558]: audit 2023-10-09T04:46:23.519352+0000 mon.a (mon.0) 897 : audit [DBG] from='client.? 172.21.15.133:0/3973112239' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T04:46:23.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:23 smithi133 bash[27558]: audit 2023-10-09T04:46:23.519505+0000 mon.a (mon.0) 898 : audit [DBG] from='client.? 172.21.15.133:0/1212185679' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T04:46:23.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:23 smithi133 bash[41291]: audit 2023-10-09T04:46:22.722871+0000 mon.a (mon.0) 895 : audit [DBG] from='client.? 172.21.15.133:0/2187717954' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T04:46:23.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:23 smithi133 bash[41291]: audit 2023-10-09T04:46:23.132803+0000 mon.a (mon.0) 896 : audit [DBG] from='client.? 172.21.15.133:0/948219495' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T04:46:23.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:23 smithi133 bash[41291]: audit 2023-10-09T04:46:23.519352+0000 mon.a (mon.0) 897 : audit [DBG] from='client.? 172.21.15.133:0/3973112239' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T04:46:23.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:23 smithi133 bash[41291]: audit 2023-10-09T04:46:23.519505+0000 mon.a (mon.0) 898 : audit [DBG] from='client.? 172.21.15.133:0/1212185679' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T04:46:24.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:23 smithi194 bash[31648]: audit 2023-10-09T04:46:22.722871+0000 mon.a (mon.0) 895 : audit [DBG] from='client.? 172.21.15.133:0/2187717954' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T04:46:24.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:23 smithi194 bash[31648]: audit 2023-10-09T04:46:23.132803+0000 mon.a (mon.0) 896 : audit [DBG] from='client.? 172.21.15.133:0/948219495' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T04:46:24.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:23 smithi194 bash[31648]: audit 2023-10-09T04:46:23.519352+0000 mon.a (mon.0) 897 : audit [DBG] from='client.? 172.21.15.133:0/3973112239' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T04:46:24.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:23 smithi194 bash[31648]: audit 2023-10-09T04:46:23.519505+0000 mon.a (mon.0) 898 : audit [DBG] from='client.? 172.21.15.133:0/1212185679' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T04:46:25.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:24 smithi133 bash[27558]: cluster 2023-10-09T04:46:23.159866+0000 mgr.y (mgr.14466) 167 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:25.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:24 smithi133 bash[41291]: cluster 2023-10-09T04:46:23.159866+0000 mgr.y (mgr.14466) 167 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:25.108 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411442 got 77309411447 for osd.2 2023-10-09T04:46:25.108 DEBUG:teuthology.parallel:result is None 2023-10-09T04:46:25.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:24 smithi194 bash[31648]: cluster 2023-10-09T04:46:23.159866+0000 mgr.y (mgr.14466) 167 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:25.534 INFO:teuthology.orchestra.run.smithi133.stdout:163208757334 2023-10-09T04:46:25.581 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234044 got 231928234050 for osd.7 2023-10-09T04:46:25.582 DEBUG:teuthology.parallel:result is None 2023-10-09T04:46:25.669 INFO:teuthology.orchestra.run.smithi133.stdout:55834574977 2023-10-09T04:46:26.220 INFO:teuthology.orchestra.run.smithi133.stdout:128849018978 2023-10-09T04:46:26.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:46:26 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:46:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:46:26.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:26 smithi194 bash[31648]: cluster 2023-10-09T04:46:25.160261+0000 mgr.y (mgr.14466) 168 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:26.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:26 smithi194 bash[31648]: audit 2023-10-09T04:46:25.532010+0000 mon.a (mon.0) 899 : audit [DBG] from='client.? 172.21.15.133:0/956289229' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T04:46:26.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:26 smithi194 bash[31648]: audit 2023-10-09T04:46:25.666978+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.133:0/2742755290' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T04:46:26.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:26 smithi194 bash[31648]: audit 2023-10-09T04:46:26.217924+0000 mon.a (mon.0) 900 : audit [DBG] from='client.? 172.21.15.133:0/4153690708' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T04:46:26.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:26 smithi133 bash[27558]: cluster 2023-10-09T04:46:25.160261+0000 mgr.y (mgr.14466) 168 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:26.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:26 smithi133 bash[27558]: audit 2023-10-09T04:46:25.532010+0000 mon.a (mon.0) 899 : audit [DBG] from='client.? 172.21.15.133:0/956289229' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T04:46:26.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:26 smithi133 bash[27558]: audit 2023-10-09T04:46:25.666978+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.133:0/2742755290' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T04:46:26.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:26 smithi133 bash[27558]: audit 2023-10-09T04:46:26.217924+0000 mon.a (mon.0) 900 : audit [DBG] from='client.? 172.21.15.133:0/4153690708' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T04:46:26.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:26 smithi133 bash[41291]: cluster 2023-10-09T04:46:25.160261+0000 mgr.y (mgr.14466) 168 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:26.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:26 smithi133 bash[41291]: audit 2023-10-09T04:46:25.532010+0000 mon.a (mon.0) 899 : audit [DBG] from='client.? 172.21.15.133:0/956289229' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T04:46:26.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:26 smithi133 bash[41291]: audit 2023-10-09T04:46:25.666978+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.133:0/2742755290' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T04:46:26.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:26 smithi133 bash[41291]: audit 2023-10-09T04:46:26.217924+0000 mon.a (mon.0) 900 : audit [DBG] from='client.? 172.21.15.133:0/4153690708' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T04:46:26.863 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738503 got 34359738507 for osd.0 2023-10-09T04:46:26.864 DEBUG:teuthology.parallel:result is None 2023-10-09T04:46:28.092 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182503 got 107374182507 for osd.3 2023-10-09T04:46:28.092 DEBUG:teuthology.parallel:result is None 2023-10-09T04:46:28.141 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495686 got 197568495692 for osd.6 2023-10-09T04:46:28.141 DEBUG:teuthology.parallel:result is None 2023-10-09T04:46:28.427 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574973 got 55834574977 for osd.1 2023-10-09T04:46:28.428 DEBUG:teuthology.parallel:result is None 2023-10-09T04:46:28.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:28 smithi133 bash[27558]: cluster 2023-10-09T04:46:27.160959+0000 mgr.y (mgr.14466) 169 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:28.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:28 smithi133 bash[41291]: cluster 2023-10-09T04:46:27.160959+0000 mgr.y (mgr.14466) 169 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:28.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:28 smithi194 bash[31648]: cluster 2023-10-09T04:46:27.160959+0000 mgr.y (mgr.14466) 169 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:29.343 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757330 got 163208757334 for osd.5 2023-10-09T04:46:29.344 DEBUG:teuthology.parallel:result is None 2023-10-09T04:46:29.364 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018974 got 128849018978 for osd.4 2023-10-09T04:46:29.364 DEBUG:teuthology.parallel:result is None 2023-10-09T04:46:29.364 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T04:46:29.364 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:46:30.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:30 smithi133 bash[27558]: cluster 2023-10-09T04:46:29.161654+0000 mgr.y (mgr.14466) 170 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:30.801 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:46:30 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:46:30] "GET /metrics HTTP/1.1" 200 238933 "" "Prometheus/2.43.0" 2023-10-09T04:46:30.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:30 smithi133 bash[41291]: cluster 2023-10-09T04:46:29.161654+0000 mgr.y (mgr.14466) 170 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:30.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:30 smithi194 bash[31648]: cluster 2023-10-09T04:46:29.161654+0000 mgr.y (mgr.14466) 170 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:32.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:32 smithi133 bash[41291]: cluster 2023-10-09T04:46:31.162642+0000 mgr.y (mgr.14466) 171 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:32.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:32 smithi133 bash[27558]: cluster 2023-10-09T04:46:31.162642+0000 mgr.y (mgr.14466) 171 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:32.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:32 smithi194 bash[31648]: cluster 2023-10-09T04:46:31.162642+0000 mgr.y (mgr.14466) 171 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:34.179 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:46:34.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:34 smithi133 bash[27558]: cluster 2023-10-09T04:46:33.163278+0000 mgr.y (mgr.14466) 172 : cluster [DBG] pgmap v149: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:34.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:34 smithi133 bash[41291]: cluster 2023-10-09T04:46:33.163278+0000 mgr.y (mgr.14466) 172 : cluster [DBG] pgmap v149: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:34.889 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:34 smithi194 bash[31648]: cluster 2023-10-09T04:46:33.163278+0000 mgr.y (mgr.14466) 172 : cluster [DBG] pgmap v149: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:35.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:35 smithi133 bash[41291]: audit 2023-10-09T04:46:35.192371+0000 mon.a (mon.0) 901 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:35.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:35 smithi133 bash[27558]: audit 2023-10-09T04:46:35.192371+0000 mon.a (mon.0) 901 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:35.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:35 smithi194 bash[31648]: audit 2023-10-09T04:46:35.192371+0000 mon.a (mon.0) 901 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:36.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:36 smithi194 bash[31648]: cluster 2023-10-09T04:46:35.163936+0000 mgr.y (mgr.14466) 173 : cluster [DBG] pgmap v150: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:36.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:46:36 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:46:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:46:36.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:36 smithi133 bash[27558]: cluster 2023-10-09T04:46:35.163936+0000 mgr.y (mgr.14466) 173 : cluster [DBG] pgmap v150: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:36.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:36 smithi133 bash[41291]: cluster 2023-10-09T04:46:35.163936+0000 mgr.y (mgr.14466) 173 : cluster [DBG] pgmap v150: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:37.592 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T04:46:37.606 INFO:teuthology.orchestra.run.smithi133.stderr:dumped all 2023-10-09T04:46:38.475 INFO:teuthology.orchestra.run.smithi133.stdout:{"pg_ready":true,"pg_map":{"version":151,"stamp":"2023-10-09T04:46:37.164204+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":259,"num_read_kb":355,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":173,"ondisk_log_size":173,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385824,"kb_used_data":7520,"kb_used_omap":0,"kb_used_meta":2378240,"kb_avail":747575392,"statfs":{"total":767960285184,"available":765517201408,"internally_reserved":0,"allocated":7700480,"data_stored":4343993,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2435317760},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004164"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.256545+0000","last_change":"2023-10-09T04:44:53.792424+0000","last_active":"2023-10-09T04:45:06.256545+0000","last_peered":"2023-10-09T04:45:06.256545+0000","last_clean":"2023-10-09T04:45:06.256545+0000","last_became_active":"2023-10-09T04:44:53.792104+0000","last_became_peered":"2023-10-09T04:44:53.792104+0000","last_unstale":"2023-10-09T04:45:06.256545+0000","last_undegraded":"2023-10-09T04:45:06.256545+0000","last_fullsized":"2023-10-09T04:45:06.256545+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:20:29.387710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'5","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.120440+0000","last_change":"2023-10-09T04:44:52.788322+0000","last_active":"2023-10-09T04:45:06.120440+0000","last_peered":"2023-10-09T04:45:06.120440+0000","last_clean":"2023-10-09T04:45:06.120440+0000","last_became_active":"2023-10-09T04:44:52.787420+0000","last_became_peered":"2023-10-09T04:44:52.787420+0000","last_unstale":"2023-10-09T04:45:06.120440+0000","last_undegraded":"2023-10-09T04:45:06.120440+0000","last_fullsized":"2023-10-09T04:45:06.120440+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:32:54.717562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.112102+0000","last_change":"2023-10-09T04:44:53.792437+0000","last_active":"2023-10-09T04:45:06.112102+0000","last_peered":"2023-10-09T04:45:06.112102+0000","last_clean":"2023-10-09T04:45:06.112102+0000","last_became_active":"2023-10-09T04:44:53.792094+0000","last_became_peered":"2023-10-09T04:44:53.792094+0000","last_unstale":"2023-10-09T04:45:06.112102+0000","last_undegraded":"2023-10-09T04:45:06.112102+0000","last_fullsized":"2023-10-09T04:45:06.112102+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:18.147611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'2","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.113009+0000","last_change":"2023-10-09T04:44:52.794030+0000","last_active":"2023-10-09T04:45:06.113009+0000","last_peered":"2023-10-09T04:45:06.113009+0000","last_clean":"2023-10-09T04:45:06.113009+0000","last_became_active":"2023-10-09T04:44:52.793602+0000","last_became_peered":"2023-10-09T04:44:52.793602+0000","last_unstale":"2023-10-09T04:45:06.113009+0000","last_undegraded":"2023-10-09T04:45:06.113009+0000","last_fullsized":"2023-10-09T04:45:06.113009+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:51.223306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.257526+0000","last_change":"2023-10-09T04:44:53.794733+0000","last_active":"2023-10-09T04:45:06.257526+0000","last_peered":"2023-10-09T04:45:06.257526+0000","last_clean":"2023-10-09T04:45:06.257526+0000","last_became_active":"2023-10-09T04:44:53.794004+0000","last_became_peered":"2023-10-09T04:44:53.794004+0000","last_unstale":"2023-10-09T04:45:06.257526+0000","last_undegraded":"2023-10-09T04:45:06.257526+0000","last_fullsized":"2023-10-09T04:45:06.257526+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:33:24.499774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.118231+0000","last_change":"2023-10-09T04:44:52.793709+0000","last_active":"2023-10-09T04:45:06.118231+0000","last_peered":"2023-10-09T04:45:06.118231+0000","last_clean":"2023-10-09T04:45:06.118231+0000","last_became_active":"2023-10-09T04:44:52.792973+0000","last_became_peered":"2023-10-09T04:44:52.792973+0000","last_unstale":"2023-10-09T04:45:06.118231+0000","last_undegraded":"2023-10-09T04:45:06.118231+0000","last_fullsized":"2023-10-09T04:45:06.118231+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:12.091089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.257448+0000","last_change":"2023-10-09T04:44:53.791051+0000","last_active":"2023-10-09T04:45:06.257448+0000","last_peered":"2023-10-09T04:45:06.257448+0000","last_clean":"2023-10-09T04:45:06.257448+0000","last_became_active":"2023-10-09T04:44:53.790741+0000","last_became_peered":"2023-10-09T04:44:53.790741+0000","last_unstale":"2023-10-09T04:45:06.257448+0000","last_undegraded":"2023-10-09T04:45:06.257448+0000","last_fullsized":"2023-10-09T04:45:06.257448+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:10:12.019609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.118016+0000","last_change":"2023-10-09T04:44:52.794672+0000","last_active":"2023-10-09T04:45:06.118016+0000","last_peered":"2023-10-09T04:45:06.118016+0000","last_clean":"2023-10-09T04:45:06.118016+0000","last_became_active":"2023-10-09T04:44:52.793285+0000","last_became_peered":"2023-10-09T04:44:52.793285+0000","last_unstale":"2023-10-09T04:45:06.118016+0000","last_undegraded":"2023-10-09T04:45:06.118016+0000","last_fullsized":"2023-10-09T04:45:06.118016+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:16.691224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.116282+0000","last_change":"2023-10-09T04:44:53.790672+0000","last_active":"2023-10-09T04:45:06.116282+0000","last_peered":"2023-10-09T04:45:06.116282+0000","last_clean":"2023-10-09T04:45:06.116282+0000","last_became_active":"2023-10-09T04:44:53.789696+0000","last_became_peered":"2023-10-09T04:44:53.789696+0000","last_unstale":"2023-10-09T04:45:06.116282+0000","last_undegraded":"2023-10-09T04:45:06.116282+0000","last_fullsized":"2023-10-09T04:45:06.116282+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:27.776552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'2","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.119784+0000","last_change":"2023-10-09T04:44:52.776358+0000","last_active":"2023-10-09T04:45:06.119784+0000","last_peered":"2023-10-09T04:45:06.119784+0000","last_clean":"2023-10-09T04:45:06.119784+0000","last_became_active":"2023-10-09T04:44:52.775894+0000","last_became_peered":"2023-10-09T04:44:52.775894+0000","last_unstale":"2023-10-09T04:45:06.119784+0000","last_undegraded":"2023-10-09T04:45:06.119784+0000","last_fullsized":"2023-10-09T04:45:06.119784+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:05:37.925341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.257215+0000","last_change":"2023-10-09T04:44:53.792412+0000","last_active":"2023-10-09T04:45:06.257215+0000","last_peered":"2023-10-09T04:45:06.257215+0000","last_clean":"2023-10-09T04:45:06.257215+0000","last_became_active":"2023-10-09T04:44:53.792009+0000","last_became_peered":"2023-10-09T04:44:53.792009+0000","last_unstale":"2023-10-09T04:45:06.257215+0000","last_undegraded":"2023-10-09T04:45:06.257215+0000","last_fullsized":"2023-10-09T04:45:06.257215+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:39.423339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.120188+0000","last_change":"2023-10-09T04:44:52.774852+0000","last_active":"2023-10-09T04:45:06.120188+0000","last_peered":"2023-10-09T04:45:06.120188+0000","last_clean":"2023-10-09T04:45:06.120188+0000","last_became_active":"2023-10-09T04:44:52.774554+0000","last_became_peered":"2023-10-09T04:44:52.774554+0000","last_unstale":"2023-10-09T04:45:06.120188+0000","last_undegraded":"2023-10-09T04:45:06.120188+0000","last_fullsized":"2023-10-09T04:45:06.120188+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:53.658339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.115098+0000","last_change":"2023-10-09T04:42:48.699613+0000","last_active":"2023-10-09T04:45:06.115098+0000","last_peered":"2023-10-09T04:45:06.115098+0000","last_clean":"2023-10-09T04:45:06.115098+0000","last_became_active":"2023-10-09T04:42:48.699095+0000","last_became_peered":"2023-10-09T04:42:48.699095+0000","last_unstale":"2023-10-09T04:45:06.115098+0000","last_undegraded":"2023-10-09T04:45:06.115098+0000","last_fullsized":"2023-10-09T04:45:06.115098+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:57.871245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.112013+0000","last_change":"2023-10-09T04:44:53.958823+0000","last_active":"2023-10-09T04:45:06.112013+0000","last_peered":"2023-10-09T04:45:06.112013+0000","last_clean":"2023-10-09T04:45:06.112013+0000","last_became_active":"2023-10-09T04:44:53.957881+0000","last_became_peered":"2023-10-09T04:44:53.957881+0000","last_unstale":"2023-10-09T04:45:06.112013+0000","last_undegraded":"2023-10-09T04:45:06.112013+0000","last_fullsized":"2023-10-09T04:45:06.112013+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:04.731794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.118724+0000","last_change":"2023-10-09T04:42:48.849748+0000","last_active":"2023-10-09T04:45:06.118724+0000","last_peered":"2023-10-09T04:45:06.118724+0000","last_clean":"2023-10-09T04:45:06.118724+0000","last_became_active":"2023-10-09T04:42:48.848834+0000","last_became_peered":"2023-10-09T04:42:48.848834+0000","last_unstale":"2023-10-09T04:45:06.118724+0000","last_undegraded":"2023-10-09T04:45:06.118724+0000","last_fullsized":"2023-10-09T04:45:06.118724+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:20.475654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.113208+0000","last_change":"2023-10-09T04:44:52.793364+0000","last_active":"2023-10-09T04:45:06.113208+0000","last_peered":"2023-10-09T04:45:06.113208+0000","last_clean":"2023-10-09T04:45:06.113208+0000","last_became_active":"2023-10-09T04:44:52.787968+0000","last_became_peered":"2023-10-09T04:44:52.787968+0000","last_unstale":"2023-10-09T04:45:06.113208+0000","last_undegraded":"2023-10-09T04:45:06.113208+0000","last_fullsized":"2023-10-09T04:45:06.113208+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:33.783576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.119888+0000","last_change":"2023-10-09T04:44:53.960925+0000","last_active":"2023-10-09T04:45:06.119888+0000","last_peered":"2023-10-09T04:45:06.119888+0000","last_clean":"2023-10-09T04:45:06.119888+0000","last_became_active":"2023-10-09T04:44:53.960514+0000","last_became_peered":"2023-10-09T04:44:53.960514+0000","last_unstale":"2023-10-09T04:45:06.119888+0000","last_undegraded":"2023-10-09T04:45:06.119888+0000","last_fullsized":"2023-10-09T04:45:06.119888+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:15.192183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.117937+0000","last_change":"2023-10-09T04:44:52.791414+0000","last_active":"2023-10-09T04:45:06.117937+0000","last_peered":"2023-10-09T04:45:06.117937+0000","last_clean":"2023-10-09T04:45:06.117937+0000","last_became_active":"2023-10-09T04:44:52.790819+0000","last_became_peered":"2023-10-09T04:44:52.790819+0000","last_unstale":"2023-10-09T04:45:06.117937+0000","last_undegraded":"2023-10-09T04:45:06.117937+0000","last_fullsized":"2023-10-09T04:45:06.117937+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:13.452987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.111829+0000","last_change":"2023-10-09T04:42:48.851447+0000","last_active":"2023-10-09T04:45:06.111829+0000","last_peered":"2023-10-09T04:45:06.111829+0000","last_clean":"2023-10-09T04:45:06.111829+0000","last_became_active":"2023-10-09T04:42:48.850687+0000","last_became_peered":"2023-10-09T04:42:48.850687+0000","last_unstale":"2023-10-09T04:45:06.111829+0000","last_undegraded":"2023-10-09T04:45:06.111829+0000","last_fullsized":"2023-10-09T04:45:06.111829+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:32:22.356405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.120288+0000","last_change":"2023-10-09T04:44:53.790281+0000","last_active":"2023-10-09T04:45:06.120288+0000","last_peered":"2023-10-09T04:45:06.120288+0000","last_clean":"2023-10-09T04:45:06.120288+0000","last_became_active":"2023-10-09T04:44:53.789959+0000","last_became_peered":"2023-10-09T04:44:53.789959+0000","last_unstale":"2023-10-09T04:45:06.120288+0000","last_undegraded":"2023-10-09T04:45:06.120288+0000","last_fullsized":"2023-10-09T04:45:06.120288+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:21.459393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.119581+0000","last_change":"2023-10-09T04:44:52.782233+0000","last_active":"2023-10-09T04:45:06.119581+0000","last_peered":"2023-10-09T04:45:06.119581+0000","last_clean":"2023-10-09T04:45:06.119581+0000","last_became_active":"2023-10-09T04:44:52.781789+0000","last_became_peered":"2023-10-09T04:44:52.781789+0000","last_unstale":"2023-10-09T04:45:06.119581+0000","last_undegraded":"2023-10-09T04:45:06.119581+0000","last_fullsized":"2023-10-09T04:45:06.119581+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:15:43.929566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.111397+0000","last_change":"2023-10-09T04:42:48.697850+0000","last_active":"2023-10-09T04:45:06.111397+0000","last_peered":"2023-10-09T04:45:06.111397+0000","last_clean":"2023-10-09T04:45:06.111397+0000","last_became_active":"2023-10-09T04:42:48.697520+0000","last_became_peered":"2023-10-09T04:42:48.697520+0000","last_unstale":"2023-10-09T04:45:06.111397+0000","last_undegraded":"2023-10-09T04:45:06.111397+0000","last_fullsized":"2023-10-09T04:45:06.111397+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:04.728265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.115056+0000","last_change":"2023-10-09T04:44:53.959705+0000","last_active":"2023-10-09T04:45:06.115056+0000","last_peered":"2023-10-09T04:45:06.115056+0000","last_clean":"2023-10-09T04:45:06.115056+0000","last_became_active":"2023-10-09T04:44:53.959092+0000","last_became_peered":"2023-10-09T04:44:53.959092+0000","last_unstale":"2023-10-09T04:45:06.115056+0000","last_undegraded":"2023-10-09T04:45:06.115056+0000","last_fullsized":"2023-10-09T04:45:06.115056+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:31:29.189897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.112362+0000","last_change":"2023-10-09T04:44:52.794247+0000","last_active":"2023-10-09T04:45:06.112362+0000","last_peered":"2023-10-09T04:45:06.112362+0000","last_clean":"2023-10-09T04:45:06.112362+0000","last_became_active":"2023-10-09T04:44:52.793941+0000","last_became_peered":"2023-10-09T04:44:52.793941+0000","last_unstale":"2023-10-09T04:45:06.112362+0000","last_undegraded":"2023-10-09T04:45:06.112362+0000","last_fullsized":"2023-10-09T04:45:06.112362+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:45.149949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":275,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.111400+0000","last_change":"2023-10-09T04:41:09.432397+0000","last_active":"2023-10-09T04:45:06.111400+0000","last_peered":"2023-10-09T04:45:06.111400+0000","last_clean":"2023-10-09T04:45:06.111400+0000","last_became_active":"2023-10-09T04:41:09.431675+0000","last_became_peered":"2023-10-09T04:41:09.431675+0000","last_unstale":"2023-10-09T04:45:06.111400+0000","last_undegraded":"2023-10-09T04:45:06.111400+0000","last_fullsized":"2023-10-09T04:45:06.111400+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_clean_scrub_stamp":"2023-10-09T04:36:44.261013+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:23.657333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":19,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.117287+0000","last_change":"2023-10-09T04:42:48.698570+0000","last_active":"2023-10-09T04:45:06.117287+0000","last_peered":"2023-10-09T04:45:06.117287+0000","last_clean":"2023-10-09T04:45:06.117287+0000","last_became_active":"2023-10-09T04:42:48.698098+0000","last_became_peered":"2023-10-09T04:42:48.698098+0000","last_unstale":"2023-10-09T04:45:06.117287+0000","last_undegraded":"2023-10-09T04:45:06.117287+0000","last_fullsized":"2023-10-09T04:45:06.117287+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:35.340926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":10,"reported_epoch":71,"state":"active+clean","last_fresh":"2023-10-09T04:45:07.121151+0000","last_change":"2023-10-09T04:45:07.121151+0000","last_active":"2023-10-09T04:45:07.121151+0000","last_peered":"2023-10-09T04:45:07.121151+0000","last_clean":"2023-10-09T04:45:07.121151+0000","last_became_active":"2023-10-09T04:45:07.120694+0000","last_became_peered":"2023-10-09T04:45:07.120694+0000","last_unstale":"2023-10-09T04:45:07.121151+0000","last_undegraded":"2023-10-09T04:45:07.121151+0000","last_fullsized":"2023-10-09T04:45:07.121151+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:10.567313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.115936+0000","last_change":"2023-10-09T04:44:52.790877+0000","last_active":"2023-10-09T04:45:06.115936+0000","last_peered":"2023-10-09T04:45:06.115936+0000","last_clean":"2023-10-09T04:45:06.115936+0000","last_became_active":"2023-10-09T04:44:52.790563+0000","last_became_peered":"2023-10-09T04:44:52.790563+0000","last_unstale":"2023-10-09T04:45:06.115936+0000","last_undegraded":"2023-10-09T04:45:06.115936+0000","last_fullsized":"2023-10-09T04:45:06.115936+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:40:33.821035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.116004+0000","last_change":"2023-10-09T04:42:48.728368+0000","last_active":"2023-10-09T04:45:06.116004+0000","last_peered":"2023-10-09T04:45:06.116004+0000","last_clean":"2023-10-09T04:45:06.116004+0000","last_became_active":"2023-10-09T04:42:48.727757+0000","last_became_peered":"2023-10-09T04:42:48.727757+0000","last_unstale":"2023-10-09T04:45:06.116004+0000","last_undegraded":"2023-10-09T04:45:06.116004+0000","last_fullsized":"2023-10-09T04:45:06.116004+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:21:02.404428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":10,"reported_epoch":71,"state":"active+clean","last_fresh":"2023-10-09T04:45:07.162112+0000","last_change":"2023-10-09T04:45:07.162112+0000","last_active":"2023-10-09T04:45:07.162112+0000","last_peered":"2023-10-09T04:45:07.162112+0000","last_clean":"2023-10-09T04:45:07.162112+0000","last_became_active":"2023-10-09T04:45:07.161385+0000","last_became_peered":"2023-10-09T04:45:07.161385+0000","last_unstale":"2023-10-09T04:45:07.162112+0000","last_undegraded":"2023-10-09T04:45:07.162112+0000","last_fullsized":"2023-10-09T04:45:07.162112+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:18.574454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.256813+0000","last_change":"2023-10-09T04:44:52.784769+0000","last_active":"2023-10-09T04:45:06.256813+0000","last_peered":"2023-10-09T04:45:06.256813+0000","last_clean":"2023-10-09T04:45:06.256813+0000","last_became_active":"2023-10-09T04:44:52.784374+0000","last_became_peered":"2023-10-09T04:44:52.784374+0000","last_unstale":"2023-10-09T04:45:06.256813+0000","last_undegraded":"2023-10-09T04:45:06.256813+0000","last_fullsized":"2023-10-09T04:45:06.256813+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:29.252800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":19,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.117288+0000","last_change":"2023-10-09T04:42:48.698230+0000","last_active":"2023-10-09T04:45:06.117288+0000","last_peered":"2023-10-09T04:45:06.117288+0000","last_clean":"2023-10-09T04:45:06.117288+0000","last_became_active":"2023-10-09T04:42:48.697694+0000","last_became_peered":"2023-10-09T04:42:48.697694+0000","last_unstale":"2023-10-09T04:45:06.117288+0000","last_undegraded":"2023-10-09T04:45:06.117288+0000","last_fullsized":"2023-10-09T04:45:06.117288+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:46:21.257527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.116232+0000","last_change":"2023-10-09T04:44:53.788563+0000","last_active":"2023-10-09T04:45:06.116232+0000","last_peered":"2023-10-09T04:45:06.116232+0000","last_clean":"2023-10-09T04:45:06.116232+0000","last_became_active":"2023-10-09T04:44:53.787983+0000","last_became_peered":"2023-10-09T04:44:53.787983+0000","last_unstale":"2023-10-09T04:45:06.116232+0000","last_undegraded":"2023-10-09T04:45:06.116232+0000","last_fullsized":"2023-10-09T04:45:06.116232+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:21:15.499148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.118121+0000","last_change":"2023-10-09T04:44:52.794966+0000","last_active":"2023-10-09T04:45:06.118121+0000","last_peered":"2023-10-09T04:45:06.118121+0000","last_clean":"2023-10-09T04:45:06.118121+0000","last_became_active":"2023-10-09T04:44:52.794640+0000","last_became_peered":"2023-10-09T04:44:52.794640+0000","last_unstale":"2023-10-09T04:45:06.118121+0000","last_undegraded":"2023-10-09T04:45:06.118121+0000","last_fullsized":"2023-10-09T04:45:06.118121+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:00:47.077423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.116766+0000","last_change":"2023-10-09T04:42:48.696990+0000","last_active":"2023-10-09T04:45:06.116766+0000","last_peered":"2023-10-09T04:45:06.116766+0000","last_clean":"2023-10-09T04:45:06.116766+0000","last_became_active":"2023-10-09T04:42:48.696505+0000","last_became_peered":"2023-10-09T04:42:48.696505+0000","last_unstale":"2023-10-09T04:45:06.116766+0000","last_undegraded":"2023-10-09T04:45:06.116766+0000","last_fullsized":"2023-10-09T04:45:06.116766+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:55.113687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.119644+0000","last_change":"2023-10-09T04:44:53.787727+0000","last_active":"2023-10-09T04:45:06.119644+0000","last_peered":"2023-10-09T04:45:06.119644+0000","last_clean":"2023-10-09T04:45:06.119644+0000","last_became_active":"2023-10-09T04:44:53.787387+0000","last_became_peered":"2023-10-09T04:44:53.787387+0000","last_unstale":"2023-10-09T04:45:06.119644+0000","last_undegraded":"2023-10-09T04:45:06.119644+0000","last_fullsized":"2023-10-09T04:45:06.119644+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:46:25.002228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.112067+0000","last_change":"2023-10-09T04:44:52.794598+0000","last_active":"2023-10-09T04:45:06.112067+0000","last_peered":"2023-10-09T04:45:06.112067+0000","last_clean":"2023-10-09T04:45:06.112067+0000","last_became_active":"2023-10-09T04:44:52.794048+0000","last_became_peered":"2023-10-09T04:44:52.794048+0000","last_unstale":"2023-10-09T04:45:06.112067+0000","last_undegraded":"2023-10-09T04:45:06.112067+0000","last_fullsized":"2023-10-09T04:45:06.112067+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:27.518572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.257584+0000","last_change":"2023-10-09T04:44:53.792760+0000","last_active":"2023-10-09T04:45:06.257584+0000","last_peered":"2023-10-09T04:45:06.257584+0000","last_clean":"2023-10-09T04:45:06.257584+0000","last_became_active":"2023-10-09T04:44:53.792031+0000","last_became_peered":"2023-10-09T04:44:53.792031+0000","last_unstale":"2023-10-09T04:45:06.257584+0000","last_undegraded":"2023-10-09T04:45:06.257584+0000","last_fullsized":"2023-10-09T04:45:06.257584+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:57.177135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.256830+0000","last_change":"2023-10-09T04:44:52.790067+0000","last_active":"2023-10-09T04:45:06.256830+0000","last_peered":"2023-10-09T04:45:06.256830+0000","last_clean":"2023-10-09T04:45:06.256830+0000","last_became_active":"2023-10-09T04:44:52.789682+0000","last_became_peered":"2023-10-09T04:44:52.789682+0000","last_unstale":"2023-10-09T04:45:06.256830+0000","last_undegraded":"2023-10-09T04:45:06.256830+0000","last_fullsized":"2023-10-09T04:45:06.256830+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:37:16.879187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.257666+0000","last_change":"2023-10-09T04:44:53.793305+0000","last_active":"2023-10-09T04:45:06.257666+0000","last_peered":"2023-10-09T04:45:06.257666+0000","last_clean":"2023-10-09T04:45:06.257666+0000","last_became_active":"2023-10-09T04:44:53.792984+0000","last_became_peered":"2023-10-09T04:44:53.792984+0000","last_unstale":"2023-10-09T04:45:06.257666+0000","last_undegraded":"2023-10-09T04:45:06.257666+0000","last_fullsized":"2023-10-09T04:45:06.257666+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:11.704709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.115432+0000","last_change":"2023-10-09T04:44:52.791331+0000","last_active":"2023-10-09T04:45:06.115432+0000","last_peered":"2023-10-09T04:45:06.115432+0000","last_clean":"2023-10-09T04:45:06.115432+0000","last_became_active":"2023-10-09T04:44:52.787485+0000","last_became_peered":"2023-10-09T04:44:52.787485+0000","last_unstale":"2023-10-09T04:45:06.115432+0000","last_undegraded":"2023-10-09T04:45:06.115432+0000","last_fullsized":"2023-10-09T04:45:06.115432+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:13.478903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.257427+0000","last_change":"2023-10-09T04:44:53.794136+0000","last_active":"2023-10-09T04:45:06.257427+0000","last_peered":"2023-10-09T04:45:06.257427+0000","last_clean":"2023-10-09T04:45:06.257427+0000","last_became_active":"2023-10-09T04:44:53.793829+0000","last_became_peered":"2023-10-09T04:44:53.793829+0000","last_unstale":"2023-10-09T04:45:06.257427+0000","last_undegraded":"2023-10-09T04:45:06.257427+0000","last_fullsized":"2023-10-09T04:45:06.257427+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:50.822987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'3","reported_seq":19,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.112571+0000","last_change":"2023-10-09T04:44:52.797846+0000","last_active":"2023-10-09T04:45:06.112571+0000","last_peered":"2023-10-09T04:45:06.112571+0000","last_clean":"2023-10-09T04:45:06.112571+0000","last_became_active":"2023-10-09T04:44:52.797486+0000","last_became_peered":"2023-10-09T04:44:52.797486+0000","last_unstale":"2023-10-09T04:45:06.112571+0000","last_undegraded":"2023-10-09T04:45:06.112571+0000","last_fullsized":"2023-10-09T04:45:06.112571+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:05.789058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.119683+0000","last_change":"2023-10-09T04:44:53.787406+0000","last_active":"2023-10-09T04:45:06.119683+0000","last_peered":"2023-10-09T04:45:06.119683+0000","last_clean":"2023-10-09T04:45:06.119683+0000","last_became_active":"2023-10-09T04:44:53.786682+0000","last_became_peered":"2023-10-09T04:44:53.786682+0000","last_unstale":"2023-10-09T04:45:06.119683+0000","last_undegraded":"2023-10-09T04:45:06.119683+0000","last_fullsized":"2023-10-09T04:45:06.119683+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:05.555795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.117949+0000","last_change":"2023-10-09T04:44:52.795694+0000","last_active":"2023-10-09T04:45:06.117949+0000","last_peered":"2023-10-09T04:45:06.117949+0000","last_clean":"2023-10-09T04:45:06.117949+0000","last_became_active":"2023-10-09T04:44:52.795230+0000","last_became_peered":"2023-10-09T04:44:52.795230+0000","last_unstale":"2023-10-09T04:45:06.117949+0000","last_undegraded":"2023-10-09T04:45:06.117949+0000","last_fullsized":"2023-10-09T04:45:06.117949+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:15.951253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":24,"reported_epoch":71,"state":"active+clean","last_fresh":"2023-10-09T04:45:07.162222+0000","last_change":"2023-10-09T04:45:07.162222+0000","last_active":"2023-10-09T04:45:07.162222+0000","last_peered":"2023-10-09T04:45:07.162222+0000","last_clean":"2023-10-09T04:45:07.162222+0000","last_became_active":"2023-10-09T04:45:07.161194+0000","last_became_peered":"2023-10-09T04:45:07.161194+0000","last_unstale":"2023-10-09T04:45:07.162222+0000","last_undegraded":"2023-10-09T04:45:07.162222+0000","last_fullsized":"2023-10-09T04:45:07.162222+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:26.794178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.111888+0000","last_change":"2023-10-09T04:44:52.787469+0000","last_active":"2023-10-09T04:45:06.111888+0000","last_peered":"2023-10-09T04:45:06.111888+0000","last_clean":"2023-10-09T04:45:06.111888+0000","last_became_active":"2023-10-09T04:44:52.787148+0000","last_became_peered":"2023-10-09T04:44:52.787148+0000","last_unstale":"2023-10-09T04:45:06.111888+0000","last_undegraded":"2023-10-09T04:45:06.111888+0000","last_fullsized":"2023-10-09T04:45:06.111888+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:12.469065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.256783+0000","last_change":"2023-10-09T04:44:53.790116+0000","last_active":"2023-10-09T04:45:06.256783+0000","last_peered":"2023-10-09T04:45:06.256783+0000","last_clean":"2023-10-09T04:45:06.256783+0000","last_became_active":"2023-10-09T04:44:53.789562+0000","last_became_peered":"2023-10-09T04:44:53.789562+0000","last_unstale":"2023-10-09T04:45:06.256783+0000","last_undegraded":"2023-10-09T04:45:06.256783+0000","last_fullsized":"2023-10-09T04:45:06.256783+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:07.107715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.112222+0000","last_change":"2023-10-09T04:44:52.780659+0000","last_active":"2023-10-09T04:45:06.112222+0000","last_peered":"2023-10-09T04:45:06.112222+0000","last_clean":"2023-10-09T04:45:06.112222+0000","last_became_active":"2023-10-09T04:44:52.780365+0000","last_became_peered":"2023-10-09T04:44:52.780365+0000","last_unstale":"2023-10-09T04:45:06.112222+0000","last_undegraded":"2023-10-09T04:45:06.112222+0000","last_fullsized":"2023-10-09T04:45:06.112222+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:59.540930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.117203+0000","last_change":"2023-10-09T04:44:53.962437+0000","last_active":"2023-10-09T04:45:06.117203+0000","last_peered":"2023-10-09T04:45:06.117203+0000","last_clean":"2023-10-09T04:45:06.117203+0000","last_became_active":"2023-10-09T04:44:53.961810+0000","last_became_peered":"2023-10-09T04:44:53.961810+0000","last_unstale":"2023-10-09T04:45:06.117203+0000","last_undegraded":"2023-10-09T04:45:06.117203+0000","last_fullsized":"2023-10-09T04:45:06.117203+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:56.436933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.112340+0000","last_change":"2023-10-09T04:44:52.792028+0000","last_active":"2023-10-09T04:45:06.112340+0000","last_peered":"2023-10-09T04:45:06.112340+0000","last_clean":"2023-10-09T04:45:06.112340+0000","last_became_active":"2023-10-09T04:44:52.791588+0000","last_became_peered":"2023-10-09T04:44:52.791588+0000","last_unstale":"2023-10-09T04:45:06.112340+0000","last_undegraded":"2023-10-09T04:45:06.112340+0000","last_fullsized":"2023-10-09T04:45:06.112340+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:57.589240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.113093+0000","last_change":"2023-10-09T04:44:53.959134+0000","last_active":"2023-10-09T04:45:06.113093+0000","last_peered":"2023-10-09T04:45:06.113093+0000","last_clean":"2023-10-09T04:45:06.113093+0000","last_became_active":"2023-10-09T04:44:53.958411+0000","last_became_peered":"2023-10-09T04:44:53.958411+0000","last_unstale":"2023-10-09T04:45:06.113093+0000","last_undegraded":"2023-10-09T04:45:06.113093+0000","last_fullsized":"2023-10-09T04:45:06.113093+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:49:59.662675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":27,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.114902+0000","last_change":"2023-10-09T04:44:52.784918+0000","last_active":"2023-10-09T04:45:06.114902+0000","last_peered":"2023-10-09T04:45:06.114902+0000","last_clean":"2023-10-09T04:45:06.114902+0000","last_became_active":"2023-10-09T04:44:52.783726+0000","last_became_peered":"2023-10-09T04:44:52.783726+0000","last_unstale":"2023-10-09T04:45:06.114902+0000","last_undegraded":"2023-10-09T04:45:06.114902+0000","last_fullsized":"2023-10-09T04:45:06.114902+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:28:13.795447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.119986+0000","last_change":"2023-10-09T04:44:53.962582+0000","last_active":"2023-10-09T04:45:06.119986+0000","last_peered":"2023-10-09T04:45:06.119986+0000","last_clean":"2023-10-09T04:45:06.119986+0000","last_became_active":"2023-10-09T04:44:53.961909+0000","last_became_peered":"2023-10-09T04:44:53.961909+0000","last_unstale":"2023-10-09T04:45:06.119986+0000","last_undegraded":"2023-10-09T04:45:06.119986+0000","last_fullsized":"2023-10-09T04:45:06.119986+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:42.808094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.111984+0000","last_change":"2023-10-09T04:44:52.792557+0000","last_active":"2023-10-09T04:45:06.111984+0000","last_peered":"2023-10-09T04:45:06.111984+0000","last_clean":"2023-10-09T04:45:06.111984+0000","last_became_active":"2023-10-09T04:44:52.791890+0000","last_became_peered":"2023-10-09T04:44:52.791890+0000","last_unstale":"2023-10-09T04:45:06.111984+0000","last_undegraded":"2023-10-09T04:45:06.111984+0000","last_fullsized":"2023-10-09T04:45:06.111984+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:04:46.379852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.118105+0000","last_change":"2023-10-09T04:44:53.962346+0000","last_active":"2023-10-09T04:45:06.118105+0000","last_peered":"2023-10-09T04:45:06.118105+0000","last_clean":"2023-10-09T04:45:06.118105+0000","last_became_active":"2023-10-09T04:44:53.961890+0000","last_became_peered":"2023-10-09T04:44:53.961890+0000","last_unstale":"2023-10-09T04:45:06.118105+0000","last_undegraded":"2023-10-09T04:45:06.118105+0000","last_fullsized":"2023-10-09T04:45:06.118105+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:01.790749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.119766+0000","last_change":"2023-10-09T04:44:52.781506+0000","last_active":"2023-10-09T04:45:06.119766+0000","last_peered":"2023-10-09T04:45:06.119766+0000","last_clean":"2023-10-09T04:45:06.119766+0000","last_became_active":"2023-10-09T04:44:52.779712+0000","last_became_peered":"2023-10-09T04:44:52.779712+0000","last_unstale":"2023-10-09T04:45:06.119766+0000","last_undegraded":"2023-10-09T04:45:06.119766+0000","last_fullsized":"2023-10-09T04:45:06.119766+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:12.774869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.114449+0000","last_change":"2023-10-09T04:44:53.959939+0000","last_active":"2023-10-09T04:45:06.114449+0000","last_peered":"2023-10-09T04:45:06.114449+0000","last_clean":"2023-10-09T04:45:06.114449+0000","last_became_active":"2023-10-09T04:44:53.957647+0000","last_became_peered":"2023-10-09T04:44:53.957647+0000","last_unstale":"2023-10-09T04:45:06.114449+0000","last_undegraded":"2023-10-09T04:45:06.114449+0000","last_fullsized":"2023-10-09T04:45:06.114449+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:11.820519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":19,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.111864+0000","last_change":"2023-10-09T04:44:52.792074+0000","last_active":"2023-10-09T04:45:06.111864+0000","last_peered":"2023-10-09T04:45:06.111864+0000","last_clean":"2023-10-09T04:45:06.111864+0000","last_became_active":"2023-10-09T04:44:52.791245+0000","last_became_peered":"2023-10-09T04:44:52.791245+0000","last_unstale":"2023-10-09T04:45:06.111864+0000","last_undegraded":"2023-10-09T04:45:06.111864+0000","last_fullsized":"2023-10-09T04:45:06.111864+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:06:12.940420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.256954+0000","last_change":"2023-10-09T04:44:53.797404+0000","last_active":"2023-10-09T04:45:06.256954+0000","last_peered":"2023-10-09T04:45:06.256954+0000","last_clean":"2023-10-09T04:45:06.256954+0000","last_became_active":"2023-10-09T04:44:53.797053+0000","last_became_peered":"2023-10-09T04:44:53.797053+0000","last_unstale":"2023-10-09T04:45:06.256954+0000","last_undegraded":"2023-10-09T04:45:06.256954+0000","last_fullsized":"2023-10-09T04:45:06.256954+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:43.827978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"69'7","reported_seq":27,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.257547+0000","last_change":"2023-10-09T04:44:52.781203+0000","last_active":"2023-10-09T04:45:06.257547+0000","last_peered":"2023-10-09T04:45:06.257547+0000","last_clean":"2023-10-09T04:45:06.257547+0000","last_became_active":"2023-10-09T04:44:52.780902+0000","last_became_peered":"2023-10-09T04:44:52.780902+0000","last_unstale":"2023-10-09T04:45:06.257547+0000","last_undegraded":"2023-10-09T04:45:06.257547+0000","last_fullsized":"2023-10-09T04:45:06.257547+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:56.332974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.116150+0000","last_change":"2023-10-09T04:44:53.793986+0000","last_active":"2023-10-09T04:45:06.116150+0000","last_peered":"2023-10-09T04:45:06.116150+0000","last_clean":"2023-10-09T04:45:06.116150+0000","last_became_active":"2023-10-09T04:44:53.793676+0000","last_became_peered":"2023-10-09T04:44:53.793676+0000","last_unstale":"2023-10-09T04:45:06.116150+0000","last_undegraded":"2023-10-09T04:45:06.116150+0000","last_fullsized":"2023-10-09T04:45:06.116150+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:18.344330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":20,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.111765+0000","last_change":"2023-10-09T04:44:52.781582+0000","last_active":"2023-10-09T04:45:06.111765+0000","last_peered":"2023-10-09T04:45:06.111765+0000","last_clean":"2023-10-09T04:45:06.111765+0000","last_became_active":"2023-10-09T04:44:52.781259+0000","last_became_peered":"2023-10-09T04:44:52.781259+0000","last_unstale":"2023-10-09T04:45:06.111765+0000","last_undegraded":"2023-10-09T04:45:06.111765+0000","last_fullsized":"2023-10-09T04:45:06.111765+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:01.648298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.112153+0000","last_change":"2023-10-09T04:44:53.958305+0000","last_active":"2023-10-09T04:45:06.112153+0000","last_peered":"2023-10-09T04:45:06.112153+0000","last_clean":"2023-10-09T04:45:06.112153+0000","last_became_active":"2023-10-09T04:44:53.957878+0000","last_became_peered":"2023-10-09T04:44:53.957878+0000","last_unstale":"2023-10-09T04:45:06.112153+0000","last_undegraded":"2023-10-09T04:45:06.112153+0000","last_fullsized":"2023-10-09T04:45:06.112153+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:43:27.473519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.118092+0000","last_change":"2023-10-09T04:44:52.790185+0000","last_active":"2023-10-09T04:45:06.118092+0000","last_peered":"2023-10-09T04:45:06.118092+0000","last_clean":"2023-10-09T04:45:06.118092+0000","last_became_active":"2023-10-09T04:44:52.789798+0000","last_became_peered":"2023-10-09T04:44:52.789798+0000","last_unstale":"2023-10-09T04:45:06.118092+0000","last_undegraded":"2023-10-09T04:45:06.118092+0000","last_fullsized":"2023-10-09T04:45:06.118092+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:17:23.671238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.112165+0000","last_change":"2023-10-09T04:44:53.794730+0000","last_active":"2023-10-09T04:45:06.112165+0000","last_peered":"2023-10-09T04:45:06.112165+0000","last_clean":"2023-10-09T04:45:06.112165+0000","last_became_active":"2023-10-09T04:44:53.793787+0000","last_became_peered":"2023-10-09T04:44:53.793787+0000","last_unstale":"2023-10-09T04:45:06.112165+0000","last_undegraded":"2023-10-09T04:45:06.112165+0000","last_fullsized":"2023-10-09T04:45:06.112165+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:58:33.016122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.120369+0000","last_change":"2023-10-09T04:44:52.781846+0000","last_active":"2023-10-09T04:45:06.120369+0000","last_peered":"2023-10-09T04:45:06.120369+0000","last_clean":"2023-10-09T04:45:06.120369+0000","last_became_active":"2023-10-09T04:44:52.781538+0000","last_became_peered":"2023-10-09T04:44:52.781538+0000","last_unstale":"2023-10-09T04:45:06.120369+0000","last_undegraded":"2023-10-09T04:45:06.120369+0000","last_fullsized":"2023-10-09T04:45:06.120369+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:33.775067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.115008+0000","last_change":"2023-10-09T04:44:53.797648+0000","last_active":"2023-10-09T04:45:06.115008+0000","last_peered":"2023-10-09T04:45:06.115008+0000","last_clean":"2023-10-09T04:45:06.115008+0000","last_became_active":"2023-10-09T04:44:53.797099+0000","last_became_peered":"2023-10-09T04:44:53.797099+0000","last_unstale":"2023-10-09T04:45:06.115008+0000","last_undegraded":"2023-10-09T04:45:06.115008+0000","last_fullsized":"2023-10-09T04:45:06.115008+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:35.903142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.112257+0000","last_change":"2023-10-09T04:44:52.794864+0000","last_active":"2023-10-09T04:45:06.112257+0000","last_peered":"2023-10-09T04:45:06.112257+0000","last_clean":"2023-10-09T04:45:06.112257+0000","last_became_active":"2023-10-09T04:44:52.793907+0000","last_became_peered":"2023-10-09T04:44:52.793907+0000","last_unstale":"2023-10-09T04:45:06.112257+0000","last_undegraded":"2023-10-09T04:45:06.112257+0000","last_fullsized":"2023-10-09T04:45:06.112257+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:56:00.825914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.120075+0000","last_change":"2023-10-09T04:44:53.958581+0000","last_active":"2023-10-09T04:45:06.120075+0000","last_peered":"2023-10-09T04:45:06.120075+0000","last_clean":"2023-10-09T04:45:06.120075+0000","last_became_active":"2023-10-09T04:44:53.957939+0000","last_became_peered":"2023-10-09T04:44:53.957939+0000","last_unstale":"2023-10-09T04:45:06.120075+0000","last_undegraded":"2023-10-09T04:45:06.120075+0000","last_fullsized":"2023-10-09T04:45:06.120075+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:12.616951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.115858+0000","last_change":"2023-10-09T04:44:52.790106+0000","last_active":"2023-10-09T04:45:06.115858+0000","last_peered":"2023-10-09T04:45:06.115858+0000","last_clean":"2023-10-09T04:45:06.115858+0000","last_became_active":"2023-10-09T04:44:52.789856+0000","last_became_peered":"2023-10-09T04:44:52.789856+0000","last_unstale":"2023-10-09T04:45:06.115858+0000","last_undegraded":"2023-10-09T04:45:06.115858+0000","last_fullsized":"2023-10-09T04:45:06.115858+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:50:07.831074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.112835+0000","last_change":"2023-10-09T04:44:53.790071+0000","last_active":"2023-10-09T04:45:06.112835+0000","last_peered":"2023-10-09T04:45:06.112835+0000","last_clean":"2023-10-09T04:45:06.112835+0000","last_became_active":"2023-10-09T04:44:53.789725+0000","last_became_peered":"2023-10-09T04:44:53.789725+0000","last_unstale":"2023-10-09T04:45:06.112835+0000","last_undegraded":"2023-10-09T04:45:06.112835+0000","last_fullsized":"2023-10-09T04:45:06.112835+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:25.624697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'2","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.257000+0000","last_change":"2023-10-09T04:44:52.793401+0000","last_active":"2023-10-09T04:45:06.257000+0000","last_peered":"2023-10-09T04:45:06.257000+0000","last_clean":"2023-10-09T04:45:06.257000+0000","last_became_active":"2023-10-09T04:44:52.792982+0000","last_became_peered":"2023-10-09T04:44:52.792982+0000","last_unstale":"2023-10-09T04:45:06.257000+0000","last_undegraded":"2023-10-09T04:45:06.257000+0000","last_fullsized":"2023-10-09T04:45:06.257000+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:39.255814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":47,"num_read_kb":49,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":45,"ondisk_log_size":45,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234053,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298600,"kb_used_data":1312,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446552,"statfs":{"total":95995035648,"available":95689269248,"internally_reserved":0,"allocated":1343488,"data_stored":912385,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":0.92800000000000005,"15min":0.94999999999999996},"min":{"1min":0.57999999999999996,"5min":0.433,"15min":0.433},"max":{"1min":1.3180000000000001,"5min":1.458,"15min":1.458},"last":1.0069999999999999},{"interface":"front","average":{"1min":1.081,"5min":0.96199999999999997,"15min":0.96799999999999997},"min":{"1min":0.59099999999999997,"5min":0.502,"15min":0.502},"max":{"1min":1.423,"5min":1.5580000000000001,"15min":1.5580000000000001},"last":1.2450000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:46:10 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":0.98699999999999999,"15min":1.0209999999999999},"min":{"1min":0.60599999999999998,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.3129999999999999,"5min":1.423,"15min":1.423},"last":1.1140000000000001},{"interface":"front","average":{"1min":0.94899999999999995,"5min":0.91600000000000004,"15min":0.93200000000000005},"min":{"1min":0.63900000000000001,"5min":0.435,"15min":0.435},"max":{"1min":1.1850000000000001,"5min":1.9950000000000001,"15min":1.9950000000000001},"last":1.163}]},{"osd":2,"last update":"Mon Oct 9 04:46:10 2023","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":0.97099999999999997,"15min":1.0169999999999999},"min":{"1min":0.67300000000000004,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.282,"5min":2.0819999999999999,"15min":2.0819999999999999},"last":1.0720000000000001},{"interface":"front","average":{"1min":1.0109999999999999,"5min":0.96499999999999997,"15min":0.98899999999999999},"min":{"1min":0.65600000000000003,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":1.2729999999999999,"5min":1.4730000000000001,"15min":1.4730000000000001},"last":0.89000000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":0.89700000000000002,"15min":0.88400000000000001},"min":{"1min":0.68400000000000005,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.226,"5min":1.4159999999999999,"15min":1.4159999999999999},"last":1.038},{"interface":"front","average":{"1min":1.1200000000000001,"5min":1.0509999999999999,"15min":1.0700000000000001},"min":{"1min":0.81499999999999995,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.3660000000000001,"5min":1.5129999999999999,"15min":1.5129999999999999},"last":1.1899999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.89600000000000002,"15min":0.83599999999999997},"min":{"1min":0.63900000000000001,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.3160000000000001,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":0.96399999999999997},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.89000000000000001,"15min":0.82399999999999995},"min":{"1min":0.73699999999999999,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.1890000000000001,"5min":1.1890000000000001,"15min":1.1890000000000001},"last":0.92200000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:46:10 2023","interfaces":[{"interface":"back","average":{"1min":1.1950000000000001,"5min":1.048,"15min":1.016},"min":{"1min":0.81999999999999995,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.3899999999999999,"5min":1.827,"15min":1.827},"last":0.78200000000000003},{"interface":"front","average":{"1min":1.1080000000000001,"5min":1.042,"15min":1.05},"min":{"1min":0.73299999999999998,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.536,"5min":1.9179999999999999,"15min":1.9179999999999999},"last":0.84199999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:46:10 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":1.0700000000000001,"15min":1.052},"min":{"1min":0.86899999999999999,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.472,"5min":1.472,"15min":1.472},"last":1.27},{"interface":"front","average":{"1min":1.091,"5min":1,"15min":0.96199999999999997},"min":{"1min":0.86199999999999999,"5min":0.505,"15min":0.505},"max":{"1min":1.359,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":1.141}]}]},{"osd":6,"up_from":46,"seq":197568495695,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298592,"kb_used_data":1304,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446560,"statfs":{"total":95995035648,"available":95689277440,"internally_reserved":0,"allocated":1335296,"data_stored":912562,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:46:27 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":0.88,"15min":0.94299999999999995},"min":{"1min":0.52200000000000002,"5min":0.437,"15min":0.437},"max":{"1min":1.3620000000000001,"5min":1.7669999999999999,"15min":1.7669999999999999},"last":0.88900000000000001},{"interface":"front","average":{"1min":0.876,"5min":0.86499999999999999,"15min":0.95099999999999996},"min":{"1min":0.59399999999999997,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.298,"5min":1.298,"15min":1.3149999999999999},"last":1.081}]},{"osd":1,"last update":"Mon Oct 9 04:46:27 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":0.874,"15min":0.90000000000000002},"min":{"1min":0.56100000000000005,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":1.2849999999999999,"5min":1.4850000000000001,"15min":1.4850000000000001},"last":0.93500000000000005},{"interface":"front","average":{"1min":0.94899999999999995,"5min":0.91600000000000004,"15min":0.92200000000000004},"min":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":1.2909999999999999,"5min":1.4059999999999999,"15min":1.4850000000000001},"last":1.056}]},{"osd":2,"last update":"Mon Oct 9 04:46:27 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":0.94199999999999995,"15min":0.94999999999999996},"min":{"1min":0.53600000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"max":{"1min":1.2689999999999999,"5min":3.238,"15min":3.238},"last":0.99199999999999999},{"interface":"front","average":{"1min":0.94799999999999995,"5min":0.91200000000000003,"15min":0.92000000000000004},"min":{"1min":0.68799999999999994,"5min":0.61799999999999999,"15min":0.61799999999999999},"max":{"1min":1.175,"5min":1.2170000000000001,"15min":1.238},"last":0.78800000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:46:27 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":0.95399999999999996,"15min":0.97999999999999998},"min":{"1min":0.60499999999999998,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.3560000000000001,"5min":1.8879999999999999,"15min":1.8879999999999999},"last":1.0189999999999999},{"interface":"front","average":{"1min":0.99099999999999999,"5min":0.97799999999999998,"15min":0.97599999999999998},"min":{"1min":0.70599999999999996,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.3089999999999999,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.90600000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:46:27 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":0.92200000000000004,"15min":0.82999999999999996},"min":{"1min":0.73799999999999999,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.198,"5min":1.4510000000000001,"15min":1.4510000000000001},"last":0.60799999999999998},{"interface":"front","average":{"1min":1.056,"5min":0.98899999999999999,"15min":0.93100000000000005},"min":{"1min":0.56899999999999995,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.3160000000000001,"5min":1.5249999999999999,"15min":1.5249999999999999},"last":0.73899999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:46:27 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":1.0009999999999999,"15min":0.92000000000000004},"min":{"1min":0.84799999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.306,"5min":1.397,"15min":1.397},"last":0.86199999999999999},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.048,"15min":0.98899999999999999},"min":{"1min":0.76100000000000001,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.296,"5min":1.6000000000000001,"15min":1.6000000000000001},"last":0.96399999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:46:13 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":1.0329999999999999,"15min":0.98299999999999998},"min":{"1min":0.77200000000000002,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.3500000000000001,"5min":1.452,"15min":1.452},"last":0.83899999999999997},{"interface":"front","average":{"1min":1.22,"5min":1.1699999999999999,"15min":1.1200000000000001},"min":{"1min":1.0269999999999999,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.393,"5min":1.5660000000000001,"15min":1.5660000000000001},"last":1.1040000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574980,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298000,"kb_used_data":712,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447152,"statfs":{"total":95995035648,"available":95689883648,"internally_reserved":0,"allocated":729088,"data_stored":320853,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:46:09 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.65000000000000002,"15min":0.61399999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.96899999999999997,"5min":1.179,"15min":1.179},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.69999999999999996,"5min":0.73699999999999999,"15min":0.66900000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":1.0269999999999999,"5min":1.2170000000000001,"15min":1.2170000000000001},"last":0.73099999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:45:52 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":0.72999999999999998,"15min":0.69499999999999995},"min":{"1min":0.442,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.0740000000000001,"5min":1.484,"15min":1.484},"last":0.84599999999999997},{"interface":"front","average":{"1min":0.78600000000000003,"5min":0.746,"15min":0.69599999999999995},"min":{"1min":0.45900000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.0609999999999999,"5min":1.371,"15min":1.371},"last":0.78100000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:45:52 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":0.78700000000000003,"15min":0.80700000000000005},"min":{"1min":0.51200000000000001,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.069,"5min":1.222,"15min":1.2330000000000001},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.873,"5min":0.79400000000000004,"15min":0.78600000000000003},"min":{"1min":0.41399999999999998,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.1559999999999999,"5min":1.3029999999999999,"15min":1.3029999999999999},"last":1.004}]},{"osd":4,"last update":"Mon Oct 9 04:45:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.0109999999999999,"15min":0.98099999999999998},"min":{"1min":0.82699999999999996,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.2230000000000001,"5min":1.607,"15min":1.607},"last":0.86299999999999999},{"interface":"front","average":{"1min":1.01,"5min":1.0209999999999999,"15min":0.98899999999999999},"min":{"1min":0.76500000000000001,"5min":0.68200000000000005,"15min":0.68200000000000005},"max":{"1min":1.488,"5min":1.488,"15min":1.488},"last":1.2629999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:46:31 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.95399999999999996,"15min":0.98999999999999999},"min":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.339,"5min":1.55,"15min":1.55},"last":1.339},{"interface":"front","average":{"1min":0.97399999999999998,"5min":1.02,"15min":1.01},"min":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.4390000000000001,"5min":1.5509999999999999,"15min":1.5509999999999999},"last":1.0569999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:46:31 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.0469999999999999,"15min":1.0589999999999999},"min":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.5449999999999999,"5min":1.639,"15min":1.639},"last":0.97199999999999998},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.0589999999999999,"15min":1.081},"min":{"1min":0.627,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.373,"5min":1.552,"15min":1.552},"last":1.0820000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:46:14 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":1.1120000000000001,"15min":1.145},"min":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.5,"5min":1.758,"15min":1.758},"last":1.117},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.121,"15min":1.165},"min":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.325,"5min":1.6899999999999999,"15min":1.6899999999999999},"last":1.236}]}]},{"osd":0,"up_from":8,"seq":34359738510,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298588,"kb_used_data":1300,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446564,"statfs":{"total":95995035648,"available":95689281536,"internally_reserved":0,"allocated":1331200,"data_stored":911797,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:46:09 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.70799999999999996,"15min":0.621},"min":{"1min":0.36499999999999999,"5min":0.35499999999999998,"15min":0.26900000000000002},"max":{"1min":0.875,"5min":1.3240000000000001,"15min":1.3240000000000001},"last":0.80000000000000004},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.72899999999999998,"15min":0.66100000000000003},"min":{"1min":0.44500000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.002,"5min":1.224,"15min":1.224},"last":0.76400000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.79300000000000004,"5min":0.76800000000000002,"15min":0.69399999999999995},"min":{"1min":0.56100000000000005,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.1819999999999999,"5min":1.319,"15min":1.319},"last":1.169},{"interface":"front","average":{"1min":0.82899999999999996,"5min":0.82099999999999995,"15min":0.71899999999999997},"min":{"1min":0.54200000000000004,"5min":0.40000000000000002,"15min":0.34499999999999997},"max":{"1min":1.2589999999999999,"5min":1.2789999999999999,"15min":1.2789999999999999},"last":1.1930000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:45:48 2023","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":0.879,"15min":0.84599999999999997},"min":{"1min":0.51200000000000001,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.361,"5min":1.4299999999999999,"15min":1.4299999999999999},"last":0.99399999999999999},{"interface":"front","average":{"1min":0.84199999999999997,"5min":0.79300000000000004,"15min":0.79800000000000004},"min":{"1min":0.502,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.2090000000000001,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":0.72699999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:45:48 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":0.94999999999999996,"15min":0.93799999999999994},"min":{"1min":0.752,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.272,"5min":1.673,"15min":1.673},"last":1.2290000000000001},{"interface":"front","average":{"1min":1,"5min":0.98599999999999999,"15min":0.95499999999999996},"min":{"1min":0.72199999999999998,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.4199999999999999,"5min":1.4199999999999999,"15min":1.4199999999999999},"last":1.0580000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:46:34 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":1.0489999999999999,"15min":1.048},"min":{"1min":0.70199999999999996,"5min":0.64000000000000001,"15min":0.64000000000000001},"max":{"1min":1.23,"5min":2.0219999999999998,"15min":2.0219999999999998},"last":1.107},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.97099999999999997,"15min":1},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":1.129,"5min":1.623,"15min":1.623},"last":1.129}]},{"osd":6,"last update":"Mon Oct 9 04:46:28 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":1.0189999999999999,"15min":1.073},"min":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.177,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":1.2609999999999999},{"interface":"front","average":{"1min":1.002,"5min":1.105,"15min":1.1439999999999999},"min":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"max":{"1min":1.3080000000000001,"5min":1.7330000000000001,"15min":1.7330000000000001},"last":1.2869999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:46:17 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":1.0660000000000001,"15min":1.1200000000000001},"min":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"max":{"1min":1.3380000000000001,"5min":1.601,"15min":1.601},"last":0.91000000000000003},{"interface":"front","average":{"1min":1.101,"5min":1.1499999999999999,"15min":1.173},"min":{"1min":0.77200000000000002,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":1.446,"5min":2.097,"15min":2.097},"last":1.339}]}]},{"osd":2,"up_from":18,"seq":77309411450,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298008,"kb_used_data":720,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447144,"statfs":{"total":95995035648,"available":95689875456,"internally_reserved":0,"allocated":737280,"data_stored":321441,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:46:01 2023","interfaces":[{"interface":"back","average":{"1min":0.69299999999999995,"5min":0.70399999999999996,"15min":0.67400000000000004},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.25,"5min":1.25,"15min":1.25},"last":0.91700000000000004},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.77200000000000002,"15min":0.71099999999999997},"min":{"1min":0.434,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.319,"5min":1.319,"15min":1.319},"last":0.625}]},{"osd":1,"last update":"Mon Oct 9 04:46:01 2023","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":0.77400000000000002,"15min":0.70899999999999996},"min":{"1min":0.44800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.425,"5min":1.425,"15min":1.425},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.76600000000000001,"5min":0.78800000000000003,"15min":0.72899999999999998},"min":{"1min":0.433,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.5580000000000001,"5min":1.5580000000000001,"15min":1.5580000000000001},"last":0.73099999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:45:50 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":0.82899999999999996,"15min":0.81000000000000005},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.5209999999999999,"5min":1.5209999999999999,"15min":1.5209999999999999},"last":0.86799999999999999},{"interface":"front","average":{"1min":0.86499999999999999,"5min":0.871,"15min":0.85499999999999998},"min":{"1min":0.498,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.462,"5min":1.462,"15min":1.462},"last":1.2669999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:45:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.032,"15min":0.97299999999999998},"min":{"1min":0.70699999999999996,"5min":0.64000000000000001,"15min":0.64000000000000001},"max":{"1min":1.597,"5min":1.72,"15min":1.72},"last":1.1619999999999999},{"interface":"front","average":{"1min":1.1539999999999999,"5min":1.0609999999999999,"15min":1.02},"min":{"1min":0.73899999999999999,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":2.1179999999999999,"5min":2.1179999999999999,"15min":2.1179999999999999},"last":1.1020000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:45:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":1.087,"15min":1.077},"min":{"1min":0.78300000000000003,"5min":0.72399999999999998,"15min":0.72399999999999998},"max":{"1min":2.0350000000000001,"5min":2.0350000000000001,"15min":2.0350000000000001},"last":1.002},{"interface":"front","average":{"1min":1.129,"5min":1.081,"15min":1.0629999999999999},"min":{"1min":0.81100000000000005,"5min":0.60599999999999998,"15min":0.60599999999999998},"max":{"1min":1.702,"5min":1.702,"15min":1.702},"last":1.2250000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:46:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.0800000000000001,"15min":1.145},"min":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"max":{"1min":2.1920000000000002,"5min":2.1920000000000002,"15min":2.1920000000000002},"last":1.1879999999999999},{"interface":"front","average":{"1min":1.022,"5min":0.98699999999999999,"15min":1.0489999999999999},"min":{"1min":0.63700000000000001,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.7729999999999999,"5min":1.7729999999999999,"15min":1.7729999999999999},"last":1.1379999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:46:13 2023","interfaces":[{"interface":"back","average":{"1min":1.214,"5min":1.163,"15min":1.2250000000000001},"min":{"1min":0.80000000000000004,"5min":0.622,"15min":0.622},"max":{"1min":1.796,"5min":1.796,"15min":1.796},"last":1.3069999999999999},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.1899999999999999,"15min":1.232},"min":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"max":{"1min":2.2639999999999998,"5min":2.2639999999999998,"15min":2.2639999999999998},"last":1.3340000000000001}]}]},{"osd":3,"up_from":25,"seq":107374182510,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298004,"kb_used_data":716,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447148,"statfs":{"total":95995035648,"available":95689879552,"internally_reserved":0,"allocated":733184,"data_stored":320943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:45:55 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.69599999999999995,"15min":0.69699999999999995},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":1.0820000000000001,"5min":1.091,"15min":1.091},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.75,"5min":0.746,"15min":0.73699999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.0589999999999999,"5min":1.266,"15min":1.266},"last":0.96699999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:45:55 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":0.75600000000000001,"15min":0.752},"min":{"1min":0.53000000000000003,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.94899999999999995,"5min":1.1819999999999999,"15min":1.1819999999999999},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.76600000000000001,"5min":0.75800000000000001,"15min":0.76100000000000001},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.022,"5min":1.7250000000000001,"15min":1.7250000000000001},"last":0.66800000000000004}]},{"osd":2,"last update":"Mon Oct 9 04:45:55 2023","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":0.84899999999999998,"15min":0.85699999999999998},"min":{"1min":0.54900000000000004,"5min":0.437,"15min":0.437},"max":{"1min":1.1759999999999999,"5min":1.8320000000000001,"15min":1.8320000000000001},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.73399999999999999,"5min":0.77600000000000002,"15min":0.80400000000000005},"min":{"1min":0.437,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.92100000000000004,"5min":1.173,"15min":1.218},"last":0.78300000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":0.98099999999999998,"15min":0.95199999999999996},"min":{"1min":0.73899999999999999,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.258,"5min":1.5009999999999999,"15min":1.5009999999999999},"last":0.91200000000000003},{"interface":"front","average":{"1min":1.0700000000000001,"5min":1.0569999999999999,"15min":0.97299999999999998},"min":{"1min":0.83599999999999997,"5min":0.67400000000000004,"15min":0.56499999999999995},"max":{"1min":1.292,"5min":2.298,"15min":2.298},"last":1.2470000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:45:35 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":1.032,"15min":1.0269999999999999},"min":{"1min":0.95499999999999996,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.2509999999999999,"5min":1.4399999999999999,"15min":1.603},"last":0.84299999999999997},{"interface":"front","average":{"1min":1.1359999999999999,"5min":1.079,"15min":1.071},"min":{"1min":0.94199999999999995,"5min":0.65000000000000002,"15min":0.65000000000000002},"max":{"1min":1.3839999999999999,"5min":1.9930000000000001,"15min":1.9930000000000001},"last":1.2110000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:46:21 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":1.0649999999999999,"15min":1.087},"min":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.3420000000000001,"5min":1.621,"15min":2.1030000000000002},"last":1.173},{"interface":"front","average":{"1min":0.97999999999999998,"5min":1.048,"15min":1.0760000000000001},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.387,"5min":1.4039999999999999,"15min":1.4039999999999999},"last":1.448}]},{"osd":7,"last update":"Mon Oct 9 04:46:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":1.091,"15min":1.161},"min":{"1min":0.71399999999999997,"5min":0.70199999999999996,"15min":0.70199999999999996},"max":{"1min":1.3380000000000001,"5min":1.367,"15min":1.367},"last":1.113},{"interface":"front","average":{"1min":1.139,"5min":1.1579999999999999,"15min":1.212},"min":{"1min":0.79500000000000004,"5min":0.76200000000000001,"15min":0.76200000000000001},"max":{"1min":1.407,"5min":1.534,"15min":1.534},"last":1.28}]}]},{"osd":4,"up_from":30,"seq":128849018981,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298024,"kb_used_data":736,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447128,"statfs":{"total":95995035648,"available":95689859072,"internally_reserved":0,"allocated":753664,"data_stored":322535,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":0.91300000000000003,"15min":0.86799999999999999},"min":{"1min":0.63300000000000001,"5min":0.54500000000000004,"15min":0.505},"max":{"1min":1.1850000000000001,"5min":1.294,"15min":1.294},"last":1.321},{"interface":"front","average":{"1min":0.98399999999999999,"5min":0.91700000000000004,"15min":0.90700000000000003},"min":{"1min":0.71699999999999997,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":1.2470000000000001,"5min":1.4039999999999999,"15min":1.8180000000000001},"last":0.997}]},{"osd":1,"last update":"Mon Oct 9 04:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":0.90800000000000003,"15min":0.86499999999999999},"min":{"1min":0.68799999999999994,"5min":0.504,"15min":0.504},"max":{"1min":1.125,"5min":1.4239999999999999,"15min":1.4239999999999999},"last":1.1120000000000001},{"interface":"front","average":{"1min":0.999,"5min":0.94599999999999995,"15min":0.90800000000000003},"min":{"1min":0.59599999999999997,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.3029999999999999,"5min":1.4099999999999999,"15min":1.9039999999999999},"last":1.2569999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":0.96299999999999997,"15min":0.92500000000000004},"min":{"1min":0.67900000000000005,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.2090000000000001,"5min":1.548,"15min":1.724},"last":1.147},{"interface":"front","average":{"1min":0.98799999999999999,"5min":0.92200000000000004,"15min":0.90300000000000002},"min":{"1min":0.80000000000000004,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.2270000000000001,"5min":1.23,"15min":1.25},"last":0.82799999999999996}]},{"osd":3,"last update":"Mon Oct 9 04:45:45 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":0.96199999999999997,"15min":0.92400000000000004},"min":{"1min":0.74199999999999999,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.3240000000000001,"5min":1.369,"15min":1.369},"last":1.1839999999999999},{"interface":"front","average":{"1min":1.0109999999999999,"5min":0.96799999999999997,"15min":0.92700000000000005},"min":{"1min":0.81899999999999995,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.202,"5min":1.377,"15min":1.377},"last":1.2210000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:45:37 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":0.91300000000000003,"15min":0.79800000000000004},"min":{"1min":0.70399999999999996,"5min":0.47499999999999998,"15min":0.45100000000000001},"max":{"1min":1.4750000000000001,"5min":1.4750000000000001,"15min":1.4750000000000001},"last":0.88600000000000001},{"interface":"front","average":{"1min":1.04,"5min":0.95999999999999996,"15min":0.81200000000000006},"min":{"1min":0.71699999999999997,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.294,"5min":1.6539999999999999,"15min":1.6539999999999999},"last":1.286}]},{"osd":6,"last update":"Mon Oct 9 04:46:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":1.004,"15min":0.83599999999999997},"min":{"1min":0.67200000000000004,"5min":0.44500000000000001,"15min":0.42599999999999999},"max":{"1min":1.4330000000000001,"5min":1.474,"15min":1.474},"last":1.397},{"interface":"front","average":{"1min":1.1459999999999999,"5min":1.0860000000000001,"15min":0.996},"min":{"1min":0.60199999999999998,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.3979999999999999,"5min":1.9299999999999999,"15min":1.9850000000000001},"last":1.0760000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:46:12 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":1.0660000000000001,"15min":1},"min":{"1min":0.81499999999999995,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.5009999999999999,"5min":1.752,"15min":1.752},"last":1.3620000000000001},{"interface":"front","average":{"1min":1.1819999999999999,"5min":1.1439999999999999,"15min":1.0580000000000001},"min":{"1min":1.0249999999999999,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.397,"5min":1.6180000000000001,"15min":1.6180000000000001},"last":1.444}]}]},{"osd":5,"up_from":38,"seq":163208757337,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298008,"kb_used_data":720,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447144,"statfs":{"total":95995035648,"available":95689875456,"internally_reserved":0,"allocated":737280,"data_stored":321477,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:46:32 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":0.89600000000000002,"15min":0.877},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":1.254,"5min":1.3720000000000001,"15min":1.3720000000000001},"last":1.115},{"interface":"front","average":{"1min":0.90700000000000003,"5min":0.92400000000000004,"15min":0.93100000000000005},"min":{"1min":0.52800000000000002,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.3320000000000001,"5min":1.389,"15min":1.7869999999999999},"last":1.157}]},{"osd":1,"last update":"Mon Oct 9 04:46:32 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":0.89800000000000002,"15min":0.93999999999999995},"min":{"1min":0.51500000000000001,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.2330000000000001,"5min":1.399,"15min":2.0510000000000002},"last":1.2230000000000001},{"interface":"front","average":{"1min":0.84699999999999998,"5min":0.90200000000000002,"15min":0.89000000000000001},"min":{"1min":0.61799999999999999,"5min":0.57899999999999996,"15min":0.57899999999999996},"max":{"1min":1.2010000000000001,"5min":1.5529999999999999,"15min":1.5529999999999999},"last":0.90100000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:46:32 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":0.90600000000000003,"15min":0.86799999999999999},"min":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.505},"max":{"1min":1.1839999999999999,"5min":1.204,"15min":1.204},"last":1.198},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.997,"15min":0.97199999999999998},"min":{"1min":0.69699999999999995,"5min":0.67000000000000004,"15min":0.60599999999999998},"max":{"1min":1.4099999999999999,"5min":1.446,"15min":1.885},"last":1.0229999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:46:32 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":0.97399999999999998,"15min":0.93300000000000005},"min":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.56299999999999994},"max":{"1min":1.3700000000000001,"5min":1.3700000000000001,"15min":1.3700000000000001},"last":1.26},{"interface":"front","average":{"1min":1.1240000000000001,"5min":1.05,"15min":1.0149999999999999},"min":{"1min":0.84499999999999997,"5min":0.61099999999999999,"15min":0.61099999999999999},"max":{"1min":1.3859999999999999,"5min":1.3859999999999999,"15min":1.9710000000000001},"last":0.80700000000000005}]},{"osd":4,"last update":"Mon Oct 9 04:46:32 2023","interfaces":[{"interface":"back","average":{"1min":0.94799999999999995,"5min":0.879,"15min":0.80100000000000005},"min":{"1min":0.72399999999999998,"5min":0.5,"15min":0.5},"max":{"1min":1.351,"5min":1.351,"15min":1.3759999999999999},"last":1.0860000000000001},{"interface":"front","average":{"1min":1.171,"5min":1.022,"15min":0.88200000000000001},"min":{"1min":0.83799999999999997,"5min":0.52500000000000002,"15min":0.36799999999999999},"max":{"1min":1.3740000000000001,"5min":1.3999999999999999,"15min":1.5269999999999999},"last":1.45}]},{"osd":6,"last update":"Mon Oct 9 04:46:26 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.042,"15min":0.93899999999999995},"min":{"1min":0.76500000000000001,"5min":0.63300000000000001,"15min":0.45700000000000002},"max":{"1min":1.4299999999999999,"5min":1.4670000000000001,"15min":1.4670000000000001},"last":1.3959999999999999},{"interface":"front","average":{"1min":1.2,"5min":1.0609999999999999,"15min":1.052},"min":{"1min":0.90000000000000002,"5min":0.55500000000000005,"15min":0.49299999999999999},"max":{"1min":1.5209999999999999,"5min":1.5209999999999999,"15min":2.169},"last":1.3720000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:46:10 2023","interfaces":[{"interface":"back","average":{"1min":1.1539999999999999,"5min":1.0980000000000001,"15min":1.0529999999999999},"min":{"1min":0.78400000000000003,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.431,"5min":1.5049999999999999,"15min":1.5049999999999999},"last":1.296},{"interface":"front","average":{"1min":1.073,"5min":1.038,"15min":0.97499999999999998},"min":{"1min":0.70899999999999996,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.335,"5min":1.4710000000000001,"15min":1.4710000000000001},"last":1.3340000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:46:38.481 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:46:38.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:38 smithi133 bash[41291]: cluster 2023-10-09T04:46:37.164795+0000 mgr.y (mgr.14466) 174 : cluster [DBG] pgmap v151: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:38.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:38 smithi133 bash[41291]: audit 2023-10-09T04:46:37.573145+0000 mgr.y (mgr.14466) 175 : audit [DBG] from='client.14814 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:46:38.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:38 smithi133 bash[27558]: cluster 2023-10-09T04:46:37.164795+0000 mgr.y (mgr.14466) 174 : cluster [DBG] pgmap v151: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:38.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:38 smithi133 bash[27558]: audit 2023-10-09T04:46:37.573145+0000 mgr.y (mgr.14466) 175 : audit [DBG] from='client.14814 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:46:38.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:38 smithi194 bash[31648]: cluster 2023-10-09T04:46:37.164795+0000 mgr.y (mgr.14466) 174 : cluster [DBG] pgmap v151: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:38.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:38 smithi194 bash[31648]: audit 2023-10-09T04:46:37.573145+0000 mgr.y (mgr.14466) 175 : audit [DBG] from='client.14814 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:46:40.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:40 smithi133 bash[41291]: cluster 2023-10-09T04:46:39.165438+0000 mgr.y (mgr.14466) 176 : cluster [DBG] pgmap v152: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:40.801 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:46:40 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:46:40] "GET /metrics HTTP/1.1" 200 238935 "" "Prometheus/2.43.0" 2023-10-09T04:46:40.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:40 smithi133 bash[27558]: cluster 2023-10-09T04:46:39.165438+0000 mgr.y (mgr.14466) 176 : cluster [DBG] pgmap v152: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:40.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:40 smithi194 bash[31648]: cluster 2023-10-09T04:46:39.165438+0000 mgr.y (mgr.14466) 176 : cluster [DBG] pgmap v152: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:42.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:42 smithi133 bash[41291]: cluster 2023-10-09T04:46:41.166424+0000 mgr.y (mgr.14466) 177 : cluster [DBG] pgmap v153: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:42.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:42 smithi133 bash[27558]: cluster 2023-10-09T04:46:41.166424+0000 mgr.y (mgr.14466) 177 : cluster [DBG] pgmap v153: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:42.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:42 smithi194 bash[31648]: cluster 2023-10-09T04:46:41.166424+0000 mgr.y (mgr.14466) 177 : cluster [DBG] pgmap v153: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:43.275 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:46:44.755 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:44 smithi133 bash[41291]: cluster 2023-10-09T04:46:43.166950+0000 mgr.y (mgr.14466) 178 : cluster [DBG] pgmap v154: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:44.755 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:44 smithi133 bash[27558]: cluster 2023-10-09T04:46:43.166950+0000 mgr.y (mgr.14466) 178 : cluster [DBG] pgmap v154: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:44.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:44 smithi194 bash[31648]: cluster 2023-10-09T04:46:43.166950+0000 mgr.y (mgr.14466) 178 : cluster [DBG] pgmap v154: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:46.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:46 smithi194 bash[31648]: cluster 2023-10-09T04:46:45.167586+0000 mgr.y (mgr.14466) 179 : cluster [DBG] pgmap v155: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:46.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:46:46 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:46:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:46:46.687 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T04:46:46.691 INFO:teuthology.orchestra.run.smithi133.stderr:dumped all 2023-10-09T04:46:46.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:46 smithi133 bash[27558]: cluster 2023-10-09T04:46:45.167586+0000 mgr.y (mgr.14466) 179 : cluster [DBG] pgmap v155: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:46.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:46 smithi133 bash[41291]: cluster 2023-10-09T04:46:45.167586+0000 mgr.y (mgr.14466) 179 : cluster [DBG] pgmap v155: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:47.581 INFO:teuthology.orchestra.run.smithi133.stdout:{"pg_ready":true,"pg_map":{"version":155,"stamp":"2023-10-09T04:46:45.167252+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":259,"num_read_kb":355,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":173,"ondisk_log_size":173,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385824,"kb_used_data":7520,"kb_used_omap":0,"kb_used_meta":2378240,"kb_avail":747575392,"statfs":{"total":767960285184,"available":765517201408,"internally_reserved":0,"allocated":7700480,"data_stored":4343993,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2435317760},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004354"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.256545+0000","last_change":"2023-10-09T04:44:53.792424+0000","last_active":"2023-10-09T04:45:06.256545+0000","last_peered":"2023-10-09T04:45:06.256545+0000","last_clean":"2023-10-09T04:45:06.256545+0000","last_became_active":"2023-10-09T04:44:53.792104+0000","last_became_peered":"2023-10-09T04:44:53.792104+0000","last_unstale":"2023-10-09T04:45:06.256545+0000","last_undegraded":"2023-10-09T04:45:06.256545+0000","last_fullsized":"2023-10-09T04:45:06.256545+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:20:29.387710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'5","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.120440+0000","last_change":"2023-10-09T04:44:52.788322+0000","last_active":"2023-10-09T04:45:06.120440+0000","last_peered":"2023-10-09T04:45:06.120440+0000","last_clean":"2023-10-09T04:45:06.120440+0000","last_became_active":"2023-10-09T04:44:52.787420+0000","last_became_peered":"2023-10-09T04:44:52.787420+0000","last_unstale":"2023-10-09T04:45:06.120440+0000","last_undegraded":"2023-10-09T04:45:06.120440+0000","last_fullsized":"2023-10-09T04:45:06.120440+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:32:54.717562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.112102+0000","last_change":"2023-10-09T04:44:53.792437+0000","last_active":"2023-10-09T04:45:06.112102+0000","last_peered":"2023-10-09T04:45:06.112102+0000","last_clean":"2023-10-09T04:45:06.112102+0000","last_became_active":"2023-10-09T04:44:53.792094+0000","last_became_peered":"2023-10-09T04:44:53.792094+0000","last_unstale":"2023-10-09T04:45:06.112102+0000","last_undegraded":"2023-10-09T04:45:06.112102+0000","last_fullsized":"2023-10-09T04:45:06.112102+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:18.147611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'2","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.113009+0000","last_change":"2023-10-09T04:44:52.794030+0000","last_active":"2023-10-09T04:45:06.113009+0000","last_peered":"2023-10-09T04:45:06.113009+0000","last_clean":"2023-10-09T04:45:06.113009+0000","last_became_active":"2023-10-09T04:44:52.793602+0000","last_became_peered":"2023-10-09T04:44:52.793602+0000","last_unstale":"2023-10-09T04:45:06.113009+0000","last_undegraded":"2023-10-09T04:45:06.113009+0000","last_fullsized":"2023-10-09T04:45:06.113009+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:51.223306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.257526+0000","last_change":"2023-10-09T04:44:53.794733+0000","last_active":"2023-10-09T04:45:06.257526+0000","last_peered":"2023-10-09T04:45:06.257526+0000","last_clean":"2023-10-09T04:45:06.257526+0000","last_became_active":"2023-10-09T04:44:53.794004+0000","last_became_peered":"2023-10-09T04:44:53.794004+0000","last_unstale":"2023-10-09T04:45:06.257526+0000","last_undegraded":"2023-10-09T04:45:06.257526+0000","last_fullsized":"2023-10-09T04:45:06.257526+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:33:24.499774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.118231+0000","last_change":"2023-10-09T04:44:52.793709+0000","last_active":"2023-10-09T04:45:06.118231+0000","last_peered":"2023-10-09T04:45:06.118231+0000","last_clean":"2023-10-09T04:45:06.118231+0000","last_became_active":"2023-10-09T04:44:52.792973+0000","last_became_peered":"2023-10-09T04:44:52.792973+0000","last_unstale":"2023-10-09T04:45:06.118231+0000","last_undegraded":"2023-10-09T04:45:06.118231+0000","last_fullsized":"2023-10-09T04:45:06.118231+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:12.091089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.257448+0000","last_change":"2023-10-09T04:44:53.791051+0000","last_active":"2023-10-09T04:45:06.257448+0000","last_peered":"2023-10-09T04:45:06.257448+0000","last_clean":"2023-10-09T04:45:06.257448+0000","last_became_active":"2023-10-09T04:44:53.790741+0000","last_became_peered":"2023-10-09T04:44:53.790741+0000","last_unstale":"2023-10-09T04:45:06.257448+0000","last_undegraded":"2023-10-09T04:45:06.257448+0000","last_fullsized":"2023-10-09T04:45:06.257448+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:10:12.019609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.118016+0000","last_change":"2023-10-09T04:44:52.794672+0000","last_active":"2023-10-09T04:45:06.118016+0000","last_peered":"2023-10-09T04:45:06.118016+0000","last_clean":"2023-10-09T04:45:06.118016+0000","last_became_active":"2023-10-09T04:44:52.793285+0000","last_became_peered":"2023-10-09T04:44:52.793285+0000","last_unstale":"2023-10-09T04:45:06.118016+0000","last_undegraded":"2023-10-09T04:45:06.118016+0000","last_fullsized":"2023-10-09T04:45:06.118016+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:16.691224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.116282+0000","last_change":"2023-10-09T04:44:53.790672+0000","last_active":"2023-10-09T04:45:06.116282+0000","last_peered":"2023-10-09T04:45:06.116282+0000","last_clean":"2023-10-09T04:45:06.116282+0000","last_became_active":"2023-10-09T04:44:53.789696+0000","last_became_peered":"2023-10-09T04:44:53.789696+0000","last_unstale":"2023-10-09T04:45:06.116282+0000","last_undegraded":"2023-10-09T04:45:06.116282+0000","last_fullsized":"2023-10-09T04:45:06.116282+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:27.776552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'2","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.119784+0000","last_change":"2023-10-09T04:44:52.776358+0000","last_active":"2023-10-09T04:45:06.119784+0000","last_peered":"2023-10-09T04:45:06.119784+0000","last_clean":"2023-10-09T04:45:06.119784+0000","last_became_active":"2023-10-09T04:44:52.775894+0000","last_became_peered":"2023-10-09T04:44:52.775894+0000","last_unstale":"2023-10-09T04:45:06.119784+0000","last_undegraded":"2023-10-09T04:45:06.119784+0000","last_fullsized":"2023-10-09T04:45:06.119784+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:05:37.925341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.257215+0000","last_change":"2023-10-09T04:44:53.792412+0000","last_active":"2023-10-09T04:45:06.257215+0000","last_peered":"2023-10-09T04:45:06.257215+0000","last_clean":"2023-10-09T04:45:06.257215+0000","last_became_active":"2023-10-09T04:44:53.792009+0000","last_became_peered":"2023-10-09T04:44:53.792009+0000","last_unstale":"2023-10-09T04:45:06.257215+0000","last_undegraded":"2023-10-09T04:45:06.257215+0000","last_fullsized":"2023-10-09T04:45:06.257215+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:39.423339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.120188+0000","last_change":"2023-10-09T04:44:52.774852+0000","last_active":"2023-10-09T04:45:06.120188+0000","last_peered":"2023-10-09T04:45:06.120188+0000","last_clean":"2023-10-09T04:45:06.120188+0000","last_became_active":"2023-10-09T04:44:52.774554+0000","last_became_peered":"2023-10-09T04:44:52.774554+0000","last_unstale":"2023-10-09T04:45:06.120188+0000","last_undegraded":"2023-10-09T04:45:06.120188+0000","last_fullsized":"2023-10-09T04:45:06.120188+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:53.658339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.115098+0000","last_change":"2023-10-09T04:42:48.699613+0000","last_active":"2023-10-09T04:45:06.115098+0000","last_peered":"2023-10-09T04:45:06.115098+0000","last_clean":"2023-10-09T04:45:06.115098+0000","last_became_active":"2023-10-09T04:42:48.699095+0000","last_became_peered":"2023-10-09T04:42:48.699095+0000","last_unstale":"2023-10-09T04:45:06.115098+0000","last_undegraded":"2023-10-09T04:45:06.115098+0000","last_fullsized":"2023-10-09T04:45:06.115098+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:57.871245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.112013+0000","last_change":"2023-10-09T04:44:53.958823+0000","last_active":"2023-10-09T04:45:06.112013+0000","last_peered":"2023-10-09T04:45:06.112013+0000","last_clean":"2023-10-09T04:45:06.112013+0000","last_became_active":"2023-10-09T04:44:53.957881+0000","last_became_peered":"2023-10-09T04:44:53.957881+0000","last_unstale":"2023-10-09T04:45:06.112013+0000","last_undegraded":"2023-10-09T04:45:06.112013+0000","last_fullsized":"2023-10-09T04:45:06.112013+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:04.731794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.118724+0000","last_change":"2023-10-09T04:42:48.849748+0000","last_active":"2023-10-09T04:45:06.118724+0000","last_peered":"2023-10-09T04:45:06.118724+0000","last_clean":"2023-10-09T04:45:06.118724+0000","last_became_active":"2023-10-09T04:42:48.848834+0000","last_became_peered":"2023-10-09T04:42:48.848834+0000","last_unstale":"2023-10-09T04:45:06.118724+0000","last_undegraded":"2023-10-09T04:45:06.118724+0000","last_fullsized":"2023-10-09T04:45:06.118724+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:20.475654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.113208+0000","last_change":"2023-10-09T04:44:52.793364+0000","last_active":"2023-10-09T04:45:06.113208+0000","last_peered":"2023-10-09T04:45:06.113208+0000","last_clean":"2023-10-09T04:45:06.113208+0000","last_became_active":"2023-10-09T04:44:52.787968+0000","last_became_peered":"2023-10-09T04:44:52.787968+0000","last_unstale":"2023-10-09T04:45:06.113208+0000","last_undegraded":"2023-10-09T04:45:06.113208+0000","last_fullsized":"2023-10-09T04:45:06.113208+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:33.783576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.119888+0000","last_change":"2023-10-09T04:44:53.960925+0000","last_active":"2023-10-09T04:45:06.119888+0000","last_peered":"2023-10-09T04:45:06.119888+0000","last_clean":"2023-10-09T04:45:06.119888+0000","last_became_active":"2023-10-09T04:44:53.960514+0000","last_became_peered":"2023-10-09T04:44:53.960514+0000","last_unstale":"2023-10-09T04:45:06.119888+0000","last_undegraded":"2023-10-09T04:45:06.119888+0000","last_fullsized":"2023-10-09T04:45:06.119888+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:15.192183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.117937+0000","last_change":"2023-10-09T04:44:52.791414+0000","last_active":"2023-10-09T04:45:06.117937+0000","last_peered":"2023-10-09T04:45:06.117937+0000","last_clean":"2023-10-09T04:45:06.117937+0000","last_became_active":"2023-10-09T04:44:52.790819+0000","last_became_peered":"2023-10-09T04:44:52.790819+0000","last_unstale":"2023-10-09T04:45:06.117937+0000","last_undegraded":"2023-10-09T04:45:06.117937+0000","last_fullsized":"2023-10-09T04:45:06.117937+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:13.452987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.111829+0000","last_change":"2023-10-09T04:42:48.851447+0000","last_active":"2023-10-09T04:45:06.111829+0000","last_peered":"2023-10-09T04:45:06.111829+0000","last_clean":"2023-10-09T04:45:06.111829+0000","last_became_active":"2023-10-09T04:42:48.850687+0000","last_became_peered":"2023-10-09T04:42:48.850687+0000","last_unstale":"2023-10-09T04:45:06.111829+0000","last_undegraded":"2023-10-09T04:45:06.111829+0000","last_fullsized":"2023-10-09T04:45:06.111829+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:32:22.356405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.120288+0000","last_change":"2023-10-09T04:44:53.790281+0000","last_active":"2023-10-09T04:45:06.120288+0000","last_peered":"2023-10-09T04:45:06.120288+0000","last_clean":"2023-10-09T04:45:06.120288+0000","last_became_active":"2023-10-09T04:44:53.789959+0000","last_became_peered":"2023-10-09T04:44:53.789959+0000","last_unstale":"2023-10-09T04:45:06.120288+0000","last_undegraded":"2023-10-09T04:45:06.120288+0000","last_fullsized":"2023-10-09T04:45:06.120288+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:21.459393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.119581+0000","last_change":"2023-10-09T04:44:52.782233+0000","last_active":"2023-10-09T04:45:06.119581+0000","last_peered":"2023-10-09T04:45:06.119581+0000","last_clean":"2023-10-09T04:45:06.119581+0000","last_became_active":"2023-10-09T04:44:52.781789+0000","last_became_peered":"2023-10-09T04:44:52.781789+0000","last_unstale":"2023-10-09T04:45:06.119581+0000","last_undegraded":"2023-10-09T04:45:06.119581+0000","last_fullsized":"2023-10-09T04:45:06.119581+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:15:43.929566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.111397+0000","last_change":"2023-10-09T04:42:48.697850+0000","last_active":"2023-10-09T04:45:06.111397+0000","last_peered":"2023-10-09T04:45:06.111397+0000","last_clean":"2023-10-09T04:45:06.111397+0000","last_became_active":"2023-10-09T04:42:48.697520+0000","last_became_peered":"2023-10-09T04:42:48.697520+0000","last_unstale":"2023-10-09T04:45:06.111397+0000","last_undegraded":"2023-10-09T04:45:06.111397+0000","last_fullsized":"2023-10-09T04:45:06.111397+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:04.728265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.115056+0000","last_change":"2023-10-09T04:44:53.959705+0000","last_active":"2023-10-09T04:45:06.115056+0000","last_peered":"2023-10-09T04:45:06.115056+0000","last_clean":"2023-10-09T04:45:06.115056+0000","last_became_active":"2023-10-09T04:44:53.959092+0000","last_became_peered":"2023-10-09T04:44:53.959092+0000","last_unstale":"2023-10-09T04:45:06.115056+0000","last_undegraded":"2023-10-09T04:45:06.115056+0000","last_fullsized":"2023-10-09T04:45:06.115056+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:31:29.189897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.112362+0000","last_change":"2023-10-09T04:44:52.794247+0000","last_active":"2023-10-09T04:45:06.112362+0000","last_peered":"2023-10-09T04:45:06.112362+0000","last_clean":"2023-10-09T04:45:06.112362+0000","last_became_active":"2023-10-09T04:44:52.793941+0000","last_became_peered":"2023-10-09T04:44:52.793941+0000","last_unstale":"2023-10-09T04:45:06.112362+0000","last_undegraded":"2023-10-09T04:45:06.112362+0000","last_fullsized":"2023-10-09T04:45:06.112362+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:45.149949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":275,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.111400+0000","last_change":"2023-10-09T04:41:09.432397+0000","last_active":"2023-10-09T04:45:06.111400+0000","last_peered":"2023-10-09T04:45:06.111400+0000","last_clean":"2023-10-09T04:45:06.111400+0000","last_became_active":"2023-10-09T04:41:09.431675+0000","last_became_peered":"2023-10-09T04:41:09.431675+0000","last_unstale":"2023-10-09T04:45:06.111400+0000","last_undegraded":"2023-10-09T04:45:06.111400+0000","last_fullsized":"2023-10-09T04:45:06.111400+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_clean_scrub_stamp":"2023-10-09T04:36:44.261013+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:23.657333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":19,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.117287+0000","last_change":"2023-10-09T04:42:48.698570+0000","last_active":"2023-10-09T04:45:06.117287+0000","last_peered":"2023-10-09T04:45:06.117287+0000","last_clean":"2023-10-09T04:45:06.117287+0000","last_became_active":"2023-10-09T04:42:48.698098+0000","last_became_peered":"2023-10-09T04:42:48.698098+0000","last_unstale":"2023-10-09T04:45:06.117287+0000","last_undegraded":"2023-10-09T04:45:06.117287+0000","last_fullsized":"2023-10-09T04:45:06.117287+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:35.340926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":10,"reported_epoch":71,"state":"active+clean","last_fresh":"2023-10-09T04:45:07.121151+0000","last_change":"2023-10-09T04:45:07.121151+0000","last_active":"2023-10-09T04:45:07.121151+0000","last_peered":"2023-10-09T04:45:07.121151+0000","last_clean":"2023-10-09T04:45:07.121151+0000","last_became_active":"2023-10-09T04:45:07.120694+0000","last_became_peered":"2023-10-09T04:45:07.120694+0000","last_unstale":"2023-10-09T04:45:07.121151+0000","last_undegraded":"2023-10-09T04:45:07.121151+0000","last_fullsized":"2023-10-09T04:45:07.121151+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:10.567313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.115936+0000","last_change":"2023-10-09T04:44:52.790877+0000","last_active":"2023-10-09T04:45:06.115936+0000","last_peered":"2023-10-09T04:45:06.115936+0000","last_clean":"2023-10-09T04:45:06.115936+0000","last_became_active":"2023-10-09T04:44:52.790563+0000","last_became_peered":"2023-10-09T04:44:52.790563+0000","last_unstale":"2023-10-09T04:45:06.115936+0000","last_undegraded":"2023-10-09T04:45:06.115936+0000","last_fullsized":"2023-10-09T04:45:06.115936+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:40:33.821035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.116004+0000","last_change":"2023-10-09T04:42:48.728368+0000","last_active":"2023-10-09T04:45:06.116004+0000","last_peered":"2023-10-09T04:45:06.116004+0000","last_clean":"2023-10-09T04:45:06.116004+0000","last_became_active":"2023-10-09T04:42:48.727757+0000","last_became_peered":"2023-10-09T04:42:48.727757+0000","last_unstale":"2023-10-09T04:45:06.116004+0000","last_undegraded":"2023-10-09T04:45:06.116004+0000","last_fullsized":"2023-10-09T04:45:06.116004+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:21:02.404428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":10,"reported_epoch":71,"state":"active+clean","last_fresh":"2023-10-09T04:45:07.162112+0000","last_change":"2023-10-09T04:45:07.162112+0000","last_active":"2023-10-09T04:45:07.162112+0000","last_peered":"2023-10-09T04:45:07.162112+0000","last_clean":"2023-10-09T04:45:07.162112+0000","last_became_active":"2023-10-09T04:45:07.161385+0000","last_became_peered":"2023-10-09T04:45:07.161385+0000","last_unstale":"2023-10-09T04:45:07.162112+0000","last_undegraded":"2023-10-09T04:45:07.162112+0000","last_fullsized":"2023-10-09T04:45:07.162112+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:18.574454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.256813+0000","last_change":"2023-10-09T04:44:52.784769+0000","last_active":"2023-10-09T04:45:06.256813+0000","last_peered":"2023-10-09T04:45:06.256813+0000","last_clean":"2023-10-09T04:45:06.256813+0000","last_became_active":"2023-10-09T04:44:52.784374+0000","last_became_peered":"2023-10-09T04:44:52.784374+0000","last_unstale":"2023-10-09T04:45:06.256813+0000","last_undegraded":"2023-10-09T04:45:06.256813+0000","last_fullsized":"2023-10-09T04:45:06.256813+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:29.252800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":19,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.117288+0000","last_change":"2023-10-09T04:42:48.698230+0000","last_active":"2023-10-09T04:45:06.117288+0000","last_peered":"2023-10-09T04:45:06.117288+0000","last_clean":"2023-10-09T04:45:06.117288+0000","last_became_active":"2023-10-09T04:42:48.697694+0000","last_became_peered":"2023-10-09T04:42:48.697694+0000","last_unstale":"2023-10-09T04:45:06.117288+0000","last_undegraded":"2023-10-09T04:45:06.117288+0000","last_fullsized":"2023-10-09T04:45:06.117288+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:46:21.257527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.116232+0000","last_change":"2023-10-09T04:44:53.788563+0000","last_active":"2023-10-09T04:45:06.116232+0000","last_peered":"2023-10-09T04:45:06.116232+0000","last_clean":"2023-10-09T04:45:06.116232+0000","last_became_active":"2023-10-09T04:44:53.787983+0000","last_became_peered":"2023-10-09T04:44:53.787983+0000","last_unstale":"2023-10-09T04:45:06.116232+0000","last_undegraded":"2023-10-09T04:45:06.116232+0000","last_fullsized":"2023-10-09T04:45:06.116232+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:21:15.499148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.118121+0000","last_change":"2023-10-09T04:44:52.794966+0000","last_active":"2023-10-09T04:45:06.118121+0000","last_peered":"2023-10-09T04:45:06.118121+0000","last_clean":"2023-10-09T04:45:06.118121+0000","last_became_active":"2023-10-09T04:44:52.794640+0000","last_became_peered":"2023-10-09T04:44:52.794640+0000","last_unstale":"2023-10-09T04:45:06.118121+0000","last_undegraded":"2023-10-09T04:45:06.118121+0000","last_fullsized":"2023-10-09T04:45:06.118121+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:00:47.077423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.116766+0000","last_change":"2023-10-09T04:42:48.696990+0000","last_active":"2023-10-09T04:45:06.116766+0000","last_peered":"2023-10-09T04:45:06.116766+0000","last_clean":"2023-10-09T04:45:06.116766+0000","last_became_active":"2023-10-09T04:42:48.696505+0000","last_became_peered":"2023-10-09T04:42:48.696505+0000","last_unstale":"2023-10-09T04:45:06.116766+0000","last_undegraded":"2023-10-09T04:45:06.116766+0000","last_fullsized":"2023-10-09T04:45:06.116766+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:55.113687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.119644+0000","last_change":"2023-10-09T04:44:53.787727+0000","last_active":"2023-10-09T04:45:06.119644+0000","last_peered":"2023-10-09T04:45:06.119644+0000","last_clean":"2023-10-09T04:45:06.119644+0000","last_became_active":"2023-10-09T04:44:53.787387+0000","last_became_peered":"2023-10-09T04:44:53.787387+0000","last_unstale":"2023-10-09T04:45:06.119644+0000","last_undegraded":"2023-10-09T04:45:06.119644+0000","last_fullsized":"2023-10-09T04:45:06.119644+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:46:25.002228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.112067+0000","last_change":"2023-10-09T04:44:52.794598+0000","last_active":"2023-10-09T04:45:06.112067+0000","last_peered":"2023-10-09T04:45:06.112067+0000","last_clean":"2023-10-09T04:45:06.112067+0000","last_became_active":"2023-10-09T04:44:52.794048+0000","last_became_peered":"2023-10-09T04:44:52.794048+0000","last_unstale":"2023-10-09T04:45:06.112067+0000","last_undegraded":"2023-10-09T04:45:06.112067+0000","last_fullsized":"2023-10-09T04:45:06.112067+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:27.518572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.257584+0000","last_change":"2023-10-09T04:44:53.792760+0000","last_active":"2023-10-09T04:45:06.257584+0000","last_peered":"2023-10-09T04:45:06.257584+0000","last_clean":"2023-10-09T04:45:06.257584+0000","last_became_active":"2023-10-09T04:44:53.792031+0000","last_became_peered":"2023-10-09T04:44:53.792031+0000","last_unstale":"2023-10-09T04:45:06.257584+0000","last_undegraded":"2023-10-09T04:45:06.257584+0000","last_fullsized":"2023-10-09T04:45:06.257584+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:57.177135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.256830+0000","last_change":"2023-10-09T04:44:52.790067+0000","last_active":"2023-10-09T04:45:06.256830+0000","last_peered":"2023-10-09T04:45:06.256830+0000","last_clean":"2023-10-09T04:45:06.256830+0000","last_became_active":"2023-10-09T04:44:52.789682+0000","last_became_peered":"2023-10-09T04:44:52.789682+0000","last_unstale":"2023-10-09T04:45:06.256830+0000","last_undegraded":"2023-10-09T04:45:06.256830+0000","last_fullsized":"2023-10-09T04:45:06.256830+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:37:16.879187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.257666+0000","last_change":"2023-10-09T04:44:53.793305+0000","last_active":"2023-10-09T04:45:06.257666+0000","last_peered":"2023-10-09T04:45:06.257666+0000","last_clean":"2023-10-09T04:45:06.257666+0000","last_became_active":"2023-10-09T04:44:53.792984+0000","last_became_peered":"2023-10-09T04:44:53.792984+0000","last_unstale":"2023-10-09T04:45:06.257666+0000","last_undegraded":"2023-10-09T04:45:06.257666+0000","last_fullsized":"2023-10-09T04:45:06.257666+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:11.704709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.115432+0000","last_change":"2023-10-09T04:44:52.791331+0000","last_active":"2023-10-09T04:45:06.115432+0000","last_peered":"2023-10-09T04:45:06.115432+0000","last_clean":"2023-10-09T04:45:06.115432+0000","last_became_active":"2023-10-09T04:44:52.787485+0000","last_became_peered":"2023-10-09T04:44:52.787485+0000","last_unstale":"2023-10-09T04:45:06.115432+0000","last_undegraded":"2023-10-09T04:45:06.115432+0000","last_fullsized":"2023-10-09T04:45:06.115432+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:13.478903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.257427+0000","last_change":"2023-10-09T04:44:53.794136+0000","last_active":"2023-10-09T04:45:06.257427+0000","last_peered":"2023-10-09T04:45:06.257427+0000","last_clean":"2023-10-09T04:45:06.257427+0000","last_became_active":"2023-10-09T04:44:53.793829+0000","last_became_peered":"2023-10-09T04:44:53.793829+0000","last_unstale":"2023-10-09T04:45:06.257427+0000","last_undegraded":"2023-10-09T04:45:06.257427+0000","last_fullsized":"2023-10-09T04:45:06.257427+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:50.822987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'3","reported_seq":19,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.112571+0000","last_change":"2023-10-09T04:44:52.797846+0000","last_active":"2023-10-09T04:45:06.112571+0000","last_peered":"2023-10-09T04:45:06.112571+0000","last_clean":"2023-10-09T04:45:06.112571+0000","last_became_active":"2023-10-09T04:44:52.797486+0000","last_became_peered":"2023-10-09T04:44:52.797486+0000","last_unstale":"2023-10-09T04:45:06.112571+0000","last_undegraded":"2023-10-09T04:45:06.112571+0000","last_fullsized":"2023-10-09T04:45:06.112571+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:05.789058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.119683+0000","last_change":"2023-10-09T04:44:53.787406+0000","last_active":"2023-10-09T04:45:06.119683+0000","last_peered":"2023-10-09T04:45:06.119683+0000","last_clean":"2023-10-09T04:45:06.119683+0000","last_became_active":"2023-10-09T04:44:53.786682+0000","last_became_peered":"2023-10-09T04:44:53.786682+0000","last_unstale":"2023-10-09T04:45:06.119683+0000","last_undegraded":"2023-10-09T04:45:06.119683+0000","last_fullsized":"2023-10-09T04:45:06.119683+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:05.555795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.117949+0000","last_change":"2023-10-09T04:44:52.795694+0000","last_active":"2023-10-09T04:45:06.117949+0000","last_peered":"2023-10-09T04:45:06.117949+0000","last_clean":"2023-10-09T04:45:06.117949+0000","last_became_active":"2023-10-09T04:44:52.795230+0000","last_became_peered":"2023-10-09T04:44:52.795230+0000","last_unstale":"2023-10-09T04:45:06.117949+0000","last_undegraded":"2023-10-09T04:45:06.117949+0000","last_fullsized":"2023-10-09T04:45:06.117949+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:15.951253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":24,"reported_epoch":71,"state":"active+clean","last_fresh":"2023-10-09T04:45:07.162222+0000","last_change":"2023-10-09T04:45:07.162222+0000","last_active":"2023-10-09T04:45:07.162222+0000","last_peered":"2023-10-09T04:45:07.162222+0000","last_clean":"2023-10-09T04:45:07.162222+0000","last_became_active":"2023-10-09T04:45:07.161194+0000","last_became_peered":"2023-10-09T04:45:07.161194+0000","last_unstale":"2023-10-09T04:45:07.162222+0000","last_undegraded":"2023-10-09T04:45:07.162222+0000","last_fullsized":"2023-10-09T04:45:07.162222+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:26.794178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.111888+0000","last_change":"2023-10-09T04:44:52.787469+0000","last_active":"2023-10-09T04:45:06.111888+0000","last_peered":"2023-10-09T04:45:06.111888+0000","last_clean":"2023-10-09T04:45:06.111888+0000","last_became_active":"2023-10-09T04:44:52.787148+0000","last_became_peered":"2023-10-09T04:44:52.787148+0000","last_unstale":"2023-10-09T04:45:06.111888+0000","last_undegraded":"2023-10-09T04:45:06.111888+0000","last_fullsized":"2023-10-09T04:45:06.111888+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:12.469065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.256783+0000","last_change":"2023-10-09T04:44:53.790116+0000","last_active":"2023-10-09T04:45:06.256783+0000","last_peered":"2023-10-09T04:45:06.256783+0000","last_clean":"2023-10-09T04:45:06.256783+0000","last_became_active":"2023-10-09T04:44:53.789562+0000","last_became_peered":"2023-10-09T04:44:53.789562+0000","last_unstale":"2023-10-09T04:45:06.256783+0000","last_undegraded":"2023-10-09T04:45:06.256783+0000","last_fullsized":"2023-10-09T04:45:06.256783+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:07.107715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.112222+0000","last_change":"2023-10-09T04:44:52.780659+0000","last_active":"2023-10-09T04:45:06.112222+0000","last_peered":"2023-10-09T04:45:06.112222+0000","last_clean":"2023-10-09T04:45:06.112222+0000","last_became_active":"2023-10-09T04:44:52.780365+0000","last_became_peered":"2023-10-09T04:44:52.780365+0000","last_unstale":"2023-10-09T04:45:06.112222+0000","last_undegraded":"2023-10-09T04:45:06.112222+0000","last_fullsized":"2023-10-09T04:45:06.112222+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:59.540930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.117203+0000","last_change":"2023-10-09T04:44:53.962437+0000","last_active":"2023-10-09T04:45:06.117203+0000","last_peered":"2023-10-09T04:45:06.117203+0000","last_clean":"2023-10-09T04:45:06.117203+0000","last_became_active":"2023-10-09T04:44:53.961810+0000","last_became_peered":"2023-10-09T04:44:53.961810+0000","last_unstale":"2023-10-09T04:45:06.117203+0000","last_undegraded":"2023-10-09T04:45:06.117203+0000","last_fullsized":"2023-10-09T04:45:06.117203+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:56.436933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.112340+0000","last_change":"2023-10-09T04:44:52.792028+0000","last_active":"2023-10-09T04:45:06.112340+0000","last_peered":"2023-10-09T04:45:06.112340+0000","last_clean":"2023-10-09T04:45:06.112340+0000","last_became_active":"2023-10-09T04:44:52.791588+0000","last_became_peered":"2023-10-09T04:44:52.791588+0000","last_unstale":"2023-10-09T04:45:06.112340+0000","last_undegraded":"2023-10-09T04:45:06.112340+0000","last_fullsized":"2023-10-09T04:45:06.112340+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:57.589240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.113093+0000","last_change":"2023-10-09T04:44:53.959134+0000","last_active":"2023-10-09T04:45:06.113093+0000","last_peered":"2023-10-09T04:45:06.113093+0000","last_clean":"2023-10-09T04:45:06.113093+0000","last_became_active":"2023-10-09T04:44:53.958411+0000","last_became_peered":"2023-10-09T04:44:53.958411+0000","last_unstale":"2023-10-09T04:45:06.113093+0000","last_undegraded":"2023-10-09T04:45:06.113093+0000","last_fullsized":"2023-10-09T04:45:06.113093+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:49:59.662675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":27,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.114902+0000","last_change":"2023-10-09T04:44:52.784918+0000","last_active":"2023-10-09T04:45:06.114902+0000","last_peered":"2023-10-09T04:45:06.114902+0000","last_clean":"2023-10-09T04:45:06.114902+0000","last_became_active":"2023-10-09T04:44:52.783726+0000","last_became_peered":"2023-10-09T04:44:52.783726+0000","last_unstale":"2023-10-09T04:45:06.114902+0000","last_undegraded":"2023-10-09T04:45:06.114902+0000","last_fullsized":"2023-10-09T04:45:06.114902+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:28:13.795447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.119986+0000","last_change":"2023-10-09T04:44:53.962582+0000","last_active":"2023-10-09T04:45:06.119986+0000","last_peered":"2023-10-09T04:45:06.119986+0000","last_clean":"2023-10-09T04:45:06.119986+0000","last_became_active":"2023-10-09T04:44:53.961909+0000","last_became_peered":"2023-10-09T04:44:53.961909+0000","last_unstale":"2023-10-09T04:45:06.119986+0000","last_undegraded":"2023-10-09T04:45:06.119986+0000","last_fullsized":"2023-10-09T04:45:06.119986+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:42.808094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.111984+0000","last_change":"2023-10-09T04:44:52.792557+0000","last_active":"2023-10-09T04:45:06.111984+0000","last_peered":"2023-10-09T04:45:06.111984+0000","last_clean":"2023-10-09T04:45:06.111984+0000","last_became_active":"2023-10-09T04:44:52.791890+0000","last_became_peered":"2023-10-09T04:44:52.791890+0000","last_unstale":"2023-10-09T04:45:06.111984+0000","last_undegraded":"2023-10-09T04:45:06.111984+0000","last_fullsized":"2023-10-09T04:45:06.111984+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:04:46.379852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.118105+0000","last_change":"2023-10-09T04:44:53.962346+0000","last_active":"2023-10-09T04:45:06.118105+0000","last_peered":"2023-10-09T04:45:06.118105+0000","last_clean":"2023-10-09T04:45:06.118105+0000","last_became_active":"2023-10-09T04:44:53.961890+0000","last_became_peered":"2023-10-09T04:44:53.961890+0000","last_unstale":"2023-10-09T04:45:06.118105+0000","last_undegraded":"2023-10-09T04:45:06.118105+0000","last_fullsized":"2023-10-09T04:45:06.118105+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:01.790749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.119766+0000","last_change":"2023-10-09T04:44:52.781506+0000","last_active":"2023-10-09T04:45:06.119766+0000","last_peered":"2023-10-09T04:45:06.119766+0000","last_clean":"2023-10-09T04:45:06.119766+0000","last_became_active":"2023-10-09T04:44:52.779712+0000","last_became_peered":"2023-10-09T04:44:52.779712+0000","last_unstale":"2023-10-09T04:45:06.119766+0000","last_undegraded":"2023-10-09T04:45:06.119766+0000","last_fullsized":"2023-10-09T04:45:06.119766+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:12.774869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.114449+0000","last_change":"2023-10-09T04:44:53.959939+0000","last_active":"2023-10-09T04:45:06.114449+0000","last_peered":"2023-10-09T04:45:06.114449+0000","last_clean":"2023-10-09T04:45:06.114449+0000","last_became_active":"2023-10-09T04:44:53.957647+0000","last_became_peered":"2023-10-09T04:44:53.957647+0000","last_unstale":"2023-10-09T04:45:06.114449+0000","last_undegraded":"2023-10-09T04:45:06.114449+0000","last_fullsized":"2023-10-09T04:45:06.114449+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:11.820519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":19,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.111864+0000","last_change":"2023-10-09T04:44:52.792074+0000","last_active":"2023-10-09T04:45:06.111864+0000","last_peered":"2023-10-09T04:45:06.111864+0000","last_clean":"2023-10-09T04:45:06.111864+0000","last_became_active":"2023-10-09T04:44:52.791245+0000","last_became_peered":"2023-10-09T04:44:52.791245+0000","last_unstale":"2023-10-09T04:45:06.111864+0000","last_undegraded":"2023-10-09T04:45:06.111864+0000","last_fullsized":"2023-10-09T04:45:06.111864+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:06:12.940420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.256954+0000","last_change":"2023-10-09T04:44:53.797404+0000","last_active":"2023-10-09T04:45:06.256954+0000","last_peered":"2023-10-09T04:45:06.256954+0000","last_clean":"2023-10-09T04:45:06.256954+0000","last_became_active":"2023-10-09T04:44:53.797053+0000","last_became_peered":"2023-10-09T04:44:53.797053+0000","last_unstale":"2023-10-09T04:45:06.256954+0000","last_undegraded":"2023-10-09T04:45:06.256954+0000","last_fullsized":"2023-10-09T04:45:06.256954+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:43.827978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"69'7","reported_seq":27,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.257547+0000","last_change":"2023-10-09T04:44:52.781203+0000","last_active":"2023-10-09T04:45:06.257547+0000","last_peered":"2023-10-09T04:45:06.257547+0000","last_clean":"2023-10-09T04:45:06.257547+0000","last_became_active":"2023-10-09T04:44:52.780902+0000","last_became_peered":"2023-10-09T04:44:52.780902+0000","last_unstale":"2023-10-09T04:45:06.257547+0000","last_undegraded":"2023-10-09T04:45:06.257547+0000","last_fullsized":"2023-10-09T04:45:06.257547+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:56.332974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.116150+0000","last_change":"2023-10-09T04:44:53.793986+0000","last_active":"2023-10-09T04:45:06.116150+0000","last_peered":"2023-10-09T04:45:06.116150+0000","last_clean":"2023-10-09T04:45:06.116150+0000","last_became_active":"2023-10-09T04:44:53.793676+0000","last_became_peered":"2023-10-09T04:44:53.793676+0000","last_unstale":"2023-10-09T04:45:06.116150+0000","last_undegraded":"2023-10-09T04:45:06.116150+0000","last_fullsized":"2023-10-09T04:45:06.116150+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:18.344330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":20,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.111765+0000","last_change":"2023-10-09T04:44:52.781582+0000","last_active":"2023-10-09T04:45:06.111765+0000","last_peered":"2023-10-09T04:45:06.111765+0000","last_clean":"2023-10-09T04:45:06.111765+0000","last_became_active":"2023-10-09T04:44:52.781259+0000","last_became_peered":"2023-10-09T04:44:52.781259+0000","last_unstale":"2023-10-09T04:45:06.111765+0000","last_undegraded":"2023-10-09T04:45:06.111765+0000","last_fullsized":"2023-10-09T04:45:06.111765+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:01.648298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.112153+0000","last_change":"2023-10-09T04:44:53.958305+0000","last_active":"2023-10-09T04:45:06.112153+0000","last_peered":"2023-10-09T04:45:06.112153+0000","last_clean":"2023-10-09T04:45:06.112153+0000","last_became_active":"2023-10-09T04:44:53.957878+0000","last_became_peered":"2023-10-09T04:44:53.957878+0000","last_unstale":"2023-10-09T04:45:06.112153+0000","last_undegraded":"2023-10-09T04:45:06.112153+0000","last_fullsized":"2023-10-09T04:45:06.112153+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:43:27.473519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.118092+0000","last_change":"2023-10-09T04:44:52.790185+0000","last_active":"2023-10-09T04:45:06.118092+0000","last_peered":"2023-10-09T04:45:06.118092+0000","last_clean":"2023-10-09T04:45:06.118092+0000","last_became_active":"2023-10-09T04:44:52.789798+0000","last_became_peered":"2023-10-09T04:44:52.789798+0000","last_unstale":"2023-10-09T04:45:06.118092+0000","last_undegraded":"2023-10-09T04:45:06.118092+0000","last_fullsized":"2023-10-09T04:45:06.118092+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:17:23.671238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.112165+0000","last_change":"2023-10-09T04:44:53.794730+0000","last_active":"2023-10-09T04:45:06.112165+0000","last_peered":"2023-10-09T04:45:06.112165+0000","last_clean":"2023-10-09T04:45:06.112165+0000","last_became_active":"2023-10-09T04:44:53.793787+0000","last_became_peered":"2023-10-09T04:44:53.793787+0000","last_unstale":"2023-10-09T04:45:06.112165+0000","last_undegraded":"2023-10-09T04:45:06.112165+0000","last_fullsized":"2023-10-09T04:45:06.112165+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:58:33.016122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.120369+0000","last_change":"2023-10-09T04:44:52.781846+0000","last_active":"2023-10-09T04:45:06.120369+0000","last_peered":"2023-10-09T04:45:06.120369+0000","last_clean":"2023-10-09T04:45:06.120369+0000","last_became_active":"2023-10-09T04:44:52.781538+0000","last_became_peered":"2023-10-09T04:44:52.781538+0000","last_unstale":"2023-10-09T04:45:06.120369+0000","last_undegraded":"2023-10-09T04:45:06.120369+0000","last_fullsized":"2023-10-09T04:45:06.120369+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:33.775067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.115008+0000","last_change":"2023-10-09T04:44:53.797648+0000","last_active":"2023-10-09T04:45:06.115008+0000","last_peered":"2023-10-09T04:45:06.115008+0000","last_clean":"2023-10-09T04:45:06.115008+0000","last_became_active":"2023-10-09T04:44:53.797099+0000","last_became_peered":"2023-10-09T04:44:53.797099+0000","last_unstale":"2023-10-09T04:45:06.115008+0000","last_undegraded":"2023-10-09T04:45:06.115008+0000","last_fullsized":"2023-10-09T04:45:06.115008+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:35.903142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.112257+0000","last_change":"2023-10-09T04:44:52.794864+0000","last_active":"2023-10-09T04:45:06.112257+0000","last_peered":"2023-10-09T04:45:06.112257+0000","last_clean":"2023-10-09T04:45:06.112257+0000","last_became_active":"2023-10-09T04:44:52.793907+0000","last_became_peered":"2023-10-09T04:44:52.793907+0000","last_unstale":"2023-10-09T04:45:06.112257+0000","last_undegraded":"2023-10-09T04:45:06.112257+0000","last_fullsized":"2023-10-09T04:45:06.112257+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:56:00.825914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.120075+0000","last_change":"2023-10-09T04:44:53.958581+0000","last_active":"2023-10-09T04:45:06.120075+0000","last_peered":"2023-10-09T04:45:06.120075+0000","last_clean":"2023-10-09T04:45:06.120075+0000","last_became_active":"2023-10-09T04:44:53.957939+0000","last_became_peered":"2023-10-09T04:44:53.957939+0000","last_unstale":"2023-10-09T04:45:06.120075+0000","last_undegraded":"2023-10-09T04:45:06.120075+0000","last_fullsized":"2023-10-09T04:45:06.120075+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:12.616951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.115858+0000","last_change":"2023-10-09T04:44:52.790106+0000","last_active":"2023-10-09T04:45:06.115858+0000","last_peered":"2023-10-09T04:45:06.115858+0000","last_clean":"2023-10-09T04:45:06.115858+0000","last_became_active":"2023-10-09T04:44:52.789856+0000","last_became_peered":"2023-10-09T04:44:52.789856+0000","last_unstale":"2023-10-09T04:45:06.115858+0000","last_undegraded":"2023-10-09T04:45:06.115858+0000","last_fullsized":"2023-10-09T04:45:06.115858+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:50:07.831074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.112835+0000","last_change":"2023-10-09T04:44:53.790071+0000","last_active":"2023-10-09T04:45:06.112835+0000","last_peered":"2023-10-09T04:45:06.112835+0000","last_clean":"2023-10-09T04:45:06.112835+0000","last_became_active":"2023-10-09T04:44:53.789725+0000","last_became_peered":"2023-10-09T04:44:53.789725+0000","last_unstale":"2023-10-09T04:45:06.112835+0000","last_undegraded":"2023-10-09T04:45:06.112835+0000","last_fullsized":"2023-10-09T04:45:06.112835+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:25.624697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'2","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:45:06.257000+0000","last_change":"2023-10-09T04:44:52.793401+0000","last_active":"2023-10-09T04:45:06.257000+0000","last_peered":"2023-10-09T04:45:06.257000+0000","last_clean":"2023-10-09T04:45:06.257000+0000","last_became_active":"2023-10-09T04:44:52.792982+0000","last_became_peered":"2023-10-09T04:44:52.792982+0000","last_unstale":"2023-10-09T04:45:06.257000+0000","last_undegraded":"2023-10-09T04:45:06.257000+0000","last_fullsized":"2023-10-09T04:45:06.257000+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:39.255814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":47,"num_read_kb":49,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":45,"ondisk_log_size":45,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234054,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298600,"kb_used_data":1312,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446552,"statfs":{"total":95995035648,"available":95689269248,"internally_reserved":0,"allocated":1343488,"data_stored":912385,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":0.92800000000000005,"15min":0.94999999999999996},"min":{"1min":0.57999999999999996,"5min":0.433,"15min":0.433},"max":{"1min":1.3180000000000001,"5min":1.458,"15min":1.458},"last":1.0620000000000001},{"interface":"front","average":{"1min":1.081,"5min":0.96199999999999997,"15min":0.96799999999999997},"min":{"1min":0.59099999999999997,"5min":0.502,"15min":0.502},"max":{"1min":1.423,"5min":1.5580000000000001,"15min":1.5580000000000001},"last":0.96799999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:46:10 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":0.98699999999999999,"15min":1.0209999999999999},"min":{"1min":0.60599999999999998,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.3129999999999999,"5min":1.423,"15min":1.423},"last":1.2010000000000001},{"interface":"front","average":{"1min":0.94899999999999995,"5min":0.91600000000000004,"15min":0.93200000000000005},"min":{"1min":0.63900000000000001,"5min":0.435,"15min":0.435},"max":{"1min":1.1850000000000001,"5min":1.9950000000000001,"15min":1.9950000000000001},"last":0.82699999999999996}]},{"osd":2,"last update":"Mon Oct 9 04:46:10 2023","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":0.97099999999999997,"15min":1.0169999999999999},"min":{"1min":0.67300000000000004,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.282,"5min":2.0819999999999999,"15min":2.0819999999999999},"last":0.996},{"interface":"front","average":{"1min":1.0109999999999999,"5min":0.96499999999999997,"15min":0.98899999999999999},"min":{"1min":0.65600000000000003,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":1.2729999999999999,"5min":1.4730000000000001,"15min":1.4730000000000001},"last":0.88200000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":0.89700000000000002,"15min":0.88400000000000001},"min":{"1min":0.68400000000000005,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.226,"5min":1.4159999999999999,"15min":1.4159999999999999},"last":1.0269999999999999},{"interface":"front","average":{"1min":1.1200000000000001,"5min":1.0509999999999999,"15min":1.0700000000000001},"min":{"1min":0.81499999999999995,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.3660000000000001,"5min":1.5129999999999999,"15min":1.5129999999999999},"last":1.2649999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.89600000000000002,"15min":0.83599999999999997},"min":{"1min":0.63900000000000001,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.3160000000000001,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.89000000000000001,"15min":0.82399999999999995},"min":{"1min":0.73699999999999999,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.1890000000000001,"5min":1.1890000000000001,"15min":1.1890000000000001},"last":1.1659999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:46:10 2023","interfaces":[{"interface":"back","average":{"1min":1.1950000000000001,"5min":1.048,"15min":1.016},"min":{"1min":0.81999999999999995,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.3899999999999999,"5min":1.827,"15min":1.827},"last":1.123},{"interface":"front","average":{"1min":1.1080000000000001,"5min":1.042,"15min":1.05},"min":{"1min":0.73299999999999998,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.536,"5min":1.9179999999999999,"15min":1.9179999999999999},"last":1.087}]},{"osd":6,"last update":"Mon Oct 9 04:46:10 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":1.0700000000000001,"15min":1.052},"min":{"1min":0.86899999999999999,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.472,"5min":1.472,"15min":1.472},"last":1.1439999999999999},{"interface":"front","average":{"1min":1.091,"5min":1,"15min":0.96199999999999997},"min":{"1min":0.86199999999999999,"5min":0.505,"15min":0.505},"max":{"1min":1.359,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":1.2430000000000001}]}]},{"osd":6,"up_from":46,"seq":197568495696,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298592,"kb_used_data":1304,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446560,"statfs":{"total":95995035648,"available":95689277440,"internally_reserved":0,"allocated":1335296,"data_stored":912562,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:46:27 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":0.88,"15min":0.94299999999999995},"min":{"1min":0.52200000000000002,"5min":0.437,"15min":0.437},"max":{"1min":1.3620000000000001,"5min":1.7669999999999999,"15min":1.7669999999999999},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.876,"5min":0.86499999999999999,"15min":0.95099999999999996},"min":{"1min":0.59399999999999997,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.298,"5min":1.298,"15min":1.3149999999999999},"last":1.155}]},{"osd":1,"last update":"Mon Oct 9 04:46:27 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":0.874,"15min":0.90000000000000002},"min":{"1min":0.56100000000000005,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":1.2849999999999999,"5min":1.4850000000000001,"15min":1.4850000000000001},"last":0.81100000000000005},{"interface":"front","average":{"1min":0.94899999999999995,"5min":0.91600000000000004,"15min":0.92200000000000004},"min":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":1.2909999999999999,"5min":1.4059999999999999,"15min":1.4850000000000001},"last":0.94599999999999995}]},{"osd":2,"last update":"Mon Oct 9 04:46:27 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":0.94199999999999995,"15min":0.94999999999999996},"min":{"1min":0.53600000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"max":{"1min":1.2689999999999999,"5min":3.238,"15min":3.238},"last":1.0940000000000001},{"interface":"front","average":{"1min":0.94799999999999995,"5min":0.91200000000000003,"15min":0.92000000000000004},"min":{"1min":0.68799999999999994,"5min":0.61799999999999999,"15min":0.61799999999999999},"max":{"1min":1.175,"5min":1.2170000000000001,"15min":1.238},"last":1.117}]},{"osd":3,"last update":"Mon Oct 9 04:46:27 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":0.95399999999999996,"15min":0.97999999999999998},"min":{"1min":0.60499999999999998,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.3560000000000001,"5min":1.8879999999999999,"15min":1.8879999999999999},"last":1.03},{"interface":"front","average":{"1min":0.99099999999999999,"5min":0.97799999999999998,"15min":0.97599999999999998},"min":{"1min":0.70599999999999996,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.3089999999999999,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":1.1899999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:46:27 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":0.92200000000000004,"15min":0.82999999999999996},"min":{"1min":0.73799999999999999,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.198,"5min":1.4510000000000001,"15min":1.4510000000000001},"last":0.98899999999999999},{"interface":"front","average":{"1min":1.056,"5min":0.98899999999999999,"15min":0.93100000000000005},"min":{"1min":0.56899999999999995,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.3160000000000001,"5min":1.5249999999999999,"15min":1.5249999999999999},"last":1.2490000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:46:27 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":1.0009999999999999,"15min":0.92000000000000004},"min":{"1min":0.84799999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.306,"5min":1.397,"15min":1.397},"last":1.2210000000000001},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.048,"15min":0.98899999999999999},"min":{"1min":0.76100000000000001,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.296,"5min":1.6000000000000001,"15min":1.6000000000000001},"last":1.3120000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:46:13 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":1.0329999999999999,"15min":0.98299999999999998},"min":{"1min":0.77200000000000002,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.3500000000000001,"5min":1.452,"15min":1.452},"last":1.286},{"interface":"front","average":{"1min":1.22,"5min":1.1699999999999999,"15min":1.1200000000000001},"min":{"1min":1.0269999999999999,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.393,"5min":1.5660000000000001,"15min":1.5660000000000001},"last":1.333}]}]},{"osd":1,"up_from":13,"seq":55834574981,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298000,"kb_used_data":712,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447152,"statfs":{"total":95995035648,"available":95689883648,"internally_reserved":0,"allocated":729088,"data_stored":320853,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:46:09 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.65000000000000002,"15min":0.61399999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.96899999999999997,"5min":1.179,"15min":1.179},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.69999999999999996,"5min":0.73699999999999999,"15min":0.66900000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":1.0269999999999999,"5min":1.2170000000000001,"15min":1.2170000000000001},"last":0.78300000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:45:52 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":0.72999999999999998,"15min":0.69499999999999995},"min":{"1min":0.442,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.0740000000000001,"5min":1.484,"15min":1.484},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.78600000000000003,"5min":0.746,"15min":0.69599999999999995},"min":{"1min":0.45900000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.0609999999999999,"5min":1.371,"15min":1.371},"last":0.67600000000000005}]},{"osd":3,"last update":"Mon Oct 9 04:45:52 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":0.78700000000000003,"15min":0.80700000000000005},"min":{"1min":0.51200000000000001,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.069,"5min":1.222,"15min":1.2330000000000001},"last":0.82999999999999996},{"interface":"front","average":{"1min":0.873,"5min":0.79400000000000004,"15min":0.78600000000000003},"min":{"1min":0.41399999999999998,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.1559999999999999,"5min":1.3029999999999999,"15min":1.3029999999999999},"last":1.0880000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:45:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.0109999999999999,"15min":0.98099999999999998},"min":{"1min":0.82699999999999996,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.2230000000000001,"5min":1.607,"15min":1.607},"last":0.92500000000000004},{"interface":"front","average":{"1min":1.01,"5min":1.0209999999999999,"15min":0.98899999999999999},"min":{"1min":0.76500000000000001,"5min":0.68200000000000005,"15min":0.68200000000000005},"max":{"1min":1.488,"5min":1.488,"15min":1.488},"last":1.1459999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:46:31 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.95399999999999996,"15min":0.98999999999999999},"min":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.339,"5min":1.55,"15min":1.55},"last":1.0049999999999999},{"interface":"front","average":{"1min":0.97399999999999998,"5min":1.02,"15min":1.01},"min":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.4390000000000001,"5min":1.5509999999999999,"15min":1.5509999999999999},"last":0.91000000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:46:31 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.0469999999999999,"15min":1.0589999999999999},"min":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.5449999999999999,"5min":1.639,"15min":1.639},"last":0.97399999999999998},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.0589999999999999,"15min":1.081},"min":{"1min":0.627,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.373,"5min":1.552,"15min":1.552},"last":1.242}]},{"osd":7,"last update":"Mon Oct 9 04:46:14 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":1.1120000000000001,"15min":1.145},"min":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.5,"5min":1.758,"15min":1.758},"last":1.0620000000000001},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.121,"15min":1.165},"min":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.325,"5min":1.6899999999999999,"15min":1.6899999999999999},"last":1.1819999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738511,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298588,"kb_used_data":1300,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446564,"statfs":{"total":95995035648,"available":95689281536,"internally_reserved":0,"allocated":1331200,"data_stored":911797,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:46:09 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.70799999999999996,"15min":0.621},"min":{"1min":0.36499999999999999,"5min":0.35499999999999998,"15min":0.26900000000000002},"max":{"1min":0.875,"5min":1.3240000000000001,"15min":1.3240000000000001},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.72899999999999998,"15min":0.66100000000000003},"min":{"1min":0.44500000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.002,"5min":1.224,"15min":1.224},"last":0.61599999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.79300000000000004,"5min":0.76800000000000002,"15min":0.69399999999999995},"min":{"1min":0.56100000000000005,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.1819999999999999,"5min":1.319,"15min":1.319},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.82899999999999996,"5min":0.82099999999999995,"15min":0.71899999999999997},"min":{"1min":0.54200000000000004,"5min":0.40000000000000002,"15min":0.34499999999999997},"max":{"1min":1.2589999999999999,"5min":1.2789999999999999,"15min":1.2789999999999999},"last":0.5}]},{"osd":3,"last update":"Mon Oct 9 04:45:48 2023","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":0.879,"15min":0.84599999999999997},"min":{"1min":0.51200000000000001,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.361,"5min":1.4299999999999999,"15min":1.4299999999999999},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.84199999999999997,"5min":0.79300000000000004,"15min":0.79800000000000004},"min":{"1min":0.502,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.2090000000000001,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":0.70099999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:45:48 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":0.94999999999999996,"15min":0.93799999999999994},"min":{"1min":0.752,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.272,"5min":1.673,"15min":1.673},"last":1.008},{"interface":"front","average":{"1min":1,"5min":0.98599999999999999,"15min":0.95499999999999996},"min":{"1min":0.72199999999999998,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.4199999999999999,"5min":1.4199999999999999,"15min":1.4199999999999999},"last":0.79500000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:46:34 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":1.0489999999999999,"15min":1.048},"min":{"1min":0.70199999999999996,"5min":0.64000000000000001,"15min":0.64000000000000001},"max":{"1min":1.23,"5min":2.0219999999999998,"15min":2.0219999999999998},"last":0.85999999999999999},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.97099999999999997,"15min":1},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":1.129,"5min":1.623,"15min":1.623},"last":0.751}]},{"osd":6,"last update":"Mon Oct 9 04:46:28 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":1.0189999999999999,"15min":1.073},"min":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.177,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":0.98199999999999998},{"interface":"front","average":{"1min":1.002,"5min":1.105,"15min":1.1439999999999999},"min":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"max":{"1min":1.3080000000000001,"5min":1.7330000000000001,"15min":1.7330000000000001},"last":1.054}]},{"osd":7,"last update":"Mon Oct 9 04:46:17 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":1.0660000000000001,"15min":1.1200000000000001},"min":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"max":{"1min":1.3380000000000001,"5min":1.601,"15min":1.601},"last":1.0620000000000001},{"interface":"front","average":{"1min":1.101,"5min":1.1499999999999999,"15min":1.173},"min":{"1min":0.77200000000000002,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":1.446,"5min":2.097,"15min":2.097},"last":0.96099999999999997}]}]},{"osd":2,"up_from":18,"seq":77309411451,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298008,"kb_used_data":720,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447144,"statfs":{"total":95995035648,"available":95689875456,"internally_reserved":0,"allocated":737280,"data_stored":321441,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:46:01 2023","interfaces":[{"interface":"back","average":{"1min":0.69299999999999995,"5min":0.70399999999999996,"15min":0.67400000000000004},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.25,"5min":1.25,"15min":1.25},"last":0.77200000000000002},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.77200000000000002,"15min":0.71099999999999997},"min":{"1min":0.434,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.319,"5min":1.319,"15min":1.319},"last":0.69699999999999995}]},{"osd":1,"last update":"Mon Oct 9 04:46:01 2023","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":0.77400000000000002,"15min":0.70899999999999996},"min":{"1min":0.44800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.425,"5min":1.425,"15min":1.425},"last":0.86199999999999999},{"interface":"front","average":{"1min":0.76600000000000001,"5min":0.78800000000000003,"15min":0.72899999999999998},"min":{"1min":0.433,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.5580000000000001,"5min":1.5580000000000001,"15min":1.5580000000000001},"last":0.82699999999999996}]},{"osd":3,"last update":"Mon Oct 9 04:45:50 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":0.82899999999999996,"15min":0.81000000000000005},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.5209999999999999,"5min":1.5209999999999999,"15min":1.5209999999999999},"last":0.91600000000000004},{"interface":"front","average":{"1min":0.86499999999999999,"5min":0.871,"15min":0.85499999999999998},"min":{"1min":0.498,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.462,"5min":1.462,"15min":1.462},"last":0.88900000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:45:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.032,"15min":0.97299999999999998},"min":{"1min":0.70699999999999996,"5min":0.64000000000000001,"15min":0.64000000000000001},"max":{"1min":1.597,"5min":1.72,"15min":1.72},"last":1.1359999999999999},{"interface":"front","average":{"1min":1.1539999999999999,"5min":1.0609999999999999,"15min":1.02},"min":{"1min":0.73899999999999999,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":2.1179999999999999,"5min":2.1179999999999999,"15min":2.1179999999999999},"last":0.93999999999999995}]},{"osd":5,"last update":"Mon Oct 9 04:46:40 2023","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":1.0700000000000001,"15min":1.073},"min":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"max":{"1min":1.454,"5min":2.0350000000000001,"15min":2.0350000000000001},"last":1.036},{"interface":"front","average":{"1min":1.054,"5min":1.0660000000000001,"15min":1.0629999999999999},"min":{"1min":0.746,"5min":0.60599999999999998,"15min":0.60599999999999998},"max":{"1min":1.3839999999999999,"5min":1.702,"15min":1.702},"last":1.2589999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:46:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.0800000000000001,"15min":1.145},"min":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"max":{"1min":2.1920000000000002,"5min":2.1920000000000002,"15min":2.1920000000000002},"last":1.196},{"interface":"front","average":{"1min":1.022,"5min":0.98699999999999999,"15min":1.0489999999999999},"min":{"1min":0.63700000000000001,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.7729999999999999,"5min":1.7729999999999999,"15min":1.7729999999999999},"last":1.167}]},{"osd":7,"last update":"Mon Oct 9 04:46:13 2023","interfaces":[{"interface":"back","average":{"1min":1.214,"5min":1.163,"15min":1.2250000000000001},"min":{"1min":0.80000000000000004,"5min":0.622,"15min":0.622},"max":{"1min":1.796,"5min":1.796,"15min":1.796},"last":1.423},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.1899999999999999,"15min":1.232},"min":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"max":{"1min":2.2639999999999998,"5min":2.2639999999999998,"15min":2.2639999999999998},"last":0.98399999999999999}]}]},{"osd":3,"up_from":25,"seq":107374182511,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298004,"kb_used_data":716,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447148,"statfs":{"total":95995035648,"available":95689879552,"internally_reserved":0,"allocated":733184,"data_stored":320943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:45:55 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.69599999999999995,"15min":0.69699999999999995},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":1.0820000000000001,"5min":1.091,"15min":1.091},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.75,"5min":0.746,"15min":0.73699999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.0589999999999999,"5min":1.266,"15min":1.266},"last":0.72999999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:45:55 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":0.75600000000000001,"15min":0.752},"min":{"1min":0.53000000000000003,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.94899999999999995,"5min":1.1819999999999999,"15min":1.1819999999999999},"last":0.85899999999999999},{"interface":"front","average":{"1min":0.76600000000000001,"5min":0.75800000000000001,"15min":0.76100000000000001},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.022,"5min":1.7250000000000001,"15min":1.7250000000000001},"last":0.96699999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:45:55 2023","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":0.84899999999999998,"15min":0.85699999999999998},"min":{"1min":0.54900000000000004,"5min":0.437,"15min":0.437},"max":{"1min":1.1759999999999999,"5min":1.8320000000000001,"15min":1.8320000000000001},"last":0.78800000000000003},{"interface":"front","average":{"1min":0.73399999999999999,"5min":0.77600000000000002,"15min":0.80400000000000005},"min":{"1min":0.437,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.92100000000000004,"5min":1.173,"15min":1.218},"last":0.82699999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":0.98099999999999998,"15min":0.95199999999999996},"min":{"1min":0.73899999999999999,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.258,"5min":1.5009999999999999,"15min":1.5009999999999999},"last":1.262},{"interface":"front","average":{"1min":1.0700000000000001,"5min":1.0569999999999999,"15min":0.97299999999999998},"min":{"1min":0.83599999999999997,"5min":0.67400000000000004,"15min":0.56499999999999995},"max":{"1min":1.292,"5min":2.298,"15min":2.298},"last":1.4139999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:46:38 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":1.0109999999999999,"15min":1.02},"min":{"1min":0.69399999999999995,"5min":0.68799999999999994,"15min":0.60999999999999999},"max":{"1min":1.3380000000000001,"5min":1.4399999999999999,"15min":1.603},"last":1.3380000000000001},{"interface":"front","average":{"1min":0.96799999999999997,"5min":1.0509999999999999,"15min":1.0649999999999999},"min":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.2789999999999999,"5min":1.5820000000000001,"15min":1.9930000000000001},"last":1.248}]},{"osd":6,"last update":"Mon Oct 9 04:46:21 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":1.0649999999999999,"15min":1.087},"min":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.3420000000000001,"5min":1.621,"15min":2.1030000000000002},"last":1.536},{"interface":"front","average":{"1min":0.97999999999999998,"5min":1.048,"15min":1.0760000000000001},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.387,"5min":1.4039999999999999,"15min":1.4039999999999999},"last":1.4319999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:46:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":1.091,"15min":1.161},"min":{"1min":0.71399999999999997,"5min":0.70199999999999996,"15min":0.70199999999999996},"max":{"1min":1.3380000000000001,"5min":1.367,"15min":1.367},"last":1.2909999999999999},{"interface":"front","average":{"1min":1.139,"5min":1.1579999999999999,"15min":1.212},"min":{"1min":0.79500000000000004,"5min":0.76200000000000001,"15min":0.76200000000000001},"max":{"1min":1.407,"5min":1.534,"15min":1.534},"last":1.359}]}]},{"osd":4,"up_from":30,"seq":128849018982,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298024,"kb_used_data":736,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447128,"statfs":{"total":95995035648,"available":95689859072,"internally_reserved":0,"allocated":753664,"data_stored":322535,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":0.91300000000000003,"15min":0.86799999999999999},"min":{"1min":0.63300000000000001,"5min":0.54500000000000004,"15min":0.505},"max":{"1min":1.1850000000000001,"5min":1.294,"15min":1.294},"last":1.1779999999999999},{"interface":"front","average":{"1min":0.98399999999999999,"5min":0.91700000000000004,"15min":0.90700000000000003},"min":{"1min":0.71699999999999997,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":1.2470000000000001,"5min":1.4039999999999999,"15min":1.8180000000000001},"last":1.0429999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":0.90800000000000003,"15min":0.86499999999999999},"min":{"1min":0.68799999999999994,"5min":0.504,"15min":0.504},"max":{"1min":1.125,"5min":1.4239999999999999,"15min":1.4239999999999999},"last":1},{"interface":"front","average":{"1min":0.999,"5min":0.94599999999999995,"15min":0.90800000000000003},"min":{"1min":0.59599999999999997,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.3029999999999999,"5min":1.4099999999999999,"15min":1.9039999999999999},"last":0.85099999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":0.96299999999999997,"15min":0.92500000000000004},"min":{"1min":0.67900000000000005,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.2090000000000001,"5min":1.548,"15min":1.724},"last":1.147},{"interface":"front","average":{"1min":0.98799999999999999,"5min":0.92200000000000004,"15min":0.90300000000000002},"min":{"1min":0.80000000000000004,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.2270000000000001,"5min":1.23,"15min":1.25},"last":1.0880000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:45:45 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":0.96199999999999997,"15min":0.92400000000000004},"min":{"1min":0.74199999999999999,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.3240000000000001,"5min":1.369,"15min":1.369},"last":1.2030000000000001},{"interface":"front","average":{"1min":1.0109999999999999,"5min":0.96799999999999997,"15min":0.92700000000000005},"min":{"1min":0.81899999999999995,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.202,"5min":1.377,"15min":1.377},"last":0.89800000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:46:37 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":0.95499999999999996,"15min":0.81499999999999995},"min":{"1min":0.70899999999999996,"5min":0.47499999999999998,"15min":0.45100000000000001},"max":{"1min":1.359,"5min":1.4750000000000001,"15min":1.4750000000000001},"last":1.3260000000000001},{"interface":"front","average":{"1min":1.024,"5min":1.006,"15min":0.83099999999999996},"min":{"1min":0.57499999999999996,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.286,"5min":1.6539999999999999,"15min":1.6539999999999999},"last":0.94499999999999995}]},{"osd":6,"last update":"Mon Oct 9 04:46:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":1.004,"15min":0.83599999999999997},"min":{"1min":0.67200000000000004,"5min":0.44500000000000001,"15min":0.42599999999999999},"max":{"1min":1.4330000000000001,"5min":1.474,"15min":1.474},"last":1.2689999999999999},{"interface":"front","average":{"1min":1.1459999999999999,"5min":1.0860000000000001,"15min":0.996},"min":{"1min":0.60199999999999998,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.3979999999999999,"5min":1.9299999999999999,"15min":1.9850000000000001},"last":1.2430000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:46:12 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":1.0660000000000001,"15min":1},"min":{"1min":0.81499999999999995,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.5009999999999999,"5min":1.752,"15min":1.752},"last":1.2949999999999999},{"interface":"front","average":{"1min":1.1819999999999999,"5min":1.1439999999999999,"15min":1.0580000000000001},"min":{"1min":1.0249999999999999,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.397,"5min":1.6180000000000001,"15min":1.6180000000000001},"last":1.1220000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757338,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298008,"kb_used_data":720,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447144,"statfs":{"total":95995035648,"available":95689875456,"internally_reserved":0,"allocated":737280,"data_stored":321477,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:46:32 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":0.89600000000000002,"15min":0.877},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":1.254,"5min":1.3720000000000001,"15min":1.3720000000000001},"last":0.88600000000000001},{"interface":"front","average":{"1min":0.90700000000000003,"5min":0.92400000000000004,"15min":0.93100000000000005},"min":{"1min":0.52800000000000002,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.3320000000000001,"5min":1.389,"15min":1.7869999999999999},"last":1.1990000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:46:32 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":0.89800000000000002,"15min":0.93999999999999995},"min":{"1min":0.51500000000000001,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.2330000000000001,"5min":1.399,"15min":2.0510000000000002},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.84699999999999998,"5min":0.90200000000000002,"15min":0.89000000000000001},"min":{"1min":0.61799999999999999,"5min":0.57899999999999996,"15min":0.57899999999999996},"max":{"1min":1.2010000000000001,"5min":1.5529999999999999,"15min":1.5529999999999999},"last":0.94299999999999995}]},{"osd":2,"last update":"Mon Oct 9 04:46:32 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":0.90600000000000003,"15min":0.86799999999999999},"min":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.505},"max":{"1min":1.1839999999999999,"5min":1.204,"15min":1.204},"last":1.232},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.997,"15min":0.97199999999999998},"min":{"1min":0.69699999999999995,"5min":0.67000000000000004,"15min":0.60599999999999998},"max":{"1min":1.4099999999999999,"5min":1.446,"15min":1.885},"last":1}]},{"osd":3,"last update":"Mon Oct 9 04:46:32 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":0.97399999999999998,"15min":0.93300000000000005},"min":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.56299999999999994},"max":{"1min":1.3700000000000001,"5min":1.3700000000000001,"15min":1.3700000000000001},"last":1.046},{"interface":"front","average":{"1min":1.1240000000000001,"5min":1.05,"15min":1.0149999999999999},"min":{"1min":0.84499999999999997,"5min":0.61099999999999999,"15min":0.61099999999999999},"max":{"1min":1.3859999999999999,"5min":1.3859999999999999,"15min":1.9710000000000001},"last":1.1140000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:46:32 2023","interfaces":[{"interface":"back","average":{"1min":0.94799999999999995,"5min":0.879,"15min":0.80100000000000005},"min":{"1min":0.72399999999999998,"5min":0.5,"15min":0.5},"max":{"1min":1.351,"5min":1.351,"15min":1.3759999999999999},"last":0.83699999999999997},{"interface":"front","average":{"1min":1.171,"5min":1.022,"15min":0.88200000000000001},"min":{"1min":0.83799999999999997,"5min":0.52500000000000002,"15min":0.36799999999999999},"max":{"1min":1.3740000000000001,"5min":1.3999999999999999,"15min":1.5269999999999999},"last":1.1619999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:46:26 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.042,"15min":0.93899999999999995},"min":{"1min":0.76500000000000001,"5min":0.63300000000000001,"15min":0.45700000000000002},"max":{"1min":1.4299999999999999,"5min":1.4670000000000001,"15min":1.4670000000000001},"last":1.2769999999999999},{"interface":"front","average":{"1min":1.2,"5min":1.0609999999999999,"15min":1.052},"min":{"1min":0.90000000000000002,"5min":0.55500000000000005,"15min":0.49299999999999999},"max":{"1min":1.5209999999999999,"5min":1.5209999999999999,"15min":2.169},"last":1.3109999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:46:10 2023","interfaces":[{"interface":"back","average":{"1min":1.1539999999999999,"5min":1.0980000000000001,"15min":1.0529999999999999},"min":{"1min":0.78400000000000003,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.431,"5min":1.5049999999999999,"15min":1.5049999999999999},"last":1.2589999999999999},{"interface":"front","average":{"1min":1.073,"5min":1.038,"15min":0.97499999999999998},"min":{"1min":0.70899999999999996,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.335,"5min":1.4710000000000001,"15min":1.4710000000000001},"last":1.0900000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:46:47.587 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T04:46:47.587 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T04:46:47.587 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T04:46:47.587 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T04:46:47.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:47 smithi194 bash[31648]: audit 2023-10-09T04:46:46.669223+0000 mgr.y (mgr.14466) 180 : audit [DBG] from='client.14820 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:46:48.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:47 smithi133 bash[41291]: audit 2023-10-09T04:46:46.669223+0000 mgr.y (mgr.14466) 180 : audit [DBG] from='client.14820 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:46:48.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:47 smithi133 bash[27558]: audit 2023-10-09T04:46:46.669223+0000 mgr.y (mgr.14466) 180 : audit [DBG] from='client.14820 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:46:48.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:48 smithi194 bash[31648]: cluster 2023-10-09T04:46:47.168434+0000 mgr.y (mgr.14466) 181 : cluster [DBG] pgmap v156: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:49.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:48 smithi133 bash[27558]: cluster 2023-10-09T04:46:47.168434+0000 mgr.y (mgr.14466) 181 : cluster [DBG] pgmap v156: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:49.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:48 smithi133 bash[41291]: cluster 2023-10-09T04:46:47.168434+0000 mgr.y (mgr.14466) 181 : cluster [DBG] pgmap v156: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:50.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:50 smithi133 bash[27558]: cluster 2023-10-09T04:46:49.169036+0000 mgr.y (mgr.14466) 182 : cluster [DBG] pgmap v157: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:50.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:50 smithi133 bash[27558]: audit 2023-10-09T04:46:50.192549+0000 mon.a (mon.0) 902 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:50.801 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:46:50 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:46:50] "GET /metrics HTTP/1.1" 200 238931 "" "Prometheus/2.43.0" 2023-10-09T04:46:50.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:50 smithi133 bash[41291]: cluster 2023-10-09T04:46:49.169036+0000 mgr.y (mgr.14466) 182 : cluster [DBG] pgmap v157: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:50.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:50 smithi133 bash[41291]: audit 2023-10-09T04:46:50.192549+0000 mon.a (mon.0) 902 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:50.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:50 smithi194 bash[31648]: cluster 2023-10-09T04:46:49.169036+0000 mgr.y (mgr.14466) 182 : cluster [DBG] pgmap v157: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:50.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:50 smithi194 bash[31648]: audit 2023-10-09T04:46:50.192549+0000 mon.a (mon.0) 902 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:52.404 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:46:52.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:52 smithi133 bash[41291]: cluster 2023-10-09T04:46:51.170107+0000 mgr.y (mgr.14466) 183 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:52.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:52 smithi133 bash[27558]: cluster 2023-10-09T04:46:51.170107+0000 mgr.y (mgr.14466) 183 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:52.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:52 smithi194 bash[31648]: cluster 2023-10-09T04:46:51.170107+0000 mgr.y (mgr.14466) 183 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:54.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:54 smithi194 bash[31648]: cluster 2023-10-09T04:46:53.170477+0000 mgr.y (mgr.14466) 184 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:55.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:54 smithi133 bash[41291]: cluster 2023-10-09T04:46:53.170477+0000 mgr.y (mgr.14466) 184 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:55.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:54 smithi133 bash[27558]: cluster 2023-10-09T04:46:53.170477+0000 mgr.y (mgr.14466) 184 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:55.983 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T04:46:55.984 INFO:teuthology.orchestra.run.smithi133.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T04:46:56.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:46:56 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:46:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:46:56.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:56 smithi194 bash[31648]: cluster 2023-10-09T04:46:55.171008+0000 mgr.y (mgr.14466) 185 : cluster [DBG] pgmap v160: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:56.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:56 smithi194 bash[31648]: audit 2023-10-09T04:46:55.980789+0000 mon.a (mon.0) 903 : audit [DBG] from='client.? 172.21.15.133:0/2484925198' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T04:46:56.846 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:56 smithi133 bash[41291]: cluster 2023-10-09T04:46:55.171008+0000 mgr.y (mgr.14466) 185 : cluster [DBG] pgmap v160: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:56.846 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:56 smithi133 bash[41291]: audit 2023-10-09T04:46:55.980789+0000 mon.a (mon.0) 903 : audit [DBG] from='client.? 172.21.15.133:0/2484925198' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T04:46:56.846 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:56 smithi133 bash[27558]: cluster 2023-10-09T04:46:55.171008+0000 mgr.y (mgr.14466) 185 : cluster [DBG] pgmap v160: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:56.846 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:56 smithi133 bash[27558]: audit 2023-10-09T04:46:55.980789+0000 mon.a (mon.0) 903 : audit [DBG] from='client.? 172.21.15.133:0/2484925198' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T04:46:56.847 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T04:46:56.847 INFO:teuthology.run_tasks:Running task print... 2023-10-09T04:46:56.856 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2023-10-09T04:46:56.857 INFO:teuthology.run_tasks:Running task parallel... 2023-10-09T04:46:56.865 INFO:teuthology.task.parallel:starting parallel... 2023-10-09T04:46:56.865 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T04:46:56.866 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-10-09T04:46:56.867 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T04:46:56.868 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T04:46:56.868 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2023-10-09T04:46:56.868 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_0 16 2023-10-09T04:46:56.869 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T04:46:56.869 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-10-09T04:46:56.869 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi133.front.sepia.ceph.com 2023-10-09T04:46:56.869 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2023-10-09T04:46:58.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:46:58 smithi194 bash[31648]: cluster 2023-10-09T04:46:57.172068+0000 mgr.y (mgr.14466) 186 : cluster [DBG] pgmap v161: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:59.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:46:58 smithi133 bash[27558]: cluster 2023-10-09T04:46:57.172068+0000 mgr.y (mgr.14466) 186 : cluster [DBG] pgmap v161: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:46:59.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:46:58 smithi133 bash[41291]: cluster 2023-10-09T04:46:57.172068+0000 mgr.y (mgr.14466) 186 : cluster [DBG] pgmap v161: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:00.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:00 smithi133 bash[41291]: cluster 2023-10-09T04:46:59.172709+0000 mgr.y (mgr.14466) 187 : cluster [DBG] pgmap v162: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:00.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:00 smithi133 bash[27558]: cluster 2023-10-09T04:46:59.172709+0000 mgr.y (mgr.14466) 187 : cluster [DBG] pgmap v162: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:00.801 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:47:00 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:47:00] "GET /metrics HTTP/1.1" 200 238931 "" "Prometheus/2.43.0" 2023-10-09T04:47:00.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:00 smithi194 bash[31648]: cluster 2023-10-09T04:46:59.172709+0000 mgr.y (mgr.14466) 187 : cluster [DBG] pgmap v162: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:01.648 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:01.649 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:02.889 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:02 smithi194 bash[31648]: cluster 2023-10-09T04:47:01.173780+0000 mgr.y (mgr.14466) 188 : cluster [DBG] pgmap v163: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:03.010 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:02 smithi133 bash[27558]: cluster 2023-10-09T04:47:01.173780+0000 mgr.y (mgr.14466) 188 : cluster [DBG] pgmap v163: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:03.011 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:02 smithi133 bash[41291]: cluster 2023-10-09T04:47:01.173780+0000 mgr.y (mgr.14466) 188 : cluster [DBG] pgmap v163: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:04.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:04 smithi194 bash[31648]: cluster 2023-10-09T04:47:03.174386+0000 mgr.y (mgr.14466) 189 : cluster [DBG] pgmap v164: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:05.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:04 smithi133 bash[41291]: cluster 2023-10-09T04:47:03.174386+0000 mgr.y (mgr.14466) 189 : cluster [DBG] pgmap v164: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:05.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:04 smithi133 bash[27558]: cluster 2023-10-09T04:47:03.174386+0000 mgr.y (mgr.14466) 189 : cluster [DBG] pgmap v164: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:05.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:05 smithi194 bash[31648]: audit 2023-10-09T04:47:05.192865+0000 mon.a (mon.0) 904 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:06.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:05 smithi133 bash[41291]: audit 2023-10-09T04:47:05.192865+0000 mon.a (mon.0) 904 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:06.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:05 smithi133 bash[27558]: audit 2023-10-09T04:47:05.192865+0000 mon.a (mon.0) 904 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:06.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:47:06 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:47:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:47:07.316 INFO:teuthology.orchestra.run.smithi133.stderr:pool 'unique_pool_0' created 2023-10-09T04:47:07.378 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:47:06 smithi133 bash[27821]: [09/Oct/2023:04:47:06] ENGINE Bus STOPPING 2023-10-09T04:47:07.378 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:47:06 smithi133 bash[27821]: [09/Oct/2023:04:47:06] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T04:47:07.378 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:47:06 smithi133 bash[27821]: [09/Oct/2023:04:47:06] ENGINE Bus STOPPED 2023-10-09T04:47:07.378 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:47:06 smithi133 bash[27821]: [09/Oct/2023:04:47:06] ENGINE Bus STARTING 2023-10-09T04:47:07.378 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:47:07 smithi133 bash[27821]: [09/Oct/2023:04:47:07] ENGINE Serving on http://:::9283 2023-10-09T04:47:07.378 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:47:07 smithi133 bash[27821]: [09/Oct/2023:04:47:07] ENGINE Bus STARTED 2023-10-09T04:47:07.379 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:06 smithi133 bash[27558]: cluster 2023-10-09T04:47:05.175030+0000 mgr.y (mgr.14466) 190 : cluster [DBG] pgmap v165: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:07.379 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:06 smithi133 bash[27558]: audit 2023-10-09T04:47:05.810025+0000 mon.a (mon.0) 905 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:47:07.379 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:06 smithi133 bash[27558]: audit 2023-10-09T04:47:06.292255+0000 mon.a (mon.0) 906 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:47:07.379 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:06 smithi133 bash[27558]: audit 2023-10-09T04:47:06.293543+0000 mon.a (mon.0) 907 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:47:07.379 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:06 smithi133 bash[27558]: audit 2023-10-09T04:47:06.302622+0000 mon.a (mon.0) 908 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:47:07.379 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:06 smithi133 bash[27558]: audit 2023-10-09T04:47:06.493608+0000 mon.a (mon.0) 909 : audit [INF] from='client.? 172.21.15.133:0/855022510' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T04:47:07.379 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:06 smithi133 bash[41291]: cluster 2023-10-09T04:47:05.175030+0000 mgr.y (mgr.14466) 190 : cluster [DBG] pgmap v165: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:07.379 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:06 smithi133 bash[41291]: audit 2023-10-09T04:47:05.810025+0000 mon.a (mon.0) 905 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:47:07.380 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:06 smithi133 bash[41291]: audit 2023-10-09T04:47:06.292255+0000 mon.a (mon.0) 906 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:47:07.380 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:06 smithi133 bash[41291]: audit 2023-10-09T04:47:06.293543+0000 mon.a (mon.0) 907 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:47:07.380 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:06 smithi133 bash[41291]: audit 2023-10-09T04:47:06.302622+0000 mon.a (mon.0) 908 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:47:07.380 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:06 smithi133 bash[41291]: audit 2023-10-09T04:47:06.493608+0000 mon.a (mon.0) 909 : audit [INF] from='client.? 172.21.15.133:0/855022510' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T04:47:07.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:06 smithi194 bash[31648]: cluster 2023-10-09T04:47:05.175030+0000 mgr.y (mgr.14466) 190 : cluster [DBG] pgmap v165: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:07.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:06 smithi194 bash[31648]: audit 2023-10-09T04:47:05.810025+0000 mon.a (mon.0) 905 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:47:07.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:06 smithi194 bash[31648]: audit 2023-10-09T04:47:06.292255+0000 mon.a (mon.0) 906 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:47:07.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:06 smithi194 bash[31648]: audit 2023-10-09T04:47:06.293543+0000 mon.a (mon.0) 907 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:47:07.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:06 smithi194 bash[31648]: audit 2023-10-09T04:47:06.302622+0000 mon.a (mon.0) 908 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:47:07.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:06 smithi194 bash[31648]: audit 2023-10-09T04:47:06.493608+0000 mon.a (mon.0) 909 : audit [INF] from='client.? 172.21.15.133:0/855022510' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T04:47:08.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:07 smithi133 bash[27558]: audit 2023-10-09T04:47:06.953039+0000 mon.a (mon.0) 910 : audit [INF] from='client.? 172.21.15.133:0/2902854662' entity='client.admin' 2023-10-09T04:47:08.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:07 smithi133 bash[27558]: audit 2023-10-09T04:47:06.989585+0000 mon.a (mon.0) 911 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:47:08.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:07 smithi133 bash[27558]: audit 2023-10-09T04:47:07.304666+0000 mon.a (mon.0) 912 : audit [INF] from='client.? 172.21.15.133:0/855022510' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T04:47:08.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:07 smithi133 bash[27558]: cluster 2023-10-09T04:47:07.304750+0000 mon.a (mon.0) 913 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-10-09T04:47:08.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:07 smithi133 bash[41291]: audit 2023-10-09T04:47:06.953039+0000 mon.a (mon.0) 910 : audit [INF] from='client.? 172.21.15.133:0/2902854662' entity='client.admin' 2023-10-09T04:47:08.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:07 smithi133 bash[41291]: audit 2023-10-09T04:47:06.989585+0000 mon.a (mon.0) 911 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:47:08.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:07 smithi133 bash[41291]: audit 2023-10-09T04:47:07.304666+0000 mon.a (mon.0) 912 : audit [INF] from='client.? 172.21.15.133:0/855022510' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T04:47:08.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:07 smithi133 bash[41291]: cluster 2023-10-09T04:47:07.304750+0000 mon.a (mon.0) 913 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-10-09T04:47:08.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:07 smithi194 bash[31648]: audit 2023-10-09T04:47:06.953039+0000 mon.a (mon.0) 910 : audit [INF] from='client.? 172.21.15.133:0/2902854662' entity='client.admin' 2023-10-09T04:47:08.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:07 smithi194 bash[31648]: audit 2023-10-09T04:47:06.989585+0000 mon.a (mon.0) 911 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:47:08.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:07 smithi194 bash[31648]: audit 2023-10-09T04:47:07.304666+0000 mon.a (mon.0) 912 : audit [INF] from='client.? 172.21.15.133:0/855022510' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T04:47:08.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:07 smithi194 bash[31648]: cluster 2023-10-09T04:47:07.304750+0000 mon.a (mon.0) 913 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-10-09T04:47:08.632 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph config set global log_to_journald false --force' 2023-10-09T04:47:08.828 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2023-10-09T04:47:09.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:08 smithi133 bash[27558]: cluster 2023-10-09T04:47:07.175763+0000 mgr.y (mgr.14466) 191 : cluster [DBG] pgmap v166: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:09.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:08 smithi133 bash[27558]: cluster 2023-10-09T04:47:08.311576+0000 mon.a (mon.0) 914 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-10-09T04:47:09.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:08 smithi133 bash[41291]: cluster 2023-10-09T04:47:07.175763+0000 mgr.y (mgr.14466) 191 : cluster [DBG] pgmap v166: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:09.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:08 smithi133 bash[41291]: cluster 2023-10-09T04:47:08.311576+0000 mon.a (mon.0) 914 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-10-09T04:47:09.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:08 smithi194 bash[31648]: cluster 2023-10-09T04:47:07.175763+0000 mgr.y (mgr.14466) 191 : cluster [DBG] pgmap v166: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:09.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:08 smithi194 bash[31648]: cluster 2023-10-09T04:47:08.311576+0000 mon.a (mon.0) 914 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-10-09T04:47:10.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:10 smithi194 bash[31648]: cluster 2023-10-09T04:47:09.176420+0000 mgr.y (mgr.14466) 192 : cluster [DBG] pgmap v169: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:10.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:10 smithi194 bash[31648]: cluster 2023-10-09T04:47:09.313171+0000 mon.a (mon.0) 915 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-10-09T04:47:10.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:10 smithi194 bash[31648]: cluster 2023-10-09T04:47:09.959335+0000 mon.a (mon.0) 916 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T04:47:10.800 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:47:10 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:47:10] "GET /metrics HTTP/1.1" 200 238934 "" "Prometheus/2.43.0" 2023-10-09T04:47:10.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:10 smithi133 bash[27558]: cluster 2023-10-09T04:47:09.176420+0000 mgr.y (mgr.14466) 192 : cluster [DBG] pgmap v169: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:10.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:10 smithi133 bash[27558]: cluster 2023-10-09T04:47:09.313171+0000 mon.a (mon.0) 915 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-10-09T04:47:10.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:10 smithi133 bash[27558]: cluster 2023-10-09T04:47:09.959335+0000 mon.a (mon.0) 916 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T04:47:10.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:10 smithi133 bash[41291]: cluster 2023-10-09T04:47:09.176420+0000 mgr.y (mgr.14466) 192 : cluster [DBG] pgmap v169: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:10.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:10 smithi133 bash[41291]: cluster 2023-10-09T04:47:09.313171+0000 mon.a (mon.0) 915 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-10-09T04:47:10.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:10 smithi133 bash[41291]: cluster 2023-10-09T04:47:09.959335+0000 mon.a (mon.0) 916 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T04:47:12.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:12 smithi133 bash[41291]: cluster 2023-10-09T04:47:11.177623+0000 mgr.y (mgr.14466) 193 : cluster [DBG] pgmap v171: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:12.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:12 smithi133 bash[27558]: cluster 2023-10-09T04:47:11.177623+0000 mgr.y (mgr.14466) 193 : cluster [DBG] pgmap v171: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:12.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:12 smithi194 bash[31648]: cluster 2023-10-09T04:47:11.177623+0000 mgr.y (mgr.14466) 193 : cluster [DBG] pgmap v171: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:13.378 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:13.378 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:14.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:14 smithi194 bash[31648]: audit 2023-10-09T04:47:13.080909+0000 mon.a (mon.0) 917 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:47:14.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:14 smithi194 bash[31648]: audit 2023-10-09T04:47:13.088496+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:47:14.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:14 smithi194 bash[31648]: cluster 2023-10-09T04:47:13.178105+0000 mgr.y (mgr.14466) 194 : cluster [DBG] pgmap v172: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:14.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:14 smithi194 bash[31648]: audit 2023-10-09T04:47:13.335637+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:47:14.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:14 smithi194 bash[31648]: audit 2023-10-09T04:47:13.345514+0000 mon.a (mon.0) 920 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:47:14.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:14 smithi194 bash[31648]: audit 2023-10-09T04:47:13.347089+0000 mon.a (mon.0) 921 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:47:14.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:14 smithi194 bash[31648]: audit 2023-10-09T04:47:13.348420+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:47:14.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:14 smithi194 bash[31648]: audit 2023-10-09T04:47:13.358104+0000 mon.a (mon.0) 923 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:47:14.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:14 smithi133 bash[27558]: audit 2023-10-09T04:47:13.080909+0000 mon.a (mon.0) 917 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:47:14.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:14 smithi133 bash[27558]: audit 2023-10-09T04:47:13.088496+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:47:14.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:14 smithi133 bash[27558]: cluster 2023-10-09T04:47:13.178105+0000 mgr.y (mgr.14466) 194 : cluster [DBG] pgmap v172: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:14.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:14 smithi133 bash[27558]: audit 2023-10-09T04:47:13.335637+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:47:14.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:14 smithi133 bash[27558]: audit 2023-10-09T04:47:13.345514+0000 mon.a (mon.0) 920 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:47:14.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:14 smithi133 bash[27558]: audit 2023-10-09T04:47:13.347089+0000 mon.a (mon.0) 921 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:47:14.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:14 smithi133 bash[27558]: audit 2023-10-09T04:47:13.348420+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:47:14.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:14 smithi133 bash[27558]: audit 2023-10-09T04:47:13.358104+0000 mon.a (mon.0) 923 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:47:14.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:14 smithi133 bash[41291]: audit 2023-10-09T04:47:13.080909+0000 mon.a (mon.0) 917 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:47:14.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:14 smithi133 bash[41291]: audit 2023-10-09T04:47:13.088496+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:47:14.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:14 smithi133 bash[41291]: cluster 2023-10-09T04:47:13.178105+0000 mgr.y (mgr.14466) 194 : cluster [DBG] pgmap v172: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:14.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:14 smithi133 bash[41291]: audit 2023-10-09T04:47:13.335637+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:47:14.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:14 smithi133 bash[41291]: audit 2023-10-09T04:47:13.345514+0000 mon.a (mon.0) 920 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:47:14.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:14 smithi133 bash[41291]: audit 2023-10-09T04:47:13.347089+0000 mon.a (mon.0) 921 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:47:14.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:14 smithi133 bash[41291]: audit 2023-10-09T04:47:13.348420+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:47:14.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:14 smithi133 bash[41291]: audit 2023-10-09T04:47:13.358104+0000 mon.a (mon.0) 923 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:47:16.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:16 smithi194 bash[31648]: cluster 2023-10-09T04:47:15.178792+0000 mgr.y (mgr.14466) 195 : cluster [DBG] pgmap v173: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:16.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:47:16 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:47:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:47:16.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:16 smithi133 bash[41291]: cluster 2023-10-09T04:47:15.178792+0000 mgr.y (mgr.14466) 195 : cluster [DBG] pgmap v173: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:16.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:16 smithi133 bash[27558]: cluster 2023-10-09T04:47:15.178792+0000 mgr.y (mgr.14466) 195 : cluster [DBG] pgmap v173: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:18.533 INFO:teuthology.orchestra.run.smithi133.stderr:enabled application 'rados' on pool 'unique_pool_0' 2023-10-09T04:47:18.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:18 smithi133 bash[27558]: cluster 2023-10-09T04:47:17.179741+0000 mgr.y (mgr.14466) 196 : cluster [DBG] pgmap v174: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:18.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:18 smithi133 bash[27558]: audit 2023-10-09T04:47:18.205109+0000 mon.a (mon.0) 924 : audit [INF] from='client.? 172.21.15.133:0/4134310869' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:47:18.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:18 smithi133 bash[41291]: cluster 2023-10-09T04:47:17.179741+0000 mgr.y (mgr.14466) 196 : cluster [DBG] pgmap v174: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:18.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:18 smithi133 bash[41291]: audit 2023-10-09T04:47:18.205109+0000 mon.a (mon.0) 924 : audit [INF] from='client.? 172.21.15.133:0/4134310869' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:47:18.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:18 smithi194 bash[31648]: cluster 2023-10-09T04:47:17.179741+0000 mgr.y (mgr.14466) 196 : cluster [DBG] pgmap v174: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:18.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:18 smithi194 bash[31648]: audit 2023-10-09T04:47:18.205109+0000 mon.a (mon.0) 924 : audit [INF] from='client.? 172.21.15.133:0/4134310869' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:47:19.463 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-10-09T04:47:19.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:19 smithi133 bash[27558]: audit 2023-10-09T04:47:18.525361+0000 mon.a (mon.0) 925 : audit [INF] from='client.? 172.21.15.133:0/4134310869' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:47:19.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:19 smithi133 bash[27558]: cluster 2023-10-09T04:47:18.525491+0000 mon.a (mon.0) 926 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-10-09T04:47:19.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:19 smithi133 bash[41291]: audit 2023-10-09T04:47:18.525361+0000 mon.a (mon.0) 925 : audit [INF] from='client.? 172.21.15.133:0/4134310869' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:47:19.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:19 smithi133 bash[41291]: cluster 2023-10-09T04:47:18.525491+0000 mon.a (mon.0) 926 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-10-09T04:47:19.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:19 smithi194 bash[31648]: audit 2023-10-09T04:47:18.525361+0000 mon.a (mon.0) 925 : audit [INF] from='client.? 172.21.15.133:0/4134310869' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:47:19.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:19 smithi194 bash[31648]: cluster 2023-10-09T04:47:18.525491+0000 mon.a (mon.0) 926 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-10-09T04:47:20.800 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:47:20 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:47:20] "GET /metrics HTTP/1.1" 200 241231 "" "Prometheus/2.43.0" 2023-10-09T04:47:20.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:20 smithi133 bash[41291]: cluster 2023-10-09T04:47:19.180298+0000 mgr.y (mgr.14466) 197 : cluster [DBG] pgmap v176: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:20.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:20 smithi133 bash[41291]: cluster 2023-10-09T04:47:19.522460+0000 mon.a (mon.0) 927 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T04:47:20.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:20 smithi133 bash[41291]: cluster 2023-10-09T04:47:19.522538+0000 mon.a (mon.0) 928 : cluster [INF] Cluster is now healthy 2023-10-09T04:47:20.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:20 smithi133 bash[41291]: audit 2023-10-09T04:47:20.193659+0000 mon.a (mon.0) 929 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:20.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:20 smithi133 bash[27558]: cluster 2023-10-09T04:47:19.180298+0000 mgr.y (mgr.14466) 197 : cluster [DBG] pgmap v176: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:20.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:20 smithi133 bash[27558]: cluster 2023-10-09T04:47:19.522460+0000 mon.a (mon.0) 927 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T04:47:20.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:20 smithi133 bash[27558]: cluster 2023-10-09T04:47:19.522538+0000 mon.a (mon.0) 928 : cluster [INF] Cluster is now healthy 2023-10-09T04:47:20.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:20 smithi133 bash[27558]: audit 2023-10-09T04:47:20.193659+0000 mon.a (mon.0) 929 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:20.841 DEBUG:teuthology.orchestra.run.smithi133:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2023-10-09T04:47:20.844 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T04:47:20.844 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T04:47:20.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:20 smithi194 bash[31648]: cluster 2023-10-09T04:47:19.180298+0000 mgr.y (mgr.14466) 197 : cluster [DBG] pgmap v176: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:47:20.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:20 smithi194 bash[31648]: cluster 2023-10-09T04:47:19.522460+0000 mon.a (mon.0) 927 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T04:47:20.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:20 smithi194 bash[31648]: cluster 2023-10-09T04:47:19.522538+0000 mon.a (mon.0) 928 : cluster [INF] Cluster is now healthy 2023-10-09T04:47:20.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:20 smithi194 bash[31648]: audit 2023-10-09T04:47:20.193659+0000 mon.a (mon.0) 929 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:20.892 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:hints = 1 2023-10-09T04:47:20.892 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T04:47:20.892 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Object prefix: benchmark_data_smithi133_87395 2023-10-09T04:47:20.892 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:47:20.892 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 0 0 0 0 0 0 - 0 2023-10-09T04:47:21.893 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 1 16 3337 3321 207.551 207.562 0.00315709 0.00479786 2023-10-09T04:47:22.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:22 smithi133 bash[41291]: cluster 2023-10-09T04:47:21.181226+0000 mgr.y (mgr.14466) 198 : cluster [DBG] pgmap v177: 89 pgs: 89 active+clean; 47 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s wr, 74 op/s 2023-10-09T04:47:22.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:22 smithi133 bash[27558]: cluster 2023-10-09T04:47:21.181226+0000 mgr.y (mgr.14466) 198 : cluster [DBG] pgmap v177: 89 pgs: 89 active+clean; 47 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s wr, 74 op/s 2023-10-09T04:47:22.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:22 smithi194 bash[31648]: cluster 2023-10-09T04:47:21.181226+0000 mgr.y (mgr.14466) 198 : cluster [DBG] pgmap v177: 89 pgs: 89 active+clean; 47 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s wr, 74 op/s 2023-10-09T04:47:22.892 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 2 16 6457 6441 201.267 195 0.00773245 0.00495762 2023-10-09T04:47:23.893 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 3 16 9417 9401 195.839 185 0.00349206 0.00509664 2023-10-09T04:47:24.240 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:24.545 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:24 smithi133 bash[41291]: cluster 2023-10-09T04:47:23.181784+0000 mgr.y (mgr.14466) 199 : cluster [DBG] pgmap v178: 89 pgs: 89 active+clean; 47 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s wr, 74 op/s 2023-10-09T04:47:24.545 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:24 smithi133 bash[27558]: cluster 2023-10-09T04:47:23.181784+0000 mgr.y (mgr.14466) 199 : cluster [DBG] pgmap v178: 89 pgs: 89 active+clean; 47 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s wr, 74 op/s 2023-10-09T04:47:24.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:24 smithi194 bash[31648]: cluster 2023-10-09T04:47:23.181784+0000 mgr.y (mgr.14466) 199 : cluster [DBG] pgmap v178: 89 pgs: 89 active+clean; 47 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s wr, 74 op/s 2023-10-09T04:47:24.893 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 4 16 12093 12077 188.687 167.25 0.00484359 0.00529286 2023-10-09T04:47:25.893 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 5 16 14740 14724 184.034 165.438 0.0112265 0.00542625 2023-10-09T04:47:26.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:47:26 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:47:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:47:26.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:26 smithi194 bash[31648]: cluster 2023-10-09T04:47:25.182285+0000 mgr.y (mgr.14466) 200 : cluster [DBG] pgmap v179: 89 pgs: 89 active+clean; 47 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s wr, 74 op/s 2023-10-09T04:47:26.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:26 smithi133 bash[41291]: cluster 2023-10-09T04:47:25.182285+0000 mgr.y (mgr.14466) 200 : cluster [DBG] pgmap v179: 89 pgs: 89 active+clean; 47 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s wr, 74 op/s 2023-10-09T04:47:26.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:26 smithi133 bash[27558]: cluster 2023-10-09T04:47:25.182285+0000 mgr.y (mgr.14466) 200 : cluster [DBG] pgmap v179: 89 pgs: 89 active+clean; 47 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s wr, 74 op/s 2023-10-09T04:47:26.893 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 6 16 17268 17252 179.692 158 0.00560153 0.00555859 2023-10-09T04:47:27.893 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 7 16 19627 19611 175.083 147.438 0.00439525 0.00570581 2023-10-09T04:47:28.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:28 smithi194 bash[31648]: cluster 2023-10-09T04:47:27.183057+0000 mgr.y (mgr.14466) 201 : cluster [DBG] pgmap v180: 89 pgs: 89 active+clean; 959 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-09T04:47:28.893 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 8 16 21936 21920 171.235 144.312 0.0043178 0.00583382 2023-10-09T04:47:29.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:28 smithi133 bash[41291]: cluster 2023-10-09T04:47:27.183057+0000 mgr.y (mgr.14466) 201 : cluster [DBG] pgmap v180: 89 pgs: 89 active+clean; 959 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-09T04:47:29.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:28 smithi133 bash[27558]: cluster 2023-10-09T04:47:27.183057+0000 mgr.y (mgr.14466) 201 : cluster [DBG] pgmap v180: 89 pgs: 89 active+clean; 959 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-09T04:47:29.893 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 9 16 23991 23975 166.478 128.438 0.00925616 0.00600076 2023-10-09T04:47:30.650 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:47:30 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:47:30] "GET /metrics HTTP/1.1" 200 241231 "" "Prometheus/2.43.0" 2023-10-09T04:47:30.651 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:30 smithi133 bash[41291]: cluster 2023-10-09T04:47:29.183596+0000 mgr.y (mgr.14466) 202 : cluster [DBG] pgmap v181: 89 pgs: 89 active+clean; 959 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-09T04:47:30.651 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:30 smithi133 bash[27558]: cluster 2023-10-09T04:47:29.183596+0000 mgr.y (mgr.14466) 202 : cluster [DBG] pgmap v181: 89 pgs: 89 active+clean; 959 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-09T04:47:30.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:30 smithi194 bash[31648]: cluster 2023-10-09T04:47:29.183596+0000 mgr.y (mgr.14466) 202 : cluster [DBG] pgmap v181: 89 pgs: 89 active+clean; 959 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-09T04:47:30.893 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 10 16 25997 25981 162.366 125.375 0.00652032 0.00615246 2023-10-09T04:47:31.294 INFO:teuthology.orchestra.run.smithi133.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T04:47:31.894 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 11 16 27923 27907 158.548 120.375 0.00898447 0.00630102 2023-10-09T04:47:32.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:32 smithi133 bash[27558]: cluster 2023-10-09T04:47:31.184328+0000 mgr.y (mgr.14466) 203 : cluster [DBG] pgmap v182: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 138 MiB/s wr, 2.20k op/s 2023-10-09T04:47:32.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:32 smithi133 bash[27558]: audit 2023-10-09T04:47:31.283108+0000 mgr.y (mgr.14466) 204 : audit [DBG] from='client.24616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:47:32.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:32 smithi133 bash[27558]: cephadm 2023-10-09T04:47:31.284244+0000 mgr.y (mgr.14466) 205 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T04:47:32.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:32 smithi133 bash[27558]: audit 2023-10-09T04:47:31.290684+0000 mon.a (mon.0) 930 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:47:32.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:32 smithi133 bash[27558]: audit 2023-10-09T04:47:31.293930+0000 mon.a (mon.0) 931 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:47:32.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:32 smithi133 bash[27558]: audit 2023-10-09T04:47:31.297035+0000 mon.a (mon.0) 932 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:47:32.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:32 smithi133 bash[27558]: audit 2023-10-09T04:47:31.298673+0000 mon.a (mon.0) 933 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:47:32.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:32 smithi133 bash[27558]: audit 2023-10-09T04:47:31.306777+0000 mon.a (mon.0) 934 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:47:32.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:32 smithi133 bash[27558]: cephadm 2023-10-09T04:47:31.386179+0000 mgr.y (mgr.14466) 206 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T04:47:32.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:32 smithi133 bash[41291]: cluster 2023-10-09T04:47:31.184328+0000 mgr.y (mgr.14466) 203 : cluster [DBG] pgmap v182: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 138 MiB/s wr, 2.20k op/s 2023-10-09T04:47:32.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:32 smithi133 bash[41291]: audit 2023-10-09T04:47:31.283108+0000 mgr.y (mgr.14466) 204 : audit [DBG] from='client.24616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:47:32.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:32 smithi133 bash[41291]: cephadm 2023-10-09T04:47:31.284244+0000 mgr.y (mgr.14466) 205 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T04:47:32.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:32 smithi133 bash[41291]: audit 2023-10-09T04:47:31.290684+0000 mon.a (mon.0) 930 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:47:32.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:32 smithi133 bash[41291]: audit 2023-10-09T04:47:31.293930+0000 mon.a (mon.0) 931 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:47:32.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:32 smithi133 bash[41291]: audit 2023-10-09T04:47:31.297035+0000 mon.a (mon.0) 932 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:47:32.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:32 smithi133 bash[41291]: audit 2023-10-09T04:47:31.298673+0000 mon.a (mon.0) 933 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:47:32.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:32 smithi133 bash[41291]: audit 2023-10-09T04:47:31.306777+0000 mon.a (mon.0) 934 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:47:32.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:32 smithi133 bash[41291]: cephadm 2023-10-09T04:47:31.386179+0000 mgr.y (mgr.14466) 206 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T04:47:32.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:32 smithi194 bash[31648]: cluster 2023-10-09T04:47:31.184328+0000 mgr.y (mgr.14466) 203 : cluster [DBG] pgmap v182: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 138 MiB/s wr, 2.20k op/s 2023-10-09T04:47:32.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:32 smithi194 bash[31648]: audit 2023-10-09T04:47:31.283108+0000 mgr.y (mgr.14466) 204 : audit [DBG] from='client.24616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:47:32.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:32 smithi194 bash[31648]: cephadm 2023-10-09T04:47:31.284244+0000 mgr.y (mgr.14466) 205 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T04:47:32.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:32 smithi194 bash[31648]: audit 2023-10-09T04:47:31.290684+0000 mon.a (mon.0) 930 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:47:32.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:32 smithi194 bash[31648]: audit 2023-10-09T04:47:31.293930+0000 mon.a (mon.0) 931 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:47:32.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:32 smithi194 bash[31648]: audit 2023-10-09T04:47:31.297035+0000 mon.a (mon.0) 932 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:47:32.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:32 smithi194 bash[31648]: audit 2023-10-09T04:47:31.298673+0000 mon.a (mon.0) 933 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:47:32.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:32 smithi194 bash[31648]: audit 2023-10-09T04:47:31.306777+0000 mon.a (mon.0) 934 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:47:32.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:32 smithi194 bash[31648]: cephadm 2023-10-09T04:47:31.386179+0000 mgr.y (mgr.14466) 206 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T04:47:32.894 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 12 16 29798 29782 155.101 117.188 0.0060038 0.00644175 2023-10-09T04:47:33.375 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T04:47:33.894 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 13 16 31508 31492 151.39 106.875 0.00660045 0.00659757 2023-10-09T04:47:34.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:34 smithi133 bash[27558]: cluster 2023-10-09T04:47:33.184773+0000 mgr.y (mgr.14466) 207 : cluster [DBG] pgmap v183: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 134 MiB/s wr, 2.14k op/s 2023-10-09T04:47:34.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:34 smithi133 bash[41291]: cluster 2023-10-09T04:47:33.184773+0000 mgr.y (mgr.14466) 207 : cluster [DBG] pgmap v183: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 134 MiB/s wr, 2.14k op/s 2023-10-09T04:47:34.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:34 smithi194 bash[31648]: cluster 2023-10-09T04:47:33.184773+0000 mgr.y (mgr.14466) 207 : cluster [DBG] pgmap v183: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 134 MiB/s wr, 2.14k op/s 2023-10-09T04:47:34.894 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 14 15 33324 33309 148.687 113.562 0.00824678 0.00671922 2023-10-09T04:47:35.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:35 smithi194 bash[31648]: audit 2023-10-09T04:47:35.194584+0000 mon.a (mon.0) 935 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:35.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:35 smithi133 bash[41291]: audit 2023-10-09T04:47:35.194584+0000 mon.a (mon.0) 935 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:35.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:35 smithi133 bash[27558]: audit 2023-10-09T04:47:35.194584+0000 mon.a (mon.0) 935 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:35.894 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 15 16 35050 35034 145.962 107.812 0.00960206 0.006845 2023-10-09T04:47:36.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:36 smithi194 bash[31648]: cluster 2023-10-09T04:47:35.186664+0000 mgr.y (mgr.14466) 208 : cluster [DBG] pgmap v184: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 134 MiB/s wr, 2.14k op/s 2023-10-09T04:47:36.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:47:36 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:47:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:47:36.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:36 smithi133 bash[41291]: cluster 2023-10-09T04:47:35.186664+0000 mgr.y (mgr.14466) 208 : cluster [DBG] pgmap v184: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 134 MiB/s wr, 2.14k op/s 2023-10-09T04:47:36.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:36 smithi133 bash[27558]: cluster 2023-10-09T04:47:35.186664+0000 mgr.y (mgr.14466) 208 : cluster [DBG] pgmap v184: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 134 MiB/s wr, 2.14k op/s 2023-10-09T04:47:36.894 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 16 16 36732 36716 143.409 105.125 0.0126406 0.00696677 2023-10-09T04:47:37.894 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 17 16 38423 38407 141.189 105.688 0.00971368 0.00707769 2023-10-09T04:47:38.272 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:38.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:38 smithi133 bash[41291]: cluster 2023-10-09T04:47:37.187400+0000 mgr.y (mgr.14466) 209 : cluster [DBG] pgmap v185: 89 pgs: 89 active+clean; 2.2 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 181 MiB/s wr, 2.89k op/s 2023-10-09T04:47:38.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:38 smithi133 bash[27558]: cluster 2023-10-09T04:47:37.187400+0000 mgr.y (mgr.14466) 209 : cluster [DBG] pgmap v185: 89 pgs: 89 active+clean; 2.2 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 181 MiB/s wr, 2.89k op/s 2023-10-09T04:47:38.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:38 smithi194 bash[31648]: cluster 2023-10-09T04:47:37.187400+0000 mgr.y (mgr.14466) 209 : cluster [DBG] pgmap v185: 89 pgs: 89 active+clean; 2.2 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 181 MiB/s wr, 2.89k op/s 2023-10-09T04:47:38.894 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 18 16 40056 40040 139.015 102.062 0.00742833 0.00718671 2023-10-09T04:47:39.897 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 19 16 41676 41660 137.027 101.25 0.00705275 0.00729231 2023-10-09T04:47:40.800 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:47:40 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:47:40] "GET /metrics HTTP/1.1" 200 242163 "" "Prometheus/2.43.0" 2023-10-09T04:47:40.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:40 smithi133 bash[41291]: cluster 2023-10-09T04:47:39.187771+0000 mgr.y (mgr.14466) 210 : cluster [DBG] pgmap v186: 89 pgs: 89 active+clean; 2.2 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-09T04:47:40.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:40 smithi133 bash[27558]: cluster 2023-10-09T04:47:39.187771+0000 mgr.y (mgr.14466) 210 : cluster [DBG] pgmap v186: 89 pgs: 89 active+clean; 2.2 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-09T04:47:40.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:40 smithi194 bash[31648]: cluster 2023-10-09T04:47:39.187771+0000 mgr.y (mgr.14466) 210 : cluster [DBG] pgmap v186: 89 pgs: 89 active+clean; 2.2 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-09T04:47:40.894 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T04:47:40.892322+0000 min lat: 0.00129931 max lat: 0.120376 avg lat: 0.00739328 2023-10-09T04:47:40.894 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:47:40.894 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 20 16 43269 43253 135.153 99.5625 0.0125259 0.00739328 2023-10-09T04:47:41.895 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 21 16 44828 44812 133.357 97.4375 0.0151536 0.00749312 2023-10-09T04:47:42.540 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:42 smithi133 bash[27558]: cluster 2023-10-09T04:47:41.188372+0000 mgr.y (mgr.14466) 211 : cluster [DBG] pgmap v187: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 147 MiB/s wr, 2.36k op/s 2023-10-09T04:47:42.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:42 smithi133 bash[41291]: cluster 2023-10-09T04:47:41.188372+0000 mgr.y (mgr.14466) 211 : cluster [DBG] pgmap v187: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 147 MiB/s wr, 2.36k op/s 2023-10-09T04:47:42.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:42 smithi194 bash[31648]: cluster 2023-10-09T04:47:41.188372+0000 mgr.y (mgr.14466) 211 : cluster [DBG] pgmap v187: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 147 MiB/s wr, 2.36k op/s 2023-10-09T04:47:42.895 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 22 16 46351 46335 131.621 95.1875 0.0121914 0.00759154 2023-10-09T04:47:43.895 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 23 16 47663 47647 129.459 82 0.00371737 0.00771814 2023-10-09T04:47:44.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:44 smithi133 bash[41291]: cluster 2023-10-09T04:47:43.188786+0000 mgr.y (mgr.14466) 212 : cluster [DBG] pgmap v188: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2023-10-09T04:47:44.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:44 smithi133 bash[27558]: cluster 2023-10-09T04:47:43.188786+0000 mgr.y (mgr.14466) 212 : cluster [DBG] pgmap v188: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2023-10-09T04:47:44.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:44 smithi194 bash[31648]: cluster 2023-10-09T04:47:43.188786+0000 mgr.y (mgr.14466) 212 : cluster [DBG] pgmap v188: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2023-10-09T04:47:44.896 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 24 15 49165 49150 127.979 93.9375 0.0121117 0.00780763 2023-10-09T04:47:45.896 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 25 16 50568 50552 126.365 87.625 0.0133177 0.00790631 2023-10-09T04:47:46.195 INFO:teuthology.orchestra.run.smithi133.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T04:47:46.196 INFO:teuthology.orchestra.run.smithi133.stdout:alertmanager.a smithi133 *:9093,9094 running (4m) 32s ago 5m 14.0M - 0.25.0 c8568f914cd2 9b1a41b191fd 2023-10-09T04:47:46.196 INFO:teuthology.orchestra.run.smithi133.stdout:grafana.a smithi194 *:3000 running (4m) 33s ago 4m 77.8M - 9.4.7 2c41d148cca3 5f0f270a8238 2023-10-09T04:47:46.196 INFO:teuthology.orchestra.run.smithi133.stdout:mds.foo.smithi133.tsuabh smithi133 running (2m) 32s ago 2m 11.4M - 17.2.6-1303-gb2903cb0 5517396c50d4 1adbb452599e 2023-10-09T04:47:46.196 INFO:teuthology.orchestra.run.smithi133.stdout:mds.foo.smithi194.yxnfqn smithi194 running (2m) 33s ago 2m 12.9M - 17.2.6-1303-gb2903cb0 5517396c50d4 745c16c1ffd2 2023-10-09T04:47:46.196 INFO:teuthology.orchestra.run.smithi133.stdout:mgr.x smithi194 *:8443 running (13m) 33s ago 13m 419M - 17.2.6-1303-gb2903cb0 5517396c50d4 e2608f2a1265 2023-10-09T04:47:46.196 INFO:teuthology.orchestra.run.smithi133.stdout:mgr.y smithi133 *:9283 running (16m) 32s ago 16m 473M - 17.2.6-1303-gb2903cb0 5517396c50d4 1fcaf44f24b8 2023-10-09T04:47:46.197 INFO:teuthology.orchestra.run.smithi133.stdout:mon.a smithi133 running (16m) 32s ago 16m 51.4M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 fdcd4c3aa009 2023-10-09T04:47:46.197 INFO:teuthology.orchestra.run.smithi133.stdout:mon.b smithi194 running (14m) 33s ago 14m 39.2M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 8059052f6835 2023-10-09T04:47:46.197 INFO:teuthology.orchestra.run.smithi133.stdout:mon.c smithi133 running (14m) 32s ago 14m 38.4M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 74897387140d 2023-10-09T04:47:46.197 INFO:teuthology.orchestra.run.smithi133.stdout:node-exporter.a smithi133 *:9100 running (5m) 32s ago 5m 8628k - 1.5.0 0da6a335fe13 ebc3ec51c93f 2023-10-09T04:47:46.197 INFO:teuthology.orchestra.run.smithi133.stdout:node-exporter.b smithi194 *:9100 running (5m) 33s ago 5m 8435k - 1.5.0 0da6a335fe13 6eaca83fe0de 2023-10-09T04:47:46.197 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0 smithi133 running (12m) 32s ago 13m 45.2M 4060M 17.2.6-1303-gb2903cb0 5517396c50d4 7dc540ae1166 2023-10-09T04:47:46.197 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1 smithi133 running (12m) 32s ago 12m 45.8M 4060M 17.2.6-1303-gb2903cb0 5517396c50d4 27ac869bf46a 2023-10-09T04:47:46.197 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2 smithi133 running (11m) 32s ago 11m 43.5M 4060M 17.2.6-1303-gb2903cb0 5517396c50d4 e9d2970cad24 2023-10-09T04:47:46.197 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3 smithi133 running (10m) 32s ago 10m 43.4M 4060M 17.2.6-1303-gb2903cb0 5517396c50d4 415368a68902 2023-10-09T04:47:46.197 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4 smithi194 running (9m) 33s ago 9m 43.1M 4060M 17.2.6-1303-gb2903cb0 5517396c50d4 c77edffeb061 2023-10-09T04:47:46.197 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5 smithi194 running (8m) 33s ago 8m 42.0M 4060M 17.2.6-1303-gb2903cb0 5517396c50d4 47540765fad9 2023-10-09T04:47:46.197 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6 smithi194 running (7m) 33s ago 7m 41.7M 4060M 17.2.6-1303-gb2903cb0 5517396c50d4 26bbe2f2859f 2023-10-09T04:47:46.198 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7 smithi194 running (6m) 33s ago 6m 40.9M 4060M 17.2.6-1303-gb2903cb0 5517396c50d4 45c343cd22ea 2023-10-09T04:47:46.198 INFO:teuthology.orchestra.run.smithi133.stdout:prometheus.a smithi194 *:9095 running (4m) 33s ago 5m 49.9M - 2.43.0 a07b618ecd1d 48786ceb620c 2023-10-09T04:47:46.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:47:46 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:47:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:47:46.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:46 smithi194 bash[31648]: cluster 2023-10-09T04:47:45.189292+0000 mgr.y (mgr.14466) 213 : cluster [DBG] pgmap v189: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2023-10-09T04:47:46.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:46 smithi133 bash[41291]: cluster 2023-10-09T04:47:45.189292+0000 mgr.y (mgr.14466) 213 : cluster [DBG] pgmap v189: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2023-10-09T04:47:46.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:46 smithi133 bash[27558]: cluster 2023-10-09T04:47:45.189292+0000 mgr.y (mgr.14466) 213 : cluster [DBG] pgmap v189: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2023-10-09T04:47:46.896 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 26 16 51987 51971 124.915 88.6875 0.0200062 0.00799935 2023-10-09T04:47:47.786 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for minority of mons to upgrade' 2023-10-09T04:47:47.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:47 smithi194 bash[31648]: audit 2023-10-09T04:47:46.171156+0000 mgr.y (mgr.14466) 214 : audit [DBG] from='client.24770 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:47:47.896 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 27 16 53416 53400 123.596 89.3125 0.0112068 0.00808441 2023-10-09T04:47:48.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:47 smithi133 bash[41291]: audit 2023-10-09T04:47:46.171156+0000 mgr.y (mgr.14466) 214 : audit [DBG] from='client.24770 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:47:48.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:47 smithi133 bash[27558]: audit 2023-10-09T04:47:46.171156+0000 mgr.y (mgr.14466) 214 : audit [DBG] from='client.24770 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:47:48.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:48 smithi194 bash[31648]: cluster 2023-10-09T04:47:47.190354+0000 mgr.y (mgr.14466) 215 : cluster [DBG] pgmap v190: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 127 MiB/s wr, 2.04k op/s 2023-10-09T04:47:48.896 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 28 16 54769 54753 122.202 84.5625 0.00575618 0.00817837 2023-10-09T04:47:49.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:48 smithi133 bash[41291]: cluster 2023-10-09T04:47:47.190354+0000 mgr.y (mgr.14466) 215 : cluster [DBG] pgmap v190: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 127 MiB/s wr, 2.04k op/s 2023-10-09T04:47:49.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:48 smithi133 bash[27558]: cluster 2023-10-09T04:47:47.190354+0000 mgr.y (mgr.14466) 215 : cluster [DBG] pgmap v190: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 127 MiB/s wr, 2.04k op/s 2023-10-09T04:47:49.896 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 29 16 56109 56093 120.876 83.75 0.00801142 0.00826687 2023-10-09T04:47:50.800 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:47:50 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:47:50] "GET /metrics HTTP/1.1" 200 242334 "" "Prometheus/2.43.0" 2023-10-09T04:47:50.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:50 smithi133 bash[41291]: cluster 2023-10-09T04:47:49.190820+0000 mgr.y (mgr.14466) 216 : cluster [DBG] pgmap v191: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T04:47:50.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:50 smithi133 bash[41291]: audit 2023-10-09T04:47:50.194662+0000 mon.a (mon.0) 936 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:50.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:50 smithi133 bash[27558]: cluster 2023-10-09T04:47:49.190820+0000 mgr.y (mgr.14466) 216 : cluster [DBG] pgmap v191: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T04:47:50.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:50 smithi133 bash[27558]: audit 2023-10-09T04:47:50.194662+0000 mon.a (mon.0) 936 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:50.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:50 smithi194 bash[31648]: cluster 2023-10-09T04:47:49.190820+0000 mgr.y (mgr.14466) 216 : cluster [DBG] pgmap v191: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T04:47:50.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:50 smithi194 bash[31648]: audit 2023-10-09T04:47:50.194662+0000 mon.a (mon.0) 936 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:50.896 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 30 16 57403 57387 119.542 80.875 0.005038 0.00835993 2023-10-09T04:47:51.896 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 31 16 58729 58713 118.359 82.875 0.0125414 0.00844374 2023-10-09T04:47:52.609 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:52.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:52 smithi194 bash[31648]: cluster 2023-10-09T04:47:51.191762+0000 mgr.y (mgr.14466) 217 : cluster [DBG] pgmap v192: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-10-09T04:47:52.896 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 32 16 60014 59998 117.17 80.3125 0.00337821 0.00852896 2023-10-09T04:47:53.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:52 smithi133 bash[27558]: cluster 2023-10-09T04:47:51.191762+0000 mgr.y (mgr.14466) 217 : cluster [DBG] pgmap v192: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-10-09T04:47:53.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:52 smithi133 bash[41291]: cluster 2023-10-09T04:47:51.191762+0000 mgr.y (mgr.14466) 217 : cluster [DBG] pgmap v192: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-10-09T04:47:53.896 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 33 16 61310 61294 116.074 81 0.00812325 0.00860998 2023-10-09T04:47:54.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:54 smithi194 bash[31648]: cluster 2023-10-09T04:47:53.192232+0000 mgr.y (mgr.14466) 218 : cluster [DBG] pgmap v193: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T04:47:54.896 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 34 16 62438 62422 114.733 70.5 0.00288989 0.00867939 2023-10-09T04:47:55.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:54 smithi133 bash[41291]: cluster 2023-10-09T04:47:53.192232+0000 mgr.y (mgr.14466) 218 : cluster [DBG] pgmap v193: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T04:47:55.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:54 smithi133 bash[27558]: cluster 2023-10-09T04:47:53.192232+0000 mgr.y (mgr.14466) 218 : cluster [DBG] pgmap v193: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T04:47:55.897 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 35 16 63676 63660 113.666 77.375 0.0116871 0.00879236 2023-10-09T04:47:56.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:56 smithi194 bash[31648]: cluster 2023-10-09T04:47:55.192619+0000 mgr.y (mgr.14466) 219 : cluster [DBG] pgmap v194: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T04:47:56.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:47:56 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:47:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:47:56.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:56 smithi133 bash[41291]: cluster 2023-10-09T04:47:55.192619+0000 mgr.y (mgr.14466) 219 : cluster [DBG] pgmap v194: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T04:47:56.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:56 smithi133 bash[27558]: cluster 2023-10-09T04:47:55.192619+0000 mgr.y (mgr.14466) 219 : cluster [DBG] pgmap v194: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T04:47:56.897 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 36 16 64776 64760 112.418 68.75 0.0281192 0.00888916 2023-10-09T04:47:57.897 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 37 16 65880 65864 111.244 69 0.00923711 0.00898306 2023-10-09T04:47:58.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:47:58 smithi194 bash[31648]: cluster 2023-10-09T04:47:57.195078+0000 mgr.y (mgr.14466) 220 : cluster [DBG] pgmap v195: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-09T04:47:58.897 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 38 16 66971 66955 110.111 68.1875 0.00383546 0.00907578 2023-10-09T04:47:59.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:47:58 smithi133 bash[27558]: cluster 2023-10-09T04:47:57.195078+0000 mgr.y (mgr.14466) 220 : cluster [DBG] pgmap v195: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-09T04:47:59.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:47:58 smithi133 bash[41291]: cluster 2023-10-09T04:47:57.195078+0000 mgr.y (mgr.14466) 220 : cluster [DBG] pgmap v195: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-09T04:47:59.897 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 39 16 67926 67910 108.818 59.6875 0.00334972 0.0091835 2023-10-09T04:48:00.610 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:00 smithi133 bash[27558]: cluster 2023-10-09T04:47:59.195514+0000 mgr.y (mgr.14466) 221 : cluster [DBG] pgmap v196: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T04:48:00.611 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:48:00 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:48:00] "GET /metrics HTTP/1.1" 200 242334 "" "Prometheus/2.43.0" 2023-10-09T04:48:00.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:00 smithi133 bash[41291]: cluster 2023-10-09T04:47:59.195514+0000 mgr.y (mgr.14466) 221 : cluster [DBG] pgmap v196: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T04:48:00.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:00 smithi194 bash[31648]: cluster 2023-10-09T04:47:59.195514+0000 mgr.y (mgr.14466) 221 : cluster [DBG] pgmap v196: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T04:48:00.897 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T04:48:00.895002+0000 min lat: 0.00129931 max lat: 0.176796 avg lat: 0.00927321 2023-10-09T04:48:00.897 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:48:00.897 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 40 16 68998 68982 107.772 67 0.0245163 0.00927321 2023-10-09T04:48:01.897 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 41 16 70073 70057 106.782 67.1875 0.0107262 0.00935893 2023-10-09T04:48:02.653 INFO:teuthology.orchestra.run.smithi133.stdout:wait for minority of mons to upgrade 2023-10-09T04:48:02.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:02 smithi194 bash[31648]: cluster 2023-10-09T04:48:01.196325+0000 mgr.y (mgr.14466) 222 : cluster [DBG] pgmap v197: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-09T04:48:02.897 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 42 16 71147 71131 105.838 67.125 0.0207855 0.00944251 2023-10-09T04:48:02.934 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:02 smithi133 bash[41291]: cluster 2023-10-09T04:48:01.196325+0000 mgr.y (mgr.14466) 222 : cluster [DBG] pgmap v197: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-09T04:48:02.934 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:02 smithi133 bash[27558]: cluster 2023-10-09T04:48:01.196325+0000 mgr.y (mgr.14466) 222 : cluster [DBG] pgmap v197: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-09T04:48:03.897 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 43 16 72196 72180 104.901 65.5625 0.00469916 0.00952656 2023-10-09T04:48:04.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:04 smithi194 bash[31648]: cluster 2023-10-09T04:48:03.196720+0000 mgr.y (mgr.14466) 223 : cluster [DBG] pgmap v198: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 961 op/s 2023-10-09T04:48:04.897 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 44 16 73225 73209 103.978 64.3125 0.0428364 0.00961155 2023-10-09T04:48:05.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:04 smithi133 bash[41291]: cluster 2023-10-09T04:48:03.196720+0000 mgr.y (mgr.14466) 223 : cluster [DBG] pgmap v198: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 961 op/s 2023-10-09T04:48:05.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:04 smithi133 bash[27558]: cluster 2023-10-09T04:48:03.196720+0000 mgr.y (mgr.14466) 223 : cluster [DBG] pgmap v198: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 961 op/s 2023-10-09T04:48:05.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:05 smithi194 bash[31648]: audit 2023-10-09T04:48:05.194952+0000 mon.a (mon.0) 937 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:05.898 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 45 16 74223 74207 103.054 62.375 0.0120423 0.00969731 2023-10-09T04:48:06.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:05 smithi133 bash[27558]: audit 2023-10-09T04:48:05.194952+0000 mon.a (mon.0) 937 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:06.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:05 smithi133 bash[41291]: audit 2023-10-09T04:48:05.194952+0000 mon.a (mon.0) 937 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:06.470 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2023-10-09T04:48:06.629 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:48:06 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:48:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:48:06.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:06 smithi133 bash[27558]: cluster 2023-10-09T04:48:05.197333+0000 mgr.y (mgr.14466) 224 : cluster [DBG] pgmap v199: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 961 op/s 2023-10-09T04:48:06.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:06 smithi133 bash[41291]: cluster 2023-10-09T04:48:05.197333+0000 mgr.y (mgr.14466) 224 : cluster [DBG] pgmap v199: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 961 op/s 2023-10-09T04:48:06.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:06 smithi194 bash[31648]: cluster 2023-10-09T04:48:05.197333+0000 mgr.y (mgr.14466) 224 : cluster [DBG] pgmap v199: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 961 op/s 2023-10-09T04:48:06.898 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 46 16 75199 75183 102.139 61 0.012629 0.00978411 2023-10-09T04:48:07.898 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 47 16 76162 76146 101.247 60.1875 0.00802571 0.00987018 2023-10-09T04:48:08.898 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 48 16 77146 77130 100.418 61.5 0.0245644 0.00995226 2023-10-09T04:48:09.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:08 smithi133 bash[41291]: cluster 2023-10-09T04:48:07.199437+0000 mgr.y (mgr.14466) 225 : cluster [DBG] pgmap v200: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T04:48:09.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:08 smithi133 bash[27558]: cluster 2023-10-09T04:48:07.199437+0000 mgr.y (mgr.14466) 225 : cluster [DBG] pgmap v200: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T04:48:09.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:08 smithi194 bash[31648]: cluster 2023-10-09T04:48:07.199437+0000 mgr.y (mgr.14466) 225 : cluster [DBG] pgmap v200: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T04:48:09.898 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 49 16 78071 78055 99.5489 57.8125 0.0188638 0.0100395 2023-10-09T04:48:10.645 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:48:10 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:48:10] "GET /metrics HTTP/1.1" 200 242397 "" "Prometheus/2.43.0" 2023-10-09T04:48:10.898 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 50 16 79115 79099 98.8628 65.25 0.00948776 0.0101088 2023-10-09T04:48:11.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:10 smithi133 bash[27558]: cluster 2023-10-09T04:48:09.200138+0000 mgr.y (mgr.14466) 226 : cluster [DBG] pgmap v201: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 876 op/s 2023-10-09T04:48:11.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:10 smithi133 bash[41291]: cluster 2023-10-09T04:48:09.200138+0000 mgr.y (mgr.14466) 226 : cluster [DBG] pgmap v201: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 876 op/s 2023-10-09T04:48:11.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:10 smithi194 bash[31648]: cluster 2023-10-09T04:48:09.200138+0000 mgr.y (mgr.14466) 226 : cluster [DBG] pgmap v201: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 876 op/s 2023-10-09T04:48:11.326 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:48:11.898 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 51 16 80221 80205 98.2795 69.125 0.0143426 0.0101695 2023-10-09T04:48:12.898 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 52 16 81326 81310 97.7176 69.0625 0.00972441 0.0102275 2023-10-09T04:48:13.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:12 smithi133 bash[41291]: cluster 2023-10-09T04:48:11.200879+0000 mgr.y (mgr.14466) 227 : cluster [DBG] pgmap v202: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2023-10-09T04:48:13.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:12 smithi133 bash[27558]: cluster 2023-10-09T04:48:11.200879+0000 mgr.y (mgr.14466) 227 : cluster [DBG] pgmap v202: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2023-10-09T04:48:13.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:12 smithi194 bash[31648]: cluster 2023-10-09T04:48:11.200879+0000 mgr.y (mgr.14466) 227 : cluster [DBG] pgmap v202: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2023-10-09T04:48:13.898 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 53 16 82425 82409 97.1697 68.6875 0.0110711 0.0102857 2023-10-09T04:48:14.899 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 54 15 83486 83471 96.5993 66.375 0.0144745 0.0103466 2023-10-09T04:48:15.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:14 smithi133 bash[41291]: cluster 2023-10-09T04:48:13.201282+0000 mgr.y (mgr.14466) 228 : cluster [DBG] pgmap v203: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s 2023-10-09T04:48:15.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:14 smithi133 bash[27558]: cluster 2023-10-09T04:48:13.201282+0000 mgr.y (mgr.14466) 228 : cluster [DBG] pgmap v203: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s 2023-10-09T04:48:15.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:14 smithi194 bash[31648]: cluster 2023-10-09T04:48:13.201282+0000 mgr.y (mgr.14466) 228 : cluster [DBG] pgmap v203: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s 2023-10-09T04:48:15.899 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 55 16 84521 84505 96.0179 64.625 0.0560109 0.0104058 2023-10-09T04:48:16.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:48:16 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:48:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:48:16.899 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 56 16 85550 85534 95.4516 64.3125 0.0180912 0.0104706 2023-10-09T04:48:17.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:16 smithi133 bash[41291]: cluster 2023-10-09T04:48:15.201733+0000 mgr.y (mgr.14466) 229 : cluster [DBG] pgmap v204: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s 2023-10-09T04:48:17.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:16 smithi133 bash[27558]: cluster 2023-10-09T04:48:15.201733+0000 mgr.y (mgr.14466) 229 : cluster [DBG] pgmap v204: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s 2023-10-09T04:48:17.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:16 smithi194 bash[31648]: cluster 2023-10-09T04:48:15.201733+0000 mgr.y (mgr.14466) 229 : cluster [DBG] pgmap v204: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s 2023-10-09T04:48:17.899 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 57 16 86541 86525 94.8635 61.9375 0.00801691 0.0105359 2023-10-09T04:48:18.899 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 58 16 87583 87567 94.3507 65.125 0.00933893 0.0105933 2023-10-09T04:48:19.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:18 smithi133 bash[27558]: cluster 2023-10-09T04:48:17.202533+0000 mgr.y (mgr.14466) 230 : cluster [DBG] pgmap v205: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T04:48:19.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:18 smithi133 bash[41291]: cluster 2023-10-09T04:48:17.202533+0000 mgr.y (mgr.14466) 230 : cluster [DBG] pgmap v205: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T04:48:19.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:18 smithi194 bash[31648]: cluster 2023-10-09T04:48:17.202533+0000 mgr.y (mgr.14466) 230 : cluster [DBG] pgmap v205: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T04:48:19.899 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 59 16 88544 88528 93.7695 60.0625 0.0141922 0.0106592 2023-10-09T04:48:20.550 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:48:20 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:48:20] "GET /metrics HTTP/1.1" 200 242463 "" "Prometheus/2.43.0" 2023-10-09T04:48:20.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:20 smithi133 bash[41291]: cluster 2023-10-09T04:48:19.203048+0000 mgr.y (mgr.14466) 231 : cluster [DBG] pgmap v206: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 861 op/s 2023-10-09T04:48:20.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:20 smithi133 bash[41291]: audit 2023-10-09T04:48:19.287777+0000 mon.a (mon.0) 938 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:48:20.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:20 smithi133 bash[41291]: cephadm 2023-10-09T04:48:19.288413+0000 mgr.y (mgr.14466) 232 : cephadm [INF] Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T04:48:20.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:20 smithi133 bash[41291]: cephadm 2023-10-09T04:48:19.288563+0000 mgr.y (mgr.14466) 233 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T04:48:20.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:20 smithi133 bash[41291]: audit 2023-10-09T04:48:19.290076+0000 mon.a (mon.0) 939 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:48:20.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:20 smithi133 bash[41291]: audit 2023-10-09T04:48:19.301860+0000 mon.a (mon.0) 940 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:48:20.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:20 smithi133 bash[41291]: cephadm 2023-10-09T04:48:19.304267+0000 mgr.y (mgr.14466) 234 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T04:48:20.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:20 smithi133 bash[41291]: cephadm 2023-10-09T04:48:19.672922+0000 mgr.y (mgr.14466) 235 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi194 2023-10-09T04:48:20.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:20 smithi133 bash[41291]: audit 2023-10-09T04:48:20.197593+0000 mon.a (mon.0) 941 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:20.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:20 smithi133 bash[27558]: cluster 2023-10-09T04:48:19.203048+0000 mgr.y (mgr.14466) 231 : cluster [DBG] pgmap v206: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 861 op/s 2023-10-09T04:48:20.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:20 smithi133 bash[27558]: audit 2023-10-09T04:48:19.287777+0000 mon.a (mon.0) 938 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:48:20.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:20 smithi133 bash[27558]: cephadm 2023-10-09T04:48:19.288413+0000 mgr.y (mgr.14466) 232 : cephadm [INF] Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T04:48:20.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:20 smithi133 bash[27558]: cephadm 2023-10-09T04:48:19.288563+0000 mgr.y (mgr.14466) 233 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T04:48:20.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:20 smithi133 bash[27558]: audit 2023-10-09T04:48:19.290076+0000 mon.a (mon.0) 939 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:48:20.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:20 smithi133 bash[27558]: audit 2023-10-09T04:48:19.301860+0000 mon.a (mon.0) 940 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:48:20.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:20 smithi133 bash[27558]: cephadm 2023-10-09T04:48:19.304267+0000 mgr.y (mgr.14466) 234 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T04:48:20.553 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:20 smithi133 bash[27558]: cephadm 2023-10-09T04:48:19.672922+0000 mgr.y (mgr.14466) 235 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi194 2023-10-09T04:48:20.553 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:20 smithi133 bash[27558]: audit 2023-10-09T04:48:20.197593+0000 mon.a (mon.0) 941 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:20.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:20 smithi194 bash[31648]: cluster 2023-10-09T04:48:19.203048+0000 mgr.y (mgr.14466) 231 : cluster [DBG] pgmap v206: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 861 op/s 2023-10-09T04:48:20.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:20 smithi194 bash[31648]: audit 2023-10-09T04:48:19.287777+0000 mon.a (mon.0) 938 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:48:20.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:20 smithi194 bash[31648]: cephadm 2023-10-09T04:48:19.288413+0000 mgr.y (mgr.14466) 232 : cephadm [INF] Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T04:48:20.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:20 smithi194 bash[31648]: cephadm 2023-10-09T04:48:19.288563+0000 mgr.y (mgr.14466) 233 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T04:48:20.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:20 smithi194 bash[31648]: audit 2023-10-09T04:48:19.290076+0000 mon.a (mon.0) 939 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:48:20.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:20 smithi194 bash[31648]: audit 2023-10-09T04:48:19.301860+0000 mon.a (mon.0) 940 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:48:20.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:20 smithi194 bash[31648]: cephadm 2023-10-09T04:48:19.304267+0000 mgr.y (mgr.14466) 234 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T04:48:20.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:20 smithi194 bash[31648]: cephadm 2023-10-09T04:48:19.672922+0000 mgr.y (mgr.14466) 235 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi194 2023-10-09T04:48:20.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:20 smithi194 bash[31648]: audit 2023-10-09T04:48:20.197593+0000 mon.a (mon.0) 941 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:20.899 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T04:48:20.896988+0000 min lat: 0.00129931 max lat: 0.176796 avg lat: 0.010725 2023-10-09T04:48:20.899 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:48:20.899 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 60 16 89491 89475 93.193 59.1875 0.0134456 0.010725 2023-10-09T04:48:21.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:21 smithi133 bash[27558]: audit 2023-10-09T04:48:21.182263+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.133:0/3232061191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:21.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:21 smithi133 bash[41291]: audit 2023-10-09T04:48:21.182263+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.133:0/3232061191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:21.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:21 smithi194 bash[31648]: audit 2023-10-09T04:48:21.182263+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.133:0/3232061191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:21.899 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 61 16 90409 90393 92.6058 57.375 0.0186787 0.0107923 2023-10-09T04:48:22.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:22 smithi133 bash[27558]: cluster 2023-10-09T04:48:21.203856+0000 mgr.y (mgr.14466) 236 : cluster [DBG] pgmap v207: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T04:48:22.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:22 smithi133 bash[41291]: cluster 2023-10-09T04:48:21.203856+0000 mgr.y (mgr.14466) 236 : cluster [DBG] pgmap v207: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T04:48:22.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:22 smithi194 bash[31648]: cluster 2023-10-09T04:48:21.203856+0000 mgr.y (mgr.14466) 236 : cluster [DBG] pgmap v207: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T04:48:22.899 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 62 16 91319 91303 92.0294 56.875 0.0194516 0.0108586 2023-10-09T04:48:23.899 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 63 16 92217 92201 91.4594 56.125 0.0270581 0.0109287 2023-10-09T04:48:24.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:24 smithi133 bash[27558]: cluster 2023-10-09T04:48:23.204320+0000 mgr.y (mgr.14466) 237 : cluster [DBG] pgmap v208: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-09T04:48:24.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:24 smithi133 bash[27558]: audit 2023-10-09T04:48:23.844687+0000 mon.a (mon.0) 942 : audit [DBG] from='client.? 172.21.15.133:0/600315209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:24.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:24 smithi133 bash[41291]: cluster 2023-10-09T04:48:23.204320+0000 mgr.y (mgr.14466) 237 : cluster [DBG] pgmap v208: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-09T04:48:24.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:24 smithi133 bash[41291]: audit 2023-10-09T04:48:23.844687+0000 mon.a (mon.0) 942 : audit [DBG] from='client.? 172.21.15.133:0/600315209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:24.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:24 smithi194 bash[31648]: cluster 2023-10-09T04:48:23.204320+0000 mgr.y (mgr.14466) 237 : cluster [DBG] pgmap v208: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-09T04:48:24.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:24 smithi194 bash[31648]: audit 2023-10-09T04:48:23.844687+0000 mon.a (mon.0) 942 : audit [DBG] from='client.? 172.21.15.133:0/600315209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:24.899 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 64 16 93120 93104 90.9121 56.4375 0.00693576 0.0109938 2023-10-09T04:48:25.900 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 65 16 94048 94032 90.4057 58 0.0120209 0.0110546 2023-10-09T04:48:26.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:48:26 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:48:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:48:26.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:26 smithi194 bash[31648]: cluster 2023-10-09T04:48:25.204804+0000 mgr.y (mgr.14466) 238 : cluster [DBG] pgmap v209: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-09T04:48:26.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:26 smithi194 bash[31648]: audit 2023-10-09T04:48:26.463078+0000 mon.a (mon.0) 943 : audit [DBG] from='client.? 172.21.15.133:0/1827735626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:26.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:26 smithi133 bash[27558]: cluster 2023-10-09T04:48:25.204804+0000 mgr.y (mgr.14466) 238 : cluster [DBG] pgmap v209: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-09T04:48:26.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:26 smithi133 bash[27558]: audit 2023-10-09T04:48:26.463078+0000 mon.a (mon.0) 943 : audit [DBG] from='client.? 172.21.15.133:0/1827735626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:26.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:26 smithi133 bash[41291]: cluster 2023-10-09T04:48:25.204804+0000 mgr.y (mgr.14466) 238 : cluster [DBG] pgmap v209: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-09T04:48:26.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:26 smithi133 bash[41291]: audit 2023-10-09T04:48:26.463078+0000 mon.a (mon.0) 943 : audit [DBG] from='client.? 172.21.15.133:0/1827735626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:26.900 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 66 16 94948 94932 89.8881 56.25 0.014422 0.0111189 2023-10-09T04:48:27.900 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 67 16 95840 95824 89.3785 55.75 0.0117003 0.0111832 2023-10-09T04:48:28.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:28 smithi194 bash[31648]: cluster 2023-10-09T04:48:27.205585+0000 mgr.y (mgr.14466) 239 : cluster [DBG] pgmap v210: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-10-09T04:48:28.900 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 68 16 96708 96692 88.8618 54.25 0.04599 0.0112467 2023-10-09T04:48:29.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:28 smithi133 bash[41291]: cluster 2023-10-09T04:48:27.205585+0000 mgr.y (mgr.14466) 239 : cluster [DBG] pgmap v210: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-10-09T04:48:29.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:28 smithi133 bash[27558]: cluster 2023-10-09T04:48:27.205585+0000 mgr.y (mgr.14466) 239 : cluster [DBG] pgmap v210: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-10-09T04:48:29.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:29 smithi194 bash[31648]: audit 2023-10-09T04:48:29.108128+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.133:0/2354365177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:29.900 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 69 16 97571 97555 88.3556 53.9375 0.0249541 0.011312 2023-10-09T04:48:30.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:29 smithi133 bash[41291]: audit 2023-10-09T04:48:29.108128+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.133:0/2354365177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:30.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:29 smithi133 bash[27558]: audit 2023-10-09T04:48:29.108128+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.133:0/2354365177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:30.800 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:48:30 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:48:30] "GET /metrics HTTP/1.1" 200 242463 "" "Prometheus/2.43.0" 2023-10-09T04:48:30.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:30 smithi133 bash[41291]: cluster 2023-10-09T04:48:29.206098+0000 mgr.y (mgr.14466) 240 : cluster [DBG] pgmap v211: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 49 MiB/s wr, 789 op/s 2023-10-09T04:48:30.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:30 smithi133 bash[27558]: cluster 2023-10-09T04:48:29.206098+0000 mgr.y (mgr.14466) 240 : cluster [DBG] pgmap v211: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 49 MiB/s wr, 789 op/s 2023-10-09T04:48:30.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:30 smithi194 bash[31648]: cluster 2023-10-09T04:48:29.206098+0000 mgr.y (mgr.14466) 240 : cluster [DBG] pgmap v211: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 49 MiB/s wr, 789 op/s 2023-10-09T04:48:30.900 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 70 16 98475 98459 87.9005 56.5 0.0191464 0.0113706 2023-10-09T04:48:31.900 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 71 16 99321 99305 87.4071 52.875 0.0217731 0.0114344 2023-10-09T04:48:32.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:32 smithi194 bash[31648]: cluster 2023-10-09T04:48:31.209437+0000 mgr.y (mgr.14466) 241 : cluster [DBG] pgmap v212: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T04:48:32.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:32 smithi194 bash[31648]: audit 2023-10-09T04:48:31.774218+0000 mon.a (mon.0) 944 : audit [DBG] from='client.? 172.21.15.133:0/2097262274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:32.900 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 72 16 100193 100177 86.95 54.5 0.0384155 0.0114952 2023-10-09T04:48:33.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:32 smithi133 bash[41291]: cluster 2023-10-09T04:48:31.209437+0000 mgr.y (mgr.14466) 241 : cluster [DBG] pgmap v212: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T04:48:33.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:32 smithi133 bash[41291]: audit 2023-10-09T04:48:31.774218+0000 mon.a (mon.0) 944 : audit [DBG] from='client.? 172.21.15.133:0/2097262274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:33.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:32 smithi133 bash[27558]: cluster 2023-10-09T04:48:31.209437+0000 mgr.y (mgr.14466) 241 : cluster [DBG] pgmap v212: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T04:48:33.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:32 smithi133 bash[27558]: audit 2023-10-09T04:48:31.774218+0000 mon.a (mon.0) 944 : audit [DBG] from='client.? 172.21.15.133:0/2097262274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:33.900 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 73 16 101068 101052 86.5079 54.6875 0.0146954 0.0115544 2023-10-09T04:48:34.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:34 smithi194 bash[31648]: cluster 2023-10-09T04:48:33.209947+0000 mgr.y (mgr.14466) 242 : cluster [DBG] pgmap v213: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s 2023-10-09T04:48:34.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:34 smithi194 bash[31648]: audit 2023-10-09T04:48:34.398307+0000 mon.a (mon.0) 945 : audit [DBG] from='client.? 172.21.15.133:0/3216725474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:34.900 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 74 16 101903 101887 86.0441 52.1875 0.0327559 0.0116165 2023-10-09T04:48:35.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:34 smithi133 bash[41291]: cluster 2023-10-09T04:48:33.209947+0000 mgr.y (mgr.14466) 242 : cluster [DBG] pgmap v213: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s 2023-10-09T04:48:35.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:34 smithi133 bash[41291]: audit 2023-10-09T04:48:34.398307+0000 mon.a (mon.0) 945 : audit [DBG] from='client.? 172.21.15.133:0/3216725474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:35.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:34 smithi133 bash[27558]: cluster 2023-10-09T04:48:33.209947+0000 mgr.y (mgr.14466) 242 : cluster [DBG] pgmap v213: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s 2023-10-09T04:48:35.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:34 smithi133 bash[27558]: audit 2023-10-09T04:48:34.398307+0000 mon.a (mon.0) 945 : audit [DBG] from='client.? 172.21.15.133:0/3216725474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:35.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:35 smithi194 bash[31648]: audit 2023-10-09T04:48:35.195388+0000 mon.a (mon.0) 946 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:35.900 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 75 16 102774 102758 85.6226 54.4375 0.0127629 0.0116735 2023-10-09T04:48:36.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:35 smithi133 bash[27558]: audit 2023-10-09T04:48:35.195388+0000 mon.a (mon.0) 946 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:36.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:35 smithi133 bash[41291]: audit 2023-10-09T04:48:35.195388+0000 mon.a (mon.0) 946 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:36.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:48:36 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:48:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:48:36.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:36 smithi194 bash[31648]: cluster 2023-10-09T04:48:35.213084+0000 mgr.y (mgr.14466) 243 : cluster [DBG] pgmap v214: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s 2023-10-09T04:48:36.901 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 76 16 103641 103625 85.2089 54.1875 0.0159901 0.0117296 2023-10-09T04:48:37.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:36 smithi133 bash[41291]: cluster 2023-10-09T04:48:35.213084+0000 mgr.y (mgr.14466) 243 : cluster [DBG] pgmap v214: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s 2023-10-09T04:48:37.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:36 smithi133 bash[27558]: cluster 2023-10-09T04:48:35.213084+0000 mgr.y (mgr.14466) 243 : cluster [DBG] pgmap v214: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s 2023-10-09T04:48:37.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:37 smithi194 bash[31648]: audit 2023-10-09T04:48:37.066811+0000 mon.a (mon.0) 947 : audit [DBG] from='client.? 172.21.15.133:0/213090777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:37.901 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 77 16 104430 104414 84.7427 49.3125 0.0186258 0.0117943 2023-10-09T04:48:38.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:37 smithi133 bash[27558]: audit 2023-10-09T04:48:37.066811+0000 mon.a (mon.0) 947 : audit [DBG] from='client.? 172.21.15.133:0/213090777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:38.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:37 smithi133 bash[41291]: audit 2023-10-09T04:48:37.066811+0000 mon.a (mon.0) 947 : audit [DBG] from='client.? 172.21.15.133:0/213090777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:38.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:38 smithi194 bash[31648]: cluster 2023-10-09T04:48:37.213710+0000 mgr.y (mgr.14466) 244 : cluster [DBG] pgmap v215: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T04:48:38.901 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 78 16 105172 105156 84.2507 46.375 0.0170357 0.0118635 2023-10-09T04:48:39.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:38 smithi133 bash[41291]: cluster 2023-10-09T04:48:37.213710+0000 mgr.y (mgr.14466) 244 : cluster [DBG] pgmap v215: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T04:48:39.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:38 smithi133 bash[27558]: cluster 2023-10-09T04:48:37.213710+0000 mgr.y (mgr.14466) 244 : cluster [DBG] pgmap v215: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T04:48:39.901 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 79 16 105895 105879 83.7562 45.1875 0.012204 0.0119336 2023-10-09T04:48:40.619 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:48:40 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:48:40] "GET /metrics HTTP/1.1" 200 242488 "" "Prometheus/2.43.0" 2023-10-09T04:48:40.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:40 smithi194 bash[31648]: cluster 2023-10-09T04:48:39.214101+0000 mgr.y (mgr.14466) 245 : cluster [DBG] pgmap v216: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 727 op/s 2023-10-09T04:48:40.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:40 smithi194 bash[31648]: audit 2023-10-09T04:48:39.700419+0000 mon.b (mon.1) 30 : audit [DBG] from='client.? 172.21.15.133:0/3504012275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:40.901 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T04:48:40.898891+0000 min lat: 0.00129931 max lat: 0.176796 avg lat: 0.0120024 2023-10-09T04:48:40.901 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:48:40.901 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 80 16 106619 106603 83.2748 45.25 0.035827 0.0120024 2023-10-09T04:48:41.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:40 smithi133 bash[27558]: cluster 2023-10-09T04:48:39.214101+0000 mgr.y (mgr.14466) 245 : cluster [DBG] pgmap v216: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 727 op/s 2023-10-09T04:48:41.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:40 smithi133 bash[27558]: audit 2023-10-09T04:48:39.700419+0000 mon.b (mon.1) 30 : audit [DBG] from='client.? 172.21.15.133:0/3504012275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:41.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:40 smithi133 bash[41291]: cluster 2023-10-09T04:48:39.214101+0000 mgr.y (mgr.14466) 245 : cluster [DBG] pgmap v216: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 727 op/s 2023-10-09T04:48:41.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:40 smithi133 bash[41291]: audit 2023-10-09T04:48:39.700419+0000 mon.b (mon.1) 30 : audit [DBG] from='client.? 172.21.15.133:0/3504012275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:41.902 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 81 16 107358 107342 82.8169 46.1875 0.0299745 0.0120685 2023-10-09T04:48:42.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:42 smithi194 bash[31648]: cluster 2023-10-09T04:48:41.214800+0000 mgr.y (mgr.14466) 246 : cluster [DBG] pgmap v217: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s 2023-10-09T04:48:42.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:42 smithi194 bash[31648]: audit 2023-10-09T04:48:42.325911+0000 mon.b (mon.1) 31 : audit [DBG] from='client.? 172.21.15.133:0/2712256802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:42.901 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 82 16 108060 108044 82.342 43.875 0.0115632 0.012137 2023-10-09T04:48:43.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:42 smithi133 bash[27558]: cluster 2023-10-09T04:48:41.214800+0000 mgr.y (mgr.14466) 246 : cluster [DBG] pgmap v217: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s 2023-10-09T04:48:43.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:42 smithi133 bash[27558]: audit 2023-10-09T04:48:42.325911+0000 mon.b (mon.1) 31 : audit [DBG] from='client.? 172.21.15.133:0/2712256802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:43.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:42 smithi133 bash[41291]: cluster 2023-10-09T04:48:41.214800+0000 mgr.y (mgr.14466) 246 : cluster [DBG] pgmap v217: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s 2023-10-09T04:48:43.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:42 smithi133 bash[41291]: audit 2023-10-09T04:48:42.325911+0000 mon.b (mon.1) 31 : audit [DBG] from='client.? 172.21.15.133:0/2712256802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:43.902 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 83 16 108764 108748 81.88 44 0.0140597 0.0122072 2023-10-09T04:48:44.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:44 smithi194 bash[31648]: cluster 2023-10-09T04:48:43.215410+0000 mgr.y (mgr.14466) 247 : cluster [DBG] pgmap v218: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 788 op/s 2023-10-09T04:48:44.901 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 84 16 109422 109406 81.3947 41.125 0.0166986 0.01228 2023-10-09T04:48:45.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:44 smithi133 bash[41291]: cluster 2023-10-09T04:48:43.215410+0000 mgr.y (mgr.14466) 247 : cluster [DBG] pgmap v218: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 788 op/s 2023-10-09T04:48:45.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:44 smithi133 bash[27558]: cluster 2023-10-09T04:48:43.215410+0000 mgr.y (mgr.14466) 247 : cluster [DBG] pgmap v218: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 788 op/s 2023-10-09T04:48:45.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:45 smithi194 bash[31648]: audit 2023-10-09T04:48:44.925652+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.133:0/2925504028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:45.902 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 85 16 110106 110090 80.94 42.75 0.0124131 0.0123476 2023-10-09T04:48:46.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:45 smithi133 bash[41291]: audit 2023-10-09T04:48:44.925652+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.133:0/2925504028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:46.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:45 smithi133 bash[27558]: audit 2023-10-09T04:48:44.925652+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.133:0/2925504028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:46.635 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:48:46 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:48:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:48:46.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:46 smithi194 bash[31648]: cluster 2023-10-09T04:48:45.215829+0000 mgr.y (mgr.14466) 248 : cluster [DBG] pgmap v219: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 42 MiB/s wr, 675 op/s 2023-10-09T04:48:46.902 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 86 16 110809 110793 80.5097 43.9375 0.0174501 0.0124144 2023-10-09T04:48:47.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:46 smithi133 bash[41291]: cluster 2023-10-09T04:48:45.215829+0000 mgr.y (mgr.14466) 248 : cluster [DBG] pgmap v219: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 42 MiB/s wr, 675 op/s 2023-10-09T04:48:47.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:46 smithi133 bash[27558]: cluster 2023-10-09T04:48:45.215829+0000 mgr.y (mgr.14466) 248 : cluster [DBG] pgmap v219: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 42 MiB/s wr, 675 op/s 2023-10-09T04:48:47.902 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 87 16 111502 111486 80.0822 43.3125 0.0102231 0.0124796 2023-10-09T04:48:48.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:47 smithi133 bash[41291]: audit 2023-10-09T04:48:47.546929+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.133:0/143774239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:48.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:47 smithi133 bash[27558]: audit 2023-10-09T04:48:47.546929+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.133:0/143774239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:48.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:47 smithi194 bash[31648]: audit 2023-10-09T04:48:47.546929+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.133:0/143774239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:48.902 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 88 16 112202 112186 79.6692 43.75 0.00901384 0.0125462 2023-10-09T04:48:49.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:48 smithi133 bash[41291]: cluster 2023-10-09T04:48:47.218004+0000 mgr.y (mgr.14466) 249 : cluster [DBG] pgmap v220: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 965 op/s 2023-10-09T04:48:49.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:48 smithi133 bash[27558]: cluster 2023-10-09T04:48:47.218004+0000 mgr.y (mgr.14466) 249 : cluster [DBG] pgmap v220: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 965 op/s 2023-10-09T04:48:49.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:48 smithi194 bash[31648]: cluster 2023-10-09T04:48:47.218004+0000 mgr.y (mgr.14466) 249 : cluster [DBG] pgmap v220: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 965 op/s 2023-10-09T04:48:49.903 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 89 16 112904 112888 79.267 43.875 0.011448 0.0126099 2023-10-09T04:48:50.656 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:48:50 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:48:50] "GET /metrics HTTP/1.1" 200 242506 "" "Prometheus/2.43.0" 2023-10-09T04:48:50.657 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:50 smithi133 bash[27558]: cluster 2023-10-09T04:48:49.218519+0000 mgr.y (mgr.14466) 250 : cluster [DBG] pgmap v221: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 38 MiB/s wr, 606 op/s 2023-10-09T04:48:50.657 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:50 smithi133 bash[27558]: audit 2023-10-09T04:48:50.168184+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.133:0/1717704862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:50.658 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:50 smithi133 bash[27558]: audit 2023-10-09T04:48:50.195367+0000 mon.a (mon.0) 948 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:50.902 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 90 5 113569 113564 78.8557 42.25 0.010083 0.0126759 2023-10-09T04:48:51.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:50 smithi133 bash[41291]: cluster 2023-10-09T04:48:49.218519+0000 mgr.y (mgr.14466) 250 : cluster [DBG] pgmap v221: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 38 MiB/s wr, 606 op/s 2023-10-09T04:48:51.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:50 smithi133 bash[41291]: audit 2023-10-09T04:48:50.168184+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.133:0/1717704862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:51.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:50 smithi133 bash[41291]: audit 2023-10-09T04:48:50.195367+0000 mon.a (mon.0) 948 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:51.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:50 smithi194 bash[31648]: cluster 2023-10-09T04:48:49.218519+0000 mgr.y (mgr.14466) 250 : cluster [DBG] pgmap v221: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 38 MiB/s wr, 606 op/s 2023-10-09T04:48:51.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:50 smithi194 bash[31648]: audit 2023-10-09T04:48:50.168184+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.133:0/1717704862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:51.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:50 smithi194 bash[31648]: audit 2023-10-09T04:48:50.195367+0000 mon.a (mon.0) 948 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:51.902 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Total time run: 90.0141 2023-10-09T04:48:51.903 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Total writes made: 113569 2023-10-09T04:48:51.903 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Write size: 65536 2023-10-09T04:48:51.903 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Object size: 65536 2023-10-09T04:48:51.903 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Bandwidth (MB/sec): 78.855 2023-10-09T04:48:51.903 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Stddev Bandwidth: 36.3757 2023-10-09T04:48:51.903 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Max bandwidth (MB/sec): 207.562 2023-10-09T04:48:51.903 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Min bandwidth (MB/sec): 41.125 2023-10-09T04:48:51.903 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Average IOPS: 1261 2023-10-09T04:48:51.903 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Stddev IOPS: 582.011 2023-10-09T04:48:51.903 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Max IOPS: 3321 2023-10-09T04:48:51.903 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Min IOPS: 658 2023-10-09T04:48:51.904 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Average Latency(s): 0.0126765 2023-10-09T04:48:51.904 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Stddev Latency(s): 0.00921395 2023-10-09T04:48:51.904 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Max latency(s): 0.176796 2023-10-09T04:48:51.904 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Min latency(s): 0.00129931 2023-10-09T04:48:51.912 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T04:48:53.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:52 smithi133 bash[27558]: cluster 2023-10-09T04:48:51.219594+0000 mgr.y (mgr.14466) 251 : cluster [DBG] pgmap v222: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 48 MiB/s wr, 761 op/s 2023-10-09T04:48:53.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:52 smithi133 bash[41291]: cluster 2023-10-09T04:48:51.219594+0000 mgr.y (mgr.14466) 251 : cluster [DBG] pgmap v222: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 48 MiB/s wr, 761 op/s 2023-10-09T04:48:53.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:52 smithi194 bash[31648]: cluster 2023-10-09T04:48:51.219594+0000 mgr.y (mgr.14466) 251 : cluster [DBG] pgmap v222: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 48 MiB/s wr, 761 op/s 2023-10-09T04:48:54.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:53 smithi133 bash[41291]: audit 2023-10-09T04:48:52.852950+0000 mon.a (mon.0) 949 : audit [DBG] from='client.? 172.21.15.133:0/1812285034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:54.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:53 smithi133 bash[27558]: audit 2023-10-09T04:48:52.852950+0000 mon.a (mon.0) 949 : audit [DBG] from='client.? 172.21.15.133:0/1812285034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:54.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:53 smithi194 bash[31648]: audit 2023-10-09T04:48:52.852950+0000 mon.a (mon.0) 949 : audit [DBG] from='client.? 172.21.15.133:0/1812285034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:55.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:54 smithi133 bash[27558]: cluster 2023-10-09T04:48:53.220222+0000 mgr.y (mgr.14466) 252 : cluster [DBG] pgmap v223: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 701 op/s 2023-10-09T04:48:55.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:54 smithi133 bash[41291]: cluster 2023-10-09T04:48:53.220222+0000 mgr.y (mgr.14466) 252 : cluster [DBG] pgmap v223: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 701 op/s 2023-10-09T04:48:55.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:54 smithi194 bash[31648]: cluster 2023-10-09T04:48:53.220222+0000 mgr.y (mgr.14466) 252 : cluster [DBG] pgmap v223: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 701 op/s 2023-10-09T04:48:56.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:55 smithi133 bash[27558]: audit 2023-10-09T04:48:55.532181+0000 mon.a (mon.0) 950 : audit [DBG] from='client.? 172.21.15.133:0/596338170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:56.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:55 smithi133 bash[41291]: audit 2023-10-09T04:48:55.532181+0000 mon.a (mon.0) 950 : audit [DBG] from='client.? 172.21.15.133:0/596338170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:56.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:55 smithi194 bash[31648]: audit 2023-10-09T04:48:55.532181+0000 mon.a (mon.0) 950 : audit [DBG] from='client.? 172.21.15.133:0/596338170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:56.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:48:56 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:48:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:48:57.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:56 smithi133 bash[41291]: cluster 2023-10-09T04:48:55.220629+0000 mgr.y (mgr.14466) 253 : cluster [DBG] pgmap v224: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 35 MiB/s wr, 561 op/s 2023-10-09T04:48:57.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:56 smithi133 bash[27558]: cluster 2023-10-09T04:48:55.220629+0000 mgr.y (mgr.14466) 253 : cluster [DBG] pgmap v224: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 35 MiB/s wr, 561 op/s 2023-10-09T04:48:57.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:56 smithi194 bash[31648]: cluster 2023-10-09T04:48:55.220629+0000 mgr.y (mgr.14466) 253 : cluster [DBG] pgmap v224: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 35 MiB/s wr, 561 op/s 2023-10-09T04:48:59.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:58 smithi133 bash[41291]: cluster 2023-10-09T04:48:57.221479+0000 mgr.y (mgr.14466) 254 : cluster [DBG] pgmap v225: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 35 MiB/s wr, 874 op/s 2023-10-09T04:48:59.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:48:58 smithi133 bash[41291]: audit 2023-10-09T04:48:58.203486+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.133:0/248621135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:59.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:58 smithi133 bash[27558]: cluster 2023-10-09T04:48:57.221479+0000 mgr.y (mgr.14466) 254 : cluster [DBG] pgmap v225: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 35 MiB/s wr, 874 op/s 2023-10-09T04:48:59.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:48:58 smithi133 bash[27558]: audit 2023-10-09T04:48:58.203486+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.133:0/248621135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:48:59.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:58 smithi194 bash[31648]: cluster 2023-10-09T04:48:57.221479+0000 mgr.y (mgr.14466) 254 : cluster [DBG] pgmap v225: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 35 MiB/s wr, 874 op/s 2023-10-09T04:48:59.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:48:58 smithi194 bash[31648]: audit 2023-10-09T04:48:58.203486+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.133:0/248621135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:00.683 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:00 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:49:00] "GET /metrics HTTP/1.1" 200 242506 "" "Prometheus/2.43.0" 2023-10-09T04:49:01.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:00 smithi133 bash[27558]: cluster 2023-10-09T04:48:59.221911+0000 mgr.y (mgr.14466) 255 : cluster [DBG] pgmap v226: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 17 MiB/s wr, 583 op/s 2023-10-09T04:49:01.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:00 smithi133 bash[41291]: cluster 2023-10-09T04:48:59.221911+0000 mgr.y (mgr.14466) 255 : cluster [DBG] pgmap v226: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 17 MiB/s wr, 583 op/s 2023-10-09T04:49:01.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:00 smithi194 bash[31648]: cluster 2023-10-09T04:48:59.221911+0000 mgr.y (mgr.14466) 255 : cluster [DBG] pgmap v226: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 17 MiB/s wr, 583 op/s 2023-10-09T04:49:02.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:01 smithi133 bash[27558]: audit 2023-10-09T04:49:00.893369+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.133:0/3579669612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:02.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:01 smithi133 bash[41291]: audit 2023-10-09T04:49:00.893369+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.133:0/3579669612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:02.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:01 smithi194 bash[31648]: audit 2023-10-09T04:49:00.893369+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.133:0/3579669612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:03.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:02 smithi133 bash[41291]: cluster 2023-10-09T04:49:01.222429+0000 mgr.y (mgr.14466) 256 : cluster [DBG] pgmap v227: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 17 MiB/s wr, 692 op/s 2023-10-09T04:49:03.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:02 smithi133 bash[27558]: cluster 2023-10-09T04:49:01.222429+0000 mgr.y (mgr.14466) 256 : cluster [DBG] pgmap v227: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 17 MiB/s wr, 692 op/s 2023-10-09T04:49:03.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:02 smithi194 bash[31648]: cluster 2023-10-09T04:49:01.222429+0000 mgr.y (mgr.14466) 256 : cluster [DBG] pgmap v227: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 17 MiB/s wr, 692 op/s 2023-10-09T04:49:04.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:03 smithi133 bash[27558]: audit 2023-10-09T04:49:03.557791+0000 mon.a (mon.0) 951 : audit [DBG] from='client.? 172.21.15.133:0/3349245370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:04.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:03 smithi133 bash[41291]: audit 2023-10-09T04:49:03.557791+0000 mon.a (mon.0) 951 : audit [DBG] from='client.? 172.21.15.133:0/3349245370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:04.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:03 smithi194 bash[31648]: audit 2023-10-09T04:49:03.557791+0000 mon.a (mon.0) 951 : audit [DBG] from='client.? 172.21.15.133:0/3349245370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:05.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:04 smithi133 bash[27558]: cluster 2023-10-09T04:49:03.223047+0000 mgr.y (mgr.14466) 257 : cluster [DBG] pgmap v228: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 7.3 MiB/s wr, 785 op/s 2023-10-09T04:49:05.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:04 smithi133 bash[41291]: cluster 2023-10-09T04:49:03.223047+0000 mgr.y (mgr.14466) 257 : cluster [DBG] pgmap v228: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 7.3 MiB/s wr, 785 op/s 2023-10-09T04:49:05.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:04 smithi194 bash[31648]: cluster 2023-10-09T04:49:03.223047+0000 mgr.y (mgr.14466) 257 : cluster [DBG] pgmap v228: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 7.3 MiB/s wr, 785 op/s 2023-10-09T04:49:06.453 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:06 smithi194 bash[31648]: cephadm 2023-10-09T04:49:05.122620+0000 mgr.y (mgr.14466) 258 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T04:49:06.453 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:06 smithi194 bash[31648]: audit 2023-10-09T04:49:05.128477+0000 mon.a (mon.0) 952 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:06.453 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:06 smithi194 bash[31648]: audit 2023-10-09T04:49:05.130287+0000 mon.a (mon.0) 953 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:49:06.453 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:06 smithi194 bash[31648]: audit 2023-10-09T04:49:05.135451+0000 mon.a (mon.0) 954 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:49:06.454 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:06 smithi194 bash[31648]: audit 2023-10-09T04:49:05.136942+0000 mon.a (mon.0) 955 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:49:06.454 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:06 smithi194 bash[31648]: cephadm 2023-10-09T04:49:05.138065+0000 mgr.y (mgr.14466) 259 : cephadm [INF] Deploying daemon mgr.x on smithi194 2023-10-09T04:49:06.454 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:06 smithi194 bash[31648]: audit 2023-10-09T04:49:05.196380+0000 mon.a (mon.0) 956 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:06.454 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:06 smithi194 bash[31648]: cluster 2023-10-09T04:49:05.223408+0000 mgr.y (mgr.14466) 260 : cluster [DBG] pgmap v229: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 668 op/s 2023-10-09T04:49:06.454 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:06 smithi194 bash[32713]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:49:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:49:06.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:06 smithi133 bash[41291]: cephadm 2023-10-09T04:49:05.122620+0000 mgr.y (mgr.14466) 258 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T04:49:06.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:06 smithi133 bash[41291]: audit 2023-10-09T04:49:05.128477+0000 mon.a (mon.0) 952 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:06.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:06 smithi133 bash[41291]: audit 2023-10-09T04:49:05.130287+0000 mon.a (mon.0) 953 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:49:06.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:06 smithi133 bash[41291]: audit 2023-10-09T04:49:05.135451+0000 mon.a (mon.0) 954 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:49:06.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:06 smithi133 bash[41291]: audit 2023-10-09T04:49:05.136942+0000 mon.a (mon.0) 955 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:49:06.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:06 smithi133 bash[41291]: cephadm 2023-10-09T04:49:05.138065+0000 mgr.y (mgr.14466) 259 : cephadm [INF] Deploying daemon mgr.x on smithi194 2023-10-09T04:49:06.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:06 smithi133 bash[41291]: audit 2023-10-09T04:49:05.196380+0000 mon.a (mon.0) 956 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:06.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:06 smithi133 bash[41291]: cluster 2023-10-09T04:49:05.223408+0000 mgr.y (mgr.14466) 260 : cluster [DBG] pgmap v229: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 668 op/s 2023-10-09T04:49:06.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:06 smithi133 bash[27558]: cephadm 2023-10-09T04:49:05.122620+0000 mgr.y (mgr.14466) 258 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T04:49:06.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:06 smithi133 bash[27558]: audit 2023-10-09T04:49:05.128477+0000 mon.a (mon.0) 952 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:06.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:06 smithi133 bash[27558]: audit 2023-10-09T04:49:05.130287+0000 mon.a (mon.0) 953 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:49:06.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:06 smithi133 bash[27558]: audit 2023-10-09T04:49:05.135451+0000 mon.a (mon.0) 954 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:49:06.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:06 smithi133 bash[27558]: audit 2023-10-09T04:49:05.136942+0000 mon.a (mon.0) 955 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:49:06.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:06 smithi133 bash[27558]: cephadm 2023-10-09T04:49:05.138065+0000 mgr.y (mgr.14466) 259 : cephadm [INF] Deploying daemon mgr.x on smithi194 2023-10-09T04:49:06.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:06 smithi133 bash[27558]: audit 2023-10-09T04:49:05.196380+0000 mon.a (mon.0) 956 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:06.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:06 smithi133 bash[27558]: cluster 2023-10-09T04:49:05.223408+0000 mgr.y (mgr.14466) 260 : cluster [DBG] pgmap v229: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 668 op/s 2023-10-09T04:49:07.520 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:07 smithi194 bash[31648]: audit 2023-10-09T04:49:06.226879+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.133:0/1159558341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:07.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:07 smithi133 bash[41291]: audit 2023-10-09T04:49:06.226879+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.133:0/1159558341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:07.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:07 smithi133 bash[27558]: audit 2023-10-09T04:49:06.226879+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.133:0/1159558341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:07.822 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:07 smithi194 systemd[1]: Stopping Ceph mgr.x for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T04:49:08.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:08 smithi194 bash[31648]: cluster 2023-10-09T04:49:07.224145+0000 mgr.y (mgr.14466) 261 : cluster [DBG] pgmap v230: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 1.01k op/s 2023-10-09T04:49:08.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:08 smithi133 bash[27558]: cluster 2023-10-09T04:49:07.224145+0000 mgr.y (mgr.14466) 261 : cluster [DBG] pgmap v230: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 1.01k op/s 2023-10-09T04:49:08.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:08 smithi133 bash[41291]: cluster 2023-10-09T04:49:07.224145+0000 mgr.y (mgr.14466) 261 : cluster [DBG] pgmap v230: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 1.01k op/s 2023-10-09T04:49:09.137 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:08 smithi194 bash[69629]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-mgr-x 2023-10-09T04:49:09.137 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:09 smithi194 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-10-09T04:49:09.137 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:09 smithi194 bash[69687]: Error response from daemon: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-mgr-x 2023-10-09T04:49:09.137 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:09 smithi194 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-10-09T04:49:09.137 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:09 smithi194 systemd[1]: Stopped Ceph mgr.x for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:49:09.479 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:09 smithi194 systemd[1]: Started Ceph mgr.x for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:49:09.479 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:09 smithi194 bash[31648]: audit 2023-10-09T04:49:08.892778+0000 mon.a (mon.0) 957 : audit [DBG] from='client.? 172.21.15.133:0/1754176161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:09.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:09 smithi133 bash[27558]: audit 2023-10-09T04:49:08.892778+0000 mon.a (mon.0) 957 : audit [DBG] from='client.? 172.21.15.133:0/1754176161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:09.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:09 smithi133 bash[41291]: audit 2023-10-09T04:49:08.892778+0000 mon.a (mon.0) 957 : audit [DBG] from='client.? 172.21.15.133:0/1754176161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:10.801 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:10 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:49:10] "GET /metrics HTTP/1.1" 200 242564 "" "Prometheus/2.43.0" 2023-10-09T04:49:10.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:10 smithi133 bash[41291]: cluster 2023-10-09T04:49:09.224559+0000 mgr.y (mgr.14466) 262 : cluster [DBG] pgmap v231: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-10-09T04:49:10.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:10 smithi133 bash[41291]: audit 2023-10-09T04:49:09.447411+0000 mon.a (mon.0) 958 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:10.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:10 smithi133 bash[41291]: audit 2023-10-09T04:49:09.456728+0000 mon.a (mon.0) 959 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:10.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:10 smithi133 bash[41291]: audit 2023-10-09T04:49:09.469969+0000 mon.a (mon.0) 960 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:49:10.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:10 smithi133 bash[27558]: cluster 2023-10-09T04:49:09.224559+0000 mgr.y (mgr.14466) 262 : cluster [DBG] pgmap v231: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-10-09T04:49:10.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:10 smithi133 bash[27558]: audit 2023-10-09T04:49:09.447411+0000 mon.a (mon.0) 958 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:10.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:10 smithi133 bash[27558]: audit 2023-10-09T04:49:09.456728+0000 mon.a (mon.0) 959 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:10.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:10 smithi133 bash[27558]: audit 2023-10-09T04:49:09.469969+0000 mon.a (mon.0) 960 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:49:10.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:10 smithi194 bash[31648]: cluster 2023-10-09T04:49:09.224559+0000 mgr.y (mgr.14466) 262 : cluster [DBG] pgmap v231: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-10-09T04:49:10.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:10 smithi194 bash[31648]: audit 2023-10-09T04:49:09.447411+0000 mon.a (mon.0) 958 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:10.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:10 smithi194 bash[31648]: audit 2023-10-09T04:49:09.456728+0000 mon.a (mon.0) 959 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:10.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:10 smithi194 bash[31648]: audit 2023-10-09T04:49:09.469969+0000 mon.a (mon.0) 960 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:49:11.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:10 smithi194 bash[69734]: debug 2023-10-09T04:49:10.920+0000 7f35ee58b200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:49:11.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:11 smithi194 bash[31648]: audit 2023-10-09T04:49:11.527250+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.133:0/2409159779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:12.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:11 smithi133 bash[41291]: audit 2023-10-09T04:49:11.527250+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.133:0/2409159779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:12.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:11 smithi133 bash[27558]: audit 2023-10-09T04:49:11.527250+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.133:0/2409159779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:12.507 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:12 smithi194 bash[69734]: debug 2023-10-09T04:49:12.203+0000 7f35ee58b200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:49:12.507 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:12 smithi194 bash[69734]: debug 2023-10-09T04:49:12.347+0000 7f35ee58b200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:49:12.507 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:12 smithi194 bash[69734]: debug 2023-10-09T04:49:12.499+0000 7f35ee58b200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:49:12.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:12 smithi194 bash[31648]: cluster 2023-10-09T04:49:11.225106+0000 mgr.y (mgr.14466) 263 : cluster [DBG] pgmap v232: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 761 op/s 2023-10-09T04:49:13.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:12 smithi133 bash[27558]: cluster 2023-10-09T04:49:11.225106+0000 mgr.y (mgr.14466) 263 : cluster [DBG] pgmap v232: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 761 op/s 2023-10-09T04:49:13.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:12 smithi133 bash[41291]: cluster 2023-10-09T04:49:11.225106+0000 mgr.y (mgr.14466) 263 : cluster [DBG] pgmap v232: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 761 op/s 2023-10-09T04:49:13.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:12 smithi194 bash[69734]: debug 2023-10-09T04:49:12.891+0000 7f35ee58b200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:49:13.887 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:13 smithi194 bash[69734]: debug 2023-10-09T04:49:13.427+0000 7f35ee58b200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:49:13.887 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:13 smithi194 bash[69734]: debug 2023-10-09T04:49:13.655+0000 7f35ee58b200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:49:14.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:13 smithi194 bash[69734]: debug 2023-10-09T04:49:13.971+0000 7f35ee58b200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:49:14.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:14 smithi194 bash[31648]: cluster 2023-10-09T04:49:13.225813+0000 mgr.y (mgr.14466) 264 : cluster [DBG] pgmap v233: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 915 op/s 2023-10-09T04:49:14.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:14 smithi194 bash[31648]: audit 2023-10-09T04:49:14.145926+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.133:0/2090307028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:15.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:14 smithi133 bash[41291]: cluster 2023-10-09T04:49:13.225813+0000 mgr.y (mgr.14466) 264 : cluster [DBG] pgmap v233: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 915 op/s 2023-10-09T04:49:15.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:14 smithi133 bash[41291]: audit 2023-10-09T04:49:14.145926+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.133:0/2090307028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:15.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:14 smithi133 bash[27558]: cluster 2023-10-09T04:49:13.225813+0000 mgr.y (mgr.14466) 264 : cluster [DBG] pgmap v233: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 915 op/s 2023-10-09T04:49:15.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:14 smithi133 bash[27558]: audit 2023-10-09T04:49:14.145926+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.133:0/2090307028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:15.887 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:15 smithi194 bash[69734]: debug 2023-10-09T04:49:15.563+0000 7f35ee58b200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:49:15.888 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:15 smithi194 bash[69734]: debug 2023-10-09T04:49:15.743+0000 7f35ee58b200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:49:16.322 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:16 smithi194 bash[31648]: cluster 2023-10-09T04:49:15.226259+0000 mgr.y (mgr.14466) 265 : cluster [DBG] pgmap v234: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-10-09T04:49:16.322 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:16 smithi194 bash[31648]: audit 2023-10-09T04:49:15.319460+0000 mon.a (mon.0) 961 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:16.322 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:16 smithi194 bash[31648]: audit 2023-10-09T04:49:15.334182+0000 mon.a (mon.0) 962 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:16.323 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:15 smithi194 bash[69734]: debug 2023-10-09T04:49:15.919+0000 7f35ee58b200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:49:16.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:16 smithi133 bash[41291]: cluster 2023-10-09T04:49:15.226259+0000 mgr.y (mgr.14466) 265 : cluster [DBG] pgmap v234: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-10-09T04:49:16.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:16 smithi133 bash[41291]: audit 2023-10-09T04:49:15.319460+0000 mon.a (mon.0) 961 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:16.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:16 smithi133 bash[41291]: audit 2023-10-09T04:49:15.334182+0000 mon.a (mon.0) 962 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:16.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:16 smithi133 bash[27558]: cluster 2023-10-09T04:49:15.226259+0000 mgr.y (mgr.14466) 265 : cluster [DBG] pgmap v234: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-10-09T04:49:16.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:16 smithi133 bash[27558]: audit 2023-10-09T04:49:15.319460+0000 mon.a (mon.0) 961 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:16.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:16 smithi133 bash[27558]: audit 2023-10-09T04:49:15.334182+0000 mon.a (mon.0) 962 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:16.981 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:16 smithi194 bash[69734]: debug 2023-10-09T04:49:16.639+0000 7f35ee58b200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:49:16.981 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:16 smithi194 bash[69734]: debug 2023-10-09T04:49:16.971+0000 7f35ee58b200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:49:17.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:17 smithi194 bash[31648]: audit 2023-10-09T04:49:16.347539+0000 mon.a (mon.0) 963 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:17.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:17 smithi194 bash[31648]: audit 2023-10-09T04:49:16.369290+0000 mon.a (mon.0) 964 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:17.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:17 smithi194 bash[31648]: audit 2023-10-09T04:49:16.767290+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.133:0/3820378919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:17.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:17 smithi194 bash[31648]: audit 2023-10-09T04:49:17.225104+0000 mon.a (mon.0) 965 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:17.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:17 smithi194 bash[31648]: audit 2023-10-09T04:49:17.234396+0000 mon.a (mon.0) 966 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:17.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:17 smithi133 bash[41291]: audit 2023-10-09T04:49:16.347539+0000 mon.a (mon.0) 963 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:17.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:17 smithi133 bash[41291]: audit 2023-10-09T04:49:16.369290+0000 mon.a (mon.0) 964 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:17.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:17 smithi133 bash[41291]: audit 2023-10-09T04:49:16.767290+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.133:0/3820378919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:17.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:17 smithi133 bash[41291]: audit 2023-10-09T04:49:17.225104+0000 mon.a (mon.0) 965 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:17.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:17 smithi133 bash[41291]: audit 2023-10-09T04:49:17.234396+0000 mon.a (mon.0) 966 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:17.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:17 smithi133 bash[27558]: audit 2023-10-09T04:49:16.347539+0000 mon.a (mon.0) 963 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:17.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:17 smithi133 bash[27558]: audit 2023-10-09T04:49:16.369290+0000 mon.a (mon.0) 964 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:17.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:17 smithi133 bash[27558]: audit 2023-10-09T04:49:16.767290+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.133:0/3820378919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:17.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:17 smithi133 bash[27558]: audit 2023-10-09T04:49:17.225104+0000 mon.a (mon.0) 965 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:17.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:17 smithi133 bash[27558]: audit 2023-10-09T04:49:17.234396+0000 mon.a (mon.0) 966 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:18.137 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:17 smithi194 bash[69734]: debug 2023-10-09T04:49:17.635+0000 7f35ee58b200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:49:18.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:18 smithi194 bash[31648]: cluster 2023-10-09T04:49:17.227099+0000 mgr.y (mgr.14466) 266 : cluster [DBG] pgmap v235: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 953 op/s 2023-10-09T04:49:18.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:18 smithi194 bash[69734]: debug 2023-10-09T04:49:18.315+0000 7f35ee58b200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:49:18.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:18 smithi133 bash[41291]: cluster 2023-10-09T04:49:17.227099+0000 mgr.y (mgr.14466) 266 : cluster [DBG] pgmap v235: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 953 op/s 2023-10-09T04:49:18.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:18 smithi133 bash[27558]: cluster 2023-10-09T04:49:17.227099+0000 mgr.y (mgr.14466) 266 : cluster [DBG] pgmap v235: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 953 op/s 2023-10-09T04:49:19.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:19 smithi194 bash[69734]: debug 2023-10-09T04:49:19.083+0000 7f35ee58b200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:49:19.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:19 smithi194 bash[69734]: debug 2023-10-09T04:49:19.235+0000 7f35ee58b200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:49:20.102 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:19 smithi194 bash[69734]: debug 2023-10-09T04:49:19.727+0000 7f35ee58b200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:49:20.102 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:19 smithi194 bash[69734]: debug 2023-10-09T04:49:19.891+0000 7f35ee58b200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:49:20.103 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:20 smithi194 bash[69734]: debug 2023-10-09T04:49:20.095+0000 7f35ee58b200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:49:20.362 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:20 smithi194 bash[69734]: debug 2023-10-09T04:49:20.259+0000 7f35ee58b200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:49:20.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:20 smithi194 bash[31648]: cluster 2023-10-09T04:49:19.227529+0000 mgr.y (mgr.14466) 267 : cluster [DBG] pgmap v236: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-10-09T04:49:20.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:20 smithi194 bash[31648]: audit 2023-10-09T04:49:19.389925+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.133:0/3818660327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:20.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:20 smithi194 bash[31648]: audit 2023-10-09T04:49:20.196095+0000 mon.a (mon.0) 967 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:20.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:20 smithi194 bash[69734]: debug 2023-10-09T04:49:20.431+0000 7f35ee58b200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:49:20.800 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:20 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:49:20] "GET /metrics HTTP/1.1" 200 242577 "" "Prometheus/2.43.0" 2023-10-09T04:49:20.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:20 smithi133 bash[41291]: cluster 2023-10-09T04:49:19.227529+0000 mgr.y (mgr.14466) 267 : cluster [DBG] pgmap v236: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-10-09T04:49:20.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:20 smithi133 bash[41291]: audit 2023-10-09T04:49:19.389925+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.133:0/3818660327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:20.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:20 smithi133 bash[41291]: audit 2023-10-09T04:49:20.196095+0000 mon.a (mon.0) 967 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:20.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:20 smithi133 bash[27558]: cluster 2023-10-09T04:49:19.227529+0000 mgr.y (mgr.14466) 267 : cluster [DBG] pgmap v236: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-10-09T04:49:20.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:20 smithi133 bash[27558]: audit 2023-10-09T04:49:19.389925+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.133:0/3818660327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:20.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:20 smithi133 bash[27558]: audit 2023-10-09T04:49:20.196095+0000 mon.a (mon.0) 967 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:21.137 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:20 smithi194 bash[69734]: debug 2023-10-09T04:49:20.715+0000 7f35ee58b200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:49:21.137 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:20 smithi194 bash[69734]: debug 2023-10-09T04:49:20.887+0000 7f35ee58b200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:49:22.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:22 smithi194 bash[31648]: cluster 2023-10-09T04:49:21.228089+0000 mgr.y (mgr.14466) 268 : cluster [DBG] pgmap v237: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-10-09T04:49:22.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:22 smithi194 bash[31648]: audit 2023-10-09T04:49:22.038655+0000 mon.b (mon.1) 32 : audit [DBG] from='client.? 172.21.15.133:0/3761781266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:22.888 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:22 smithi194 bash[69734]: debug 2023-10-09T04:49:22.587+0000 7f35ee58b200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:49:22.888 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:22 smithi194 bash[69734]: [09/Oct/2023:04:49:22] ENGINE Bus STARTING 2023-10-09T04:49:22.888 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:22 smithi194 bash[69734]: CherryPy Checker: 2023-10-09T04:49:22.888 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:22 smithi194 bash[69734]: The Application mounted at '' has an empty config. 2023-10-09T04:49:22.889 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:22 smithi194 bash[69734]: [09/Oct/2023:04:49:22] ENGINE Serving on http://:::9283 2023-10-09T04:49:22.889 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:22 smithi194 bash[69734]: [09/Oct/2023:04:49:22] ENGINE Bus STARTED 2023-10-09T04:49:23.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:22 smithi133 bash[27558]: cluster 2023-10-09T04:49:21.228089+0000 mgr.y (mgr.14466) 268 : cluster [DBG] pgmap v237: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-10-09T04:49:23.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:22 smithi133 bash[27558]: audit 2023-10-09T04:49:22.038655+0000 mon.b (mon.1) 32 : audit [DBG] from='client.? 172.21.15.133:0/3761781266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:23.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:22 smithi133 bash[41291]: cluster 2023-10-09T04:49:21.228089+0000 mgr.y (mgr.14466) 268 : cluster [DBG] pgmap v237: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-10-09T04:49:23.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:22 smithi133 bash[41291]: audit 2023-10-09T04:49:22.038655+0000 mon.b (mon.1) 32 : audit [DBG] from='client.? 172.21.15.133:0/3761781266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:23.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:23 smithi194 bash[31648]: cluster 2023-10-09T04:49:22.616630+0000 mon.a (mon.0) 968 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T04:49:23.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:23 smithi194 bash[31648]: cluster 2023-10-09T04:49:22.616779+0000 mon.a (mon.0) 969 : cluster [DBG] Standby manager daemon x started 2023-10-09T04:49:23.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:23 smithi194 bash[31648]: audit 2023-10-09T04:49:22.617701+0000 mon.a (mon.0) 970 : audit [DBG] from='mgr.? 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:49:23.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:23 smithi194 bash[31648]: audit 2023-10-09T04:49:22.624359+0000 mon.a (mon.0) 971 : audit [DBG] from='mgr.? 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:49:23.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:23 smithi194 bash[31648]: audit 2023-10-09T04:49:22.637046+0000 mon.a (mon.0) 972 : audit [DBG] from='mgr.? 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:49:23.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:23 smithi194 bash[31648]: audit 2023-10-09T04:49:22.638459+0000 mon.a (mon.0) 973 : audit [DBG] from='mgr.? 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:49:24.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:23 smithi133 bash[27558]: cluster 2023-10-09T04:49:22.616630+0000 mon.a (mon.0) 968 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T04:49:24.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:23 smithi133 bash[27558]: cluster 2023-10-09T04:49:22.616779+0000 mon.a (mon.0) 969 : cluster [DBG] Standby manager daemon x started 2023-10-09T04:49:24.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:23 smithi133 bash[27558]: audit 2023-10-09T04:49:22.617701+0000 mon.a (mon.0) 970 : audit [DBG] from='mgr.? 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:49:24.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:23 smithi133 bash[27558]: audit 2023-10-09T04:49:22.624359+0000 mon.a (mon.0) 971 : audit [DBG] from='mgr.? 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:49:24.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:23 smithi133 bash[27558]: audit 2023-10-09T04:49:22.637046+0000 mon.a (mon.0) 972 : audit [DBG] from='mgr.? 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:49:24.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:23 smithi133 bash[27558]: audit 2023-10-09T04:49:22.638459+0000 mon.a (mon.0) 973 : audit [DBG] from='mgr.? 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:49:24.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:23 smithi133 bash[41291]: cluster 2023-10-09T04:49:22.616630+0000 mon.a (mon.0) 968 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T04:49:24.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:23 smithi133 bash[41291]: cluster 2023-10-09T04:49:22.616779+0000 mon.a (mon.0) 969 : cluster [DBG] Standby manager daemon x started 2023-10-09T04:49:24.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:23 smithi133 bash[41291]: audit 2023-10-09T04:49:22.617701+0000 mon.a (mon.0) 970 : audit [DBG] from='mgr.? 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:49:24.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:23 smithi133 bash[41291]: audit 2023-10-09T04:49:22.624359+0000 mon.a (mon.0) 971 : audit [DBG] from='mgr.? 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:49:24.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:23 smithi133 bash[41291]: audit 2023-10-09T04:49:22.637046+0000 mon.a (mon.0) 972 : audit [DBG] from='mgr.? 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:49:24.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:23 smithi133 bash[41291]: audit 2023-10-09T04:49:22.638459+0000 mon.a (mon.0) 973 : audit [DBG] from='mgr.? 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:49:24.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:24 smithi194 bash[31648]: cluster 2023-10-09T04:49:23.228753+0000 mgr.y (mgr.14466) 269 : cluster [DBG] pgmap v238: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 840 op/s 2023-10-09T04:49:24.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:24 smithi194 bash[31648]: cluster 2023-10-09T04:49:23.568151+0000 mon.a (mon.0) 974 : cluster [DBG] mgrmap e20: y(active, since 7m), standbys: x 2023-10-09T04:49:25.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:24 smithi133 bash[41291]: cluster 2023-10-09T04:49:23.228753+0000 mgr.y (mgr.14466) 269 : cluster [DBG] pgmap v238: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 840 op/s 2023-10-09T04:49:25.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:24 smithi133 bash[41291]: cluster 2023-10-09T04:49:23.568151+0000 mon.a (mon.0) 974 : cluster [DBG] mgrmap e20: y(active, since 7m), standbys: x 2023-10-09T04:49:25.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:24 smithi133 bash[27558]: cluster 2023-10-09T04:49:23.228753+0000 mgr.y (mgr.14466) 269 : cluster [DBG] pgmap v238: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 840 op/s 2023-10-09T04:49:25.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:24 smithi133 bash[27558]: cluster 2023-10-09T04:49:23.568151+0000 mon.a (mon.0) 974 : cluster [DBG] mgrmap e20: y(active, since 7m), standbys: x 2023-10-09T04:49:25.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:25 smithi194 bash[31648]: audit 2023-10-09T04:49:24.750122+0000 mon.b (mon.1) 33 : audit [DBG] from='client.? 172.21.15.133:0/1879026996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:26.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:25 smithi133 bash[27558]: audit 2023-10-09T04:49:24.750122+0000 mon.b (mon.1) 33 : audit [DBG] from='client.? 172.21.15.133:0/1879026996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:26.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:25 smithi133 bash[41291]: audit 2023-10-09T04:49:24.750122+0000 mon.b (mon.1) 33 : audit [DBG] from='client.? 172.21.15.133:0/1879026996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:26.353 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:26 smithi194 bash[69734]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:49:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:49:26.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:26 smithi194 bash[31648]: cluster 2023-10-09T04:49:25.229213+0000 mgr.y (mgr.14466) 270 : cluster [DBG] pgmap v239: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 577 op/s 2023-10-09T04:49:27.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:26 smithi133 bash[27558]: cluster 2023-10-09T04:49:25.229213+0000 mgr.y (mgr.14466) 270 : cluster [DBG] pgmap v239: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 577 op/s 2023-10-09T04:49:27.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:26 smithi133 bash[41291]: cluster 2023-10-09T04:49:25.229213+0000 mgr.y (mgr.14466) 270 : cluster [DBG] pgmap v239: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 577 op/s 2023-10-09T04:49:27.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:27 smithi194 bash[31648]: audit 2023-10-09T04:49:27.404091+0000 mon.a (mon.0) 975 : audit [DBG] from='client.? 172.21.15.133:0/340372945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:28.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:27 smithi133 bash[41291]: audit 2023-10-09T04:49:27.404091+0000 mon.a (mon.0) 975 : audit [DBG] from='client.? 172.21.15.133:0/340372945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:28.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:27 smithi133 bash[27558]: audit 2023-10-09T04:49:27.404091+0000 mon.a (mon.0) 975 : audit [DBG] from='client.? 172.21.15.133:0/340372945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:28.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:28 smithi194 bash[31648]: cluster 2023-10-09T04:49:27.229977+0000 mgr.y (mgr.14466) 271 : cluster [DBG] pgmap v240: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 895 op/s 2023-10-09T04:49:29.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:28 smithi133 bash[41291]: cluster 2023-10-09T04:49:27.229977+0000 mgr.y (mgr.14466) 271 : cluster [DBG] pgmap v240: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 895 op/s 2023-10-09T04:49:29.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:28 smithi133 bash[27558]: cluster 2023-10-09T04:49:27.229977+0000 mgr.y (mgr.14466) 271 : cluster [DBG] pgmap v240: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 895 op/s 2023-10-09T04:49:30.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:30 smithi133 bash[27558]: cluster 2023-10-09T04:49:29.230405+0000 mgr.y (mgr.14466) 272 : cluster [DBG] pgmap v241: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-10-09T04:49:30.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:30 smithi133 bash[27558]: audit 2023-10-09T04:49:30.051927+0000 mon.a (mon.0) 976 : audit [DBG] from='client.? 172.21.15.133:0/4255047155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:30.801 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:30 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:49:30] "GET /metrics HTTP/1.1" 200 242577 "" "Prometheus/2.43.0" 2023-10-09T04:49:30.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:30 smithi133 bash[41291]: cluster 2023-10-09T04:49:29.230405+0000 mgr.y (mgr.14466) 272 : cluster [DBG] pgmap v241: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-10-09T04:49:30.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:30 smithi133 bash[41291]: audit 2023-10-09T04:49:30.051927+0000 mon.a (mon.0) 976 : audit [DBG] from='client.? 172.21.15.133:0/4255047155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:30.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:30 smithi194 bash[31648]: cluster 2023-10-09T04:49:29.230405+0000 mgr.y (mgr.14466) 272 : cluster [DBG] pgmap v241: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-10-09T04:49:30.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:30 smithi194 bash[31648]: audit 2023-10-09T04:49:30.051927+0000 mon.a (mon.0) 976 : audit [DBG] from='client.? 172.21.15.133:0/4255047155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:31.228 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:30 smithi133 bash[27821]: debug 2023-10-09T04:49:30.907+0000 7fe23caf7700 -1 mgr handle_mgr_map I was active but no longer am 2023-10-09T04:49:31.228 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:30 smithi133 bash[27821]: ignoring --setuser ceph since I am not root 2023-10-09T04:49:31.228 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:30 smithi133 bash[27821]: ignoring --setgroup ceph since I am not root 2023-10-09T04:49:31.228 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:30 smithi133 bash[27821]: debug 2023-10-09T04:49:30.991+0000 7f6fdf68f700 1 -- 172.21.15.133:0/3307179012 <== mon.0 v2:172.21.15.133:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x563e456ce9c0 con 0x563e46454800 2023-10-09T04:49:31.228 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27821]: debug 2023-10-09T04:49:31.219+0000 7f6fe797f1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:49:31.235 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:30 smithi194 bash[69734]: [09/Oct/2023:04:49:30] ENGINE Bus STOPPING 2023-10-09T04:49:31.235 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:30 smithi194 bash[69734]: [09/Oct/2023:04:49:30] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T04:49:31.235 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:30 smithi194 bash[69734]: [09/Oct/2023:04:49:30] ENGINE Bus STOPPED 2023-10-09T04:49:31.235 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[69734]: [09/Oct/2023:04:49:31] ENGINE Bus STARTING 2023-10-09T04:49:31.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[69734]: [09/Oct/2023:04:49:31] ENGINE Serving on http://:::9283 2023-10-09T04:49:31.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[69734]: [09/Oct/2023:04:49:31] ENGINE Bus STARTED 2023-10-09T04:49:31.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[69734]: debug 2023-10-09T04:49:31.351+0000 7f35561fa700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:49:31.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[69734]: debug 2023-10-09T04:49:31.359+0000 7f35561fa700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:49:31.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[69734]: debug 2023-10-09T04:49:31.359+0000 7f35561fa700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:49:31.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[69734]: debug 2023-10-09T04:49:31.359+0000 7f35561fa700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:49:31.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[69734]: debug 2023-10-09T04:49:31.359+0000 7f35561fa700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:49:31.638 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[69734]: debug 2023-10-09T04:49:31.371+0000 7f35491e0700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:49:31.638 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[69734]: debug 2023-10-09T04:49:31.371+0000 7f35491e0700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:49:31.638 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[69734]: debug 2023-10-09T04:49:31.371+0000 7f35491e0700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:49:31.638 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[69734]: debug 2023-10-09T04:49:31.371+0000 7f35491e0700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:49:31.638 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[69734]: debug 2023-10-09T04:49:31.371+0000 7f35491e0700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:49:32.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:30.797416+0000 mon.a (mon.0) 977 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:32.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:30.804123+0000 mon.a (mon.0) 978 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:32.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:30.807098+0000 mon.a (mon.0) 979 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:49:32.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:30.808107+0000 mon.a (mon.0) 980 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:49:32.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:30.816596+0000 mon.a (mon.0) 981 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:32.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:30.890141+0000 mon.a (mon.0) 982 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:49:32.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:30.896479+0000 mon.a (mon.0) 983 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T04:49:32.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: cluster 2023-10-09T04:49:30.914552+0000 mon.a (mon.0) 984 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-10-09T04:49:32.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:30.915787+0000 mon.a (mon.0) 985 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T04:49:32.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: cluster 2023-10-09T04:49:30.915849+0000 mon.a (mon.0) 986 : cluster [DBG] mgrmap e21: x(active, starting, since 0.0187908s) 2023-10-09T04:49:32.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:30.916994+0000 mon.a (mon.0) 987 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:49:32.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:30.917181+0000 mon.a (mon.0) 988 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:49:32.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:30.917430+0000 mon.a (mon.0) 989 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:49:32.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:30.917845+0000 mon.a (mon.0) 990 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi133.tsuabh"}]: dispatch 2023-10-09T04:49:32.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:30.918104+0000 mon.a (mon.0) 991 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi194.yxnfqn"}]: dispatch 2023-10-09T04:49:32.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:30.918922+0000 mon.a (mon.0) 992 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:49:32.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:30.919244+0000 mon.a (mon.0) 993 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:49:32.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:30.919525+0000 mon.a (mon.0) 994 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:49:32.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:30.919797+0000 mon.a (mon.0) 995 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:49:32.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:30.920180+0000 mon.a (mon.0) 996 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:49:32.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:30.920555+0000 mon.a (mon.0) 997 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:49:32.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:30.920891+0000 mon.a (mon.0) 998 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:49:32.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:30.921223+0000 mon.a (mon.0) 999 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:49:32.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:30.921726+0000 mon.a (mon.0) 1000 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:49:32.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:30.929391+0000 mon.a (mon.0) 1001 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:49:32.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:30.929918+0000 mon.a (mon.0) 1002 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:49:32.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:30.930527+0000 mon.a (mon.0) 1003 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:49:32.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: cluster 2023-10-09T04:49:30.986363+0000 mon.a (mon.0) 1004 : cluster [INF] Manager daemon x is now available 2023-10-09T04:49:32.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:31.022782+0000 mon.a (mon.0) 1005 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:32.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:31.043588+0000 mon.a (mon.0) 1006 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:49:32.056 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:31.062125+0000 mon.a (mon.0) 1007 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:49:32.056 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:31.093688+0000 mon.a (mon.0) 1008 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:49:32.056 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:31.112165+0000 mon.a (mon.0) 1009 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:32.056 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:31.224992+0000 mon.a (mon.0) 1010 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T04:49:32.056 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:31.327330+0000 mon.a (mon.0) 1011 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:32.056 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[41291]: audit 2023-10-09T04:49:31.349570+0000 mon.a (mon.0) 1012 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:32.056 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:30.797416+0000 mon.a (mon.0) 977 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:32.057 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:30.804123+0000 mon.a (mon.0) 978 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:32.057 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:30.807098+0000 mon.a (mon.0) 979 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:49:32.057 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:30.808107+0000 mon.a (mon.0) 980 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:49:32.057 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:30.816596+0000 mon.a (mon.0) 981 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:32.057 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:30.890141+0000 mon.a (mon.0) 982 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:49:32.057 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:30.896479+0000 mon.a (mon.0) 983 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T04:49:32.058 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: cluster 2023-10-09T04:49:30.914552+0000 mon.a (mon.0) 984 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-10-09T04:49:32.058 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:30.915787+0000 mon.a (mon.0) 985 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T04:49:32.058 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: cluster 2023-10-09T04:49:30.915849+0000 mon.a (mon.0) 986 : cluster [DBG] mgrmap e21: x(active, starting, since 0.0187908s) 2023-10-09T04:49:32.058 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:30.916994+0000 mon.a (mon.0) 987 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:49:32.058 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:30.917181+0000 mon.a (mon.0) 988 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:49:32.058 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:30.917430+0000 mon.a (mon.0) 989 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:49:32.058 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:30.917845+0000 mon.a (mon.0) 990 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi133.tsuabh"}]: dispatch 2023-10-09T04:49:32.059 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:30.918104+0000 mon.a (mon.0) 991 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi194.yxnfqn"}]: dispatch 2023-10-09T04:49:32.059 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:30.918922+0000 mon.a (mon.0) 992 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:49:32.059 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:30.919244+0000 mon.a (mon.0) 993 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:49:32.059 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:30.919525+0000 mon.a (mon.0) 994 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:49:32.059 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:30.919797+0000 mon.a (mon.0) 995 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:49:32.059 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:30.920180+0000 mon.a (mon.0) 996 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:49:32.059 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:30.920555+0000 mon.a (mon.0) 997 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:49:32.059 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:30.920891+0000 mon.a (mon.0) 998 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:49:32.059 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:30.921223+0000 mon.a (mon.0) 999 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:49:32.060 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:30.921726+0000 mon.a (mon.0) 1000 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:49:32.060 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:30.929391+0000 mon.a (mon.0) 1001 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:49:32.060 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:30.929918+0000 mon.a (mon.0) 1002 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:49:32.060 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:30.930527+0000 mon.a (mon.0) 1003 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:49:32.060 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: cluster 2023-10-09T04:49:30.986363+0000 mon.a (mon.0) 1004 : cluster [INF] Manager daemon x is now available 2023-10-09T04:49:32.060 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:31.022782+0000 mon.a (mon.0) 1005 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:32.060 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:31.043588+0000 mon.a (mon.0) 1006 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:49:32.060 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:31.062125+0000 mon.a (mon.0) 1007 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:49:32.061 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:31.093688+0000 mon.a (mon.0) 1008 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:49:32.061 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:31.112165+0000 mon.a (mon.0) 1009 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:32.061 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:31.224992+0000 mon.a (mon.0) 1010 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T04:49:32.061 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:31.327330+0000 mon.a (mon.0) 1011 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:32.061 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:31 smithi133 bash[27558]: audit 2023-10-09T04:49:31.349570+0000 mon.a (mon.0) 1012 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:32.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:30.797416+0000 mon.a (mon.0) 977 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:32.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:30.804123+0000 mon.a (mon.0) 978 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:32.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:30.807098+0000 mon.a (mon.0) 979 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:49:32.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:30.808107+0000 mon.a (mon.0) 980 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:49:32.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:30.816596+0000 mon.a (mon.0) 981 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' 2023-10-09T04:49:32.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:30.890141+0000 mon.a (mon.0) 982 : audit [DBG] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:49:32.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:30.896479+0000 mon.a (mon.0) 983 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T04:49:32.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: cluster 2023-10-09T04:49:30.914552+0000 mon.a (mon.0) 984 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-10-09T04:49:32.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:30.915787+0000 mon.a (mon.0) 985 : audit [INF] from='mgr.14466 172.21.15.133:0/3038544779' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T04:49:32.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: cluster 2023-10-09T04:49:30.915849+0000 mon.a (mon.0) 986 : cluster [DBG] mgrmap e21: x(active, starting, since 0.0187908s) 2023-10-09T04:49:32.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:30.916994+0000 mon.a (mon.0) 987 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:49:32.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:30.917181+0000 mon.a (mon.0) 988 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:49:32.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:30.917430+0000 mon.a (mon.0) 989 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:49:32.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:30.917845+0000 mon.a (mon.0) 990 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi133.tsuabh"}]: dispatch 2023-10-09T04:49:32.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:30.918104+0000 mon.a (mon.0) 991 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi194.yxnfqn"}]: dispatch 2023-10-09T04:49:32.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:30.918922+0000 mon.a (mon.0) 992 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:49:32.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:30.919244+0000 mon.a (mon.0) 993 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:49:32.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:30.919525+0000 mon.a (mon.0) 994 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:49:32.140 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:30.919797+0000 mon.a (mon.0) 995 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:49:32.140 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:30.920180+0000 mon.a (mon.0) 996 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:49:32.140 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:30.920555+0000 mon.a (mon.0) 997 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:49:32.140 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:30.920891+0000 mon.a (mon.0) 998 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:49:32.140 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:30.921223+0000 mon.a (mon.0) 999 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:49:32.140 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:30.921726+0000 mon.a (mon.0) 1000 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:49:32.140 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:30.929391+0000 mon.a (mon.0) 1001 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:49:32.140 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:30.929918+0000 mon.a (mon.0) 1002 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:49:32.140 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:30.930527+0000 mon.a (mon.0) 1003 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:49:32.140 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: cluster 2023-10-09T04:49:30.986363+0000 mon.a (mon.0) 1004 : cluster [INF] Manager daemon x is now available 2023-10-09T04:49:32.141 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:31.022782+0000 mon.a (mon.0) 1005 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:32.141 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:31.043588+0000 mon.a (mon.0) 1006 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:49:32.141 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:31.062125+0000 mon.a (mon.0) 1007 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:49:32.141 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:31.093688+0000 mon.a (mon.0) 1008 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:49:32.141 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:31.112165+0000 mon.a (mon.0) 1009 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:32.141 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:31.224992+0000 mon.a (mon.0) 1010 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T04:49:32.141 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:31.327330+0000 mon.a (mon.0) 1011 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:32.141 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:31 smithi194 bash[31648]: audit 2023-10-09T04:49:31.349570+0000 mon.a (mon.0) 1012 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:32.799 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:32 smithi133 bash[27821]: debug 2023-10-09T04:49:32.483+0000 7f6fe797f1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:49:32.799 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:32 smithi133 bash[27821]: debug 2023-10-09T04:49:32.635+0000 7f6fe797f1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:49:32.799 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:32 smithi133 bash[27821]: debug 2023-10-09T04:49:32.791+0000 7f6fe797f1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:49:33.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:32 smithi133 bash[27558]: cluster 2023-10-09T04:49:31.915067+0000 mon.a (mon.0) 1013 : cluster [DBG] mgrmap e22: x(active, since 1.018s) 2023-10-09T04:49:33.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:32 smithi133 bash[27558]: cephadm 2023-10-09T04:49:31.920099+0000 mgr.x (mgr.14967) 1 : cephadm [INF] Deploying cephadm binary to smithi194 2023-10-09T04:49:33.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:32 smithi133 bash[27558]: cephadm 2023-10-09T04:49:31.921931+0000 mgr.x (mgr.14967) 2 : cephadm [INF] Deploying cephadm binary to smithi133 2023-10-09T04:49:33.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:32 smithi133 bash[27558]: cluster 2023-10-09T04:49:31.998438+0000 mgr.x (mgr.14967) 3 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail 2023-10-09T04:49:33.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:32 smithi133 bash[27558]: cephadm 2023-10-09T04:49:32.556957+0000 mgr.x (mgr.14967) 4 : cephadm [INF] [09/Oct/2023:04:49:32] ENGINE Bus STARTING 2023-10-09T04:49:33.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:32 smithi133 bash[27558]: cephadm 2023-10-09T04:49:32.695195+0000 mgr.x (mgr.14967) 5 : cephadm [INF] [09/Oct/2023:04:49:32] ENGINE Serving on http://172.21.15.194:8765 2023-10-09T04:49:33.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:32 smithi133 bash[27558]: audit 2023-10-09T04:49:32.803841+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.133:0/594825508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:33.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:32 smithi133 bash[41291]: cluster 2023-10-09T04:49:31.915067+0000 mon.a (mon.0) 1013 : cluster [DBG] mgrmap e22: x(active, since 1.018s) 2023-10-09T04:49:33.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:32 smithi133 bash[41291]: cephadm 2023-10-09T04:49:31.920099+0000 mgr.x (mgr.14967) 1 : cephadm [INF] Deploying cephadm binary to smithi194 2023-10-09T04:49:33.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:32 smithi133 bash[41291]: cephadm 2023-10-09T04:49:31.921931+0000 mgr.x (mgr.14967) 2 : cephadm [INF] Deploying cephadm binary to smithi133 2023-10-09T04:49:33.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:32 smithi133 bash[41291]: cluster 2023-10-09T04:49:31.998438+0000 mgr.x (mgr.14967) 3 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail 2023-10-09T04:49:33.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:32 smithi133 bash[41291]: cephadm 2023-10-09T04:49:32.556957+0000 mgr.x (mgr.14967) 4 : cephadm [INF] [09/Oct/2023:04:49:32] ENGINE Bus STARTING 2023-10-09T04:49:33.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:32 smithi133 bash[41291]: cephadm 2023-10-09T04:49:32.695195+0000 mgr.x (mgr.14967) 5 : cephadm [INF] [09/Oct/2023:04:49:32] ENGINE Serving on http://172.21.15.194:8765 2023-10-09T04:49:33.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:32 smithi133 bash[41291]: audit 2023-10-09T04:49:32.803841+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.133:0/594825508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:33.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:32 smithi194 bash[31648]: cluster 2023-10-09T04:49:31.915067+0000 mon.a (mon.0) 1013 : cluster [DBG] mgrmap e22: x(active, since 1.018s) 2023-10-09T04:49:33.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:32 smithi194 bash[31648]: cephadm 2023-10-09T04:49:31.920099+0000 mgr.x (mgr.14967) 1 : cephadm [INF] Deploying cephadm binary to smithi194 2023-10-09T04:49:33.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:32 smithi194 bash[31648]: cephadm 2023-10-09T04:49:31.921931+0000 mgr.x (mgr.14967) 2 : cephadm [INF] Deploying cephadm binary to smithi133 2023-10-09T04:49:33.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:32 smithi194 bash[31648]: cluster 2023-10-09T04:49:31.998438+0000 mgr.x (mgr.14967) 3 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail 2023-10-09T04:49:33.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:32 smithi194 bash[31648]: cephadm 2023-10-09T04:49:32.556957+0000 mgr.x (mgr.14967) 4 : cephadm [INF] [09/Oct/2023:04:49:32] ENGINE Bus STARTING 2023-10-09T04:49:33.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:32 smithi194 bash[31648]: cephadm 2023-10-09T04:49:32.695195+0000 mgr.x (mgr.14967) 5 : cephadm [INF] [09/Oct/2023:04:49:32] ENGINE Serving on http://172.21.15.194:8765 2023-10-09T04:49:33.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:32 smithi194 bash[31648]: audit 2023-10-09T04:49:32.803841+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.133:0/594825508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:33.551 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:33 smithi133 bash[27821]: debug 2023-10-09T04:49:33.163+0000 7f6fe797f1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:49:33.930 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:33 smithi133 bash[27558]: cluster 2023-10-09T04:49:32.931116+0000 mgr.x (mgr.14967) 6 : cluster [DBG] pgmap v4: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail 2023-10-09T04:49:33.931 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:33 smithi133 bash[27558]: cluster 2023-10-09T04:49:32.939937+0000 mon.a (mon.0) 1014 : cluster [DBG] mgrmap e23: x(active, since 2s) 2023-10-09T04:49:33.931 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:33 smithi133 bash[27558]: cephadm 2023-10-09T04:49:33.039480+0000 mgr.x (mgr.14967) 7 : cephadm [INF] [09/Oct/2023:04:49:33] ENGINE Serving on https://172.21.15.194:7150 2023-10-09T04:49:33.931 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:33 smithi133 bash[27558]: cephadm 2023-10-09T04:49:33.039609+0000 mgr.x (mgr.14967) 8 : cephadm [INF] [09/Oct/2023:04:49:33] ENGINE Bus STARTED 2023-10-09T04:49:33.931 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:33 smithi133 bash[27821]: debug 2023-10-09T04:49:33.651+0000 7f6fe797f1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:49:33.931 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:33 smithi133 bash[27821]: debug 2023-10-09T04:49:33.779+0000 7f6fe797f1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:49:34.301 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:34 smithi133 bash[27821]: debug 2023-10-09T04:49:34.167+0000 7f6fe797f1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:49:34.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:33 smithi133 bash[41291]: cluster 2023-10-09T04:49:32.931116+0000 mgr.x (mgr.14967) 6 : cluster [DBG] pgmap v4: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail 2023-10-09T04:49:34.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:33 smithi133 bash[41291]: cluster 2023-10-09T04:49:32.939937+0000 mon.a (mon.0) 1014 : cluster [DBG] mgrmap e23: x(active, since 2s) 2023-10-09T04:49:34.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:33 smithi133 bash[41291]: cephadm 2023-10-09T04:49:33.039480+0000 mgr.x (mgr.14967) 7 : cephadm [INF] [09/Oct/2023:04:49:33] ENGINE Serving on https://172.21.15.194:7150 2023-10-09T04:49:34.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:33 smithi133 bash[41291]: cephadm 2023-10-09T04:49:33.039609+0000 mgr.x (mgr.14967) 8 : cephadm [INF] [09/Oct/2023:04:49:33] ENGINE Bus STARTED 2023-10-09T04:49:34.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:33 smithi194 bash[31648]: cluster 2023-10-09T04:49:32.931116+0000 mgr.x (mgr.14967) 6 : cluster [DBG] pgmap v4: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail 2023-10-09T04:49:34.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:33 smithi194 bash[31648]: cluster 2023-10-09T04:49:32.939937+0000 mon.a (mon.0) 1014 : cluster [DBG] mgrmap e23: x(active, since 2s) 2023-10-09T04:49:34.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:33 smithi194 bash[31648]: cephadm 2023-10-09T04:49:33.039480+0000 mgr.x (mgr.14967) 7 : cephadm [INF] [09/Oct/2023:04:49:33] ENGINE Serving on https://172.21.15.194:7150 2023-10-09T04:49:34.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:33 smithi194 bash[31648]: cephadm 2023-10-09T04:49:33.039609+0000 mgr.x (mgr.14967) 8 : cephadm [INF] [09/Oct/2023:04:49:33] ENGINE Bus STARTED 2023-10-09T04:49:36.026 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:35 smithi133 bash[27821]: debug 2023-10-09T04:49:35.655+0000 7f6fe797f1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:49:36.026 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:35 smithi133 bash[27821]: debug 2023-10-09T04:49:35.839+0000 7f6fe797f1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:49:36.026 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:36 smithi133 bash[27821]: debug 2023-10-09T04:49:36.019+0000 7f6fe797f1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:49:36.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:36 smithi194 bash[31648]: cluster 2023-10-09T04:49:34.931544+0000 mgr.x (mgr.14967) 9 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail 2023-10-09T04:49:36.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:36 smithi194 bash[31648]: audit 2023-10-09T04:49:35.560450+0000 mon.a (mon.0) 1015 : audit [DBG] from='client.? 172.21.15.133:0/419577244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:36.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:36 smithi194 bash[69734]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:49:36] "GET /metrics HTTP/1.1" 200 33229 "" "Prometheus/2.43.0" 2023-10-09T04:49:36.990 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:36 smithi133 bash[27821]: debug 2023-10-09T04:49:36.687+0000 7f6fe797f1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:49:36.990 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:36 smithi133 bash[27821]: debug 2023-10-09T04:49:36.983+0000 7f6fe797f1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:49:36.990 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:36 smithi133 bash[41291]: cluster 2023-10-09T04:49:34.931544+0000 mgr.x (mgr.14967) 9 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail 2023-10-09T04:49:36.990 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:36 smithi133 bash[41291]: audit 2023-10-09T04:49:35.560450+0000 mon.a (mon.0) 1015 : audit [DBG] from='client.? 172.21.15.133:0/419577244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:36.991 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:36 smithi133 bash[27558]: cluster 2023-10-09T04:49:34.931544+0000 mgr.x (mgr.14967) 9 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail 2023-10-09T04:49:36.991 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:36 smithi133 bash[27558]: audit 2023-10-09T04:49:35.560450+0000 mon.a (mon.0) 1015 : audit [DBG] from='client.? 172.21.15.133:0/419577244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:37.801 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:37 smithi133 bash[27821]: debug 2023-10-09T04:49:37.507+0000 7f6fe797f1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:49:38.551 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:38 smithi133 bash[27821]: debug 2023-10-09T04:49:38.231+0000 7f6fe797f1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:49:38.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:38 smithi194 bash[31648]: cluster 2023-10-09T04:49:36.931978+0000 mgr.x (mgr.14967) 10 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail 2023-10-09T04:49:38.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:38 smithi194 bash[31648]: audit 2023-10-09T04:49:38.491970+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.133:0/2168485272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:38.888 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:49:38 smithi194 bash[67868]: ts=2023-10-09T04:49:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi194.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"172.21.15.133:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:49:38.888 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:49:38 smithi194 bash[67868]: ts=2023-10-09T04:49:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephPGImbalance index=15 msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on (job) group_left () avg by (job) (ceph_osd_numpg\n > 0)) / on (job) group_left () avg by (job) (ceph_osd_numpg > 0)) * on (ceph_daemon)\n group_left (hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi194.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"172.21.15.133:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:49:39.042 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:38 smithi133 bash[41291]: cluster 2023-10-09T04:49:36.931978+0000 mgr.x (mgr.14967) 10 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail 2023-10-09T04:49:39.042 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:38 smithi133 bash[41291]: audit 2023-10-09T04:49:38.491970+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.133:0/2168485272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:39.042 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:38 smithi133 bash[27558]: cluster 2023-10-09T04:49:36.931978+0000 mgr.x (mgr.14967) 10 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail 2023-10-09T04:49:39.043 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:38 smithi133 bash[27558]: audit 2023-10-09T04:49:38.491970+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.133:0/2168485272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:39.301 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:39 smithi133 bash[27821]: debug 2023-10-09T04:49:39.031+0000 7f6fe797f1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:49:39.301 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:39 smithi133 bash[27821]: debug 2023-10-09T04:49:39.187+0000 7f6fe797f1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:49:39.801 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:39 smithi133 bash[27821]: debug 2023-10-09T04:49:39.411+0000 7f6fe797f1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:49:40.203 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:39 smithi133 bash[27821]: debug 2023-10-09T04:49:39.819+0000 7f6fe797f1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:49:40.203 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:40 smithi133 bash[27821]: debug 2023-10-09T04:49:40.027+0000 7f6fe797f1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:49:40.203 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:40 smithi133 bash[27821]: debug 2023-10-09T04:49:40.195+0000 7f6fe797f1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:49:40.551 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:40 smithi133 bash[27821]: debug 2023-10-09T04:49:40.371+0000 7f6fe797f1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:49:40.848 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:40 smithi133 bash[27821]: debug 2023-10-09T04:49:40.659+0000 7f6fe797f1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:49:40.848 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:40 smithi133 bash[27558]: cluster 2023-10-09T04:49:38.932804+0000 mgr.x (mgr.14967) 11 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 474 op/s 2023-10-09T04:49:40.848 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:40 smithi133 bash[41291]: cluster 2023-10-09T04:49:38.932804+0000 mgr.x (mgr.14967) 11 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 474 op/s 2023-10-09T04:49:40.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:40 smithi194 bash[31648]: cluster 2023-10-09T04:49:38.932804+0000 mgr.x (mgr.14967) 11 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 474 op/s 2023-10-09T04:49:41.305 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:40 smithi133 bash[27821]: debug 2023-10-09T04:49:40.839+0000 7f6fe797f1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:49:41.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:41 smithi133 bash[27558]: audit 2023-10-09T04:49:41.212786+0000 mon.a (mon.0) 1016 : audit [DBG] from='client.? 172.21.15.133:0/2846885577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:41.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:41 smithi133 bash[41291]: audit 2023-10-09T04:49:41.212786+0000 mon.a (mon.0) 1016 : audit [DBG] from='client.? 172.21.15.133:0/2846885577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:41.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:41 smithi194 bash[31648]: audit 2023-10-09T04:49:41.212786+0000 mon.a (mon.0) 1016 : audit [DBG] from='client.? 172.21.15.133:0/2846885577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:42.583 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:49:42 smithi133 bash[75109]: ts=2023-10-09T04:49:42.323Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi133.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.133:8443: connect: connection refused" 2023-10-09T04:49:42.583 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:42 smithi133 bash[27558]: cluster 2023-10-09T04:49:40.933238+0000 mgr.x (mgr.14967) 12 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 368 op/s 2023-10-09T04:49:42.583 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:42 smithi133 bash[27558]: audit 2023-10-09T04:49:41.711716+0000 mon.a (mon.0) 1017 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:42.583 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:42 smithi133 bash[27558]: audit 2023-10-09T04:49:41.722665+0000 mon.a (mon.0) 1018 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:42.583 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:42 smithi133 bash[27558]: audit 2023-10-09T04:49:42.477114+0000 mon.a (mon.0) 1019 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:42.584 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:42 smithi133 bash[27558]: audit 2023-10-09T04:49:42.492816+0000 mon.a (mon.0) 1020 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:42.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:42 smithi194 bash[31648]: cluster 2023-10-09T04:49:40.933238+0000 mgr.x (mgr.14967) 12 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 368 op/s 2023-10-09T04:49:42.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:42 smithi194 bash[31648]: audit 2023-10-09T04:49:41.711716+0000 mon.a (mon.0) 1017 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:42.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:42 smithi194 bash[31648]: audit 2023-10-09T04:49:41.722665+0000 mon.a (mon.0) 1018 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:42.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:42 smithi194 bash[31648]: audit 2023-10-09T04:49:42.477114+0000 mon.a (mon.0) 1019 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:42.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:42 smithi194 bash[31648]: audit 2023-10-09T04:49:42.492816+0000 mon.a (mon.0) 1020 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:43.051 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:42 smithi133 bash[27821]: debug 2023-10-09T04:49:42.627+0000 7f6fe797f1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:49:43.051 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:42 smithi133 bash[27821]: [09/Oct/2023:04:49:42] ENGINE Bus STARTING 2023-10-09T04:49:43.051 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:42 smithi133 bash[27821]: CherryPy Checker: 2023-10-09T04:49:43.051 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:42 smithi133 bash[27821]: The Application mounted at '' has an empty config. 2023-10-09T04:49:43.051 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:42 smithi133 bash[27821]: [09/Oct/2023:04:49:42] ENGINE Serving on http://:::9283 2023-10-09T04:49:43.051 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:42 smithi133 bash[27821]: [09/Oct/2023:04:49:42] ENGINE Bus STARTED 2023-10-09T04:49:43.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:42 smithi133 bash[41291]: cluster 2023-10-09T04:49:40.933238+0000 mgr.x (mgr.14967) 12 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 368 op/s 2023-10-09T04:49:43.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:42 smithi133 bash[41291]: audit 2023-10-09T04:49:41.711716+0000 mon.a (mon.0) 1017 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:43.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:42 smithi133 bash[41291]: audit 2023-10-09T04:49:41.722665+0000 mon.a (mon.0) 1018 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:43.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:42 smithi133 bash[41291]: audit 2023-10-09T04:49:42.477114+0000 mon.a (mon.0) 1019 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:43.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:42 smithi133 bash[41291]: audit 2023-10-09T04:49:42.492816+0000 mon.a (mon.0) 1020 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:43.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:43 smithi133 bash[27558]: cluster 2023-10-09T04:49:42.633348+0000 mon.a (mon.0) 1021 : cluster [DBG] Standby manager daemon y started 2023-10-09T04:49:43.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:43 smithi133 bash[27558]: audit 2023-10-09T04:49:42.642468+0000 mon.a (mon.0) 1022 : audit [DBG] from='mgr.? 172.21.15.133:0/260654217' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T04:49:43.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:43 smithi133 bash[27558]: audit 2023-10-09T04:49:42.648004+0000 mon.a (mon.0) 1023 : audit [DBG] from='mgr.? 172.21.15.133:0/260654217' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:49:43.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:43 smithi133 bash[27558]: audit 2023-10-09T04:49:42.649933+0000 mon.a (mon.0) 1024 : audit [DBG] from='mgr.? 172.21.15.133:0/260654217' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T04:49:43.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:43 smithi133 bash[27558]: audit 2023-10-09T04:49:42.650529+0000 mon.a (mon.0) 1025 : audit [DBG] from='mgr.? 172.21.15.133:0/260654217' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:49:43.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:43 smithi133 bash[27558]: audit 2023-10-09T04:49:43.021470+0000 mon.a (mon.0) 1026 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:43.803 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:43 smithi133 bash[27558]: audit 2023-10-09T04:49:43.028958+0000 mon.a (mon.0) 1027 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:43.803 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:43 smithi133 bash[27558]: audit 2023-10-09T04:49:43.031518+0000 mon.a (mon.0) 1028 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:49:43.803 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:43 smithi133 bash[27558]: audit 2023-10-09T04:49:43.032592+0000 mon.a (mon.0) 1029 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:49:43.803 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:43 smithi133 bash[27558]: audit 2023-10-09T04:49:43.037665+0000 mon.a (mon.0) 1030 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:49:43.803 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:43 smithi133 bash[27558]: audit 2023-10-09T04:49:43.041875+0000 mon.a (mon.0) 1031 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:49:43.803 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:43 smithi133 bash[27558]: audit 2023-10-09T04:49:43.048173+0000 mon.a (mon.0) 1032 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:43.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:43 smithi133 bash[41291]: cluster 2023-10-09T04:49:42.633348+0000 mon.a (mon.0) 1021 : cluster [DBG] Standby manager daemon y started 2023-10-09T04:49:43.804 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:43 smithi133 bash[41291]: audit 2023-10-09T04:49:42.642468+0000 mon.a (mon.0) 1022 : audit [DBG] from='mgr.? 172.21.15.133:0/260654217' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T04:49:43.804 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:43 smithi133 bash[41291]: audit 2023-10-09T04:49:42.648004+0000 mon.a (mon.0) 1023 : audit [DBG] from='mgr.? 172.21.15.133:0/260654217' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:49:43.804 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:43 smithi133 bash[41291]: audit 2023-10-09T04:49:42.649933+0000 mon.a (mon.0) 1024 : audit [DBG] from='mgr.? 172.21.15.133:0/260654217' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T04:49:43.804 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:43 smithi133 bash[41291]: audit 2023-10-09T04:49:42.650529+0000 mon.a (mon.0) 1025 : audit [DBG] from='mgr.? 172.21.15.133:0/260654217' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:49:43.804 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:43 smithi133 bash[41291]: audit 2023-10-09T04:49:43.021470+0000 mon.a (mon.0) 1026 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:43.804 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:43 smithi133 bash[41291]: audit 2023-10-09T04:49:43.028958+0000 mon.a (mon.0) 1027 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:43.804 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:43 smithi133 bash[41291]: audit 2023-10-09T04:49:43.031518+0000 mon.a (mon.0) 1028 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:49:43.804 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:43 smithi133 bash[41291]: audit 2023-10-09T04:49:43.032592+0000 mon.a (mon.0) 1029 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:49:43.804 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:43 smithi133 bash[41291]: audit 2023-10-09T04:49:43.037665+0000 mon.a (mon.0) 1030 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:49:43.805 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:43 smithi133 bash[41291]: audit 2023-10-09T04:49:43.041875+0000 mon.a (mon.0) 1031 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:49:43.805 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:43 smithi133 bash[41291]: audit 2023-10-09T04:49:43.048173+0000 mon.a (mon.0) 1032 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:43.805 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:49:43 smithi133 bash[75109]: ts=2023-10-09T04:49:43.485Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=3 2023-10-09T04:49:43.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:43 smithi194 bash[31648]: cluster 2023-10-09T04:49:42.633348+0000 mon.a (mon.0) 1021 : cluster [DBG] Standby manager daemon y started 2023-10-09T04:49:43.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:43 smithi194 bash[31648]: audit 2023-10-09T04:49:42.642468+0000 mon.a (mon.0) 1022 : audit [DBG] from='mgr.? 172.21.15.133:0/260654217' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T04:49:43.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:43 smithi194 bash[31648]: audit 2023-10-09T04:49:42.648004+0000 mon.a (mon.0) 1023 : audit [DBG] from='mgr.? 172.21.15.133:0/260654217' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:49:43.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:43 smithi194 bash[31648]: audit 2023-10-09T04:49:42.649933+0000 mon.a (mon.0) 1024 : audit [DBG] from='mgr.? 172.21.15.133:0/260654217' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T04:49:43.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:43 smithi194 bash[31648]: audit 2023-10-09T04:49:42.650529+0000 mon.a (mon.0) 1025 : audit [DBG] from='mgr.? 172.21.15.133:0/260654217' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:49:43.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:43 smithi194 bash[31648]: audit 2023-10-09T04:49:43.021470+0000 mon.a (mon.0) 1026 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:43.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:43 smithi194 bash[31648]: audit 2023-10-09T04:49:43.028958+0000 mon.a (mon.0) 1027 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:43.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:43 smithi194 bash[31648]: audit 2023-10-09T04:49:43.031518+0000 mon.a (mon.0) 1028 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:49:43.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:43 smithi194 bash[31648]: audit 2023-10-09T04:49:43.032592+0000 mon.a (mon.0) 1029 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:49:43.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:43 smithi194 bash[31648]: audit 2023-10-09T04:49:43.037665+0000 mon.a (mon.0) 1030 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:49:43.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:43 smithi194 bash[31648]: audit 2023-10-09T04:49:43.041875+0000 mon.a (mon.0) 1031 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:49:43.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:43 smithi194 bash[31648]: audit 2023-10-09T04:49:43.048173+0000 mon.a (mon.0) 1032 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:44.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:44 smithi194 bash[31648]: cluster 2023-10-09T04:49:42.933955+0000 mgr.x (mgr.14967) 13 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-09T04:49:44.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:44 smithi194 bash[31648]: cephadm 2023-10-09T04:49:43.044689+0000 mgr.x (mgr.14967) 14 : cephadm [INF] Adjusting osd_memory_target on smithi133 to 2524M 2023-10-09T04:49:44.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:44 smithi194 bash[31648]: cluster 2023-10-09T04:49:43.597620+0000 mon.a (mon.0) 1033 : cluster [DBG] mgrmap e24: x(active, since 12s), standbys: y 2023-10-09T04:49:44.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:44 smithi194 bash[31648]: audit 2023-10-09T04:49:43.605610+0000 mon.a (mon.0) 1034 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:49:44.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:44 smithi194 bash[31648]: audit 2023-10-09T04:49:43.751510+0000 mon.a (mon.0) 1035 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:44.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:44 smithi194 bash[31648]: audit 2023-10-09T04:49:43.763614+0000 mon.a (mon.0) 1036 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:44.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:44 smithi194 bash[31648]: audit 2023-10-09T04:49:43.884875+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.133:0/1888806715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:45.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:44 smithi133 bash[41291]: cluster 2023-10-09T04:49:42.933955+0000 mgr.x (mgr.14967) 13 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-09T04:49:45.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:44 smithi133 bash[41291]: cephadm 2023-10-09T04:49:43.044689+0000 mgr.x (mgr.14967) 14 : cephadm [INF] Adjusting osd_memory_target on smithi133 to 2524M 2023-10-09T04:49:45.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:44 smithi133 bash[41291]: cluster 2023-10-09T04:49:43.597620+0000 mon.a (mon.0) 1033 : cluster [DBG] mgrmap e24: x(active, since 12s), standbys: y 2023-10-09T04:49:45.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:44 smithi133 bash[41291]: audit 2023-10-09T04:49:43.605610+0000 mon.a (mon.0) 1034 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:49:45.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:44 smithi133 bash[41291]: audit 2023-10-09T04:49:43.751510+0000 mon.a (mon.0) 1035 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:45.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:44 smithi133 bash[41291]: audit 2023-10-09T04:49:43.763614+0000 mon.a (mon.0) 1036 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:45.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:44 smithi133 bash[41291]: audit 2023-10-09T04:49:43.884875+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.133:0/1888806715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:45.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:44 smithi133 bash[27558]: cluster 2023-10-09T04:49:42.933955+0000 mgr.x (mgr.14967) 13 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-09T04:49:45.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:44 smithi133 bash[27558]: cephadm 2023-10-09T04:49:43.044689+0000 mgr.x (mgr.14967) 14 : cephadm [INF] Adjusting osd_memory_target on smithi133 to 2524M 2023-10-09T04:49:45.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:44 smithi133 bash[27558]: cluster 2023-10-09T04:49:43.597620+0000 mon.a (mon.0) 1033 : cluster [DBG] mgrmap e24: x(active, since 12s), standbys: y 2023-10-09T04:49:45.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:44 smithi133 bash[27558]: audit 2023-10-09T04:49:43.605610+0000 mon.a (mon.0) 1034 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:49:45.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:44 smithi133 bash[27558]: audit 2023-10-09T04:49:43.751510+0000 mon.a (mon.0) 1035 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:45.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:44 smithi133 bash[27558]: audit 2023-10-09T04:49:43.763614+0000 mon.a (mon.0) 1036 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:45.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:44 smithi133 bash[27558]: audit 2023-10-09T04:49:43.884875+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.133:0/1888806715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:46.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:46 smithi194 bash[69734]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:49:46] "GET /metrics HTTP/1.1" 200 36175 "" "Prometheus/2.43.0" 2023-10-09T04:49:46.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:46 smithi194 bash[31648]: cluster 2023-10-09T04:49:44.934454+0000 mgr.x (mgr.14967) 15 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 547 op/s 2023-10-09T04:49:46.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:46 smithi194 bash[31648]: audit 2023-10-09T04:49:46.111076+0000 mon.a (mon.0) 1037 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:46.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:46 smithi194 bash[31648]: audit 2023-10-09T04:49:46.539511+0000 mon.a (mon.0) 1038 : audit [DBG] from='client.? 172.21.15.133:0/3739115141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:47.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:46 smithi133 bash[41291]: cluster 2023-10-09T04:49:44.934454+0000 mgr.x (mgr.14967) 15 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 547 op/s 2023-10-09T04:49:47.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:46 smithi133 bash[41291]: audit 2023-10-09T04:49:46.111076+0000 mon.a (mon.0) 1037 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:47.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:46 smithi133 bash[41291]: audit 2023-10-09T04:49:46.539511+0000 mon.a (mon.0) 1038 : audit [DBG] from='client.? 172.21.15.133:0/3739115141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:47.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:46 smithi133 bash[27558]: cluster 2023-10-09T04:49:44.934454+0000 mgr.x (mgr.14967) 15 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 547 op/s 2023-10-09T04:49:47.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:46 smithi133 bash[27558]: audit 2023-10-09T04:49:46.111076+0000 mon.a (mon.0) 1037 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:47.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:46 smithi133 bash[27558]: audit 2023-10-09T04:49:46.539511+0000 mon.a (mon.0) 1038 : audit [DBG] from='client.? 172.21.15.133:0/3739115141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:48.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:48 smithi194 bash[31648]: cluster 2023-10-09T04:49:46.934919+0000 mgr.x (mgr.14967) 16 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 547 op/s 2023-10-09T04:49:49.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:48 smithi133 bash[27558]: cluster 2023-10-09T04:49:46.934919+0000 mgr.x (mgr.14967) 16 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 547 op/s 2023-10-09T04:49:49.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:48 smithi133 bash[41291]: cluster 2023-10-09T04:49:46.934919+0000 mgr.x (mgr.14967) 16 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 547 op/s 2023-10-09T04:49:49.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:49 smithi194 bash[31648]: audit 2023-10-09T04:49:49.176407+0000 mon.c (mon.2) 40 : audit [DBG] from='client.? 172.21.15.133:0/3990195261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:50.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:49 smithi133 bash[41291]: audit 2023-10-09T04:49:49.176407+0000 mon.c (mon.2) 40 : audit [DBG] from='client.? 172.21.15.133:0/3990195261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:50.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:49 smithi133 bash[27558]: audit 2023-10-09T04:49:49.176407+0000 mon.c (mon.2) 40 : audit [DBG] from='client.? 172.21.15.133:0/3990195261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:50.613 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:49:50 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:49:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:49:50.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:50 smithi194 bash[31648]: cluster 2023-10-09T04:49:48.937904+0000 mgr.x (mgr.14967) 17 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 833 op/s 2023-10-09T04:49:51.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:50 smithi133 bash[27558]: cluster 2023-10-09T04:49:48.937904+0000 mgr.x (mgr.14967) 17 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 833 op/s 2023-10-09T04:49:51.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:50 smithi133 bash[41291]: cluster 2023-10-09T04:49:48.937904+0000 mgr.x (mgr.14967) 17 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 833 op/s 2023-10-09T04:49:52.882 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:52 smithi194 bash[31648]: cluster 2023-10-09T04:49:50.938340+0000 mgr.x (mgr.14967) 18 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 559 op/s 2023-10-09T04:49:52.882 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:52 smithi194 bash[31648]: audit 2023-10-09T04:49:51.830414+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.133:0/1961344839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:53.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:52 smithi133 bash[41291]: cluster 2023-10-09T04:49:50.938340+0000 mgr.x (mgr.14967) 18 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 559 op/s 2023-10-09T04:49:53.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:52 smithi133 bash[41291]: audit 2023-10-09T04:49:51.830414+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.133:0/1961344839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:53.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:52 smithi133 bash[27558]: cluster 2023-10-09T04:49:50.938340+0000 mgr.x (mgr.14967) 18 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 559 op/s 2023-10-09T04:49:53.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:52 smithi133 bash[27558]: audit 2023-10-09T04:49:51.830414+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.133:0/1961344839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:54.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:54 smithi194 bash[31648]: cluster 2023-10-09T04:49:52.939082+0000 mgr.x (mgr.14967) 19 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 864 op/s 2023-10-09T04:49:54.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:54 smithi194 bash[31648]: audit 2023-10-09T04:49:54.514352+0000 mon.a (mon.0) 1039 : audit [DBG] from='client.? 172.21.15.133:0/1918709853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:55.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:54 smithi133 bash[41291]: cluster 2023-10-09T04:49:52.939082+0000 mgr.x (mgr.14967) 19 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 864 op/s 2023-10-09T04:49:55.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:54 smithi133 bash[41291]: audit 2023-10-09T04:49:54.514352+0000 mon.a (mon.0) 1039 : audit [DBG] from='client.? 172.21.15.133:0/1918709853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:55.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:54 smithi133 bash[27558]: cluster 2023-10-09T04:49:52.939082+0000 mgr.x (mgr.14967) 19 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 864 op/s 2023-10-09T04:49:55.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:54 smithi133 bash[27558]: audit 2023-10-09T04:49:54.514352+0000 mon.a (mon.0) 1039 : audit [DBG] from='client.? 172.21.15.133:0/1918709853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:56.631 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:49:56 smithi194 bash[69734]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:49:56] "GET /metrics HTTP/1.1" 200 36175 "" "Prometheus/2.43.0" 2023-10-09T04:49:56.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:56 smithi194 bash[31648]: cluster 2023-10-09T04:49:54.939530+0000 mgr.x (mgr.14967) 20 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-10-09T04:49:57.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:56 smithi133 bash[27558]: cluster 2023-10-09T04:49:54.939530+0000 mgr.x (mgr.14967) 20 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-10-09T04:49:57.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:56 smithi133 bash[41291]: cluster 2023-10-09T04:49:54.939530+0000 mgr.x (mgr.14967) 20 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-10-09T04:49:58.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:57 smithi133 bash[41291]: audit 2023-10-09T04:49:57.207632+0000 mon.b (mon.1) 34 : audit [DBG] from='client.? 172.21.15.133:0/553408319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:58.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:57 smithi133 bash[27558]: audit 2023-10-09T04:49:57.207632+0000 mon.b (mon.1) 34 : audit [DBG] from='client.? 172.21.15.133:0/553408319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:58.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:57 smithi194 bash[31648]: audit 2023-10-09T04:49:57.207632+0000 mon.b (mon.1) 34 : audit [DBG] from='client.? 172.21.15.133:0/553408319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:49:59.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:58 smithi133 bash[41291]: cluster 2023-10-09T04:49:56.940005+0000 mgr.x (mgr.14967) 21 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-10-09T04:49:59.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:58 smithi133 bash[41291]: audit 2023-10-09T04:49:58.220871+0000 mon.a (mon.0) 1040 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:59.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:58 smithi133 bash[41291]: audit 2023-10-09T04:49:58.230603+0000 mon.a (mon.0) 1041 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:59.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:58 smithi133 bash[41291]: audit 2023-10-09T04:49:58.232435+0000 mon.a (mon.0) 1042 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:49:59.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:58 smithi133 bash[41291]: audit 2023-10-09T04:49:58.233777+0000 mon.a (mon.0) 1043 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:49:59.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:58 smithi133 bash[41291]: audit 2023-10-09T04:49:58.234894+0000 mon.a (mon.0) 1044 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:49:59.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:58 smithi133 bash[41291]: audit 2023-10-09T04:49:58.235895+0000 mon.a (mon.0) 1045 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:49:59.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:58 smithi133 bash[41291]: audit 2023-10-09T04:49:58.244845+0000 mon.a (mon.0) 1046 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:59.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:58 smithi133 bash[41291]: audit 2023-10-09T04:49:58.247888+0000 mon.a (mon.0) 1047 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:49:59.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:58 smithi133 bash[41291]: audit 2023-10-09T04:49:58.254267+0000 mon.a (mon.0) 1048 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:49:59.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:58 smithi133 bash[27558]: cluster 2023-10-09T04:49:56.940005+0000 mgr.x (mgr.14967) 21 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-10-09T04:49:59.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:58 smithi133 bash[27558]: audit 2023-10-09T04:49:58.220871+0000 mon.a (mon.0) 1040 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:59.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:58 smithi133 bash[27558]: audit 2023-10-09T04:49:58.230603+0000 mon.a (mon.0) 1041 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:59.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:58 smithi133 bash[27558]: audit 2023-10-09T04:49:58.232435+0000 mon.a (mon.0) 1042 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:49:59.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:58 smithi133 bash[27558]: audit 2023-10-09T04:49:58.233777+0000 mon.a (mon.0) 1043 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:49:59.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:58 smithi133 bash[27558]: audit 2023-10-09T04:49:58.234894+0000 mon.a (mon.0) 1044 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:49:59.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:58 smithi133 bash[27558]: audit 2023-10-09T04:49:58.235895+0000 mon.a (mon.0) 1045 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:49:59.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:58 smithi133 bash[27558]: audit 2023-10-09T04:49:58.244845+0000 mon.a (mon.0) 1046 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:59.054 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:58 smithi133 bash[27558]: audit 2023-10-09T04:49:58.247888+0000 mon.a (mon.0) 1047 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:49:59.054 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:58 smithi133 bash[27558]: audit 2023-10-09T04:49:58.254267+0000 mon.a (mon.0) 1048 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:49:59.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:58 smithi194 bash[31648]: cluster 2023-10-09T04:49:56.940005+0000 mgr.x (mgr.14967) 21 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-10-09T04:49:59.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:58 smithi194 bash[31648]: audit 2023-10-09T04:49:58.220871+0000 mon.a (mon.0) 1040 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:59.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:58 smithi194 bash[31648]: audit 2023-10-09T04:49:58.230603+0000 mon.a (mon.0) 1041 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:59.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:58 smithi194 bash[31648]: audit 2023-10-09T04:49:58.232435+0000 mon.a (mon.0) 1042 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:49:59.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:58 smithi194 bash[31648]: audit 2023-10-09T04:49:58.233777+0000 mon.a (mon.0) 1043 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:49:59.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:58 smithi194 bash[31648]: audit 2023-10-09T04:49:58.234894+0000 mon.a (mon.0) 1044 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:49:59.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:58 smithi194 bash[31648]: audit 2023-10-09T04:49:58.235895+0000 mon.a (mon.0) 1045 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:49:59.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:58 smithi194 bash[31648]: audit 2023-10-09T04:49:58.244845+0000 mon.a (mon.0) 1046 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:49:59.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:58 smithi194 bash[31648]: audit 2023-10-09T04:49:58.247888+0000 mon.a (mon.0) 1047 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:49:59.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:58 smithi194 bash[31648]: audit 2023-10-09T04:49:58.254267+0000 mon.a (mon.0) 1048 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:50:00.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[27558]: cephadm 2023-10-09T04:49:58.238542+0000 mgr.x (mgr.14967) 22 : cephadm [INF] Adjusting osd_memory_target on smithi194 to 2524M 2023-10-09T04:50:00.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[27558]: cephadm 2023-10-09T04:49:58.259536+0000 mgr.x (mgr.14967) 23 : cephadm [INF] Updating smithi133:/etc/ceph/ceph.conf 2023-10-09T04:50:00.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[27558]: cephadm 2023-10-09T04:49:58.260369+0000 mgr.x (mgr.14967) 24 : cephadm [INF] Updating smithi194:/etc/ceph/ceph.conf 2023-10-09T04:50:00.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[27558]: cephadm 2023-10-09T04:49:58.477264+0000 mgr.x (mgr.14967) 25 : cephadm [INF] Updating smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:50:00.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[27558]: cephadm 2023-10-09T04:49:58.478693+0000 mgr.x (mgr.14967) 26 : cephadm [INF] Updating smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:50:00.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[27558]: cephadm 2023-10-09T04:49:58.696520+0000 mgr.x (mgr.14967) 27 : cephadm [INF] Updating smithi194:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:50:00.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[27558]: cephadm 2023-10-09T04:49:58.697432+0000 mgr.x (mgr.14967) 28 : cephadm [INF] Updating smithi133:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:50:00.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[27558]: audit 2023-10-09T04:49:59.088016+0000 mon.a (mon.0) 1049 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:00.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[27558]: audit 2023-10-09T04:49:59.097319+0000 mon.a (mon.0) 1050 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:00.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[27558]: audit 2023-10-09T04:49:59.145884+0000 mon.a (mon.0) 1051 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:00.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[27558]: audit 2023-10-09T04:49:59.154019+0000 mon.a (mon.0) 1052 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:00.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[27558]: audit 2023-10-09T04:49:59.160497+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:00.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[27558]: audit 2023-10-09T04:49:59.235720+0000 mon.a (mon.0) 1054 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:00.054 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[27558]: audit 2023-10-09T04:49:59.245700+0000 mon.a (mon.0) 1055 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:00.054 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[27558]: audit 2023-10-09T04:49:59.256052+0000 mon.a (mon.0) 1056 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:00.054 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[27558]: audit 2023-10-09T04:49:59.263717+0000 mon.a (mon.0) 1057 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:00.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[41291]: cephadm 2023-10-09T04:49:58.238542+0000 mgr.x (mgr.14967) 22 : cephadm [INF] Adjusting osd_memory_target on smithi194 to 2524M 2023-10-09T04:50:00.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[41291]: cephadm 2023-10-09T04:49:58.259536+0000 mgr.x (mgr.14967) 23 : cephadm [INF] Updating smithi133:/etc/ceph/ceph.conf 2023-10-09T04:50:00.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[41291]: cephadm 2023-10-09T04:49:58.260369+0000 mgr.x (mgr.14967) 24 : cephadm [INF] Updating smithi194:/etc/ceph/ceph.conf 2023-10-09T04:50:00.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[41291]: cephadm 2023-10-09T04:49:58.477264+0000 mgr.x (mgr.14967) 25 : cephadm [INF] Updating smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:50:00.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[41291]: cephadm 2023-10-09T04:49:58.478693+0000 mgr.x (mgr.14967) 26 : cephadm [INF] Updating smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:50:00.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[41291]: cephadm 2023-10-09T04:49:58.696520+0000 mgr.x (mgr.14967) 27 : cephadm [INF] Updating smithi194:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:50:00.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[41291]: cephadm 2023-10-09T04:49:58.697432+0000 mgr.x (mgr.14967) 28 : cephadm [INF] Updating smithi133:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:50:00.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[41291]: audit 2023-10-09T04:49:59.088016+0000 mon.a (mon.0) 1049 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:00.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[41291]: audit 2023-10-09T04:49:59.097319+0000 mon.a (mon.0) 1050 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:00.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[41291]: audit 2023-10-09T04:49:59.145884+0000 mon.a (mon.0) 1051 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:00.056 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[41291]: audit 2023-10-09T04:49:59.154019+0000 mon.a (mon.0) 1052 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:00.056 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[41291]: audit 2023-10-09T04:49:59.160497+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:00.056 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[41291]: audit 2023-10-09T04:49:59.235720+0000 mon.a (mon.0) 1054 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:00.056 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[41291]: audit 2023-10-09T04:49:59.245700+0000 mon.a (mon.0) 1055 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:00.056 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[41291]: audit 2023-10-09T04:49:59.256052+0000 mon.a (mon.0) 1056 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:00.056 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:49:59 smithi133 bash[41291]: audit 2023-10-09T04:49:59.263717+0000 mon.a (mon.0) 1057 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:00.056 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:49:59 smithi133 systemd[1]: Stopping Ceph node-exporter.a for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T04:50:00.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:59 smithi194 bash[31648]: cephadm 2023-10-09T04:49:58.238542+0000 mgr.x (mgr.14967) 22 : cephadm [INF] Adjusting osd_memory_target on smithi194 to 2524M 2023-10-09T04:50:00.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:59 smithi194 bash[31648]: cephadm 2023-10-09T04:49:58.259536+0000 mgr.x (mgr.14967) 23 : cephadm [INF] Updating smithi133:/etc/ceph/ceph.conf 2023-10-09T04:50:00.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:59 smithi194 bash[31648]: cephadm 2023-10-09T04:49:58.260369+0000 mgr.x (mgr.14967) 24 : cephadm [INF] Updating smithi194:/etc/ceph/ceph.conf 2023-10-09T04:50:00.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:59 smithi194 bash[31648]: cephadm 2023-10-09T04:49:58.477264+0000 mgr.x (mgr.14967) 25 : cephadm [INF] Updating smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:50:00.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:59 smithi194 bash[31648]: cephadm 2023-10-09T04:49:58.478693+0000 mgr.x (mgr.14967) 26 : cephadm [INF] Updating smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:50:00.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:59 smithi194 bash[31648]: cephadm 2023-10-09T04:49:58.696520+0000 mgr.x (mgr.14967) 27 : cephadm [INF] Updating smithi194:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:50:00.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:59 smithi194 bash[31648]: cephadm 2023-10-09T04:49:58.697432+0000 mgr.x (mgr.14967) 28 : cephadm [INF] Updating smithi133:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:50:00.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:59 smithi194 bash[31648]: audit 2023-10-09T04:49:59.088016+0000 mon.a (mon.0) 1049 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:00.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:59 smithi194 bash[31648]: audit 2023-10-09T04:49:59.097319+0000 mon.a (mon.0) 1050 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:00.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:59 smithi194 bash[31648]: audit 2023-10-09T04:49:59.145884+0000 mon.a (mon.0) 1051 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:00.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:59 smithi194 bash[31648]: audit 2023-10-09T04:49:59.154019+0000 mon.a (mon.0) 1052 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:00.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:59 smithi194 bash[31648]: audit 2023-10-09T04:49:59.160497+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:00.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:59 smithi194 bash[31648]: audit 2023-10-09T04:49:59.235720+0000 mon.a (mon.0) 1054 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:00.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:59 smithi194 bash[31648]: audit 2023-10-09T04:49:59.245700+0000 mon.a (mon.0) 1055 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:00.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:59 smithi194 bash[31648]: audit 2023-10-09T04:49:59.256052+0000 mon.a (mon.0) 1056 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:00.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:49:59 smithi194 bash[31648]: audit 2023-10-09T04:49:59.263717+0000 mon.a (mon.0) 1057 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:00.347 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:00 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:50:00] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:50:01.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:00 smithi133 bash[27558]: cephadm 2023-10-09T04:49:58.914235+0000 mgr.x (mgr.14967) 29 : cephadm [INF] Updating smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:50:01.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:00 smithi133 bash[27558]: cephadm 2023-10-09T04:49:58.926779+0000 mgr.x (mgr.14967) 30 : cephadm [INF] Updating smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:50:01.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:00 smithi133 bash[27558]: cluster 2023-10-09T04:49:58.940751+0000 mgr.x (mgr.14967) 31 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 884 op/s 2023-10-09T04:50:01.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:00 smithi133 bash[27558]: cephadm 2023-10-09T04:49:59.266763+0000 mgr.x (mgr.14967) 32 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T04:50:01.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:00 smithi133 bash[27558]: cephadm 2023-10-09T04:49:59.267121+0000 mgr.x (mgr.14967) 33 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi133 2023-10-09T04:50:01.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:00 smithi133 bash[27558]: audit 2023-10-09T04:49:59.986992+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.133:0/176740087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:01.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:00 smithi133 bash[27558]: cluster 2023-10-09T04:50:00.000120+0000 mon.a (mon.0) 1058 : cluster [INF] overall HEALTH_OK 2023-10-09T04:50:01.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:00 smithi133 bash[41291]: cephadm 2023-10-09T04:49:58.914235+0000 mgr.x (mgr.14967) 29 : cephadm [INF] Updating smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:50:01.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:00 smithi133 bash[41291]: cephadm 2023-10-09T04:49:58.926779+0000 mgr.x (mgr.14967) 30 : cephadm [INF] Updating smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:50:01.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:00 smithi133 bash[41291]: cluster 2023-10-09T04:49:58.940751+0000 mgr.x (mgr.14967) 31 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 884 op/s 2023-10-09T04:50:01.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:00 smithi133 bash[41291]: cephadm 2023-10-09T04:49:59.266763+0000 mgr.x (mgr.14967) 32 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T04:50:01.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:00 smithi133 bash[41291]: cephadm 2023-10-09T04:49:59.267121+0000 mgr.x (mgr.14967) 33 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi133 2023-10-09T04:50:01.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:00 smithi133 bash[41291]: audit 2023-10-09T04:49:59.986992+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.133:0/176740087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:01.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:00 smithi133 bash[41291]: cluster 2023-10-09T04:50:00.000120+0000 mon.a (mon.0) 1058 : cluster [INF] overall HEALTH_OK 2023-10-09T04:50:01.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:00 smithi194 bash[31648]: cephadm 2023-10-09T04:49:58.914235+0000 mgr.x (mgr.14967) 29 : cephadm [INF] Updating smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:50:01.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:00 smithi194 bash[31648]: cephadm 2023-10-09T04:49:58.926779+0000 mgr.x (mgr.14967) 30 : cephadm [INF] Updating smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:50:01.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:00 smithi194 bash[31648]: cluster 2023-10-09T04:49:58.940751+0000 mgr.x (mgr.14967) 31 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 884 op/s 2023-10-09T04:50:01.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:00 smithi194 bash[31648]: cephadm 2023-10-09T04:49:59.266763+0000 mgr.x (mgr.14967) 32 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T04:50:01.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:00 smithi194 bash[31648]: cephadm 2023-10-09T04:49:59.267121+0000 mgr.x (mgr.14967) 33 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi133 2023-10-09T04:50:01.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:00 smithi194 bash[31648]: audit 2023-10-09T04:49:59.986992+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.133:0/176740087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:01.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:00 smithi194 bash[31648]: cluster 2023-10-09T04:50:00.000120+0000 mon.a (mon.0) 1058 : cluster [INF] overall HEALTH_OK 2023-10-09T04:50:01.315 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:01 smithi133 bash[90030]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-node-exporter-a 2023-10-09T04:50:01.662 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:01 smithi133 bash[27558]: audit 2023-10-09T04:50:01.111489+0000 mon.a (mon.0) 1059 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:01.662 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:01 smithi133 bash[27558]: audit 2023-10-09T04:50:01.377576+0000 mon.a (mon.0) 1060 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:01.662 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:01 smithi133 bash[27558]: audit 2023-10-09T04:50:01.386153+0000 mon.a (mon.0) 1061 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:01.663 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:01 smithi133 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T04:50:01.663 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:01 smithi133 bash[90090]: Error response from daemon: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-node-exporter-a 2023-10-09T04:50:01.663 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:01 smithi133 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-10-09T04:50:01.663 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:01 smithi133 systemd[1]: Stopped Ceph node-exporter.a for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:50:01.663 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:01 smithi133 systemd[1]: Started Ceph node-exporter.a for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:50:02.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:01 smithi133 bash[41291]: audit 2023-10-09T04:50:01.111489+0000 mon.a (mon.0) 1059 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:02.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:01 smithi133 bash[41291]: audit 2023-10-09T04:50:01.377576+0000 mon.a (mon.0) 1060 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:02.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:01 smithi133 bash[41291]: audit 2023-10-09T04:50:01.386153+0000 mon.a (mon.0) 1061 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:02.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:01 smithi194 bash[31648]: audit 2023-10-09T04:50:01.111489+0000 mon.a (mon.0) 1059 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:02.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:01 smithi194 bash[31648]: audit 2023-10-09T04:50:01.377576+0000 mon.a (mon.0) 1060 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:02.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:01 smithi194 bash[31648]: audit 2023-10-09T04:50:01.386153+0000 mon.a (mon.0) 1061 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:03.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:02 smithi133 bash[41291]: cluster 2023-10-09T04:50:00.941174+0000 mgr.x (mgr.14967) 34 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 598 op/s 2023-10-09T04:50:03.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:02 smithi133 bash[41291]: cephadm 2023-10-09T04:50:01.391652+0000 mgr.x (mgr.14967) 35 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T04:50:03.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:02 smithi133 bash[41291]: cephadm 2023-10-09T04:50:01.414959+0000 mgr.x (mgr.14967) 36 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi133 2023-10-09T04:50:03.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:02 smithi133 bash[27558]: cluster 2023-10-09T04:50:00.941174+0000 mgr.x (mgr.14967) 34 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 598 op/s 2023-10-09T04:50:03.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:02 smithi133 bash[27558]: cephadm 2023-10-09T04:50:01.391652+0000 mgr.x (mgr.14967) 35 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T04:50:03.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:02 smithi133 bash[27558]: cephadm 2023-10-09T04:50:01.414959+0000 mgr.x (mgr.14967) 36 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi133 2023-10-09T04:50:03.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:02 smithi194 bash[31648]: cluster 2023-10-09T04:50:00.941174+0000 mgr.x (mgr.14967) 34 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 598 op/s 2023-10-09T04:50:03.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:02 smithi194 bash[31648]: cephadm 2023-10-09T04:50:01.391652+0000 mgr.x (mgr.14967) 35 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T04:50:03.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:02 smithi194 bash[31648]: cephadm 2023-10-09T04:50:01.414959+0000 mgr.x (mgr.14967) 36 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi133 2023-10-09T04:50:03.925 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[41291]: audit 2023-10-09T04:50:02.658394+0000 mon.a (mon.0) 1062 : audit [DBG] from='client.? 172.21.15.133:0/2821125439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:03.926 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[27558]: audit 2023-10-09T04:50:02.658394+0000 mon.a (mon.0) 1062 : audit [DBG] from='client.? 172.21.15.133:0/2821125439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:04.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:03 smithi194 bash[31648]: audit 2023-10-09T04:50:02.658394+0000 mon.a (mon.0) 1062 : audit [DBG] from='client.? 172.21.15.133:0/2821125439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:04.301 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.981Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-10-09T04:50:04.302 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.981Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-10-09T04:50:04.302 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.982Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-10-09T04:50:04.302 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.982Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-10-09T04:50:04.302 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.983Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-10-09T04:50:04.302 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T04:50:04.302 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-10-09T04:50:04.302 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=arp 2023-10-09T04:50:04.302 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=bcache 2023-10-09T04:50:04.302 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=bonding 2023-10-09T04:50:04.302 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=btrfs 2023-10-09T04:50:04.303 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=conntrack 2023-10-09T04:50:04.303 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=cpu 2023-10-09T04:50:04.303 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-10-09T04:50:04.303 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=diskstats 2023-10-09T04:50:04.303 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=dmi 2023-10-09T04:50:04.303 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=edac 2023-10-09T04:50:04.303 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=entropy 2023-10-09T04:50:04.303 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-10-09T04:50:04.303 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=filefd 2023-10-09T04:50:04.303 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=filesystem 2023-10-09T04:50:04.303 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=hwmon 2023-10-09T04:50:04.303 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=infiniband 2023-10-09T04:50:04.304 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=ipvs 2023-10-09T04:50:04.304 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=loadavg 2023-10-09T04:50:04.304 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=mdadm 2023-10-09T04:50:04.304 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=meminfo 2023-10-09T04:50:04.304 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=netclass 2023-10-09T04:50:04.304 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=netdev 2023-10-09T04:50:04.304 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=netstat 2023-10-09T04:50:04.304 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=nfs 2023-10-09T04:50:04.304 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=nfsd 2023-10-09T04:50:04.304 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=nvme 2023-10-09T04:50:04.304 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=os 2023-10-09T04:50:04.304 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-10-09T04:50:04.305 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=pressure 2023-10-09T04:50:04.305 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=rapl 2023-10-09T04:50:04.305 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=schedstat 2023-10-09T04:50:04.305 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=selinux 2023-10-09T04:50:04.305 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=sockstat 2023-10-09T04:50:04.305 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=softnet 2023-10-09T04:50:04.305 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=stat 2023-10-09T04:50:04.305 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=tapestats 2023-10-09T04:50:04.305 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=textfile 2023-10-09T04:50:04.305 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-10-09T04:50:04.305 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=time 2023-10-09T04:50:04.305 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-10-09T04:50:04.305 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=uname 2023-10-09T04:50:04.306 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=vmstat 2023-10-09T04:50:04.306 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=xfs 2023-10-09T04:50:04.306 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=node_exporter.go:117 level=info collector=zfs 2023-10-09T04:50:04.306 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-10-09T04:50:04.306 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 04:50:03 smithi133 bash[90113]: ts=2023-10-09T04:50:03.984Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-10-09T04:50:04.672 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:04 smithi133 bash[27558]: cluster 2023-10-09T04:50:02.941924+0000 mgr.x (mgr.14967) 37 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 887 op/s 2023-10-09T04:50:05.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:04 smithi133 bash[41291]: cluster 2023-10-09T04:50:02.941924+0000 mgr.x (mgr.14967) 37 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 887 op/s 2023-10-09T04:50:05.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:04 smithi194 bash[31648]: cluster 2023-10-09T04:50:02.941924+0000 mgr.x (mgr.14967) 37 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 887 op/s 2023-10-09T04:50:06.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:05 smithi133 bash[41291]: audit 2023-10-09T04:50:05.338604+0000 mon.b (mon.1) 35 : audit [DBG] from='client.? 172.21.15.133:0/1677789244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:06.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:05 smithi133 bash[27558]: audit 2023-10-09T04:50:05.338604+0000 mon.b (mon.1) 35 : audit [DBG] from='client.? 172.21.15.133:0/1677789244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:06.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:05 smithi194 bash[31648]: audit 2023-10-09T04:50:05.338604+0000 mon.b (mon.1) 35 : audit [DBG] from='client.? 172.21.15.133:0/1677789244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:06.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:50:06 smithi194 bash[69734]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:50:06] "GET /metrics HTTP/1.1" 200 36173 "" "Prometheus/2.43.0" 2023-10-09T04:50:06.682 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:06 smithi133 bash[41291]: cluster 2023-10-09T04:50:04.942351+0000 mgr.x (mgr.14967) 38 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-10-09T04:50:07.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:06 smithi133 bash[27558]: cluster 2023-10-09T04:50:04.942351+0000 mgr.x (mgr.14967) 38 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-10-09T04:50:07.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:06 smithi194 bash[31648]: cluster 2023-10-09T04:50:04.942351+0000 mgr.x (mgr.14967) 38 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-10-09T04:50:07.801 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:50:07 smithi133 systemd[1]: Stopping Ceph alertmanager.a for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T04:50:07.801 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:50:07 smithi133 bash[75109]: ts=2023-10-09T04:50:07.581Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-10-09T04:50:08.608 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:50:08 smithi133 bash[90401]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T04:50:08.608 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:50:08 smithi133 bash[90484]: Error response from daemon: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T04:50:08.608 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:50:08 smithi133 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T04:50:08.608 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:50:08 smithi133 systemd[1]: Stopped Ceph alertmanager.a for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:50:08.608 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:50:08 smithi133 systemd[1]: Started Ceph alertmanager.a for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:50:09.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:08 smithi133 bash[41291]: cluster 2023-10-09T04:50:06.946823+0000 mgr.x (mgr.14967) 39 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-10-09T04:50:09.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:08 smithi133 bash[41291]: audit 2023-10-09T04:50:08.023527+0000 mon.a (mon.0) 1063 : audit [DBG] from='client.? 172.21.15.133:0/3726422080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:09.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:08 smithi133 bash[41291]: audit 2023-10-09T04:50:08.568838+0000 mon.a (mon.0) 1064 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:09.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:08 smithi133 bash[41291]: audit 2023-10-09T04:50:08.581300+0000 mon.a (mon.0) 1065 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:09.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:08 smithi133 bash[27558]: cluster 2023-10-09T04:50:06.946823+0000 mgr.x (mgr.14967) 39 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-10-09T04:50:09.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:08 smithi133 bash[27558]: audit 2023-10-09T04:50:08.023527+0000 mon.a (mon.0) 1063 : audit [DBG] from='client.? 172.21.15.133:0/3726422080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:09.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:08 smithi133 bash[27558]: audit 2023-10-09T04:50:08.568838+0000 mon.a (mon.0) 1064 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:09.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:08 smithi133 bash[27558]: audit 2023-10-09T04:50:08.581300+0000 mon.a (mon.0) 1065 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:09.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:08 smithi194 bash[31648]: cluster 2023-10-09T04:50:06.946823+0000 mgr.x (mgr.14967) 39 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-10-09T04:50:09.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:08 smithi194 bash[31648]: audit 2023-10-09T04:50:08.023527+0000 mon.a (mon.0) 1063 : audit [DBG] from='client.? 172.21.15.133:0/3726422080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:09.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:08 smithi194 bash[31648]: audit 2023-10-09T04:50:08.568838+0000 mon.a (mon.0) 1064 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:09.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:08 smithi194 bash[31648]: audit 2023-10-09T04:50:08.581300+0000 mon.a (mon.0) 1065 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:09.955 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:09 smithi133 bash[41291]: cephadm 2023-10-09T04:50:08.621340+0000 mgr.x (mgr.14967) 40 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T04:50:09.955 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:09 smithi133 bash[27558]: cephadm 2023-10-09T04:50:08.621340+0000 mgr.x (mgr.14967) 40 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T04:50:10.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:09 smithi194 bash[31648]: cephadm 2023-10-09T04:50:08.621340+0000 mgr.x (mgr.14967) 40 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T04:50:10.209 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:50:10 smithi133 bash[90509]: ts=2023-10-09T04:50:10.114Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-10-09T04:50:10.209 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:50:10 smithi133 bash[90509]: ts=2023-10-09T04:50:10.115Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-10-09T04:50:10.209 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:50:10 smithi133 bash[90509]: ts=2023-10-09T04:50:10.126Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.133 port=9094 2023-10-09T04:50:10.209 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:50:10 smithi133 bash[90509]: ts=2023-10-09T04:50:10.129Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T04:50:10.301 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:50:10 smithi133 bash[90509]: ts=2023-10-09T04:50:10.206Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T04:50:10.301 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:50:10 smithi133 bash[90509]: ts=2023-10-09T04:50:10.206Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T04:50:10.301 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:50:10 smithi133 bash[90509]: ts=2023-10-09T04:50:10.211Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-10-09T04:50:10.301 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:50:10 smithi133 bash[90509]: ts=2023-10-09T04:50:10.211Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-10-09T04:50:10.551 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:10 smithi133 bash[27821]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:50:10] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:50:11.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:10 smithi133 bash[41291]: cephadm 2023-10-09T04:50:08.883008+0000 mgr.x (mgr.14967) 41 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi194 2023-10-09T04:50:11.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:10 smithi133 bash[41291]: cluster 2023-10-09T04:50:08.949980+0000 mgr.x (mgr.14967) 42 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 877 op/s 2023-10-09T04:50:11.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:10 smithi133 bash[27558]: cephadm 2023-10-09T04:50:08.883008+0000 mgr.x (mgr.14967) 41 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi194 2023-10-09T04:50:11.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:10 smithi133 bash[27558]: cluster 2023-10-09T04:50:08.949980+0000 mgr.x (mgr.14967) 42 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 877 op/s 2023-10-09T04:50:11.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:10 smithi194 bash[31648]: cephadm 2023-10-09T04:50:08.883008+0000 mgr.x (mgr.14967) 41 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi194 2023-10-09T04:50:11.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:10 smithi194 bash[31648]: cluster 2023-10-09T04:50:08.949980+0000 mgr.x (mgr.14967) 42 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 877 op/s 2023-10-09T04:50:12.033 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:11 smithi194 bash[31648]: audit 2023-10-09T04:50:10.734411+0000 mon.b (mon.1) 36 : audit [DBG] from='client.? 172.21.15.133:0/328145693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:12.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:11 smithi133 bash[41291]: audit 2023-10-09T04:50:10.734411+0000 mon.b (mon.1) 36 : audit [DBG] from='client.? 172.21.15.133:0/328145693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:12.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:11 smithi133 bash[27558]: audit 2023-10-09T04:50:10.734411+0000 mon.b (mon.1) 36 : audit [DBG] from='client.? 172.21.15.133:0/328145693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:12.551 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:50:12 smithi133 bash[90509]: ts=2023-10-09T04:50:12.129Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000738204s 2023-10-09T04:50:13.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:12 smithi133 bash[41291]: cluster 2023-10-09T04:50:10.950393+0000 mgr.x (mgr.14967) 43 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 584 op/s 2023-10-09T04:50:13.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:12 smithi133 bash[27558]: cluster 2023-10-09T04:50:10.950393+0000 mgr.x (mgr.14967) 43 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 584 op/s 2023-10-09T04:50:13.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:12 smithi194 bash[31648]: cluster 2023-10-09T04:50:10.950393+0000 mgr.x (mgr.14967) 43 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 584 op/s 2023-10-09T04:50:13.980 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:13 smithi194 bash[31648]: audit 2023-10-09T04:50:13.370891+0000 mon.b (mon.1) 37 : audit [DBG] from='client.? 172.21.15.133:0/281850943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:14.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:13 smithi133 bash[41291]: audit 2023-10-09T04:50:13.370891+0000 mon.b (mon.1) 37 : audit [DBG] from='client.? 172.21.15.133:0/281850943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:14.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:13 smithi133 bash[27558]: audit 2023-10-09T04:50:13.370891+0000 mon.b (mon.1) 37 : audit [DBG] from='client.? 172.21.15.133:0/281850943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:15.044 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:14 smithi194 bash[31648]: cluster 2023-10-09T04:50:12.951005+0000 mgr.x (mgr.14967) 44 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 877 op/s 2023-10-09T04:50:15.045 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:15 smithi194 systemd[1]: Stopping Ceph prometheus.a for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T04:50:15.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:14 smithi133 bash[41291]: cluster 2023-10-09T04:50:12.951005+0000 mgr.x (mgr.14967) 44 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 877 op/s 2023-10-09T04:50:15.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:14 smithi133 bash[27558]: cluster 2023-10-09T04:50:12.951005+0000 mgr.x (mgr.14967) 44 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 877 op/s 2023-10-09T04:50:15.387 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:15 smithi194 bash[67868]: ts=2023-10-09T04:50:15.197Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T04:50:15.387 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:15 smithi194 bash[67868]: ts=2023-10-09T04:50:15.197Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-10-09T04:50:15.387 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:15 smithi194 bash[67868]: ts=2023-10-09T04:50:15.197Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-10-09T04:50:15.387 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:15 smithi194 bash[67868]: ts=2023-10-09T04:50:15.197Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T04:50:15.387 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:15 smithi194 bash[67868]: ts=2023-10-09T04:50:15.197Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T04:50:15.387 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:15 smithi194 bash[67868]: ts=2023-10-09T04:50:15.197Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-10-09T04:50:15.387 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:15 smithi194 bash[67868]: ts=2023-10-09T04:50:15.197Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-10-09T04:50:15.388 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:15 smithi194 bash[67868]: ts=2023-10-09T04:50:15.197Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-10-09T04:50:15.388 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:15 smithi194 bash[67868]: ts=2023-10-09T04:50:15.198Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-10-09T04:50:15.388 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:15 smithi194 bash[67868]: ts=2023-10-09T04:50:15.198Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T04:50:15.388 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:15 smithi194 bash[67868]: ts=2023-10-09T04:50:15.198Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-10-09T04:50:15.388 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:15 smithi194 bash[67868]: ts=2023-10-09T04:50:15.198Z caller=main.go:1147 level=info msg="See you next time!" 2023-10-09T04:50:16.154 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:15 smithi194 bash[80059]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T04:50:16.154 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:16 smithi194 bash[80115]: Error response from daemon: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T04:50:16.638 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:16 smithi194 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T04:50:16.638 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:16 smithi194 systemd[1]: Stopped Ceph prometheus.a for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:50:16.638 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:16 smithi194 systemd[1]: Started Ceph prometheus.a for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:50:16.639 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:16 smithi194 systemd[1]: Stopping Ceph node-exporter.b for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T04:50:17.022 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:16 smithi194 bash[31648]: cluster 2023-10-09T04:50:14.951449+0000 mgr.x (mgr.14967) 45 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-10-09T04:50:17.022 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:16 smithi194 bash[31648]: audit 2023-10-09T04:50:16.070129+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.133:0/984281703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:17.023 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:16 smithi194 bash[31648]: audit 2023-10-09T04:50:16.111787+0000 mon.a (mon.0) 1066 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:17.023 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:16 smithi194 bash[31648]: audit 2023-10-09T04:50:16.202929+0000 mon.a (mon.0) 1067 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:17.023 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:16 smithi194 bash[31648]: audit 2023-10-09T04:50:16.212600+0000 mon.a (mon.0) 1068 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:17.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:16 smithi133 bash[41291]: cluster 2023-10-09T04:50:14.951449+0000 mgr.x (mgr.14967) 45 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-10-09T04:50:17.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:16 smithi133 bash[41291]: audit 2023-10-09T04:50:16.070129+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.133:0/984281703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:17.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:16 smithi133 bash[41291]: audit 2023-10-09T04:50:16.111787+0000 mon.a (mon.0) 1066 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:17.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:16 smithi133 bash[41291]: audit 2023-10-09T04:50:16.202929+0000 mon.a (mon.0) 1067 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:17.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:16 smithi133 bash[41291]: audit 2023-10-09T04:50:16.212600+0000 mon.a (mon.0) 1068 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:17.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:16 smithi133 bash[27558]: cluster 2023-10-09T04:50:14.951449+0000 mgr.x (mgr.14967) 45 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-10-09T04:50:17.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:16 smithi133 bash[27558]: audit 2023-10-09T04:50:16.070129+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.133:0/984281703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:17.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:16 smithi133 bash[27558]: audit 2023-10-09T04:50:16.111787+0000 mon.a (mon.0) 1066 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:17.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:16 smithi133 bash[27558]: audit 2023-10-09T04:50:16.202929+0000 mon.a (mon.0) 1067 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:17.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:16 smithi133 bash[27558]: audit 2023-10-09T04:50:16.212600+0000 mon.a (mon.0) 1068 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:18.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:17 smithi133 bash[27558]: cephadm 2023-10-09T04:50:16.246135+0000 mgr.x (mgr.14967) 46 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T04:50:18.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:17 smithi133 bash[27558]: cephadm 2023-10-09T04:50:16.246681+0000 mgr.x (mgr.14967) 47 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi194 2023-10-09T04:50:18.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:17 smithi133 bash[41291]: cephadm 2023-10-09T04:50:16.246135+0000 mgr.x (mgr.14967) 46 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T04:50:18.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:17 smithi133 bash[41291]: cephadm 2023-10-09T04:50:16.246681+0000 mgr.x (mgr.14967) 47 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi194 2023-10-09T04:50:18.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:17 smithi194 bash[31648]: cephadm 2023-10-09T04:50:16.246135+0000 mgr.x (mgr.14967) 46 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T04:50:18.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:17 smithi194 bash[31648]: cephadm 2023-10-09T04:50:16.246681+0000 mgr.x (mgr.14967) 47 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi194 2023-10-09T04:50:18.137 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:17 smithi194 bash[80183]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-node-exporter-b 2023-10-09T04:50:18.451 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:18 smithi194 bash[80137]: ts=2023-10-09T04:50:18.352Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-10-09T04:50:18.451 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:18 smithi194 bash[80137]: ts=2023-10-09T04:50:18.352Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-10-09T04:50:18.451 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:18 smithi194 bash[80137]: ts=2023-10-09T04:50:18.352Z caller=main.go:570 level=info host_details="(Linux 5.15.0-86-generic #96~20.04.1-Ubuntu SMP Thu Sep 21 13:23:37 UTC 2023 x86_64 smithi194 (none))" 2023-10-09T04:50:18.451 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:18 smithi194 bash[80137]: ts=2023-10-09T04:50:18.352Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T04:50:18.451 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:18 smithi194 bash[80137]: ts=2023-10-09T04:50:18.352Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T04:50:18.451 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:18 smithi194 bash[80137]: ts=2023-10-09T04:50:18.354Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T04:50:18.451 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:18 smithi194 bash[80137]: ts=2023-10-09T04:50:18.355Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-10-09T04:50:18.451 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:18 smithi194 bash[80137]: ts=2023-10-09T04:50:18.356Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-10-09T04:50:18.451 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:18 smithi194 bash[80137]: ts=2023-10-09T04:50:18.358Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-10-09T04:50:18.451 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:18 smithi194 bash[80137]: ts=2023-10-09T04:50:18.361Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T04:50:18.452 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:18 smithi194 bash[80137]: ts=2023-10-09T04:50:18.361Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=3.527µs 2023-10-09T04:50:18.452 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:18 smithi194 bash[80137]: ts=2023-10-09T04:50:18.361Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T04:50:18.452 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:18 smithi194 bash[80137]: ts=2023-10-09T04:50:18.368Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2023-10-09T04:50:18.452 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:18 smithi194 bash[80137]: ts=2023-10-09T04:50:18.448Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2023-10-09T04:50:18.452 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:18 smithi194 bash[80137]: ts=2023-10-09T04:50:18.448Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2023-10-09T04:50:18.452 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:18 smithi194 bash[80137]: ts=2023-10-09T04:50:18.448Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=49.751µs wal_replay_duration=87.525369ms wbl_replay_duration=134ns total_replay_duration=87.677586ms 2023-10-09T04:50:18.452 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:18 smithi194 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-10-09T04:50:18.452 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:18 smithi194 bash[80267]: Error response from daemon: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-node-exporter-b 2023-10-09T04:50:18.452 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:18 smithi194 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-10-09T04:50:18.452 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:18 smithi194 systemd[1]: Stopped Ceph node-exporter.b for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:50:18.452 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:18 smithi194 systemd[1]: Started Ceph node-exporter.b for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:50:18.730 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:18 smithi194 bash[80137]: ts=2023-10-09T04:50:18.452Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T04:50:18.731 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:18 smithi194 bash[80137]: ts=2023-10-09T04:50:18.452Z caller=main.go:1029 level=info msg="TSDB started" 2023-10-09T04:50:18.731 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:18 smithi194 bash[80137]: ts=2023-10-09T04:50:18.452Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T04:50:18.731 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:18 smithi194 bash[80137]: ts=2023-10-09T04:50:18.495Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=43.26081ms db_storage=1.361µs remote_storage=1.817µs web_handler=478ns query_engine=833ns scrape=431.016µs scrape_sd=140.643µs notify=27.807µs notify_sd=19.369µs rules=42.226655ms tracing=7.781µs 2023-10-09T04:50:18.731 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:18 smithi194 bash[80137]: ts=2023-10-09T04:50:18.495Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-10-09T04:50:18.731 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:18 smithi194 bash[80137]: ts=2023-10-09T04:50:18.495Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-10-09T04:50:18.731 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:18 smithi194 bash[31648]: cluster 2023-10-09T04:50:16.951925+0000 mgr.x (mgr.14967) 48 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-10-09T04:50:18.731 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:18 smithi194 bash[31648]: audit 2023-10-09T04:50:18.267019+0000 mon.a (mon.0) 1069 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:18.732 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:18 smithi194 bash[31648]: audit 2023-10-09T04:50:18.276442+0000 mon.a (mon.0) 1070 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:18.732 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:18 smithi194 bash[31648]: audit 2023-10-09T04:50:18.698967+0000 mon.a (mon.0) 1071 : audit [DBG] from='client.? 172.21.15.133:0/2873674242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:19.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:18 smithi133 bash[41291]: cluster 2023-10-09T04:50:16.951925+0000 mgr.x (mgr.14967) 48 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-10-09T04:50:19.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:18 smithi133 bash[41291]: audit 2023-10-09T04:50:18.267019+0000 mon.a (mon.0) 1069 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:19.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:18 smithi133 bash[41291]: audit 2023-10-09T04:50:18.276442+0000 mon.a (mon.0) 1070 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:19.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:18 smithi133 bash[41291]: audit 2023-10-09T04:50:18.698967+0000 mon.a (mon.0) 1071 : audit [DBG] from='client.? 172.21.15.133:0/2873674242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:19.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:18 smithi133 bash[27558]: cluster 2023-10-09T04:50:16.951925+0000 mgr.x (mgr.14967) 48 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-10-09T04:50:19.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:18 smithi133 bash[27558]: audit 2023-10-09T04:50:18.267019+0000 mon.a (mon.0) 1069 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:19.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:18 smithi133 bash[27558]: audit 2023-10-09T04:50:18.276442+0000 mon.a (mon.0) 1070 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:19.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:18 smithi133 bash[27558]: audit 2023-10-09T04:50:18.698967+0000 mon.a (mon.0) 1071 : audit [DBG] from='client.? 172.21.15.133:0/2873674242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:20.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:19 smithi133 bash[41291]: cephadm 2023-10-09T04:50:18.323767+0000 mgr.x (mgr.14967) 49 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-10-09T04:50:20.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:19 smithi133 bash[41291]: cephadm 2023-10-09T04:50:18.344083+0000 mgr.x (mgr.14967) 50 : cephadm [INF] Reconfiguring daemon grafana.a on smithi194 2023-10-09T04:50:20.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:19 smithi133 bash[27558]: cephadm 2023-10-09T04:50:18.323767+0000 mgr.x (mgr.14967) 49 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-10-09T04:50:20.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:19 smithi133 bash[27558]: cephadm 2023-10-09T04:50:18.344083+0000 mgr.x (mgr.14967) 50 : cephadm [INF] Reconfiguring daemon grafana.a on smithi194 2023-10-09T04:50:20.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:19 smithi194 bash[31648]: cephadm 2023-10-09T04:50:18.323767+0000 mgr.x (mgr.14967) 49 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-10-09T04:50:20.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:19 smithi194 bash[31648]: cephadm 2023-10-09T04:50:18.344083+0000 mgr.x (mgr.14967) 50 : cephadm [INF] Reconfiguring daemon grafana.a on smithi194 2023-10-09T04:50:20.551 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:50:20 smithi133 bash[90509]: ts=2023-10-09T04:50:20.132Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002938394s 2023-10-09T04:50:20.637 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.360Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-10-09T04:50:20.637 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.360Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-10-09T04:50:20.638 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.365Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-10-09T04:50:20.638 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.365Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-10-09T04:50:20.638 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.365Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-10-09T04:50:20.638 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.365Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T04:50:20.638 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-10-09T04:50:20.638 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=arp 2023-10-09T04:50:20.638 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=bcache 2023-10-09T04:50:20.638 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=bonding 2023-10-09T04:50:20.638 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=btrfs 2023-10-09T04:50:20.638 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=conntrack 2023-10-09T04:50:20.638 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=cpu 2023-10-09T04:50:20.639 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-10-09T04:50:20.639 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=diskstats 2023-10-09T04:50:20.639 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=dmi 2023-10-09T04:50:20.639 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=edac 2023-10-09T04:50:20.639 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=entropy 2023-10-09T04:50:20.639 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-10-09T04:50:20.639 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=filefd 2023-10-09T04:50:20.639 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=filesystem 2023-10-09T04:50:20.639 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=hwmon 2023-10-09T04:50:20.639 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=infiniband 2023-10-09T04:50:20.639 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=ipvs 2023-10-09T04:50:20.639 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=loadavg 2023-10-09T04:50:20.640 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=mdadm 2023-10-09T04:50:20.640 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=meminfo 2023-10-09T04:50:20.640 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=netclass 2023-10-09T04:50:20.640 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=netdev 2023-10-09T04:50:20.640 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=netstat 2023-10-09T04:50:20.640 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=nfs 2023-10-09T04:50:20.640 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=nfsd 2023-10-09T04:50:20.640 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=nvme 2023-10-09T04:50:20.640 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=os 2023-10-09T04:50:20.640 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-10-09T04:50:20.640 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=pressure 2023-10-09T04:50:20.640 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=rapl 2023-10-09T04:50:20.640 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=schedstat 2023-10-09T04:50:20.641 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=selinux 2023-10-09T04:50:20.641 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=sockstat 2023-10-09T04:50:20.641 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=softnet 2023-10-09T04:50:20.641 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=stat 2023-10-09T04:50:20.641 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=tapestats 2023-10-09T04:50:20.641 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=textfile 2023-10-09T04:50:20.641 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-10-09T04:50:20.641 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=time 2023-10-09T04:50:20.641 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-10-09T04:50:20.641 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=uname 2023-10-09T04:50:20.641 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=vmstat 2023-10-09T04:50:20.641 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=xfs 2023-10-09T04:50:20.642 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=node_exporter.go:117 level=info collector=zfs 2023-10-09T04:50:20.642 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.366Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-10-09T04:50:20.642 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[80313]: ts=2023-10-09T04:50:20.367Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-10-09T04:50:21.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:20 smithi133 bash[27558]: cluster 2023-10-09T04:50:18.952772+0000 mgr.x (mgr.14967) 51 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 872 op/s 2023-10-09T04:50:21.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:20 smithi133 bash[41291]: cluster 2023-10-09T04:50:18.952772+0000 mgr.x (mgr.14967) 51 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 872 op/s 2023-10-09T04:50:21.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:20 smithi194 bash[31648]: cluster 2023-10-09T04:50:18.952772+0000 mgr.x (mgr.14967) 51 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 872 op/s 2023-10-09T04:50:22.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:21 smithi133 bash[41291]: cluster 2023-10-09T04:50:20.953246+0000 mgr.x (mgr.14967) 52 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 577 op/s 2023-10-09T04:50:22.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:21 smithi133 bash[41291]: audit 2023-10-09T04:50:21.385173+0000 mon.b (mon.1) 38 : audit [DBG] from='client.? 172.21.15.133:0/3693005148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:22.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:21 smithi133 bash[27558]: cluster 2023-10-09T04:50:20.953246+0000 mgr.x (mgr.14967) 52 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 577 op/s 2023-10-09T04:50:22.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:21 smithi133 bash[27558]: audit 2023-10-09T04:50:21.385173+0000 mon.b (mon.1) 38 : audit [DBG] from='client.? 172.21.15.133:0/3693005148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:22.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:21 smithi194 bash[31648]: cluster 2023-10-09T04:50:20.953246+0000 mgr.x (mgr.14967) 52 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 577 op/s 2023-10-09T04:50:22.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:21 smithi194 bash[31648]: audit 2023-10-09T04:50:21.385173+0000 mon.b (mon.1) 38 : audit [DBG] from='client.? 172.21.15.133:0/3693005148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:23.637 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:23 smithi194 systemd[1]: Stopping Ceph grafana.a for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T04:50:24.027 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:24 smithi194 bash[31648]: cluster 2023-10-09T04:50:22.953812+0000 mgr.x (mgr.14967) 53 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-10-09T04:50:24.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:24 smithi133 bash[27558]: cluster 2023-10-09T04:50:22.953812+0000 mgr.x (mgr.14967) 53 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-10-09T04:50:24.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:24 smithi133 bash[41291]: cluster 2023-10-09T04:50:22.953812+0000 mgr.x (mgr.14967) 53 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-10-09T04:50:24.340 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:24 smithi194 bash[80563]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-grafana-a 2023-10-09T04:50:24.637 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:24 smithi194 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@grafana.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T04:50:24.637 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:24 smithi194 bash[80617]: Error response from daemon: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-grafana-a 2023-10-09T04:50:24.637 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:24 smithi194 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@grafana.a.service: Failed with result 'exit-code'. 2023-10-09T04:50:24.637 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:24 smithi194 systemd[1]: Stopped Ceph grafana.a for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:50:24.637 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:24 smithi194 systemd[1]: Started Ceph grafana.a for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:50:25.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:25 smithi133 bash[27558]: audit 2023-10-09T04:50:24.040542+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.133:0/2934860418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:25.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:25 smithi133 bash[27558]: audit 2023-10-09T04:50:24.398716+0000 mon.a (mon.0) 1072 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:25.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:25 smithi133 bash[27558]: audit 2023-10-09T04:50:24.417006+0000 mon.a (mon.0) 1073 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:25.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:25 smithi133 bash[27558]: audit 2023-10-09T04:50:24.464876+0000 mon.a (mon.0) 1074 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:50:25.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:25 smithi133 bash[27558]: audit 2023-10-09T04:50:24.467678+0000 mgr.x (mgr.14967) 54 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:50:25.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:25 smithi133 bash[27558]: audit 2023-10-09T04:50:24.471343+0000 mon.a (mon.0) 1075 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:50:25.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:25 smithi133 bash[27558]: audit 2023-10-09T04:50:24.474064+0000 mgr.x (mgr.14967) 55 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:50:25.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:25 smithi133 bash[27558]: audit 2023-10-09T04:50:24.484385+0000 mon.a (mon.0) 1076 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:50:25.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:25 smithi133 bash[27558]: audit 2023-10-09T04:50:24.487159+0000 mgr.x (mgr.14967) 56 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:50:25.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:25 smithi133 bash[27558]: audit 2023-10-09T04:50:24.496561+0000 mon.a (mon.0) 1077 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:50:25.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:25 smithi133 bash[27558]: cephadm 2023-10-09T04:50:24.511790+0000 mgr.x (mgr.14967) 57 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:50:25.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:25 smithi133 bash[27558]: cephadm 2023-10-09T04:50:24.512474+0000 mgr.x (mgr.14967) 58 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:50:25.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:25 smithi133 bash[41291]: audit 2023-10-09T04:50:24.040542+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.133:0/2934860418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:25.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:25 smithi133 bash[41291]: audit 2023-10-09T04:50:24.398716+0000 mon.a (mon.0) 1072 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:25.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:25 smithi133 bash[41291]: audit 2023-10-09T04:50:24.417006+0000 mon.a (mon.0) 1073 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:25.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:25 smithi133 bash[41291]: audit 2023-10-09T04:50:24.464876+0000 mon.a (mon.0) 1074 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:50:25.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:25 smithi133 bash[41291]: audit 2023-10-09T04:50:24.467678+0000 mgr.x (mgr.14967) 54 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:50:25.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:25 smithi133 bash[41291]: audit 2023-10-09T04:50:24.471343+0000 mon.a (mon.0) 1075 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:50:25.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:25 smithi133 bash[41291]: audit 2023-10-09T04:50:24.474064+0000 mgr.x (mgr.14967) 55 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:50:25.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:25 smithi133 bash[41291]: audit 2023-10-09T04:50:24.484385+0000 mon.a (mon.0) 1076 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:50:25.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:25 smithi133 bash[41291]: audit 2023-10-09T04:50:24.487159+0000 mgr.x (mgr.14967) 56 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:50:25.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:25 smithi133 bash[41291]: audit 2023-10-09T04:50:24.496561+0000 mon.a (mon.0) 1077 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:50:25.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:25 smithi133 bash[41291]: cephadm 2023-10-09T04:50:24.511790+0000 mgr.x (mgr.14967) 57 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:50:25.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:25 smithi133 bash[41291]: cephadm 2023-10-09T04:50:24.512474+0000 mgr.x (mgr.14967) 58 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:50:25.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:25 smithi194 bash[31648]: audit 2023-10-09T04:50:24.040542+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.133:0/2934860418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:25.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:25 smithi194 bash[31648]: audit 2023-10-09T04:50:24.398716+0000 mon.a (mon.0) 1072 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:25.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:25 smithi194 bash[31648]: audit 2023-10-09T04:50:24.417006+0000 mon.a (mon.0) 1073 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:25.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:25 smithi194 bash[31648]: audit 2023-10-09T04:50:24.464876+0000 mon.a (mon.0) 1074 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:50:25.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:25 smithi194 bash[31648]: audit 2023-10-09T04:50:24.467678+0000 mgr.x (mgr.14967) 54 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:50:25.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:25 smithi194 bash[31648]: audit 2023-10-09T04:50:24.471343+0000 mon.a (mon.0) 1075 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:50:25.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:25 smithi194 bash[31648]: audit 2023-10-09T04:50:24.474064+0000 mgr.x (mgr.14967) 55 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:50:25.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:25 smithi194 bash[31648]: audit 2023-10-09T04:50:24.484385+0000 mon.a (mon.0) 1076 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:50:25.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:25 smithi194 bash[31648]: audit 2023-10-09T04:50:24.487159+0000 mgr.x (mgr.14967) 56 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:50:25.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:25 smithi194 bash[31648]: audit 2023-10-09T04:50:24.496561+0000 mon.a (mon.0) 1077 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:50:25.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:25 smithi194 bash[31648]: cephadm 2023-10-09T04:50:24.511790+0000 mgr.x (mgr.14967) 57 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:50:25.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:25 smithi194 bash[31648]: cephadm 2023-10-09T04:50:24.512474+0000 mgr.x (mgr.14967) 58 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:50:26.387 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:25 smithi194 bash[80638]: logger=settings t=2023-10-09T04:50:25.950910929Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2023-10-09T04:50:26.387 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:25 smithi194 bash[80638]: logger=settings t=2023-10-09T04:50:25.951214749Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2023-10-09T04:50:26.388 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:25 smithi194 bash[80638]: logger=settings t=2023-10-09T04:50:25.951247448Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2023-10-09T04:50:26.388 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:25 smithi194 bash[80638]: logger=settings t=2023-10-09T04:50:25.951264281Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2023-10-09T04:50:26.388 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:25 smithi194 bash[80638]: logger=settings t=2023-10-09T04:50:25.951284056Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2023-10-09T04:50:26.388 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:25 smithi194 bash[80638]: logger=settings t=2023-10-09T04:50:25.951300734Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2023-10-09T04:50:26.388 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:25 smithi194 bash[80638]: logger=settings t=2023-10-09T04:50:25.951315581Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-10-09T04:50:26.388 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:25 smithi194 bash[80638]: logger=settings t=2023-10-09T04:50:25.951331095Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-10-09T04:50:26.388 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:25 smithi194 bash[80638]: logger=settings t=2023-10-09T04:50:25.951354576Z level=info msg="Path Home" path=/usr/share/grafana 2023-10-09T04:50:26.389 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:25 smithi194 bash[80638]: logger=settings t=2023-10-09T04:50:25.951369713Z level=info msg="Path Data" path=/var/lib/grafana 2023-10-09T04:50:26.389 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:25 smithi194 bash[80638]: logger=settings t=2023-10-09T04:50:25.951383366Z level=info msg="Path Logs" path=/var/log/grafana 2023-10-09T04:50:26.389 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:25 smithi194 bash[80638]: logger=settings t=2023-10-09T04:50:25.951403065Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2023-10-09T04:50:26.389 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:25 smithi194 bash[80638]: logger=settings t=2023-10-09T04:50:25.95141922Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2023-10-09T04:50:26.389 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:25 smithi194 bash[80638]: logger=settings t=2023-10-09T04:50:25.951431154Z level=info msg="App mode production" 2023-10-09T04:50:26.389 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:25 smithi194 bash[80638]: logger=sqlstore t=2023-10-09T04:50:25.951539199Z level=info msg="Connecting to DB" dbtype=sqlite3 2023-10-09T04:50:26.389 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:25 smithi194 bash[80638]: logger=sqlstore t=2023-10-09T04:50:25.951614609Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-10-09T04:50:26.390 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:25 smithi194 bash[80638]: logger=migrator t=2023-10-09T04:50:25.995628494Z level=info msg="Starting DB migrations" 2023-10-09T04:50:26.390 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:26 smithi194 bash[80638]: logger=migrator t=2023-10-09T04:50:26.008070252Z level=info msg="migrations completed" performed=0 skipped=481 duration=965.851µs 2023-10-09T04:50:26.390 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:26 smithi194 bash[80638]: logger=sqlstore t=2023-10-09T04:50:26.009484646Z level=info msg="Created default organization" 2023-10-09T04:50:26.390 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:26 smithi194 bash[80638]: logger=plugin.loader t=2023-10-09T04:50:26.116265075Z level=info msg="Plugin registered" pluginID=input 2023-10-09T04:50:26.390 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:26 smithi194 bash[80638]: logger=plugin.loader t=2023-10-09T04:50:26.159676154Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2023-10-09T04:50:26.390 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:26 smithi194 bash[80638]: logger=plugin.loader t=2023-10-09T04:50:26.160317333Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2023-10-09T04:50:26.391 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:26 smithi194 bash[80638]: logger=secrets t=2023-10-09T04:50:26.160725429Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2023-10-09T04:50:26.391 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:26 smithi194 bash[80638]: logger=query_data t=2023-10-09T04:50:26.163921011Z level=info msg="Query Service initialization" 2023-10-09T04:50:26.391 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:26 smithi194 bash[80638]: logger=live.push_http t=2023-10-09T04:50:26.167447126Z level=info msg="Live Push Gateway initialization" 2023-10-09T04:50:26.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:26 smithi194 bash[31648]: cluster 2023-10-09T04:50:24.954242+0000 mgr.x (mgr.14967) 59 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-10-09T04:50:26.936 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:26 smithi133 bash[27558]: cluster 2023-10-09T04:50:24.954242+0000 mgr.x (mgr.14967) 59 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-10-09T04:50:26.936 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:26 smithi133 bash[41291]: cluster 2023-10-09T04:50:24.954242+0000 mgr.x (mgr.14967) 59 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-10-09T04:50:27.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:50:26 smithi194 bash[69734]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:50:26] "GET /metrics HTTP/1.1" 200 36172 "" "Prometheus/2.43.0" 2023-10-09T04:50:27.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:27 smithi194 bash[31648]: audit 2023-10-09T04:50:26.704880+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.133:0/3912261493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:27.887 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:27 smithi194 bash[80638]: logger=infra.usagestats.collector t=2023-10-09T04:50:27.564200275Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2023-10-09T04:50:27.887 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:27 smithi194 bash[80638]: logger=provisioning.datasources t=2023-10-09T04:50:27.701533578Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2023-10-09T04:50:27.887 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:27 smithi194 bash[80638]: logger=provisioning.datasources t=2023-10-09T04:50:27.701582142Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2023-10-09T04:50:27.888 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:27 smithi194 bash[80638]: logger=provisioning.datasources t=2023-10-09T04:50:27.712126925Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2023-10-09T04:50:27.914 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:27 smithi133 bash[27558]: audit 2023-10-09T04:50:26.704880+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.133:0/3912261493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:27.914 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:27 smithi133 bash[41291]: audit 2023-10-09T04:50:26.704880+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.133:0/3912261493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:28.387 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:27 smithi194 bash[80638]: logger=provisioning.alerting t=2023-10-09T04:50:27.905140741Z level=info msg="starting to provision alerting" 2023-10-09T04:50:28.387 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:27 smithi194 bash[80638]: logger=provisioning.alerting t=2023-10-09T04:50:27.905486791Z level=info msg="finished to provision alerting" 2023-10-09T04:50:28.387 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:27 smithi194 bash[80638]: logger=ngalert.state.manager t=2023-10-09T04:50:27.905749972Z level=info msg="Warming state cache for startup" 2023-10-09T04:50:28.387 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:27 smithi194 bash[80638]: logger=grafanaStorageLogger t=2023-10-09T04:50:27.905950768Z level=info msg="storage starting" 2023-10-09T04:50:28.387 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:27 smithi194 bash[80638]: logger=http.server t=2023-10-09T04:50:27.910623062Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2023-10-09T04:50:28.387 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:27 smithi194 bash[80638]: logger=ngalert.state.manager t=2023-10-09T04:50:27.95912269Z level=info msg="State cache has been initialized" states=0 duration=53.369763ms 2023-10-09T04:50:28.388 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:27 smithi194 bash[80638]: logger=ticker t=2023-10-09T04:50:27.959986855Z level=info msg=starting first_tick=2023-10-09T04:50:30Z 2023-10-09T04:50:28.388 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 04:50:27 smithi194 bash[80638]: logger=ngalert.multiorg.alertmanager t=2023-10-09T04:50:27.960233003Z level=info msg="starting MultiOrg Alertmanager" 2023-10-09T04:50:28.593 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:28 smithi133 bash[27558]: cluster 2023-10-09T04:50:26.954694+0000 mgr.x (mgr.14967) 60 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-10-09T04:50:28.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:28 smithi194 bash[31648]: cluster 2023-10-09T04:50:26.954694+0000 mgr.x (mgr.14967) 60 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-10-09T04:50:28.887 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:28 smithi194 bash[80137]: ts=2023-10-09T04:50:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi194:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi194.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:50:29.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:28 smithi133 bash[41291]: cluster 2023-10-09T04:50:26.954694+0000 mgr.x (mgr.14967) 60 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-10-09T04:50:30.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:30 smithi133 bash[41291]: cluster 2023-10-09T04:50:28.955592+0000 mgr.x (mgr.14967) 61 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 867 op/s 2023-10-09T04:50:30.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:30 smithi133 bash[41291]: cephadm 2023-10-09T04:50:29.052664+0000 mgr.x (mgr.14967) 62 : cephadm [INF] Upgrade: Updating mgr.y 2023-10-09T04:50:30.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:30 smithi133 bash[41291]: audit 2023-10-09T04:50:29.059380+0000 mon.a (mon.0) 1078 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:30.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:30 smithi133 bash[41291]: audit 2023-10-09T04:50:29.061161+0000 mon.a (mon.0) 1079 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:50:30.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:30 smithi133 bash[41291]: audit 2023-10-09T04:50:29.062905+0000 mon.a (mon.0) 1080 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:50:30.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:30 smithi133 bash[41291]: audit 2023-10-09T04:50:29.064480+0000 mon.a (mon.0) 1081 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:50:30.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:30 smithi133 bash[41291]: cephadm 2023-10-09T04:50:29.071348+0000 mgr.x (mgr.14967) 63 : cephadm [INF] Deploying daemon mgr.y on smithi133 2023-10-09T04:50:30.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:30 smithi133 bash[41291]: audit 2023-10-09T04:50:29.449977+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.133:0/2292635595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:30.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:30 smithi133 bash[27558]: cluster 2023-10-09T04:50:28.955592+0000 mgr.x (mgr.14967) 61 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 867 op/s 2023-10-09T04:50:30.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:30 smithi133 bash[27558]: cephadm 2023-10-09T04:50:29.052664+0000 mgr.x (mgr.14967) 62 : cephadm [INF] Upgrade: Updating mgr.y 2023-10-09T04:50:30.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:30 smithi133 bash[27558]: audit 2023-10-09T04:50:29.059380+0000 mon.a (mon.0) 1078 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:30.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:30 smithi133 bash[27558]: audit 2023-10-09T04:50:29.061161+0000 mon.a (mon.0) 1079 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:50:30.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:30 smithi133 bash[27558]: audit 2023-10-09T04:50:29.062905+0000 mon.a (mon.0) 1080 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:50:30.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:30 smithi133 bash[27558]: audit 2023-10-09T04:50:29.064480+0000 mon.a (mon.0) 1081 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:50:30.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:30 smithi133 bash[27558]: cephadm 2023-10-09T04:50:29.071348+0000 mgr.x (mgr.14967) 63 : cephadm [INF] Deploying daemon mgr.y on smithi133 2023-10-09T04:50:30.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:30 smithi133 bash[27558]: audit 2023-10-09T04:50:29.449977+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.133:0/2292635595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:30.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:30 smithi194 bash[31648]: cluster 2023-10-09T04:50:28.955592+0000 mgr.x (mgr.14967) 61 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 867 op/s 2023-10-09T04:50:30.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:30 smithi194 bash[31648]: cephadm 2023-10-09T04:50:29.052664+0000 mgr.x (mgr.14967) 62 : cephadm [INF] Upgrade: Updating mgr.y 2023-10-09T04:50:30.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:30 smithi194 bash[31648]: audit 2023-10-09T04:50:29.059380+0000 mon.a (mon.0) 1078 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:30.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:30 smithi194 bash[31648]: audit 2023-10-09T04:50:29.061161+0000 mon.a (mon.0) 1079 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:50:30.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:30 smithi194 bash[31648]: audit 2023-10-09T04:50:29.062905+0000 mon.a (mon.0) 1080 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:50:30.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:30 smithi194 bash[31648]: audit 2023-10-09T04:50:29.064480+0000 mon.a (mon.0) 1081 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:50:30.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:30 smithi194 bash[31648]: cephadm 2023-10-09T04:50:29.071348+0000 mgr.x (mgr.14967) 63 : cephadm [INF] Deploying daemon mgr.y on smithi133 2023-10-09T04:50:30.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:30 smithi194 bash[31648]: audit 2023-10-09T04:50:29.449977+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.133:0/2292635595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:31.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:31 smithi194 bash[31648]: audit 2023-10-09T04:50:31.112147+0000 mon.a (mon.0) 1082 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:31.909 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:31 smithi133 bash[27558]: audit 2023-10-09T04:50:31.112147+0000 mon.a (mon.0) 1082 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:31.910 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:31 smithi133 bash[41291]: audit 2023-10-09T04:50:31.112147+0000 mon.a (mon.0) 1082 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:32.261 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:32 smithi133 systemd[1]: Stopping Ceph mgr.y for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T04:50:32.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:32 smithi133 bash[27558]: cluster 2023-10-09T04:50:30.956113+0000 mgr.x (mgr.14967) 64 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-10-09T04:50:32.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:32 smithi133 bash[27558]: audit 2023-10-09T04:50:32.182978+0000 mon.c (mon.2) 47 : audit [DBG] from='client.? 172.21.15.133:0/321173391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:32.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:32 smithi133 bash[41291]: cluster 2023-10-09T04:50:30.956113+0000 mgr.x (mgr.14967) 64 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-10-09T04:50:32.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:32 smithi133 bash[41291]: audit 2023-10-09T04:50:32.182978+0000 mon.c (mon.2) 47 : audit [DBG] from='client.? 172.21.15.133:0/321173391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:32.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:32 smithi194 bash[31648]: cluster 2023-10-09T04:50:30.956113+0000 mgr.x (mgr.14967) 64 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-10-09T04:50:32.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:32 smithi194 bash[31648]: audit 2023-10-09T04:50:32.182978+0000 mon.c (mon.2) 47 : audit [DBG] from='client.? 172.21.15.133:0/321173391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:33.512 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:33 smithi133 bash[91017]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-mgr-y 2023-10-09T04:50:33.513 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:33 smithi133 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mgr.y.service: Main process exited, code=exited, status=143/n/a 2023-10-09T04:50:33.513 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:33 smithi133 bash[91076]: Error response from daemon: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-mgr-y 2023-10-09T04:50:33.513 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:33 smithi133 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mgr.y.service: Failed with result 'exit-code'. 2023-10-09T04:50:33.513 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:33 smithi133 systemd[1]: Stopped Ceph mgr.y for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:50:33.782 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:33 smithi133 systemd[1]: Started Ceph mgr.y for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:50:33.782 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:50:33 smithi133 bash[90509]: ts=2023-10-09T04:50:33.509Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi133.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.133:8443: connect: connection refused" 2023-10-09T04:50:34.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:34 smithi194 bash[31648]: cluster 2023-10-09T04:50:32.957128+0000 mgr.x (mgr.14967) 65 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 870 op/s 2023-10-09T04:50:34.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:34 smithi194 bash[31648]: audit 2023-10-09T04:50:33.832486+0000 mon.a (mon.0) 1083 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:34.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:34 smithi194 bash[31648]: audit 2023-10-09T04:50:33.851484+0000 mon.a (mon.0) 1084 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:34.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:34 smithi194 bash[31648]: audit 2023-10-09T04:50:33.859785+0000 mon.a (mon.0) 1085 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:50:34.940 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:34 smithi133 bash[41291]: cluster 2023-10-09T04:50:32.957128+0000 mgr.x (mgr.14967) 65 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 870 op/s 2023-10-09T04:50:34.940 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:34 smithi133 bash[41291]: audit 2023-10-09T04:50:33.832486+0000 mon.a (mon.0) 1083 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:34.940 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:34 smithi133 bash[41291]: audit 2023-10-09T04:50:33.851484+0000 mon.a (mon.0) 1084 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:34.940 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:34 smithi133 bash[41291]: audit 2023-10-09T04:50:33.859785+0000 mon.a (mon.0) 1085 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:50:34.941 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:34 smithi133 bash[27558]: cluster 2023-10-09T04:50:32.957128+0000 mgr.x (mgr.14967) 65 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 870 op/s 2023-10-09T04:50:34.941 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:34 smithi133 bash[27558]: audit 2023-10-09T04:50:33.832486+0000 mon.a (mon.0) 1083 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:34.941 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:34 smithi133 bash[27558]: audit 2023-10-09T04:50:33.851484+0000 mon.a (mon.0) 1084 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:34.941 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:34 smithi133 bash[27558]: audit 2023-10-09T04:50:33.859785+0000 mon.a (mon.0) 1085 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:50:35.606 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:35 smithi133 bash[91128]: debug 2023-10-09T04:50:35.318+0000 7f18755fd200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:50:35.606 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:35 smithi133 bash[41291]: audit 2023-10-09T04:50:34.896663+0000 mon.c (mon.2) 48 : audit [DBG] from='client.? 172.21.15.133:0/494852720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:35.606 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:35 smithi133 bash[27558]: audit 2023-10-09T04:50:34.896663+0000 mon.c (mon.2) 48 : audit [DBG] from='client.? 172.21.15.133:0/494852720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:35.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:35 smithi194 bash[31648]: audit 2023-10-09T04:50:34.896663+0000 mon.c (mon.2) 48 : audit [DBG] from='client.? 172.21.15.133:0/494852720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:36.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:36 smithi194 bash[31648]: cluster 2023-10-09T04:50:34.957617+0000 mgr.x (mgr.14967) 66 : cluster [DBG] pgmap v35: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-10-09T04:50:36.929 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:36 smithi133 bash[27558]: cluster 2023-10-09T04:50:34.957617+0000 mgr.x (mgr.14967) 66 : cluster [DBG] pgmap v35: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-10-09T04:50:36.930 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:36 smithi133 bash[91128]: debug 2023-10-09T04:50:36.633+0000 7f18755fd200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:50:36.930 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:36 smithi133 bash[91128]: debug 2023-10-09T04:50:36.777+0000 7f18755fd200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:50:36.930 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:36 smithi133 bash[91128]: debug 2023-10-09T04:50:36.921+0000 7f18755fd200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:50:36.930 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:36 smithi133 bash[41291]: cluster 2023-10-09T04:50:34.957617+0000 mgr.x (mgr.14967) 66 : cluster [DBG] pgmap v35: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-10-09T04:50:37.386 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:50:36 smithi194 bash[69734]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:50:36] "GET /metrics HTTP/1.1" 200 36168 "" "Prometheus/2.43.0" 2023-10-09T04:50:37.801 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:37 smithi133 bash[91128]: debug 2023-10-09T04:50:37.333+0000 7f18755fd200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:50:38.301 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:37 smithi133 bash[91128]: debug 2023-10-09T04:50:37.849+0000 7f18755fd200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:50:38.301 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:38 smithi133 bash[91128]: debug 2023-10-09T04:50:38.081+0000 7f18755fd200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:50:38.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:38 smithi133 bash[27558]: cluster 2023-10-09T04:50:36.960160+0000 mgr.x (mgr.14967) 67 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-10-09T04:50:38.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:38 smithi133 bash[27558]: audit 2023-10-09T04:50:37.679191+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.133:0/2001518090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:38.802 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:38 smithi133 bash[91128]: debug 2023-10-09T04:50:38.389+0000 7f18755fd200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:50:38.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:38 smithi133 bash[41291]: cluster 2023-10-09T04:50:36.960160+0000 mgr.x (mgr.14967) 67 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-10-09T04:50:38.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:38 smithi133 bash[41291]: audit 2023-10-09T04:50:37.679191+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.133:0/2001518090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:38.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:38 smithi194 bash[31648]: cluster 2023-10-09T04:50:36.960160+0000 mgr.x (mgr.14967) 67 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-10-09T04:50:38.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:38 smithi194 bash[31648]: audit 2023-10-09T04:50:37.679191+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.133:0/2001518090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:38.891 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:38 smithi194 bash[80137]: ts=2023-10-09T04:50:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi194:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi194.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:50:40.208 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:39 smithi133 bash[91128]: debug 2023-10-09T04:50:39.873+0000 7f18755fd200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:50:40.209 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:40 smithi133 bash[91128]: debug 2023-10-09T04:50:40.073+0000 7f18755fd200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:50:40.551 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:40 smithi133 bash[91128]: debug 2023-10-09T04:50:40.289+0000 7f18755fd200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:50:40.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:40 smithi194 bash[31648]: cluster 2023-10-09T04:50:38.961241+0000 mgr.x (mgr.14967) 68 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 852 op/s 2023-10-09T04:50:40.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:40 smithi194 bash[31648]: audit 2023-10-09T04:50:40.432960+0000 mon.a (mon.0) 1086 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:40.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:40 smithi194 bash[31648]: audit 2023-10-09T04:50:40.433304+0000 mon.a (mon.0) 1087 : audit [DBG] from='client.? 172.21.15.133:0/3211934001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:40.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:40 smithi194 bash[31648]: audit 2023-10-09T04:50:40.445159+0000 mon.a (mon.0) 1088 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:41.010 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:40 smithi133 bash[41291]: cluster 2023-10-09T04:50:38.961241+0000 mgr.x (mgr.14967) 68 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 852 op/s 2023-10-09T04:50:41.010 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:40 smithi133 bash[41291]: audit 2023-10-09T04:50:40.432960+0000 mon.a (mon.0) 1086 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:41.010 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:40 smithi133 bash[41291]: audit 2023-10-09T04:50:40.433304+0000 mon.a (mon.0) 1087 : audit [DBG] from='client.? 172.21.15.133:0/3211934001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:41.010 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:40 smithi133 bash[41291]: audit 2023-10-09T04:50:40.445159+0000 mon.a (mon.0) 1088 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:41.010 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:40 smithi133 bash[27558]: cluster 2023-10-09T04:50:38.961241+0000 mgr.x (mgr.14967) 68 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 852 op/s 2023-10-09T04:50:41.010 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:40 smithi133 bash[27558]: audit 2023-10-09T04:50:40.432960+0000 mon.a (mon.0) 1086 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:41.011 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:40 smithi133 bash[27558]: audit 2023-10-09T04:50:40.433304+0000 mon.a (mon.0) 1087 : audit [DBG] from='client.? 172.21.15.133:0/3211934001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:41.011 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:40 smithi133 bash[27558]: audit 2023-10-09T04:50:40.445159+0000 mon.a (mon.0) 1088 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:41.301 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:41 smithi133 bash[91128]: debug 2023-10-09T04:50:40.997+0000 7f18755fd200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:50:41.640 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:41 smithi133 bash[27558]: audit 2023-10-09T04:50:40.638014+0000 mon.a (mon.0) 1089 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:41.640 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:41 smithi133 bash[27558]: audit 2023-10-09T04:50:40.651178+0000 mon.a (mon.0) 1090 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:41.640 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:41 smithi133 bash[27558]: audit 2023-10-09T04:50:41.442041+0000 mon.a (mon.0) 1091 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:41.641 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:41 smithi133 bash[27558]: audit 2023-10-09T04:50:41.450644+0000 mon.a (mon.0) 1092 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:41.641 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:41 smithi133 bash[91128]: debug 2023-10-09T04:50:41.313+0000 7f18755fd200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:50:41.988 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:41 smithi133 bash[91128]: debug 2023-10-09T04:50:41.977+0000 7f18755fd200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:50:41.988 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:41 smithi133 bash[41291]: audit 2023-10-09T04:50:40.638014+0000 mon.a (mon.0) 1089 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:41.988 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:41 smithi133 bash[41291]: audit 2023-10-09T04:50:40.651178+0000 mon.a (mon.0) 1090 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:41.988 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:41 smithi133 bash[41291]: audit 2023-10-09T04:50:41.442041+0000 mon.a (mon.0) 1091 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:41.989 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:41 smithi133 bash[41291]: audit 2023-10-09T04:50:41.450644+0000 mon.a (mon.0) 1092 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:42.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:41 smithi194 bash[31648]: audit 2023-10-09T04:50:40.638014+0000 mon.a (mon.0) 1089 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:42.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:41 smithi194 bash[31648]: audit 2023-10-09T04:50:40.651178+0000 mon.a (mon.0) 1090 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:42.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:41 smithi194 bash[31648]: audit 2023-10-09T04:50:41.442041+0000 mon.a (mon.0) 1091 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:42.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:41 smithi194 bash[31648]: audit 2023-10-09T04:50:41.450644+0000 mon.a (mon.0) 1092 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:43.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:42 smithi133 bash[27558]: cluster 2023-10-09T04:50:40.961669+0000 mgr.x (mgr.14967) 69 : cluster [DBG] pgmap v38: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-10-09T04:50:43.051 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:42 smithi133 bash[91128]: debug 2023-10-09T04:50:42.677+0000 7f18755fd200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:50:43.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:42 smithi133 bash[41291]: cluster 2023-10-09T04:50:40.961669+0000 mgr.x (mgr.14967) 69 : cluster [DBG] pgmap v38: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-10-09T04:50:43.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:42 smithi194 bash[31648]: cluster 2023-10-09T04:50:40.961669+0000 mgr.x (mgr.14967) 69 : cluster [DBG] pgmap v38: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-10-09T04:50:43.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:43 smithi133 bash[27558]: audit 2023-10-09T04:50:43.113834+0000 mon.c (mon.2) 50 : audit [DBG] from='client.? 172.21.15.133:0/793395849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:43.802 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:43 smithi133 bash[91128]: debug 2023-10-09T04:50:43.457+0000 7f18755fd200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:50:43.802 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:43 smithi133 bash[91128]: debug 2023-10-09T04:50:43.601+0000 7f18755fd200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:50:43.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:43 smithi133 bash[41291]: audit 2023-10-09T04:50:43.113834+0000 mon.c (mon.2) 50 : audit [DBG] from='client.? 172.21.15.133:0/793395849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:43.802 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:50:43 smithi133 bash[90509]: ts=2023-10-09T04:50:43.507Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi133.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.133:8443: connect: connection refused" 2023-10-09T04:50:43.802 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:50:43 smithi133 bash[90509]: ts=2023-10-09T04:50:43.508Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi133.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.133:8443: connect: connection refused" 2023-10-09T04:50:44.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:43 smithi194 bash[31648]: audit 2023-10-09T04:50:43.113834+0000 mon.c (mon.2) 50 : audit [DBG] from='client.? 172.21.15.133:0/793395849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:44.480 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:44 smithi133 bash[91128]: debug 2023-10-09T04:50:44.077+0000 7f18755fd200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:50:44.480 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:44 smithi133 bash[91128]: debug 2023-10-09T04:50:44.241+0000 7f18755fd200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:50:44.481 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:44 smithi133 bash[91128]: debug 2023-10-09T04:50:44.469+0000 7f18755fd200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:50:44.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:44 smithi133 bash[27558]: cluster 2023-10-09T04:50:42.962503+0000 mgr.x (mgr.14967) 70 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 2.1 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 828 op/s 2023-10-09T04:50:44.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:44 smithi133 bash[41291]: cluster 2023-10-09T04:50:42.962503+0000 mgr.x (mgr.14967) 70 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 2.1 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 828 op/s 2023-10-09T04:50:44.802 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:44 smithi133 bash[91128]: debug 2023-10-09T04:50:44.637+0000 7f18755fd200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:50:45.117 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:44 smithi133 bash[91128]: debug 2023-10-09T04:50:44.821+0000 7f18755fd200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:50:45.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:44 smithi194 bash[31648]: cluster 2023-10-09T04:50:42.962503+0000 mgr.x (mgr.14967) 70 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 2.1 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 828 op/s 2023-10-09T04:50:45.551 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:45 smithi133 bash[91128]: debug 2023-10-09T04:50:45.109+0000 7f18755fd200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:50:45.551 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:45 smithi133 bash[91128]: debug 2023-10-09T04:50:45.285+0000 7f18755fd200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:50:46.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:46 smithi133 bash[27558]: cluster 2023-10-09T04:50:44.962950+0000 mgr.x (mgr.14967) 71 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 2.1 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 541 op/s 2023-10-09T04:50:46.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:46 smithi133 bash[27558]: audit 2023-10-09T04:50:45.860837+0000 mon.a (mon.0) 1093 : audit [DBG] from='client.? 172.21.15.133:0/1391149834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:46.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:46 smithi133 bash[27558]: audit 2023-10-09T04:50:46.113695+0000 mon.a (mon.0) 1094 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:46.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:46 smithi133 bash[41291]: cluster 2023-10-09T04:50:44.962950+0000 mgr.x (mgr.14967) 71 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 2.1 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 541 op/s 2023-10-09T04:50:46.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:46 smithi133 bash[41291]: audit 2023-10-09T04:50:45.860837+0000 mon.a (mon.0) 1093 : audit [DBG] from='client.? 172.21.15.133:0/1391149834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:46.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:46 smithi133 bash[41291]: audit 2023-10-09T04:50:46.113695+0000 mon.a (mon.0) 1094 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:46.989 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:46 smithi194 bash[31648]: cluster 2023-10-09T04:50:44.962950+0000 mgr.x (mgr.14967) 71 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 2.1 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 541 op/s 2023-10-09T04:50:46.989 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:46 smithi194 bash[31648]: audit 2023-10-09T04:50:45.860837+0000 mon.a (mon.0) 1093 : audit [DBG] from='client.? 172.21.15.133:0/1391149834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:46.989 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:46 smithi194 bash[31648]: audit 2023-10-09T04:50:46.113695+0000 mon.a (mon.0) 1094 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:46.990 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:50:46 smithi194 bash[69734]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:50:46] "GET /metrics HTTP/1.1" 200 36158 "" "Prometheus/2.43.0" 2023-10-09T04:50:47.551 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:47 smithi133 bash[91128]: debug 2023-10-09T04:50:47.061+0000 7f18755fd200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:50:47.551 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:47 smithi133 bash[91128]: [09/Oct/2023:04:50:47] ENGINE Bus STARTING 2023-10-09T04:50:47.551 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:47 smithi133 bash[91128]: CherryPy Checker: 2023-10-09T04:50:47.551 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:47 smithi133 bash[91128]: The Application mounted at '' has an empty config. 2023-10-09T04:50:47.551 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:47 smithi133 bash[91128]: [09/Oct/2023:04:50:47] ENGINE Serving on http://:::9283 2023-10-09T04:50:47.551 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:47 smithi133 bash[91128]: [09/Oct/2023:04:50:47] ENGINE Bus STARTED 2023-10-09T04:50:48.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:47 smithi133 bash[27558]: cluster 2023-10-09T04:50:47.071064+0000 mon.a (mon.0) 1095 : cluster [DBG] Standby manager daemon y restarted 2023-10-09T04:50:48.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:47 smithi133 bash[27558]: cluster 2023-10-09T04:50:47.071211+0000 mon.a (mon.0) 1096 : cluster [DBG] Standby manager daemon y started 2023-10-09T04:50:48.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:47 smithi133 bash[27558]: audit 2023-10-09T04:50:47.075745+0000 mon.c (mon.2) 51 : audit [DBG] from='mgr.? 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T04:50:48.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:47 smithi133 bash[27558]: audit 2023-10-09T04:50:47.079503+0000 mon.c (mon.2) 52 : audit [DBG] from='mgr.? 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:50:48.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:47 smithi133 bash[27558]: audit 2023-10-09T04:50:47.087447+0000 mon.c (mon.2) 53 : audit [DBG] from='mgr.? 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T04:50:48.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:47 smithi133 bash[27558]: audit 2023-10-09T04:50:47.088372+0000 mon.c (mon.2) 54 : audit [DBG] from='mgr.? 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:50:48.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:47 smithi133 bash[41291]: cluster 2023-10-09T04:50:47.071064+0000 mon.a (mon.0) 1095 : cluster [DBG] Standby manager daemon y restarted 2023-10-09T04:50:48.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:47 smithi133 bash[41291]: cluster 2023-10-09T04:50:47.071211+0000 mon.a (mon.0) 1096 : cluster [DBG] Standby manager daemon y started 2023-10-09T04:50:48.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:47 smithi133 bash[41291]: audit 2023-10-09T04:50:47.075745+0000 mon.c (mon.2) 51 : audit [DBG] from='mgr.? 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T04:50:48.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:47 smithi133 bash[41291]: audit 2023-10-09T04:50:47.079503+0000 mon.c (mon.2) 52 : audit [DBG] from='mgr.? 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:50:48.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:47 smithi133 bash[41291]: audit 2023-10-09T04:50:47.087447+0000 mon.c (mon.2) 53 : audit [DBG] from='mgr.? 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T04:50:48.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:47 smithi133 bash[41291]: audit 2023-10-09T04:50:47.088372+0000 mon.c (mon.2) 54 : audit [DBG] from='mgr.? 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:50:48.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:47 smithi194 bash[31648]: cluster 2023-10-09T04:50:47.071064+0000 mon.a (mon.0) 1095 : cluster [DBG] Standby manager daemon y restarted 2023-10-09T04:50:48.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:47 smithi194 bash[31648]: cluster 2023-10-09T04:50:47.071211+0000 mon.a (mon.0) 1096 : cluster [DBG] Standby manager daemon y started 2023-10-09T04:50:48.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:47 smithi194 bash[31648]: audit 2023-10-09T04:50:47.075745+0000 mon.c (mon.2) 51 : audit [DBG] from='mgr.? 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T04:50:48.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:47 smithi194 bash[31648]: audit 2023-10-09T04:50:47.079503+0000 mon.c (mon.2) 52 : audit [DBG] from='mgr.? 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:50:48.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:47 smithi194 bash[31648]: audit 2023-10-09T04:50:47.087447+0000 mon.c (mon.2) 53 : audit [DBG] from='mgr.? 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T04:50:48.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:47 smithi194 bash[31648]: audit 2023-10-09T04:50:47.088372+0000 mon.c (mon.2) 54 : audit [DBG] from='mgr.? 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:50:48.939 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:48 smithi133 bash[41291]: cluster 2023-10-09T04:50:46.963382+0000 mgr.x (mgr.14967) 72 : cluster [DBG] pgmap v41: 89 pgs: 89 active+clean; 2.1 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 541 op/s 2023-10-09T04:50:48.939 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:48 smithi133 bash[41291]: cluster 2023-10-09T04:50:47.688224+0000 mon.a (mon.0) 1097 : cluster [DBG] mgrmap e25: x(active, since 76s), standbys: y 2023-10-09T04:50:48.939 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:48 smithi133 bash[41291]: audit 2023-10-09T04:50:48.515379+0000 mon.a (mon.0) 1098 : audit [DBG] from='client.? 172.21.15.133:0/802146592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:48.939 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:48 smithi133 bash[27558]: cluster 2023-10-09T04:50:46.963382+0000 mgr.x (mgr.14967) 72 : cluster [DBG] pgmap v41: 89 pgs: 89 active+clean; 2.1 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 541 op/s 2023-10-09T04:50:48.939 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:48 smithi133 bash[27558]: cluster 2023-10-09T04:50:47.688224+0000 mon.a (mon.0) 1097 : cluster [DBG] mgrmap e25: x(active, since 76s), standbys: y 2023-10-09T04:50:48.939 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:48 smithi133 bash[27558]: audit 2023-10-09T04:50:48.515379+0000 mon.a (mon.0) 1098 : audit [DBG] from='client.? 172.21.15.133:0/802146592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:49.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:48 smithi194 bash[31648]: cluster 2023-10-09T04:50:46.963382+0000 mgr.x (mgr.14967) 72 : cluster [DBG] pgmap v41: 89 pgs: 89 active+clean; 2.1 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 541 op/s 2023-10-09T04:50:49.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:48 smithi194 bash[31648]: cluster 2023-10-09T04:50:47.688224+0000 mon.a (mon.0) 1097 : cluster [DBG] mgrmap e25: x(active, since 76s), standbys: y 2023-10-09T04:50:49.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:48 smithi194 bash[31648]: audit 2023-10-09T04:50:48.515379+0000 mon.a (mon.0) 1098 : audit [DBG] from='client.? 172.21.15.133:0/802146592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:49.137 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:50:48 smithi194 bash[80137]: ts=2023-10-09T04:50:48.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi194:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi194.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:50:50.301 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 04:50:49 smithi133 bash[90509]: ts=2023-10-09T04:50:49.938Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=6 2023-10-09T04:50:51.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:50 smithi133 bash[41291]: cluster 2023-10-09T04:50:48.964084+0000 mgr.x (mgr.14967) 73 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2023-10-09T04:50:51.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:50 smithi133 bash[27558]: cluster 2023-10-09T04:50:48.964084+0000 mgr.x (mgr.14967) 73 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2023-10-09T04:50:51.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:50 smithi194 bash[31648]: cluster 2023-10-09T04:50:48.964084+0000 mgr.x (mgr.14967) 73 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2023-10-09T04:50:52.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:51 smithi133 bash[41291]: audit 2023-10-09T04:50:51.273760+0000 mon.a (mon.0) 1099 : audit [DBG] from='client.? 172.21.15.133:0/1473843972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:52.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:51 smithi133 bash[27558]: audit 2023-10-09T04:50:51.273760+0000 mon.a (mon.0) 1099 : audit [DBG] from='client.? 172.21.15.133:0/1473843972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:52.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:51 smithi194 bash[31648]: audit 2023-10-09T04:50:51.273760+0000 mon.a (mon.0) 1099 : audit [DBG] from='client.? 172.21.15.133:0/1473843972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:53.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:52 smithi133 bash[41291]: cluster 2023-10-09T04:50:50.964558+0000 mgr.x (mgr.14967) 74 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 541 op/s 2023-10-09T04:50:53.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:52 smithi133 bash[27558]: cluster 2023-10-09T04:50:50.964558+0000 mgr.x (mgr.14967) 74 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 541 op/s 2023-10-09T04:50:53.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:52 smithi194 bash[31648]: cluster 2023-10-09T04:50:50.964558+0000 mgr.x (mgr.14967) 74 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 541 op/s 2023-10-09T04:50:55.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:54 smithi133 bash[41291]: cluster 2023-10-09T04:50:52.965399+0000 mgr.x (mgr.14967) 75 : cluster [DBG] pgmap v44: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 836 op/s 2023-10-09T04:50:55.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:54 smithi133 bash[41291]: audit 2023-10-09T04:50:53.977515+0000 mon.a (mon.0) 1100 : audit [DBG] from='client.? 172.21.15.133:0/3355892316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:55.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:54 smithi133 bash[27558]: cluster 2023-10-09T04:50:52.965399+0000 mgr.x (mgr.14967) 75 : cluster [DBG] pgmap v44: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 836 op/s 2023-10-09T04:50:55.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:54 smithi133 bash[27558]: audit 2023-10-09T04:50:53.977515+0000 mon.a (mon.0) 1100 : audit [DBG] from='client.? 172.21.15.133:0/3355892316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:55.069 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[69734]: debug 2023-10-09T04:50:55.065+0000 7f35e1c38700 -1 mgr handle_mgr_map I was active but no longer am 2023-10-09T04:50:55.069 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:54 smithi194 bash[31648]: cluster 2023-10-09T04:50:52.965399+0000 mgr.x (mgr.14967) 75 : cluster [DBG] pgmap v44: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 836 op/s 2023-10-09T04:50:55.069 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:54 smithi194 bash[31648]: audit 2023-10-09T04:50:53.977515+0000 mon.a (mon.0) 1100 : audit [DBG] from='client.? 172.21.15.133:0/3355892316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:55.377 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[69734]: ignoring --setuser ceph since I am not root 2023-10-09T04:50:55.377 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[69734]: ignoring --setgroup ceph since I am not root 2023-10-09T04:50:55.377 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[69734]: debug 2023-10-09T04:50:55.145+0000 7f468ba77700 1 -- 172.21.15.194:0/112364084 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x55855966f1e0 con 0x55855a3dac00 2023-10-09T04:50:55.377 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[69734]: debug 2023-10-09T04:50:55.373+0000 7f46957a7200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:50:55.424 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[91128]: [09/Oct/2023:04:50:55] ENGINE Bus STOPPING 2023-10-09T04:50:55.424 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[91128]: [09/Oct/2023:04:50:55] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T04:50:55.424 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[91128]: [09/Oct/2023:04:50:55] ENGINE Bus STOPPED 2023-10-09T04:50:55.424 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[91128]: [09/Oct/2023:04:50:55] ENGINE Bus STARTING 2023-10-09T04:50:55.801 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[91128]: debug 2023-10-09T04:50:55.465+0000 7f17d72a3700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:50:55.802 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[91128]: debug 2023-10-09T04:50:55.465+0000 7f17d72a3700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:50:55.802 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[91128]: debug 2023-10-09T04:50:55.465+0000 7f17d72a3700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:50:55.802 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[91128]: debug 2023-10-09T04:50:55.465+0000 7f17d72a3700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:50:55.802 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[91128]: debug 2023-10-09T04:50:55.465+0000 7f17d72a3700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:50:55.802 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[91128]: [09/Oct/2023:04:50:55] ENGINE Serving on http://:::9283 2023-10-09T04:50:55.802 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[91128]: [09/Oct/2023:04:50:55] ENGINE Bus STARTED 2023-10-09T04:50:55.802 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[91128]: debug 2023-10-09T04:50:55.485+0000 7f17d429d700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:50:55.802 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[91128]: debug 2023-10-09T04:50:55.485+0000 7f17d429d700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:50:55.802 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[91128]: debug 2023-10-09T04:50:55.485+0000 7f17d429d700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:50:55.802 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[91128]: debug 2023-10-09T04:50:55.485+0000 7f17d429d700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:50:55.802 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[91128]: debug 2023-10-09T04:50:55.485+0000 7f17d429d700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:50:56.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:54.936828+0000 mon.a (mon.0) 1101 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:56.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:54.948843+0000 mon.a (mon.0) 1102 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:56.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:54.953437+0000 mon.a (mon.0) 1103 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:50:56.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:54.954558+0000 mon.a (mon.0) 1104 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:50:56.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:54.960473+0000 mon.a (mon.0) 1105 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:56.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:55.039946+0000 mon.a (mon.0) 1106 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:50:56.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:55.051806+0000 mon.a (mon.0) 1107 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T04:50:56.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: cluster 2023-10-09T04:50:55.062169+0000 mon.a (mon.0) 1108 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-10-09T04:50:56.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:55.062838+0000 mon.a (mon.0) 1109 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T04:50:56.304 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: cluster 2023-10-09T04:50:55.062916+0000 mon.a (mon.0) 1110 : cluster [DBG] mgrmap e26: y(active, starting, since 0.009942s) 2023-10-09T04:50:56.304 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:55.065480+0000 mon.c (mon.2) 55 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:50:56.304 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:55.065856+0000 mon.c (mon.2) 56 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:50:56.304 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:55.066159+0000 mon.c (mon.2) 57 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:50:56.304 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:55.066924+0000 mon.c (mon.2) 58 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi133.tsuabh"}]: dispatch 2023-10-09T04:50:56.304 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:55.067481+0000 mon.c (mon.2) 59 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi194.yxnfqn"}]: dispatch 2023-10-09T04:50:56.304 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:55.068377+0000 mon.c (mon.2) 60 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:50:56.304 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:55.069422+0000 mon.c (mon.2) 61 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:50:56.304 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:55.069994+0000 mon.c (mon.2) 62 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:50:56.304 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:55.070591+0000 mon.c (mon.2) 63 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:50:56.305 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:55.071065+0000 mon.c (mon.2) 64 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:50:56.305 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:55.071463+0000 mon.c (mon.2) 65 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:50:56.305 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:55.071876+0000 mon.c (mon.2) 66 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:50:56.305 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:55.072169+0000 mon.c (mon.2) 67 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:50:56.305 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:55.072617+0000 mon.c (mon.2) 68 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:50:56.305 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:55.073008+0000 mon.c (mon.2) 69 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:50:56.305 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:55.073396+0000 mon.c (mon.2) 70 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:50:56.305 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:55.074006+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:50:56.305 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: cluster 2023-10-09T04:50:55.131752+0000 mon.a (mon.0) 1111 : cluster [INF] Manager daemon y is now available 2023-10-09T04:50:56.305 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:55.190621+0000 mon.c (mon.2) 72 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:50:56.306 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:55.200376+0000 mon.c (mon.2) 73 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:50:56.306 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:55.242278+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:56.306 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:55.261063+0000 mon.c (mon.2) 75 : audit [INF] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:50:56.306 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:55.261533+0000 mon.a (mon.0) 1112 : audit [INF] from='mgr.25046 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:50:56.306 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:55.369975+0000 mon.c (mon.2) 76 : audit [INF] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:50:56.306 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[27558]: audit 2023-10-09T04:50:55.370607+0000 mon.a (mon.0) 1113 : audit [INF] from='mgr.25046 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:50:56.306 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:54.936828+0000 mon.a (mon.0) 1101 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:56.306 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:54.948843+0000 mon.a (mon.0) 1102 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:56.307 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:54.953437+0000 mon.a (mon.0) 1103 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:50:56.307 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:54.954558+0000 mon.a (mon.0) 1104 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:50:56.307 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:54.960473+0000 mon.a (mon.0) 1105 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:56.307 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:55.039946+0000 mon.a (mon.0) 1106 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:50:56.307 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:55.051806+0000 mon.a (mon.0) 1107 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T04:50:56.307 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: cluster 2023-10-09T04:50:55.062169+0000 mon.a (mon.0) 1108 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-10-09T04:50:56.307 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:55.062838+0000 mon.a (mon.0) 1109 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T04:50:56.307 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: cluster 2023-10-09T04:50:55.062916+0000 mon.a (mon.0) 1110 : cluster [DBG] mgrmap e26: y(active, starting, since 0.009942s) 2023-10-09T04:50:56.307 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:55.065480+0000 mon.c (mon.2) 55 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:50:56.308 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:55.065856+0000 mon.c (mon.2) 56 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:50:56.308 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:55.066159+0000 mon.c (mon.2) 57 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:50:56.308 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:55.066924+0000 mon.c (mon.2) 58 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi133.tsuabh"}]: dispatch 2023-10-09T04:50:56.308 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:55.067481+0000 mon.c (mon.2) 59 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi194.yxnfqn"}]: dispatch 2023-10-09T04:50:56.309 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:55.068377+0000 mon.c (mon.2) 60 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:50:56.309 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:55.069422+0000 mon.c (mon.2) 61 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:50:56.309 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:55.069994+0000 mon.c (mon.2) 62 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:50:56.309 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:55.070591+0000 mon.c (mon.2) 63 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:50:56.309 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:55.071065+0000 mon.c (mon.2) 64 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:50:56.309 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:55.071463+0000 mon.c (mon.2) 65 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:50:56.310 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:55.071876+0000 mon.c (mon.2) 66 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:50:56.310 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:55.072169+0000 mon.c (mon.2) 67 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:50:56.310 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:55.072617+0000 mon.c (mon.2) 68 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:50:56.310 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:55.073008+0000 mon.c (mon.2) 69 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:50:56.310 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:55.073396+0000 mon.c (mon.2) 70 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:50:56.310 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:55.074006+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:50:56.310 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: cluster 2023-10-09T04:50:55.131752+0000 mon.a (mon.0) 1111 : cluster [INF] Manager daemon y is now available 2023-10-09T04:50:56.310 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:55.190621+0000 mon.c (mon.2) 72 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:50:56.310 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:55.200376+0000 mon.c (mon.2) 73 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:50:56.310 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:55.242278+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:56.311 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:55.261063+0000 mon.c (mon.2) 75 : audit [INF] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:50:56.311 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:55.261533+0000 mon.a (mon.0) 1112 : audit [INF] from='mgr.25046 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:50:56.311 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:55.369975+0000 mon.c (mon.2) 76 : audit [INF] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:50:56.311 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:55 smithi133 bash[41291]: audit 2023-10-09T04:50:55.370607+0000 mon.a (mon.0) 1113 : audit [INF] from='mgr.25046 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:50:56.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:54.936828+0000 mon.a (mon.0) 1101 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:56.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:54.948843+0000 mon.a (mon.0) 1102 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:56.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:54.953437+0000 mon.a (mon.0) 1103 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:50:56.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:54.954558+0000 mon.a (mon.0) 1104 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:50:56.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:54.960473+0000 mon.a (mon.0) 1105 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' 2023-10-09T04:50:56.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:55.039946+0000 mon.a (mon.0) 1106 : audit [DBG] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:50:56.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:55.051806+0000 mon.a (mon.0) 1107 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T04:50:56.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: cluster 2023-10-09T04:50:55.062169+0000 mon.a (mon.0) 1108 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-10-09T04:50:56.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:55.062838+0000 mon.a (mon.0) 1109 : audit [INF] from='mgr.14967 172.21.15.194:0/1914671929' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T04:50:56.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: cluster 2023-10-09T04:50:55.062916+0000 mon.a (mon.0) 1110 : cluster [DBG] mgrmap e26: y(active, starting, since 0.009942s) 2023-10-09T04:50:56.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:55.065480+0000 mon.c (mon.2) 55 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:50:56.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:55.065856+0000 mon.c (mon.2) 56 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:50:56.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:55.066159+0000 mon.c (mon.2) 57 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:50:56.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:55.066924+0000 mon.c (mon.2) 58 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi133.tsuabh"}]: dispatch 2023-10-09T04:50:56.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:55.067481+0000 mon.c (mon.2) 59 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi194.yxnfqn"}]: dispatch 2023-10-09T04:50:56.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:55.068377+0000 mon.c (mon.2) 60 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:50:56.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:55.069422+0000 mon.c (mon.2) 61 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:50:56.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:55.069994+0000 mon.c (mon.2) 62 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:50:56.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:55.070591+0000 mon.c (mon.2) 63 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:50:56.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:55.071065+0000 mon.c (mon.2) 64 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:50:56.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:55.071463+0000 mon.c (mon.2) 65 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:50:56.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:55.071876+0000 mon.c (mon.2) 66 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:50:56.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:55.072169+0000 mon.c (mon.2) 67 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:50:56.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:55.072617+0000 mon.c (mon.2) 68 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:50:56.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:55.073008+0000 mon.c (mon.2) 69 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:50:56.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:55.073396+0000 mon.c (mon.2) 70 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:50:56.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:55.074006+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:50:56.390 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: cluster 2023-10-09T04:50:55.131752+0000 mon.a (mon.0) 1111 : cluster [INF] Manager daemon y is now available 2023-10-09T04:50:56.390 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:55.190621+0000 mon.c (mon.2) 72 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:50:56.390 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:55.200376+0000 mon.c (mon.2) 73 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:50:56.390 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:55.242278+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:56.390 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:55.261063+0000 mon.c (mon.2) 75 : audit [INF] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:50:56.390 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:55.261533+0000 mon.a (mon.0) 1112 : audit [INF] from='mgr.25046 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:50:56.390 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:55.369975+0000 mon.c (mon.2) 76 : audit [INF] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:50:56.390 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:55 smithi194 bash[31648]: audit 2023-10-09T04:50:55.370607+0000 mon.a (mon.0) 1113 : audit [INF] from='mgr.25046 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:50:57.056 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:50:56 smithi194 bash[69734]: debug 2023-10-09T04:50:56.745+0000 7f46957a7200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:50:57.056 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:50:56 smithi194 bash[69734]: debug 2023-10-09T04:50:56.893+0000 7f46957a7200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:50:57.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:50:57 smithi194 bash[69734]: debug 2023-10-09T04:50:57.053+0000 7f46957a7200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:50:57.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:57 smithi194 bash[31648]: cephadm 2023-10-09T04:50:56.030260+0000 mgr.y (mgr.25046) 1 : cephadm [INF] [09/Oct/2023:04:50:56] ENGINE Bus STARTING 2023-10-09T04:50:57.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:57 smithi194 bash[31648]: cluster 2023-10-09T04:50:56.068208+0000 mon.a (mon.0) 1114 : cluster [DBG] mgrmap e27: y(active, since 1.01523s) 2023-10-09T04:50:57.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:57 smithi194 bash[31648]: audit 2023-10-09T04:50:56.695535+0000 mon.b (mon.1) 39 : audit [DBG] from='client.? 172.21.15.133:0/2689049110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:57.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:57 smithi133 bash[27558]: cephadm 2023-10-09T04:50:56.030260+0000 mgr.y (mgr.25046) 1 : cephadm [INF] [09/Oct/2023:04:50:56] ENGINE Bus STARTING 2023-10-09T04:50:57.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:57 smithi133 bash[27558]: cluster 2023-10-09T04:50:56.068208+0000 mon.a (mon.0) 1114 : cluster [DBG] mgrmap e27: y(active, since 1.01523s) 2023-10-09T04:50:57.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:57 smithi133 bash[27558]: audit 2023-10-09T04:50:56.695535+0000 mon.b (mon.1) 39 : audit [DBG] from='client.? 172.21.15.133:0/2689049110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:57.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:57 smithi133 bash[41291]: cephadm 2023-10-09T04:50:56.030260+0000 mgr.y (mgr.25046) 1 : cephadm [INF] [09/Oct/2023:04:50:56] ENGINE Bus STARTING 2023-10-09T04:50:57.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:57 smithi133 bash[41291]: cluster 2023-10-09T04:50:56.068208+0000 mon.a (mon.0) 1114 : cluster [DBG] mgrmap e27: y(active, since 1.01523s) 2023-10-09T04:50:57.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:57 smithi133 bash[41291]: audit 2023-10-09T04:50:56.695535+0000 mon.b (mon.1) 39 : audit [DBG] from='client.? 172.21.15.133:0/2689049110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:50:57.887 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:50:57 smithi194 bash[69734]: debug 2023-10-09T04:50:57.437+0000 7f46957a7200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:50:58.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:50:57 smithi194 bash[69734]: debug 2023-10-09T04:50:57.957+0000 7f46957a7200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:50:58.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:50:58 smithi194 bash[69734]: debug 2023-10-09T04:50:58.193+0000 7f46957a7200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:50:58.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:58 smithi194 bash[31648]: cephadm 2023-10-09T04:50:56.143156+0000 mgr.y (mgr.25046) 2 : cephadm [INF] [09/Oct/2023:04:50:56] ENGINE Serving on http://172.21.15.133:8765 2023-10-09T04:50:58.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:58 smithi194 bash[31648]: cluster 2023-10-09T04:50:56.160857+0000 mgr.y (mgr.25046) 3 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:50:58.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:58 smithi194 bash[31648]: cephadm 2023-10-09T04:50:56.277884+0000 mgr.y (mgr.25046) 4 : cephadm [INF] [09/Oct/2023:04:50:56] ENGINE Serving on https://172.21.15.133:7150 2023-10-09T04:50:58.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:58 smithi194 bash[31648]: cephadm 2023-10-09T04:50:56.278064+0000 mgr.y (mgr.25046) 5 : cephadm [INF] [09/Oct/2023:04:50:56] ENGINE Bus STARTED 2023-10-09T04:50:58.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:58 smithi194 bash[31648]: cluster 2023-10-09T04:50:57.069585+0000 mgr.y (mgr.25046) 6 : cluster [DBG] pgmap v4: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:50:58.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:58 smithi133 bash[41291]: cephadm 2023-10-09T04:50:56.143156+0000 mgr.y (mgr.25046) 2 : cephadm [INF] [09/Oct/2023:04:50:56] ENGINE Serving on http://172.21.15.133:8765 2023-10-09T04:50:58.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:58 smithi133 bash[41291]: cluster 2023-10-09T04:50:56.160857+0000 mgr.y (mgr.25046) 3 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:50:58.554 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:58 smithi133 bash[41291]: cephadm 2023-10-09T04:50:56.277884+0000 mgr.y (mgr.25046) 4 : cephadm [INF] [09/Oct/2023:04:50:56] ENGINE Serving on https://172.21.15.133:7150 2023-10-09T04:50:58.554 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:58 smithi133 bash[41291]: cephadm 2023-10-09T04:50:56.278064+0000 mgr.y (mgr.25046) 5 : cephadm [INF] [09/Oct/2023:04:50:56] ENGINE Bus STARTED 2023-10-09T04:50:58.554 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:58 smithi133 bash[41291]: cluster 2023-10-09T04:50:57.069585+0000 mgr.y (mgr.25046) 6 : cluster [DBG] pgmap v4: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:50:58.554 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:58 smithi133 bash[27558]: cephadm 2023-10-09T04:50:56.143156+0000 mgr.y (mgr.25046) 2 : cephadm [INF] [09/Oct/2023:04:50:56] ENGINE Serving on http://172.21.15.133:8765 2023-10-09T04:50:58.554 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:58 smithi133 bash[27558]: cluster 2023-10-09T04:50:56.160857+0000 mgr.y (mgr.25046) 3 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:50:58.554 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:58 smithi133 bash[27558]: cephadm 2023-10-09T04:50:56.277884+0000 mgr.y (mgr.25046) 4 : cephadm [INF] [09/Oct/2023:04:50:56] ENGINE Serving on https://172.21.15.133:7150 2023-10-09T04:50:58.554 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:58 smithi133 bash[27558]: cephadm 2023-10-09T04:50:56.278064+0000 mgr.y (mgr.25046) 5 : cephadm [INF] [09/Oct/2023:04:50:56] ENGINE Bus STARTED 2023-10-09T04:50:58.554 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:58 smithi133 bash[27558]: cluster 2023-10-09T04:50:57.069585+0000 mgr.y (mgr.25046) 6 : cluster [DBG] pgmap v4: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:50:58.887 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:50:58 smithi194 bash[69734]: debug 2023-10-09T04:50:58.517+0000 7f46957a7200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:50:59.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:50:59 smithi194 bash[31648]: cluster 2023-10-09T04:50:58.092867+0000 mon.a (mon.0) 1115 : cluster [DBG] mgrmap e28: y(active, since 3s) 2023-10-09T04:50:59.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:50:59 smithi133 bash[41291]: cluster 2023-10-09T04:50:58.092867+0000 mon.a (mon.0) 1115 : cluster [DBG] mgrmap e28: y(active, since 3s) 2023-10-09T04:50:59.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:50:59 smithi133 bash[27558]: cluster 2023-10-09T04:50:58.092867+0000 mon.a (mon.0) 1115 : cluster [DBG] mgrmap e28: y(active, since 3s) 2023-10-09T04:51:00.357 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:00 smithi194 bash[69734]: debug 2023-10-09T04:50:59.997+0000 7f46957a7200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:51:00.357 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:00 smithi194 bash[69734]: debug 2023-10-09T04:51:00.169+0000 7f46957a7200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:51:00.357 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:00 smithi194 bash[69734]: debug 2023-10-09T04:51:00.353+0000 7f46957a7200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:51:00.357 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:00 smithi194 bash[31648]: cluster 2023-10-09T04:50:59.070944+0000 mgr.y (mgr.25046) 7 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:51:00.357 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:00 smithi194 bash[31648]: audit 2023-10-09T04:50:59.362882+0000 mon.a (mon.0) 1116 : audit [DBG] from='client.? 172.21.15.133:0/2654554397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:00.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:00 smithi133 bash[27558]: cluster 2023-10-09T04:50:59.070944+0000 mgr.y (mgr.25046) 7 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:51:00.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:00 smithi133 bash[27558]: audit 2023-10-09T04:50:59.362882+0000 mon.a (mon.0) 1116 : audit [DBG] from='client.? 172.21.15.133:0/2654554397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:00.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:00 smithi133 bash[41291]: cluster 2023-10-09T04:50:59.070944+0000 mgr.y (mgr.25046) 7 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:51:00.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:00 smithi133 bash[41291]: audit 2023-10-09T04:50:59.362882+0000 mon.a (mon.0) 1116 : audit [DBG] from='client.? 172.21.15.133:0/2654554397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:01.239 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:01 smithi194 bash[69734]: debug 2023-10-09T04:51:01.077+0000 7f46957a7200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:51:01.605 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:01 smithi194 bash[69734]: debug 2023-10-09T04:51:01.401+0000 7f46957a7200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:51:01.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:01 smithi194 bash[31648]: cluster 2023-10-09T04:51:01.071382+0000 mgr.y (mgr.25046) 8 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:51:02.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:01 smithi133 bash[27558]: cluster 2023-10-09T04:51:01.071382+0000 mgr.y (mgr.25046) 8 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:51:02.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:01 smithi133 bash[41291]: cluster 2023-10-09T04:51:01.071382+0000 mgr.y (mgr.25046) 8 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:51:02.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:02 smithi194 bash[69734]: debug 2023-10-09T04:51:02.101+0000 7f46957a7200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:51:02.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:02 smithi194 bash[31648]: audit 2023-10-09T04:51:02.050897+0000 mon.a (mon.0) 1117 : audit [DBG] from='client.? 172.21.15.133:0/2853562433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:02.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:02 smithi194 bash[31648]: audit 2023-10-09T04:51:02.055160+0000 mon.a (mon.0) 1118 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:02.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:02 smithi194 bash[31648]: audit 2023-10-09T04:51:02.068539+0000 mon.a (mon.0) 1119 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:02.887 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:02 smithi194 bash[69734]: debug 2023-10-09T04:51:02.841+0000 7f46957a7200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:51:03.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:02 smithi133 bash[41291]: audit 2023-10-09T04:51:02.050897+0000 mon.a (mon.0) 1117 : audit [DBG] from='client.? 172.21.15.133:0/2853562433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:03.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:02 smithi133 bash[41291]: audit 2023-10-09T04:51:02.055160+0000 mon.a (mon.0) 1118 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:03.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:02 smithi133 bash[41291]: audit 2023-10-09T04:51:02.068539+0000 mon.a (mon.0) 1119 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:03.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:02 smithi133 bash[27558]: audit 2023-10-09T04:51:02.050897+0000 mon.a (mon.0) 1117 : audit [DBG] from='client.? 172.21.15.133:0/2853562433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:03.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:02 smithi133 bash[27558]: audit 2023-10-09T04:51:02.055160+0000 mon.a (mon.0) 1118 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:03.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:02 smithi133 bash[27558]: audit 2023-10-09T04:51:02.068539+0000 mon.a (mon.0) 1119 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:04.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:03 smithi133 bash[41291]: audit 2023-10-09T04:51:02.782293+0000 mon.a (mon.0) 1120 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:04.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:03 smithi133 bash[41291]: audit 2023-10-09T04:51:02.801002+0000 mon.a (mon.0) 1121 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:04.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:03 smithi133 bash[41291]: audit 2023-10-09T04:51:03.023046+0000 mon.a (mon.0) 1122 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:04.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:03 smithi133 bash[41291]: audit 2023-10-09T04:51:03.039190+0000 mon.a (mon.0) 1123 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:04.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:03 smithi133 bash[41291]: cluster 2023-10-09T04:51:03.077988+0000 mgr.y (mgr.25046) 9 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 458 op/s 2023-10-09T04:51:04.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:03 smithi133 bash[27558]: audit 2023-10-09T04:51:02.782293+0000 mon.a (mon.0) 1120 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:04.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:03 smithi133 bash[27558]: audit 2023-10-09T04:51:02.801002+0000 mon.a (mon.0) 1121 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:04.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:03 smithi133 bash[27558]: audit 2023-10-09T04:51:03.023046+0000 mon.a (mon.0) 1122 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:04.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:03 smithi133 bash[27558]: audit 2023-10-09T04:51:03.039190+0000 mon.a (mon.0) 1123 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:04.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:03 smithi133 bash[27558]: cluster 2023-10-09T04:51:03.077988+0000 mgr.y (mgr.25046) 9 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 458 op/s 2023-10-09T04:51:04.137 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:03 smithi194 bash[69734]: debug 2023-10-09T04:51:03.661+0000 7f46957a7200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:51:04.137 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:03 smithi194 bash[69734]: debug 2023-10-09T04:51:03.817+0000 7f46957a7200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:51:04.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:03 smithi194 bash[31648]: audit 2023-10-09T04:51:02.782293+0000 mon.a (mon.0) 1120 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:04.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:03 smithi194 bash[31648]: audit 2023-10-09T04:51:02.801002+0000 mon.a (mon.0) 1121 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:04.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:03 smithi194 bash[31648]: audit 2023-10-09T04:51:03.023046+0000 mon.a (mon.0) 1122 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:04.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:03 smithi194 bash[31648]: audit 2023-10-09T04:51:03.039190+0000 mon.a (mon.0) 1123 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:04.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:03 smithi194 bash[31648]: cluster 2023-10-09T04:51:03.077988+0000 mgr.y (mgr.25046) 9 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 458 op/s 2023-10-09T04:51:04.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:04 smithi194 bash[69734]: debug 2023-10-09T04:51:04.313+0000 7f46957a7200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:51:04.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:04 smithi194 bash[69734]: debug 2023-10-09T04:51:04.473+0000 7f46957a7200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:51:05.019 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:04 smithi194 bash[31648]: audit 2023-10-09T04:51:03.818572+0000 mon.a (mon.0) 1124 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:05.020 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:04 smithi194 bash[31648]: audit 2023-10-09T04:51:03.831810+0000 mon.a (mon.0) 1125 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:05.020 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:04 smithi194 bash[31648]: audit 2023-10-09T04:51:04.713844+0000 mon.a (mon.0) 1126 : audit [DBG] from='client.? 172.21.15.133:0/2526529810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:05.020 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:04 smithi194 bash[69734]: debug 2023-10-09T04:51:04.677+0000 7f46957a7200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:51:05.020 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:04 smithi194 bash[69734]: debug 2023-10-09T04:51:04.837+0000 7f46957a7200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:51:05.020 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:05 smithi194 bash[69734]: debug 2023-10-09T04:51:05.013+0000 7f46957a7200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:51:05.301 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:05 smithi194 bash[69734]: debug 2023-10-09T04:51:05.297+0000 7f46957a7200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:51:05.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:04 smithi133 bash[41291]: audit 2023-10-09T04:51:03.818572+0000 mon.a (mon.0) 1124 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:05.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:04 smithi133 bash[41291]: audit 2023-10-09T04:51:03.831810+0000 mon.a (mon.0) 1125 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:05.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:04 smithi133 bash[41291]: audit 2023-10-09T04:51:04.713844+0000 mon.a (mon.0) 1126 : audit [DBG] from='client.? 172.21.15.133:0/2526529810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:05.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:04 smithi133 bash[27558]: audit 2023-10-09T04:51:03.818572+0000 mon.a (mon.0) 1124 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:05.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:04 smithi133 bash[27558]: audit 2023-10-09T04:51:03.831810+0000 mon.a (mon.0) 1125 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:05.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:04 smithi133 bash[27558]: audit 2023-10-09T04:51:04.713844+0000 mon.a (mon.0) 1126 : audit [DBG] from='client.? 172.21.15.133:0/2526529810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:05.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:05 smithi194 bash[69734]: debug 2023-10-09T04:51:05.477+0000 7f46957a7200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:51:06.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:05 smithi194 bash[31648]: cluster 2023-10-09T04:51:05.078430+0000 mgr.y (mgr.25046) 10 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 355 op/s 2023-10-09T04:51:06.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:05 smithi133 bash[27558]: cluster 2023-10-09T04:51:05.078430+0000 mgr.y (mgr.25046) 10 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 355 op/s 2023-10-09T04:51:06.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:05 smithi133 bash[41291]: cluster 2023-10-09T04:51:05.078430+0000 mgr.y (mgr.25046) 10 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 355 op/s 2023-10-09T04:51:07.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:07 smithi133 bash[27558]: cluster 2023-10-09T04:51:07.079150+0000 mgr.y (mgr.25046) 11 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2023-10-09T04:51:07.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:07 smithi133 bash[41291]: cluster 2023-10-09T04:51:07.079150+0000 mgr.y (mgr.25046) 11 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2023-10-09T04:51:07.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:07 smithi194 bash[31648]: cluster 2023-10-09T04:51:07.079150+0000 mgr.y (mgr.25046) 11 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2023-10-09T04:51:07.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:07 smithi194 bash[69734]: debug 2023-10-09T04:51:07.305+0000 7f46957a7200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:51:07.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:07 smithi194 bash[69734]: [09/Oct/2023:04:51:07] ENGINE Bus STARTING 2023-10-09T04:51:07.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:07 smithi194 bash[69734]: CherryPy Checker: 2023-10-09T04:51:07.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:07 smithi194 bash[69734]: The Application mounted at '' has an empty config. 2023-10-09T04:51:07.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:07 smithi194 bash[69734]: [09/Oct/2023:04:51:07] ENGINE Serving on http://:::9283 2023-10-09T04:51:07.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:07 smithi194 bash[69734]: [09/Oct/2023:04:51:07] ENGINE Bus STARTED 2023-10-09T04:51:08.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:08 smithi133 bash[27558]: cluster 2023-10-09T04:51:07.330064+0000 mon.a (mon.0) 1127 : cluster [DBG] Standby manager daemon x started 2023-10-09T04:51:08.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:08 smithi133 bash[27558]: audit 2023-10-09T04:51:07.339389+0000 mon.a (mon.0) 1128 : audit [DBG] from='mgr.? 172.21.15.194:0/2921303287' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:51:08.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:08 smithi133 bash[27558]: audit 2023-10-09T04:51:07.340497+0000 mon.a (mon.0) 1129 : audit [DBG] from='mgr.? 172.21.15.194:0/2921303287' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:51:08.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:08 smithi133 bash[27558]: audit 2023-10-09T04:51:07.348057+0000 mon.a (mon.0) 1130 : audit [DBG] from='mgr.? 172.21.15.194:0/2921303287' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:51:08.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:08 smithi133 bash[27558]: audit 2023-10-09T04:51:07.348672+0000 mon.a (mon.0) 1131 : audit [DBG] from='mgr.? 172.21.15.194:0/2921303287' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:51:08.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:08 smithi133 bash[27558]: audit 2023-10-09T04:51:07.404232+0000 mon.b (mon.1) 40 : audit [DBG] from='client.? 172.21.15.133:0/3788226300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:08.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:08 smithi133 bash[41291]: cluster 2023-10-09T04:51:07.330064+0000 mon.a (mon.0) 1127 : cluster [DBG] Standby manager daemon x started 2023-10-09T04:51:08.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:08 smithi133 bash[41291]: audit 2023-10-09T04:51:07.339389+0000 mon.a (mon.0) 1128 : audit [DBG] from='mgr.? 172.21.15.194:0/2921303287' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:51:08.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:08 smithi133 bash[41291]: audit 2023-10-09T04:51:07.340497+0000 mon.a (mon.0) 1129 : audit [DBG] from='mgr.? 172.21.15.194:0/2921303287' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:51:08.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:08 smithi133 bash[41291]: audit 2023-10-09T04:51:07.348057+0000 mon.a (mon.0) 1130 : audit [DBG] from='mgr.? 172.21.15.194:0/2921303287' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:51:08.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:08 smithi133 bash[41291]: audit 2023-10-09T04:51:07.348672+0000 mon.a (mon.0) 1131 : audit [DBG] from='mgr.? 172.21.15.194:0/2921303287' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:51:08.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:08 smithi133 bash[41291]: audit 2023-10-09T04:51:07.404232+0000 mon.b (mon.1) 40 : audit [DBG] from='client.? 172.21.15.133:0/3788226300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:08.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:08 smithi194 bash[31648]: cluster 2023-10-09T04:51:07.330064+0000 mon.a (mon.0) 1127 : cluster [DBG] Standby manager daemon x started 2023-10-09T04:51:08.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:08 smithi194 bash[31648]: audit 2023-10-09T04:51:07.339389+0000 mon.a (mon.0) 1128 : audit [DBG] from='mgr.? 172.21.15.194:0/2921303287' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:51:08.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:08 smithi194 bash[31648]: audit 2023-10-09T04:51:07.340497+0000 mon.a (mon.0) 1129 : audit [DBG] from='mgr.? 172.21.15.194:0/2921303287' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:51:08.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:08 smithi194 bash[31648]: audit 2023-10-09T04:51:07.348057+0000 mon.a (mon.0) 1130 : audit [DBG] from='mgr.? 172.21.15.194:0/2921303287' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:51:08.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:08 smithi194 bash[31648]: audit 2023-10-09T04:51:07.348672+0000 mon.a (mon.0) 1131 : audit [DBG] from='mgr.? 172.21.15.194:0/2921303287' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:51:08.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:08 smithi194 bash[31648]: audit 2023-10-09T04:51:07.404232+0000 mon.b (mon.1) 40 : audit [DBG] from='client.? 172.21.15.133:0/3788226300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:09.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:09 smithi133 bash[27558]: cluster 2023-10-09T04:51:08.176051+0000 mon.a (mon.0) 1132 : cluster [DBG] mgrmap e29: y(active, since 13s), standbys: x 2023-10-09T04:51:09.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:09 smithi133 bash[27558]: audit 2023-10-09T04:51:08.182045+0000 mon.c (mon.2) 77 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:51:09.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:09 smithi133 bash[27558]: cluster 2023-10-09T04:51:09.079629+0000 mgr.y (mgr.25046) 12 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 515 op/s 2023-10-09T04:51:09.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:09 smithi133 bash[41291]: cluster 2023-10-09T04:51:08.176051+0000 mon.a (mon.0) 1132 : cluster [DBG] mgrmap e29: y(active, since 13s), standbys: x 2023-10-09T04:51:09.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:09 smithi133 bash[41291]: audit 2023-10-09T04:51:08.182045+0000 mon.c (mon.2) 77 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:51:09.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:09 smithi133 bash[41291]: cluster 2023-10-09T04:51:09.079629+0000 mgr.y (mgr.25046) 12 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 515 op/s 2023-10-09T04:51:09.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:09 smithi194 bash[31648]: cluster 2023-10-09T04:51:08.176051+0000 mon.a (mon.0) 1132 : cluster [DBG] mgrmap e29: y(active, since 13s), standbys: x 2023-10-09T04:51:09.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:09 smithi194 bash[31648]: audit 2023-10-09T04:51:08.182045+0000 mon.c (mon.2) 77 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:51:09.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:09 smithi194 bash[31648]: cluster 2023-10-09T04:51:09.079629+0000 mgr.y (mgr.25046) 12 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 515 op/s 2023-10-09T04:51:10.524 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:10 smithi133 bash[27558]: audit 2023-10-09T04:51:10.069793+0000 mon.a (mon.0) 1133 : audit [DBG] from='client.? 172.21.15.133:0/1316217418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:10.525 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:10 smithi133 bash[41291]: audit 2023-10-09T04:51:10.069793+0000 mon.a (mon.0) 1133 : audit [DBG] from='client.? 172.21.15.133:0/1316217418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:10.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:10 smithi194 bash[31648]: audit 2023-10-09T04:51:10.069793+0000 mon.a (mon.0) 1133 : audit [DBG] from='client.? 172.21.15.133:0/1316217418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:11.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:11 smithi133 bash[41291]: audit 2023-10-09T04:51:10.235554+0000 mon.c (mon.2) 78 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:11.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:11 smithi133 bash[41291]: cluster 2023-10-09T04:51:11.080018+0000 mgr.y (mgr.25046) 13 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 515 op/s 2023-10-09T04:51:11.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:11 smithi133 bash[27558]: audit 2023-10-09T04:51:10.235554+0000 mon.c (mon.2) 78 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:11.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:11 smithi133 bash[27558]: cluster 2023-10-09T04:51:11.080018+0000 mgr.y (mgr.25046) 13 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 515 op/s 2023-10-09T04:51:11.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:11 smithi194 bash[31648]: audit 2023-10-09T04:51:10.235554+0000 mon.c (mon.2) 78 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:11.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:11 smithi194 bash[31648]: cluster 2023-10-09T04:51:11.080018+0000 mgr.y (mgr.25046) 13 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 515 op/s 2023-10-09T04:51:13.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:12 smithi133 bash[27558]: audit 2023-10-09T04:51:12.773294+0000 mon.a (mon.0) 1134 : audit [DBG] from='client.? 172.21.15.133:0/109813051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:13.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:12 smithi133 bash[41291]: audit 2023-10-09T04:51:12.773294+0000 mon.a (mon.0) 1134 : audit [DBG] from='client.? 172.21.15.133:0/109813051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:13.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:12 smithi194 bash[31648]: audit 2023-10-09T04:51:12.773294+0000 mon.a (mon.0) 1134 : audit [DBG] from='client.? 172.21.15.133:0/109813051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:14.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:13 smithi194 bash[31648]: cluster 2023-10-09T04:51:13.080795+0000 mgr.y (mgr.25046) 14 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 970 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2023-10-09T04:51:14.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:13 smithi133 bash[41291]: cluster 2023-10-09T04:51:13.080795+0000 mgr.y (mgr.25046) 14 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 970 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2023-10-09T04:51:14.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:13 smithi133 bash[27558]: cluster 2023-10-09T04:51:13.080795+0000 mgr.y (mgr.25046) 14 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 970 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2023-10-09T04:51:15.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:15 smithi133 bash[41291]: cluster 2023-10-09T04:51:15.081287+0000 mgr.y (mgr.25046) 15 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 970 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 538 op/s 2023-10-09T04:51:15.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:15 smithi133 bash[27558]: cluster 2023-10-09T04:51:15.081287+0000 mgr.y (mgr.25046) 15 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 970 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 538 op/s 2023-10-09T04:51:15.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:15 smithi194 bash[31648]: cluster 2023-10-09T04:51:15.081287+0000 mgr.y (mgr.25046) 15 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 970 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 538 op/s 2023-10-09T04:51:16.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:16 smithi133 bash[41291]: audit 2023-10-09T04:51:15.513006+0000 mon.c (mon.2) 79 : audit [DBG] from='client.? 172.21.15.133:0/2483470444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:16.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:16 smithi133 bash[27558]: audit 2023-10-09T04:51:15.513006+0000 mon.c (mon.2) 79 : audit [DBG] from='client.? 172.21.15.133:0/2483470444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:16.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:16 smithi194 bash[31648]: audit 2023-10-09T04:51:15.513006+0000 mon.c (mon.2) 79 : audit [DBG] from='client.? 172.21.15.133:0/2483470444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:17.386 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:16 smithi194 bash[69734]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:51:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:51:18.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:17 smithi133 bash[41291]: audit 2023-10-09T04:51:16.783902+0000 mon.a (mon.0) 1135 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:18.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:17 smithi133 bash[41291]: audit 2023-10-09T04:51:16.797908+0000 mon.a (mon.0) 1136 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:18.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:17 smithi133 bash[41291]: audit 2023-10-09T04:51:16.807072+0000 mon.c (mon.2) 80 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:51:18.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:17 smithi133 bash[41291]: audit 2023-10-09T04:51:16.808391+0000 mon.c (mon.2) 81 : audit [INF] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:51:18.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:17 smithi133 bash[41291]: cephadm 2023-10-09T04:51:16.811137+0000 mgr.y (mgr.25046) 16 : cephadm [INF] Updating smithi133:/etc/ceph/ceph.conf 2023-10-09T04:51:18.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:17 smithi133 bash[41291]: cephadm 2023-10-09T04:51:16.811431+0000 mgr.y (mgr.25046) 17 : cephadm [INF] Updating smithi194:/etc/ceph/ceph.conf 2023-10-09T04:51:18.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:17 smithi133 bash[41291]: cephadm 2023-10-09T04:51:17.027151+0000 mgr.y (mgr.25046) 18 : cephadm [INF] Updating smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:51:18.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:17 smithi133 bash[41291]: cephadm 2023-10-09T04:51:17.029812+0000 mgr.y (mgr.25046) 19 : cephadm [INF] Updating smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:51:18.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:17 smithi133 bash[41291]: cluster 2023-10-09T04:51:17.082034+0000 mgr.y (mgr.25046) 20 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 752 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 829 op/s 2023-10-09T04:51:18.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:17 smithi133 bash[27558]: audit 2023-10-09T04:51:16.783902+0000 mon.a (mon.0) 1135 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:18.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:17 smithi133 bash[27558]: audit 2023-10-09T04:51:16.797908+0000 mon.a (mon.0) 1136 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:18.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:17 smithi133 bash[27558]: audit 2023-10-09T04:51:16.807072+0000 mon.c (mon.2) 80 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:51:18.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:17 smithi133 bash[27558]: audit 2023-10-09T04:51:16.808391+0000 mon.c (mon.2) 81 : audit [INF] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:51:18.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:17 smithi133 bash[27558]: cephadm 2023-10-09T04:51:16.811137+0000 mgr.y (mgr.25046) 16 : cephadm [INF] Updating smithi133:/etc/ceph/ceph.conf 2023-10-09T04:51:18.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:17 smithi133 bash[27558]: cephadm 2023-10-09T04:51:16.811431+0000 mgr.y (mgr.25046) 17 : cephadm [INF] Updating smithi194:/etc/ceph/ceph.conf 2023-10-09T04:51:18.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:17 smithi133 bash[27558]: cephadm 2023-10-09T04:51:17.027151+0000 mgr.y (mgr.25046) 18 : cephadm [INF] Updating smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:51:18.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:17 smithi133 bash[27558]: cephadm 2023-10-09T04:51:17.029812+0000 mgr.y (mgr.25046) 19 : cephadm [INF] Updating smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:51:18.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:17 smithi133 bash[27558]: cluster 2023-10-09T04:51:17.082034+0000 mgr.y (mgr.25046) 20 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 752 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 829 op/s 2023-10-09T04:51:18.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:17 smithi194 bash[31648]: audit 2023-10-09T04:51:16.783902+0000 mon.a (mon.0) 1135 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:18.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:17 smithi194 bash[31648]: audit 2023-10-09T04:51:16.797908+0000 mon.a (mon.0) 1136 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:18.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:17 smithi194 bash[31648]: audit 2023-10-09T04:51:16.807072+0000 mon.c (mon.2) 80 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:51:18.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:17 smithi194 bash[31648]: audit 2023-10-09T04:51:16.808391+0000 mon.c (mon.2) 81 : audit [INF] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:51:18.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:17 smithi194 bash[31648]: cephadm 2023-10-09T04:51:16.811137+0000 mgr.y (mgr.25046) 16 : cephadm [INF] Updating smithi133:/etc/ceph/ceph.conf 2023-10-09T04:51:18.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:17 smithi194 bash[31648]: cephadm 2023-10-09T04:51:16.811431+0000 mgr.y (mgr.25046) 17 : cephadm [INF] Updating smithi194:/etc/ceph/ceph.conf 2023-10-09T04:51:18.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:17 smithi194 bash[31648]: cephadm 2023-10-09T04:51:17.027151+0000 mgr.y (mgr.25046) 18 : cephadm [INF] Updating smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:51:18.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:17 smithi194 bash[31648]: cephadm 2023-10-09T04:51:17.029812+0000 mgr.y (mgr.25046) 19 : cephadm [INF] Updating smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:51:18.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:17 smithi194 bash[31648]: cluster 2023-10-09T04:51:17.082034+0000 mgr.y (mgr.25046) 20 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 752 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 829 op/s 2023-10-09T04:51:18.859 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:18 smithi194 bash[31648]: cephadm 2023-10-09T04:51:17.253754+0000 mgr.y (mgr.25046) 21 : cephadm [INF] Updating smithi194:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:51:18.859 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:18 smithi194 bash[31648]: cephadm 2023-10-09T04:51:17.263713+0000 mgr.y (mgr.25046) 22 : cephadm [INF] Updating smithi133:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:51:18.860 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:18 smithi194 bash[31648]: cephadm 2023-10-09T04:51:17.537080+0000 mgr.y (mgr.25046) 23 : cephadm [INF] Updating smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:51:18.860 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:18 smithi194 bash[31648]: cephadm 2023-10-09T04:51:17.537467+0000 mgr.y (mgr.25046) 24 : cephadm [INF] Updating smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:51:18.860 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:18 smithi194 bash[31648]: audit 2023-10-09T04:51:17.855750+0000 mon.a (mon.0) 1137 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:18.860 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:18 smithi194 bash[31648]: audit 2023-10-09T04:51:17.865289+0000 mon.a (mon.0) 1138 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:18.860 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:18 smithi194 bash[31648]: audit 2023-10-09T04:51:17.876091+0000 mon.a (mon.0) 1139 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:18.860 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:18 smithi194 bash[31648]: audit 2023-10-09T04:51:17.889081+0000 mon.a (mon.0) 1140 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:18.860 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:18 smithi194 bash[31648]: audit 2023-10-09T04:51:17.901749+0000 mon.a (mon.0) 1141 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:18.860 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:18 smithi194 bash[31648]: cephadm 2023-10-09T04:51:18.007558+0000 mgr.y (mgr.25046) 25 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T04:51:18.860 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:18 smithi194 bash[31648]: audit 2023-10-09T04:51:18.253616+0000 mon.a (mon.0) 1142 : audit [DBG] from='client.? 172.21.15.133:0/4149076144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:18.861 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:18 smithi194 bash[80137]: ts=2023-10-09T04:51:18.470Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.194:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.194:8765: connect: connection refused" 2023-10-09T04:51:18.861 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:18 smithi194 bash[80137]: ts=2023-10-09T04:51:18.471Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.194:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.194:8765: connect: connection refused" 2023-10-09T04:51:18.861 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:18 smithi194 bash[80137]: ts=2023-10-09T04:51:18.480Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.194:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.194:8765: connect: connection refused" 2023-10-09T04:51:18.861 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:18 smithi194 bash[80137]: ts=2023-10-09T04:51:18.480Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.194:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.194:8765: connect: connection refused" 2023-10-09T04:51:19.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:18 smithi133 bash[41291]: cephadm 2023-10-09T04:51:17.253754+0000 mgr.y (mgr.25046) 21 : cephadm [INF] Updating smithi194:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:51:19.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:18 smithi133 bash[41291]: cephadm 2023-10-09T04:51:17.263713+0000 mgr.y (mgr.25046) 22 : cephadm [INF] Updating smithi133:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:51:19.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:18 smithi133 bash[41291]: cephadm 2023-10-09T04:51:17.537080+0000 mgr.y (mgr.25046) 23 : cephadm [INF] Updating smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:51:19.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:18 smithi133 bash[41291]: cephadm 2023-10-09T04:51:17.537467+0000 mgr.y (mgr.25046) 24 : cephadm [INF] Updating smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:51:19.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:18 smithi133 bash[41291]: audit 2023-10-09T04:51:17.855750+0000 mon.a (mon.0) 1137 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:19.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:18 smithi133 bash[41291]: audit 2023-10-09T04:51:17.865289+0000 mon.a (mon.0) 1138 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:19.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:18 smithi133 bash[41291]: audit 2023-10-09T04:51:17.876091+0000 mon.a (mon.0) 1139 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:19.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:18 smithi133 bash[41291]: audit 2023-10-09T04:51:17.889081+0000 mon.a (mon.0) 1140 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:19.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:18 smithi133 bash[41291]: audit 2023-10-09T04:51:17.901749+0000 mon.a (mon.0) 1141 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:19.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:18 smithi133 bash[41291]: cephadm 2023-10-09T04:51:18.007558+0000 mgr.y (mgr.25046) 25 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T04:51:19.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:18 smithi133 bash[41291]: audit 2023-10-09T04:51:18.253616+0000 mon.a (mon.0) 1142 : audit [DBG] from='client.? 172.21.15.133:0/4149076144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:19.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:18 smithi133 bash[27558]: cephadm 2023-10-09T04:51:17.253754+0000 mgr.y (mgr.25046) 21 : cephadm [INF] Updating smithi194:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:51:19.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:18 smithi133 bash[27558]: cephadm 2023-10-09T04:51:17.263713+0000 mgr.y (mgr.25046) 22 : cephadm [INF] Updating smithi133:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:51:19.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:18 smithi133 bash[27558]: cephadm 2023-10-09T04:51:17.537080+0000 mgr.y (mgr.25046) 23 : cephadm [INF] Updating smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:51:19.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:18 smithi133 bash[27558]: cephadm 2023-10-09T04:51:17.537467+0000 mgr.y (mgr.25046) 24 : cephadm [INF] Updating smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:51:19.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:18 smithi133 bash[27558]: audit 2023-10-09T04:51:17.855750+0000 mon.a (mon.0) 1137 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:19.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:18 smithi133 bash[27558]: audit 2023-10-09T04:51:17.865289+0000 mon.a (mon.0) 1138 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:19.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:18 smithi133 bash[27558]: audit 2023-10-09T04:51:17.876091+0000 mon.a (mon.0) 1139 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:19.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:18 smithi133 bash[27558]: audit 2023-10-09T04:51:17.889081+0000 mon.a (mon.0) 1140 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:19.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:18 smithi133 bash[27558]: audit 2023-10-09T04:51:17.901749+0000 mon.a (mon.0) 1141 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:19.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:18 smithi133 bash[27558]: cephadm 2023-10-09T04:51:18.007558+0000 mgr.y (mgr.25046) 25 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T04:51:19.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:18 smithi133 bash[27558]: audit 2023-10-09T04:51:18.253616+0000 mon.a (mon.0) 1142 : audit [DBG] from='client.? 172.21.15.133:0/4149076144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:20.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:19 smithi194 bash[31648]: cephadm 2023-10-09T04:51:18.285690+0000 mgr.y (mgr.25046) 26 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi194 2023-10-09T04:51:20.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:19 smithi194 bash[31648]: cluster 2023-10-09T04:51:19.082471+0000 mgr.y (mgr.25046) 27 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 752 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-09T04:51:20.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:19 smithi133 bash[27558]: cephadm 2023-10-09T04:51:18.285690+0000 mgr.y (mgr.25046) 26 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi194 2023-10-09T04:51:20.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:19 smithi133 bash[27558]: cluster 2023-10-09T04:51:19.082471+0000 mgr.y (mgr.25046) 27 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 752 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-09T04:51:20.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:19 smithi133 bash[41291]: cephadm 2023-10-09T04:51:18.285690+0000 mgr.y (mgr.25046) 26 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi194 2023-10-09T04:51:20.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:19 smithi133 bash[41291]: cluster 2023-10-09T04:51:19.082471+0000 mgr.y (mgr.25046) 27 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 752 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-09T04:51:21.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:21 smithi194 bash[31648]: audit 2023-10-09T04:51:20.962946+0000 mon.a (mon.0) 1143 : audit [DBG] from='client.? 172.21.15.133:0/2564394180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:21.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:21 smithi194 bash[31648]: cluster 2023-10-09T04:51:21.082899+0000 mgr.y (mgr.25046) 28 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 752 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-09T04:51:22.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:21 smithi133 bash[27558]: audit 2023-10-09T04:51:20.962946+0000 mon.a (mon.0) 1143 : audit [DBG] from='client.? 172.21.15.133:0/2564394180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:22.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:21 smithi133 bash[27558]: cluster 2023-10-09T04:51:21.082899+0000 mgr.y (mgr.25046) 28 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 752 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-09T04:51:22.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:21 smithi133 bash[41291]: audit 2023-10-09T04:51:20.962946+0000 mon.a (mon.0) 1143 : audit [DBG] from='client.? 172.21.15.133:0/2564394180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:22.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:21 smithi133 bash[41291]: cluster 2023-10-09T04:51:21.082899+0000 mgr.y (mgr.25046) 28 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 752 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-09T04:51:23.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:23 smithi133 bash[27558]: cluster 2023-10-09T04:51:23.083742+0000 mgr.y (mgr.25046) 29 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 538 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 864 op/s 2023-10-09T04:51:23.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:23 smithi133 bash[41291]: cluster 2023-10-09T04:51:23.083742+0000 mgr.y (mgr.25046) 29 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 538 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 864 op/s 2023-10-09T04:51:23.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:23 smithi194 bash[31648]: cluster 2023-10-09T04:51:23.083742+0000 mgr.y (mgr.25046) 29 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 538 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 864 op/s 2023-10-09T04:51:24.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:24 smithi194 bash[31648]: audit 2023-10-09T04:51:23.692210+0000 mon.b (mon.1) 41 : audit [DBG] from='client.? 172.21.15.133:0/402331833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:24.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:24 smithi133 bash[27558]: audit 2023-10-09T04:51:23.692210+0000 mon.b (mon.1) 41 : audit [DBG] from='client.? 172.21.15.133:0/402331833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:24.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:24 smithi133 bash[41291]: audit 2023-10-09T04:51:23.692210+0000 mon.b (mon.1) 41 : audit [DBG] from='client.? 172.21.15.133:0/402331833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:25.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:25 smithi133 bash[41291]: cluster 2023-10-09T04:51:25.084173+0000 mgr.y (mgr.25046) 30 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 538 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-10-09T04:51:25.749 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:25 smithi133 bash[27558]: cluster 2023-10-09T04:51:25.084173+0000 mgr.y (mgr.25046) 30 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 538 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-10-09T04:51:25.749 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:25 smithi194 bash[31648]: cluster 2023-10-09T04:51:25.084173+0000 mgr.y (mgr.25046) 30 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 538 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-10-09T04:51:26.137 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:25 smithi194 systemd[1]: Stopping Ceph prometheus.a for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T04:51:26.137 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:25 smithi194 bash[80137]: ts=2023-10-09T04:51:25.996Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T04:51:26.137 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:25 smithi194 bash[80137]: ts=2023-10-09T04:51:25.996Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-10-09T04:51:26.137 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:25 smithi194 bash[80137]: ts=2023-10-09T04:51:25.996Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-10-09T04:51:26.137 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:25 smithi194 bash[80137]: ts=2023-10-09T04:51:25.996Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T04:51:26.137 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:25 smithi194 bash[80137]: ts=2023-10-09T04:51:25.996Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T04:51:26.137 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:25 smithi194 bash[80137]: ts=2023-10-09T04:51:25.996Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-10-09T04:51:26.138 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:25 smithi194 bash[80137]: ts=2023-10-09T04:51:25.996Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-10-09T04:51:26.138 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:25 smithi194 bash[80137]: ts=2023-10-09T04:51:25.996Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-10-09T04:51:26.138 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:25 smithi194 bash[80137]: ts=2023-10-09T04:51:25.997Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-10-09T04:51:26.138 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:25 smithi194 bash[80137]: ts=2023-10-09T04:51:25.997Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T04:51:26.138 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:25 smithi194 bash[80137]: ts=2023-10-09T04:51:25.997Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-10-09T04:51:26.138 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:25 smithi194 bash[80137]: ts=2023-10-09T04:51:25.997Z caller=main.go:1147 level=info msg="See you next time!" 2023-10-09T04:51:26.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:26 smithi133 bash[41291]: audit 2023-10-09T04:51:25.235453+0000 mon.c (mon.2) 82 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:26.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:26 smithi133 bash[27558]: audit 2023-10-09T04:51:25.235453+0000 mon.c (mon.2) 82 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:26.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:26 smithi194 bash[31648]: audit 2023-10-09T04:51:25.235453+0000 mon.c (mon.2) 82 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:26.967 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:26 smithi194 bash[81802]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T04:51:26.968 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:26 smithi194 bash[81863]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T04:51:26.968 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:26 smithi194 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T04:51:26.968 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:26 smithi194 systemd[1]: Stopped Ceph prometheus.a for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:51:26.968 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:26 smithi194 systemd[1]: Started Ceph prometheus.a for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:51:27.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:27 smithi194 bash[31648]: audit 2023-10-09T04:51:26.393623+0000 mon.c (mon.2) 83 : audit [DBG] from='client.? 172.21.15.133:0/102684244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:27.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:27 smithi194 bash[31648]: audit 2023-10-09T04:51:26.914629+0000 mon.a (mon.0) 1144 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:27.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:27 smithi194 bash[31648]: audit 2023-10-09T04:51:26.928561+0000 mon.a (mon.0) 1145 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:27.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:27 smithi194 bash[31648]: audit 2023-10-09T04:51:26.973564+0000 mon.c (mon.2) 84 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:51:27.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:27 smithi194 bash[31648]: audit 2023-10-09T04:51:26.974605+0000 mgr.y (mgr.25046) 31 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:51:27.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:27 smithi194 bash[31648]: audit 2023-10-09T04:51:26.989536+0000 mon.c (mon.2) 85 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:51:27.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:27 smithi194 bash[31648]: cluster 2023-10-09T04:51:27.085142+0000 mgr.y (mgr.25046) 32 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 312 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 877 op/s 2023-10-09T04:51:27.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:27 smithi133 bash[27558]: audit 2023-10-09T04:51:26.393623+0000 mon.c (mon.2) 83 : audit [DBG] from='client.? 172.21.15.133:0/102684244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:27.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:27 smithi133 bash[27558]: audit 2023-10-09T04:51:26.914629+0000 mon.a (mon.0) 1144 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:27.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:27 smithi133 bash[27558]: audit 2023-10-09T04:51:26.928561+0000 mon.a (mon.0) 1145 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:27.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:27 smithi133 bash[27558]: audit 2023-10-09T04:51:26.973564+0000 mon.c (mon.2) 84 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:51:27.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:27 smithi133 bash[27558]: audit 2023-10-09T04:51:26.974605+0000 mgr.y (mgr.25046) 31 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:51:27.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:27 smithi133 bash[27558]: audit 2023-10-09T04:51:26.989536+0000 mon.c (mon.2) 85 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:51:27.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:27 smithi133 bash[27558]: cluster 2023-10-09T04:51:27.085142+0000 mgr.y (mgr.25046) 32 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 312 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 877 op/s 2023-10-09T04:51:27.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:27 smithi133 bash[41291]: audit 2023-10-09T04:51:26.393623+0000 mon.c (mon.2) 83 : audit [DBG] from='client.? 172.21.15.133:0/102684244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:27.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:27 smithi133 bash[41291]: audit 2023-10-09T04:51:26.914629+0000 mon.a (mon.0) 1144 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:27.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:27 smithi133 bash[41291]: audit 2023-10-09T04:51:26.928561+0000 mon.a (mon.0) 1145 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:27.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:27 smithi133 bash[41291]: audit 2023-10-09T04:51:26.973564+0000 mon.c (mon.2) 84 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:51:27.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:27 smithi133 bash[41291]: audit 2023-10-09T04:51:26.974605+0000 mgr.y (mgr.25046) 31 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:51:27.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:27 smithi133 bash[41291]: audit 2023-10-09T04:51:26.989536+0000 mon.c (mon.2) 85 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:51:27.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:27 smithi133 bash[41291]: cluster 2023-10-09T04:51:27.085142+0000 mgr.y (mgr.25046) 32 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 312 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 877 op/s 2023-10-09T04:51:28.944 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:28 smithi194 bash[81885]: ts=2023-10-09T04:51:28.879Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-10-09T04:51:28.944 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:28 smithi194 bash[81885]: ts=2023-10-09T04:51:28.879Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-10-09T04:51:28.944 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:28 smithi194 bash[81885]: ts=2023-10-09T04:51:28.879Z caller=main.go:570 level=info host_details="(Linux 5.15.0-86-generic #96~20.04.1-Ubuntu SMP Thu Sep 21 13:23:37 UTC 2023 x86_64 smithi194 (none))" 2023-10-09T04:51:28.944 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:28 smithi194 bash[81885]: ts=2023-10-09T04:51:28.879Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T04:51:28.944 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:28 smithi194 bash[81885]: ts=2023-10-09T04:51:28.879Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T04:51:28.944 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:28 smithi194 bash[81885]: ts=2023-10-09T04:51:28.882Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T04:51:28.945 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:28 smithi194 bash[81885]: ts=2023-10-09T04:51:28.882Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-10-09T04:51:28.945 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:28 smithi194 bash[81885]: ts=2023-10-09T04:51:28.889Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-10-09T04:51:28.945 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:28 smithi194 bash[81885]: ts=2023-10-09T04:51:28.889Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-10-09T04:51:28.945 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:28 smithi194 bash[81885]: ts=2023-10-09T04:51:28.891Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T04:51:28.945 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:28 smithi194 bash[81885]: ts=2023-10-09T04:51:28.891Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.438µs 2023-10-09T04:51:28.945 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:28 smithi194 bash[81885]: ts=2023-10-09T04:51:28.891Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T04:51:28.945 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:28 smithi194 bash[81885]: ts=2023-10-09T04:51:28.891Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2023-10-09T04:51:28.945 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:28 smithi194 bash[81885]: ts=2023-10-09T04:51:28.942Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2023-10-09T04:51:29.003 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:28 smithi194 bash[81885]: ts=2023-10-09T04:51:28.952Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2023-10-09T04:51:29.004 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:28 smithi194 bash[81885]: ts=2023-10-09T04:51:28.952Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2023-10-09T04:51:29.004 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:28 smithi194 bash[81885]: ts=2023-10-09T04:51:28.952Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=31.545µs wal_replay_duration=61.137937ms wbl_replay_duration=138ns total_replay_duration=61.241616ms 2023-10-09T04:51:29.004 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:28 smithi194 bash[81885]: ts=2023-10-09T04:51:28.960Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T04:51:29.004 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:28 smithi194 bash[81885]: ts=2023-10-09T04:51:28.960Z caller=main.go:1029 level=info msg="TSDB started" 2023-10-09T04:51:29.004 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:28 smithi194 bash[81885]: ts=2023-10-09T04:51:28.960Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T04:51:29.004 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:29 smithi194 bash[81885]: ts=2023-10-09T04:51:29.002Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=41.521495ms db_storage=1.597µs remote_storage=2.129µs web_handler=872ns query_engine=1.168µs scrape=413.644µs scrape_sd=245.439µs notify=17.141µs notify_sd=19.372µs rules=40.415725ms tracing=8.993µs 2023-10-09T04:51:29.211 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:29 smithi194 bash[31648]: cluster 2023-10-09T04:51:29.086015+0000 mgr.y (mgr.25046) 33 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 312 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-10-09T04:51:29.212 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:29 smithi194 bash[31648]: audit 2023-10-09T04:51:29.091569+0000 mon.b (mon.1) 42 : audit [DBG] from='client.? 172.21.15.133:0/65156566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:29.212 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:29 smithi194 bash[81885]: ts=2023-10-09T04:51:29.002Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-10-09T04:51:29.212 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:29 smithi194 bash[81885]: ts=2023-10-09T04:51:29.002Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-10-09T04:51:29.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:29 smithi133 bash[41291]: cluster 2023-10-09T04:51:29.086015+0000 mgr.y (mgr.25046) 33 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 312 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-10-09T04:51:29.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:29 smithi133 bash[41291]: audit 2023-10-09T04:51:29.091569+0000 mon.b (mon.1) 42 : audit [DBG] from='client.? 172.21.15.133:0/65156566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:29.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:29 smithi133 bash[27558]: cluster 2023-10-09T04:51:29.086015+0000 mgr.y (mgr.25046) 33 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 312 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-10-09T04:51:29.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:29 smithi133 bash[27558]: audit 2023-10-09T04:51:29.091569+0000 mon.b (mon.1) 42 : audit [DBG] from='client.? 172.21.15.133:0/65156566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:30.404 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:30 smithi194 bash[31648]: cephadm 2023-10-09T04:51:30.020518+0000 mgr.y (mgr.25046) 34 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T04:51:30.404 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:30 smithi194 bash[31648]: audit 2023-10-09T04:51:30.021051+0000 mon.c (mon.2) 86 : audit [INF] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:51:30.404 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:30 smithi194 bash[31648]: audit 2023-10-09T04:51:30.021605+0000 mon.a (mon.0) 1146 : audit [INF] from='mgr.25046 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:51:30.404 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:30 smithi194 bash[31648]: audit 2023-10-09T04:51:30.022822+0000 mon.c (mon.2) 87 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:51:30.404 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:30 smithi194 bash[31648]: audit 2023-10-09T04:51:30.024126+0000 mon.c (mon.2) 88 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:51:30.405 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:30 smithi194 bash[31648]: cephadm 2023-10-09T04:51:30.025302+0000 mgr.y (mgr.25046) 35 : cephadm [INF] Deploying daemon mgr.x on smithi194 2023-10-09T04:51:30.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:30 smithi133 bash[41291]: cephadm 2023-10-09T04:51:30.020518+0000 mgr.y (mgr.25046) 34 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T04:51:30.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:30 smithi133 bash[41291]: audit 2023-10-09T04:51:30.021051+0000 mon.c (mon.2) 86 : audit [INF] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:51:30.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:30 smithi133 bash[41291]: audit 2023-10-09T04:51:30.021605+0000 mon.a (mon.0) 1146 : audit [INF] from='mgr.25046 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:51:30.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:30 smithi133 bash[41291]: audit 2023-10-09T04:51:30.022822+0000 mon.c (mon.2) 87 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:51:30.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:30 smithi133 bash[41291]: audit 2023-10-09T04:51:30.024126+0000 mon.c (mon.2) 88 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:51:30.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:30 smithi133 bash[41291]: cephadm 2023-10-09T04:51:30.025302+0000 mgr.y (mgr.25046) 35 : cephadm [INF] Deploying daemon mgr.x on smithi194 2023-10-09T04:51:30.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:30 smithi133 bash[27558]: cephadm 2023-10-09T04:51:30.020518+0000 mgr.y (mgr.25046) 34 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T04:51:30.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:30 smithi133 bash[27558]: audit 2023-10-09T04:51:30.021051+0000 mon.c (mon.2) 86 : audit [INF] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:51:30.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:30 smithi133 bash[27558]: audit 2023-10-09T04:51:30.021605+0000 mon.a (mon.0) 1146 : audit [INF] from='mgr.25046 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:51:30.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:30 smithi133 bash[27558]: audit 2023-10-09T04:51:30.022822+0000 mon.c (mon.2) 87 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:51:30.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:30 smithi133 bash[27558]: audit 2023-10-09T04:51:30.024126+0000 mon.c (mon.2) 88 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:51:30.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:30 smithi133 bash[27558]: cephadm 2023-10-09T04:51:30.025302+0000 mgr.y (mgr.25046) 35 : cephadm [INF] Deploying daemon mgr.x on smithi194 2023-10-09T04:51:31.639 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:31 smithi194 bash[31648]: cluster 2023-10-09T04:51:31.086457+0000 mgr.y (mgr.25046) 36 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 312 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-10-09T04:51:32.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:31 smithi133 bash[41291]: cluster 2023-10-09T04:51:31.086457+0000 mgr.y (mgr.25046) 36 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 312 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-10-09T04:51:32.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:31 smithi133 bash[27558]: cluster 2023-10-09T04:51:31.086457+0000 mgr.y (mgr.25046) 36 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 312 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-10-09T04:51:32.774 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:32 smithi194 systemd[1]: Stopping Ceph mgr.x for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T04:51:32.775 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:32 smithi194 bash[31648]: audit 2023-10-09T04:51:31.773984+0000 mon.a (mon.0) 1147 : audit [DBG] from='client.? 172.21.15.133:0/841589226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:33.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:32 smithi133 bash[41291]: audit 2023-10-09T04:51:31.773984+0000 mon.a (mon.0) 1147 : audit [DBG] from='client.? 172.21.15.133:0/841589226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:33.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:32 smithi133 bash[27558]: audit 2023-10-09T04:51:31.773984+0000 mon.a (mon.0) 1147 : audit [DBG] from='client.? 172.21.15.133:0/841589226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:33.319 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Removed 113569 objects 2023-10-09T04:51:33.319 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Clean up completed and total clean up time :161.407 2023-10-09T04:51:33.887 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:33 smithi194 bash[82183]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-mgr-x 2023-10-09T04:51:33.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:33 smithi194 bash[31648]: cluster 2023-10-09T04:51:33.089969+0000 mgr.y (mgr.25046) 37 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 95 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 875 op/s 2023-10-09T04:51:34.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:33 smithi133 bash[41291]: cluster 2023-10-09T04:51:33.089969+0000 mgr.y (mgr.25046) 37 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 95 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 875 op/s 2023-10-09T04:51:34.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:33 smithi133 bash[27558]: cluster 2023-10-09T04:51:33.089969+0000 mgr.y (mgr.25046) 37 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 95 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 875 op/s 2023-10-09T04:51:34.200 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:33 smithi194 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-10-09T04:51:34.200 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:33 smithi194 bash[82245]: Error response from daemon: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-mgr-x 2023-10-09T04:51:34.200 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:33 smithi194 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-10-09T04:51:34.200 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:33 smithi194 systemd[1]: Stopped Ceph mgr.x for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:51:34.201 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:34 smithi194 systemd[1]: Started Ceph mgr.x for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:51:34.551 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:51:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:51:34] "GET /metrics HTTP/1.1" 200 36163 "" "Prometheus/2.43.0" 2023-10-09T04:51:35.493 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:35 smithi194 bash[82301]: debug 2023-10-09T04:51:35.325+0000 7f30ab154700 1 -- 172.21.15.194:0/928782783 <== mon.1 v2:172.21.15.194:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55632b60f1e0 con 0x55632c38e800 2023-10-09T04:51:35.493 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:35 smithi194 bash[31648]: audit 2023-10-09T04:51:34.239483+0000 mon.a (mon.0) 1148 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:35.494 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:35 smithi194 bash[31648]: audit 2023-10-09T04:51:34.247717+0000 mon.a (mon.0) 1149 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:35.494 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:35 smithi194 bash[31648]: audit 2023-10-09T04:51:34.259735+0000 mon.a (mon.0) 1150 : audit [DBG] from='client.? 172.21.15.133:0/566108339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:35.494 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:35 smithi194 bash[31648]: audit 2023-10-09T04:51:34.264638+0000 mon.c (mon.2) 89 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:51:35.494 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:35 smithi194 bash[31648]: cluster 2023-10-09T04:51:35.090728+0000 mgr.y (mgr.25046) 38 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 95 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-10-09T04:51:35.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:35 smithi133 bash[41291]: audit 2023-10-09T04:51:34.239483+0000 mon.a (mon.0) 1148 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:35.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:35 smithi133 bash[41291]: audit 2023-10-09T04:51:34.247717+0000 mon.a (mon.0) 1149 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:35.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:35 smithi133 bash[41291]: audit 2023-10-09T04:51:34.259735+0000 mon.a (mon.0) 1150 : audit [DBG] from='client.? 172.21.15.133:0/566108339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:35.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:35 smithi133 bash[41291]: audit 2023-10-09T04:51:34.264638+0000 mon.c (mon.2) 89 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:51:35.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:35 smithi133 bash[41291]: cluster 2023-10-09T04:51:35.090728+0000 mgr.y (mgr.25046) 38 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 95 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-10-09T04:51:35.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:35 smithi133 bash[27558]: audit 2023-10-09T04:51:34.239483+0000 mon.a (mon.0) 1148 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:35.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:35 smithi133 bash[27558]: audit 2023-10-09T04:51:34.247717+0000 mon.a (mon.0) 1149 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:35.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:35 smithi133 bash[27558]: audit 2023-10-09T04:51:34.259735+0000 mon.a (mon.0) 1150 : audit [DBG] from='client.? 172.21.15.133:0/566108339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:35.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:35 smithi133 bash[27558]: audit 2023-10-09T04:51:34.264638+0000 mon.c (mon.2) 89 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:51:35.553 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:35 smithi133 bash[27558]: cluster 2023-10-09T04:51:35.090728+0000 mgr.y (mgr.25046) 38 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 95 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-10-09T04:51:36.774 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:35 smithi194 bash[82301]: debug 2023-10-09T04:51:35.489+0000 7f30b52a2200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:51:36.774 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:36 smithi194 bash[82301]: debug 2023-10-09T04:51:36.441+0000 7f30b52a2200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:51:36.774 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:36 smithi194 bash[82301]: debug 2023-10-09T04:51:36.541+0000 7f30b52a2200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:51:36.774 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:36 smithi194 bash[82301]: debug 2023-10-09T04:51:36.649+0000 7f30b52a2200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:51:37.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:36 smithi133 bash[27558]: audit 2023-10-09T04:51:36.717484+0000 mon.c (mon.2) 90 : audit [DBG] from='client.? 172.21.15.133:0/427982236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:37.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:36 smithi133 bash[41291]: audit 2023-10-09T04:51:36.717484+0000 mon.c (mon.2) 90 : audit [DBG] from='client.? 172.21.15.133:0/427982236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:37.098 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2023-10-09T04:51:37.099 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2023-10-09T04:51:37.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:36 smithi194 bash[31648]: audit 2023-10-09T04:51:36.717484+0000 mon.c (mon.2) 90 : audit [DBG] from='client.? 172.21.15.133:0/427982236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:37.137 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:36 smithi194 bash[82301]: debug 2023-10-09T04:51:36.921+0000 7f30b52a2200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:51:37.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:37 smithi194 bash[82301]: debug 2023-10-09T04:51:37.285+0000 7f30b52a2200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:51:37.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:37 smithi194 bash[82301]: debug 2023-10-09T04:51:37.441+0000 7f30b52a2200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:51:38.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:37 smithi133 bash[27558]: cluster 2023-10-09T04:51:37.091969+0000 mgr.y (mgr.25046) 39 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-10-09T04:51:38.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:37 smithi133 bash[41291]: cluster 2023-10-09T04:51:37.091969+0000 mgr.y (mgr.25046) 39 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-10-09T04:51:38.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:37 smithi194 bash[31648]: cluster 2023-10-09T04:51:37.091969+0000 mgr.y (mgr.25046) 39 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-10-09T04:51:38.137 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:37 smithi194 bash[82301]: debug 2023-10-09T04:51:37.665+0000 7f30b52a2200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:51:38.914 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:38 smithi194 bash[82301]: debug 2023-10-09T04:51:38.681+0000 7f30b52a2200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:51:38.915 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:38 smithi194 bash[82301]: debug 2023-10-09T04:51:38.793+0000 7f30b52a2200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:51:38.915 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:38 smithi194 bash[81885]: ts=2023-10-09T04:51:38.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi194.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi133:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:51:39.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:38 smithi194 bash[82301]: debug 2023-10-09T04:51:38.909+0000 7f30b52a2200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:51:39.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:39 smithi194 bash[31648]: cluster 2023-10-09T04:51:39.092602+0000 mgr.y (mgr.25046) 40 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 414 op/s 2023-10-09T04:51:39.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:39 smithi133 bash[27558]: cluster 2023-10-09T04:51:39.092602+0000 mgr.y (mgr.25046) 40 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 414 op/s 2023-10-09T04:51:39.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:39 smithi133 bash[41291]: cluster 2023-10-09T04:51:39.092602+0000 mgr.y (mgr.25046) 40 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 414 op/s 2023-10-09T04:51:39.677 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:39 smithi194 bash[82301]: debug 2023-10-09T04:51:39.409+0000 7f30b52a2200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:51:39.677 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:39 smithi194 bash[82301]: debug 2023-10-09T04:51:39.673+0000 7f30b52a2200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:51:40.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:40 smithi133 bash[27558]: audit 2023-10-09T04:51:39.173079+0000 mon.c (mon.2) 91 : audit [DBG] from='client.? 172.21.15.133:0/688068654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:40.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:40 smithi133 bash[41291]: audit 2023-10-09T04:51:39.173079+0000 mon.c (mon.2) 91 : audit [DBG] from='client.? 172.21.15.133:0/688068654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:40.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:40 smithi194 bash[31648]: audit 2023-10-09T04:51:39.173079+0000 mon.c (mon.2) 91 : audit [DBG] from='client.? 172.21.15.133:0/688068654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:40.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:40 smithi194 bash[82301]: debug 2023-10-09T04:51:40.181+0000 7f30b52a2200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:51:41.137 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:40 smithi194 bash[82301]: debug 2023-10-09T04:51:40.769+0000 7f30b52a2200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:51:41.537 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:41 smithi194 bash[82301]: debug 2023-10-09T04:51:41.413+0000 7f30b52a2200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:51:41.537 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:41 smithi194 bash[82301]: debug 2023-10-09T04:51:41.533+0000 7f30b52a2200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:51:41.538 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:41 smithi194 bash[31648]: audit 2023-10-09T04:51:40.236614+0000 mon.c (mon.2) 92 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:41.538 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:41 smithi194 bash[31648]: audit 2023-10-09T04:51:40.663639+0000 mon.a (mon.0) 1151 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:41.538 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:41 smithi194 bash[31648]: audit 2023-10-09T04:51:40.673599+0000 mon.a (mon.0) 1152 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:41.538 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:41 smithi194 bash[31648]: cluster 2023-10-09T04:51:41.093348+0000 mgr.y (mgr.25046) 41 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 414 op/s 2023-10-09T04:51:41.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:41 smithi133 bash[27558]: audit 2023-10-09T04:51:40.236614+0000 mon.c (mon.2) 92 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:41.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:41 smithi133 bash[27558]: audit 2023-10-09T04:51:40.663639+0000 mon.a (mon.0) 1151 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:41.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:41 smithi133 bash[27558]: audit 2023-10-09T04:51:40.673599+0000 mon.a (mon.0) 1152 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:41.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:41 smithi133 bash[27558]: cluster 2023-10-09T04:51:41.093348+0000 mgr.y (mgr.25046) 41 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 414 op/s 2023-10-09T04:51:41.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:41 smithi133 bash[41291]: audit 2023-10-09T04:51:40.236614+0000 mon.c (mon.2) 92 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:41.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:41 smithi133 bash[41291]: audit 2023-10-09T04:51:40.663639+0000 mon.a (mon.0) 1151 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:41.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:41 smithi133 bash[41291]: audit 2023-10-09T04:51:40.673599+0000 mon.a (mon.0) 1152 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:41.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:41 smithi133 bash[41291]: cluster 2023-10-09T04:51:41.093348+0000 mgr.y (mgr.25046) 41 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 414 op/s 2023-10-09T04:51:41.879 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:51:42.213 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:41 smithi194 bash[82301]: debug 2023-10-09T04:51:41.925+0000 7f30b52a2200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:51:42.213 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:42 smithi194 bash[82301]: debug 2023-10-09T04:51:42.049+0000 7f30b52a2200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:51:42.474 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:42 smithi194 bash[82301]: debug 2023-10-09T04:51:42.209+0000 7f30b52a2200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:51:42.475 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:42 smithi194 bash[82301]: debug 2023-10-09T04:51:42.337+0000 7f30b52a2200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:51:42.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:42 smithi133 bash[27558]: audit 2023-10-09T04:51:41.471239+0000 mon.a (mon.0) 1153 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:42.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:42 smithi133 bash[27558]: audit 2023-10-09T04:51:41.479998+0000 mon.a (mon.0) 1154 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:42.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:42 smithi133 bash[27558]: audit 2023-10-09T04:51:41.633778+0000 mon.c (mon.2) 93 : audit [DBG] from='client.? 172.21.15.133:0/2607468210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:42.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:42 smithi133 bash[41291]: audit 2023-10-09T04:51:41.471239+0000 mon.a (mon.0) 1153 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:42.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:42 smithi133 bash[41291]: audit 2023-10-09T04:51:41.479998+0000 mon.a (mon.0) 1154 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:42.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:42 smithi133 bash[41291]: audit 2023-10-09T04:51:41.633778+0000 mon.c (mon.2) 93 : audit [DBG] from='client.? 172.21.15.133:0/2607468210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:42.824 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:42 smithi194 bash[31648]: audit 2023-10-09T04:51:41.471239+0000 mon.a (mon.0) 1153 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:42.824 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:42 smithi194 bash[31648]: audit 2023-10-09T04:51:41.479998+0000 mon.a (mon.0) 1154 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:42.824 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:42 smithi194 bash[31648]: audit 2023-10-09T04:51:41.633778+0000 mon.c (mon.2) 93 : audit [DBG] from='client.? 172.21.15.133:0/2607468210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:42.824 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:42 smithi194 bash[82301]: debug 2023-10-09T04:51:42.477+0000 7f30b52a2200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:51:42.824 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:42 smithi194 bash[82301]: debug 2023-10-09T04:51:42.685+0000 7f30b52a2200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:51:43.137 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:42 smithi194 bash[82301]: debug 2023-10-09T04:51:42.821+0000 7f30b52a2200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:51:43.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:43 smithi133 bash[27558]: cluster 2023-10-09T04:51:43.094536+0000 mgr.y (mgr.25046) 42 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 414 op/s 2023-10-09T04:51:43.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:43 smithi133 bash[41291]: cluster 2023-10-09T04:51:43.094536+0000 mgr.y (mgr.25046) 42 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 414 op/s 2023-10-09T04:51:43.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:43 smithi194 bash[31648]: cluster 2023-10-09T04:51:43.094536+0000 mgr.y (mgr.25046) 42 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 414 op/s 2023-10-09T04:51:44.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:44 smithi194 bash[82301]: debug 2023-10-09T04:51:44.080+0000 7f30b52a2200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:51:44.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:44 smithi194 bash[82301]: [09/Oct/2023:04:51:44] ENGINE Bus STARTING 2023-10-09T04:51:44.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:44 smithi194 bash[82301]: CherryPy Checker: 2023-10-09T04:51:44.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:44 smithi194 bash[82301]: The Application mounted at '' has an empty config. 2023-10-09T04:51:44.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:44 smithi194 bash[82301]: [09/Oct/2023:04:51:44] ENGINE Serving on http://:::9283 2023-10-09T04:51:44.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:51:44 smithi194 bash[82301]: [09/Oct/2023:04:51:44] ENGINE Bus STARTED 2023-10-09T04:51:44.551 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:51:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:51:44] "GET /metrics HTTP/1.1" 200 36117 "" "Prometheus/2.43.0" 2023-10-09T04:51:44.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:44 smithi133 bash[27558]: audit 2023-10-09T04:51:44.067805+0000 mon.c (mon.2) 94 : audit [DBG] from='client.? 172.21.15.133:0/1594549368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:44.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:44 smithi133 bash[27558]: cluster 2023-10-09T04:51:44.103825+0000 mon.a (mon.0) 1155 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T04:51:44.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:44 smithi133 bash[27558]: cluster 2023-10-09T04:51:44.104003+0000 mon.a (mon.0) 1156 : cluster [DBG] Standby manager daemon x started 2023-10-09T04:51:44.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:44 smithi133 bash[27558]: audit 2023-10-09T04:51:44.110214+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.? 172.21.15.194:0/472205933' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:51:44.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:44 smithi133 bash[27558]: audit 2023-10-09T04:51:44.112419+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.? 172.21.15.194:0/472205933' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:51:44.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:44 smithi133 bash[27558]: audit 2023-10-09T04:51:44.118565+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.? 172.21.15.194:0/472205933' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:51:44.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:44 smithi133 bash[27558]: audit 2023-10-09T04:51:44.119101+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.? 172.21.15.194:0/472205933' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:51:44.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:44 smithi133 bash[41291]: audit 2023-10-09T04:51:44.067805+0000 mon.c (mon.2) 94 : audit [DBG] from='client.? 172.21.15.133:0/1594549368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:44.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:44 smithi133 bash[41291]: cluster 2023-10-09T04:51:44.103825+0000 mon.a (mon.0) 1155 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T04:51:44.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:44 smithi133 bash[41291]: cluster 2023-10-09T04:51:44.104003+0000 mon.a (mon.0) 1156 : cluster [DBG] Standby manager daemon x started 2023-10-09T04:51:44.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:44 smithi133 bash[41291]: audit 2023-10-09T04:51:44.110214+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.? 172.21.15.194:0/472205933' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:51:44.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:44 smithi133 bash[41291]: audit 2023-10-09T04:51:44.112419+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.? 172.21.15.194:0/472205933' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:51:44.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:44 smithi133 bash[41291]: audit 2023-10-09T04:51:44.118565+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.? 172.21.15.194:0/472205933' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:51:44.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:44 smithi133 bash[41291]: audit 2023-10-09T04:51:44.119101+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.? 172.21.15.194:0/472205933' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:51:44.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:44 smithi194 bash[31648]: audit 2023-10-09T04:51:44.067805+0000 mon.c (mon.2) 94 : audit [DBG] from='client.? 172.21.15.133:0/1594549368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:44.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:44 smithi194 bash[31648]: cluster 2023-10-09T04:51:44.103825+0000 mon.a (mon.0) 1155 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T04:51:44.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:44 smithi194 bash[31648]: cluster 2023-10-09T04:51:44.104003+0000 mon.a (mon.0) 1156 : cluster [DBG] Standby manager daemon x started 2023-10-09T04:51:44.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:44 smithi194 bash[31648]: audit 2023-10-09T04:51:44.110214+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.? 172.21.15.194:0/472205933' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:51:44.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:44 smithi194 bash[31648]: audit 2023-10-09T04:51:44.112419+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.? 172.21.15.194:0/472205933' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:51:44.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:44 smithi194 bash[31648]: audit 2023-10-09T04:51:44.118565+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.? 172.21.15.194:0/472205933' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:51:44.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:44 smithi194 bash[31648]: audit 2023-10-09T04:51:44.119101+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.? 172.21.15.194:0/472205933' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:51:45.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:45 smithi133 bash[27558]: cluster 2023-10-09T04:51:44.495054+0000 mon.a (mon.0) 1157 : cluster [DBG] mgrmap e30: y(active, since 49s), standbys: x 2023-10-09T04:51:45.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:45 smithi133 bash[27558]: cluster 2023-10-09T04:51:45.095257+0000 mgr.y (mgr.25046) 43 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 125 op/s 2023-10-09T04:51:45.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:45 smithi133 bash[41291]: cluster 2023-10-09T04:51:44.495054+0000 mon.a (mon.0) 1157 : cluster [DBG] mgrmap e30: y(active, since 49s), standbys: x 2023-10-09T04:51:45.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:45 smithi133 bash[41291]: cluster 2023-10-09T04:51:45.095257+0000 mgr.y (mgr.25046) 43 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 125 op/s 2023-10-09T04:51:45.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:45 smithi194 bash[31648]: cluster 2023-10-09T04:51:44.495054+0000 mon.a (mon.0) 1157 : cluster [DBG] mgrmap e30: y(active, since 49s), standbys: x 2023-10-09T04:51:45.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:45 smithi194 bash[31648]: cluster 2023-10-09T04:51:45.095257+0000 mgr.y (mgr.25046) 43 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 125 op/s 2023-10-09T04:51:46.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:46 smithi194 bash[31648]: audit 2023-10-09T04:51:46.511020+0000 mon.c (mon.2) 95 : audit [DBG] from='client.? 172.21.15.133:0/884883763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:47.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:46 smithi133 bash[41291]: audit 2023-10-09T04:51:46.511020+0000 mon.c (mon.2) 95 : audit [DBG] from='client.? 172.21.15.133:0/884883763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:47.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:46 smithi133 bash[27558]: audit 2023-10-09T04:51:46.511020+0000 mon.c (mon.2) 95 : audit [DBG] from='client.? 172.21.15.133:0/884883763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:47.622 INFO:teuthology.orchestra.run.smithi133.stderr:pool 'unique_pool_0' removed 2023-10-09T04:51:47.861 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:47 smithi133 bash[27558]: audit 2023-10-09T04:51:47.004311+0000 mon.a (mon.0) 1158 : audit [INF] from='client.? 172.21.15.133:0/3180626617' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:51:47.861 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:47 smithi133 bash[27558]: cluster 2023-10-09T04:51:47.096339+0000 mgr.y (mgr.25046) 44 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 125 op/s 2023-10-09T04:51:47.862 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:47 smithi133 bash[41291]: audit 2023-10-09T04:51:47.004311+0000 mon.a (mon.0) 1158 : audit [INF] from='client.? 172.21.15.133:0/3180626617' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:51:47.862 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:47 smithi133 bash[41291]: cluster 2023-10-09T04:51:47.096339+0000 mgr.y (mgr.25046) 44 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 125 op/s 2023-10-09T04:51:47.883 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:47 smithi194 bash[31648]: audit 2023-10-09T04:51:47.004311+0000 mon.a (mon.0) 1158 : audit [INF] from='client.? 172.21.15.133:0/3180626617' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:51:47.883 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:47 smithi194 bash[31648]: cluster 2023-10-09T04:51:47.096339+0000 mgr.y (mgr.25046) 44 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 125 op/s 2023-10-09T04:51:48.526 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T04:51:48.526 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T04:51:48.527 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2023-10-09T04:51:48.527 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_1 16 2023-10-09T04:51:48.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:48 smithi133 bash[41291]: audit 2023-10-09T04:51:47.612560+0000 mon.a (mon.0) 1159 : audit [INF] from='client.? 172.21.15.133:0/3180626617' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:51:48.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:48 smithi133 bash[41291]: cluster 2023-10-09T04:51:47.612649+0000 mon.a (mon.0) 1160 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2023-10-09T04:51:48.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:48 smithi133 bash[27558]: audit 2023-10-09T04:51:47.612560+0000 mon.a (mon.0) 1159 : audit [INF] from='client.? 172.21.15.133:0/3180626617' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:51:48.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:48 smithi133 bash[27558]: cluster 2023-10-09T04:51:47.612649+0000 mon.a (mon.0) 1160 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2023-10-09T04:51:48.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:48 smithi194 bash[31648]: audit 2023-10-09T04:51:47.612560+0000 mon.a (mon.0) 1159 : audit [INF] from='client.? 172.21.15.133:0/3180626617' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:51:48.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:48 smithi194 bash[31648]: cluster 2023-10-09T04:51:47.612649+0000 mon.a (mon.0) 1160 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2023-10-09T04:51:48.887 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:48 smithi194 bash[81885]: ts=2023-10-09T04:51:48.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi194.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi133:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:51:49.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:49 smithi194 bash[31648]: audit 2023-10-09T04:51:48.991690+0000 mon.c (mon.2) 96 : audit [DBG] from='client.? 172.21.15.133:0/98423992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:49.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:49 smithi194 bash[31648]: cluster 2023-10-09T04:51:49.096993+0000 mgr.y (mgr.25046) 45 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:51:50.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:49 smithi133 bash[27558]: audit 2023-10-09T04:51:48.991690+0000 mon.c (mon.2) 96 : audit [DBG] from='client.? 172.21.15.133:0/98423992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:50.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:49 smithi133 bash[27558]: cluster 2023-10-09T04:51:49.096993+0000 mgr.y (mgr.25046) 45 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:51:50.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:49 smithi133 bash[41291]: audit 2023-10-09T04:51:48.991690+0000 mon.c (mon.2) 96 : audit [DBG] from='client.? 172.21.15.133:0/98423992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:50.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:49 smithi133 bash[41291]: cluster 2023-10-09T04:51:49.096993+0000 mgr.y (mgr.25046) 45 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:51:51.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:51 smithi194 bash[31648]: cluster 2023-10-09T04:51:51.097596+0000 mgr.y (mgr.25046) 46 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:51:51.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:51 smithi194 bash[31648]: audit 2023-10-09T04:51:51.449753+0000 mon.a (mon.0) 1161 : audit [DBG] from='client.? 172.21.15.133:0/2960755591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:52.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:51 smithi133 bash[27558]: cluster 2023-10-09T04:51:51.097596+0000 mgr.y (mgr.25046) 46 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:51:52.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:51 smithi133 bash[27558]: audit 2023-10-09T04:51:51.449753+0000 mon.a (mon.0) 1161 : audit [DBG] from='client.? 172.21.15.133:0/2960755591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:52.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:51 smithi133 bash[41291]: cluster 2023-10-09T04:51:51.097596+0000 mgr.y (mgr.25046) 46 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:51:52.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:51 smithi133 bash[41291]: audit 2023-10-09T04:51:51.449753+0000 mon.a (mon.0) 1161 : audit [DBG] from='client.? 172.21.15.133:0/2960755591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:53.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[27558]: audit 2023-10-09T04:51:51.780966+0000 mon.a (mon.0) 1162 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:53.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[27558]: audit 2023-10-09T04:51:51.791048+0000 mon.a (mon.0) 1163 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:53.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[27558]: audit 2023-10-09T04:51:51.793566+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:51:53.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[27558]: audit 2023-10-09T04:51:51.794671+0000 mon.c (mon.2) 98 : audit [INF] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:51:53.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[27558]: audit 2023-10-09T04:51:51.801906+0000 mon.a (mon.0) 1164 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:53.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[27558]: audit 2023-10-09T04:51:51.884069+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:51:53.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[27558]: audit 2023-10-09T04:51:51.889244+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:51:53.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[27558]: cephadm 2023-10-09T04:51:51.890538+0000 mgr.y (mgr.25046) 47 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-10-09T04:51:53.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[27558]: audit 2023-10-09T04:51:51.898560+0000 mon.a (mon.0) 1165 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:53.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[27558]: audit 2023-10-09T04:51:51.900141+0000 mon.c (mon.2) 101 : audit [INF] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T04:51:53.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[27558]: audit 2023-10-09T04:51:51.900678+0000 mon.a (mon.0) 1166 : audit [INF] from='mgr.25046 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T04:51:53.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[27558]: audit 2023-10-09T04:51:51.905055+0000 mon.a (mon.0) 1167 : audit [INF] from='mgr.25046 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T04:51:53.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[27558]: audit 2023-10-09T04:51:51.908720+0000 mon.c (mon.2) 102 : audit [INF] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T04:51:53.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[27558]: audit 2023-10-09T04:51:51.909230+0000 mon.a (mon.0) 1168 : audit [INF] from='mgr.25046 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T04:51:53.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[27558]: audit 2023-10-09T04:51:51.914474+0000 mon.a (mon.0) 1169 : audit [INF] from='mgr.25046 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T04:51:53.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[27558]: audit 2023-10-09T04:51:51.918403+0000 mon.c (mon.2) 103 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T04:51:53.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[27558]: audit 2023-10-09T04:51:51.919646+0000 mon.c (mon.2) 104 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T04:51:53.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[27558]: cephadm 2023-10-09T04:51:51.920589+0000 mgr.y (mgr.25046) 48 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-10-09T04:51:53.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[41291]: audit 2023-10-09T04:51:51.780966+0000 mon.a (mon.0) 1162 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:53.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[41291]: audit 2023-10-09T04:51:51.791048+0000 mon.a (mon.0) 1163 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:53.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[41291]: audit 2023-10-09T04:51:51.793566+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:51:53.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[41291]: audit 2023-10-09T04:51:51.794671+0000 mon.c (mon.2) 98 : audit [INF] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:51:53.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[41291]: audit 2023-10-09T04:51:51.801906+0000 mon.a (mon.0) 1164 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:53.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[41291]: audit 2023-10-09T04:51:51.884069+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:51:53.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[41291]: audit 2023-10-09T04:51:51.889244+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:51:53.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[41291]: cephadm 2023-10-09T04:51:51.890538+0000 mgr.y (mgr.25046) 47 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-10-09T04:51:53.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[41291]: audit 2023-10-09T04:51:51.898560+0000 mon.a (mon.0) 1165 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:53.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[41291]: audit 2023-10-09T04:51:51.900141+0000 mon.c (mon.2) 101 : audit [INF] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T04:51:53.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[41291]: audit 2023-10-09T04:51:51.900678+0000 mon.a (mon.0) 1166 : audit [INF] from='mgr.25046 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T04:51:53.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[41291]: audit 2023-10-09T04:51:51.905055+0000 mon.a (mon.0) 1167 : audit [INF] from='mgr.25046 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T04:51:53.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[41291]: audit 2023-10-09T04:51:51.908720+0000 mon.c (mon.2) 102 : audit [INF] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T04:51:53.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[41291]: audit 2023-10-09T04:51:51.909230+0000 mon.a (mon.0) 1168 : audit [INF] from='mgr.25046 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T04:51:53.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[41291]: audit 2023-10-09T04:51:51.914474+0000 mon.a (mon.0) 1169 : audit [INF] from='mgr.25046 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T04:51:53.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[41291]: audit 2023-10-09T04:51:51.918403+0000 mon.c (mon.2) 103 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T04:51:53.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[41291]: audit 2023-10-09T04:51:51.919646+0000 mon.c (mon.2) 104 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T04:51:53.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:52 smithi133 bash[41291]: cephadm 2023-10-09T04:51:51.920589+0000 mgr.y (mgr.25046) 48 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-10-09T04:51:53.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:52 smithi194 bash[31648]: audit 2023-10-09T04:51:51.780966+0000 mon.a (mon.0) 1162 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:53.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:52 smithi194 bash[31648]: audit 2023-10-09T04:51:51.791048+0000 mon.a (mon.0) 1163 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:53.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:52 smithi194 bash[31648]: audit 2023-10-09T04:51:51.793566+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:51:53.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:52 smithi194 bash[31648]: audit 2023-10-09T04:51:51.794671+0000 mon.c (mon.2) 98 : audit [INF] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:51:53.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:52 smithi194 bash[31648]: audit 2023-10-09T04:51:51.801906+0000 mon.a (mon.0) 1164 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:53.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:52 smithi194 bash[31648]: audit 2023-10-09T04:51:51.884069+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:51:53.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:52 smithi194 bash[31648]: audit 2023-10-09T04:51:51.889244+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:51:53.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:52 smithi194 bash[31648]: cephadm 2023-10-09T04:51:51.890538+0000 mgr.y (mgr.25046) 47 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-10-09T04:51:53.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:52 smithi194 bash[31648]: audit 2023-10-09T04:51:51.898560+0000 mon.a (mon.0) 1165 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:53.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:52 smithi194 bash[31648]: audit 2023-10-09T04:51:51.900141+0000 mon.c (mon.2) 101 : audit [INF] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T04:51:53.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:52 smithi194 bash[31648]: audit 2023-10-09T04:51:51.900678+0000 mon.a (mon.0) 1166 : audit [INF] from='mgr.25046 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T04:51:53.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:52 smithi194 bash[31648]: audit 2023-10-09T04:51:51.905055+0000 mon.a (mon.0) 1167 : audit [INF] from='mgr.25046 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T04:51:53.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:52 smithi194 bash[31648]: audit 2023-10-09T04:51:51.908720+0000 mon.c (mon.2) 102 : audit [INF] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T04:51:53.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:52 smithi194 bash[31648]: audit 2023-10-09T04:51:51.909230+0000 mon.a (mon.0) 1168 : audit [INF] from='mgr.25046 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T04:51:53.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:52 smithi194 bash[31648]: audit 2023-10-09T04:51:51.914474+0000 mon.a (mon.0) 1169 : audit [INF] from='mgr.25046 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T04:51:53.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:52 smithi194 bash[31648]: audit 2023-10-09T04:51:51.918403+0000 mon.c (mon.2) 103 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T04:51:53.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:52 smithi194 bash[31648]: audit 2023-10-09T04:51:51.919646+0000 mon.c (mon.2) 104 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T04:51:53.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:52 smithi194 bash[31648]: cephadm 2023-10-09T04:51:51.920589+0000 mgr.y (mgr.25046) 48 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-10-09T04:51:53.315 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:51:54.070 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:53 smithi133 bash[27558]: cluster 2023-10-09T04:51:53.098638+0000 mgr.y (mgr.25046) 49 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:51:54.070 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:53 smithi133 bash[41291]: cluster 2023-10-09T04:51:53.098638+0000 mgr.y (mgr.25046) 49 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:51:54.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:53 smithi194 bash[31648]: cluster 2023-10-09T04:51:53.098638+0000 mgr.y (mgr.25046) 49 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:51:54.551 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:51:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:51:54] "GET /metrics HTTP/1.1" 200 36117 "" "Prometheus/2.43.0" 2023-10-09T04:51:55.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:54 smithi133 bash[27558]: audit 2023-10-09T04:51:53.881909+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.133:0/1063646749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:55.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:54 smithi133 bash[41291]: audit 2023-10-09T04:51:53.881909+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.133:0/1063646749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:55.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:54 smithi194 bash[31648]: audit 2023-10-09T04:51:53.881909+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.133:0/1063646749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:56.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:55 smithi133 bash[27558]: cluster 2023-10-09T04:51:55.099353+0000 mgr.y (mgr.25046) 50 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:51:56.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:55 smithi133 bash[27558]: audit 2023-10-09T04:51:55.236369+0000 mon.c (mon.2) 106 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:56.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:55 smithi133 bash[41291]: cluster 2023-10-09T04:51:55.099353+0000 mgr.y (mgr.25046) 50 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:51:56.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:55 smithi133 bash[41291]: audit 2023-10-09T04:51:55.236369+0000 mon.c (mon.2) 106 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:56.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:55 smithi194 bash[31648]: cluster 2023-10-09T04:51:55.099353+0000 mgr.y (mgr.25046) 50 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:51:56.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:55 smithi194 bash[31648]: audit 2023-10-09T04:51:55.236369+0000 mon.c (mon.2) 106 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:57.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:56 smithi194 bash[31648]: cephadm 2023-10-09T04:51:55.851775+0000 mgr.y (mgr.25046) 51 : cephadm [INF] Upgrade: Updating mon.a 2023-10-09T04:51:57.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:56 smithi194 bash[31648]: audit 2023-10-09T04:51:55.859402+0000 mon.a (mon.0) 1170 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:57.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:56 smithi194 bash[31648]: audit 2023-10-09T04:51:55.862438+0000 mon.c (mon.2) 107 : audit [INF] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:51:57.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:56 smithi194 bash[31648]: audit 2023-10-09T04:51:55.863542+0000 mon.c (mon.2) 108 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:51:57.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:56 smithi194 bash[31648]: audit 2023-10-09T04:51:55.864968+0000 mon.c (mon.2) 109 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:51:57.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:56 smithi194 bash[31648]: cephadm 2023-10-09T04:51:55.866287+0000 mgr.y (mgr.25046) 52 : cephadm [INF] Deploying daemon mon.a on smithi133 2023-10-09T04:51:57.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:56 smithi194 bash[31648]: audit 2023-10-09T04:51:56.364466+0000 mon.c (mon.2) 110 : audit [DBG] from='client.? 172.21.15.133:0/3058350275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:57.186 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:56 smithi133 bash[41291]: cephadm 2023-10-09T04:51:55.851775+0000 mgr.y (mgr.25046) 51 : cephadm [INF] Upgrade: Updating mon.a 2023-10-09T04:51:57.186 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:56 smithi133 bash[41291]: audit 2023-10-09T04:51:55.859402+0000 mon.a (mon.0) 1170 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:57.186 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:56 smithi133 bash[41291]: audit 2023-10-09T04:51:55.862438+0000 mon.c (mon.2) 107 : audit [INF] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:51:57.186 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:56 smithi133 bash[41291]: audit 2023-10-09T04:51:55.863542+0000 mon.c (mon.2) 108 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:51:57.187 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:56 smithi133 bash[41291]: audit 2023-10-09T04:51:55.864968+0000 mon.c (mon.2) 109 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:51:57.187 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:56 smithi133 bash[41291]: cephadm 2023-10-09T04:51:55.866287+0000 mgr.y (mgr.25046) 52 : cephadm [INF] Deploying daemon mon.a on smithi133 2023-10-09T04:51:57.187 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:56 smithi133 bash[41291]: audit 2023-10-09T04:51:56.364466+0000 mon.c (mon.2) 110 : audit [DBG] from='client.? 172.21.15.133:0/3058350275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:57.187 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:56 smithi133 bash[27558]: cephadm 2023-10-09T04:51:55.851775+0000 mgr.y (mgr.25046) 51 : cephadm [INF] Upgrade: Updating mon.a 2023-10-09T04:51:57.187 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:56 smithi133 bash[27558]: audit 2023-10-09T04:51:55.859402+0000 mon.a (mon.0) 1170 : audit [INF] from='mgr.25046 ' entity='mgr.y' 2023-10-09T04:51:57.187 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:56 smithi133 bash[27558]: audit 2023-10-09T04:51:55.862438+0000 mon.c (mon.2) 107 : audit [INF] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:51:57.187 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:56 smithi133 bash[27558]: audit 2023-10-09T04:51:55.863542+0000 mon.c (mon.2) 108 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:51:57.187 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:56 smithi133 bash[27558]: audit 2023-10-09T04:51:55.864968+0000 mon.c (mon.2) 109 : audit [DBG] from='mgr.25046 172.21.15.133:0/4085512372' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:51:57.187 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:56 smithi133 bash[27558]: cephadm 2023-10-09T04:51:55.866287+0000 mgr.y (mgr.25046) 52 : cephadm [INF] Deploying daemon mon.a on smithi133 2023-10-09T04:51:57.187 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:56 smithi133 bash[27558]: audit 2023-10-09T04:51:56.364466+0000 mon.c (mon.2) 110 : audit [DBG] from='client.? 172.21.15.133:0/3058350275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:58.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:57 smithi194 bash[31648]: cluster 2023-10-09T04:51:57.100530+0000 mgr.y (mgr.25046) 53 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:51:58.329 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:57 smithi133 bash[27558]: cluster 2023-10-09T04:51:57.100530+0000 mgr.y (mgr.25046) 53 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:51:58.329 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:57 smithi133 bash[41291]: cluster 2023-10-09T04:51:57.100530+0000 mgr.y (mgr.25046) 53 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:51:59.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:58 smithi194 bash[31648]: audit 2023-10-09T04:51:58.807512+0000 mon.a (mon.0) 1171 : audit [DBG] from='client.? 172.21.15.133:0/416134199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:59.137 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:51:58 smithi194 bash[81885]: ts=2023-10-09T04:51:58.664Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi194.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi133:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:51:59.276 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:58 smithi133 bash[27558]: audit 2023-10-09T04:51:58.807512+0000 mon.a (mon.0) 1171 : audit [DBG] from='client.? 172.21.15.133:0/416134199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:51:59.276 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:58 smithi133 bash[41291]: audit 2023-10-09T04:51:58.807512+0000 mon.a (mon.0) 1171 : audit [DBG] from='client.? 172.21.15.133:0/416134199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:52:00.136 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:51:59 smithi133 bash[27558]: cluster 2023-10-09T04:51:59.101154+0000 mgr.y (mgr.25046) 54 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:52:00.137 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:51:59 smithi133 bash[41291]: cluster 2023-10-09T04:51:59.101154+0000 mgr.y (mgr.25046) 54 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:52:00.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:51:59 smithi194 bash[31648]: cluster 2023-10-09T04:51:59.101154+0000 mgr.y (mgr.25046) 54 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:52:00.705 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:00 smithi133 systemd[1]: Stopping Ceph mon.a for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T04:52:00.705 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:00 smithi133 bash[27558]: debug 2023-10-09T04:52:00.540+0000 7fa58a63b700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T04:52:00.705 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:00 smithi133 bash[27558]: debug 2023-10-09T04:52:00.540+0000 7fa58a63b700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-10-09T04:52:01.517 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:01 smithi133 bash[102712]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T04:52:01.517 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:01 smithi133 bash[102801]: Error response from daemon: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T04:52:01.517 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:01 smithi133 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mon.a.service: Succeeded. 2023-10-09T04:52:01.517 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:01 smithi133 systemd[1]: Stopped Ceph mon.a for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:52:01.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:01 smithi133 systemd[1]: Started Ceph mon.a for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:52:03.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.880+0000 7f5ec3507c80 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T04:52:03.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.880+0000 7f5ec3507c80 0 ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T04:52:03.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.880+0000 7f5ec3507c80 0 pidfile_write: ignore empty --pid-file 2023-10-09T04:52:03.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 0 load: jerasure load: lrc 2023-10-09T04:52:03.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: RocksDB version: 7.9.2 2023-10-09T04:52:03.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Git sha 0 2023-10-09T04:52:03.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T04:52:03.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: DB SUMMARY 2023-10-09T04:52:03.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: DB Session ID: 9UR5APMCBILTSLAX47RA 2023-10-09T04:52:03.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: CURRENT file: CURRENT 2023-10-09T04:52:03.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: IDENTITY file: IDENTITY 2023-10-09T04:52:03.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: MANIFEST file: MANIFEST-000015 size: 1947 Bytes 2023-10-09T04:52:03.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000045.sst 2023-10-09T04:52:03.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000043.log size: 799379 ; 2023-10-09T04:52:03.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.error_if_exists: 0 2023-10-09T04:52:03.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.create_if_missing: 0 2023-10-09T04:52:03.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T04:52:03.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T04:52:03.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T04:52:03.054 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T04:52:03.054 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.env: 0x55fde9d24720 2023-10-09T04:52:03.054 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.fs: PosixFileSystem 2023-10-09T04:52:03.054 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.info_log: 0x55fdecfd3480 2023-10-09T04:52:03.054 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T04:52:03.054 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.statistics: (nil) 2023-10-09T04:52:03.054 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.use_fsync: 0 2023-10-09T04:52:03.054 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T04:52:03.055 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T04:52:03.055 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T04:52:03.055 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T04:52:03.056 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T04:52:03.056 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T04:52:03.056 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T04:52:03.056 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T04:52:03.056 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T04:52:03.056 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T04:52:03.056 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T04:52:03.056 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.db_log_dir: 2023-10-09T04:52:03.056 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.wal_dir: 2023-10-09T04:52:03.056 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T04:52:03.056 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T04:52:03.057 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T04:52:03.057 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T04:52:03.057 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T04:52:03.057 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T04:52:03.057 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T04:52:03.057 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T04:52:03.057 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.write_buffer_manager: 0x55fdec260c80 2023-10-09T04:52:03.057 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T04:52:03.057 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T04:52:03.057 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T04:52:03.057 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T04:52:03.057 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T04:52:03.058 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T04:52:03.058 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T04:52:03.058 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T04:52:03.058 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.unordered_write: 0 2023-10-09T04:52:03.058 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T04:52:03.058 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T04:52:03.058 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T04:52:03.058 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T04:52:03.058 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.row_cache: None 2023-10-09T04:52:03.058 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.wal_filter: None 2023-10-09T04:52:03.058 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T04:52:03.058 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T04:52:03.059 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.two_write_queues: 0 2023-10-09T04:52:03.059 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T04:52:03.059 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.wal_compression: 0 2023-10-09T04:52:03.059 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.atomic_flush: 0 2023-10-09T04:52:03.059 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T04:52:03.059 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T04:52:03.059 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T04:52:03.059 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T04:52:03.059 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T04:52:03.059 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.best_efforts_recovery: 0 2023-10-09T04:52:03.059 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T04:52:03.059 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T04:52:03.059 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.allow_data_in_errors: 0 2023-10-09T04:52:03.060 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.db_host_id: __hostname__ 2023-10-09T04:52:03.060 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T04:52:03.060 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T04:52:03.060 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T04:52:03.060 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T04:52:03.060 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T04:52:03.060 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T04:52:03.060 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T04:52:03.060 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T04:52:03.060 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T04:52:03.060 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T04:52:03.060 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T04:52:03.061 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T04:52:03.061 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.max_open_files: -1 2023-10-09T04:52:03.061 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T04:52:03.061 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T04:52:03.061 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T04:52:03.061 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T04:52:03.061 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T04:52:03.061 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Compression algorithms supported: 2023-10-09T04:52:03.061 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T04:52:03.061 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: kZSTD supported: 0 2023-10-09T04:52:03.061 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: kXpressCompression supported: 0 2023-10-09T04:52:03.061 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T04:52:03.062 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: kZlibCompression supported: 1 2023-10-09T04:52:03.062 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T04:52:03.062 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T04:52:03.062 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T04:52:03.062 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T04:52:03.062 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T04:52:03.062 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2023-10-09T04:52:03.062 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T04:52:03.062 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T04:52:03.062 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.merge_operator: 2023-10-09T04:52:03.062 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.compaction_filter: None 2023-10-09T04:52:03.062 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T04:52:03.062 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.sst_partitioner_factory: None 2023-10-09T04:52:03.063 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T04:52:03.063 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T04:52:03.063 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55fdecfd3580) 2023-10-09T04:52:03.063 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: cache_index_and_filter_blocks: 1 2023-10-09T04:52:03.063 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T04:52:03.063 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T04:52:03.063 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: pin_top_level_index_and_filter: 1 2023-10-09T04:52:03.063 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: index_type: 0 2023-10-09T04:52:03.063 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: data_block_index_type: 0 2023-10-09T04:52:03.063 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: index_shortening: 1 2023-10-09T04:52:03.063 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: data_block_hash_table_util_ratio: 0.750000 2023-10-09T04:52:03.063 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: checksum: 4 2023-10-09T04:52:03.064 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: no_block_cache: 0 2023-10-09T04:52:03.064 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: block_cache: 0x55fdec2e31f0 2023-10-09T04:52:03.064 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: block_cache_name: BinnedLRUCache 2023-10-09T04:52:03.064 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: block_cache_options: 2023-10-09T04:52:03.064 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: capacity : 536870912 2023-10-09T04:52:03.064 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: num_shard_bits : 4 2023-10-09T04:52:03.064 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: strict_capacity_limit : 0 2023-10-09T04:52:03.064 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: high_pri_pool_ratio: 0.000 2023-10-09T04:52:03.064 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: block_cache_compressed: (nil) 2023-10-09T04:52:03.064 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: persistent_cache: (nil) 2023-10-09T04:52:03.064 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: block_size: 4096 2023-10-09T04:52:03.064 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: block_size_deviation: 10 2023-10-09T04:52:03.064 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: block_restart_interval: 16 2023-10-09T04:52:03.065 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: index_block_restart_interval: 1 2023-10-09T04:52:03.065 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: metadata_block_size: 4096 2023-10-09T04:52:03.065 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: partition_filters: 0 2023-10-09T04:52:03.065 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: use_delta_encoding: 1 2023-10-09T04:52:03.065 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: filter_policy: bloomfilter 2023-10-09T04:52:03.065 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: whole_key_filtering: 1 2023-10-09T04:52:03.065 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: verify_compression: 0 2023-10-09T04:52:03.065 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: read_amp_bytes_per_bit: 0 2023-10-09T04:52:03.065 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: format_version: 5 2023-10-09T04:52:03.065 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: enable_index_compression: 1 2023-10-09T04:52:03.065 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: block_align: 0 2023-10-09T04:52:03.065 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: max_auto_readahead_size: 262144 2023-10-09T04:52:03.066 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: prepopulate_block_cache: 0 2023-10-09T04:52:03.066 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: initial_auto_readahead_size: 8192 2023-10-09T04:52:03.066 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: num_file_reads_for_auto_readahead: 2 2023-10-09T04:52:03.066 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T04:52:03.066 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T04:52:03.066 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.compression: NoCompression 2023-10-09T04:52:03.066 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T04:52:03.066 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T04:52:03.066 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T04:52:03.066 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.num_levels: 7 2023-10-09T04:52:03.066 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T04:52:03.066 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T04:52:03.067 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T04:52:03.067 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T04:52:03.067 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T04:52:03.067 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T04:52:03.067 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T04:52:03.067 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:52:03.067 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T04:52:03.067 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T04:52:03.067 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T04:52:03.067 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T04:52:03.067 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T04:52:03.067 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T04:52:03.068 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T04:52:03.068 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T04:52:03.068 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:52:03.068 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T04:52:03.068 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T04:52:03.068 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T04:52:03.068 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T04:52:03.068 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T04:52:03.068 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T04:52:03.068 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T04:52:03.068 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T04:52:03.068 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T04:52:03.069 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T04:52:03.069 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T04:52:03.069 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T04:52:03.069 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T04:52:03.069 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T04:52:03.069 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T04:52:03.069 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T04:52:03.069 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T04:52:03.069 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T04:52:03.069 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T04:52:03.069 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T04:52:03.070 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T04:52:03.070 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T04:52:03.070 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.arena_block_size: 1048576 2023-10-09T04:52:03.070 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T04:52:03.070 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T04:52:03.070 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T04:52:03.070 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T04:52:03.070 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T04:52:03.070 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T04:52:03.070 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T04:52:03.071 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T04:52:03.071 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T04:52:03.071 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T04:52:03.071 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T04:52:03.071 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T04:52:03.072 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T04:52:03.072 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.table_properties_collectors: 2023-10-09T04:52:03.072 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T04:52:03.072 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T04:52:03.072 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T04:52:03.072 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T04:52:03.072 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T04:52:03.072 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.bloom_locality: 0 2023-10-09T04:52:03.072 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T04:52:03.072 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T04:52:03.072 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T04:52:03.072 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.force_consistency_checks: 1 2023-10-09T04:52:03.072 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T04:52:03.073 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.ttl: 2592000 2023-10-09T04:52:03.073 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T04:52:03.073 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T04:52:03.073 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T04:52:03.073 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.enable_blob_files: false 2023-10-09T04:52:03.073 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.min_blob_size: 0 2023-10-09T04:52:03.073 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.blob_file_size: 268435456 2023-10-09T04:52:03.073 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T04:52:03.073 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T04:52:03.073 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T04:52:03.073 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T04:52:03.073 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T04:52:03.073 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.blob_file_starting_level: 0 2023-10-09T04:52:03.074 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5ec3507c80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T04:52:03.074 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.884+0000 7f5eb8524700 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 45.sst 2023-10-09T04:52:03.074 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.888+0000 7f5ec3507c80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T04:52:03.074 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.888+0000 7f5ec3507c80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 47, last_sequence is 19383, log_number is 43,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T04:52:03.074 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.888+0000 7f5ec3507c80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 43 2023-10-09T04:52:03.074 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.888+0000 7f5ec3507c80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: f3f60ce5-6dcc-4737-bd6d-6fc3a9754473 2023-10-09T04:52:03.074 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.888+0000 7f5ec3507c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696827122889605, "job": 1, "event": "recovery_started", "wal_files": [43]} 2023-10-09T04:52:03.074 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.888+0000 7f5ec3507c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #43 mode 2 2023-10-09T04:52:03.074 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.892+0000 7f5ec3507c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696827122894201, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 48, "file_size": 743585, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 19375, "largest_seqno": 20023, "table_properties": {"data_size": 740201, "index_size": 1666, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 709, "raw_key_size": 6488, "raw_average_key_size": 23, "raw_value_size": 734241, "raw_average_value_size": 2622, "num_data_blocks": 75, "num_entries": 280, "num_filter_entries": 280, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696827122, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "f3f60ce5-6dcc-4737-bd6d-6fc3a9754473", "db_session_id": "9UR5APMCBILTSLAX47RA", "orig_file_number": 48, "seqno_to_time_mapping": "N/A"}} 2023-10-09T04:52:03.074 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.892+0000 7f5ec3507c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696827122894306, "job": 1, "event": "recovery_finished"} 2023-10-09T04:52:03.074 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.892+0000 7f5ec3507c80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 50 2023-10-09T04:52:03.074 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.892+0000 7f5ec3507c80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T04:52:03.075 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.892+0000 7f5ec3507c80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000043.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T04:52:03.075 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.892+0000 7f5ec3507c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55fdec384000 2023-10-09T04:52:03.075 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.892+0000 7f5ec3507c80 4 rocksdb: DB pointer 0x55fdec372000 2023-10-09T04:52:03.075 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.892+0000 7f5eb1d17700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T04:52:03.075 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.892+0000 7f5eb1d17700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T04:52:03.075 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: ** DB Stats ** 2023-10-09T04:52:03.075 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:52:03.075 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T04:52:03.075 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T04:52:03.075 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:52:03.075 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T04:52:03.075 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T04:52:03.075 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:52:03.076 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: ** Compaction Stats [default] ** 2023-10-09T04:52:03.076 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T04:52:03.076 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T04:52:03.076 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: L0 1/0 726.16 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 280.7 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-09T04:52:03.076 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: L6 1/0 6.45 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T04:52:03.076 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: Sum 2/0 7.16 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 280.7 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-09T04:52:03.076 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 280.7 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-09T04:52:03.076 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: ** Compaction Stats [default] ** 2023-10-09T04:52:03.076 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T04:52:03.076 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:52:03.076 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 280.7 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-09T04:52:03.076 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T04:52:03.076 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:52:03.076 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: Flush(GB): cumulative 0.001, interval 0.001 2023-10-09T04:52:03.077 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T04:52:03.077 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T04:52:03.077 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T04:52:03.077 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T04:52:03.077 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: Cumulative compaction: 0.00 GB write, 92.20 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:52:03.077 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: Interval compaction: 0.00 GB write, 92.20 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:52:03.077 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T04:52:03.077 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: Block cache BinnedLRUCache@0x55fdec2e31f0#7 capacity: 512.00 MB usage: 2.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 2.3e-05 secs_since: 0 2023-10-09T04:52:03.077 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: Block cache entry stats(count,size,portion): FilterBlock(1,0.78 KB,0.000149012%) IndexBlock(1,1.83 KB,0.000348687%) Misc(1,0.00 KB,0%) 2023-10-09T04:52:03.077 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T04:52:03.077 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.892+0000 7f5ec3507c80 0 starting mon.a rank 0 at public addrs [v2:172.21.15.133:3300/0,v1:172.21.15.133:6789/0] at bind addrs [v2:172.21.15.133:3300/0,v1:172.21.15.133:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 855b5206-665c-11ee-8db5-212e2dc638e7 2023-10-09T04:52:03.078 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.896+0000 7f5ec3507c80 1 mon.a@-1(???) e3 preinit fsid 855b5206-665c-11ee-8db5-212e2dc638e7 2023-10-09T04:52:03.078 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.896+0000 7f5ec3507c80 0 mon.a@-1(???).mds e10 new map 2023-10-09T04:52:03.078 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.896+0000 7f5ec3507c80 0 mon.a@-1(???).mds e10 print_map 2023-10-09T04:52:03.078 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: e10 2023-10-09T04:52:03.078 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T04:52:03.078 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T04:52:03.078 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: legacy client fscid: 1 2023-10-09T04:52:03.078 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: 2023-10-09T04:52:03.078 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: Filesystem 'foo' (1) 2023-10-09T04:52:03.078 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: fs_name foo 2023-10-09T04:52:03.078 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: epoch 10 2023-10-09T04:52:03.078 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T04:52:03.079 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: created 2023-10-09T04:44:52.764614+0000 2023-10-09T04:52:03.079 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: modified 2023-10-09T04:45:03.032645+0000 2023-10-09T04:52:03.079 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: tableserver 0 2023-10-09T04:52:03.079 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: root 0 2023-10-09T04:52:03.079 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: session_timeout 60 2023-10-09T04:52:03.079 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: session_autoclose 300 2023-10-09T04:52:03.079 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: max_file_size 1099511627776 2023-10-09T04:52:03.079 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: required_client_features {} 2023-10-09T04:52:03.079 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: last_failure 0 2023-10-09T04:52:03.079 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: last_failure_osd_epoch 69 2023-10-09T04:52:03.079 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T04:52:03.079 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: max_mds 1 2023-10-09T04:52:03.080 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: in 0 2023-10-09T04:52:03.080 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: up {0=24502} 2023-10-09T04:52:03.080 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: failed 2023-10-09T04:52:03.080 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: damaged 2023-10-09T04:52:03.080 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: stopped 2023-10-09T04:52:03.080 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: data_pools [4] 2023-10-09T04:52:03.080 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: metadata_pool 3 2023-10-09T04:52:03.080 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: inline_data disabled 2023-10-09T04:52:03.080 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: balancer 2023-10-09T04:52:03.080 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: bal_rank_mask -1 2023-10-09T04:52:03.080 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: standby_count_wanted 1 2023-10-09T04:52:03.080 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: [mds.foo.smithi194.yxnfqn{0:24502} state up:active seq 4 join_fscid=1 addr [v2:172.21.15.194:6832/1679470116,v1:172.21.15.194:6833/1679470116] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T04:52:03.081 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: 2023-10-09T04:52:03.081 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: 2023-10-09T04:52:03.081 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: Standby daemons: 2023-10-09T04:52:03.081 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: 2023-10-09T04:52:03.081 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: [mds.foo.smithi133.tsuabh{-1:14682} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.133:6834/3356898887,v1:172.21.15.133:6835/3356898887] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T04:52:03.081 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.896+0000 7f5ec3507c80 0 mon.a@-1(???).osd e78 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T04:52:03.081 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.896+0000 7f5ec3507c80 0 mon.a@-1(???).osd e78 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T04:52:03.081 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.896+0000 7f5ec3507c80 0 mon.a@-1(???).osd e78 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T04:52:03.081 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.896+0000 7f5ec3507c80 0 mon.a@-1(???).osd e78 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T04:52:03.081 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:02 smithi133 bash[102858]: debug 2023-10-09T04:52:02.896+0000 7f5ec3507c80 1 mon.a@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2023-10-09T04:52:04.551 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:52:04] "GET /metrics HTTP/1.1" 200 33813 "" "Prometheus/2.43.0" 2023-10-09T04:52:04.860 INFO:teuthology.orchestra.run.smithi133.stderr:pool 'unique_pool_1' created 2023-10-09T04:52:05.083 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:04 smithi133 bash[91128]: ignoring --setuser ceph since I am not root 2023-10-09T04:52:05.083 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:04 smithi133 bash[91128]: ignoring --setgroup ceph since I am not root 2023-10-09T04:52:05.083 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:04 smithi133 bash[91128]: debug 2023-10-09T04:52:04.868+0000 7ff49b905700 1 -- 172.21.15.133:0/4121107444 <== mon.0 v2:172.21.15.133:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5573fe0991e0 con 0x5573fee18c00 2023-10-09T04:52:05.084 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:05 smithi133 bash[91128]: debug 2023-10-09T04:52:05.012+0000 7ff4a5635200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:52:05.084 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:04 smithi133 bash[41291]: audit 2023-10-09T04:52:00.552121+0000 mon.c (mon.2) 111 : audit [INF] from='client.? 172.21.15.133:0/799734787' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T04:52:05.084 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:04 smithi133 bash[41291]: cluster 2023-10-09T04:52:01.101757+0000 mgr.y (mgr.25046) 55 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:52:05.084 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:04 smithi133 bash[41291]: audit 2023-10-09T04:52:01.246445+0000 mon.c (mon.2) 112 : audit [DBG] from='client.? 172.21.15.133:0/3867838143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:52:05.084 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:04 smithi133 bash[41291]: cluster 2023-10-09T04:52:03.102759+0000 mgr.y (mgr.25046) 56 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:52:05.084 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:04 smithi133 bash[41291]: audit 2023-10-09T04:52:03.701063+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.133:0/2786309958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:52:05.084 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:04 smithi133 bash[41291]: cluster 2023-10-09T04:52:04.320883+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2023-10-09T04:52:05.084 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:04 smithi133 bash[41291]: cluster 2023-10-09T04:52:04.343600+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:52:05.085 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:04 smithi133 bash[41291]: audit 2023-10-09T04:52:04.346289+0000 mon.a (mon.0) 3 : audit [INF] from='client.? ' entity='' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T04:52:05.085 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:04 smithi133 bash[41291]: debug 2023-10-09T04:52:04.840+0000 7f5ee1550700 -1 mon.c@2(peon).osd e79 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T04:52:05.085 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:04 smithi133 bash[102858]: audit 2023-10-09T04:52:00.552121+0000 mon.c (mon.2) 111 : audit 1 from='client.? 172.21.15.133:0/799734787' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T04:52:05.085 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:04 smithi133 bash[102858]: cluster 2023-10-09T04:52:01.101757+0000 mgr.y (mgr.25046) 55 : cluster 0 pgmap v37: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:52:05.085 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:04 smithi133 bash[102858]: audit 2023-10-09T04:52:01.246445+0000 mon.c (mon.2) 112 : audit 0 from='client.? 172.21.15.133:0/3867838143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:52:05.085 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:04 smithi133 bash[102858]: cluster 2023-10-09T04:52:03.102759+0000 mgr.y (mgr.25046) 56 : cluster 0 pgmap v38: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:52:05.085 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:04 smithi133 bash[102858]: audit 2023-10-09T04:52:03.701063+0000 mon.c (mon.2) 113 : audit 0 from='client.? 172.21.15.133:0/2786309958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:52:05.085 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:04 smithi133 bash[102858]: cluster 2023-10-09T04:52:04.320883+0000 mon.a (mon.0) 1 : cluster 1 mon.a calling monitor election 2023-10-09T04:52:05.085 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:04 smithi133 bash[102858]: cluster 2023-10-09T04:52:04.343600+0000 mon.a (mon.0) 2 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:52:05.086 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:04 smithi133 bash[102858]: audit 2023-10-09T04:52:04.346289+0000 mon.a (mon.0) 3 : audit 1 from='client.? ' entity='' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T04:52:05.137 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:04 smithi194 bash[82301]: ignoring --setuser ceph since I am not root 2023-10-09T04:52:05.137 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:04 smithi194 bash[82301]: ignoring --setgroup ceph since I am not root 2023-10-09T04:52:05.137 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:04 smithi194 bash[82301]: debug 2023-10-09T04:52:04.872+0000 7ff3d3e12700 1 -- 172.21.15.194:0/3437546964 <== mon.1 v2:172.21.15.194:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f721abb1e0 con 0x55f72283a800 2023-10-09T04:52:05.137 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:05 smithi194 bash[82301]: debug 2023-10-09T04:52:05.028+0000 7ff3ddb42200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:52:05.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:04 smithi194 bash[31648]: audit 2023-10-09T04:52:00.552121+0000 mon.c (mon.2) 111 : audit [INF] from='client.? 172.21.15.133:0/799734787' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T04:52:05.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:04 smithi194 bash[31648]: cluster 2023-10-09T04:52:01.101757+0000 mgr.y (mgr.25046) 55 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:52:05.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:04 smithi194 bash[31648]: audit 2023-10-09T04:52:01.246445+0000 mon.c (mon.2) 112 : audit [DBG] from='client.? 172.21.15.133:0/3867838143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:52:05.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:04 smithi194 bash[31648]: cluster 2023-10-09T04:52:03.102759+0000 mgr.y (mgr.25046) 56 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:52:05.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:04 smithi194 bash[31648]: audit 2023-10-09T04:52:03.701063+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.133:0/2786309958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:52:05.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:04 smithi194 bash[31648]: cluster 2023-10-09T04:52:04.320883+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2023-10-09T04:52:05.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:04 smithi194 bash[31648]: cluster 2023-10-09T04:52:04.343600+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:52:05.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:04 smithi194 bash[31648]: audit 2023-10-09T04:52:04.346289+0000 mon.a (mon.0) 3 : audit [INF] from='client.? ' entity='' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T04:52:05.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:04 smithi194 bash[31648]: debug 2023-10-09T04:52:04.840+0000 7f7a49f0f700 -1 mon.b@1(peon).osd e79 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T04:52:05.810 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2023-10-09T04:52:06.056 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:05 smithi194 bash[82301]: debug 2023-10-09T04:52:05.948+0000 7ff3ddb42200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:52:06.057 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:05 smithi194 bash[31648]: cluster 2023-10-09T04:52:04.351917+0000 mon.a (mon.0) 4 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.133:3300/0,v1:172.21.15.133:6789/0],b=[v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0],c=[v2:172.21.15.133:3301/0,v1:172.21.15.133:6790/0]} removed_ranks: {} 2023-10-09T04:52:06.057 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:05 smithi194 bash[31648]: cluster 2023-10-09T04:52:04.352592+0000 mon.a (mon.0) 5 : cluster [DBG] fsmap foo:1 {0=foo.smithi194.yxnfqn=up:active} 1 up:standby 2023-10-09T04:52:06.057 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:05 smithi194 bash[31648]: cluster 2023-10-09T04:52:04.352666+0000 mon.a (mon.0) 6 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2023-10-09T04:52:06.057 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:05 smithi194 bash[31648]: cluster 2023-10-09T04:52:04.354129+0000 mon.a (mon.0) 7 : cluster [DBG] mgrmap e30: y(active, since 69s), standbys: x 2023-10-09T04:52:06.057 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:05 smithi194 bash[31648]: cluster 2023-10-09T04:52:04.805191+0000 mon.a (mon.0) 8 : cluster [DBG] mgrmap e31: y(active, since 69s), standbys: x 2023-10-09T04:52:06.057 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:05 smithi194 bash[31648]: cluster 2023-10-09T04:52:04.805244+0000 mon.a (mon.0) 9 : cluster [INF] overall HEALTH_OK 2023-10-09T04:52:06.057 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:05 smithi194 bash[31648]: audit 2023-10-09T04:52:04.805395+0000 mon.a (mon.0) 10 : audit [INF] from='mgr.25046 ' entity='' 2023-10-09T04:52:06.057 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:05 smithi194 bash[31648]: audit 2023-10-09T04:52:04.844689+0000 mon.a (mon.0) 11 : audit [INF] from='client.? ' entity='' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T04:52:06.058 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:05 smithi194 bash[31648]: cluster 2023-10-09T04:52:04.844770+0000 mon.a (mon.0) 12 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2023-10-09T04:52:06.058 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:05 smithi194 bash[31648]: cluster 2023-10-09T04:52:04.850608+0000 osd.2 (osd.2) 3 : cluster [WRN] failed to encode map e79 with expected crc 2023-10-09T04:52:06.058 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:05 smithi194 bash[31648]: cluster 2023-10-09T04:52:04.850822+0000 osd.4 (osd.4) 3 : cluster [WRN] failed to encode map e79 with expected crc 2023-10-09T04:52:06.058 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:05 smithi194 bash[31648]: debug 2023-10-09T04:52:05.856+0000 7f7a49f0f700 -1 mon.b@1(peon).osd e80 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T04:52:06.158 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:05 smithi133 bash[91128]: debug 2023-10-09T04:52:05.932+0000 7ff4a5635200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:52:06.158 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:06 smithi133 bash[91128]: debug 2023-10-09T04:52:06.040+0000 7ff4a5635200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:52:06.158 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:05 smithi133 bash[102858]: cluster 2023-10-09T04:52:04.351917+0000 mon.a (mon.0) 4 : cluster 0 monmap e3: 3 mons at {a=[v2:172.21.15.133:3300/0,v1:172.21.15.133:6789/0],b=[v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0],c=[v2:172.21.15.133:3301/0,v1:172.21.15.133:6790/0]} removed_ranks: {} 2023-10-09T04:52:06.159 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:05 smithi133 bash[102858]: cluster 2023-10-09T04:52:04.352592+0000 mon.a (mon.0) 5 : cluster 0 fsmap foo:1 {0=foo.smithi194.yxnfqn=up:active} 1 up:standby 2023-10-09T04:52:06.159 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:05 smithi133 bash[102858]: cluster 2023-10-09T04:52:04.352666+0000 mon.a (mon.0) 6 : cluster 0 osdmap e78: 8 total, 8 up, 8 in 2023-10-09T04:52:06.159 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:05 smithi133 bash[102858]: cluster 2023-10-09T04:52:04.354129+0000 mon.a (mon.0) 7 : cluster 0 mgrmap e30: y(active, since 69s), standbys: x 2023-10-09T04:52:06.159 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:05 smithi133 bash[102858]: cluster 2023-10-09T04:52:04.805191+0000 mon.a (mon.0) 8 : cluster 0 mgrmap e31: y(active, since 69s), standbys: x 2023-10-09T04:52:06.159 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:05 smithi133 bash[102858]: cluster 2023-10-09T04:52:04.805244+0000 mon.a (mon.0) 9 : cluster 1 overall HEALTH_OK 2023-10-09T04:52:06.159 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:05 smithi133 bash[102858]: audit 2023-10-09T04:52:04.805395+0000 mon.a (mon.0) 10 : audit 1 from='mgr.25046 ' entity='' 2023-10-09T04:52:06.159 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:05 smithi133 bash[102858]: audit 2023-10-09T04:52:04.844689+0000 mon.a (mon.0) 11 : audit 1 from='client.? ' entity='' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T04:52:06.159 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:05 smithi133 bash[102858]: cluster 2023-10-09T04:52:04.844770+0000 mon.a (mon.0) 12 : cluster 0 osdmap e79: 8 total, 8 up, 8 in 2023-10-09T04:52:06.159 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:05 smithi133 bash[102858]: cluster 2023-10-09T04:52:04.850608+0000 osd.2 (osd.2) 3 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T04:52:06.159 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:05 smithi133 bash[102858]: cluster 2023-10-09T04:52:04.850822+0000 osd.4 (osd.4) 3 : cluster 3 failed to encode map e79 with expected crc 2023-10-09T04:52:06.160 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:05 smithi133 bash[41291]: cluster 2023-10-09T04:52:04.351917+0000 mon.a (mon.0) 4 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.133:3300/0,v1:172.21.15.133:6789/0],b=[v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0],c=[v2:172.21.15.133:3301/0,v1:172.21.15.133:6790/0]} removed_ranks: {} 2023-10-09T04:52:06.160 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:05 smithi133 bash[41291]: cluster 2023-10-09T04:52:04.352592+0000 mon.a (mon.0) 5 : cluster [DBG] fsmap foo:1 {0=foo.smithi194.yxnfqn=up:active} 1 up:standby 2023-10-09T04:52:06.160 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:05 smithi133 bash[41291]: cluster 2023-10-09T04:52:04.352666+0000 mon.a (mon.0) 6 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2023-10-09T04:52:06.160 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:05 smithi133 bash[41291]: cluster 2023-10-09T04:52:04.354129+0000 mon.a (mon.0) 7 : cluster [DBG] mgrmap e30: y(active, since 69s), standbys: x 2023-10-09T04:52:06.160 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:05 smithi133 bash[41291]: cluster 2023-10-09T04:52:04.805191+0000 mon.a (mon.0) 8 : cluster [DBG] mgrmap e31: y(active, since 69s), standbys: x 2023-10-09T04:52:06.160 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:05 smithi133 bash[41291]: cluster 2023-10-09T04:52:04.805244+0000 mon.a (mon.0) 9 : cluster [INF] overall HEALTH_OK 2023-10-09T04:52:06.160 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:05 smithi133 bash[41291]: audit 2023-10-09T04:52:04.805395+0000 mon.a (mon.0) 10 : audit [INF] from='mgr.25046 ' entity='' 2023-10-09T04:52:06.160 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:05 smithi133 bash[41291]: audit 2023-10-09T04:52:04.844689+0000 mon.a (mon.0) 11 : audit [INF] from='client.? ' entity='' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T04:52:06.160 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:05 smithi133 bash[41291]: cluster 2023-10-09T04:52:04.844770+0000 mon.a (mon.0) 12 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2023-10-09T04:52:06.160 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:05 smithi133 bash[41291]: cluster 2023-10-09T04:52:04.850608+0000 osd.2 (osd.2) 3 : cluster [WRN] failed to encode map e79 with expected crc 2023-10-09T04:52:06.161 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:05 smithi133 bash[41291]: cluster 2023-10-09T04:52:04.850822+0000 osd.4 (osd.4) 3 : cluster [WRN] failed to encode map e79 with expected crc 2023-10-09T04:52:06.161 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:05 smithi133 bash[41291]: debug 2023-10-09T04:52:05.856+0000 7f5ee1550700 -1 mon.c@2(peon).osd e80 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T04:52:06.207 INFO:teuthology.orchestra.run.smithi133.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 1 2023-10-09T04:52:06.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:06 smithi194 bash[82301]: debug 2023-10-09T04:52:06.052+0000 7ff3ddb42200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:52:06.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:06 smithi194 bash[82301]: debug 2023-10-09T04:52:06.152+0000 7ff3ddb42200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:52:06.444 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:06 smithi133 bash[91128]: debug 2023-10-09T04:52:06.152+0000 7ff4a5635200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:52:06.787 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:06 smithi194 bash[82301]: debug 2023-10-09T04:52:06.424+0000 7ff3ddb42200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:52:06.788 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:06 smithi133 bash[91128]: debug 2023-10-09T04:52:06.436+0000 7ff4a5635200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:52:06.789 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:06 smithi133 bash[91128]: debug 2023-10-09T04:52:06.780+0000 7ff4a5635200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:52:07.137 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:06 smithi194 bash[82301]: debug 2023-10-09T04:52:06.780+0000 7ff3ddb42200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:52:07.137 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:06 smithi194 bash[82301]: debug 2023-10-09T04:52:06.944+0000 7ff3ddb42200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:52:07.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:06 smithi194 bash[31648]: debug 2023-10-09T04:52:06.860+0000 7f7a49f0f700 -1 mon.b@1(peon).osd e81 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T04:52:07.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:06 smithi194 bash[31648]: cluster 2023-10-09T04:52:05.861779+0000 mon.a (mon.0) 13 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2023-10-09T04:52:07.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:06 smithi194 bash[31648]: cluster 2023-10-09T04:52:05.866674+0000 osd.1 (osd.1) 3 : cluster [WRN] failed to encode map e80 with expected crc 2023-10-09T04:52:07.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:06 smithi194 bash[31648]: cluster 2023-10-09T04:52:05.868510+0000 osd.4 (osd.4) 4 : cluster [WRN] failed to encode map e80 with expected crc 2023-10-09T04:52:07.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:06 smithi194 bash[31648]: cluster 2023-10-09T04:52:05.869450+0000 osd.2 (osd.2) 4 : cluster [WRN] failed to encode map e80 with expected crc 2023-10-09T04:52:07.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:06 smithi194 bash[31648]: cluster 2023-10-09T04:52:05.869548+0000 osd.0 (osd.0) 3 : cluster [WRN] failed to encode map e80 with expected crc 2023-10-09T04:52:07.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:06 smithi194 bash[31648]: cluster 2023-10-09T04:52:05.869643+0000 osd.1 (osd.1) 4 : cluster [WRN] failed to encode map e80 with expected crc 2023-10-09T04:52:07.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:06 smithi194 bash[31648]: cluster 2023-10-09T04:52:05.869764+0000 osd.3 (osd.3) 3 : cluster [WRN] failed to encode map e80 with expected crc 2023-10-09T04:52:07.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:06 smithi194 bash[31648]: cluster 2023-10-09T04:52:05.870779+0000 osd.7 (osd.7) 3 : cluster [WRN] failed to encode map e80 with expected crc 2023-10-09T04:52:07.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:06 smithi194 bash[31648]: cluster 2023-10-09T04:52:05.870782+0000 osd.5 (osd.5) 3 : cluster [WRN] failed to encode map e80 with expected crc 2023-10-09T04:52:07.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:06 smithi194 bash[31648]: cluster 2023-10-09T04:52:05.871354+0000 osd.4 (osd.4) 5 : cluster [WRN] failed to encode map e80 with expected crc 2023-10-09T04:52:07.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:06 smithi194 bash[31648]: audit 2023-10-09T04:52:06.185595+0000 mon.c (mon.2) 114 : audit [DBG] from='client.? 172.21.15.133:0/1087246646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:52:07.172 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:06 smithi133 bash[91128]: debug 2023-10-09T04:52:06.948+0000 7ff4a5635200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:52:07.172 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:06 smithi133 bash[41291]: debug 2023-10-09T04:52:06.856+0000 7f5ee1550700 -1 mon.c@2(peon).osd e81 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T04:52:07.172 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:06 smithi133 bash[41291]: cluster 2023-10-09T04:52:05.861779+0000 mon.a (mon.0) 13 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2023-10-09T04:52:07.172 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:06 smithi133 bash[41291]: cluster 2023-10-09T04:52:05.866674+0000 osd.1 (osd.1) 3 : cluster [WRN] failed to encode map e80 with expected crc 2023-10-09T04:52:07.172 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:06 smithi133 bash[41291]: cluster 2023-10-09T04:52:05.868510+0000 osd.4 (osd.4) 4 : cluster [WRN] failed to encode map e80 with expected crc 2023-10-09T04:52:07.172 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:06 smithi133 bash[41291]: cluster 2023-10-09T04:52:05.869450+0000 osd.2 (osd.2) 4 : cluster [WRN] failed to encode map e80 with expected crc 2023-10-09T04:52:07.173 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:06 smithi133 bash[41291]: cluster 2023-10-09T04:52:05.869548+0000 osd.0 (osd.0) 3 : cluster [WRN] failed to encode map e80 with expected crc 2023-10-09T04:52:07.173 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:06 smithi133 bash[41291]: cluster 2023-10-09T04:52:05.869643+0000 osd.1 (osd.1) 4 : cluster [WRN] failed to encode map e80 with expected crc 2023-10-09T04:52:07.173 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:06 smithi133 bash[41291]: cluster 2023-10-09T04:52:05.869764+0000 osd.3 (osd.3) 3 : cluster [WRN] failed to encode map e80 with expected crc 2023-10-09T04:52:07.173 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:06 smithi133 bash[41291]: cluster 2023-10-09T04:52:05.870779+0000 osd.7 (osd.7) 3 : cluster [WRN] failed to encode map e80 with expected crc 2023-10-09T04:52:07.173 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:06 smithi133 bash[41291]: cluster 2023-10-09T04:52:05.870782+0000 osd.5 (osd.5) 3 : cluster [WRN] failed to encode map e80 with expected crc 2023-10-09T04:52:07.173 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:06 smithi133 bash[41291]: cluster 2023-10-09T04:52:05.871354+0000 osd.4 (osd.4) 5 : cluster [WRN] failed to encode map e80 with expected crc 2023-10-09T04:52:07.173 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:06 smithi133 bash[41291]: audit 2023-10-09T04:52:06.185595+0000 mon.c (mon.2) 114 : audit [DBG] from='client.? 172.21.15.133:0/1087246646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:52:07.173 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:06 smithi133 bash[102858]: cluster 2023-10-09T04:52:05.861779+0000 mon.a (mon.0) 13 : cluster 0 osdmap e80: 8 total, 8 up, 8 in 2023-10-09T04:52:07.173 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:06 smithi133 bash[102858]: cluster 2023-10-09T04:52:05.866674+0000 osd.1 (osd.1) 3 : cluster 3 failed to encode map e80 with expected crc 2023-10-09T04:52:07.173 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:06 smithi133 bash[102858]: cluster 2023-10-09T04:52:05.868510+0000 osd.4 (osd.4) 4 : cluster 3 failed to encode map e80 with expected crc 2023-10-09T04:52:07.173 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:06 smithi133 bash[102858]: cluster 2023-10-09T04:52:05.869450+0000 osd.2 (osd.2) 4 : cluster 3 failed to encode map e80 with expected crc 2023-10-09T04:52:07.174 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:06 smithi133 bash[102858]: cluster 2023-10-09T04:52:05.869548+0000 osd.0 (osd.0) 3 : cluster 3 failed to encode map e80 with expected crc 2023-10-09T04:52:07.174 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:06 smithi133 bash[102858]: cluster 2023-10-09T04:52:05.869643+0000 osd.1 (osd.1) 4 : cluster 3 failed to encode map e80 with expected crc 2023-10-09T04:52:07.174 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:06 smithi133 bash[102858]: cluster 2023-10-09T04:52:05.869764+0000 osd.3 (osd.3) 3 : cluster 3 failed to encode map e80 with expected crc 2023-10-09T04:52:07.174 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:06 smithi133 bash[102858]: cluster 2023-10-09T04:52:05.870779+0000 osd.7 (osd.7) 3 : cluster 3 failed to encode map e80 with expected crc 2023-10-09T04:52:07.174 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:06 smithi133 bash[102858]: cluster 2023-10-09T04:52:05.870782+0000 osd.5 (osd.5) 3 : cluster 3 failed to encode map e80 with expected crc 2023-10-09T04:52:07.174 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:06 smithi133 bash[102858]: cluster 2023-10-09T04:52:05.871354+0000 osd.4 (osd.4) 5 : cluster 3 failed to encode map e80 with expected crc 2023-10-09T04:52:07.174 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:06 smithi133 bash[102858]: audit 2023-10-09T04:52:06.185595+0000 mon.c (mon.2) 114 : audit 0 from='client.? 172.21.15.133:0/1087246646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:52:07.233 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T04:52:07.551 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:07 smithi133 bash[91128]: debug 2023-10-09T04:52:07.168+0000 7ff4a5635200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:52:07.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:07 smithi194 bash[82301]: debug 2023-10-09T04:52:07.156+0000 7ff3ddb42200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:52:08.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:07 smithi194 bash[31648]: cluster 2023-10-09T04:52:06.867372+0000 mon.a (mon.0) 14 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2023-10-09T04:52:08.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:07 smithi194 bash[31648]: cluster 2023-10-09T04:52:06.869345+0000 osd.2 (osd.2) 5 : cluster [WRN] failed to encode map e81 with expected crc 2023-10-09T04:52:08.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:07 smithi194 bash[31648]: cluster 2023-10-09T04:52:06.872691+0000 osd.4 (osd.4) 6 : cluster [WRN] failed to encode map e81 with expected crc 2023-10-09T04:52:08.217 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:07 smithi133 bash[41291]: cluster 2023-10-09T04:52:06.867372+0000 mon.a (mon.0) 14 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2023-10-09T04:52:08.217 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:07 smithi133 bash[41291]: cluster 2023-10-09T04:52:06.869345+0000 osd.2 (osd.2) 5 : cluster [WRN] failed to encode map e81 with expected crc 2023-10-09T04:52:08.217 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:07 smithi133 bash[41291]: cluster 2023-10-09T04:52:06.872691+0000 osd.4 (osd.4) 6 : cluster [WRN] failed to encode map e81 with expected crc 2023-10-09T04:52:08.217 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:07 smithi133 bash[102858]: cluster 2023-10-09T04:52:06.867372+0000 mon.a (mon.0) 14 : cluster 0 osdmap e81: 8 total, 8 up, 8 in 2023-10-09T04:52:08.217 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:07 smithi133 bash[102858]: cluster 2023-10-09T04:52:06.869345+0000 osd.2 (osd.2) 5 : cluster 3 failed to encode map e81 with expected crc 2023-10-09T04:52:08.217 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:07 smithi133 bash[102858]: cluster 2023-10-09T04:52:06.872691+0000 osd.4 (osd.4) 6 : cluster 3 failed to encode map e81 with expected crc 2023-10-09T04:52:08.551 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:08 smithi133 bash[91128]: debug 2023-10-09T04:52:08.208+0000 7ff4a5635200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:52:08.551 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:08 smithi133 bash[91128]: debug 2023-10-09T04:52:08.328+0000 7ff4a5635200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:52:08.551 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:08 smithi133 bash[91128]: debug 2023-10-09T04:52:08.448+0000 7ff4a5635200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:52:08.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:08 smithi194 bash[82301]: debug 2023-10-09T04:52:08.160+0000 7ff3ddb42200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:52:08.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:08 smithi194 bash[82301]: debug 2023-10-09T04:52:08.276+0000 7ff3ddb42200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:52:08.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:08 smithi194 bash[82301]: debug 2023-10-09T04:52:08.392+0000 7ff3ddb42200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:52:09.135 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:08 smithi194 bash[82301]: debug 2023-10-09T04:52:08.888+0000 7ff3ddb42200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:52:09.135 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:09 smithi194 bash[82301]: debug 2023-10-09T04:52:09.128+0000 7ff3ddb42200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:52:09.135 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:52:08 smithi194 bash[81885]: ts=2023-10-09T04:52:08.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi194.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi133:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:52:09.301 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:08 smithi133 bash[91128]: debug 2023-10-09T04:52:08.936+0000 7ff4a5635200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:52:09.301 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:09 smithi133 bash[91128]: debug 2023-10-09T04:52:09.168+0000 7ff4a5635200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:52:09.887 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:09 smithi194 bash[82301]: debug 2023-10-09T04:52:09.628+0000 7ff3ddb42200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:52:10.051 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:09 smithi133 bash[91128]: debug 2023-10-09T04:52:09.632+0000 7ff4a5635200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:52:10.551 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:10 smithi133 bash[91128]: debug 2023-10-09T04:52:10.132+0000 7ff4a5635200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:52:10.584 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:52:10.637 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:10 smithi194 bash[82301]: debug 2023-10-09T04:52:10.132+0000 7ff3ddb42200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:52:11.051 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:10 smithi133 bash[91128]: debug 2023-10-09T04:52:10.700+0000 7ff4a5635200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:52:11.051 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:10 smithi133 bash[91128]: debug 2023-10-09T04:52:10.808+0000 7ff4a5635200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:52:11.136 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:10 smithi194 bash[82301]: debug 2023-10-09T04:52:10.728+0000 7ff3ddb42200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:52:11.137 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:10 smithi194 bash[82301]: debug 2023-10-09T04:52:10.840+0000 7ff3ddb42200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:52:11.436 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:11 smithi133 bash[91128]: debug 2023-10-09T04:52:11.164+0000 7ff4a5635200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:52:11.436 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:11 smithi133 bash[91128]: debug 2023-10-09T04:52:11.284+0000 7ff4a5635200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:52:11.464 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:11 smithi194 bash[82301]: debug 2023-10-09T04:52:11.200+0000 7ff3ddb42200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:52:11.464 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:11 smithi194 bash[82301]: debug 2023-10-09T04:52:11.312+0000 7ff3ddb42200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:52:11.596 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:52:11.713 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:11 smithi133 bash[91128]: debug 2023-10-09T04:52:11.428+0000 7ff4a5635200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:52:11.713 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:11 smithi133 bash[91128]: debug 2023-10-09T04:52:11.548+0000 7ff4a5635200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:52:11.722 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:11 smithi194 bash[82301]: debug 2023-10-09T04:52:11.460+0000 7ff3ddb42200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:52:11.722 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:11 smithi194 bash[82301]: debug 2023-10-09T04:52:11.584+0000 7ff3ddb42200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:52:12.009 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:11 smithi133 bash[91128]: debug 2023-10-09T04:52:11.680+0000 7ff4a5635200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:52:12.009 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:11 smithi133 bash[91128]: debug 2023-10-09T04:52:11.872+0000 7ff4a5635200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:52:12.039 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:11 smithi194 bash[82301]: debug 2023-10-09T04:52:11.716+0000 7ff3ddb42200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:52:12.039 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:11 smithi194 bash[82301]: debug 2023-10-09T04:52:11.904+0000 7ff3ddb42200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:52:12.301 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:12 smithi133 bash[91128]: debug 2023-10-09T04:52:12.004+0000 7ff4a5635200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:52:12.387 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:12 smithi194 bash[82301]: debug 2023-10-09T04:52:12.036+0000 7ff3ddb42200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:52:13.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: debug 2023-10-09T04:52:13.304+0000 7f5ee1550700 -1 mon.c@2(peon).osd e82 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T04:52:13.552 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[91128]: debug 2023-10-09T04:52:13.296+0000 7ff4a5635200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:52:13.552 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[91128]: [09/Oct/2023:04:52:13] ENGINE Bus STARTING 2023-10-09T04:52:13.552 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[91128]: debug 2023-10-09T04:52:13.496+0000 7ff4021ce700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:52:13.552 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[91128]: debug 2023-10-09T04:52:13.496+0000 7ff4021ce700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:52:13.552 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[91128]: debug 2023-10-09T04:52:13.496+0000 7ff4021ce700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:52:13.552 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[91128]: debug 2023-10-09T04:52:13.496+0000 7ff4021ce700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:52:13.552 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[91128]: debug 2023-10-09T04:52:13.496+0000 7ff4021ce700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:52:13.552 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[91128]: debug 2023-10-09T04:52:13.504+0000 7ff4081da700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:52:13.552 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[91128]: debug 2023-10-09T04:52:13.504+0000 7ff4081da700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:52:13.553 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[91128]: debug 2023-10-09T04:52:13.504+0000 7ff4081da700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:52:13.553 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[91128]: debug 2023-10-09T04:52:13.504+0000 7ff4081da700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:52:13.553 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[91128]: debug 2023-10-09T04:52:13.504+0000 7ff4081da700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:52:13.574 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: debug 2023-10-09T04:52:13.304+0000 7f7a49f0f700 -1 mon.b@1(peon).osd e82 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T04:52:13.574 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[82301]: debug 2023-10-09T04:52:13.284+0000 7ff3ddb42200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:52:13.575 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[82301]: [09/Oct/2023:04:52:13] ENGINE Bus STARTING 2023-10-09T04:52:13.575 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[82301]: CherryPy Checker: 2023-10-09T04:52:13.575 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[82301]: The Application mounted at '' has an empty config. 2023-10-09T04:52:13.575 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[82301]: [09/Oct/2023:04:52:13] ENGINE Serving on http://:::9283 2023-10-09T04:52:13.575 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[82301]: [09/Oct/2023:04:52:13] ENGINE Bus STARTED 2023-10-09T04:52:13.801 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[91128]: CherryPy Checker: 2023-10-09T04:52:13.801 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[91128]: The Application mounted at '' has an empty config. 2023-10-09T04:52:13.801 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[91128]: [09/Oct/2023:04:52:13] ENGINE Serving on http://:::9283 2023-10-09T04:52:13.801 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[91128]: [09/Oct/2023:04:52:13] ENGINE Bus STARTED 2023-10-09T04:52:14.239 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: cluster 2023-10-09T04:52:13.301872+0000 mon.a (mon.0) 15 : cluster [INF] Active manager daemon y restarted 2023-10-09T04:52:14.240 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: cluster 2023-10-09T04:52:13.302231+0000 mon.a (mon.0) 16 : cluster [INF] Activating manager daemon y 2023-10-09T04:52:14.240 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: cluster 2023-10-09T04:52:13.308186+0000 mon.a (mon.0) 17 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2023-10-09T04:52:14.240 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: cluster 2023-10-09T04:52:13.308576+0000 mon.a (mon.0) 18 : cluster [DBG] mgrmap e32: y(active, starting, since 0.0064261s), standbys: x 2023-10-09T04:52:14.240 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: cluster 2023-10-09T04:52:13.312568+0000 osd.3 (osd.3) 4 : cluster [WRN] failed to encode map e82 with expected crc 2023-10-09T04:52:14.240 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: cluster 2023-10-09T04:52:13.317541+0000 osd.4 (osd.4) 7 : cluster [WRN] failed to encode map e82 with expected crc 2023-10-09T04:52:14.240 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: cluster 2023-10-09T04:52:13.320831+0000 mon.a (mon.0) 19 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T04:52:14.240 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: cluster 2023-10-09T04:52:13.320943+0000 mon.a (mon.0) 20 : cluster [DBG] Standby manager daemon x started 2023-10-09T04:52:14.240 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: audit 2023-10-09T04:52:13.322448+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.? 172.21.15.194:0/57901141' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:52:14.240 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: cluster 2023-10-09T04:52:13.322858+0000 osd.4 (osd.4) 8 : cluster [WRN] failed to encode map e82 with expected crc 2023-10-09T04:52:14.240 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: audit 2023-10-09T04:52:13.322882+0000 mon.a (mon.0) 21 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:52:14.240 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: audit 2023-10-09T04:52:13.323157+0000 mon.a (mon.0) 22 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:52:14.241 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: audit 2023-10-09T04:52:13.323347+0000 mon.a (mon.0) 23 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:52:14.241 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: audit 2023-10-09T04:52:13.324027+0000 mon.a (mon.0) 24 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi133.tsuabh"}]: dispatch 2023-10-09T04:52:14.241 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: audit 2023-10-09T04:52:13.324360+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.? 172.21.15.194:0/57901141' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:52:14.241 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: audit 2023-10-09T04:52:13.324382+0000 mon.a (mon.0) 25 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi194.yxnfqn"}]: dispatch 2023-10-09T04:52:14.241 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: audit 2023-10-09T04:52:13.324721+0000 mon.a (mon.0) 26 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:52:14.241 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: audit 2023-10-09T04:52:13.324998+0000 mon.a (mon.0) 27 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:52:14.241 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: audit 2023-10-09T04:52:13.325186+0000 mon.a (mon.0) 28 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:52:14.241 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: audit 2023-10-09T04:52:13.325481+0000 mon.a (mon.0) 29 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:52:14.241 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: cluster 2023-10-09T04:52:13.301872+0000 mon.a (mon.0) 15 : cluster 1 Active manager daemon y restarted 2023-10-09T04:52:14.242 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: cluster 2023-10-09T04:52:13.302231+0000 mon.a (mon.0) 16 : cluster 1 Activating manager daemon y 2023-10-09T04:52:14.242 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: cluster 2023-10-09T04:52:13.308186+0000 mon.a (mon.0) 17 : cluster 0 osdmap e82: 8 total, 8 up, 8 in 2023-10-09T04:52:14.242 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: cluster 2023-10-09T04:52:13.308576+0000 mon.a (mon.0) 18 : cluster 0 mgrmap e32: y(active, starting, since 0.0064261s), standbys: x 2023-10-09T04:52:14.242 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: cluster 2023-10-09T04:52:13.312568+0000 osd.3 (osd.3) 4 : cluster 3 failed to encode map e82 with expected crc 2023-10-09T04:52:14.242 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: cluster 2023-10-09T04:52:13.317541+0000 osd.4 (osd.4) 7 : cluster 3 failed to encode map e82 with expected crc 2023-10-09T04:52:14.242 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: cluster 2023-10-09T04:52:13.320831+0000 mon.a (mon.0) 19 : cluster 0 Standby manager daemon x restarted 2023-10-09T04:52:14.242 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: cluster 2023-10-09T04:52:13.320943+0000 mon.a (mon.0) 20 : cluster 0 Standby manager daemon x started 2023-10-09T04:52:14.242 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: audit 2023-10-09T04:52:13.322448+0000 mon.b (mon.1) 47 : audit 0 from='mgr.? 172.21.15.194:0/57901141' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:52:14.242 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: cluster 2023-10-09T04:52:13.322858+0000 osd.4 (osd.4) 8 : cluster 3 failed to encode map e82 with expected crc 2023-10-09T04:52:14.242 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: audit 2023-10-09T04:52:13.322882+0000 mon.a (mon.0) 21 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:52:14.242 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: audit 2023-10-09T04:52:13.323157+0000 mon.a (mon.0) 22 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:52:14.243 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: audit 2023-10-09T04:52:13.323347+0000 mon.a (mon.0) 23 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:52:14.243 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: audit 2023-10-09T04:52:13.324027+0000 mon.a (mon.0) 24 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi133.tsuabh"}]: dispatch 2023-10-09T04:52:14.243 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: audit 2023-10-09T04:52:13.324360+0000 mon.b (mon.1) 48 : audit 0 from='mgr.? 172.21.15.194:0/57901141' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:52:14.243 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: audit 2023-10-09T04:52:13.324382+0000 mon.a (mon.0) 25 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi194.yxnfqn"}]: dispatch 2023-10-09T04:52:14.243 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: audit 2023-10-09T04:52:13.324721+0000 mon.a (mon.0) 26 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:52:14.243 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: audit 2023-10-09T04:52:13.324998+0000 mon.a (mon.0) 27 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:52:14.243 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: audit 2023-10-09T04:52:13.325186+0000 mon.a (mon.0) 28 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:52:14.243 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: audit 2023-10-09T04:52:13.325481+0000 mon.a (mon.0) 29 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:52:14.243 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: audit 2023-10-09T04:52:13.325721+0000 mon.a (mon.0) 30 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:52:14.243 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: audit 2023-10-09T04:52:13.325721+0000 mon.a (mon.0) 30 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:52:14.244 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: audit 2023-10-09T04:52:13.325997+0000 mon.a (mon.0) 31 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:52:14.244 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: audit 2023-10-09T04:52:13.326232+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.? 172.21.15.194:0/57901141' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:52:14.244 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: audit 2023-10-09T04:52:13.326315+0000 mon.a (mon.0) 32 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:52:14.244 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: audit 2023-10-09T04:52:13.326596+0000 mon.a (mon.0) 33 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:52:14.244 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: audit 2023-10-09T04:52:13.326900+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:52:14.244 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: audit 2023-10-09T04:52:13.326932+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.? 172.21.15.194:0/57901141' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:52:14.244 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: audit 2023-10-09T04:52:13.327193+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:52:14.244 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: audit 2023-10-09T04:52:13.327497+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:52:14.244 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: audit 2023-10-09T04:52:13.327833+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:52:14.244 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: audit 2023-10-09T04:52:13.328530+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:52:14.244 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: cluster 2023-10-09T04:52:13.338333+0000 mon.a (mon.0) 39 : cluster [INF] Manager daemon y is now available 2023-10-09T04:52:14.244 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: audit 2023-10-09T04:52:13.369818+0000 mon.a (mon.0) 40 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:52:14.245 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: audit 2023-10-09T04:52:13.372050+0000 mon.a (mon.0) 41 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:52:14.245 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: audit 2023-10-09T04:52:13.400914+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:14.245 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: audit 2023-10-09T04:52:13.406931+0000 mon.a (mon.0) 43 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:52:14.245 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: cluster 2023-10-09T04:52:13.430359+0000 osd.2 (osd.2) 6 : cluster [WRN] failed to encode map e82 with expected crc 2023-10-09T04:52:14.245 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[41291]: audit 2023-10-09T04:52:13.449799+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:52:14.245 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: audit 2023-10-09T04:52:13.325997+0000 mon.a (mon.0) 31 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:52:14.245 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: audit 2023-10-09T04:52:13.326232+0000 mon.b (mon.1) 49 : audit 0 from='mgr.? 172.21.15.194:0/57901141' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:52:14.245 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: audit 2023-10-09T04:52:13.326315+0000 mon.a (mon.0) 32 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:52:14.245 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: audit 2023-10-09T04:52:13.326596+0000 mon.a (mon.0) 33 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:52:14.245 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: audit 2023-10-09T04:52:13.326900+0000 mon.a (mon.0) 34 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:52:14.246 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: audit 2023-10-09T04:52:13.326932+0000 mon.b (mon.1) 50 : audit 0 from='mgr.? 172.21.15.194:0/57901141' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:52:14.246 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: audit 2023-10-09T04:52:13.327193+0000 mon.a (mon.0) 35 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:52:14.246 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: audit 2023-10-09T04:52:13.327497+0000 mon.a (mon.0) 36 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:52:14.246 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: audit 2023-10-09T04:52:13.327833+0000 mon.a (mon.0) 37 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:52:14.246 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: audit 2023-10-09T04:52:13.328530+0000 mon.a (mon.0) 38 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:52:14.246 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: cluster 2023-10-09T04:52:13.338333+0000 mon.a (mon.0) 39 : cluster 1 Manager daemon y is now available 2023-10-09T04:52:14.246 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: audit 2023-10-09T04:52:13.369818+0000 mon.a (mon.0) 40 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:52:14.246 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: audit 2023-10-09T04:52:13.372050+0000 mon.a (mon.0) 41 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:52:14.246 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: audit 2023-10-09T04:52:13.400914+0000 mon.a (mon.0) 42 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:14.246 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: audit 2023-10-09T04:52:13.406931+0000 mon.a (mon.0) 43 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:52:14.246 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: cluster 2023-10-09T04:52:13.430359+0000 osd.2 (osd.2) 6 : cluster 3 failed to encode map e82 with expected crc 2023-10-09T04:52:14.247 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:13 smithi133 bash[102858]: audit 2023-10-09T04:52:13.449799+0000 mon.a (mon.0) 44 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:52:14.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: cluster 2023-10-09T04:52:13.301872+0000 mon.a (mon.0) 15 : cluster [INF] Active manager daemon y restarted 2023-10-09T04:52:14.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: cluster 2023-10-09T04:52:13.302231+0000 mon.a (mon.0) 16 : cluster [INF] Activating manager daemon y 2023-10-09T04:52:14.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: cluster 2023-10-09T04:52:13.308186+0000 mon.a (mon.0) 17 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2023-10-09T04:52:14.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: cluster 2023-10-09T04:52:13.308576+0000 mon.a (mon.0) 18 : cluster [DBG] mgrmap e32: y(active, starting, since 0.0064261s), standbys: x 2023-10-09T04:52:14.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: cluster 2023-10-09T04:52:13.312568+0000 osd.3 (osd.3) 4 : cluster [WRN] failed to encode map e82 with expected crc 2023-10-09T04:52:14.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: cluster 2023-10-09T04:52:13.317541+0000 osd.4 (osd.4) 7 : cluster [WRN] failed to encode map e82 with expected crc 2023-10-09T04:52:14.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: cluster 2023-10-09T04:52:13.320831+0000 mon.a (mon.0) 19 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T04:52:14.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: cluster 2023-10-09T04:52:13.320943+0000 mon.a (mon.0) 20 : cluster [DBG] Standby manager daemon x started 2023-10-09T04:52:14.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: audit 2023-10-09T04:52:13.322448+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.? 172.21.15.194:0/57901141' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:52:14.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: cluster 2023-10-09T04:52:13.322858+0000 osd.4 (osd.4) 8 : cluster [WRN] failed to encode map e82 with expected crc 2023-10-09T04:52:14.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: audit 2023-10-09T04:52:13.322882+0000 mon.a (mon.0) 21 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:52:14.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: audit 2023-10-09T04:52:13.323157+0000 mon.a (mon.0) 22 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:52:14.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: audit 2023-10-09T04:52:13.323347+0000 mon.a (mon.0) 23 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:52:14.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: audit 2023-10-09T04:52:13.324027+0000 mon.a (mon.0) 24 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi133.tsuabh"}]: dispatch 2023-10-09T04:52:14.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: audit 2023-10-09T04:52:13.324360+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.? 172.21.15.194:0/57901141' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:52:14.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: audit 2023-10-09T04:52:13.324382+0000 mon.a (mon.0) 25 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi194.yxnfqn"}]: dispatch 2023-10-09T04:52:14.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: audit 2023-10-09T04:52:13.324721+0000 mon.a (mon.0) 26 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:52:14.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: audit 2023-10-09T04:52:13.324998+0000 mon.a (mon.0) 27 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:52:14.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: audit 2023-10-09T04:52:13.325186+0000 mon.a (mon.0) 28 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:52:14.390 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: audit 2023-10-09T04:52:13.325481+0000 mon.a (mon.0) 29 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:52:14.390 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: audit 2023-10-09T04:52:13.325721+0000 mon.a (mon.0) 30 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:52:14.390 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: audit 2023-10-09T04:52:13.325997+0000 mon.a (mon.0) 31 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:52:14.390 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: audit 2023-10-09T04:52:13.326232+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.? 172.21.15.194:0/57901141' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:52:14.390 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: audit 2023-10-09T04:52:13.326315+0000 mon.a (mon.0) 32 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:52:14.390 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: audit 2023-10-09T04:52:13.326596+0000 mon.a (mon.0) 33 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:52:14.390 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: audit 2023-10-09T04:52:13.326900+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:52:14.390 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: audit 2023-10-09T04:52:13.326932+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.? 172.21.15.194:0/57901141' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:52:14.390 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: audit 2023-10-09T04:52:13.327193+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:52:14.390 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: audit 2023-10-09T04:52:13.327497+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:52:14.390 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: audit 2023-10-09T04:52:13.327833+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:52:14.390 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: audit 2023-10-09T04:52:13.328530+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:52:14.391 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: cluster 2023-10-09T04:52:13.338333+0000 mon.a (mon.0) 39 : cluster [INF] Manager daemon y is now available 2023-10-09T04:52:14.391 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: audit 2023-10-09T04:52:13.369818+0000 mon.a (mon.0) 40 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:52:14.391 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: audit 2023-10-09T04:52:13.372050+0000 mon.a (mon.0) 41 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:52:14.391 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: audit 2023-10-09T04:52:13.400914+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:14.391 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: audit 2023-10-09T04:52:13.406931+0000 mon.a (mon.0) 43 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:52:14.391 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: cluster 2023-10-09T04:52:13.430359+0000 osd.2 (osd.2) 6 : cluster [WRN] failed to encode map e82 with expected crc 2023-10-09T04:52:14.391 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:13 smithi194 bash[31648]: audit 2023-10-09T04:52:13.449799+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:52:14.551 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:52:14] "GET /metrics HTTP/1.1" 200 33437 "" "Prometheus/2.43.0" 2023-10-09T04:52:14.551 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:14 smithi133 bash[91128]: debug 2023-10-09T04:52:14.312+0000 7ff45d5ed700 -1 mgr.server handle_report got status from non-daemon mon.a 2023-10-09T04:52:15.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:15 smithi133 bash[102858]: cluster 2023-10-09T04:52:13.415364+0000 osd.5 (osd.5) 4 : cluster 3 failed to encode map e82 with expected crc 2023-10-09T04:52:15.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:15 smithi133 bash[102858]: cluster 2023-10-09T04:52:13.443058+0000 osd.1 (osd.1) 5 : cluster 3 failed to encode map e82 with expected crc 2023-10-09T04:52:15.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:15 smithi133 bash[102858]: cephadm 2023-10-09T04:52:13.947257+0000 mgr.y (mgr.25277) 1 : cephadm 1 [09/Oct/2023:04:52:13] ENGINE Bus STARTING 2023-10-09T04:52:15.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:15 smithi133 bash[102858]: cephadm 2023-10-09T04:52:14.064089+0000 mgr.y (mgr.25277) 2 : cephadm 1 [09/Oct/2023:04:52:14] ENGINE Serving on https://172.21.15.133:7150 2023-10-09T04:52:15.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:15 smithi133 bash[102858]: cephadm 2023-10-09T04:52:14.166655+0000 mgr.y (mgr.25277) 3 : cephadm 1 [09/Oct/2023:04:52:14] ENGINE Serving on http://172.21.15.133:8765 2023-10-09T04:52:15.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:15 smithi133 bash[102858]: cephadm 2023-10-09T04:52:14.167027+0000 mgr.y (mgr.25277) 4 : cephadm 1 [09/Oct/2023:04:52:14] ENGINE Bus STARTED 2023-10-09T04:52:15.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:15 smithi133 bash[102858]: audit 2023-10-09T04:52:14.171842+0000 mon.a (mon.0) 45 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:15.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:15 smithi133 bash[102858]: audit 2023-10-09T04:52:14.179051+0000 mon.a (mon.0) 46 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:15.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:15 smithi133 bash[102858]: cluster 2023-10-09T04:52:14.313030+0000 mon.a (mon.0) 47 : cluster 0 mgrmap e33: y(active, since 1.01088s), standbys: x 2023-10-09T04:52:15.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:15 smithi133 bash[102858]: cluster 2023-10-09T04:52:14.326526+0000 mgr.y (mgr.25277) 5 : cluster 0 pgmap v3: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:52:15.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:15 smithi133 bash[41291]: cluster 2023-10-09T04:52:13.415364+0000 osd.5 (osd.5) 4 : cluster [WRN] failed to encode map e82 with expected crc 2023-10-09T04:52:15.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:15 smithi133 bash[41291]: cluster 2023-10-09T04:52:13.443058+0000 osd.1 (osd.1) 5 : cluster [WRN] failed to encode map e82 with expected crc 2023-10-09T04:52:15.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:15 smithi133 bash[41291]: cephadm 2023-10-09T04:52:13.947257+0000 mgr.y (mgr.25277) 1 : cephadm [INF] [09/Oct/2023:04:52:13] ENGINE Bus STARTING 2023-10-09T04:52:15.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:15 smithi133 bash[41291]: cephadm 2023-10-09T04:52:14.064089+0000 mgr.y (mgr.25277) 2 : cephadm [INF] [09/Oct/2023:04:52:14] ENGINE Serving on https://172.21.15.133:7150 2023-10-09T04:52:15.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:15 smithi133 bash[41291]: cephadm 2023-10-09T04:52:14.166655+0000 mgr.y (mgr.25277) 3 : cephadm [INF] [09/Oct/2023:04:52:14] ENGINE Serving on http://172.21.15.133:8765 2023-10-09T04:52:15.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:15 smithi133 bash[41291]: cephadm 2023-10-09T04:52:14.167027+0000 mgr.y (mgr.25277) 4 : cephadm [INF] [09/Oct/2023:04:52:14] ENGINE Bus STARTED 2023-10-09T04:52:15.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:15 smithi133 bash[41291]: audit 2023-10-09T04:52:14.171842+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:15.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:15 smithi133 bash[41291]: audit 2023-10-09T04:52:14.179051+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:15.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:15 smithi133 bash[41291]: cluster 2023-10-09T04:52:14.313030+0000 mon.a (mon.0) 47 : cluster [DBG] mgrmap e33: y(active, since 1.01088s), standbys: x 2023-10-09T04:52:15.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:15 smithi133 bash[41291]: cluster 2023-10-09T04:52:14.326526+0000 mgr.y (mgr.25277) 5 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:52:15.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:15 smithi194 bash[31648]: cluster 2023-10-09T04:52:13.415364+0000 osd.5 (osd.5) 4 : cluster [WRN] failed to encode map e82 with expected crc 2023-10-09T04:52:15.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:15 smithi194 bash[31648]: cluster 2023-10-09T04:52:13.443058+0000 osd.1 (osd.1) 5 : cluster [WRN] failed to encode map e82 with expected crc 2023-10-09T04:52:15.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:15 smithi194 bash[31648]: cephadm 2023-10-09T04:52:13.947257+0000 mgr.y (mgr.25277) 1 : cephadm [INF] [09/Oct/2023:04:52:13] ENGINE Bus STARTING 2023-10-09T04:52:15.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:15 smithi194 bash[31648]: cephadm 2023-10-09T04:52:14.064089+0000 mgr.y (mgr.25277) 2 : cephadm [INF] [09/Oct/2023:04:52:14] ENGINE Serving on https://172.21.15.133:7150 2023-10-09T04:52:15.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:15 smithi194 bash[31648]: cephadm 2023-10-09T04:52:14.166655+0000 mgr.y (mgr.25277) 3 : cephadm [INF] [09/Oct/2023:04:52:14] ENGINE Serving on http://172.21.15.133:8765 2023-10-09T04:52:15.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:15 smithi194 bash[31648]: cephadm 2023-10-09T04:52:14.167027+0000 mgr.y (mgr.25277) 4 : cephadm [INF] [09/Oct/2023:04:52:14] ENGINE Bus STARTED 2023-10-09T04:52:15.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:15 smithi194 bash[31648]: audit 2023-10-09T04:52:14.171842+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:15.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:15 smithi194 bash[31648]: audit 2023-10-09T04:52:14.179051+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:15.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:15 smithi194 bash[31648]: cluster 2023-10-09T04:52:14.313030+0000 mon.a (mon.0) 47 : cluster [DBG] mgrmap e33: y(active, since 1.01088s), standbys: x 2023-10-09T04:52:15.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:15 smithi194 bash[31648]: cluster 2023-10-09T04:52:14.326526+0000 mgr.y (mgr.25277) 5 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:52:16.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:16 smithi194 bash[31648]: cluster 2023-10-09T04:52:15.326391+0000 mgr.y (mgr.25277) 6 : cluster [DBG] pgmap v4: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:52:16.684 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:16 smithi133 bash[102858]: cluster 2023-10-09T04:52:15.326391+0000 mgr.y (mgr.25277) 6 : cluster 0 pgmap v4: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:52:16.684 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:16 smithi133 bash[41291]: cluster 2023-10-09T04:52:15.326391+0000 mgr.y (mgr.25277) 6 : cluster [DBG] pgmap v4: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:52:17.142 INFO:teuthology.orchestra.run.smithi133.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T04:52:17.143 INFO:teuthology.orchestra.run.smithi133.stdout:alertmanager.a smithi133 *:9093,9094 running (2m) - 9m 14.9M - 0.25.0 c8568f914cd2 f7641cfa53d2 2023-10-09T04:52:17.143 INFO:teuthology.orchestra.run.smithi133.stdout:grafana.a smithi194 *:3000 running (112s) - 9m 77.7M - 9.4.7 2c41d148cca3 18d67948e55e 2023-10-09T04:52:17.143 INFO:teuthology.orchestra.run.smithi133.stdout:mds.foo.smithi133.tsuabh smithi133 running (7m) - 7m 12.8M - 17.2.6-1303-gb2903cb0 5517396c50d4 1adbb452599e 2023-10-09T04:52:17.143 INFO:teuthology.orchestra.run.smithi133.stdout:mds.foo.smithi194.yxnfqn smithi194 running (7m) - 7m 14.9M - 17.2.6-1303-gb2903cb0 5517396c50d4 745c16c1ffd2 2023-10-09T04:52:17.143 INFO:teuthology.orchestra.run.smithi133.stdout:mgr.x smithi194 *:8443,9283,8765 running (42s) - 18m 75.1M - 18.2.0-598-gb4fa663f 3060897fdf97 e7277fed699e 2023-10-09T04:52:17.143 INFO:teuthology.orchestra.run.smithi133.stdout:mgr.y smithi133 *:8443,9283,8765 running (103s) - 21m 458M - 18.2.0-598-gb4fa663f 3060897fdf97 caf6152f91c6 2023-10-09T04:52:17.143 INFO:teuthology.orchestra.run.smithi133.stdout:mon.a smithi133 running (21m) - 21m 55.8M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 fdcd4c3aa009 2023-10-09T04:52:17.143 INFO:teuthology.orchestra.run.smithi133.stdout:mon.b smithi194 running (18m) - 18m 42.2M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 8059052f6835 2023-10-09T04:52:17.143 INFO:teuthology.orchestra.run.smithi133.stdout:mon.c smithi133 running (18m) - 18m 43.5M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 74897387140d 2023-10-09T04:52:17.143 INFO:teuthology.orchestra.run.smithi133.stdout:node-exporter.a smithi133 *:9100 running (2m) - 10m 7671k - 1.5.0 0da6a335fe13 f8bdde5940e7 2023-10-09T04:52:17.144 INFO:teuthology.orchestra.run.smithi133.stdout:node-exporter.b smithi194 *:9100 running (118s) - 10m 7759k - 1.5.0 0da6a335fe13 ffab717ac947 2023-10-09T04:52:17.144 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0 smithi133 running (17m) - 17m 366M 2524M 17.2.6-1303-gb2903cb0 5517396c50d4 7dc540ae1166 2023-10-09T04:52:17.144 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1 smithi133 running (16m) - 16m 242M 2524M 17.2.6-1303-gb2903cb0 5517396c50d4 27ac869bf46a 2023-10-09T04:52:17.144 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2 smithi133 running (15m) - 15m 332M 2524M 17.2.6-1303-gb2903cb0 5517396c50d4 e9d2970cad24 2023-10-09T04:52:17.144 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3 smithi133 running (14m) - 14m 170M 2524M 17.2.6-1303-gb2903cb0 5517396c50d4 415368a68902 2023-10-09T04:52:17.144 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4 smithi194 running (13m) - 13m 367M 2524M 17.2.6-1303-gb2903cb0 5517396c50d4 c77edffeb061 2023-10-09T04:52:17.144 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5 smithi194 running (13m) - 13m 404M 2524M 17.2.6-1303-gb2903cb0 5517396c50d4 47540765fad9 2023-10-09T04:52:17.144 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6 smithi194 running (12m) - 12m 207M 2524M 17.2.6-1303-gb2903cb0 5517396c50d4 26bbe2f2859f 2023-10-09T04:52:17.144 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7 smithi194 running (11m) - 11m 261M 2524M 17.2.6-1303-gb2903cb0 5517396c50d4 45c343cd22ea 2023-10-09T04:52:17.144 INFO:teuthology.orchestra.run.smithi133.stdout:prometheus.a smithi194 *:9095 running (50s) - 10m 37.8M - 2.43.0 a07b618ecd1d f40ed7def7ab 2023-10-09T04:52:17.337 INFO:teuthology.orchestra.run.smithi133.stderr:enabled application 'rados' on pool 'unique_pool_1' 2023-10-09T04:52:17.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:17 smithi133 bash[102858]: cluster 2023-10-09T04:52:16.327654+0000 mon.a (mon.0) 48 : cluster 0 mgrmap e34: y(active, since 3s), standbys: x 2023-10-09T04:52:17.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:17 smithi133 bash[102858]: audit 2023-10-09T04:52:17.101086+0000 mon.a (mon.0) 49 : audit 1 from='client.? 172.21.15.133:0/1675037656' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:52:17.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:17 smithi133 bash[41291]: debug 2023-10-09T04:52:17.324+0000 7f5ee1550700 -1 mon.c@2(peon).osd e83 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T04:52:17.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:17 smithi133 bash[41291]: cluster 2023-10-09T04:52:16.327654+0000 mon.a (mon.0) 48 : cluster [DBG] mgrmap e34: y(active, since 3s), standbys: x 2023-10-09T04:52:17.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:17 smithi133 bash[41291]: audit 2023-10-09T04:52:17.101086+0000 mon.a (mon.0) 49 : audit [INF] from='client.? 172.21.15.133:0/1675037656' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:52:17.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:17 smithi194 bash[31648]: debug 2023-10-09T04:52:17.324+0000 7f7a49f0f700 -1 mon.b@1(peon).osd e83 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T04:52:17.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:17 smithi194 bash[31648]: cluster 2023-10-09T04:52:16.327654+0000 mon.a (mon.0) 48 : cluster [DBG] mgrmap e34: y(active, since 3s), standbys: x 2023-10-09T04:52:17.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:17 smithi194 bash[31648]: audit 2023-10-09T04:52:17.101086+0000 mon.a (mon.0) 49 : audit [INF] from='client.? 172.21.15.133:0/1675037656' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:52:18.266 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T04:52:18.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:18 smithi194 bash[31648]: audit 2023-10-09T04:52:17.120846+0000 mgr.y (mgr.25277) 7 : audit [DBG] from='client.25334 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:52:18.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:18 smithi194 bash[31648]: cluster 2023-10-09T04:52:17.327097+0000 mgr.y (mgr.25277) 8 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:52:18.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:18 smithi194 bash[31648]: audit 2023-10-09T04:52:17.329486+0000 mon.a (mon.0) 50 : audit [INF] from='client.? 172.21.15.133:0/1675037656' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:52:18.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:18 smithi194 bash[31648]: cluster 2023-10-09T04:52:17.329610+0000 mon.a (mon.0) 51 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-10-09T04:52:18.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:18 smithi194 bash[31648]: cluster 2023-10-09T04:52:17.332957+0000 osd.2 (osd.2) 7 : cluster [WRN] failed to encode map e83 with expected crc 2023-10-09T04:52:18.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:18 smithi194 bash[31648]: cluster 2023-10-09T04:52:17.335297+0000 osd.6 (osd.6) 3 : cluster [WRN] failed to encode map e83 with expected crc 2023-10-09T04:52:18.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:18 smithi133 bash[102858]: audit 2023-10-09T04:52:17.120846+0000 mgr.y (mgr.25277) 7 : audit 0 from='client.25334 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:52:18.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:18 smithi133 bash[102858]: cluster 2023-10-09T04:52:17.327097+0000 mgr.y (mgr.25277) 8 : cluster 0 pgmap v5: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:52:18.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:18 smithi133 bash[102858]: audit 2023-10-09T04:52:17.329486+0000 mon.a (mon.0) 50 : audit 1 from='client.? 172.21.15.133:0/1675037656' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:52:18.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:18 smithi133 bash[102858]: cluster 2023-10-09T04:52:17.329610+0000 mon.a (mon.0) 51 : cluster 0 osdmap e83: 8 total, 8 up, 8 in 2023-10-09T04:52:18.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:18 smithi133 bash[102858]: cluster 2023-10-09T04:52:17.332957+0000 osd.2 (osd.2) 7 : cluster 3 failed to encode map e83 with expected crc 2023-10-09T04:52:18.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:18 smithi133 bash[102858]: cluster 2023-10-09T04:52:17.335297+0000 osd.6 (osd.6) 3 : cluster 3 failed to encode map e83 with expected crc 2023-10-09T04:52:18.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:18 smithi133 bash[41291]: audit 2023-10-09T04:52:17.120846+0000 mgr.y (mgr.25277) 7 : audit [DBG] from='client.25334 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:52:18.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:18 smithi133 bash[41291]: cluster 2023-10-09T04:52:17.327097+0000 mgr.y (mgr.25277) 8 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:52:18.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:18 smithi133 bash[41291]: audit 2023-10-09T04:52:17.329486+0000 mon.a (mon.0) 50 : audit [INF] from='client.? 172.21.15.133:0/1675037656' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:52:18.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:18 smithi133 bash[41291]: cluster 2023-10-09T04:52:17.329610+0000 mon.a (mon.0) 51 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-10-09T04:52:18.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:18 smithi133 bash[41291]: cluster 2023-10-09T04:52:17.332957+0000 osd.2 (osd.2) 7 : cluster [WRN] failed to encode map e83 with expected crc 2023-10-09T04:52:18.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:18 smithi133 bash[41291]: cluster 2023-10-09T04:52:17.335297+0000 osd.6 (osd.6) 3 : cluster [WRN] failed to encode map e83 with expected crc 2023-10-09T04:52:19.137 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:52:18 smithi194 bash[81885]: ts=2023-10-09T04:52:18.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi194.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi133:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:52:19.467 DEBUG:teuthology.orchestra.run.smithi133:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2023-10-09T04:52:19.469 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T04:52:19.469 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T04:52:19.515 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:hints = 1 2023-10-09T04:52:19.515 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T04:52:19.515 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Object prefix: benchmark_data_smithi133_103890 2023-10-09T04:52:19.516 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:52:19.516 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 0 0 0 0 0 0 - 0 2023-10-09T04:52:20.516 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 1 16 2574 2558 159.864 159.875 0.00473899 0.00622281 2023-10-09T04:52:20.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:20 smithi194 bash[31648]: cluster 2023-10-09T04:52:19.328107+0000 mgr.y (mgr.25277) 9 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:52:20.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:20 smithi133 bash[41291]: cluster 2023-10-09T04:52:19.328107+0000 mgr.y (mgr.25277) 9 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:52:20.804 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:20 smithi133 bash[102858]: cluster 2023-10-09T04:52:19.328107+0000 mgr.y (mgr.25277) 9 : cluster 0 pgmap v7: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:52:21.463 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:21 smithi133 bash[102858]: audit 2023-10-09T04:52:20.459260+0000 mon.a (mon.0) 52 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:21.463 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:21 smithi133 bash[102858]: audit 2023-10-09T04:52:20.468867+0000 mon.a (mon.0) 53 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:21.516 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 2 16 4936 4920 153.739 147.625 0.00467688 0.00649313 2023-10-09T04:52:21.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:21 smithi133 bash[41291]: audit 2023-10-09T04:52:20.459260+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:21.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:21 smithi133 bash[41291]: audit 2023-10-09T04:52:20.468867+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:21.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:21 smithi133 bash[41291]: audit 2023-10-09T04:52:21.336463+0000 mon.a (mon.0) 54 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:21.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:21 smithi133 bash[41291]: audit 2023-10-09T04:52:21.353779+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:21.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:21 smithi133 bash[102858]: audit 2023-10-09T04:52:21.336463+0000 mon.a (mon.0) 54 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:21.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:21 smithi133 bash[102858]: audit 2023-10-09T04:52:21.353779+0000 mon.a (mon.0) 55 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:21.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:21 smithi194 bash[31648]: audit 2023-10-09T04:52:20.459260+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:21.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:21 smithi194 bash[31648]: audit 2023-10-09T04:52:20.468867+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:21.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:21 smithi194 bash[31648]: audit 2023-10-09T04:52:21.336463+0000 mon.a (mon.0) 54 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:21.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:21 smithi194 bash[31648]: audit 2023-10-09T04:52:21.353779+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:22.516 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 3 16 7479 7463 155.466 158.938 0.0058002 0.00641856 2023-10-09T04:52:22.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:22 smithi133 bash[102858]: cluster 2023-10-09T04:52:21.328681+0000 mgr.y (mgr.25277) 10 : cluster 0 pgmap v8: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:52:22.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:22 smithi133 bash[102858]: audit 2023-10-09T04:52:21.520192+0000 mon.a (mon.0) 56 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:22.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:22 smithi133 bash[102858]: audit 2023-10-09T04:52:21.530331+0000 mon.a (mon.0) 57 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:22.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:22 smithi133 bash[41291]: cluster 2023-10-09T04:52:21.328681+0000 mgr.y (mgr.25277) 10 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:52:22.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:22 smithi133 bash[41291]: audit 2023-10-09T04:52:21.520192+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:22.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:22 smithi133 bash[41291]: audit 2023-10-09T04:52:21.530331+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:22.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:22 smithi194 bash[31648]: cluster 2023-10-09T04:52:21.328681+0000 mgr.y (mgr.25277) 10 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:52:22.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:22 smithi194 bash[31648]: audit 2023-10-09T04:52:21.520192+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:22.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:22 smithi194 bash[31648]: audit 2023-10-09T04:52:21.530331+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:23.032 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:52:23.516 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 4 16 10101 10085 157.565 163.875 0.00480388 0.00633755 2023-10-09T04:52:23.534 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:23 smithi133 bash[102858]: audit 2023-10-09T04:52:22.529931+0000 mon.a (mon.0) 58 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:23.534 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:23 smithi133 bash[102858]: audit 2023-10-09T04:52:22.541890+0000 mon.a (mon.0) 59 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:23.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:23 smithi133 bash[41291]: audit 2023-10-09T04:52:22.529931+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:23.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:23 smithi133 bash[41291]: audit 2023-10-09T04:52:22.541890+0000 mon.a (mon.0) 59 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:23.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:23 smithi194 bash[31648]: audit 2023-10-09T04:52:22.529931+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:23.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:23 smithi194 bash[31648]: audit 2023-10-09T04:52:22.541890+0000 mon.a (mon.0) 59 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:24.516 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 5 16 12695 12679 158.474 162.125 0.00505313 0.0063008 2023-10-09T04:52:24.537 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:52:24] "GET /metrics HTTP/1.1" 200 33437 "" "Prometheus/2.43.0" 2023-10-09T04:52:24.812 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:24 smithi133 bash[41291]: cluster 2023-10-09T04:52:23.329318+0000 mgr.y (mgr.25277) 11 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:52:24.812 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:24 smithi133 bash[102858]: cluster 2023-10-09T04:52:23.329318+0000 mgr.y (mgr.25277) 11 : cluster 0 pgmap v9: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:52:24.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:24 smithi194 bash[31648]: cluster 2023-10-09T04:52:23.329318+0000 mgr.y (mgr.25277) 11 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:52:25.516 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 6 16 15267 15251 158.85 160.75 0.00760376 0.00628764 2023-10-09T04:52:26.517 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 7 16 17661 17645 157.53 149.625 0.00799493 0.00634085 2023-10-09T04:52:26.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:26 smithi133 bash[41291]: cluster 2023-10-09T04:52:25.329997+0000 mgr.y (mgr.25277) 12 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 762 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T04:52:26.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:26 smithi133 bash[102858]: cluster 2023-10-09T04:52:25.329997+0000 mgr.y (mgr.25277) 12 : cluster 0 pgmap v10: 89 pgs: 89 active+clean; 762 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T04:52:26.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:26 smithi194 bash[31648]: cluster 2023-10-09T04:52:25.329997+0000 mgr.y (mgr.25277) 12 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 762 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T04:52:27.517 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 8 16 19803 19787 154.572 133.875 0.00405279 0.00646016 2023-10-09T04:52:28.517 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 9 16 21924 21908 152.125 132.562 0.00886882 0.00656768 2023-10-09T04:52:28.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:28 smithi133 bash[41291]: cluster 2023-10-09T04:52:27.331291+0000 mgr.y (mgr.25277) 13 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 762 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T04:52:28.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:28 smithi133 bash[41291]: audit 2023-10-09T04:52:28.402204+0000 mon.a (mon.0) 60 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:28.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:28 smithi133 bash[102858]: cluster 2023-10-09T04:52:27.331291+0000 mgr.y (mgr.25277) 13 : cluster 0 pgmap v11: 89 pgs: 89 active+clean; 762 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T04:52:28.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:28 smithi133 bash[102858]: audit 2023-10-09T04:52:28.402204+0000 mon.a (mon.0) 60 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:28.887 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:52:28 smithi194 bash[81885]: ts=2023-10-09T04:52:28.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi194.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi133:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:52:28.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:28 smithi194 bash[31648]: cluster 2023-10-09T04:52:27.331291+0000 mgr.y (mgr.25277) 13 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 762 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T04:52:28.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:28 smithi194 bash[31648]: audit 2023-10-09T04:52:28.402204+0000 mon.a (mon.0) 60 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:29.517 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 10 16 24085 24069 150.417 135.062 0.00892479 0.00664188 2023-10-09T04:52:30.517 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 11 16 26007 25991 147.662 120.125 0.00295653 0.00676479 2023-10-09T04:52:30.622 INFO:teuthology.orchestra.run.smithi133.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:52:30.837 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:30 smithi133 bash[102858]: cluster 2023-10-09T04:52:29.332055+0000 mgr.y (mgr.25277) 14 : cluster 0 pgmap v12: 89 pgs: 89 active+clean; 1.4 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T04:52:30.837 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:30 smithi133 bash[41291]: cluster 2023-10-09T04:52:29.332055+0000 mgr.y (mgr.25277) 14 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 1.4 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T04:52:30.845 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:30 smithi194 bash[31648]: cluster 2023-10-09T04:52:29.332055+0000 mgr.y (mgr.25277) 14 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 1.4 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T04:52:31.517 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 12 16 27659 27643 143.96 103.25 0.00227402 0.0068689 2023-10-09T04:52:31.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:31 smithi194 bash[31648]: audit 2023-10-09T04:52:30.604936+0000 mgr.y (mgr.25277) 15 : audit [DBG] from='client.34167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:52:31.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:31 smithi194 bash[31648]: cephadm 2023-10-09T04:52:30.605918+0000 mgr.y (mgr.25277) 16 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:52:31.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:31 smithi194 bash[31648]: audit 2023-10-09T04:52:30.617570+0000 mon.a (mon.0) 61 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:32.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:31 smithi133 bash[102858]: audit 2023-10-09T04:52:30.604936+0000 mgr.y (mgr.25277) 15 : audit 0 from='client.34167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:52:32.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:31 smithi133 bash[102858]: cephadm 2023-10-09T04:52:30.605918+0000 mgr.y (mgr.25277) 16 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:52:32.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:31 smithi133 bash[102858]: audit 2023-10-09T04:52:30.617570+0000 mon.a (mon.0) 61 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:32.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:31 smithi133 bash[41291]: audit 2023-10-09T04:52:30.604936+0000 mgr.y (mgr.25277) 15 : audit [DBG] from='client.34167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:52:32.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:31 smithi133 bash[41291]: cephadm 2023-10-09T04:52:30.605918+0000 mgr.y (mgr.25277) 16 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:52:32.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:31 smithi133 bash[41291]: audit 2023-10-09T04:52:30.617570+0000 mon.a (mon.0) 61 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:32.098 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T04:52:32.517 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 13 16 29239 29223 140.482 98.75 0.00583946 0.00711092 2023-10-09T04:52:32.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:32 smithi194 bash[31648]: cluster 2023-10-09T04:52:31.332450+0000 mgr.y (mgr.25277) 17 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 1.4 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T04:52:33.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:32 smithi133 bash[41291]: cluster 2023-10-09T04:52:31.332450+0000 mgr.y (mgr.25277) 17 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 1.4 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T04:52:33.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:32 smithi133 bash[102858]: cluster 2023-10-09T04:52:31.332450+0000 mgr.y (mgr.25277) 17 : cluster 0 pgmap v13: 89 pgs: 89 active+clean; 1.4 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T04:52:33.517 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 14 16 31069 31053 138.616 114.375 0.00627488 0.0072079 2023-10-09T04:52:34.286 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:33 smithi133 bash[102858]: cluster 2023-10-09T04:52:33.332939+0000 mgr.y (mgr.25277) 18 : cluster 0 pgmap v14: 89 pgs: 89 active+clean; 1.4 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T04:52:34.287 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:52:34] "GET /metrics HTTP/1.1" 200 36159 "" "Prometheus/2.43.0" 2023-10-09T04:52:34.287 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:33 smithi133 bash[41291]: cluster 2023-10-09T04:52:33.332939+0000 mgr.y (mgr.25277) 18 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 1.4 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T04:52:34.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:33 smithi194 bash[31648]: cluster 2023-10-09T04:52:33.332939+0000 mgr.y (mgr.25277) 18 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 1.4 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T04:52:34.518 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 15 16 32811 32795 136.633 108.875 0.00714316 0.00731267 2023-10-09T04:52:35.517 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 16 16 34486 34470 134.636 104.688 0.0137183 0.00742178 2023-10-09T04:52:36.517 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 17 16 36206 36190 133.039 107.5 0.004526 0.00751016 2023-10-09T04:52:36.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:36 smithi133 bash[41291]: cluster 2023-10-09T04:52:35.333692+0000 mgr.y (mgr.25277) 19 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 169 MiB/s wr, 2.71k op/s 2023-10-09T04:52:36.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:36 smithi133 bash[102858]: cluster 2023-10-09T04:52:35.333692+0000 mgr.y (mgr.25277) 19 : cluster 0 pgmap v15: 89 pgs: 89 active+clean; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 169 MiB/s wr, 2.71k op/s 2023-10-09T04:52:36.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:36 smithi194 bash[31648]: cluster 2023-10-09T04:52:35.333692+0000 mgr.y (mgr.25277) 19 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 169 MiB/s wr, 2.71k op/s 2023-10-09T04:52:37.001 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:52:37.518 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 18 16 37668 37652 130.724 91.375 0.00839179 0.00764339 2023-10-09T04:52:38.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:37 smithi133 bash[102858]: audit 2023-10-09T04:52:36.770231+0000 mon.a (mon.0) 62 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:38.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:37 smithi133 bash[102858]: audit 2023-10-09T04:52:36.780555+0000 mon.a (mon.0) 63 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:38.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:37 smithi133 bash[102858]: audit 2023-10-09T04:52:36.782818+0000 mon.a (mon.0) 64 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:52:38.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:37 smithi133 bash[102858]: audit 2023-10-09T04:52:36.784140+0000 mon.a (mon.0) 65 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:52:38.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:37 smithi133 bash[102858]: cephadm 2023-10-09T04:52:36.787694+0000 mgr.y (mgr.25277) 20 : cephadm 1 Updating smithi133:/etc/ceph/ceph.conf 2023-10-09T04:52:38.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:37 smithi133 bash[102858]: cephadm 2023-10-09T04:52:36.788533+0000 mgr.y (mgr.25277) 21 : cephadm 1 Updating smithi194:/etc/ceph/ceph.conf 2023-10-09T04:52:38.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:37 smithi133 bash[102858]: audit 2023-10-09T04:52:37.712226+0000 mon.a (mon.0) 66 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:38.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:37 smithi133 bash[102858]: audit 2023-10-09T04:52:37.721788+0000 mon.a (mon.0) 67 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:38.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:37 smithi133 bash[41291]: audit 2023-10-09T04:52:36.770231+0000 mon.a (mon.0) 62 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:38.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:37 smithi133 bash[41291]: audit 2023-10-09T04:52:36.780555+0000 mon.a (mon.0) 63 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:38.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:37 smithi133 bash[41291]: audit 2023-10-09T04:52:36.782818+0000 mon.a (mon.0) 64 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:52:38.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:37 smithi133 bash[41291]: audit 2023-10-09T04:52:36.784140+0000 mon.a (mon.0) 65 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:52:38.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:37 smithi133 bash[41291]: cephadm 2023-10-09T04:52:36.787694+0000 mgr.y (mgr.25277) 20 : cephadm [INF] Updating smithi133:/etc/ceph/ceph.conf 2023-10-09T04:52:38.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:37 smithi133 bash[41291]: cephadm 2023-10-09T04:52:36.788533+0000 mgr.y (mgr.25277) 21 : cephadm [INF] Updating smithi194:/etc/ceph/ceph.conf 2023-10-09T04:52:38.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:37 smithi133 bash[41291]: audit 2023-10-09T04:52:37.712226+0000 mon.a (mon.0) 66 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:38.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:37 smithi133 bash[41291]: audit 2023-10-09T04:52:37.721788+0000 mon.a (mon.0) 67 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:38.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:37 smithi194 bash[31648]: audit 2023-10-09T04:52:36.770231+0000 mon.a (mon.0) 62 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:38.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:37 smithi194 bash[31648]: audit 2023-10-09T04:52:36.780555+0000 mon.a (mon.0) 63 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:38.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:37 smithi194 bash[31648]: audit 2023-10-09T04:52:36.782818+0000 mon.a (mon.0) 64 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:52:38.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:37 smithi194 bash[31648]: audit 2023-10-09T04:52:36.784140+0000 mon.a (mon.0) 65 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:52:38.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:37 smithi194 bash[31648]: cephadm 2023-10-09T04:52:36.787694+0000 mgr.y (mgr.25277) 20 : cephadm [INF] Updating smithi133:/etc/ceph/ceph.conf 2023-10-09T04:52:38.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:37 smithi194 bash[31648]: cephadm 2023-10-09T04:52:36.788533+0000 mgr.y (mgr.25277) 21 : cephadm [INF] Updating smithi194:/etc/ceph/ceph.conf 2023-10-09T04:52:38.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:37 smithi194 bash[31648]: audit 2023-10-09T04:52:37.712226+0000 mon.a (mon.0) 66 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:38.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:37 smithi194 bash[31648]: audit 2023-10-09T04:52:37.721788+0000 mon.a (mon.0) 67 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:38.518 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 19 16 39202 39186 128.889 95.875 0.0108178 0.00775363 2023-10-09T04:52:39.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:38 smithi133 bash[41291]: cephadm 2023-10-09T04:52:37.002990+0000 mgr.y (mgr.25277) 22 : cephadm [INF] Updating smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:52:39.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:38 smithi133 bash[41291]: cephadm 2023-10-09T04:52:37.003237+0000 mgr.y (mgr.25277) 23 : cephadm [INF] Updating smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:52:39.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:38 smithi133 bash[41291]: cephadm 2023-10-09T04:52:37.218581+0000 mgr.y (mgr.25277) 24 : cephadm [INF] Updating smithi194:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:52:39.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:38 smithi133 bash[41291]: cephadm 2023-10-09T04:52:37.323437+0000 mgr.y (mgr.25277) 25 : cephadm [INF] Updating smithi133:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:52:39.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:38 smithi133 bash[41291]: cluster 2023-10-09T04:52:37.334429+0000 mgr.y (mgr.25277) 26 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-09T04:52:39.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:38 smithi133 bash[41291]: cephadm 2023-10-09T04:52:37.487715+0000 mgr.y (mgr.25277) 27 : cephadm [INF] Updating smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:52:39.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:38 smithi133 bash[41291]: cephadm 2023-10-09T04:52:37.639892+0000 mgr.y (mgr.25277) 28 : cephadm [INF] Updating smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:52:39.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:38 smithi133 bash[41291]: audit 2023-10-09T04:52:37.927198+0000 mon.a (mon.0) 68 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:39.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:38 smithi133 bash[41291]: audit 2023-10-09T04:52:37.945587+0000 mon.a (mon.0) 69 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:39.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:38 smithi133 bash[41291]: audit 2023-10-09T04:52:37.955822+0000 mon.a (mon.0) 70 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:39.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:38 smithi133 bash[41291]: audit 2023-10-09T04:52:38.041859+0000 mon.a (mon.0) 71 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:52:39.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:38 smithi133 bash[41291]: audit 2023-10-09T04:52:38.047287+0000 mon.a (mon.0) 72 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:52:39.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:38 smithi133 bash[41291]: audit 2023-10-09T04:52:38.049108+0000 mon.a (mon.0) 73 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:52:39.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:38 smithi133 bash[41291]: audit 2023-10-09T04:52:38.056348+0000 mon.a (mon.0) 74 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:39.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:38 smithi133 bash[102858]: cephadm 2023-10-09T04:52:37.002990+0000 mgr.y (mgr.25277) 22 : cephadm 1 Updating smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:52:39.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:38 smithi133 bash[102858]: cephadm 2023-10-09T04:52:37.003237+0000 mgr.y (mgr.25277) 23 : cephadm 1 Updating smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:52:39.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:38 smithi133 bash[102858]: cephadm 2023-10-09T04:52:37.218581+0000 mgr.y (mgr.25277) 24 : cephadm 1 Updating smithi194:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:52:39.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:38 smithi133 bash[102858]: cephadm 2023-10-09T04:52:37.323437+0000 mgr.y (mgr.25277) 25 : cephadm 1 Updating smithi133:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:52:39.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:38 smithi133 bash[102858]: cluster 2023-10-09T04:52:37.334429+0000 mgr.y (mgr.25277) 26 : cluster 0 pgmap v16: 89 pgs: 89 active+clean; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-09T04:52:39.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:38 smithi133 bash[102858]: cephadm 2023-10-09T04:52:37.487715+0000 mgr.y (mgr.25277) 27 : cephadm 1 Updating smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:52:39.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:38 smithi133 bash[102858]: cephadm 2023-10-09T04:52:37.639892+0000 mgr.y (mgr.25277) 28 : cephadm 1 Updating smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:52:39.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:38 smithi133 bash[102858]: audit 2023-10-09T04:52:37.927198+0000 mon.a (mon.0) 68 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:39.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:38 smithi133 bash[102858]: audit 2023-10-09T04:52:37.945587+0000 mon.a (mon.0) 69 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:39.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:38 smithi133 bash[102858]: audit 2023-10-09T04:52:37.955822+0000 mon.a (mon.0) 70 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:39.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:38 smithi133 bash[102858]: audit 2023-10-09T04:52:38.041859+0000 mon.a (mon.0) 71 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:52:39.054 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:38 smithi133 bash[102858]: audit 2023-10-09T04:52:38.047287+0000 mon.a (mon.0) 72 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:52:39.054 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:38 smithi133 bash[102858]: audit 2023-10-09T04:52:38.049108+0000 mon.a (mon.0) 73 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:52:39.054 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:38 smithi133 bash[102858]: audit 2023-10-09T04:52:38.056348+0000 mon.a (mon.0) 74 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:39.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:38 smithi194 bash[31648]: cephadm 2023-10-09T04:52:37.002990+0000 mgr.y (mgr.25277) 22 : cephadm [INF] Updating smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:52:39.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:38 smithi194 bash[31648]: cephadm 2023-10-09T04:52:37.003237+0000 mgr.y (mgr.25277) 23 : cephadm [INF] Updating smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:52:39.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:38 smithi194 bash[31648]: cephadm 2023-10-09T04:52:37.218581+0000 mgr.y (mgr.25277) 24 : cephadm [INF] Updating smithi194:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:52:39.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:38 smithi194 bash[31648]: cephadm 2023-10-09T04:52:37.323437+0000 mgr.y (mgr.25277) 25 : cephadm [INF] Updating smithi133:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:52:39.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:38 smithi194 bash[31648]: cluster 2023-10-09T04:52:37.334429+0000 mgr.y (mgr.25277) 26 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-09T04:52:39.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:38 smithi194 bash[31648]: cephadm 2023-10-09T04:52:37.487715+0000 mgr.y (mgr.25277) 27 : cephadm [INF] Updating smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:52:39.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:38 smithi194 bash[31648]: cephadm 2023-10-09T04:52:37.639892+0000 mgr.y (mgr.25277) 28 : cephadm [INF] Updating smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:52:39.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:38 smithi194 bash[31648]: audit 2023-10-09T04:52:37.927198+0000 mon.a (mon.0) 68 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:39.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:38 smithi194 bash[31648]: audit 2023-10-09T04:52:37.945587+0000 mon.a (mon.0) 69 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:39.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:38 smithi194 bash[31648]: audit 2023-10-09T04:52:37.955822+0000 mon.a (mon.0) 70 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:39.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:38 smithi194 bash[31648]: audit 2023-10-09T04:52:38.041859+0000 mon.a (mon.0) 71 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:52:39.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:38 smithi194 bash[31648]: audit 2023-10-09T04:52:38.047287+0000 mon.a (mon.0) 72 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:52:39.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:38 smithi194 bash[31648]: audit 2023-10-09T04:52:38.049108+0000 mon.a (mon.0) 73 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:52:39.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:38 smithi194 bash[31648]: audit 2023-10-09T04:52:38.056348+0000 mon.a (mon.0) 74 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:52:39.138 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:52:38 smithi194 bash[81885]: ts=2023-10-09T04:52:38.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi194.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi133:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:52:39.518 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T04:52:39.515129+0000 min lat: 0.00143184 max lat: 0.219204 avg lat: 0.00782804 2023-10-09T04:52:39.518 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:52:39.518 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 20 16 40869 40853 127.653 104.188 0.0086232 0.00782804 2023-10-09T04:52:40.518 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 21 16 42431 42415 126.223 97.625 0.00597399 0.00791693 2023-10-09T04:52:41.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:40 smithi133 bash[102858]: cluster 2023-10-09T04:52:39.335078+0000 mgr.y (mgr.25277) 29 : cluster 0 pgmap v17: 89 pgs: 89 active+clean; 2.4 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2023-10-09T04:52:41.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:40 smithi133 bash[41291]: cluster 2023-10-09T04:52:39.335078+0000 mgr.y (mgr.25277) 29 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 2.4 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2023-10-09T04:52:41.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:40 smithi194 bash[31648]: cluster 2023-10-09T04:52:39.335078+0000 mgr.y (mgr.25277) 29 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 2.4 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2023-10-09T04:52:41.518 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 22 16 43556 43540 123.681 70.3125 0.0125648 0.00807957 2023-10-09T04:52:42.518 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 23 16 45018 45002 122.276 91.375 0.00871361 0.00817137 2023-10-09T04:52:43.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:42 smithi133 bash[41291]: cluster 2023-10-09T04:52:41.335633+0000 mgr.y (mgr.25277) 30 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T04:52:43.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:42 smithi133 bash[102858]: cluster 2023-10-09T04:52:41.335633+0000 mgr.y (mgr.25277) 30 : cluster 0 pgmap v18: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T04:52:43.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:42 smithi194 bash[31648]: cluster 2023-10-09T04:52:41.335633+0000 mgr.y (mgr.25277) 30 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T04:52:43.518 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 24 16 46457 46441 120.928 89.9375 0.0122316 0.00826206 2023-10-09T04:52:44.283 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:52:44] "GET /metrics HTTP/1.1" 200 36159 "" "Prometheus/2.43.0" 2023-10-09T04:52:44.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:43 smithi133 bash[41291]: cluster 2023-10-09T04:52:43.336108+0000 mgr.y (mgr.25277) 31 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T04:52:44.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:43 smithi133 bash[41291]: audit 2023-10-09T04:52:43.403262+0000 mon.a (mon.0) 75 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:44.284 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:43 smithi133 bash[102858]: cluster 2023-10-09T04:52:43.336108+0000 mgr.y (mgr.25277) 31 : cluster 0 pgmap v19: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T04:52:44.284 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:43 smithi133 bash[102858]: audit 2023-10-09T04:52:43.403262+0000 mon.a (mon.0) 75 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:44.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:43 smithi194 bash[31648]: cluster 2023-10-09T04:52:43.336108+0000 mgr.y (mgr.25277) 31 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T04:52:44.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:43 smithi194 bash[31648]: audit 2023-10-09T04:52:43.403262+0000 mon.a (mon.0) 75 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:44.518 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 25 16 47916 47900 119.738 91.1875 0.00771961 0.0083449 2023-10-09T04:52:45.518 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 26 16 49352 49336 118.585 89.75 0.0081264 0.00842663 2023-10-09T04:52:46.518 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 27 16 50763 50747 117.459 88.1875 0.00779155 0.00850787 2023-10-09T04:52:46.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:46 smithi133 bash[41291]: cluster 2023-10-09T04:52:45.336838+0000 mgr.y (mgr.25277) 32 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-10-09T04:52:46.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:46 smithi133 bash[102858]: cluster 2023-10-09T04:52:45.336838+0000 mgr.y (mgr.25277) 32 : cluster 0 pgmap v20: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-10-09T04:52:46.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:46 smithi194 bash[31648]: cluster 2023-10-09T04:52:45.336838+0000 mgr.y (mgr.25277) 32 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-10-09T04:52:47.518 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 28 16 52151 52135 116.361 86.75 0.0085339 0.00858866 2023-10-09T04:52:48.518 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 29 16 53537 53521 115.336 86.625 0.00777764 0.00866413 2023-10-09T04:52:48.662 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:48 smithi194 bash[31648]: cluster 2023-10-09T04:52:47.337282+0000 mgr.y (mgr.25277) 33 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T04:52:48.663 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:52:48 smithi194 bash[81885]: ts=2023-10-09T04:52:48.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi194.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi133:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:52:48.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:48 smithi133 bash[102858]: cluster 2023-10-09T04:52:47.337282+0000 mgr.y (mgr.25277) 33 : cluster 0 pgmap v21: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T04:52:48.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:48 smithi133 bash[41291]: cluster 2023-10-09T04:52:47.337282+0000 mgr.y (mgr.25277) 33 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T04:52:49.519 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 30 16 54880 54864 114.289 83.9375 0.00705306 0.00874439 2023-10-09T04:52:50.519 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 31 16 56211 56195 113.285 83.1875 0.00586333 0.00882007 2023-10-09T04:52:50.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:50 smithi133 bash[41291]: cluster 2023-10-09T04:52:49.337915+0000 mgr.y (mgr.25277) 34 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2023-10-09T04:52:50.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:50 smithi133 bash[102858]: cluster 2023-10-09T04:52:49.337915+0000 mgr.y (mgr.25277) 34 : cluster 0 pgmap v22: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2023-10-09T04:52:50.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:50 smithi194 bash[31648]: cluster 2023-10-09T04:52:49.337915+0000 mgr.y (mgr.25277) 34 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2023-10-09T04:52:51.519 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 32 16 57512 57496 112.286 81.3125 0.00937593 0.00889996 2023-10-09T04:52:52.519 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 33 16 58807 58791 111.336 80.9375 0.0276655 0.00897576 2023-10-09T04:52:52.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:52 smithi133 bash[41291]: cluster 2023-10-09T04:52:51.338462+0000 mgr.y (mgr.25277) 35 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T04:52:52.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:52 smithi133 bash[102858]: cluster 2023-10-09T04:52:51.338462+0000 mgr.y (mgr.25277) 35 : cluster 0 pgmap v23: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T04:52:52.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:52 smithi194 bash[31648]: cluster 2023-10-09T04:52:51.338462+0000 mgr.y (mgr.25277) 35 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T04:52:53.519 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 34 16 60090 60074 110.419 80.1875 0.00942415 0.00905019 2023-10-09T04:52:54.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:53 smithi133 bash[102858]: cluster 2023-10-09T04:52:53.338947+0000 mgr.y (mgr.25277) 36 : cluster 0 pgmap v24: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T04:52:54.283 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:52:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:52:54] "GET /metrics HTTP/1.1" 200 36159 "" "Prometheus/2.43.0" 2023-10-09T04:52:54.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:53 smithi133 bash[41291]: cluster 2023-10-09T04:52:53.338947+0000 mgr.y (mgr.25277) 36 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T04:52:54.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:53 smithi194 bash[31648]: cluster 2023-10-09T04:52:53.338947+0000 mgr.y (mgr.25277) 36 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T04:52:54.519 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 35 16 61365 61349 109.541 79.6875 0.00997743 0.00912329 2023-10-09T04:52:55.519 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 36 16 62604 62588 108.649 77.4375 0.00756813 0.00919753 2023-10-09T04:52:56.519 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 37 15 63817 63802 107.763 75.875 0.0144003 0.00927421 2023-10-09T04:52:56.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:56 smithi133 bash[102858]: cluster 2023-10-09T04:52:55.339862+0000 mgr.y (mgr.25277) 37 : cluster 0 pgmap v25: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-10-09T04:52:56.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:56 smithi133 bash[41291]: cluster 2023-10-09T04:52:55.339862+0000 mgr.y (mgr.25277) 37 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-10-09T04:52:56.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:56 smithi194 bash[31648]: cluster 2023-10-09T04:52:55.339862+0000 mgr.y (mgr.25277) 37 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-10-09T04:52:57.519 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 38 16 65038 65022 106.934 76.25 0.00601981 0.00934534 2023-10-09T04:52:58.519 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 39 16 66228 66212 106.099 74.375 0.0138744 0.00941976 2023-10-09T04:52:58.663 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:58 smithi194 bash[31648]: cluster 2023-10-09T04:52:57.340306+0000 mgr.y (mgr.25277) 38 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T04:52:58.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:58 smithi133 bash[102858]: cluster 2023-10-09T04:52:57.340306+0000 mgr.y (mgr.25277) 38 : cluster 0 pgmap v26: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T04:52:58.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:58 smithi133 bash[41291]: cluster 2023-10-09T04:52:57.340306+0000 mgr.y (mgr.25277) 38 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T04:52:59.137 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:52:58 smithi194 bash[81885]: ts=2023-10-09T04:52:58.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi194.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi133:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:52:59.520 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T04:52:59.517069+0000 min lat: 0.00143184 max lat: 0.319036 avg lat: 0.00949167 2023-10-09T04:52:59.520 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:52:59.520 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 40 16 67412 67396 105.296 74 0.0236067 0.00949167 2023-10-09T04:52:59.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:52:59 smithi133 bash[102858]: audit 2023-10-09T04:52:58.408280+0000 mon.a (mon.0) 76 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:59.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:52:59 smithi133 bash[41291]: audit 2023-10-09T04:52:58.408280+0000 mon.a (mon.0) 76 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:52:59 smithi194 bash[31648]: audit 2023-10-09T04:52:58.408280+0000 mon.a (mon.0) 76 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:00.520 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 41 16 68543 68527 104.452 70.6875 0.0180174 0.00956756 2023-10-09T04:53:00.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:00 smithi133 bash[102858]: cluster 2023-10-09T04:52:59.341080+0000 mgr.y (mgr.25277) 39 : cluster 0 pgmap v27: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-09T04:53:00.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:00 smithi133 bash[41291]: cluster 2023-10-09T04:52:59.341080+0000 mgr.y (mgr.25277) 39 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-09T04:53:00.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:00 smithi194 bash[31648]: cluster 2023-10-09T04:52:59.341080+0000 mgr.y (mgr.25277) 39 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-09T04:53:01.520 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 42 16 69698 69682 103.683 72.1875 0.00936273 0.009639 2023-10-09T04:53:02.520 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 43 16 70776 70760 102.839 67.375 0.0143301 0.00971791 2023-10-09T04:53:02.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:02 smithi133 bash[102858]: cluster 2023-10-09T04:53:01.341642+0000 mgr.y (mgr.25277) 40 : cluster 0 pgmap v28: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T04:53:02.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:02 smithi133 bash[41291]: cluster 2023-10-09T04:53:01.341642+0000 mgr.y (mgr.25277) 40 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T04:53:02.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:02 smithi194 bash[31648]: cluster 2023-10-09T04:53:01.341642+0000 mgr.y (mgr.25277) 40 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T04:53:03.520 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 44 16 71907 71891 102.108 70.6875 0.00475739 0.00978769 2023-10-09T04:53:04.282 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:03 smithi133 bash[102858]: cluster 2023-10-09T04:53:03.342212+0000 mgr.y (mgr.25277) 41 : cluster 0 pgmap v29: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-10-09T04:53:04.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:03 smithi133 bash[41291]: cluster 2023-10-09T04:53:03.342212+0000 mgr.y (mgr.25277) 41 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-10-09T04:53:04.283 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:53:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:53:04] "GET /metrics HTTP/1.1" 200 36162 "" "Prometheus/2.43.0" 2023-10-09T04:53:04.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:03 smithi194 bash[31648]: cluster 2023-10-09T04:53:03.342212+0000 mgr.y (mgr.25277) 41 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-10-09T04:53:04.520 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 45 16 73062 73046 101.443 72.1875 0.0105764 0.00985219 2023-10-09T04:53:05.520 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 46 16 74212 74196 100.8 71.875 0.00996441 0.00991454 2023-10-09T04:53:06.520 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 47 16 75357 75341 100.178 71.5625 0.00935068 0.00997662 2023-10-09T04:53:06.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:06 smithi133 bash[41291]: cluster 2023-10-09T04:53:05.342974+0000 mgr.y (mgr.25277) 42 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T04:53:06.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:06 smithi133 bash[102858]: cluster 2023-10-09T04:53:05.342974+0000 mgr.y (mgr.25277) 42 : cluster 0 pgmap v30: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T04:53:06.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:06 smithi194 bash[31648]: cluster 2023-10-09T04:53:05.342974+0000 mgr.y (mgr.25277) 42 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T04:53:07.520 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 48 16 76464 76448 99.5321 69.1875 0.0208074 0.010041 2023-10-09T04:53:08.520 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 49 16 77570 77554 98.9114 69.125 0.0161729 0.0101048 2023-10-09T04:53:08.663 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:08 smithi194 bash[31648]: cluster 2023-10-09T04:53:07.343427+0000 mgr.y (mgr.25277) 43 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s 2023-10-09T04:53:08.664 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:53:08 smithi194 bash[81885]: ts=2023-10-09T04:53:08.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi194.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi133:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:53:08.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:08 smithi133 bash[102858]: cluster 2023-10-09T04:53:07.343427+0000 mgr.y (mgr.25277) 43 : cluster 0 pgmap v31: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s 2023-10-09T04:53:08.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:08 smithi133 bash[41291]: cluster 2023-10-09T04:53:07.343427+0000 mgr.y (mgr.25277) 43 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s 2023-10-09T04:53:09.520 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 50 16 78636 78620 98.2656 66.625 0.0119768 0.0101711 2023-10-09T04:53:10.520 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 51 16 79688 79672 97.6279 65.75 0.00918575 0.0102373 2023-10-09T04:53:10.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:10 smithi133 bash[102858]: cluster 2023-10-09T04:53:09.344038+0000 mgr.y (mgr.25277) 44 : cluster 0 pgmap v32: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T04:53:10.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:10 smithi133 bash[41291]: cluster 2023-10-09T04:53:09.344038+0000 mgr.y (mgr.25277) 44 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T04:53:10.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:10 smithi194 bash[31648]: cluster 2023-10-09T04:53:09.344038+0000 mgr.y (mgr.25277) 44 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T04:53:11.521 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 52 16 80729 80713 97.0015 65.0625 0.0128568 0.0103037 2023-10-09T04:53:12.521 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 53 16 81752 81736 96.3776 63.9375 0.0121411 0.0103695 2023-10-09T04:53:12.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:12 smithi133 bash[41291]: cluster 2023-10-09T04:53:11.344730+0000 mgr.y (mgr.25277) 45 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T04:53:12.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:12 smithi133 bash[102858]: cluster 2023-10-09T04:53:11.344730+0000 mgr.y (mgr.25277) 45 : cluster 0 pgmap v33: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T04:53:12.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:12 smithi194 bash[31648]: cluster 2023-10-09T04:53:11.344730+0000 mgr.y (mgr.25277) 45 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T04:53:13.521 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 54 16 82757 82741 95.7559 62.8125 0.0157835 0.0104376 2023-10-09T04:53:14.283 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:53:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:53:14] "GET /metrics HTTP/1.1" 200 36161 "" "Prometheus/2.43.0" 2023-10-09T04:53:14.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:13 smithi133 bash[41291]: cluster 2023-10-09T04:53:13.345187+0000 mgr.y (mgr.25277) 46 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 938 op/s 2023-10-09T04:53:14.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:13 smithi133 bash[41291]: audit 2023-10-09T04:53:13.407455+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:14.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:13 smithi133 bash[102858]: cluster 2023-10-09T04:53:13.345187+0000 mgr.y (mgr.25277) 46 : cluster 0 pgmap v34: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 938 op/s 2023-10-09T04:53:14.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:13 smithi133 bash[102858]: audit 2023-10-09T04:53:13.407455+0000 mon.a (mon.0) 77 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:14.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:13 smithi194 bash[31648]: cluster 2023-10-09T04:53:13.345187+0000 mgr.y (mgr.25277) 46 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 938 op/s 2023-10-09T04:53:14.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:13 smithi194 bash[31648]: audit 2023-10-09T04:53:13.407455+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:14.521 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 55 16 83789 83773 95.1875 64.5 0.00804755 0.0104996 2023-10-09T04:53:15.521 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 56 16 84804 84788 94.6204 63.4375 0.00650296 0.0105627 2023-10-09T04:53:16.521 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 57 16 85825 85809 94.0798 63.8125 0.0106144 0.0106228 2023-10-09T04:53:16.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:16 smithi133 bash[102858]: cluster 2023-10-09T04:53:15.346168+0000 mgr.y (mgr.25277) 47 : cluster 0 pgmap v35: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 85 MiB/s wr, 1.37k op/s 2023-10-09T04:53:16.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:16 smithi133 bash[41291]: cluster 2023-10-09T04:53:15.346168+0000 mgr.y (mgr.25277) 47 : cluster [DBG] pgmap v35: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 85 MiB/s wr, 1.37k op/s 2023-10-09T04:53:16.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:16 smithi194 bash[31648]: cluster 2023-10-09T04:53:15.346168+0000 mgr.y (mgr.25277) 47 : cluster [DBG] pgmap v35: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 85 MiB/s wr, 1.37k op/s 2023-10-09T04:53:17.521 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 58 16 86833 86817 93.5438 63 0.00866386 0.0106844 2023-10-09T04:53:18.521 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 59 16 87809 87793 92.9921 61 0.0139119 0.0107482 2023-10-09T04:53:18.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:18 smithi133 bash[41291]: cluster 2023-10-09T04:53:17.346587+0000 mgr.y (mgr.25277) 48 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 56 MiB/s wr, 895 op/s 2023-10-09T04:53:18.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:18 smithi133 bash[102858]: cluster 2023-10-09T04:53:17.346587+0000 mgr.y (mgr.25277) 48 : cluster 0 pgmap v36: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 56 MiB/s wr, 895 op/s 2023-10-09T04:53:18.887 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:53:18 smithi194 bash[81885]: ts=2023-10-09T04:53:18.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi194.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi133:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:53:18.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:18 smithi194 bash[31648]: cluster 2023-10-09T04:53:17.346587+0000 mgr.y (mgr.25277) 48 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 56 MiB/s wr, 895 op/s 2023-10-09T04:53:19.521 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T04:53:19.518993+0000 min lat: 0.00143184 max lat: 0.319036 avg lat: 0.0108094 2023-10-09T04:53:19.521 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:53:19.522 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 60 16 88786 88770 92.4598 61.0625 0.0139883 0.0108094 2023-10-09T04:53:20.523 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 61 16 89767 89751 91.9491 61.3125 0.00966561 0.0108694 2023-10-09T04:53:20.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:20 smithi133 bash[41291]: cluster 2023-10-09T04:53:19.347064+0000 mgr.y (mgr.25277) 49 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s 2023-10-09T04:53:20.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:20 smithi133 bash[102858]: cluster 2023-10-09T04:53:19.347064+0000 mgr.y (mgr.25277) 49 : cluster 0 pgmap v37: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s 2023-10-09T04:53:20.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:20 smithi194 bash[31648]: cluster 2023-10-09T04:53:19.347064+0000 mgr.y (mgr.25277) 49 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s 2023-10-09T04:53:21.521 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 62 16 90726 90710 91.4327 59.9375 0.00734423 0.0109314 2023-10-09T04:53:22.522 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 63 16 91664 91648 90.9119 58.625 0.0209952 0.0109934 2023-10-09T04:53:22.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:22 smithi133 bash[102858]: cluster 2023-10-09T04:53:21.347775+0000 mgr.y (mgr.25277) 50 : cluster 0 pgmap v38: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T04:53:22.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:22 smithi133 bash[41291]: cluster 2023-10-09T04:53:21.347775+0000 mgr.y (mgr.25277) 50 : cluster [DBG] pgmap v38: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T04:53:22.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:22 smithi194 bash[31648]: cluster 2023-10-09T04:53:21.347775+0000 mgr.y (mgr.25277) 50 : cluster [DBG] pgmap v38: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T04:53:23.522 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 64 16 92610 92594 90.4151 59.125 0.00580296 0.0110543 2023-10-09T04:53:24.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:23 smithi133 bash[102858]: cluster 2023-10-09T04:53:23.348221+0000 mgr.y (mgr.25277) 51 : cluster 0 pgmap v39: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s 2023-10-09T04:53:24.283 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:53:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:53:24] "GET /metrics HTTP/1.1" 200 36161 "" "Prometheus/2.43.0" 2023-10-09T04:53:24.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:23 smithi133 bash[41291]: cluster 2023-10-09T04:53:23.348221+0000 mgr.y (mgr.25277) 51 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s 2023-10-09T04:53:24.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:23 smithi194 bash[31648]: cluster 2023-10-09T04:53:23.348221+0000 mgr.y (mgr.25277) 51 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s 2023-10-09T04:53:24.522 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 65 16 93563 93547 89.9404 59.5625 0.025302 0.0111136 2023-10-09T04:53:25.522 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 66 16 94507 94491 89.4715 59 0.0253479 0.0111702 2023-10-09T04:53:26.522 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 67 16 95434 95418 89.0007 57.9375 0.0191255 0.0112303 2023-10-09T04:53:26.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:26 smithi133 bash[102858]: cluster 2023-10-09T04:53:25.348832+0000 mgr.y (mgr.25277) 52 : cluster 0 pgmap v40: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T04:53:26.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:26 smithi133 bash[41291]: cluster 2023-10-09T04:53:25.348832+0000 mgr.y (mgr.25277) 52 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T04:53:26.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:26 smithi194 bash[31648]: cluster 2023-10-09T04:53:25.348832+0000 mgr.y (mgr.25277) 52 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T04:53:27.522 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 68 16 96354 96338 88.5374 57.5 0.0105754 0.0112892 2023-10-09T04:53:28.522 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 69 16 97279 97263 88.092 57.8125 0.0320927 0.011346 2023-10-09T04:53:28.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:28 smithi133 bash[102858]: cluster 2023-10-09T04:53:27.349247+0000 mgr.y (mgr.25277) 53 : cluster 0 pgmap v41: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s 2023-10-09T04:53:28.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:28 smithi133 bash[102858]: audit 2023-10-09T04:53:28.407824+0000 mon.a (mon.0) 78 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:28.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:28 smithi133 bash[41291]: cluster 2023-10-09T04:53:27.349247+0000 mgr.y (mgr.25277) 53 : cluster [DBG] pgmap v41: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s 2023-10-09T04:53:28.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:28 smithi133 bash[41291]: audit 2023-10-09T04:53:28.407824+0000 mon.a (mon.0) 78 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:28.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:28 smithi194 bash[31648]: cluster 2023-10-09T04:53:27.349247+0000 mgr.y (mgr.25277) 53 : cluster [DBG] pgmap v41: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s 2023-10-09T04:53:28.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:28 smithi194 bash[31648]: audit 2023-10-09T04:53:28.407824+0000 mon.a (mon.0) 78 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:28.887 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:53:28 smithi194 bash[81885]: ts=2023-10-09T04:53:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi194.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi133:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:53:29.522 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 70 16 98192 98176 87.6487 57.0625 0.0195237 0.0114039 2023-10-09T04:53:30.522 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 71 16 99105 99089 87.2178 57.0625 0.032582 0.0114596 2023-10-09T04:53:30.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:30 smithi133 bash[102858]: cluster 2023-10-09T04:53:29.349970+0000 mgr.y (mgr.25277) 54 : cluster 0 pgmap v42: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s 2023-10-09T04:53:30.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:30 smithi133 bash[41291]: cluster 2023-10-09T04:53:29.349970+0000 mgr.y (mgr.25277) 54 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s 2023-10-09T04:53:30.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:30 smithi194 bash[31648]: cluster 2023-10-09T04:53:29.349970+0000 mgr.y (mgr.25277) 54 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s 2023-10-09T04:53:31.522 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 72 16 100022 100006 86.8024 57.3125 0.0258826 0.0115142 2023-10-09T04:53:32.522 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 73 16 100906 100890 86.3701 55.25 0.0125142 0.0115726 2023-10-09T04:53:32.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:32 smithi133 bash[41291]: cluster 2023-10-09T04:53:31.350928+0000 mgr.y (mgr.25277) 55 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T04:53:32.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:32 smithi133 bash[102858]: cluster 2023-10-09T04:53:31.350928+0000 mgr.y (mgr.25277) 55 : cluster 0 pgmap v43: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T04:53:32.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:32 smithi194 bash[31648]: cluster 2023-10-09T04:53:31.350928+0000 mgr.y (mgr.25277) 55 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T04:53:33.522 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 74 16 101807 101791 85.9638 56.3125 0.0208233 0.0116271 2023-10-09T04:53:34.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:33 smithi133 bash[102858]: cluster 2023-10-09T04:53:33.351367+0000 mgr.y (mgr.25277) 56 : cluster 0 pgmap v44: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s 2023-10-09T04:53:34.283 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:53:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:53:34] "GET /metrics HTTP/1.1" 200 36157 "" "Prometheus/2.43.0" 2023-10-09T04:53:34.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:33 smithi133 bash[41291]: cluster 2023-10-09T04:53:33.351367+0000 mgr.y (mgr.25277) 56 : cluster [DBG] pgmap v44: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s 2023-10-09T04:53:34.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:33 smithi194 bash[31648]: cluster 2023-10-09T04:53:33.351367+0000 mgr.y (mgr.25277) 56 : cluster [DBG] pgmap v44: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s 2023-10-09T04:53:34.523 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 75 16 102687 102671 85.5509 55 0.00887934 0.0116831 2023-10-09T04:53:35.523 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 76 16 103578 103562 85.1579 55.6875 0.0129836 0.0117371 2023-10-09T04:53:36.523 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 77 16 104476 104460 84.7808 56.125 0.00730684 0.0117894 2023-10-09T04:53:36.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:36 smithi133 bash[102858]: cluster 2023-10-09T04:53:35.352091+0000 mgr.y (mgr.25277) 57 : cluster 0 pgmap v45: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-10-09T04:53:36.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:36 smithi133 bash[41291]: cluster 2023-10-09T04:53:35.352091+0000 mgr.y (mgr.25277) 57 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-10-09T04:53:36.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:36 smithi194 bash[31648]: cluster 2023-10-09T04:53:35.352091+0000 mgr.y (mgr.25277) 57 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-10-09T04:53:37.523 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 78 16 105340 105324 84.3861 54 0.0268975 0.0118446 2023-10-09T04:53:38.529 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 79 16 106191 106175 83.9842 53.1875 0.0167487 0.0119005 2023-10-09T04:53:38.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:38 smithi133 bash[41291]: cluster 2023-10-09T04:53:37.352491+0000 mgr.y (mgr.25277) 58 : cluster [DBG] pgmap v46: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 761 op/s 2023-10-09T04:53:38.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:38 smithi133 bash[41291]: audit 2023-10-09T04:53:38.145482+0000 mon.a (mon.0) 79 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:53:38.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:38 smithi133 bash[102858]: cluster 2023-10-09T04:53:37.352491+0000 mgr.y (mgr.25277) 58 : cluster 0 pgmap v46: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 761 op/s 2023-10-09T04:53:38.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:38 smithi133 bash[102858]: audit 2023-10-09T04:53:38.145482+0000 mon.a (mon.0) 79 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:53:38.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:38 smithi194 bash[31648]: cluster 2023-10-09T04:53:37.352491+0000 mgr.y (mgr.25277) 58 : cluster [DBG] pgmap v46: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 761 op/s 2023-10-09T04:53:38.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:38 smithi194 bash[31648]: audit 2023-10-09T04:53:38.145482+0000 mon.a (mon.0) 79 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:53:38.888 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:53:38 smithi194 bash[81885]: ts=2023-10-09T04:53:38.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi194.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi133:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:53:39.529 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T04:53:39.527459+0000 min lat: 0.00143184 max lat: 0.319036 avg lat: 0.0119573 2023-10-09T04:53:39.530 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:53:39.530 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 80 16 107027 107011 83.5874 52.25 0.00867124 0.0119573 2023-10-09T04:53:39.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:39 smithi194 bash[31648]: audit 2023-10-09T04:53:38.627888+0000 mon.a (mon.0) 80 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:53:39.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:39 smithi194 bash[31648]: audit 2023-10-09T04:53:38.638776+0000 mon.a (mon.0) 81 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:53:40.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:39 smithi133 bash[102858]: audit 2023-10-09T04:53:38.627888+0000 mon.a (mon.0) 80 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:53:40.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:39 smithi133 bash[102858]: audit 2023-10-09T04:53:38.638776+0000 mon.a (mon.0) 81 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:53:40.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:39 smithi133 bash[41291]: audit 2023-10-09T04:53:38.627888+0000 mon.a (mon.0) 80 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:53:40.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:39 smithi133 bash[41291]: audit 2023-10-09T04:53:38.638776+0000 mon.a (mon.0) 81 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:53:40.530 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 81 16 107883 107867 83.216 53.5 0.0137396 0.0120104 2023-10-09T04:53:41.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:40 smithi133 bash[102858]: cluster 2023-10-09T04:53:39.352952+0000 mgr.y (mgr.25277) 59 : cluster 0 pgmap v47: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 761 op/s 2023-10-09T04:53:41.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:40 smithi133 bash[41291]: cluster 2023-10-09T04:53:39.352952+0000 mgr.y (mgr.25277) 59 : cluster [DBG] pgmap v47: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 761 op/s 2023-10-09T04:53:41.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:40 smithi194 bash[31648]: cluster 2023-10-09T04:53:39.352952+0000 mgr.y (mgr.25277) 59 : cluster [DBG] pgmap v47: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 761 op/s 2023-10-09T04:53:41.530 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 82 16 108708 108692 82.8299 51.5625 0.0103251 0.0120673 2023-10-09T04:53:42.530 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 83 16 109539 109523 82.4577 51.9375 0.0149641 0.012122 2023-10-09T04:53:43.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:42 smithi133 bash[102858]: cluster 2023-10-09T04:53:41.353634+0000 mgr.y (mgr.25277) 60 : cluster 0 pgmap v48: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T04:53:43.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:42 smithi133 bash[41291]: cluster 2023-10-09T04:53:41.353634+0000 mgr.y (mgr.25277) 60 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T04:53:43.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:42 smithi194 bash[31648]: cluster 2023-10-09T04:53:41.353634+0000 mgr.y (mgr.25277) 60 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T04:53:43.530 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 84 16 110324 110308 82.0601 49.0625 0.0141823 0.0121802 2023-10-09T04:53:43.957 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:43 smithi133 bash[102858]: cluster 2023-10-09T04:53:43.354080+0000 mgr.y (mgr.25277) 61 : cluster 0 pgmap v49: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 46 MiB/s wr, 737 op/s 2023-10-09T04:53:43.957 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:43 smithi133 bash[102858]: audit 2023-10-09T04:53:43.408196+0000 mon.a (mon.0) 82 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:44.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:43 smithi133 bash[41291]: cluster 2023-10-09T04:53:43.354080+0000 mgr.y (mgr.25277) 61 : cluster [DBG] pgmap v49: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 46 MiB/s wr, 737 op/s 2023-10-09T04:53:44.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:43 smithi133 bash[41291]: audit 2023-10-09T04:53:43.408196+0000 mon.a (mon.0) 82 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:44.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:43 smithi194 bash[31648]: cluster 2023-10-09T04:53:43.354080+0000 mgr.y (mgr.25277) 61 : cluster [DBG] pgmap v49: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 46 MiB/s wr, 737 op/s 2023-10-09T04:53:44.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:43 smithi194 bash[31648]: audit 2023-10-09T04:53:43.408196+0000 mon.a (mon.0) 82 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:44.530 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 85 16 111009 110993 81.5983 42.8125 0.0086451 0.0122491 2023-10-09T04:53:44.550 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:53:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:53:44] "GET /metrics HTTP/1.1" 200 36157 "" "Prometheus/2.43.0" 2023-10-09T04:53:45.530 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 86 16 111776 111760 81.2069 47.9375 0.00776246 0.0123078 2023-10-09T04:53:45.575 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T04:53:45.803 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:45 smithi133 bash[102858]: audit 2023-10-09T04:53:44.452928+0000 mon.a (mon.0) 83 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:53:45.804 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:45 smithi133 bash[102858]: audit 2023-10-09T04:53:44.464440+0000 mon.a (mon.0) 84 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:53:45.804 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:45 smithi133 bash[102858]: cluster 2023-10-09T04:53:44.972989+0000 mon.a (mon.0) 85 : cluster 0 mgrmap e35: y(active, since 91s), standbys: x 2023-10-09T04:53:45.804 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:45 smithi133 bash[102858]: audit 2023-10-09T04:53:45.355010+0000 mon.a (mon.0) 86 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:53:45.804 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:45 smithi133 bash[102858]: audit 2023-10-09T04:53:45.363450+0000 mon.a (mon.0) 87 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:53:45.804 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:45 smithi133 bash[41291]: audit 2023-10-09T04:53:44.452928+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:53:45.804 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:45 smithi133 bash[41291]: audit 2023-10-09T04:53:44.464440+0000 mon.a (mon.0) 84 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:53:45.804 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:45 smithi133 bash[41291]: cluster 2023-10-09T04:53:44.972989+0000 mon.a (mon.0) 85 : cluster [DBG] mgrmap e35: y(active, since 91s), standbys: x 2023-10-09T04:53:45.804 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:45 smithi133 bash[41291]: audit 2023-10-09T04:53:45.355010+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:53:45.805 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:45 smithi133 bash[41291]: audit 2023-10-09T04:53:45.363450+0000 mon.a (mon.0) 87 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:53:45.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:45 smithi194 bash[31648]: audit 2023-10-09T04:53:44.452928+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:53:45.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:45 smithi194 bash[31648]: audit 2023-10-09T04:53:44.464440+0000 mon.a (mon.0) 84 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:53:45.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:45 smithi194 bash[31648]: cluster 2023-10-09T04:53:44.972989+0000 mon.a (mon.0) 85 : cluster [DBG] mgrmap e35: y(active, since 91s), standbys: x 2023-10-09T04:53:45.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:45 smithi194 bash[31648]: audit 2023-10-09T04:53:45.355010+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:53:45.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:45 smithi194 bash[31648]: audit 2023-10-09T04:53:45.363450+0000 mon.a (mon.0) 87 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:53:46.530 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 87 16 112535 112519 80.8187 47.4375 0.00490505 0.0123667 2023-10-09T04:53:46.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:46 smithi133 bash[41291]: cluster 2023-10-09T04:53:45.354762+0000 mgr.y (mgr.25277) 62 : cluster [DBG] pgmap v50: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:53:46.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:46 smithi133 bash[41291]: audit 2023-10-09T04:53:45.951916+0000 mon.a (mon.0) 88 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:53:46.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:46 smithi133 bash[41291]: audit 2023-10-09T04:53:45.952991+0000 mon.a (mon.0) 89 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:53:46.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:46 smithi133 bash[41291]: audit 2023-10-09T04:53:45.961194+0000 mon.a (mon.0) 90 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:53:46.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:46 smithi133 bash[102858]: cluster 2023-10-09T04:53:45.354762+0000 mgr.y (mgr.25277) 62 : cluster 0 pgmap v50: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:53:46.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:46 smithi133 bash[102858]: audit 2023-10-09T04:53:45.951916+0000 mon.a (mon.0) 88 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:53:46.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:46 smithi133 bash[102858]: audit 2023-10-09T04:53:45.952991+0000 mon.a (mon.0) 89 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:53:46.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:46 smithi133 bash[102858]: audit 2023-10-09T04:53:45.961194+0000 mon.a (mon.0) 90 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:53:46.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:46 smithi194 bash[31648]: cluster 2023-10-09T04:53:45.354762+0000 mgr.y (mgr.25277) 62 : cluster [DBG] pgmap v50: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:53:46.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:46 smithi194 bash[31648]: audit 2023-10-09T04:53:45.951916+0000 mon.a (mon.0) 88 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:53:46.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:46 smithi194 bash[31648]: audit 2023-10-09T04:53:45.952991+0000 mon.a (mon.0) 89 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:53:46.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:46 smithi194 bash[31648]: audit 2023-10-09T04:53:45.961194+0000 mon.a (mon.0) 90 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:53:47.530 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 88 16 113354 113338 80.482 51.1875 0.0407302 0.0124197 2023-10-09T04:53:48.530 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 89 16 114154 114138 80.1395 50 0.0217092 0.0124726 2023-10-09T04:53:48.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:48 smithi133 bash[41291]: cluster 2023-10-09T04:53:47.355183+0000 mgr.y (mgr.25277) 63 : cluster [DBG] pgmap v51: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 695 op/s 2023-10-09T04:53:48.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:48 smithi133 bash[102858]: cluster 2023-10-09T04:53:47.355183+0000 mgr.y (mgr.25277) 63 : cluster 0 pgmap v51: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 695 op/s 2023-10-09T04:53:48.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:48 smithi194 bash[31648]: cluster 2023-10-09T04:53:47.355183+0000 mgr.y (mgr.25277) 63 : cluster [DBG] pgmap v51: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 695 op/s 2023-10-09T04:53:48.887 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:53:48 smithi194 bash[81885]: ts=2023-10-09T04:53:48.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi194.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi133:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:53:49.531 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Total time run: 90.0097 2023-10-09T04:53:49.531 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Total writes made: 114939 2023-10-09T04:53:49.531 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Write size: 65536 2023-10-09T04:53:49.531 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Object size: 65536 2023-10-09T04:53:49.531 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Bandwidth (MB/sec): 79.8101 2023-10-09T04:53:49.532 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Stddev Bandwidth: 30.5123 2023-10-09T04:53:49.532 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Max bandwidth (MB/sec): 163.875 2023-10-09T04:53:49.532 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Min bandwidth (MB/sec): 42.8125 2023-10-09T04:53:49.532 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Average IOPS: 1276 2023-10-09T04:53:49.532 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Stddev IOPS: 488.197 2023-10-09T04:53:49.533 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Max IOPS: 2622 2023-10-09T04:53:49.533 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Min IOPS: 685 2023-10-09T04:53:49.533 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Average Latency(s): 0.0125253 2023-10-09T04:53:49.533 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Stddev Latency(s): 0.0087735 2023-10-09T04:53:49.533 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Max latency(s): 0.319036 2023-10-09T04:53:49.533 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Min latency(s): 0.00143184 2023-10-09T04:53:49.540 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T04:53:50.401 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:50.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:50 smithi133 bash[41291]: cluster 2023-10-09T04:53:49.355625+0000 mgr.y (mgr.25277) 64 : cluster [DBG] pgmap v52: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 695 op/s 2023-10-09T04:53:50.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:50 smithi133 bash[102858]: cluster 2023-10-09T04:53:49.355625+0000 mgr.y (mgr.25277) 64 : cluster 0 pgmap v52: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 695 op/s 2023-10-09T04:53:50.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:50 smithi194 bash[31648]: cluster 2023-10-09T04:53:49.355625+0000 mgr.y (mgr.25277) 64 : cluster [DBG] pgmap v52: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 695 op/s 2023-10-09T04:53:52.691 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:52 smithi133 bash[41291]: cluster 2023-10-09T04:53:51.356181+0000 mgr.y (mgr.25277) 65 : cluster [DBG] pgmap v53: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T04:53:52.692 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:52 smithi133 bash[102858]: cluster 2023-10-09T04:53:51.356181+0000 mgr.y (mgr.25277) 65 : cluster 0 pgmap v53: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T04:53:52.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:52 smithi194 bash[31648]: cluster 2023-10-09T04:53:51.356181+0000 mgr.y (mgr.25277) 65 : cluster [DBG] pgmap v53: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T04:53:54.283 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:53:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:53:54] "GET /metrics HTTP/1.1" 200 36157 "" "Prometheus/2.43.0" 2023-10-09T04:53:54.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:53 smithi133 bash[41291]: cluster 2023-10-09T04:53:53.356595+0000 mgr.y (mgr.25277) 66 : cluster [DBG] pgmap v54: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 41 MiB/s wr, 661 op/s 2023-10-09T04:53:54.284 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:53 smithi133 bash[102858]: cluster 2023-10-09T04:53:53.356595+0000 mgr.y (mgr.25277) 66 : cluster 0 pgmap v54: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 41 MiB/s wr, 661 op/s 2023-10-09T04:53:54.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:53 smithi194 bash[31648]: cluster 2023-10-09T04:53:53.356595+0000 mgr.y (mgr.25277) 66 : cluster [DBG] pgmap v54: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 41 MiB/s wr, 661 op/s 2023-10-09T04:53:55.545 INFO:teuthology.orchestra.run.smithi133.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:53:56.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:56 smithi133 bash[102858]: cluster 2023-10-09T04:53:55.357289+0000 mgr.y (mgr.25277) 67 : cluster 0 pgmap v55: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 1.01k op/s 2023-10-09T04:53:56.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:56 smithi133 bash[102858]: audit 2023-10-09T04:53:55.528008+0000 mgr.y (mgr.25277) 68 : audit 0 from='client.25129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:53:56.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:56 smithi133 bash[102858]: cephadm 2023-10-09T04:53:55.528456+0000 mgr.y (mgr.25277) 69 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:53:56.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:56 smithi133 bash[102858]: audit 2023-10-09T04:53:55.537041+0000 mon.a (mon.0) 91 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:53:56.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:56 smithi133 bash[102858]: audit 2023-10-09T04:53:55.555084+0000 mon.a (mon.0) 92 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:53:56.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:56 smithi133 bash[102858]: audit 2023-10-09T04:53:55.557516+0000 mon.a (mon.0) 93 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:53:56.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:56 smithi133 bash[102858]: audit 2023-10-09T04:53:55.558757+0000 mon.a (mon.0) 94 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:53:56.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:56 smithi133 bash[102858]: audit 2023-10-09T04:53:55.567944+0000 mon.a (mon.0) 95 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:53:56.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:56 smithi133 bash[102858]: audit 2023-10-09T04:53:55.632208+0000 mon.a (mon.0) 96 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:53:56.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:56 smithi133 bash[102858]: audit 2023-10-09T04:53:55.636695+0000 mon.a (mon.0) 97 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:53:56.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:56 smithi133 bash[102858]: audit 2023-10-09T04:53:55.638937+0000 mon.a (mon.0) 98 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T04:53:56.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:56 smithi133 bash[102858]: audit 2023-10-09T04:53:55.643591+0000 mon.a (mon.0) 99 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T04:53:56.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:56 smithi133 bash[102858]: cephadm 2023-10-09T04:53:55.644454+0000 mgr.y (mgr.25277) 70 : cephadm 1 Upgrade: It appears safe to stop mon.c 2023-10-09T04:53:56.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:56 smithi133 bash[41291]: cluster 2023-10-09T04:53:55.357289+0000 mgr.y (mgr.25277) 67 : cluster [DBG] pgmap v55: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 1.01k op/s 2023-10-09T04:53:56.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:56 smithi133 bash[41291]: audit 2023-10-09T04:53:55.528008+0000 mgr.y (mgr.25277) 68 : audit [DBG] from='client.25129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:53:56.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:56 smithi133 bash[41291]: cephadm 2023-10-09T04:53:55.528456+0000 mgr.y (mgr.25277) 69 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:53:56.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:56 smithi133 bash[41291]: audit 2023-10-09T04:53:55.537041+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:53:56.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:56 smithi133 bash[41291]: audit 2023-10-09T04:53:55.555084+0000 mon.a (mon.0) 92 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:53:56.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:56 smithi133 bash[41291]: audit 2023-10-09T04:53:55.557516+0000 mon.a (mon.0) 93 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:53:56.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:56 smithi133 bash[41291]: audit 2023-10-09T04:53:55.558757+0000 mon.a (mon.0) 94 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:53:56.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:56 smithi133 bash[41291]: audit 2023-10-09T04:53:55.567944+0000 mon.a (mon.0) 95 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:53:56.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:56 smithi133 bash[41291]: audit 2023-10-09T04:53:55.632208+0000 mon.a (mon.0) 96 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:53:56.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:56 smithi133 bash[41291]: audit 2023-10-09T04:53:55.636695+0000 mon.a (mon.0) 97 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:53:56.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:56 smithi133 bash[41291]: audit 2023-10-09T04:53:55.638937+0000 mon.a (mon.0) 98 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T04:53:56.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:56 smithi133 bash[41291]: audit 2023-10-09T04:53:55.643591+0000 mon.a (mon.0) 99 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T04:53:56.804 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:56 smithi133 bash[41291]: cephadm 2023-10-09T04:53:55.644454+0000 mgr.y (mgr.25277) 70 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2023-10-09T04:53:56.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:56 smithi194 bash[31648]: cluster 2023-10-09T04:53:55.357289+0000 mgr.y (mgr.25277) 67 : cluster [DBG] pgmap v55: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 1.01k op/s 2023-10-09T04:53:56.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:56 smithi194 bash[31648]: audit 2023-10-09T04:53:55.528008+0000 mgr.y (mgr.25277) 68 : audit [DBG] from='client.25129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:53:56.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:56 smithi194 bash[31648]: cephadm 2023-10-09T04:53:55.528456+0000 mgr.y (mgr.25277) 69 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:53:56.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:56 smithi194 bash[31648]: audit 2023-10-09T04:53:55.537041+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:53:56.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:56 smithi194 bash[31648]: audit 2023-10-09T04:53:55.555084+0000 mon.a (mon.0) 92 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:53:56.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:56 smithi194 bash[31648]: audit 2023-10-09T04:53:55.557516+0000 mon.a (mon.0) 93 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:53:56.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:56 smithi194 bash[31648]: audit 2023-10-09T04:53:55.558757+0000 mon.a (mon.0) 94 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:53:56.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:56 smithi194 bash[31648]: audit 2023-10-09T04:53:55.567944+0000 mon.a (mon.0) 95 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:53:56.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:56 smithi194 bash[31648]: audit 2023-10-09T04:53:55.632208+0000 mon.a (mon.0) 96 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:53:56.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:56 smithi194 bash[31648]: audit 2023-10-09T04:53:55.636695+0000 mon.a (mon.0) 97 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:53:56.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:56 smithi194 bash[31648]: audit 2023-10-09T04:53:55.638937+0000 mon.a (mon.0) 98 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T04:53:56.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:56 smithi194 bash[31648]: audit 2023-10-09T04:53:55.643591+0000 mon.a (mon.0) 99 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T04:53:56.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:56 smithi194 bash[31648]: cephadm 2023-10-09T04:53:55.644454+0000 mgr.y (mgr.25277) 70 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2023-10-09T04:53:57.292 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for majority of mons to upgrade' 2023-10-09T04:53:58.547 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:58 smithi133 bash[102858]: cluster 2023-10-09T04:53:57.357749+0000 mgr.y (mgr.25277) 71 : cluster 0 pgmap v56: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 681 op/s 2023-10-09T04:53:58.547 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:53:58 smithi133 bash[102858]: audit 2023-10-09T04:53:58.408477+0000 mon.a (mon.0) 100 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:58.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:58 smithi133 bash[41291]: cluster 2023-10-09T04:53:57.357749+0000 mgr.y (mgr.25277) 71 : cluster [DBG] pgmap v56: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 681 op/s 2023-10-09T04:53:58.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:53:58 smithi133 bash[41291]: audit 2023-10-09T04:53:58.408477+0000 mon.a (mon.0) 100 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:58.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:58 smithi194 bash[31648]: cluster 2023-10-09T04:53:57.357749+0000 mgr.y (mgr.25277) 71 : cluster [DBG] pgmap v56: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 681 op/s 2023-10-09T04:53:58.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:53:58 smithi194 bash[31648]: audit 2023-10-09T04:53:58.408477+0000 mon.a (mon.0) 100 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:58.887 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:53:58 smithi194 bash[81885]: ts=2023-10-09T04:53:58.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi194.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi133:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:54:00.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:00 smithi133 bash[41291]: cephadm 2023-10-09T04:53:59.140740+0000 mgr.y (mgr.25277) 72 : cephadm [INF] Upgrade: Updating mon.c 2023-10-09T04:54:00.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:00 smithi133 bash[41291]: audit 2023-10-09T04:53:59.151297+0000 mon.a (mon.0) 101 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:00.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:00 smithi133 bash[41291]: audit 2023-10-09T04:53:59.152602+0000 mon.a (mon.0) 102 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:54:00.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:00 smithi133 bash[41291]: audit 2023-10-09T04:53:59.153624+0000 mon.a (mon.0) 103 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:54:00.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:00 smithi133 bash[41291]: audit 2023-10-09T04:53:59.154637+0000 mon.a (mon.0) 104 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:54:00.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:00 smithi133 bash[41291]: cephadm 2023-10-09T04:53:59.156252+0000 mgr.y (mgr.25277) 73 : cephadm [INF] Deploying daemon mon.c on smithi133 2023-10-09T04:54:00.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:00 smithi133 bash[41291]: cluster 2023-10-09T04:53:59.358142+0000 mgr.y (mgr.25277) 74 : cluster [DBG] pgmap v57: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 681 op/s 2023-10-09T04:54:00.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:00 smithi133 bash[102858]: cephadm 2023-10-09T04:53:59.140740+0000 mgr.y (mgr.25277) 72 : cephadm 1 Upgrade: Updating mon.c 2023-10-09T04:54:00.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:00 smithi133 bash[102858]: audit 2023-10-09T04:53:59.151297+0000 mon.a (mon.0) 101 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:00.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:00 smithi133 bash[102858]: audit 2023-10-09T04:53:59.152602+0000 mon.a (mon.0) 102 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:54:00.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:00 smithi133 bash[102858]: audit 2023-10-09T04:53:59.153624+0000 mon.a (mon.0) 103 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:54:00.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:00 smithi133 bash[102858]: audit 2023-10-09T04:53:59.154637+0000 mon.a (mon.0) 104 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:54:00.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:00 smithi133 bash[102858]: cephadm 2023-10-09T04:53:59.156252+0000 mgr.y (mgr.25277) 73 : cephadm 1 Deploying daemon mon.c on smithi133 2023-10-09T04:54:00.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:00 smithi133 bash[102858]: cluster 2023-10-09T04:53:59.358142+0000 mgr.y (mgr.25277) 74 : cluster 0 pgmap v57: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 681 op/s 2023-10-09T04:54:00.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:00 smithi194 bash[31648]: cephadm 2023-10-09T04:53:59.140740+0000 mgr.y (mgr.25277) 72 : cephadm [INF] Upgrade: Updating mon.c 2023-10-09T04:54:00.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:00 smithi194 bash[31648]: audit 2023-10-09T04:53:59.151297+0000 mon.a (mon.0) 101 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:00.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:00 smithi194 bash[31648]: audit 2023-10-09T04:53:59.152602+0000 mon.a (mon.0) 102 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:54:00.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:00 smithi194 bash[31648]: audit 2023-10-09T04:53:59.153624+0000 mon.a (mon.0) 103 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:54:00.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:00 smithi194 bash[31648]: audit 2023-10-09T04:53:59.154637+0000 mon.a (mon.0) 104 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:54:00.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:00 smithi194 bash[31648]: cephadm 2023-10-09T04:53:59.156252+0000 mgr.y (mgr.25277) 73 : cephadm [INF] Deploying daemon mon.c on smithi133 2023-10-09T04:54:00.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:00 smithi194 bash[31648]: cluster 2023-10-09T04:53:59.358142+0000 mgr.y (mgr.25277) 74 : cluster [DBG] pgmap v57: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 681 op/s 2023-10-09T04:54:02.471 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:02 smithi133 systemd[1]: Stopping Ceph mon.c for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T04:54:02.471 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:02 smithi133 bash[41291]: debug 2023-10-09T04:54:02.302+0000 7f5ee6d5b700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T04:54:02.471 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:02 smithi133 bash[41291]: debug 2023-10-09T04:54:02.302+0000 7f5ee6d5b700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2023-10-09T04:54:02.627 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:03.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:03 smithi133 bash[105447]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-mon-c 2023-10-09T04:54:04.372 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:54:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:54:04] "GET /metrics HTTP/1.1" 200 36165 "" "Prometheus/2.43.0" 2023-10-09T04:54:04.372 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:04 smithi133 bash[105511]: Error response from daemon: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-mon-c 2023-10-09T04:54:04.372 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:04 smithi133 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mon.c.service: Succeeded. 2023-10-09T04:54:04.372 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:04 smithi133 systemd[1]: Stopped Ceph mon.c for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:54:04.373 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:04 smithi133 systemd[1]: Started Ceph mon.c for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:54:07.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.090+0000 7f3d09d73c80 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T04:54:07.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.090+0000 7f3d09d73c80 0 ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T04:54:07.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.090+0000 7f3d09d73c80 0 pidfile_write: ignore empty --pid-file 2023-10-09T04:54:07.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.094+0000 7f3d09d73c80 0 load: jerasure load: lrc 2023-10-09T04:54:07.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: RocksDB version: 7.9.2 2023-10-09T04:54:07.305 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Git sha 0 2023-10-09T04:54:07.305 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T04:54:07.305 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: DB SUMMARY 2023-10-09T04:54:07.305 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: DB Session ID: 23D2CJN27YLMU70S1YFQ 2023-10-09T04:54:07.305 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: CURRENT file: CURRENT 2023-10-09T04:54:07.305 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: IDENTITY file: IDENTITY 2023-10-09T04:54:07.305 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: MANIFEST file: MANIFEST-000009 size: 2067 Bytes 2023-10-09T04:54:07.306 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000042.sst 2023-10-09T04:54:07.306 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000040.log size: 219384 ; 2023-10-09T04:54:07.306 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.error_if_exists: 0 2023-10-09T04:54:07.306 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.create_if_missing: 0 2023-10-09T04:54:07.306 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T04:54:07.306 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T04:54:07.306 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T04:54:07.306 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T04:54:07.306 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.env: 0x55fc8abcd720 2023-10-09T04:54:07.306 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.fs: PosixFileSystem 2023-10-09T04:54:07.307 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.info_log: 0x55fc8d7e9480 2023-10-09T04:54:07.307 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T04:54:07.307 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.statistics: (nil) 2023-10-09T04:54:07.307 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.use_fsync: 0 2023-10-09T04:54:07.307 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T04:54:07.307 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T04:54:07.307 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T04:54:07.307 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T04:54:07.307 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T04:54:07.308 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T04:54:07.308 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T04:54:07.308 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T04:54:07.308 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T04:54:07.308 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T04:54:07.308 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T04:54:07.308 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.db_log_dir: 2023-10-09T04:54:07.308 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.wal_dir: 2023-10-09T04:54:07.308 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T04:54:07.308 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T04:54:07.308 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T04:54:07.309 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T04:54:07.309 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T04:54:07.309 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T04:54:07.309 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T04:54:07.309 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T04:54:07.309 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.write_buffer_manager: 0x55fc8ca76c80 2023-10-09T04:54:07.309 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T04:54:07.309 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T04:54:07.309 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T04:54:07.309 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T04:54:07.310 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T04:54:07.310 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T04:54:07.310 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T04:54:07.310 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T04:54:07.310 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.unordered_write: 0 2023-10-09T04:54:07.310 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T04:54:07.310 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T04:54:07.310 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T04:54:07.310 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T04:54:07.310 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.row_cache: None 2023-10-09T04:54:07.311 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.wal_filter: None 2023-10-09T04:54:07.311 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T04:54:07.311 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T04:54:07.311 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.two_write_queues: 0 2023-10-09T04:54:07.311 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T04:54:07.311 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.wal_compression: 0 2023-10-09T04:54:07.311 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.atomic_flush: 0 2023-10-09T04:54:07.311 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T04:54:07.311 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T04:54:07.311 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T04:54:07.312 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T04:54:07.312 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T04:54:07.312 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.best_efforts_recovery: 0 2023-10-09T04:54:07.312 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T04:54:07.312 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T04:54:07.312 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.allow_data_in_errors: 0 2023-10-09T04:54:07.312 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.db_host_id: __hostname__ 2023-10-09T04:54:07.312 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T04:54:07.312 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T04:54:07.313 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T04:54:07.313 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T04:54:07.313 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T04:54:07.313 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T04:54:07.313 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T04:54:07.313 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T04:54:07.313 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T04:54:07.313 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T04:54:07.313 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T04:54:07.313 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T04:54:07.314 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.max_open_files: -1 2023-10-09T04:54:07.314 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T04:54:07.314 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T04:54:07.314 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T04:54:07.314 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T04:54:07.314 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T04:54:07.314 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Compression algorithms supported: 2023-10-09T04:54:07.314 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T04:54:07.314 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: kZSTD supported: 0 2023-10-09T04:54:07.315 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: kXpressCompression supported: 0 2023-10-09T04:54:07.315 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T04:54:07.315 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: kZlibCompression supported: 1 2023-10-09T04:54:07.315 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T04:54:07.315 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T04:54:07.315 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T04:54:07.315 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T04:54:07.315 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T04:54:07.316 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2023-10-09T04:54:07.316 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T04:54:07.316 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T04:54:07.316 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.merge_operator: 2023-10-09T04:54:07.316 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.compaction_filter: None 2023-10-09T04:54:07.316 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T04:54:07.316 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.sst_partitioner_factory: None 2023-10-09T04:54:07.316 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T04:54:07.316 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T04:54:07.317 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.098+0000 7f3d09d73c80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55fc8d7e9580) 2023-10-09T04:54:07.317 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: cache_index_and_filter_blocks: 1 2023-10-09T04:54:07.317 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T04:54:07.317 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T04:54:07.317 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: pin_top_level_index_and_filter: 1 2023-10-09T04:54:07.317 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: index_type: 0 2023-10-09T04:54:07.318 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: data_block_index_type: 0 2023-10-09T04:54:07.318 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: index_shortening: 1 2023-10-09T04:54:07.318 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: data_block_hash_table_util_ratio: 0.750000 2023-10-09T04:54:07.318 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: checksum: 4 2023-10-09T04:54:07.318 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: no_block_cache: 0 2023-10-09T04:54:07.318 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: block_cache: 0x55fc8caf91f0 2023-10-09T04:54:07.318 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: block_cache_name: BinnedLRUCache 2023-10-09T04:54:07.318 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: block_cache_options: 2023-10-09T04:54:07.318 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: capacity : 536870912 2023-10-09T04:54:07.318 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: num_shard_bits : 4 2023-10-09T04:54:07.318 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: strict_capacity_limit : 0 2023-10-09T04:54:07.319 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: high_pri_pool_ratio: 0.000 2023-10-09T04:54:07.319 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: block_cache_compressed: (nil) 2023-10-09T04:54:07.319 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: persistent_cache: (nil) 2023-10-09T04:54:07.319 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: block_size: 4096 2023-10-09T04:54:07.319 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: block_size_deviation: 10 2023-10-09T04:54:07.319 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: block_restart_interval: 16 2023-10-09T04:54:07.319 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: index_block_restart_interval: 1 2023-10-09T04:54:07.319 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: metadata_block_size: 4096 2023-10-09T04:54:07.319 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: partition_filters: 0 2023-10-09T04:54:07.319 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: use_delta_encoding: 1 2023-10-09T04:54:07.319 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: filter_policy: bloomfilter 2023-10-09T04:54:07.319 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: whole_key_filtering: 1 2023-10-09T04:54:07.320 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: verify_compression: 0 2023-10-09T04:54:07.320 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: read_amp_bytes_per_bit: 0 2023-10-09T04:54:07.320 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: format_version: 5 2023-10-09T04:54:07.320 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: enable_index_compression: 1 2023-10-09T04:54:07.320 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: block_align: 0 2023-10-09T04:54:07.320 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: max_auto_readahead_size: 262144 2023-10-09T04:54:07.320 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: prepopulate_block_cache: 0 2023-10-09T04:54:07.320 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: initial_auto_readahead_size: 8192 2023-10-09T04:54:07.320 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: num_file_reads_for_auto_readahead: 2 2023-10-09T04:54:07.320 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T04:54:07.320 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T04:54:07.321 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.compression: NoCompression 2023-10-09T04:54:07.321 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T04:54:07.321 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T04:54:07.321 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T04:54:07.321 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.num_levels: 7 2023-10-09T04:54:07.321 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T04:54:07.321 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T04:54:07.321 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T04:54:07.321 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T04:54:07.321 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T04:54:07.321 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T04:54:07.321 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T04:54:07.322 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:54:07.322 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T04:54:07.322 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T04:54:07.322 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T04:54:07.322 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T04:54:07.322 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T04:54:07.322 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T04:54:07.322 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T04:54:07.322 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T04:54:07.322 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:54:07.322 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T04:54:07.323 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T04:54:07.323 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T04:54:07.323 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T04:54:07.323 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T04:54:07.323 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T04:54:07.323 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T04:54:07.323 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T04:54:07.323 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T04:54:07.323 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T04:54:07.323 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T04:54:07.324 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T04:54:07.324 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T04:54:07.324 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T04:54:07.324 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T04:54:07.324 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T04:54:07.324 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T04:54:07.324 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T04:54:07.324 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T04:54:07.324 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T04:54:07.324 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T04:54:07.325 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T04:54:07.325 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.arena_block_size: 1048576 2023-10-09T04:54:07.325 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T04:54:07.325 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T04:54:07.325 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T04:54:07.325 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T04:54:07.325 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T04:54:07.325 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T04:54:07.325 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T04:54:07.325 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T04:54:07.326 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T04:54:07.326 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T04:54:07.326 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T04:54:07.326 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T04:54:07.326 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T04:54:07.326 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.table_properties_collectors: 2023-10-09T04:54:07.326 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T04:54:07.326 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T04:54:07.326 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T04:54:07.326 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T04:54:07.327 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T04:54:07.327 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.bloom_locality: 0 2023-10-09T04:54:07.327 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T04:54:07.327 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T04:54:07.327 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T04:54:07.327 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.force_consistency_checks: 1 2023-10-09T04:54:07.327 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T04:54:07.327 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.ttl: 2592000 2023-10-09T04:54:07.327 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T04:54:07.327 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T04:54:07.328 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T04:54:07.328 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.enable_blob_files: false 2023-10-09T04:54:07.328 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.min_blob_size: 0 2023-10-09T04:54:07.328 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.blob_file_size: 268435456 2023-10-09T04:54:07.328 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T04:54:07.328 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T04:54:07.328 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T04:54:07.328 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T04:54:07.328 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T04:54:07.328 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.blob_file_starting_level: 0 2023-10-09T04:54:07.329 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3d09d73c80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T04:54:07.329 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.102+0000 7f3cff591700 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 42.sst 2023-10-09T04:54:07.329 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.106+0000 7f3d09d73c80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T04:54:07.329 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.106+0000 7f3d09d73c80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 44, last_sequence is 23035, log_number is 40,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T04:54:07.329 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.106+0000 7f3d09d73c80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 40 2023-10-09T04:54:07.329 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.106+0000 7f3d09d73c80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 77528be9-ed8d-4284-b080-16d184e7344c 2023-10-09T04:54:07.329 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.106+0000 7f3d09d73c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696827247110281, "job": 1, "event": "recovery_started", "wal_files": [40]} 2023-10-09T04:54:07.329 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.106+0000 7f3d09d73c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #40 mode 2 2023-10-09T04:54:07.329 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.106+0000 7f3d09d73c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696827247112142, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 45, "file_size": 144340, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 23040, "largest_seqno": 23245, "table_properties": {"data_size": 142605, "index_size": 467, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 261, "raw_key_size": 2161, "raw_average_key_size": 24, "raw_value_size": 140610, "raw_average_value_size": 1562, "num_data_blocks": 21, "num_entries": 90, "num_filter_entries": 90, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696827247, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "77528be9-ed8d-4284-b080-16d184e7344c", "db_session_id": "23D2CJN27YLMU70S1YFQ", "orig_file_number": 45, "seqno_to_time_mapping": "N/A"}} 2023-10-09T04:54:07.330 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.106+0000 7f3d09d73c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696827247112298, "job": 1, "event": "recovery_finished"} 2023-10-09T04:54:07.330 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.106+0000 7f3d09d73c80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 47 2023-10-09T04:54:07.330 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.106+0000 7f3d09d73c80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T04:54:07.330 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.114+0000 7f3d09d73c80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000040.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T04:54:07.330 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.114+0000 7f3d09d73c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55fc8cb9a000 2023-10-09T04:54:07.331 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.114+0000 7f3d09d73c80 4 rocksdb: DB pointer 0x55fc8cb88000 2023-10-09T04:54:07.331 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.114+0000 7f3cf8583700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T04:54:07.331 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.114+0000 7f3cf8583700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T04:54:07.331 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: ** DB Stats ** 2023-10-09T04:54:07.331 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:54:07.331 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T04:54:07.331 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T04:54:07.331 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:54:07.331 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T04:54:07.332 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T04:54:07.332 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:54:07.332 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: ** Compaction Stats [default] ** 2023-10-09T04:54:07.332 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T04:54:07.332 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T04:54:07.332 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: L0 1/0 140.96 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 130.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-10-09T04:54:07.332 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: L6 1/0 8.58 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T04:54:07.332 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: Sum 2/0 8.72 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 130.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-10-09T04:54:07.332 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 130.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-10-09T04:54:07.332 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: ** Compaction Stats [default] ** 2023-10-09T04:54:07.333 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T04:54:07.333 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:54:07.333 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 130.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-10-09T04:54:07.333 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T04:54:07.333 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:54:07.333 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T04:54:07.333 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T04:54:07.333 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T04:54:07.333 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T04:54:07.333 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T04:54:07.334 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: Cumulative compaction: 0.00 GB write, 8.79 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:54:07.334 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: Interval compaction: 0.00 GB write, 8.79 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:54:07.334 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T04:54:07.334 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: Block cache BinnedLRUCache@0x55fc8caf91f0#7 capacity: 512.00 MB usage: 0.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.5e-05 secs_since: 0 2023-10-09T04:54:07.334 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: Block cache entry stats(count,size,portion): FilterBlock(1,0.31 KB,5.96046e-05%) IndexBlock(1,0.55 KB,0.000104308%) Misc(1,0.00 KB,0%) 2023-10-09T04:54:07.334 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T04:54:07.334 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.118+0000 7f3d09d73c80 0 starting mon.c rank 2 at public addrs [v2:172.21.15.133:3301/0,v1:172.21.15.133:6790/0] at bind addrs [v2:172.21.15.133:3301/0,v1:172.21.15.133:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 855b5206-665c-11ee-8db5-212e2dc638e7 2023-10-09T04:54:07.334 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.118+0000 7f3d09d73c80 1 mon.c@-1(???) e3 preinit fsid 855b5206-665c-11ee-8db5-212e2dc638e7 2023-10-09T04:54:07.334 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.118+0000 7f3d09d73c80 0 mon.c@-1(???).mds e10 new map 2023-10-09T04:54:07.334 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.118+0000 7f3d09d73c80 0 mon.c@-1(???).mds e10 print_map 2023-10-09T04:54:07.334 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: e10 2023-10-09T04:54:07.335 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T04:54:07.335 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T04:54:07.335 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: legacy client fscid: 1 2023-10-09T04:54:07.335 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: 2023-10-09T04:54:07.335 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: Filesystem 'foo' (1) 2023-10-09T04:54:07.335 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: fs_name foo 2023-10-09T04:54:07.335 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: epoch 10 2023-10-09T04:54:07.335 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T04:54:07.335 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: created 2023-10-09T04:44:52.764614+0000 2023-10-09T04:54:07.335 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: modified 2023-10-09T04:45:03.032645+0000 2023-10-09T04:54:07.336 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: tableserver 0 2023-10-09T04:54:07.336 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: root 0 2023-10-09T04:54:07.336 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: session_timeout 60 2023-10-09T04:54:07.336 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: session_autoclose 300 2023-10-09T04:54:07.336 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: max_file_size 1099511627776 2023-10-09T04:54:07.336 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: required_client_features {} 2023-10-09T04:54:07.336 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: last_failure 0 2023-10-09T04:54:07.336 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: last_failure_osd_epoch 69 2023-10-09T04:54:07.336 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T04:54:07.336 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: max_mds 1 2023-10-09T04:54:07.336 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: in 0 2023-10-09T04:54:07.337 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: up {0=24502} 2023-10-09T04:54:07.337 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: failed 2023-10-09T04:54:07.337 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: damaged 2023-10-09T04:54:07.337 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: stopped 2023-10-09T04:54:07.337 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: data_pools [4] 2023-10-09T04:54:07.337 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: metadata_pool 3 2023-10-09T04:54:07.337 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: inline_data disabled 2023-10-09T04:54:07.337 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: balancer 2023-10-09T04:54:07.337 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: bal_rank_mask -1 2023-10-09T04:54:07.337 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: standby_count_wanted 1 2023-10-09T04:54:07.337 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: [mds.foo.smithi194.yxnfqn{0:24502} state up:active seq 4 join_fscid=1 addr [v2:172.21.15.194:6832/1679470116,v1:172.21.15.194:6833/1679470116] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T04:54:07.338 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: 2023-10-09T04:54:07.338 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: 2023-10-09T04:54:07.338 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: Standby daemons: 2023-10-09T04:54:07.338 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: 2023-10-09T04:54:07.338 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: [mds.foo.smithi133.tsuabh{-1:14682} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.133:6834/3356898887,v1:172.21.15.133:6835/3356898887] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T04:54:07.338 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.118+0000 7f3d09d73c80 0 mon.c@-1(???).osd e83 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T04:54:07.338 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.118+0000 7f3d09d73c80 0 mon.c@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T04:54:07.338 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.118+0000 7f3d09d73c80 0 mon.c@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T04:54:07.338 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.122+0000 7f3d09d73c80 0 mon.c@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T04:54:07.339 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.122+0000 7f3d09d73c80 1 mon.c@-1(???).paxosservice(auth 1..23) refresh upgraded, format 0 -> 3 2023-10-09T04:54:07.339 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.126+0000 7f3d09d73c80 20 expand_channel_meta expand map: {default=false} 2023-10-09T04:54:07.339 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.126+0000 7f3d09d73c80 20 expand_channel_meta from 'false' to 'false' 2023-10-09T04:54:07.339 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.126+0000 7f3d09d73c80 20 expand_channel_meta expanded map: {default=false} 2023-10-09T04:54:07.339 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.126+0000 7f3d09d73c80 20 expand_channel_meta expand map: {default=info} 2023-10-09T04:54:07.339 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.126+0000 7f3d09d73c80 20 expand_channel_meta from 'info' to 'info' 2023-10-09T04:54:07.339 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.126+0000 7f3d09d73c80 20 expand_channel_meta expanded map: {default=info} 2023-10-09T04:54:07.339 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.126+0000 7f3d09d73c80 20 expand_channel_meta expand map: {default=daemon} 2023-10-09T04:54:07.339 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.126+0000 7f3d09d73c80 20 expand_channel_meta from 'daemon' to 'daemon' 2023-10-09T04:54:07.339 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.126+0000 7f3d09d73c80 20 expand_channel_meta expanded map: {default=daemon} 2023-10-09T04:54:07.339 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.126+0000 7f3d09d73c80 20 expand_channel_meta expand map: {default=debug} 2023-10-09T04:54:07.339 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.126+0000 7f3d09d73c80 20 expand_channel_meta from 'debug' to 'debug' 2023-10-09T04:54:07.340 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:07 smithi133 bash[105589]: debug 2023-10-09T04:54:07.126+0000 7f3d09d73c80 20 expand_channel_meta expanded map: {default=debug} 2023-10-09T04:54:08.538 INFO:teuthology.orchestra.run.smithi133.stdout:wait for majority of mons to upgrade 2023-10-09T04:54:09.137 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:54:08 smithi194 bash[81885]: ts=2023-10-09T04:54:08.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi194.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi133:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:54:09.565 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2023-10-09T04:54:12.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:12 smithi194 bash[31648]: cluster 2023-10-09T04:54:01.358859+0000 mgr.y (mgr.25277) 75 : cluster [DBG] pgmap v58: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 1.01k op/s 2023-10-09T04:54:12.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:12 smithi133 bash[102858]: cluster 2023-10-09T04:54:01.358859+0000 mgr.y (mgr.25277) 75 : cluster 0 pgmap v58: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 1.01k op/s 2023-10-09T04:54:13.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:13 smithi133 bash[105589]: cluster 2023-10-09T04:54:01.358859+0000 mgr.y (mgr.25277) 75 : cluster 0 pgmap v58: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 1.01k op/s 2023-10-09T04:54:14.285 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:13 smithi133 bash[102858]: cluster 2023-10-09T04:54:07.346609+0000 mon.c (mon.2) 1 : cluster 1 mon.c calling monitor election 2023-10-09T04:54:14.285 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:13 smithi133 bash[102858]: cluster 2023-10-09T04:54:13.354476+0000 mon.a (mon.0) 118 : cluster 1 mon.a calling monitor election 2023-10-09T04:54:14.285 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:13 smithi133 bash[102858]: cluster 2023-10-09T04:54:13.355265+0000 mon.b (mon.1) 52 : cluster 1 mon.b calling monitor election 2023-10-09T04:54:14.285 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:13 smithi133 bash[102858]: cluster 2023-10-09T04:54:13.362050+0000 mgr.y (mgr.25277) 81 : cluster 0 pgmap v64: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-10-09T04:54:14.285 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:13 smithi133 bash[102858]: cluster 2023-10-09T04:54:13.377793+0000 mon.a (mon.0) 119 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:54:14.285 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:13 smithi133 bash[102858]: cluster 2023-10-09T04:54:13.382630+0000 mon.a (mon.0) 120 : cluster 0 monmap e3: 3 mons at {a=[v2:172.21.15.133:3300/0,v1:172.21.15.133:6789/0],b=[v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0],c=[v2:172.21.15.133:3301/0,v1:172.21.15.133:6790/0]} removed_ranks: {} 2023-10-09T04:54:14.285 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:13 smithi133 bash[102858]: cluster 2023-10-09T04:54:13.383340+0000 mon.a (mon.0) 121 : cluster 0 fsmap foo:1 {0=foo.smithi194.yxnfqn=up:active} 1 up:standby 2023-10-09T04:54:14.286 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:13 smithi133 bash[102858]: cluster 2023-10-09T04:54:13.383391+0000 mon.a (mon.0) 122 : cluster 0 osdmap e83: 8 total, 8 up, 8 in 2023-10-09T04:54:14.286 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:13 smithi133 bash[102858]: cluster 2023-10-09T04:54:13.384083+0000 mon.a (mon.0) 123 : cluster 0 mgrmap e35: y(active, since 2m), standbys: x 2023-10-09T04:54:14.286 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:13 smithi133 bash[102858]: cluster 2023-10-09T04:54:13.384295+0000 mon.a (mon.0) 124 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T04:54:14.286 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:13 smithi133 bash[102858]: cluster 2023-10-09T04:54:13.384329+0000 mon.a (mon.0) 125 : cluster 1 Cluster is now healthy 2023-10-09T04:54:14.286 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:13 smithi133 bash[102858]: cluster 2023-10-09T04:54:13.393202+0000 mon.a (mon.0) 126 : cluster 1 overall HEALTH_OK 2023-10-09T04:54:14.286 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:13 smithi133 bash[102858]: audit 2023-10-09T04:54:13.408956+0000 mon.a (mon.0) 127 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:14.286 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:54:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:54:14] "GET /metrics HTTP/1.1" 200 36161 "" "Prometheus/2.43.0" 2023-10-09T04:54:14.287 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:13 smithi133 bash[105589]: cluster 2023-10-09T04:54:07.346609+0000 mon.c (mon.2) 1 : cluster 1 mon.c calling monitor election 2023-10-09T04:54:14.287 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:13 smithi133 bash[105589]: cluster 2023-10-09T04:54:13.354476+0000 mon.a (mon.0) 118 : cluster 1 mon.a calling monitor election 2023-10-09T04:54:14.287 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:13 smithi133 bash[105589]: cluster 2023-10-09T04:54:13.355265+0000 mon.b (mon.1) 52 : cluster 1 mon.b calling monitor election 2023-10-09T04:54:14.287 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:13 smithi133 bash[105589]: cluster 2023-10-09T04:54:13.362050+0000 mgr.y (mgr.25277) 81 : cluster 0 pgmap v64: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-10-09T04:54:14.287 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:13 smithi133 bash[105589]: cluster 2023-10-09T04:54:13.377793+0000 mon.a (mon.0) 119 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:54:14.287 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:13 smithi133 bash[105589]: cluster 2023-10-09T04:54:13.382630+0000 mon.a (mon.0) 120 : cluster 0 monmap e3: 3 mons at {a=[v2:172.21.15.133:3300/0,v1:172.21.15.133:6789/0],b=[v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0],c=[v2:172.21.15.133:3301/0,v1:172.21.15.133:6790/0]} removed_ranks: {} 2023-10-09T04:54:14.287 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:13 smithi133 bash[105589]: cluster 2023-10-09T04:54:13.383340+0000 mon.a (mon.0) 121 : cluster 0 fsmap foo:1 {0=foo.smithi194.yxnfqn=up:active} 1 up:standby 2023-10-09T04:54:14.288 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:13 smithi133 bash[105589]: cluster 2023-10-09T04:54:13.383391+0000 mon.a (mon.0) 122 : cluster 0 osdmap e83: 8 total, 8 up, 8 in 2023-10-09T04:54:14.288 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:13 smithi133 bash[105589]: cluster 2023-10-09T04:54:13.384083+0000 mon.a (mon.0) 123 : cluster 0 mgrmap e35: y(active, since 2m), standbys: x 2023-10-09T04:54:14.288 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:13 smithi133 bash[105589]: cluster 2023-10-09T04:54:13.384295+0000 mon.a (mon.0) 124 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T04:54:14.288 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:13 smithi133 bash[105589]: cluster 2023-10-09T04:54:13.384329+0000 mon.a (mon.0) 125 : cluster 1 Cluster is now healthy 2023-10-09T04:54:14.288 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:13 smithi133 bash[105589]: cluster 2023-10-09T04:54:13.393202+0000 mon.a (mon.0) 126 : cluster 1 overall HEALTH_OK 2023-10-09T04:54:14.288 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:13 smithi133 bash[105589]: audit 2023-10-09T04:54:13.408956+0000 mon.a (mon.0) 127 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:14.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:13 smithi194 bash[31648]: cluster 2023-10-09T04:54:07.346609+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-09T04:54:14.534 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:13 smithi194 bash[31648]: cluster 2023-10-09T04:54:13.354476+0000 mon.a (mon.0) 118 : cluster [INF] mon.a calling monitor election 2023-10-09T04:54:14.534 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:13 smithi194 bash[31648]: cluster 2023-10-09T04:54:13.355265+0000 mon.b (mon.1) 52 : cluster [INF] mon.b calling monitor election 2023-10-09T04:54:14.534 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:13 smithi194 bash[31648]: cluster 2023-10-09T04:54:13.362050+0000 mgr.y (mgr.25277) 81 : cluster [DBG] pgmap v64: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-10-09T04:54:14.534 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:13 smithi194 bash[31648]: cluster 2023-10-09T04:54:13.377793+0000 mon.a (mon.0) 119 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:54:14.534 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:13 smithi194 bash[31648]: cluster 2023-10-09T04:54:13.382630+0000 mon.a (mon.0) 120 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.133:3300/0,v1:172.21.15.133:6789/0],b=[v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0],c=[v2:172.21.15.133:3301/0,v1:172.21.15.133:6790/0]} removed_ranks: {} 2023-10-09T04:54:14.534 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:13 smithi194 bash[31648]: cluster 2023-10-09T04:54:13.383340+0000 mon.a (mon.0) 121 : cluster [DBG] fsmap foo:1 {0=foo.smithi194.yxnfqn=up:active} 1 up:standby 2023-10-09T04:54:14.534 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:13 smithi194 bash[31648]: cluster 2023-10-09T04:54:13.383391+0000 mon.a (mon.0) 122 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-10-09T04:54:14.535 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:13 smithi194 bash[31648]: cluster 2023-10-09T04:54:13.384083+0000 mon.a (mon.0) 123 : cluster [DBG] mgrmap e35: y(active, since 2m), standbys: x 2023-10-09T04:54:14.535 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:13 smithi194 bash[31648]: cluster 2023-10-09T04:54:13.384295+0000 mon.a (mon.0) 124 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T04:54:14.535 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:13 smithi194 bash[31648]: cluster 2023-10-09T04:54:13.384329+0000 mon.a (mon.0) 125 : cluster [INF] Cluster is now healthy 2023-10-09T04:54:14.535 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:13 smithi194 bash[31648]: cluster 2023-10-09T04:54:13.393202+0000 mon.a (mon.0) 126 : cluster [INF] overall HEALTH_OK 2023-10-09T04:54:14.535 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:13 smithi194 bash[31648]: audit 2023-10-09T04:54:13.408956+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:14.536 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:16.785 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:16 smithi133 bash[102858]: cluster 2023-10-09T04:54:15.362666+0000 mgr.y (mgr.25277) 82 : cluster 0 pgmap v65: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 978 op/s 2023-10-09T04:54:17.101 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:16 smithi133 bash[105589]: cluster 2023-10-09T04:54:15.362666+0000 mgr.y (mgr.25277) 82 : cluster 0 pgmap v65: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 978 op/s 2023-10-09T04:54:17.102 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:16 smithi194 bash[31648]: cluster 2023-10-09T04:54:15.362666+0000 mgr.y (mgr.25277) 82 : cluster [DBG] pgmap v65: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 978 op/s 2023-10-09T04:54:17.550 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:54:17 smithi133 bash[91128]: debug 2023-10-09T04:54:17.134+0000 7ff45d5ed700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-10-09T04:54:19.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:18 smithi194 bash[31648]: cluster 2023-10-09T04:54:17.363212+0000 mgr.y (mgr.25277) 83 : cluster [DBG] pgmap v66: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-10-09T04:54:19.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:18 smithi194 bash[31648]: audit 2023-10-09T04:54:17.814400+0000 mon.a (mon.0) 128 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:19.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:18 smithi194 bash[31648]: audit 2023-10-09T04:54:17.822616+0000 mon.a (mon.0) 129 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:19.138 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:54:18 smithi194 bash[81885]: ts=2023-10-09T04:54:18.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi194.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi133:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:54:19.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:18 smithi133 bash[102858]: cluster 2023-10-09T04:54:17.363212+0000 mgr.y (mgr.25277) 83 : cluster 0 pgmap v66: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-10-09T04:54:19.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:18 smithi133 bash[102858]: audit 2023-10-09T04:54:17.814400+0000 mon.a (mon.0) 128 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:19.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:18 smithi133 bash[102858]: audit 2023-10-09T04:54:17.822616+0000 mon.a (mon.0) 129 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:19.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:18 smithi133 bash[105589]: cluster 2023-10-09T04:54:17.363212+0000 mgr.y (mgr.25277) 83 : cluster 0 pgmap v66: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-10-09T04:54:19.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:18 smithi133 bash[105589]: audit 2023-10-09T04:54:17.814400+0000 mon.a (mon.0) 128 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:19.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:18 smithi133 bash[105589]: audit 2023-10-09T04:54:17.822616+0000 mon.a (mon.0) 129 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:19.709 INFO:teuthology.orchestra.run.smithi133.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 2 2023-10-09T04:54:20.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:19 smithi133 bash[105589]: audit 2023-10-09T04:54:18.844511+0000 mon.a (mon.0) 130 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:20.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:19 smithi133 bash[105589]: audit 2023-10-09T04:54:18.855897+0000 mon.a (mon.0) 131 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:20.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:19 smithi133 bash[105589]: audit 2023-10-09T04:54:19.673384+0000 mon.a (mon.0) 132 : audit 0 from='client.? 172.21.15.133:0/89334061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:54:20.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:19 smithi133 bash[102858]: audit 2023-10-09T04:54:18.844511+0000 mon.a (mon.0) 130 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:20.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:19 smithi133 bash[102858]: audit 2023-10-09T04:54:18.855897+0000 mon.a (mon.0) 131 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:20.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:19 smithi133 bash[102858]: audit 2023-10-09T04:54:19.673384+0000 mon.a (mon.0) 132 : audit 0 from='client.? 172.21.15.133:0/89334061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:54:20.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:19 smithi194 bash[31648]: audit 2023-10-09T04:54:18.844511+0000 mon.a (mon.0) 130 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:20.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:19 smithi194 bash[31648]: audit 2023-10-09T04:54:18.855897+0000 mon.a (mon.0) 131 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:20.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:19 smithi194 bash[31648]: audit 2023-10-09T04:54:19.673384+0000 mon.a (mon.0) 132 : audit [DBG] from='client.? 172.21.15.133:0/89334061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:54:20.742 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T04:54:21.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:20 smithi133 bash[102858]: cluster 2023-10-09T04:54:19.363631+0000 mgr.y (mgr.25277) 84 : cluster 0 pgmap v67: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-10-09T04:54:21.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:20 smithi133 bash[105589]: cluster 2023-10-09T04:54:19.363631+0000 mgr.y (mgr.25277) 84 : cluster 0 pgmap v67: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-10-09T04:54:21.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:20 smithi194 bash[31648]: cluster 2023-10-09T04:54:19.363631+0000 mgr.y (mgr.25277) 84 : cluster [DBG] pgmap v67: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-10-09T04:54:23.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:22 smithi194 bash[31648]: cluster 2023-10-09T04:54:21.364309+0000 mgr.y (mgr.25277) 85 : cluster [DBG] pgmap v68: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 951 op/s 2023-10-09T04:54:23.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:22 smithi133 bash[105589]: cluster 2023-10-09T04:54:21.364309+0000 mgr.y (mgr.25277) 85 : cluster 0 pgmap v68: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 951 op/s 2023-10-09T04:54:23.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:22 smithi133 bash[102858]: cluster 2023-10-09T04:54:21.364309+0000 mgr.y (mgr.25277) 85 : cluster 0 pgmap v68: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 951 op/s 2023-10-09T04:54:24.284 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:54:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:54:24] "GET /metrics HTTP/1.1" 200 36161 "" "Prometheus/2.43.0" 2023-10-09T04:54:24.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:23 smithi133 bash[105589]: cluster 2023-10-09T04:54:23.364689+0000 mgr.y (mgr.25277) 86 : cluster 0 pgmap v69: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-10-09T04:54:24.284 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:23 smithi133 bash[102858]: cluster 2023-10-09T04:54:23.364689+0000 mgr.y (mgr.25277) 86 : cluster 0 pgmap v69: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-10-09T04:54:24.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:23 smithi194 bash[31648]: cluster 2023-10-09T04:54:23.364689+0000 mgr.y (mgr.25277) 86 : cluster [DBG] pgmap v69: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-10-09T04:54:25.305 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:26.711 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:26 smithi133 bash[105589]: cluster 2023-10-09T04:54:25.365514+0000 mgr.y (mgr.25277) 87 : cluster 0 pgmap v70: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 941 op/s 2023-10-09T04:54:26.712 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:26 smithi133 bash[102858]: cluster 2023-10-09T04:54:25.365514+0000 mgr.y (mgr.25277) 87 : cluster 0 pgmap v70: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 941 op/s 2023-10-09T04:54:26.889 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:26 smithi194 bash[31648]: cluster 2023-10-09T04:54:25.365514+0000 mgr.y (mgr.25277) 87 : cluster [DBG] pgmap v70: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 941 op/s 2023-10-09T04:54:28.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:28 smithi133 bash[105589]: cluster 2023-10-09T04:54:27.365898+0000 mgr.y (mgr.25277) 88 : cluster 0 pgmap v71: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-10-09T04:54:28.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:28 smithi133 bash[105589]: audit 2023-10-09T04:54:28.409013+0000 mon.a (mon.0) 133 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:28.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:28 smithi133 bash[102858]: cluster 2023-10-09T04:54:27.365898+0000 mgr.y (mgr.25277) 88 : cluster 0 pgmap v71: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-10-09T04:54:28.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:28 smithi133 bash[102858]: audit 2023-10-09T04:54:28.409013+0000 mon.a (mon.0) 133 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:28.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:28 smithi194 bash[31648]: cluster 2023-10-09T04:54:27.365898+0000 mgr.y (mgr.25277) 88 : cluster [DBG] pgmap v71: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-10-09T04:54:28.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:28 smithi194 bash[31648]: audit 2023-10-09T04:54:28.409013+0000 mon.a (mon.0) 133 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:28.887 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:54:28 smithi194 bash[81885]: ts=2023-10-09T04:54:28.664Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi194.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi133:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:54:30.698 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:30 smithi133 bash[102858]: cluster 2023-10-09T04:54:29.366282+0000 mgr.y (mgr.25277) 89 : cluster 0 pgmap v72: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-10-09T04:54:30.698 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:30 smithi133 bash[105589]: cluster 2023-10-09T04:54:29.366282+0000 mgr.y (mgr.25277) 89 : cluster 0 pgmap v72: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-10-09T04:54:30.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:30 smithi194 bash[31648]: cluster 2023-10-09T04:54:29.366282+0000 mgr.y (mgr.25277) 89 : cluster [DBG] pgmap v72: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-10-09T04:54:31.498 INFO:teuthology.orchestra.run.smithi133.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T04:54:31.498 INFO:teuthology.orchestra.run.smithi133.stdout:alertmanager.a smithi133 *:9093,9094 running (4m) - 12m 14.4M - 0.25.0 c8568f914cd2 f7641cfa53d2 2023-10-09T04:54:31.498 INFO:teuthology.orchestra.run.smithi133.stdout:grafana.a smithi194 *:3000 running (4m) - 11m 73.4M - 9.4.7 2c41d148cca3 18d67948e55e 2023-10-09T04:54:31.498 INFO:teuthology.orchestra.run.smithi133.stdout:mds.foo.smithi133.tsuabh smithi133 running (9m) - 9m 13.6M - 17.2.6-1303-gb2903cb0 5517396c50d4 1adbb452599e 2023-10-09T04:54:31.499 INFO:teuthology.orchestra.run.smithi133.stdout:mds.foo.smithi194.yxnfqn smithi194 running (9m) - 9m 15.7M - 17.2.6-1303-gb2903cb0 5517396c50d4 745c16c1ffd2 2023-10-09T04:54:31.499 INFO:teuthology.orchestra.run.smithi133.stdout:mgr.x smithi194 *:8443,9283,8765 running (2m) - 20m 426M - 18.2.0-598-gb4fa663f 3060897fdf97 e7277fed699e 2023-10-09T04:54:31.499 INFO:teuthology.orchestra.run.smithi133.stdout:mgr.y smithi133 *:8443,9283,8765 running (3m) - 23m 478M - 18.2.0-598-gb4fa663f 3060897fdf97 caf6152f91c6 2023-10-09T04:54:31.499 INFO:teuthology.orchestra.run.smithi133.stdout:mon.a smithi133 running (2m) - 23m 40.2M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 ee45bbc6b194 2023-10-09T04:54:31.499 INFO:teuthology.orchestra.run.smithi133.stdout:mon.b smithi194 running (21m) - 21m 50.4M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 8059052f6835 2023-10-09T04:54:31.499 INFO:teuthology.orchestra.run.smithi133.stdout:mon.c smithi133 running (27s) - 20m 16.9M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 eb3f3f9009e4 2023-10-09T04:54:31.499 INFO:teuthology.orchestra.run.smithi133.stdout:node-exporter.a smithi133 *:9100 running (4m) - 12m 7860k - 1.5.0 0da6a335fe13 f8bdde5940e7 2023-10-09T04:54:31.499 INFO:teuthology.orchestra.run.smithi133.stdout:node-exporter.b smithi194 *:9100 running (4m) - 12m 8292k - 1.5.0 0da6a335fe13 ffab717ac947 2023-10-09T04:54:31.499 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0 smithi133 running (19m) - 19m 394M 2524M 17.2.6-1303-gb2903cb0 5517396c50d4 7dc540ae1166 2023-10-09T04:54:31.499 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1 smithi133 running (18m) - 18m 334M 2524M 17.2.6-1303-gb2903cb0 5517396c50d4 27ac869bf46a 2023-10-09T04:54:31.499 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2 smithi133 running (17m) - 17m 487M 2524M 17.2.6-1303-gb2903cb0 5517396c50d4 e9d2970cad24 2023-10-09T04:54:31.499 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3 smithi133 running (17m) - 17m 473M 2524M 17.2.6-1303-gb2903cb0 5517396c50d4 415368a68902 2023-10-09T04:54:31.500 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4 smithi194 running (16m) - 16m 398M 2524M 17.2.6-1303-gb2903cb0 5517396c50d4 c77edffeb061 2023-10-09T04:54:31.500 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5 smithi194 running (15m) - 15m 433M 2524M 17.2.6-1303-gb2903cb0 5517396c50d4 47540765fad9 2023-10-09T04:54:31.500 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6 smithi194 running (14m) - 14m 276M 2524M 17.2.6-1303-gb2903cb0 5517396c50d4 26bbe2f2859f 2023-10-09T04:54:31.500 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7 smithi194 running (13m) - 13m 377M 2524M 17.2.6-1303-gb2903cb0 5517396c50d4 45c343cd22ea 2023-10-09T04:54:31.500 INFO:teuthology.orchestra.run.smithi133.stdout:prometheus.a smithi194 *:9095 running (3m) - 12m 49.1M - 2.43.0 a07b618ecd1d f40ed7def7ab 2023-10-09T04:54:32.627 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T04:54:32.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:32 smithi133 bash[102858]: cluster 2023-10-09T04:54:31.367032+0000 mgr.y (mgr.25277) 90 : cluster 0 pgmap v73: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 935 op/s 2023-10-09T04:54:32.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:32 smithi133 bash[102858]: audit 2023-10-09T04:54:31.478948+0000 mgr.y (mgr.25277) 91 : audit 0 from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:54:32.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:32 smithi133 bash[105589]: cluster 2023-10-09T04:54:31.367032+0000 mgr.y (mgr.25277) 90 : cluster 0 pgmap v73: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 935 op/s 2023-10-09T04:54:32.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:32 smithi133 bash[105589]: audit 2023-10-09T04:54:31.478948+0000 mgr.y (mgr.25277) 91 : audit 0 from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:54:32.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:32 smithi194 bash[31648]: cluster 2023-10-09T04:54:31.367032+0000 mgr.y (mgr.25277) 90 : cluster [DBG] pgmap v73: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 935 op/s 2023-10-09T04:54:32.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:32 smithi194 bash[31648]: audit 2023-10-09T04:54:31.478948+0000 mgr.y (mgr.25277) 91 : audit [DBG] from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:54:33.958 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:33 smithi133 bash[102858]: cluster 2023-10-09T04:54:33.367455+0000 mgr.y (mgr.25277) 92 : cluster 0 pgmap v74: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-10-09T04:54:34.282 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:33 smithi133 bash[105589]: cluster 2023-10-09T04:54:33.367455+0000 mgr.y (mgr.25277) 92 : cluster 0 pgmap v74: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-10-09T04:54:34.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:33 smithi194 bash[31648]: cluster 2023-10-09T04:54:33.367455+0000 mgr.y (mgr.25277) 92 : cluster [DBG] pgmap v74: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-10-09T04:54:34.550 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:54:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:54:34] "GET /metrics HTTP/1.1" 200 36169 "" "Prometheus/2.43.0" 2023-10-09T04:54:35.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:35 smithi133 bash[102858]: audit 2023-10-09T04:54:34.464286+0000 mon.a (mon.0) 134 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:35.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:35 smithi133 bash[102858]: audit 2023-10-09T04:54:34.475502+0000 mon.a (mon.0) 135 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:35.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:35 smithi133 bash[102858]: audit 2023-10-09T04:54:34.479822+0000 mon.a (mon.0) 136 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:54:35.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:35 smithi133 bash[102858]: audit 2023-10-09T04:54:34.484837+0000 mon.a (mon.0) 137 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:54:35.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:35 smithi133 bash[102858]: audit 2023-10-09T04:54:34.493912+0000 mon.a (mon.0) 138 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:35.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:35 smithi133 bash[102858]: audit 2023-10-09T04:54:34.577663+0000 mon.a (mon.0) 139 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:54:35.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:35 smithi133 bash[102858]: audit 2023-10-09T04:54:34.581268+0000 mon.a (mon.0) 140 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:54:35.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:35 smithi133 bash[102858]: audit 2023-10-09T04:54:34.583245+0000 mon.a (mon.0) 141 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T04:54:35.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:35 smithi133 bash[102858]: audit 2023-10-09T04:54:34.584300+0000 mon.a (mon.0) 142 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T04:54:35.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:35 smithi133 bash[102858]: cephadm 2023-10-09T04:54:34.585408+0000 mgr.y (mgr.25277) 93 : cephadm 1 Upgrade: It appears safe to stop mon.b 2023-10-09T04:54:35.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:35 smithi133 bash[105589]: audit 2023-10-09T04:54:34.464286+0000 mon.a (mon.0) 134 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:35.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:35 smithi133 bash[105589]: audit 2023-10-09T04:54:34.475502+0000 mon.a (mon.0) 135 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:35.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:35 smithi133 bash[105589]: audit 2023-10-09T04:54:34.479822+0000 mon.a (mon.0) 136 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:54:35.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:35 smithi133 bash[105589]: audit 2023-10-09T04:54:34.484837+0000 mon.a (mon.0) 137 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:54:35.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:35 smithi133 bash[105589]: audit 2023-10-09T04:54:34.493912+0000 mon.a (mon.0) 138 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:35.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:35 smithi133 bash[105589]: audit 2023-10-09T04:54:34.577663+0000 mon.a (mon.0) 139 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:54:35.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:35 smithi133 bash[105589]: audit 2023-10-09T04:54:34.581268+0000 mon.a (mon.0) 140 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:54:35.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:35 smithi133 bash[105589]: audit 2023-10-09T04:54:34.583245+0000 mon.a (mon.0) 141 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T04:54:35.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:35 smithi133 bash[105589]: audit 2023-10-09T04:54:34.584300+0000 mon.a (mon.0) 142 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T04:54:35.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:35 smithi133 bash[105589]: cephadm 2023-10-09T04:54:34.585408+0000 mgr.y (mgr.25277) 93 : cephadm 1 Upgrade: It appears safe to stop mon.b 2023-10-09T04:54:35.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:35 smithi194 bash[31648]: audit 2023-10-09T04:54:34.464286+0000 mon.a (mon.0) 134 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:35.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:35 smithi194 bash[31648]: audit 2023-10-09T04:54:34.475502+0000 mon.a (mon.0) 135 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:35.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:35 smithi194 bash[31648]: audit 2023-10-09T04:54:34.479822+0000 mon.a (mon.0) 136 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:54:35.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:35 smithi194 bash[31648]: audit 2023-10-09T04:54:34.484837+0000 mon.a (mon.0) 137 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:54:35.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:35 smithi194 bash[31648]: audit 2023-10-09T04:54:34.493912+0000 mon.a (mon.0) 138 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:35.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:35 smithi194 bash[31648]: audit 2023-10-09T04:54:34.577663+0000 mon.a (mon.0) 139 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:54:35.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:35 smithi194 bash[31648]: audit 2023-10-09T04:54:34.581268+0000 mon.a (mon.0) 140 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:54:35.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:35 smithi194 bash[31648]: audit 2023-10-09T04:54:34.583245+0000 mon.a (mon.0) 141 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T04:54:35.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:35 smithi194 bash[31648]: audit 2023-10-09T04:54:34.584300+0000 mon.a (mon.0) 142 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T04:54:35.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:35 smithi194 bash[31648]: cephadm 2023-10-09T04:54:34.585408+0000 mgr.y (mgr.25277) 93 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2023-10-09T04:54:36.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:36 smithi133 bash[102858]: cluster 2023-10-09T04:54:35.368168+0000 mgr.y (mgr.25277) 94 : cluster 0 pgmap v75: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 928 op/s 2023-10-09T04:54:36.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:36 smithi133 bash[105589]: cluster 2023-10-09T04:54:35.368168+0000 mgr.y (mgr.25277) 94 : cluster 0 pgmap v75: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 928 op/s 2023-10-09T04:54:36.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:36 smithi194 bash[31648]: cluster 2023-10-09T04:54:35.368168+0000 mgr.y (mgr.25277) 94 : cluster [DBG] pgmap v75: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 928 op/s 2023-10-09T04:54:37.959 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:38.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:38 smithi133 bash[102858]: cluster 2023-10-09T04:54:37.368610+0000 mgr.y (mgr.25277) 95 : cluster 0 pgmap v76: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-10-09T04:54:38.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:38 smithi133 bash[105589]: cluster 2023-10-09T04:54:37.368610+0000 mgr.y (mgr.25277) 95 : cluster 0 pgmap v76: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-10-09T04:54:38.877 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:38 smithi194 bash[31648]: cluster 2023-10-09T04:54:37.368610+0000 mgr.y (mgr.25277) 95 : cluster [DBG] pgmap v76: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-10-09T04:54:38.878 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:54:38 smithi194 bash[81885]: ts=2023-10-09T04:54:38.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi194.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi133:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:54:41.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:40 smithi133 bash[102858]: cluster 2023-10-09T04:54:39.369072+0000 mgr.y (mgr.25277) 96 : cluster 0 pgmap v77: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-10-09T04:54:41.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:40 smithi133 bash[102858]: cephadm 2023-10-09T04:54:39.721658+0000 mgr.y (mgr.25277) 97 : cephadm 1 Upgrade: Updating mon.b 2023-10-09T04:54:41.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:40 smithi133 bash[102858]: audit 2023-10-09T04:54:39.730715+0000 mon.a (mon.0) 143 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:41.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:40 smithi133 bash[102858]: audit 2023-10-09T04:54:39.731980+0000 mon.a (mon.0) 144 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:54:41.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:40 smithi133 bash[102858]: audit 2023-10-09T04:54:39.733900+0000 mon.a (mon.0) 145 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:54:41.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:40 smithi133 bash[102858]: audit 2023-10-09T04:54:39.734991+0000 mon.a (mon.0) 146 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:54:41.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:40 smithi133 bash[102858]: cephadm 2023-10-09T04:54:39.736275+0000 mgr.y (mgr.25277) 98 : cephadm 1 Deploying daemon mon.b on smithi194 2023-10-09T04:54:41.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:40 smithi133 bash[105589]: cluster 2023-10-09T04:54:39.369072+0000 mgr.y (mgr.25277) 96 : cluster 0 pgmap v77: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-10-09T04:54:41.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:40 smithi133 bash[105589]: cephadm 2023-10-09T04:54:39.721658+0000 mgr.y (mgr.25277) 97 : cephadm 1 Upgrade: Updating mon.b 2023-10-09T04:54:41.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:40 smithi133 bash[105589]: audit 2023-10-09T04:54:39.730715+0000 mon.a (mon.0) 143 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:41.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:40 smithi133 bash[105589]: audit 2023-10-09T04:54:39.731980+0000 mon.a (mon.0) 144 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:54:41.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:40 smithi133 bash[105589]: audit 2023-10-09T04:54:39.733900+0000 mon.a (mon.0) 145 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:54:41.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:40 smithi133 bash[105589]: audit 2023-10-09T04:54:39.734991+0000 mon.a (mon.0) 146 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:54:41.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:40 smithi133 bash[105589]: cephadm 2023-10-09T04:54:39.736275+0000 mgr.y (mgr.25277) 98 : cephadm 1 Deploying daemon mon.b on smithi194 2023-10-09T04:54:41.118 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:40 smithi194 bash[31648]: cluster 2023-10-09T04:54:39.369072+0000 mgr.y (mgr.25277) 96 : cluster [DBG] pgmap v77: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-10-09T04:54:41.118 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:40 smithi194 bash[31648]: cephadm 2023-10-09T04:54:39.721658+0000 mgr.y (mgr.25277) 97 : cephadm [INF] Upgrade: Updating mon.b 2023-10-09T04:54:41.118 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:40 smithi194 bash[31648]: audit 2023-10-09T04:54:39.730715+0000 mon.a (mon.0) 143 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:41.118 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:40 smithi194 bash[31648]: audit 2023-10-09T04:54:39.731980+0000 mon.a (mon.0) 144 : audit [INF] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:54:41.118 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:40 smithi194 bash[31648]: audit 2023-10-09T04:54:39.733900+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:54:41.119 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:40 smithi194 bash[31648]: audit 2023-10-09T04:54:39.734991+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:54:41.119 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:40 smithi194 bash[31648]: cephadm 2023-10-09T04:54:39.736275+0000 mgr.y (mgr.25277) 98 : cephadm [INF] Deploying daemon mon.b on smithi194 2023-10-09T04:54:42.415 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:42 smithi194 systemd[1]: Stopping Ceph mon.b for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T04:54:42.774 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:42 smithi194 bash[31648]: debug 2023-10-09T04:54:42.553+0000 7f7a4f71a700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T04:54:42.774 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:42 smithi194 bash[31648]: debug 2023-10-09T04:54:42.553+0000 7f7a4f71a700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2023-10-09T04:54:43.890 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:43 smithi194 bash[92561]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-mon-b 2023-10-09T04:54:43.890 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:43 smithi194 bash[92615]: Error response from daemon: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-mon-b 2023-10-09T04:54:43.890 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:43 smithi194 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mon.b.service: Succeeded. 2023-10-09T04:54:43.890 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:43 smithi194 systemd[1]: Stopped Ceph mon.b for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:54:44.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:43 smithi194 systemd[1]: Started Ceph mon.b for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:54:44.550 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:54:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:54:44] "GET /metrics HTTP/1.1" 200 36171 "" "Prometheus/2.43.0" 2023-10-09T04:54:45.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.293+0000 7f092564ec80 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T04:54:45.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.293+0000 7f092564ec80 0 ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T04:54:45.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.293+0000 7f092564ec80 0 pidfile_write: ignore empty --pid-file 2023-10-09T04:54:45.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 0 load: jerasure load: lrc 2023-10-09T04:54:45.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: RocksDB version: 7.9.2 2023-10-09T04:54:45.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Git sha 0 2023-10-09T04:54:45.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T04:54:45.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: DB SUMMARY 2023-10-09T04:54:45.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: DB Session ID: CM6XNQAFU77OTFI2Y4WC 2023-10-09T04:54:45.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: CURRENT file: CURRENT 2023-10-09T04:54:45.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: IDENTITY file: IDENTITY 2023-10-09T04:54:45.639 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: MANIFEST file: MANIFEST-000009 size: 2067 Bytes 2023-10-09T04:54:45.639 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000042.sst 2023-10-09T04:54:45.639 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000040.log size: 920224 ; 2023-10-09T04:54:45.639 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.error_if_exists: 0 2023-10-09T04:54:45.639 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.create_if_missing: 0 2023-10-09T04:54:45.639 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T04:54:45.639 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T04:54:45.639 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T04:54:45.639 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T04:54:45.639 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.env: 0x55ce08108720 2023-10-09T04:54:45.639 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.fs: PosixFileSystem 2023-10-09T04:54:45.639 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.info_log: 0x55ce09ddf480 2023-10-09T04:54:45.640 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T04:54:45.640 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.statistics: (nil) 2023-10-09T04:54:45.640 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.use_fsync: 0 2023-10-09T04:54:45.640 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T04:54:45.640 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T04:54:45.640 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T04:54:45.640 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T04:54:45.640 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T04:54:45.640 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T04:54:45.640 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T04:54:45.640 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T04:54:45.640 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T04:54:45.641 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T04:54:45.641 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T04:54:45.641 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.db_log_dir: 2023-10-09T04:54:45.641 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.wal_dir: 2023-10-09T04:54:45.641 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T04:54:45.641 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T04:54:45.641 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T04:54:45.641 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T04:54:45.641 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T04:54:45.641 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T04:54:45.641 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T04:54:45.641 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T04:54:45.642 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.write_buffer_manager: 0x55ce0906cc80 2023-10-09T04:54:45.642 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T04:54:45.642 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T04:54:45.642 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T04:54:45.642 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T04:54:45.642 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T04:54:45.642 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T04:54:45.643 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T04:54:45.643 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T04:54:45.643 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.unordered_write: 0 2023-10-09T04:54:45.644 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T04:54:45.644 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T04:54:45.644 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T04:54:45.644 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T04:54:45.644 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.row_cache: None 2023-10-09T04:54:45.644 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.wal_filter: None 2023-10-09T04:54:45.644 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T04:54:45.644 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T04:54:45.644 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.two_write_queues: 0 2023-10-09T04:54:45.644 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T04:54:45.644 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.wal_compression: 0 2023-10-09T04:54:45.644 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.atomic_flush: 0 2023-10-09T04:54:45.645 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T04:54:45.645 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T04:54:45.645 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T04:54:45.645 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T04:54:45.645 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T04:54:45.645 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.best_efforts_recovery: 0 2023-10-09T04:54:45.645 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T04:54:45.645 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T04:54:45.645 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.allow_data_in_errors: 0 2023-10-09T04:54:45.645 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.db_host_id: __hostname__ 2023-10-09T04:54:45.645 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T04:54:45.646 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T04:54:45.646 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T04:54:45.646 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T04:54:45.646 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T04:54:45.646 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T04:54:45.646 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T04:54:45.646 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T04:54:45.646 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T04:54:45.646 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T04:54:45.646 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T04:54:45.646 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T04:54:45.646 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.max_open_files: -1 2023-10-09T04:54:45.647 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T04:54:45.647 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T04:54:45.647 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T04:54:45.647 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T04:54:45.647 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T04:54:45.647 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Compression algorithms supported: 2023-10-09T04:54:45.647 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T04:54:45.647 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: kZSTD supported: 0 2023-10-09T04:54:45.647 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: kXpressCompression supported: 0 2023-10-09T04:54:45.647 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T04:54:45.647 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: kZlibCompression supported: 1 2023-10-09T04:54:45.647 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T04:54:45.648 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T04:54:45.648 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T04:54:45.648 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T04:54:45.648 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T04:54:45.648 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2023-10-09T04:54:45.648 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T04:54:45.648 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T04:54:45.648 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.merge_operator: 2023-10-09T04:54:45.648 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.compaction_filter: None 2023-10-09T04:54:45.648 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T04:54:45.648 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.sst_partitioner_factory: None 2023-10-09T04:54:45.648 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T04:54:45.649 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T04:54:45.649 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ce09ddf580) 2023-10-09T04:54:45.649 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: cache_index_and_filter_blocks: 1 2023-10-09T04:54:45.649 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T04:54:45.649 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T04:54:45.649 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: pin_top_level_index_and_filter: 1 2023-10-09T04:54:45.649 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: index_type: 0 2023-10-09T04:54:45.649 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: data_block_index_type: 0 2023-10-09T04:54:45.649 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: index_shortening: 1 2023-10-09T04:54:45.649 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: data_block_hash_table_util_ratio: 0.750000 2023-10-09T04:54:45.649 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: checksum: 4 2023-10-09T04:54:45.649 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: no_block_cache: 0 2023-10-09T04:54:45.650 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: block_cache: 0x55ce090ef1f0 2023-10-09T04:54:45.650 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: block_cache_name: BinnedLRUCache 2023-10-09T04:54:45.650 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: block_cache_options: 2023-10-09T04:54:45.650 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: capacity : 536870912 2023-10-09T04:54:45.650 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: num_shard_bits : 4 2023-10-09T04:54:45.650 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: strict_capacity_limit : 0 2023-10-09T04:54:45.650 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: high_pri_pool_ratio: 0.000 2023-10-09T04:54:45.650 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: block_cache_compressed: (nil) 2023-10-09T04:54:45.650 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: persistent_cache: (nil) 2023-10-09T04:54:45.650 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: block_size: 4096 2023-10-09T04:54:45.650 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: block_size_deviation: 10 2023-10-09T04:54:45.651 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: block_restart_interval: 16 2023-10-09T04:54:45.651 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: index_block_restart_interval: 1 2023-10-09T04:54:45.651 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: metadata_block_size: 4096 2023-10-09T04:54:45.651 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: partition_filters: 0 2023-10-09T04:54:45.651 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: use_delta_encoding: 1 2023-10-09T04:54:45.651 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: filter_policy: bloomfilter 2023-10-09T04:54:45.651 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: whole_key_filtering: 1 2023-10-09T04:54:45.651 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: verify_compression: 0 2023-10-09T04:54:45.651 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: read_amp_bytes_per_bit: 0 2023-10-09T04:54:45.651 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: format_version: 5 2023-10-09T04:54:45.651 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: enable_index_compression: 1 2023-10-09T04:54:45.651 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: block_align: 0 2023-10-09T04:54:45.651 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: max_auto_readahead_size: 262144 2023-10-09T04:54:45.652 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: prepopulate_block_cache: 0 2023-10-09T04:54:45.652 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: initial_auto_readahead_size: 8192 2023-10-09T04:54:45.652 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: num_file_reads_for_auto_readahead: 2 2023-10-09T04:54:45.652 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T04:54:45.652 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T04:54:45.652 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.compression: NoCompression 2023-10-09T04:54:45.652 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T04:54:45.652 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T04:54:45.652 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T04:54:45.652 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.num_levels: 7 2023-10-09T04:54:45.652 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T04:54:45.652 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T04:54:45.653 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T04:54:45.653 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T04:54:45.653 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T04:54:45.653 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T04:54:45.653 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T04:54:45.653 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:54:45.653 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T04:54:45.653 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T04:54:45.653 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T04:54:45.653 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T04:54:45.653 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T04:54:45.653 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T04:54:45.654 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T04:54:45.654 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T04:54:45.654 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:54:45.654 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T04:54:45.654 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T04:54:45.654 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T04:54:45.654 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T04:54:45.655 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T04:54:45.655 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T04:54:45.655 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T04:54:45.655 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T04:54:45.655 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T04:54:45.655 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T04:54:45.655 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T04:54:45.655 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T04:54:45.655 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T04:54:45.655 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T04:54:45.655 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T04:54:45.655 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T04:54:45.656 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T04:54:45.656 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T04:54:45.656 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T04:54:45.656 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T04:54:45.656 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T04:54:45.656 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T04:54:45.656 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.arena_block_size: 1048576 2023-10-09T04:54:45.656 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T04:54:45.656 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T04:54:45.656 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T04:54:45.656 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T04:54:45.656 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T04:54:45.657 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T04:54:45.657 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T04:54:45.657 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T04:54:45.657 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T04:54:45.657 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T04:54:45.657 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T04:54:45.657 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T04:54:45.657 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T04:54:45.657 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.table_properties_collectors: 2023-10-09T04:54:45.657 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T04:54:45.657 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T04:54:45.657 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T04:54:45.658 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T04:54:45.658 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T04:54:45.658 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.bloom_locality: 0 2023-10-09T04:54:45.658 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T04:54:45.658 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T04:54:45.658 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T04:54:45.658 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.force_consistency_checks: 1 2023-10-09T04:54:45.658 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T04:54:45.658 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.ttl: 2592000 2023-10-09T04:54:45.658 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T04:54:45.658 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T04:54:45.658 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T04:54:45.659 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.enable_blob_files: false 2023-10-09T04:54:45.659 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.min_blob_size: 0 2023-10-09T04:54:45.661 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.blob_file_size: 268435456 2023-10-09T04:54:45.661 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T04:54:45.661 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T04:54:45.662 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T04:54:45.662 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T04:54:45.662 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T04:54:45.662 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.blob_file_starting_level: 0 2023-10-09T04:54:45.662 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.301+0000 7f092564ec80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T04:54:45.662 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.305+0000 7f091ae6c700 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 42.sst 2023-10-09T04:54:45.662 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.313+0000 7f092564ec80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T04:54:45.662 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.313+0000 7f092564ec80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 44, last_sequence is 23081, log_number is 40,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T04:54:45.662 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.313+0000 7f092564ec80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 40 2023-10-09T04:54:45.662 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.313+0000 7f092564ec80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 9f5b740a-006c-4b3c-8322-326a0f3d621f 2023-10-09T04:54:45.662 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.313+0000 7f092564ec80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696827285316372, "job": 1, "event": "recovery_started", "wal_files": [40]} 2023-10-09T04:54:45.663 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.313+0000 7f092564ec80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #40 mode 2 2023-10-09T04:54:45.663 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.321+0000 7f092564ec80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696827285325861, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 45, "file_size": 563569, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 23086, "largest_seqno": 23782, "table_properties": {"data_size": 560282, "index_size": 1633, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 645, "raw_key_size": 5670, "raw_average_key_size": 22, "raw_value_size": 554840, "raw_average_value_size": 2246, "num_data_blocks": 75, "num_entries": 247, "num_filter_entries": 247, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696827285, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "9f5b740a-006c-4b3c-8322-326a0f3d621f", "db_session_id": "CM6XNQAFU77OTFI2Y4WC", "orig_file_number": 45, "seqno_to_time_mapping": "N/A"}} 2023-10-09T04:54:45.663 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.321+0000 7f092564ec80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696827285325976, "job": 1, "event": "recovery_finished"} 2023-10-09T04:54:45.663 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.321+0000 7f092564ec80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 47 2023-10-09T04:54:45.663 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.321+0000 7f092564ec80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T04:54:45.663 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.329+0000 7f092564ec80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000040.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T04:54:45.663 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.329+0000 7f092564ec80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55ce09190000 2023-10-09T04:54:45.663 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.329+0000 7f092564ec80 4 rocksdb: DB pointer 0x55ce0917e000 2023-10-09T04:54:45.663 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.329+0000 7f0913e5e700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T04:54:45.663 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.329+0000 7f0913e5e700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T04:54:45.663 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: ** DB Stats ** 2023-10-09T04:54:45.664 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:54:45.664 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T04:54:45.664 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T04:54:45.664 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:54:45.664 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T04:54:45.664 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T04:54:45.664 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:54:45.664 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: ** Compaction Stats [default] ** 2023-10-09T04:54:45.664 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T04:54:45.664 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T04:54:45.664 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: L0 1/0 550.36 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 114.9 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-10-09T04:54:45.664 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: L6 1/0 8.58 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T04:54:45.665 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: Sum 2/0 9.12 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 114.9 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-10-09T04:54:45.665 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 114.9 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-10-09T04:54:45.665 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: ** Compaction Stats [default] ** 2023-10-09T04:54:45.665 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T04:54:45.665 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:54:45.665 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 114.9 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-10-09T04:54:45.665 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T04:54:45.665 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:54:45.665 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: Flush(GB): cumulative 0.001, interval 0.001 2023-10-09T04:54:45.665 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T04:54:45.665 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T04:54:45.665 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T04:54:45.666 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T04:54:45.666 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: Cumulative compaction: 0.00 GB write, 19.81 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:54:45.666 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: Interval compaction: 0.00 GB write, 19.81 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:54:45.666 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T04:54:45.666 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: Block cache BinnedLRUCache@0x55ce090ef1f0#7 capacity: 512.00 MB usage: 41.23 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.4e-05 secs_since: 0 2023-10-09T04:54:45.666 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: Block cache entry stats(count,size,portion): DataBlock(1,6.58 KB,0.00125468%) FilterBlock(2,12.75 KB,0.00243187%) IndexBlock(2,21.91 KB,0.00417829%) Misc(1,0.00 KB,0%) 2023-10-09T04:54:45.666 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T04:54:45.666 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.329+0000 7f092564ec80 0 starting mon.b rank 1 at public addrs [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] at bind addrs [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 855b5206-665c-11ee-8db5-212e2dc638e7 2023-10-09T04:54:45.666 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.333+0000 7f092564ec80 1 mon.b@-1(???) e3 preinit fsid 855b5206-665c-11ee-8db5-212e2dc638e7 2023-10-09T04:54:45.666 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.333+0000 7f092564ec80 0 mon.b@-1(???).mds e10 new map 2023-10-09T04:54:45.666 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.333+0000 7f092564ec80 0 mon.b@-1(???).mds e10 print_map 2023-10-09T04:54:45.666 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: e10 2023-10-09T04:54:45.667 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T04:54:45.667 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T04:54:45.667 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: legacy client fscid: 1 2023-10-09T04:54:45.667 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: 2023-10-09T04:54:45.667 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: Filesystem 'foo' (1) 2023-10-09T04:54:45.667 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: fs_name foo 2023-10-09T04:54:45.667 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: epoch 10 2023-10-09T04:54:45.667 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T04:54:45.667 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: created 2023-10-09T04:44:52.764614+0000 2023-10-09T04:54:45.667 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: modified 2023-10-09T04:45:03.032645+0000 2023-10-09T04:54:45.667 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: tableserver 0 2023-10-09T04:54:45.667 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: root 0 2023-10-09T04:54:45.668 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: session_timeout 60 2023-10-09T04:54:45.668 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: session_autoclose 300 2023-10-09T04:54:45.668 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: max_file_size 1099511627776 2023-10-09T04:54:45.668 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: required_client_features {} 2023-10-09T04:54:45.668 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: last_failure 0 2023-10-09T04:54:45.668 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: last_failure_osd_epoch 69 2023-10-09T04:54:45.668 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T04:54:45.668 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: max_mds 1 2023-10-09T04:54:45.668 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: in 0 2023-10-09T04:54:45.668 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: up {0=24502} 2023-10-09T04:54:45.668 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: failed 2023-10-09T04:54:45.668 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: damaged 2023-10-09T04:54:45.669 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: stopped 2023-10-09T04:54:45.669 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: data_pools [4] 2023-10-09T04:54:45.669 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: metadata_pool 3 2023-10-09T04:54:45.669 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: inline_data disabled 2023-10-09T04:54:45.669 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: balancer 2023-10-09T04:54:45.669 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: bal_rank_mask -1 2023-10-09T04:54:45.669 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: standby_count_wanted 1 2023-10-09T04:54:45.669 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: [mds.foo.smithi194.yxnfqn{0:24502} state up:active seq 4 join_fscid=1 addr [v2:172.21.15.194:6832/1679470116,v1:172.21.15.194:6833/1679470116] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T04:54:45.669 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: 2023-10-09T04:54:45.669 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: 2023-10-09T04:54:45.669 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: Standby daemons: 2023-10-09T04:54:45.669 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: 2023-10-09T04:54:45.670 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: [mds.foo.smithi133.tsuabh{-1:14682} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.133:6834/3356898887,v1:172.21.15.133:6835/3356898887] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T04:54:45.670 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.333+0000 7f092564ec80 0 mon.b@-1(???).osd e83 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T04:54:45.670 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.333+0000 7f092564ec80 0 mon.b@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T04:54:45.670 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.333+0000 7f092564ec80 0 mon.b@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T04:54:45.670 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.333+0000 7f092564ec80 0 mon.b@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T04:54:45.670 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: debug 2023-10-09T04:54:45.333+0000 7f092564ec80 1 mon.b@-1(???).paxosservice(auth 1..24) refresh upgraded, format 0 -> 3 2023-10-09T04:54:46.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:45 smithi133 bash[102858]: cluster 2023-10-09T04:54:41.369836+0000 mgr.y (mgr.25277) 99 : cluster 0 pgmap v78: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 912 op/s 2023-10-09T04:54:46.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:45 smithi133 bash[105589]: cluster 2023-10-09T04:54:41.369836+0000 mgr.y (mgr.25277) 99 : cluster 0 pgmap v78: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 912 op/s 2023-10-09T04:54:46.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:45 smithi194 bash[92666]: cluster 2023-10-09T04:54:41.369836+0000 mgr.y (mgr.25277) 99 : cluster 0 pgmap v78: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 912 op/s 2023-10-09T04:54:49.137 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:54:48 smithi194 bash[81885]: ts=2023-10-09T04:54:48.663Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi194.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi133:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:54:50.800 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:54:50 smithi133 bash[91128]: debug 2023-10-09T04:54:50.346+0000 7ff45d5ed700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-10-09T04:54:50.854 INFO:teuthology.orchestra.run.smithi133.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:54:51.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:50 smithi194 bash[92666]: cluster 2023-10-09T04:54:45.370986+0000 mgr.y (mgr.25277) 103 : cluster 0 pgmap v80: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2023-10-09T04:54:51.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:50 smithi194 bash[92666]: cluster 2023-10-09T04:54:45.800759+0000 mon.a (mon.0) 152 : cluster 0 osdmap e83: 8 total, 8 up, 8 in 2023-10-09T04:54:51.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:50 smithi194 bash[92666]: cluster 2023-10-09T04:54:45.801851+0000 mon.a (mon.0) 153 : cluster 0 mgrmap e35: y(active, since 2m), standbys: x 2023-10-09T04:54:51.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:50 smithi194 bash[92666]: cluster 2023-10-09T04:54:45.810635+0000 mon.a (mon.0) 154 : cluster 1 overall HEALTH_OK 2023-10-09T04:54:51.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:50 smithi194 bash[92666]: cluster 2023-10-09T04:54:45.811392+0000 mon.a (mon.0) 155 : cluster 1 mon.a calling monitor election 2023-10-09T04:54:51.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:50 smithi194 bash[92666]: cluster 2023-10-09T04:54:45.812525+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2023-10-09T04:54:51.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:50 smithi194 bash[92666]: audit 2023-10-09T04:54:45.814037+0000 mon.a (mon.0) 156 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:54:51.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:50 smithi194 bash[92666]: audit 2023-10-09T04:54:45.814213+0000 mon.a (mon.0) 157 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:54:51.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:50 smithi194 bash[92666]: audit 2023-10-09T04:54:45.814399+0000 mon.a (mon.0) 158 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:54:51.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:50 smithi194 bash[92666]: cluster 2023-10-09T04:54:47.372346+0000 mgr.y (mgr.25277) 104 : cluster 0 pgmap v81: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-09T04:54:51.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:50 smithi194 bash[92666]: cluster 2023-10-09T04:54:49.372757+0000 mgr.y (mgr.25277) 105 : cluster 0 pgmap v82: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-09T04:54:51.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:50 smithi194 bash[92666]: cluster 2023-10-09T04:54:50.812969+0000 mon.a (mon.0) 159 : cluster 1 mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T04:54:51.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:50 smithi194 bash[92666]: cluster 2023-10-09T04:54:50.818796+0000 mon.a (mon.0) 160 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.133:3300/0,v1:172.21.15.133:6789/0],b=[v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0],c=[v2:172.21.15.133:3301/0,v1:172.21.15.133:6790/0]} removed_ranks: {} 2023-10-09T04:54:51.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:50 smithi194 bash[92666]: cluster 2023-10-09T04:54:50.819832+0000 mon.a (mon.0) 161 : cluster 0 fsmap foo:1 {0=foo.smithi194.yxnfqn=up:active} 1 up:standby 2023-10-09T04:54:51.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:50 smithi194 bash[92666]: cluster 2023-10-09T04:54:50.819906+0000 mon.a (mon.0) 162 : cluster 0 osdmap e83: 8 total, 8 up, 8 in 2023-10-09T04:54:51.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:50 smithi194 bash[92666]: cluster 2023-10-09T04:54:50.820876+0000 mon.a (mon.0) 163 : cluster 0 mgrmap e35: y(active, since 2m), standbys: x 2023-10-09T04:54:51.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:50 smithi194 bash[92666]: cluster 2023-10-09T04:54:50.821177+0000 mon.a (mon.0) 164 : cluster 3 Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-10-09T04:54:51.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:50 smithi194 bash[92666]: cluster 2023-10-09T04:54:50.843929+0000 mon.a (mon.0) 165 : cluster 3 Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-10-09T04:54:51.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:50 smithi194 bash[92666]: cluster 2023-10-09T04:54:50.843983+0000 mon.a (mon.0) 166 : cluster 3 [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-10-09T04:54:51.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:50 smithi194 bash[92666]: cluster 2023-10-09T04:54:50.844008+0000 mon.a (mon.0) 167 : cluster 3 mon.c (rank 2) addr [v2:172.21.15.133:3301/0,v1:172.21.15.133:6790/0] is down (out of quorum) 2023-10-09T04:54:51.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:50 smithi194 bash[92666]: audit 2023-10-09T04:54:50.851233+0000 mon.a (mon.0) 168 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:51.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:50 smithi194 bash[92666]: audit 2023-10-09T04:54:50.858618+0000 mon.a (mon.0) 169 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:51.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:50 smithi194 bash[92666]: audit 2023-10-09T04:54:50.867803+0000 mon.a (mon.0) 170 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:51.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:50 smithi133 bash[102858]: cluster 2023-10-09T04:54:45.370986+0000 mgr.y (mgr.25277) 103 : cluster 0 pgmap v80: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2023-10-09T04:54:51.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:50 smithi133 bash[102858]: cluster 2023-10-09T04:54:45.800759+0000 mon.a (mon.0) 152 : cluster 0 osdmap e83: 8 total, 8 up, 8 in 2023-10-09T04:54:51.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:50 smithi133 bash[102858]: cluster 2023-10-09T04:54:45.801851+0000 mon.a (mon.0) 153 : cluster 0 mgrmap e35: y(active, since 2m), standbys: x 2023-10-09T04:54:51.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:50 smithi133 bash[102858]: cluster 2023-10-09T04:54:45.810635+0000 mon.a (mon.0) 154 : cluster 1 overall HEALTH_OK 2023-10-09T04:54:51.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:50 smithi133 bash[102858]: cluster 2023-10-09T04:54:45.811392+0000 mon.a (mon.0) 155 : cluster 1 mon.a calling monitor election 2023-10-09T04:54:51.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:50 smithi133 bash[102858]: cluster 2023-10-09T04:54:45.812525+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2023-10-09T04:54:51.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:50 smithi133 bash[102858]: audit 2023-10-09T04:54:45.814037+0000 mon.a (mon.0) 156 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:54:51.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:50 smithi133 bash[102858]: audit 2023-10-09T04:54:45.814213+0000 mon.a (mon.0) 157 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:54:51.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:50 smithi133 bash[102858]: audit 2023-10-09T04:54:45.814399+0000 mon.a (mon.0) 158 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:54:51.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:50 smithi133 bash[102858]: cluster 2023-10-09T04:54:47.372346+0000 mgr.y (mgr.25277) 104 : cluster 0 pgmap v81: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-09T04:54:51.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:50 smithi133 bash[102858]: cluster 2023-10-09T04:54:49.372757+0000 mgr.y (mgr.25277) 105 : cluster 0 pgmap v82: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-09T04:54:51.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:50 smithi133 bash[102858]: cluster 2023-10-09T04:54:50.812969+0000 mon.a (mon.0) 159 : cluster 1 mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T04:54:51.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:50 smithi133 bash[102858]: cluster 2023-10-09T04:54:50.818796+0000 mon.a (mon.0) 160 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.133:3300/0,v1:172.21.15.133:6789/0],b=[v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0],c=[v2:172.21.15.133:3301/0,v1:172.21.15.133:6790/0]} removed_ranks: {} 2023-10-09T04:54:51.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:50 smithi133 bash[102858]: cluster 2023-10-09T04:54:50.819832+0000 mon.a (mon.0) 161 : cluster 0 fsmap foo:1 {0=foo.smithi194.yxnfqn=up:active} 1 up:standby 2023-10-09T04:54:51.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:50 smithi133 bash[102858]: cluster 2023-10-09T04:54:50.819906+0000 mon.a (mon.0) 162 : cluster 0 osdmap e83: 8 total, 8 up, 8 in 2023-10-09T04:54:51.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:50 smithi133 bash[102858]: cluster 2023-10-09T04:54:50.820876+0000 mon.a (mon.0) 163 : cluster 0 mgrmap e35: y(active, since 2m), standbys: x 2023-10-09T04:54:51.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:50 smithi133 bash[102858]: cluster 2023-10-09T04:54:50.821177+0000 mon.a (mon.0) 164 : cluster 3 Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-10-09T04:54:51.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:50 smithi133 bash[102858]: cluster 2023-10-09T04:54:50.843929+0000 mon.a (mon.0) 165 : cluster 3 Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-10-09T04:54:51.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:50 smithi133 bash[102858]: cluster 2023-10-09T04:54:50.843983+0000 mon.a (mon.0) 166 : cluster 3 [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-10-09T04:54:51.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:50 smithi133 bash[102858]: cluster 2023-10-09T04:54:50.844008+0000 mon.a (mon.0) 167 : cluster 3 mon.c (rank 2) addr [v2:172.21.15.133:3301/0,v1:172.21.15.133:6790/0] is down (out of quorum) 2023-10-09T04:54:51.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:50 smithi133 bash[102858]: audit 2023-10-09T04:54:50.851233+0000 mon.a (mon.0) 168 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:51.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:50 smithi133 bash[102858]: audit 2023-10-09T04:54:50.858618+0000 mon.a (mon.0) 169 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:51.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:50 smithi133 bash[102858]: audit 2023-10-09T04:54:50.867803+0000 mon.a (mon.0) 170 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:52.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:51 smithi133 bash[102858]: audit 2023-10-09T04:54:50.909725+0000 mon.a (mon.0) 171 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:54:52.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:51 smithi194 bash[92666]: audit 2023-10-09T04:54:50.909725+0000 mon.a (mon.0) 171 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:54:52.763 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T04:54:53.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[102858]: cluster 2023-10-09T04:54:45.814794+0000 mon.c (mon.2) 2 : cluster 1 mon.c calling monitor election 2023-10-09T04:54:53.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[102858]: cluster 2023-10-09T04:54:52.762110+0000 mon.a (mon.0) 172 : cluster 1 mon.a calling monitor election 2023-10-09T04:54:53.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[102858]: cluster 2023-10-09T04:54:52.763027+0000 mon.b (mon.1) 3 : cluster 1 mon.b calling monitor election 2023-10-09T04:54:53.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[102858]: cluster 2023-10-09T04:54:52.783418+0000 mon.a (mon.0) 173 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:54:53.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[102858]: cluster 2023-10-09T04:54:52.790237+0000 mon.a (mon.0) 174 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.133:3300/0,v1:172.21.15.133:6789/0],b=[v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0],c=[v2:172.21.15.133:3301/0,v1:172.21.15.133:6790/0]} removed_ranks: {} 2023-10-09T04:54:53.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[102858]: cluster 2023-10-09T04:54:52.791093+0000 mon.a (mon.0) 175 : cluster 0 fsmap foo:1 {0=foo.smithi194.yxnfqn=up:active} 1 up:standby 2023-10-09T04:54:53.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[102858]: cluster 2023-10-09T04:54:52.791252+0000 mon.a (mon.0) 176 : cluster 0 osdmap e83: 8 total, 8 up, 8 in 2023-10-09T04:54:53.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[102858]: cluster 2023-10-09T04:54:52.792300+0000 mon.a (mon.0) 177 : cluster 0 mgrmap e35: y(active, since 2m), standbys: x 2023-10-09T04:54:53.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[102858]: cluster 2023-10-09T04:54:52.793011+0000 mon.a (mon.0) 178 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T04:54:53.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[102858]: cluster 2023-10-09T04:54:52.793094+0000 mon.a (mon.0) 179 : cluster 1 Cluster is now healthy 2023-10-09T04:54:53.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[102858]: cluster 2023-10-09T04:54:52.812326+0000 mon.a (mon.0) 180 : cluster 1 overall HEALTH_OK 2023-10-09T04:54:53.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: cluster 2023-10-09T04:54:45.370986+0000 mgr.y (mgr.25277) 103 : cluster 0 pgmap v80: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2023-10-09T04:54:53.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: cluster 2023-10-09T04:54:45.800759+0000 mon.a (mon.0) 152 : cluster 0 osdmap e83: 8 total, 8 up, 8 in 2023-10-09T04:54:53.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: cluster 2023-10-09T04:54:45.801851+0000 mon.a (mon.0) 153 : cluster 0 mgrmap e35: y(active, since 2m), standbys: x 2023-10-09T04:54:53.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: cluster 2023-10-09T04:54:45.810635+0000 mon.a (mon.0) 154 : cluster 1 overall HEALTH_OK 2023-10-09T04:54:53.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: cluster 2023-10-09T04:54:45.811392+0000 mon.a (mon.0) 155 : cluster 1 mon.a calling monitor election 2023-10-09T04:54:53.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: cluster 2023-10-09T04:54:45.812525+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2023-10-09T04:54:53.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: audit 2023-10-09T04:54:45.814037+0000 mon.a (mon.0) 156 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:54:53.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: audit 2023-10-09T04:54:45.814213+0000 mon.a (mon.0) 157 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:54:53.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: audit 2023-10-09T04:54:45.814399+0000 mon.a (mon.0) 158 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:54:53.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: cluster 2023-10-09T04:54:47.372346+0000 mgr.y (mgr.25277) 104 : cluster 0 pgmap v81: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-09T04:54:53.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: cluster 2023-10-09T04:54:49.372757+0000 mgr.y (mgr.25277) 105 : cluster 0 pgmap v82: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-09T04:54:53.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: cluster 2023-10-09T04:54:50.812969+0000 mon.a (mon.0) 159 : cluster 1 mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T04:54:53.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: cluster 2023-10-09T04:54:50.818796+0000 mon.a (mon.0) 160 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.133:3300/0,v1:172.21.15.133:6789/0],b=[v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0],c=[v2:172.21.15.133:3301/0,v1:172.21.15.133:6790/0]} removed_ranks: {} 2023-10-09T04:54:53.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: cluster 2023-10-09T04:54:50.819832+0000 mon.a (mon.0) 161 : cluster 0 fsmap foo:1 {0=foo.smithi194.yxnfqn=up:active} 1 up:standby 2023-10-09T04:54:53.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: cluster 2023-10-09T04:54:50.819906+0000 mon.a (mon.0) 162 : cluster 0 osdmap e83: 8 total, 8 up, 8 in 2023-10-09T04:54:53.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: cluster 2023-10-09T04:54:50.820876+0000 mon.a (mon.0) 163 : cluster 0 mgrmap e35: y(active, since 2m), standbys: x 2023-10-09T04:54:53.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: cluster 2023-10-09T04:54:50.821177+0000 mon.a (mon.0) 164 : cluster 3 Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-10-09T04:54:53.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: cluster 2023-10-09T04:54:50.843929+0000 mon.a (mon.0) 165 : cluster 3 Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-10-09T04:54:53.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: cluster 2023-10-09T04:54:50.843983+0000 mon.a (mon.0) 166 : cluster 3 [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-10-09T04:54:53.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: cluster 2023-10-09T04:54:50.844008+0000 mon.a (mon.0) 167 : cluster 3 mon.c (rank 2) addr [v2:172.21.15.133:3301/0,v1:172.21.15.133:6790/0] is down (out of quorum) 2023-10-09T04:54:53.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: audit 2023-10-09T04:54:50.851233+0000 mon.a (mon.0) 168 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:53.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: audit 2023-10-09T04:54:50.858618+0000 mon.a (mon.0) 169 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:53.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: audit 2023-10-09T04:54:50.867803+0000 mon.a (mon.0) 170 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:53.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: audit 2023-10-09T04:54:50.909725+0000 mon.a (mon.0) 171 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:54:53.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: cluster 2023-10-09T04:54:45.814794+0000 mon.c (mon.2) 2 : cluster 1 mon.c calling monitor election 2023-10-09T04:54:53.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: cluster 2023-10-09T04:54:52.762110+0000 mon.a (mon.0) 172 : cluster 1 mon.a calling monitor election 2023-10-09T04:54:53.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: cluster 2023-10-09T04:54:52.763027+0000 mon.b (mon.1) 3 : cluster 1 mon.b calling monitor election 2023-10-09T04:54:53.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: cluster 2023-10-09T04:54:52.783418+0000 mon.a (mon.0) 173 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:54:53.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: cluster 2023-10-09T04:54:52.790237+0000 mon.a (mon.0) 174 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.133:3300/0,v1:172.21.15.133:6789/0],b=[v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0],c=[v2:172.21.15.133:3301/0,v1:172.21.15.133:6790/0]} removed_ranks: {} 2023-10-09T04:54:53.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: cluster 2023-10-09T04:54:52.791093+0000 mon.a (mon.0) 175 : cluster 0 fsmap foo:1 {0=foo.smithi194.yxnfqn=up:active} 1 up:standby 2023-10-09T04:54:53.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: cluster 2023-10-09T04:54:52.791252+0000 mon.a (mon.0) 176 : cluster 0 osdmap e83: 8 total, 8 up, 8 in 2023-10-09T04:54:53.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: cluster 2023-10-09T04:54:52.792300+0000 mon.a (mon.0) 177 : cluster 0 mgrmap e35: y(active, since 2m), standbys: x 2023-10-09T04:54:53.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: cluster 2023-10-09T04:54:52.793011+0000 mon.a (mon.0) 178 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T04:54:53.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: cluster 2023-10-09T04:54:52.793094+0000 mon.a (mon.0) 179 : cluster 1 Cluster is now healthy 2023-10-09T04:54:53.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:52 smithi133 bash[105589]: cluster 2023-10-09T04:54:52.812326+0000 mon.a (mon.0) 180 : cluster 1 overall HEALTH_OK 2023-10-09T04:54:53.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:52 smithi194 bash[92666]: cluster 2023-10-09T04:54:45.814794+0000 mon.c (mon.2) 2 : cluster 1 mon.c calling monitor election 2023-10-09T04:54:53.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:52 smithi194 bash[92666]: cluster 2023-10-09T04:54:52.762110+0000 mon.a (mon.0) 172 : cluster 1 mon.a calling monitor election 2023-10-09T04:54:53.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:52 smithi194 bash[92666]: cluster 2023-10-09T04:54:52.763027+0000 mon.b (mon.1) 3 : cluster 1 mon.b calling monitor election 2023-10-09T04:54:53.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:52 smithi194 bash[92666]: cluster 2023-10-09T04:54:52.783418+0000 mon.a (mon.0) 173 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:54:53.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:52 smithi194 bash[92666]: cluster 2023-10-09T04:54:52.790237+0000 mon.a (mon.0) 174 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.133:3300/0,v1:172.21.15.133:6789/0],b=[v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0],c=[v2:172.21.15.133:3301/0,v1:172.21.15.133:6790/0]} removed_ranks: {} 2023-10-09T04:54:53.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:52 smithi194 bash[92666]: cluster 2023-10-09T04:54:52.791093+0000 mon.a (mon.0) 175 : cluster 0 fsmap foo:1 {0=foo.smithi194.yxnfqn=up:active} 1 up:standby 2023-10-09T04:54:53.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:52 smithi194 bash[92666]: cluster 2023-10-09T04:54:52.791252+0000 mon.a (mon.0) 176 : cluster 0 osdmap e83: 8 total, 8 up, 8 in 2023-10-09T04:54:53.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:52 smithi194 bash[92666]: cluster 2023-10-09T04:54:52.792300+0000 mon.a (mon.0) 177 : cluster 0 mgrmap e35: y(active, since 2m), standbys: x 2023-10-09T04:54:53.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:52 smithi194 bash[92666]: cluster 2023-10-09T04:54:52.793011+0000 mon.a (mon.0) 178 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T04:54:53.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:52 smithi194 bash[92666]: cluster 2023-10-09T04:54:52.793094+0000 mon.a (mon.0) 179 : cluster 1 Cluster is now healthy 2023-10-09T04:54:53.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:52 smithi194 bash[92666]: cluster 2023-10-09T04:54:52.812326+0000 mon.a (mon.0) 180 : cluster 1 overall HEALTH_OK 2023-10-09T04:54:54.286 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:54:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:54:54] "GET /metrics HTTP/1.1" 200 36171 "" "Prometheus/2.43.0" 2023-10-09T04:54:54.286 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:53 smithi133 bash[105589]: cluster 2023-10-09T04:54:53.373976+0000 mgr.y (mgr.25277) 107 : cluster 0 pgmap v84: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-10-09T04:54:54.287 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:53 smithi133 bash[102858]: cluster 2023-10-09T04:54:53.373976+0000 mgr.y (mgr.25277) 107 : cluster 0 pgmap v84: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-10-09T04:54:54.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:53 smithi194 bash[92666]: cluster 2023-10-09T04:54:53.373976+0000 mgr.y (mgr.25277) 107 : cluster 0 pgmap v84: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-10-09T04:54:56.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:56 smithi194 bash[92666]: cluster 2023-10-09T04:54:55.380280+0000 mgr.y (mgr.25277) 108 : cluster 0 pgmap v85: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2023-10-09T04:54:56.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:56 smithi133 bash[105589]: cluster 2023-10-09T04:54:55.380280+0000 mgr.y (mgr.25277) 108 : cluster 0 pgmap v85: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2023-10-09T04:54:56.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:56 smithi133 bash[102858]: cluster 2023-10-09T04:54:55.380280+0000 mgr.y (mgr.25277) 108 : cluster 0 pgmap v85: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2023-10-09T04:54:57.599 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:59.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:58 smithi133 bash[102858]: cluster 2023-10-09T04:54:57.381636+0000 mgr.y (mgr.25277) 109 : cluster 0 pgmap v86: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-10-09T04:54:59.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:58 smithi133 bash[102858]: audit 2023-10-09T04:54:57.712992+0000 mon.a (mon.0) 181 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:59.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:58 smithi133 bash[102858]: audit 2023-10-09T04:54:57.725653+0000 mon.a (mon.0) 182 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:59.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:58 smithi133 bash[102858]: audit 2023-10-09T04:54:58.410216+0000 mon.a (mon.0) 183 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:59.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:58 smithi133 bash[102858]: audit 2023-10-09T04:54:58.677248+0000 mon.a (mon.0) 184 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:59.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:54:58 smithi133 bash[102858]: audit 2023-10-09T04:54:58.693658+0000 mon.a (mon.0) 185 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:59.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:58 smithi133 bash[105589]: cluster 2023-10-09T04:54:57.381636+0000 mgr.y (mgr.25277) 109 : cluster 0 pgmap v86: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-10-09T04:54:59.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:58 smithi133 bash[105589]: audit 2023-10-09T04:54:57.712992+0000 mon.a (mon.0) 181 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:59.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:58 smithi133 bash[105589]: audit 2023-10-09T04:54:57.725653+0000 mon.a (mon.0) 182 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:59.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:58 smithi133 bash[105589]: audit 2023-10-09T04:54:58.410216+0000 mon.a (mon.0) 183 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:59.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:58 smithi133 bash[105589]: audit 2023-10-09T04:54:58.677248+0000 mon.a (mon.0) 184 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:59.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:54:58 smithi133 bash[105589]: audit 2023-10-09T04:54:58.693658+0000 mon.a (mon.0) 185 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:59.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:58 smithi194 bash[92666]: cluster 2023-10-09T04:54:57.381636+0000 mgr.y (mgr.25277) 109 : cluster 0 pgmap v86: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-10-09T04:54:59.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:58 smithi194 bash[92666]: audit 2023-10-09T04:54:57.712992+0000 mon.a (mon.0) 181 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:59.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:58 smithi194 bash[92666]: audit 2023-10-09T04:54:57.725653+0000 mon.a (mon.0) 182 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:59.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:58 smithi194 bash[92666]: audit 2023-10-09T04:54:58.410216+0000 mon.a (mon.0) 183 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:59.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:58 smithi194 bash[92666]: audit 2023-10-09T04:54:58.677248+0000 mon.a (mon.0) 184 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:59.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:54:58 smithi194 bash[92666]: audit 2023-10-09T04:54:58.693658+0000 mon.a (mon.0) 185 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:54:59.138 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 04:54:58 smithi194 bash[81885]: ts=2023-10-09T04:54:58.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi194.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.133\", device_class=\"ssd\", hostname=\"smithi133\", instance=\"smithi133:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.133\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:55:01.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:00 smithi133 bash[102858]: cluster 2023-10-09T04:54:59.382130+0000 mgr.y (mgr.25277) 110 : cluster 0 pgmap v87: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-09T04:55:01.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:00 smithi133 bash[105589]: cluster 2023-10-09T04:54:59.382130+0000 mgr.y (mgr.25277) 110 : cluster 0 pgmap v87: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-09T04:55:01.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:00 smithi194 bash[92666]: cluster 2023-10-09T04:54:59.382130+0000 mgr.y (mgr.25277) 110 : cluster 0 pgmap v87: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-09T04:55:03.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:02 smithi133 bash[102858]: cluster 2023-10-09T04:55:01.382817+0000 mgr.y (mgr.25277) 111 : cluster 0 pgmap v88: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-10-09T04:55:03.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:02 smithi133 bash[105589]: cluster 2023-10-09T04:55:01.382817+0000 mgr.y (mgr.25277) 111 : cluster 0 pgmap v88: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-10-09T04:55:03.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:02 smithi194 bash[92666]: cluster 2023-10-09T04:55:01.382817+0000 mgr.y (mgr.25277) 111 : cluster 0 pgmap v88: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-10-09T04:55:03.958 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:03 smithi194 bash[92666]: cluster 2023-10-09T04:55:03.383203+0000 mgr.y (mgr.25277) 112 : cluster 0 pgmap v89: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-10-09T04:55:04.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:03 smithi133 bash[102858]: cluster 2023-10-09T04:55:03.383203+0000 mgr.y (mgr.25277) 112 : cluster 0 pgmap v89: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-10-09T04:55:04.284 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:55:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:55:04] "GET /metrics HTTP/1.1" 200 36166 "" "Prometheus/2.43.0" 2023-10-09T04:55:04.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:03 smithi133 bash[105589]: cluster 2023-10-09T04:55:03.383203+0000 mgr.y (mgr.25277) 112 : cluster 0 pgmap v89: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-10-09T04:55:06.712 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:06 smithi194 bash[92666]: cluster 2023-10-09T04:55:05.383972+0000 mgr.y (mgr.25277) 113 : cluster 0 pgmap v90: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2023-10-09T04:55:06.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:06 smithi133 bash[105589]: cluster 2023-10-09T04:55:05.383972+0000 mgr.y (mgr.25277) 113 : cluster 0 pgmap v90: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2023-10-09T04:55:06.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:06 smithi133 bash[102858]: cluster 2023-10-09T04:55:05.383972+0000 mgr.y (mgr.25277) 113 : cluster 0 pgmap v90: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2023-10-09T04:55:08.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:08 smithi133 bash[102858]: cluster 2023-10-09T04:55:07.384430+0000 mgr.y (mgr.25277) 114 : cluster 0 pgmap v91: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-10-09T04:55:08.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:08 smithi133 bash[105589]: cluster 2023-10-09T04:55:07.384430+0000 mgr.y (mgr.25277) 114 : cluster 0 pgmap v91: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-10-09T04:55:08.847 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:08 smithi194 bash[92666]: cluster 2023-10-09T04:55:07.384430+0000 mgr.y (mgr.25277) 114 : cluster 0 pgmap v91: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-10-09T04:55:10.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:10 smithi133 bash[102858]: cluster 2023-10-09T04:55:09.384861+0000 mgr.y (mgr.25277) 115 : cluster 0 pgmap v92: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-10-09T04:55:10.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:10 smithi133 bash[105589]: cluster 2023-10-09T04:55:09.384861+0000 mgr.y (mgr.25277) 115 : cluster 0 pgmap v92: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-10-09T04:55:10.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:10 smithi194 bash[92666]: cluster 2023-10-09T04:55:09.384861+0000 mgr.y (mgr.25277) 115 : cluster 0 pgmap v92: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-10-09T04:55:12.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:12 smithi133 bash[102858]: cluster 2023-10-09T04:55:11.385654+0000 mgr.y (mgr.25277) 116 : cluster 0 pgmap v93: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-10-09T04:55:12.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:12 smithi133 bash[102858]: audit 2023-10-09T04:55:12.281699+0000 mon.a (mon.0) 186 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:12.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:12 smithi133 bash[102858]: audit 2023-10-09T04:55:12.290098+0000 mon.a (mon.0) 187 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:12.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:12 smithi133 bash[102858]: audit 2023-10-09T04:55:12.296361+0000 mon.a (mon.0) 188 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:12.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:12 smithi133 bash[102858]: audit 2023-10-09T04:55:12.298246+0000 mon.a (mon.0) 189 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:55:12.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:12 smithi133 bash[102858]: audit 2023-10-09T04:55:12.307584+0000 mon.a (mon.0) 190 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:12.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:12 smithi133 bash[102858]: audit 2023-10-09T04:55:12.387100+0000 mon.a (mon.0) 191 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:55:12.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:12 smithi133 bash[102858]: audit 2023-10-09T04:55:12.388692+0000 mon.a (mon.0) 192 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:55:12.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:12 smithi133 bash[102858]: audit 2023-10-09T04:55:12.389945+0000 mon.a (mon.0) 193 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:12.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:12 smithi133 bash[105589]: cluster 2023-10-09T04:55:11.385654+0000 mgr.y (mgr.25277) 116 : cluster 0 pgmap v93: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-10-09T04:55:12.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:12 smithi133 bash[105589]: audit 2023-10-09T04:55:12.281699+0000 mon.a (mon.0) 186 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:12.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:12 smithi133 bash[105589]: audit 2023-10-09T04:55:12.290098+0000 mon.a (mon.0) 187 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:12.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:12 smithi133 bash[105589]: audit 2023-10-09T04:55:12.296361+0000 mon.a (mon.0) 188 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:12.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:12 smithi133 bash[105589]: audit 2023-10-09T04:55:12.298246+0000 mon.a (mon.0) 189 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:55:12.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:12 smithi133 bash[105589]: audit 2023-10-09T04:55:12.307584+0000 mon.a (mon.0) 190 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:12.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:12 smithi133 bash[105589]: audit 2023-10-09T04:55:12.387100+0000 mon.a (mon.0) 191 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:55:12.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:12 smithi133 bash[105589]: audit 2023-10-09T04:55:12.388692+0000 mon.a (mon.0) 192 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:55:12.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:12 smithi133 bash[105589]: audit 2023-10-09T04:55:12.389945+0000 mon.a (mon.0) 193 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:12.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:12 smithi194 bash[92666]: cluster 2023-10-09T04:55:11.385654+0000 mgr.y (mgr.25277) 116 : cluster 0 pgmap v93: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-10-09T04:55:12.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:12 smithi194 bash[92666]: audit 2023-10-09T04:55:12.281699+0000 mon.a (mon.0) 186 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:12.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:12 smithi194 bash[92666]: audit 2023-10-09T04:55:12.290098+0000 mon.a (mon.0) 187 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:12.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:12 smithi194 bash[92666]: audit 2023-10-09T04:55:12.296361+0000 mon.a (mon.0) 188 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:12.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:12 smithi194 bash[92666]: audit 2023-10-09T04:55:12.298246+0000 mon.a (mon.0) 189 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:55:12.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:12 smithi194 bash[92666]: audit 2023-10-09T04:55:12.307584+0000 mon.a (mon.0) 190 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:12.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:12 smithi194 bash[92666]: audit 2023-10-09T04:55:12.387100+0000 mon.a (mon.0) 191 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:55:12.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:12 smithi194 bash[92666]: audit 2023-10-09T04:55:12.388692+0000 mon.a (mon.0) 192 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:55:12.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:12 smithi194 bash[92666]: audit 2023-10-09T04:55:12.389945+0000 mon.a (mon.0) 193 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:13.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:13 smithi133 bash[102858]: cephadm 2023-10-09T04:55:12.386394+0000 mgr.y (mgr.25277) 117 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2023-10-09T04:55:13.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:13 smithi133 bash[102858]: cephadm 2023-10-09T04:55:12.391254+0000 mgr.y (mgr.25277) 118 : cephadm 1 Reconfiguring daemon mon.a on smithi133 2023-10-09T04:55:13.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:13 smithi133 bash[102858]: audit 2023-10-09T04:55:13.410239+0000 mon.a (mon.0) 194 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:13.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:13 smithi133 bash[105589]: cephadm 2023-10-09T04:55:12.386394+0000 mgr.y (mgr.25277) 117 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2023-10-09T04:55:13.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:13 smithi133 bash[105589]: cephadm 2023-10-09T04:55:12.391254+0000 mgr.y (mgr.25277) 118 : cephadm 1 Reconfiguring daemon mon.a on smithi133 2023-10-09T04:55:13.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:13 smithi133 bash[105589]: audit 2023-10-09T04:55:13.410239+0000 mon.a (mon.0) 194 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:13.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:13 smithi194 bash[92666]: cephadm 2023-10-09T04:55:12.386394+0000 mgr.y (mgr.25277) 117 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2023-10-09T04:55:13.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:13 smithi194 bash[92666]: cephadm 2023-10-09T04:55:12.391254+0000 mgr.y (mgr.25277) 118 : cephadm 1 Reconfiguring daemon mon.a on smithi133 2023-10-09T04:55:13.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:13 smithi194 bash[92666]: audit 2023-10-09T04:55:13.410239+0000 mon.a (mon.0) 194 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:14.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:14 smithi133 bash[102858]: cluster 2023-10-09T04:55:13.386046+0000 mgr.y (mgr.25277) 119 : cluster 0 pgmap v94: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-10-09T04:55:14.550 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:55:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:55:14] "GET /metrics HTTP/1.1" 200 36167 "" "Prometheus/2.43.0" 2023-10-09T04:55:14.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:14 smithi133 bash[105589]: cluster 2023-10-09T04:55:13.386046+0000 mgr.y (mgr.25277) 119 : cluster 0 pgmap v94: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-10-09T04:55:14.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:14 smithi194 bash[92666]: cluster 2023-10-09T04:55:13.386046+0000 mgr.y (mgr.25277) 119 : cluster 0 pgmap v94: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-10-09T04:55:16.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:16 smithi133 bash[102858]: cluster 2023-10-09T04:55:15.386836+0000 mgr.y (mgr.25277) 120 : cluster 0 pgmap v95: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 897 op/s 2023-10-09T04:55:16.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:16 smithi133 bash[105589]: cluster 2023-10-09T04:55:15.386836+0000 mgr.y (mgr.25277) 120 : cluster 0 pgmap v95: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 897 op/s 2023-10-09T04:55:16.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:16 smithi194 bash[92666]: cluster 2023-10-09T04:55:15.386836+0000 mgr.y (mgr.25277) 120 : cluster 0 pgmap v95: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 897 op/s 2023-10-09T04:55:17.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:17 smithi133 bash[105589]: audit 2023-10-09T04:55:16.531777+0000 mon.a (mon.0) 195 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:17.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:17 smithi133 bash[105589]: audit 2023-10-09T04:55:16.539646+0000 mon.a (mon.0) 196 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:17.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:17 smithi133 bash[105589]: cephadm 2023-10-09T04:55:16.586687+0000 mgr.y (mgr.25277) 121 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2023-10-09T04:55:17.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:17 smithi133 bash[105589]: audit 2023-10-09T04:55:16.587197+0000 mon.a (mon.0) 197 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:55:17.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:17 smithi133 bash[105589]: audit 2023-10-09T04:55:16.588389+0000 mon.a (mon.0) 198 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:55:17.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:17 smithi133 bash[105589]: audit 2023-10-09T04:55:16.589449+0000 mon.a (mon.0) 199 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:17.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:17 smithi133 bash[105589]: cephadm 2023-10-09T04:55:16.590680+0000 mgr.y (mgr.25277) 122 : cephadm 1 Reconfiguring daemon mgr.y on smithi133 2023-10-09T04:55:17.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:17 smithi133 bash[102858]: audit 2023-10-09T04:55:16.531777+0000 mon.a (mon.0) 195 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:17.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:17 smithi133 bash[102858]: audit 2023-10-09T04:55:16.539646+0000 mon.a (mon.0) 196 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:17.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:17 smithi133 bash[102858]: cephadm 2023-10-09T04:55:16.586687+0000 mgr.y (mgr.25277) 121 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2023-10-09T04:55:17.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:17 smithi133 bash[102858]: audit 2023-10-09T04:55:16.587197+0000 mon.a (mon.0) 197 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:55:17.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:17 smithi133 bash[102858]: audit 2023-10-09T04:55:16.588389+0000 mon.a (mon.0) 198 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:55:17.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:17 smithi133 bash[102858]: audit 2023-10-09T04:55:16.589449+0000 mon.a (mon.0) 199 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:17.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:17 smithi133 bash[102858]: cephadm 2023-10-09T04:55:16.590680+0000 mgr.y (mgr.25277) 122 : cephadm 1 Reconfiguring daemon mgr.y on smithi133 2023-10-09T04:55:17.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:17 smithi194 bash[92666]: audit 2023-10-09T04:55:16.531777+0000 mon.a (mon.0) 195 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:17.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:17 smithi194 bash[92666]: audit 2023-10-09T04:55:16.539646+0000 mon.a (mon.0) 196 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:17.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:17 smithi194 bash[92666]: cephadm 2023-10-09T04:55:16.586687+0000 mgr.y (mgr.25277) 121 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2023-10-09T04:55:17.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:17 smithi194 bash[92666]: audit 2023-10-09T04:55:16.587197+0000 mon.a (mon.0) 197 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:55:17.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:17 smithi194 bash[92666]: audit 2023-10-09T04:55:16.588389+0000 mon.a (mon.0) 198 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:55:17.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:17 smithi194 bash[92666]: audit 2023-10-09T04:55:16.589449+0000 mon.a (mon.0) 199 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:17.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:17 smithi194 bash[92666]: cephadm 2023-10-09T04:55:16.590680+0000 mgr.y (mgr.25277) 122 : cephadm 1 Reconfiguring daemon mgr.y on smithi133 2023-10-09T04:55:18.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:18 smithi133 bash[102858]: cluster 2023-10-09T04:55:17.387203+0000 mgr.y (mgr.25277) 123 : cluster 0 pgmap v96: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-09T04:55:18.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:18 smithi133 bash[105589]: cluster 2023-10-09T04:55:17.387203+0000 mgr.y (mgr.25277) 123 : cluster 0 pgmap v96: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-09T04:55:18.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:18 smithi194 bash[92666]: cluster 2023-10-09T04:55:17.387203+0000 mgr.y (mgr.25277) 123 : cluster 0 pgmap v96: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-09T04:55:20.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:20 smithi133 bash[105589]: cluster 2023-10-09T04:55:19.387615+0000 mgr.y (mgr.25277) 124 : cluster 0 pgmap v97: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-09T04:55:20.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:20 smithi133 bash[105589]: audit 2023-10-09T04:55:19.397314+0000 mon.a (mon.0) 200 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:20.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:20 smithi133 bash[105589]: audit 2023-10-09T04:55:19.405652+0000 mon.a (mon.0) 201 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:20.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:20 smithi133 bash[105589]: cephadm 2023-10-09T04:55:19.430336+0000 mgr.y (mgr.25277) 125 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2023-10-09T04:55:20.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:20 smithi133 bash[105589]: audit 2023-10-09T04:55:19.430917+0000 mon.a (mon.0) 202 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:55:20.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:20 smithi133 bash[105589]: audit 2023-10-09T04:55:19.434435+0000 mon.a (mon.0) 203 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:55:20.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:20 smithi133 bash[105589]: audit 2023-10-09T04:55:19.435649+0000 mon.a (mon.0) 204 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:20.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:20 smithi133 bash[105589]: cephadm 2023-10-09T04:55:19.436953+0000 mgr.y (mgr.25277) 126 : cephadm 1 Reconfiguring daemon mon.c on smithi133 2023-10-09T04:55:20.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:20 smithi133 bash[102858]: cluster 2023-10-09T04:55:19.387615+0000 mgr.y (mgr.25277) 124 : cluster 0 pgmap v97: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-09T04:55:20.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:20 smithi133 bash[102858]: audit 2023-10-09T04:55:19.397314+0000 mon.a (mon.0) 200 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:20.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:20 smithi133 bash[102858]: audit 2023-10-09T04:55:19.405652+0000 mon.a (mon.0) 201 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:20.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:20 smithi133 bash[102858]: cephadm 2023-10-09T04:55:19.430336+0000 mgr.y (mgr.25277) 125 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2023-10-09T04:55:20.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:20 smithi133 bash[102858]: audit 2023-10-09T04:55:19.430917+0000 mon.a (mon.0) 202 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:55:20.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:20 smithi133 bash[102858]: audit 2023-10-09T04:55:19.434435+0000 mon.a (mon.0) 203 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:55:20.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:20 smithi133 bash[102858]: audit 2023-10-09T04:55:19.435649+0000 mon.a (mon.0) 204 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:20.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:20 smithi133 bash[102858]: cephadm 2023-10-09T04:55:19.436953+0000 mgr.y (mgr.25277) 126 : cephadm 1 Reconfiguring daemon mon.c on smithi133 2023-10-09T04:55:20.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:20 smithi194 bash[92666]: cluster 2023-10-09T04:55:19.387615+0000 mgr.y (mgr.25277) 124 : cluster 0 pgmap v97: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-09T04:55:20.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:20 smithi194 bash[92666]: audit 2023-10-09T04:55:19.397314+0000 mon.a (mon.0) 200 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:20.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:20 smithi194 bash[92666]: audit 2023-10-09T04:55:19.405652+0000 mon.a (mon.0) 201 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:20.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:20 smithi194 bash[92666]: cephadm 2023-10-09T04:55:19.430336+0000 mgr.y (mgr.25277) 125 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2023-10-09T04:55:20.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:20 smithi194 bash[92666]: audit 2023-10-09T04:55:19.430917+0000 mon.a (mon.0) 202 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:55:20.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:20 smithi194 bash[92666]: audit 2023-10-09T04:55:19.434435+0000 mon.a (mon.0) 203 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:55:20.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:20 smithi194 bash[92666]: audit 2023-10-09T04:55:19.435649+0000 mon.a (mon.0) 204 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:20.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:20 smithi194 bash[92666]: cephadm 2023-10-09T04:55:19.436953+0000 mgr.y (mgr.25277) 126 : cephadm 1 Reconfiguring daemon mon.c on smithi133 2023-10-09T04:55:22.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:22 smithi133 bash[105589]: cluster 2023-10-09T04:55:21.388321+0000 mgr.y (mgr.25277) 127 : cluster 0 pgmap v98: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 905 op/s 2023-10-09T04:55:22.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:22 smithi133 bash[105589]: audit 2023-10-09T04:55:22.056619+0000 mon.a (mon.0) 205 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:22.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:22 smithi133 bash[105589]: audit 2023-10-09T04:55:22.068277+0000 mon.a (mon.0) 206 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:22.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:22 smithi133 bash[105589]: audit 2023-10-09T04:55:22.109025+0000 mon.a (mon.0) 207 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:55:22.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:22 smithi133 bash[105589]: audit 2023-10-09T04:55:22.110263+0000 mon.a (mon.0) 208 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:22.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:22 smithi133 bash[102858]: cluster 2023-10-09T04:55:21.388321+0000 mgr.y (mgr.25277) 127 : cluster 0 pgmap v98: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 905 op/s 2023-10-09T04:55:22.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:22 smithi133 bash[102858]: audit 2023-10-09T04:55:22.056619+0000 mon.a (mon.0) 205 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:22.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:22 smithi133 bash[102858]: audit 2023-10-09T04:55:22.068277+0000 mon.a (mon.0) 206 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:22.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:22 smithi133 bash[102858]: audit 2023-10-09T04:55:22.109025+0000 mon.a (mon.0) 207 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:55:22.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:22 smithi133 bash[102858]: audit 2023-10-09T04:55:22.110263+0000 mon.a (mon.0) 208 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:22.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:22 smithi194 bash[92666]: cluster 2023-10-09T04:55:21.388321+0000 mgr.y (mgr.25277) 127 : cluster 0 pgmap v98: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 905 op/s 2023-10-09T04:55:22.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:22 smithi194 bash[92666]: audit 2023-10-09T04:55:22.056619+0000 mon.a (mon.0) 205 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:22.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:22 smithi194 bash[92666]: audit 2023-10-09T04:55:22.068277+0000 mon.a (mon.0) 206 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:22.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:22 smithi194 bash[92666]: audit 2023-10-09T04:55:22.109025+0000 mon.a (mon.0) 207 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:55:22.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:22 smithi194 bash[92666]: audit 2023-10-09T04:55:22.110263+0000 mon.a (mon.0) 208 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:23.760 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:23 smithi133 bash[105589]: cephadm 2023-10-09T04:55:22.108429+0000 mgr.y (mgr.25277) 128 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2023-10-09T04:55:23.760 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:23 smithi133 bash[105589]: cephadm 2023-10-09T04:55:22.113273+0000 mgr.y (mgr.25277) 129 : cephadm 1 Reconfiguring daemon osd.0 on smithi133 2023-10-09T04:55:23.760 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:23 smithi133 bash[102858]: cephadm 2023-10-09T04:55:22.108429+0000 mgr.y (mgr.25277) 128 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2023-10-09T04:55:23.760 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:23 smithi133 bash[102858]: cephadm 2023-10-09T04:55:22.113273+0000 mgr.y (mgr.25277) 129 : cephadm 1 Reconfiguring daemon osd.0 on smithi133 2023-10-09T04:55:23.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:23 smithi194 bash[92666]: cephadm 2023-10-09T04:55:22.108429+0000 mgr.y (mgr.25277) 128 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2023-10-09T04:55:23.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:23 smithi194 bash[92666]: cephadm 2023-10-09T04:55:22.113273+0000 mgr.y (mgr.25277) 129 : cephadm 1 Reconfiguring daemon osd.0 on smithi133 2023-10-09T04:55:24.409 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:55:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:55:24] "GET /metrics HTTP/1.1" 200 36167 "" "Prometheus/2.43.0" 2023-10-09T04:55:24.410 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:24 smithi133 bash[102858]: cluster 2023-10-09T04:55:23.388757+0000 mgr.y (mgr.25277) 130 : cluster 0 pgmap v99: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-10-09T04:55:24.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:24 smithi133 bash[105589]: cluster 2023-10-09T04:55:23.388757+0000 mgr.y (mgr.25277) 130 : cluster 0 pgmap v99: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-10-09T04:55:24.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:24 smithi194 bash[92666]: cluster 2023-10-09T04:55:23.388757+0000 mgr.y (mgr.25277) 130 : cluster 0 pgmap v99: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-10-09T04:55:26.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:25 smithi133 bash[105589]: audit 2023-10-09T04:55:24.942636+0000 mon.a (mon.0) 209 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:26.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:25 smithi133 bash[105589]: audit 2023-10-09T04:55:24.950240+0000 mon.a (mon.0) 210 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:26.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:25 smithi133 bash[105589]: cephadm 2023-10-09T04:55:24.993295+0000 mgr.y (mgr.25277) 131 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2023-10-09T04:55:26.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:25 smithi133 bash[105589]: audit 2023-10-09T04:55:24.993929+0000 mon.a (mon.0) 211 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:55:26.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:25 smithi133 bash[105589]: audit 2023-10-09T04:55:24.995429+0000 mon.a (mon.0) 212 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:26.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:25 smithi133 bash[105589]: cephadm 2023-10-09T04:55:24.999346+0000 mgr.y (mgr.25277) 132 : cephadm 1 Reconfiguring daemon osd.1 on smithi133 2023-10-09T04:55:26.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:25 smithi133 bash[105589]: cluster 2023-10-09T04:55:25.389503+0000 mgr.y (mgr.25277) 133 : cluster 0 pgmap v100: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2023-10-09T04:55:26.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:25 smithi133 bash[102858]: audit 2023-10-09T04:55:24.942636+0000 mon.a (mon.0) 209 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:26.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:25 smithi133 bash[102858]: audit 2023-10-09T04:55:24.950240+0000 mon.a (mon.0) 210 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:26.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:25 smithi133 bash[102858]: cephadm 2023-10-09T04:55:24.993295+0000 mgr.y (mgr.25277) 131 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2023-10-09T04:55:26.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:25 smithi133 bash[102858]: audit 2023-10-09T04:55:24.993929+0000 mon.a (mon.0) 211 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:55:26.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:25 smithi133 bash[102858]: audit 2023-10-09T04:55:24.995429+0000 mon.a (mon.0) 212 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:26.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:25 smithi133 bash[102858]: cephadm 2023-10-09T04:55:24.999346+0000 mgr.y (mgr.25277) 132 : cephadm 1 Reconfiguring daemon osd.1 on smithi133 2023-10-09T04:55:26.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:25 smithi133 bash[102858]: cluster 2023-10-09T04:55:25.389503+0000 mgr.y (mgr.25277) 133 : cluster 0 pgmap v100: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2023-10-09T04:55:26.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:25 smithi194 bash[92666]: audit 2023-10-09T04:55:24.942636+0000 mon.a (mon.0) 209 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:26.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:25 smithi194 bash[92666]: audit 2023-10-09T04:55:24.950240+0000 mon.a (mon.0) 210 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:26.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:25 smithi194 bash[92666]: cephadm 2023-10-09T04:55:24.993295+0000 mgr.y (mgr.25277) 131 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2023-10-09T04:55:26.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:25 smithi194 bash[92666]: audit 2023-10-09T04:55:24.993929+0000 mon.a (mon.0) 211 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:55:26.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:25 smithi194 bash[92666]: audit 2023-10-09T04:55:24.995429+0000 mon.a (mon.0) 212 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:26.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:25 smithi194 bash[92666]: cephadm 2023-10-09T04:55:24.999346+0000 mgr.y (mgr.25277) 132 : cephadm 1 Reconfiguring daemon osd.1 on smithi133 2023-10-09T04:55:26.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:25 smithi194 bash[92666]: cluster 2023-10-09T04:55:25.389503+0000 mgr.y (mgr.25277) 133 : cluster 0 pgmap v100: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2023-10-09T04:55:28.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:28 smithi194 bash[92666]: cluster 2023-10-09T04:55:27.390359+0000 mgr.y (mgr.25277) 134 : cluster 0 pgmap v101: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2023-10-09T04:55:28.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:28 smithi194 bash[92666]: audit 2023-10-09T04:55:27.628595+0000 mon.a (mon.0) 213 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:28.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:28 smithi194 bash[92666]: audit 2023-10-09T04:55:27.648103+0000 mon.a (mon.0) 214 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:28.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:28 smithi194 bash[92666]: cephadm 2023-10-09T04:55:27.683330+0000 mgr.y (mgr.25277) 135 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2023-10-09T04:55:28.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:28 smithi194 bash[92666]: audit 2023-10-09T04:55:27.683741+0000 mon.a (mon.0) 215 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T04:55:28.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:28 smithi194 bash[92666]: audit 2023-10-09T04:55:27.684732+0000 mon.a (mon.0) 216 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:28.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:28 smithi194 bash[92666]: cephadm 2023-10-09T04:55:27.688226+0000 mgr.y (mgr.25277) 136 : cephadm 1 Reconfiguring daemon osd.2 on smithi133 2023-10-09T04:55:28.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:28 smithi194 bash[92666]: audit 2023-10-09T04:55:28.410783+0000 mon.a (mon.0) 217 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:29.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:28 smithi133 bash[102858]: cluster 2023-10-09T04:55:27.390359+0000 mgr.y (mgr.25277) 134 : cluster 0 pgmap v101: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2023-10-09T04:55:29.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:28 smithi133 bash[102858]: audit 2023-10-09T04:55:27.628595+0000 mon.a (mon.0) 213 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:29.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:28 smithi133 bash[102858]: audit 2023-10-09T04:55:27.648103+0000 mon.a (mon.0) 214 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:29.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:28 smithi133 bash[102858]: cephadm 2023-10-09T04:55:27.683330+0000 mgr.y (mgr.25277) 135 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2023-10-09T04:55:29.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:28 smithi133 bash[102858]: audit 2023-10-09T04:55:27.683741+0000 mon.a (mon.0) 215 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T04:55:29.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:28 smithi133 bash[102858]: audit 2023-10-09T04:55:27.684732+0000 mon.a (mon.0) 216 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:29.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:28 smithi133 bash[102858]: cephadm 2023-10-09T04:55:27.688226+0000 mgr.y (mgr.25277) 136 : cephadm 1 Reconfiguring daemon osd.2 on smithi133 2023-10-09T04:55:29.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:28 smithi133 bash[102858]: audit 2023-10-09T04:55:28.410783+0000 mon.a (mon.0) 217 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:29.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:28 smithi133 bash[105589]: cluster 2023-10-09T04:55:27.390359+0000 mgr.y (mgr.25277) 134 : cluster 0 pgmap v101: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2023-10-09T04:55:29.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:28 smithi133 bash[105589]: audit 2023-10-09T04:55:27.628595+0000 mon.a (mon.0) 213 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:29.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:28 smithi133 bash[105589]: audit 2023-10-09T04:55:27.648103+0000 mon.a (mon.0) 214 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:29.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:28 smithi133 bash[105589]: cephadm 2023-10-09T04:55:27.683330+0000 mgr.y (mgr.25277) 135 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2023-10-09T04:55:29.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:28 smithi133 bash[105589]: audit 2023-10-09T04:55:27.683741+0000 mon.a (mon.0) 215 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T04:55:29.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:28 smithi133 bash[105589]: audit 2023-10-09T04:55:27.684732+0000 mon.a (mon.0) 216 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:29.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:28 smithi133 bash[105589]: cephadm 2023-10-09T04:55:27.688226+0000 mgr.y (mgr.25277) 136 : cephadm 1 Reconfiguring daemon osd.2 on smithi133 2023-10-09T04:55:29.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:28 smithi133 bash[105589]: audit 2023-10-09T04:55:28.410783+0000 mon.a (mon.0) 217 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:30.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:30 smithi194 bash[92666]: cluster 2023-10-09T04:55:29.390746+0000 mgr.y (mgr.25277) 137 : cluster 0 pgmap v102: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2023-10-09T04:55:30.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:30 smithi194 bash[92666]: audit 2023-10-09T04:55:30.328599+0000 mon.a (mon.0) 218 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:30.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:30 smithi194 bash[92666]: audit 2023-10-09T04:55:30.338582+0000 mon.a (mon.0) 219 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:30.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:30 smithi194 bash[92666]: audit 2023-10-09T04:55:30.382925+0000 mon.a (mon.0) 220 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T04:55:30.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:30 smithi194 bash[92666]: audit 2023-10-09T04:55:30.383937+0000 mon.a (mon.0) 221 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:31.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:30 smithi133 bash[105589]: cluster 2023-10-09T04:55:29.390746+0000 mgr.y (mgr.25277) 137 : cluster 0 pgmap v102: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2023-10-09T04:55:31.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:30 smithi133 bash[105589]: audit 2023-10-09T04:55:30.328599+0000 mon.a (mon.0) 218 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:31.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:30 smithi133 bash[105589]: audit 2023-10-09T04:55:30.338582+0000 mon.a (mon.0) 219 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:31.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:30 smithi133 bash[105589]: audit 2023-10-09T04:55:30.382925+0000 mon.a (mon.0) 220 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T04:55:31.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:30 smithi133 bash[105589]: audit 2023-10-09T04:55:30.383937+0000 mon.a (mon.0) 221 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:31.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:30 smithi133 bash[102858]: cluster 2023-10-09T04:55:29.390746+0000 mgr.y (mgr.25277) 137 : cluster 0 pgmap v102: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2023-10-09T04:55:31.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:30 smithi133 bash[102858]: audit 2023-10-09T04:55:30.328599+0000 mon.a (mon.0) 218 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:31.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:30 smithi133 bash[102858]: audit 2023-10-09T04:55:30.338582+0000 mon.a (mon.0) 219 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:31.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:30 smithi133 bash[102858]: audit 2023-10-09T04:55:30.382925+0000 mon.a (mon.0) 220 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T04:55:31.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:30 smithi133 bash[102858]: audit 2023-10-09T04:55:30.383937+0000 mon.a (mon.0) 221 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:31.946 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:31 smithi133 bash[102858]: cephadm 2023-10-09T04:55:30.382448+0000 mgr.y (mgr.25277) 138 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2023-10-09T04:55:31.946 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:31 smithi133 bash[102858]: cephadm 2023-10-09T04:55:30.386787+0000 mgr.y (mgr.25277) 139 : cephadm 1 Reconfiguring daemon osd.3 on smithi133 2023-10-09T04:55:31.947 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:31 smithi133 bash[105589]: cephadm 2023-10-09T04:55:30.382448+0000 mgr.y (mgr.25277) 138 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2023-10-09T04:55:31.947 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:31 smithi133 bash[105589]: cephadm 2023-10-09T04:55:30.386787+0000 mgr.y (mgr.25277) 139 : cephadm 1 Reconfiguring daemon osd.3 on smithi133 2023-10-09T04:55:32.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:31 smithi194 bash[92666]: cephadm 2023-10-09T04:55:30.382448+0000 mgr.y (mgr.25277) 138 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2023-10-09T04:55:32.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:31 smithi194 bash[92666]: cephadm 2023-10-09T04:55:30.386787+0000 mgr.y (mgr.25277) 139 : cephadm 1 Reconfiguring daemon osd.3 on smithi133 2023-10-09T04:55:33.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:32 smithi133 bash[102858]: cluster 2023-10-09T04:55:31.391417+0000 mgr.y (mgr.25277) 140 : cluster 0 pgmap v103: 89 pgs: 89 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 905 op/s 2023-10-09T04:55:33.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:32 smithi133 bash[105589]: cluster 2023-10-09T04:55:31.391417+0000 mgr.y (mgr.25277) 140 : cluster 0 pgmap v103: 89 pgs: 89 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 905 op/s 2023-10-09T04:55:33.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:32 smithi194 bash[92666]: cluster 2023-10-09T04:55:31.391417+0000 mgr.y (mgr.25277) 140 : cluster 0 pgmap v103: 89 pgs: 89 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 905 op/s 2023-10-09T04:55:34.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:34 smithi133 bash[105589]: audit 2023-10-09T04:55:33.139430+0000 mon.a (mon.0) 222 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:34.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:34 smithi133 bash[105589]: audit 2023-10-09T04:55:33.151879+0000 mon.a (mon.0) 223 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:34.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:34 smithi133 bash[105589]: cephadm 2023-10-09T04:55:33.160197+0000 mgr.y (mgr.25277) 141 : cephadm 1 Reconfiguring mds.foo.smithi133.tsuabh (monmap changed)... 2023-10-09T04:55:34.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:34 smithi133 bash[105589]: audit 2023-10-09T04:55:33.160650+0000 mon.a (mon.0) 224 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi133.tsuabh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:55:34.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:34 smithi133 bash[105589]: audit 2023-10-09T04:55:33.161899+0000 mon.a (mon.0) 225 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:34.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:34 smithi133 bash[105589]: cephadm 2023-10-09T04:55:33.163160+0000 mgr.y (mgr.25277) 142 : cephadm 1 Reconfiguring daemon mds.foo.smithi133.tsuabh on smithi133 2023-10-09T04:55:34.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:34 smithi133 bash[105589]: cluster 2023-10-09T04:55:33.391761+0000 mgr.y (mgr.25277) 143 : cluster 0 pgmap v104: 89 pgs: 89 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-09T04:55:34.284 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:55:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:55:34] "GET /metrics HTTP/1.1" 200 36164 "" "Prometheus/2.43.0" 2023-10-09T04:55:34.284 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:34 smithi133 bash[102858]: audit 2023-10-09T04:55:33.139430+0000 mon.a (mon.0) 222 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:34.285 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:34 smithi133 bash[102858]: audit 2023-10-09T04:55:33.151879+0000 mon.a (mon.0) 223 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:34.285 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:34 smithi133 bash[102858]: cephadm 2023-10-09T04:55:33.160197+0000 mgr.y (mgr.25277) 141 : cephadm 1 Reconfiguring mds.foo.smithi133.tsuabh (monmap changed)... 2023-10-09T04:55:34.285 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:34 smithi133 bash[102858]: audit 2023-10-09T04:55:33.160650+0000 mon.a (mon.0) 224 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi133.tsuabh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:55:34.285 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:34 smithi133 bash[102858]: audit 2023-10-09T04:55:33.161899+0000 mon.a (mon.0) 225 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:34.285 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:34 smithi133 bash[102858]: cephadm 2023-10-09T04:55:33.163160+0000 mgr.y (mgr.25277) 142 : cephadm 1 Reconfiguring daemon mds.foo.smithi133.tsuabh on smithi133 2023-10-09T04:55:34.285 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:34 smithi133 bash[102858]: cluster 2023-10-09T04:55:33.391761+0000 mgr.y (mgr.25277) 143 : cluster 0 pgmap v104: 89 pgs: 89 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-09T04:55:34.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:34 smithi194 bash[92666]: audit 2023-10-09T04:55:33.139430+0000 mon.a (mon.0) 222 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:34.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:34 smithi194 bash[92666]: audit 2023-10-09T04:55:33.151879+0000 mon.a (mon.0) 223 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:34.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:34 smithi194 bash[92666]: cephadm 2023-10-09T04:55:33.160197+0000 mgr.y (mgr.25277) 141 : cephadm 1 Reconfiguring mds.foo.smithi133.tsuabh (monmap changed)... 2023-10-09T04:55:34.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:34 smithi194 bash[92666]: audit 2023-10-09T04:55:33.160650+0000 mon.a (mon.0) 224 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi133.tsuabh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:55:34.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:34 smithi194 bash[92666]: audit 2023-10-09T04:55:33.161899+0000 mon.a (mon.0) 225 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:34.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:34 smithi194 bash[92666]: cephadm 2023-10-09T04:55:33.163160+0000 mgr.y (mgr.25277) 142 : cephadm 1 Reconfiguring daemon mds.foo.smithi133.tsuabh on smithi133 2023-10-09T04:55:34.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:34 smithi194 bash[92666]: cluster 2023-10-09T04:55:33.391761+0000 mgr.y (mgr.25277) 143 : cluster 0 pgmap v104: 89 pgs: 89 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-09T04:55:37.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:36 smithi133 bash[102858]: cluster 2023-10-09T04:55:35.392461+0000 mgr.y (mgr.25277) 144 : cluster 0 pgmap v105: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 902 op/s 2023-10-09T04:55:37.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:36 smithi133 bash[102858]: audit 2023-10-09T04:55:35.784275+0000 mon.a (mon.0) 226 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:37.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:36 smithi133 bash[102858]: audit 2023-10-09T04:55:35.792031+0000 mon.a (mon.0) 227 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:37.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:36 smithi133 bash[102858]: cephadm 2023-10-09T04:55:35.798223+0000 mgr.y (mgr.25277) 145 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2023-10-09T04:55:37.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:36 smithi133 bash[102858]: audit 2023-10-09T04:55:35.798708+0000 mon.a (mon.0) 228 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:55:37.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:36 smithi133 bash[102858]: audit 2023-10-09T04:55:35.799733+0000 mon.a (mon.0) 229 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:55:37.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:36 smithi133 bash[102858]: audit 2023-10-09T04:55:35.800638+0000 mon.a (mon.0) 230 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:37.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:36 smithi133 bash[102858]: cephadm 2023-10-09T04:55:35.801797+0000 mgr.y (mgr.25277) 146 : cephadm 1 Reconfiguring daemon mon.b on smithi194 2023-10-09T04:55:37.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:36 smithi133 bash[105589]: cluster 2023-10-09T04:55:35.392461+0000 mgr.y (mgr.25277) 144 : cluster 0 pgmap v105: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 902 op/s 2023-10-09T04:55:37.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:36 smithi133 bash[105589]: audit 2023-10-09T04:55:35.784275+0000 mon.a (mon.0) 226 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:37.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:36 smithi133 bash[105589]: audit 2023-10-09T04:55:35.792031+0000 mon.a (mon.0) 227 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:37.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:36 smithi133 bash[105589]: cephadm 2023-10-09T04:55:35.798223+0000 mgr.y (mgr.25277) 145 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2023-10-09T04:55:37.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:36 smithi133 bash[105589]: audit 2023-10-09T04:55:35.798708+0000 mon.a (mon.0) 228 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:55:37.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:36 smithi133 bash[105589]: audit 2023-10-09T04:55:35.799733+0000 mon.a (mon.0) 229 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:55:37.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:36 smithi133 bash[105589]: audit 2023-10-09T04:55:35.800638+0000 mon.a (mon.0) 230 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:37.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:36 smithi133 bash[105589]: cephadm 2023-10-09T04:55:35.801797+0000 mgr.y (mgr.25277) 146 : cephadm 1 Reconfiguring daemon mon.b on smithi194 2023-10-09T04:55:37.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:36 smithi194 bash[92666]: cluster 2023-10-09T04:55:35.392461+0000 mgr.y (mgr.25277) 144 : cluster 0 pgmap v105: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 902 op/s 2023-10-09T04:55:37.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:36 smithi194 bash[92666]: audit 2023-10-09T04:55:35.784275+0000 mon.a (mon.0) 226 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:37.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:36 smithi194 bash[92666]: audit 2023-10-09T04:55:35.792031+0000 mon.a (mon.0) 227 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:37.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:36 smithi194 bash[92666]: cephadm 2023-10-09T04:55:35.798223+0000 mgr.y (mgr.25277) 145 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2023-10-09T04:55:37.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:36 smithi194 bash[92666]: audit 2023-10-09T04:55:35.798708+0000 mon.a (mon.0) 228 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:55:37.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:36 smithi194 bash[92666]: audit 2023-10-09T04:55:35.799733+0000 mon.a (mon.0) 229 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:55:37.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:36 smithi194 bash[92666]: audit 2023-10-09T04:55:35.800638+0000 mon.a (mon.0) 230 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:37.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:36 smithi194 bash[92666]: cephadm 2023-10-09T04:55:35.801797+0000 mgr.y (mgr.25277) 146 : cephadm 1 Reconfiguring daemon mon.b on smithi194 2023-10-09T04:55:39.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:38 smithi133 bash[102858]: cluster 2023-10-09T04:55:37.392922+0000 mgr.y (mgr.25277) 147 : cluster 0 pgmap v106: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2023-10-09T04:55:39.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:38 smithi133 bash[105589]: cluster 2023-10-09T04:55:37.392922+0000 mgr.y (mgr.25277) 147 : cluster 0 pgmap v106: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2023-10-09T04:55:39.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:38 smithi194 bash[92666]: cluster 2023-10-09T04:55:37.392922+0000 mgr.y (mgr.25277) 147 : cluster 0 pgmap v106: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2023-10-09T04:55:41.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:40 smithi133 bash[102858]: cluster 2023-10-09T04:55:39.393412+0000 mgr.y (mgr.25277) 148 : cluster 0 pgmap v107: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2023-10-09T04:55:41.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:40 smithi133 bash[102858]: audit 2023-10-09T04:55:40.394037+0000 mon.a (mon.0) 231 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:41.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:40 smithi133 bash[102858]: audit 2023-10-09T04:55:40.403104+0000 mon.a (mon.0) 232 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:41.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:40 smithi133 bash[102858]: audit 2023-10-09T04:55:40.437709+0000 mon.a (mon.0) 233 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:55:41.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:40 smithi133 bash[102858]: audit 2023-10-09T04:55:40.439000+0000 mon.a (mon.0) 234 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:55:41.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:40 smithi133 bash[102858]: audit 2023-10-09T04:55:40.440201+0000 mon.a (mon.0) 235 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:41.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:40 smithi133 bash[105589]: cluster 2023-10-09T04:55:39.393412+0000 mgr.y (mgr.25277) 148 : cluster 0 pgmap v107: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2023-10-09T04:55:41.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:40 smithi133 bash[105589]: audit 2023-10-09T04:55:40.394037+0000 mon.a (mon.0) 231 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:41.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:40 smithi133 bash[105589]: audit 2023-10-09T04:55:40.403104+0000 mon.a (mon.0) 232 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:41.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:40 smithi133 bash[105589]: audit 2023-10-09T04:55:40.437709+0000 mon.a (mon.0) 233 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:55:41.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:40 smithi133 bash[105589]: audit 2023-10-09T04:55:40.439000+0000 mon.a (mon.0) 234 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:55:41.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:40 smithi133 bash[105589]: audit 2023-10-09T04:55:40.440201+0000 mon.a (mon.0) 235 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:41.119 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:40 smithi194 bash[92666]: cluster 2023-10-09T04:55:39.393412+0000 mgr.y (mgr.25277) 148 : cluster 0 pgmap v107: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2023-10-09T04:55:41.119 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:40 smithi194 bash[92666]: audit 2023-10-09T04:55:40.394037+0000 mon.a (mon.0) 231 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:41.120 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:40 smithi194 bash[92666]: audit 2023-10-09T04:55:40.403104+0000 mon.a (mon.0) 232 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:41.120 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:40 smithi194 bash[92666]: audit 2023-10-09T04:55:40.437709+0000 mon.a (mon.0) 233 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:55:41.120 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:40 smithi194 bash[92666]: audit 2023-10-09T04:55:40.439000+0000 mon.a (mon.0) 234 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:55:41.120 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:40 smithi194 bash[92666]: audit 2023-10-09T04:55:40.440201+0000 mon.a (mon.0) 235 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:42.124 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:41 smithi194 bash[92666]: cephadm 2023-10-09T04:55:40.436037+0000 mgr.y (mgr.25277) 149 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2023-10-09T04:55:42.124 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:41 smithi194 bash[92666]: cephadm 2023-10-09T04:55:40.441933+0000 mgr.y (mgr.25277) 150 : cephadm 1 Reconfiguring daemon mgr.x on smithi194 2023-10-09T04:55:42.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:41 smithi133 bash[102858]: cephadm 2023-10-09T04:55:40.436037+0000 mgr.y (mgr.25277) 149 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2023-10-09T04:55:42.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:41 smithi133 bash[102858]: cephadm 2023-10-09T04:55:40.441933+0000 mgr.y (mgr.25277) 150 : cephadm 1 Reconfiguring daemon mgr.x on smithi194 2023-10-09T04:55:42.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:41 smithi133 bash[105589]: cephadm 2023-10-09T04:55:40.436037+0000 mgr.y (mgr.25277) 149 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2023-10-09T04:55:42.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:41 smithi133 bash[105589]: cephadm 2023-10-09T04:55:40.441933+0000 mgr.y (mgr.25277) 150 : cephadm 1 Reconfiguring daemon mgr.x on smithi194 2023-10-09T04:55:43.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:42 smithi194 bash[92666]: cluster 2023-10-09T04:55:41.394161+0000 mgr.y (mgr.25277) 151 : cluster 0 pgmap v108: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 885 op/s 2023-10-09T04:55:43.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:42 smithi133 bash[102858]: cluster 2023-10-09T04:55:41.394161+0000 mgr.y (mgr.25277) 151 : cluster 0 pgmap v108: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 885 op/s 2023-10-09T04:55:43.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:42 smithi133 bash[105589]: cluster 2023-10-09T04:55:41.394161+0000 mgr.y (mgr.25277) 151 : cluster 0 pgmap v108: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 885 op/s 2023-10-09T04:55:44.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:43 smithi133 bash[102858]: audit 2023-10-09T04:55:42.955606+0000 mon.a (mon.0) 236 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:44.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:43 smithi133 bash[102858]: audit 2023-10-09T04:55:42.978593+0000 mon.a (mon.0) 237 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:44.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:43 smithi133 bash[102858]: cephadm 2023-10-09T04:55:43.022880+0000 mgr.y (mgr.25277) 152 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2023-10-09T04:55:44.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:43 smithi133 bash[102858]: audit 2023-10-09T04:55:43.024393+0000 mon.a (mon.0) 238 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T04:55:44.284 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:43 smithi133 bash[102858]: audit 2023-10-09T04:55:43.025515+0000 mon.a (mon.0) 239 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:44.284 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:43 smithi133 bash[102858]: cephadm 2023-10-09T04:55:43.028601+0000 mgr.y (mgr.25277) 153 : cephadm 1 Reconfiguring daemon osd.4 on smithi194 2023-10-09T04:55:44.284 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:43 smithi133 bash[102858]: cluster 2023-10-09T04:55:43.394606+0000 mgr.y (mgr.25277) 154 : cluster 0 pgmap v109: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-10-09T04:55:44.284 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:43 smithi133 bash[102858]: audit 2023-10-09T04:55:43.410862+0000 mon.a (mon.0) 240 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:44.284 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:55:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:55:44] "GET /metrics HTTP/1.1" 200 36156 "" "Prometheus/2.43.0" 2023-10-09T04:55:44.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:43 smithi133 bash[105589]: audit 2023-10-09T04:55:42.955606+0000 mon.a (mon.0) 236 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:44.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:43 smithi133 bash[105589]: audit 2023-10-09T04:55:42.978593+0000 mon.a (mon.0) 237 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:44.285 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:43 smithi133 bash[105589]: cephadm 2023-10-09T04:55:43.022880+0000 mgr.y (mgr.25277) 152 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2023-10-09T04:55:44.285 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:43 smithi133 bash[105589]: audit 2023-10-09T04:55:43.024393+0000 mon.a (mon.0) 238 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T04:55:44.285 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:43 smithi133 bash[105589]: audit 2023-10-09T04:55:43.025515+0000 mon.a (mon.0) 239 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:44.285 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:43 smithi133 bash[105589]: cephadm 2023-10-09T04:55:43.028601+0000 mgr.y (mgr.25277) 153 : cephadm 1 Reconfiguring daemon osd.4 on smithi194 2023-10-09T04:55:44.285 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:43 smithi133 bash[105589]: cluster 2023-10-09T04:55:43.394606+0000 mgr.y (mgr.25277) 154 : cluster 0 pgmap v109: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-10-09T04:55:44.285 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:43 smithi133 bash[105589]: audit 2023-10-09T04:55:43.410862+0000 mon.a (mon.0) 240 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:44.376 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:43 smithi194 bash[92666]: audit 2023-10-09T04:55:42.955606+0000 mon.a (mon.0) 236 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:44.376 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:43 smithi194 bash[92666]: audit 2023-10-09T04:55:42.978593+0000 mon.a (mon.0) 237 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:44.377 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:43 smithi194 bash[92666]: cephadm 2023-10-09T04:55:43.022880+0000 mgr.y (mgr.25277) 152 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2023-10-09T04:55:44.377 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:43 smithi194 bash[92666]: audit 2023-10-09T04:55:43.024393+0000 mon.a (mon.0) 238 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T04:55:44.377 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:43 smithi194 bash[92666]: audit 2023-10-09T04:55:43.025515+0000 mon.a (mon.0) 239 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:44.377 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:43 smithi194 bash[92666]: cephadm 2023-10-09T04:55:43.028601+0000 mgr.y (mgr.25277) 153 : cephadm 1 Reconfiguring daemon osd.4 on smithi194 2023-10-09T04:55:44.377 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:43 smithi194 bash[92666]: cluster 2023-10-09T04:55:43.394606+0000 mgr.y (mgr.25277) 154 : cluster 0 pgmap v109: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-10-09T04:55:44.377 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:43 smithi194 bash[92666]: audit 2023-10-09T04:55:43.410862+0000 mon.a (mon.0) 240 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:46.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:46 smithi133 bash[102858]: cluster 2023-10-09T04:55:45.395347+0000 mgr.y (mgr.25277) 155 : cluster 0 pgmap v110: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2023-10-09T04:55:46.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:46 smithi133 bash[102858]: audit 2023-10-09T04:55:45.503053+0000 mon.a (mon.0) 241 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:46.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:46 smithi133 bash[102858]: audit 2023-10-09T04:55:45.522337+0000 mon.a (mon.0) 242 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:46.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:46 smithi133 bash[102858]: cephadm 2023-10-09T04:55:45.560023+0000 mgr.y (mgr.25277) 156 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2023-10-09T04:55:46.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:46 smithi133 bash[102858]: audit 2023-10-09T04:55:45.560449+0000 mon.a (mon.0) 243 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T04:55:46.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:46 smithi133 bash[102858]: audit 2023-10-09T04:55:45.561482+0000 mon.a (mon.0) 244 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:46.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:46 smithi133 bash[102858]: cephadm 2023-10-09T04:55:45.564584+0000 mgr.y (mgr.25277) 157 : cephadm 1 Reconfiguring daemon osd.5 on smithi194 2023-10-09T04:55:46.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:46 smithi133 bash[105589]: cluster 2023-10-09T04:55:45.395347+0000 mgr.y (mgr.25277) 155 : cluster 0 pgmap v110: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2023-10-09T04:55:46.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:46 smithi133 bash[105589]: audit 2023-10-09T04:55:45.503053+0000 mon.a (mon.0) 241 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:46.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:46 smithi133 bash[105589]: audit 2023-10-09T04:55:45.522337+0000 mon.a (mon.0) 242 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:46.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:46 smithi133 bash[105589]: cephadm 2023-10-09T04:55:45.560023+0000 mgr.y (mgr.25277) 156 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2023-10-09T04:55:46.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:46 smithi133 bash[105589]: audit 2023-10-09T04:55:45.560449+0000 mon.a (mon.0) 243 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T04:55:46.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:46 smithi133 bash[105589]: audit 2023-10-09T04:55:45.561482+0000 mon.a (mon.0) 244 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:46.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:46 smithi133 bash[105589]: cephadm 2023-10-09T04:55:45.564584+0000 mgr.y (mgr.25277) 157 : cephadm 1 Reconfiguring daemon osd.5 on smithi194 2023-10-09T04:55:46.803 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:46 smithi194 bash[92666]: cluster 2023-10-09T04:55:45.395347+0000 mgr.y (mgr.25277) 155 : cluster 0 pgmap v110: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2023-10-09T04:55:46.803 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:46 smithi194 bash[92666]: audit 2023-10-09T04:55:45.503053+0000 mon.a (mon.0) 241 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:46.803 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:46 smithi194 bash[92666]: audit 2023-10-09T04:55:45.522337+0000 mon.a (mon.0) 242 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:46.803 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:46 smithi194 bash[92666]: cephadm 2023-10-09T04:55:45.560023+0000 mgr.y (mgr.25277) 156 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2023-10-09T04:55:46.803 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:46 smithi194 bash[92666]: audit 2023-10-09T04:55:45.560449+0000 mon.a (mon.0) 243 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T04:55:46.804 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:46 smithi194 bash[92666]: audit 2023-10-09T04:55:45.561482+0000 mon.a (mon.0) 244 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:46.804 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:46 smithi194 bash[92666]: cephadm 2023-10-09T04:55:45.564584+0000 mgr.y (mgr.25277) 157 : cephadm 1 Reconfiguring daemon osd.5 on smithi194 2023-10-09T04:55:49.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:48 smithi194 bash[92666]: cluster 2023-10-09T04:55:47.395714+0000 mgr.y (mgr.25277) 158 : cluster 0 pgmap v111: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-10-09T04:55:49.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:48 smithi194 bash[92666]: audit 2023-10-09T04:55:47.848757+0000 mon.a (mon.0) 245 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:49.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:48 smithi194 bash[92666]: audit 2023-10-09T04:55:47.857732+0000 mon.a (mon.0) 246 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:49.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:48 smithi194 bash[92666]: cephadm 2023-10-09T04:55:47.887295+0000 mgr.y (mgr.25277) 159 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2023-10-09T04:55:49.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:48 smithi194 bash[92666]: audit 2023-10-09T04:55:47.887770+0000 mon.a (mon.0) 247 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T04:55:49.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:48 smithi194 bash[92666]: audit 2023-10-09T04:55:47.888884+0000 mon.a (mon.0) 248 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:49.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:48 smithi194 bash[92666]: cephadm 2023-10-09T04:55:47.891937+0000 mgr.y (mgr.25277) 160 : cephadm 1 Reconfiguring daemon osd.6 on smithi194 2023-10-09T04:55:49.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:48 smithi133 bash[102858]: cluster 2023-10-09T04:55:47.395714+0000 mgr.y (mgr.25277) 158 : cluster 0 pgmap v111: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-10-09T04:55:49.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:48 smithi133 bash[102858]: audit 2023-10-09T04:55:47.848757+0000 mon.a (mon.0) 245 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:49.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:48 smithi133 bash[102858]: audit 2023-10-09T04:55:47.857732+0000 mon.a (mon.0) 246 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:49.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:48 smithi133 bash[102858]: cephadm 2023-10-09T04:55:47.887295+0000 mgr.y (mgr.25277) 159 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2023-10-09T04:55:49.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:48 smithi133 bash[102858]: audit 2023-10-09T04:55:47.887770+0000 mon.a (mon.0) 247 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T04:55:49.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:48 smithi133 bash[102858]: audit 2023-10-09T04:55:47.888884+0000 mon.a (mon.0) 248 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:49.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:48 smithi133 bash[102858]: cephadm 2023-10-09T04:55:47.891937+0000 mgr.y (mgr.25277) 160 : cephadm 1 Reconfiguring daemon osd.6 on smithi194 2023-10-09T04:55:49.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:48 smithi133 bash[105589]: cluster 2023-10-09T04:55:47.395714+0000 mgr.y (mgr.25277) 158 : cluster 0 pgmap v111: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-10-09T04:55:49.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:48 smithi133 bash[105589]: audit 2023-10-09T04:55:47.848757+0000 mon.a (mon.0) 245 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:49.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:48 smithi133 bash[105589]: audit 2023-10-09T04:55:47.857732+0000 mon.a (mon.0) 246 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:49.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:48 smithi133 bash[105589]: cephadm 2023-10-09T04:55:47.887295+0000 mgr.y (mgr.25277) 159 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2023-10-09T04:55:49.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:48 smithi133 bash[105589]: audit 2023-10-09T04:55:47.887770+0000 mon.a (mon.0) 247 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T04:55:49.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:48 smithi133 bash[105589]: audit 2023-10-09T04:55:47.888884+0000 mon.a (mon.0) 248 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:49.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:48 smithi133 bash[105589]: cephadm 2023-10-09T04:55:47.891937+0000 mgr.y (mgr.25277) 160 : cephadm 1 Reconfiguring daemon osd.6 on smithi194 2023-10-09T04:55:51.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:50 smithi194 bash[92666]: cluster 2023-10-09T04:55:49.396161+0000 mgr.y (mgr.25277) 161 : cluster 0 pgmap v112: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-10-09T04:55:51.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:50 smithi194 bash[92666]: audit 2023-10-09T04:55:50.301255+0000 mon.a (mon.0) 249 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:51.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:50 smithi194 bash[92666]: audit 2023-10-09T04:55:50.310481+0000 mon.a (mon.0) 250 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:51.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:50 smithi194 bash[92666]: audit 2023-10-09T04:55:50.317426+0000 mon.a (mon.0) 251 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T04:55:51.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:50 smithi194 bash[92666]: audit 2023-10-09T04:55:50.318352+0000 mon.a (mon.0) 252 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:51.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:50 smithi133 bash[102858]: cluster 2023-10-09T04:55:49.396161+0000 mgr.y (mgr.25277) 161 : cluster 0 pgmap v112: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-10-09T04:55:51.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:50 smithi133 bash[102858]: audit 2023-10-09T04:55:50.301255+0000 mon.a (mon.0) 249 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:51.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:50 smithi133 bash[102858]: audit 2023-10-09T04:55:50.310481+0000 mon.a (mon.0) 250 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:51.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:50 smithi133 bash[102858]: audit 2023-10-09T04:55:50.317426+0000 mon.a (mon.0) 251 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T04:55:51.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:50 smithi133 bash[102858]: audit 2023-10-09T04:55:50.318352+0000 mon.a (mon.0) 252 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:51.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:50 smithi133 bash[105589]: cluster 2023-10-09T04:55:49.396161+0000 mgr.y (mgr.25277) 161 : cluster 0 pgmap v112: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-10-09T04:55:51.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:50 smithi133 bash[105589]: audit 2023-10-09T04:55:50.301255+0000 mon.a (mon.0) 249 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:51.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:50 smithi133 bash[105589]: audit 2023-10-09T04:55:50.310481+0000 mon.a (mon.0) 250 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:51.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:50 smithi133 bash[105589]: audit 2023-10-09T04:55:50.317426+0000 mon.a (mon.0) 251 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T04:55:51.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:50 smithi133 bash[105589]: audit 2023-10-09T04:55:50.318352+0000 mon.a (mon.0) 252 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:52.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:51 smithi194 bash[92666]: cephadm 2023-10-09T04:55:50.316980+0000 mgr.y (mgr.25277) 162 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2023-10-09T04:55:52.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:51 smithi194 bash[92666]: cephadm 2023-10-09T04:55:50.321002+0000 mgr.y (mgr.25277) 163 : cephadm 1 Reconfiguring daemon osd.7 on smithi194 2023-10-09T04:55:52.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:51 smithi133 bash[102858]: cephadm 2023-10-09T04:55:50.316980+0000 mgr.y (mgr.25277) 162 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2023-10-09T04:55:52.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:51 smithi133 bash[102858]: cephadm 2023-10-09T04:55:50.321002+0000 mgr.y (mgr.25277) 163 : cephadm 1 Reconfiguring daemon osd.7 on smithi194 2023-10-09T04:55:52.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:51 smithi133 bash[105589]: cephadm 2023-10-09T04:55:50.316980+0000 mgr.y (mgr.25277) 162 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2023-10-09T04:55:52.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:51 smithi133 bash[105589]: cephadm 2023-10-09T04:55:50.321002+0000 mgr.y (mgr.25277) 163 : cephadm 1 Reconfiguring daemon osd.7 on smithi194 2023-10-09T04:55:53.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:52 smithi194 bash[92666]: cluster 2023-10-09T04:55:51.396899+0000 mgr.y (mgr.25277) 164 : cluster 0 pgmap v113: 89 pgs: 89 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 882 op/s 2023-10-09T04:55:53.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:52 smithi194 bash[92666]: audit 2023-10-09T04:55:52.814385+0000 mon.a (mon.0) 253 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:53.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:52 smithi194 bash[92666]: audit 2023-10-09T04:55:52.827035+0000 mon.a (mon.0) 254 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:53.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:52 smithi194 bash[92666]: audit 2023-10-09T04:55:52.843850+0000 mon.a (mon.0) 255 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi194.yxnfqn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:55:53.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:52 smithi194 bash[92666]: audit 2023-10-09T04:55:52.844949+0000 mon.a (mon.0) 256 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:53.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:52 smithi133 bash[105589]: cluster 2023-10-09T04:55:51.396899+0000 mgr.y (mgr.25277) 164 : cluster 0 pgmap v113: 89 pgs: 89 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 882 op/s 2023-10-09T04:55:53.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:52 smithi133 bash[105589]: audit 2023-10-09T04:55:52.814385+0000 mon.a (mon.0) 253 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:53.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:52 smithi133 bash[105589]: audit 2023-10-09T04:55:52.827035+0000 mon.a (mon.0) 254 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:53.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:52 smithi133 bash[105589]: audit 2023-10-09T04:55:52.843850+0000 mon.a (mon.0) 255 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi194.yxnfqn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:55:53.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:52 smithi133 bash[105589]: audit 2023-10-09T04:55:52.844949+0000 mon.a (mon.0) 256 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:53.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:52 smithi133 bash[102858]: cluster 2023-10-09T04:55:51.396899+0000 mgr.y (mgr.25277) 164 : cluster 0 pgmap v113: 89 pgs: 89 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 882 op/s 2023-10-09T04:55:53.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:52 smithi133 bash[102858]: audit 2023-10-09T04:55:52.814385+0000 mon.a (mon.0) 253 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:53.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:52 smithi133 bash[102858]: audit 2023-10-09T04:55:52.827035+0000 mon.a (mon.0) 254 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:53.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:52 smithi133 bash[102858]: audit 2023-10-09T04:55:52.843850+0000 mon.a (mon.0) 255 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi194.yxnfqn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:55:53.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:52 smithi133 bash[102858]: audit 2023-10-09T04:55:52.844949+0000 mon.a (mon.0) 256 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:54.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:53 smithi194 bash[92666]: cephadm 2023-10-09T04:55:52.843314+0000 mgr.y (mgr.25277) 165 : cephadm 1 Reconfiguring mds.foo.smithi194.yxnfqn (monmap changed)... 2023-10-09T04:55:54.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:53 smithi194 bash[92666]: cephadm 2023-10-09T04:55:52.846233+0000 mgr.y (mgr.25277) 166 : cephadm 1 Reconfiguring daemon mds.foo.smithi194.yxnfqn on smithi194 2023-10-09T04:55:54.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:53 smithi133 bash[105589]: cephadm 2023-10-09T04:55:52.843314+0000 mgr.y (mgr.25277) 165 : cephadm 1 Reconfiguring mds.foo.smithi194.yxnfqn (monmap changed)... 2023-10-09T04:55:54.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:53 smithi133 bash[105589]: cephadm 2023-10-09T04:55:52.846233+0000 mgr.y (mgr.25277) 166 : cephadm 1 Reconfiguring daemon mds.foo.smithi194.yxnfqn on smithi194 2023-10-09T04:55:54.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:53 smithi133 bash[102858]: cephadm 2023-10-09T04:55:52.843314+0000 mgr.y (mgr.25277) 165 : cephadm 1 Reconfiguring mds.foo.smithi194.yxnfqn (monmap changed)... 2023-10-09T04:55:54.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:53 smithi133 bash[102858]: cephadm 2023-10-09T04:55:52.846233+0000 mgr.y (mgr.25277) 166 : cephadm 1 Reconfiguring daemon mds.foo.smithi194.yxnfqn on smithi194 2023-10-09T04:55:54.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:55:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:55:54] "GET /metrics HTTP/1.1" 200 36156 "" "Prometheus/2.43.0" 2023-10-09T04:55:55.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:54 smithi194 bash[92666]: cluster 2023-10-09T04:55:53.397375+0000 mgr.y (mgr.25277) 167 : cluster 0 pgmap v114: 89 pgs: 89 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-09T04:55:55.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:54 smithi133 bash[102858]: cluster 2023-10-09T04:55:53.397375+0000 mgr.y (mgr.25277) 167 : cluster 0 pgmap v114: 89 pgs: 89 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-09T04:55:55.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:54 smithi133 bash[105589]: cluster 2023-10-09T04:55:53.397375+0000 mgr.y (mgr.25277) 167 : cluster 0 pgmap v114: 89 pgs: 89 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-09T04:55:56.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:56 smithi133 bash[102858]: audit 2023-10-09T04:55:55.255053+0000 mon.a (mon.0) 257 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:56.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:56 smithi133 bash[102858]: audit 2023-10-09T04:55:55.267293+0000 mon.a (mon.0) 258 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:56.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:56 smithi133 bash[102858]: audit 2023-10-09T04:55:55.279954+0000 mon.a (mon.0) 259 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:55:56.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:56 smithi133 bash[102858]: cluster 2023-10-09T04:55:55.398136+0000 mgr.y (mgr.25277) 168 : cluster 0 pgmap v115: 89 pgs: 89 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 878 op/s 2023-10-09T04:55:56.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:56 smithi133 bash[105589]: audit 2023-10-09T04:55:55.255053+0000 mon.a (mon.0) 257 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:56.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:56 smithi133 bash[105589]: audit 2023-10-09T04:55:55.267293+0000 mon.a (mon.0) 258 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:56.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:56 smithi133 bash[105589]: audit 2023-10-09T04:55:55.279954+0000 mon.a (mon.0) 259 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:55:56.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:56 smithi133 bash[105589]: cluster 2023-10-09T04:55:55.398136+0000 mgr.y (mgr.25277) 168 : cluster 0 pgmap v115: 89 pgs: 89 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 878 op/s 2023-10-09T04:55:56.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:56 smithi194 bash[92666]: audit 2023-10-09T04:55:55.255053+0000 mon.a (mon.0) 257 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:56.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:56 smithi194 bash[92666]: audit 2023-10-09T04:55:55.267293+0000 mon.a (mon.0) 258 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:55:56.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:56 smithi194 bash[92666]: audit 2023-10-09T04:55:55.279954+0000 mon.a (mon.0) 259 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:55:56.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:56 smithi194 bash[92666]: cluster 2023-10-09T04:55:55.398136+0000 mgr.y (mgr.25277) 168 : cluster 0 pgmap v115: 89 pgs: 89 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 878 op/s 2023-10-09T04:55:58.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:58 smithi133 bash[102858]: cluster 2023-10-09T04:55:57.398476+0000 mgr.y (mgr.25277) 169 : cluster 0 pgmap v116: 89 pgs: 89 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-09T04:55:58.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:55:58 smithi133 bash[102858]: audit 2023-10-09T04:55:58.411046+0000 mon.a (mon.0) 260 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:58.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:58 smithi133 bash[105589]: cluster 2023-10-09T04:55:57.398476+0000 mgr.y (mgr.25277) 169 : cluster 0 pgmap v116: 89 pgs: 89 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-09T04:55:58.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:55:58 smithi133 bash[105589]: audit 2023-10-09T04:55:58.411046+0000 mon.a (mon.0) 260 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:58.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:58 smithi194 bash[92666]: cluster 2023-10-09T04:55:57.398476+0000 mgr.y (mgr.25277) 169 : cluster 0 pgmap v116: 89 pgs: 89 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-09T04:55:58.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:55:58 smithi194 bash[92666]: audit 2023-10-09T04:55:58.411046+0000 mon.a (mon.0) 260 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:00.803 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:00 smithi133 bash[102858]: cluster 2023-10-09T04:55:59.398917+0000 mgr.y (mgr.25277) 170 : cluster 0 pgmap v117: 89 pgs: 89 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-09T04:56:00.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:00 smithi133 bash[105589]: cluster 2023-10-09T04:55:59.398917+0000 mgr.y (mgr.25277) 170 : cluster 0 pgmap v117: 89 pgs: 89 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-09T04:56:00.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:00 smithi194 bash[92666]: cluster 2023-10-09T04:55:59.398917+0000 mgr.y (mgr.25277) 170 : cluster 0 pgmap v117: 89 pgs: 89 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-09T04:56:03.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:02 smithi133 bash[102858]: cluster 2023-10-09T04:56:01.401520+0000 mgr.y (mgr.25277) 171 : cluster 0 pgmap v118: 89 pgs: 89 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 873 op/s 2023-10-09T04:56:03.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:02 smithi133 bash[102858]: audit 2023-10-09T04:56:01.784232+0000 mon.a (mon.0) 261 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:03.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:02 smithi133 bash[102858]: audit 2023-10-09T04:56:01.793029+0000 mon.a (mon.0) 262 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:03.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:02 smithi133 bash[102858]: audit 2023-10-09T04:56:02.373588+0000 mon.a (mon.0) 263 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:56:03.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:02 smithi133 bash[102858]: audit 2023-10-09T04:56:02.376987+0000 mon.a (mon.0) 264 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:56:03.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:02 smithi133 bash[102858]: audit 2023-10-09T04:56:02.385270+0000 mon.a (mon.0) 265 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:03.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:02 smithi133 bash[105589]: cluster 2023-10-09T04:56:01.401520+0000 mgr.y (mgr.25277) 171 : cluster 0 pgmap v118: 89 pgs: 89 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 873 op/s 2023-10-09T04:56:03.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:02 smithi133 bash[105589]: audit 2023-10-09T04:56:01.784232+0000 mon.a (mon.0) 261 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:03.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:02 smithi133 bash[105589]: audit 2023-10-09T04:56:01.793029+0000 mon.a (mon.0) 262 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:03.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:02 smithi133 bash[105589]: audit 2023-10-09T04:56:02.373588+0000 mon.a (mon.0) 263 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:56:03.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:02 smithi133 bash[105589]: audit 2023-10-09T04:56:02.376987+0000 mon.a (mon.0) 264 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:56:03.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:02 smithi133 bash[105589]: audit 2023-10-09T04:56:02.385270+0000 mon.a (mon.0) 265 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:03.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:02 smithi194 bash[92666]: cluster 2023-10-09T04:56:01.401520+0000 mgr.y (mgr.25277) 171 : cluster 0 pgmap v118: 89 pgs: 89 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 873 op/s 2023-10-09T04:56:03.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:02 smithi194 bash[92666]: audit 2023-10-09T04:56:01.784232+0000 mon.a (mon.0) 261 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:03.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:02 smithi194 bash[92666]: audit 2023-10-09T04:56:01.793029+0000 mon.a (mon.0) 262 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:03.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:02 smithi194 bash[92666]: audit 2023-10-09T04:56:02.373588+0000 mon.a (mon.0) 263 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:56:03.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:02 smithi194 bash[92666]: audit 2023-10-09T04:56:02.376987+0000 mon.a (mon.0) 264 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:56:03.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:02 smithi194 bash[92666]: audit 2023-10-09T04:56:02.385270+0000 mon.a (mon.0) 265 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:04.282 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:03 smithi133 bash[105589]: cluster 2023-10-09T04:56:03.403543+0000 mgr.y (mgr.25277) 172 : cluster 0 pgmap v119: 89 pgs: 89 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2023-10-09T04:56:04.282 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:03 smithi133 bash[102858]: cluster 2023-10-09T04:56:03.403543+0000 mgr.y (mgr.25277) 172 : cluster 0 pgmap v119: 89 pgs: 89 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2023-10-09T04:56:04.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:03 smithi194 bash[92666]: cluster 2023-10-09T04:56:03.403543+0000 mgr.y (mgr.25277) 172 : cluster 0 pgmap v119: 89 pgs: 89 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2023-10-09T04:56:04.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:56:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:56:04] "GET /metrics HTTP/1.1" 200 36157 "" "Prometheus/2.43.0" 2023-10-09T04:56:05.441 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T04:56:06.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:06 smithi133 bash[105589]: cluster 2023-10-09T04:56:05.404355+0000 mgr.y (mgr.25277) 173 : cluster 0 pgmap v120: 89 pgs: 89 active+clean; 1008 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 862 op/s 2023-10-09T04:56:06.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:06 smithi133 bash[102858]: cluster 2023-10-09T04:56:05.404355+0000 mgr.y (mgr.25277) 173 : cluster 0 pgmap v120: 89 pgs: 89 active+clean; 1008 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 862 op/s 2023-10-09T04:56:06.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:06 smithi194 bash[92666]: cluster 2023-10-09T04:56:05.404355+0000 mgr.y (mgr.25277) 173 : cluster 0 pgmap v120: 89 pgs: 89 active+clean; 1008 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 862 op/s 2023-10-09T04:56:08.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:08 smithi133 bash[105589]: cluster 2023-10-09T04:56:07.407680+0000 mgr.y (mgr.25277) 174 : cluster 0 pgmap v121: 89 pgs: 89 active+clean; 1008 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-10-09T04:56:08.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:08 smithi133 bash[102858]: cluster 2023-10-09T04:56:07.407680+0000 mgr.y (mgr.25277) 174 : cluster 0 pgmap v121: 89 pgs: 89 active+clean; 1008 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-10-09T04:56:08.890 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:08 smithi194 bash[92666]: cluster 2023-10-09T04:56:07.407680+0000 mgr.y (mgr.25277) 174 : cluster 0 pgmap v121: 89 pgs: 89 active+clean; 1008 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-10-09T04:56:10.278 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:10.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:10 smithi133 bash[105589]: cluster 2023-10-09T04:56:09.408131+0000 mgr.y (mgr.25277) 175 : cluster 0 pgmap v122: 89 pgs: 89 active+clean; 1008 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-10-09T04:56:10.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:10 smithi133 bash[102858]: cluster 2023-10-09T04:56:09.408131+0000 mgr.y (mgr.25277) 175 : cluster 0 pgmap v122: 89 pgs: 89 active+clean; 1008 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-10-09T04:56:10.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:10 smithi194 bash[92666]: cluster 2023-10-09T04:56:09.408131+0000 mgr.y (mgr.25277) 175 : cluster 0 pgmap v122: 89 pgs: 89 active+clean; 1008 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-10-09T04:56:12.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:12 smithi133 bash[105589]: cluster 2023-10-09T04:56:11.408838+0000 mgr.y (mgr.25277) 176 : cluster 0 pgmap v123: 89 pgs: 89 active+clean; 782 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 885 op/s 2023-10-09T04:56:12.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:12 smithi133 bash[102858]: cluster 2023-10-09T04:56:11.408838+0000 mgr.y (mgr.25277) 176 : cluster 0 pgmap v123: 89 pgs: 89 active+clean; 782 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 885 op/s 2023-10-09T04:56:12.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:12 smithi194 bash[92666]: cluster 2023-10-09T04:56:11.408838+0000 mgr.y (mgr.25277) 176 : cluster 0 pgmap v123: 89 pgs: 89 active+clean; 782 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 885 op/s 2023-10-09T04:56:14.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:13 smithi133 bash[105589]: cluster 2023-10-09T04:56:13.411080+0000 mgr.y (mgr.25277) 177 : cluster 0 pgmap v124: 89 pgs: 89 active+clean; 782 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-10-09T04:56:14.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:13 smithi133 bash[105589]: audit 2023-10-09T04:56:13.411482+0000 mon.a (mon.0) 266 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:14.284 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:13 smithi133 bash[102858]: cluster 2023-10-09T04:56:13.411080+0000 mgr.y (mgr.25277) 177 : cluster 0 pgmap v124: 89 pgs: 89 active+clean; 782 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-10-09T04:56:14.284 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:13 smithi133 bash[102858]: audit 2023-10-09T04:56:13.411482+0000 mon.a (mon.0) 266 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:14.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:13 smithi194 bash[92666]: cluster 2023-10-09T04:56:13.411080+0000 mgr.y (mgr.25277) 177 : cluster 0 pgmap v124: 89 pgs: 89 active+clean; 782 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-10-09T04:56:14.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:13 smithi194 bash[92666]: audit 2023-10-09T04:56:13.411482+0000 mon.a (mon.0) 266 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:14.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:56:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:56:14] "GET /metrics HTTP/1.1" 200 36159 "" "Prometheus/2.43.0" 2023-10-09T04:56:15.129 INFO:teuthology.orchestra.run.smithi133.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:56:16.225 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for all mons to upgrade' 2023-10-09T04:56:16.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:16 smithi194 bash[92666]: audit 2023-10-09T04:56:15.119121+0000 mgr.y (mgr.25277) 178 : audit 0 from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:56:16.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:16 smithi194 bash[92666]: cephadm 2023-10-09T04:56:15.119561+0000 mgr.y (mgr.25277) 179 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:56:16.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:16 smithi194 bash[92666]: audit 2023-10-09T04:56:15.127234+0000 mon.a (mon.0) 267 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:16.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:16 smithi194 bash[92666]: audit 2023-10-09T04:56:15.148297+0000 mon.a (mon.0) 268 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:56:16.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:16 smithi194 bash[92666]: cluster 2023-10-09T04:56:15.411925+0000 mgr.y (mgr.25277) 180 : cluster 0 pgmap v125: 89 pgs: 89 active+clean; 558 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 889 op/s 2023-10-09T04:56:16.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:16 smithi133 bash[102858]: audit 2023-10-09T04:56:15.119121+0000 mgr.y (mgr.25277) 178 : audit 0 from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:56:16.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:16 smithi133 bash[102858]: cephadm 2023-10-09T04:56:15.119561+0000 mgr.y (mgr.25277) 179 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:56:16.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:16 smithi133 bash[102858]: audit 2023-10-09T04:56:15.127234+0000 mon.a (mon.0) 267 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:16.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:16 smithi133 bash[102858]: audit 2023-10-09T04:56:15.148297+0000 mon.a (mon.0) 268 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:56:16.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:16 smithi133 bash[102858]: cluster 2023-10-09T04:56:15.411925+0000 mgr.y (mgr.25277) 180 : cluster 0 pgmap v125: 89 pgs: 89 active+clean; 558 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 889 op/s 2023-10-09T04:56:16.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:16 smithi133 bash[105589]: audit 2023-10-09T04:56:15.119121+0000 mgr.y (mgr.25277) 178 : audit 0 from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:56:16.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:16 smithi133 bash[105589]: cephadm 2023-10-09T04:56:15.119561+0000 mgr.y (mgr.25277) 179 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:56:16.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:16 smithi133 bash[105589]: audit 2023-10-09T04:56:15.127234+0000 mon.a (mon.0) 267 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:16.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:16 smithi133 bash[105589]: audit 2023-10-09T04:56:15.148297+0000 mon.a (mon.0) 268 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:56:16.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:16 smithi133 bash[105589]: cluster 2023-10-09T04:56:15.411925+0000 mgr.y (mgr.25277) 180 : cluster 0 pgmap v125: 89 pgs: 89 active+clean; 558 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 889 op/s 2023-10-09T04:56:18.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:18 smithi133 bash[102858]: cluster 2023-10-09T04:56:17.412317+0000 mgr.y (mgr.25277) 181 : cluster 0 pgmap v126: 89 pgs: 89 active+clean; 558 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-09T04:56:18.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:18 smithi133 bash[105589]: cluster 2023-10-09T04:56:17.412317+0000 mgr.y (mgr.25277) 181 : cluster 0 pgmap v126: 89 pgs: 89 active+clean; 558 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-09T04:56:18.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:18 smithi194 bash[92666]: cluster 2023-10-09T04:56:17.412317+0000 mgr.y (mgr.25277) 181 : cluster 0 pgmap v126: 89 pgs: 89 active+clean; 558 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-09T04:56:20.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:20 smithi133 bash[102858]: cluster 2023-10-09T04:56:19.412748+0000 mgr.y (mgr.25277) 182 : cluster 0 pgmap v127: 89 pgs: 89 active+clean; 558 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-09T04:56:20.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:20 smithi133 bash[105589]: cluster 2023-10-09T04:56:19.412748+0000 mgr.y (mgr.25277) 182 : cluster 0 pgmap v127: 89 pgs: 89 active+clean; 558 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-09T04:56:20.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:20 smithi194 bash[92666]: cluster 2023-10-09T04:56:19.412748+0000 mgr.y (mgr.25277) 182 : cluster 0 pgmap v127: 89 pgs: 89 active+clean; 558 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-09T04:56:21.051 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:23.063 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[102858]: cluster 2023-10-09T04:56:21.413489+0000 mgr.y (mgr.25277) 183 : cluster 0 pgmap v128: 89 pgs: 89 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2023-10-09T04:56:23.064 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[102858]: audit 2023-10-09T04:56:21.811657+0000 mon.a (mon.0) 269 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:23.064 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[102858]: audit 2023-10-09T04:56:21.820209+0000 mon.a (mon.0) 270 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:23.064 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[102858]: audit 2023-10-09T04:56:22.405752+0000 mon.a (mon.0) 271 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:56:23.064 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[102858]: audit 2023-10-09T04:56:22.408288+0000 mon.a (mon.0) 272 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:56:23.064 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[102858]: audit 2023-10-09T04:56:22.415156+0000 mon.a (mon.0) 273 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:23.064 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[102858]: audit 2023-10-09T04:56:22.509357+0000 mon.a (mon.0) 274 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:56:23.064 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[102858]: audit 2023-10-09T04:56:22.516490+0000 mon.a (mon.0) 275 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:56:23.064 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[102858]: audit 2023-10-09T04:56:22.519095+0000 mon.a (mon.0) 276 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:56:23.064 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[102858]: audit 2023-10-09T04:56:22.525953+0000 mon.a (mon.0) 277 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:23.065 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[102858]: audit 2023-10-09T04:56:22.527711+0000 mon.a (mon.0) 278 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T04:56:23.065 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[102858]: audit 2023-10-09T04:56:22.537890+0000 mon.a (mon.0) 279 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T04:56:23.065 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[102858]: audit 2023-10-09T04:56:22.539575+0000 mon.a (mon.0) 280 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T04:56:23.065 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[102858]: audit 2023-10-09T04:56:22.554024+0000 mon.a (mon.0) 281 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T04:56:23.065 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[102858]: audit 2023-10-09T04:56:22.555452+0000 mon.a (mon.0) 282 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T04:56:23.065 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[102858]: audit 2023-10-09T04:56:22.565197+0000 mon.a (mon.0) 283 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T04:56:23.065 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[102858]: audit 2023-10-09T04:56:22.567669+0000 mon.a (mon.0) 284 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:56:23.065 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[102858]: audit 2023-10-09T04:56:22.579357+0000 mon.a (mon.0) 285 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:23.065 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[102858]: audit 2023-10-09T04:56:22.587331+0000 mon.a (mon.0) 286 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T04:56:23.066 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[105589]: cluster 2023-10-09T04:56:21.413489+0000 mgr.y (mgr.25277) 183 : cluster 0 pgmap v128: 89 pgs: 89 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2023-10-09T04:56:23.066 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[105589]: audit 2023-10-09T04:56:21.811657+0000 mon.a (mon.0) 269 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:23.066 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[105589]: audit 2023-10-09T04:56:21.820209+0000 mon.a (mon.0) 270 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:23.066 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[105589]: audit 2023-10-09T04:56:22.405752+0000 mon.a (mon.0) 271 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:56:23.066 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[105589]: audit 2023-10-09T04:56:22.408288+0000 mon.a (mon.0) 272 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:56:23.066 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[105589]: audit 2023-10-09T04:56:22.415156+0000 mon.a (mon.0) 273 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:23.066 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[105589]: audit 2023-10-09T04:56:22.509357+0000 mon.a (mon.0) 274 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:56:23.066 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[105589]: audit 2023-10-09T04:56:22.516490+0000 mon.a (mon.0) 275 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:56:23.066 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[105589]: audit 2023-10-09T04:56:22.519095+0000 mon.a (mon.0) 276 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:56:23.066 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[105589]: audit 2023-10-09T04:56:22.525953+0000 mon.a (mon.0) 277 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:23.066 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[105589]: audit 2023-10-09T04:56:22.527711+0000 mon.a (mon.0) 278 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T04:56:23.067 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[105589]: audit 2023-10-09T04:56:22.537890+0000 mon.a (mon.0) 279 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T04:56:23.067 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[105589]: audit 2023-10-09T04:56:22.539575+0000 mon.a (mon.0) 280 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T04:56:23.067 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[105589]: audit 2023-10-09T04:56:22.554024+0000 mon.a (mon.0) 281 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T04:56:23.067 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[105589]: audit 2023-10-09T04:56:22.555452+0000 mon.a (mon.0) 282 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T04:56:23.067 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[105589]: audit 2023-10-09T04:56:22.565197+0000 mon.a (mon.0) 283 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T04:56:23.067 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[105589]: audit 2023-10-09T04:56:22.567669+0000 mon.a (mon.0) 284 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:56:23.067 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[105589]: audit 2023-10-09T04:56:22.579357+0000 mon.a (mon.0) 285 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:23.067 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:22 smithi133 bash[105589]: audit 2023-10-09T04:56:22.587331+0000 mon.a (mon.0) 286 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T04:56:23.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:22 smithi194 bash[92666]: cluster 2023-10-09T04:56:21.413489+0000 mgr.y (mgr.25277) 183 : cluster 0 pgmap v128: 89 pgs: 89 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2023-10-09T04:56:23.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:22 smithi194 bash[92666]: audit 2023-10-09T04:56:21.811657+0000 mon.a (mon.0) 269 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:23.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:22 smithi194 bash[92666]: audit 2023-10-09T04:56:21.820209+0000 mon.a (mon.0) 270 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:23.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:22 smithi194 bash[92666]: audit 2023-10-09T04:56:22.405752+0000 mon.a (mon.0) 271 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:56:23.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:22 smithi194 bash[92666]: audit 2023-10-09T04:56:22.408288+0000 mon.a (mon.0) 272 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:56:23.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:22 smithi194 bash[92666]: audit 2023-10-09T04:56:22.415156+0000 mon.a (mon.0) 273 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:23.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:22 smithi194 bash[92666]: audit 2023-10-09T04:56:22.509357+0000 mon.a (mon.0) 274 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:56:23.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:22 smithi194 bash[92666]: audit 2023-10-09T04:56:22.516490+0000 mon.a (mon.0) 275 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:56:23.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:22 smithi194 bash[92666]: audit 2023-10-09T04:56:22.519095+0000 mon.a (mon.0) 276 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:56:23.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:22 smithi194 bash[92666]: audit 2023-10-09T04:56:22.525953+0000 mon.a (mon.0) 277 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:23.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:22 smithi194 bash[92666]: audit 2023-10-09T04:56:22.527711+0000 mon.a (mon.0) 278 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T04:56:23.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:22 smithi194 bash[92666]: audit 2023-10-09T04:56:22.537890+0000 mon.a (mon.0) 279 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T04:56:23.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:22 smithi194 bash[92666]: audit 2023-10-09T04:56:22.539575+0000 mon.a (mon.0) 280 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T04:56:23.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:22 smithi194 bash[92666]: audit 2023-10-09T04:56:22.554024+0000 mon.a (mon.0) 281 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T04:56:23.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:22 smithi194 bash[92666]: audit 2023-10-09T04:56:22.555452+0000 mon.a (mon.0) 282 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T04:56:23.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:22 smithi194 bash[92666]: audit 2023-10-09T04:56:22.565197+0000 mon.a (mon.0) 283 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T04:56:23.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:22 smithi194 bash[92666]: audit 2023-10-09T04:56:22.567669+0000 mon.a (mon.0) 284 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:56:23.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:22 smithi194 bash[92666]: audit 2023-10-09T04:56:22.579357+0000 mon.a (mon.0) 285 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:23.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:22 smithi194 bash[92666]: audit 2023-10-09T04:56:22.587331+0000 mon.a (mon.0) 286 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T04:56:24.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:23 smithi194 bash[92666]: cephadm 2023-10-09T04:56:22.520113+0000 mgr.y (mgr.25277) 184 : cephadm 1 Upgrade: Setting container_image for all mon 2023-10-09T04:56:24.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:23 smithi194 bash[92666]: cephadm 2023-10-09T04:56:22.568966+0000 mgr.y (mgr.25277) 185 : cephadm 1 Upgrade: Setting container_image for all crash 2023-10-09T04:56:24.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:23 smithi194 bash[92666]: audit 2023-10-09T04:56:22.587567+0000 mgr.y (mgr.25277) 186 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T04:56:24.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:23 smithi194 bash[92666]: cephadm 2023-10-09T04:56:22.605153+0000 mgr.y (mgr.25277) 187 : cephadm 1 Upgrade: osd.0 is safe to restart 2023-10-09T04:56:24.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:23 smithi133 bash[102858]: cephadm 2023-10-09T04:56:22.520113+0000 mgr.y (mgr.25277) 184 : cephadm 1 Upgrade: Setting container_image for all mon 2023-10-09T04:56:24.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:23 smithi133 bash[102858]: cephadm 2023-10-09T04:56:22.568966+0000 mgr.y (mgr.25277) 185 : cephadm 1 Upgrade: Setting container_image for all crash 2023-10-09T04:56:24.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:23 smithi133 bash[102858]: audit 2023-10-09T04:56:22.587567+0000 mgr.y (mgr.25277) 186 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T04:56:24.284 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:23 smithi133 bash[102858]: cephadm 2023-10-09T04:56:22.605153+0000 mgr.y (mgr.25277) 187 : cephadm 1 Upgrade: osd.0 is safe to restart 2023-10-09T04:56:24.284 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:56:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:56:24] "GET /metrics HTTP/1.1" 200 36159 "" "Prometheus/2.43.0" 2023-10-09T04:56:24.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:23 smithi133 bash[105589]: cephadm 2023-10-09T04:56:22.520113+0000 mgr.y (mgr.25277) 184 : cephadm 1 Upgrade: Setting container_image for all mon 2023-10-09T04:56:24.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:23 smithi133 bash[105589]: cephadm 2023-10-09T04:56:22.568966+0000 mgr.y (mgr.25277) 185 : cephadm 1 Upgrade: Setting container_image for all crash 2023-10-09T04:56:24.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:23 smithi133 bash[105589]: audit 2023-10-09T04:56:22.587567+0000 mgr.y (mgr.25277) 186 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T04:56:24.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:23 smithi133 bash[105589]: cephadm 2023-10-09T04:56:22.605153+0000 mgr.y (mgr.25277) 187 : cephadm 1 Upgrade: osd.0 is safe to restart 2023-10-09T04:56:25.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:24 smithi194 bash[92666]: cluster 2023-10-09T04:56:23.413898+0000 mgr.y (mgr.25277) 188 : cluster 0 pgmap v129: 89 pgs: 89 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-09T04:56:25.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:24 smithi133 bash[102858]: cluster 2023-10-09T04:56:23.413898+0000 mgr.y (mgr.25277) 188 : cluster 0 pgmap v129: 89 pgs: 89 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-09T04:56:25.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:24 smithi133 bash[105589]: cluster 2023-10-09T04:56:23.413898+0000 mgr.y (mgr.25277) 188 : cluster 0 pgmap v129: 89 pgs: 89 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-09T04:56:27.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:26 smithi194 bash[92666]: cluster 2023-10-09T04:56:25.414575+0000 mgr.y (mgr.25277) 189 : cluster 0 pgmap v130: 89 pgs: 89 active+clean; 116 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 887 op/s 2023-10-09T04:56:27.253 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Removed 114939 objects 2023-10-09T04:56:27.253 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Clean up completed and total clean up time :157.713 2023-10-09T04:56:27.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:26 smithi133 bash[102858]: cluster 2023-10-09T04:56:25.414575+0000 mgr.y (mgr.25277) 189 : cluster 0 pgmap v130: 89 pgs: 89 active+clean; 116 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 887 op/s 2023-10-09T04:56:27.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:26 smithi133 bash[105589]: cluster 2023-10-09T04:56:25.414575+0000 mgr.y (mgr.25277) 189 : cluster 0 pgmap v130: 89 pgs: 89 active+clean; 116 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 887 op/s 2023-10-09T04:56:28.396 INFO:teuthology.orchestra.run.smithi133.stdout:wait for all mons to upgrade 2023-10-09T04:56:29.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:28 smithi194 bash[92666]: cluster 2023-10-09T04:56:27.415197+0000 mgr.y (mgr.25277) 190 : cluster 0 pgmap v131: 89 pgs: 89 active+clean; 116 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-10-09T04:56:29.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:28 smithi194 bash[92666]: audit 2023-10-09T04:56:28.264295+0000 mon.a (mon.0) 287 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:29.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:28 smithi194 bash[92666]: audit 2023-10-09T04:56:28.265555+0000 mon.a (mon.0) 288 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:56:29.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:28 smithi194 bash[92666]: audit 2023-10-09T04:56:28.266641+0000 mon.a (mon.0) 289 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:56:29.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:28 smithi194 bash[92666]: audit 2023-10-09T04:56:28.412266+0000 mon.a (mon.0) 290 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:29.236 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:28 smithi133 bash[102858]: cluster 2023-10-09T04:56:27.415197+0000 mgr.y (mgr.25277) 190 : cluster 0 pgmap v131: 89 pgs: 89 active+clean; 116 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-10-09T04:56:29.236 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:28 smithi133 bash[102858]: audit 2023-10-09T04:56:28.264295+0000 mon.a (mon.0) 287 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:29.236 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:28 smithi133 bash[102858]: audit 2023-10-09T04:56:28.265555+0000 mon.a (mon.0) 288 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:56:29.236 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:28 smithi133 bash[102858]: audit 2023-10-09T04:56:28.266641+0000 mon.a (mon.0) 289 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:56:29.236 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:28 smithi133 bash[102858]: audit 2023-10-09T04:56:28.412266+0000 mon.a (mon.0) 290 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:29.237 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:28 smithi133 bash[105589]: cluster 2023-10-09T04:56:27.415197+0000 mgr.y (mgr.25277) 190 : cluster 0 pgmap v131: 89 pgs: 89 active+clean; 116 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-10-09T04:56:29.237 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:28 smithi133 bash[105589]: audit 2023-10-09T04:56:28.264295+0000 mon.a (mon.0) 287 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:29.237 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:28 smithi133 bash[105589]: audit 2023-10-09T04:56:28.265555+0000 mon.a (mon.0) 288 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:56:29.237 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:28 smithi133 bash[105589]: audit 2023-10-09T04:56:28.266641+0000 mon.a (mon.0) 289 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:56:29.237 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:28 smithi133 bash[105589]: audit 2023-10-09T04:56:28.412266+0000 mon.a (mon.0) 290 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:29.627 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2023-10-09T04:56:29.628 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2023-10-09T04:56:30.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:29 smithi133 bash[105589]: cephadm 2023-10-09T04:56:28.255203+0000 mgr.y (mgr.25277) 191 : cephadm 1 Upgrade: Updating osd.0 2023-10-09T04:56:30.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:29 smithi133 bash[105589]: cephadm 2023-10-09T04:56:28.269182+0000 mgr.y (mgr.25277) 192 : cephadm 1 Deploying daemon osd.0 on smithi133 2023-10-09T04:56:30.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:29 smithi133 bash[102858]: cephadm 2023-10-09T04:56:28.255203+0000 mgr.y (mgr.25277) 191 : cephadm 1 Upgrade: Updating osd.0 2023-10-09T04:56:30.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:29 smithi133 bash[102858]: cephadm 2023-10-09T04:56:28.269182+0000 mgr.y (mgr.25277) 192 : cephadm 1 Deploying daemon osd.0 on smithi133 2023-10-09T04:56:30.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:29 smithi194 bash[92666]: cephadm 2023-10-09T04:56:28.255203+0000 mgr.y (mgr.25277) 191 : cephadm 1 Upgrade: Updating osd.0 2023-10-09T04:56:30.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:29 smithi194 bash[92666]: cephadm 2023-10-09T04:56:28.269182+0000 mgr.y (mgr.25277) 192 : cephadm 1 Deploying daemon osd.0 on smithi133 2023-10-09T04:56:30.358 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2023-10-09T04:56:31.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:30 smithi133 bash[105589]: cluster 2023-10-09T04:56:29.415672+0000 mgr.y (mgr.25277) 193 : cluster 0 pgmap v132: 89 pgs: 89 active+clean; 116 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-10-09T04:56:31.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:30 smithi133 bash[102858]: cluster 2023-10-09T04:56:29.415672+0000 mgr.y (mgr.25277) 193 : cluster 0 pgmap v132: 89 pgs: 89 active+clean; 116 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-10-09T04:56:31.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:30 smithi194 bash[92666]: cluster 2023-10-09T04:56:29.415672+0000 mgr.y (mgr.25277) 193 : cluster 0 pgmap v132: 89 pgs: 89 active+clean; 116 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-10-09T04:56:32.991 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:32 smithi133 bash[102858]: cluster 2023-10-09T04:56:31.416831+0000 mgr.y (mgr.25277) 194 : cluster 0 pgmap v133: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-10-09T04:56:32.991 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:32 smithi133 bash[105589]: cluster 2023-10-09T04:56:31.416831+0000 mgr.y (mgr.25277) 194 : cluster 0 pgmap v133: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-10-09T04:56:33.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:32 smithi194 bash[92666]: cluster 2023-10-09T04:56:31.416831+0000 mgr.y (mgr.25277) 194 : cluster 0 pgmap v133: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-10-09T04:56:34.258 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:33 smithi133 bash[105589]: cluster 2023-10-09T04:56:33.417496+0000 mgr.y (mgr.25277) 195 : cluster 0 pgmap v134: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 441 op/s 2023-10-09T04:56:34.258 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:33 smithi133 bash[102858]: cluster 2023-10-09T04:56:33.417496+0000 mgr.y (mgr.25277) 195 : cluster 0 pgmap v134: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 441 op/s 2023-10-09T04:56:34.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:33 smithi194 bash[92666]: cluster 2023-10-09T04:56:33.417496+0000 mgr.y (mgr.25277) 195 : cluster 0 pgmap v134: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 441 op/s 2023-10-09T04:56:34.392 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:34.393 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:34.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:56:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:56:34] "GET /metrics HTTP/1.1" 200 36150 "" "Prometheus/2.43.0" 2023-10-09T04:56:34.550 INFO:journalctl@ceph.osd.0.smithi133.stdout:Oct 09 04:56:34 smithi133 systemd[1]: Stopping Ceph osd.0 for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T04:56:34.550 INFO:journalctl@ceph.osd.0.smithi133.stdout:Oct 09 04:56:34 smithi133 bash[44591]: debug 2023-10-09T04:56:34.408+0000 7fc9738c5700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T04:56:34.550 INFO:journalctl@ceph.osd.0.smithi133.stdout:Oct 09 04:56:34 smithi133 bash[44591]: debug 2023-10-09T04:56:34.408+0000 7fc9738c5700 -1 osd.0 83 *** Got signal Terminated *** 2023-10-09T04:56:34.550 INFO:journalctl@ceph.osd.0.smithi133.stdout:Oct 09 04:56:34 smithi133 bash[44591]: debug 2023-10-09T04:56:34.408+0000 7fc9738c5700 -1 osd.0 83 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T04:56:35.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:34 smithi133 bash[102858]: cluster 2023-10-09T04:56:34.412856+0000 mon.a (mon.0) 291 : cluster 1 osd.0 marked itself down and dead 2023-10-09T04:56:35.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:34 smithi133 bash[105589]: cluster 2023-10-09T04:56:34.412856+0000 mon.a (mon.0) 291 : cluster 1 osd.0 marked itself down and dead 2023-10-09T04:56:35.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:34 smithi194 bash[92666]: cluster 2023-10-09T04:56:34.412856+0000 mon.a (mon.0) 291 : cluster 1 osd.0 marked itself down and dead 2023-10-09T04:56:36.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:35 smithi133 bash[105589]: cluster 2023-10-09T04:56:34.986265+0000 mon.a (mon.0) 292 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:56:36.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:35 smithi133 bash[105589]: cluster 2023-10-09T04:56:35.001736+0000 osd.7 (osd.7) 4 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T04:56:36.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:35 smithi133 bash[105589]: cluster 2023-10-09T04:56:35.002451+0000 mon.a (mon.0) 293 : cluster 0 osdmap e84: 8 total, 7 up, 8 in 2023-10-09T04:56:36.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:35 smithi133 bash[105589]: cluster 2023-10-09T04:56:35.004546+0000 osd.5 (osd.5) 5 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T04:56:36.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:35 smithi133 bash[105589]: cluster 2023-10-09T04:56:35.418033+0000 mgr.y (mgr.25277) 196 : cluster 0 pgmap v136: 89 pgs: 7 peering, 71 active+clean, 11 stale+active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 184 op/s 2023-10-09T04:56:36.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:35 smithi133 bash[102858]: cluster 2023-10-09T04:56:34.986265+0000 mon.a (mon.0) 292 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:56:36.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:35 smithi133 bash[102858]: cluster 2023-10-09T04:56:35.001736+0000 osd.7 (osd.7) 4 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T04:56:36.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:35 smithi133 bash[102858]: cluster 2023-10-09T04:56:35.002451+0000 mon.a (mon.0) 293 : cluster 0 osdmap e84: 8 total, 7 up, 8 in 2023-10-09T04:56:36.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:35 smithi133 bash[102858]: cluster 2023-10-09T04:56:35.004546+0000 osd.5 (osd.5) 5 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T04:56:36.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:35 smithi133 bash[102858]: cluster 2023-10-09T04:56:35.418033+0000 mgr.y (mgr.25277) 196 : cluster 0 pgmap v136: 89 pgs: 7 peering, 71 active+clean, 11 stale+active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 184 op/s 2023-10-09T04:56:36.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:35 smithi194 bash[92666]: cluster 2023-10-09T04:56:34.986265+0000 mon.a (mon.0) 292 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:56:36.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:35 smithi194 bash[92666]: cluster 2023-10-09T04:56:35.001736+0000 osd.7 (osd.7) 4 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T04:56:36.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:35 smithi194 bash[92666]: cluster 2023-10-09T04:56:35.002451+0000 mon.a (mon.0) 293 : cluster 0 osdmap e84: 8 total, 7 up, 8 in 2023-10-09T04:56:36.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:35 smithi194 bash[92666]: cluster 2023-10-09T04:56:35.004546+0000 osd.5 (osd.5) 5 : cluster 3 failed to encode map e84 with expected crc 2023-10-09T04:56:36.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:35 smithi194 bash[92666]: cluster 2023-10-09T04:56:35.418033+0000 mgr.y (mgr.25277) 196 : cluster 0 pgmap v136: 89 pgs: 7 peering, 71 active+clean, 11 stale+active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 184 op/s 2023-10-09T04:56:36.922 INFO:journalctl@ceph.osd.0.smithi133.stdout:Oct 09 04:56:36 smithi133 bash[112587]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd-0 2023-10-09T04:56:37.261 INFO:journalctl@ceph.osd.0.smithi133.stdout:Oct 09 04:56:36 smithi133 bash[112750]: Error response from daemon: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd-0 2023-10-09T04:56:37.262 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:36 smithi133 bash[105589]: cluster 2023-10-09T04:56:35.997688+0000 mon.a (mon.0) 294 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 6 pgs peering (PG_AVAILABILITY) 2023-10-09T04:56:37.262 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:36 smithi133 bash[105589]: cluster 2023-10-09T04:56:36.005480+0000 osd.6 (osd.6) 4 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:56:37.262 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:36 smithi133 bash[105589]: cluster 2023-10-09T04:56:36.006163+0000 osd.5 (osd.5) 6 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:56:37.262 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:36 smithi133 bash[105589]: cluster 2023-10-09T04:56:36.006956+0000 mon.a (mon.0) 295 : cluster 0 osdmap e85: 8 total, 7 up, 8 in 2023-10-09T04:56:37.262 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:36 smithi133 bash[105589]: cluster 2023-10-09T04:56:36.007324+0000 osd.3 (osd.3) 5 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:56:37.262 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:36 smithi133 bash[105589]: cluster 2023-10-09T04:56:36.007411+0000 osd.2 (osd.2) 8 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:56:37.262 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:36 smithi133 bash[105589]: cluster 2023-10-09T04:56:36.007440+0000 osd.1 (osd.1) 6 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:56:37.262 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:36 smithi133 bash[105589]: cluster 2023-10-09T04:56:36.007947+0000 osd.7 (osd.7) 5 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:56:37.263 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:36 smithi133 bash[105589]: cluster 2023-10-09T04:56:36.008040+0000 osd.6 (osd.6) 5 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:56:37.263 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:36 smithi133 bash[105589]: cluster 2023-10-09T04:56:36.008391+0000 osd.4 (osd.4) 9 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:56:37.263 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:36 smithi133 bash[105589]: cluster 2023-10-09T04:56:36.009659+0000 osd.5 (osd.5) 7 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:56:37.263 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:36 smithi133 bash[102858]: cluster 2023-10-09T04:56:35.997688+0000 mon.a (mon.0) 294 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 6 pgs peering (PG_AVAILABILITY) 2023-10-09T04:56:37.263 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:36 smithi133 bash[102858]: cluster 2023-10-09T04:56:36.005480+0000 osd.6 (osd.6) 4 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:56:37.263 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:36 smithi133 bash[102858]: cluster 2023-10-09T04:56:36.006163+0000 osd.5 (osd.5) 6 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:56:37.263 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:36 smithi133 bash[102858]: cluster 2023-10-09T04:56:36.006956+0000 mon.a (mon.0) 295 : cluster 0 osdmap e85: 8 total, 7 up, 8 in 2023-10-09T04:56:37.263 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:36 smithi133 bash[102858]: cluster 2023-10-09T04:56:36.007324+0000 osd.3 (osd.3) 5 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:56:37.263 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:36 smithi133 bash[102858]: cluster 2023-10-09T04:56:36.007411+0000 osd.2 (osd.2) 8 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:56:37.263 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:36 smithi133 bash[102858]: cluster 2023-10-09T04:56:36.007440+0000 osd.1 (osd.1) 6 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:56:37.264 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:36 smithi133 bash[102858]: cluster 2023-10-09T04:56:36.007947+0000 osd.7 (osd.7) 5 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:56:37.264 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:36 smithi133 bash[102858]: cluster 2023-10-09T04:56:36.008040+0000 osd.6 (osd.6) 5 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:56:37.264 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:36 smithi133 bash[102858]: cluster 2023-10-09T04:56:36.008391+0000 osd.4 (osd.4) 9 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:56:37.264 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:36 smithi133 bash[102858]: cluster 2023-10-09T04:56:36.009659+0000 osd.5 (osd.5) 7 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:56:37.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:36 smithi194 bash[92666]: cluster 2023-10-09T04:56:35.997688+0000 mon.a (mon.0) 294 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 6 pgs peering (PG_AVAILABILITY) 2023-10-09T04:56:37.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:36 smithi194 bash[92666]: cluster 2023-10-09T04:56:36.005480+0000 osd.6 (osd.6) 4 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:56:37.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:36 smithi194 bash[92666]: cluster 2023-10-09T04:56:36.006163+0000 osd.5 (osd.5) 6 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:56:37.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:36 smithi194 bash[92666]: cluster 2023-10-09T04:56:36.006956+0000 mon.a (mon.0) 295 : cluster 0 osdmap e85: 8 total, 7 up, 8 in 2023-10-09T04:56:37.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:36 smithi194 bash[92666]: cluster 2023-10-09T04:56:36.007324+0000 osd.3 (osd.3) 5 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:56:37.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:36 smithi194 bash[92666]: cluster 2023-10-09T04:56:36.007411+0000 osd.2 (osd.2) 8 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:56:37.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:36 smithi194 bash[92666]: cluster 2023-10-09T04:56:36.007440+0000 osd.1 (osd.1) 6 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:56:37.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:36 smithi194 bash[92666]: cluster 2023-10-09T04:56:36.007947+0000 osd.7 (osd.7) 5 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:56:37.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:36 smithi194 bash[92666]: cluster 2023-10-09T04:56:36.008040+0000 osd.6 (osd.6) 5 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:56:37.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:36 smithi194 bash[92666]: cluster 2023-10-09T04:56:36.008391+0000 osd.4 (osd.4) 9 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:56:37.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:36 smithi194 bash[92666]: cluster 2023-10-09T04:56:36.009659+0000 osd.5 (osd.5) 7 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:56:38.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:37 smithi133 bash[102858]: cluster 2023-10-09T04:56:37.418790+0000 mgr.y (mgr.25277) 197 : cluster 0 pgmap v138: 89 pgs: 7 peering, 71 active+clean, 11 stale+active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 231 op/s 2023-10-09T04:56:38.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:38 smithi133 bash[105589]: cluster 2023-10-09T04:56:37.418790+0000 mgr.y (mgr.25277) 197 : cluster 0 pgmap v138: 89 pgs: 7 peering, 71 active+clean, 11 stale+active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 231 op/s 2023-10-09T04:56:38.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:38 smithi194 bash[92666]: cluster 2023-10-09T04:56:37.418790+0000 mgr.y (mgr.25277) 197 : cluster 0 pgmap v138: 89 pgs: 7 peering, 71 active+clean, 11 stale+active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 231 op/s 2023-10-09T04:56:40.759 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:40 smithi133 bash[105589]: cluster 2023-10-09T04:56:39.419518+0000 mgr.y (mgr.25277) 198 : cluster 0 pgmap v139: 89 pgs: 7 peering, 71 active+clean, 11 stale+active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:56:40.759 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:40 smithi133 bash[102858]: cluster 2023-10-09T04:56:39.419518+0000 mgr.y (mgr.25277) 198 : cluster 0 pgmap v139: 89 pgs: 7 peering, 71 active+clean, 11 stale+active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:56:40.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:40 smithi194 bash[92666]: cluster 2023-10-09T04:56:39.419518+0000 mgr.y (mgr.25277) 198 : cluster 0 pgmap v139: 89 pgs: 7 peering, 71 active+clean, 11 stale+active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:56:41.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:41 smithi133 bash[105589]: cluster 2023-10-09T04:56:41.474998+0000 mon.a (mon.0) 296 : cluster 3 Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:56:41.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:41 smithi133 bash[105589]: cluster 2023-10-09T04:56:41.475060+0000 mon.a (mon.0) 297 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 6 pgs peering) 2023-10-09T04:56:41.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:41 smithi133 bash[102858]: cluster 2023-10-09T04:56:41.474998+0000 mon.a (mon.0) 296 : cluster 3 Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:56:41.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:41 smithi133 bash[102858]: cluster 2023-10-09T04:56:41.475060+0000 mon.a (mon.0) 297 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 6 pgs peering) 2023-10-09T04:56:41.800 INFO:journalctl@ceph.osd.0.smithi133.stdout:Oct 09 04:56:41 smithi133 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@osd.0.service: Succeeded. 2023-10-09T04:56:41.800 INFO:journalctl@ceph.osd.0.smithi133.stdout:Oct 09 04:56:41 smithi133 systemd[1]: Stopped Ceph osd.0 for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:56:41.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:41 smithi194 bash[92666]: cluster 2023-10-09T04:56:41.474998+0000 mon.a (mon.0) 296 : cluster 3 Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:56:41.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:41 smithi194 bash[92666]: cluster 2023-10-09T04:56:41.475060+0000 mon.a (mon.0) 297 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 6 pgs peering) 2023-10-09T04:56:42.163 INFO:journalctl@ceph.osd.0.smithi133.stdout:Oct 09 04:56:41 smithi133 systemd[1]: Started Ceph osd.0 for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:56:42.455 INFO:teuthology.orchestra.run.smithi133.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 3 2023-10-09T04:56:42.961 INFO:teuthology.orchestra.run.smithi133.stderr:pool 'unique_pool_1' removed 2023-10-09T04:56:43.186 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:42 smithi133 bash[102858]: cluster 2023-10-09T04:56:41.420906+0000 mgr.y (mgr.25277) 199 : cluster 0 pgmap v140: 89 pgs: 7 active+undersized+degraded, 23 active+undersized, 59 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:56:43.187 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:42 smithi133 bash[102858]: audit 2023-10-09T04:56:41.932827+0000 mon.a (mon.0) 298 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:43.187 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:42 smithi133 bash[102858]: audit 2023-10-09T04:56:41.941390+0000 mon.a (mon.0) 299 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:43.187 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:42 smithi133 bash[102858]: audit 2023-10-09T04:56:41.968966+0000 mon.a (mon.0) 300 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:56:43.187 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:42 smithi133 bash[102858]: audit 2023-10-09T04:56:42.437789+0000 mon.a (mon.0) 301 : audit 0 from='client.? 172.21.15.133:0/3180876787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:56:43.187 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:42 smithi133 bash[102858]: audit 2023-10-09T04:56:42.616119+0000 mon.c (mon.2) 3 : audit 1 from='client.? 172.21.15.133:0/92677036' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:56:43.187 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:42 smithi133 bash[102858]: audit 2023-10-09T04:56:42.616507+0000 mon.a (mon.0) 302 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:56:43.187 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:42 smithi133 bash[105589]: cluster 2023-10-09T04:56:41.420906+0000 mgr.y (mgr.25277) 199 : cluster 0 pgmap v140: 89 pgs: 7 active+undersized+degraded, 23 active+undersized, 59 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:56:43.187 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:42 smithi133 bash[105589]: audit 2023-10-09T04:56:41.932827+0000 mon.a (mon.0) 298 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:43.188 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:42 smithi133 bash[105589]: audit 2023-10-09T04:56:41.941390+0000 mon.a (mon.0) 299 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:43.188 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:42 smithi133 bash[105589]: audit 2023-10-09T04:56:41.968966+0000 mon.a (mon.0) 300 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:56:43.188 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:42 smithi133 bash[105589]: audit 2023-10-09T04:56:42.437789+0000 mon.a (mon.0) 301 : audit 0 from='client.? 172.21.15.133:0/3180876787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:56:43.188 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:42 smithi133 bash[105589]: audit 2023-10-09T04:56:42.616119+0000 mon.c (mon.2) 3 : audit 1 from='client.? 172.21.15.133:0/92677036' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:56:43.188 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:42 smithi133 bash[105589]: audit 2023-10-09T04:56:42.616507+0000 mon.a (mon.0) 302 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:56:43.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:42 smithi194 bash[92666]: cluster 2023-10-09T04:56:41.420906+0000 mgr.y (mgr.25277) 199 : cluster 0 pgmap v140: 89 pgs: 7 active+undersized+degraded, 23 active+undersized, 59 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:56:43.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:42 smithi194 bash[92666]: audit 2023-10-09T04:56:41.932827+0000 mon.a (mon.0) 298 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:43.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:42 smithi194 bash[92666]: audit 2023-10-09T04:56:41.941390+0000 mon.a (mon.0) 299 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:43.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:42 smithi194 bash[92666]: audit 2023-10-09T04:56:41.968966+0000 mon.a (mon.0) 300 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:56:43.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:42 smithi194 bash[92666]: audit 2023-10-09T04:56:42.437789+0000 mon.a (mon.0) 301 : audit 0 from='client.? 172.21.15.133:0/3180876787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:56:43.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:42 smithi194 bash[92666]: audit 2023-10-09T04:56:42.616119+0000 mon.c (mon.2) 3 : audit 1 from='client.? 172.21.15.133:0/92677036' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:56:43.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:42 smithi194 bash[92666]: audit 2023-10-09T04:56:42.616507+0000 mon.a (mon.0) 302 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:56:44.282 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:43 smithi133 bash[102858]: cluster 2023-10-09T04:56:42.948927+0000 osd.1 (osd.1) 7 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:56:44.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:43 smithi133 bash[102858]: cluster 2023-10-09T04:56:42.949476+0000 osd.5 (osd.5) 8 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:56:44.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:43 smithi133 bash[102858]: audit 2023-10-09T04:56:42.951720+0000 mon.a (mon.0) 303 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:56:44.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:43 smithi133 bash[102858]: cluster 2023-10-09T04:56:42.951836+0000 mon.a (mon.0) 304 : cluster 0 osdmap e86: 8 total, 7 up, 8 in 2023-10-09T04:56:44.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:43 smithi133 bash[102858]: cluster 2023-10-09T04:56:43.336828+0000 osd.2 (osd.2) 9 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:56:44.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:43 smithi133 bash[102858]: audit 2023-10-09T04:56:43.416564+0000 mon.a (mon.0) 305 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:44.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:43 smithi133 bash[102858]: audit 2023-10-09T04:56:43.417612+0000 mon.a (mon.0) 306 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:44.284 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:43 smithi133 bash[102858]: cluster 2023-10-09T04:56:43.460413+0000 osd.3 (osd.3) 6 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:56:44.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:43 smithi133 bash[105589]: cluster 2023-10-09T04:56:42.948927+0000 osd.1 (osd.1) 7 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:56:44.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:43 smithi133 bash[105589]: cluster 2023-10-09T04:56:42.949476+0000 osd.5 (osd.5) 8 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:56:44.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:43 smithi133 bash[105589]: audit 2023-10-09T04:56:42.951720+0000 mon.a (mon.0) 303 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:56:44.285 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:43 smithi133 bash[105589]: cluster 2023-10-09T04:56:42.951836+0000 mon.a (mon.0) 304 : cluster 0 osdmap e86: 8 total, 7 up, 8 in 2023-10-09T04:56:44.285 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:43 smithi133 bash[105589]: cluster 2023-10-09T04:56:43.336828+0000 osd.2 (osd.2) 9 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:56:44.285 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:43 smithi133 bash[105589]: audit 2023-10-09T04:56:43.416564+0000 mon.a (mon.0) 305 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:44.285 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:43 smithi133 bash[105589]: audit 2023-10-09T04:56:43.417612+0000 mon.a (mon.0) 306 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:44.285 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:43 smithi133 bash[105589]: cluster 2023-10-09T04:56:43.460413+0000 osd.3 (osd.3) 6 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:56:44.310 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T04:56:44.363 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:56:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:56:44] "GET /metrics HTTP/1.1" 200 33935 "" "Prometheus/2.43.0" 2023-10-09T04:56:44.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:43 smithi194 bash[92666]: cluster 2023-10-09T04:56:42.948927+0000 osd.1 (osd.1) 7 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:56:44.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:43 smithi194 bash[92666]: cluster 2023-10-09T04:56:42.949476+0000 osd.5 (osd.5) 8 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:56:44.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:43 smithi194 bash[92666]: audit 2023-10-09T04:56:42.951720+0000 mon.a (mon.0) 303 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:56:44.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:43 smithi194 bash[92666]: cluster 2023-10-09T04:56:42.951836+0000 mon.a (mon.0) 304 : cluster 0 osdmap e86: 8 total, 7 up, 8 in 2023-10-09T04:56:44.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:43 smithi194 bash[92666]: cluster 2023-10-09T04:56:43.336828+0000 osd.2 (osd.2) 9 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:56:44.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:43 smithi194 bash[92666]: audit 2023-10-09T04:56:43.416564+0000 mon.a (mon.0) 305 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:44.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:43 smithi194 bash[92666]: audit 2023-10-09T04:56:43.417612+0000 mon.a (mon.0) 306 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:44.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:43 smithi194 bash[92666]: cluster 2023-10-09T04:56:43.460413+0000 osd.3 (osd.3) 6 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:56:45.152 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T04:56:45.153 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T04:56:45.153 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2023-10-09T04:56:45.154 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_2 16 2023-10-09T04:56:45.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:44 smithi133 bash[105589]: cluster 2023-10-09T04:56:43.422295+0000 mgr.y (mgr.25277) 200 : cluster 0 pgmap v142: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:56:45.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:44 smithi133 bash[102858]: cluster 2023-10-09T04:56:43.422295+0000 mgr.y (mgr.25277) 200 : cluster 0 pgmap v142: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:56:45.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:44 smithi194 bash[92666]: cluster 2023-10-09T04:56:43.422295+0000 mgr.y (mgr.25277) 200 : cluster 0 pgmap v142: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:56:45.926 INFO:journalctl@ceph.osd.0.smithi133.stdout:Oct 09 04:56:45 smithi133 bash[113015]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T04:56:45.926 INFO:journalctl@ceph.osd.0.smithi133.stdout:Oct 09 04:56:45 smithi133 bash[113015]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-09T04:56:45.926 INFO:journalctl@ceph.osd.0.smithi133.stdout:Oct 09 04:56:45 smithi133 bash[113015]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-10-09T04:56:45.926 INFO:journalctl@ceph.osd.0.smithi133.stdout:Oct 09 04:56:45 smithi133 bash[113015]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-10-09T04:56:45.927 INFO:journalctl@ceph.osd.0.smithi133.stdout:Oct 09 04:56:45 smithi133 bash[113015]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-10-09T04:56:45.927 INFO:journalctl@ceph.osd.0.smithi133.stdout:Oct 09 04:56:45 smithi133 bash[113015]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-10-09T04:56:45.927 INFO:journalctl@ceph.osd.0.smithi133.stdout:Oct 09 04:56:45 smithi133 bash[113015]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-09T04:56:45.927 INFO:journalctl@ceph.osd.0.smithi133.stdout:Oct 09 04:56:45 smithi133 bash[113015]: --> ceph-volume lvm activate successful for osd ID: 0 2023-10-09T04:56:47.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:46 smithi133 bash[102858]: cluster 2023-10-09T04:56:45.423474+0000 mgr.y (mgr.25277) 201 : cluster 0 pgmap v143: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:56:47.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:46 smithi133 bash[105589]: cluster 2023-10-09T04:56:45.423474+0000 mgr.y (mgr.25277) 201 : cluster 0 pgmap v143: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:56:47.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:46 smithi194 bash[92666]: cluster 2023-10-09T04:56:45.423474+0000 mgr.y (mgr.25277) 201 : cluster 0 pgmap v143: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:56:48.049 INFO:journalctl@ceph.osd.0.smithi133.stdout:Oct 09 04:56:47 smithi133 bash[113504]: debug 2023-10-09T04:56:47.708+0000 7fdc257d5700 1 -- 172.21.15.133:0/780376271 <== mon.2 v2:172.21.15.133:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55e86fcf04e0 con 0x55e86fd4ac00 2023-10-09T04:56:48.549 INFO:journalctl@ceph.osd.0.smithi133.stdout:Oct 09 04:56:48 smithi133 bash[113504]: debug 2023-10-09T04:56:48.268+0000 7fdc2e95b640 -1 Falling back to public interface 2023-10-09T04:56:49.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:48 smithi133 bash[102858]: cluster 2023-10-09T04:56:47.424212+0000 mgr.y (mgr.25277) 202 : cluster 0 pgmap v144: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:56:49.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:48 smithi133 bash[102858]: audit 2023-10-09T04:56:48.424771+0000 mon.a (mon.0) 307 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:49.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:48 smithi133 bash[105589]: cluster 2023-10-09T04:56:47.424212+0000 mgr.y (mgr.25277) 202 : cluster 0 pgmap v144: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:56:49.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:48 smithi133 bash[105589]: audit 2023-10-09T04:56:48.424771+0000 mon.a (mon.0) 307 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:49.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:48 smithi194 bash[92666]: cluster 2023-10-09T04:56:47.424212+0000 mgr.y (mgr.25277) 202 : cluster 0 pgmap v144: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:56:49.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:48 smithi194 bash[92666]: audit 2023-10-09T04:56:48.424771+0000 mon.a (mon.0) 307 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:50.192 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:50.193 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:51.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:50 smithi133 bash[102858]: cluster 2023-10-09T04:56:49.424984+0000 mgr.y (mgr.25277) 203 : cluster 0 pgmap v145: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:56:51.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:50 smithi133 bash[105589]: cluster 2023-10-09T04:56:49.424984+0000 mgr.y (mgr.25277) 203 : cluster 0 pgmap v145: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:56:51.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:50 smithi194 bash[92666]: cluster 2023-10-09T04:56:49.424984+0000 mgr.y (mgr.25277) 203 : cluster 0 pgmap v145: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:56:52.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:52 smithi133 bash[102858]: audit 2023-10-09T04:56:51.214765+0000 mon.a (mon.0) 308 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:52.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:52 smithi133 bash[102858]: audit 2023-10-09T04:56:51.221032+0000 mon.a (mon.0) 309 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:52.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:52 smithi133 bash[102858]: cluster 2023-10-09T04:56:51.426237+0000 mgr.y (mgr.25277) 204 : cluster 0 pgmap v146: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:56:52.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:52 smithi133 bash[102858]: audit 2023-10-09T04:56:52.012167+0000 mon.a (mon.0) 310 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:52.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:52 smithi133 bash[102858]: audit 2023-10-09T04:56:52.017755+0000 mon.a (mon.0) 311 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:52.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:52 smithi133 bash[105589]: audit 2023-10-09T04:56:51.214765+0000 mon.a (mon.0) 308 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:52.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:52 smithi133 bash[105589]: audit 2023-10-09T04:56:51.221032+0000 mon.a (mon.0) 309 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:52.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:52 smithi133 bash[105589]: cluster 2023-10-09T04:56:51.426237+0000 mgr.y (mgr.25277) 204 : cluster 0 pgmap v146: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:56:52.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:52 smithi133 bash[105589]: audit 2023-10-09T04:56:52.012167+0000 mon.a (mon.0) 310 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:52.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:52 smithi133 bash[105589]: audit 2023-10-09T04:56:52.017755+0000 mon.a (mon.0) 311 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:52.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:52 smithi194 bash[92666]: audit 2023-10-09T04:56:51.214765+0000 mon.a (mon.0) 308 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:52.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:52 smithi194 bash[92666]: audit 2023-10-09T04:56:51.221032+0000 mon.a (mon.0) 309 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:52.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:52 smithi194 bash[92666]: cluster 2023-10-09T04:56:51.426237+0000 mgr.y (mgr.25277) 204 : cluster 0 pgmap v146: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:56:52.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:52 smithi194 bash[92666]: audit 2023-10-09T04:56:52.012167+0000 mon.a (mon.0) 310 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:52.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:52 smithi194 bash[92666]: audit 2023-10-09T04:56:52.017755+0000 mon.a (mon.0) 311 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:53.985 INFO:journalctl@ceph.osd.0.smithi133.stdout:Oct 09 04:56:53 smithi133 bash[113504]: debug 2023-10-09T04:56:53.548+0000 7fdc2e95b640 -1 osd.0 83 log_to_monitors true 2023-10-09T04:56:54.284 INFO:journalctl@ceph.osd.0.smithi133.stdout:Oct 09 04:56:54 smithi133 bash[113504]: debug 2023-10-09T04:56:54.036+0000 7fdc1ddc3700 -1 osd.0 83 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T04:56:54.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:53 smithi133 bash[105589]: cluster 2023-10-09T04:56:53.426708+0000 mgr.y (mgr.25277) 205 : cluster 0 pgmap v147: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:56:54.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:53 smithi133 bash[105589]: audit 2023-10-09T04:56:53.553486+0000 mon.a (mon.0) 312 : audit 1 from='osd.0 [v2:172.21.15.133:6802/2098404708,v1:172.21.15.133:6803/2098404708]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T04:56:54.284 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:53 smithi133 bash[102858]: cluster 2023-10-09T04:56:53.426708+0000 mgr.y (mgr.25277) 205 : cluster 0 pgmap v147: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:56:54.284 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:53 smithi133 bash[102858]: audit 2023-10-09T04:56:53.553486+0000 mon.a (mon.0) 312 : audit 1 from='osd.0 [v2:172.21.15.133:6802/2098404708,v1:172.21.15.133:6803/2098404708]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T04:56:54.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:53 smithi194 bash[92666]: cluster 2023-10-09T04:56:53.426708+0000 mgr.y (mgr.25277) 205 : cluster 0 pgmap v147: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:56:54.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:53 smithi194 bash[92666]: audit 2023-10-09T04:56:53.553486+0000 mon.a (mon.0) 312 : audit 1 from='osd.0 [v2:172.21.15.133:6802/2098404708,v1:172.21.15.133:6803/2098404708]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T04:56:54.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:56:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:56:54] "GET /metrics HTTP/1.1" 200 33935 "" "Prometheus/2.43.0" 2023-10-09T04:56:55.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:55 smithi133 bash[102858]: cluster 2023-10-09T04:56:53.994781+0000 osd.3 (osd.3) 7 : cluster 3 failed to encode map e87 with expected crc 2023-10-09T04:56:55.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:55 smithi133 bash[102858]: cluster 2023-10-09T04:56:53.995902+0000 osd.5 (osd.5) 9 : cluster 3 failed to encode map e87 with expected crc 2023-10-09T04:56:55.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:55 smithi133 bash[102858]: audit 2023-10-09T04:56:53.996884+0000 mon.a (mon.0) 313 : audit 1 from='osd.0 [v2:172.21.15.133:6802/2098404708,v1:172.21.15.133:6803/2098404708]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T04:56:55.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:55 smithi133 bash[102858]: cluster 2023-10-09T04:56:53.996964+0000 mon.a (mon.0) 314 : cluster 0 osdmap e87: 8 total, 7 up, 8 in 2023-10-09T04:56:55.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:55 smithi133 bash[102858]: audit 2023-10-09T04:56:53.997691+0000 mon.a (mon.0) 315 : audit 1 from='osd.0 [v2:172.21.15.133:6802/2098404708,v1:172.21.15.133:6803/2098404708]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:56:55.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:55 smithi133 bash[102858]: cluster 2023-10-09T04:56:54.388316+0000 osd.1 (osd.1) 8 : cluster 3 failed to encode map e87 with expected crc 2023-10-09T04:56:55.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:55 smithi133 bash[105589]: cluster 2023-10-09T04:56:53.994781+0000 osd.3 (osd.3) 7 : cluster 3 failed to encode map e87 with expected crc 2023-10-09T04:56:55.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:55 smithi133 bash[105589]: cluster 2023-10-09T04:56:53.995902+0000 osd.5 (osd.5) 9 : cluster 3 failed to encode map e87 with expected crc 2023-10-09T04:56:55.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:55 smithi133 bash[105589]: audit 2023-10-09T04:56:53.996884+0000 mon.a (mon.0) 313 : audit 1 from='osd.0 [v2:172.21.15.133:6802/2098404708,v1:172.21.15.133:6803/2098404708]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T04:56:55.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:55 smithi133 bash[105589]: cluster 2023-10-09T04:56:53.996964+0000 mon.a (mon.0) 314 : cluster 0 osdmap e87: 8 total, 7 up, 8 in 2023-10-09T04:56:55.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:55 smithi133 bash[105589]: audit 2023-10-09T04:56:53.997691+0000 mon.a (mon.0) 315 : audit 1 from='osd.0 [v2:172.21.15.133:6802/2098404708,v1:172.21.15.133:6803/2098404708]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:56:55.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:55 smithi133 bash[105589]: cluster 2023-10-09T04:56:54.388316+0000 osd.1 (osd.1) 8 : cluster 3 failed to encode map e87 with expected crc 2023-10-09T04:56:55.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:55 smithi194 bash[92666]: cluster 2023-10-09T04:56:53.994781+0000 osd.3 (osd.3) 7 : cluster 3 failed to encode map e87 with expected crc 2023-10-09T04:56:55.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:55 smithi194 bash[92666]: cluster 2023-10-09T04:56:53.995902+0000 osd.5 (osd.5) 9 : cluster 3 failed to encode map e87 with expected crc 2023-10-09T04:56:55.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:55 smithi194 bash[92666]: audit 2023-10-09T04:56:53.996884+0000 mon.a (mon.0) 313 : audit 1 from='osd.0 [v2:172.21.15.133:6802/2098404708,v1:172.21.15.133:6803/2098404708]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T04:56:55.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:55 smithi194 bash[92666]: cluster 2023-10-09T04:56:53.996964+0000 mon.a (mon.0) 314 : cluster 0 osdmap e87: 8 total, 7 up, 8 in 2023-10-09T04:56:55.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:55 smithi194 bash[92666]: audit 2023-10-09T04:56:53.997691+0000 mon.a (mon.0) 315 : audit 1 from='osd.0 [v2:172.21.15.133:6802/2098404708,v1:172.21.15.133:6803/2098404708]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:56:55.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:55 smithi194 bash[92666]: cluster 2023-10-09T04:56:54.388316+0000 osd.1 (osd.1) 8 : cluster 3 failed to encode map e87 with expected crc 2023-10-09T04:56:55.636 INFO:teuthology.orchestra.run.smithi133.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T04:56:55.637 INFO:teuthology.orchestra.run.smithi133.stdout:alertmanager.a smithi133 *:9093,9094 running (6m) - 14m 14.7M - 0.25.0 c8568f914cd2 f7641cfa53d2 2023-10-09T04:56:55.637 INFO:teuthology.orchestra.run.smithi133.stdout:grafana.a smithi194 *:3000 running (6m) - 13m 76.4M - 9.4.7 2c41d148cca3 18d67948e55e 2023-10-09T04:56:55.637 INFO:teuthology.orchestra.run.smithi133.stdout:mds.foo.smithi133.tsuabh smithi133 running (11m) - 11m 14.1M - 17.2.6-1303-gb2903cb0 5517396c50d4 1adbb452599e 2023-10-09T04:56:55.637 INFO:teuthology.orchestra.run.smithi133.stdout:mds.foo.smithi194.yxnfqn smithi194 running (11m) - 11m 16.4M - 17.2.6-1303-gb2903cb0 5517396c50d4 745c16c1ffd2 2023-10-09T04:56:55.637 INFO:teuthology.orchestra.run.smithi133.stdout:mgr.x smithi194 *:8443,9283,8765 running (5m) - 22m 427M - 18.2.0-598-gb4fa663f 3060897fdf97 e7277fed699e 2023-10-09T04:56:55.637 INFO:teuthology.orchestra.run.smithi133.stdout:mgr.y smithi133 *:8443,9283,8765 running (6m) - 25m 484M - 18.2.0-598-gb4fa663f 3060897fdf97 caf6152f91c6 2023-10-09T04:56:55.637 INFO:teuthology.orchestra.run.smithi133.stdout:mon.a smithi133 running (4m) - 25m 46.5M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 ee45bbc6b194 2023-10-09T04:56:55.637 INFO:teuthology.orchestra.run.smithi133.stdout:mon.b smithi194 running (2m) - 23m 29.9M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 ed14c6f4a04d 2023-10-09T04:56:55.637 INFO:teuthology.orchestra.run.smithi133.stdout:mon.c smithi133 running (2m) - 23m 34.0M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 eb3f3f9009e4 2023-10-09T04:56:55.637 INFO:teuthology.orchestra.run.smithi133.stdout:node-exporter.a smithi133 *:9100 running (6m) - 14m 8504k - 1.5.0 0da6a335fe13 f8bdde5940e7 2023-10-09T04:56:55.637 INFO:teuthology.orchestra.run.smithi133.stdout:node-exporter.b smithi194 *:9100 running (6m) - 14m 8204k - 1.5.0 0da6a335fe13 ffab717ac947 2023-10-09T04:56:55.638 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0 smithi133 running (8s) - 22m - 2524M 18.2.0-598-gb4fa663f 3060897fdf97 021c5eccb109 2023-10-09T04:56:55.638 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1 smithi133 running (21m) - 21m 327M 2524M 17.2.6-1303-gb2903cb0 5517396c50d4 27ac869bf46a 2023-10-09T04:56:55.638 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2 smithi133 running (20m) - 20m 451M 2524M 17.2.6-1303-gb2903cb0 5517396c50d4 e9d2970cad24 2023-10-09T04:56:55.638 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3 smithi133 running (19m) - 19m 429M 2524M 17.2.6-1303-gb2903cb0 5517396c50d4 415368a68902 2023-10-09T04:56:55.638 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4 smithi194 running (18m) - 18m 440M 2524M 17.2.6-1303-gb2903cb0 5517396c50d4 c77edffeb061 2023-10-09T04:56:55.638 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5 smithi194 running (17m) - 17m 470M 2524M 17.2.6-1303-gb2903cb0 5517396c50d4 47540765fad9 2023-10-09T04:56:55.638 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6 smithi194 running (16m) - 16m 311M 2524M 17.2.6-1303-gb2903cb0 5517396c50d4 26bbe2f2859f 2023-10-09T04:56:55.638 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7 smithi194 running (15m) - 15m 428M 2524M 17.2.6-1303-gb2903cb0 5517396c50d4 45c343cd22ea 2023-10-09T04:56:55.638 INFO:teuthology.orchestra.run.smithi133.stdout:prometheus.a smithi194 *:9095 running (5m) - 14m 50.7M - 2.43.0 a07b618ecd1d f40ed7def7ab 2023-10-09T04:56:56.075 INFO:teuthology.orchestra.run.smithi133.stderr:pool 'unique_pool_2' created 2023-10-09T04:56:56.108 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:56 smithi133 bash[102858]: cluster 2023-10-09T04:56:54.696891+0000 osd.7 (osd.7) 6 : cluster 3 failed to encode map e87 with expected crc 2023-10-09T04:56:56.108 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:56 smithi133 bash[102858]: cluster 2023-10-09T04:56:54.992369+0000 mon.a (mon.0) 316 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:56:56.108 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:56 smithi133 bash[102858]: cluster 2023-10-09T04:56:54.996827+0000 osd.1 (osd.1) 9 : cluster 3 failed to encode map e88 with expected crc 2023-10-09T04:56:56.108 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:56 smithi133 bash[102858]: cluster 2023-10-09T04:56:55.000267+0000 mon.a (mon.0) 317 : cluster 1 osd.0 [v2:172.21.15.133:6802/2098404708,v1:172.21.15.133:6803/2098404708] boot 2023-10-09T04:56:56.108 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:56 smithi133 bash[102858]: cluster 2023-10-09T04:56:55.000351+0000 mon.a (mon.0) 318 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2023-10-09T04:56:56.108 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:56 smithi133 bash[102858]: audit 2023-10-09T04:56:55.000953+0000 mon.a (mon.0) 319 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:56:56.109 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:56 smithi133 bash[102858]: audit 2023-10-09T04:56:55.596818+0000 mon.a (mon.0) 320 : audit 1 from='client.? 172.21.15.133:0/3429714558' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T04:56:56.109 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:56 smithi133 bash[105589]: cluster 2023-10-09T04:56:54.696891+0000 osd.7 (osd.7) 6 : cluster 3 failed to encode map e87 with expected crc 2023-10-09T04:56:56.109 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:56 smithi133 bash[105589]: cluster 2023-10-09T04:56:54.992369+0000 mon.a (mon.0) 316 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:56:56.109 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:56 smithi133 bash[105589]: cluster 2023-10-09T04:56:54.996827+0000 osd.1 (osd.1) 9 : cluster 3 failed to encode map e88 with expected crc 2023-10-09T04:56:56.109 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:56 smithi133 bash[105589]: cluster 2023-10-09T04:56:55.000267+0000 mon.a (mon.0) 317 : cluster 1 osd.0 [v2:172.21.15.133:6802/2098404708,v1:172.21.15.133:6803/2098404708] boot 2023-10-09T04:56:56.109 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:56 smithi133 bash[105589]: cluster 2023-10-09T04:56:55.000351+0000 mon.a (mon.0) 318 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2023-10-09T04:56:56.109 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:56 smithi133 bash[105589]: audit 2023-10-09T04:56:55.000953+0000 mon.a (mon.0) 319 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:56:56.109 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:56 smithi133 bash[105589]: audit 2023-10-09T04:56:55.596818+0000 mon.a (mon.0) 320 : audit 1 from='client.? 172.21.15.133:0/3429714558' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T04:56:56.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:56 smithi194 bash[92666]: cluster 2023-10-09T04:56:54.696891+0000 osd.7 (osd.7) 6 : cluster 3 failed to encode map e87 with expected crc 2023-10-09T04:56:56.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:56 smithi194 bash[92666]: cluster 2023-10-09T04:56:54.992369+0000 mon.a (mon.0) 316 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:56:56.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:56 smithi194 bash[92666]: cluster 2023-10-09T04:56:54.996827+0000 osd.1 (osd.1) 9 : cluster 3 failed to encode map e88 with expected crc 2023-10-09T04:56:56.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:56 smithi194 bash[92666]: cluster 2023-10-09T04:56:55.000267+0000 mon.a (mon.0) 317 : cluster 1 osd.0 [v2:172.21.15.133:6802/2098404708,v1:172.21.15.133:6803/2098404708] boot 2023-10-09T04:56:56.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:56 smithi194 bash[92666]: cluster 2023-10-09T04:56:55.000351+0000 mon.a (mon.0) 318 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2023-10-09T04:56:56.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:56 smithi194 bash[92666]: audit 2023-10-09T04:56:55.000953+0000 mon.a (mon.0) 319 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:56:56.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:56 smithi194 bash[92666]: audit 2023-10-09T04:56:55.596818+0000 mon.a (mon.0) 320 : audit 1 from='client.? 172.21.15.133:0/3429714558' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T04:56:56.740 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T04:56:57.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[102858]: cluster 2023-10-09T04:56:54.997430+0000 osd.5 (osd.5) 10 : cluster 3 failed to encode map e88 with expected crc 2023-10-09T04:56:57.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[102858]: cluster 2023-10-09T04:56:55.427438+0000 mgr.y (mgr.25277) 206 : cluster 0 pgmap v150: 73 pgs: 5 peering, 5 active+undersized+degraded, 16 active+undersized, 47 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10/72 objects degraded (13.889%) 2023-10-09T04:56:57.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[102858]: audit 2023-10-09T04:56:55.616159+0000 mgr.y (mgr.25277) 207 : audit 0 from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:56:57.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[102858]: cluster 2023-10-09T04:56:56.041366+0000 mon.a (mon.0) 321 : cluster 3 Health check update: Degraded data redundancy: 10/72 objects degraded (13.889%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:56:57.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[102858]: cluster 2023-10-09T04:56:56.053157+0000 osd.7 (osd.7) 7 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:56:57.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[102858]: cluster 2023-10-09T04:56:56.056266+0000 osd.5 (osd.5) 11 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:56:57.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[102858]: audit 2023-10-09T04:56:56.057434+0000 mon.a (mon.0) 322 : audit 1 from='client.? 172.21.15.133:0/3429714558' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T04:56:57.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[102858]: cluster 2023-10-09T04:56:56.057513+0000 mon.a (mon.0) 323 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2023-10-09T04:56:57.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[102858]: cluster 2023-10-09T04:56:56.057733+0000 osd.2 (osd.2) 10 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:56:57.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[102858]: cluster 2023-10-09T04:56:56.057737+0000 osd.3 (osd.3) 8 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:56:57.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[102858]: cluster 2023-10-09T04:56:56.058212+0000 osd.7 (osd.7) 8 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:56:57.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[102858]: cluster 2023-10-09T04:56:56.058350+0000 osd.6 (osd.6) 6 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:56:57.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[102858]: cluster 2023-10-09T04:56:56.058690+0000 osd.4 (osd.4) 10 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:56:57.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[102858]: cluster 2023-10-09T04:56:56.059485+0000 osd.1 (osd.1) 10 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:56:57.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[102858]: cluster 2023-10-09T04:56:56.062752+0000 osd.4 (osd.4) 11 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:56:57.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[102858]: cluster 2023-10-09T04:56:56.064330+0000 osd.1 (osd.1) 11 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:56:57.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[105589]: cluster 2023-10-09T04:56:54.997430+0000 osd.5 (osd.5) 10 : cluster 3 failed to encode map e88 with expected crc 2023-10-09T04:56:57.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[105589]: cluster 2023-10-09T04:56:55.427438+0000 mgr.y (mgr.25277) 206 : cluster 0 pgmap v150: 73 pgs: 5 peering, 5 active+undersized+degraded, 16 active+undersized, 47 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10/72 objects degraded (13.889%) 2023-10-09T04:56:57.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[105589]: audit 2023-10-09T04:56:55.616159+0000 mgr.y (mgr.25277) 207 : audit 0 from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:56:57.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[105589]: cluster 2023-10-09T04:56:56.041366+0000 mon.a (mon.0) 321 : cluster 3 Health check update: Degraded data redundancy: 10/72 objects degraded (13.889%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:56:57.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[105589]: cluster 2023-10-09T04:56:56.053157+0000 osd.7 (osd.7) 7 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:56:57.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[105589]: cluster 2023-10-09T04:56:56.056266+0000 osd.5 (osd.5) 11 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:56:57.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[105589]: audit 2023-10-09T04:56:56.057434+0000 mon.a (mon.0) 322 : audit 1 from='client.? 172.21.15.133:0/3429714558' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T04:56:57.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[105589]: cluster 2023-10-09T04:56:56.057513+0000 mon.a (mon.0) 323 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2023-10-09T04:56:57.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[105589]: cluster 2023-10-09T04:56:56.057733+0000 osd.2 (osd.2) 10 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:56:57.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[105589]: cluster 2023-10-09T04:56:56.057737+0000 osd.3 (osd.3) 8 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:56:57.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[105589]: cluster 2023-10-09T04:56:56.058212+0000 osd.7 (osd.7) 8 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:56:57.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[105589]: cluster 2023-10-09T04:56:56.058350+0000 osd.6 (osd.6) 6 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:56:57.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[105589]: cluster 2023-10-09T04:56:56.058690+0000 osd.4 (osd.4) 10 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:56:57.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[105589]: cluster 2023-10-09T04:56:56.059485+0000 osd.1 (osd.1) 10 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:56:57.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[105589]: cluster 2023-10-09T04:56:56.062752+0000 osd.4 (osd.4) 11 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:56:57.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:57 smithi133 bash[105589]: cluster 2023-10-09T04:56:56.064330+0000 osd.1 (osd.1) 11 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:56:57.304 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2023-10-09T04:56:57.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:57 smithi194 bash[92666]: cluster 2023-10-09T04:56:54.997430+0000 osd.5 (osd.5) 10 : cluster 3 failed to encode map e88 with expected crc 2023-10-09T04:56:57.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:57 smithi194 bash[92666]: cluster 2023-10-09T04:56:55.427438+0000 mgr.y (mgr.25277) 206 : cluster 0 pgmap v150: 73 pgs: 5 peering, 5 active+undersized+degraded, 16 active+undersized, 47 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10/72 objects degraded (13.889%) 2023-10-09T04:56:57.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:57 smithi194 bash[92666]: audit 2023-10-09T04:56:55.616159+0000 mgr.y (mgr.25277) 207 : audit 0 from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:56:57.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:57 smithi194 bash[92666]: cluster 2023-10-09T04:56:56.041366+0000 mon.a (mon.0) 321 : cluster 3 Health check update: Degraded data redundancy: 10/72 objects degraded (13.889%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:56:57.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:57 smithi194 bash[92666]: cluster 2023-10-09T04:56:56.053157+0000 osd.7 (osd.7) 7 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:56:57.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:57 smithi194 bash[92666]: cluster 2023-10-09T04:56:56.056266+0000 osd.5 (osd.5) 11 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:56:57.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:57 smithi194 bash[92666]: audit 2023-10-09T04:56:56.057434+0000 mon.a (mon.0) 322 : audit 1 from='client.? 172.21.15.133:0/3429714558' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T04:56:57.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:57 smithi194 bash[92666]: cluster 2023-10-09T04:56:56.057513+0000 mon.a (mon.0) 323 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2023-10-09T04:56:57.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:57 smithi194 bash[92666]: cluster 2023-10-09T04:56:56.057733+0000 osd.2 (osd.2) 10 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:56:57.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:57 smithi194 bash[92666]: cluster 2023-10-09T04:56:56.057737+0000 osd.3 (osd.3) 8 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:56:57.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:57 smithi194 bash[92666]: cluster 2023-10-09T04:56:56.058212+0000 osd.7 (osd.7) 8 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:56:57.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:57 smithi194 bash[92666]: cluster 2023-10-09T04:56:56.058350+0000 osd.6 (osd.6) 6 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:56:57.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:57 smithi194 bash[92666]: cluster 2023-10-09T04:56:56.058690+0000 osd.4 (osd.4) 10 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:56:57.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:57 smithi194 bash[92666]: cluster 2023-10-09T04:56:56.059485+0000 osd.1 (osd.1) 10 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:56:57.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:57 smithi194 bash[92666]: cluster 2023-10-09T04:56:56.062752+0000 osd.4 (osd.4) 11 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:56:57.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:57 smithi194 bash[92666]: cluster 2023-10-09T04:56:56.064330+0000 osd.1 (osd.1) 11 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:56:58.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:58 smithi194 bash[92666]: cluster 2023-10-09T04:56:57.061202+0000 osd.2 (osd.2) 11 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:56:58.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:58 smithi194 bash[92666]: cluster 2023-10-09T04:56:57.062034+0000 osd.5 (osd.5) 12 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:56:58.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:58 smithi194 bash[92666]: cluster 2023-10-09T04:56:57.064225+0000 mon.a (mon.0) 324 : cluster 0 osdmap e90: 8 total, 8 up, 8 in 2023-10-09T04:56:58.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:58 smithi194 bash[92666]: cluster 2023-10-09T04:56:57.064337+0000 osd.2 (osd.2) 12 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:56:58.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:58 smithi194 bash[92666]: cluster 2023-10-09T04:56:57.064358+0000 osd.3 (osd.3) 9 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:56:58.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:58 smithi194 bash[92666]: cluster 2023-10-09T04:56:57.064849+0000 osd.1 (osd.1) 12 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:56:58.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:58 smithi194 bash[92666]: cluster 2023-10-09T04:56:57.064933+0000 osd.7 (osd.7) 9 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:56:58.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:58 smithi194 bash[92666]: cluster 2023-10-09T04:56:57.065085+0000 osd.6 (osd.6) 7 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:56:58.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:58 smithi194 bash[92666]: cluster 2023-10-09T04:56:57.065151+0000 osd.4 (osd.4) 12 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:56:58.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:58 smithi194 bash[92666]: cluster 2023-10-09T04:56:57.065344+0000 osd.5 (osd.5) 13 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:56:58.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:58 smithi194 bash[92666]: cluster 2023-10-09T04:56:57.428184+0000 mgr.y (mgr.25277) 208 : cluster 0 pgmap v153: 89 pgs: 16 unknown, 5 peering, 5 active+undersized+degraded, 16 active+undersized, 47 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10/72 objects degraded (13.889%) 2023-10-09T04:56:58.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:58 smithi133 bash[102858]: cluster 2023-10-09T04:56:57.061202+0000 osd.2 (osd.2) 11 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:56:58.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:58 smithi133 bash[102858]: cluster 2023-10-09T04:56:57.062034+0000 osd.5 (osd.5) 12 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:56:58.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:58 smithi133 bash[102858]: cluster 2023-10-09T04:56:57.064225+0000 mon.a (mon.0) 324 : cluster 0 osdmap e90: 8 total, 8 up, 8 in 2023-10-09T04:56:58.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:58 smithi133 bash[102858]: cluster 2023-10-09T04:56:57.064337+0000 osd.2 (osd.2) 12 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:56:58.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:58 smithi133 bash[102858]: cluster 2023-10-09T04:56:57.064358+0000 osd.3 (osd.3) 9 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:56:58.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:58 smithi133 bash[102858]: cluster 2023-10-09T04:56:57.064849+0000 osd.1 (osd.1) 12 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:56:58.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:58 smithi133 bash[102858]: cluster 2023-10-09T04:56:57.064933+0000 osd.7 (osd.7) 9 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:56:58.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:58 smithi133 bash[102858]: cluster 2023-10-09T04:56:57.065085+0000 osd.6 (osd.6) 7 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:56:58.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:58 smithi133 bash[102858]: cluster 2023-10-09T04:56:57.065151+0000 osd.4 (osd.4) 12 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:56:58.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:58 smithi133 bash[102858]: cluster 2023-10-09T04:56:57.065344+0000 osd.5 (osd.5) 13 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:56:58.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:58 smithi133 bash[102858]: cluster 2023-10-09T04:56:57.428184+0000 mgr.y (mgr.25277) 208 : cluster 0 pgmap v153: 89 pgs: 16 unknown, 5 peering, 5 active+undersized+degraded, 16 active+undersized, 47 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10/72 objects degraded (13.889%) 2023-10-09T04:56:58.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:58 smithi133 bash[105589]: cluster 2023-10-09T04:56:57.061202+0000 osd.2 (osd.2) 11 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:56:58.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:58 smithi133 bash[105589]: cluster 2023-10-09T04:56:57.062034+0000 osd.5 (osd.5) 12 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:56:58.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:58 smithi133 bash[105589]: cluster 2023-10-09T04:56:57.064225+0000 mon.a (mon.0) 324 : cluster 0 osdmap e90: 8 total, 8 up, 8 in 2023-10-09T04:56:58.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:58 smithi133 bash[105589]: cluster 2023-10-09T04:56:57.064337+0000 osd.2 (osd.2) 12 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:56:58.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:58 smithi133 bash[105589]: cluster 2023-10-09T04:56:57.064358+0000 osd.3 (osd.3) 9 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:56:58.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:58 smithi133 bash[105589]: cluster 2023-10-09T04:56:57.064849+0000 osd.1 (osd.1) 12 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:56:58.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:58 smithi133 bash[105589]: cluster 2023-10-09T04:56:57.064933+0000 osd.7 (osd.7) 9 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:56:58.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:58 smithi133 bash[105589]: cluster 2023-10-09T04:56:57.065085+0000 osd.6 (osd.6) 7 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:56:58.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:58 smithi133 bash[105589]: cluster 2023-10-09T04:56:57.065151+0000 osd.4 (osd.4) 12 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:56:58.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:58 smithi133 bash[105589]: cluster 2023-10-09T04:56:57.065344+0000 osd.5 (osd.5) 13 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:56:58.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:58 smithi133 bash[105589]: cluster 2023-10-09T04:56:57.428184+0000 mgr.y (mgr.25277) 208 : cluster 0 pgmap v153: 89 pgs: 16 unknown, 5 peering, 5 active+undersized+degraded, 16 active+undersized, 47 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10/72 objects degraded (13.889%) 2023-10-09T04:56:59.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:59 smithi133 bash[105589]: cluster 2023-10-09T04:56:58.069955+0000 osd.4 (osd.4) 13 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:56:59.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:59 smithi133 bash[105589]: cluster 2023-10-09T04:56:58.069997+0000 osd.5 (osd.5) 14 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:56:59.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:59 smithi133 bash[105589]: cluster 2023-10-09T04:56:58.072622+0000 mon.a (mon.0) 325 : cluster 0 osdmap e91: 8 total, 8 up, 8 in 2023-10-09T04:56:59.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:59 smithi133 bash[105589]: audit 2023-10-09T04:56:58.417757+0000 mon.a (mon.0) 326 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:59.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:56:59 smithi133 bash[105589]: audit 2023-10-09T04:56:58.419018+0000 mon.a (mon.0) 327 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:59.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:59 smithi133 bash[102858]: cluster 2023-10-09T04:56:58.069955+0000 osd.4 (osd.4) 13 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:56:59.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:59 smithi133 bash[102858]: cluster 2023-10-09T04:56:58.069997+0000 osd.5 (osd.5) 14 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:56:59.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:59 smithi133 bash[102858]: cluster 2023-10-09T04:56:58.072622+0000 mon.a (mon.0) 325 : cluster 0 osdmap e91: 8 total, 8 up, 8 in 2023-10-09T04:56:59.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:59 smithi133 bash[102858]: audit 2023-10-09T04:56:58.417757+0000 mon.a (mon.0) 326 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:59.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:56:59 smithi133 bash[102858]: audit 2023-10-09T04:56:58.419018+0000 mon.a (mon.0) 327 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:59.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:59 smithi194 bash[92666]: cluster 2023-10-09T04:56:58.069955+0000 osd.4 (osd.4) 13 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:56:59.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:59 smithi194 bash[92666]: cluster 2023-10-09T04:56:58.069997+0000 osd.5 (osd.5) 14 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:56:59.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:59 smithi194 bash[92666]: cluster 2023-10-09T04:56:58.072622+0000 mon.a (mon.0) 325 : cluster 0 osdmap e91: 8 total, 8 up, 8 in 2023-10-09T04:56:59.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:59 smithi194 bash[92666]: audit 2023-10-09T04:56:58.417757+0000 mon.a (mon.0) 326 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:56:59.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:56:59 smithi194 bash[92666]: audit 2023-10-09T04:56:58.419018+0000 mon.a (mon.0) 327 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:00.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:00 smithi133 bash[105589]: cluster 2023-10-09T04:56:59.428921+0000 mgr.y (mgr.25277) 209 : cluster 0 pgmap v155: 89 pgs: 12 unknown, 5 peering, 3 active+undersized+degraded, 7 active+undersized, 62 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7/72 objects degraded (9.722%) 2023-10-09T04:57:00.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:00 smithi133 bash[102858]: cluster 2023-10-09T04:56:59.428921+0000 mgr.y (mgr.25277) 209 : cluster 0 pgmap v155: 89 pgs: 12 unknown, 5 peering, 3 active+undersized+degraded, 7 active+undersized, 62 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7/72 objects degraded (9.722%) 2023-10-09T04:57:00.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:00 smithi194 bash[92666]: cluster 2023-10-09T04:56:59.428921+0000 mgr.y (mgr.25277) 209 : cluster 0 pgmap v155: 89 pgs: 12 unknown, 5 peering, 3 active+undersized+degraded, 7 active+undersized, 62 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7/72 objects degraded (9.722%) 2023-10-09T04:57:01.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:01 smithi133 bash[102858]: cluster 2023-10-09T04:57:01.481728+0000 mon.a (mon.0) 328 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/72 objects degraded (9.722%), 3 pgs degraded) 2023-10-09T04:57:01.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:01 smithi133 bash[102858]: cluster 2023-10-09T04:57:01.481783+0000 mon.a (mon.0) 329 : cluster 1 Cluster is now healthy 2023-10-09T04:57:01.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:01 smithi133 bash[105589]: cluster 2023-10-09T04:57:01.481728+0000 mon.a (mon.0) 328 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/72 objects degraded (9.722%), 3 pgs degraded) 2023-10-09T04:57:01.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:01 smithi133 bash[105589]: cluster 2023-10-09T04:57:01.481783+0000 mon.a (mon.0) 329 : cluster 1 Cluster is now healthy 2023-10-09T04:57:01.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:01 smithi194 bash[92666]: cluster 2023-10-09T04:57:01.481728+0000 mon.a (mon.0) 328 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/72 objects degraded (9.722%), 3 pgs degraded) 2023-10-09T04:57:01.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:01 smithi194 bash[92666]: cluster 2023-10-09T04:57:01.481783+0000 mon.a (mon.0) 329 : cluster 1 Cluster is now healthy 2023-10-09T04:57:02.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:02 smithi133 bash[102858]: cluster 2023-10-09T04:57:01.429987+0000 mgr.y (mgr.25277) 210 : cluster 0 pgmap v156: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:57:02.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:02 smithi133 bash[105589]: cluster 2023-10-09T04:57:01.429987+0000 mgr.y (mgr.25277) 210 : cluster 0 pgmap v156: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:57:02.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:02 smithi194 bash[92666]: cluster 2023-10-09T04:57:01.429987+0000 mgr.y (mgr.25277) 210 : cluster 0 pgmap v156: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:57:03.805 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:57:03.808 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:57:04.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:57:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:57:04] "GET /metrics HTTP/1.1" 200 36073 "" "Prometheus/2.43.0" 2023-10-09T04:57:04.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:04 smithi133 bash[102858]: audit 2023-10-09T04:57:03.414436+0000 mon.a (mon.0) 330 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:04.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:04 smithi133 bash[102858]: audit 2023-10-09T04:57:03.423969+0000 mon.a (mon.0) 331 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:04.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:04 smithi133 bash[102858]: audit 2023-10-09T04:57:03.425608+0000 mon.a (mon.0) 332 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:57:04.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:04 smithi133 bash[102858]: audit 2023-10-09T04:57:03.426932+0000 mon.a (mon.0) 333 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:57:04.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:04 smithi133 bash[102858]: cluster 2023-10-09T04:57:03.430579+0000 mgr.y (mgr.25277) 211 : cluster 0 pgmap v157: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:57:04.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:04 smithi133 bash[102858]: audit 2023-10-09T04:57:03.435051+0000 mon.a (mon.0) 334 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:04.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:04 smithi133 bash[102858]: audit 2023-10-09T04:57:03.519492+0000 mon.a (mon.0) 335 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:57:04.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:04 smithi133 bash[102858]: audit 2023-10-09T04:57:03.523194+0000 mon.a (mon.0) 336 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:57:04.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:04 smithi133 bash[102858]: audit 2023-10-09T04:57:03.525269+0000 mon.a (mon.0) 337 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:57:04.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:04 smithi133 bash[102858]: audit 2023-10-09T04:57:03.526925+0000 mon.a (mon.0) 338 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:57:04.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:04 smithi133 bash[102858]: audit 2023-10-09T04:57:03.529965+0000 mon.a (mon.0) 339 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:57:04.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:04 smithi133 bash[102858]: audit 2023-10-09T04:57:03.530248+0000 mgr.y (mgr.25277) 212 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:57:04.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:04 smithi133 bash[102858]: cephadm 2023-10-09T04:57:03.531766+0000 mgr.y (mgr.25277) 213 : cephadm 1 Upgrade: osd.1 is safe to restart 2023-10-09T04:57:04.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:04 smithi133 bash[105589]: audit 2023-10-09T04:57:03.414436+0000 mon.a (mon.0) 330 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:04.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:04 smithi133 bash[105589]: audit 2023-10-09T04:57:03.423969+0000 mon.a (mon.0) 331 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:04.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:04 smithi133 bash[105589]: audit 2023-10-09T04:57:03.425608+0000 mon.a (mon.0) 332 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:57:04.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:04 smithi133 bash[105589]: audit 2023-10-09T04:57:03.426932+0000 mon.a (mon.0) 333 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:57:04.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:04 smithi133 bash[105589]: cluster 2023-10-09T04:57:03.430579+0000 mgr.y (mgr.25277) 211 : cluster 0 pgmap v157: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:57:04.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:04 smithi133 bash[105589]: audit 2023-10-09T04:57:03.435051+0000 mon.a (mon.0) 334 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:04.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:04 smithi133 bash[105589]: audit 2023-10-09T04:57:03.519492+0000 mon.a (mon.0) 335 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:57:04.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:04 smithi133 bash[105589]: audit 2023-10-09T04:57:03.523194+0000 mon.a (mon.0) 336 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:57:04.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:04 smithi133 bash[105589]: audit 2023-10-09T04:57:03.525269+0000 mon.a (mon.0) 337 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:57:04.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:04 smithi133 bash[105589]: audit 2023-10-09T04:57:03.526925+0000 mon.a (mon.0) 338 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:57:04.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:04 smithi133 bash[105589]: audit 2023-10-09T04:57:03.529965+0000 mon.a (mon.0) 339 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:57:04.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:04 smithi133 bash[105589]: audit 2023-10-09T04:57:03.530248+0000 mgr.y (mgr.25277) 212 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:57:04.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:04 smithi133 bash[105589]: cephadm 2023-10-09T04:57:03.531766+0000 mgr.y (mgr.25277) 213 : cephadm 1 Upgrade: osd.1 is safe to restart 2023-10-09T04:57:04.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:04 smithi194 bash[92666]: audit 2023-10-09T04:57:03.414436+0000 mon.a (mon.0) 330 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:04.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:04 smithi194 bash[92666]: audit 2023-10-09T04:57:03.423969+0000 mon.a (mon.0) 331 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:04.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:04 smithi194 bash[92666]: audit 2023-10-09T04:57:03.425608+0000 mon.a (mon.0) 332 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:57:04.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:04 smithi194 bash[92666]: audit 2023-10-09T04:57:03.426932+0000 mon.a (mon.0) 333 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:57:04.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:04 smithi194 bash[92666]: cluster 2023-10-09T04:57:03.430579+0000 mgr.y (mgr.25277) 211 : cluster 0 pgmap v157: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:57:04.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:04 smithi194 bash[92666]: audit 2023-10-09T04:57:03.435051+0000 mon.a (mon.0) 334 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:04.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:04 smithi194 bash[92666]: audit 2023-10-09T04:57:03.519492+0000 mon.a (mon.0) 335 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:57:04.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:04 smithi194 bash[92666]: audit 2023-10-09T04:57:03.523194+0000 mon.a (mon.0) 336 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:57:04.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:04 smithi194 bash[92666]: audit 2023-10-09T04:57:03.525269+0000 mon.a (mon.0) 337 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:57:04.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:04 smithi194 bash[92666]: audit 2023-10-09T04:57:03.526925+0000 mon.a (mon.0) 338 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:57:04.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:04 smithi194 bash[92666]: audit 2023-10-09T04:57:03.529965+0000 mon.a (mon.0) 339 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:57:04.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:04 smithi194 bash[92666]: audit 2023-10-09T04:57:03.530248+0000 mgr.y (mgr.25277) 212 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:57:04.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:04 smithi194 bash[92666]: cephadm 2023-10-09T04:57:03.531766+0000 mgr.y (mgr.25277) 213 : cephadm 1 Upgrade: osd.1 is safe to restart 2023-10-09T04:57:06.655 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:06 smithi133 bash[102858]: cluster 2023-10-09T04:57:05.431821+0000 mgr.y (mgr.25277) 214 : cluster 0 pgmap v158: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:57:06.655 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:06 smithi133 bash[105589]: cluster 2023-10-09T04:57:05.431821+0000 mgr.y (mgr.25277) 214 : cluster 0 pgmap v158: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:57:06.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:06 smithi194 bash[92666]: cluster 2023-10-09T04:57:05.431821+0000 mgr.y (mgr.25277) 214 : cluster 0 pgmap v158: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:57:08.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:08 smithi133 bash[105589]: cluster 2023-10-09T04:57:07.432258+0000 mgr.y (mgr.25277) 215 : cluster 0 pgmap v159: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:57:08.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:08 smithi133 bash[105589]: audit 2023-10-09T04:57:08.219015+0000 mon.a (mon.0) 340 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:08.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:08 smithi133 bash[105589]: audit 2023-10-09T04:57:08.220586+0000 mon.a (mon.0) 341 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:57:08.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:08 smithi133 bash[105589]: audit 2023-10-09T04:57:08.221583+0000 mon.a (mon.0) 342 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:57:08.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:08 smithi133 bash[105589]: audit 2023-10-09T04:57:08.447127+0000 mon.a (mon.0) 343 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:08.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:08 smithi133 bash[102858]: cluster 2023-10-09T04:57:07.432258+0000 mgr.y (mgr.25277) 215 : cluster 0 pgmap v159: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:57:08.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:08 smithi133 bash[102858]: audit 2023-10-09T04:57:08.219015+0000 mon.a (mon.0) 340 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:08.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:08 smithi133 bash[102858]: audit 2023-10-09T04:57:08.220586+0000 mon.a (mon.0) 341 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:57:08.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:08 smithi133 bash[102858]: audit 2023-10-09T04:57:08.221583+0000 mon.a (mon.0) 342 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:57:08.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:08 smithi133 bash[102858]: audit 2023-10-09T04:57:08.447127+0000 mon.a (mon.0) 343 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:08.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:08 smithi194 bash[92666]: cluster 2023-10-09T04:57:07.432258+0000 mgr.y (mgr.25277) 215 : cluster 0 pgmap v159: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:57:08.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:08 smithi194 bash[92666]: audit 2023-10-09T04:57:08.219015+0000 mon.a (mon.0) 340 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:08.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:08 smithi194 bash[92666]: audit 2023-10-09T04:57:08.220586+0000 mon.a (mon.0) 341 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:57:08.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:08 smithi194 bash[92666]: audit 2023-10-09T04:57:08.221583+0000 mon.a (mon.0) 342 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:57:08.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:08 smithi194 bash[92666]: audit 2023-10-09T04:57:08.447127+0000 mon.a (mon.0) 343 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:09.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:09 smithi133 bash[102858]: cephadm 2023-10-09T04:57:08.206693+0000 mgr.y (mgr.25277) 216 : cephadm 1 Upgrade: Updating osd.1 2023-10-09T04:57:09.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:09 smithi133 bash[102858]: cephadm 2023-10-09T04:57:08.224254+0000 mgr.y (mgr.25277) 217 : cephadm 1 Deploying daemon osd.1 on smithi133 2023-10-09T04:57:09.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:09 smithi133 bash[105589]: cephadm 2023-10-09T04:57:08.206693+0000 mgr.y (mgr.25277) 216 : cephadm 1 Upgrade: Updating osd.1 2023-10-09T04:57:09.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:09 smithi133 bash[105589]: cephadm 2023-10-09T04:57:08.224254+0000 mgr.y (mgr.25277) 217 : cephadm 1 Deploying daemon osd.1 on smithi133 2023-10-09T04:57:09.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:09 smithi194 bash[92666]: cephadm 2023-10-09T04:57:08.206693+0000 mgr.y (mgr.25277) 216 : cephadm 1 Upgrade: Updating osd.1 2023-10-09T04:57:09.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:09 smithi194 bash[92666]: cephadm 2023-10-09T04:57:08.224254+0000 mgr.y (mgr.25277) 217 : cephadm 1 Deploying daemon osd.1 on smithi133 2023-10-09T04:57:10.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:10 smithi133 bash[105589]: cluster 2023-10-09T04:57:09.433028+0000 mgr.y (mgr.25277) 218 : cluster 0 pgmap v160: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:57:10.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:10 smithi133 bash[102858]: cluster 2023-10-09T04:57:09.433028+0000 mgr.y (mgr.25277) 218 : cluster 0 pgmap v160: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:57:10.849 INFO:teuthology.orchestra.run.smithi133.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:57:10.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:10 smithi194 bash[92666]: cluster 2023-10-09T04:57:09.433028+0000 mgr.y (mgr.25277) 218 : cluster 0 pgmap v160: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:57:11.523 INFO:teuthology.orchestra.run.smithi133.stderr:enabled application 'rados' on pool 'unique_pool_2' 2023-10-09T04:57:11.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:11 smithi133 bash[105589]: audit 2023-10-09T04:57:10.687636+0000 mon.a (mon.0) 344 : audit 1 from='client.? 172.21.15.133:0/3894780132' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:57:11.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:11 smithi133 bash[105589]: audit 2023-10-09T04:57:10.842091+0000 mgr.y (mgr.25277) 219 : audit 0 from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:57:11.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:11 smithi133 bash[105589]: cephadm 2023-10-09T04:57:10.842441+0000 mgr.y (mgr.25277) 220 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:57:11.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:11 smithi133 bash[105589]: audit 2023-10-09T04:57:10.846873+0000 mon.a (mon.0) 345 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:11.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:11 smithi133 bash[102858]: audit 2023-10-09T04:57:10.687636+0000 mon.a (mon.0) 344 : audit 1 from='client.? 172.21.15.133:0/3894780132' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:57:11.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:11 smithi133 bash[102858]: audit 2023-10-09T04:57:10.842091+0000 mgr.y (mgr.25277) 219 : audit 0 from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:57:11.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:11 smithi133 bash[102858]: cephadm 2023-10-09T04:57:10.842441+0000 mgr.y (mgr.25277) 220 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:57:11.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:11 smithi133 bash[102858]: audit 2023-10-09T04:57:10.846873+0000 mon.a (mon.0) 345 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:11.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:11 smithi194 bash[92666]: audit 2023-10-09T04:57:10.687636+0000 mon.a (mon.0) 344 : audit 1 from='client.? 172.21.15.133:0/3894780132' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:57:11.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:11 smithi194 bash[92666]: audit 2023-10-09T04:57:10.842091+0000 mgr.y (mgr.25277) 219 : audit 0 from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:57:11.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:11 smithi194 bash[92666]: cephadm 2023-10-09T04:57:10.842441+0000 mgr.y (mgr.25277) 220 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:57:11.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:11 smithi194 bash[92666]: audit 2023-10-09T04:57:10.846873+0000 mon.a (mon.0) 345 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:12.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:12 smithi133 bash[105589]: cluster 2023-10-09T04:57:11.434159+0000 mgr.y (mgr.25277) 221 : cluster 0 pgmap v161: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:57:12.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:12 smithi133 bash[105589]: cluster 2023-10-09T04:57:11.511991+0000 osd.1 (osd.1) 13 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T04:57:12.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:12 smithi133 bash[105589]: cluster 2023-10-09T04:57:11.513240+0000 osd.5 (osd.5) 15 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T04:57:12.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:12 smithi133 bash[105589]: audit 2023-10-09T04:57:11.516585+0000 mon.a (mon.0) 346 : audit 1 from='client.? 172.21.15.133:0/3894780132' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:57:12.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:12 smithi133 bash[105589]: cluster 2023-10-09T04:57:11.516723+0000 mon.a (mon.0) 347 : cluster 0 osdmap e92: 8 total, 8 up, 8 in 2023-10-09T04:57:12.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:12 smithi133 bash[102858]: cluster 2023-10-09T04:57:11.434159+0000 mgr.y (mgr.25277) 221 : cluster 0 pgmap v161: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:57:12.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:12 smithi133 bash[102858]: cluster 2023-10-09T04:57:11.511991+0000 osd.1 (osd.1) 13 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T04:57:12.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:12 smithi133 bash[102858]: cluster 2023-10-09T04:57:11.513240+0000 osd.5 (osd.5) 15 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T04:57:12.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:12 smithi133 bash[102858]: audit 2023-10-09T04:57:11.516585+0000 mon.a (mon.0) 346 : audit 1 from='client.? 172.21.15.133:0/3894780132' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:57:12.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:12 smithi133 bash[102858]: cluster 2023-10-09T04:57:11.516723+0000 mon.a (mon.0) 347 : cluster 0 osdmap e92: 8 total, 8 up, 8 in 2023-10-09T04:57:12.886 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T04:57:12.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:12 smithi194 bash[92666]: cluster 2023-10-09T04:57:11.434159+0000 mgr.y (mgr.25277) 221 : cluster 0 pgmap v161: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:57:12.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:12 smithi194 bash[92666]: cluster 2023-10-09T04:57:11.511991+0000 osd.1 (osd.1) 13 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T04:57:12.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:12 smithi194 bash[92666]: cluster 2023-10-09T04:57:11.513240+0000 osd.5 (osd.5) 15 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T04:57:12.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:12 smithi194 bash[92666]: audit 2023-10-09T04:57:11.516585+0000 mon.a (mon.0) 346 : audit 1 from='client.? 172.21.15.133:0/3894780132' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:57:12.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:12 smithi194 bash[92666]: cluster 2023-10-09T04:57:11.516723+0000 mon.a (mon.0) 347 : cluster 0 osdmap e92: 8 total, 8 up, 8 in 2023-10-09T04:57:14.361 DEBUG:teuthology.orchestra.run.smithi133:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2023-10-09T04:57:14.364 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T04:57:14.364 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T04:57:14.404 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:hints = 1 2023-10-09T04:57:14.404 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T04:57:14.404 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Object prefix: benchmark_data_smithi133_119345 2023-10-09T04:57:14.405 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:57:14.405 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 0 0 0 0 0 0 - 0 2023-10-09T04:57:14.430 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:14 smithi133 bash[102858]: audit 2023-10-09T04:57:13.420743+0000 mon.a (mon.0) 348 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:14.430 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:14 smithi133 bash[102858]: audit 2023-10-09T04:57:13.422115+0000 mon.a (mon.0) 349 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:14.430 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:14 smithi133 bash[102858]: cluster 2023-10-09T04:57:13.437925+0000 mgr.y (mgr.25277) 222 : cluster 0 pgmap v163: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:57:14.431 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:57:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:57:14] "GET /metrics HTTP/1.1" 200 36222 "" "Prometheus/2.43.0" 2023-10-09T04:57:14.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:14 smithi133 bash[105589]: audit 2023-10-09T04:57:13.420743+0000 mon.a (mon.0) 348 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:14.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:14 smithi133 bash[105589]: audit 2023-10-09T04:57:13.422115+0000 mon.a (mon.0) 349 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:14.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:14 smithi133 bash[105589]: cluster 2023-10-09T04:57:13.437925+0000 mgr.y (mgr.25277) 222 : cluster 0 pgmap v163: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:57:14.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:14 smithi194 bash[92666]: audit 2023-10-09T04:57:13.420743+0000 mon.a (mon.0) 348 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:14.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:14 smithi194 bash[92666]: audit 2023-10-09T04:57:13.422115+0000 mon.a (mon.0) 349 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:14.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:14 smithi194 bash[92666]: cluster 2023-10-09T04:57:13.437925+0000 mgr.y (mgr.25277) 222 : cluster 0 pgmap v163: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:57:15.405 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 1 16 2868 2852 178.24 178.25 0.00600302 0.00558484 2023-10-09T04:57:15.799 INFO:journalctl@ceph.osd.1.smithi133.stdout:Oct 09 04:57:15 smithi133 systemd[1]: Stopping Ceph osd.1 for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T04:57:15.800 INFO:journalctl@ceph.osd.1.smithi133.stdout:Oct 09 04:57:15 smithi133 bash[51683]: debug 2023-10-09T04:57:15.624+0000 7fb77ec1c700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T04:57:15.800 INFO:journalctl@ceph.osd.1.smithi133.stdout:Oct 09 04:57:15 smithi133 bash[51683]: debug 2023-10-09T04:57:15.624+0000 7fb77ec1c700 -1 osd.1 92 *** Got signal Terminated *** 2023-10-09T04:57:15.800 INFO:journalctl@ceph.osd.1.smithi133.stdout:Oct 09 04:57:15 smithi133 bash[51683]: debug 2023-10-09T04:57:15.624+0000 7fb77ec1c700 -1 osd.1 92 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T04:57:16.405 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 2 16 5676 5660 176.862 175.5 0.00278286 0.00563866 2023-10-09T04:57:16.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:16 smithi133 bash[102858]: cluster 2023-10-09T04:57:15.438685+0000 mgr.y (mgr.25277) 223 : cluster 0 pgmap v164: 89 pgs: 89 active+clean; 51 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s wr, 79 op/s 2023-10-09T04:57:16.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:16 smithi133 bash[102858]: cluster 2023-10-09T04:57:15.625986+0000 mon.a (mon.0) 350 : cluster 1 osd.1 marked itself down and dead 2023-10-09T04:57:16.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:16 smithi133 bash[105589]: cluster 2023-10-09T04:57:15.438685+0000 mgr.y (mgr.25277) 223 : cluster 0 pgmap v164: 89 pgs: 89 active+clean; 51 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s wr, 79 op/s 2023-10-09T04:57:16.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:16 smithi133 bash[105589]: cluster 2023-10-09T04:57:15.625986+0000 mon.a (mon.0) 350 : cluster 1 osd.1 marked itself down and dead 2023-10-09T04:57:16.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:16 smithi194 bash[92666]: cluster 2023-10-09T04:57:15.438685+0000 mgr.y (mgr.25277) 223 : cluster 0 pgmap v164: 89 pgs: 89 active+clean; 51 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s wr, 79 op/s 2023-10-09T04:57:16.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:16 smithi194 bash[92666]: cluster 2023-10-09T04:57:15.625986+0000 mon.a (mon.0) 350 : cluster 1 osd.1 marked itself down and dead 2023-10-09T04:57:17.406 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 3 16 5989 5973 124.425 19.5625 0.0022084 0.00567174 2023-10-09T04:57:17.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:17 smithi133 bash[102858]: cluster 2023-10-09T04:57:16.495873+0000 mon.a (mon.0) 351 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:57:17.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:17 smithi133 bash[102858]: cluster 2023-10-09T04:57:16.503843+0000 osd.5 (osd.5) 16 : cluster 3 failed to encode map e93 with expected crc 2023-10-09T04:57:17.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:17 smithi133 bash[102858]: cluster 2023-10-09T04:57:16.504561+0000 osd.6 (osd.6) 8 : cluster 3 failed to encode map e93 with expected crc 2023-10-09T04:57:17.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:17 smithi133 bash[102858]: cluster 2023-10-09T04:57:16.507206+0000 mon.a (mon.0) 352 : cluster 0 osdmap e93: 8 total, 7 up, 8 in 2023-10-09T04:57:17.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:17 smithi133 bash[105589]: cluster 2023-10-09T04:57:16.495873+0000 mon.a (mon.0) 351 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:57:17.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:17 smithi133 bash[105589]: cluster 2023-10-09T04:57:16.503843+0000 osd.5 (osd.5) 16 : cluster 3 failed to encode map e93 with expected crc 2023-10-09T04:57:17.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:17 smithi133 bash[105589]: cluster 2023-10-09T04:57:16.504561+0000 osd.6 (osd.6) 8 : cluster 3 failed to encode map e93 with expected crc 2023-10-09T04:57:17.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:17 smithi133 bash[105589]: cluster 2023-10-09T04:57:16.507206+0000 mon.a (mon.0) 352 : cluster 0 osdmap e93: 8 total, 7 up, 8 in 2023-10-09T04:57:17.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:17 smithi194 bash[92666]: cluster 2023-10-09T04:57:16.495873+0000 mon.a (mon.0) 351 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:57:17.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:17 smithi194 bash[92666]: cluster 2023-10-09T04:57:16.503843+0000 osd.5 (osd.5) 16 : cluster 3 failed to encode map e93 with expected crc 2023-10-09T04:57:17.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:17 smithi194 bash[92666]: cluster 2023-10-09T04:57:16.504561+0000 osd.6 (osd.6) 8 : cluster 3 failed to encode map e93 with expected crc 2023-10-09T04:57:17.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:17 smithi194 bash[92666]: cluster 2023-10-09T04:57:16.507206+0000 mon.a (mon.0) 352 : cluster 0 osdmap e93: 8 total, 7 up, 8 in 2023-10-09T04:57:18.405 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 4 16 8807 8791 137.346 176.125 0.0026139 0.007272 2023-10-09T04:57:18.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:18 smithi133 bash[105589]: cluster 2023-10-09T04:57:17.439250+0000 mgr.y (mgr.25277) 224 : cluster 0 pgmap v166: 89 pgs: 9 stale+active+clean, 80 active+clean; 51 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s wr, 99 op/s 2023-10-09T04:57:18.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:18 smithi133 bash[105589]: cluster 2023-10-09T04:57:17.507669+0000 osd.2 (osd.2) 13 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:57:18.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:18 smithi133 bash[105589]: cluster 2023-10-09T04:57:17.508948+0000 osd.5 (osd.5) 17 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:57:18.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:18 smithi133 bash[105589]: cluster 2023-10-09T04:57:17.510211+0000 mon.a (mon.0) 353 : cluster 0 osdmap e94: 8 total, 7 up, 8 in 2023-10-09T04:57:18.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:18 smithi133 bash[105589]: cluster 2023-10-09T04:57:17.510440+0000 osd.2 (osd.2) 14 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:57:18.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:18 smithi133 bash[105589]: cluster 2023-10-09T04:57:17.510671+0000 osd.3 (osd.3) 10 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:57:18.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:18 smithi133 bash[105589]: cluster 2023-10-09T04:57:17.511058+0000 osd.7 (osd.7) 10 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:57:18.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:18 smithi133 bash[105589]: cluster 2023-10-09T04:57:17.511058+0000 osd.4 (osd.4) 14 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:57:18.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:18 smithi133 bash[105589]: cluster 2023-10-09T04:57:17.511139+0000 osd.6 (osd.6) 9 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:57:18.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:18 smithi133 bash[105589]: cluster 2023-10-09T04:57:17.512213+0000 osd.5 (osd.5) 18 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:57:18.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:18 smithi133 bash[102858]: cluster 2023-10-09T04:57:17.439250+0000 mgr.y (mgr.25277) 224 : cluster 0 pgmap v166: 89 pgs: 9 stale+active+clean, 80 active+clean; 51 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s wr, 99 op/s 2023-10-09T04:57:18.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:18 smithi133 bash[102858]: cluster 2023-10-09T04:57:17.507669+0000 osd.2 (osd.2) 13 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:57:18.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:18 smithi133 bash[102858]: cluster 2023-10-09T04:57:17.508948+0000 osd.5 (osd.5) 17 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:57:18.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:18 smithi133 bash[102858]: cluster 2023-10-09T04:57:17.510211+0000 mon.a (mon.0) 353 : cluster 0 osdmap e94: 8 total, 7 up, 8 in 2023-10-09T04:57:18.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:18 smithi133 bash[102858]: cluster 2023-10-09T04:57:17.510440+0000 osd.2 (osd.2) 14 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:57:18.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:18 smithi133 bash[102858]: cluster 2023-10-09T04:57:17.510671+0000 osd.3 (osd.3) 10 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:57:18.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:18 smithi133 bash[102858]: cluster 2023-10-09T04:57:17.511058+0000 osd.7 (osd.7) 10 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:57:18.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:18 smithi133 bash[102858]: cluster 2023-10-09T04:57:17.511058+0000 osd.4 (osd.4) 14 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:57:18.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:18 smithi133 bash[102858]: cluster 2023-10-09T04:57:17.511139+0000 osd.6 (osd.6) 9 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:57:18.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:18 smithi133 bash[102858]: cluster 2023-10-09T04:57:17.512213+0000 osd.5 (osd.5) 18 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:57:18.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:18 smithi194 bash[92666]: cluster 2023-10-09T04:57:17.439250+0000 mgr.y (mgr.25277) 224 : cluster 0 pgmap v166: 89 pgs: 9 stale+active+clean, 80 active+clean; 51 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s wr, 99 op/s 2023-10-09T04:57:18.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:18 smithi194 bash[92666]: cluster 2023-10-09T04:57:17.507669+0000 osd.2 (osd.2) 13 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:57:18.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:18 smithi194 bash[92666]: cluster 2023-10-09T04:57:17.508948+0000 osd.5 (osd.5) 17 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:57:18.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:18 smithi194 bash[92666]: cluster 2023-10-09T04:57:17.510211+0000 mon.a (mon.0) 353 : cluster 0 osdmap e94: 8 total, 7 up, 8 in 2023-10-09T04:57:18.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:18 smithi194 bash[92666]: cluster 2023-10-09T04:57:17.510440+0000 osd.2 (osd.2) 14 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:57:18.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:18 smithi194 bash[92666]: cluster 2023-10-09T04:57:17.510671+0000 osd.3 (osd.3) 10 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:57:18.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:18 smithi194 bash[92666]: cluster 2023-10-09T04:57:17.511058+0000 osd.7 (osd.7) 10 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:57:18.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:18 smithi194 bash[92666]: cluster 2023-10-09T04:57:17.511058+0000 osd.4 (osd.4) 14 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:57:18.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:18 smithi194 bash[92666]: cluster 2023-10-09T04:57:17.511139+0000 osd.6 (osd.6) 9 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:57:18.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:18 smithi194 bash[92666]: cluster 2023-10-09T04:57:17.512213+0000 osd.5 (osd.5) 18 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:57:19.406 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 5 16 10852 10836 135.437 127.812 0.00204585 0.00736938 2023-10-09T04:57:19.457 INFO:journalctl@ceph.osd.1.smithi133.stdout:Oct 09 04:57:19 smithi133 bash[119421]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd-1 2023-10-09T04:57:19.457 INFO:journalctl@ceph.osd.1.smithi133.stdout:Oct 09 04:57:19 smithi133 bash[119494]: Error response from daemon: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd-1 2023-10-09T04:57:19.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:19 smithi133 bash[105589]: cluster 2023-10-09T04:57:19.508638+0000 mon.a (mon.0) 354 : cluster 3 Health check failed: Degraded data redundancy: 1214/9717 objects degraded (12.494%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:19.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:19 smithi133 bash[102858]: cluster 2023-10-09T04:57:19.508638+0000 mon.a (mon.0) 354 : cluster 3 Health check failed: Degraded data redundancy: 1214/9717 objects degraded (12.494%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:19.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:19 smithi194 bash[92666]: cluster 2023-10-09T04:57:19.508638+0000 mon.a (mon.0) 354 : cluster 3 Health check failed: Degraded data redundancy: 1214/9717 objects degraded (12.494%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:19.892 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:57:20.406 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 6 16 12517 12501 130.206 104.062 0.00358294 0.00767308 2023-10-09T04:57:20.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:20 smithi133 bash[102858]: cluster 2023-10-09T04:57:19.440038+0000 mgr.y (mgr.25277) 225 : cluster 0 pgmap v168: 89 pgs: 3 active+undersized+degraded, 8 stale+active+clean, 3 active+undersized, 75 active+clean; 202 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s wr, 405 op/s; 1214/9717 objects degraded (12.494%) 2023-10-09T04:57:20.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:20 smithi133 bash[105589]: cluster 2023-10-09T04:57:19.440038+0000 mgr.y (mgr.25277) 225 : cluster 0 pgmap v168: 89 pgs: 3 active+undersized+degraded, 8 stale+active+clean, 3 active+undersized, 75 active+clean; 202 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s wr, 405 op/s; 1214/9717 objects degraded (12.494%) 2023-10-09T04:57:20.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:20 smithi194 bash[92666]: cluster 2023-10-09T04:57:19.440038+0000 mgr.y (mgr.25277) 225 : cluster 0 pgmap v168: 89 pgs: 3 active+undersized+degraded, 8 stale+active+clean, 3 active+undersized, 75 active+clean; 202 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s wr, 405 op/s; 1214/9717 objects degraded (12.494%) 2023-10-09T04:57:21.405 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 7 16 15288 15272 136.344 173.188 0.00580549 0.00732739 2023-10-09T04:57:22.406 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 8 16 17483 17467 136.448 137.188 0.00657759 0.00732186 2023-10-09T04:57:22.868 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:22 smithi133 bash[102858]: cluster 2023-10-09T04:57:21.440930+0000 mgr.y (mgr.25277) 226 : cluster 0 pgmap v169: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 695 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 4065/33396 objects degraded (12.172%) 2023-10-09T04:57:22.868 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:22 smithi133 bash[105589]: cluster 2023-10-09T04:57:21.440930+0000 mgr.y (mgr.25277) 226 : cluster 0 pgmap v169: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 695 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 4065/33396 objects degraded (12.172%) 2023-10-09T04:57:22.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:22 smithi194 bash[92666]: cluster 2023-10-09T04:57:21.440930+0000 mgr.y (mgr.25277) 226 : cluster 0 pgmap v169: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 695 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 4065/33396 objects degraded (12.172%) 2023-10-09T04:57:23.406 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 9 16 19656 19640 136.376 135.812 0.00498165 0.00732545 2023-10-09T04:57:24.282 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:57:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:57:24] "GET /metrics HTTP/1.1" 200 36222 "" "Prometheus/2.43.0" 2023-10-09T04:57:24.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:24 smithi133 bash[105589]: cluster 2023-10-09T04:57:23.441498+0000 mgr.y (mgr.25277) 227 : cluster 0 pgmap v170: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 695 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s; 4065/33396 objects degraded (12.172%) 2023-10-09T04:57:24.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:23 smithi133 bash[102858]: cluster 2023-10-09T04:57:23.441498+0000 mgr.y (mgr.25277) 227 : cluster 0 pgmap v170: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 695 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s; 4065/33396 objects degraded (12.172%) 2023-10-09T04:57:24.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:24 smithi194 bash[92666]: cluster 2023-10-09T04:57:23.441498+0000 mgr.y (mgr.25277) 227 : cluster 0 pgmap v170: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 695 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s; 4065/33396 objects degraded (12.172%) 2023-10-09T04:57:24.406 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 10 16 22137 22121 138.243 155.062 0.00436077 0.0072277 2023-10-09T04:57:25.406 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 11 16 24345 24329 138.22 138 0.00511926 0.00722698 2023-10-09T04:57:25.549 INFO:journalctl@ceph.osd.1.smithi133.stdout:Oct 09 04:57:25 smithi133 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@osd.1.service: Succeeded. 2023-10-09T04:57:25.550 INFO:journalctl@ceph.osd.1.smithi133.stdout:Oct 09 04:57:25 smithi133 systemd[1]: Stopped Ceph osd.1 for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:57:25.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:25 smithi133 bash[105589]: cluster 2023-10-09T04:57:25.495723+0000 mon.a (mon.0) 355 : cluster 3 Health check update: Degraded data redundancy: 8321/67626 objects degraded (12.304%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:25.697 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:25 smithi133 bash[102858]: cluster 2023-10-09T04:57:25.495723+0000 mon.a (mon.0) 355 : cluster 3 Health check update: Degraded data redundancy: 8321/67626 objects degraded (12.304%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:25.697 INFO:journalctl@ceph.osd.1.smithi133.stdout:Oct 09 04:57:25 smithi133 systemd[1]: Started Ceph osd.1 for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T04:57:25.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:25 smithi194 bash[92666]: cluster 2023-10-09T04:57:25.495723+0000 mon.a (mon.0) 355 : cluster 3 Health check update: Degraded data redundancy: 8321/67626 objects degraded (12.304%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:26.406 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 12 16 26151 26135 136.107 112.875 0.0027228 0.00734149 2023-10-09T04:57:27.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:26 smithi133 bash[105589]: cluster 2023-10-09T04:57:25.442860+0000 mgr.y (mgr.25277) 228 : cluster 0 pgmap v171: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 1.4 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 152 MiB/s wr, 2.43k op/s; 8321/67626 objects degraded (12.304%) 2023-10-09T04:57:27.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:26 smithi133 bash[105589]: audit 2023-10-09T04:57:25.662797+0000 mon.a (mon.0) 356 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:27.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:26 smithi133 bash[105589]: audit 2023-10-09T04:57:25.675043+0000 mon.a (mon.0) 357 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:27.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:26 smithi133 bash[105589]: audit 2023-10-09T04:57:25.691282+0000 mon.a (mon.0) 358 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:57:27.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:26 smithi133 bash[102858]: cluster 2023-10-09T04:57:25.442860+0000 mgr.y (mgr.25277) 228 : cluster 0 pgmap v171: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 1.4 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 152 MiB/s wr, 2.43k op/s; 8321/67626 objects degraded (12.304%) 2023-10-09T04:57:27.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:26 smithi133 bash[102858]: audit 2023-10-09T04:57:25.662797+0000 mon.a (mon.0) 356 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:27.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:26 smithi133 bash[102858]: audit 2023-10-09T04:57:25.675043+0000 mon.a (mon.0) 357 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:27.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:26 smithi133 bash[102858]: audit 2023-10-09T04:57:25.691282+0000 mon.a (mon.0) 358 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:57:27.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:26 smithi194 bash[92666]: cluster 2023-10-09T04:57:25.442860+0000 mgr.y (mgr.25277) 228 : cluster 0 pgmap v171: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 1.4 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 152 MiB/s wr, 2.43k op/s; 8321/67626 objects degraded (12.304%) 2023-10-09T04:57:27.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:26 smithi194 bash[92666]: audit 2023-10-09T04:57:25.662797+0000 mon.a (mon.0) 356 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:27.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:26 smithi194 bash[92666]: audit 2023-10-09T04:57:25.675043+0000 mon.a (mon.0) 357 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:27.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:26 smithi194 bash[92666]: audit 2023-10-09T04:57:25.691282+0000 mon.a (mon.0) 358 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:57:27.406 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 13 16 28319 28303 136.06 135.5 0.00467383 0.00734476 2023-10-09T04:57:28.407 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 14 16 30395 30379 135.598 129.75 0.00432445 0.00736872 2023-10-09T04:57:29.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:28 smithi133 bash[105589]: cluster 2023-10-09T04:57:27.443447+0000 mgr.y (mgr.25277) 229 : cluster 0 pgmap v172: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 1.4 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 136 MiB/s wr, 2.17k op/s; 8321/67626 objects degraded (12.304%) 2023-10-09T04:57:29.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:28 smithi133 bash[105589]: audit 2023-10-09T04:57:28.422134+0000 mon.a (mon.0) 359 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:29.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:28 smithi133 bash[105589]: audit 2023-10-09T04:57:28.423856+0000 mon.a (mon.0) 360 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:29.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:28 smithi133 bash[102858]: cluster 2023-10-09T04:57:27.443447+0000 mgr.y (mgr.25277) 229 : cluster 0 pgmap v172: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 1.4 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 136 MiB/s wr, 2.17k op/s; 8321/67626 objects degraded (12.304%) 2023-10-09T04:57:29.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:28 smithi133 bash[102858]: audit 2023-10-09T04:57:28.422134+0000 mon.a (mon.0) 359 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:29.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:28 smithi133 bash[102858]: audit 2023-10-09T04:57:28.423856+0000 mon.a (mon.0) 360 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:29.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:28 smithi194 bash[92666]: cluster 2023-10-09T04:57:27.443447+0000 mgr.y (mgr.25277) 229 : cluster 0 pgmap v172: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 1.4 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 136 MiB/s wr, 2.17k op/s; 8321/67626 objects degraded (12.304%) 2023-10-09T04:57:29.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:28 smithi194 bash[92666]: audit 2023-10-09T04:57:28.422134+0000 mon.a (mon.0) 359 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:29.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:28 smithi194 bash[92666]: audit 2023-10-09T04:57:28.423856+0000 mon.a (mon.0) 360 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:29.407 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 15 16 32154 32138 133.885 109.938 0.00752564 0.00746232 2023-10-09T04:57:29.609 INFO:journalctl@ceph.osd.1.smithi133.stdout:Oct 09 04:57:29 smithi133 bash[119728]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T04:57:29.609 INFO:journalctl@ceph.osd.1.smithi133.stdout:Oct 09 04:57:29 smithi133 bash[119728]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-09T04:57:29.610 INFO:journalctl@ceph.osd.1.smithi133.stdout:Oct 09 04:57:29 smithi133 bash[119728]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-10-09T04:57:29.610 INFO:journalctl@ceph.osd.1.smithi133.stdout:Oct 09 04:57:29 smithi133 bash[119728]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-10-09T04:57:29.610 INFO:journalctl@ceph.osd.1.smithi133.stdout:Oct 09 04:57:29 smithi133 bash[119728]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-10-09T04:57:29.610 INFO:journalctl@ceph.osd.1.smithi133.stdout:Oct 09 04:57:29 smithi133 bash[119728]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-10-09T04:57:29.610 INFO:journalctl@ceph.osd.1.smithi133.stdout:Oct 09 04:57:29 smithi133 bash[119728]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-09T04:57:29.610 INFO:journalctl@ceph.osd.1.smithi133.stdout:Oct 09 04:57:29 smithi133 bash[119728]: --> ceph-volume lvm activate successful for osd ID: 1 2023-10-09T04:57:30.408 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 16 16 34115 34099 133.177 122.562 0.00598671 0.00750254 2023-10-09T04:57:30.973 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:30 smithi133 bash[102858]: cluster 2023-10-09T04:57:29.444030+0000 mgr.y (mgr.25277) 230 : cluster 0 pgmap v173: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 1.5 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s; 9571/75213 objects degraded (12.725%) 2023-10-09T04:57:30.973 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:30 smithi133 bash[105589]: cluster 2023-10-09T04:57:29.444030+0000 mgr.y (mgr.25277) 230 : cluster 0 pgmap v173: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 1.5 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s; 9571/75213 objects degraded (12.725%) 2023-10-09T04:57:31.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:30 smithi194 bash[92666]: cluster 2023-10-09T04:57:29.444030+0000 mgr.y (mgr.25277) 230 : cluster 0 pgmap v173: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 1.5 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s; 9571/75213 objects degraded (12.725%) 2023-10-09T04:57:31.408 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 17 16 35939 35923 132.046 114 0.00702439 0.00756719 2023-10-09T04:57:32.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:31 smithi133 bash[105589]: cluster 2023-10-09T04:57:31.679972+0000 mon.a (mon.0) 361 : cluster 3 Health check update: Degraded data redundancy: 12033/97563 objects degraded (12.334%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:32.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:31 smithi133 bash[102858]: cluster 2023-10-09T04:57:31.679972+0000 mon.a (mon.0) 361 : cluster 3 Health check update: Degraded data redundancy: 12033/97563 objects degraded (12.334%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:32.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:31 smithi194 bash[92666]: cluster 2023-10-09T04:57:31.679972+0000 mon.a (mon.0) 361 : cluster 3 Health check update: Degraded data redundancy: 12033/97563 objects degraded (12.334%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:32.408 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 18 16 37355 37339 129.627 88.5 0.00873081 0.00770868 2023-10-09T04:57:32.741 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:32 smithi133 bash[102858]: cluster 2023-10-09T04:57:31.445587+0000 mgr.y (mgr.25277) 231 : cluster 0 pgmap v174: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 2.0 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 152 MiB/s wr, 2.44k op/s; 12033/97563 objects degraded (12.334%) 2023-10-09T04:57:32.741 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:32 smithi133 bash[102858]: audit 2023-10-09T04:57:32.335923+0000 mon.a (mon.0) 362 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:32.741 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:32 smithi133 bash[105589]: cluster 2023-10-09T04:57:31.445587+0000 mgr.y (mgr.25277) 231 : cluster 0 pgmap v174: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 2.0 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 152 MiB/s wr, 2.44k op/s; 12033/97563 objects degraded (12.334%) 2023-10-09T04:57:32.741 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:32 smithi133 bash[105589]: audit 2023-10-09T04:57:32.335923+0000 mon.a (mon.0) 362 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:32.741 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:32 smithi133 bash[105589]: audit 2023-10-09T04:57:32.354897+0000 mon.a (mon.0) 363 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:33.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:32 smithi133 bash[102858]: audit 2023-10-09T04:57:32.354897+0000 mon.a (mon.0) 363 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:33.050 INFO:journalctl@ceph.osd.1.smithi133.stdout:Oct 09 04:57:32 smithi133 bash[120074]: debug 2023-10-09T04:57:32.979+0000 7f9d44f1c640 -1 Falling back to public interface 2023-10-09T04:57:33.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:32 smithi194 bash[92666]: cluster 2023-10-09T04:57:31.445587+0000 mgr.y (mgr.25277) 231 : cluster 0 pgmap v174: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 2.0 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 152 MiB/s wr, 2.44k op/s; 12033/97563 objects degraded (12.334%) 2023-10-09T04:57:33.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:32 smithi194 bash[92666]: audit 2023-10-09T04:57:32.335923+0000 mon.a (mon.0) 362 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:33.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:32 smithi194 bash[92666]: audit 2023-10-09T04:57:32.354897+0000 mon.a (mon.0) 363 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:33.408 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 19 16 38818 38802 127.615 91.4375 0.0131982 0.00782915 2023-10-09T04:57:34.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:33 smithi133 bash[102858]: audit 2023-10-09T04:57:32.945858+0000 mon.a (mon.0) 364 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:34.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:33 smithi133 bash[102858]: audit 2023-10-09T04:57:32.955731+0000 mon.a (mon.0) 365 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:34.283 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:57:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:57:34] "GET /metrics HTTP/1.1" 200 36283 "" "Prometheus/2.43.0" 2023-10-09T04:57:34.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:33 smithi133 bash[105589]: audit 2023-10-09T04:57:32.945858+0000 mon.a (mon.0) 364 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:34.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:33 smithi133 bash[105589]: audit 2023-10-09T04:57:32.955731+0000 mon.a (mon.0) 365 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:34.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:33 smithi194 bash[92666]: audit 2023-10-09T04:57:32.945858+0000 mon.a (mon.0) 364 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:34.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:33 smithi194 bash[92666]: audit 2023-10-09T04:57:32.955731+0000 mon.a (mon.0) 365 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:34.408 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T04:57:34.407406+0000 min lat: 0.00148753 max lat: 1.02837 avg lat: 0.00791839 2023-10-09T04:57:34.408 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:57:34.408 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 20 16 40403 40387 126.187 99.0625 0.00645239 0.00791839 2023-10-09T04:57:35.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:34 smithi133 bash[102858]: cluster 2023-10-09T04:57:33.446122+0000 mgr.y (mgr.25277) 232 : cluster 0 pgmap v175: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 2.0 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s; 12033/97563 objects degraded (12.334%) 2023-10-09T04:57:35.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:34 smithi133 bash[102858]: audit 2023-10-09T04:57:34.056188+0000 mon.a (mon.0) 366 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:35.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:34 smithi133 bash[102858]: audit 2023-10-09T04:57:34.063905+0000 mon.a (mon.0) 367 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:35.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:34 smithi133 bash[105589]: cluster 2023-10-09T04:57:33.446122+0000 mgr.y (mgr.25277) 232 : cluster 0 pgmap v175: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 2.0 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s; 12033/97563 objects degraded (12.334%) 2023-10-09T04:57:35.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:34 smithi133 bash[105589]: audit 2023-10-09T04:57:34.056188+0000 mon.a (mon.0) 366 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:35.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:34 smithi133 bash[105589]: audit 2023-10-09T04:57:34.063905+0000 mon.a (mon.0) 367 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:35.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:34 smithi194 bash[92666]: cluster 2023-10-09T04:57:33.446122+0000 mgr.y (mgr.25277) 232 : cluster 0 pgmap v175: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 2.0 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s; 12033/97563 objects degraded (12.334%) 2023-10-09T04:57:35.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:34 smithi194 bash[92666]: audit 2023-10-09T04:57:34.056188+0000 mon.a (mon.0) 366 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:35.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:34 smithi194 bash[92666]: audit 2023-10-09T04:57:34.063905+0000 mon.a (mon.0) 367 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:35.408 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 21 16 42102 42086 125.234 106.188 0.00437008 0.00797936 2023-10-09T04:57:36.409 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 22 16 43678 43662 124.018 98.5 0.0148232 0.00805831 2023-10-09T04:57:37.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:36 smithi133 bash[102858]: cluster 2023-10-09T04:57:35.447145+0000 mgr.y (mgr.25277) 233 : cluster 0 pgmap v176: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 2.5 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 154 MiB/s wr, 2.47k op/s; 15112/122250 objects degraded (12.362%) 2023-10-09T04:57:37.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:36 smithi133 bash[105589]: cluster 2023-10-09T04:57:35.447145+0000 mgr.y (mgr.25277) 233 : cluster 0 pgmap v176: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 2.5 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 154 MiB/s wr, 2.47k op/s; 15112/122250 objects degraded (12.362%) 2023-10-09T04:57:37.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:36 smithi194 bash[92666]: cluster 2023-10-09T04:57:35.447145+0000 mgr.y (mgr.25277) 233 : cluster 0 pgmap v176: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 2.5 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 154 MiB/s wr, 2.47k op/s; 15112/122250 objects degraded (12.362%) 2023-10-09T04:57:37.409 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 23 16 45168 45152 122.675 93.125 0.0114787 0.00814576 2023-10-09T04:57:38.409 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 24 16 46637 46621 121.389 91.8125 0.00841512 0.00823183 2023-10-09T04:57:39.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:38 smithi133 bash[102858]: cluster 2023-10-09T04:57:37.447572+0000 mgr.y (mgr.25277) 234 : cluster 0 pgmap v177: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 2.5 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s; 15112/122250 objects degraded (12.362%) 2023-10-09T04:57:39.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:38 smithi133 bash[102858]: cluster 2023-10-09T04:57:37.991397+0000 mon.a (mon.0) 368 : cluster 3 Health check update: Degraded data redundancy: 15112/122250 objects degraded (12.362%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:39.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:38 smithi133 bash[105589]: cluster 2023-10-09T04:57:37.447572+0000 mgr.y (mgr.25277) 234 : cluster 0 pgmap v177: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 2.5 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s; 15112/122250 objects degraded (12.362%) 2023-10-09T04:57:39.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:38 smithi133 bash[105589]: cluster 2023-10-09T04:57:37.991397+0000 mon.a (mon.0) 368 : cluster 3 Health check update: Degraded data redundancy: 15112/122250 objects degraded (12.362%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:39.300 INFO:journalctl@ceph.osd.1.smithi133.stdout:Oct 09 04:57:38 smithi133 bash[120074]: debug 2023-10-09T04:57:38.947+0000 7f9d44f1c640 -1 osd.1 92 log_to_monitors true 2023-10-09T04:57:39.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:38 smithi194 bash[92666]: cluster 2023-10-09T04:57:37.447572+0000 mgr.y (mgr.25277) 234 : cluster 0 pgmap v177: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 2.5 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s; 15112/122250 objects degraded (12.362%) 2023-10-09T04:57:39.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:38 smithi194 bash[92666]: cluster 2023-10-09T04:57:37.991397+0000 mon.a (mon.0) 368 : cluster 3 Health check update: Degraded data redundancy: 15112/122250 objects degraded (12.362%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:39.409 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 25 16 48119 48103 120.238 92.625 0.00980915 0.00831186 2023-10-09T04:57:40.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:39 smithi133 bash[102858]: audit 2023-10-09T04:57:38.968124+0000 mon.b (mon.1) 4 : audit 1 from='osd.1 [v2:172.21.15.133:6810/2668659948,v1:172.21.15.133:6811/2668659948]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T04:57:40.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:39 smithi133 bash[102858]: audit 2023-10-09T04:57:38.971237+0000 mon.a (mon.0) 369 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T04:57:40.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:39 smithi133 bash[105589]: audit 2023-10-09T04:57:38.968124+0000 mon.b (mon.1) 4 : audit 1 from='osd.1 [v2:172.21.15.133:6810/2668659948,v1:172.21.15.133:6811/2668659948]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T04:57:40.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:39 smithi133 bash[105589]: audit 2023-10-09T04:57:38.971237+0000 mon.a (mon.0) 369 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T04:57:40.301 INFO:journalctl@ceph.osd.1.smithi133.stdout:Oct 09 04:57:40 smithi133 bash[120074]: debug 2023-10-09T04:57:40.031+0000 7f9d33382700 -1 osd.1 92 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T04:57:40.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:39 smithi194 bash[92666]: audit 2023-10-09T04:57:38.968124+0000 mon.b (mon.1) 4 : audit 1 from='osd.1 [v2:172.21.15.133:6810/2668659948,v1:172.21.15.133:6811/2668659948]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T04:57:40.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:39 smithi194 bash[92666]: audit 2023-10-09T04:57:38.971237+0000 mon.a (mon.0) 369 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T04:57:40.409 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 26 16 49702 49686 119.418 98.9375 0.0102513 0.00836887 2023-10-09T04:57:41.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:40 smithi133 bash[102858]: cluster 2023-10-09T04:57:39.448189+0000 mgr.y (mgr.25277) 235 : cluster 0 pgmap v178: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 2.6 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s; 16077/128082 objects degraded (12.552%) 2023-10-09T04:57:41.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:40 smithi133 bash[102858]: cluster 2023-10-09T04:57:39.969892+0000 osd.3 (osd.3) 11 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T04:57:41.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:40 smithi133 bash[102858]: audit 2023-10-09T04:57:39.973337+0000 mon.a (mon.0) 370 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T04:57:41.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:40 smithi133 bash[102858]: cluster 2023-10-09T04:57:39.973424+0000 mon.a (mon.0) 371 : cluster 0 osdmap e95: 8 total, 7 up, 8 in 2023-10-09T04:57:41.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:40 smithi133 bash[102858]: cluster 2023-10-09T04:57:39.973515+0000 osd.5 (osd.5) 19 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T04:57:41.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:40 smithi133 bash[102858]: audit 2023-10-09T04:57:39.979002+0000 mon.a (mon.0) 372 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:57:41.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:40 smithi133 bash[102858]: audit 2023-10-09T04:57:39.979030+0000 mon.b (mon.1) 5 : audit 1 from='osd.1 [v2:172.21.15.133:6810/2668659948,v1:172.21.15.133:6811/2668659948]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:57:41.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:40 smithi133 bash[105589]: cluster 2023-10-09T04:57:39.448189+0000 mgr.y (mgr.25277) 235 : cluster 0 pgmap v178: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 2.6 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s; 16077/128082 objects degraded (12.552%) 2023-10-09T04:57:41.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:40 smithi133 bash[105589]: cluster 2023-10-09T04:57:39.969892+0000 osd.3 (osd.3) 11 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T04:57:41.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:40 smithi133 bash[105589]: audit 2023-10-09T04:57:39.973337+0000 mon.a (mon.0) 370 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T04:57:41.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:40 smithi133 bash[105589]: cluster 2023-10-09T04:57:39.973424+0000 mon.a (mon.0) 371 : cluster 0 osdmap e95: 8 total, 7 up, 8 in 2023-10-09T04:57:41.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:40 smithi133 bash[105589]: cluster 2023-10-09T04:57:39.973515+0000 osd.5 (osd.5) 19 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T04:57:41.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:40 smithi133 bash[105589]: audit 2023-10-09T04:57:39.979002+0000 mon.a (mon.0) 372 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:57:41.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:40 smithi133 bash[105589]: audit 2023-10-09T04:57:39.979030+0000 mon.b (mon.1) 5 : audit 1 from='osd.1 [v2:172.21.15.133:6810/2668659948,v1:172.21.15.133:6811/2668659948]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:57:41.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:40 smithi194 bash[92666]: cluster 2023-10-09T04:57:39.448189+0000 mgr.y (mgr.25277) 235 : cluster 0 pgmap v178: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 2.6 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s; 16077/128082 objects degraded (12.552%) 2023-10-09T04:57:41.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:40 smithi194 bash[92666]: cluster 2023-10-09T04:57:39.969892+0000 osd.3 (osd.3) 11 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T04:57:41.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:40 smithi194 bash[92666]: audit 2023-10-09T04:57:39.973337+0000 mon.a (mon.0) 370 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T04:57:41.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:40 smithi194 bash[92666]: cluster 2023-10-09T04:57:39.973424+0000 mon.a (mon.0) 371 : cluster 0 osdmap e95: 8 total, 7 up, 8 in 2023-10-09T04:57:41.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:40 smithi194 bash[92666]: cluster 2023-10-09T04:57:39.973515+0000 osd.5 (osd.5) 19 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T04:57:41.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:40 smithi194 bash[92666]: audit 2023-10-09T04:57:39.979002+0000 mon.a (mon.0) 372 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:57:41.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:40 smithi194 bash[92666]: audit 2023-10-09T04:57:39.979030+0000 mon.b (mon.1) 5 : audit 1 from='osd.1 [v2:172.21.15.133:6810/2668659948,v1:172.21.15.133:6811/2668659948]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T04:57:41.409 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 27 16 50712 50696 117.333 63.125 0.00283764 0.00839412 2023-10-09T04:57:42.265 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:42 smithi133 bash[102858]: cluster 2023-10-09T04:57:40.969368+0000 mon.a (mon.0) 373 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:57:42.265 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:42 smithi133 bash[102858]: cluster 2023-10-09T04:57:40.977063+0000 osd.2 (osd.2) 15 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:57:42.265 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:42 smithi133 bash[102858]: cluster 2023-10-09T04:57:40.982941+0000 osd.5 (osd.5) 20 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:57:42.265 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:42 smithi133 bash[102858]: cluster 2023-10-09T04:57:40.983728+0000 mon.a (mon.0) 374 : cluster 1 osd.1 [v2:172.21.15.133:6810/2668659948,v1:172.21.15.133:6811/2668659948] boot 2023-10-09T04:57:42.265 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:42 smithi133 bash[102858]: cluster 2023-10-09T04:57:40.983958+0000 mon.a (mon.0) 375 : cluster 0 osdmap e96: 8 total, 8 up, 8 in 2023-10-09T04:57:42.265 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:42 smithi133 bash[102858]: audit 2023-10-09T04:57:40.985155+0000 mon.a (mon.0) 376 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:57:42.265 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:42 smithi133 bash[102858]: cluster 2023-10-09T04:57:41.449042+0000 mgr.y (mgr.25277) 236 : cluster 0 pgmap v181: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 3.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s; 17970/145308 objects degraded (12.367%) 2023-10-09T04:57:42.266 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:42 smithi133 bash[105589]: cluster 2023-10-09T04:57:40.969368+0000 mon.a (mon.0) 373 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:57:42.266 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:42 smithi133 bash[105589]: cluster 2023-10-09T04:57:40.977063+0000 osd.2 (osd.2) 15 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:57:42.266 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:42 smithi133 bash[105589]: cluster 2023-10-09T04:57:40.982941+0000 osd.5 (osd.5) 20 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:57:42.266 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:42 smithi133 bash[105589]: cluster 2023-10-09T04:57:40.983728+0000 mon.a (mon.0) 374 : cluster 1 osd.1 [v2:172.21.15.133:6810/2668659948,v1:172.21.15.133:6811/2668659948] boot 2023-10-09T04:57:42.266 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:42 smithi133 bash[105589]: cluster 2023-10-09T04:57:40.983958+0000 mon.a (mon.0) 375 : cluster 0 osdmap e96: 8 total, 8 up, 8 in 2023-10-09T04:57:42.266 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:42 smithi133 bash[105589]: audit 2023-10-09T04:57:40.985155+0000 mon.a (mon.0) 376 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:57:42.266 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:42 smithi133 bash[105589]: cluster 2023-10-09T04:57:41.449042+0000 mgr.y (mgr.25277) 236 : cluster 0 pgmap v181: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 3.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s; 17970/145308 objects degraded (12.367%) 2023-10-09T04:57:42.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:42 smithi194 bash[92666]: cluster 2023-10-09T04:57:40.969368+0000 mon.a (mon.0) 373 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:57:42.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:42 smithi194 bash[92666]: cluster 2023-10-09T04:57:40.977063+0000 osd.2 (osd.2) 15 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:57:42.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:42 smithi194 bash[92666]: cluster 2023-10-09T04:57:40.982941+0000 osd.5 (osd.5) 20 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:57:42.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:42 smithi194 bash[92666]: cluster 2023-10-09T04:57:40.983728+0000 mon.a (mon.0) 374 : cluster 1 osd.1 [v2:172.21.15.133:6810/2668659948,v1:172.21.15.133:6811/2668659948] boot 2023-10-09T04:57:42.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:42 smithi194 bash[92666]: cluster 2023-10-09T04:57:40.983958+0000 mon.a (mon.0) 375 : cluster 0 osdmap e96: 8 total, 8 up, 8 in 2023-10-09T04:57:42.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:42 smithi194 bash[92666]: audit 2023-10-09T04:57:40.985155+0000 mon.a (mon.0) 376 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:57:42.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:42 smithi194 bash[92666]: cluster 2023-10-09T04:57:41.449042+0000 mgr.y (mgr.25277) 236 : cluster 0 pgmap v181: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 3.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s; 17970/145308 objects degraded (12.367%) 2023-10-09T04:57:42.409 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 28 16 50712 50696 113.143 0 - 0.00839412 2023-10-09T04:57:43.014 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[102858]: cluster 2023-10-09T04:57:41.990455+0000 osd.3 (osd.3) 12 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.014 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[102858]: cluster 2023-10-09T04:57:41.992737+0000 osd.5 (osd.5) 21 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.014 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[102858]: cluster 2023-10-09T04:57:41.994091+0000 mon.a (mon.0) 377 : cluster 0 osdmap e97: 8 total, 8 up, 8 in 2023-10-09T04:57:43.014 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[102858]: cluster 2023-10-09T04:57:41.994546+0000 osd.2 (osd.2) 16 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.014 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[102858]: cluster 2023-10-09T04:57:41.994964+0000 osd.6 (osd.6) 10 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.014 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[102858]: cluster 2023-10-09T04:57:41.995160+0000 osd.4 (osd.4) 15 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.014 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[102858]: cluster 2023-10-09T04:57:41.995427+0000 osd.7 (osd.7) 11 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.015 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[102858]: cluster 2023-10-09T04:57:41.996302+0000 osd.3 (osd.3) 13 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.015 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[102858]: cluster 2023-10-09T04:57:41.998795+0000 osd.6 (osd.6) 11 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.015 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[102858]: cluster 2023-10-09T04:57:41.999051+0000 osd.3 (osd.3) 14 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.015 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[102858]: cluster 2023-10-09T04:57:41.999340+0000 osd.5 (osd.5) 22 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.015 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[102858]: cluster 2023-10-09T04:57:42.001252+0000 osd.4 (osd.4) 16 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.015 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[102858]: cluster 2023-10-09T04:57:42.001392+0000 osd.7 (osd.7) 12 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.015 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[102858]: cluster 2023-10-09T04:57:42.003205+0000 osd.5 (osd.5) 23 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.015 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[102858]: cluster 2023-10-09T04:57:42.004668+0000 osd.7 (osd.7) 13 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.015 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[102858]: cluster 2023-10-09T04:57:42.005999+0000 osd.4 (osd.4) 17 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.015 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[102858]: cluster 2023-10-09T04:57:42.992629+0000 mon.a (mon.0) 378 : cluster 3 Health check update: Degraded data redundancy: 17970/145308 objects degraded (12.367%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:43.015 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[102858]: cluster 2023-10-09T04:57:42.999410+0000 mon.a (mon.0) 379 : cluster 0 osdmap e98: 8 total, 8 up, 8 in 2023-10-09T04:57:43.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[105589]: cluster 2023-10-09T04:57:41.990455+0000 osd.3 (osd.3) 12 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[105589]: cluster 2023-10-09T04:57:41.992737+0000 osd.5 (osd.5) 21 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[105589]: cluster 2023-10-09T04:57:41.994091+0000 mon.a (mon.0) 377 : cluster 0 osdmap e97: 8 total, 8 up, 8 in 2023-10-09T04:57:43.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[105589]: cluster 2023-10-09T04:57:41.994546+0000 osd.2 (osd.2) 16 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[105589]: cluster 2023-10-09T04:57:41.994964+0000 osd.6 (osd.6) 10 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[105589]: cluster 2023-10-09T04:57:41.995160+0000 osd.4 (osd.4) 15 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[105589]: cluster 2023-10-09T04:57:41.995427+0000 osd.7 (osd.7) 11 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[105589]: cluster 2023-10-09T04:57:41.996302+0000 osd.3 (osd.3) 13 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[105589]: cluster 2023-10-09T04:57:41.998795+0000 osd.6 (osd.6) 11 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[105589]: cluster 2023-10-09T04:57:41.999051+0000 osd.3 (osd.3) 14 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[105589]: cluster 2023-10-09T04:57:41.999340+0000 osd.5 (osd.5) 22 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[105589]: cluster 2023-10-09T04:57:42.001252+0000 osd.4 (osd.4) 16 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[105589]: cluster 2023-10-09T04:57:42.001392+0000 osd.7 (osd.7) 12 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[105589]: cluster 2023-10-09T04:57:42.003205+0000 osd.5 (osd.5) 23 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[105589]: cluster 2023-10-09T04:57:42.004668+0000 osd.7 (osd.7) 13 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[105589]: cluster 2023-10-09T04:57:42.005999+0000 osd.4 (osd.4) 17 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[105589]: cluster 2023-10-09T04:57:42.992629+0000 mon.a (mon.0) 378 : cluster 3 Health check update: Degraded data redundancy: 17970/145308 objects degraded (12.367%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:43.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:43 smithi133 bash[105589]: cluster 2023-10-09T04:57:42.999410+0000 mon.a (mon.0) 379 : cluster 0 osdmap e98: 8 total, 8 up, 8 in 2023-10-09T04:57:43.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:43 smithi194 bash[92666]: cluster 2023-10-09T04:57:41.990455+0000 osd.3 (osd.3) 12 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:43 smithi194 bash[92666]: cluster 2023-10-09T04:57:41.992737+0000 osd.5 (osd.5) 21 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:43 smithi194 bash[92666]: cluster 2023-10-09T04:57:41.994091+0000 mon.a (mon.0) 377 : cluster 0 osdmap e97: 8 total, 8 up, 8 in 2023-10-09T04:57:43.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:43 smithi194 bash[92666]: cluster 2023-10-09T04:57:41.994546+0000 osd.2 (osd.2) 16 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:43 smithi194 bash[92666]: cluster 2023-10-09T04:57:41.994964+0000 osd.6 (osd.6) 10 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:43 smithi194 bash[92666]: cluster 2023-10-09T04:57:41.995160+0000 osd.4 (osd.4) 15 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:43 smithi194 bash[92666]: cluster 2023-10-09T04:57:41.995427+0000 osd.7 (osd.7) 11 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:43 smithi194 bash[92666]: cluster 2023-10-09T04:57:41.996302+0000 osd.3 (osd.3) 13 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:43 smithi194 bash[92666]: cluster 2023-10-09T04:57:41.998795+0000 osd.6 (osd.6) 11 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:43 smithi194 bash[92666]: cluster 2023-10-09T04:57:41.999051+0000 osd.3 (osd.3) 14 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:43 smithi194 bash[92666]: cluster 2023-10-09T04:57:41.999340+0000 osd.5 (osd.5) 22 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:43 smithi194 bash[92666]: cluster 2023-10-09T04:57:42.001252+0000 osd.4 (osd.4) 16 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:43 smithi194 bash[92666]: cluster 2023-10-09T04:57:42.001392+0000 osd.7 (osd.7) 12 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:43 smithi194 bash[92666]: cluster 2023-10-09T04:57:42.003205+0000 osd.5 (osd.5) 23 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:43 smithi194 bash[92666]: cluster 2023-10-09T04:57:42.004668+0000 osd.7 (osd.7) 13 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:43 smithi194 bash[92666]: cluster 2023-10-09T04:57:42.005999+0000 osd.4 (osd.4) 17 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:57:43.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:43 smithi194 bash[92666]: cluster 2023-10-09T04:57:42.992629+0000 mon.a (mon.0) 378 : cluster 3 Health check update: Degraded data redundancy: 17970/145308 objects degraded (12.367%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:43.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:43 smithi194 bash[92666]: cluster 2023-10-09T04:57:42.999410+0000 mon.a (mon.0) 379 : cluster 0 osdmap e98: 8 total, 8 up, 8 in 2023-10-09T04:57:43.409 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 29 16 50908 50892 109.664 6.125 0.034848 0.00911218 2023-10-09T04:57:44.410 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 30 16 51874 51858 108.021 60.375 0.0162602 0.00924801 2023-10-09T04:57:44.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:57:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:57:44] "GET /metrics HTTP/1.1" 200 36290 "" "Prometheus/2.43.0" 2023-10-09T04:57:44.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:44 smithi133 bash[102858]: cluster 2023-10-09T04:57:42.994943+0000 osd.3 (osd.3) 15 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:44 smithi133 bash[102858]: cluster 2023-10-09T04:57:42.996449+0000 osd.5 (osd.5) 24 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:44 smithi133 bash[102858]: cluster 2023-10-09T04:57:42.999333+0000 osd.3 (osd.3) 16 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:44 smithi133 bash[102858]: cluster 2023-10-09T04:57:43.000848+0000 osd.6 (osd.6) 12 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:44 smithi133 bash[102858]: cluster 2023-10-09T04:57:43.001327+0000 osd.5 (osd.5) 25 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:44 smithi133 bash[102858]: cluster 2023-10-09T04:57:43.002231+0000 osd.3 (osd.3) 17 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:44 smithi133 bash[102858]: cluster 2023-10-09T04:57:43.004193+0000 osd.6 (osd.6) 13 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:44 smithi133 bash[102858]: cluster 2023-10-09T04:57:43.004252+0000 osd.5 (osd.5) 26 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:44 smithi133 bash[102858]: cluster 2023-10-09T04:57:43.006736+0000 osd.6 (osd.6) 14 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:44 smithi133 bash[102858]: cluster 2023-10-09T04:57:43.019124+0000 osd.2 (osd.2) 17 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:44 smithi133 bash[102858]: cluster 2023-10-09T04:57:43.025686+0000 osd.4 (osd.4) 18 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:44 smithi133 bash[102858]: cluster 2023-10-09T04:57:43.062252+0000 osd.7 (osd.7) 14 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:44 smithi133 bash[102858]: audit 2023-10-09T04:57:43.419859+0000 mon.a (mon.0) 380 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:44.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:44 smithi133 bash[102858]: audit 2023-10-09T04:57:43.421488+0000 mon.a (mon.0) 381 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:44.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:44 smithi133 bash[102858]: cluster 2023-10-09T04:57:43.449644+0000 mgr.y (mgr.25277) 237 : cluster 0 pgmap v184: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 3.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s; 17970/145308 objects degraded (12.367%) 2023-10-09T04:57:44.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:44 smithi133 bash[105589]: cluster 2023-10-09T04:57:42.994943+0000 osd.3 (osd.3) 15 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:44 smithi133 bash[105589]: cluster 2023-10-09T04:57:42.996449+0000 osd.5 (osd.5) 24 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:44 smithi133 bash[105589]: cluster 2023-10-09T04:57:42.999333+0000 osd.3 (osd.3) 16 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:44 smithi133 bash[105589]: cluster 2023-10-09T04:57:43.000848+0000 osd.6 (osd.6) 12 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:44 smithi133 bash[105589]: cluster 2023-10-09T04:57:43.001327+0000 osd.5 (osd.5) 25 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:44 smithi133 bash[105589]: cluster 2023-10-09T04:57:43.002231+0000 osd.3 (osd.3) 17 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:44 smithi133 bash[105589]: cluster 2023-10-09T04:57:43.004193+0000 osd.6 (osd.6) 13 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:44 smithi133 bash[105589]: cluster 2023-10-09T04:57:43.004252+0000 osd.5 (osd.5) 26 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:44 smithi133 bash[105589]: cluster 2023-10-09T04:57:43.006736+0000 osd.6 (osd.6) 14 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:44 smithi133 bash[105589]: cluster 2023-10-09T04:57:43.019124+0000 osd.2 (osd.2) 17 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:44 smithi133 bash[105589]: cluster 2023-10-09T04:57:43.025686+0000 osd.4 (osd.4) 18 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:44 smithi133 bash[105589]: cluster 2023-10-09T04:57:43.062252+0000 osd.7 (osd.7) 14 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:44 smithi133 bash[105589]: audit 2023-10-09T04:57:43.419859+0000 mon.a (mon.0) 380 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:44.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:44 smithi133 bash[105589]: audit 2023-10-09T04:57:43.421488+0000 mon.a (mon.0) 381 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:44.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:44 smithi133 bash[105589]: cluster 2023-10-09T04:57:43.449644+0000 mgr.y (mgr.25277) 237 : cluster 0 pgmap v184: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 3.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s; 17970/145308 objects degraded (12.367%) 2023-10-09T04:57:44.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:44 smithi194 bash[92666]: cluster 2023-10-09T04:57:42.994943+0000 osd.3 (osd.3) 15 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:44 smithi194 bash[92666]: cluster 2023-10-09T04:57:42.996449+0000 osd.5 (osd.5) 24 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:44 smithi194 bash[92666]: cluster 2023-10-09T04:57:42.999333+0000 osd.3 (osd.3) 16 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:44 smithi194 bash[92666]: cluster 2023-10-09T04:57:43.000848+0000 osd.6 (osd.6) 12 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:44 smithi194 bash[92666]: cluster 2023-10-09T04:57:43.001327+0000 osd.5 (osd.5) 25 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:44 smithi194 bash[92666]: cluster 2023-10-09T04:57:43.002231+0000 osd.3 (osd.3) 17 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:44 smithi194 bash[92666]: cluster 2023-10-09T04:57:43.004193+0000 osd.6 (osd.6) 13 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:44 smithi194 bash[92666]: cluster 2023-10-09T04:57:43.004252+0000 osd.5 (osd.5) 26 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:44 smithi194 bash[92666]: cluster 2023-10-09T04:57:43.006736+0000 osd.6 (osd.6) 14 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:44 smithi194 bash[92666]: cluster 2023-10-09T04:57:43.019124+0000 osd.2 (osd.2) 17 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:44 smithi194 bash[92666]: cluster 2023-10-09T04:57:43.025686+0000 osd.4 (osd.4) 18 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:44 smithi194 bash[92666]: cluster 2023-10-09T04:57:43.062252+0000 osd.7 (osd.7) 14 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:57:44.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:44 smithi194 bash[92666]: audit 2023-10-09T04:57:43.419859+0000 mon.a (mon.0) 380 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:44.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:44 smithi194 bash[92666]: audit 2023-10-09T04:57:43.421488+0000 mon.a (mon.0) 381 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:44.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:44 smithi194 bash[92666]: cluster 2023-10-09T04:57:43.449644+0000 mgr.y (mgr.25277) 237 : cluster 0 pgmap v184: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 3.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s; 17970/145308 objects degraded (12.367%) 2023-10-09T04:57:45.410 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 31 16 52785 52769 106.373 56.9375 0.00933547 0.00939402 2023-10-09T04:57:46.410 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 32 16 53695 53679 104.826 56.875 0.00725794 0.00953124 2023-10-09T04:57:46.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:46 smithi133 bash[102858]: cluster 2023-10-09T04:57:45.450496+0000 mgr.y (mgr.25277) 238 : cluster 0 pgmap v185: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 58 MiB/s wr, 922 op/s; 16641/156201 objects degraded (10.654%); 315 KiB/s, 4 objects/s recovering 2023-10-09T04:57:46.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:46 smithi133 bash[105589]: cluster 2023-10-09T04:57:45.450496+0000 mgr.y (mgr.25277) 238 : cluster 0 pgmap v185: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 58 MiB/s wr, 922 op/s; 16641/156201 objects degraded (10.654%); 315 KiB/s, 4 objects/s recovering 2023-10-09T04:57:46.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:46 smithi194 bash[92666]: cluster 2023-10-09T04:57:45.450496+0000 mgr.y (mgr.25277) 238 : cluster 0 pgmap v185: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 58 MiB/s wr, 922 op/s; 16641/156201 objects degraded (10.654%); 315 KiB/s, 4 objects/s recovering 2023-10-09T04:57:47.410 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 33 16 54572 54556 103.31 54.8125 0.0547302 0.00967218 2023-10-09T04:57:48.410 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 34 16 55472 55456 101.926 56.25 0.00864277 0.00980212 2023-10-09T04:57:48.520 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:48 smithi133 bash[102858]: cluster 2023-10-09T04:57:47.451080+0000 mgr.y (mgr.25277) 239 : cluster 0 pgmap v186: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 35 MiB/s wr, 560 op/s; 16641/156201 objects degraded (10.654%); 267 KiB/s, 4 objects/s recovering 2023-10-09T04:57:48.520 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:48 smithi133 bash[102858]: cluster 2023-10-09T04:57:47.993567+0000 mon.a (mon.0) 382 : cluster 3 Health check update: Degraded data redundancy: 16641/156201 objects degraded (10.654%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:48.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:48 smithi133 bash[105589]: cluster 2023-10-09T04:57:47.451080+0000 mgr.y (mgr.25277) 239 : cluster 0 pgmap v186: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 35 MiB/s wr, 560 op/s; 16641/156201 objects degraded (10.654%); 267 KiB/s, 4 objects/s recovering 2023-10-09T04:57:48.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:48 smithi133 bash[105589]: cluster 2023-10-09T04:57:47.993567+0000 mon.a (mon.0) 382 : cluster 3 Health check update: Degraded data redundancy: 16641/156201 objects degraded (10.654%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:48.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:48 smithi194 bash[92666]: cluster 2023-10-09T04:57:47.451080+0000 mgr.y (mgr.25277) 239 : cluster 0 pgmap v186: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 35 MiB/s wr, 560 op/s; 16641/156201 objects degraded (10.654%); 267 KiB/s, 4 objects/s recovering 2023-10-09T04:57:48.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:48 smithi194 bash[92666]: cluster 2023-10-09T04:57:47.993567+0000 mon.a (mon.0) 382 : cluster 3 Health check update: Degraded data redundancy: 16641/156201 objects degraded (10.654%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:49.410 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 35 16 56443 56427 100.748 60.6875 0.0109245 0.00991866 2023-10-09T04:57:50.410 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 36 16 57346 57330 99.5169 56.4375 0.0110752 0.0100421 2023-10-09T04:57:50.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:50 smithi133 bash[102858]: cluster 2023-10-09T04:57:49.451684+0000 mgr.y (mgr.25277) 240 : cluster 0 pgmap v187: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 37 MiB/s wr, 594 op/s; 16495/159585 objects degraded (10.336%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T04:57:50.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:50 smithi133 bash[102858]: audit 2023-10-09T04:57:50.003851+0000 mon.a (mon.0) 383 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:50.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:50 smithi133 bash[102858]: audit 2023-10-09T04:57:50.016482+0000 mon.a (mon.0) 384 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:50.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:50 smithi133 bash[102858]: audit 2023-10-09T04:57:50.018398+0000 mon.a (mon.0) 385 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:57:50.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:50 smithi133 bash[102858]: audit 2023-10-09T04:57:50.019814+0000 mon.a (mon.0) 386 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:57:50.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:50 smithi133 bash[102858]: audit 2023-10-09T04:57:50.030364+0000 mon.a (mon.0) 387 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:50.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:50 smithi133 bash[102858]: audit 2023-10-09T04:57:50.130359+0000 mon.a (mon.0) 388 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:57:50.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:50 smithi133 bash[102858]: audit 2023-10-09T04:57:50.132943+0000 mon.a (mon.0) 389 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:57:50.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:50 smithi133 bash[102858]: audit 2023-10-09T04:57:50.134125+0000 mon.a (mon.0) 390 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:57:50.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:50 smithi133 bash[102858]: audit 2023-10-09T04:57:50.144620+0000 mon.a (mon.0) 391 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:50.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:50 smithi133 bash[105589]: cluster 2023-10-09T04:57:49.451684+0000 mgr.y (mgr.25277) 240 : cluster 0 pgmap v187: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 37 MiB/s wr, 594 op/s; 16495/159585 objects degraded (10.336%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T04:57:50.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:50 smithi133 bash[105589]: audit 2023-10-09T04:57:50.003851+0000 mon.a (mon.0) 383 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:50.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:50 smithi133 bash[105589]: audit 2023-10-09T04:57:50.016482+0000 mon.a (mon.0) 384 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:50.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:50 smithi133 bash[105589]: audit 2023-10-09T04:57:50.018398+0000 mon.a (mon.0) 385 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:57:50.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:50 smithi133 bash[105589]: audit 2023-10-09T04:57:50.019814+0000 mon.a (mon.0) 386 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:57:50.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:50 smithi133 bash[105589]: audit 2023-10-09T04:57:50.030364+0000 mon.a (mon.0) 387 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:50.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:50 smithi133 bash[105589]: audit 2023-10-09T04:57:50.130359+0000 mon.a (mon.0) 388 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:57:50.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:50 smithi133 bash[105589]: audit 2023-10-09T04:57:50.132943+0000 mon.a (mon.0) 389 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:57:50.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:50 smithi133 bash[105589]: audit 2023-10-09T04:57:50.134125+0000 mon.a (mon.0) 390 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:57:50.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:50 smithi133 bash[105589]: audit 2023-10-09T04:57:50.144620+0000 mon.a (mon.0) 391 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:50.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:50 smithi194 bash[92666]: cluster 2023-10-09T04:57:49.451684+0000 mgr.y (mgr.25277) 240 : cluster 0 pgmap v187: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 37 MiB/s wr, 594 op/s; 16495/159585 objects degraded (10.336%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T04:57:50.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:50 smithi194 bash[92666]: audit 2023-10-09T04:57:50.003851+0000 mon.a (mon.0) 383 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:50.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:50 smithi194 bash[92666]: audit 2023-10-09T04:57:50.016482+0000 mon.a (mon.0) 384 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:50.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:50 smithi194 bash[92666]: audit 2023-10-09T04:57:50.018398+0000 mon.a (mon.0) 385 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:57:50.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:50 smithi194 bash[92666]: audit 2023-10-09T04:57:50.019814+0000 mon.a (mon.0) 386 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:57:50.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:50 smithi194 bash[92666]: audit 2023-10-09T04:57:50.030364+0000 mon.a (mon.0) 387 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:50.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:50 smithi194 bash[92666]: audit 2023-10-09T04:57:50.130359+0000 mon.a (mon.0) 388 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:57:50.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:50 smithi194 bash[92666]: audit 2023-10-09T04:57:50.132943+0000 mon.a (mon.0) 389 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:57:50.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:50 smithi194 bash[92666]: audit 2023-10-09T04:57:50.134125+0000 mon.a (mon.0) 390 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:57:50.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:50 smithi194 bash[92666]: audit 2023-10-09T04:57:50.144620+0000 mon.a (mon.0) 391 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:57:51.410 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 37 16 58330 58314 98.4893 61.5 0.0151219 0.0101461 2023-10-09T04:57:52.410 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 38 16 59271 59255 97.4451 58.8125 0.00995557 0.0102545 2023-10-09T04:57:52.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:52 smithi133 bash[102858]: cluster 2023-10-09T04:57:51.452444+0000 mgr.y (mgr.25277) 241 : cluster 0 pgmap v188: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s; 16495/169884 objects degraded (9.710%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T04:57:52.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:52 smithi133 bash[105589]: cluster 2023-10-09T04:57:51.452444+0000 mgr.y (mgr.25277) 241 : cluster 0 pgmap v188: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s; 16495/169884 objects degraded (9.710%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T04:57:52.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:52 smithi194 bash[92666]: cluster 2023-10-09T04:57:51.452444+0000 mgr.y (mgr.25277) 241 : cluster 0 pgmap v188: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s; 16495/169884 objects degraded (9.710%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T04:57:53.410 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 39 15 60191 60176 96.4223 57.5625 0.0183425 0.0103638 2023-10-09T04:57:53.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:53 smithi133 bash[102858]: cluster 2023-10-09T04:57:52.994813+0000 mon.a (mon.0) 392 : cluster 3 Health check update: Degraded data redundancy: 16495/169884 objects degraded (9.710%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:53.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:53 smithi133 bash[105589]: cluster 2023-10-09T04:57:52.994813+0000 mon.a (mon.0) 392 : cluster 3 Health check update: Degraded data redundancy: 16495/169884 objects degraded (9.710%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:53.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:53 smithi194 bash[92666]: cluster 2023-10-09T04:57:52.994813+0000 mon.a (mon.0) 392 : cluster 3 Health check update: Degraded data redundancy: 16495/169884 objects degraded (9.710%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:54.410 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T04:57:54.409349+0000 min lat: 0.00148753 max lat: 2.25006 avg lat: 0.0104685 2023-10-09T04:57:54.411 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:57:54.411 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 40 16 61125 61109 95.4696 58.3125 0.0134441 0.0104685 2023-10-09T04:57:54.535 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:57:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:57:54] "GET /metrics HTTP/1.1" 200 36290 "" "Prometheus/2.43.0" 2023-10-09T04:57:54.535 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:54 smithi133 bash[102858]: cluster 2023-10-09T04:57:53.456201+0000 mgr.y (mgr.25277) 242 : cluster 0 pgmap v189: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 49 MiB/s wr, 783 op/s; 16495/169884 objects degraded (9.710%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:57:54.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:54 smithi133 bash[105589]: cluster 2023-10-09T04:57:53.456201+0000 mgr.y (mgr.25277) 242 : cluster 0 pgmap v189: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 49 MiB/s wr, 783 op/s; 16495/169884 objects degraded (9.710%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:57:54.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:54 smithi194 bash[92666]: cluster 2023-10-09T04:57:53.456201+0000 mgr.y (mgr.25277) 242 : cluster 0 pgmap v189: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 49 MiB/s wr, 783 op/s; 16495/169884 objects degraded (9.710%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:57:55.410 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 41 16 62060 62044 94.5662 58.4375 0.00963352 0.0105686 2023-10-09T04:57:56.411 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 42 16 62981 62965 93.6851 57.5625 0.017906 0.0106674 2023-10-09T04:57:56.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:56 smithi133 bash[102858]: cluster 2023-10-09T04:57:55.457280+0000 mgr.y (mgr.25277) 243 : cluster 0 pgmap v190: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 16343/184047 objects degraded (8.880%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:57:56.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:56 smithi133 bash[105589]: cluster 2023-10-09T04:57:55.457280+0000 mgr.y (mgr.25277) 243 : cluster 0 pgmap v190: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 16343/184047 objects degraded (8.880%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:57:56.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:56 smithi194 bash[92666]: cluster 2023-10-09T04:57:55.457280+0000 mgr.y (mgr.25277) 243 : cluster 0 pgmap v190: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 16343/184047 objects degraded (8.880%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:57:57.411 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 43 16 63880 63864 92.8129 56.1875 0.0116487 0.0107662 2023-10-09T04:57:58.411 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 44 16 64776 64760 91.9762 56 0.023365 0.0108657 2023-10-09T04:57:58.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:58 smithi194 bash[92666]: cluster 2023-10-09T04:57:57.457878+0000 mgr.y (mgr.25277) 244 : cluster 0 pgmap v191: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 48 MiB/s wr, 773 op/s; 16343/184047 objects degraded (8.880%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T04:57:58.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:58 smithi194 bash[92666]: cluster 2023-10-09T04:57:57.996596+0000 mon.a (mon.0) 393 : cluster 3 Health check update: Degraded data redundancy: 16343/184047 objects degraded (8.880%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:58.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:57:58 smithi194 bash[92666]: audit 2023-10-09T04:57:58.413449+0000 mon.a (mon.0) 394 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:59.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:58 smithi133 bash[102858]: cluster 2023-10-09T04:57:57.457878+0000 mgr.y (mgr.25277) 244 : cluster 0 pgmap v191: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 48 MiB/s wr, 773 op/s; 16343/184047 objects degraded (8.880%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T04:57:59.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:58 smithi133 bash[102858]: cluster 2023-10-09T04:57:57.996596+0000 mon.a (mon.0) 393 : cluster 3 Health check update: Degraded data redundancy: 16343/184047 objects degraded (8.880%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:59.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:57:58 smithi133 bash[102858]: audit 2023-10-09T04:57:58.413449+0000 mon.a (mon.0) 394 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:59.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:58 smithi133 bash[105589]: cluster 2023-10-09T04:57:57.457878+0000 mgr.y (mgr.25277) 244 : cluster 0 pgmap v191: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 48 MiB/s wr, 773 op/s; 16343/184047 objects degraded (8.880%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T04:57:59.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:58 smithi133 bash[105589]: cluster 2023-10-09T04:57:57.996596+0000 mon.a (mon.0) 393 : cluster 3 Health check update: Degraded data redundancy: 16343/184047 objects degraded (8.880%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:59.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:57:58 smithi133 bash[105589]: audit 2023-10-09T04:57:58.413449+0000 mon.a (mon.0) 394 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:59.411 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 45 16 65667 65651 91.1697 55.6875 0.0274428 0.010962 2023-10-09T04:58:00.411 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 46 16 66547 66531 90.3833 55 0.00733093 0.0110549 2023-10-09T04:58:00.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:00 smithi194 bash[92666]: cluster 2023-10-09T04:57:59.460908+0000 mgr.y (mgr.25277) 245 : cluster 0 pgmap v192: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s; 16175/187440 objects degraded (8.629%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T04:58:01.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:00 smithi133 bash[102858]: cluster 2023-10-09T04:57:59.460908+0000 mgr.y (mgr.25277) 245 : cluster 0 pgmap v192: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s; 16175/187440 objects degraded (8.629%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T04:58:01.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:00 smithi133 bash[105589]: cluster 2023-10-09T04:57:59.460908+0000 mgr.y (mgr.25277) 245 : cluster 0 pgmap v192: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s; 16175/187440 objects degraded (8.629%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T04:58:01.411 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 47 16 67426 67410 89.629 54.9375 0.0111856 0.0111493 2023-10-09T04:58:02.411 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 48 16 68314 68298 88.9179 55.5 0.00950512 0.0112398 2023-10-09T04:58:02.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:02 smithi194 bash[92666]: cluster 2023-10-09T04:58:01.461788+0000 mgr.y (mgr.25277) 246 : cluster 0 pgmap v193: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 16175/197604 objects degraded (8.186%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T04:58:03.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:02 smithi133 bash[102858]: cluster 2023-10-09T04:58:01.461788+0000 mgr.y (mgr.25277) 246 : cluster 0 pgmap v193: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 16175/197604 objects degraded (8.186%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T04:58:03.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:02 smithi133 bash[105589]: cluster 2023-10-09T04:58:01.461788+0000 mgr.y (mgr.25277) 246 : cluster 0 pgmap v193: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 16175/197604 objects degraded (8.186%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T04:58:03.411 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 49 15 69185 69170 88.2154 54.5 0.00900781 0.0113269 2023-10-09T04:58:03.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:03 smithi194 bash[92666]: cluster 2023-10-09T04:58:02.997998+0000 mon.a (mon.0) 395 : cluster 3 Health check update: Degraded data redundancy: 16175/197604 objects degraded (8.186%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:04.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:03 smithi133 bash[105589]: cluster 2023-10-09T04:58:02.997998+0000 mon.a (mon.0) 395 : cluster 3 Health check update: Degraded data redundancy: 16175/197604 objects degraded (8.186%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:04.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:03 smithi133 bash[102858]: cluster 2023-10-09T04:58:02.997998+0000 mon.a (mon.0) 395 : cluster 3 Health check update: Degraded data redundancy: 16175/197604 objects degraded (8.186%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:04.411 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 50 16 70059 70043 87.5423 54.5625 0.0106106 0.0114163 2023-10-09T04:58:04.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:58:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:58:04] "GET /metrics HTTP/1.1" 200 36232 "" "Prometheus/2.43.0" 2023-10-09T04:58:04.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:04 smithi194 bash[92666]: cluster 2023-10-09T04:58:03.462266+0000 mgr.y (mgr.25277) 247 : cluster 0 pgmap v194: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 48 MiB/s wr, 769 op/s; 16175/197604 objects degraded (8.186%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T04:58:05.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:04 smithi133 bash[105589]: cluster 2023-10-09T04:58:03.462266+0000 mgr.y (mgr.25277) 247 : cluster 0 pgmap v194: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 48 MiB/s wr, 769 op/s; 16175/197604 objects degraded (8.186%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T04:58:05.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:04 smithi133 bash[102858]: cluster 2023-10-09T04:58:03.462266+0000 mgr.y (mgr.25277) 247 : cluster 0 pgmap v194: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 48 MiB/s wr, 769 op/s; 16175/197604 objects degraded (8.186%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T04:58:05.411 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 51 16 70918 70902 86.8784 53.6875 0.00556762 0.011503 2023-10-09T04:58:06.412 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 52 16 71750 71734 86.2076 52 0.015533 0.0115937 2023-10-09T04:58:06.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:06 smithi194 bash[92666]: cluster 2023-10-09T04:58:05.463171+0000 mgr.y (mgr.25277) 248 : cluster 0 pgmap v195: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s; 16013/210732 objects degraded (7.599%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T04:58:07.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:06 smithi133 bash[105589]: cluster 2023-10-09T04:58:05.463171+0000 mgr.y (mgr.25277) 248 : cluster 0 pgmap v195: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s; 16013/210732 objects degraded (7.599%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T04:58:07.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:06 smithi133 bash[102858]: cluster 2023-10-09T04:58:05.463171+0000 mgr.y (mgr.25277) 248 : cluster 0 pgmap v195: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s; 16013/210732 objects degraded (7.599%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T04:58:07.412 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 53 16 72571 72555 85.5491 51.3125 0.0160029 0.0116829 2023-10-09T04:58:08.412 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 54 16 73421 73405 84.9486 53.125 0.0228056 0.0117652 2023-10-09T04:58:08.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:08 smithi194 bash[92666]: cluster 2023-10-09T04:58:07.463723+0000 mgr.y (mgr.25277) 249 : cluster 0 pgmap v196: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 46 MiB/s wr, 740 op/s; 16013/210732 objects degraded (7.599%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:58:08.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:08 smithi194 bash[92666]: cluster 2023-10-09T04:58:07.999333+0000 mon.a (mon.0) 396 : cluster 3 Health check update: Degraded data redundancy: 16013/210732 objects degraded (7.599%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:09.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:08 smithi133 bash[105589]: cluster 2023-10-09T04:58:07.463723+0000 mgr.y (mgr.25277) 249 : cluster 0 pgmap v196: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 46 MiB/s wr, 740 op/s; 16013/210732 objects degraded (7.599%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:58:09.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:08 smithi133 bash[105589]: cluster 2023-10-09T04:58:07.999333+0000 mon.a (mon.0) 396 : cluster 3 Health check update: Degraded data redundancy: 16013/210732 objects degraded (7.599%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:09.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:08 smithi133 bash[102858]: cluster 2023-10-09T04:58:07.463723+0000 mgr.y (mgr.25277) 249 : cluster 0 pgmap v196: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 46 MiB/s wr, 740 op/s; 16013/210732 objects degraded (7.599%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:58:09.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:08 smithi133 bash[102858]: cluster 2023-10-09T04:58:07.999333+0000 mon.a (mon.0) 396 : cluster 3 Health check update: Degraded data redundancy: 16013/210732 objects degraded (7.599%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:09.412 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 55 16 74279 74263 84.379 53.625 0.0146326 0.0118447 2023-10-09T04:58:10.412 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 56 16 75129 75113 83.8208 53.125 0.00805497 0.0119244 2023-10-09T04:58:10.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:10 smithi194 bash[92666]: cluster 2023-10-09T04:58:09.464384+0000 mgr.y (mgr.25277) 250 : cluster 0 pgmap v197: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 52 MiB/s wr, 827 op/s; 15839/213840 objects degraded (7.407%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:58:11.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:10 smithi133 bash[102858]: cluster 2023-10-09T04:58:09.464384+0000 mgr.y (mgr.25277) 250 : cluster 0 pgmap v197: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 52 MiB/s wr, 827 op/s; 15839/213840 objects degraded (7.407%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:58:11.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:10 smithi133 bash[105589]: cluster 2023-10-09T04:58:09.464384+0000 mgr.y (mgr.25277) 250 : cluster 0 pgmap v197: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 52 MiB/s wr, 827 op/s; 15839/213840 objects degraded (7.407%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:58:11.412 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 57 16 75933 75917 83.2318 50.25 0.0387342 0.0120078 2023-10-09T04:58:12.412 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 58 16 76800 76784 82.7309 54.1875 0.0166554 0.0120809 2023-10-09T04:58:12.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:12 smithi194 bash[92666]: cluster 2023-10-09T04:58:11.465151+0000 mgr.y (mgr.25277) 251 : cluster 0 pgmap v198: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 15839/223515 objects degraded (7.086%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:58:13.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:12 smithi133 bash[105589]: cluster 2023-10-09T04:58:11.465151+0000 mgr.y (mgr.25277) 251 : cluster 0 pgmap v198: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 15839/223515 objects degraded (7.086%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:58:13.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:12 smithi133 bash[102858]: cluster 2023-10-09T04:58:11.465151+0000 mgr.y (mgr.25277) 251 : cluster 0 pgmap v198: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 15839/223515 objects degraded (7.086%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:58:13.412 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 59 16 77648 77632 82.227 53 0.0255351 0.0121547 2023-10-09T04:58:13.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:13 smithi194 bash[92666]: cluster 2023-10-09T04:58:13.000558+0000 mon.a (mon.0) 397 : cluster 3 Health check update: Degraded data redundancy: 15839/223515 objects degraded (7.086%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:13.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:13 smithi194 bash[92666]: audit 2023-10-09T04:58:13.416110+0000 mon.a (mon.0) 398 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:14.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:13 smithi133 bash[102858]: cluster 2023-10-09T04:58:13.000558+0000 mon.a (mon.0) 397 : cluster 3 Health check update: Degraded data redundancy: 15839/223515 objects degraded (7.086%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:14.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:13 smithi133 bash[102858]: audit 2023-10-09T04:58:13.416110+0000 mon.a (mon.0) 398 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:14.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:13 smithi133 bash[105589]: cluster 2023-10-09T04:58:13.000558+0000 mon.a (mon.0) 397 : cluster 3 Health check update: Degraded data redundancy: 15839/223515 objects degraded (7.086%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:14.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:13 smithi133 bash[105589]: audit 2023-10-09T04:58:13.416110+0000 mon.a (mon.0) 398 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:14.412 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T04:58:14.411305+0000 min lat: 0.00148753 max lat: 2.25006 avg lat: 0.0122276 2023-10-09T04:58:14.413 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:58:14.413 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 60 16 78487 78471 81.7304 52.4375 0.0141666 0.0122276 2023-10-09T04:58:14.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:58:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:58:14] "GET /metrics HTTP/1.1" 200 36298 "" "Prometheus/2.43.0" 2023-10-09T04:58:14.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:14 smithi194 bash[92666]: cluster 2023-10-09T04:58:13.465780+0000 mgr.y (mgr.25277) 252 : cluster 0 pgmap v199: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 45 MiB/s wr, 719 op/s; 15839/223515 objects degraded (7.086%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:58:15.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:14 smithi133 bash[102858]: cluster 2023-10-09T04:58:13.465780+0000 mgr.y (mgr.25277) 252 : cluster 0 pgmap v199: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 45 MiB/s wr, 719 op/s; 15839/223515 objects degraded (7.086%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:58:15.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:14 smithi133 bash[105589]: cluster 2023-10-09T04:58:13.465780+0000 mgr.y (mgr.25277) 252 : cluster 0 pgmap v199: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 45 MiB/s wr, 719 op/s; 15839/223515 objects degraded (7.086%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:58:15.412 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 61 16 79311 79295 81.2347 51.5 0.020435 0.0123036 2023-10-09T04:58:16.413 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 62 16 80163 80147 80.7833 53.25 0.0189251 0.0123721 2023-10-09T04:58:16.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:16 smithi194 bash[92666]: cluster 2023-10-09T04:58:15.466624+0000 mgr.y (mgr.25277) 253 : cluster 0 pgmap v200: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 15644/236166 objects degraded (6.624%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T04:58:17.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:16 smithi133 bash[102858]: cluster 2023-10-09T04:58:15.466624+0000 mgr.y (mgr.25277) 253 : cluster 0 pgmap v200: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 15644/236166 objects degraded (6.624%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T04:58:17.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:16 smithi133 bash[105589]: cluster 2023-10-09T04:58:15.466624+0000 mgr.y (mgr.25277) 253 : cluster 0 pgmap v200: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 15644/236166 objects degraded (6.624%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T04:58:17.413 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 63 16 80969 80953 80.3005 50.375 0.019531 0.0124462 2023-10-09T04:58:18.413 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 64 16 81772 81756 79.83 50.1875 0.00989463 0.0125189 2023-10-09T04:58:18.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:18 smithi194 bash[92666]: cluster 2023-10-09T04:58:17.467081+0000 mgr.y (mgr.25277) 254 : cluster 0 pgmap v201: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 44 MiB/s wr, 706 op/s; 15644/236166 objects degraded (6.624%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:58:18.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:18 smithi194 bash[92666]: cluster 2023-10-09T04:58:18.007796+0000 mon.a (mon.0) 399 : cluster 3 Health check update: Degraded data redundancy: 15644/236166 objects degraded (6.624%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:19.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:18 smithi133 bash[102858]: cluster 2023-10-09T04:58:17.467081+0000 mgr.y (mgr.25277) 254 : cluster 0 pgmap v201: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 44 MiB/s wr, 706 op/s; 15644/236166 objects degraded (6.624%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:58:19.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:18 smithi133 bash[102858]: cluster 2023-10-09T04:58:18.007796+0000 mon.a (mon.0) 399 : cluster 3 Health check update: Degraded data redundancy: 15644/236166 objects degraded (6.624%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:19.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:18 smithi133 bash[105589]: cluster 2023-10-09T04:58:17.467081+0000 mgr.y (mgr.25277) 254 : cluster 0 pgmap v201: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 44 MiB/s wr, 706 op/s; 15644/236166 objects degraded (6.624%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:58:19.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:18 smithi133 bash[105589]: cluster 2023-10-09T04:58:18.007796+0000 mon.a (mon.0) 399 : cluster 3 Health check update: Degraded data redundancy: 15644/236166 objects degraded (6.624%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:19.413 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 65 16 82567 82551 79.3662 49.6875 0.0110263 0.0125932 2023-10-09T04:58:20.413 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 66 16 83369 83353 78.9231 50.125 0.0160155 0.0126644 2023-10-09T04:58:20.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:20 smithi194 bash[92666]: cluster 2023-10-09T04:58:19.467654+0000 mgr.y (mgr.25277) 255 : cluster 0 pgmap v202: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 50 MiB/s wr, 793 op/s; 15446/239319 objects degraded (6.454%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T04:58:21.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:20 smithi133 bash[102858]: cluster 2023-10-09T04:58:19.467654+0000 mgr.y (mgr.25277) 255 : cluster 0 pgmap v202: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 50 MiB/s wr, 793 op/s; 15446/239319 objects degraded (6.454%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T04:58:21.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:20 smithi133 bash[105589]: cluster 2023-10-09T04:58:19.467654+0000 mgr.y (mgr.25277) 255 : cluster 0 pgmap v202: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 50 MiB/s wr, 793 op/s; 15446/239319 objects degraded (6.454%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T04:58:21.413 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 67 16 84150 84134 78.4736 48.8125 0.0695276 0.0127352 2023-10-09T04:58:22.413 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 68 16 84934 84918 78.0401 49 0.00940845 0.0128069 2023-10-09T04:58:22.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:22 smithi194 bash[92666]: cluster 2023-10-09T04:58:21.468530+0000 mgr.y (mgr.25277) 256 : cluster 0 pgmap v203: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s; 15446/248376 objects degraded (6.219%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:58:23.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:22 smithi133 bash[102858]: cluster 2023-10-09T04:58:21.468530+0000 mgr.y (mgr.25277) 256 : cluster 0 pgmap v203: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s; 15446/248376 objects degraded (6.219%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:58:23.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:22 smithi133 bash[105589]: cluster 2023-10-09T04:58:21.468530+0000 mgr.y (mgr.25277) 256 : cluster 0 pgmap v203: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s; 15446/248376 objects degraded (6.219%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:58:23.413 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 69 16 85687 85671 77.5911 47.0625 0.00781534 0.0128809 2023-10-09T04:58:23.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:23 smithi194 bash[92666]: cluster 2023-10-09T04:58:23.009006+0000 mon.a (mon.0) 400 : cluster 3 Health check update: Degraded data redundancy: 15446/248376 objects degraded (6.219%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:24.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:23 smithi133 bash[102858]: cluster 2023-10-09T04:58:23.009006+0000 mon.a (mon.0) 400 : cluster 3 Health check update: Degraded data redundancy: 15446/248376 objects degraded (6.219%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:24.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:23 smithi133 bash[105589]: cluster 2023-10-09T04:58:23.009006+0000 mon.a (mon.0) 400 : cluster 3 Health check update: Degraded data redundancy: 15446/248376 objects degraded (6.219%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:24.413 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 70 16 86466 86450 77.1781 48.6875 0.0200381 0.0129502 2023-10-09T04:58:24.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:58:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:58:24] "GET /metrics HTTP/1.1" 200 36298 "" "Prometheus/2.43.0" 2023-10-09T04:58:24.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:24 smithi194 bash[92666]: cluster 2023-10-09T04:58:23.469037+0000 mgr.y (mgr.25277) 257 : cluster 0 pgmap v204: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 43 MiB/s wr, 690 op/s; 15446/248376 objects degraded (6.219%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:58:25.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:24 smithi133 bash[105589]: cluster 2023-10-09T04:58:23.469037+0000 mgr.y (mgr.25277) 257 : cluster 0 pgmap v204: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 43 MiB/s wr, 690 op/s; 15446/248376 objects degraded (6.219%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:58:25.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:24 smithi133 bash[102858]: cluster 2023-10-09T04:58:23.469037+0000 mgr.y (mgr.25277) 257 : cluster 0 pgmap v204: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 43 MiB/s wr, 690 op/s; 15446/248376 objects degraded (6.219%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:58:25.413 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 71 16 87258 87242 76.7882 49.5 0.0196896 0.0130151 2023-10-09T04:58:26.413 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 72 16 88058 88042 76.4161 50 0.0115171 0.0130796 2023-10-09T04:58:27.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:26 smithi133 bash[105589]: cluster 2023-10-09T04:58:25.469837+0000 mgr.y (mgr.25277) 258 : cluster 0 pgmap v205: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 15254/260094 objects degraded (5.865%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:58:27.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:26 smithi133 bash[102858]: cluster 2023-10-09T04:58:25.469837+0000 mgr.y (mgr.25277) 258 : cluster 0 pgmap v205: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 15254/260094 objects degraded (5.865%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:58:27.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:26 smithi194 bash[92666]: cluster 2023-10-09T04:58:25.469837+0000 mgr.y (mgr.25277) 258 : cluster 0 pgmap v205: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 15254/260094 objects degraded (5.865%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:58:27.414 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 73 16 88841 88825 76.0396 48.9375 0.0188438 0.0131446 2023-10-09T04:58:28.414 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 74 15 89592 89577 75.6471 47 0.0734257 0.0132123 2023-10-09T04:58:28.899 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:28 smithi133 bash[105589]: cluster 2023-10-09T04:58:27.472973+0000 mgr.y (mgr.25277) 259 : cluster 0 pgmap v206: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 42 MiB/s wr, 664 op/s; 15254/260094 objects degraded (5.865%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:58:28.900 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:28 smithi133 bash[105589]: cluster 2023-10-09T04:58:28.013013+0000 mon.a (mon.0) 401 : cluster 3 Health check update: Degraded data redundancy: 15254/260094 objects degraded (5.865%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:28.900 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:28 smithi133 bash[105589]: audit 2023-10-09T04:58:28.416705+0000 mon.a (mon.0) 402 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:28.900 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:28 smithi133 bash[102858]: cluster 2023-10-09T04:58:27.472973+0000 mgr.y (mgr.25277) 259 : cluster 0 pgmap v206: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 42 MiB/s wr, 664 op/s; 15254/260094 objects degraded (5.865%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:58:28.900 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:28 smithi133 bash[102858]: cluster 2023-10-09T04:58:28.013013+0000 mon.a (mon.0) 401 : cluster 3 Health check update: Degraded data redundancy: 15254/260094 objects degraded (5.865%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:28.900 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:28 smithi133 bash[102858]: audit 2023-10-09T04:58:28.416705+0000 mon.a (mon.0) 402 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:29.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:28 smithi194 bash[92666]: cluster 2023-10-09T04:58:27.472973+0000 mgr.y (mgr.25277) 259 : cluster 0 pgmap v206: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 42 MiB/s wr, 664 op/s; 15254/260094 objects degraded (5.865%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:58:29.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:28 smithi194 bash[92666]: cluster 2023-10-09T04:58:28.013013+0000 mon.a (mon.0) 401 : cluster 3 Health check update: Degraded data redundancy: 15254/260094 objects degraded (5.865%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:29.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:28 smithi194 bash[92666]: audit 2023-10-09T04:58:28.416705+0000 mon.a (mon.0) 402 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:29.414 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 75 16 90353 90337 75.2718 47.5 0.00908067 0.0132787 2023-10-09T04:58:30.414 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 76 16 91134 91118 74.9236 48.8125 0.0252255 0.0133402 2023-10-09T04:58:31.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:30 smithi133 bash[102858]: cluster 2023-10-09T04:58:29.473579+0000 mgr.y (mgr.25277) 260 : cluster 0 pgmap v207: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s; 15068/263145 objects degraded (5.726%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T04:58:31.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:30 smithi133 bash[105589]: cluster 2023-10-09T04:58:29.473579+0000 mgr.y (mgr.25277) 260 : cluster 0 pgmap v207: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s; 15068/263145 objects degraded (5.726%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T04:58:31.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:30 smithi194 bash[92666]: cluster 2023-10-09T04:58:29.473579+0000 mgr.y (mgr.25277) 260 : cluster 0 pgmap v207: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s; 15068/263145 objects degraded (5.726%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T04:58:31.414 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 77 16 91914 91898 74.5836 48.75 0.0305066 0.0134002 2023-10-09T04:58:32.414 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 78 16 92578 92562 74.1594 41.5 0.00860868 0.0134556 2023-10-09T04:58:32.943 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T04:58:32.943 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:32 smithi133 bash[105589]: cluster 2023-10-09T04:58:31.474418+0000 mgr.y (mgr.25277) 261 : cluster 0 pgmap v208: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s; 15068/271740 objects degraded (5.545%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T04:58:32.944 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:32 smithi133 bash[102858]: cluster 2023-10-09T04:58:31.474418+0000 mgr.y (mgr.25277) 261 : cluster 0 pgmap v208: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s; 15068/271740 objects degraded (5.545%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T04:58:33.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:32 smithi194 bash[92666]: cluster 2023-10-09T04:58:31.474418+0000 mgr.y (mgr.25277) 261 : cluster 0 pgmap v208: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s; 15068/271740 objects degraded (5.545%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T04:58:33.414 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 79 16 93303 93287 73.7942 45.3125 0.0259196 0.0135447 2023-10-09T04:58:34.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:33 smithi133 bash[102858]: cluster 2023-10-09T04:58:33.014229+0000 mon.a (mon.0) 403 : cluster 3 Health check update: Degraded data redundancy: 15068/271740 objects degraded (5.545%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:34.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:33 smithi133 bash[105589]: cluster 2023-10-09T04:58:33.014229+0000 mon.a (mon.0) 403 : cluster 3 Health check update: Degraded data redundancy: 15068/271740 objects degraded (5.545%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:34.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:33 smithi194 bash[92666]: cluster 2023-10-09T04:58:33.014229+0000 mon.a (mon.0) 403 : cluster 3 Health check update: Degraded data redundancy: 15068/271740 objects degraded (5.545%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:34.414 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T04:58:34.413294+0000 min lat: 0.00148753 max lat: 2.25006 avg lat: 0.0136072 2023-10-09T04:58:34.415 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:58:34.415 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 80 16 94050 94034 73.4553 46.6875 0.0082873 0.0136072 2023-10-09T04:58:34.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:58:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:58:34] "GET /metrics HTTP/1.1" 200 36299 "" "Prometheus/2.43.0" 2023-10-09T04:58:35.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:34 smithi133 bash[102858]: cluster 2023-10-09T04:58:33.474910+0000 mgr.y (mgr.25277) 262 : cluster 0 pgmap v209: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 41 MiB/s wr, 648 op/s; 15068/271740 objects degraded (5.545%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T04:58:35.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:34 smithi133 bash[105589]: cluster 2023-10-09T04:58:33.474910+0000 mgr.y (mgr.25277) 262 : cluster 0 pgmap v209: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 41 MiB/s wr, 648 op/s; 15068/271740 objects degraded (5.545%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T04:58:35.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:34 smithi194 bash[92666]: cluster 2023-10-09T04:58:33.474910+0000 mgr.y (mgr.25277) 262 : cluster 0 pgmap v209: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 41 MiB/s wr, 648 op/s; 15068/271740 objects degraded (5.545%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T04:58:35.414 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 81 16 94801 94785 73.1279 46.9375 0.0114597 0.0136681 2023-10-09T04:58:36.415 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 82 16 95553 95537 72.8092 47 0.0184875 0.0137282 2023-10-09T04:58:37.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:36 smithi133 bash[102858]: cluster 2023-10-09T04:58:35.478668+0000 mgr.y (mgr.25277) 263 : cluster 0 pgmap v210: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s; 14816/282792 objects degraded (5.239%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T04:58:37.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:36 smithi133 bash[105589]: cluster 2023-10-09T04:58:35.478668+0000 mgr.y (mgr.25277) 263 : cluster 0 pgmap v210: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s; 14816/282792 objects degraded (5.239%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T04:58:37.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:36 smithi194 bash[92666]: cluster 2023-10-09T04:58:35.478668+0000 mgr.y (mgr.25277) 263 : cluster 0 pgmap v210: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s; 14816/282792 objects degraded (5.239%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T04:58:37.415 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 83 16 96281 96265 72.4801 45.5 0.0130628 0.0137878 2023-10-09T04:58:37.815 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:58:38.415 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 84 16 97008 96992 72.1581 45.4375 0.0229829 0.0138517 2023-10-09T04:58:39.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:38 smithi133 bash[102858]: cluster 2023-10-09T04:58:37.479484+0000 mgr.y (mgr.25277) 264 : cluster 0 pgmap v211: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 39 MiB/s wr, 629 op/s; 14816/282792 objects degraded (5.239%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T04:58:39.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:38 smithi133 bash[102858]: cluster 2023-10-09T04:58:38.015537+0000 mon.a (mon.0) 404 : cluster 3 Health check update: Degraded data redundancy: 14816/282792 objects degraded (5.239%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:39.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:38 smithi133 bash[105589]: cluster 2023-10-09T04:58:37.479484+0000 mgr.y (mgr.25277) 264 : cluster 0 pgmap v211: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 39 MiB/s wr, 629 op/s; 14816/282792 objects degraded (5.239%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T04:58:39.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:38 smithi133 bash[105589]: cluster 2023-10-09T04:58:38.015537+0000 mon.a (mon.0) 404 : cluster 3 Health check update: Degraded data redundancy: 14816/282792 objects degraded (5.239%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:39.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:38 smithi194 bash[92666]: cluster 2023-10-09T04:58:37.479484+0000 mgr.y (mgr.25277) 264 : cluster 0 pgmap v211: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 39 MiB/s wr, 629 op/s; 14816/282792 objects degraded (5.239%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T04:58:39.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:38 smithi194 bash[92666]: cluster 2023-10-09T04:58:38.015537+0000 mon.a (mon.0) 404 : cluster 3 Health check update: Degraded data redundancy: 14816/282792 objects degraded (5.239%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:39.415 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 85 16 97775 97759 71.8731 47.9375 0.0198532 0.0139074 2023-10-09T04:58:40.415 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 86 16 98518 98502 71.5773 46.4375 0.0238529 0.0139649 2023-10-09T04:58:40.673 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:40 smithi133 bash[102858]: cluster 2023-10-09T04:58:39.480270+0000 mgr.y (mgr.25277) 265 : cluster 0 pgmap v212: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 44 MiB/s wr, 706 op/s; 14562/285540 objects degraded (5.100%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T04:58:41.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:40 smithi133 bash[105589]: cluster 2023-10-09T04:58:39.480270+0000 mgr.y (mgr.25277) 265 : cluster 0 pgmap v212: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 44 MiB/s wr, 706 op/s; 14562/285540 objects degraded (5.100%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T04:58:41.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:40 smithi194 bash[92666]: cluster 2023-10-09T04:58:39.480270+0000 mgr.y (mgr.25277) 265 : cluster 0 pgmap v212: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 44 MiB/s wr, 706 op/s; 14562/285540 objects degraded (5.100%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T04:58:41.415 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 87 16 99249 99233 71.2797 45.6875 0.0160216 0.0140233 2023-10-09T04:58:42.415 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 88 16 99998 99982 71.0016 46.8125 0.0486547 0.014078 2023-10-09T04:58:42.942 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:42 smithi133 bash[105589]: cluster 2023-10-09T04:58:41.480978+0000 mgr.y (mgr.25277) 266 : cluster 0 pgmap v213: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s; 14562/294006 objects degraded (4.953%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T04:58:42.942 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:42 smithi133 bash[102858]: cluster 2023-10-09T04:58:41.480978+0000 mgr.y (mgr.25277) 266 : cluster 0 pgmap v213: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s; 14562/294006 objects degraded (4.953%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T04:58:43.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:42 smithi194 bash[92666]: cluster 2023-10-09T04:58:41.480978+0000 mgr.y (mgr.25277) 266 : cluster 0 pgmap v213: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s; 14562/294006 objects degraded (4.953%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T04:58:43.415 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 89 16 100702 100686 70.6982 44 0.0182302 0.0141366 2023-10-09T04:58:43.643 INFO:teuthology.orchestra.run.smithi133.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:58:44.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:43 smithi133 bash[102858]: cluster 2023-10-09T04:58:43.016636+0000 mon.a (mon.0) 405 : cluster 3 Health check update: Degraded data redundancy: 14562/294006 objects degraded (4.953%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:44.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:43 smithi133 bash[102858]: audit 2023-10-09T04:58:43.416794+0000 mon.a (mon.0) 406 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:44.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:43 smithi133 bash[102858]: audit 2023-10-09T04:58:43.641320+0000 mon.a (mon.0) 407 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:58:44.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:43 smithi133 bash[102858]: audit 2023-10-09T04:58:43.653806+0000 mon.a (mon.0) 408 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:58:44.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:43 smithi133 bash[105589]: cluster 2023-10-09T04:58:43.016636+0000 mon.a (mon.0) 405 : cluster 3 Health check update: Degraded data redundancy: 14562/294006 objects degraded (4.953%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:44.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:43 smithi133 bash[105589]: audit 2023-10-09T04:58:43.416794+0000 mon.a (mon.0) 406 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:44.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:43 smithi133 bash[105589]: audit 2023-10-09T04:58:43.641320+0000 mon.a (mon.0) 407 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:58:44.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:43 smithi133 bash[105589]: audit 2023-10-09T04:58:43.653806+0000 mon.a (mon.0) 408 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:58:44.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:43 smithi194 bash[92666]: cluster 2023-10-09T04:58:43.016636+0000 mon.a (mon.0) 405 : cluster 3 Health check update: Degraded data redundancy: 14562/294006 objects degraded (4.953%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:44.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:43 smithi194 bash[92666]: audit 2023-10-09T04:58:43.416794+0000 mon.a (mon.0) 406 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:44.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:43 smithi194 bash[92666]: audit 2023-10-09T04:58:43.641320+0000 mon.a (mon.0) 407 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:58:44.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:43 smithi194 bash[92666]: audit 2023-10-09T04:58:43.653806+0000 mon.a (mon.0) 408 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:58:44.282 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:58:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:58:44] "GET /metrics HTTP/1.1" 200 36303 "" "Prometheus/2.43.0" 2023-10-09T04:58:44.415 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 90 8 101372 101364 70.3835 42.375 0.0219524 0.0142016 2023-10-09T04:58:44.962 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:44 smithi133 bash[105589]: cluster 2023-10-09T04:58:43.481531+0000 mgr.y (mgr.25277) 267 : cluster 0 pgmap v214: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 39 MiB/s wr, 618 op/s; 14562/294006 objects degraded (4.953%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T04:58:44.962 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:44 smithi133 bash[105589]: audit 2023-10-09T04:58:43.630326+0000 mgr.y (mgr.25277) 268 : audit 0 from='client.54145 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:58:44.962 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:44 smithi133 bash[105589]: cephadm 2023-10-09T04:58:43.630884+0000 mgr.y (mgr.25277) 269 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:58:44.963 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:44 smithi133 bash[102858]: cluster 2023-10-09T04:58:43.481531+0000 mgr.y (mgr.25277) 267 : cluster 0 pgmap v214: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 39 MiB/s wr, 618 op/s; 14562/294006 objects degraded (4.953%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T04:58:44.963 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:44 smithi133 bash[102858]: audit 2023-10-09T04:58:43.630326+0000 mgr.y (mgr.25277) 268 : audit 0 from='client.54145 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:58:44.963 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:44 smithi133 bash[102858]: cephadm 2023-10-09T04:58:43.630884+0000 mgr.y (mgr.25277) 269 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:58:44.964 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for half of osds to upgrade' 2023-10-09T04:58:45.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:44 smithi194 bash[92666]: cluster 2023-10-09T04:58:43.481531+0000 mgr.y (mgr.25277) 267 : cluster 0 pgmap v214: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 39 MiB/s wr, 618 op/s; 14562/294006 objects degraded (4.953%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T04:58:45.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:44 smithi194 bash[92666]: audit 2023-10-09T04:58:43.630326+0000 mgr.y (mgr.25277) 268 : audit 0 from='client.54145 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:58:45.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:44 smithi194 bash[92666]: cephadm 2023-10-09T04:58:43.630884+0000 mgr.y (mgr.25277) 269 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:58:45.415 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Total time run: 90.0197 2023-10-09T04:58:45.416 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Total writes made: 101372 2023-10-09T04:58:45.416 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Write size: 65536 2023-10-09T04:58:45.416 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Object size: 65536 2023-10-09T04:58:45.416 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Bandwidth (MB/sec): 70.3818 2023-10-09T04:58:45.416 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Stddev Bandwidth: 37.5388 2023-10-09T04:58:45.416 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Max bandwidth (MB/sec): 178.25 2023-10-09T04:58:45.416 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Min bandwidth (MB/sec): 0 2023-10-09T04:58:45.416 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Average IOPS: 1126 2023-10-09T04:58:45.416 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Stddev IOPS: 600.621 2023-10-09T04:58:45.416 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Max IOPS: 2852 2023-10-09T04:58:45.417 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Min IOPS: 0 2023-10-09T04:58:45.417 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Average Latency(s): 0.0142022 2023-10-09T04:58:45.417 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Stddev Latency(s): 0.0324254 2023-10-09T04:58:45.417 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Max latency(s): 2.25006 2023-10-09T04:58:45.417 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Min latency(s): 0.00148753 2023-10-09T04:58:45.431 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T04:58:47.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:46 smithi133 bash[102858]: cluster 2023-10-09T04:58:45.482441+0000 mgr.y (mgr.25277) 270 : cluster 0 pgmap v215: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s; 14281/304179 objects degraded (4.695%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T04:58:47.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:46 smithi133 bash[105589]: cluster 2023-10-09T04:58:45.482441+0000 mgr.y (mgr.25277) 270 : cluster 0 pgmap v215: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s; 14281/304179 objects degraded (4.695%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T04:58:47.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:46 smithi194 bash[92666]: cluster 2023-10-09T04:58:45.482441+0000 mgr.y (mgr.25277) 270 : cluster 0 pgmap v215: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s; 14281/304179 objects degraded (4.695%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T04:58:49.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:48 smithi133 bash[105589]: cluster 2023-10-09T04:58:47.483030+0000 mgr.y (mgr.25277) 271 : cluster 0 pgmap v216: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 37 MiB/s wr, 593 op/s; 14281/304179 objects degraded (4.695%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T04:58:49.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:48 smithi133 bash[105589]: cluster 2023-10-09T04:58:48.018164+0000 mon.a (mon.0) 409 : cluster 3 Health check update: Degraded data redundancy: 14281/304179 objects degraded (4.695%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:49.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:48 smithi133 bash[102858]: cluster 2023-10-09T04:58:47.483030+0000 mgr.y (mgr.25277) 271 : cluster 0 pgmap v216: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 37 MiB/s wr, 593 op/s; 14281/304179 objects degraded (4.695%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T04:58:49.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:48 smithi133 bash[102858]: cluster 2023-10-09T04:58:48.018164+0000 mon.a (mon.0) 409 : cluster 3 Health check update: Degraded data redundancy: 14281/304179 objects degraded (4.695%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:49.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:48 smithi194 bash[92666]: cluster 2023-10-09T04:58:47.483030+0000 mgr.y (mgr.25277) 271 : cluster 0 pgmap v216: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 37 MiB/s wr, 593 op/s; 14281/304179 objects degraded (4.695%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T04:58:49.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:48 smithi194 bash[92666]: cluster 2023-10-09T04:58:48.018164+0000 mon.a (mon.0) 409 : cluster 3 Health check update: Degraded data redundancy: 14281/304179 objects degraded (4.695%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:49.674 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:58:50.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:50 smithi133 bash[102858]: cluster 2023-10-09T04:58:49.025560+0000 osd.6 (osd.6) 15 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:58:50.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:50 smithi133 bash[102858]: cluster 2023-10-09T04:58:49.027765+0000 osd.5 (osd.5) 27 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:58:50.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:50 smithi133 bash[102858]: cluster 2023-10-09T04:58:49.027971+0000 mon.a (mon.0) 410 : cluster 0 osdmap e99: 8 total, 8 up, 8 in 2023-10-09T04:58:50.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:50 smithi133 bash[102858]: cluster 2023-10-09T04:58:49.043921+0000 osd.4 (osd.4) 19 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:58:50.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:50 smithi133 bash[102858]: cluster 2023-10-09T04:58:49.044011+0000 osd.3 (osd.3) 18 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:58:50.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:50 smithi133 bash[102858]: cluster 2023-10-09T04:58:49.483799+0000 mgr.y (mgr.25277) 272 : cluster 0 pgmap v218: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 39 MiB/s wr, 689 op/s; 13858/302130 objects degraded (4.587%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T04:58:50.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:50 smithi133 bash[102858]: audit 2023-10-09T04:58:49.909192+0000 mon.a (mon.0) 411 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:58:50.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:50 smithi133 bash[102858]: audit 2023-10-09T04:58:49.916672+0000 mon.a (mon.0) 412 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:58:50.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:50 smithi133 bash[105589]: cluster 2023-10-09T04:58:49.025560+0000 osd.6 (osd.6) 15 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:58:50.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:50 smithi133 bash[105589]: cluster 2023-10-09T04:58:49.027765+0000 osd.5 (osd.5) 27 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:58:50.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:50 smithi133 bash[105589]: cluster 2023-10-09T04:58:49.027971+0000 mon.a (mon.0) 410 : cluster 0 osdmap e99: 8 total, 8 up, 8 in 2023-10-09T04:58:50.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:50 smithi133 bash[105589]: cluster 2023-10-09T04:58:49.043921+0000 osd.4 (osd.4) 19 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:58:50.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:50 smithi133 bash[105589]: cluster 2023-10-09T04:58:49.044011+0000 osd.3 (osd.3) 18 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:58:50.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:50 smithi133 bash[105589]: cluster 2023-10-09T04:58:49.483799+0000 mgr.y (mgr.25277) 272 : cluster 0 pgmap v218: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 39 MiB/s wr, 689 op/s; 13858/302130 objects degraded (4.587%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T04:58:50.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:50 smithi133 bash[105589]: audit 2023-10-09T04:58:49.909192+0000 mon.a (mon.0) 411 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:58:50.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:50 smithi133 bash[105589]: audit 2023-10-09T04:58:49.916672+0000 mon.a (mon.0) 412 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:58:50.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:50 smithi194 bash[92666]: cluster 2023-10-09T04:58:49.025560+0000 osd.6 (osd.6) 15 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:58:50.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:50 smithi194 bash[92666]: cluster 2023-10-09T04:58:49.027765+0000 osd.5 (osd.5) 27 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:58:50.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:50 smithi194 bash[92666]: cluster 2023-10-09T04:58:49.027971+0000 mon.a (mon.0) 410 : cluster 0 osdmap e99: 8 total, 8 up, 8 in 2023-10-09T04:58:50.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:50 smithi194 bash[92666]: cluster 2023-10-09T04:58:49.043921+0000 osd.4 (osd.4) 19 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:58:50.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:50 smithi194 bash[92666]: cluster 2023-10-09T04:58:49.044011+0000 osd.3 (osd.3) 18 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:58:50.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:50 smithi194 bash[92666]: cluster 2023-10-09T04:58:49.483799+0000 mgr.y (mgr.25277) 272 : cluster 0 pgmap v218: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 39 MiB/s wr, 689 op/s; 13858/302130 objects degraded (4.587%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T04:58:50.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:50 smithi194 bash[92666]: audit 2023-10-09T04:58:49.909192+0000 mon.a (mon.0) 411 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:58:50.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:50 smithi194 bash[92666]: audit 2023-10-09T04:58:49.916672+0000 mon.a (mon.0) 412 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:58:51.300 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:58:50 smithi133 bash[91128]: debug 2023-10-09T04:58:50.818+0000 7ff45d5ed700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:58:51.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[105589]: cluster 2023-10-09T04:58:50.038080+0000 osd.5 (osd.5) 28 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T04:58:51.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[105589]: cluster 2023-10-09T04:58:50.038174+0000 osd.4 (osd.4) 20 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T04:58:51.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[105589]: cluster 2023-10-09T04:58:50.043392+0000 mon.a (mon.0) 413 : cluster 0 osdmap e100: 8 total, 8 up, 8 in 2023-10-09T04:58:51.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[105589]: cluster 2023-10-09T04:58:50.045187+0000 osd.4 (osd.4) 21 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T04:58:51.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[105589]: cluster 2023-10-09T04:58:50.048831+0000 osd.3 (osd.3) 19 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T04:58:51.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[105589]: cluster 2023-10-09T04:58:50.070798+0000 osd.2 (osd.2) 18 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T04:58:51.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[105589]: cluster 2023-10-09T04:58:50.071214+0000 osd.7 (osd.7) 15 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T04:58:51.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[105589]: cluster 2023-10-09T04:58:50.085820+0000 osd.6 (osd.6) 16 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T04:58:51.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[105589]: audit 2023-10-09T04:58:50.104644+0000 mon.a (mon.0) 414 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:58:51.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[105589]: audit 2023-10-09T04:58:50.115900+0000 mon.a (mon.0) 415 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:58:51.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[105589]: audit 2023-10-09T04:58:50.708142+0000 mon.a (mon.0) 416 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:58:51.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[105589]: audit 2023-10-09T04:58:50.709198+0000 mon.a (mon.0) 417 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:58:51.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[105589]: audit 2023-10-09T04:58:50.718824+0000 mon.a (mon.0) 418 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:58:51.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[105589]: audit 2023-10-09T04:58:50.804080+0000 mon.a (mon.0) 419 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:58:51.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[105589]: audit 2023-10-09T04:58:50.811708+0000 mon.a (mon.0) 420 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:58:51.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[105589]: audit 2023-10-09T04:58:50.814802+0000 mon.a (mon.0) 421 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:58:51.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[105589]: audit 2023-10-09T04:58:50.816945+0000 mon.a (mon.0) 422 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:58:51.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[105589]: audit 2023-10-09T04:58:50.822123+0000 mon.a (mon.0) 423 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:58:51.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[105589]: audit 2023-10-09T04:58:50.822372+0000 mgr.y (mgr.25277) 273 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:58:51.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[105589]: cephadm 2023-10-09T04:58:50.823757+0000 mgr.y (mgr.25277) 274 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:58:51.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[102858]: cluster 2023-10-09T04:58:50.038080+0000 osd.5 (osd.5) 28 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T04:58:51.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[102858]: cluster 2023-10-09T04:58:50.038174+0000 osd.4 (osd.4) 20 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T04:58:51.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[102858]: cluster 2023-10-09T04:58:50.043392+0000 mon.a (mon.0) 413 : cluster 0 osdmap e100: 8 total, 8 up, 8 in 2023-10-09T04:58:51.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[102858]: cluster 2023-10-09T04:58:50.045187+0000 osd.4 (osd.4) 21 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T04:58:51.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[102858]: cluster 2023-10-09T04:58:50.048831+0000 osd.3 (osd.3) 19 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T04:58:51.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[102858]: cluster 2023-10-09T04:58:50.070798+0000 osd.2 (osd.2) 18 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T04:58:51.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[102858]: cluster 2023-10-09T04:58:50.071214+0000 osd.7 (osd.7) 15 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T04:58:51.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[102858]: cluster 2023-10-09T04:58:50.085820+0000 osd.6 (osd.6) 16 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T04:58:51.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[102858]: audit 2023-10-09T04:58:50.104644+0000 mon.a (mon.0) 414 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:58:51.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[102858]: audit 2023-10-09T04:58:50.115900+0000 mon.a (mon.0) 415 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:58:51.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[102858]: audit 2023-10-09T04:58:50.708142+0000 mon.a (mon.0) 416 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:58:51.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[102858]: audit 2023-10-09T04:58:50.709198+0000 mon.a (mon.0) 417 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:58:51.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[102858]: audit 2023-10-09T04:58:50.718824+0000 mon.a (mon.0) 418 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:58:51.303 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[102858]: audit 2023-10-09T04:58:50.804080+0000 mon.a (mon.0) 419 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:58:51.304 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[102858]: audit 2023-10-09T04:58:50.811708+0000 mon.a (mon.0) 420 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:58:51.304 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[102858]: audit 2023-10-09T04:58:50.814802+0000 mon.a (mon.0) 421 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:58:51.304 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[102858]: audit 2023-10-09T04:58:50.816945+0000 mon.a (mon.0) 422 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:58:51.304 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[102858]: audit 2023-10-09T04:58:50.822123+0000 mon.a (mon.0) 423 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:58:51.304 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[102858]: audit 2023-10-09T04:58:50.822372+0000 mgr.y (mgr.25277) 273 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:58:51.304 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:51 smithi133 bash[102858]: cephadm 2023-10-09T04:58:50.823757+0000 mgr.y (mgr.25277) 274 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:58:51.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:51 smithi194 bash[92666]: cluster 2023-10-09T04:58:50.038080+0000 osd.5 (osd.5) 28 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T04:58:51.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:51 smithi194 bash[92666]: cluster 2023-10-09T04:58:50.038174+0000 osd.4 (osd.4) 20 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T04:58:51.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:51 smithi194 bash[92666]: cluster 2023-10-09T04:58:50.043392+0000 mon.a (mon.0) 413 : cluster 0 osdmap e100: 8 total, 8 up, 8 in 2023-10-09T04:58:51.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:51 smithi194 bash[92666]: cluster 2023-10-09T04:58:50.045187+0000 osd.4 (osd.4) 21 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T04:58:51.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:51 smithi194 bash[92666]: cluster 2023-10-09T04:58:50.048831+0000 osd.3 (osd.3) 19 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T04:58:51.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:51 smithi194 bash[92666]: cluster 2023-10-09T04:58:50.070798+0000 osd.2 (osd.2) 18 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T04:58:51.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:51 smithi194 bash[92666]: cluster 2023-10-09T04:58:50.071214+0000 osd.7 (osd.7) 15 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T04:58:51.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:51 smithi194 bash[92666]: cluster 2023-10-09T04:58:50.085820+0000 osd.6 (osd.6) 16 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T04:58:51.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:51 smithi194 bash[92666]: audit 2023-10-09T04:58:50.104644+0000 mon.a (mon.0) 414 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:58:51.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:51 smithi194 bash[92666]: audit 2023-10-09T04:58:50.115900+0000 mon.a (mon.0) 415 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:58:51.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:51 smithi194 bash[92666]: audit 2023-10-09T04:58:50.708142+0000 mon.a (mon.0) 416 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:58:51.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:51 smithi194 bash[92666]: audit 2023-10-09T04:58:50.709198+0000 mon.a (mon.0) 417 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:58:51.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:51 smithi194 bash[92666]: audit 2023-10-09T04:58:50.718824+0000 mon.a (mon.0) 418 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:58:51.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:51 smithi194 bash[92666]: audit 2023-10-09T04:58:50.804080+0000 mon.a (mon.0) 419 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:58:51.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:51 smithi194 bash[92666]: audit 2023-10-09T04:58:50.811708+0000 mon.a (mon.0) 420 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:58:51.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:51 smithi194 bash[92666]: audit 2023-10-09T04:58:50.814802+0000 mon.a (mon.0) 421 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:58:51.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:51 smithi194 bash[92666]: audit 2023-10-09T04:58:50.816945+0000 mon.a (mon.0) 422 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:58:51.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:51 smithi194 bash[92666]: audit 2023-10-09T04:58:50.822123+0000 mon.a (mon.0) 423 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:58:51.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:51 smithi194 bash[92666]: audit 2023-10-09T04:58:50.822372+0000 mgr.y (mgr.25277) 273 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:58:51.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:51 smithi194 bash[92666]: cephadm 2023-10-09T04:58:50.823757+0000 mgr.y (mgr.25277) 274 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:58:52.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:52 smithi133 bash[105589]: cluster 2023-10-09T04:58:51.484477+0000 mgr.y (mgr.25277) 275 : cluster 0 pgmap v220: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 127 B/s rd, 27 MiB/s wr, 782 op/s; 13769/295590 objects degraded (4.658%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T04:58:52.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:52 smithi133 bash[102858]: cluster 2023-10-09T04:58:51.484477+0000 mgr.y (mgr.25277) 275 : cluster 0 pgmap v220: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 127 B/s rd, 27 MiB/s wr, 782 op/s; 13769/295590 objects degraded (4.658%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T04:58:52.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:52 smithi194 bash[92666]: cluster 2023-10-09T04:58:51.484477+0000 mgr.y (mgr.25277) 275 : cluster 0 pgmap v220: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 127 B/s rd, 27 MiB/s wr, 782 op/s; 13769/295590 objects degraded (4.658%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T04:58:53.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:53 smithi133 bash[105589]: cluster 2023-10-09T04:58:53.019269+0000 mon.a (mon.0) 424 : cluster 3 Health check update: Degraded data redundancy: 13769/295590 objects degraded (4.658%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:53.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:53 smithi133 bash[102858]: cluster 2023-10-09T04:58:53.019269+0000 mon.a (mon.0) 424 : cluster 3 Health check update: Degraded data redundancy: 13769/295590 objects degraded (4.658%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:53.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:53 smithi194 bash[92666]: cluster 2023-10-09T04:58:53.019269+0000 mon.a (mon.0) 424 : cluster 3 Health check update: Degraded data redundancy: 13769/295590 objects degraded (4.658%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:54.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:54 smithi194 bash[92666]: cluster 2023-10-09T04:58:53.485023+0000 mgr.y (mgr.25277) 276 : cluster 0 pgmap v221: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 127 B/s rd, 24 KiB/s wr, 358 op/s; 13769/295590 objects degraded (4.658%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T04:58:54.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:58:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:58:54] "GET /metrics HTTP/1.1" 200 36303 "" "Prometheus/2.43.0" 2023-10-09T04:58:54.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:54 smithi133 bash[105589]: cluster 2023-10-09T04:58:53.485023+0000 mgr.y (mgr.25277) 276 : cluster 0 pgmap v221: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 127 B/s rd, 24 KiB/s wr, 358 op/s; 13769/295590 objects degraded (4.658%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T04:58:54.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:54 smithi133 bash[102858]: cluster 2023-10-09T04:58:53.485023+0000 mgr.y (mgr.25277) 276 : cluster 0 pgmap v221: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 127 B/s rd, 24 KiB/s wr, 358 op/s; 13769/295590 objects degraded (4.658%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T04:58:54.769 INFO:teuthology.orchestra.run.smithi133.stdout:wait for half of osds to upgrade 2023-10-09T04:58:55.873 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2023-10-09T04:58:56.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:56 smithi133 bash[105589]: cluster 2023-10-09T04:58:55.485855+0000 mgr.y (mgr.25277) 277 : cluster 0 pgmap v222: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 127 B/s rd, 24 KiB/s wr, 788 op/s; 13656/285279 objects degraded (4.787%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T04:58:56.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:56 smithi133 bash[102858]: cluster 2023-10-09T04:58:55.485855+0000 mgr.y (mgr.25277) 277 : cluster 0 pgmap v222: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 127 B/s rd, 24 KiB/s wr, 788 op/s; 13656/285279 objects degraded (4.787%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T04:58:56.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:56 smithi194 bash[92666]: cluster 2023-10-09T04:58:55.485855+0000 mgr.y (mgr.25277) 277 : cluster 0 pgmap v222: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 127 B/s rd, 24 KiB/s wr, 788 op/s; 13656/285279 objects degraded (4.787%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T04:58:58.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:58 smithi133 bash[102858]: cluster 2023-10-09T04:58:57.489724+0000 mgr.y (mgr.25277) 278 : cluster 0 pgmap v223: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 121 B/s rd, 23 KiB/s wr, 745 op/s; 13656/285279 objects degraded (4.787%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T04:58:58.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:58 smithi133 bash[102858]: cluster 2023-10-09T04:58:58.020377+0000 mon.a (mon.0) 425 : cluster 3 Health check update: Degraded data redundancy: 13656/285279 objects degraded (4.787%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:58.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:58:58 smithi133 bash[102858]: audit 2023-10-09T04:58:58.417166+0000 mon.a (mon.0) 426 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:58.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:58 smithi133 bash[105589]: cluster 2023-10-09T04:58:57.489724+0000 mgr.y (mgr.25277) 278 : cluster 0 pgmap v223: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 121 B/s rd, 23 KiB/s wr, 745 op/s; 13656/285279 objects degraded (4.787%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T04:58:58.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:58 smithi133 bash[105589]: cluster 2023-10-09T04:58:58.020377+0000 mon.a (mon.0) 425 : cluster 3 Health check update: Degraded data redundancy: 13656/285279 objects degraded (4.787%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:58.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:58:58 smithi133 bash[105589]: audit 2023-10-09T04:58:58.417166+0000 mon.a (mon.0) 426 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:58.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:58 smithi194 bash[92666]: cluster 2023-10-09T04:58:57.489724+0000 mgr.y (mgr.25277) 278 : cluster 0 pgmap v223: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 121 B/s rd, 23 KiB/s wr, 745 op/s; 13656/285279 objects degraded (4.787%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T04:58:58.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:58 smithi194 bash[92666]: cluster 2023-10-09T04:58:58.020377+0000 mon.a (mon.0) 425 : cluster 3 Health check update: Degraded data redundancy: 13656/285279 objects degraded (4.787%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:58.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:58:58 smithi194 bash[92666]: audit 2023-10-09T04:58:58.417166+0000 mon.a (mon.0) 426 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:00.717 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:59:00.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:00 smithi194 bash[92666]: cluster 2023-10-09T04:58:59.490380+0000 mgr.y (mgr.25277) 279 : cluster 0 pgmap v224: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 102 B/s rd, 102 B/s wr, 653 op/s; 13656/282531 objects degraded (4.833%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:59:01.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:00 smithi133 bash[102858]: cluster 2023-10-09T04:58:59.490380+0000 mgr.y (mgr.25277) 279 : cluster 0 pgmap v224: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 102 B/s rd, 102 B/s wr, 653 op/s; 13656/282531 objects degraded (4.833%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:59:01.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:00 smithi133 bash[105589]: cluster 2023-10-09T04:58:59.490380+0000 mgr.y (mgr.25277) 279 : cluster 0 pgmap v224: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 102 B/s rd, 102 B/s wr, 653 op/s; 13656/282531 objects degraded (4.833%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:59:02.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:02 smithi194 bash[92666]: cluster 2023-10-09T04:59:01.491224+0000 mgr.y (mgr.25277) 280 : cluster 0 pgmap v225: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 13550/274104 objects degraded (4.943%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T04:59:03.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:02 smithi133 bash[102858]: cluster 2023-10-09T04:59:01.491224+0000 mgr.y (mgr.25277) 280 : cluster 0 pgmap v225: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 13550/274104 objects degraded (4.943%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T04:59:03.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:02 smithi133 bash[105589]: cluster 2023-10-09T04:59:01.491224+0000 mgr.y (mgr.25277) 280 : cluster 0 pgmap v225: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 13550/274104 objects degraded (4.943%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T04:59:03.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:03 smithi194 bash[92666]: cluster 2023-10-09T04:59:03.021531+0000 mon.a (mon.0) 427 : cluster 3 Health check update: Degraded data redundancy: 13550/274104 objects degraded (4.943%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:04.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:03 smithi133 bash[102858]: cluster 2023-10-09T04:59:03.021531+0000 mon.a (mon.0) 427 : cluster 3 Health check update: Degraded data redundancy: 13550/274104 objects degraded (4.943%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:04.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:03 smithi133 bash[105589]: cluster 2023-10-09T04:59:03.021531+0000 mon.a (mon.0) 427 : cluster 3 Health check update: Degraded data redundancy: 13550/274104 objects degraded (4.943%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:04.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:59:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:59:04] "GET /metrics HTTP/1.1" 200 36308 "" "Prometheus/2.43.0" 2023-10-09T04:59:04.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:04 smithi194 bash[92666]: cluster 2023-10-09T04:59:03.491751+0000 mgr.y (mgr.25277) 281 : cluster 0 pgmap v226: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 13550/274104 objects degraded (4.943%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T04:59:04.905 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:04 smithi133 bash[102858]: cluster 2023-10-09T04:59:03.491751+0000 mgr.y (mgr.25277) 281 : cluster 0 pgmap v226: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 13550/274104 objects degraded (4.943%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T04:59:04.905 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:04 smithi133 bash[105589]: cluster 2023-10-09T04:59:03.491751+0000 mgr.y (mgr.25277) 281 : cluster 0 pgmap v226: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 13550/274104 objects degraded (4.943%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T04:59:06.299 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:59:05 smithi133 bash[91128]: debug 2023-10-09T04:59:05.838+0000 7ff45d5ed700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:59:06.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:06 smithi194 bash[92666]: cluster 2023-10-09T04:59:05.492573+0000 mgr.y (mgr.25277) 282 : cluster 0 pgmap v227: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 902 op/s; 13434/263064 objects degraded (5.107%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:59:06.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:06 smithi194 bash[92666]: audit 2023-10-09T04:59:05.841764+0000 mon.a (mon.0) 428 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:59:06.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:06 smithi194 bash[92666]: audit 2023-10-09T04:59:05.841989+0000 mgr.y (mgr.25277) 283 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:59:06.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:06 smithi194 bash[92666]: cephadm 2023-10-09T04:59:05.843416+0000 mgr.y (mgr.25277) 284 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:59:06.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:06 smithi194 bash[92666]: audit 2023-10-09T04:59:06.490417+0000 mon.c (mon.2) 4 : audit 0 from='client.? 172.21.15.133:0/2882660194' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:07.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:06 smithi133 bash[102858]: cluster 2023-10-09T04:59:05.492573+0000 mgr.y (mgr.25277) 282 : cluster 0 pgmap v227: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 902 op/s; 13434/263064 objects degraded (5.107%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:59:07.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:06 smithi133 bash[102858]: audit 2023-10-09T04:59:05.841764+0000 mon.a (mon.0) 428 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:59:07.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:06 smithi133 bash[102858]: audit 2023-10-09T04:59:05.841989+0000 mgr.y (mgr.25277) 283 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:59:07.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:06 smithi133 bash[102858]: cephadm 2023-10-09T04:59:05.843416+0000 mgr.y (mgr.25277) 284 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:59:07.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:06 smithi133 bash[102858]: audit 2023-10-09T04:59:06.490417+0000 mon.c (mon.2) 4 : audit 0 from='client.? 172.21.15.133:0/2882660194' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:07.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:06 smithi133 bash[105589]: cluster 2023-10-09T04:59:05.492573+0000 mgr.y (mgr.25277) 282 : cluster 0 pgmap v227: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 902 op/s; 13434/263064 objects degraded (5.107%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:59:07.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:06 smithi133 bash[105589]: audit 2023-10-09T04:59:05.841764+0000 mon.a (mon.0) 428 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:59:07.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:06 smithi133 bash[105589]: audit 2023-10-09T04:59:05.841989+0000 mgr.y (mgr.25277) 283 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:59:07.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:06 smithi133 bash[105589]: cephadm 2023-10-09T04:59:05.843416+0000 mgr.y (mgr.25277) 284 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:59:07.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:06 smithi133 bash[105589]: audit 2023-10-09T04:59:06.490417+0000 mon.c (mon.2) 4 : audit 0 from='client.? 172.21.15.133:0/2882660194' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:08.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:08 smithi194 bash[92666]: cluster 2023-10-09T04:59:07.493085+0000 mgr.y (mgr.25277) 285 : cluster 0 pgmap v228: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 13434/263064 objects degraded (5.107%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:59:08.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:08 smithi194 bash[92666]: cluster 2023-10-09T04:59:08.022848+0000 mon.a (mon.0) 429 : cluster 3 Health check update: Degraded data redundancy: 13434/263064 objects degraded (5.107%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:09.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:08 smithi133 bash[105589]: cluster 2023-10-09T04:59:07.493085+0000 mgr.y (mgr.25277) 285 : cluster 0 pgmap v228: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 13434/263064 objects degraded (5.107%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:59:09.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:08 smithi133 bash[105589]: cluster 2023-10-09T04:59:08.022848+0000 mon.a (mon.0) 429 : cluster 3 Health check update: Degraded data redundancy: 13434/263064 objects degraded (5.107%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:09.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:08 smithi133 bash[102858]: cluster 2023-10-09T04:59:07.493085+0000 mgr.y (mgr.25277) 285 : cluster 0 pgmap v228: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 13434/263064 objects degraded (5.107%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:59:09.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:08 smithi133 bash[102858]: cluster 2023-10-09T04:59:08.022848+0000 mon.a (mon.0) 429 : cluster 3 Health check update: Degraded data redundancy: 13434/263064 objects degraded (5.107%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:09.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:09 smithi194 bash[92666]: audit 2023-10-09T04:59:09.133831+0000 mon.a (mon.0) 430 : audit 0 from='client.? 172.21.15.133:0/2728608401' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:10.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:09 smithi133 bash[102858]: audit 2023-10-09T04:59:09.133831+0000 mon.a (mon.0) 430 : audit 0 from='client.? 172.21.15.133:0/2728608401' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:10.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:09 smithi133 bash[105589]: audit 2023-10-09T04:59:09.133831+0000 mon.a (mon.0) 430 : audit 0 from='client.? 172.21.15.133:0/2728608401' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:10.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:10 smithi194 bash[92666]: cluster 2023-10-09T04:59:09.493906+0000 mgr.y (mgr.25277) 286 : cluster 0 pgmap v229: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 13434/263064 objects degraded (5.107%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:59:11.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:10 smithi133 bash[102858]: cluster 2023-10-09T04:59:09.493906+0000 mgr.y (mgr.25277) 286 : cluster 0 pgmap v229: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 13434/263064 objects degraded (5.107%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:59:11.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:10 smithi133 bash[105589]: cluster 2023-10-09T04:59:09.493906+0000 mgr.y (mgr.25277) 286 : cluster 0 pgmap v229: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 13434/263064 objects degraded (5.107%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:59:12.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:12 smithi194 bash[92666]: cluster 2023-10-09T04:59:11.494766+0000 mgr.y (mgr.25277) 287 : cluster 0 pgmap v230: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 13322/252696 objects degraded (5.272%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:59:12.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:12 smithi194 bash[92666]: audit 2023-10-09T04:59:11.770974+0000 mon.c (mon.2) 5 : audit 0 from='client.? 172.21.15.133:0/1927912059' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:13.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:12 smithi133 bash[102858]: cluster 2023-10-09T04:59:11.494766+0000 mgr.y (mgr.25277) 287 : cluster 0 pgmap v230: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 13322/252696 objects degraded (5.272%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:59:13.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:12 smithi133 bash[102858]: audit 2023-10-09T04:59:11.770974+0000 mon.c (mon.2) 5 : audit 0 from='client.? 172.21.15.133:0/1927912059' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:13.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:12 smithi133 bash[105589]: cluster 2023-10-09T04:59:11.494766+0000 mgr.y (mgr.25277) 287 : cluster 0 pgmap v230: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 13322/252696 objects degraded (5.272%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:59:13.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:12 smithi133 bash[105589]: audit 2023-10-09T04:59:11.770974+0000 mon.c (mon.2) 5 : audit 0 from='client.? 172.21.15.133:0/1927912059' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:13.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:13 smithi194 bash[92666]: cluster 2023-10-09T04:59:13.023870+0000 mon.a (mon.0) 431 : cluster 3 Health check update: Degraded data redundancy: 13322/252696 objects degraded (5.272%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:13.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:13 smithi194 bash[92666]: audit 2023-10-09T04:59:13.417375+0000 mon.a (mon.0) 432 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:14.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:13 smithi133 bash[102858]: cluster 2023-10-09T04:59:13.023870+0000 mon.a (mon.0) 431 : cluster 3 Health check update: Degraded data redundancy: 13322/252696 objects degraded (5.272%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:14.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:13 smithi133 bash[102858]: audit 2023-10-09T04:59:13.417375+0000 mon.a (mon.0) 432 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:14.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:13 smithi133 bash[105589]: cluster 2023-10-09T04:59:13.023870+0000 mon.a (mon.0) 431 : cluster 3 Health check update: Degraded data redundancy: 13322/252696 objects degraded (5.272%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:14.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:13 smithi133 bash[105589]: audit 2023-10-09T04:59:13.417375+0000 mon.a (mon.0) 432 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:14.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:59:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:59:14] "GET /metrics HTTP/1.1" 200 36304 "" "Prometheus/2.43.0" 2023-10-09T04:59:14.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:14 smithi194 bash[92666]: cluster 2023-10-09T04:59:13.497496+0000 mgr.y (mgr.25277) 288 : cluster 0 pgmap v231: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 13322/252696 objects degraded (5.272%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:59:14.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:14 smithi194 bash[92666]: audit 2023-10-09T04:59:14.349847+0000 mon.c (mon.2) 6 : audit 0 from='client.? 172.21.15.133:0/968075596' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:15.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:14 smithi133 bash[102858]: cluster 2023-10-09T04:59:13.497496+0000 mgr.y (mgr.25277) 288 : cluster 0 pgmap v231: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 13322/252696 objects degraded (5.272%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:59:15.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:14 smithi133 bash[102858]: audit 2023-10-09T04:59:14.349847+0000 mon.c (mon.2) 6 : audit 0 from='client.? 172.21.15.133:0/968075596' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:15.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:14 smithi133 bash[105589]: cluster 2023-10-09T04:59:13.497496+0000 mgr.y (mgr.25277) 288 : cluster 0 pgmap v231: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 13322/252696 objects degraded (5.272%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:59:15.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:14 smithi133 bash[105589]: audit 2023-10-09T04:59:14.349847+0000 mon.c (mon.2) 6 : audit 0 from='client.? 172.21.15.133:0/968075596' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:16.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:16 smithi194 bash[92666]: cluster 2023-10-09T04:59:15.498352+0000 mgr.y (mgr.25277) 289 : cluster 0 pgmap v232: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 13322/244470 objects degraded (5.449%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:59:17.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:16 smithi133 bash[102858]: cluster 2023-10-09T04:59:15.498352+0000 mgr.y (mgr.25277) 289 : cluster 0 pgmap v232: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 13322/244470 objects degraded (5.449%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:59:17.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:16 smithi133 bash[105589]: cluster 2023-10-09T04:59:15.498352+0000 mgr.y (mgr.25277) 289 : cluster 0 pgmap v232: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 13322/244470 objects degraded (5.449%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:59:17.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:17 smithi194 bash[92666]: audit 2023-10-09T04:59:16.940317+0000 mon.a (mon.0) 433 : audit 0 from='client.? 172.21.15.133:0/1250608519' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:18.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:17 smithi133 bash[102858]: audit 2023-10-09T04:59:16.940317+0000 mon.a (mon.0) 433 : audit 0 from='client.? 172.21.15.133:0/1250608519' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:18.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:17 smithi133 bash[105589]: audit 2023-10-09T04:59:16.940317+0000 mon.a (mon.0) 433 : audit 0 from='client.? 172.21.15.133:0/1250608519' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:18.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:18 smithi194 bash[92666]: cluster 2023-10-09T04:59:17.498842+0000 mgr.y (mgr.25277) 290 : cluster 0 pgmap v233: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 13204/242514 objects degraded (5.445%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:59:18.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:18 smithi194 bash[92666]: cluster 2023-10-09T04:59:18.025978+0000 mon.a (mon.0) 434 : cluster 3 Health check update: Degraded data redundancy: 13204/242514 objects degraded (5.445%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:19.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:18 smithi133 bash[105589]: cluster 2023-10-09T04:59:17.498842+0000 mgr.y (mgr.25277) 290 : cluster 0 pgmap v233: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 13204/242514 objects degraded (5.445%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:59:19.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:18 smithi133 bash[105589]: cluster 2023-10-09T04:59:18.025978+0000 mon.a (mon.0) 434 : cluster 3 Health check update: Degraded data redundancy: 13204/242514 objects degraded (5.445%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:19.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:18 smithi133 bash[102858]: cluster 2023-10-09T04:59:17.498842+0000 mgr.y (mgr.25277) 290 : cluster 0 pgmap v233: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 13204/242514 objects degraded (5.445%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:59:19.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:18 smithi133 bash[102858]: cluster 2023-10-09T04:59:18.025978+0000 mon.a (mon.0) 434 : cluster 3 Health check update: Degraded data redundancy: 13204/242514 objects degraded (5.445%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:20.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:19 smithi133 bash[102858]: audit 2023-10-09T04:59:19.557813+0000 mon.c (mon.2) 7 : audit 0 from='client.? 172.21.15.133:0/1548154530' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:20.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:19 smithi133 bash[105589]: audit 2023-10-09T04:59:19.557813+0000 mon.c (mon.2) 7 : audit 0 from='client.? 172.21.15.133:0/1548154530' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:20.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:19 smithi194 bash[92666]: audit 2023-10-09T04:59:19.557813+0000 mon.c (mon.2) 7 : audit 0 from='client.? 172.21.15.133:0/1548154530' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:21.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:20 smithi133 bash[102858]: cluster 2023-10-09T04:59:19.499354+0000 mgr.y (mgr.25277) 291 : cluster 0 pgmap v234: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 13204/242514 objects degraded (5.445%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:59:21.050 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:59:20 smithi133 bash[91128]: debug 2023-10-09T04:59:20.854+0000 7ff45d5ed700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:59:21.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:20 smithi133 bash[105589]: cluster 2023-10-09T04:59:19.499354+0000 mgr.y (mgr.25277) 291 : cluster 0 pgmap v234: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 13204/242514 objects degraded (5.445%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:59:21.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:20 smithi194 bash[92666]: cluster 2023-10-09T04:59:19.499354+0000 mgr.y (mgr.25277) 291 : cluster 0 pgmap v234: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 13204/242514 objects degraded (5.445%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:59:22.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:21 smithi133 bash[102858]: audit 2023-10-09T04:59:20.858075+0000 mon.a (mon.0) 435 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:59:22.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:21 smithi133 bash[102858]: audit 2023-10-09T04:59:20.858376+0000 mgr.y (mgr.25277) 292 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:59:22.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:21 smithi133 bash[102858]: cephadm 2023-10-09T04:59:20.860005+0000 mgr.y (mgr.25277) 293 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:59:22.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:21 smithi133 bash[105589]: audit 2023-10-09T04:59:20.858075+0000 mon.a (mon.0) 435 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:59:22.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:21 smithi133 bash[105589]: audit 2023-10-09T04:59:20.858376+0000 mgr.y (mgr.25277) 292 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:59:22.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:21 smithi133 bash[105589]: cephadm 2023-10-09T04:59:20.860005+0000 mgr.y (mgr.25277) 293 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:59:22.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:21 smithi194 bash[92666]: audit 2023-10-09T04:59:20.858075+0000 mon.a (mon.0) 435 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:59:22.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:21 smithi194 bash[92666]: audit 2023-10-09T04:59:20.858376+0000 mgr.y (mgr.25277) 292 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:59:22.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:21 smithi194 bash[92666]: cephadm 2023-10-09T04:59:20.860005+0000 mgr.y (mgr.25277) 293 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:59:23.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:22 smithi133 bash[105589]: cluster 2023-10-09T04:59:21.502105+0000 mgr.y (mgr.25277) 294 : cluster 0 pgmap v235: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 852 op/s; 13080/232359 objects degraded (5.629%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T04:59:23.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:22 smithi133 bash[105589]: audit 2023-10-09T04:59:22.189411+0000 mon.a (mon.0) 436 : audit 0 from='client.? 172.21.15.133:0/2214091123' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:23.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:22 smithi133 bash[102858]: cluster 2023-10-09T04:59:21.502105+0000 mgr.y (mgr.25277) 294 : cluster 0 pgmap v235: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 852 op/s; 13080/232359 objects degraded (5.629%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T04:59:23.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:22 smithi133 bash[102858]: audit 2023-10-09T04:59:22.189411+0000 mon.a (mon.0) 436 : audit 0 from='client.? 172.21.15.133:0/2214091123' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:23.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:22 smithi194 bash[92666]: cluster 2023-10-09T04:59:21.502105+0000 mgr.y (mgr.25277) 294 : cluster 0 pgmap v235: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 852 op/s; 13080/232359 objects degraded (5.629%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T04:59:23.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:22 smithi194 bash[92666]: audit 2023-10-09T04:59:22.189411+0000 mon.a (mon.0) 436 : audit 0 from='client.? 172.21.15.133:0/2214091123' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:24.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:23 smithi133 bash[102858]: cluster 2023-10-09T04:59:23.027173+0000 mon.a (mon.0) 437 : cluster 3 Health check update: Degraded data redundancy: 13080/232359 objects degraded (5.629%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:24.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:23 smithi133 bash[105589]: cluster 2023-10-09T04:59:23.027173+0000 mon.a (mon.0) 437 : cluster 3 Health check update: Degraded data redundancy: 13080/232359 objects degraded (5.629%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:24.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:23 smithi194 bash[92666]: cluster 2023-10-09T04:59:23.027173+0000 mon.a (mon.0) 437 : cluster 3 Health check update: Degraded data redundancy: 13080/232359 objects degraded (5.629%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:24.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:59:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:59:24] "GET /metrics HTTP/1.1" 200 36304 "" "Prometheus/2.43.0" 2023-10-09T04:59:25.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:24 smithi133 bash[105589]: cluster 2023-10-09T04:59:23.502630+0000 mgr.y (mgr.25277) 295 : cluster 0 pgmap v236: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 13080/232359 objects degraded (5.629%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:59:25.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:24 smithi133 bash[102858]: cluster 2023-10-09T04:59:23.502630+0000 mgr.y (mgr.25277) 295 : cluster 0 pgmap v236: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 13080/232359 objects degraded (5.629%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:59:25.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:24 smithi194 bash[92666]: cluster 2023-10-09T04:59:23.502630+0000 mgr.y (mgr.25277) 295 : cluster 0 pgmap v236: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 13080/232359 objects degraded (5.629%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:59:26.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:25 smithi133 bash[105589]: audit 2023-10-09T04:59:24.830135+0000 mon.a (mon.0) 438 : audit 0 from='client.? 172.21.15.133:0/3346664259' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:26.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:25 smithi133 bash[102858]: audit 2023-10-09T04:59:24.830135+0000 mon.a (mon.0) 438 : audit 0 from='client.? 172.21.15.133:0/3346664259' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:26.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:25 smithi194 bash[92666]: audit 2023-10-09T04:59:24.830135+0000 mon.a (mon.0) 438 : audit 0 from='client.? 172.21.15.133:0/3346664259' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:27.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:26 smithi133 bash[102858]: cluster 2023-10-09T04:59:25.503415+0000 mgr.y (mgr.25277) 296 : cluster 0 pgmap v237: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 13080/224340 objects degraded (5.830%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:59:27.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:26 smithi133 bash[105589]: cluster 2023-10-09T04:59:25.503415+0000 mgr.y (mgr.25277) 296 : cluster 0 pgmap v237: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 13080/224340 objects degraded (5.830%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:59:27.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:26 smithi194 bash[92666]: cluster 2023-10-09T04:59:25.503415+0000 mgr.y (mgr.25277) 296 : cluster 0 pgmap v237: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 13080/224340 objects degraded (5.830%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:59:28.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:27 smithi133 bash[102858]: audit 2023-10-09T04:59:27.480481+0000 mon.c (mon.2) 8 : audit 0 from='client.? 172.21.15.133:0/2248384564' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:28.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:27 smithi133 bash[105589]: audit 2023-10-09T04:59:27.480481+0000 mon.c (mon.2) 8 : audit 0 from='client.? 172.21.15.133:0/2248384564' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:28.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:27 smithi194 bash[92666]: audit 2023-10-09T04:59:27.480481+0000 mon.c (mon.2) 8 : audit 0 from='client.? 172.21.15.133:0/2248384564' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:29.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:28 smithi133 bash[102858]: cluster 2023-10-09T04:59:27.504015+0000 mgr.y (mgr.25277) 297 : cluster 0 pgmap v238: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 12964/222462 objects degraded (5.828%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T04:59:29.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:28 smithi133 bash[102858]: cluster 2023-10-09T04:59:28.028514+0000 mon.a (mon.0) 439 : cluster 3 Health check update: Degraded data redundancy: 12964/222462 objects degraded (5.828%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:29.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:28 smithi133 bash[102858]: audit 2023-10-09T04:59:28.420856+0000 mon.a (mon.0) 440 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:29.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:28 smithi133 bash[105589]: cluster 2023-10-09T04:59:27.504015+0000 mgr.y (mgr.25277) 297 : cluster 0 pgmap v238: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 12964/222462 objects degraded (5.828%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T04:59:29.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:28 smithi133 bash[105589]: cluster 2023-10-09T04:59:28.028514+0000 mon.a (mon.0) 439 : cluster 3 Health check update: Degraded data redundancy: 12964/222462 objects degraded (5.828%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:29.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:28 smithi133 bash[105589]: audit 2023-10-09T04:59:28.420856+0000 mon.a (mon.0) 440 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:29.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:28 smithi194 bash[92666]: cluster 2023-10-09T04:59:27.504015+0000 mgr.y (mgr.25277) 297 : cluster 0 pgmap v238: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 12964/222462 objects degraded (5.828%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T04:59:29.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:28 smithi194 bash[92666]: cluster 2023-10-09T04:59:28.028514+0000 mon.a (mon.0) 439 : cluster 3 Health check update: Degraded data redundancy: 12964/222462 objects degraded (5.828%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:29.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:28 smithi194 bash[92666]: audit 2023-10-09T04:59:28.420856+0000 mon.a (mon.0) 440 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:31.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:30 smithi133 bash[102858]: cluster 2023-10-09T04:59:29.504751+0000 mgr.y (mgr.25277) 298 : cluster 0 pgmap v239: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 12964/222462 objects degraded (5.828%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:59:31.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:30 smithi133 bash[102858]: audit 2023-10-09T04:59:30.114721+0000 mon.c (mon.2) 9 : audit 0 from='client.? 172.21.15.133:0/666905123' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:31.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:30 smithi133 bash[105589]: cluster 2023-10-09T04:59:29.504751+0000 mgr.y (mgr.25277) 298 : cluster 0 pgmap v239: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 12964/222462 objects degraded (5.828%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:59:31.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:30 smithi133 bash[105589]: audit 2023-10-09T04:59:30.114721+0000 mon.c (mon.2) 9 : audit 0 from='client.? 172.21.15.133:0/666905123' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:31.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:30 smithi194 bash[92666]: cluster 2023-10-09T04:59:29.504751+0000 mgr.y (mgr.25277) 298 : cluster 0 pgmap v239: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 12964/222462 objects degraded (5.828%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:59:31.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:30 smithi194 bash[92666]: audit 2023-10-09T04:59:30.114721+0000 mon.c (mon.2) 9 : audit 0 from='client.? 172.21.15.133:0/666905123' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:33.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:32 smithi133 bash[102858]: cluster 2023-10-09T04:59:31.505576+0000 mgr.y (mgr.25277) 299 : cluster 0 pgmap v240: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 12839/212682 objects degraded (6.037%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:59:33.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:32 smithi133 bash[102858]: audit 2023-10-09T04:59:32.685049+0000 mon.c (mon.2) 10 : audit 0 from='client.? 172.21.15.133:0/2072900136' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:33.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:32 smithi133 bash[105589]: cluster 2023-10-09T04:59:31.505576+0000 mgr.y (mgr.25277) 299 : cluster 0 pgmap v240: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 12839/212682 objects degraded (6.037%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:59:33.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:32 smithi133 bash[105589]: audit 2023-10-09T04:59:32.685049+0000 mon.c (mon.2) 10 : audit 0 from='client.? 172.21.15.133:0/2072900136' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:33.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:32 smithi194 bash[92666]: cluster 2023-10-09T04:59:31.505576+0000 mgr.y (mgr.25277) 299 : cluster 0 pgmap v240: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 12839/212682 objects degraded (6.037%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:59:33.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:32 smithi194 bash[92666]: audit 2023-10-09T04:59:32.685049+0000 mon.c (mon.2) 10 : audit 0 from='client.? 172.21.15.133:0/2072900136' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:34.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:33 smithi133 bash[102858]: cluster 2023-10-09T04:59:33.032735+0000 mon.a (mon.0) 441 : cluster 3 Health check update: Degraded data redundancy: 12839/212682 objects degraded (6.037%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:34.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:33 smithi133 bash[105589]: cluster 2023-10-09T04:59:33.032735+0000 mon.a (mon.0) 441 : cluster 3 Health check update: Degraded data redundancy: 12839/212682 objects degraded (6.037%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:34.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:33 smithi194 bash[92666]: cluster 2023-10-09T04:59:33.032735+0000 mon.a (mon.0) 441 : cluster 3 Health check update: Degraded data redundancy: 12839/212682 objects degraded (6.037%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:34.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:59:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:59:34] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.43.0" 2023-10-09T04:59:35.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:34 smithi133 bash[105589]: cluster 2023-10-09T04:59:33.506145+0000 mgr.y (mgr.25277) 300 : cluster 0 pgmap v241: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 12839/212682 objects degraded (6.037%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:59:35.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:34 smithi133 bash[102858]: cluster 2023-10-09T04:59:33.506145+0000 mgr.y (mgr.25277) 300 : cluster 0 pgmap v241: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 12839/212682 objects degraded (6.037%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:59:35.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:34 smithi194 bash[92666]: cluster 2023-10-09T04:59:33.506145+0000 mgr.y (mgr.25277) 300 : cluster 0 pgmap v241: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 12839/212682 objects degraded (6.037%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:59:36.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:35 smithi133 bash[102858]: audit 2023-10-09T04:59:35.279403+0000 mon.c (mon.2) 11 : audit 0 from='client.? 172.21.15.133:0/3998502639' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:36.050 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:59:35 smithi133 bash[91128]: debug 2023-10-09T04:59:35.869+0000 7ff45d5ed700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:59:36.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:35 smithi133 bash[105589]: audit 2023-10-09T04:59:35.279403+0000 mon.c (mon.2) 11 : audit 0 from='client.? 172.21.15.133:0/3998502639' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:36.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:35 smithi194 bash[92666]: audit 2023-10-09T04:59:35.279403+0000 mon.c (mon.2) 11 : audit 0 from='client.? 172.21.15.133:0/3998502639' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:37.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:36 smithi133 bash[102858]: cluster 2023-10-09T04:59:35.506898+0000 mgr.y (mgr.25277) 301 : cluster 0 pgmap v242: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 12839/204549 objects degraded (6.277%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:59:37.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:36 smithi133 bash[102858]: audit 2023-10-09T04:59:35.875801+0000 mon.a (mon.0) 442 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:59:37.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:36 smithi133 bash[102858]: audit 2023-10-09T04:59:35.876354+0000 mgr.y (mgr.25277) 302 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:59:37.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:36 smithi133 bash[102858]: cephadm 2023-10-09T04:59:35.877652+0000 mgr.y (mgr.25277) 303 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:59:37.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:36 smithi133 bash[105589]: cluster 2023-10-09T04:59:35.506898+0000 mgr.y (mgr.25277) 301 : cluster 0 pgmap v242: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 12839/204549 objects degraded (6.277%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:59:37.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:36 smithi133 bash[105589]: audit 2023-10-09T04:59:35.875801+0000 mon.a (mon.0) 442 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:59:37.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:36 smithi133 bash[105589]: audit 2023-10-09T04:59:35.876354+0000 mgr.y (mgr.25277) 302 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:59:37.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:36 smithi133 bash[105589]: cephadm 2023-10-09T04:59:35.877652+0000 mgr.y (mgr.25277) 303 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:59:37.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:36 smithi194 bash[92666]: cluster 2023-10-09T04:59:35.506898+0000 mgr.y (mgr.25277) 301 : cluster 0 pgmap v242: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 12839/204549 objects degraded (6.277%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:59:37.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:36 smithi194 bash[92666]: audit 2023-10-09T04:59:35.875801+0000 mon.a (mon.0) 442 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:59:37.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:36 smithi194 bash[92666]: audit 2023-10-09T04:59:35.876354+0000 mgr.y (mgr.25277) 302 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:59:37.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:36 smithi194 bash[92666]: cephadm 2023-10-09T04:59:35.877652+0000 mgr.y (mgr.25277) 303 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:59:39.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:38 smithi133 bash[105589]: cluster 2023-10-09T04:59:37.507525+0000 mgr.y (mgr.25277) 304 : cluster 0 pgmap v243: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 12697/202734 objects degraded (6.263%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T04:59:39.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:38 smithi133 bash[105589]: audit 2023-10-09T04:59:37.860804+0000 mon.c (mon.2) 12 : audit 0 from='client.? 172.21.15.133:0/957912745' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:39.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:38 smithi133 bash[105589]: cluster 2023-10-09T04:59:38.033759+0000 mon.a (mon.0) 443 : cluster 3 Health check update: Degraded data redundancy: 12697/202734 objects degraded (6.263%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:39.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:38 smithi133 bash[102858]: cluster 2023-10-09T04:59:37.507525+0000 mgr.y (mgr.25277) 304 : cluster 0 pgmap v243: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 12697/202734 objects degraded (6.263%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T04:59:39.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:38 smithi133 bash[102858]: audit 2023-10-09T04:59:37.860804+0000 mon.c (mon.2) 12 : audit 0 from='client.? 172.21.15.133:0/957912745' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:39.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:38 smithi133 bash[102858]: cluster 2023-10-09T04:59:38.033759+0000 mon.a (mon.0) 443 : cluster 3 Health check update: Degraded data redundancy: 12697/202734 objects degraded (6.263%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:39.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:38 smithi194 bash[92666]: cluster 2023-10-09T04:59:37.507525+0000 mgr.y (mgr.25277) 304 : cluster 0 pgmap v243: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 12697/202734 objects degraded (6.263%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T04:59:39.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:38 smithi194 bash[92666]: audit 2023-10-09T04:59:37.860804+0000 mon.c (mon.2) 12 : audit 0 from='client.? 172.21.15.133:0/957912745' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:39.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:38 smithi194 bash[92666]: cluster 2023-10-09T04:59:38.033759+0000 mon.a (mon.0) 443 : cluster 3 Health check update: Degraded data redundancy: 12697/202734 objects degraded (6.263%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:41.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:40 smithi133 bash[102858]: cluster 2023-10-09T04:59:39.508054+0000 mgr.y (mgr.25277) 305 : cluster 0 pgmap v244: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 12697/202734 objects degraded (6.263%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:59:41.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:40 smithi133 bash[102858]: audit 2023-10-09T04:59:40.474120+0000 mon.a (mon.0) 444 : audit 0 from='client.? 172.21.15.133:0/2398719153' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:41.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:40 smithi133 bash[105589]: cluster 2023-10-09T04:59:39.508054+0000 mgr.y (mgr.25277) 305 : cluster 0 pgmap v244: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 12697/202734 objects degraded (6.263%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:59:41.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:40 smithi133 bash[105589]: audit 2023-10-09T04:59:40.474120+0000 mon.a (mon.0) 444 : audit 0 from='client.? 172.21.15.133:0/2398719153' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:41.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:40 smithi194 bash[92666]: cluster 2023-10-09T04:59:39.508054+0000 mgr.y (mgr.25277) 305 : cluster 0 pgmap v244: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 12697/202734 objects degraded (6.263%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:59:41.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:40 smithi194 bash[92666]: audit 2023-10-09T04:59:40.474120+0000 mon.a (mon.0) 444 : audit 0 from='client.? 172.21.15.133:0/2398719153' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:43.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:42 smithi133 bash[102858]: cluster 2023-10-09T04:59:41.508910+0000 mgr.y (mgr.25277) 306 : cluster 0 pgmap v245: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 12562/192663 objects degraded (6.520%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T04:59:43.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:42 smithi133 bash[105589]: cluster 2023-10-09T04:59:41.508910+0000 mgr.y (mgr.25277) 306 : cluster 0 pgmap v245: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 12562/192663 objects degraded (6.520%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T04:59:43.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:42 smithi194 bash[92666]: cluster 2023-10-09T04:59:41.508910+0000 mgr.y (mgr.25277) 306 : cluster 0 pgmap v245: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 12562/192663 objects degraded (6.520%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T04:59:44.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:43 smithi133 bash[102858]: cluster 2023-10-09T04:59:43.034934+0000 mon.a (mon.0) 445 : cluster 3 Health check update: Degraded data redundancy: 12562/192663 objects degraded (6.520%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:44.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:43 smithi133 bash[102858]: audit 2023-10-09T04:59:43.064286+0000 mon.a (mon.0) 446 : audit 0 from='client.? 172.21.15.133:0/1199945265' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:44.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:43 smithi133 bash[102858]: audit 2023-10-09T04:59:43.421265+0000 mon.a (mon.0) 447 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:44.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:43 smithi133 bash[105589]: cluster 2023-10-09T04:59:43.034934+0000 mon.a (mon.0) 445 : cluster 3 Health check update: Degraded data redundancy: 12562/192663 objects degraded (6.520%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:44.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:43 smithi133 bash[105589]: audit 2023-10-09T04:59:43.064286+0000 mon.a (mon.0) 446 : audit 0 from='client.? 172.21.15.133:0/1199945265' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:44.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:43 smithi133 bash[105589]: audit 2023-10-09T04:59:43.421265+0000 mon.a (mon.0) 447 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:44.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:43 smithi194 bash[92666]: cluster 2023-10-09T04:59:43.034934+0000 mon.a (mon.0) 445 : cluster 3 Health check update: Degraded data redundancy: 12562/192663 objects degraded (6.520%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:44.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:43 smithi194 bash[92666]: audit 2023-10-09T04:59:43.064286+0000 mon.a (mon.0) 446 : audit 0 from='client.? 172.21.15.133:0/1199945265' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:44.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:43 smithi194 bash[92666]: audit 2023-10-09T04:59:43.421265+0000 mon.a (mon.0) 447 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:44.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:59:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:59:44] "GET /metrics HTTP/1.1" 200 36236 "" "Prometheus/2.43.0" 2023-10-09T04:59:45.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:44 smithi133 bash[102858]: cluster 2023-10-09T04:59:43.509454+0000 mgr.y (mgr.25277) 307 : cluster 0 pgmap v246: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 12562/192663 objects degraded (6.520%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:59:45.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:44 smithi133 bash[105589]: cluster 2023-10-09T04:59:43.509454+0000 mgr.y (mgr.25277) 307 : cluster 0 pgmap v246: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 12562/192663 objects degraded (6.520%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:59:45.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:44 smithi194 bash[92666]: cluster 2023-10-09T04:59:43.509454+0000 mgr.y (mgr.25277) 307 : cluster 0 pgmap v246: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 12562/192663 objects degraded (6.520%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:59:46.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:45 smithi133 bash[102858]: audit 2023-10-09T04:59:45.669462+0000 mon.c (mon.2) 13 : audit 0 from='client.? 172.21.15.133:0/1007295333' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:46.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:45 smithi133 bash[105589]: audit 2023-10-09T04:59:45.669462+0000 mon.c (mon.2) 13 : audit 0 from='client.? 172.21.15.133:0/1007295333' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:46.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:45 smithi194 bash[92666]: audit 2023-10-09T04:59:45.669462+0000 mon.c (mon.2) 13 : audit 0 from='client.? 172.21.15.133:0/1007295333' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:47.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:46 smithi133 bash[102858]: cluster 2023-10-09T04:59:45.510266+0000 mgr.y (mgr.25277) 308 : cluster 0 pgmap v247: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 12562/184614 objects degraded (6.804%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:59:47.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:46 smithi133 bash[105589]: cluster 2023-10-09T04:59:45.510266+0000 mgr.y (mgr.25277) 308 : cluster 0 pgmap v247: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 12562/184614 objects degraded (6.804%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:59:47.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:46 smithi194 bash[92666]: cluster 2023-10-09T04:59:45.510266+0000 mgr.y (mgr.25277) 308 : cluster 0 pgmap v247: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 12562/184614 objects degraded (6.804%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:59:49.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:48 smithi133 bash[102858]: cluster 2023-10-09T04:59:47.510877+0000 mgr.y (mgr.25277) 309 : cluster 0 pgmap v248: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 12414/182796 objects degraded (6.791%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:59:49.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:48 smithi133 bash[102858]: cluster 2023-10-09T04:59:48.036279+0000 mon.a (mon.0) 448 : cluster 3 Health check update: Degraded data redundancy: 12414/182796 objects degraded (6.791%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:49.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:48 smithi133 bash[102858]: audit 2023-10-09T04:59:48.290265+0000 mon.c (mon.2) 14 : audit 0 from='client.? 172.21.15.133:0/282613548' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:49.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:48 smithi133 bash[105589]: cluster 2023-10-09T04:59:47.510877+0000 mgr.y (mgr.25277) 309 : cluster 0 pgmap v248: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 12414/182796 objects degraded (6.791%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:59:49.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:48 smithi133 bash[105589]: cluster 2023-10-09T04:59:48.036279+0000 mon.a (mon.0) 448 : cluster 3 Health check update: Degraded data redundancy: 12414/182796 objects degraded (6.791%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:49.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:48 smithi133 bash[105589]: audit 2023-10-09T04:59:48.290265+0000 mon.c (mon.2) 14 : audit 0 from='client.? 172.21.15.133:0/282613548' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:49.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:48 smithi194 bash[92666]: cluster 2023-10-09T04:59:47.510877+0000 mgr.y (mgr.25277) 309 : cluster 0 pgmap v248: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 12414/182796 objects degraded (6.791%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:59:49.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:48 smithi194 bash[92666]: cluster 2023-10-09T04:59:48.036279+0000 mon.a (mon.0) 448 : cluster 3 Health check update: Degraded data redundancy: 12414/182796 objects degraded (6.791%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:49.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:48 smithi194 bash[92666]: audit 2023-10-09T04:59:48.290265+0000 mon.c (mon.2) 14 : audit 0 from='client.? 172.21.15.133:0/282613548' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:51.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:50 smithi133 bash[102858]: cluster 2023-10-09T04:59:49.511448+0000 mgr.y (mgr.25277) 310 : cluster 0 pgmap v249: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 553 op/s; 12414/182796 objects degraded (6.791%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T04:59:51.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:50 smithi133 bash[105589]: cluster 2023-10-09T04:59:49.511448+0000 mgr.y (mgr.25277) 310 : cluster 0 pgmap v249: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 553 op/s; 12414/182796 objects degraded (6.791%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T04:59:51.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:50 smithi194 bash[92666]: cluster 2023-10-09T04:59:49.511448+0000 mgr.y (mgr.25277) 310 : cluster 0 pgmap v249: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 553 op/s; 12414/182796 objects degraded (6.791%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T04:59:52.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:51 smithi133 bash[102858]: audit 2023-10-09T04:59:50.889497+0000 mon.b (mon.1) 6 : audit 0 from='client.? 172.21.15.133:0/569689575' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:52.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:51 smithi133 bash[102858]: audit 2023-10-09T04:59:50.890522+0000 mon.a (mon.0) 449 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:59:52.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:51 smithi133 bash[105589]: audit 2023-10-09T04:59:50.889497+0000 mon.b (mon.1) 6 : audit 0 from='client.? 172.21.15.133:0/569689575' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:52.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:51 smithi133 bash[105589]: audit 2023-10-09T04:59:50.890522+0000 mon.a (mon.0) 449 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:59:52.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:51 smithi194 bash[92666]: audit 2023-10-09T04:59:50.889497+0000 mon.b (mon.1) 6 : audit 0 from='client.? 172.21.15.133:0/569689575' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:52.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:51 smithi194 bash[92666]: audit 2023-10-09T04:59:50.890522+0000 mon.a (mon.0) 449 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:59:53.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:52 smithi133 bash[102858]: cluster 2023-10-09T04:59:51.512320+0000 mgr.y (mgr.25277) 311 : cluster 0 pgmap v250: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 12242/173121 objects degraded (7.071%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T04:59:53.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:52 smithi133 bash[105589]: cluster 2023-10-09T04:59:51.512320+0000 mgr.y (mgr.25277) 311 : cluster 0 pgmap v250: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 12242/173121 objects degraded (7.071%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T04:59:53.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:52 smithi194 bash[92666]: cluster 2023-10-09T04:59:51.512320+0000 mgr.y (mgr.25277) 311 : cluster 0 pgmap v250: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 12242/173121 objects degraded (7.071%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T04:59:54.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:53 smithi133 bash[102858]: cluster 2023-10-09T04:59:53.038856+0000 mon.a (mon.0) 450 : cluster 3 Health check update: Degraded data redundancy: 12242/173121 objects degraded (7.071%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:54.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:53 smithi133 bash[102858]: audit 2023-10-09T04:59:53.463984+0000 mon.c (mon.2) 15 : audit 0 from='client.? 172.21.15.133:0/4022036484' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:54.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:53 smithi133 bash[105589]: cluster 2023-10-09T04:59:53.038856+0000 mon.a (mon.0) 450 : cluster 3 Health check update: Degraded data redundancy: 12242/173121 objects degraded (7.071%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:54.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:53 smithi133 bash[105589]: audit 2023-10-09T04:59:53.463984+0000 mon.c (mon.2) 15 : audit 0 from='client.? 172.21.15.133:0/4022036484' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:54.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:53 smithi194 bash[92666]: cluster 2023-10-09T04:59:53.038856+0000 mon.a (mon.0) 450 : cluster 3 Health check update: Degraded data redundancy: 12242/173121 objects degraded (7.071%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:54.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:53 smithi194 bash[92666]: audit 2023-10-09T04:59:53.463984+0000 mon.c (mon.2) 15 : audit 0 from='client.? 172.21.15.133:0/4022036484' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:54.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:59:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:04:59:54] "GET /metrics HTTP/1.1" 200 36236 "" "Prometheus/2.43.0" 2023-10-09T04:59:55.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:54 smithi133 bash[102858]: cluster 2023-10-09T04:59:53.512966+0000 mgr.y (mgr.25277) 312 : cluster 0 pgmap v251: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 12242/173121 objects degraded (7.071%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T04:59:55.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:54 smithi133 bash[105589]: cluster 2023-10-09T04:59:53.512966+0000 mgr.y (mgr.25277) 312 : cluster 0 pgmap v251: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 12242/173121 objects degraded (7.071%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T04:59:55.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:54 smithi194 bash[92666]: cluster 2023-10-09T04:59:53.512966+0000 mgr.y (mgr.25277) 312 : cluster 0 pgmap v251: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 12242/173121 objects degraded (7.071%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T04:59:57.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:56 smithi194 bash[92666]: cluster 2023-10-09T04:59:55.513752+0000 mgr.y (mgr.25277) 313 : cluster 0 pgmap v252: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 12242/165300 objects degraded (7.406%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T04:59:57.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:56 smithi194 bash[92666]: audit 2023-10-09T04:59:56.073454+0000 mon.a (mon.0) 451 : audit 0 from='client.? 172.21.15.133:0/3018706111' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:57.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:56 smithi133 bash[105589]: cluster 2023-10-09T04:59:55.513752+0000 mgr.y (mgr.25277) 313 : cluster 0 pgmap v252: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 12242/165300 objects degraded (7.406%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T04:59:57.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:56 smithi133 bash[105589]: audit 2023-10-09T04:59:56.073454+0000 mon.a (mon.0) 451 : audit 0 from='client.? 172.21.15.133:0/3018706111' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:57.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:56 smithi133 bash[102858]: cluster 2023-10-09T04:59:55.513752+0000 mgr.y (mgr.25277) 313 : cluster 0 pgmap v252: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 12242/165300 objects degraded (7.406%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T04:59:57.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:56 smithi133 bash[102858]: audit 2023-10-09T04:59:56.073454+0000 mon.a (mon.0) 451 : audit 0 from='client.? 172.21.15.133:0/3018706111' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:58.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 04:59:58 smithi133 bash[91128]: debug 2023-10-09T04:59:58.241+0000 7ff45d5ed700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:59:58.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:58 smithi133 bash[102858]: audit 2023-10-09T04:59:57.429100+0000 mon.a (mon.0) 452 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:59:58.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:58 smithi133 bash[102858]: audit 2023-10-09T04:59:57.441659+0000 mon.a (mon.0) 453 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:59:58.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:58 smithi133 bash[102858]: cluster 2023-10-09T04:59:57.514606+0000 mgr.y (mgr.25277) 314 : cluster 0 pgmap v253: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 12072/163533 objects degraded (7.382%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T04:59:58.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:58 smithi133 bash[102858]: audit 2023-10-09T04:59:57.522549+0000 mon.a (mon.0) 454 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:59:58.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:58 smithi133 bash[102858]: audit 2023-10-09T04:59:57.533613+0000 mon.a (mon.0) 455 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:59:58.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:58 smithi133 bash[102858]: cluster 2023-10-09T04:59:58.039831+0000 mon.a (mon.0) 456 : cluster 3 Health check update: Degraded data redundancy: 12242/165300 objects degraded (7.406%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:58.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:58 smithi133 bash[102858]: audit 2023-10-09T04:59:58.138233+0000 mon.a (mon.0) 457 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:59:58.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:58 smithi133 bash[102858]: audit 2023-10-09T04:59:58.139327+0000 mon.a (mon.0) 458 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:59:58.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:58 smithi133 bash[102858]: audit 2023-10-09T04:59:58.145482+0000 mon.a (mon.0) 459 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:59:58.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:58 smithi133 bash[102858]: audit 2023-10-09T04:59:58.227814+0000 mon.a (mon.0) 460 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:59:58.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:58 smithi133 bash[102858]: audit 2023-10-09T04:59:58.233125+0000 mon.a (mon.0) 461 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:59:58.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:58 smithi133 bash[102858]: audit 2023-10-09T04:59:58.239464+0000 mon.a (mon.0) 462 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:59:58.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:58 smithi133 bash[102858]: audit 2023-10-09T04:59:58.241874+0000 mon.a (mon.0) 463 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:59:58.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:58 smithi133 bash[102858]: audit 2023-10-09T04:59:58.245109+0000 mon.a (mon.0) 464 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:59:58.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:58 smithi133 bash[102858]: audit 2023-10-09T04:59:58.423130+0000 mon.a (mon.0) 465 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:58.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:58 smithi133 bash[105589]: audit 2023-10-09T04:59:57.429100+0000 mon.a (mon.0) 452 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:59:58.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:58 smithi133 bash[105589]: audit 2023-10-09T04:59:57.441659+0000 mon.a (mon.0) 453 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:59:58.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:58 smithi133 bash[105589]: cluster 2023-10-09T04:59:57.514606+0000 mgr.y (mgr.25277) 314 : cluster 0 pgmap v253: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 12072/163533 objects degraded (7.382%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T04:59:58.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:58 smithi133 bash[105589]: audit 2023-10-09T04:59:57.522549+0000 mon.a (mon.0) 454 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:59:58.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:58 smithi133 bash[105589]: audit 2023-10-09T04:59:57.533613+0000 mon.a (mon.0) 455 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:59:58.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:58 smithi133 bash[105589]: cluster 2023-10-09T04:59:58.039831+0000 mon.a (mon.0) 456 : cluster 3 Health check update: Degraded data redundancy: 12242/165300 objects degraded (7.406%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:58.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:58 smithi133 bash[105589]: audit 2023-10-09T04:59:58.138233+0000 mon.a (mon.0) 457 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:59:58.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:58 smithi133 bash[105589]: audit 2023-10-09T04:59:58.139327+0000 mon.a (mon.0) 458 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:59:58.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:58 smithi133 bash[105589]: audit 2023-10-09T04:59:58.145482+0000 mon.a (mon.0) 459 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:59:58.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:58 smithi133 bash[105589]: audit 2023-10-09T04:59:58.227814+0000 mon.a (mon.0) 460 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:59:58.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:58 smithi133 bash[105589]: audit 2023-10-09T04:59:58.233125+0000 mon.a (mon.0) 461 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:59:58.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:58 smithi133 bash[105589]: audit 2023-10-09T04:59:58.239464+0000 mon.a (mon.0) 462 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:59:58.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:58 smithi133 bash[105589]: audit 2023-10-09T04:59:58.241874+0000 mon.a (mon.0) 463 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:59:58.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:58 smithi133 bash[105589]: audit 2023-10-09T04:59:58.245109+0000 mon.a (mon.0) 464 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:59:58.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:58 smithi133 bash[105589]: audit 2023-10-09T04:59:58.423130+0000 mon.a (mon.0) 465 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:58.889 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:58 smithi194 bash[92666]: audit 2023-10-09T04:59:57.429100+0000 mon.a (mon.0) 452 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:59:58.889 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:58 smithi194 bash[92666]: audit 2023-10-09T04:59:57.441659+0000 mon.a (mon.0) 453 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:59:58.889 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:58 smithi194 bash[92666]: cluster 2023-10-09T04:59:57.514606+0000 mgr.y (mgr.25277) 314 : cluster 0 pgmap v253: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 12072/163533 objects degraded (7.382%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T04:59:58.889 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:58 smithi194 bash[92666]: audit 2023-10-09T04:59:57.522549+0000 mon.a (mon.0) 454 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:59:58.889 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:58 smithi194 bash[92666]: audit 2023-10-09T04:59:57.533613+0000 mon.a (mon.0) 455 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:59:58.889 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:58 smithi194 bash[92666]: cluster 2023-10-09T04:59:58.039831+0000 mon.a (mon.0) 456 : cluster 3 Health check update: Degraded data redundancy: 12242/165300 objects degraded (7.406%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:58.889 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:58 smithi194 bash[92666]: audit 2023-10-09T04:59:58.138233+0000 mon.a (mon.0) 457 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:59:58.889 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:58 smithi194 bash[92666]: audit 2023-10-09T04:59:58.139327+0000 mon.a (mon.0) 458 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:59:58.889 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:58 smithi194 bash[92666]: audit 2023-10-09T04:59:58.145482+0000 mon.a (mon.0) 459 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T04:59:58.889 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:58 smithi194 bash[92666]: audit 2023-10-09T04:59:58.227814+0000 mon.a (mon.0) 460 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:59:58.890 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:58 smithi194 bash[92666]: audit 2023-10-09T04:59:58.233125+0000 mon.a (mon.0) 461 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:59:58.890 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:58 smithi194 bash[92666]: audit 2023-10-09T04:59:58.239464+0000 mon.a (mon.0) 462 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:59:58.890 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:58 smithi194 bash[92666]: audit 2023-10-09T04:59:58.241874+0000 mon.a (mon.0) 463 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:59:58.890 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:58 smithi194 bash[92666]: audit 2023-10-09T04:59:58.245109+0000 mon.a (mon.0) 464 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:59:58.890 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:58 smithi194 bash[92666]: audit 2023-10-09T04:59:58.423130+0000 mon.a (mon.0) 465 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:59.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:59 smithi133 bash[102858]: audit 2023-10-09T04:59:58.245412+0000 mgr.y (mgr.25277) 315 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:59:59.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:59 smithi133 bash[102858]: cephadm 2023-10-09T04:59:58.249136+0000 mgr.y (mgr.25277) 316 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:59:59.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 04:59:59 smithi133 bash[102858]: audit 2023-10-09T04:59:58.732209+0000 mon.c (mon.2) 16 : audit 0 from='client.? 172.21.15.133:0/1059467334' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:59.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:59 smithi133 bash[105589]: audit 2023-10-09T04:59:58.245412+0000 mgr.y (mgr.25277) 315 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:59:59.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:59 smithi133 bash[105589]: cephadm 2023-10-09T04:59:58.249136+0000 mgr.y (mgr.25277) 316 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:59:59.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 04:59:59 smithi133 bash[105589]: audit 2023-10-09T04:59:58.732209+0000 mon.c (mon.2) 16 : audit 0 from='client.? 172.21.15.133:0/1059467334' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:59:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:59 smithi194 bash[92666]: audit 2023-10-09T04:59:58.245412+0000 mgr.y (mgr.25277) 315 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:59:59.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:59 smithi194 bash[92666]: cephadm 2023-10-09T04:59:58.249136+0000 mgr.y (mgr.25277) 316 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:59:59.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 04:59:59 smithi194 bash[92666]: audit 2023-10-09T04:59:58.732209+0000 mon.c (mon.2) 16 : audit 0 from='client.? 172.21.15.133:0/1059467334' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:00.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:00 smithi133 bash[102858]: cluster 2023-10-09T04:59:59.515067+0000 mgr.y (mgr.25277) 317 : cluster 0 pgmap v254: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 12072/163533 objects degraded (7.382%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T05:00:00.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:00 smithi133 bash[102858]: cluster 2023-10-09T05:00:00.000117+0000 mon.a (mon.0) 466 : cluster 3 Health detail: HEALTH_WARN Degraded data redundancy: 12072/163533 objects degraded (7.382%), 5 pgs degraded, 5 pgs undersized 2023-10-09T05:00:00.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:00 smithi133 bash[102858]: cluster 2023-10-09T05:00:00.000153+0000 mon.a (mon.0) 467 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 12072/163533 objects degraded (7.382%), 5 pgs degraded, 5 pgs undersized 2023-10-09T05:00:00.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:00 smithi133 bash[102858]: cluster 2023-10-09T05:00:00.000168+0000 mon.a (mon.0) 468 : cluster 3 pg 7.0 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,3] 2023-10-09T05:00:00.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:00 smithi133 bash[102858]: cluster 2023-10-09T05:00:00.000198+0000 mon.a (mon.0) 469 : cluster 3 pg 7.2 is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [3,2] 2023-10-09T05:00:00.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:00 smithi133 bash[102858]: cluster 2023-10-09T05:00:00.000216+0000 mon.a (mon.0) 470 : cluster 3 pg 7.9 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,4] 2023-10-09T05:00:00.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:00 smithi133 bash[102858]: cluster 2023-10-09T05:00:00.000232+0000 mon.a (mon.0) 471 : cluster 3 pg 7.b is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,6] 2023-10-09T05:00:00.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:00 smithi133 bash[102858]: cluster 2023-10-09T05:00:00.000247+0000 mon.a (mon.0) 472 : cluster 3 pg 7.d is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,0] 2023-10-09T05:00:00.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:00 smithi133 bash[105589]: cluster 2023-10-09T04:59:59.515067+0000 mgr.y (mgr.25277) 317 : cluster 0 pgmap v254: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 12072/163533 objects degraded (7.382%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T05:00:00.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:00 smithi133 bash[105589]: cluster 2023-10-09T05:00:00.000117+0000 mon.a (mon.0) 466 : cluster 3 Health detail: HEALTH_WARN Degraded data redundancy: 12072/163533 objects degraded (7.382%), 5 pgs degraded, 5 pgs undersized 2023-10-09T05:00:00.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:00 smithi133 bash[105589]: cluster 2023-10-09T05:00:00.000153+0000 mon.a (mon.0) 467 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 12072/163533 objects degraded (7.382%), 5 pgs degraded, 5 pgs undersized 2023-10-09T05:00:00.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:00 smithi133 bash[105589]: cluster 2023-10-09T05:00:00.000168+0000 mon.a (mon.0) 468 : cluster 3 pg 7.0 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,3] 2023-10-09T05:00:00.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:00 smithi133 bash[105589]: cluster 2023-10-09T05:00:00.000198+0000 mon.a (mon.0) 469 : cluster 3 pg 7.2 is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [3,2] 2023-10-09T05:00:00.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:00 smithi133 bash[105589]: cluster 2023-10-09T05:00:00.000216+0000 mon.a (mon.0) 470 : cluster 3 pg 7.9 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,4] 2023-10-09T05:00:00.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:00 smithi133 bash[105589]: cluster 2023-10-09T05:00:00.000232+0000 mon.a (mon.0) 471 : cluster 3 pg 7.b is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,6] 2023-10-09T05:00:00.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:00 smithi133 bash[105589]: cluster 2023-10-09T05:00:00.000247+0000 mon.a (mon.0) 472 : cluster 3 pg 7.d is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,0] 2023-10-09T05:00:00.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:00 smithi194 bash[92666]: cluster 2023-10-09T04:59:59.515067+0000 mgr.y (mgr.25277) 317 : cluster 0 pgmap v254: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 12072/163533 objects degraded (7.382%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T05:00:00.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:00 smithi194 bash[92666]: cluster 2023-10-09T05:00:00.000117+0000 mon.a (mon.0) 466 : cluster 3 Health detail: HEALTH_WARN Degraded data redundancy: 12072/163533 objects degraded (7.382%), 5 pgs degraded, 5 pgs undersized 2023-10-09T05:00:00.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:00 smithi194 bash[92666]: cluster 2023-10-09T05:00:00.000153+0000 mon.a (mon.0) 467 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 12072/163533 objects degraded (7.382%), 5 pgs degraded, 5 pgs undersized 2023-10-09T05:00:00.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:00 smithi194 bash[92666]: cluster 2023-10-09T05:00:00.000168+0000 mon.a (mon.0) 468 : cluster 3 pg 7.0 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,3] 2023-10-09T05:00:00.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:00 smithi194 bash[92666]: cluster 2023-10-09T05:00:00.000198+0000 mon.a (mon.0) 469 : cluster 3 pg 7.2 is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [3,2] 2023-10-09T05:00:00.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:00 smithi194 bash[92666]: cluster 2023-10-09T05:00:00.000216+0000 mon.a (mon.0) 470 : cluster 3 pg 7.9 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,4] 2023-10-09T05:00:00.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:00 smithi194 bash[92666]: cluster 2023-10-09T05:00:00.000232+0000 mon.a (mon.0) 471 : cluster 3 pg 7.b is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,6] 2023-10-09T05:00:00.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:00 smithi194 bash[92666]: cluster 2023-10-09T05:00:00.000247+0000 mon.a (mon.0) 472 : cluster 3 pg 7.d is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,0] 2023-10-09T05:00:01.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:01 smithi133 bash[105589]: audit 2023-10-09T05:00:01.343628+0000 mon.c (mon.2) 17 : audit 0 from='client.? 172.21.15.133:0/2565607552' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:01.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:01 smithi133 bash[102858]: audit 2023-10-09T05:00:01.343628+0000 mon.c (mon.2) 17 : audit 0 from='client.? 172.21.15.133:0/2565607552' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:01.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:01 smithi194 bash[92666]: audit 2023-10-09T05:00:01.343628+0000 mon.c (mon.2) 17 : audit 0 from='client.? 172.21.15.133:0/2565607552' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:02.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:02 smithi133 bash[105589]: cluster 2023-10-09T05:00:01.515750+0000 mgr.y (mgr.25277) 318 : cluster 0 pgmap v255: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 11907/154683 objects degraded (7.698%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T05:00:02.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:02 smithi133 bash[102858]: cluster 2023-10-09T05:00:01.515750+0000 mgr.y (mgr.25277) 318 : cluster 0 pgmap v255: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 11907/154683 objects degraded (7.698%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T05:00:02.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:02 smithi194 bash[92666]: cluster 2023-10-09T05:00:01.515750+0000 mgr.y (mgr.25277) 318 : cluster 0 pgmap v255: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 11907/154683 objects degraded (7.698%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T05:00:03.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:03 smithi133 bash[102858]: cluster 2023-10-09T05:00:03.040932+0000 mon.a (mon.0) 473 : cluster 3 Health check update: Degraded data redundancy: 11907/154683 objects degraded (7.698%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:03.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:03 smithi133 bash[105589]: cluster 2023-10-09T05:00:03.040932+0000 mon.a (mon.0) 473 : cluster 3 Health check update: Degraded data redundancy: 11907/154683 objects degraded (7.698%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:03.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:03 smithi194 bash[92666]: cluster 2023-10-09T05:00:03.040932+0000 mon.a (mon.0) 473 : cluster 3 Health check update: Degraded data redundancy: 11907/154683 objects degraded (7.698%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:04.541 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:00:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:00:04] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.43.0" 2023-10-09T05:00:04.542 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:04 smithi133 bash[102858]: cluster 2023-10-09T05:00:03.516234+0000 mgr.y (mgr.25277) 319 : cluster 0 pgmap v256: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 11907/154683 objects degraded (7.698%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T05:00:04.542 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:04 smithi133 bash[102858]: audit 2023-10-09T05:00:03.945809+0000 mon.a (mon.0) 474 : audit 0 from='client.? 172.21.15.133:0/4258282956' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:04.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:04 smithi133 bash[105589]: cluster 2023-10-09T05:00:03.516234+0000 mgr.y (mgr.25277) 319 : cluster 0 pgmap v256: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 11907/154683 objects degraded (7.698%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T05:00:04.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:04 smithi133 bash[105589]: audit 2023-10-09T05:00:03.945809+0000 mon.a (mon.0) 474 : audit 0 from='client.? 172.21.15.133:0/4258282956' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:04.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:04 smithi194 bash[92666]: cluster 2023-10-09T05:00:03.516234+0000 mgr.y (mgr.25277) 319 : cluster 0 pgmap v256: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 11907/154683 objects degraded (7.698%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T05:00:04.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:04 smithi194 bash[92666]: audit 2023-10-09T05:00:03.945809+0000 mon.a (mon.0) 474 : audit 0 from='client.? 172.21.15.133:0/4258282956' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:06.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:06 smithi133 bash[102858]: cluster 2023-10-09T05:00:05.516968+0000 mgr.y (mgr.25277) 320 : cluster 0 pgmap v257: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 11907/146928 objects degraded (8.104%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T05:00:06.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:06 smithi133 bash[102858]: audit 2023-10-09T05:00:06.537358+0000 mon.b (mon.1) 7 : audit 0 from='client.? 172.21.15.133:0/425417854' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:06.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:06 smithi133 bash[105589]: cluster 2023-10-09T05:00:05.516968+0000 mgr.y (mgr.25277) 320 : cluster 0 pgmap v257: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 11907/146928 objects degraded (8.104%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T05:00:06.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:06 smithi133 bash[105589]: audit 2023-10-09T05:00:06.537358+0000 mon.b (mon.1) 7 : audit 0 from='client.? 172.21.15.133:0/425417854' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:06.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:06 smithi194 bash[92666]: cluster 2023-10-09T05:00:05.516968+0000 mgr.y (mgr.25277) 320 : cluster 0 pgmap v257: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 11907/146928 objects degraded (8.104%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T05:00:06.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:06 smithi194 bash[92666]: audit 2023-10-09T05:00:06.537358+0000 mon.b (mon.1) 7 : audit 0 from='client.? 172.21.15.133:0/425417854' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:08.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:08 smithi133 bash[102858]: cluster 2023-10-09T05:00:07.517562+0000 mgr.y (mgr.25277) 321 : cluster 0 pgmap v258: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 11722/145149 objects degraded (8.076%); 1.8 MiB/s, 43 objects/s recovering 2023-10-09T05:00:08.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:08 smithi133 bash[102858]: cluster 2023-10-09T05:00:08.042199+0000 mon.a (mon.0) 475 : cluster 3 Health check update: Degraded data redundancy: 11722/145149 objects degraded (8.076%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:08.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:08 smithi133 bash[105589]: cluster 2023-10-09T05:00:07.517562+0000 mgr.y (mgr.25277) 321 : cluster 0 pgmap v258: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 11722/145149 objects degraded (8.076%); 1.8 MiB/s, 43 objects/s recovering 2023-10-09T05:00:08.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:08 smithi133 bash[105589]: cluster 2023-10-09T05:00:08.042199+0000 mon.a (mon.0) 475 : cluster 3 Health check update: Degraded data redundancy: 11722/145149 objects degraded (8.076%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:08.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:08 smithi194 bash[92666]: cluster 2023-10-09T05:00:07.517562+0000 mgr.y (mgr.25277) 321 : cluster 0 pgmap v258: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 11722/145149 objects degraded (8.076%); 1.8 MiB/s, 43 objects/s recovering 2023-10-09T05:00:08.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:08 smithi194 bash[92666]: cluster 2023-10-09T05:00:08.042199+0000 mon.a (mon.0) 475 : cluster 3 Health check update: Degraded data redundancy: 11722/145149 objects degraded (8.076%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:09.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:09 smithi194 bash[92666]: audit 2023-10-09T05:00:09.145685+0000 mon.c (mon.2) 18 : audit 0 from='client.? 172.21.15.133:0/2403754259' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:10.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:09 smithi133 bash[102858]: audit 2023-10-09T05:00:09.145685+0000 mon.c (mon.2) 18 : audit 0 from='client.? 172.21.15.133:0/2403754259' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:10.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:09 smithi133 bash[105589]: audit 2023-10-09T05:00:09.145685+0000 mon.c (mon.2) 18 : audit 0 from='client.? 172.21.15.133:0/2403754259' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:10.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:10 smithi194 bash[92666]: cluster 2023-10-09T05:00:09.518106+0000 mgr.y (mgr.25277) 322 : cluster 0 pgmap v259: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 11722/145149 objects degraded (8.076%); 933 KiB/s, 29 objects/s recovering 2023-10-09T05:00:11.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:10 smithi133 bash[102858]: cluster 2023-10-09T05:00:09.518106+0000 mgr.y (mgr.25277) 322 : cluster 0 pgmap v259: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 11722/145149 objects degraded (8.076%); 933 KiB/s, 29 objects/s recovering 2023-10-09T05:00:11.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:10 smithi133 bash[105589]: cluster 2023-10-09T05:00:09.518106+0000 mgr.y (mgr.25277) 322 : cluster 0 pgmap v259: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 11722/145149 objects degraded (8.076%); 933 KiB/s, 29 objects/s recovering 2023-10-09T05:00:12.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:12 smithi194 bash[92666]: cluster 2023-10-09T05:00:11.519069+0000 mgr.y (mgr.25277) 323 : cluster 0 pgmap v260: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 11518/135573 objects degraded (8.496%); 933 KiB/s, 46 objects/s recovering 2023-10-09T05:00:12.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:12 smithi194 bash[92666]: audit 2023-10-09T05:00:11.777859+0000 mon.c (mon.2) 19 : audit 0 from='client.? 172.21.15.133:0/462734605' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:13.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:12 smithi133 bash[102858]: cluster 2023-10-09T05:00:11.519069+0000 mgr.y (mgr.25277) 323 : cluster 0 pgmap v260: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 11518/135573 objects degraded (8.496%); 933 KiB/s, 46 objects/s recovering 2023-10-09T05:00:13.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:12 smithi133 bash[102858]: audit 2023-10-09T05:00:11.777859+0000 mon.c (mon.2) 19 : audit 0 from='client.? 172.21.15.133:0/462734605' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:13.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:12 smithi133 bash[105589]: cluster 2023-10-09T05:00:11.519069+0000 mgr.y (mgr.25277) 323 : cluster 0 pgmap v260: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 11518/135573 objects degraded (8.496%); 933 KiB/s, 46 objects/s recovering 2023-10-09T05:00:13.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:12 smithi133 bash[105589]: audit 2023-10-09T05:00:11.777859+0000 mon.c (mon.2) 19 : audit 0 from='client.? 172.21.15.133:0/462734605' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:13.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:00:13 smithi133 bash[91128]: debug 2023-10-09T05:00:13.265+0000 7ff45d5ed700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T05:00:13.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:13 smithi194 bash[92666]: cluster 2023-10-09T05:00:13.049505+0000 mon.a (mon.0) 476 : cluster 3 Health check update: Degraded data redundancy: 11518/135573 objects degraded (8.496%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:13.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:13 smithi194 bash[92666]: audit 2023-10-09T05:00:13.266526+0000 mon.a (mon.0) 477 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:00:13.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:13 smithi194 bash[92666]: audit 2023-10-09T05:00:13.421859+0000 mon.a (mon.0) 478 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:14.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:13 smithi133 bash[102858]: cluster 2023-10-09T05:00:13.049505+0000 mon.a (mon.0) 476 : cluster 3 Health check update: Degraded data redundancy: 11518/135573 objects degraded (8.496%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:14.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:13 smithi133 bash[102858]: audit 2023-10-09T05:00:13.266526+0000 mon.a (mon.0) 477 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:00:14.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:13 smithi133 bash[102858]: audit 2023-10-09T05:00:13.421859+0000 mon.a (mon.0) 478 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:14.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:13 smithi133 bash[105589]: cluster 2023-10-09T05:00:13.049505+0000 mon.a (mon.0) 476 : cluster 3 Health check update: Degraded data redundancy: 11518/135573 objects degraded (8.496%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:14.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:13 smithi133 bash[105589]: audit 2023-10-09T05:00:13.266526+0000 mon.a (mon.0) 477 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:00:14.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:13 smithi133 bash[105589]: audit 2023-10-09T05:00:13.421859+0000 mon.a (mon.0) 478 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:14.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:00:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:00:14] "GET /metrics HTTP/1.1" 200 36300 "" "Prometheus/2.43.0" 2023-10-09T05:00:14.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:14 smithi194 bash[92666]: audit 2023-10-09T05:00:13.267349+0000 mgr.y (mgr.25277) 324 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:00:14.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:14 smithi194 bash[92666]: cephadm 2023-10-09T05:00:13.270161+0000 mgr.y (mgr.25277) 325 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T05:00:14.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:14 smithi194 bash[92666]: cluster 2023-10-09T05:00:13.519621+0000 mgr.y (mgr.25277) 326 : cluster 0 pgmap v261: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 11518/135573 objects degraded (8.496%); 53 KiB/s, 32 objects/s recovering 2023-10-09T05:00:14.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:14 smithi194 bash[92666]: audit 2023-10-09T05:00:14.383349+0000 mon.c (mon.2) 20 : audit 0 from='client.? 172.21.15.133:0/1867892061' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:15.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:14 smithi133 bash[102858]: audit 2023-10-09T05:00:13.267349+0000 mgr.y (mgr.25277) 324 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:00:15.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:14 smithi133 bash[102858]: cephadm 2023-10-09T05:00:13.270161+0000 mgr.y (mgr.25277) 325 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T05:00:15.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:14 smithi133 bash[102858]: cluster 2023-10-09T05:00:13.519621+0000 mgr.y (mgr.25277) 326 : cluster 0 pgmap v261: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 11518/135573 objects degraded (8.496%); 53 KiB/s, 32 objects/s recovering 2023-10-09T05:00:15.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:14 smithi133 bash[102858]: audit 2023-10-09T05:00:14.383349+0000 mon.c (mon.2) 20 : audit 0 from='client.? 172.21.15.133:0/1867892061' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:15.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:14 smithi133 bash[105589]: audit 2023-10-09T05:00:13.267349+0000 mgr.y (mgr.25277) 324 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:00:15.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:14 smithi133 bash[105589]: cephadm 2023-10-09T05:00:13.270161+0000 mgr.y (mgr.25277) 325 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T05:00:15.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:14 smithi133 bash[105589]: cluster 2023-10-09T05:00:13.519621+0000 mgr.y (mgr.25277) 326 : cluster 0 pgmap v261: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 11518/135573 objects degraded (8.496%); 53 KiB/s, 32 objects/s recovering 2023-10-09T05:00:15.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:14 smithi133 bash[105589]: audit 2023-10-09T05:00:14.383349+0000 mon.c (mon.2) 20 : audit 0 from='client.? 172.21.15.133:0/1867892061' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:16.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:16 smithi194 bash[92666]: cluster 2023-10-09T05:00:15.520526+0000 mgr.y (mgr.25277) 327 : cluster 0 pgmap v262: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 11518/127764 objects degraded (9.015%); 53 KiB/s, 32 objects/s recovering 2023-10-09T05:00:17.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:16 smithi133 bash[105589]: cluster 2023-10-09T05:00:15.520526+0000 mgr.y (mgr.25277) 327 : cluster 0 pgmap v262: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 11518/127764 objects degraded (9.015%); 53 KiB/s, 32 objects/s recovering 2023-10-09T05:00:17.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:16 smithi133 bash[102858]: cluster 2023-10-09T05:00:15.520526+0000 mgr.y (mgr.25277) 327 : cluster 0 pgmap v262: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 11518/127764 objects degraded (9.015%); 53 KiB/s, 32 objects/s recovering 2023-10-09T05:00:17.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:17 smithi194 bash[92666]: audit 2023-10-09T05:00:16.974459+0000 mon.c (mon.2) 21 : audit 0 from='client.? 172.21.15.133:0/2553522423' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:18.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:17 smithi133 bash[102858]: audit 2023-10-09T05:00:16.974459+0000 mon.c (mon.2) 21 : audit 0 from='client.? 172.21.15.133:0/2553522423' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:18.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:17 smithi133 bash[105589]: audit 2023-10-09T05:00:16.974459+0000 mon.c (mon.2) 21 : audit 0 from='client.? 172.21.15.133:0/2553522423' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:18.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:18 smithi194 bash[92666]: cluster 2023-10-09T05:00:17.521161+0000 mgr.y (mgr.25277) 328 : cluster 0 pgmap v263: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 11324/125850 objects degraded (8.998%); 53 KiB/s, 48 objects/s recovering 2023-10-09T05:00:18.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:18 smithi194 bash[92666]: cluster 2023-10-09T05:00:18.050778+0000 mon.a (mon.0) 479 : cluster 3 Health check update: Degraded data redundancy: 11324/125850 objects degraded (8.998%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:19.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:18 smithi133 bash[102858]: cluster 2023-10-09T05:00:17.521161+0000 mgr.y (mgr.25277) 328 : cluster 0 pgmap v263: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 11324/125850 objects degraded (8.998%); 53 KiB/s, 48 objects/s recovering 2023-10-09T05:00:19.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:18 smithi133 bash[102858]: cluster 2023-10-09T05:00:18.050778+0000 mon.a (mon.0) 479 : cluster 3 Health check update: Degraded data redundancy: 11324/125850 objects degraded (8.998%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:19.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:18 smithi133 bash[105589]: cluster 2023-10-09T05:00:17.521161+0000 mgr.y (mgr.25277) 328 : cluster 0 pgmap v263: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 11324/125850 objects degraded (8.998%); 53 KiB/s, 48 objects/s recovering 2023-10-09T05:00:19.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:18 smithi133 bash[105589]: cluster 2023-10-09T05:00:18.050778+0000 mon.a (mon.0) 479 : cluster 3 Health check update: Degraded data redundancy: 11324/125850 objects degraded (8.998%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:20.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:20 smithi194 bash[92666]: cluster 2023-10-09T05:00:19.521711+0000 mgr.y (mgr.25277) 329 : cluster 0 pgmap v264: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 11324/125850 objects degraded (8.998%); 0 B/s, 33 objects/s recovering 2023-10-09T05:00:20.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:20 smithi194 bash[92666]: audit 2023-10-09T05:00:19.604186+0000 mon.c (mon.2) 22 : audit 0 from='client.? 172.21.15.133:0/354279630' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:21.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:20 smithi133 bash[102858]: cluster 2023-10-09T05:00:19.521711+0000 mgr.y (mgr.25277) 329 : cluster 0 pgmap v264: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 11324/125850 objects degraded (8.998%); 0 B/s, 33 objects/s recovering 2023-10-09T05:00:21.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:20 smithi133 bash[102858]: audit 2023-10-09T05:00:19.604186+0000 mon.c (mon.2) 22 : audit 0 from='client.? 172.21.15.133:0/354279630' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:21.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:20 smithi133 bash[105589]: cluster 2023-10-09T05:00:19.521711+0000 mgr.y (mgr.25277) 329 : cluster 0 pgmap v264: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 11324/125850 objects degraded (8.998%); 0 B/s, 33 objects/s recovering 2023-10-09T05:00:21.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:20 smithi133 bash[105589]: audit 2023-10-09T05:00:19.604186+0000 mon.c (mon.2) 22 : audit 0 from='client.? 172.21.15.133:0/354279630' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:22.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:22 smithi194 bash[92666]: cluster 2023-10-09T05:00:21.522522+0000 mgr.y (mgr.25277) 330 : cluster 0 pgmap v265: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 802 op/s; 11124/116232 objects degraded (9.571%); 0 B/s, 49 objects/s recovering 2023-10-09T05:00:22.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:22 smithi194 bash[92666]: cluster 2023-10-09T05:00:21.602863+0000 osd.2 (osd.2) 19 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T05:00:22.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:22 smithi194 bash[92666]: cluster 2023-10-09T05:00:21.604035+0000 osd.5 (osd.5) 29 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T05:00:22.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:22 smithi194 bash[92666]: cluster 2023-10-09T05:00:21.608904+0000 osd.3 (osd.3) 20 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T05:00:22.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:22 smithi194 bash[92666]: cluster 2023-10-09T05:00:21.609630+0000 mon.a (mon.0) 480 : cluster 0 osdmap e101: 8 total, 8 up, 8 in 2023-10-09T05:00:22.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:22 smithi194 bash[92666]: cluster 2023-10-09T05:00:21.610995+0000 osd.3 (osd.3) 21 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T05:00:22.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:22 smithi194 bash[92666]: audit 2023-10-09T05:00:22.063072+0000 mon.b (mon.1) 8 : audit 0 from='client.? 172.21.15.133:0/821512045' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:23.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:22 smithi133 bash[102858]: cluster 2023-10-09T05:00:21.522522+0000 mgr.y (mgr.25277) 330 : cluster 0 pgmap v265: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 802 op/s; 11124/116232 objects degraded (9.571%); 0 B/s, 49 objects/s recovering 2023-10-09T05:00:23.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:22 smithi133 bash[102858]: cluster 2023-10-09T05:00:21.602863+0000 osd.2 (osd.2) 19 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T05:00:23.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:22 smithi133 bash[102858]: cluster 2023-10-09T05:00:21.604035+0000 osd.5 (osd.5) 29 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T05:00:23.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:22 smithi133 bash[102858]: cluster 2023-10-09T05:00:21.608904+0000 osd.3 (osd.3) 20 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T05:00:23.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:22 smithi133 bash[102858]: cluster 2023-10-09T05:00:21.609630+0000 mon.a (mon.0) 480 : cluster 0 osdmap e101: 8 total, 8 up, 8 in 2023-10-09T05:00:23.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:22 smithi133 bash[102858]: cluster 2023-10-09T05:00:21.610995+0000 osd.3 (osd.3) 21 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T05:00:23.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:22 smithi133 bash[102858]: audit 2023-10-09T05:00:22.063072+0000 mon.b (mon.1) 8 : audit 0 from='client.? 172.21.15.133:0/821512045' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:23.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:22 smithi133 bash[105589]: cluster 2023-10-09T05:00:21.522522+0000 mgr.y (mgr.25277) 330 : cluster 0 pgmap v265: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 802 op/s; 11124/116232 objects degraded (9.571%); 0 B/s, 49 objects/s recovering 2023-10-09T05:00:23.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:22 smithi133 bash[105589]: cluster 2023-10-09T05:00:21.602863+0000 osd.2 (osd.2) 19 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T05:00:23.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:22 smithi133 bash[105589]: cluster 2023-10-09T05:00:21.604035+0000 osd.5 (osd.5) 29 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T05:00:23.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:22 smithi133 bash[105589]: cluster 2023-10-09T05:00:21.608904+0000 osd.3 (osd.3) 20 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T05:00:23.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:22 smithi133 bash[105589]: cluster 2023-10-09T05:00:21.609630+0000 mon.a (mon.0) 480 : cluster 0 osdmap e101: 8 total, 8 up, 8 in 2023-10-09T05:00:23.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:22 smithi133 bash[105589]: cluster 2023-10-09T05:00:21.610995+0000 osd.3 (osd.3) 21 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T05:00:23.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:22 smithi133 bash[105589]: audit 2023-10-09T05:00:22.063072+0000 mon.b (mon.1) 8 : audit 0 from='client.? 172.21.15.133:0/821512045' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:23.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:23 smithi194 bash[92666]: cluster 2023-10-09T05:00:22.612459+0000 osd.4 (osd.4) 22 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T05:00:23.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:23 smithi194 bash[92666]: cluster 2023-10-09T05:00:22.612561+0000 osd.5 (osd.5) 30 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T05:00:23.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:23 smithi194 bash[92666]: cluster 2023-10-09T05:00:22.614038+0000 osd.3 (osd.3) 22 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T05:00:23.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:23 smithi194 bash[92666]: cluster 2023-10-09T05:00:22.614186+0000 mon.a (mon.0) 481 : cluster 0 osdmap e102: 8 total, 8 up, 8 in 2023-10-09T05:00:23.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:23 smithi194 bash[92666]: cluster 2023-10-09T05:00:22.617279+0000 osd.3 (osd.3) 23 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T05:00:23.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:23 smithi194 bash[92666]: cluster 2023-10-09T05:00:22.620859+0000 osd.6 (osd.6) 17 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T05:00:23.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:23 smithi194 bash[92666]: cluster 2023-10-09T05:00:22.663705+0000 osd.7 (osd.7) 16 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T05:00:23.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:23 smithi194 bash[92666]: cluster 2023-10-09T05:00:23.054187+0000 mon.a (mon.0) 482 : cluster 3 Health check update: Degraded data redundancy: 11124/116232 objects degraded (9.571%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:24.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:23 smithi133 bash[102858]: cluster 2023-10-09T05:00:22.612459+0000 osd.4 (osd.4) 22 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T05:00:24.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:23 smithi133 bash[102858]: cluster 2023-10-09T05:00:22.612561+0000 osd.5 (osd.5) 30 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T05:00:24.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:23 smithi133 bash[102858]: cluster 2023-10-09T05:00:22.614038+0000 osd.3 (osd.3) 22 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T05:00:24.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:23 smithi133 bash[102858]: cluster 2023-10-09T05:00:22.614186+0000 mon.a (mon.0) 481 : cluster 0 osdmap e102: 8 total, 8 up, 8 in 2023-10-09T05:00:24.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:23 smithi133 bash[102858]: cluster 2023-10-09T05:00:22.617279+0000 osd.3 (osd.3) 23 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T05:00:24.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:23 smithi133 bash[102858]: cluster 2023-10-09T05:00:22.620859+0000 osd.6 (osd.6) 17 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T05:00:24.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:23 smithi133 bash[102858]: cluster 2023-10-09T05:00:22.663705+0000 osd.7 (osd.7) 16 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T05:00:24.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:23 smithi133 bash[102858]: cluster 2023-10-09T05:00:23.054187+0000 mon.a (mon.0) 482 : cluster 3 Health check update: Degraded data redundancy: 11124/116232 objects degraded (9.571%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:24.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:23 smithi133 bash[105589]: cluster 2023-10-09T05:00:22.612459+0000 osd.4 (osd.4) 22 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T05:00:24.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:23 smithi133 bash[105589]: cluster 2023-10-09T05:00:22.612561+0000 osd.5 (osd.5) 30 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T05:00:24.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:23 smithi133 bash[105589]: cluster 2023-10-09T05:00:22.614038+0000 osd.3 (osd.3) 22 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T05:00:24.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:23 smithi133 bash[105589]: cluster 2023-10-09T05:00:22.614186+0000 mon.a (mon.0) 481 : cluster 0 osdmap e102: 8 total, 8 up, 8 in 2023-10-09T05:00:24.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:23 smithi133 bash[105589]: cluster 2023-10-09T05:00:22.617279+0000 osd.3 (osd.3) 23 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T05:00:24.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:23 smithi133 bash[105589]: cluster 2023-10-09T05:00:22.620859+0000 osd.6 (osd.6) 17 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T05:00:24.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:23 smithi133 bash[105589]: cluster 2023-10-09T05:00:22.663705+0000 osd.7 (osd.7) 16 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T05:00:24.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:23 smithi133 bash[105589]: cluster 2023-10-09T05:00:23.054187+0000 mon.a (mon.0) 482 : cluster 3 Health check update: Degraded data redundancy: 11124/116232 objects degraded (9.571%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:24.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:00:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:00:24] "GET /metrics HTTP/1.1" 200 36300 "" "Prometheus/2.43.0" 2023-10-09T05:00:24.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:24 smithi194 bash[92666]: cluster 2023-10-09T05:00:23.523066+0000 mgr.y (mgr.25277) 331 : cluster 0 pgmap v268: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 480 op/s; 11124/116232 objects degraded (9.571%); 0 B/s, 49 objects/s recovering 2023-10-09T05:00:25.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:24 smithi133 bash[102858]: cluster 2023-10-09T05:00:23.523066+0000 mgr.y (mgr.25277) 331 : cluster 0 pgmap v268: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 480 op/s; 11124/116232 objects degraded (9.571%); 0 B/s, 49 objects/s recovering 2023-10-09T05:00:25.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:24 smithi133 bash[105589]: cluster 2023-10-09T05:00:23.523066+0000 mgr.y (mgr.25277) 331 : cluster 0 pgmap v268: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 480 op/s; 11124/116232 objects degraded (9.571%); 0 B/s, 49 objects/s recovering 2023-10-09T05:00:25.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:25 smithi194 bash[92666]: audit 2023-10-09T05:00:24.644140+0000 mon.c (mon.2) 23 : audit 0 from='client.? 172.21.15.133:0/3770799603' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:26.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:25 smithi133 bash[102858]: audit 2023-10-09T05:00:24.644140+0000 mon.c (mon.2) 23 : audit 0 from='client.? 172.21.15.133:0/3770799603' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:26.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:25 smithi133 bash[105589]: audit 2023-10-09T05:00:24.644140+0000 mon.c (mon.2) 23 : audit 0 from='client.? 172.21.15.133:0/3770799603' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:26.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:26 smithi194 bash[92666]: cluster 2023-10-09T05:00:25.523840+0000 mgr.y (mgr.25277) 332 : cluster 0 pgmap v269: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 10843/109866 objects degraded (9.869%); 0 B/s, 60 objects/s recovering 2023-10-09T05:00:27.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:26 smithi133 bash[102858]: cluster 2023-10-09T05:00:25.523840+0000 mgr.y (mgr.25277) 332 : cluster 0 pgmap v269: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 10843/109866 objects degraded (9.869%); 0 B/s, 60 objects/s recovering 2023-10-09T05:00:27.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:26 smithi133 bash[105589]: cluster 2023-10-09T05:00:25.523840+0000 mgr.y (mgr.25277) 332 : cluster 0 pgmap v269: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 10843/109866 objects degraded (9.869%); 0 B/s, 60 objects/s recovering 2023-10-09T05:00:27.919 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:27 smithi194 bash[92666]: audit 2023-10-09T05:00:27.284970+0000 mon.c (mon.2) 24 : audit 0 from='client.? 172.21.15.133:0/1261527708' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:27.919 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 05:00:27 smithi194 bash[80638]: logger=sqlstore.transactions t=2023-10-09T05:00:27.917251784Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-10-09T05:00:28.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:27 smithi133 bash[102858]: audit 2023-10-09T05:00:27.284970+0000 mon.c (mon.2) 24 : audit 0 from='client.? 172.21.15.133:0/1261527708' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:28.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:27 smithi133 bash[105589]: audit 2023-10-09T05:00:27.284970+0000 mon.c (mon.2) 24 : audit 0 from='client.? 172.21.15.133:0/1261527708' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:28.386 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 05:00:28 smithi194 bash[80638]: logger=cleanup t=2023-10-09T05:00:28.047334568Z level=info msg="Completed cleanup jobs" duration=141.36873ms 2023-10-09T05:00:29.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:28 smithi133 bash[102858]: cluster 2023-10-09T05:00:27.524428+0000 mgr.y (mgr.25277) 333 : cluster 0 pgmap v270: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 10798/108336 objects degraded (9.967%); 0 B/s, 65 objects/s recovering 2023-10-09T05:00:29.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:28 smithi133 bash[102858]: cluster 2023-10-09T05:00:28.055416+0000 mon.a (mon.0) 483 : cluster 3 Health check update: Degraded data redundancy: 10798/108336 objects degraded (9.967%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:29.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:28 smithi133 bash[102858]: audit 2023-10-09T05:00:28.284589+0000 mon.a (mon.0) 484 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:00:29.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:28 smithi133 bash[102858]: audit 2023-10-09T05:00:28.422215+0000 mon.a (mon.0) 485 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:29.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:28 smithi133 bash[105589]: cluster 2023-10-09T05:00:27.524428+0000 mgr.y (mgr.25277) 333 : cluster 0 pgmap v270: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 10798/108336 objects degraded (9.967%); 0 B/s, 65 objects/s recovering 2023-10-09T05:00:29.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:28 smithi133 bash[105589]: cluster 2023-10-09T05:00:28.055416+0000 mon.a (mon.0) 483 : cluster 3 Health check update: Degraded data redundancy: 10798/108336 objects degraded (9.967%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:29.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:28 smithi133 bash[105589]: audit 2023-10-09T05:00:28.284589+0000 mon.a (mon.0) 484 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:00:29.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:28 smithi133 bash[105589]: audit 2023-10-09T05:00:28.422215+0000 mon.a (mon.0) 485 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:29.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:28 smithi194 bash[92666]: cluster 2023-10-09T05:00:27.524428+0000 mgr.y (mgr.25277) 333 : cluster 0 pgmap v270: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 10798/108336 objects degraded (9.967%); 0 B/s, 65 objects/s recovering 2023-10-09T05:00:29.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:28 smithi194 bash[92666]: cluster 2023-10-09T05:00:28.055416+0000 mon.a (mon.0) 483 : cluster 3 Health check update: Degraded data redundancy: 10798/108336 objects degraded (9.967%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:29.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:28 smithi194 bash[92666]: audit 2023-10-09T05:00:28.284589+0000 mon.a (mon.0) 484 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:00:29.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:28 smithi194 bash[92666]: audit 2023-10-09T05:00:28.422215+0000 mon.a (mon.0) 485 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:30.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:29 smithi133 bash[102858]: audit 2023-10-09T05:00:28.284816+0000 mgr.y (mgr.25277) 334 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:00:30.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:29 smithi133 bash[102858]: cephadm 2023-10-09T05:00:28.287335+0000 mgr.y (mgr.25277) 335 : cephadm 1 Upgrade: osd.2 is safe to restart 2023-10-09T05:00:30.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:29 smithi133 bash[105589]: audit 2023-10-09T05:00:28.284816+0000 mgr.y (mgr.25277) 334 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:00:30.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:29 smithi133 bash[105589]: cephadm 2023-10-09T05:00:28.287335+0000 mgr.y (mgr.25277) 335 : cephadm 1 Upgrade: osd.2 is safe to restart 2023-10-09T05:00:30.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:29 smithi194 bash[92666]: audit 2023-10-09T05:00:28.284816+0000 mgr.y (mgr.25277) 334 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:00:30.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:29 smithi194 bash[92666]: cephadm 2023-10-09T05:00:28.287335+0000 mgr.y (mgr.25277) 335 : cephadm 1 Upgrade: osd.2 is safe to restart 2023-10-09T05:00:31.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:30 smithi133 bash[102858]: cluster 2023-10-09T05:00:29.524953+0000 mgr.y (mgr.25277) 336 : cluster 0 pgmap v271: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 328 op/s; 10798/108336 objects degraded (9.967%); 0 B/s, 40 objects/s recovering 2023-10-09T05:00:31.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:30 smithi133 bash[102858]: audit 2023-10-09T05:00:29.909583+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.133:0/1581021116' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:31.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:30 smithi133 bash[105589]: cluster 2023-10-09T05:00:29.524953+0000 mgr.y (mgr.25277) 336 : cluster 0 pgmap v271: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 328 op/s; 10798/108336 objects degraded (9.967%); 0 B/s, 40 objects/s recovering 2023-10-09T05:00:31.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:30 smithi133 bash[105589]: audit 2023-10-09T05:00:29.909583+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.133:0/1581021116' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:31.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:30 smithi194 bash[92666]: cluster 2023-10-09T05:00:29.524953+0000 mgr.y (mgr.25277) 336 : cluster 0 pgmap v271: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 328 op/s; 10798/108336 objects degraded (9.967%); 0 B/s, 40 objects/s recovering 2023-10-09T05:00:31.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:30 smithi194 bash[92666]: audit 2023-10-09T05:00:29.909583+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.133:0/1581021116' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:33.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:32 smithi133 bash[102858]: cluster 2023-10-09T05:00:31.525913+0000 mgr.y (mgr.25277) 337 : cluster 0 pgmap v272: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 10615/98853 objects degraded (10.738%); 0 B/s, 51 objects/s recovering 2023-10-09T05:00:33.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:32 smithi133 bash[102858]: audit 2023-10-09T05:00:32.503903+0000 mon.a (mon.0) 486 : audit 0 from='client.? 172.21.15.133:0/992027227' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:33.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:32 smithi133 bash[105589]: cluster 2023-10-09T05:00:31.525913+0000 mgr.y (mgr.25277) 337 : cluster 0 pgmap v272: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 10615/98853 objects degraded (10.738%); 0 B/s, 51 objects/s recovering 2023-10-09T05:00:33.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:32 smithi133 bash[105589]: audit 2023-10-09T05:00:32.503903+0000 mon.a (mon.0) 486 : audit 0 from='client.? 172.21.15.133:0/992027227' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:33.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:32 smithi194 bash[92666]: cluster 2023-10-09T05:00:31.525913+0000 mgr.y (mgr.25277) 337 : cluster 0 pgmap v272: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 10615/98853 objects degraded (10.738%); 0 B/s, 51 objects/s recovering 2023-10-09T05:00:33.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:32 smithi194 bash[92666]: audit 2023-10-09T05:00:32.503903+0000 mon.a (mon.0) 486 : audit 0 from='client.? 172.21.15.133:0/992027227' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:33.682 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:33 smithi133 bash[105589]: cluster 2023-10-09T05:00:33.056612+0000 mon.a (mon.0) 487 : cluster 3 Health check update: Degraded data redundancy: 10615/98853 objects degraded (10.738%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:33.683 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:33 smithi133 bash[102858]: cluster 2023-10-09T05:00:33.056612+0000 mon.a (mon.0) 487 : cluster 3 Health check update: Degraded data redundancy: 10615/98853 objects degraded (10.738%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:34.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:33 smithi194 bash[92666]: cluster 2023-10-09T05:00:33.056612+0000 mon.a (mon.0) 487 : cluster 3 Health check update: Degraded data redundancy: 10615/98853 objects degraded (10.738%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:34.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:00:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:00:34] "GET /metrics HTTP/1.1" 200 36286 "" "Prometheus/2.43.0" 2023-10-09T05:00:35.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:34 smithi133 bash[105589]: cluster 2023-10-09T05:00:33.526409+0000 mgr.y (mgr.25277) 338 : cluster 0 pgmap v273: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 10615/98853 objects degraded (10.738%); 0 B/s, 46 objects/s recovering 2023-10-09T05:00:35.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:34 smithi133 bash[105589]: audit 2023-10-09T05:00:34.441053+0000 mon.a (mon.0) 488 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:00:35.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:34 smithi133 bash[105589]: audit 2023-10-09T05:00:34.442671+0000 mon.a (mon.0) 489 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T05:00:35.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:34 smithi133 bash[105589]: audit 2023-10-09T05:00:34.444010+0000 mon.a (mon.0) 490 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:00:35.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:34 smithi133 bash[102858]: cluster 2023-10-09T05:00:33.526409+0000 mgr.y (mgr.25277) 338 : cluster 0 pgmap v273: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 10615/98853 objects degraded (10.738%); 0 B/s, 46 objects/s recovering 2023-10-09T05:00:35.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:34 smithi133 bash[102858]: audit 2023-10-09T05:00:34.441053+0000 mon.a (mon.0) 488 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:00:35.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:34 smithi133 bash[102858]: audit 2023-10-09T05:00:34.442671+0000 mon.a (mon.0) 489 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T05:00:35.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:34 smithi133 bash[102858]: audit 2023-10-09T05:00:34.444010+0000 mon.a (mon.0) 490 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:00:35.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:34 smithi194 bash[92666]: cluster 2023-10-09T05:00:33.526409+0000 mgr.y (mgr.25277) 338 : cluster 0 pgmap v273: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 10615/98853 objects degraded (10.738%); 0 B/s, 46 objects/s recovering 2023-10-09T05:00:35.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:34 smithi194 bash[92666]: audit 2023-10-09T05:00:34.441053+0000 mon.a (mon.0) 488 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:00:35.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:34 smithi194 bash[92666]: audit 2023-10-09T05:00:34.442671+0000 mon.a (mon.0) 489 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T05:00:35.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:34 smithi194 bash[92666]: audit 2023-10-09T05:00:34.444010+0000 mon.a (mon.0) 490 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:00:36.027 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:35 smithi133 bash[102858]: cephadm 2023-10-09T05:00:34.427822+0000 mgr.y (mgr.25277) 339 : cephadm 1 Upgrade: Updating osd.2 2023-10-09T05:00:36.028 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:35 smithi133 bash[102858]: cephadm 2023-10-09T05:00:34.449164+0000 mgr.y (mgr.25277) 340 : cephadm 1 Deploying daemon osd.2 on smithi133 2023-10-09T05:00:36.028 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:35 smithi133 bash[102858]: audit 2023-10-09T05:00:35.143374+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.133:0/3578593777' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:36.028 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:35 smithi133 bash[105589]: cephadm 2023-10-09T05:00:34.427822+0000 mgr.y (mgr.25277) 339 : cephadm 1 Upgrade: Updating osd.2 2023-10-09T05:00:36.028 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:35 smithi133 bash[105589]: cephadm 2023-10-09T05:00:34.449164+0000 mgr.y (mgr.25277) 340 : cephadm 1 Deploying daemon osd.2 on smithi133 2023-10-09T05:00:36.028 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:35 smithi133 bash[105589]: audit 2023-10-09T05:00:35.143374+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.133:0/3578593777' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:36.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:35 smithi194 bash[92666]: cephadm 2023-10-09T05:00:34.427822+0000 mgr.y (mgr.25277) 339 : cephadm 1 Upgrade: Updating osd.2 2023-10-09T05:00:36.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:35 smithi194 bash[92666]: cephadm 2023-10-09T05:00:34.449164+0000 mgr.y (mgr.25277) 340 : cephadm 1 Deploying daemon osd.2 on smithi133 2023-10-09T05:00:36.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:35 smithi194 bash[92666]: audit 2023-10-09T05:00:35.143374+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.133:0/3578593777' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:36.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:36 smithi133 bash[102858]: cluster 2023-10-09T05:00:35.527189+0000 mgr.y (mgr.25277) 341 : cluster 0 pgmap v274: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 10436/91077 objects degraded (11.458%); 0 B/s, 57 objects/s recovering 2023-10-09T05:00:36.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:36 smithi133 bash[105589]: cluster 2023-10-09T05:00:35.527189+0000 mgr.y (mgr.25277) 341 : cluster 0 pgmap v274: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 10436/91077 objects degraded (11.458%); 0 B/s, 57 objects/s recovering 2023-10-09T05:00:37.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:36 smithi194 bash[92666]: cluster 2023-10-09T05:00:35.527189+0000 mgr.y (mgr.25277) 341 : cluster 0 pgmap v274: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 10436/91077 objects degraded (11.458%); 0 B/s, 57 objects/s recovering 2023-10-09T05:00:38.697 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:38 smithi133 bash[102858]: cluster 2023-10-09T05:00:37.527820+0000 mgr.y (mgr.25277) 342 : cluster 0 pgmap v275: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 10436/89397 objects degraded (11.674%); 0 B/s, 33 objects/s recovering 2023-10-09T05:00:38.697 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:38 smithi133 bash[102858]: audit 2023-10-09T05:00:37.717772+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.133:0/2487753858' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:38.697 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:38 smithi133 bash[102858]: cluster 2023-10-09T05:00:38.057798+0000 mon.a (mon.0) 491 : cluster 3 Health check update: Degraded data redundancy: 10436/89397 objects degraded (11.674%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:39.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:38 smithi133 bash[105589]: cluster 2023-10-09T05:00:37.527820+0000 mgr.y (mgr.25277) 342 : cluster 0 pgmap v275: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 10436/89397 objects degraded (11.674%); 0 B/s, 33 objects/s recovering 2023-10-09T05:00:39.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:38 smithi133 bash[105589]: audit 2023-10-09T05:00:37.717772+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.133:0/2487753858' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:39.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:38 smithi133 bash[105589]: cluster 2023-10-09T05:00:38.057798+0000 mon.a (mon.0) 491 : cluster 3 Health check update: Degraded data redundancy: 10436/89397 objects degraded (11.674%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:39.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:38 smithi194 bash[92666]: cluster 2023-10-09T05:00:37.527820+0000 mgr.y (mgr.25277) 342 : cluster 0 pgmap v275: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 10436/89397 objects degraded (11.674%); 0 B/s, 33 objects/s recovering 2023-10-09T05:00:39.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:38 smithi194 bash[92666]: audit 2023-10-09T05:00:37.717772+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.133:0/2487753858' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:39.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:38 smithi194 bash[92666]: cluster 2023-10-09T05:00:38.057798+0000 mon.a (mon.0) 491 : cluster 3 Health check update: Degraded data redundancy: 10436/89397 objects degraded (11.674%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:40.717 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:40 smithi133 bash[102858]: cluster 2023-10-09T05:00:39.528318+0000 mgr.y (mgr.25277) 343 : cluster 0 pgmap v276: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 525 op/s; 10436/89397 objects degraded (11.674%); 0 B/s, 30 objects/s recovering 2023-10-09T05:00:40.718 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:40 smithi133 bash[102858]: audit 2023-10-09T05:00:40.346758+0000 mon.c (mon.2) 28 : audit 0 from='client.? 172.21.15.133:0/3210735393' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:40.718 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:40 smithi133 bash[102858]: cluster 2023-10-09T05:00:40.481152+0000 mon.a (mon.0) 492 : cluster 1 osd.2 marked itself down and dead 2023-10-09T05:00:40.718 INFO:journalctl@ceph.osd.2.smithi133.stdout:Oct 09 05:00:40 smithi133 systemd[1]: Stopping Ceph osd.2 for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T05:00:40.718 INFO:journalctl@ceph.osd.2.smithi133.stdout:Oct 09 05:00:40 smithi133 bash[58798]: debug 2023-10-09T05:00:40.476+0000 7fa102905700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T05:00:40.718 INFO:journalctl@ceph.osd.2.smithi133.stdout:Oct 09 05:00:40 smithi133 bash[58798]: debug 2023-10-09T05:00:40.476+0000 7fa102905700 -1 osd.2 102 *** Got signal Terminated *** 2023-10-09T05:00:40.718 INFO:journalctl@ceph.osd.2.smithi133.stdout:Oct 09 05:00:40 smithi133 bash[58798]: debug 2023-10-09T05:00:40.476+0000 7fa102905700 -1 osd.2 102 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:00:41.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:40 smithi133 bash[105589]: cluster 2023-10-09T05:00:39.528318+0000 mgr.y (mgr.25277) 343 : cluster 0 pgmap v276: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 525 op/s; 10436/89397 objects degraded (11.674%); 0 B/s, 30 objects/s recovering 2023-10-09T05:00:41.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:40 smithi133 bash[105589]: audit 2023-10-09T05:00:40.346758+0000 mon.c (mon.2) 28 : audit 0 from='client.? 172.21.15.133:0/3210735393' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:41.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:40 smithi133 bash[105589]: cluster 2023-10-09T05:00:40.481152+0000 mon.a (mon.0) 492 : cluster 1 osd.2 marked itself down and dead 2023-10-09T05:00:41.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:40 smithi194 bash[92666]: cluster 2023-10-09T05:00:39.528318+0000 mgr.y (mgr.25277) 343 : cluster 0 pgmap v276: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 525 op/s; 10436/89397 objects degraded (11.674%); 0 B/s, 30 objects/s recovering 2023-10-09T05:00:41.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:40 smithi194 bash[92666]: audit 2023-10-09T05:00:40.346758+0000 mon.c (mon.2) 28 : audit 0 from='client.? 172.21.15.133:0/3210735393' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:41.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:40 smithi194 bash[92666]: cluster 2023-10-09T05:00:40.481152+0000 mon.a (mon.0) 492 : cluster 1 osd.2 marked itself down and dead 2023-10-09T05:00:41.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:41 smithi133 bash[102858]: cluster 2023-10-09T05:00:40.696031+0000 mon.a (mon.0) 493 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:00:41.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:41 smithi133 bash[102858]: cluster 2023-10-09T05:00:40.700205+0000 osd.5 (osd.5) 31 : cluster 3 failed to encode map e103 with expected crc 2023-10-09T05:00:41.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:41 smithi133 bash[102858]: cluster 2023-10-09T05:00:40.701976+0000 mon.a (mon.0) 494 : cluster 0 osdmap e103: 8 total, 7 up, 8 in 2023-10-09T05:00:41.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:41 smithi133 bash[102858]: cluster 2023-10-09T05:00:40.703524+0000 osd.4 (osd.4) 23 : cluster 3 failed to encode map e103 with expected crc 2023-10-09T05:00:41.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:41 smithi133 bash[102858]: cluster 2023-10-09T05:00:40.705306+0000 osd.6 (osd.6) 18 : cluster 3 failed to encode map e103 with expected crc 2023-10-09T05:00:41.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:41 smithi133 bash[102858]: cluster 2023-10-09T05:00:40.706113+0000 osd.3 (osd.3) 24 : cluster 3 failed to encode map e103 with expected crc 2023-10-09T05:00:41.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:41 smithi133 bash[102858]: cluster 2023-10-09T05:00:40.707524+0000 osd.7 (osd.7) 17 : cluster 3 failed to encode map e103 with expected crc 2023-10-09T05:00:41.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:41 smithi133 bash[102858]: cluster 2023-10-09T05:00:41.706394+0000 osd.5 (osd.5) 32 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:41.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:41 smithi133 bash[102858]: cluster 2023-10-09T05:00:41.709178+0000 mon.a (mon.0) 495 : cluster 0 osdmap e104: 8 total, 7 up, 8 in 2023-10-09T05:00:41.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:41 smithi133 bash[102858]: cluster 2023-10-09T05:00:41.711294+0000 osd.5 (osd.5) 33 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:41.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:41 smithi133 bash[102858]: cluster 2023-10-09T05:00:41.713342+0000 osd.5 (osd.5) 34 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:41.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:41 smithi133 bash[105589]: cluster 2023-10-09T05:00:40.696031+0000 mon.a (mon.0) 493 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:00:41.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:41 smithi133 bash[105589]: cluster 2023-10-09T05:00:40.700205+0000 osd.5 (osd.5) 31 : cluster 3 failed to encode map e103 with expected crc 2023-10-09T05:00:41.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:41 smithi133 bash[105589]: cluster 2023-10-09T05:00:40.701976+0000 mon.a (mon.0) 494 : cluster 0 osdmap e103: 8 total, 7 up, 8 in 2023-10-09T05:00:41.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:41 smithi133 bash[105589]: cluster 2023-10-09T05:00:40.703524+0000 osd.4 (osd.4) 23 : cluster 3 failed to encode map e103 with expected crc 2023-10-09T05:00:41.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:41 smithi133 bash[105589]: cluster 2023-10-09T05:00:40.705306+0000 osd.6 (osd.6) 18 : cluster 3 failed to encode map e103 with expected crc 2023-10-09T05:00:41.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:41 smithi133 bash[105589]: cluster 2023-10-09T05:00:40.706113+0000 osd.3 (osd.3) 24 : cluster 3 failed to encode map e103 with expected crc 2023-10-09T05:00:41.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:41 smithi133 bash[105589]: cluster 2023-10-09T05:00:40.707524+0000 osd.7 (osd.7) 17 : cluster 3 failed to encode map e103 with expected crc 2023-10-09T05:00:41.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:41 smithi133 bash[105589]: cluster 2023-10-09T05:00:41.706394+0000 osd.5 (osd.5) 32 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:41.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:41 smithi133 bash[105589]: cluster 2023-10-09T05:00:41.709178+0000 mon.a (mon.0) 495 : cluster 0 osdmap e104: 8 total, 7 up, 8 in 2023-10-09T05:00:41.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:41 smithi133 bash[105589]: cluster 2023-10-09T05:00:41.711294+0000 osd.5 (osd.5) 33 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:41.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:41 smithi133 bash[105589]: cluster 2023-10-09T05:00:41.713342+0000 osd.5 (osd.5) 34 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:42.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:41 smithi194 bash[92666]: cluster 2023-10-09T05:00:40.696031+0000 mon.a (mon.0) 493 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:00:42.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:41 smithi194 bash[92666]: cluster 2023-10-09T05:00:40.700205+0000 osd.5 (osd.5) 31 : cluster 3 failed to encode map e103 with expected crc 2023-10-09T05:00:42.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:41 smithi194 bash[92666]: cluster 2023-10-09T05:00:40.701976+0000 mon.a (mon.0) 494 : cluster 0 osdmap e103: 8 total, 7 up, 8 in 2023-10-09T05:00:42.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:41 smithi194 bash[92666]: cluster 2023-10-09T05:00:40.703524+0000 osd.4 (osd.4) 23 : cluster 3 failed to encode map e103 with expected crc 2023-10-09T05:00:42.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:41 smithi194 bash[92666]: cluster 2023-10-09T05:00:40.705306+0000 osd.6 (osd.6) 18 : cluster 3 failed to encode map e103 with expected crc 2023-10-09T05:00:42.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:41 smithi194 bash[92666]: cluster 2023-10-09T05:00:40.706113+0000 osd.3 (osd.3) 24 : cluster 3 failed to encode map e103 with expected crc 2023-10-09T05:00:42.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:41 smithi194 bash[92666]: cluster 2023-10-09T05:00:40.707524+0000 osd.7 (osd.7) 17 : cluster 3 failed to encode map e103 with expected crc 2023-10-09T05:00:42.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:41 smithi194 bash[92666]: cluster 2023-10-09T05:00:41.706394+0000 osd.5 (osd.5) 32 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:42.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:41 smithi194 bash[92666]: cluster 2023-10-09T05:00:41.709178+0000 mon.a (mon.0) 495 : cluster 0 osdmap e104: 8 total, 7 up, 8 in 2023-10-09T05:00:42.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:41 smithi194 bash[92666]: cluster 2023-10-09T05:00:41.711294+0000 osd.5 (osd.5) 33 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:42.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:41 smithi194 bash[92666]: cluster 2023-10-09T05:00:41.713342+0000 osd.5 (osd.5) 34 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:42.494 INFO:journalctl@ceph.osd.2.smithi133.stdout:Oct 09 05:00:42 smithi133 bash[128053]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd-2 2023-10-09T05:00:42.495 INFO:journalctl@ceph.osd.2.smithi133.stdout:Oct 09 05:00:42 smithi133 bash[128121]: Error response from daemon: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd-2 2023-10-09T05:00:42.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:42 smithi133 bash[102858]: cluster 2023-10-09T05:00:41.529421+0000 mgr.y (mgr.25277) 344 : cluster 0 pgmap v278: 89 pgs: 6 peering, 4 stale+active+clean, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 75 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 10240/80052 objects degraded (12.792%); 0 B/s, 37 objects/s recovering 2023-10-09T05:00:42.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:42 smithi133 bash[102858]: cluster 2023-10-09T05:00:41.705572+0000 osd.3 (osd.3) 25 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:42.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:42 smithi133 bash[102858]: cluster 2023-10-09T05:00:41.708809+0000 osd.3 (osd.3) 26 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:42.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:42 smithi133 bash[102858]: cluster 2023-10-09T05:00:41.709318+0000 osd.4 (osd.4) 24 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:42.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:42 smithi133 bash[102858]: cluster 2023-10-09T05:00:41.710362+0000 osd.7 (osd.7) 18 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:42.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:42 smithi133 bash[102858]: cluster 2023-10-09T05:00:41.710505+0000 osd.6 (osd.6) 19 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:42.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:42 smithi133 bash[102858]: cluster 2023-10-09T05:00:41.710881+0000 osd.4 (osd.4) 25 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:42.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:42 smithi133 bash[102858]: cluster 2023-10-09T05:00:41.711338+0000 osd.3 (osd.3) 27 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:42.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:42 smithi133 bash[102858]: cluster 2023-10-09T05:00:41.713076+0000 osd.3 (osd.3) 28 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:42.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:42 smithi133 bash[102858]: cluster 2023-10-09T05:00:41.714502+0000 mon.a (mon.0) 496 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:00:42.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:42 smithi133 bash[102858]: cluster 2023-10-09T05:00:41.714982+0000 osd.7 (osd.7) 19 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:42.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:42 smithi133 bash[105589]: cluster 2023-10-09T05:00:41.529421+0000 mgr.y (mgr.25277) 344 : cluster 0 pgmap v278: 89 pgs: 6 peering, 4 stale+active+clean, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 75 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 10240/80052 objects degraded (12.792%); 0 B/s, 37 objects/s recovering 2023-10-09T05:00:42.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:42 smithi133 bash[105589]: cluster 2023-10-09T05:00:41.705572+0000 osd.3 (osd.3) 25 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:42.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:42 smithi133 bash[105589]: cluster 2023-10-09T05:00:41.708809+0000 osd.3 (osd.3) 26 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:42.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:42 smithi133 bash[105589]: cluster 2023-10-09T05:00:41.709318+0000 osd.4 (osd.4) 24 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:42.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:42 smithi133 bash[105589]: cluster 2023-10-09T05:00:41.710362+0000 osd.7 (osd.7) 18 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:42.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:42 smithi133 bash[105589]: cluster 2023-10-09T05:00:41.710505+0000 osd.6 (osd.6) 19 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:42.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:42 smithi133 bash[105589]: cluster 2023-10-09T05:00:41.710881+0000 osd.4 (osd.4) 25 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:42.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:42 smithi133 bash[105589]: cluster 2023-10-09T05:00:41.711338+0000 osd.3 (osd.3) 27 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:42.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:42 smithi133 bash[105589]: cluster 2023-10-09T05:00:41.713076+0000 osd.3 (osd.3) 28 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:42.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:42 smithi133 bash[105589]: cluster 2023-10-09T05:00:41.714502+0000 mon.a (mon.0) 496 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:00:42.804 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:42 smithi133 bash[105589]: cluster 2023-10-09T05:00:41.714982+0000 osd.7 (osd.7) 19 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:43.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:42 smithi194 bash[92666]: cluster 2023-10-09T05:00:41.529421+0000 mgr.y (mgr.25277) 344 : cluster 0 pgmap v278: 89 pgs: 6 peering, 4 stale+active+clean, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 75 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 10240/80052 objects degraded (12.792%); 0 B/s, 37 objects/s recovering 2023-10-09T05:00:43.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:42 smithi194 bash[92666]: cluster 2023-10-09T05:00:41.705572+0000 osd.3 (osd.3) 25 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:43.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:42 smithi194 bash[92666]: cluster 2023-10-09T05:00:41.708809+0000 osd.3 (osd.3) 26 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:43.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:42 smithi194 bash[92666]: cluster 2023-10-09T05:00:41.709318+0000 osd.4 (osd.4) 24 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:43.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:42 smithi194 bash[92666]: cluster 2023-10-09T05:00:41.710362+0000 osd.7 (osd.7) 18 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:43.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:42 smithi194 bash[92666]: cluster 2023-10-09T05:00:41.710505+0000 osd.6 (osd.6) 19 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:43.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:42 smithi194 bash[92666]: cluster 2023-10-09T05:00:41.710881+0000 osd.4 (osd.4) 25 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:43.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:42 smithi194 bash[92666]: cluster 2023-10-09T05:00:41.711338+0000 osd.3 (osd.3) 27 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:43.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:42 smithi194 bash[92666]: cluster 2023-10-09T05:00:41.713076+0000 osd.3 (osd.3) 28 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:43.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:42 smithi194 bash[92666]: cluster 2023-10-09T05:00:41.714502+0000 mon.a (mon.0) 496 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:00:43.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:42 smithi194 bash[92666]: cluster 2023-10-09T05:00:41.714982+0000 osd.7 (osd.7) 19 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T05:00:44.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:43 smithi133 bash[102858]: audit 2023-10-09T05:00:42.913942+0000 mon.c (mon.2) 29 : audit 0 from='client.? 172.21.15.133:0/889468100' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:44.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:43 smithi133 bash[102858]: cluster 2023-10-09T05:00:43.058934+0000 mon.a (mon.0) 497 : cluster 3 Health check update: Degraded data redundancy: 10240/80052 objects degraded (12.792%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:44.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:43 smithi133 bash[102858]: audit 2023-10-09T05:00:43.429873+0000 mon.a (mon.0) 498 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:00:44.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:43 smithi133 bash[102858]: audit 2023-10-09T05:00:43.431185+0000 mon.a (mon.0) 499 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:44.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:43 smithi133 bash[105589]: audit 2023-10-09T05:00:42.913942+0000 mon.c (mon.2) 29 : audit 0 from='client.? 172.21.15.133:0/889468100' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:44.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:43 smithi133 bash[105589]: cluster 2023-10-09T05:00:43.058934+0000 mon.a (mon.0) 497 : cluster 3 Health check update: Degraded data redundancy: 10240/80052 objects degraded (12.792%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:44.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:43 smithi133 bash[105589]: audit 2023-10-09T05:00:43.429873+0000 mon.a (mon.0) 498 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:00:44.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:43 smithi133 bash[105589]: audit 2023-10-09T05:00:43.431185+0000 mon.a (mon.0) 499 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:44.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:43 smithi194 bash[92666]: audit 2023-10-09T05:00:42.913942+0000 mon.c (mon.2) 29 : audit 0 from='client.? 172.21.15.133:0/889468100' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:44.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:43 smithi194 bash[92666]: cluster 2023-10-09T05:00:43.058934+0000 mon.a (mon.0) 497 : cluster 3 Health check update: Degraded data redundancy: 10240/80052 objects degraded (12.792%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:44.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:43 smithi194 bash[92666]: audit 2023-10-09T05:00:43.429873+0000 mon.a (mon.0) 498 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:00:44.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:43 smithi194 bash[92666]: audit 2023-10-09T05:00:43.431185+0000 mon.a (mon.0) 499 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:44.537 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:00:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:00:44] "GET /metrics HTTP/1.1" 200 36354 "" "Prometheus/2.43.0" 2023-10-09T05:00:44.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:44 smithi133 bash[102858]: cluster 2023-10-09T05:00:43.530059+0000 mgr.y (mgr.25277) 345 : cluster 0 pgmap v280: 89 pgs: 6 peering, 4 stale+active+clean, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 75 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 459 op/s; 10240/80052 objects degraded (12.792%); 0 B/s, 24 objects/s recovering 2023-10-09T05:00:44.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:44 smithi133 bash[105589]: cluster 2023-10-09T05:00:43.530059+0000 mgr.y (mgr.25277) 345 : cluster 0 pgmap v280: 89 pgs: 6 peering, 4 stale+active+clean, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 75 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 459 op/s; 10240/80052 objects degraded (12.792%); 0 B/s, 24 objects/s recovering 2023-10-09T05:00:45.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:44 smithi194 bash[92666]: cluster 2023-10-09T05:00:43.530059+0000 mgr.y (mgr.25277) 345 : cluster 0 pgmap v280: 89 pgs: 6 peering, 4 stale+active+clean, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 75 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 459 op/s; 10240/80052 objects degraded (12.792%); 0 B/s, 24 objects/s recovering 2023-10-09T05:00:45.903 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:45 smithi133 bash[102858]: audit 2023-10-09T05:00:45.498582+0000 mon.a (mon.0) 500 : audit 0 from='client.? 172.21.15.133:0/607801468' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:45.904 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:45 smithi133 bash[105589]: audit 2023-10-09T05:00:45.498582+0000 mon.a (mon.0) 500 : audit 0 from='client.? 172.21.15.133:0/607801468' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:45.904 INFO:journalctl@ceph.osd.2.smithi133.stdout:Oct 09 05:00:45 smithi133 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@osd.2.service: Succeeded. 2023-10-09T05:00:45.904 INFO:journalctl@ceph.osd.2.smithi133.stdout:Oct 09 05:00:45 smithi133 systemd[1]: Stopped Ceph osd.2 for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T05:00:45.904 INFO:journalctl@ceph.osd.2.smithi133.stdout:Oct 09 05:00:45 smithi133 systemd[1]: Started Ceph osd.2 for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T05:00:46.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:45 smithi194 bash[92666]: audit 2023-10-09T05:00:45.498582+0000 mon.a (mon.0) 500 : audit 0 from='client.? 172.21.15.133:0/607801468' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:47.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:46 smithi133 bash[102858]: cluster 2023-10-09T05:00:45.530887+0000 mgr.y (mgr.25277) 346 : cluster 0 pgmap v281: 89 pgs: 63 active+clean, 10 active+undersized, 5 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 stale+active+clean, 6 peering; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 14468/73605 objects degraded (19.656%); 0 B/s, 56 objects/s recovering 2023-10-09T05:00:47.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:46 smithi133 bash[102858]: audit 2023-10-09T05:00:45.951471+0000 mon.a (mon.0) 501 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:00:47.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:46 smithi133 bash[102858]: audit 2023-10-09T05:00:45.969456+0000 mon.a (mon.0) 502 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:00:47.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:46 smithi133 bash[102858]: audit 2023-10-09T05:00:46.020081+0000 mon.a (mon.0) 503 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:00:47.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:46 smithi133 bash[105589]: cluster 2023-10-09T05:00:45.530887+0000 mgr.y (mgr.25277) 346 : cluster 0 pgmap v281: 89 pgs: 63 active+clean, 10 active+undersized, 5 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 stale+active+clean, 6 peering; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 14468/73605 objects degraded (19.656%); 0 B/s, 56 objects/s recovering 2023-10-09T05:00:47.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:46 smithi133 bash[105589]: audit 2023-10-09T05:00:45.951471+0000 mon.a (mon.0) 501 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:00:47.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:46 smithi133 bash[105589]: audit 2023-10-09T05:00:45.969456+0000 mon.a (mon.0) 502 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:00:47.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:46 smithi133 bash[105589]: audit 2023-10-09T05:00:46.020081+0000 mon.a (mon.0) 503 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:00:47.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:46 smithi194 bash[92666]: cluster 2023-10-09T05:00:45.530887+0000 mgr.y (mgr.25277) 346 : cluster 0 pgmap v281: 89 pgs: 63 active+clean, 10 active+undersized, 5 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 stale+active+clean, 6 peering; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 14468/73605 objects degraded (19.656%); 0 B/s, 56 objects/s recovering 2023-10-09T05:00:47.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:46 smithi194 bash[92666]: audit 2023-10-09T05:00:45.951471+0000 mon.a (mon.0) 501 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:00:47.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:46 smithi194 bash[92666]: audit 2023-10-09T05:00:45.969456+0000 mon.a (mon.0) 502 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:00:47.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:46 smithi194 bash[92666]: audit 2023-10-09T05:00:46.020081+0000 mon.a (mon.0) 503 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:00:48.235 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:48 smithi133 bash[102858]: cluster 2023-10-09T05:00:47.947637+0000 mon.a (mon.0) 504 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:00:48.236 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:48 smithi133 bash[105589]: cluster 2023-10-09T05:00:47.947637+0000 mon.a (mon.0) 504 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:00:48.236 INFO:journalctl@ceph.osd.2.smithi133.stdout:Oct 09 05:00:47 smithi133 bash[128320]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T05:00:48.236 INFO:journalctl@ceph.osd.2.smithi133.stdout:Oct 09 05:00:47 smithi133 bash[128320]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-09T05:00:48.236 INFO:journalctl@ceph.osd.2.smithi133.stdout:Oct 09 05:00:47 smithi133 bash[128320]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-10-09T05:00:48.236 INFO:journalctl@ceph.osd.2.smithi133.stdout:Oct 09 05:00:47 smithi133 bash[128320]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-10-09T05:00:48.237 INFO:journalctl@ceph.osd.2.smithi133.stdout:Oct 09 05:00:47 smithi133 bash[128320]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-10-09T05:00:48.237 INFO:journalctl@ceph.osd.2.smithi133.stdout:Oct 09 05:00:47 smithi133 bash[128320]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-10-09T05:00:48.237 INFO:journalctl@ceph.osd.2.smithi133.stdout:Oct 09 05:00:47 smithi133 bash[128320]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-09T05:00:48.237 INFO:journalctl@ceph.osd.2.smithi133.stdout:Oct 09 05:00:47 smithi133 bash[128320]: --> ceph-volume lvm activate successful for osd ID: 2 2023-10-09T05:00:48.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:48 smithi194 bash[92666]: cluster 2023-10-09T05:00:47.947637+0000 mon.a (mon.0) 504 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:00:49.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:49 smithi133 bash[102858]: cluster 2023-10-09T05:00:47.531498+0000 mgr.y (mgr.25277) 347 : cluster 0 pgmap v282: 89 pgs: 60 active+clean, 17 active+undersized, 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 18746/71907 objects degraded (26.070%); 0 B/s, 56 objects/s recovering 2023-10-09T05:00:49.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:49 smithi133 bash[102858]: cluster 2023-10-09T05:00:48.060565+0000 mon.a (mon.0) 505 : cluster 3 Health check update: Degraded data redundancy: 18746/71907 objects degraded (26.070%), 12 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:49.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:49 smithi133 bash[102858]: audit 2023-10-09T05:00:48.133111+0000 mon.a (mon.0) 506 : audit 0 from='client.? 172.21.15.133:0/1785406432' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:49.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:49 smithi133 bash[105589]: cluster 2023-10-09T05:00:47.531498+0000 mgr.y (mgr.25277) 347 : cluster 0 pgmap v282: 89 pgs: 60 active+clean, 17 active+undersized, 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 18746/71907 objects degraded (26.070%); 0 B/s, 56 objects/s recovering 2023-10-09T05:00:49.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:49 smithi133 bash[105589]: cluster 2023-10-09T05:00:48.060565+0000 mon.a (mon.0) 505 : cluster 3 Health check update: Degraded data redundancy: 18746/71907 objects degraded (26.070%), 12 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:49.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:49 smithi133 bash[105589]: audit 2023-10-09T05:00:48.133111+0000 mon.a (mon.0) 506 : audit 0 from='client.? 172.21.15.133:0/1785406432' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:49.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:49 smithi194 bash[92666]: cluster 2023-10-09T05:00:47.531498+0000 mgr.y (mgr.25277) 347 : cluster 0 pgmap v282: 89 pgs: 60 active+clean, 17 active+undersized, 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 18746/71907 objects degraded (26.070%); 0 B/s, 56 objects/s recovering 2023-10-09T05:00:49.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:49 smithi194 bash[92666]: cluster 2023-10-09T05:00:48.060565+0000 mon.a (mon.0) 505 : cluster 3 Health check update: Degraded data redundancy: 18746/71907 objects degraded (26.070%), 12 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:49.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:49 smithi194 bash[92666]: audit 2023-10-09T05:00:48.133111+0000 mon.a (mon.0) 506 : audit 0 from='client.? 172.21.15.133:0/1785406432' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:50.799 INFO:journalctl@ceph.osd.2.smithi133.stdout:Oct 09 05:00:50 smithi133 bash[128609]: debug 2023-10-09T05:00:50.488+0000 7f868f247700 1 -- 172.21.15.133:0/2234337167 <== mon.2 v2:172.21.15.133:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55df05af44e0 con 0x55df05b4f000 2023-10-09T05:00:51.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:51 smithi133 bash[102858]: cluster 2023-10-09T05:00:49.531945+0000 mgr.y (mgr.25277) 348 : cluster 0 pgmap v283: 89 pgs: 60 active+clean, 17 active+undersized, 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 373 op/s; 18746/71907 objects degraded (26.070%); 0 B/s, 29 objects/s recovering 2023-10-09T05:00:51.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:51 smithi133 bash[102858]: audit 2023-10-09T05:00:50.733050+0000 mon.a (mon.0) 507 : audit 0 from='client.? 172.21.15.133:0/3871341844' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:51.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:51 smithi133 bash[105589]: cluster 2023-10-09T05:00:49.531945+0000 mgr.y (mgr.25277) 348 : cluster 0 pgmap v283: 89 pgs: 60 active+clean, 17 active+undersized, 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 373 op/s; 18746/71907 objects degraded (26.070%); 0 B/s, 29 objects/s recovering 2023-10-09T05:00:51.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:51 smithi133 bash[105589]: audit 2023-10-09T05:00:50.733050+0000 mon.a (mon.0) 507 : audit 0 from='client.? 172.21.15.133:0/3871341844' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:51.300 INFO:journalctl@ceph.osd.2.smithi133.stdout:Oct 09 05:00:51 smithi133 bash[128609]: debug 2023-10-09T05:00:51.048+0000 7f86983cd640 -1 Falling back to public interface 2023-10-09T05:00:51.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:51 smithi194 bash[92666]: cluster 2023-10-09T05:00:49.531945+0000 mgr.y (mgr.25277) 348 : cluster 0 pgmap v283: 89 pgs: 60 active+clean, 17 active+undersized, 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 373 op/s; 18746/71907 objects degraded (26.070%); 0 B/s, 29 objects/s recovering 2023-10-09T05:00:51.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:51 smithi194 bash[92666]: audit 2023-10-09T05:00:50.733050+0000 mon.a (mon.0) 507 : audit 0 from='client.? 172.21.15.133:0/3871341844' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:53.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:53 smithi133 bash[102858]: cluster 2023-10-09T05:00:51.532788+0000 mgr.y (mgr.25277) 349 : cluster 0 pgmap v284: 89 pgs: 60 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 17189/60972 objects degraded (28.192%); 0 B/s, 43 objects/s recovering 2023-10-09T05:00:53.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:53 smithi133 bash[105589]: cluster 2023-10-09T05:00:51.532788+0000 mgr.y (mgr.25277) 349 : cluster 0 pgmap v284: 89 pgs: 60 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 17189/60972 objects degraded (28.192%); 0 B/s, 43 objects/s recovering 2023-10-09T05:00:53.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:53 smithi194 bash[92666]: cluster 2023-10-09T05:00:51.532788+0000 mgr.y (mgr.25277) 349 : cluster 0 pgmap v284: 89 pgs: 60 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 17189/60972 objects degraded (28.192%); 0 B/s, 43 objects/s recovering 2023-10-09T05:00:54.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:54 smithi133 bash[102858]: cluster 2023-10-09T05:00:53.062441+0000 mon.a (mon.0) 508 : cluster 3 Health check update: Degraded data redundancy: 17189/60972 objects degraded (28.192%), 12 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:54.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:54 smithi133 bash[102858]: audit 2023-10-09T05:00:53.398013+0000 mon.a (mon.0) 509 : audit 0 from='client.? 172.21.15.133:0/79075502' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:54.283 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:00:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:00:54] "GET /metrics HTTP/1.1" 200 36354 "" "Prometheus/2.43.0" 2023-10-09T05:00:54.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:54 smithi133 bash[105589]: cluster 2023-10-09T05:00:53.062441+0000 mon.a (mon.0) 508 : cluster 3 Health check update: Degraded data redundancy: 17189/60972 objects degraded (28.192%), 12 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:54.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:54 smithi133 bash[105589]: audit 2023-10-09T05:00:53.398013+0000 mon.a (mon.0) 509 : audit 0 from='client.? 172.21.15.133:0/79075502' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:54.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:54 smithi194 bash[92666]: cluster 2023-10-09T05:00:53.062441+0000 mon.a (mon.0) 508 : cluster 3 Health check update: Degraded data redundancy: 17189/60972 objects degraded (28.192%), 12 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:54.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:54 smithi194 bash[92666]: audit 2023-10-09T05:00:53.398013+0000 mon.a (mon.0) 509 : audit 0 from='client.? 172.21.15.133:0/79075502' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:55.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:55 smithi133 bash[102858]: cluster 2023-10-09T05:00:53.533372+0000 mgr.y (mgr.25277) 350 : cluster 0 pgmap v285: 89 pgs: 60 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 17189/60972 objects degraded (28.192%); 0 B/s, 36 objects/s recovering 2023-10-09T05:00:55.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:55 smithi133 bash[102858]: audit 2023-10-09T05:00:54.287912+0000 mon.a (mon.0) 510 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:00:55.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:55 smithi133 bash[102858]: audit 2023-10-09T05:00:54.299802+0000 mon.a (mon.0) 511 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:00:55.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:55 smithi133 bash[105589]: cluster 2023-10-09T05:00:53.533372+0000 mgr.y (mgr.25277) 350 : cluster 0 pgmap v285: 89 pgs: 60 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 17189/60972 objects degraded (28.192%); 0 B/s, 36 objects/s recovering 2023-10-09T05:00:55.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:55 smithi133 bash[105589]: audit 2023-10-09T05:00:54.287912+0000 mon.a (mon.0) 510 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:00:55.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:55 smithi133 bash[105589]: audit 2023-10-09T05:00:54.299802+0000 mon.a (mon.0) 511 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:00:55.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:55 smithi194 bash[92666]: cluster 2023-10-09T05:00:53.533372+0000 mgr.y (mgr.25277) 350 : cluster 0 pgmap v285: 89 pgs: 60 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 17189/60972 objects degraded (28.192%); 0 B/s, 36 objects/s recovering 2023-10-09T05:00:55.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:55 smithi194 bash[92666]: audit 2023-10-09T05:00:54.287912+0000 mon.a (mon.0) 510 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:00:55.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:55 smithi194 bash[92666]: audit 2023-10-09T05:00:54.299802+0000 mon.a (mon.0) 511 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:00:56.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:56 smithi194 bash[92666]: audit 2023-10-09T05:00:55.355565+0000 mon.a (mon.0) 512 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:00:56.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:56 smithi194 bash[92666]: audit 2023-10-09T05:00:55.365812+0000 mon.a (mon.0) 513 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:00:56.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:56 smithi194 bash[92666]: cluster 2023-10-09T05:00:55.534251+0000 mgr.y (mgr.25277) 351 : cluster 0 pgmap v286: 89 pgs: 60 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 16428/52764 objects degraded (31.135%); 0 B/s, 49 objects/s recovering 2023-10-09T05:00:56.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:56 smithi194 bash[92666]: audit 2023-10-09T05:00:55.993770+0000 mon.c (mon.2) 30 : audit 0 from='client.? 172.21.15.133:0/648033341' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:56.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:56 smithi133 bash[102858]: audit 2023-10-09T05:00:55.355565+0000 mon.a (mon.0) 512 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:00:56.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:56 smithi133 bash[102858]: audit 2023-10-09T05:00:55.365812+0000 mon.a (mon.0) 513 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:00:56.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:56 smithi133 bash[102858]: cluster 2023-10-09T05:00:55.534251+0000 mgr.y (mgr.25277) 351 : cluster 0 pgmap v286: 89 pgs: 60 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 16428/52764 objects degraded (31.135%); 0 B/s, 49 objects/s recovering 2023-10-09T05:00:56.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:56 smithi133 bash[102858]: audit 2023-10-09T05:00:55.993770+0000 mon.c (mon.2) 30 : audit 0 from='client.? 172.21.15.133:0/648033341' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:56.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:56 smithi133 bash[105589]: audit 2023-10-09T05:00:55.355565+0000 mon.a (mon.0) 512 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:00:56.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:56 smithi133 bash[105589]: audit 2023-10-09T05:00:55.365812+0000 mon.a (mon.0) 513 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:00:56.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:56 smithi133 bash[105589]: cluster 2023-10-09T05:00:55.534251+0000 mgr.y (mgr.25277) 351 : cluster 0 pgmap v286: 89 pgs: 60 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 16428/52764 objects degraded (31.135%); 0 B/s, 49 objects/s recovering 2023-10-09T05:00:56.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:56 smithi133 bash[105589]: audit 2023-10-09T05:00:55.993770+0000 mon.c (mon.2) 30 : audit 0 from='client.? 172.21.15.133:0/648033341' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:00:58.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:58 smithi194 bash[92666]: cluster 2023-10-09T05:00:57.534925+0000 mgr.y (mgr.25277) 352 : cluster 0 pgmap v287: 89 pgs: 60 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 15815/50925 objects degraded (31.055%); 0 B/s, 27 objects/s recovering 2023-10-09T05:00:58.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:58 smithi194 bash[92666]: cluster 2023-10-09T05:00:58.065290+0000 mon.a (mon.0) 514 : cluster 3 Health check update: Degraded data redundancy: 15815/50925 objects degraded (31.055%), 12 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:58.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:58 smithi194 bash[92666]: audit 2023-10-09T05:00:58.435004+0000 mon.a (mon.0) 515 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:00:58.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:58 smithi194 bash[92666]: audit 2023-10-09T05:00:58.437757+0000 mon.a (mon.0) 516 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:59.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:58 smithi133 bash[102858]: cluster 2023-10-09T05:00:57.534925+0000 mgr.y (mgr.25277) 352 : cluster 0 pgmap v287: 89 pgs: 60 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 15815/50925 objects degraded (31.055%); 0 B/s, 27 objects/s recovering 2023-10-09T05:00:59.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:58 smithi133 bash[102858]: cluster 2023-10-09T05:00:58.065290+0000 mon.a (mon.0) 514 : cluster 3 Health check update: Degraded data redundancy: 15815/50925 objects degraded (31.055%), 12 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:59.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:58 smithi133 bash[102858]: audit 2023-10-09T05:00:58.435004+0000 mon.a (mon.0) 515 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:00:59.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:58 smithi133 bash[102858]: audit 2023-10-09T05:00:58.437757+0000 mon.a (mon.0) 516 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:59.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:58 smithi133 bash[105589]: cluster 2023-10-09T05:00:57.534925+0000 mgr.y (mgr.25277) 352 : cluster 0 pgmap v287: 89 pgs: 60 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 15815/50925 objects degraded (31.055%); 0 B/s, 27 objects/s recovering 2023-10-09T05:00:59.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:58 smithi133 bash[105589]: cluster 2023-10-09T05:00:58.065290+0000 mon.a (mon.0) 514 : cluster 3 Health check update: Degraded data redundancy: 15815/50925 objects degraded (31.055%), 12 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:59.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:58 smithi133 bash[105589]: audit 2023-10-09T05:00:58.435004+0000 mon.a (mon.0) 515 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:00:59.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:58 smithi133 bash[105589]: audit 2023-10-09T05:00:58.437757+0000 mon.a (mon.0) 516 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:00:59 smithi194 bash[92666]: audit 2023-10-09T05:00:58.588919+0000 mon.a (mon.0) 517 : audit 0 from='client.? 172.21.15.133:0/483107523' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:00.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:00:59 smithi133 bash[102858]: audit 2023-10-09T05:00:58.588919+0000 mon.a (mon.0) 517 : audit 0 from='client.? 172.21.15.133:0/483107523' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:00.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:00:59 smithi133 bash[105589]: audit 2023-10-09T05:00:58.588919+0000 mon.a (mon.0) 517 : audit 0 from='client.? 172.21.15.133:0/483107523' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:00.553 INFO:journalctl@ceph.osd.2.smithi133.stdout:Oct 09 05:01:00 smithi133 bash[128609]: debug 2023-10-09T05:01:00.084+0000 7f86983cd640 -1 osd.2 102 log_to_monitors true 2023-10-09T05:01:00.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:00 smithi194 bash[92666]: cluster 2023-10-09T05:00:59.535746+0000 mgr.y (mgr.25277) 353 : cluster 0 pgmap v288: 89 pgs: 60 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 15815/50925 objects degraded (31.055%); 0 B/s, 27 objects/s recovering 2023-10-09T05:01:00.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:00 smithi194 bash[92666]: audit 2023-10-09T05:01:00.103910+0000 mon.c (mon.2) 31 : audit 1 from='osd.2 [v2:172.21.15.133:6818/3595178639,v1:172.21.15.133:6819/3595178639]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T05:01:00.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:00 smithi194 bash[92666]: audit 2023-10-09T05:01:00.104538+0000 mon.a (mon.0) 518 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T05:01:01.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:00 smithi133 bash[102858]: cluster 2023-10-09T05:00:59.535746+0000 mgr.y (mgr.25277) 353 : cluster 0 pgmap v288: 89 pgs: 60 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 15815/50925 objects degraded (31.055%); 0 B/s, 27 objects/s recovering 2023-10-09T05:01:01.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:00 smithi133 bash[102858]: audit 2023-10-09T05:01:00.103910+0000 mon.c (mon.2) 31 : audit 1 from='osd.2 [v2:172.21.15.133:6818/3595178639,v1:172.21.15.133:6819/3595178639]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T05:01:01.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:00 smithi133 bash[102858]: audit 2023-10-09T05:01:00.104538+0000 mon.a (mon.0) 518 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T05:01:01.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:00 smithi133 bash[105589]: cluster 2023-10-09T05:00:59.535746+0000 mgr.y (mgr.25277) 353 : cluster 0 pgmap v288: 89 pgs: 60 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 15815/50925 objects degraded (31.055%); 0 B/s, 27 objects/s recovering 2023-10-09T05:01:01.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:00 smithi133 bash[105589]: audit 2023-10-09T05:01:00.103910+0000 mon.c (mon.2) 31 : audit 1 from='osd.2 [v2:172.21.15.133:6818/3595178639,v1:172.21.15.133:6819/3595178639]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T05:01:01.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:00 smithi133 bash[105589]: audit 2023-10-09T05:01:00.104538+0000 mon.a (mon.0) 518 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T05:01:01.051 INFO:journalctl@ceph.osd.2.smithi133.stdout:Oct 09 05:01:00 smithi133 bash[128609]: debug 2023-10-09T05:01:00.708+0000 7f8688837700 -1 osd.2 102 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:01:02.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:01 smithi133 bash[105589]: audit 2023-10-09T05:01:00.608736+0000 mon.a (mon.0) 519 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T05:01:02.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:01 smithi133 bash[105589]: cluster 2023-10-09T05:01:00.608937+0000 mon.a (mon.0) 520 : cluster 0 osdmap e105: 8 total, 7 up, 8 in 2023-10-09T05:01:02.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:01 smithi133 bash[105589]: cluster 2023-10-09T05:01:00.611104+0000 osd.5 (osd.5) 35 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T05:01:02.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:01 smithi133 bash[105589]: cluster 2023-10-09T05:01:00.616390+0000 osd.6 (osd.6) 20 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T05:01:02.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:01 smithi133 bash[105589]: cluster 2023-10-09T05:01:00.616595+0000 osd.3 (osd.3) 29 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T05:01:02.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:01 smithi133 bash[105589]: cluster 2023-10-09T05:01:00.617521+0000 osd.4 (osd.4) 26 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T05:01:02.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:01 smithi133 bash[105589]: cluster 2023-10-09T05:01:00.619757+0000 osd.5 (osd.5) 36 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T05:01:02.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:01 smithi133 bash[105589]: cluster 2023-10-09T05:01:00.621494+0000 osd.7 (osd.7) 20 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T05:01:02.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:01 smithi133 bash[105589]: cluster 2023-10-09T05:01:00.622526+0000 osd.5 (osd.5) 37 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T05:01:02.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:01 smithi133 bash[105589]: audit 2023-10-09T05:01:00.622689+0000 mon.c (mon.2) 32 : audit 1 from='osd.2 [v2:172.21.15.133:6818/3595178639,v1:172.21.15.133:6819/3595178639]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T05:01:02.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:01 smithi133 bash[105589]: audit 2023-10-09T05:01:00.638676+0000 mon.a (mon.0) 521 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T05:01:02.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:01 smithi133 bash[105589]: audit 2023-10-09T05:01:01.183940+0000 mon.a (mon.0) 522 : audit 0 from='client.? 172.21.15.133:0/172701507' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:02.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:01 smithi133 bash[102858]: audit 2023-10-09T05:01:00.608736+0000 mon.a (mon.0) 519 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T05:01:02.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:01 smithi133 bash[102858]: cluster 2023-10-09T05:01:00.608937+0000 mon.a (mon.0) 520 : cluster 0 osdmap e105: 8 total, 7 up, 8 in 2023-10-09T05:01:02.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:01 smithi133 bash[102858]: cluster 2023-10-09T05:01:00.611104+0000 osd.5 (osd.5) 35 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T05:01:02.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:01 smithi133 bash[102858]: cluster 2023-10-09T05:01:00.616390+0000 osd.6 (osd.6) 20 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T05:01:02.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:01 smithi133 bash[102858]: cluster 2023-10-09T05:01:00.616595+0000 osd.3 (osd.3) 29 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T05:01:02.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:01 smithi133 bash[102858]: cluster 2023-10-09T05:01:00.617521+0000 osd.4 (osd.4) 26 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T05:01:02.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:01 smithi133 bash[102858]: cluster 2023-10-09T05:01:00.619757+0000 osd.5 (osd.5) 36 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T05:01:02.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:01 smithi133 bash[102858]: cluster 2023-10-09T05:01:00.621494+0000 osd.7 (osd.7) 20 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T05:01:02.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:01 smithi133 bash[102858]: cluster 2023-10-09T05:01:00.622526+0000 osd.5 (osd.5) 37 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T05:01:02.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:01 smithi133 bash[102858]: audit 2023-10-09T05:01:00.622689+0000 mon.c (mon.2) 32 : audit 1 from='osd.2 [v2:172.21.15.133:6818/3595178639,v1:172.21.15.133:6819/3595178639]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T05:01:02.053 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:01 smithi133 bash[102858]: audit 2023-10-09T05:01:00.638676+0000 mon.a (mon.0) 521 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T05:01:02.054 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:01 smithi133 bash[102858]: audit 2023-10-09T05:01:01.183940+0000 mon.a (mon.0) 522 : audit 0 from='client.? 172.21.15.133:0/172701507' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:02.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:01 smithi194 bash[92666]: audit 2023-10-09T05:01:00.608736+0000 mon.a (mon.0) 519 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T05:01:02.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:01 smithi194 bash[92666]: cluster 2023-10-09T05:01:00.608937+0000 mon.a (mon.0) 520 : cluster 0 osdmap e105: 8 total, 7 up, 8 in 2023-10-09T05:01:02.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:01 smithi194 bash[92666]: cluster 2023-10-09T05:01:00.611104+0000 osd.5 (osd.5) 35 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T05:01:02.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:01 smithi194 bash[92666]: cluster 2023-10-09T05:01:00.616390+0000 osd.6 (osd.6) 20 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T05:01:02.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:01 smithi194 bash[92666]: cluster 2023-10-09T05:01:00.616595+0000 osd.3 (osd.3) 29 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T05:01:02.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:01 smithi194 bash[92666]: cluster 2023-10-09T05:01:00.617521+0000 osd.4 (osd.4) 26 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T05:01:02.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:01 smithi194 bash[92666]: cluster 2023-10-09T05:01:00.619757+0000 osd.5 (osd.5) 36 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T05:01:02.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:01 smithi194 bash[92666]: cluster 2023-10-09T05:01:00.621494+0000 osd.7 (osd.7) 20 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T05:01:02.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:01 smithi194 bash[92666]: cluster 2023-10-09T05:01:00.622526+0000 osd.5 (osd.5) 37 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T05:01:02.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:01 smithi194 bash[92666]: audit 2023-10-09T05:01:00.622689+0000 mon.c (mon.2) 32 : audit 1 from='osd.2 [v2:172.21.15.133:6818/3595178639,v1:172.21.15.133:6819/3595178639]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T05:01:02.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:01 smithi194 bash[92666]: audit 2023-10-09T05:01:00.638676+0000 mon.a (mon.0) 521 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T05:01:02.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:01 smithi194 bash[92666]: audit 2023-10-09T05:01:01.183940+0000 mon.a (mon.0) 522 : audit 0 from='client.? 172.21.15.133:0/172701507' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:02.923 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:02 smithi133 bash[105589]: cluster 2023-10-09T05:01:01.536358+0000 mgr.y (mgr.25277) 354 : cluster 0 pgmap v290: 89 pgs: 60 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped; 858 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 14472/41217 objects degraded (35.112%); 0 B/s, 32 objects/s recovering 2023-10-09T05:01:02.923 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:02 smithi133 bash[105589]: cluster 2023-10-09T05:01:01.624343+0000 mon.a (mon.0) 523 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:01:02.923 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:02 smithi133 bash[105589]: cluster 2023-10-09T05:01:01.639197+0000 osd.5 (osd.5) 38 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T05:01:02.923 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:02 smithi133 bash[105589]: cluster 2023-10-09T05:01:01.640241+0000 osd.7 (osd.7) 21 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T05:01:02.924 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:02 smithi133 bash[105589]: cluster 2023-10-09T05:01:01.643518+0000 osd.4 (osd.4) 27 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T05:01:02.924 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:02 smithi133 bash[105589]: cluster 2023-10-09T05:01:01.644589+0000 osd.5 (osd.5) 39 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T05:01:02.924 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:02 smithi133 bash[105589]: cluster 2023-10-09T05:01:01.644659+0000 mon.a (mon.0) 524 : cluster 1 osd.2 [v2:172.21.15.133:6818/3595178639,v1:172.21.15.133:6819/3595178639] boot 2023-10-09T05:01:02.924 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:02 smithi133 bash[105589]: cluster 2023-10-09T05:01:01.644951+0000 mon.a (mon.0) 525 : cluster 0 osdmap e106: 8 total, 8 up, 8 in 2023-10-09T05:01:02.924 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:02 smithi133 bash[105589]: audit 2023-10-09T05:01:01.646619+0000 mon.a (mon.0) 526 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:01:02.924 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:02 smithi133 bash[105589]: cluster 2023-10-09T05:01:01.647099+0000 osd.6 (osd.6) 21 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T05:01:02.925 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:02 smithi133 bash[102858]: cluster 2023-10-09T05:01:01.536358+0000 mgr.y (mgr.25277) 354 : cluster 0 pgmap v290: 89 pgs: 60 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped; 858 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 14472/41217 objects degraded (35.112%); 0 B/s, 32 objects/s recovering 2023-10-09T05:01:02.925 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:02 smithi133 bash[102858]: cluster 2023-10-09T05:01:01.624343+0000 mon.a (mon.0) 523 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:01:02.925 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:02 smithi133 bash[102858]: cluster 2023-10-09T05:01:01.639197+0000 osd.5 (osd.5) 38 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T05:01:02.925 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:02 smithi133 bash[102858]: cluster 2023-10-09T05:01:01.640241+0000 osd.7 (osd.7) 21 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T05:01:02.925 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:02 smithi133 bash[102858]: cluster 2023-10-09T05:01:01.643518+0000 osd.4 (osd.4) 27 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T05:01:02.925 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:02 smithi133 bash[102858]: cluster 2023-10-09T05:01:01.644589+0000 osd.5 (osd.5) 39 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T05:01:02.925 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:02 smithi133 bash[102858]: cluster 2023-10-09T05:01:01.644659+0000 mon.a (mon.0) 524 : cluster 1 osd.2 [v2:172.21.15.133:6818/3595178639,v1:172.21.15.133:6819/3595178639] boot 2023-10-09T05:01:02.926 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:02 smithi133 bash[102858]: cluster 2023-10-09T05:01:01.644951+0000 mon.a (mon.0) 525 : cluster 0 osdmap e106: 8 total, 8 up, 8 in 2023-10-09T05:01:02.926 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:02 smithi133 bash[102858]: audit 2023-10-09T05:01:01.646619+0000 mon.a (mon.0) 526 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:01:02.926 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:02 smithi133 bash[102858]: cluster 2023-10-09T05:01:01.647099+0000 osd.6 (osd.6) 21 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T05:01:03.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:02 smithi194 bash[92666]: cluster 2023-10-09T05:01:01.536358+0000 mgr.y (mgr.25277) 354 : cluster 0 pgmap v290: 89 pgs: 60 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped; 858 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 14472/41217 objects degraded (35.112%); 0 B/s, 32 objects/s recovering 2023-10-09T05:01:03.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:02 smithi194 bash[92666]: cluster 2023-10-09T05:01:01.624343+0000 mon.a (mon.0) 523 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:01:03.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:02 smithi194 bash[92666]: cluster 2023-10-09T05:01:01.639197+0000 osd.5 (osd.5) 38 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T05:01:03.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:02 smithi194 bash[92666]: cluster 2023-10-09T05:01:01.640241+0000 osd.7 (osd.7) 21 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T05:01:03.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:02 smithi194 bash[92666]: cluster 2023-10-09T05:01:01.643518+0000 osd.4 (osd.4) 27 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T05:01:03.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:02 smithi194 bash[92666]: cluster 2023-10-09T05:01:01.644589+0000 osd.5 (osd.5) 39 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T05:01:03.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:02 smithi194 bash[92666]: cluster 2023-10-09T05:01:01.644659+0000 mon.a (mon.0) 524 : cluster 1 osd.2 [v2:172.21.15.133:6818/3595178639,v1:172.21.15.133:6819/3595178639] boot 2023-10-09T05:01:03.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:02 smithi194 bash[92666]: cluster 2023-10-09T05:01:01.644951+0000 mon.a (mon.0) 525 : cluster 0 osdmap e106: 8 total, 8 up, 8 in 2023-10-09T05:01:03.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:02 smithi194 bash[92666]: audit 2023-10-09T05:01:01.646619+0000 mon.a (mon.0) 526 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:01:03.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:02 smithi194 bash[92666]: cluster 2023-10-09T05:01:01.647099+0000 osd.6 (osd.6) 21 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T05:01:04.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[102858]: cluster 2023-10-09T05:01:02.639782+0000 osd.6 (osd.6) 22 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[102858]: cluster 2023-10-09T05:01:02.639990+0000 osd.5 (osd.5) 40 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[102858]: cluster 2023-10-09T05:01:02.640943+0000 mon.a (mon.0) 527 : cluster 0 osdmap e107: 8 total, 8 up, 8 in 2023-10-09T05:01:04.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[102858]: cluster 2023-10-09T05:01:02.641813+0000 osd.3 (osd.3) 30 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[102858]: cluster 2023-10-09T05:01:02.641990+0000 osd.7 (osd.7) 22 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[102858]: cluster 2023-10-09T05:01:02.642190+0000 osd.4 (osd.4) 28 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[102858]: cluster 2023-10-09T05:01:02.644730+0000 osd.6 (osd.6) 23 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[102858]: cluster 2023-10-09T05:01:02.645310+0000 osd.4 (osd.4) 29 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[102858]: cluster 2023-10-09T05:01:02.645639+0000 osd.3 (osd.3) 31 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[102858]: cluster 2023-10-09T05:01:02.646236+0000 osd.7 (osd.7) 23 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[102858]: cluster 2023-10-09T05:01:02.646598+0000 osd.4 (osd.4) 30 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[102858]: cluster 2023-10-09T05:01:02.647090+0000 osd.5 (osd.5) 41 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[102858]: cluster 2023-10-09T05:01:02.647282+0000 osd.3 (osd.3) 32 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[102858]: cluster 2023-10-09T05:01:02.647791+0000 osd.7 (osd.7) 24 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[102858]: cluster 2023-10-09T05:01:02.648802+0000 osd.6 (osd.6) 24 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[102858]: cluster 2023-10-09T05:01:02.649083+0000 osd.5 (osd.5) 42 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[102858]: cluster 2023-10-09T05:01:02.649404+0000 osd.3 (osd.3) 33 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[102858]: cluster 2023-10-09T05:01:02.650274+0000 osd.5 (osd.5) 43 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[102858]: cluster 2023-10-09T05:01:02.652466+0000 osd.5 (osd.5) 44 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[102858]: cluster 2023-10-09T05:01:03.066367+0000 mon.a (mon.0) 528 : cluster 3 Health check update: Degraded data redundancy: 14472/41217 objects degraded (35.112%), 12 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:04.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[102858]: audit 2023-10-09T05:01:03.566467+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.133:0/4126778810' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:04.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[105589]: cluster 2023-10-09T05:01:02.639782+0000 osd.6 (osd.6) 22 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[105589]: cluster 2023-10-09T05:01:02.639990+0000 osd.5 (osd.5) 40 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[105589]: cluster 2023-10-09T05:01:02.640943+0000 mon.a (mon.0) 527 : cluster 0 osdmap e107: 8 total, 8 up, 8 in 2023-10-09T05:01:04.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[105589]: cluster 2023-10-09T05:01:02.641813+0000 osd.3 (osd.3) 30 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[105589]: cluster 2023-10-09T05:01:02.641990+0000 osd.7 (osd.7) 22 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[105589]: cluster 2023-10-09T05:01:02.642190+0000 osd.4 (osd.4) 28 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[105589]: cluster 2023-10-09T05:01:02.644730+0000 osd.6 (osd.6) 23 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[105589]: cluster 2023-10-09T05:01:02.645310+0000 osd.4 (osd.4) 29 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[105589]: cluster 2023-10-09T05:01:02.645639+0000 osd.3 (osd.3) 31 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[105589]: cluster 2023-10-09T05:01:02.646236+0000 osd.7 (osd.7) 23 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[105589]: cluster 2023-10-09T05:01:02.646598+0000 osd.4 (osd.4) 30 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[105589]: cluster 2023-10-09T05:01:02.647090+0000 osd.5 (osd.5) 41 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[105589]: cluster 2023-10-09T05:01:02.647282+0000 osd.3 (osd.3) 32 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[105589]: cluster 2023-10-09T05:01:02.647791+0000 osd.7 (osd.7) 24 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[105589]: cluster 2023-10-09T05:01:02.648802+0000 osd.6 (osd.6) 24 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[105589]: cluster 2023-10-09T05:01:02.649083+0000 osd.5 (osd.5) 42 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[105589]: cluster 2023-10-09T05:01:02.649404+0000 osd.3 (osd.3) 33 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.054 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[105589]: cluster 2023-10-09T05:01:02.650274+0000 osd.5 (osd.5) 43 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[105589]: cluster 2023-10-09T05:01:02.652466+0000 osd.5 (osd.5) 44 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[105589]: cluster 2023-10-09T05:01:03.066367+0000 mon.a (mon.0) 528 : cluster 3 Health check update: Degraded data redundancy: 14472/41217 objects degraded (35.112%), 12 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:04.055 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:03 smithi133 bash[105589]: audit 2023-10-09T05:01:03.566467+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.133:0/4126778810' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:04.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:03 smithi194 bash[92666]: cluster 2023-10-09T05:01:02.639782+0000 osd.6 (osd.6) 22 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:03 smithi194 bash[92666]: cluster 2023-10-09T05:01:02.639990+0000 osd.5 (osd.5) 40 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:03 smithi194 bash[92666]: cluster 2023-10-09T05:01:02.640943+0000 mon.a (mon.0) 527 : cluster 0 osdmap e107: 8 total, 8 up, 8 in 2023-10-09T05:01:04.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:03 smithi194 bash[92666]: cluster 2023-10-09T05:01:02.641813+0000 osd.3 (osd.3) 30 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:03 smithi194 bash[92666]: cluster 2023-10-09T05:01:02.641990+0000 osd.7 (osd.7) 22 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:03 smithi194 bash[92666]: cluster 2023-10-09T05:01:02.642190+0000 osd.4 (osd.4) 28 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:03 smithi194 bash[92666]: cluster 2023-10-09T05:01:02.644730+0000 osd.6 (osd.6) 23 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:03 smithi194 bash[92666]: cluster 2023-10-09T05:01:02.645310+0000 osd.4 (osd.4) 29 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:03 smithi194 bash[92666]: cluster 2023-10-09T05:01:02.645639+0000 osd.3 (osd.3) 31 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:03 smithi194 bash[92666]: cluster 2023-10-09T05:01:02.646236+0000 osd.7 (osd.7) 23 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:03 smithi194 bash[92666]: cluster 2023-10-09T05:01:02.646598+0000 osd.4 (osd.4) 30 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:03 smithi194 bash[92666]: cluster 2023-10-09T05:01:02.647090+0000 osd.5 (osd.5) 41 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:03 smithi194 bash[92666]: cluster 2023-10-09T05:01:02.647282+0000 osd.3 (osd.3) 32 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:03 smithi194 bash[92666]: cluster 2023-10-09T05:01:02.647791+0000 osd.7 (osd.7) 24 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:03 smithi194 bash[92666]: cluster 2023-10-09T05:01:02.648802+0000 osd.6 (osd.6) 24 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:03 smithi194 bash[92666]: cluster 2023-10-09T05:01:02.649083+0000 osd.5 (osd.5) 42 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:03 smithi194 bash[92666]: cluster 2023-10-09T05:01:02.649404+0000 osd.3 (osd.3) 33 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:03 smithi194 bash[92666]: cluster 2023-10-09T05:01:02.650274+0000 osd.5 (osd.5) 43 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:03 smithi194 bash[92666]: cluster 2023-10-09T05:01:02.652466+0000 osd.5 (osd.5) 44 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T05:01:04.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:03 smithi194 bash[92666]: cluster 2023-10-09T05:01:03.066367+0000 mon.a (mon.0) 528 : cluster 3 Health check update: Degraded data redundancy: 14472/41217 objects degraded (35.112%), 12 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:04.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:03 smithi194 bash[92666]: audit 2023-10-09T05:01:03.566467+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.133:0/4126778810' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:04.550 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:01:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:01:04] "GET /metrics HTTP/1.1" 200 36358 "" "Prometheus/2.43.0" 2023-10-09T05:01:05.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:04 smithi133 bash[102858]: cluster 2023-10-09T05:01:03.536796+0000 mgr.y (mgr.25277) 355 : cluster 0 pgmap v293: 89 pgs: 60 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped; 858 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 14472/41217 objects degraded (35.112%); 0 B/s, 28 objects/s recovering 2023-10-09T05:01:05.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:04 smithi133 bash[102858]: cluster 2023-10-09T05:01:03.652927+0000 osd.6 (osd.6) 25 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T05:01:05.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:04 smithi133 bash[102858]: cluster 2023-10-09T05:01:03.654911+0000 osd.3 (osd.3) 34 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T05:01:05.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:04 smithi133 bash[102858]: cluster 2023-10-09T05:01:03.655551+0000 osd.5 (osd.5) 45 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T05:01:05.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:04 smithi133 bash[102858]: cluster 2023-10-09T05:01:03.655883+0000 mon.a (mon.0) 529 : cluster 0 osdmap e108: 8 total, 8 up, 8 in 2023-10-09T05:01:05.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:04 smithi133 bash[102858]: cluster 2023-10-09T05:01:03.655900+0000 osd.4 (osd.4) 31 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T05:01:05.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:04 smithi133 bash[102858]: cluster 2023-10-09T05:01:03.656015+0000 osd.7 (osd.7) 25 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T05:01:05.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:04 smithi133 bash[102858]: cluster 2023-10-09T05:01:03.657151+0000 osd.5 (osd.5) 46 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T05:01:05.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:04 smithi133 bash[102858]: cluster 2023-10-09T05:01:03.657155+0000 osd.3 (osd.3) 35 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T05:01:05.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:04 smithi133 bash[102858]: cluster 2023-10-09T05:01:03.658946+0000 osd.7 (osd.7) 26 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T05:01:05.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:04 smithi133 bash[102858]: cluster 2023-10-09T05:01:03.658961+0000 osd.3 (osd.3) 36 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T05:01:05.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:04 smithi133 bash[102858]: cluster 2023-10-09T05:01:03.659617+0000 osd.4 (osd.4) 32 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T05:01:05.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:04 smithi133 bash[105589]: cluster 2023-10-09T05:01:03.536796+0000 mgr.y (mgr.25277) 355 : cluster 0 pgmap v293: 89 pgs: 60 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped; 858 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 14472/41217 objects degraded (35.112%); 0 B/s, 28 objects/s recovering 2023-10-09T05:01:05.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:04 smithi133 bash[105589]: cluster 2023-10-09T05:01:03.652927+0000 osd.6 (osd.6) 25 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T05:01:05.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:04 smithi133 bash[105589]: cluster 2023-10-09T05:01:03.654911+0000 osd.3 (osd.3) 34 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T05:01:05.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:04 smithi133 bash[105589]: cluster 2023-10-09T05:01:03.655551+0000 osd.5 (osd.5) 45 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T05:01:05.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:04 smithi133 bash[105589]: cluster 2023-10-09T05:01:03.655883+0000 mon.a (mon.0) 529 : cluster 0 osdmap e108: 8 total, 8 up, 8 in 2023-10-09T05:01:05.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:04 smithi133 bash[105589]: cluster 2023-10-09T05:01:03.655900+0000 osd.4 (osd.4) 31 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T05:01:05.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:04 smithi133 bash[105589]: cluster 2023-10-09T05:01:03.656015+0000 osd.7 (osd.7) 25 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T05:01:05.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:04 smithi133 bash[105589]: cluster 2023-10-09T05:01:03.657151+0000 osd.5 (osd.5) 46 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T05:01:05.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:04 smithi133 bash[105589]: cluster 2023-10-09T05:01:03.657155+0000 osd.3 (osd.3) 35 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T05:01:05.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:04 smithi133 bash[105589]: cluster 2023-10-09T05:01:03.658946+0000 osd.7 (osd.7) 26 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T05:01:05.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:04 smithi133 bash[105589]: cluster 2023-10-09T05:01:03.658961+0000 osd.3 (osd.3) 36 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T05:01:05.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:04 smithi133 bash[105589]: cluster 2023-10-09T05:01:03.659617+0000 osd.4 (osd.4) 32 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T05:01:05.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:04 smithi194 bash[92666]: cluster 2023-10-09T05:01:03.536796+0000 mgr.y (mgr.25277) 355 : cluster 0 pgmap v293: 89 pgs: 60 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped; 858 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 14472/41217 objects degraded (35.112%); 0 B/s, 28 objects/s recovering 2023-10-09T05:01:05.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:04 smithi194 bash[92666]: cluster 2023-10-09T05:01:03.652927+0000 osd.6 (osd.6) 25 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T05:01:05.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:04 smithi194 bash[92666]: cluster 2023-10-09T05:01:03.654911+0000 osd.3 (osd.3) 34 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T05:01:05.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:04 smithi194 bash[92666]: cluster 2023-10-09T05:01:03.655551+0000 osd.5 (osd.5) 45 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T05:01:05.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:04 smithi194 bash[92666]: cluster 2023-10-09T05:01:03.655883+0000 mon.a (mon.0) 529 : cluster 0 osdmap e108: 8 total, 8 up, 8 in 2023-10-09T05:01:05.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:04 smithi194 bash[92666]: cluster 2023-10-09T05:01:03.655900+0000 osd.4 (osd.4) 31 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T05:01:05.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:04 smithi194 bash[92666]: cluster 2023-10-09T05:01:03.656015+0000 osd.7 (osd.7) 25 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T05:01:05.140 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:04 smithi194 bash[92666]: cluster 2023-10-09T05:01:03.657151+0000 osd.5 (osd.5) 46 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T05:01:05.140 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:04 smithi194 bash[92666]: cluster 2023-10-09T05:01:03.657155+0000 osd.3 (osd.3) 35 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T05:01:05.140 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:04 smithi194 bash[92666]: cluster 2023-10-09T05:01:03.658946+0000 osd.7 (osd.7) 26 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T05:01:05.140 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:04 smithi194 bash[92666]: cluster 2023-10-09T05:01:03.658961+0000 osd.3 (osd.3) 36 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T05:01:05.140 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:04 smithi194 bash[92666]: cluster 2023-10-09T05:01:03.659617+0000 osd.4 (osd.4) 32 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T05:01:07.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:06 smithi133 bash[102858]: cluster 2023-10-09T05:01:05.537634+0000 mgr.y (mgr.25277) 356 : cluster 0 pgmap v295: 89 pgs: 71 active+clean, 8 active+undersized, 3 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped; 756 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 451 op/s; 13708/36333 objects degraded (37.729%); 0 B/s, 141 objects/s recovering 2023-10-09T05:01:07.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:06 smithi133 bash[102858]: audit 2023-10-09T05:01:06.153630+0000 mon.b (mon.1) 9 : audit 0 from='client.? 172.21.15.133:0/1022173325' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:07.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:06 smithi133 bash[105589]: cluster 2023-10-09T05:01:05.537634+0000 mgr.y (mgr.25277) 356 : cluster 0 pgmap v295: 89 pgs: 71 active+clean, 8 active+undersized, 3 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped; 756 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 451 op/s; 13708/36333 objects degraded (37.729%); 0 B/s, 141 objects/s recovering 2023-10-09T05:01:07.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:06 smithi133 bash[105589]: audit 2023-10-09T05:01:06.153630+0000 mon.b (mon.1) 9 : audit 0 from='client.? 172.21.15.133:0/1022173325' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:07.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:06 smithi194 bash[92666]: cluster 2023-10-09T05:01:05.537634+0000 mgr.y (mgr.25277) 356 : cluster 0 pgmap v295: 89 pgs: 71 active+clean, 8 active+undersized, 3 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped; 756 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 451 op/s; 13708/36333 objects degraded (37.729%); 0 B/s, 141 objects/s recovering 2023-10-09T05:01:07.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:06 smithi194 bash[92666]: audit 2023-10-09T05:01:06.153630+0000 mon.b (mon.1) 9 : audit 0 from='client.? 172.21.15.133:0/1022173325' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:08.670 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:08 smithi133 bash[105589]: cluster 2023-10-09T05:01:07.539072+0000 mgr.y (mgr.25277) 357 : cluster 0 pgmap v296: 89 pgs: 1 active+recovering+undersized+remapped, 79 active+clean, 9 active+recovery_wait+undersized+degraded+remapped; 736 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 323 op/s; 13058/35391 objects degraded (36.896%); 0 B/s, 129 objects/s recovering 2023-10-09T05:01:08.670 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:08 smithi133 bash[105589]: cluster 2023-10-09T05:01:08.067587+0000 mon.a (mon.0) 530 : cluster 3 Health check update: Degraded data redundancy: 13058/35391 objects degraded (36.896%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:08.670 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:08 smithi133 bash[102858]: cluster 2023-10-09T05:01:07.539072+0000 mgr.y (mgr.25277) 357 : cluster 0 pgmap v296: 89 pgs: 1 active+recovering+undersized+remapped, 79 active+clean, 9 active+recovery_wait+undersized+degraded+remapped; 736 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 323 op/s; 13058/35391 objects degraded (36.896%); 0 B/s, 129 objects/s recovering 2023-10-09T05:01:08.671 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:08 smithi133 bash[102858]: cluster 2023-10-09T05:01:08.067587+0000 mon.a (mon.0) 530 : cluster 3 Health check update: Degraded data redundancy: 13058/35391 objects degraded (36.896%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:09.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:08 smithi194 bash[92666]: cluster 2023-10-09T05:01:07.539072+0000 mgr.y (mgr.25277) 357 : cluster 0 pgmap v296: 89 pgs: 1 active+recovering+undersized+remapped, 79 active+clean, 9 active+recovery_wait+undersized+degraded+remapped; 736 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 323 op/s; 13058/35391 objects degraded (36.896%); 0 B/s, 129 objects/s recovering 2023-10-09T05:01:09.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:08 smithi194 bash[92666]: cluster 2023-10-09T05:01:08.067587+0000 mon.a (mon.0) 530 : cluster 3 Health check update: Degraded data redundancy: 13058/35391 objects degraded (36.896%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:10.033 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:09 smithi133 bash[102858]: audit 2023-10-09T05:01:08.736228+0000 mon.a (mon.0) 531 : audit 0 from='client.? 172.21.15.133:0/3838399605' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:10.033 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:01:10 smithi133 bash[91128]: debug 2023-10-09T05:01:10.028+0000 7ff45d5ed700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2023-10-09T05:01:10.034 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:09 smithi133 bash[105589]: audit 2023-10-09T05:01:08.736228+0000 mon.a (mon.0) 531 : audit 0 from='client.? 172.21.15.133:0/3838399605' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:10.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:09 smithi194 bash[92666]: audit 2023-10-09T05:01:08.736228+0000 mon.a (mon.0) 531 : audit 0 from='client.? 172.21.15.133:0/3838399605' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:11.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:10 smithi133 bash[102858]: cluster 2023-10-09T05:01:09.539648+0000 mgr.y (mgr.25277) 358 : cluster 0 pgmap v297: 89 pgs: 1 active+recovering+undersized+remapped, 79 active+clean, 9 active+recovery_wait+undersized+degraded+remapped; 736 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 245 op/s; 13058/35391 objects degraded (36.896%); 0 B/s, 98 objects/s recovering 2023-10-09T05:01:11.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:10 smithi133 bash[102858]: audit 2023-10-09T05:01:09.901222+0000 mon.a (mon.0) 532 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:01:11.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:10 smithi133 bash[102858]: audit 2023-10-09T05:01:09.912649+0000 mon.a (mon.0) 533 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:01:11.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:10 smithi133 bash[102858]: audit 2023-10-09T05:01:09.914291+0000 mon.a (mon.0) 534 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:01:11.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:10 smithi133 bash[102858]: audit 2023-10-09T05:01:09.917925+0000 mon.a (mon.0) 535 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:01:11.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:10 smithi133 bash[102858]: audit 2023-10-09T05:01:09.926417+0000 mon.a (mon.0) 536 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:01:11.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:10 smithi133 bash[102858]: audit 2023-10-09T05:01:10.013381+0000 mon.a (mon.0) 537 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:01:11.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:10 smithi133 bash[102858]: audit 2023-10-09T05:01:10.019820+0000 mon.a (mon.0) 538 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:01:11.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:10 smithi133 bash[102858]: audit 2023-10-09T05:01:10.023669+0000 mon.a (mon.0) 539 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:01:11.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:10 smithi133 bash[102858]: audit 2023-10-09T05:01:10.026527+0000 mon.a (mon.0) 540 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:01:11.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:10 smithi133 bash[102858]: audit 2023-10-09T05:01:10.030099+0000 mon.a (mon.0) 541 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:01:11.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:10 smithi133 bash[102858]: audit 2023-10-09T05:01:10.030382+0000 mgr.y (mgr.25277) 359 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:01:11.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:10 smithi133 bash[102858]: cephadm 2023-10-09T05:01:10.033220+0000 mgr.y (mgr.25277) 360 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2023-10-09T05:01:11.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:10 smithi133 bash[105589]: cluster 2023-10-09T05:01:09.539648+0000 mgr.y (mgr.25277) 358 : cluster 0 pgmap v297: 89 pgs: 1 active+recovering+undersized+remapped, 79 active+clean, 9 active+recovery_wait+undersized+degraded+remapped; 736 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 245 op/s; 13058/35391 objects degraded (36.896%); 0 B/s, 98 objects/s recovering 2023-10-09T05:01:11.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:10 smithi133 bash[105589]: audit 2023-10-09T05:01:09.901222+0000 mon.a (mon.0) 532 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:01:11.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:10 smithi133 bash[105589]: audit 2023-10-09T05:01:09.912649+0000 mon.a (mon.0) 533 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:01:11.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:10 smithi133 bash[105589]: audit 2023-10-09T05:01:09.914291+0000 mon.a (mon.0) 534 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:01:11.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:10 smithi133 bash[105589]: audit 2023-10-09T05:01:09.917925+0000 mon.a (mon.0) 535 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:01:11.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:10 smithi133 bash[105589]: audit 2023-10-09T05:01:09.926417+0000 mon.a (mon.0) 536 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:01:11.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:10 smithi133 bash[105589]: audit 2023-10-09T05:01:10.013381+0000 mon.a (mon.0) 537 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:01:11.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:10 smithi133 bash[105589]: audit 2023-10-09T05:01:10.019820+0000 mon.a (mon.0) 538 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:01:11.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:10 smithi133 bash[105589]: audit 2023-10-09T05:01:10.023669+0000 mon.a (mon.0) 539 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:01:11.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:10 smithi133 bash[105589]: audit 2023-10-09T05:01:10.026527+0000 mon.a (mon.0) 540 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:01:11.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:10 smithi133 bash[105589]: audit 2023-10-09T05:01:10.030099+0000 mon.a (mon.0) 541 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:01:11.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:10 smithi133 bash[105589]: audit 2023-10-09T05:01:10.030382+0000 mgr.y (mgr.25277) 359 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:01:11.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:10 smithi133 bash[105589]: cephadm 2023-10-09T05:01:10.033220+0000 mgr.y (mgr.25277) 360 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2023-10-09T05:01:11.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:10 smithi194 bash[92666]: cluster 2023-10-09T05:01:09.539648+0000 mgr.y (mgr.25277) 358 : cluster 0 pgmap v297: 89 pgs: 1 active+recovering+undersized+remapped, 79 active+clean, 9 active+recovery_wait+undersized+degraded+remapped; 736 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 245 op/s; 13058/35391 objects degraded (36.896%); 0 B/s, 98 objects/s recovering 2023-10-09T05:01:11.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:10 smithi194 bash[92666]: audit 2023-10-09T05:01:09.901222+0000 mon.a (mon.0) 532 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:01:11.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:10 smithi194 bash[92666]: audit 2023-10-09T05:01:09.912649+0000 mon.a (mon.0) 533 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:01:11.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:10 smithi194 bash[92666]: audit 2023-10-09T05:01:09.914291+0000 mon.a (mon.0) 534 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:01:11.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:10 smithi194 bash[92666]: audit 2023-10-09T05:01:09.917925+0000 mon.a (mon.0) 535 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:01:11.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:10 smithi194 bash[92666]: audit 2023-10-09T05:01:09.926417+0000 mon.a (mon.0) 536 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:01:11.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:10 smithi194 bash[92666]: audit 2023-10-09T05:01:10.013381+0000 mon.a (mon.0) 537 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:01:11.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:10 smithi194 bash[92666]: audit 2023-10-09T05:01:10.019820+0000 mon.a (mon.0) 538 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:01:11.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:10 smithi194 bash[92666]: audit 2023-10-09T05:01:10.023669+0000 mon.a (mon.0) 539 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:01:11.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:10 smithi194 bash[92666]: audit 2023-10-09T05:01:10.026527+0000 mon.a (mon.0) 540 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:01:11.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:10 smithi194 bash[92666]: audit 2023-10-09T05:01:10.030099+0000 mon.a (mon.0) 541 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:01:11.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:10 smithi194 bash[92666]: audit 2023-10-09T05:01:10.030382+0000 mgr.y (mgr.25277) 359 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:01:11.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:10 smithi194 bash[92666]: cephadm 2023-10-09T05:01:10.033220+0000 mgr.y (mgr.25277) 360 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2023-10-09T05:01:12.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:11 smithi133 bash[102858]: audit 2023-10-09T05:01:11.348919+0000 mon.c (mon.2) 34 : audit 0 from='client.? 172.21.15.133:0/3607211492' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:12.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:11 smithi133 bash[105589]: audit 2023-10-09T05:01:11.348919+0000 mon.c (mon.2) 34 : audit 0 from='client.? 172.21.15.133:0/3607211492' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:12.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:11 smithi194 bash[92666]: audit 2023-10-09T05:01:11.348919+0000 mon.c (mon.2) 34 : audit 0 from='client.? 172.21.15.133:0/3607211492' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:13.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:12 smithi133 bash[105589]: cluster 2023-10-09T05:01:11.540513+0000 mgr.y (mgr.25277) 361 : cluster 0 pgmap v298: 89 pgs: 1 active+recovering+undersized+remapped, 79 active+clean, 9 active+recovery_wait+undersized+degraded+remapped; 546 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 13058/26253 objects degraded (49.739%); 0 B/s, 107 objects/s recovering 2023-10-09T05:01:13.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:12 smithi133 bash[102858]: cluster 2023-10-09T05:01:11.540513+0000 mgr.y (mgr.25277) 361 : cluster 0 pgmap v298: 89 pgs: 1 active+recovering+undersized+remapped, 79 active+clean, 9 active+recovery_wait+undersized+degraded+remapped; 546 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 13058/26253 objects degraded (49.739%); 0 B/s, 107 objects/s recovering 2023-10-09T05:01:13.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:12 smithi194 bash[92666]: cluster 2023-10-09T05:01:11.540513+0000 mgr.y (mgr.25277) 361 : cluster 0 pgmap v298: 89 pgs: 1 active+recovering+undersized+remapped, 79 active+clean, 9 active+recovery_wait+undersized+degraded+remapped; 546 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 13058/26253 objects degraded (49.739%); 0 B/s, 107 objects/s recovering 2023-10-09T05:01:14.282 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:13 smithi133 bash[102858]: cluster 2023-10-09T05:01:13.068649+0000 mon.a (mon.0) 542 : cluster 3 Health check update: Degraded data redundancy: 13058/26253 objects degraded (49.739%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:14.282 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:13 smithi133 bash[102858]: audit 2023-10-09T05:01:13.438353+0000 mon.a (mon.0) 543 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:01:14.282 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:13 smithi133 bash[102858]: audit 2023-10-09T05:01:13.439835+0000 mon.a (mon.0) 544 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:14.283 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:01:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:01:14] "GET /metrics HTTP/1.1" 200 36354 "" "Prometheus/2.43.0" 2023-10-09T05:01:14.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:13 smithi133 bash[105589]: cluster 2023-10-09T05:01:13.068649+0000 mon.a (mon.0) 542 : cluster 3 Health check update: Degraded data redundancy: 13058/26253 objects degraded (49.739%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:14.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:13 smithi133 bash[105589]: audit 2023-10-09T05:01:13.438353+0000 mon.a (mon.0) 543 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:01:14.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:13 smithi133 bash[105589]: audit 2023-10-09T05:01:13.439835+0000 mon.a (mon.0) 544 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:14.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:13 smithi194 bash[92666]: cluster 2023-10-09T05:01:13.068649+0000 mon.a (mon.0) 542 : cluster 3 Health check update: Degraded data redundancy: 13058/26253 objects degraded (49.739%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:14.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:13 smithi194 bash[92666]: audit 2023-10-09T05:01:13.438353+0000 mon.a (mon.0) 543 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:01:14.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:13 smithi194 bash[92666]: audit 2023-10-09T05:01:13.439835+0000 mon.a (mon.0) 544 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:15.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:14 smithi133 bash[102858]: cluster 2023-10-09T05:01:13.543089+0000 mgr.y (mgr.25277) 362 : cluster 0 pgmap v299: 89 pgs: 1 active+recovering+undersized+remapped, 79 active+clean, 9 active+recovery_wait+undersized+degraded+remapped; 546 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 13058/26253 objects degraded (49.739%); 0 B/s, 95 objects/s recovering 2023-10-09T05:01:15.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:14 smithi133 bash[102858]: audit 2023-10-09T05:01:13.972712+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.133:0/1145922810' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:15.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:14 smithi133 bash[105589]: cluster 2023-10-09T05:01:13.543089+0000 mgr.y (mgr.25277) 362 : cluster 0 pgmap v299: 89 pgs: 1 active+recovering+undersized+remapped, 79 active+clean, 9 active+recovery_wait+undersized+degraded+remapped; 546 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 13058/26253 objects degraded (49.739%); 0 B/s, 95 objects/s recovering 2023-10-09T05:01:15.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:14 smithi133 bash[105589]: audit 2023-10-09T05:01:13.972712+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.133:0/1145922810' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:15.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:14 smithi194 bash[92666]: cluster 2023-10-09T05:01:13.543089+0000 mgr.y (mgr.25277) 362 : cluster 0 pgmap v299: 89 pgs: 1 active+recovering+undersized+remapped, 79 active+clean, 9 active+recovery_wait+undersized+degraded+remapped; 546 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 13058/26253 objects degraded (49.739%); 0 B/s, 95 objects/s recovering 2023-10-09T05:01:15.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:14 smithi194 bash[92666]: audit 2023-10-09T05:01:13.972712+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.133:0/1145922810' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:17.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:16 smithi133 bash[105589]: cluster 2023-10-09T05:01:15.543866+0000 mgr.y (mgr.25277) 363 : cluster 0 pgmap v300: 89 pgs: 1 active+recovering+undersized+remapped, 79 active+clean, 9 active+recovery_wait+undersized+degraded+remapped; 380 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 13058/18282 objects degraded (71.425%); 0 B/s, 80 objects/s recovering 2023-10-09T05:01:17.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:16 smithi133 bash[105589]: audit 2023-10-09T05:01:16.626490+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.133:0/3596495653' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:17.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:16 smithi133 bash[102858]: cluster 2023-10-09T05:01:15.543866+0000 mgr.y (mgr.25277) 363 : cluster 0 pgmap v300: 89 pgs: 1 active+recovering+undersized+remapped, 79 active+clean, 9 active+recovery_wait+undersized+degraded+remapped; 380 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 13058/18282 objects degraded (71.425%); 0 B/s, 80 objects/s recovering 2023-10-09T05:01:17.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:16 smithi133 bash[102858]: audit 2023-10-09T05:01:16.626490+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.133:0/3596495653' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:17.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:16 smithi194 bash[92666]: cluster 2023-10-09T05:01:15.543866+0000 mgr.y (mgr.25277) 363 : cluster 0 pgmap v300: 89 pgs: 1 active+recovering+undersized+remapped, 79 active+clean, 9 active+recovery_wait+undersized+degraded+remapped; 380 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 13058/18282 objects degraded (71.425%); 0 B/s, 80 objects/s recovering 2023-10-09T05:01:17.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:16 smithi194 bash[92666]: audit 2023-10-09T05:01:16.626490+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.133:0/3596495653' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:19.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:18 smithi133 bash[102858]: cluster 2023-10-09T05:01:17.544645+0000 mgr.y (mgr.25277) 364 : cluster 0 pgmap v301: 89 pgs: 1 active+recovering+undersized+remapped, 79 active+clean, 9 active+recovery_wait+undersized+degraded+remapped; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 13058/16497 objects degraded (79.154%); 0 B/s, 40 objects/s recovering 2023-10-09T05:01:19.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:18 smithi133 bash[102858]: cluster 2023-10-09T05:01:18.069816+0000 mon.a (mon.0) 545 : cluster 3 Health check update: Degraded data redundancy: 13058/16497 objects degraded (79.154%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:19.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:18 smithi133 bash[105589]: cluster 2023-10-09T05:01:17.544645+0000 mgr.y (mgr.25277) 364 : cluster 0 pgmap v301: 89 pgs: 1 active+recovering+undersized+remapped, 79 active+clean, 9 active+recovery_wait+undersized+degraded+remapped; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 13058/16497 objects degraded (79.154%); 0 B/s, 40 objects/s recovering 2023-10-09T05:01:19.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:18 smithi133 bash[105589]: cluster 2023-10-09T05:01:18.069816+0000 mon.a (mon.0) 545 : cluster 3 Health check update: Degraded data redundancy: 13058/16497 objects degraded (79.154%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:19.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:18 smithi194 bash[92666]: cluster 2023-10-09T05:01:17.544645+0000 mgr.y (mgr.25277) 364 : cluster 0 pgmap v301: 89 pgs: 1 active+recovering+undersized+remapped, 79 active+clean, 9 active+recovery_wait+undersized+degraded+remapped; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 13058/16497 objects degraded (79.154%); 0 B/s, 40 objects/s recovering 2023-10-09T05:01:19.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:18 smithi194 bash[92666]: cluster 2023-10-09T05:01:18.069816+0000 mon.a (mon.0) 545 : cluster 3 Health check update: Degraded data redundancy: 13058/16497 objects degraded (79.154%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:20.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:19 smithi133 bash[102858]: audit 2023-10-09T05:01:19.269882+0000 mon.b (mon.1) 10 : audit 0 from='client.? 172.21.15.133:0/3568121037' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:20.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:19 smithi133 bash[105589]: audit 2023-10-09T05:01:19.269882+0000 mon.b (mon.1) 10 : audit 0 from='client.? 172.21.15.133:0/3568121037' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:20.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:19 smithi194 bash[92666]: audit 2023-10-09T05:01:19.269882+0000 mon.b (mon.1) 10 : audit 0 from='client.? 172.21.15.133:0/3568121037' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:21.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:20 smithi133 bash[102858]: cluster 2023-10-09T05:01:19.545236+0000 mgr.y (mgr.25277) 365 : cluster 0 pgmap v302: 89 pgs: 1 active+recovering+undersized+remapped, 79 active+clean, 9 active+recovery_wait+undersized+degraded+remapped; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 13058/16497 objects degraded (79.154%); 0 B/s, 34 objects/s recovering 2023-10-09T05:01:21.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:20 smithi133 bash[105589]: cluster 2023-10-09T05:01:19.545236+0000 mgr.y (mgr.25277) 365 : cluster 0 pgmap v302: 89 pgs: 1 active+recovering+undersized+remapped, 79 active+clean, 9 active+recovery_wait+undersized+degraded+remapped; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 13058/16497 objects degraded (79.154%); 0 B/s, 34 objects/s recovering 2023-10-09T05:01:21.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:20 smithi194 bash[92666]: cluster 2023-10-09T05:01:19.545236+0000 mgr.y (mgr.25277) 365 : cluster 0 pgmap v302: 89 pgs: 1 active+recovering+undersized+remapped, 79 active+clean, 9 active+recovery_wait+undersized+degraded+remapped; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 13058/16497 objects degraded (79.154%); 0 B/s, 34 objects/s recovering 2023-10-09T05:01:22.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:21 smithi133 bash[102858]: audit 2023-10-09T05:01:21.902902+0000 mon.a (mon.0) 546 : audit 0 from='client.? 172.21.15.133:0/3164665535' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:22.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:21 smithi133 bash[105589]: audit 2023-10-09T05:01:21.902902+0000 mon.a (mon.0) 546 : audit 0 from='client.? 172.21.15.133:0/3164665535' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:22.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:21 smithi194 bash[92666]: audit 2023-10-09T05:01:21.902902+0000 mon.a (mon.0) 546 : audit 0 from='client.? 172.21.15.133:0/3164665535' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:23.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:22 smithi133 bash[102858]: cluster 2023-10-09T05:01:21.546079+0000 mgr.y (mgr.25277) 366 : cluster 0 pgmap v303: 89 pgs: 1 active+recovering+undersized+remapped, 79 active+clean, 9 active+recovery_wait+undersized+degraded+remapped; 142 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 13058/6849 objects degraded (190.656%); 0 B/s, 54 objects/s recovering 2023-10-09T05:01:23.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:22 smithi133 bash[105589]: cluster 2023-10-09T05:01:21.546079+0000 mgr.y (mgr.25277) 366 : cluster 0 pgmap v303: 89 pgs: 1 active+recovering+undersized+remapped, 79 active+clean, 9 active+recovery_wait+undersized+degraded+remapped; 142 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 13058/6849 objects degraded (190.656%); 0 B/s, 54 objects/s recovering 2023-10-09T05:01:23.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:22 smithi194 bash[92666]: cluster 2023-10-09T05:01:21.546079+0000 mgr.y (mgr.25277) 366 : cluster 0 pgmap v303: 89 pgs: 1 active+recovering+undersized+remapped, 79 active+clean, 9 active+recovery_wait+undersized+degraded+remapped; 142 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 13058/6849 objects degraded (190.656%); 0 B/s, 54 objects/s recovering 2023-10-09T05:01:24.219 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Removed 101372 objects 2023-10-09T05:01:24.219 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Clean up completed and total clean up time :158.788 2023-10-09T05:01:24.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:23 smithi133 bash[105589]: cluster 2023-10-09T05:01:23.070682+0000 mon.a (mon.0) 547 : cluster 3 Health check update: Degraded data redundancy: 13058/6849 objects degraded (190.656%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:24.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:23 smithi133 bash[105589]: cluster 2023-10-09T05:01:23.082629+0000 osd.3 (osd.3) 37 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T05:01:24.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:23 smithi133 bash[105589]: cluster 2023-10-09T05:01:23.083081+0000 mon.a (mon.0) 548 : cluster 0 osdmap e109: 8 total, 8 up, 8 in 2023-10-09T05:01:24.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:23 smithi133 bash[105589]: cluster 2023-10-09T05:01:23.088037+0000 osd.3 (osd.3) 38 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T05:01:24.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:23 smithi133 bash[105589]: cluster 2023-10-09T05:01:23.090770+0000 osd.3 (osd.3) 39 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T05:01:24.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:23 smithi133 bash[105589]: cluster 2023-10-09T05:01:23.092771+0000 osd.3 (osd.3) 40 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T05:01:24.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:23 smithi133 bash[105589]: cluster 2023-10-09T05:01:23.093632+0000 osd.5 (osd.5) 47 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T05:01:24.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:23 smithi133 bash[105589]: cluster 2023-10-09T05:01:23.095862+0000 osd.6 (osd.6) 26 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T05:01:24.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:23 smithi133 bash[105589]: cluster 2023-10-09T05:01:23.099574+0000 osd.6 (osd.6) 27 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T05:01:24.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:23 smithi133 bash[105589]: cluster 2023-10-09T05:01:23.100109+0000 osd.7 (osd.7) 27 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T05:01:24.285 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:23 smithi133 bash[105589]: cluster 2023-10-09T05:01:23.101676+0000 osd.4 (osd.4) 33 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T05:01:24.285 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:23 smithi133 bash[105589]: cluster 2023-10-09T05:01:23.105121+0000 osd.7 (osd.7) 28 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T05:01:24.285 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:23 smithi133 bash[102858]: cluster 2023-10-09T05:01:23.070682+0000 mon.a (mon.0) 547 : cluster 3 Health check update: Degraded data redundancy: 13058/6849 objects degraded (190.656%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:24.285 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:23 smithi133 bash[102858]: cluster 2023-10-09T05:01:23.082629+0000 osd.3 (osd.3) 37 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T05:01:24.285 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:23 smithi133 bash[102858]: cluster 2023-10-09T05:01:23.083081+0000 mon.a (mon.0) 548 : cluster 0 osdmap e109: 8 total, 8 up, 8 in 2023-10-09T05:01:24.285 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:23 smithi133 bash[102858]: cluster 2023-10-09T05:01:23.088037+0000 osd.3 (osd.3) 38 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T05:01:24.285 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:23 smithi133 bash[102858]: cluster 2023-10-09T05:01:23.090770+0000 osd.3 (osd.3) 39 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T05:01:24.286 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:23 smithi133 bash[102858]: cluster 2023-10-09T05:01:23.092771+0000 osd.3 (osd.3) 40 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T05:01:24.286 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:23 smithi133 bash[102858]: cluster 2023-10-09T05:01:23.093632+0000 osd.5 (osd.5) 47 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T05:01:24.286 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:23 smithi133 bash[102858]: cluster 2023-10-09T05:01:23.095862+0000 osd.6 (osd.6) 26 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T05:01:24.286 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:23 smithi133 bash[102858]: cluster 2023-10-09T05:01:23.099574+0000 osd.6 (osd.6) 27 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T05:01:24.286 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:23 smithi133 bash[102858]: cluster 2023-10-09T05:01:23.100109+0000 osd.7 (osd.7) 27 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T05:01:24.286 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:23 smithi133 bash[102858]: cluster 2023-10-09T05:01:23.101676+0000 osd.4 (osd.4) 33 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T05:01:24.286 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:23 smithi133 bash[102858]: cluster 2023-10-09T05:01:23.105121+0000 osd.7 (osd.7) 28 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T05:01:24.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:23 smithi194 bash[92666]: cluster 2023-10-09T05:01:23.070682+0000 mon.a (mon.0) 547 : cluster 3 Health check update: Degraded data redundancy: 13058/6849 objects degraded (190.656%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:24.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:23 smithi194 bash[92666]: cluster 2023-10-09T05:01:23.082629+0000 osd.3 (osd.3) 37 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T05:01:24.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:23 smithi194 bash[92666]: cluster 2023-10-09T05:01:23.083081+0000 mon.a (mon.0) 548 : cluster 0 osdmap e109: 8 total, 8 up, 8 in 2023-10-09T05:01:24.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:23 smithi194 bash[92666]: cluster 2023-10-09T05:01:23.088037+0000 osd.3 (osd.3) 38 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T05:01:24.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:23 smithi194 bash[92666]: cluster 2023-10-09T05:01:23.090770+0000 osd.3 (osd.3) 39 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T05:01:24.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:23 smithi194 bash[92666]: cluster 2023-10-09T05:01:23.092771+0000 osd.3 (osd.3) 40 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T05:01:24.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:23 smithi194 bash[92666]: cluster 2023-10-09T05:01:23.093632+0000 osd.5 (osd.5) 47 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T05:01:24.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:23 smithi194 bash[92666]: cluster 2023-10-09T05:01:23.095862+0000 osd.6 (osd.6) 26 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T05:01:24.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:23 smithi194 bash[92666]: cluster 2023-10-09T05:01:23.099574+0000 osd.6 (osd.6) 27 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T05:01:24.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:23 smithi194 bash[92666]: cluster 2023-10-09T05:01:23.100109+0000 osd.7 (osd.7) 27 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T05:01:24.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:23 smithi194 bash[92666]: cluster 2023-10-09T05:01:23.101676+0000 osd.4 (osd.4) 33 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T05:01:24.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:23 smithi194 bash[92666]: cluster 2023-10-09T05:01:23.105121+0000 osd.7 (osd.7) 28 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T05:01:24.521 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2023-10-09T05:01:24.522 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2023-10-09T05:01:24.533 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:01:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:01:24] "GET /metrics HTTP/1.1" 200 36354 "" "Prometheus/2.43.0" 2023-10-09T05:01:25.299 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:01:25 smithi133 bash[91128]: debug 2023-10-09T05:01:25.043+0000 7ff45d5ed700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2023-10-09T05:01:25.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:24 smithi133 bash[102858]: cluster 2023-10-09T05:01:23.546696+0000 mgr.y (mgr.25277) 367 : cluster 0 pgmap v305: 89 pgs: 1 active+recovering+undersized+remapped, 79 active+clean, 9 active+recovery_wait+undersized+degraded+remapped; 142 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 13058/6849 objects degraded (190.656%); 0 B/s, 47 objects/s recovering 2023-10-09T05:01:25.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:24 smithi133 bash[102858]: cluster 2023-10-09T05:01:24.098236+0000 osd.6 (osd.6) 28 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T05:01:25.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:24 smithi133 bash[102858]: cluster 2023-10-09T05:01:24.099845+0000 mon.a (mon.0) 549 : cluster 0 osdmap e110: 8 total, 8 up, 8 in 2023-10-09T05:01:25.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:24 smithi133 bash[102858]: cluster 2023-10-09T05:01:24.100296+0000 osd.3 (osd.3) 41 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T05:01:25.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:24 smithi133 bash[102858]: cluster 2023-10-09T05:01:24.102536+0000 osd.3 (osd.3) 42 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T05:01:25.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:24 smithi133 bash[102858]: cluster 2023-10-09T05:01:24.113162+0000 osd.5 (osd.5) 48 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T05:01:25.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:24 smithi133 bash[102858]: cluster 2023-10-09T05:01:24.316831+0000 osd.7 (osd.7) 29 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T05:01:25.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:24 smithi133 bash[102858]: cluster 2023-10-09T05:01:24.316882+0000 osd.4 (osd.4) 34 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T05:01:25.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:24 smithi133 bash[102858]: audit 2023-10-09T05:01:24.369995+0000 mon.c (mon.2) 37 : audit 0 from='client.? 172.21.15.133:0/511607505' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:25.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:24 smithi133 bash[105589]: cluster 2023-10-09T05:01:23.546696+0000 mgr.y (mgr.25277) 367 : cluster 0 pgmap v305: 89 pgs: 1 active+recovering+undersized+remapped, 79 active+clean, 9 active+recovery_wait+undersized+degraded+remapped; 142 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 13058/6849 objects degraded (190.656%); 0 B/s, 47 objects/s recovering 2023-10-09T05:01:25.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:24 smithi133 bash[105589]: cluster 2023-10-09T05:01:24.098236+0000 osd.6 (osd.6) 28 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T05:01:25.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:24 smithi133 bash[105589]: cluster 2023-10-09T05:01:24.099845+0000 mon.a (mon.0) 549 : cluster 0 osdmap e110: 8 total, 8 up, 8 in 2023-10-09T05:01:25.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:24 smithi133 bash[105589]: cluster 2023-10-09T05:01:24.100296+0000 osd.3 (osd.3) 41 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T05:01:25.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:24 smithi133 bash[105589]: cluster 2023-10-09T05:01:24.102536+0000 osd.3 (osd.3) 42 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T05:01:25.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:24 smithi133 bash[105589]: cluster 2023-10-09T05:01:24.113162+0000 osd.5 (osd.5) 48 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T05:01:25.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:24 smithi133 bash[105589]: cluster 2023-10-09T05:01:24.316831+0000 osd.7 (osd.7) 29 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T05:01:25.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:24 smithi133 bash[105589]: cluster 2023-10-09T05:01:24.316882+0000 osd.4 (osd.4) 34 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T05:01:25.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:24 smithi133 bash[105589]: audit 2023-10-09T05:01:24.369995+0000 mon.c (mon.2) 37 : audit 0 from='client.? 172.21.15.133:0/511607505' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:25.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:24 smithi194 bash[92666]: cluster 2023-10-09T05:01:23.546696+0000 mgr.y (mgr.25277) 367 : cluster 0 pgmap v305: 89 pgs: 1 active+recovering+undersized+remapped, 79 active+clean, 9 active+recovery_wait+undersized+degraded+remapped; 142 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 13058/6849 objects degraded (190.656%); 0 B/s, 47 objects/s recovering 2023-10-09T05:01:25.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:24 smithi194 bash[92666]: cluster 2023-10-09T05:01:24.098236+0000 osd.6 (osd.6) 28 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T05:01:25.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:24 smithi194 bash[92666]: cluster 2023-10-09T05:01:24.099845+0000 mon.a (mon.0) 549 : cluster 0 osdmap e110: 8 total, 8 up, 8 in 2023-10-09T05:01:25.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:24 smithi194 bash[92666]: cluster 2023-10-09T05:01:24.100296+0000 osd.3 (osd.3) 41 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T05:01:25.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:24 smithi194 bash[92666]: cluster 2023-10-09T05:01:24.102536+0000 osd.3 (osd.3) 42 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T05:01:25.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:24 smithi194 bash[92666]: cluster 2023-10-09T05:01:24.113162+0000 osd.5 (osd.5) 48 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T05:01:25.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:24 smithi194 bash[92666]: cluster 2023-10-09T05:01:24.316831+0000 osd.7 (osd.7) 29 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T05:01:25.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:24 smithi194 bash[92666]: cluster 2023-10-09T05:01:24.316882+0000 osd.4 (osd.4) 34 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T05:01:25.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:24 smithi194 bash[92666]: audit 2023-10-09T05:01:24.369995+0000 mon.c (mon.2) 37 : audit 0 from='client.? 172.21.15.133:0/511607505' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:26.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:25 smithi133 bash[102858]: audit 2023-10-09T05:01:25.045948+0000 mon.a (mon.0) 550 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:01:26.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:25 smithi133 bash[102858]: audit 2023-10-09T05:01:25.046241+0000 mgr.y (mgr.25277) 368 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:01:26.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:25 smithi133 bash[105589]: audit 2023-10-09T05:01:25.045948+0000 mon.a (mon.0) 550 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:01:26.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:25 smithi133 bash[105589]: audit 2023-10-09T05:01:25.046241+0000 mgr.y (mgr.25277) 368 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:01:26.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:25 smithi194 bash[92666]: audit 2023-10-09T05:01:25.045948+0000 mon.a (mon.0) 550 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:01:26.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:25 smithi194 bash[92666]: audit 2023-10-09T05:01:25.046241+0000 mgr.y (mgr.25277) 368 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:01:27.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:26 smithi133 bash[105589]: cephadm 2023-10-09T05:01:25.047840+0000 mgr.y (mgr.25277) 369 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2023-10-09T05:01:27.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:26 smithi133 bash[105589]: cluster 2023-10-09T05:01:25.547845+0000 mgr.y (mgr.25277) 370 : cluster 0 pgmap v307: 89 pgs: 2 active+recovering+undersized+remapped, 79 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 22 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 12032/1083 objects degraded (1110.988%); 0 B/s, 75 objects/s recovering 2023-10-09T05:01:27.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:26 smithi133 bash[105589]: audit 2023-10-09T05:01:26.809837+0000 mon.c (mon.2) 38 : audit 0 from='client.? 172.21.15.133:0/372992536' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:27.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:26 smithi133 bash[102858]: cephadm 2023-10-09T05:01:25.047840+0000 mgr.y (mgr.25277) 369 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2023-10-09T05:01:27.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:26 smithi133 bash[102858]: cluster 2023-10-09T05:01:25.547845+0000 mgr.y (mgr.25277) 370 : cluster 0 pgmap v307: 89 pgs: 2 active+recovering+undersized+remapped, 79 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 22 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 12032/1083 objects degraded (1110.988%); 0 B/s, 75 objects/s recovering 2023-10-09T05:01:27.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:26 smithi133 bash[102858]: audit 2023-10-09T05:01:26.809837+0000 mon.c (mon.2) 38 : audit 0 from='client.? 172.21.15.133:0/372992536' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:27.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:26 smithi194 bash[92666]: cephadm 2023-10-09T05:01:25.047840+0000 mgr.y (mgr.25277) 369 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2023-10-09T05:01:27.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:26 smithi194 bash[92666]: cluster 2023-10-09T05:01:25.547845+0000 mgr.y (mgr.25277) 370 : cluster 0 pgmap v307: 89 pgs: 2 active+recovering+undersized+remapped, 79 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 22 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 12032/1083 objects degraded (1110.988%); 0 B/s, 75 objects/s recovering 2023-10-09T05:01:27.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:26 smithi194 bash[92666]: audit 2023-10-09T05:01:26.809837+0000 mon.c (mon.2) 38 : audit 0 from='client.? 172.21.15.133:0/372992536' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:29.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:28 smithi133 bash[105589]: cluster 2023-10-09T05:01:27.548674+0000 mgr.y (mgr.25277) 371 : cluster 0 pgmap v308: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 12032/72 objects degraded (16711.111%); 0 B/s, 88 objects/s recovering 2023-10-09T05:01:29.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:28 smithi133 bash[105589]: cluster 2023-10-09T05:01:28.072506+0000 mon.a (mon.0) 551 : cluster 3 Health check update: Degraded data redundancy: 12032/72 objects degraded (16711.111%), 8 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:29.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:28 smithi133 bash[105589]: audit 2023-10-09T05:01:28.423920+0000 mon.a (mon.0) 552 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:29.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:28 smithi133 bash[102858]: cluster 2023-10-09T05:01:27.548674+0000 mgr.y (mgr.25277) 371 : cluster 0 pgmap v308: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 12032/72 objects degraded (16711.111%); 0 B/s, 88 objects/s recovering 2023-10-09T05:01:29.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:28 smithi133 bash[102858]: cluster 2023-10-09T05:01:28.072506+0000 mon.a (mon.0) 551 : cluster 3 Health check update: Degraded data redundancy: 12032/72 objects degraded (16711.111%), 8 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:29.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:28 smithi133 bash[102858]: audit 2023-10-09T05:01:28.423920+0000 mon.a (mon.0) 552 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:29.319 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:29.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:28 smithi194 bash[92666]: cluster 2023-10-09T05:01:27.548674+0000 mgr.y (mgr.25277) 371 : cluster 0 pgmap v308: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 12032/72 objects degraded (16711.111%); 0 B/s, 88 objects/s recovering 2023-10-09T05:01:29.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:28 smithi194 bash[92666]: cluster 2023-10-09T05:01:28.072506+0000 mon.a (mon.0) 551 : cluster 3 Health check update: Degraded data redundancy: 12032/72 objects degraded (16711.111%), 8 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:29.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:28 smithi194 bash[92666]: audit 2023-10-09T05:01:28.423920+0000 mon.a (mon.0) 552 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:30.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:30 smithi194 bash[92666]: cluster 2023-10-09T05:01:29.078370+0000 osd.3 (osd.3) 43 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T05:01:30.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:30 smithi194 bash[92666]: cluster 2023-10-09T05:01:29.082571+0000 mon.a (mon.0) 553 : cluster 0 osdmap e111: 8 total, 8 up, 8 in 2023-10-09T05:01:30.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:30 smithi194 bash[92666]: cluster 2023-10-09T05:01:29.083967+0000 osd.5 (osd.5) 49 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T05:01:30.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:30 smithi194 bash[92666]: cluster 2023-10-09T05:01:29.086765+0000 osd.4 (osd.4) 35 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T05:01:30.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:30 smithi194 bash[92666]: cluster 2023-10-09T05:01:29.087792+0000 osd.5 (osd.5) 50 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T05:01:30.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:30 smithi194 bash[92666]: cluster 2023-10-09T05:01:29.136069+0000 osd.6 (osd.6) 29 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T05:01:30.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:30 smithi194 bash[92666]: audit 2023-10-09T05:01:29.251477+0000 mon.a (mon.0) 554 : audit 0 from='client.? 172.21.15.133:0/3313010317' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:30.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:30 smithi194 bash[92666]: cluster 2023-10-09T05:01:29.549176+0000 mgr.y (mgr.25277) 372 : cluster 0 pgmap v310: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 349 op/s; 12032/72 objects degraded (16711.111%); 0 B/s, 71 objects/s recovering 2023-10-09T05:01:30.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:30 smithi133 bash[105589]: cluster 2023-10-09T05:01:29.078370+0000 osd.3 (osd.3) 43 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T05:01:30.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:30 smithi133 bash[105589]: cluster 2023-10-09T05:01:29.082571+0000 mon.a (mon.0) 553 : cluster 0 osdmap e111: 8 total, 8 up, 8 in 2023-10-09T05:01:30.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:30 smithi133 bash[105589]: cluster 2023-10-09T05:01:29.083967+0000 osd.5 (osd.5) 49 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T05:01:30.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:30 smithi133 bash[105589]: cluster 2023-10-09T05:01:29.086765+0000 osd.4 (osd.4) 35 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T05:01:30.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:30 smithi133 bash[105589]: cluster 2023-10-09T05:01:29.087792+0000 osd.5 (osd.5) 50 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T05:01:30.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:30 smithi133 bash[105589]: cluster 2023-10-09T05:01:29.136069+0000 osd.6 (osd.6) 29 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T05:01:30.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:30 smithi133 bash[105589]: audit 2023-10-09T05:01:29.251477+0000 mon.a (mon.0) 554 : audit 0 from='client.? 172.21.15.133:0/3313010317' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:30.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:30 smithi133 bash[105589]: cluster 2023-10-09T05:01:29.549176+0000 mgr.y (mgr.25277) 372 : cluster 0 pgmap v310: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 349 op/s; 12032/72 objects degraded (16711.111%); 0 B/s, 71 objects/s recovering 2023-10-09T05:01:30.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:30 smithi133 bash[102858]: cluster 2023-10-09T05:01:29.078370+0000 osd.3 (osd.3) 43 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T05:01:30.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:30 smithi133 bash[102858]: cluster 2023-10-09T05:01:29.082571+0000 mon.a (mon.0) 553 : cluster 0 osdmap e111: 8 total, 8 up, 8 in 2023-10-09T05:01:30.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:30 smithi133 bash[102858]: cluster 2023-10-09T05:01:29.083967+0000 osd.5 (osd.5) 49 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T05:01:30.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:30 smithi133 bash[102858]: cluster 2023-10-09T05:01:29.086765+0000 osd.4 (osd.4) 35 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T05:01:30.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:30 smithi133 bash[102858]: cluster 2023-10-09T05:01:29.087792+0000 osd.5 (osd.5) 50 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T05:01:30.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:30 smithi133 bash[102858]: cluster 2023-10-09T05:01:29.136069+0000 osd.6 (osd.6) 29 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T05:01:30.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:30 smithi133 bash[102858]: audit 2023-10-09T05:01:29.251477+0000 mon.a (mon.0) 554 : audit 0 from='client.? 172.21.15.133:0/3313010317' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:30.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:30 smithi133 bash[102858]: cluster 2023-10-09T05:01:29.549176+0000 mgr.y (mgr.25277) 372 : cluster 0 pgmap v310: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 349 op/s; 12032/72 objects degraded (16711.111%); 0 B/s, 71 objects/s recovering 2023-10-09T05:01:31.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:31 smithi194 bash[92666]: cluster 2023-10-09T05:01:30.089406+0000 osd.3 (osd.3) 44 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T05:01:31.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:31 smithi194 bash[92666]: cluster 2023-10-09T05:01:30.093646+0000 mon.a (mon.0) 555 : cluster 0 osdmap e112: 8 total, 8 up, 8 in 2023-10-09T05:01:31.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:31 smithi194 bash[92666]: cluster 2023-10-09T05:01:30.097509+0000 osd.5 (osd.5) 51 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T05:01:31.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:31 smithi194 bash[92666]: cluster 2023-10-09T05:01:30.102093+0000 osd.4 (osd.4) 36 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T05:01:31.556 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:31 smithi133 bash[105589]: cluster 2023-10-09T05:01:30.089406+0000 osd.3 (osd.3) 44 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T05:01:31.556 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:31 smithi133 bash[105589]: cluster 2023-10-09T05:01:30.093646+0000 mon.a (mon.0) 555 : cluster 0 osdmap e112: 8 total, 8 up, 8 in 2023-10-09T05:01:31.557 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:31 smithi133 bash[105589]: cluster 2023-10-09T05:01:30.097509+0000 osd.5 (osd.5) 51 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T05:01:31.557 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:31 smithi133 bash[105589]: cluster 2023-10-09T05:01:30.102093+0000 osd.4 (osd.4) 36 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T05:01:31.557 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:31 smithi133 bash[102858]: cluster 2023-10-09T05:01:30.089406+0000 osd.3 (osd.3) 44 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T05:01:31.557 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:31 smithi133 bash[102858]: cluster 2023-10-09T05:01:30.093646+0000 mon.a (mon.0) 555 : cluster 0 osdmap e112: 8 total, 8 up, 8 in 2023-10-09T05:01:31.557 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:31 smithi133 bash[102858]: cluster 2023-10-09T05:01:30.097509+0000 osd.5 (osd.5) 51 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T05:01:31.557 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:31 smithi133 bash[102858]: cluster 2023-10-09T05:01:30.102093+0000 osd.4 (osd.4) 36 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T05:01:32.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:32 smithi194 bash[92666]: cluster 2023-10-09T05:01:31.549939+0000 mgr.y (mgr.25277) 373 : cluster 0 pgmap v312: 89 pgs: 1 active+recovering+undersized+remapped, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 303 op/s; 10630/72 objects degraded (14763.889%); 0 B/s, 154 objects/s recovering 2023-10-09T05:01:32.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:32 smithi194 bash[92666]: audit 2023-10-09T05:01:31.731169+0000 mon.c (mon.2) 39 : audit 0 from='client.? 172.21.15.133:0/1377909933' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:32.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:32 smithi133 bash[102858]: cluster 2023-10-09T05:01:31.549939+0000 mgr.y (mgr.25277) 373 : cluster 0 pgmap v312: 89 pgs: 1 active+recovering+undersized+remapped, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 303 op/s; 10630/72 objects degraded (14763.889%); 0 B/s, 154 objects/s recovering 2023-10-09T05:01:32.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:32 smithi133 bash[102858]: audit 2023-10-09T05:01:31.731169+0000 mon.c (mon.2) 39 : audit 0 from='client.? 172.21.15.133:0/1377909933' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:32.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:32 smithi133 bash[105589]: cluster 2023-10-09T05:01:31.549939+0000 mgr.y (mgr.25277) 373 : cluster 0 pgmap v312: 89 pgs: 1 active+recovering+undersized+remapped, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 303 op/s; 10630/72 objects degraded (14763.889%); 0 B/s, 154 objects/s recovering 2023-10-09T05:01:32.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:32 smithi133 bash[105589]: audit 2023-10-09T05:01:31.731169+0000 mon.c (mon.2) 39 : audit 0 from='client.? 172.21.15.133:0/1377909933' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:33.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:33 smithi194 bash[92666]: cluster 2023-10-09T05:01:32.097092+0000 osd.7 (osd.7) 30 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T05:01:33.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:33 smithi194 bash[92666]: cluster 2023-10-09T05:01:32.099755+0000 mon.a (mon.0) 556 : cluster 0 osdmap e113: 8 total, 8 up, 8 in 2023-10-09T05:01:33.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:33 smithi194 bash[92666]: cluster 2023-10-09T05:01:32.100867+0000 osd.7 (osd.7) 31 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T05:01:33.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:33 smithi194 bash[92666]: cluster 2023-10-09T05:01:32.100888+0000 osd.4 (osd.4) 37 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T05:01:33.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:33 smithi194 bash[92666]: cluster 2023-10-09T05:01:32.104162+0000 osd.4 (osd.4) 38 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T05:01:33.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:33 smithi194 bash[92666]: cluster 2023-10-09T05:01:32.691985+0000 osd.5 (osd.5) 52 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T05:01:33.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:33 smithi194 bash[92666]: cluster 2023-10-09T05:01:32.694576+0000 osd.5 (osd.5) 53 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T05:01:33.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:33 smithi194 bash[92666]: cluster 2023-10-09T05:01:32.696088+0000 osd.5 (osd.5) 54 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T05:01:33.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:33 smithi194 bash[92666]: cluster 2023-10-09T05:01:33.074174+0000 mon.a (mon.0) 557 : cluster 3 Health check update: Degraded data redundancy: 10630/72 objects degraded (14763.889%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:33.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:33 smithi133 bash[105589]: cluster 2023-10-09T05:01:32.097092+0000 osd.7 (osd.7) 30 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T05:01:33.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:33 smithi133 bash[105589]: cluster 2023-10-09T05:01:32.099755+0000 mon.a (mon.0) 556 : cluster 0 osdmap e113: 8 total, 8 up, 8 in 2023-10-09T05:01:33.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:33 smithi133 bash[105589]: cluster 2023-10-09T05:01:32.100867+0000 osd.7 (osd.7) 31 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T05:01:33.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:33 smithi133 bash[105589]: cluster 2023-10-09T05:01:32.100888+0000 osd.4 (osd.4) 37 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T05:01:33.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:33 smithi133 bash[105589]: cluster 2023-10-09T05:01:32.104162+0000 osd.4 (osd.4) 38 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T05:01:33.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:33 smithi133 bash[105589]: cluster 2023-10-09T05:01:32.691985+0000 osd.5 (osd.5) 52 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T05:01:33.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:33 smithi133 bash[105589]: cluster 2023-10-09T05:01:32.694576+0000 osd.5 (osd.5) 53 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T05:01:33.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:33 smithi133 bash[105589]: cluster 2023-10-09T05:01:32.696088+0000 osd.5 (osd.5) 54 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T05:01:33.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:33 smithi133 bash[105589]: cluster 2023-10-09T05:01:33.074174+0000 mon.a (mon.0) 557 : cluster 3 Health check update: Degraded data redundancy: 10630/72 objects degraded (14763.889%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:33.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:33 smithi133 bash[102858]: cluster 2023-10-09T05:01:32.097092+0000 osd.7 (osd.7) 30 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T05:01:33.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:33 smithi133 bash[102858]: cluster 2023-10-09T05:01:32.099755+0000 mon.a (mon.0) 556 : cluster 0 osdmap e113: 8 total, 8 up, 8 in 2023-10-09T05:01:33.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:33 smithi133 bash[102858]: cluster 2023-10-09T05:01:32.100867+0000 osd.7 (osd.7) 31 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T05:01:33.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:33 smithi133 bash[102858]: cluster 2023-10-09T05:01:32.100888+0000 osd.4 (osd.4) 37 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T05:01:33.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:33 smithi133 bash[102858]: cluster 2023-10-09T05:01:32.104162+0000 osd.4 (osd.4) 38 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T05:01:33.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:33 smithi133 bash[102858]: cluster 2023-10-09T05:01:32.691985+0000 osd.5 (osd.5) 52 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T05:01:33.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:33 smithi133 bash[102858]: cluster 2023-10-09T05:01:32.694576+0000 osd.5 (osd.5) 53 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T05:01:33.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:33 smithi133 bash[102858]: cluster 2023-10-09T05:01:32.696088+0000 osd.5 (osd.5) 54 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T05:01:33.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:33 smithi133 bash[102858]: cluster 2023-10-09T05:01:33.074174+0000 mon.a (mon.0) 557 : cluster 3 Health check update: Degraded data redundancy: 10630/72 objects degraded (14763.889%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:34.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:34 smithi194 bash[92666]: cluster 2023-10-09T05:01:32.250828+0000 osd.3 (osd.3) 45 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T05:01:34.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:34 smithi194 bash[92666]: cluster 2023-10-09T05:01:33.103683+0000 osd.3 (osd.3) 46 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T05:01:34.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:34 smithi194 bash[92666]: cluster 2023-10-09T05:01:33.105608+0000 osd.5 (osd.5) 55 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T05:01:34.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:34 smithi194 bash[92666]: cluster 2023-10-09T05:01:33.106497+0000 mon.a (mon.0) 558 : cluster 0 osdmap e114: 8 total, 8 up, 8 in 2023-10-09T05:01:34.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:34 smithi194 bash[92666]: cluster 2023-10-09T05:01:33.112781+0000 osd.7 (osd.7) 32 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T05:01:34.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:34 smithi194 bash[92666]: cluster 2023-10-09T05:01:33.112904+0000 osd.4 (osd.4) 39 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T05:01:34.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:34 smithi194 bash[92666]: cluster 2023-10-09T05:01:33.318155+0000 osd.6 (osd.6) 30 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T05:01:34.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:34 smithi194 bash[92666]: cluster 2023-10-09T05:01:33.320880+0000 osd.6 (osd.6) 31 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T05:01:34.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:34 smithi194 bash[92666]: cluster 2023-10-09T05:01:33.550472+0000 mgr.y (mgr.25277) 374 : cluster 0 pgmap v315: 89 pgs: 1 active+recovering+undersized+remapped, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10630/72 objects degraded (14763.889%); 0 B/s, 153 objects/s recovering 2023-10-09T05:01:34.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:01:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:01:34] "GET /metrics HTTP/1.1" 200 36346 "" "Prometheus/2.43.0" 2023-10-09T05:01:34.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:34 smithi133 bash[102858]: cluster 2023-10-09T05:01:32.250828+0000 osd.3 (osd.3) 45 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T05:01:34.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:34 smithi133 bash[102858]: cluster 2023-10-09T05:01:33.103683+0000 osd.3 (osd.3) 46 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T05:01:34.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:34 smithi133 bash[102858]: cluster 2023-10-09T05:01:33.105608+0000 osd.5 (osd.5) 55 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T05:01:34.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:34 smithi133 bash[102858]: cluster 2023-10-09T05:01:33.106497+0000 mon.a (mon.0) 558 : cluster 0 osdmap e114: 8 total, 8 up, 8 in 2023-10-09T05:01:34.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:34 smithi133 bash[102858]: cluster 2023-10-09T05:01:33.112781+0000 osd.7 (osd.7) 32 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T05:01:34.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:34 smithi133 bash[102858]: cluster 2023-10-09T05:01:33.112904+0000 osd.4 (osd.4) 39 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T05:01:34.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:34 smithi133 bash[102858]: cluster 2023-10-09T05:01:33.318155+0000 osd.6 (osd.6) 30 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T05:01:34.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:34 smithi133 bash[102858]: cluster 2023-10-09T05:01:33.320880+0000 osd.6 (osd.6) 31 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T05:01:34.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:34 smithi133 bash[102858]: cluster 2023-10-09T05:01:33.550472+0000 mgr.y (mgr.25277) 374 : cluster 0 pgmap v315: 89 pgs: 1 active+recovering+undersized+remapped, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10630/72 objects degraded (14763.889%); 0 B/s, 153 objects/s recovering 2023-10-09T05:01:34.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:34 smithi133 bash[105589]: cluster 2023-10-09T05:01:32.250828+0000 osd.3 (osd.3) 45 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T05:01:34.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:34 smithi133 bash[105589]: cluster 2023-10-09T05:01:33.103683+0000 osd.3 (osd.3) 46 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T05:01:34.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:34 smithi133 bash[105589]: cluster 2023-10-09T05:01:33.105608+0000 osd.5 (osd.5) 55 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T05:01:34.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:34 smithi133 bash[105589]: cluster 2023-10-09T05:01:33.106497+0000 mon.a (mon.0) 558 : cluster 0 osdmap e114: 8 total, 8 up, 8 in 2023-10-09T05:01:34.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:34 smithi133 bash[105589]: cluster 2023-10-09T05:01:33.112781+0000 osd.7 (osd.7) 32 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T05:01:34.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:34 smithi133 bash[105589]: cluster 2023-10-09T05:01:33.112904+0000 osd.4 (osd.4) 39 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T05:01:34.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:34 smithi133 bash[105589]: cluster 2023-10-09T05:01:33.318155+0000 osd.6 (osd.6) 30 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T05:01:34.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:34 smithi133 bash[105589]: cluster 2023-10-09T05:01:33.320880+0000 osd.6 (osd.6) 31 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T05:01:34.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:34 smithi133 bash[105589]: cluster 2023-10-09T05:01:33.550472+0000 mgr.y (mgr.25277) 374 : cluster 0 pgmap v315: 89 pgs: 1 active+recovering+undersized+remapped, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10630/72 objects degraded (14763.889%); 0 B/s, 153 objects/s recovering 2023-10-09T05:01:35.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:35 smithi133 bash[105589]: audit 2023-10-09T05:01:34.179111+0000 mon.c (mon.2) 40 : audit 0 from='client.? 172.21.15.133:0/2273058938' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:35.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:35 smithi133 bash[102858]: audit 2023-10-09T05:01:34.179111+0000 mon.c (mon.2) 40 : audit 0 from='client.? 172.21.15.133:0/2273058938' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:35.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:35 smithi194 bash[92666]: audit 2023-10-09T05:01:34.179111+0000 mon.c (mon.2) 40 : audit 0 from='client.? 172.21.15.133:0/2273058938' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:36.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:36 smithi194 bash[92666]: cluster 2023-10-09T05:01:35.119710+0000 osd.3 (osd.3) 47 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T05:01:36.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:36 smithi194 bash[92666]: cluster 2023-10-09T05:01:35.122198+0000 mon.a (mon.0) 559 : cluster 0 osdmap e115: 8 total, 8 up, 8 in 2023-10-09T05:01:36.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:36 smithi194 bash[92666]: cluster 2023-10-09T05:01:35.123759+0000 osd.7 (osd.7) 33 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T05:01:36.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:36 smithi194 bash[92666]: cluster 2023-10-09T05:01:35.126473+0000 osd.7 (osd.7) 34 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T05:01:36.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:36 smithi194 bash[92666]: cluster 2023-10-09T05:01:35.259996+0000 osd.6 (osd.6) 32 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T05:01:36.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:36 smithi194 bash[92666]: cluster 2023-10-09T05:01:35.393795+0000 osd.5 (osd.5) 56 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T05:01:36.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:36 smithi194 bash[92666]: cluster 2023-10-09T05:01:35.393856+0000 osd.4 (osd.4) 40 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T05:01:36.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:36 smithi194 bash[92666]: cluster 2023-10-09T05:01:35.551165+0000 mgr.y (mgr.25277) 375 : cluster 0 pgmap v317: 89 pgs: 1 active+recovering+undersized+remapped, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8735/72 objects degraded (12131.944%); 0 B/s, 424 objects/s recovering 2023-10-09T05:01:36.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:36 smithi133 bash[105589]: cluster 2023-10-09T05:01:35.119710+0000 osd.3 (osd.3) 47 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T05:01:36.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:36 smithi133 bash[105589]: cluster 2023-10-09T05:01:35.122198+0000 mon.a (mon.0) 559 : cluster 0 osdmap e115: 8 total, 8 up, 8 in 2023-10-09T05:01:36.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:36 smithi133 bash[105589]: cluster 2023-10-09T05:01:35.123759+0000 osd.7 (osd.7) 33 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T05:01:36.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:36 smithi133 bash[105589]: cluster 2023-10-09T05:01:35.126473+0000 osd.7 (osd.7) 34 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T05:01:36.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:36 smithi133 bash[105589]: cluster 2023-10-09T05:01:35.259996+0000 osd.6 (osd.6) 32 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T05:01:36.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:36 smithi133 bash[105589]: cluster 2023-10-09T05:01:35.393795+0000 osd.5 (osd.5) 56 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T05:01:36.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:36 smithi133 bash[105589]: cluster 2023-10-09T05:01:35.393856+0000 osd.4 (osd.4) 40 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T05:01:36.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:36 smithi133 bash[105589]: cluster 2023-10-09T05:01:35.551165+0000 mgr.y (mgr.25277) 375 : cluster 0 pgmap v317: 89 pgs: 1 active+recovering+undersized+remapped, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8735/72 objects degraded (12131.944%); 0 B/s, 424 objects/s recovering 2023-10-09T05:01:36.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:36 smithi133 bash[102858]: cluster 2023-10-09T05:01:35.119710+0000 osd.3 (osd.3) 47 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T05:01:36.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:36 smithi133 bash[102858]: cluster 2023-10-09T05:01:35.122198+0000 mon.a (mon.0) 559 : cluster 0 osdmap e115: 8 total, 8 up, 8 in 2023-10-09T05:01:36.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:36 smithi133 bash[102858]: cluster 2023-10-09T05:01:35.123759+0000 osd.7 (osd.7) 33 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T05:01:36.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:36 smithi133 bash[102858]: cluster 2023-10-09T05:01:35.126473+0000 osd.7 (osd.7) 34 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T05:01:36.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:36 smithi133 bash[102858]: cluster 2023-10-09T05:01:35.259996+0000 osd.6 (osd.6) 32 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T05:01:36.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:36 smithi133 bash[102858]: cluster 2023-10-09T05:01:35.393795+0000 osd.5 (osd.5) 56 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T05:01:36.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:36 smithi133 bash[102858]: cluster 2023-10-09T05:01:35.393856+0000 osd.4 (osd.4) 40 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T05:01:36.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:36 smithi133 bash[102858]: cluster 2023-10-09T05:01:35.551165+0000 mgr.y (mgr.25277) 375 : cluster 0 pgmap v317: 89 pgs: 1 active+recovering+undersized+remapped, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8735/72 objects degraded (12131.944%); 0 B/s, 424 objects/s recovering 2023-10-09T05:01:37.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:37 smithi194 bash[92666]: cluster 2023-10-09T05:01:36.129893+0000 osd.6 (osd.6) 33 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T05:01:37.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:37 smithi194 bash[92666]: cluster 2023-10-09T05:01:36.134816+0000 mon.a (mon.0) 560 : cluster 0 osdmap e116: 8 total, 8 up, 8 in 2023-10-09T05:01:37.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:37 smithi194 bash[92666]: cluster 2023-10-09T05:01:36.135313+0000 osd.7 (osd.7) 35 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T05:01:37.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:37 smithi194 bash[92666]: cluster 2023-10-09T05:01:36.139033+0000 osd.7 (osd.7) 36 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T05:01:37.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:37 smithi194 bash[92666]: cluster 2023-10-09T05:01:36.150631+0000 osd.3 (osd.3) 48 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T05:01:37.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:37 smithi194 bash[92666]: audit 2023-10-09T05:01:36.606716+0000 mon.c (mon.2) 41 : audit 0 from='client.? 172.21.15.133:0/2778136666' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:37.418 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:37 smithi133 bash[102858]: cluster 2023-10-09T05:01:36.129893+0000 osd.6 (osd.6) 33 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T05:01:37.418 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:37 smithi133 bash[102858]: cluster 2023-10-09T05:01:36.134816+0000 mon.a (mon.0) 560 : cluster 0 osdmap e116: 8 total, 8 up, 8 in 2023-10-09T05:01:37.418 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:37 smithi133 bash[102858]: cluster 2023-10-09T05:01:36.135313+0000 osd.7 (osd.7) 35 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T05:01:37.418 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:37 smithi133 bash[102858]: cluster 2023-10-09T05:01:36.139033+0000 osd.7 (osd.7) 36 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T05:01:37.418 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:37 smithi133 bash[102858]: cluster 2023-10-09T05:01:36.150631+0000 osd.3 (osd.3) 48 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T05:01:37.418 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:37 smithi133 bash[102858]: audit 2023-10-09T05:01:36.606716+0000 mon.c (mon.2) 41 : audit 0 from='client.? 172.21.15.133:0/2778136666' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:37.418 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:37 smithi133 bash[105589]: cluster 2023-10-09T05:01:36.129893+0000 osd.6 (osd.6) 33 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T05:01:37.419 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:37 smithi133 bash[105589]: cluster 2023-10-09T05:01:36.134816+0000 mon.a (mon.0) 560 : cluster 0 osdmap e116: 8 total, 8 up, 8 in 2023-10-09T05:01:37.419 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:37 smithi133 bash[105589]: cluster 2023-10-09T05:01:36.135313+0000 osd.7 (osd.7) 35 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T05:01:37.419 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:37 smithi133 bash[105589]: cluster 2023-10-09T05:01:36.139033+0000 osd.7 (osd.7) 36 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T05:01:37.419 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:37 smithi133 bash[105589]: cluster 2023-10-09T05:01:36.150631+0000 osd.3 (osd.3) 48 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T05:01:37.419 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:37 smithi133 bash[105589]: audit 2023-10-09T05:01:36.606716+0000 mon.c (mon.2) 41 : audit 0 from='client.? 172.21.15.133:0/2778136666' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:38.106 INFO:teuthology.orchestra.run.smithi133.stderr:pool 'unique_pool_2' removed 2023-10-09T05:01:38.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:38 smithi133 bash[102858]: cluster 2023-10-09T05:01:37.551701+0000 mgr.y (mgr.25277) 376 : cluster 0 pgmap v319: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8735/72 objects degraded (12131.944%); 0 B/s, 449 objects/s recovering 2023-10-09T05:01:38.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:38 smithi133 bash[102858]: audit 2023-10-09T05:01:37.909084+0000 mon.a (mon.0) 561 : audit 1 from='client.? 172.21.15.133:0/967431103' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:01:38.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:38 smithi133 bash[102858]: cluster 2023-10-09T05:01:38.075111+0000 mon.a (mon.0) 562 : cluster 3 Health check update: Degraded data redundancy: 8735/72 objects degraded (12131.944%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:38.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:38 smithi133 bash[102858]: audit 2023-10-09T05:01:38.083684+0000 mon.a (mon.0) 563 : audit 1 from='client.? 172.21.15.133:0/967431103' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:01:38.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:38 smithi133 bash[102858]: cluster 2023-10-09T05:01:38.083772+0000 mon.a (mon.0) 564 : cluster 0 osdmap e117: 8 total, 8 up, 8 in 2023-10-09T05:01:38.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:38 smithi133 bash[105589]: cluster 2023-10-09T05:01:37.551701+0000 mgr.y (mgr.25277) 376 : cluster 0 pgmap v319: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8735/72 objects degraded (12131.944%); 0 B/s, 449 objects/s recovering 2023-10-09T05:01:38.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:38 smithi133 bash[105589]: audit 2023-10-09T05:01:37.909084+0000 mon.a (mon.0) 561 : audit 1 from='client.? 172.21.15.133:0/967431103' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:01:38.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:38 smithi133 bash[105589]: cluster 2023-10-09T05:01:38.075111+0000 mon.a (mon.0) 562 : cluster 3 Health check update: Degraded data redundancy: 8735/72 objects degraded (12131.944%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:38.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:38 smithi133 bash[105589]: audit 2023-10-09T05:01:38.083684+0000 mon.a (mon.0) 563 : audit 1 from='client.? 172.21.15.133:0/967431103' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:01:38.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:38 smithi133 bash[105589]: cluster 2023-10-09T05:01:38.083772+0000 mon.a (mon.0) 564 : cluster 0 osdmap e117: 8 total, 8 up, 8 in 2023-10-09T05:01:38.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:38 smithi194 bash[92666]: cluster 2023-10-09T05:01:37.551701+0000 mgr.y (mgr.25277) 376 : cluster 0 pgmap v319: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8735/72 objects degraded (12131.944%); 0 B/s, 449 objects/s recovering 2023-10-09T05:01:38.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:38 smithi194 bash[92666]: audit 2023-10-09T05:01:37.909084+0000 mon.a (mon.0) 561 : audit 1 from='client.? 172.21.15.133:0/967431103' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:01:38.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:38 smithi194 bash[92666]: cluster 2023-10-09T05:01:38.075111+0000 mon.a (mon.0) 562 : cluster 3 Health check update: Degraded data redundancy: 8735/72 objects degraded (12131.944%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:38.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:38 smithi194 bash[92666]: audit 2023-10-09T05:01:38.083684+0000 mon.a (mon.0) 563 : audit 1 from='client.? 172.21.15.133:0/967431103' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:01:38.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:38 smithi194 bash[92666]: cluster 2023-10-09T05:01:38.083772+0000 mon.a (mon.0) 564 : cluster 0 osdmap e117: 8 total, 8 up, 8 in 2023-10-09T05:01:39.074 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T05:01:39.075 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T05:01:39.075 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2023-10-09T05:01:39.075 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_3 16 2023-10-09T05:01:39.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:39 smithi133 bash[102858]: cluster 2023-10-09T05:01:38.079297+0000 osd.3 (osd.3) 49 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T05:01:39.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:39 smithi133 bash[102858]: cluster 2023-10-09T05:01:38.219286+0000 osd.4 (osd.4) 41 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T05:01:39.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:39 smithi133 bash[102858]: cluster 2023-10-09T05:01:38.219380+0000 osd.5 (osd.5) 57 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T05:01:39.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:39 smithi133 bash[102858]: cluster 2023-10-09T05:01:38.219440+0000 osd.6 (osd.6) 34 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T05:01:39.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:39 smithi133 bash[102858]: cluster 2023-10-09T05:01:38.222152+0000 osd.7 (osd.7) 37 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T05:01:39.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:39 smithi133 bash[102858]: audit 2023-10-09T05:01:38.961452+0000 mon.a (mon.0) 565 : audit 0 from='client.? 172.21.15.133:0/2803344997' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:39.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:39 smithi133 bash[105589]: cluster 2023-10-09T05:01:38.079297+0000 osd.3 (osd.3) 49 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T05:01:39.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:39 smithi133 bash[105589]: cluster 2023-10-09T05:01:38.219286+0000 osd.4 (osd.4) 41 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T05:01:39.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:39 smithi133 bash[105589]: cluster 2023-10-09T05:01:38.219380+0000 osd.5 (osd.5) 57 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T05:01:39.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:39 smithi133 bash[105589]: cluster 2023-10-09T05:01:38.219440+0000 osd.6 (osd.6) 34 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T05:01:39.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:39 smithi133 bash[105589]: cluster 2023-10-09T05:01:38.222152+0000 osd.7 (osd.7) 37 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T05:01:39.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:39 smithi133 bash[105589]: audit 2023-10-09T05:01:38.961452+0000 mon.a (mon.0) 565 : audit 0 from='client.? 172.21.15.133:0/2803344997' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:39.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:39 smithi194 bash[92666]: cluster 2023-10-09T05:01:38.079297+0000 osd.3 (osd.3) 49 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T05:01:39.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:39 smithi194 bash[92666]: cluster 2023-10-09T05:01:38.219286+0000 osd.4 (osd.4) 41 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T05:01:39.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:39 smithi194 bash[92666]: cluster 2023-10-09T05:01:38.219380+0000 osd.5 (osd.5) 57 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T05:01:39.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:39 smithi194 bash[92666]: cluster 2023-10-09T05:01:38.219440+0000 osd.6 (osd.6) 34 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T05:01:39.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:39 smithi194 bash[92666]: cluster 2023-10-09T05:01:38.222152+0000 osd.7 (osd.7) 37 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T05:01:39.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:39 smithi194 bash[92666]: audit 2023-10-09T05:01:38.961452+0000 mon.a (mon.0) 565 : audit 0 from='client.? 172.21.15.133:0/2803344997' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:40.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:40 smithi133 bash[102858]: cluster 2023-10-09T05:01:39.552291+0000 mgr.y (mgr.25277) 377 : cluster 0 pgmap v321: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:01:40.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:40 smithi133 bash[102858]: audit 2023-10-09T05:01:40.057548+0000 mon.a (mon.0) 566 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:01:40.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:40 smithi133 bash[105589]: cluster 2023-10-09T05:01:39.552291+0000 mgr.y (mgr.25277) 377 : cluster 0 pgmap v321: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:01:40.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:40 smithi133 bash[105589]: audit 2023-10-09T05:01:40.057548+0000 mon.a (mon.0) 566 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:01:40.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:40 smithi194 bash[92666]: cluster 2023-10-09T05:01:39.552291+0000 mgr.y (mgr.25277) 377 : cluster 0 pgmap v321: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:01:40.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:40 smithi194 bash[92666]: audit 2023-10-09T05:01:40.057548+0000 mon.a (mon.0) 566 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:01:41.468 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:41 smithi133 bash[105589]: audit 2023-10-09T05:01:40.057929+0000 mgr.y (mgr.25277) 378 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:01:41.469 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:41 smithi133 bash[105589]: cephadm 2023-10-09T05:01:40.060128+0000 mgr.y (mgr.25277) 379 : cephadm 1 Upgrade: osd.3 is safe to restart 2023-10-09T05:01:41.469 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:41 smithi133 bash[105589]: cluster 2023-10-09T05:01:40.138775+0000 mon.a (mon.0) 567 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8735/72 objects degraded (12131.944%), 6 pgs degraded, 4 pgs undersized) 2023-10-09T05:01:41.469 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:41 smithi133 bash[105589]: cluster 2023-10-09T05:01:40.138832+0000 mon.a (mon.0) 568 : cluster 1 Cluster is now healthy 2023-10-09T05:01:41.469 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:41 smithi133 bash[102858]: audit 2023-10-09T05:01:40.057929+0000 mgr.y (mgr.25277) 378 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:01:41.469 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:41 smithi133 bash[102858]: cephadm 2023-10-09T05:01:40.060128+0000 mgr.y (mgr.25277) 379 : cephadm 1 Upgrade: osd.3 is safe to restart 2023-10-09T05:01:41.469 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:41 smithi133 bash[102858]: cluster 2023-10-09T05:01:40.138775+0000 mon.a (mon.0) 567 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8735/72 objects degraded (12131.944%), 6 pgs degraded, 4 pgs undersized) 2023-10-09T05:01:41.469 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:41 smithi133 bash[102858]: cluster 2023-10-09T05:01:40.138832+0000 mon.a (mon.0) 568 : cluster 1 Cluster is now healthy 2023-10-09T05:01:41.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:41 smithi194 bash[92666]: audit 2023-10-09T05:01:40.057929+0000 mgr.y (mgr.25277) 378 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:01:41.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:41 smithi194 bash[92666]: cephadm 2023-10-09T05:01:40.060128+0000 mgr.y (mgr.25277) 379 : cephadm 1 Upgrade: osd.3 is safe to restart 2023-10-09T05:01:41.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:41 smithi194 bash[92666]: cluster 2023-10-09T05:01:40.138775+0000 mon.a (mon.0) 567 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8735/72 objects degraded (12131.944%), 6 pgs degraded, 4 pgs undersized) 2023-10-09T05:01:41.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:41 smithi194 bash[92666]: cluster 2023-10-09T05:01:40.138832+0000 mon.a (mon.0) 568 : cluster 1 Cluster is now healthy 2023-10-09T05:01:42.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:42 smithi133 bash[105589]: audit 2023-10-09T05:01:41.316236+0000 mon.a (mon.0) 569 : audit 0 from='client.? 172.21.15.133:0/1583556873' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:42.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:42 smithi133 bash[105589]: cluster 2023-10-09T05:01:41.553426+0000 mgr.y (mgr.25277) 380 : cluster 0 pgmap v322: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:01:42.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:42 smithi133 bash[102858]: audit 2023-10-09T05:01:41.316236+0000 mon.a (mon.0) 569 : audit 0 from='client.? 172.21.15.133:0/1583556873' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:42.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:42 smithi133 bash[102858]: cluster 2023-10-09T05:01:41.553426+0000 mgr.y (mgr.25277) 380 : cluster 0 pgmap v322: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:01:42.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:42 smithi194 bash[92666]: audit 2023-10-09T05:01:41.316236+0000 mon.a (mon.0) 569 : audit 0 from='client.? 172.21.15.133:0/1583556873' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:42.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:42 smithi194 bash[92666]: cluster 2023-10-09T05:01:41.553426+0000 mgr.y (mgr.25277) 380 : cluster 0 pgmap v322: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:01:43.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:43 smithi133 bash[105589]: cephadm 2023-10-09T05:01:42.480046+0000 mgr.y (mgr.25277) 381 : cephadm 1 Upgrade: Updating osd.3 2023-10-09T05:01:43.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:43 smithi133 bash[105589]: audit 2023-10-09T05:01:42.489226+0000 mon.a (mon.0) 570 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:01:43.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:43 smithi133 bash[105589]: audit 2023-10-09T05:01:42.490925+0000 mon.a (mon.0) 571 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T05:01:43.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:43 smithi133 bash[105589]: audit 2023-10-09T05:01:42.492102+0000 mon.a (mon.0) 572 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:01:43.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:43 smithi133 bash[105589]: cephadm 2023-10-09T05:01:42.495556+0000 mgr.y (mgr.25277) 382 : cephadm 1 Deploying daemon osd.3 on smithi133 2023-10-09T05:01:43.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:43 smithi133 bash[105589]: audit 2023-10-09T05:01:43.430837+0000 mon.a (mon.0) 573 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:01:43.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:43 smithi133 bash[105589]: audit 2023-10-09T05:01:43.432088+0000 mon.a (mon.0) 574 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:43.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:43 smithi133 bash[102858]: cephadm 2023-10-09T05:01:42.480046+0000 mgr.y (mgr.25277) 381 : cephadm 1 Upgrade: Updating osd.3 2023-10-09T05:01:43.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:43 smithi133 bash[102858]: audit 2023-10-09T05:01:42.489226+0000 mon.a (mon.0) 570 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:01:43.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:43 smithi133 bash[102858]: audit 2023-10-09T05:01:42.490925+0000 mon.a (mon.0) 571 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T05:01:43.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:43 smithi133 bash[102858]: audit 2023-10-09T05:01:42.492102+0000 mon.a (mon.0) 572 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:01:43.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:43 smithi133 bash[102858]: cephadm 2023-10-09T05:01:42.495556+0000 mgr.y (mgr.25277) 382 : cephadm 1 Deploying daemon osd.3 on smithi133 2023-10-09T05:01:43.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:43 smithi133 bash[102858]: audit 2023-10-09T05:01:43.430837+0000 mon.a (mon.0) 573 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:01:43.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:43 smithi133 bash[102858]: audit 2023-10-09T05:01:43.432088+0000 mon.a (mon.0) 574 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:43.885 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:43.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:43 smithi194 bash[92666]: cephadm 2023-10-09T05:01:42.480046+0000 mgr.y (mgr.25277) 381 : cephadm 1 Upgrade: Updating osd.3 2023-10-09T05:01:43.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:43 smithi194 bash[92666]: audit 2023-10-09T05:01:42.489226+0000 mon.a (mon.0) 570 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:01:43.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:43 smithi194 bash[92666]: audit 2023-10-09T05:01:42.490925+0000 mon.a (mon.0) 571 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T05:01:43.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:43 smithi194 bash[92666]: audit 2023-10-09T05:01:42.492102+0000 mon.a (mon.0) 572 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:01:43.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:43 smithi194 bash[92666]: cephadm 2023-10-09T05:01:42.495556+0000 mgr.y (mgr.25277) 382 : cephadm 1 Deploying daemon osd.3 on smithi133 2023-10-09T05:01:43.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:43 smithi194 bash[92666]: audit 2023-10-09T05:01:43.430837+0000 mon.a (mon.0) 573 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:01:43.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:43 smithi194 bash[92666]: audit 2023-10-09T05:01:43.432088+0000 mon.a (mon.0) 574 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:44.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:01:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:01:44] "GET /metrics HTTP/1.1" 200 34003 "" "Prometheus/2.43.0" 2023-10-09T05:01:44.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:44 smithi133 bash[102858]: audit 2023-10-09T05:01:43.521743+0000 mon.a (mon.0) 575 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:01:44.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:44 smithi133 bash[102858]: cluster 2023-10-09T05:01:43.553981+0000 mgr.y (mgr.25277) 383 : cluster 0 pgmap v323: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:01:44.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:44 smithi133 bash[102858]: audit 2023-10-09T05:01:43.692330+0000 mon.c (mon.2) 42 : audit 0 from='client.? 172.21.15.133:0/3973504873' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:44.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:44 smithi133 bash[105589]: audit 2023-10-09T05:01:43.521743+0000 mon.a (mon.0) 575 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:01:44.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:44 smithi133 bash[105589]: cluster 2023-10-09T05:01:43.553981+0000 mgr.y (mgr.25277) 383 : cluster 0 pgmap v323: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:01:44.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:44 smithi133 bash[105589]: audit 2023-10-09T05:01:43.692330+0000 mon.c (mon.2) 42 : audit 0 from='client.? 172.21.15.133:0/3973504873' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:44.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:44 smithi194 bash[92666]: audit 2023-10-09T05:01:43.521743+0000 mon.a (mon.0) 575 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:01:44.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:44 smithi194 bash[92666]: cluster 2023-10-09T05:01:43.553981+0000 mgr.y (mgr.25277) 383 : cluster 0 pgmap v323: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:01:44.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:44 smithi194 bash[92666]: audit 2023-10-09T05:01:43.692330+0000 mon.c (mon.2) 42 : audit 0 from='client.? 172.21.15.133:0/3973504873' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:46.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:46 smithi133 bash[102858]: cluster 2023-10-09T05:01:45.554932+0000 mgr.y (mgr.25277) 384 : cluster 0 pgmap v324: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:01:46.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:46 smithi133 bash[102858]: audit 2023-10-09T05:01:46.064804+0000 mon.a (mon.0) 576 : audit 0 from='client.? 172.21.15.133:0/3032387678' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:46.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:46 smithi133 bash[105589]: cluster 2023-10-09T05:01:45.554932+0000 mgr.y (mgr.25277) 384 : cluster 0 pgmap v324: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:01:46.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:46 smithi133 bash[105589]: audit 2023-10-09T05:01:46.064804+0000 mon.a (mon.0) 576 : audit 0 from='client.? 172.21.15.133:0/3032387678' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:46.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:46 smithi194 bash[92666]: cluster 2023-10-09T05:01:45.554932+0000 mgr.y (mgr.25277) 384 : cluster 0 pgmap v324: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:01:46.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:46 smithi194 bash[92666]: audit 2023-10-09T05:01:46.064804+0000 mon.a (mon.0) 576 : audit 0 from='client.? 172.21.15.133:0/3032387678' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:48.765 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:48 smithi133 bash[102858]: cluster 2023-10-09T05:01:47.555650+0000 mgr.y (mgr.25277) 385 : cluster 0 pgmap v325: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:01:48.765 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:48 smithi133 bash[102858]: audit 2023-10-09T05:01:48.413174+0000 mon.a (mon.0) 577 : audit 0 from='client.? 172.21.15.133:0/1133188450' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:48.765 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:48 smithi133 bash[105589]: cluster 2023-10-09T05:01:47.555650+0000 mgr.y (mgr.25277) 385 : cluster 0 pgmap v325: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:01:48.765 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:48 smithi133 bash[105589]: audit 2023-10-09T05:01:48.413174+0000 mon.a (mon.0) 577 : audit 0 from='client.? 172.21.15.133:0/1133188450' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:48.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:48 smithi194 bash[92666]: cluster 2023-10-09T05:01:47.555650+0000 mgr.y (mgr.25277) 385 : cluster 0 pgmap v325: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:01:48.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:48 smithi194 bash[92666]: audit 2023-10-09T05:01:48.413174+0000 mon.a (mon.0) 577 : audit 0 from='client.? 172.21.15.133:0/1133188450' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:50.299 INFO:journalctl@ceph.osd.3.smithi133.stdout:Oct 09 05:01:50 smithi133 systemd[1]: Stopping Ceph osd.3 for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T05:01:50.643 INFO:journalctl@ceph.osd.3.smithi133.stdout:Oct 09 05:01:50 smithi133 bash[66026]: debug 2023-10-09T05:01:50.367+0000 7fd81358e700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T05:01:50.643 INFO:journalctl@ceph.osd.3.smithi133.stdout:Oct 09 05:01:50 smithi133 bash[66026]: debug 2023-10-09T05:01:50.367+0000 7fd81358e700 -1 osd.3 117 *** Got signal Terminated *** 2023-10-09T05:01:50.643 INFO:journalctl@ceph.osd.3.smithi133.stdout:Oct 09 05:01:50 smithi133 bash[66026]: debug 2023-10-09T05:01:50.367+0000 7fd81358e700 -1 osd.3 117 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:01:50.716 INFO:teuthology.orchestra.run.smithi133.stderr:pool 'unique_pool_3' created 2023-10-09T05:01:50.955 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:50 smithi133 bash[105589]: cluster 2023-10-09T05:01:49.556130+0000 mgr.y (mgr.25277) 386 : cluster 0 pgmap v326: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:01:50.955 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:50 smithi133 bash[102858]: cluster 2023-10-09T05:01:49.556130+0000 mgr.y (mgr.25277) 386 : cluster 0 pgmap v326: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:01:51.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:50 smithi194 bash[92666]: cluster 2023-10-09T05:01:49.556130+0000 mgr.y (mgr.25277) 386 : cluster 0 pgmap v326: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:01:51.772 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:51 smithi133 bash[102858]: audit 2023-10-09T05:01:50.189609+0000 mon.a (mon.0) 578 : audit 1 from='client.? 172.21.15.133:0/2697352323' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T05:01:51.772 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:51 smithi133 bash[102858]: cluster 2023-10-09T05:01:50.639482+0000 mon.a (mon.0) 579 : cluster 1 osd.3 marked itself down and dead 2023-10-09T05:01:51.772 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:51 smithi133 bash[102858]: cluster 2023-10-09T05:01:50.690037+0000 mon.a (mon.0) 580 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:01:51.773 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:51 smithi133 bash[102858]: cluster 2023-10-09T05:01:50.694473+0000 osd.6 (osd.6) 35 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T05:01:51.773 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:51 smithi133 bash[102858]: cluster 2023-10-09T05:01:50.695124+0000 osd.5 (osd.5) 58 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T05:01:51.773 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:51 smithi133 bash[102858]: audit 2023-10-09T05:01:50.698372+0000 mon.a (mon.0) 581 : audit 1 from='client.? 172.21.15.133:0/2697352323' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T05:01:51.773 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:51 smithi133 bash[102858]: cluster 2023-10-09T05:01:50.698475+0000 mon.a (mon.0) 582 : cluster 0 osdmap e118: 8 total, 7 up, 8 in 2023-10-09T05:01:51.773 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:51 smithi133 bash[102858]: cluster 2023-10-09T05:01:50.698843+0000 osd.6 (osd.6) 36 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T05:01:51.773 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:51 smithi133 bash[102858]: cluster 2023-10-09T05:01:50.700021+0000 osd.4 (osd.4) 42 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T05:01:51.773 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:51 smithi133 bash[102858]: cluster 2023-10-09T05:01:50.700090+0000 osd.5 (osd.5) 59 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T05:01:51.773 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:51 smithi133 bash[102858]: cluster 2023-10-09T05:01:50.702263+0000 osd.7 (osd.7) 38 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T05:01:51.773 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:51 smithi133 bash[102858]: cluster 2023-10-09T05:01:50.703084+0000 osd.4 (osd.4) 43 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T05:01:51.774 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:51 smithi133 bash[102858]: cluster 2023-10-09T05:01:50.703612+0000 osd.5 (osd.5) 60 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T05:01:51.774 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:51 smithi133 bash[102858]: audit 2023-10-09T05:01:50.894815+0000 mon.a (mon.0) 583 : audit 0 from='client.? 172.21.15.133:0/1352108056' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:51.775 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:51 smithi133 bash[105589]: audit 2023-10-09T05:01:50.189609+0000 mon.a (mon.0) 578 : audit 1 from='client.? 172.21.15.133:0/2697352323' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T05:01:51.775 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:51 smithi133 bash[105589]: cluster 2023-10-09T05:01:50.639482+0000 mon.a (mon.0) 579 : cluster 1 osd.3 marked itself down and dead 2023-10-09T05:01:51.775 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:51 smithi133 bash[105589]: cluster 2023-10-09T05:01:50.690037+0000 mon.a (mon.0) 580 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:01:51.775 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:51 smithi133 bash[105589]: cluster 2023-10-09T05:01:50.694473+0000 osd.6 (osd.6) 35 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T05:01:51.775 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:51 smithi133 bash[105589]: cluster 2023-10-09T05:01:50.695124+0000 osd.5 (osd.5) 58 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T05:01:51.775 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:51 smithi133 bash[105589]: audit 2023-10-09T05:01:50.698372+0000 mon.a (mon.0) 581 : audit 1 from='client.? 172.21.15.133:0/2697352323' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T05:01:51.775 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:51 smithi133 bash[105589]: cluster 2023-10-09T05:01:50.698475+0000 mon.a (mon.0) 582 : cluster 0 osdmap e118: 8 total, 7 up, 8 in 2023-10-09T05:01:51.775 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:51 smithi133 bash[105589]: cluster 2023-10-09T05:01:50.698843+0000 osd.6 (osd.6) 36 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T05:01:51.775 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:51 smithi133 bash[105589]: cluster 2023-10-09T05:01:50.700021+0000 osd.4 (osd.4) 42 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T05:01:51.775 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:51 smithi133 bash[105589]: cluster 2023-10-09T05:01:50.700090+0000 osd.5 (osd.5) 59 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T05:01:51.776 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:51 smithi133 bash[105589]: cluster 2023-10-09T05:01:50.702263+0000 osd.7 (osd.7) 38 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T05:01:51.776 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:51 smithi133 bash[105589]: cluster 2023-10-09T05:01:50.703084+0000 osd.4 (osd.4) 43 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T05:01:51.776 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:51 smithi133 bash[105589]: cluster 2023-10-09T05:01:50.703612+0000 osd.5 (osd.5) 60 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T05:01:51.776 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:51 smithi133 bash[105589]: audit 2023-10-09T05:01:50.894815+0000 mon.a (mon.0) 583 : audit 0 from='client.? 172.21.15.133:0/1352108056' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:51.778 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2023-10-09T05:01:52.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:51 smithi194 bash[92666]: audit 2023-10-09T05:01:50.189609+0000 mon.a (mon.0) 578 : audit 1 from='client.? 172.21.15.133:0/2697352323' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T05:01:52.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:51 smithi194 bash[92666]: cluster 2023-10-09T05:01:50.639482+0000 mon.a (mon.0) 579 : cluster 1 osd.3 marked itself down and dead 2023-10-09T05:01:52.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:51 smithi194 bash[92666]: cluster 2023-10-09T05:01:50.690037+0000 mon.a (mon.0) 580 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:01:52.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:51 smithi194 bash[92666]: cluster 2023-10-09T05:01:50.694473+0000 osd.6 (osd.6) 35 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T05:01:52.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:51 smithi194 bash[92666]: cluster 2023-10-09T05:01:50.695124+0000 osd.5 (osd.5) 58 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T05:01:52.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:51 smithi194 bash[92666]: audit 2023-10-09T05:01:50.698372+0000 mon.a (mon.0) 581 : audit 1 from='client.? 172.21.15.133:0/2697352323' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T05:01:52.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:51 smithi194 bash[92666]: cluster 2023-10-09T05:01:50.698475+0000 mon.a (mon.0) 582 : cluster 0 osdmap e118: 8 total, 7 up, 8 in 2023-10-09T05:01:52.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:51 smithi194 bash[92666]: cluster 2023-10-09T05:01:50.698843+0000 osd.6 (osd.6) 36 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T05:01:52.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:51 smithi194 bash[92666]: cluster 2023-10-09T05:01:50.700021+0000 osd.4 (osd.4) 42 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T05:01:52.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:51 smithi194 bash[92666]: cluster 2023-10-09T05:01:50.700090+0000 osd.5 (osd.5) 59 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T05:01:52.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:51 smithi194 bash[92666]: cluster 2023-10-09T05:01:50.702263+0000 osd.7 (osd.7) 38 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T05:01:52.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:51 smithi194 bash[92666]: cluster 2023-10-09T05:01:50.703084+0000 osd.4 (osd.4) 43 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T05:01:52.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:51 smithi194 bash[92666]: cluster 2023-10-09T05:01:50.703612+0000 osd.5 (osd.5) 60 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T05:01:52.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:51 smithi194 bash[92666]: audit 2023-10-09T05:01:50.894815+0000 mon.a (mon.0) 583 : audit 0 from='client.? 172.21.15.133:0/1352108056' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:52.343 INFO:journalctl@ceph.osd.3.smithi133.stdout:Oct 09 05:01:52 smithi133 bash[134830]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd-3 2023-10-09T05:01:52.709 INFO:journalctl@ceph.osd.3.smithi133.stdout:Oct 09 05:01:52 smithi133 bash[134971]: Error response from daemon: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd-3 2023-10-09T05:01:53.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:52 smithi133 bash[105589]: cluster 2023-10-09T05:01:51.556867+0000 mgr.y (mgr.25277) 387 : cluster 0 pgmap v328: 89 pgs: 16 unknown, 13 stale+active+clean, 60 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:01:53.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:52 smithi133 bash[105589]: cluster 2023-10-09T05:01:51.707308+0000 osd.7 (osd.7) 39 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T05:01:53.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:52 smithi133 bash[105589]: cluster 2023-10-09T05:01:51.708012+0000 mon.a (mon.0) 584 : cluster 0 osdmap e119: 8 total, 7 up, 8 in 2023-10-09T05:01:53.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:52 smithi133 bash[105589]: cluster 2023-10-09T05:01:51.709386+0000 osd.6 (osd.6) 37 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T05:01:53.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:52 smithi133 bash[105589]: cluster 2023-10-09T05:01:51.709490+0000 osd.4 (osd.4) 44 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T05:01:53.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:52 smithi133 bash[105589]: cluster 2023-10-09T05:01:51.710635+0000 osd.5 (osd.5) 61 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T05:01:53.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:52 smithi133 bash[105589]: cluster 2023-10-09T05:01:51.710636+0000 osd.7 (osd.7) 40 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T05:01:53.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:52 smithi133 bash[105589]: cluster 2023-10-09T05:01:51.714179+0000 osd.4 (osd.4) 45 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T05:01:53.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:52 smithi133 bash[105589]: cluster 2023-10-09T05:01:51.716502+0000 osd.5 (osd.5) 62 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T05:01:53.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:52 smithi133 bash[105589]: cluster 2023-10-09T05:01:51.716909+0000 osd.4 (osd.4) 46 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T05:01:53.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:52 smithi133 bash[102858]: cluster 2023-10-09T05:01:51.556867+0000 mgr.y (mgr.25277) 387 : cluster 0 pgmap v328: 89 pgs: 16 unknown, 13 stale+active+clean, 60 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:01:53.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:52 smithi133 bash[102858]: cluster 2023-10-09T05:01:51.707308+0000 osd.7 (osd.7) 39 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T05:01:53.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:52 smithi133 bash[102858]: cluster 2023-10-09T05:01:51.708012+0000 mon.a (mon.0) 584 : cluster 0 osdmap e119: 8 total, 7 up, 8 in 2023-10-09T05:01:53.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:52 smithi133 bash[102858]: cluster 2023-10-09T05:01:51.709386+0000 osd.6 (osd.6) 37 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T05:01:53.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:52 smithi133 bash[102858]: cluster 2023-10-09T05:01:51.709490+0000 osd.4 (osd.4) 44 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T05:01:53.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:52 smithi133 bash[102858]: cluster 2023-10-09T05:01:51.710635+0000 osd.5 (osd.5) 61 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T05:01:53.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:52 smithi133 bash[102858]: cluster 2023-10-09T05:01:51.710636+0000 osd.7 (osd.7) 40 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T05:01:53.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:52 smithi133 bash[102858]: cluster 2023-10-09T05:01:51.714179+0000 osd.4 (osd.4) 45 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T05:01:53.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:52 smithi133 bash[102858]: cluster 2023-10-09T05:01:51.716502+0000 osd.5 (osd.5) 62 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T05:01:53.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:52 smithi133 bash[102858]: cluster 2023-10-09T05:01:51.716909+0000 osd.4 (osd.4) 46 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T05:01:53.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:52 smithi194 bash[92666]: cluster 2023-10-09T05:01:51.556867+0000 mgr.y (mgr.25277) 387 : cluster 0 pgmap v328: 89 pgs: 16 unknown, 13 stale+active+clean, 60 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:01:53.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:52 smithi194 bash[92666]: cluster 2023-10-09T05:01:51.707308+0000 osd.7 (osd.7) 39 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T05:01:53.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:52 smithi194 bash[92666]: cluster 2023-10-09T05:01:51.708012+0000 mon.a (mon.0) 584 : cluster 0 osdmap e119: 8 total, 7 up, 8 in 2023-10-09T05:01:53.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:52 smithi194 bash[92666]: cluster 2023-10-09T05:01:51.709386+0000 osd.6 (osd.6) 37 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T05:01:53.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:52 smithi194 bash[92666]: cluster 2023-10-09T05:01:51.709490+0000 osd.4 (osd.4) 44 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T05:01:53.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:52 smithi194 bash[92666]: cluster 2023-10-09T05:01:51.710635+0000 osd.5 (osd.5) 61 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T05:01:53.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:52 smithi194 bash[92666]: cluster 2023-10-09T05:01:51.710636+0000 osd.7 (osd.7) 40 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T05:01:53.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:52 smithi194 bash[92666]: cluster 2023-10-09T05:01:51.714179+0000 osd.4 (osd.4) 45 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T05:01:53.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:52 smithi194 bash[92666]: cluster 2023-10-09T05:01:51.716502+0000 osd.5 (osd.5) 62 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T05:01:53.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:52 smithi194 bash[92666]: cluster 2023-10-09T05:01:51.716909+0000 osd.4 (osd.4) 46 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T05:01:54.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:53 smithi133 bash[105589]: cluster 2023-10-09T05:01:52.717833+0000 osd.4 (osd.4) 47 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T05:01:54.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:53 smithi133 bash[105589]: cluster 2023-10-09T05:01:52.720498+0000 mon.a (mon.0) 585 : cluster 0 osdmap e120: 8 total, 7 up, 8 in 2023-10-09T05:01:54.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:53 smithi133 bash[105589]: audit 2023-10-09T05:01:53.262039+0000 mon.c (mon.2) 43 : audit 0 from='client.? 172.21.15.133:0/3843848939' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:54.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:53 smithi133 bash[105589]: cluster 2023-10-09T05:01:53.322521+0000 osd.7 (osd.7) 41 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T05:01:54.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:53 smithi133 bash[105589]: cluster 2023-10-09T05:01:53.322776+0000 osd.6 (osd.6) 38 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T05:01:54.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:53 smithi133 bash[102858]: cluster 2023-10-09T05:01:52.717833+0000 osd.4 (osd.4) 47 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T05:01:54.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:53 smithi133 bash[102858]: cluster 2023-10-09T05:01:52.720498+0000 mon.a (mon.0) 585 : cluster 0 osdmap e120: 8 total, 7 up, 8 in 2023-10-09T05:01:54.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:53 smithi133 bash[102858]: audit 2023-10-09T05:01:53.262039+0000 mon.c (mon.2) 43 : audit 0 from='client.? 172.21.15.133:0/3843848939' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:54.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:53 smithi133 bash[102858]: cluster 2023-10-09T05:01:53.322521+0000 osd.7 (osd.7) 41 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T05:01:54.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:53 smithi133 bash[102858]: cluster 2023-10-09T05:01:53.322776+0000 osd.6 (osd.6) 38 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T05:01:54.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:53 smithi194 bash[92666]: cluster 2023-10-09T05:01:52.717833+0000 osd.4 (osd.4) 47 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T05:01:54.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:53 smithi194 bash[92666]: cluster 2023-10-09T05:01:52.720498+0000 mon.a (mon.0) 585 : cluster 0 osdmap e120: 8 total, 7 up, 8 in 2023-10-09T05:01:54.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:53 smithi194 bash[92666]: audit 2023-10-09T05:01:53.262039+0000 mon.c (mon.2) 43 : audit 0 from='client.? 172.21.15.133:0/3843848939' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:54.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:53 smithi194 bash[92666]: cluster 2023-10-09T05:01:53.322521+0000 osd.7 (osd.7) 41 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T05:01:54.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:53 smithi194 bash[92666]: cluster 2023-10-09T05:01:53.322776+0000 osd.6 (osd.6) 38 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T05:01:54.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:01:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:01:54] "GET /metrics HTTP/1.1" 200 34003 "" "Prometheus/2.43.0" 2023-10-09T05:01:55.049 INFO:journalctl@ceph.osd.3.smithi133.stdout:Oct 09 05:01:54 smithi133 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@osd.3.service: Succeeded. 2023-10-09T05:01:55.050 INFO:journalctl@ceph.osd.3.smithi133.stdout:Oct 09 05:01:54 smithi133 systemd[1]: Stopped Ceph osd.3 for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T05:01:55.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:54 smithi133 bash[102858]: cluster 2023-10-09T05:01:53.322562+0000 osd.5 (osd.5) 63 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T05:01:55.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:54 smithi133 bash[102858]: cluster 2023-10-09T05:01:53.557475+0000 mgr.y (mgr.25277) 388 : cluster 0 pgmap v331: 89 pgs: 16 unknown, 13 stale+active+clean, 60 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:01:55.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:54 smithi133 bash[105589]: cluster 2023-10-09T05:01:53.322562+0000 osd.5 (osd.5) 63 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T05:01:55.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:54 smithi133 bash[105589]: cluster 2023-10-09T05:01:53.557475+0000 mgr.y (mgr.25277) 388 : cluster 0 pgmap v331: 89 pgs: 16 unknown, 13 stale+active+clean, 60 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:01:55.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:54 smithi194 bash[92666]: cluster 2023-10-09T05:01:53.322562+0000 osd.5 (osd.5) 63 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T05:01:56.429 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:54 smithi194 bash[92666]: cluster 2023-10-09T05:01:53.557475+0000 mgr.y (mgr.25277) 388 : cluster 0 pgmap v331: 89 pgs: 16 unknown, 13 stale+active+clean, 60 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:01:56.429 INFO:journalctl@ceph.osd.3.smithi133.stdout:Oct 09 05:01:55 smithi133 systemd[1]: Started Ceph osd.3 for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T05:01:56.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:56 smithi133 bash[102858]: audit 2023-10-09T05:01:55.228444+0000 mon.a (mon.0) 586 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:01:56.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:56 smithi133 bash[102858]: audit 2023-10-09T05:01:55.236697+0000 mon.a (mon.0) 587 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:01:56.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:56 smithi133 bash[102858]: audit 2023-10-09T05:01:55.287577+0000 mon.a (mon.0) 588 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:01:56.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:56 smithi133 bash[102858]: cluster 2023-10-09T05:01:55.558266+0000 mgr.y (mgr.25277) 389 : cluster 0 pgmap v332: 89 pgs: 3 active+undersized+degraded, 23 active+undersized, 5 unknown, 5 stale+active+clean, 53 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:01:56.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:56 smithi133 bash[102858]: audit 2023-10-09T05:01:55.624586+0000 mon.a (mon.0) 589 : audit 0 from='client.? 172.21.15.133:0/625355728' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:56.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:56 smithi133 bash[105589]: audit 2023-10-09T05:01:55.228444+0000 mon.a (mon.0) 586 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:01:56.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:56 smithi133 bash[105589]: audit 2023-10-09T05:01:55.236697+0000 mon.a (mon.0) 587 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:01:56.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:56 smithi133 bash[105589]: audit 2023-10-09T05:01:55.287577+0000 mon.a (mon.0) 588 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:01:56.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:56 smithi133 bash[105589]: cluster 2023-10-09T05:01:55.558266+0000 mgr.y (mgr.25277) 389 : cluster 0 pgmap v332: 89 pgs: 3 active+undersized+degraded, 23 active+undersized, 5 unknown, 5 stale+active+clean, 53 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:01:56.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:56 smithi133 bash[105589]: audit 2023-10-09T05:01:55.624586+0000 mon.a (mon.0) 589 : audit 0 from='client.? 172.21.15.133:0/625355728' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:56.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:56 smithi194 bash[92666]: audit 2023-10-09T05:01:55.228444+0000 mon.a (mon.0) 586 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:01:56.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:56 smithi194 bash[92666]: audit 2023-10-09T05:01:55.236697+0000 mon.a (mon.0) 587 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:01:56.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:56 smithi194 bash[92666]: audit 2023-10-09T05:01:55.287577+0000 mon.a (mon.0) 588 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:01:56.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:56 smithi194 bash[92666]: cluster 2023-10-09T05:01:55.558266+0000 mgr.y (mgr.25277) 389 : cluster 0 pgmap v332: 89 pgs: 3 active+undersized+degraded, 23 active+undersized, 5 unknown, 5 stale+active+clean, 53 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:01:56.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:56 smithi194 bash[92666]: audit 2023-10-09T05:01:55.624586+0000 mon.a (mon.0) 589 : audit 0 from='client.? 172.21.15.133:0/625355728' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:57.049 INFO:journalctl@ceph.osd.3.smithi133.stdout:Oct 09 05:01:56 smithi133 bash[135159]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T05:01:57.050 INFO:journalctl@ceph.osd.3.smithi133.stdout:Oct 09 05:01:56 smithi133 bash[135159]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-09T05:01:57.050 INFO:journalctl@ceph.osd.3.smithi133.stdout:Oct 09 05:01:56 smithi133 bash[135159]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2023-10-09T05:01:57.050 INFO:journalctl@ceph.osd.3.smithi133.stdout:Oct 09 05:01:56 smithi133 bash[135159]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2023-10-09T05:01:57.050 INFO:journalctl@ceph.osd.3.smithi133.stdout:Oct 09 05:01:56 smithi133 bash[135159]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2023-10-09T05:01:57.050 INFO:journalctl@ceph.osd.3.smithi133.stdout:Oct 09 05:01:56 smithi133 bash[135159]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-10-09T05:01:57.050 INFO:journalctl@ceph.osd.3.smithi133.stdout:Oct 09 05:01:56 smithi133 bash[135159]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-09T05:01:57.050 INFO:journalctl@ceph.osd.3.smithi133.stdout:Oct 09 05:01:56 smithi133 bash[135159]: --> ceph-volume lvm activate successful for osd ID: 3 2023-10-09T05:01:57.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:57 smithi133 bash[102858]: cluster 2023-10-09T05:01:56.232836+0000 mon.a (mon.0) 590 : cluster 3 Health check failed: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T05:01:57.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:57 smithi133 bash[105589]: cluster 2023-10-09T05:01:56.232836+0000 mon.a (mon.0) 590 : cluster 3 Health check failed: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T05:01:57.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:57 smithi194 bash[92666]: cluster 2023-10-09T05:01:56.232836+0000 mon.a (mon.0) 590 : cluster 3 Health check failed: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T05:01:58.082 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:58.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:58 smithi133 bash[105589]: cluster 2023-10-09T05:01:57.559130+0000 mgr.y (mgr.25277) 390 : cluster 0 pgmap v333: 89 pgs: 3 active+undersized+degraded, 34 active+undersized, 52 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:01:58.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:58 smithi133 bash[105589]: audit 2023-10-09T05:01:57.977568+0000 mon.c (mon.2) 44 : audit 0 from='client.? 172.21.15.133:0/137580815' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:58.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:58 smithi133 bash[102858]: cluster 2023-10-09T05:01:57.559130+0000 mgr.y (mgr.25277) 390 : cluster 0 pgmap v333: 89 pgs: 3 active+undersized+degraded, 34 active+undersized, 52 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:01:58.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:58 smithi133 bash[102858]: audit 2023-10-09T05:01:57.977568+0000 mon.c (mon.2) 44 : audit 0 from='client.? 172.21.15.133:0/137580815' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:58.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:58 smithi194 bash[92666]: cluster 2023-10-09T05:01:57.559130+0000 mgr.y (mgr.25277) 390 : cluster 0 pgmap v333: 89 pgs: 3 active+undersized+degraded, 34 active+undersized, 52 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:01:58.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:58 smithi194 bash[92666]: audit 2023-10-09T05:01:57.977568+0000 mon.c (mon.2) 44 : audit 0 from='client.? 172.21.15.133:0/137580815' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:01:59.710 INFO:journalctl@ceph.osd.3.smithi133.stdout:Oct 09 05:01:59 smithi133 bash[135470]: debug 2023-10-09T05:01:59.567+0000 7fc35809f700 1 -- 172.21.15.133:0/3751727306 <== mon.2 v2:172.21.15.133:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5562676164e0 con 0x556267671000 2023-10-09T05:01:59.711 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:59 smithi133 bash[105589]: audit 2023-10-09T05:01:58.429731+0000 mon.a (mon.0) 591 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:01:59.711 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:01:59 smithi133 bash[105589]: audit 2023-10-09T05:01:58.430934+0000 mon.a (mon.0) 592 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:59.711 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:59 smithi133 bash[102858]: audit 2023-10-09T05:01:58.429731+0000 mon.a (mon.0) 591 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:01:59.711 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:01:59 smithi133 bash[102858]: audit 2023-10-09T05:01:58.430934+0000 mon.a (mon.0) 592 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:59 smithi194 bash[92666]: audit 2023-10-09T05:01:58.429731+0000 mon.a (mon.0) 591 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:01:59.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:01:59 smithi194 bash[92666]: audit 2023-10-09T05:01:58.430934+0000 mon.a (mon.0) 592 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:00.299 INFO:journalctl@ceph.osd.3.smithi133.stdout:Oct 09 05:02:00 smithi133 bash[135470]: debug 2023-10-09T05:02:00.111+0000 7fc361225640 -1 Falling back to public interface 2023-10-09T05:02:00.441 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:00 smithi133 bash[102858]: cluster 2023-10-09T05:01:59.559805+0000 mgr.y (mgr.25277) 391 : cluster 0 pgmap v334: 89 pgs: 3 active+undersized+degraded, 34 active+undersized, 52 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:02:00.441 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:00 smithi133 bash[102858]: audit 2023-10-09T05:02:00.348103+0000 mon.c (mon.2) 45 : audit 0 from='client.? 172.21.15.133:0/241824892' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:02:00.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:00 smithi133 bash[105589]: cluster 2023-10-09T05:01:59.559805+0000 mgr.y (mgr.25277) 391 : cluster 0 pgmap v334: 89 pgs: 3 active+undersized+degraded, 34 active+undersized, 52 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:02:00.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:00 smithi133 bash[105589]: audit 2023-10-09T05:02:00.348103+0000 mon.c (mon.2) 45 : audit 0 from='client.? 172.21.15.133:0/241824892' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:02:00.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:00 smithi194 bash[92666]: cluster 2023-10-09T05:01:59.559805+0000 mgr.y (mgr.25277) 391 : cluster 0 pgmap v334: 89 pgs: 3 active+undersized+degraded, 34 active+undersized, 52 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:02:00.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:00 smithi194 bash[92666]: audit 2023-10-09T05:02:00.348103+0000 mon.c (mon.2) 45 : audit 0 from='client.? 172.21.15.133:0/241824892' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:02:02.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:02 smithi133 bash[102858]: audit 2023-10-09T05:02:01.227155+0000 mon.a (mon.0) 593 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:02.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:02 smithi133 bash[102858]: audit 2023-10-09T05:02:01.236065+0000 mon.a (mon.0) 594 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:02.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:02 smithi133 bash[102858]: cluster 2023-10-09T05:02:01.560488+0000 mgr.y (mgr.25277) 392 : cluster 0 pgmap v335: 89 pgs: 3 active+undersized+degraded, 34 active+undersized, 52 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:02:02.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:02 smithi133 bash[102858]: audit 2023-10-09T05:02:01.783877+0000 mon.a (mon.0) 595 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:02.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:02 smithi133 bash[102858]: audit 2023-10-09T05:02:01.792103+0000 mon.a (mon.0) 596 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:02.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:02 smithi133 bash[102858]: audit 2023-10-09T05:02:02.019594+0000 mon.a (mon.0) 597 : audit 1 from='client.? 172.21.15.133:0/1289696942' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:02:02.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:02 smithi133 bash[102858]: audit 2023-10-09T05:02:02.088308+0000 mon.a (mon.0) 598 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:02.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:02 smithi133 bash[102858]: audit 2023-10-09T05:02:02.094983+0000 mon.a (mon.0) 599 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:02.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:02 smithi133 bash[105589]: audit 2023-10-09T05:02:01.227155+0000 mon.a (mon.0) 593 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:02.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:02 smithi133 bash[105589]: audit 2023-10-09T05:02:01.236065+0000 mon.a (mon.0) 594 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:02.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:02 smithi133 bash[105589]: cluster 2023-10-09T05:02:01.560488+0000 mgr.y (mgr.25277) 392 : cluster 0 pgmap v335: 89 pgs: 3 active+undersized+degraded, 34 active+undersized, 52 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:02:02.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:02 smithi133 bash[105589]: audit 2023-10-09T05:02:01.783877+0000 mon.a (mon.0) 595 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:02.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:02 smithi133 bash[105589]: audit 2023-10-09T05:02:01.792103+0000 mon.a (mon.0) 596 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:02.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:02 smithi133 bash[105589]: audit 2023-10-09T05:02:02.019594+0000 mon.a (mon.0) 597 : audit 1 from='client.? 172.21.15.133:0/1289696942' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:02:02.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:02 smithi133 bash[105589]: audit 2023-10-09T05:02:02.088308+0000 mon.a (mon.0) 598 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:02.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:02 smithi133 bash[105589]: audit 2023-10-09T05:02:02.094983+0000 mon.a (mon.0) 599 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:02.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:02 smithi194 bash[92666]: audit 2023-10-09T05:02:01.227155+0000 mon.a (mon.0) 593 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:02.685 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:02 smithi194 bash[92666]: audit 2023-10-09T05:02:01.236065+0000 mon.a (mon.0) 594 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:02.685 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:02 smithi194 bash[92666]: cluster 2023-10-09T05:02:01.560488+0000 mgr.y (mgr.25277) 392 : cluster 0 pgmap v335: 89 pgs: 3 active+undersized+degraded, 34 active+undersized, 52 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:02:02.686 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:02 smithi194 bash[92666]: audit 2023-10-09T05:02:01.783877+0000 mon.a (mon.0) 595 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:02.686 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:02 smithi194 bash[92666]: audit 2023-10-09T05:02:01.792103+0000 mon.a (mon.0) 596 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:02.686 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:02 smithi194 bash[92666]: audit 2023-10-09T05:02:02.019594+0000 mon.a (mon.0) 597 : audit 1 from='client.? 172.21.15.133:0/1289696942' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:02:02.686 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:02 smithi194 bash[92666]: audit 2023-10-09T05:02:02.088308+0000 mon.a (mon.0) 598 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:02.686 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:02 smithi194 bash[92666]: audit 2023-10-09T05:02:02.094983+0000 mon.a (mon.0) 599 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:02.804 INFO:teuthology.orchestra.run.smithi133.stderr:enabled application 'rados' on pool 'unique_pool_3' 2023-10-09T05:02:03.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:03 smithi133 bash[105589]: audit 2023-10-09T05:02:02.708175+0000 mon.a (mon.0) 600 : audit 0 from='client.? 172.21.15.133:0/2169584527' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:02:03.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:03 smithi133 bash[105589]: audit 2023-10-09T05:02:02.797792+0000 mon.a (mon.0) 601 : audit 1 from='client.? 172.21.15.133:0/1289696942' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:02:03.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:03 smithi133 bash[105589]: cluster 2023-10-09T05:02:02.797884+0000 mon.a (mon.0) 602 : cluster 0 osdmap e121: 8 total, 7 up, 8 in 2023-10-09T05:02:03.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:03 smithi133 bash[102858]: audit 2023-10-09T05:02:02.708175+0000 mon.a (mon.0) 600 : audit 0 from='client.? 172.21.15.133:0/2169584527' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:02:03.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:03 smithi133 bash[102858]: audit 2023-10-09T05:02:02.797792+0000 mon.a (mon.0) 601 : audit 1 from='client.? 172.21.15.133:0/1289696942' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:02:03.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:03 smithi133 bash[102858]: cluster 2023-10-09T05:02:02.797884+0000 mon.a (mon.0) 602 : cluster 0 osdmap e121: 8 total, 7 up, 8 in 2023-10-09T05:02:03.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:03 smithi194 bash[92666]: audit 2023-10-09T05:02:02.708175+0000 mon.a (mon.0) 600 : audit 0 from='client.? 172.21.15.133:0/2169584527' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:02:03.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:03 smithi194 bash[92666]: audit 2023-10-09T05:02:02.797792+0000 mon.a (mon.0) 601 : audit 1 from='client.? 172.21.15.133:0/1289696942' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:02:03.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:03 smithi194 bash[92666]: cluster 2023-10-09T05:02:02.797884+0000 mon.a (mon.0) 602 : cluster 0 osdmap e121: 8 total, 7 up, 8 in 2023-10-09T05:02:04.547 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:02:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:02:04] "GET /metrics HTTP/1.1" 200 36284 "" "Prometheus/2.43.0" 2023-10-09T05:02:04.678 DEBUG:teuthology.orchestra.run.smithi133:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2023-10-09T05:02:04.681 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T05:02:04.681 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T05:02:04.723 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:hints = 1 2023-10-09T05:02:04.723 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T05:02:04.724 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Object prefix: benchmark_data_smithi133_136590 2023-10-09T05:02:04.724 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:02:04.724 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 0 0 0 0 0 0 - 0 2023-10-09T05:02:04.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:04 smithi133 bash[105589]: cluster 2023-10-09T05:02:02.794569+0000 osd.5 (osd.5) 64 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T05:02:04.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:04 smithi133 bash[105589]: cluster 2023-10-09T05:02:03.021586+0000 osd.4 (osd.4) 48 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T05:02:04.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:04 smithi133 bash[105589]: cluster 2023-10-09T05:02:03.024600+0000 osd.4 (osd.4) 49 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T05:02:04.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:04 smithi133 bash[105589]: audit 2023-10-09T05:02:03.541901+0000 mon.a (mon.0) 603 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:04.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:04 smithi133 bash[105589]: cluster 2023-10-09T05:02:03.560847+0000 mgr.y (mgr.25277) 393 : cluster 0 pgmap v337: 89 pgs: 3 active+undersized+degraded, 34 active+undersized, 52 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:02:04.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:04 smithi133 bash[105589]: cluster 2023-10-09T05:02:03.999193+0000 osd.6 (osd.6) 39 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T05:02:04.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:04 smithi133 bash[102858]: cluster 2023-10-09T05:02:02.794569+0000 osd.5 (osd.5) 64 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T05:02:04.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:04 smithi133 bash[102858]: cluster 2023-10-09T05:02:03.021586+0000 osd.4 (osd.4) 48 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T05:02:04.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:04 smithi133 bash[102858]: cluster 2023-10-09T05:02:03.024600+0000 osd.4 (osd.4) 49 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T05:02:04.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:04 smithi133 bash[102858]: audit 2023-10-09T05:02:03.541901+0000 mon.a (mon.0) 603 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:04.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:04 smithi133 bash[102858]: cluster 2023-10-09T05:02:03.560847+0000 mgr.y (mgr.25277) 393 : cluster 0 pgmap v337: 89 pgs: 3 active+undersized+degraded, 34 active+undersized, 52 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:02:04.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:04 smithi133 bash[102858]: cluster 2023-10-09T05:02:03.999193+0000 osd.6 (osd.6) 39 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T05:02:04.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:04 smithi194 bash[92666]: cluster 2023-10-09T05:02:02.794569+0000 osd.5 (osd.5) 64 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T05:02:04.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:04 smithi194 bash[92666]: cluster 2023-10-09T05:02:03.021586+0000 osd.4 (osd.4) 48 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T05:02:04.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:04 smithi194 bash[92666]: cluster 2023-10-09T05:02:03.024600+0000 osd.4 (osd.4) 49 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T05:02:04.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:04 smithi194 bash[92666]: audit 2023-10-09T05:02:03.541901+0000 mon.a (mon.0) 603 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:04.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:04 smithi194 bash[92666]: cluster 2023-10-09T05:02:03.560847+0000 mgr.y (mgr.25277) 393 : cluster 0 pgmap v337: 89 pgs: 3 active+undersized+degraded, 34 active+undersized, 52 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:02:04.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:04 smithi194 bash[92666]: cluster 2023-10-09T05:02:03.999193+0000 osd.6 (osd.6) 39 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T05:02:05.725 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 1 16 3066 3050 190.614 190.625 0.00614494 0.00522881 2023-10-09T05:02:05.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:05 smithi194 bash[92666]: cluster 2023-10-09T05:02:03.999173+0000 osd.7 (osd.7) 42 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T05:02:05.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:05 smithi194 bash[92666]: audit 2023-10-09T05:02:05.331366+0000 mon.a (mon.0) 604 : audit 0 from='client.? 172.21.15.133:0/3078669350' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:02:06.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:05 smithi133 bash[102858]: cluster 2023-10-09T05:02:03.999173+0000 osd.7 (osd.7) 42 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T05:02:06.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:05 smithi133 bash[102858]: audit 2023-10-09T05:02:05.331366+0000 mon.a (mon.0) 604 : audit 0 from='client.? 172.21.15.133:0/3078669350' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:02:06.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:05 smithi133 bash[105589]: cluster 2023-10-09T05:02:03.999173+0000 osd.7 (osd.7) 42 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T05:02:06.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:05 smithi133 bash[105589]: audit 2023-10-09T05:02:05.331366+0000 mon.a (mon.0) 604 : audit 0 from='client.? 172.21.15.133:0/3078669350' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:02:06.724 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 2 16 6227 6211 194.079 197.562 0.00582682 0.00514134 2023-10-09T05:02:06.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:06 smithi194 bash[92666]: cluster 2023-10-09T05:02:05.561673+0000 mgr.y (mgr.25277) 394 : cluster 0 pgmap v338: 89 pgs: 4 active+undersized+degraded, 33 active+undersized, 52 active+clean; 16 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s wr, 24 op/s; 88/807 objects degraded (10.905%) 2023-10-09T05:02:07.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:06 smithi133 bash[105589]: cluster 2023-10-09T05:02:05.561673+0000 mgr.y (mgr.25277) 394 : cluster 0 pgmap v338: 89 pgs: 4 active+undersized+degraded, 33 active+undersized, 52 active+clean; 16 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s wr, 24 op/s; 88/807 objects degraded (10.905%) 2023-10-09T05:02:07.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:06 smithi133 bash[102858]: cluster 2023-10-09T05:02:05.561673+0000 mgr.y (mgr.25277) 394 : cluster 0 pgmap v338: 89 pgs: 4 active+undersized+degraded, 33 active+undersized, 52 active+clean; 16 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s wr, 24 op/s; 88/807 objects degraded (10.905%) 2023-10-09T05:02:07.724 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 3 16 8978 8962 186.693 171.938 0.0073044 0.00534227 2023-10-09T05:02:07.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:07 smithi133 bash[105589]: cluster 2023-10-09T05:02:06.549480+0000 mon.a (mon.0) 605 : cluster 3 Health check update: Degraded data redundancy: 88/807 objects degraded (10.905%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T05:02:07.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:07 smithi133 bash[105589]: audit 2023-10-09T05:02:07.321452+0000 mon.a (mon.0) 606 : audit 1 from='osd.3 [v2:172.21.15.133:6826/1082233372,v1:172.21.15.133:6827/1082233372]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T05:02:07.800 INFO:journalctl@ceph.osd.3.smithi133.stdout:Oct 09 05:02:07 smithi133 bash[135470]: debug 2023-10-09T05:02:07.303+0000 7fc361225640 -1 osd.3 117 log_to_monitors true 2023-10-09T05:02:07.800 INFO:journalctl@ceph.osd.3.smithi133.stdout:Oct 09 05:02:07 smithi133 bash[135470]: debug 2023-10-09T05:02:07.627+0000 7fc35068d700 -1 osd.3 117 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:02:07.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:07 smithi133 bash[102858]: cluster 2023-10-09T05:02:06.549480+0000 mon.a (mon.0) 605 : cluster 3 Health check update: Degraded data redundancy: 88/807 objects degraded (10.905%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T05:02:07.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:07 smithi133 bash[102858]: audit 2023-10-09T05:02:07.321452+0000 mon.a (mon.0) 606 : audit 1 from='osd.3 [v2:172.21.15.133:6826/1082233372,v1:172.21.15.133:6827/1082233372]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T05:02:07.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:07 smithi194 bash[92666]: cluster 2023-10-09T05:02:06.549480+0000 mon.a (mon.0) 605 : cluster 3 Health check update: Degraded data redundancy: 88/807 objects degraded (10.905%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T05:02:07.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:07 smithi194 bash[92666]: audit 2023-10-09T05:02:07.321452+0000 mon.a (mon.0) 606 : audit 1 from='osd.3 [v2:172.21.15.133:6826/1082233372,v1:172.21.15.133:6827/1082233372]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T05:02:08.725 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 4 16 11747 11731 183.278 173.062 0.00172987 0.00534032 2023-10-09T05:02:08.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:08 smithi194 bash[92666]: cluster 2023-10-09T05:02:07.562306+0000 mgr.y (mgr.25277) 395 : cluster 0 pgmap v339: 89 pgs: 6 active+undersized+degraded, 31 active+undersized, 52 active+clean; 67 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s wr, 106 op/s; 440/3276 objects degraded (13.431%) 2023-10-09T05:02:08.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:08 smithi194 bash[92666]: cluster 2023-10-09T05:02:07.578614+0000 osd.6 (osd.6) 40 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T05:02:08.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:08 smithi194 bash[92666]: audit 2023-10-09T05:02:07.578628+0000 mon.a (mon.0) 607 : audit 1 from='osd.3 [v2:172.21.15.133:6826/1082233372,v1:172.21.15.133:6827/1082233372]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T05:02:08.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:08 smithi194 bash[92666]: cluster 2023-10-09T05:02:07.579109+0000 mon.a (mon.0) 608 : cluster 0 osdmap e122: 8 total, 7 up, 8 in 2023-10-09T05:02:08.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:08 smithi194 bash[92666]: audit 2023-10-09T05:02:07.580428+0000 mon.a (mon.0) 609 : audit 1 from='osd.3 [v2:172.21.15.133:6826/1082233372,v1:172.21.15.133:6827/1082233372]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T05:02:08.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:08 smithi194 bash[92666]: cluster 2023-10-09T05:02:07.581439+0000 osd.4 (osd.4) 50 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T05:02:08.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:08 smithi194 bash[92666]: cluster 2023-10-09T05:02:07.583032+0000 osd.5 (osd.5) 65 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T05:02:08.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:08 smithi194 bash[92666]: cluster 2023-10-09T05:02:07.583583+0000 osd.4 (osd.4) 51 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T05:02:08.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:08 smithi194 bash[92666]: cluster 2023-10-09T05:02:07.586547+0000 osd.5 (osd.5) 66 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T05:02:08.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:08 smithi194 bash[92666]: cluster 2023-10-09T05:02:07.589471+0000 osd.7 (osd.7) 43 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T05:02:08.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:08 smithi194 bash[92666]: audit 2023-10-09T05:02:07.927486+0000 mon.c (mon.2) 46 : audit 0 from='client.? 172.21.15.133:0/1807465485' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:02:09.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:08 smithi133 bash[105589]: cluster 2023-10-09T05:02:07.562306+0000 mgr.y (mgr.25277) 395 : cluster 0 pgmap v339: 89 pgs: 6 active+undersized+degraded, 31 active+undersized, 52 active+clean; 67 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s wr, 106 op/s; 440/3276 objects degraded (13.431%) 2023-10-09T05:02:09.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:08 smithi133 bash[105589]: cluster 2023-10-09T05:02:07.578614+0000 osd.6 (osd.6) 40 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T05:02:09.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:08 smithi133 bash[105589]: audit 2023-10-09T05:02:07.578628+0000 mon.a (mon.0) 607 : audit 1 from='osd.3 [v2:172.21.15.133:6826/1082233372,v1:172.21.15.133:6827/1082233372]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T05:02:09.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:08 smithi133 bash[105589]: cluster 2023-10-09T05:02:07.579109+0000 mon.a (mon.0) 608 : cluster 0 osdmap e122: 8 total, 7 up, 8 in 2023-10-09T05:02:09.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:08 smithi133 bash[105589]: audit 2023-10-09T05:02:07.580428+0000 mon.a (mon.0) 609 : audit 1 from='osd.3 [v2:172.21.15.133:6826/1082233372,v1:172.21.15.133:6827/1082233372]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T05:02:09.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:08 smithi133 bash[105589]: cluster 2023-10-09T05:02:07.581439+0000 osd.4 (osd.4) 50 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T05:02:09.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:08 smithi133 bash[105589]: cluster 2023-10-09T05:02:07.583032+0000 osd.5 (osd.5) 65 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T05:02:09.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:08 smithi133 bash[105589]: cluster 2023-10-09T05:02:07.583583+0000 osd.4 (osd.4) 51 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T05:02:09.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:08 smithi133 bash[105589]: cluster 2023-10-09T05:02:07.586547+0000 osd.5 (osd.5) 66 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T05:02:09.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:08 smithi133 bash[105589]: cluster 2023-10-09T05:02:07.589471+0000 osd.7 (osd.7) 43 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T05:02:09.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:08 smithi133 bash[105589]: audit 2023-10-09T05:02:07.927486+0000 mon.c (mon.2) 46 : audit 0 from='client.? 172.21.15.133:0/1807465485' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:02:09.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:08 smithi133 bash[102858]: cluster 2023-10-09T05:02:07.562306+0000 mgr.y (mgr.25277) 395 : cluster 0 pgmap v339: 89 pgs: 6 active+undersized+degraded, 31 active+undersized, 52 active+clean; 67 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s wr, 106 op/s; 440/3276 objects degraded (13.431%) 2023-10-09T05:02:09.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:08 smithi133 bash[102858]: cluster 2023-10-09T05:02:07.578614+0000 osd.6 (osd.6) 40 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T05:02:09.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:08 smithi133 bash[102858]: audit 2023-10-09T05:02:07.578628+0000 mon.a (mon.0) 607 : audit 1 from='osd.3 [v2:172.21.15.133:6826/1082233372,v1:172.21.15.133:6827/1082233372]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T05:02:09.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:08 smithi133 bash[102858]: cluster 2023-10-09T05:02:07.579109+0000 mon.a (mon.0) 608 : cluster 0 osdmap e122: 8 total, 7 up, 8 in 2023-10-09T05:02:09.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:08 smithi133 bash[102858]: audit 2023-10-09T05:02:07.580428+0000 mon.a (mon.0) 609 : audit 1 from='osd.3 [v2:172.21.15.133:6826/1082233372,v1:172.21.15.133:6827/1082233372]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi133", "root=default"]}]: dispatch 2023-10-09T05:02:09.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:08 smithi133 bash[102858]: cluster 2023-10-09T05:02:07.581439+0000 osd.4 (osd.4) 50 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T05:02:09.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:08 smithi133 bash[102858]: cluster 2023-10-09T05:02:07.583032+0000 osd.5 (osd.5) 65 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T05:02:09.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:08 smithi133 bash[102858]: cluster 2023-10-09T05:02:07.583583+0000 osd.4 (osd.4) 51 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T05:02:09.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:08 smithi133 bash[102858]: cluster 2023-10-09T05:02:07.586547+0000 osd.5 (osd.5) 66 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T05:02:09.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:08 smithi133 bash[102858]: cluster 2023-10-09T05:02:07.589471+0000 osd.7 (osd.7) 43 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T05:02:09.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:08 smithi133 bash[102858]: audit 2023-10-09T05:02:07.927486+0000 mon.c (mon.2) 46 : audit 0 from='client.? 172.21.15.133:0/1807465485' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:02:09.725 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 5 16 11747 11731 146.621 0 - 0.00534032 2023-10-09T05:02:09.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:09 smithi194 bash[92666]: cluster 2023-10-09T05:02:08.617486+0000 mon.a (mon.0) 610 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:02:09.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:09 smithi194 bash[92666]: cluster 2023-10-09T05:02:08.622576+0000 osd.5 (osd.5) 67 : cluster 3 failed to encode map e123 with expected crc 2023-10-09T05:02:09.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:09 smithi194 bash[92666]: cluster 2023-10-09T05:02:08.632155+0000 mon.a (mon.0) 611 : cluster 1 osd.3 [v2:172.21.15.133:6826/1082233372,v1:172.21.15.133:6827/1082233372] boot 2023-10-09T05:02:09.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:09 smithi194 bash[92666]: cluster 2023-10-09T05:02:08.632476+0000 mon.a (mon.0) 612 : cluster 0 osdmap e123: 8 total, 8 up, 8 in 2023-10-09T05:02:09.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:09 smithi194 bash[92666]: audit 2023-10-09T05:02:08.634297+0000 mon.a (mon.0) 613 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:02:09.961 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:09 smithi133 bash[102858]: cluster 2023-10-09T05:02:08.617486+0000 mon.a (mon.0) 610 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:02:09.961 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:09 smithi133 bash[102858]: cluster 2023-10-09T05:02:08.622576+0000 osd.5 (osd.5) 67 : cluster 3 failed to encode map e123 with expected crc 2023-10-09T05:02:09.961 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:09 smithi133 bash[102858]: cluster 2023-10-09T05:02:08.632155+0000 mon.a (mon.0) 611 : cluster 1 osd.3 [v2:172.21.15.133:6826/1082233372,v1:172.21.15.133:6827/1082233372] boot 2023-10-09T05:02:09.961 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:09 smithi133 bash[102858]: cluster 2023-10-09T05:02:08.632476+0000 mon.a (mon.0) 612 : cluster 0 osdmap e123: 8 total, 8 up, 8 in 2023-10-09T05:02:09.961 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:09 smithi133 bash[102858]: audit 2023-10-09T05:02:08.634297+0000 mon.a (mon.0) 613 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:02:09.961 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:09 smithi133 bash[105589]: cluster 2023-10-09T05:02:08.617486+0000 mon.a (mon.0) 610 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:02:09.962 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:09 smithi133 bash[105589]: cluster 2023-10-09T05:02:08.622576+0000 osd.5 (osd.5) 67 : cluster 3 failed to encode map e123 with expected crc 2023-10-09T05:02:09.962 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:09 smithi133 bash[105589]: cluster 2023-10-09T05:02:08.632155+0000 mon.a (mon.0) 611 : cluster 1 osd.3 [v2:172.21.15.133:6826/1082233372,v1:172.21.15.133:6827/1082233372] boot 2023-10-09T05:02:09.962 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:09 smithi133 bash[105589]: cluster 2023-10-09T05:02:08.632476+0000 mon.a (mon.0) 612 : cluster 0 osdmap e123: 8 total, 8 up, 8 in 2023-10-09T05:02:09.962 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:09 smithi133 bash[105589]: audit 2023-10-09T05:02:08.634297+0000 mon.a (mon.0) 613 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:02:10.323 INFO:teuthology.orchestra.run.smithi133.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 4 2023-10-09T05:02:10.725 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 6 16 11747 11731 122.184 0 - 0.00534032 2023-10-09T05:02:10.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:10 smithi194 bash[92666]: cluster 2023-10-09T05:02:09.563055+0000 mgr.y (mgr.25277) 396 : cluster 0 pgmap v342: 89 pgs: 6 active+undersized+degraded, 31 active+undersized, 52 active+clean; 67 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s wr, 157 op/s; 440/3276 objects degraded (13.431%) 2023-10-09T05:02:10.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:10 smithi194 bash[92666]: cluster 2023-10-09T05:02:09.628758+0000 osd.6 (osd.6) 41 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T05:02:10.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:10 smithi194 bash[92666]: cluster 2023-10-09T05:02:09.629243+0000 osd.5 (osd.5) 68 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T05:02:10.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:10 smithi194 bash[92666]: cluster 2023-10-09T05:02:09.629989+0000 mon.a (mon.0) 614 : cluster 0 osdmap e124: 8 total, 8 up, 8 in 2023-10-09T05:02:10.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:10 smithi194 bash[92666]: cluster 2023-10-09T05:02:09.631396+0000 osd.7 (osd.7) 44 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T05:02:10.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:10 smithi194 bash[92666]: cluster 2023-10-09T05:02:09.631499+0000 osd.4 (osd.4) 52 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T05:02:10.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:10 smithi194 bash[92666]: cluster 2023-10-09T05:02:09.634634+0000 osd.5 (osd.5) 69 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T05:02:10.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:10 smithi194 bash[92666]: cluster 2023-10-09T05:02:09.635093+0000 osd.4 (osd.4) 53 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T05:02:10.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:10 smithi194 bash[92666]: cluster 2023-10-09T05:02:09.635862+0000 osd.6 (osd.6) 42 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T05:02:10.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:10 smithi194 bash[92666]: cluster 2023-10-09T05:02:09.638081+0000 osd.5 (osd.5) 70 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T05:02:10.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:10 smithi194 bash[92666]: cluster 2023-10-09T05:02:09.639847+0000 osd.4 (osd.4) 54 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T05:02:10.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:10 smithi194 bash[92666]: cluster 2023-10-09T05:02:09.639964+0000 osd.6 (osd.6) 43 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T05:02:10.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:10 smithi194 bash[92666]: audit 2023-10-09T05:02:10.303471+0000 mon.c (mon.2) 47 : audit 0 from='client.? 172.21.15.133:0/1875637411' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:02:11.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:10 smithi133 bash[105589]: cluster 2023-10-09T05:02:09.563055+0000 mgr.y (mgr.25277) 396 : cluster 0 pgmap v342: 89 pgs: 6 active+undersized+degraded, 31 active+undersized, 52 active+clean; 67 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s wr, 157 op/s; 440/3276 objects degraded (13.431%) 2023-10-09T05:02:11.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:10 smithi133 bash[105589]: cluster 2023-10-09T05:02:09.628758+0000 osd.6 (osd.6) 41 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T05:02:11.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:10 smithi133 bash[105589]: cluster 2023-10-09T05:02:09.629243+0000 osd.5 (osd.5) 68 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T05:02:11.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:10 smithi133 bash[105589]: cluster 2023-10-09T05:02:09.629989+0000 mon.a (mon.0) 614 : cluster 0 osdmap e124: 8 total, 8 up, 8 in 2023-10-09T05:02:11.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:10 smithi133 bash[105589]: cluster 2023-10-09T05:02:09.631396+0000 osd.7 (osd.7) 44 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T05:02:11.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:10 smithi133 bash[105589]: cluster 2023-10-09T05:02:09.631499+0000 osd.4 (osd.4) 52 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T05:02:11.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:10 smithi133 bash[105589]: cluster 2023-10-09T05:02:09.634634+0000 osd.5 (osd.5) 69 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T05:02:11.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:10 smithi133 bash[105589]: cluster 2023-10-09T05:02:09.635093+0000 osd.4 (osd.4) 53 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T05:02:11.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:10 smithi133 bash[105589]: cluster 2023-10-09T05:02:09.635862+0000 osd.6 (osd.6) 42 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T05:02:11.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:10 smithi133 bash[105589]: cluster 2023-10-09T05:02:09.638081+0000 osd.5 (osd.5) 70 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T05:02:11.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:10 smithi133 bash[105589]: cluster 2023-10-09T05:02:09.639847+0000 osd.4 (osd.4) 54 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T05:02:11.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:10 smithi133 bash[105589]: cluster 2023-10-09T05:02:09.639964+0000 osd.6 (osd.6) 43 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T05:02:11.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:10 smithi133 bash[105589]: audit 2023-10-09T05:02:10.303471+0000 mon.c (mon.2) 47 : audit 0 from='client.? 172.21.15.133:0/1875637411' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:02:11.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:10 smithi133 bash[102858]: cluster 2023-10-09T05:02:09.563055+0000 mgr.y (mgr.25277) 396 : cluster 0 pgmap v342: 89 pgs: 6 active+undersized+degraded, 31 active+undersized, 52 active+clean; 67 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s wr, 157 op/s; 440/3276 objects degraded (13.431%) 2023-10-09T05:02:11.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:10 smithi133 bash[102858]: cluster 2023-10-09T05:02:09.628758+0000 osd.6 (osd.6) 41 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T05:02:11.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:10 smithi133 bash[102858]: cluster 2023-10-09T05:02:09.629243+0000 osd.5 (osd.5) 68 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T05:02:11.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:10 smithi133 bash[102858]: cluster 2023-10-09T05:02:09.629989+0000 mon.a (mon.0) 614 : cluster 0 osdmap e124: 8 total, 8 up, 8 in 2023-10-09T05:02:11.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:10 smithi133 bash[102858]: cluster 2023-10-09T05:02:09.631396+0000 osd.7 (osd.7) 44 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T05:02:11.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:10 smithi133 bash[102858]: cluster 2023-10-09T05:02:09.631499+0000 osd.4 (osd.4) 52 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T05:02:11.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:10 smithi133 bash[102858]: cluster 2023-10-09T05:02:09.634634+0000 osd.5 (osd.5) 69 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T05:02:11.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:10 smithi133 bash[102858]: cluster 2023-10-09T05:02:09.635093+0000 osd.4 (osd.4) 53 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T05:02:11.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:10 smithi133 bash[102858]: cluster 2023-10-09T05:02:09.635862+0000 osd.6 (osd.6) 42 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T05:02:11.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:10 smithi133 bash[102858]: cluster 2023-10-09T05:02:09.638081+0000 osd.5 (osd.5) 70 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T05:02:11.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:10 smithi133 bash[102858]: cluster 2023-10-09T05:02:09.639847+0000 osd.4 (osd.4) 54 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T05:02:11.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:10 smithi133 bash[102858]: cluster 2023-10-09T05:02:09.639964+0000 osd.6 (osd.6) 43 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T05:02:11.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:10 smithi133 bash[102858]: audit 2023-10-09T05:02:10.303471+0000 mon.c (mon.2) 47 : audit 0 from='client.? 172.21.15.133:0/1875637411' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:02:11.725 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 7 16 13713 13697 122.281 40.9583 0.0104346 0.0081689 2023-10-09T05:02:12.056 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:11 smithi133 bash[105589]: cluster 2023-10-09T05:02:10.635532+0000 osd.6 (osd.6) 44 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T05:02:12.056 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:11 smithi133 bash[105589]: cluster 2023-10-09T05:02:10.636419+0000 mon.a (mon.0) 615 : cluster 0 osdmap e125: 8 total, 8 up, 8 in 2023-10-09T05:02:12.056 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:11 smithi133 bash[105589]: cluster 2023-10-09T05:02:10.636721+0000 osd.5 (osd.5) 71 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T05:02:12.056 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:11 smithi133 bash[105589]: cluster 2023-10-09T05:02:10.637706+0000 osd.4 (osd.4) 55 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T05:02:12.056 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:11 smithi133 bash[105589]: cluster 2023-10-09T05:02:10.640256+0000 osd.5 (osd.5) 72 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T05:02:12.056 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:11 smithi133 bash[105589]: cluster 2023-10-09T05:02:10.640630+0000 osd.4 (osd.4) 56 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T05:02:12.056 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:11 smithi133 bash[102858]: cluster 2023-10-09T05:02:10.635532+0000 osd.6 (osd.6) 44 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T05:02:12.056 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:11 smithi133 bash[102858]: cluster 2023-10-09T05:02:10.636419+0000 mon.a (mon.0) 615 : cluster 0 osdmap e125: 8 total, 8 up, 8 in 2023-10-09T05:02:12.057 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:11 smithi133 bash[102858]: cluster 2023-10-09T05:02:10.636721+0000 osd.5 (osd.5) 71 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T05:02:12.057 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:11 smithi133 bash[102858]: cluster 2023-10-09T05:02:10.637706+0000 osd.4 (osd.4) 55 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T05:02:12.057 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:11 smithi133 bash[102858]: cluster 2023-10-09T05:02:10.640256+0000 osd.5 (osd.5) 72 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T05:02:12.057 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:11 smithi133 bash[102858]: cluster 2023-10-09T05:02:10.640630+0000 osd.4 (osd.4) 56 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T05:02:12.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:11 smithi194 bash[92666]: cluster 2023-10-09T05:02:10.635532+0000 osd.6 (osd.6) 44 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T05:02:12.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:11 smithi194 bash[92666]: cluster 2023-10-09T05:02:10.636419+0000 mon.a (mon.0) 615 : cluster 0 osdmap e125: 8 total, 8 up, 8 in 2023-10-09T05:02:12.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:11 smithi194 bash[92666]: cluster 2023-10-09T05:02:10.636721+0000 osd.5 (osd.5) 71 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T05:02:12.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:11 smithi194 bash[92666]: cluster 2023-10-09T05:02:10.637706+0000 osd.4 (osd.4) 55 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T05:02:12.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:11 smithi194 bash[92666]: cluster 2023-10-09T05:02:10.640256+0000 osd.5 (osd.5) 72 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T05:02:12.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:11 smithi194 bash[92666]: cluster 2023-10-09T05:02:10.640630+0000 osd.4 (osd.4) 56 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T05:02:12.725 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 8 16 15549 15533 121.338 114.75 0.00735084 0.0082324 2023-10-09T05:02:13.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:12 smithi133 bash[105589]: cluster 2023-10-09T05:02:10.647478+0000 osd.7 (osd.7) 45 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T05:02:13.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:12 smithi133 bash[105589]: cluster 2023-10-09T05:02:11.563657+0000 mgr.y (mgr.25277) 397 : cluster 0 pgmap v345: 89 pgs: 5 remapped+peering, 1 active, 4 peering, 3 activating, 2 active+undersized+degraded, 12 active+undersized, 62 active+clean; 685 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 154 MiB/s wr, 2.47k op/s; 2/32934 objects degraded (0.006%) 2023-10-09T05:02:13.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:12 smithi133 bash[105589]: cluster 2023-10-09T05:02:11.633956+0000 mon.a (mon.0) 616 : cluster 3 Health check update: Degraded data redundancy: 2/32934 objects degraded (0.006%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:02:13.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:12 smithi133 bash[102858]: cluster 2023-10-09T05:02:10.647478+0000 osd.7 (osd.7) 45 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T05:02:13.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:12 smithi133 bash[102858]: cluster 2023-10-09T05:02:11.563657+0000 mgr.y (mgr.25277) 397 : cluster 0 pgmap v345: 89 pgs: 5 remapped+peering, 1 active, 4 peering, 3 activating, 2 active+undersized+degraded, 12 active+undersized, 62 active+clean; 685 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 154 MiB/s wr, 2.47k op/s; 2/32934 objects degraded (0.006%) 2023-10-09T05:02:13.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:12 smithi133 bash[102858]: cluster 2023-10-09T05:02:11.633956+0000 mon.a (mon.0) 616 : cluster 3 Health check update: Degraded data redundancy: 2/32934 objects degraded (0.006%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:02:13.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:12 smithi194 bash[92666]: cluster 2023-10-09T05:02:10.647478+0000 osd.7 (osd.7) 45 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T05:02:13.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:12 smithi194 bash[92666]: cluster 2023-10-09T05:02:11.563657+0000 mgr.y (mgr.25277) 397 : cluster 0 pgmap v345: 89 pgs: 5 remapped+peering, 1 active, 4 peering, 3 activating, 2 active+undersized+degraded, 12 active+undersized, 62 active+clean; 685 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 154 MiB/s wr, 2.47k op/s; 2/32934 objects degraded (0.006%) 2023-10-09T05:02:13.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:12 smithi194 bash[92666]: cluster 2023-10-09T05:02:11.633956+0000 mon.a (mon.0) 616 : cluster 3 Health check update: Degraded data redundancy: 2/32934 objects degraded (0.006%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:02:13.263 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T05:02:13.725 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 9 16 17468 17452 121.181 119.938 0.0061968 0.00824311 2023-10-09T05:02:14.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:14 smithi133 bash[102858]: audit 2023-10-09T05:02:13.430109+0000 mon.a (mon.0) 617 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:14.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:14 smithi133 bash[102858]: audit 2023-10-09T05:02:13.432119+0000 mon.a (mon.0) 618 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:14.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:14 smithi133 bash[102858]: cluster 2023-10-09T05:02:13.565905+0000 mgr.y (mgr.25277) 398 : cluster 0 pgmap v346: 89 pgs: 5 remapped+peering, 1 active, 4 peering, 3 activating, 1 active+undersized, 75 active+clean; 685 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-09T05:02:14.550 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:02:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:02:14] "GET /metrics HTTP/1.1" 200 36336 "" "Prometheus/2.43.0" 2023-10-09T05:02:14.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:14 smithi133 bash[105589]: audit 2023-10-09T05:02:13.430109+0000 mon.a (mon.0) 617 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:14.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:14 smithi133 bash[105589]: audit 2023-10-09T05:02:13.432119+0000 mon.a (mon.0) 618 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:14.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:14 smithi133 bash[105589]: cluster 2023-10-09T05:02:13.565905+0000 mgr.y (mgr.25277) 398 : cluster 0 pgmap v346: 89 pgs: 5 remapped+peering, 1 active, 4 peering, 3 activating, 1 active+undersized, 75 active+clean; 685 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-09T05:02:14.725 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 10 16 19466 19450 121.55 124.875 0.00421659 0.00821822 2023-10-09T05:02:14.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:14 smithi194 bash[92666]: audit 2023-10-09T05:02:13.430109+0000 mon.a (mon.0) 617 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:14.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:14 smithi194 bash[92666]: audit 2023-10-09T05:02:13.432119+0000 mon.a (mon.0) 618 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:14.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:14 smithi194 bash[92666]: cluster 2023-10-09T05:02:13.565905+0000 mgr.y (mgr.25277) 398 : cluster 0 pgmap v346: 89 pgs: 5 remapped+peering, 1 active, 4 peering, 3 activating, 1 active+undersized, 75 active+clean; 685 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-09T05:02:15.725 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 11 16 21264 21248 120.715 112.375 0.00613845 0.00827252 2023-10-09T05:02:15.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:15 smithi133 bash[105589]: cluster 2023-10-09T05:02:14.426636+0000 mon.a (mon.0) 619 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/32934 objects degraded (0.006%), 2 pgs degraded) 2023-10-09T05:02:15.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:15 smithi133 bash[105589]: cluster 2023-10-09T05:02:14.426676+0000 mon.a (mon.0) 620 : cluster 1 Cluster is now healthy 2023-10-09T05:02:15.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:15 smithi133 bash[102858]: cluster 2023-10-09T05:02:14.426636+0000 mon.a (mon.0) 619 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/32934 objects degraded (0.006%), 2 pgs degraded) 2023-10-09T05:02:15.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:15 smithi133 bash[102858]: cluster 2023-10-09T05:02:14.426676+0000 mon.a (mon.0) 620 : cluster 1 Cluster is now healthy 2023-10-09T05:02:15.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:15 smithi194 bash[92666]: cluster 2023-10-09T05:02:14.426636+0000 mon.a (mon.0) 619 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/32934 objects degraded (0.006%), 2 pgs degraded) 2023-10-09T05:02:15.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:15 smithi194 bash[92666]: cluster 2023-10-09T05:02:14.426676+0000 mon.a (mon.0) 620 : cluster 1 Cluster is now healthy 2023-10-09T05:02:16.726 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 12 16 22818 22802 118.748 97.125 0.00197949 0.00831737 2023-10-09T05:02:16.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:16 smithi133 bash[102858]: cluster 2023-10-09T05:02:15.566796+0000 mgr.y (mgr.25277) 399 : cluster 0 pgmap v347: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 83 active+clean; 1.0 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 140 MiB/s wr, 2.23k op/s; 1577/49803 objects degraded (3.166%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T05:02:16.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:16 smithi133 bash[105589]: cluster 2023-10-09T05:02:15.566796+0000 mgr.y (mgr.25277) 399 : cluster 0 pgmap v347: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 83 active+clean; 1.0 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 140 MiB/s wr, 2.23k op/s; 1577/49803 objects degraded (3.166%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T05:02:16.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:16 smithi194 bash[92666]: cluster 2023-10-09T05:02:15.566796+0000 mgr.y (mgr.25277) 399 : cluster 0 pgmap v347: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 83 active+clean; 1.0 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 140 MiB/s wr, 2.23k op/s; 1577/49803 objects degraded (3.166%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T05:02:17.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:17 smithi133 bash[105589]: cluster 2023-10-09T05:02:16.442122+0000 mon.a (mon.0) 621 : cluster 3 Health check failed: Degraded data redundancy: 1577/49803 objects degraded (3.166%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T05:02:17.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:17 smithi133 bash[105589]: cluster 2023-10-09T05:02:16.452859+0000 osd.6 (osd.6) 45 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T05:02:17.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:17 smithi133 bash[105589]: cluster 2023-10-09T05:02:16.455093+0000 osd.5 (osd.5) 73 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T05:02:17.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:17 smithi133 bash[105589]: cluster 2023-10-09T05:02:16.463178+0000 mon.a (mon.0) 622 : cluster 0 osdmap e126: 8 total, 8 up, 8 in 2023-10-09T05:02:17.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:17 smithi133 bash[102858]: cluster 2023-10-09T05:02:16.442122+0000 mon.a (mon.0) 621 : cluster 3 Health check failed: Degraded data redundancy: 1577/49803 objects degraded (3.166%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T05:02:17.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:17 smithi133 bash[102858]: cluster 2023-10-09T05:02:16.452859+0000 osd.6 (osd.6) 45 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T05:02:17.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:17 smithi133 bash[102858]: cluster 2023-10-09T05:02:16.455093+0000 osd.5 (osd.5) 73 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T05:02:17.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:17 smithi133 bash[102858]: cluster 2023-10-09T05:02:16.463178+0000 mon.a (mon.0) 622 : cluster 0 osdmap e126: 8 total, 8 up, 8 in 2023-10-09T05:02:17.725 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 13 16 23287 23271 111.869 29.3125 0.0122923 0.00893128 2023-10-09T05:02:17.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:17 smithi194 bash[92666]: cluster 2023-10-09T05:02:16.442122+0000 mon.a (mon.0) 621 : cluster 3 Health check failed: Degraded data redundancy: 1577/49803 objects degraded (3.166%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T05:02:17.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:17 smithi194 bash[92666]: cluster 2023-10-09T05:02:16.452859+0000 osd.6 (osd.6) 45 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T05:02:17.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:17 smithi194 bash[92666]: cluster 2023-10-09T05:02:16.455093+0000 osd.5 (osd.5) 73 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T05:02:17.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:17 smithi194 bash[92666]: cluster 2023-10-09T05:02:16.463178+0000 mon.a (mon.0) 622 : cluster 0 osdmap e126: 8 total, 8 up, 8 in 2023-10-09T05:02:18.725 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 14 16 25156 25140 112.221 116.812 0.00279774 0.00889663 2023-10-09T05:02:18.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:18 smithi133 bash[102858]: cluster 2023-10-09T05:02:17.454319+0000 osd.7 (osd.7) 46 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T05:02:18.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:18 smithi133 bash[102858]: cluster 2023-10-09T05:02:17.454579+0000 osd.5 (osd.5) 74 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T05:02:18.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:18 smithi133 bash[102858]: cluster 2023-10-09T05:02:17.456574+0000 mon.a (mon.0) 623 : cluster 0 osdmap e127: 8 total, 8 up, 8 in 2023-10-09T05:02:18.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:18 smithi133 bash[102858]: cluster 2023-10-09T05:02:17.469616+0000 osd.6 (osd.6) 46 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T05:02:18.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:18 smithi133 bash[102858]: cluster 2023-10-09T05:02:17.474334+0000 osd.4 (osd.4) 57 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T05:02:18.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:18 smithi133 bash[102858]: cluster 2023-10-09T05:02:17.567327+0000 mgr.y (mgr.25277) 400 : cluster 0 pgmap v350: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 3832/60480 objects degraded (6.336%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T05:02:18.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:18 smithi133 bash[105589]: cluster 2023-10-09T05:02:17.454319+0000 osd.7 (osd.7) 46 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T05:02:18.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:18 smithi133 bash[105589]: cluster 2023-10-09T05:02:17.454579+0000 osd.5 (osd.5) 74 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T05:02:18.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:18 smithi133 bash[105589]: cluster 2023-10-09T05:02:17.456574+0000 mon.a (mon.0) 623 : cluster 0 osdmap e127: 8 total, 8 up, 8 in 2023-10-09T05:02:18.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:18 smithi133 bash[105589]: cluster 2023-10-09T05:02:17.469616+0000 osd.6 (osd.6) 46 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T05:02:18.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:18 smithi133 bash[105589]: cluster 2023-10-09T05:02:17.474334+0000 osd.4 (osd.4) 57 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T05:02:18.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:18 smithi133 bash[105589]: cluster 2023-10-09T05:02:17.567327+0000 mgr.y (mgr.25277) 400 : cluster 0 pgmap v350: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 3832/60480 objects degraded (6.336%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T05:02:18.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:18 smithi194 bash[92666]: cluster 2023-10-09T05:02:17.454319+0000 osd.7 (osd.7) 46 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T05:02:18.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:18 smithi194 bash[92666]: cluster 2023-10-09T05:02:17.454579+0000 osd.5 (osd.5) 74 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T05:02:18.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:18 smithi194 bash[92666]: cluster 2023-10-09T05:02:17.456574+0000 mon.a (mon.0) 623 : cluster 0 osdmap e127: 8 total, 8 up, 8 in 2023-10-09T05:02:18.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:18 smithi194 bash[92666]: cluster 2023-10-09T05:02:17.469616+0000 osd.6 (osd.6) 46 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T05:02:18.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:18 smithi194 bash[92666]: cluster 2023-10-09T05:02:17.474334+0000 osd.4 (osd.4) 57 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T05:02:18.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:18 smithi194 bash[92666]: cluster 2023-10-09T05:02:17.567327+0000 mgr.y (mgr.25277) 400 : cluster 0 pgmap v350: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 3832/60480 objects degraded (6.336%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T05:02:19.725 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 15 16 26679 26663 111.085 95.1875 0.0123862 0.00899474 2023-10-09T05:02:19.778 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:02:19.799 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:02:19 smithi133 bash[91128]: debug 2023-10-09T05:02:19.435+0000 7ff45d5ed700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T05:02:20.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:20 smithi194 bash[92666]: audit 2023-10-09T05:02:19.320708+0000 mon.a (mon.0) 624 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:20.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:20 smithi194 bash[92666]: audit 2023-10-09T05:02:19.331357+0000 mon.a (mon.0) 625 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:20.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:20 smithi194 bash[92666]: audit 2023-10-09T05:02:19.332589+0000 mon.a (mon.0) 626 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:02:20.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:20 smithi194 bash[92666]: audit 2023-10-09T05:02:19.334231+0000 mon.a (mon.0) 627 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:02:20.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:20 smithi194 bash[92666]: audit 2023-10-09T05:02:19.341397+0000 mon.a (mon.0) 628 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:20.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:20 smithi194 bash[92666]: audit 2023-10-09T05:02:19.424752+0000 mon.a (mon.0) 629 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:02:20.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:20 smithi194 bash[92666]: audit 2023-10-09T05:02:19.429709+0000 mon.a (mon.0) 630 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:02:20.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:20 smithi194 bash[92666]: audit 2023-10-09T05:02:19.432432+0000 mon.a (mon.0) 631 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:02:20.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:20 smithi194 bash[92666]: audit 2023-10-09T05:02:19.435384+0000 mon.a (mon.0) 632 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:02:20.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:20 smithi194 bash[92666]: audit 2023-10-09T05:02:19.439101+0000 mon.a (mon.0) 633 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T05:02:20.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:20 smithi194 bash[92666]: audit 2023-10-09T05:02:19.439358+0000 mgr.y (mgr.25277) 401 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T05:02:20.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:20 smithi194 bash[92666]: cephadm 2023-10-09T05:02:19.440598+0000 mgr.y (mgr.25277) 402 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T05:02:20.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:20 smithi194 bash[92666]: cluster 2023-10-09T05:02:19.567851+0000 mgr.y (mgr.25277) 403 : cluster 0 pgmap v351: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 3832/60480 objects degraded (6.336%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T05:02:20.726 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 16 16 28407 28391 110.891 108 0.00387878 0.00901029 2023-10-09T05:02:20.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:20 smithi133 bash[105589]: audit 2023-10-09T05:02:19.320708+0000 mon.a (mon.0) 624 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:20.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:20 smithi133 bash[105589]: audit 2023-10-09T05:02:19.331357+0000 mon.a (mon.0) 625 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:20.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:20 smithi133 bash[105589]: audit 2023-10-09T05:02:19.332589+0000 mon.a (mon.0) 626 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:02:20.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:20 smithi133 bash[105589]: audit 2023-10-09T05:02:19.334231+0000 mon.a (mon.0) 627 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:02:20.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:20 smithi133 bash[105589]: audit 2023-10-09T05:02:19.341397+0000 mon.a (mon.0) 628 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:20.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:20 smithi133 bash[105589]: audit 2023-10-09T05:02:19.424752+0000 mon.a (mon.0) 629 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:02:20.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:20 smithi133 bash[105589]: audit 2023-10-09T05:02:19.429709+0000 mon.a (mon.0) 630 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:02:20.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:20 smithi133 bash[105589]: audit 2023-10-09T05:02:19.432432+0000 mon.a (mon.0) 631 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:02:20.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:20 smithi133 bash[105589]: audit 2023-10-09T05:02:19.435384+0000 mon.a (mon.0) 632 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:02:20.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:20 smithi133 bash[105589]: audit 2023-10-09T05:02:19.439101+0000 mon.a (mon.0) 633 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T05:02:20.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:20 smithi133 bash[105589]: audit 2023-10-09T05:02:19.439358+0000 mgr.y (mgr.25277) 401 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T05:02:20.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:20 smithi133 bash[105589]: cephadm 2023-10-09T05:02:19.440598+0000 mgr.y (mgr.25277) 402 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T05:02:20.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:20 smithi133 bash[105589]: cluster 2023-10-09T05:02:19.567851+0000 mgr.y (mgr.25277) 403 : cluster 0 pgmap v351: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 3832/60480 objects degraded (6.336%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T05:02:20.804 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:20 smithi133 bash[102858]: audit 2023-10-09T05:02:19.320708+0000 mon.a (mon.0) 624 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:20.804 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:20 smithi133 bash[102858]: audit 2023-10-09T05:02:19.331357+0000 mon.a (mon.0) 625 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:20.804 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:20 smithi133 bash[102858]: audit 2023-10-09T05:02:19.332589+0000 mon.a (mon.0) 626 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:02:20.804 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:20 smithi133 bash[102858]: audit 2023-10-09T05:02:19.334231+0000 mon.a (mon.0) 627 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:02:20.804 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:20 smithi133 bash[102858]: audit 2023-10-09T05:02:19.341397+0000 mon.a (mon.0) 628 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:20.804 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:20 smithi133 bash[102858]: audit 2023-10-09T05:02:19.424752+0000 mon.a (mon.0) 629 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:02:20.804 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:20 smithi133 bash[102858]: audit 2023-10-09T05:02:19.429709+0000 mon.a (mon.0) 630 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:02:20.804 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:20 smithi133 bash[102858]: audit 2023-10-09T05:02:19.432432+0000 mon.a (mon.0) 631 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:02:20.804 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:20 smithi133 bash[102858]: audit 2023-10-09T05:02:19.435384+0000 mon.a (mon.0) 632 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:02:20.805 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:20 smithi133 bash[102858]: audit 2023-10-09T05:02:19.439101+0000 mon.a (mon.0) 633 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T05:02:20.805 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:20 smithi133 bash[102858]: audit 2023-10-09T05:02:19.439358+0000 mgr.y (mgr.25277) 401 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T05:02:20.805 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:20 smithi133 bash[102858]: cephadm 2023-10-09T05:02:19.440598+0000 mgr.y (mgr.25277) 402 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T05:02:20.805 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:20 smithi133 bash[102858]: cluster 2023-10-09T05:02:19.567851+0000 mgr.y (mgr.25277) 403 : cluster 0 pgmap v351: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 3832/60480 objects degraded (6.336%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T05:02:21.726 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 17 16 30180 30164 110.886 110.812 0.00663145 0.00901227 2023-10-09T05:02:22.028 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:21 smithi133 bash[102858]: cluster 2023-10-09T05:02:21.619830+0000 mon.a (mon.0) 634 : cluster 3 Health check update: Degraded data redundancy: 3209/81993 objects degraded (3.914%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:02:22.029 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:21 smithi133 bash[105589]: cluster 2023-10-09T05:02:21.619830+0000 mon.a (mon.0) 634 : cluster 3 Health check update: Degraded data redundancy: 3209/81993 objects degraded (3.914%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:02:22.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:21 smithi194 bash[92666]: cluster 2023-10-09T05:02:21.619830+0000 mon.a (mon.0) 634 : cluster 3 Health check update: Degraded data redundancy: 3209/81993 objects degraded (3.914%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:02:22.726 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 18 16 31776 31760 110.267 99.75 0.00699337 0.00906269 2023-10-09T05:02:23.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:22 smithi133 bash[102858]: cluster 2023-10-09T05:02:21.568574+0000 mgr.y (mgr.25277) 404 : cluster 0 pgmap v352: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.7 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s; 3209/81993 objects degraded (3.914%); 9.6 MiB/s, 154 objects/s recovering 2023-10-09T05:02:23.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:22 smithi133 bash[105589]: cluster 2023-10-09T05:02:21.568574+0000 mgr.y (mgr.25277) 404 : cluster 0 pgmap v352: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.7 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s; 3209/81993 objects degraded (3.914%); 9.6 MiB/s, 154 objects/s recovering 2023-10-09T05:02:23.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:22 smithi194 bash[92666]: cluster 2023-10-09T05:02:21.568574+0000 mgr.y (mgr.25277) 404 : cluster 0 pgmap v352: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.7 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s; 3209/81993 objects degraded (3.914%); 9.6 MiB/s, 154 objects/s recovering 2023-10-09T05:02:23.728 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 19 16 32562 32546 107.049 49.125 0.00253993 0.00905782 2023-10-09T05:02:24.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:24 smithi194 bash[92666]: cluster 2023-10-09T05:02:23.102677+0000 mon.a (mon.0) 635 : cluster 0 osdmap e128: 8 total, 8 up, 8 in 2023-10-09T05:02:24.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:24 smithi194 bash[92666]: cluster 2023-10-09T05:02:23.105540+0000 osd.4 (osd.4) 58 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T05:02:24.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:24 smithi194 bash[92666]: cluster 2023-10-09T05:02:23.105756+0000 osd.7 (osd.7) 47 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T05:02:24.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:24 smithi194 bash[92666]: cluster 2023-10-09T05:02:23.107434+0000 osd.5 (osd.5) 75 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T05:02:24.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:24 smithi194 bash[92666]: cluster 2023-10-09T05:02:23.112072+0000 osd.6 (osd.6) 47 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T05:02:24.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:24 smithi194 bash[92666]: cluster 2023-10-09T05:02:23.112249+0000 osd.5 (osd.5) 76 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T05:02:24.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:24 smithi194 bash[92666]: cluster 2023-10-09T05:02:23.569011+0000 mgr.y (mgr.25277) 405 : cluster 0 pgmap v354: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.7 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 3209/81993 objects degraded (3.914%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T05:02:24.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:02:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:02:24] "GET /metrics HTTP/1.1" 200 36336 "" "Prometheus/2.43.0" 2023-10-09T05:02:24.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:24 smithi133 bash[102858]: cluster 2023-10-09T05:02:23.102677+0000 mon.a (mon.0) 635 : cluster 0 osdmap e128: 8 total, 8 up, 8 in 2023-10-09T05:02:24.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:24 smithi133 bash[102858]: cluster 2023-10-09T05:02:23.105540+0000 osd.4 (osd.4) 58 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T05:02:24.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:24 smithi133 bash[102858]: cluster 2023-10-09T05:02:23.105756+0000 osd.7 (osd.7) 47 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T05:02:24.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:24 smithi133 bash[102858]: cluster 2023-10-09T05:02:23.107434+0000 osd.5 (osd.5) 75 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T05:02:24.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:24 smithi133 bash[102858]: cluster 2023-10-09T05:02:23.112072+0000 osd.6 (osd.6) 47 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T05:02:24.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:24 smithi133 bash[102858]: cluster 2023-10-09T05:02:23.112249+0000 osd.5 (osd.5) 76 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T05:02:24.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:24 smithi133 bash[102858]: cluster 2023-10-09T05:02:23.569011+0000 mgr.y (mgr.25277) 405 : cluster 0 pgmap v354: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.7 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 3209/81993 objects degraded (3.914%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T05:02:24.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:24 smithi133 bash[105589]: cluster 2023-10-09T05:02:23.102677+0000 mon.a (mon.0) 635 : cluster 0 osdmap e128: 8 total, 8 up, 8 in 2023-10-09T05:02:24.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:24 smithi133 bash[105589]: cluster 2023-10-09T05:02:23.105540+0000 osd.4 (osd.4) 58 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T05:02:24.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:24 smithi133 bash[105589]: cluster 2023-10-09T05:02:23.105756+0000 osd.7 (osd.7) 47 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T05:02:24.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:24 smithi133 bash[105589]: cluster 2023-10-09T05:02:23.107434+0000 osd.5 (osd.5) 75 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T05:02:24.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:24 smithi133 bash[105589]: cluster 2023-10-09T05:02:23.112072+0000 osd.6 (osd.6) 47 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T05:02:24.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:24 smithi133 bash[105589]: cluster 2023-10-09T05:02:23.112249+0000 osd.5 (osd.5) 76 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T05:02:24.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:24 smithi133 bash[105589]: cluster 2023-10-09T05:02:23.569011+0000 mgr.y (mgr.25277) 405 : cluster 0 pgmap v354: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.7 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 3209/81993 objects degraded (3.914%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T05:02:24.726 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T05:02:24.725023+0000 min lat: 0.00115667 max lat: 2.19705 avg lat: 0.00954273 2023-10-09T05:02:24.726 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:02:24.726 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 20 16 33519 33503 104.687 59.8125 0.0209807 0.00954273 2023-10-09T05:02:25.332 INFO:teuthology.orchestra.run.smithi133.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:02:25.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:25 smithi194 bash[92666]: cluster 2023-10-09T05:02:24.114847+0000 mon.a (mon.0) 636 : cluster 0 osdmap e129: 8 total, 8 up, 8 in 2023-10-09T05:02:25.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:25 smithi194 bash[92666]: cluster 2023-10-09T05:02:24.121846+0000 osd.4 (osd.4) 59 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T05:02:25.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:25 smithi194 bash[92666]: cluster 2023-10-09T05:02:24.130747+0000 osd.6 (osd.6) 48 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T05:02:25.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:25 smithi194 bash[92666]: cluster 2023-10-09T05:02:24.131997+0000 osd.5 (osd.5) 77 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T05:02:25.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:25 smithi194 bash[92666]: cluster 2023-10-09T05:02:24.134230+0000 osd.7 (osd.7) 48 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T05:02:25.399 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:25 smithi133 bash[105589]: cluster 2023-10-09T05:02:24.114847+0000 mon.a (mon.0) 636 : cluster 0 osdmap e129: 8 total, 8 up, 8 in 2023-10-09T05:02:25.399 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:25 smithi133 bash[105589]: cluster 2023-10-09T05:02:24.121846+0000 osd.4 (osd.4) 59 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T05:02:25.399 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:25 smithi133 bash[105589]: cluster 2023-10-09T05:02:24.130747+0000 osd.6 (osd.6) 48 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T05:02:25.399 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:25 smithi133 bash[105589]: cluster 2023-10-09T05:02:24.131997+0000 osd.5 (osd.5) 77 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T05:02:25.400 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:25 smithi133 bash[105589]: cluster 2023-10-09T05:02:24.134230+0000 osd.7 (osd.7) 48 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T05:02:25.401 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:25 smithi133 bash[102858]: cluster 2023-10-09T05:02:24.114847+0000 mon.a (mon.0) 636 : cluster 0 osdmap e129: 8 total, 8 up, 8 in 2023-10-09T05:02:25.401 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:25 smithi133 bash[102858]: cluster 2023-10-09T05:02:24.121846+0000 osd.4 (osd.4) 59 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T05:02:25.401 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:25 smithi133 bash[102858]: cluster 2023-10-09T05:02:24.130747+0000 osd.6 (osd.6) 48 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T05:02:25.401 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:25 smithi133 bash[102858]: cluster 2023-10-09T05:02:24.131997+0000 osd.5 (osd.5) 77 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T05:02:25.401 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:25 smithi133 bash[102858]: cluster 2023-10-09T05:02:24.134230+0000 osd.7 (osd.7) 48 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T05:02:25.726 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 21 16 34999 34983 104.106 92.5 0.00418638 0.00959745 2023-10-09T05:02:26.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:26 smithi194 bash[92666]: audit 2023-10-09T05:02:25.323777+0000 mgr.y (mgr.25277) 406 : audit 0 from='client.44582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:02:26.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:26 smithi194 bash[92666]: cephadm 2023-10-09T05:02:25.324169+0000 mgr.y (mgr.25277) 407 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:02:26.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:26 smithi194 bash[92666]: audit 2023-10-09T05:02:25.329548+0000 mon.a (mon.0) 637 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:26.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:26 smithi194 bash[92666]: cluster 2023-10-09T05:02:25.571266+0000 mgr.y (mgr.25277) 408 : cluster 0 pgmap v356: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.9 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s; 2945/94773 objects degraded (3.107%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T05:02:26.727 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T05:02:26.727 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:26 smithi133 bash[102858]: audit 2023-10-09T05:02:25.323777+0000 mgr.y (mgr.25277) 406 : audit 0 from='client.44582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:02:26.727 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:26 smithi133 bash[102858]: cephadm 2023-10-09T05:02:25.324169+0000 mgr.y (mgr.25277) 407 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:02:26.727 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:26 smithi133 bash[102858]: audit 2023-10-09T05:02:25.329548+0000 mon.a (mon.0) 637 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:26.728 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:26 smithi133 bash[102858]: cluster 2023-10-09T05:02:25.571266+0000 mgr.y (mgr.25277) 408 : cluster 0 pgmap v356: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.9 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s; 2945/94773 objects degraded (3.107%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T05:02:26.728 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:26 smithi133 bash[105589]: audit 2023-10-09T05:02:25.323777+0000 mgr.y (mgr.25277) 406 : audit 0 from='client.44582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:02:26.728 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:26 smithi133 bash[105589]: cephadm 2023-10-09T05:02:25.324169+0000 mgr.y (mgr.25277) 407 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:02:26.728 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:26 smithi133 bash[105589]: audit 2023-10-09T05:02:25.329548+0000 mon.a (mon.0) 637 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:26.728 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:26 smithi133 bash[105589]: cluster 2023-10-09T05:02:25.571266+0000 mgr.y (mgr.25277) 408 : cluster 0 pgmap v356: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.9 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s; 2945/94773 objects degraded (3.107%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T05:02:26.729 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 22 16 36595 36579 103.906 99.75 0.0157118 0.00961786 2023-10-09T05:02:27.726 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 23 16 38142 38126 103.592 96.6875 0.00871919 0.00964693 2023-10-09T05:02:28.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:27 smithi133 bash[105589]: cluster 2023-10-09T05:02:27.624509+0000 mon.a (mon.0) 638 : cluster 3 Health check update: Degraded data redundancy: 2945/101058 objects degraded (2.914%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:02:28.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:27 smithi133 bash[102858]: cluster 2023-10-09T05:02:27.624509+0000 mon.a (mon.0) 638 : cluster 3 Health check update: Degraded data redundancy: 2945/101058 objects degraded (2.914%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:02:28.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:27 smithi194 bash[92666]: cluster 2023-10-09T05:02:27.624509+0000 mon.a (mon.0) 638 : cluster 3 Health check update: Degraded data redundancy: 2945/101058 objects degraded (2.914%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:02:28.726 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 24 16 39704 39688 103.343 97.625 0.0109072 0.00967033 2023-10-09T05:02:29.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:28 smithi133 bash[105589]: cluster 2023-10-09T05:02:27.571921+0000 mgr.y (mgr.25277) 409 : cluster 0 pgmap v357: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s; 2945/101058 objects degraded (2.914%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T05:02:29.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:28 smithi133 bash[105589]: audit 2023-10-09T05:02:28.425769+0000 mon.a (mon.0) 639 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:29.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:28 smithi133 bash[102858]: cluster 2023-10-09T05:02:27.571921+0000 mgr.y (mgr.25277) 409 : cluster 0 pgmap v357: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s; 2945/101058 objects degraded (2.914%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T05:02:29.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:28 smithi133 bash[102858]: audit 2023-10-09T05:02:28.425769+0000 mon.a (mon.0) 639 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:29.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:28 smithi194 bash[92666]: cluster 2023-10-09T05:02:27.571921+0000 mgr.y (mgr.25277) 409 : cluster 0 pgmap v357: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s; 2945/101058 objects degraded (2.914%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T05:02:29.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:28 smithi194 bash[92666]: audit 2023-10-09T05:02:28.425769+0000 mon.a (mon.0) 639 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:29.727 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 25 16 41213 41197 102.981 94.3125 0.00709711 0.00970283 2023-10-09T05:02:30.727 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 26 16 41391 41375 99.4483 11.125 1.0308 0.00973113 2023-10-09T05:02:31.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:30 smithi133 bash[105589]: cluster 2023-10-09T05:02:29.572561+0000 mgr.y (mgr.25277) 410 : cluster 0 pgmap v358: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s; 2740/102813 objects degraded (2.665%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T05:02:31.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:30 smithi133 bash[105589]: cluster 2023-10-09T05:02:29.693857+0000 osd.6 (osd.6) 49 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T05:02:31.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:30 smithi133 bash[105589]: cluster 2023-10-09T05:02:29.694797+0000 mon.a (mon.0) 640 : cluster 0 osdmap e130: 8 total, 8 up, 8 in 2023-10-09T05:02:31.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:30 smithi133 bash[105589]: cluster 2023-10-09T05:02:29.696157+0000 osd.4 (osd.4) 60 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T05:02:31.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:30 smithi133 bash[105589]: cluster 2023-10-09T05:02:29.701619+0000 osd.5 (osd.5) 78 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T05:02:31.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:30 smithi133 bash[105589]: cluster 2023-10-09T05:02:29.705727+0000 osd.5 (osd.5) 79 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T05:02:31.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:30 smithi133 bash[102858]: cluster 2023-10-09T05:02:29.572561+0000 mgr.y (mgr.25277) 410 : cluster 0 pgmap v358: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s; 2740/102813 objects degraded (2.665%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T05:02:31.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:30 smithi133 bash[102858]: cluster 2023-10-09T05:02:29.693857+0000 osd.6 (osd.6) 49 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T05:02:31.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:30 smithi133 bash[102858]: cluster 2023-10-09T05:02:29.694797+0000 mon.a (mon.0) 640 : cluster 0 osdmap e130: 8 total, 8 up, 8 in 2023-10-09T05:02:31.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:30 smithi133 bash[102858]: cluster 2023-10-09T05:02:29.696157+0000 osd.4 (osd.4) 60 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T05:02:31.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:30 smithi133 bash[102858]: cluster 2023-10-09T05:02:29.701619+0000 osd.5 (osd.5) 78 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T05:02:31.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:30 smithi133 bash[102858]: cluster 2023-10-09T05:02:29.705727+0000 osd.5 (osd.5) 79 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T05:02:31.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:30 smithi194 bash[92666]: cluster 2023-10-09T05:02:29.572561+0000 mgr.y (mgr.25277) 410 : cluster 0 pgmap v358: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s; 2740/102813 objects degraded (2.665%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T05:02:31.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:30 smithi194 bash[92666]: cluster 2023-10-09T05:02:29.693857+0000 osd.6 (osd.6) 49 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T05:02:31.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:30 smithi194 bash[92666]: cluster 2023-10-09T05:02:29.694797+0000 mon.a (mon.0) 640 : cluster 0 osdmap e130: 8 total, 8 up, 8 in 2023-10-09T05:02:31.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:30 smithi194 bash[92666]: cluster 2023-10-09T05:02:29.696157+0000 osd.4 (osd.4) 60 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T05:02:31.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:30 smithi194 bash[92666]: cluster 2023-10-09T05:02:29.701619+0000 osd.5 (osd.5) 78 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T05:02:31.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:30 smithi194 bash[92666]: cluster 2023-10-09T05:02:29.705727+0000 osd.5 (osd.5) 79 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T05:02:31.543 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:02:31.727 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 27 16 42788 42772 98.9984 87.3125 0.0583521 0.0100815 2023-10-09T05:02:32.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:31 smithi133 bash[105589]: cluster 2023-10-09T05:02:30.704038+0000 osd.4 (osd.4) 61 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T05:02:32.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:31 smithi133 bash[105589]: cluster 2023-10-09T05:02:30.707600+0000 mon.a (mon.0) 641 : cluster 0 osdmap e131: 8 total, 8 up, 8 in 2023-10-09T05:02:32.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:31 smithi133 bash[105589]: cluster 2023-10-09T05:02:30.709274+0000 osd.4 (osd.4) 62 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T05:02:32.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:31 smithi133 bash[105589]: cluster 2023-10-09T05:02:30.713527+0000 osd.5 (osd.5) 80 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T05:02:32.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:31 smithi133 bash[105589]: cluster 2023-10-09T05:02:30.718960+0000 osd.6 (osd.6) 50 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T05:02:32.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:31 smithi133 bash[102858]: cluster 2023-10-09T05:02:30.704038+0000 osd.4 (osd.4) 61 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T05:02:32.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:31 smithi133 bash[102858]: cluster 2023-10-09T05:02:30.707600+0000 mon.a (mon.0) 641 : cluster 0 osdmap e131: 8 total, 8 up, 8 in 2023-10-09T05:02:32.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:31 smithi133 bash[102858]: cluster 2023-10-09T05:02:30.709274+0000 osd.4 (osd.4) 62 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T05:02:32.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:31 smithi133 bash[102858]: cluster 2023-10-09T05:02:30.713527+0000 osd.5 (osd.5) 80 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T05:02:32.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:31 smithi133 bash[102858]: cluster 2023-10-09T05:02:30.718960+0000 osd.6 (osd.6) 50 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T05:02:32.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:31 smithi194 bash[92666]: cluster 2023-10-09T05:02:30.704038+0000 osd.4 (osd.4) 61 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T05:02:32.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:31 smithi194 bash[92666]: cluster 2023-10-09T05:02:30.707600+0000 mon.a (mon.0) 641 : cluster 0 osdmap e131: 8 total, 8 up, 8 in 2023-10-09T05:02:32.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:31 smithi194 bash[92666]: cluster 2023-10-09T05:02:30.709274+0000 osd.4 (osd.4) 62 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T05:02:32.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:31 smithi194 bash[92666]: cluster 2023-10-09T05:02:30.713527+0000 osd.5 (osd.5) 80 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T05:02:32.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:31 smithi194 bash[92666]: cluster 2023-10-09T05:02:30.718960+0000 osd.6 (osd.6) 50 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T05:02:32.727 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 28 16 44216 44200 98.6499 89.25 0.0144855 0.0101302 2023-10-09T05:02:33.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:32 smithi133 bash[105589]: cluster 2023-10-09T05:02:31.573066+0000 mgr.y (mgr.25277) 411 : cluster 0 pgmap v361: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s; 2156/123258 objects degraded (1.749%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T05:02:33.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:32 smithi133 bash[102858]: cluster 2023-10-09T05:02:31.573066+0000 mgr.y (mgr.25277) 411 : cluster 0 pgmap v361: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s; 2156/123258 objects degraded (1.749%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T05:02:33.146 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:32 smithi194 bash[92666]: cluster 2023-10-09T05:02:31.573066+0000 mgr.y (mgr.25277) 411 : cluster 0 pgmap v361: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s; 2156/123258 objects degraded (1.749%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T05:02:33.727 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 29 15 45418 45403 97.8406 75.1875 0.0123504 0.0102147 2023-10-09T05:02:34.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:33 smithi133 bash[102858]: cluster 2023-10-09T05:02:33.091948+0000 mon.a (mon.0) 642 : cluster 3 Health check update: Degraded data redundancy: 2156/123258 objects degraded (1.749%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T05:02:34.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:33 smithi133 bash[105589]: cluster 2023-10-09T05:02:33.091948+0000 mon.a (mon.0) 642 : cluster 3 Health check update: Degraded data redundancy: 2156/123258 objects degraded (1.749%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T05:02:34.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:33 smithi194 bash[92666]: cluster 2023-10-09T05:02:33.091948+0000 mon.a (mon.0) 642 : cluster 3 Health check update: Degraded data redundancy: 2156/123258 objects degraded (1.749%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T05:02:34.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:02:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:02:34] "GET /metrics HTTP/1.1" 200 36357 "" "Prometheus/2.43.0" 2023-10-09T05:02:34.727 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 30 16 46818 46802 97.4935 87.4375 0.0196132 0.0102506 2023-10-09T05:02:35.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:34 smithi133 bash[102858]: cluster 2023-10-09T05:02:33.573690+0000 mgr.y (mgr.25277) 412 : cluster 0 pgmap v362: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 2156/124965 objects degraded (1.725%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T05:02:35.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:34 smithi133 bash[102858]: audit 2023-10-09T05:02:34.454065+0000 mon.a (mon.0) 643 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:02:35.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:34 smithi133 bash[105589]: cluster 2023-10-09T05:02:33.573690+0000 mgr.y (mgr.25277) 412 : cluster 0 pgmap v362: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 2156/124965 objects degraded (1.725%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T05:02:35.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:34 smithi133 bash[105589]: audit 2023-10-09T05:02:34.454065+0000 mon.a (mon.0) 643 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:02:35.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:34 smithi194 bash[92666]: cluster 2023-10-09T05:02:33.573690+0000 mgr.y (mgr.25277) 412 : cluster 0 pgmap v362: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 2156/124965 objects degraded (1.725%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T05:02:35.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:34 smithi194 bash[92666]: audit 2023-10-09T05:02:34.454065+0000 mon.a (mon.0) 643 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:02:35.728 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 31 16 48165 48149 97.0632 84.1875 0.00389921 0.0102954 2023-10-09T05:02:35.984 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:35 smithi133 bash[102858]: audit 2023-10-09T05:02:34.981527+0000 mon.a (mon.0) 644 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:35.984 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:35 smithi133 bash[102858]: audit 2023-10-09T05:02:34.994478+0000 mon.a (mon.0) 645 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:35.984 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:35 smithi133 bash[102858]: audit 2023-10-09T05:02:34.996678+0000 mon.a (mon.0) 646 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:02:35.984 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:35 smithi133 bash[102858]: audit 2023-10-09T05:02:34.998095+0000 mon.a (mon.0) 647 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:02:35.984 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:35 smithi133 bash[102858]: audit 2023-10-09T05:02:35.006374+0000 mon.a (mon.0) 648 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:35.985 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:35 smithi133 bash[102858]: audit 2023-10-09T05:02:35.088013+0000 mon.a (mon.0) 649 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:02:35.985 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:35 smithi133 bash[102858]: audit 2023-10-09T05:02:35.093052+0000 mon.a (mon.0) 650 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:02:35.985 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:35 smithi133 bash[102858]: audit 2023-10-09T05:02:35.094825+0000 mon.a (mon.0) 651 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:02:35.985 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:35 smithi133 bash[102858]: audit 2023-10-09T05:02:35.104045+0000 mon.a (mon.0) 652 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:36.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:35 smithi133 bash[105589]: audit 2023-10-09T05:02:34.981527+0000 mon.a (mon.0) 644 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:36.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:35 smithi133 bash[105589]: audit 2023-10-09T05:02:34.994478+0000 mon.a (mon.0) 645 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:36.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:35 smithi133 bash[105589]: audit 2023-10-09T05:02:34.996678+0000 mon.a (mon.0) 646 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:02:36.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:35 smithi133 bash[105589]: audit 2023-10-09T05:02:34.998095+0000 mon.a (mon.0) 647 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:02:36.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:35 smithi133 bash[105589]: audit 2023-10-09T05:02:35.006374+0000 mon.a (mon.0) 648 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:36.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:35 smithi133 bash[105589]: audit 2023-10-09T05:02:35.088013+0000 mon.a (mon.0) 649 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:02:36.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:35 smithi133 bash[105589]: audit 2023-10-09T05:02:35.093052+0000 mon.a (mon.0) 650 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:02:36.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:35 smithi133 bash[105589]: audit 2023-10-09T05:02:35.094825+0000 mon.a (mon.0) 651 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:02:36.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:35 smithi133 bash[105589]: audit 2023-10-09T05:02:35.104045+0000 mon.a (mon.0) 652 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:36.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:35 smithi194 bash[92666]: audit 2023-10-09T05:02:34.981527+0000 mon.a (mon.0) 644 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:36.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:35 smithi194 bash[92666]: audit 2023-10-09T05:02:34.994478+0000 mon.a (mon.0) 645 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:36.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:35 smithi194 bash[92666]: audit 2023-10-09T05:02:34.996678+0000 mon.a (mon.0) 646 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:02:36.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:35 smithi194 bash[92666]: audit 2023-10-09T05:02:34.998095+0000 mon.a (mon.0) 647 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:02:36.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:35 smithi194 bash[92666]: audit 2023-10-09T05:02:35.006374+0000 mon.a (mon.0) 648 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:36.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:35 smithi194 bash[92666]: audit 2023-10-09T05:02:35.088013+0000 mon.a (mon.0) 649 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:02:36.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:35 smithi194 bash[92666]: audit 2023-10-09T05:02:35.093052+0000 mon.a (mon.0) 650 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:02:36.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:35 smithi194 bash[92666]: audit 2023-10-09T05:02:35.094825+0000 mon.a (mon.0) 651 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:02:36.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:35 smithi194 bash[92666]: audit 2023-10-09T05:02:35.104045+0000 mon.a (mon.0) 652 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:02:36.728 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 32 16 48954 48938 95.5709 49.3125 0.00434588 0.0102959 2023-10-09T05:02:37.266 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:36 smithi133 bash[102858]: cluster 2023-10-09T05:02:35.574409+0000 mgr.y (mgr.25277) 413 : cluster 0 pgmap v363: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s; 1549/134580 objects degraded (1.151%); 11 MiB/s, 174 objects/s recovering 2023-10-09T05:02:37.266 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:36 smithi133 bash[102858]: cluster 2023-10-09T05:02:36.113917+0000 mon.a (mon.0) 653 : cluster 0 osdmap e132: 8 total, 8 up, 8 in 2023-10-09T05:02:37.266 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:36 smithi133 bash[102858]: cluster 2023-10-09T05:02:36.116589+0000 osd.5 (osd.5) 81 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T05:02:37.266 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:36 smithi133 bash[102858]: cluster 2023-10-09T05:02:36.121194+0000 osd.4 (osd.4) 63 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T05:02:37.266 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:36 smithi133 bash[102858]: cluster 2023-10-09T05:02:36.121578+0000 osd.5 (osd.5) 82 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T05:02:37.266 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:36 smithi133 bash[102858]: cluster 2023-10-09T05:02:36.127565+0000 osd.6 (osd.6) 51 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T05:02:37.267 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:36 smithi133 bash[105589]: cluster 2023-10-09T05:02:35.574409+0000 mgr.y (mgr.25277) 413 : cluster 0 pgmap v363: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s; 1549/134580 objects degraded (1.151%); 11 MiB/s, 174 objects/s recovering 2023-10-09T05:02:37.267 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:36 smithi133 bash[105589]: cluster 2023-10-09T05:02:36.113917+0000 mon.a (mon.0) 653 : cluster 0 osdmap e132: 8 total, 8 up, 8 in 2023-10-09T05:02:37.267 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:36 smithi133 bash[105589]: cluster 2023-10-09T05:02:36.116589+0000 osd.5 (osd.5) 81 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T05:02:37.267 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:36 smithi133 bash[105589]: cluster 2023-10-09T05:02:36.121194+0000 osd.4 (osd.4) 63 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T05:02:37.267 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:36 smithi133 bash[105589]: cluster 2023-10-09T05:02:36.121578+0000 osd.5 (osd.5) 82 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T05:02:37.267 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:36 smithi133 bash[105589]: cluster 2023-10-09T05:02:36.127565+0000 osd.6 (osd.6) 51 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T05:02:37.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:36 smithi194 bash[92666]: cluster 2023-10-09T05:02:35.574409+0000 mgr.y (mgr.25277) 413 : cluster 0 pgmap v363: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s; 1549/134580 objects degraded (1.151%); 11 MiB/s, 174 objects/s recovering 2023-10-09T05:02:37.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:36 smithi194 bash[92666]: cluster 2023-10-09T05:02:36.113917+0000 mon.a (mon.0) 653 : cluster 0 osdmap e132: 8 total, 8 up, 8 in 2023-10-09T05:02:37.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:36 smithi194 bash[92666]: cluster 2023-10-09T05:02:36.116589+0000 osd.5 (osd.5) 81 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T05:02:37.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:36 smithi194 bash[92666]: cluster 2023-10-09T05:02:36.121194+0000 osd.4 (osd.4) 63 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T05:02:37.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:36 smithi194 bash[92666]: cluster 2023-10-09T05:02:36.121578+0000 osd.5 (osd.5) 82 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T05:02:37.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:36 smithi194 bash[92666]: cluster 2023-10-09T05:02:36.127565+0000 osd.6 (osd.6) 51 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T05:02:37.728 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 33 16 49660 49644 94.0118 44.125 0.0248384 0.0106299 2023-10-09T05:02:37.963 INFO:teuthology.orchestra.run.smithi133.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T05:02:37.964 INFO:teuthology.orchestra.run.smithi133.stdout:alertmanager.a smithi133 *:9093,9094 running (12m) - 20m 15.0M - 0.25.0 c8568f914cd2 f7641cfa53d2 2023-10-09T05:02:37.964 INFO:teuthology.orchestra.run.smithi133.stdout:grafana.a smithi194 *:3000 running (12m) - 19m 74.5M - 9.4.7 2c41d148cca3 18d67948e55e 2023-10-09T05:02:37.964 INFO:teuthology.orchestra.run.smithi133.stdout:mds.foo.smithi133.tsuabh smithi133 running (17m) - 17m 15.0M - 17.2.6-1303-gb2903cb0 5517396c50d4 1adbb452599e 2023-10-09T05:02:37.964 INFO:teuthology.orchestra.run.smithi133.stdout:mds.foo.smithi194.yxnfqn smithi194 running (17m) - 17m 18.0M - 17.2.6-1303-gb2903cb0 5517396c50d4 745c16c1ffd2 2023-10-09T05:02:37.964 INFO:teuthology.orchestra.run.smithi133.stdout:mgr.x smithi194 *:8443,9283,8765 running (11m) - 28m 430M - 18.2.0-598-gb4fa663f 3060897fdf97 e7277fed699e 2023-10-09T05:02:37.964 INFO:teuthology.orchestra.run.smithi133.stdout:mgr.y smithi133 *:8443,9283,8765 running (12m) - 31m 489M - 18.2.0-598-gb4fa663f 3060897fdf97 caf6152f91c6 2023-10-09T05:02:37.964 INFO:teuthology.orchestra.run.smithi133.stdout:mon.a smithi133 running (10m) - 31m 58.9M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 ee45bbc6b194 2023-10-09T05:02:37.964 INFO:teuthology.orchestra.run.smithi133.stdout:mon.b smithi194 running (7m) - 29m 40.6M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 ed14c6f4a04d 2023-10-09T05:02:37.964 INFO:teuthology.orchestra.run.smithi133.stdout:mon.c smithi133 running (8m) - 29m 42.1M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 eb3f3f9009e4 2023-10-09T05:02:37.965 INFO:teuthology.orchestra.run.smithi133.stdout:node-exporter.a smithi133 *:9100 running (12m) - 20m 8387k - 1.5.0 0da6a335fe13 f8bdde5940e7 2023-10-09T05:02:37.965 INFO:teuthology.orchestra.run.smithi133.stdout:node-exporter.b smithi194 *:9100 running (12m) - 20m 8464k - 1.5.0 0da6a335fe13 ffab717ac947 2023-10-09T05:02:37.965 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0 smithi133 running (5m) - 27m 1265M 2524M 18.2.0-598-gb4fa663f 3060897fdf97 021c5eccb109 2023-10-09T05:02:37.965 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1 smithi133 running (5m) - 26m 400M 2524M 18.2.0-598-gb4fa663f 3060897fdf97 4728310689ed 2023-10-09T05:02:37.965 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2 smithi133 running (108s) - 26m 418M 2524M 18.2.0-598-gb4fa663f 3060897fdf97 e32c40f5649e 2023-10-09T05:02:37.965 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3 smithi133 running (40s) - 25m - 2524M 18.2.0-598-gb4fa663f 3060897fdf97 809bb1f1cba2 2023-10-09T05:02:37.965 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4 smithi194 running (24m) - 24m 565M 2524M 17.2.6-1303-gb2903cb0 5517396c50d4 c77edffeb061 2023-10-09T05:02:37.965 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5 smithi194 running (23m) - 23m 925M 2524M 17.2.6-1303-gb2903cb0 5517396c50d4 47540765fad9 2023-10-09T05:02:37.965 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6 smithi194 running (22m) - 22m 740M 2524M 17.2.6-1303-gb2903cb0 5517396c50d4 26bbe2f2859f 2023-10-09T05:02:37.965 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7 smithi194 running (21m) - 21m 541M 2524M 17.2.6-1303-gb2903cb0 5517396c50d4 45c343cd22ea 2023-10-09T05:02:37.965 INFO:teuthology.orchestra.run.smithi133.stdout:prometheus.a smithi194 *:9095 running (11m) - 20m 52.6M - 2.43.0 a07b618ecd1d f40ed7def7ab 2023-10-09T05:02:38.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:38 smithi133 bash[105589]: cluster 2023-10-09T05:02:37.116739+0000 osd.5 (osd.5) 83 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T05:02:38.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:38 smithi133 bash[105589]: cluster 2023-10-09T05:02:37.117644+0000 mon.a (mon.0) 654 : cluster 0 osdmap e133: 8 total, 8 up, 8 in 2023-10-09T05:02:38.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:38 smithi133 bash[105589]: cluster 2023-10-09T05:02:37.119939+0000 osd.5 (osd.5) 84 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T05:02:38.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:38 smithi133 bash[105589]: cluster 2023-10-09T05:02:37.141824+0000 osd.4 (osd.4) 64 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T05:02:38.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:38 smithi133 bash[105589]: cluster 2023-10-09T05:02:37.146072+0000 osd.6 (osd.6) 52 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T05:02:38.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:38 smithi133 bash[105589]: cluster 2023-10-09T05:02:37.574821+0000 mgr.y (mgr.25277) 414 : cluster 0 pgmap v366: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 53 MiB/s wr, 855 op/s; 1525/140901 objects degraded (1.082%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T05:02:38.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:38 smithi133 bash[105589]: audit 2023-10-09T05:02:37.936866+0000 mgr.y (mgr.25277) 415 : audit 0 from='client.54499 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:02:38.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:38 smithi133 bash[105589]: cluster 2023-10-09T05:02:38.093121+0000 mon.a (mon.0) 655 : cluster 3 Health check update: Degraded data redundancy: 1549/134580 objects degraded (1.151%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T05:02:38.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:38 smithi133 bash[102858]: cluster 2023-10-09T05:02:37.116739+0000 osd.5 (osd.5) 83 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T05:02:38.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:38 smithi133 bash[102858]: cluster 2023-10-09T05:02:37.117644+0000 mon.a (mon.0) 654 : cluster 0 osdmap e133: 8 total, 8 up, 8 in 2023-10-09T05:02:38.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:38 smithi133 bash[102858]: cluster 2023-10-09T05:02:37.119939+0000 osd.5 (osd.5) 84 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T05:02:38.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:38 smithi133 bash[102858]: cluster 2023-10-09T05:02:37.141824+0000 osd.4 (osd.4) 64 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T05:02:38.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:38 smithi133 bash[102858]: cluster 2023-10-09T05:02:37.146072+0000 osd.6 (osd.6) 52 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T05:02:38.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:38 smithi133 bash[102858]: cluster 2023-10-09T05:02:37.574821+0000 mgr.y (mgr.25277) 414 : cluster 0 pgmap v366: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 53 MiB/s wr, 855 op/s; 1525/140901 objects degraded (1.082%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T05:02:38.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:38 smithi133 bash[102858]: audit 2023-10-09T05:02:37.936866+0000 mgr.y (mgr.25277) 415 : audit 0 from='client.54499 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:02:38.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:38 smithi133 bash[102858]: cluster 2023-10-09T05:02:38.093121+0000 mon.a (mon.0) 655 : cluster 3 Health check update: Degraded data redundancy: 1549/134580 objects degraded (1.151%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T05:02:38.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:38 smithi194 bash[92666]: cluster 2023-10-09T05:02:37.116739+0000 osd.5 (osd.5) 83 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T05:02:38.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:38 smithi194 bash[92666]: cluster 2023-10-09T05:02:37.117644+0000 mon.a (mon.0) 654 : cluster 0 osdmap e133: 8 total, 8 up, 8 in 2023-10-09T05:02:38.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:38 smithi194 bash[92666]: cluster 2023-10-09T05:02:37.119939+0000 osd.5 (osd.5) 84 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T05:02:38.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:38 smithi194 bash[92666]: cluster 2023-10-09T05:02:37.141824+0000 osd.4 (osd.4) 64 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T05:02:38.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:38 smithi194 bash[92666]: cluster 2023-10-09T05:02:37.146072+0000 osd.6 (osd.6) 52 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T05:02:38.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:38 smithi194 bash[92666]: cluster 2023-10-09T05:02:37.574821+0000 mgr.y (mgr.25277) 414 : cluster 0 pgmap v366: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 53 MiB/s wr, 855 op/s; 1525/140901 objects degraded (1.082%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T05:02:38.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:38 smithi194 bash[92666]: audit 2023-10-09T05:02:37.936866+0000 mgr.y (mgr.25277) 415 : audit 0 from='client.54499 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:02:38.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:38 smithi194 bash[92666]: cluster 2023-10-09T05:02:38.093121+0000 mon.a (mon.0) 655 : cluster 3 Health check update: Degraded data redundancy: 1549/134580 objects degraded (1.151%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T05:02:38.728 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 34 16 50919 50903 93.5608 78.6875 0.00602732 0.0106818 2023-10-09T05:02:39.466 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T05:02:39.728 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 35 16 52203 52187 93.1803 80.25 0.0406936 0.0107234 2023-10-09T05:02:40.728 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 36 16 53512 53496 92.8643 81.8125 0.0144874 0.0107623 2023-10-09T05:02:40.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:40 smithi194 bash[92666]: cluster 2023-10-09T05:02:39.575543+0000 mgr.y (mgr.25277) 416 : cluster 0 pgmap v367: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s; 1525/142884 objects degraded (1.067%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T05:02:41.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:40 smithi133 bash[105589]: cluster 2023-10-09T05:02:39.575543+0000 mgr.y (mgr.25277) 416 : cluster 0 pgmap v367: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s; 1525/142884 objects degraded (1.067%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T05:02:41.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:40 smithi133 bash[102858]: cluster 2023-10-09T05:02:39.575543+0000 mgr.y (mgr.25277) 416 : cluster 0 pgmap v367: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s; 1525/142884 objects degraded (1.067%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T05:02:41.728 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 37 16 54814 54798 92.5536 81.375 0.0104012 0.0107987 2023-10-09T05:02:42.729 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 38 16 56096 56080 92.2263 80.125 0.0119958 0.0108365 2023-10-09T05:02:42.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:42 smithi194 bash[92666]: cluster 2023-10-09T05:02:41.576430+0000 mgr.y (mgr.25277) 417 : cluster 0 pgmap v368: 89 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s; 1015/157083 objects degraded (0.646%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T05:02:43.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:42 smithi133 bash[102858]: cluster 2023-10-09T05:02:41.576430+0000 mgr.y (mgr.25277) 417 : cluster 0 pgmap v368: 89 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s; 1015/157083 objects degraded (0.646%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T05:02:43.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:42 smithi133 bash[105589]: cluster 2023-10-09T05:02:41.576430+0000 mgr.y (mgr.25277) 417 : cluster 0 pgmap v368: 89 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s; 1015/157083 objects degraded (0.646%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T05:02:43.728 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 39 16 57336 57320 91.8485 77.5 0.00940616 0.0108814 2023-10-09T05:02:43.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:43 smithi194 bash[92666]: cluster 2023-10-09T05:02:43.094320+0000 mon.a (mon.0) 656 : cluster 3 Health check update: Degraded data redundancy: 1015/157083 objects degraded (0.646%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:02:43.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:43 smithi194 bash[92666]: audit 2023-10-09T05:02:43.426063+0000 mon.a (mon.0) 657 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:44.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:43 smithi133 bash[102858]: cluster 2023-10-09T05:02:43.094320+0000 mon.a (mon.0) 656 : cluster 3 Health check update: Degraded data redundancy: 1015/157083 objects degraded (0.646%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:02:44.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:43 smithi133 bash[102858]: audit 2023-10-09T05:02:43.426063+0000 mon.a (mon.0) 657 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:44.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:43 smithi133 bash[105589]: cluster 2023-10-09T05:02:43.094320+0000 mon.a (mon.0) 656 : cluster 3 Health check update: Degraded data redundancy: 1015/157083 objects degraded (0.646%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:02:44.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:43 smithi133 bash[105589]: audit 2023-10-09T05:02:43.426063+0000 mon.a (mon.0) 657 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:44.346 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:02:44.550 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:02:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:02:44] "GET /metrics HTTP/1.1" 200 36366 "" "Prometheus/2.43.0" 2023-10-09T05:02:44.729 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T05:02:44.727588+0000 min lat: 0.00115667 max lat: 2.19705 avg lat: 0.0109229 2023-10-09T05:02:44.729 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:02:44.729 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 40 16 58584 58568 91.5021 78 0.0122641 0.0109229 2023-10-09T05:02:45.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:44 smithi133 bash[105589]: cluster 2023-10-09T05:02:43.577097+0000 mgr.y (mgr.25277) 418 : cluster 0 pgmap v369: 89 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 1015/159498 objects degraded (0.636%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T05:02:45.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:44 smithi133 bash[102858]: cluster 2023-10-09T05:02:43.577097+0000 mgr.y (mgr.25277) 418 : cluster 0 pgmap v369: 89 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 1015/159498 objects degraded (0.636%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T05:02:45.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:44 smithi194 bash[92666]: cluster 2023-10-09T05:02:43.577097+0000 mgr.y (mgr.25277) 418 : cluster 0 pgmap v369: 89 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 1015/159498 objects degraded (0.636%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T05:02:45.728 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 41 16 59296 59280 90.3556 44.5 0.00375243 0.010924 2023-10-09T05:02:46.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:46 smithi194 bash[92666]: cluster 2023-10-09T05:02:45.107892+0000 osd.6 (osd.6) 53 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T05:02:46.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:46 smithi194 bash[92666]: cluster 2023-10-09T05:02:45.111054+0000 mon.a (mon.0) 658 : cluster 0 osdmap e134: 8 total, 8 up, 8 in 2023-10-09T05:02:46.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:46 smithi194 bash[92666]: cluster 2023-10-09T05:02:45.111410+0000 osd.5 (osd.5) 85 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T05:02:46.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:46 smithi194 bash[92666]: cluster 2023-10-09T05:02:45.114085+0000 osd.6 (osd.6) 54 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T05:02:46.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:46 smithi194 bash[92666]: cluster 2023-10-09T05:02:45.114135+0000 osd.5 (osd.5) 86 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T05:02:46.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:46 smithi194 bash[92666]: cluster 2023-10-09T05:02:45.121319+0000 osd.4 (osd.4) 65 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T05:02:46.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:46 smithi194 bash[92666]: cluster 2023-10-09T05:02:45.577832+0000 mgr.y (mgr.25277) 419 : cluster 0 pgmap v371: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 1015/170211 objects degraded (0.596%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T05:02:46.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:46 smithi133 bash[105589]: cluster 2023-10-09T05:02:45.107892+0000 osd.6 (osd.6) 53 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T05:02:46.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:46 smithi133 bash[105589]: cluster 2023-10-09T05:02:45.111054+0000 mon.a (mon.0) 658 : cluster 0 osdmap e134: 8 total, 8 up, 8 in 2023-10-09T05:02:46.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:46 smithi133 bash[105589]: cluster 2023-10-09T05:02:45.111410+0000 osd.5 (osd.5) 85 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T05:02:46.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:46 smithi133 bash[105589]: cluster 2023-10-09T05:02:45.114085+0000 osd.6 (osd.6) 54 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T05:02:46.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:46 smithi133 bash[105589]: cluster 2023-10-09T05:02:45.114135+0000 osd.5 (osd.5) 86 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T05:02:46.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:46 smithi133 bash[105589]: cluster 2023-10-09T05:02:45.121319+0000 osd.4 (osd.4) 65 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T05:02:46.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:46 smithi133 bash[105589]: cluster 2023-10-09T05:02:45.577832+0000 mgr.y (mgr.25277) 419 : cluster 0 pgmap v371: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 1015/170211 objects degraded (0.596%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T05:02:46.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:46 smithi133 bash[102858]: cluster 2023-10-09T05:02:45.107892+0000 osd.6 (osd.6) 53 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T05:02:46.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:46 smithi133 bash[102858]: cluster 2023-10-09T05:02:45.111054+0000 mon.a (mon.0) 658 : cluster 0 osdmap e134: 8 total, 8 up, 8 in 2023-10-09T05:02:46.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:46 smithi133 bash[102858]: cluster 2023-10-09T05:02:45.111410+0000 osd.5 (osd.5) 85 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T05:02:46.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:46 smithi133 bash[102858]: cluster 2023-10-09T05:02:45.114085+0000 osd.6 (osd.6) 54 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T05:02:46.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:46 smithi133 bash[102858]: cluster 2023-10-09T05:02:45.114135+0000 osd.5 (osd.5) 86 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T05:02:46.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:46 smithi133 bash[102858]: cluster 2023-10-09T05:02:45.121319+0000 osd.4 (osd.4) 65 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T05:02:46.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:46 smithi133 bash[102858]: cluster 2023-10-09T05:02:45.577832+0000 mgr.y (mgr.25277) 419 : cluster 0 pgmap v371: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 1015/170211 objects degraded (0.596%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T05:02:46.729 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 42 16 60015 59999 89.2742 44.9375 0.00746503 0.011196 2023-10-09T05:02:47.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:47 smithi194 bash[92666]: cluster 2023-10-09T05:02:46.118347+0000 osd.6 (osd.6) 55 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T05:02:47.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:47 smithi194 bash[92666]: cluster 2023-10-09T05:02:46.119429+0000 osd.5 (osd.5) 87 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T05:02:47.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:47 smithi194 bash[92666]: cluster 2023-10-09T05:02:46.121381+0000 osd.6 (osd.6) 56 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T05:02:47.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:47 smithi194 bash[92666]: cluster 2023-10-09T05:02:46.122720+0000 mon.a (mon.0) 659 : cluster 0 osdmap e135: 8 total, 8 up, 8 in 2023-10-09T05:02:47.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:47 smithi194 bash[92666]: cluster 2023-10-09T05:02:46.138788+0000 osd.4 (osd.4) 66 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T05:02:47.401 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:47 smithi133 bash[102858]: cluster 2023-10-09T05:02:46.118347+0000 osd.6 (osd.6) 55 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T05:02:47.401 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:47 smithi133 bash[102858]: cluster 2023-10-09T05:02:46.119429+0000 osd.5 (osd.5) 87 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T05:02:47.401 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:47 smithi133 bash[102858]: cluster 2023-10-09T05:02:46.121381+0000 osd.6 (osd.6) 56 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T05:02:47.401 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:47 smithi133 bash[102858]: cluster 2023-10-09T05:02:46.122720+0000 mon.a (mon.0) 659 : cluster 0 osdmap e135: 8 total, 8 up, 8 in 2023-10-09T05:02:47.401 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:47 smithi133 bash[102858]: cluster 2023-10-09T05:02:46.138788+0000 osd.4 (osd.4) 66 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T05:02:47.402 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:47 smithi133 bash[105589]: cluster 2023-10-09T05:02:46.118347+0000 osd.6 (osd.6) 55 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T05:02:47.402 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:47 smithi133 bash[105589]: cluster 2023-10-09T05:02:46.119429+0000 osd.5 (osd.5) 87 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T05:02:47.402 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:47 smithi133 bash[105589]: cluster 2023-10-09T05:02:46.121381+0000 osd.6 (osd.6) 56 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T05:02:47.402 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:47 smithi133 bash[105589]: cluster 2023-10-09T05:02:46.122720+0000 mon.a (mon.0) 659 : cluster 0 osdmap e135: 8 total, 8 up, 8 in 2023-10-09T05:02:47.402 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:47 smithi133 bash[105589]: cluster 2023-10-09T05:02:46.138788+0000 osd.4 (osd.4) 66 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T05:02:47.729 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 43 16 61227 61211 88.9595 75.75 0.00820148 0.0112351 2023-10-09T05:02:48.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:48 smithi194 bash[92666]: cluster 2023-10-09T05:02:47.578524+0000 mgr.y (mgr.25277) 420 : cluster 0 pgmap v373: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 640/175644 objects degraded (0.364%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T05:02:48.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:48 smithi194 bash[92666]: cluster 2023-10-09T05:02:48.095461+0000 mon.a (mon.0) 660 : cluster 3 Health check update: Degraded data redundancy: 1015/170211 objects degraded (0.596%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:02:48.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:48 smithi133 bash[105589]: cluster 2023-10-09T05:02:47.578524+0000 mgr.y (mgr.25277) 420 : cluster 0 pgmap v373: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 640/175644 objects degraded (0.364%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T05:02:48.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:48 smithi133 bash[105589]: cluster 2023-10-09T05:02:48.095461+0000 mon.a (mon.0) 660 : cluster 3 Health check update: Degraded data redundancy: 1015/170211 objects degraded (0.596%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:02:48.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:48 smithi133 bash[102858]: cluster 2023-10-09T05:02:47.578524+0000 mgr.y (mgr.25277) 420 : cluster 0 pgmap v373: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 640/175644 objects degraded (0.364%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T05:02:48.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:48 smithi133 bash[102858]: cluster 2023-10-09T05:02:48.095461+0000 mon.a (mon.0) 660 : cluster 3 Health check update: Degraded data redundancy: 1015/170211 objects degraded (0.596%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:02:48.729 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 44 16 62427 62411 88.6421 75 0.00350932 0.0112754 2023-10-09T05:02:49.729 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 45 16 63647 63631 88.3665 76.25 0.0164814 0.0113105 2023-10-09T05:02:50.729 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 46 16 64826 64810 88.0473 73.6875 0.00553348 0.0113508 2023-10-09T05:02:51.006 INFO:teuthology.orchestra.run.smithi133.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T05:02:51.006 INFO:teuthology.orchestra.run.smithi133.stdout:alertmanager.a smithi133 *:9093,9094 running (12m) - 20m 15.0M - 0.25.0 c8568f914cd2 f7641cfa53d2 2023-10-09T05:02:51.006 INFO:teuthology.orchestra.run.smithi133.stdout:grafana.a smithi194 *:3000 running (12m) - 19m 74.5M - 9.4.7 2c41d148cca3 18d67948e55e 2023-10-09T05:02:51.006 INFO:teuthology.orchestra.run.smithi133.stdout:mds.foo.smithi133.tsuabh smithi133 running (17m) - 17m 15.0M - 17.2.6-1303-gb2903cb0 5517396c50d4 1adbb452599e 2023-10-09T05:02:51.006 INFO:teuthology.orchestra.run.smithi133.stdout:mds.foo.smithi194.yxnfqn smithi194 running (17m) - 17m 18.0M - 17.2.6-1303-gb2903cb0 5517396c50d4 745c16c1ffd2 2023-10-09T05:02:51.006 INFO:teuthology.orchestra.run.smithi133.stdout:mgr.x smithi194 *:8443,9283,8765 running (11m) - 28m 430M - 18.2.0-598-gb4fa663f 3060897fdf97 e7277fed699e 2023-10-09T05:02:51.006 INFO:teuthology.orchestra.run.smithi133.stdout:mgr.y smithi133 *:8443,9283,8765 running (12m) - 31m 489M - 18.2.0-598-gb4fa663f 3060897fdf97 caf6152f91c6 2023-10-09T05:02:51.007 INFO:teuthology.orchestra.run.smithi133.stdout:mon.a smithi133 running (10m) - 31m 58.9M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 ee45bbc6b194 2023-10-09T05:02:51.007 INFO:teuthology.orchestra.run.smithi133.stdout:mon.b smithi194 running (8m) - 29m 40.6M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 ed14c6f4a04d 2023-10-09T05:02:51.007 INFO:teuthology.orchestra.run.smithi133.stdout:mon.c smithi133 running (8m) - 29m 42.1M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 eb3f3f9009e4 2023-10-09T05:02:51.007 INFO:teuthology.orchestra.run.smithi133.stdout:node-exporter.a smithi133 *:9100 running (12m) - 20m 8387k - 1.5.0 0da6a335fe13 f8bdde5940e7 2023-10-09T05:02:51.007 INFO:teuthology.orchestra.run.smithi133.stdout:node-exporter.b smithi194 *:9100 running (12m) - 20m 8464k - 1.5.0 0da6a335fe13 ffab717ac947 2023-10-09T05:02:51.007 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0 smithi133 running (6m) - 28m 1265M 2524M 18.2.0-598-gb4fa663f 3060897fdf97 021c5eccb109 2023-10-09T05:02:51.007 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1 smithi133 running (5m) - 27m 400M 2524M 18.2.0-598-gb4fa663f 3060897fdf97 4728310689ed 2023-10-09T05:02:51.007 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2 smithi133 running (2m) - 26m 418M 2524M 18.2.0-598-gb4fa663f 3060897fdf97 e32c40f5649e 2023-10-09T05:02:51.007 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3 smithi133 running (53s) - 25m - 2524M 18.2.0-598-gb4fa663f 3060897fdf97 809bb1f1cba2 2023-10-09T05:02:51.007 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4 smithi194 running (24m) - 24m 565M 2524M 17.2.6-1303-gb2903cb0 5517396c50d4 c77edffeb061 2023-10-09T05:02:51.008 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5 smithi194 running (23m) - 23m 925M 2524M 17.2.6-1303-gb2903cb0 5517396c50d4 47540765fad9 2023-10-09T05:02:51.008 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6 smithi194 running (22m) - 22m 740M 2524M 17.2.6-1303-gb2903cb0 5517396c50d4 26bbe2f2859f 2023-10-09T05:02:51.008 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7 smithi194 running (21m) - 21m 541M 2524M 17.2.6-1303-gb2903cb0 5517396c50d4 45c343cd22ea 2023-10-09T05:02:51.008 INFO:teuthology.orchestra.run.smithi133.stdout:prometheus.a smithi194 *:9095 running (11m) - 20m 52.6M - 2.43.0 a07b618ecd1d f40ed7def7ab 2023-10-09T05:02:51.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:50 smithi133 bash[105589]: cluster 2023-10-09T05:02:49.579140+0000 mgr.y (mgr.25277) 421 : cluster 0 pgmap v374: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s; 640/177660 objects degraded (0.360%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:02:51.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:50 smithi133 bash[102858]: cluster 2023-10-09T05:02:49.579140+0000 mgr.y (mgr.25277) 421 : cluster 0 pgmap v374: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s; 640/177660 objects degraded (0.360%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:02:51.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:50 smithi194 bash[92666]: cluster 2023-10-09T05:02:49.579140+0000 mgr.y (mgr.25277) 421 : cluster 0 pgmap v374: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s; 640/177660 objects degraded (0.360%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:02:51.729 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 47 16 65950 65934 87.6685 70.25 0.00530438 0.0113998 2023-10-09T05:02:52.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:51 smithi133 bash[102858]: audit 2023-10-09T05:02:50.980931+0000 mgr.y (mgr.25277) 422 : audit 0 from='client.44585 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:02:52.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:51 smithi133 bash[105589]: audit 2023-10-09T05:02:50.980931+0000 mgr.y (mgr.25277) 422 : audit 0 from='client.44585 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:02:52.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:51 smithi194 bash[92666]: audit 2023-10-09T05:02:50.980931+0000 mgr.y (mgr.25277) 422 : audit 0 from='client.44585 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:02:52.357 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph versions' 2023-10-09T05:02:52.651 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:52 smithi133 bash[102858]: cluster 2023-10-09T05:02:51.579965+0000 mgr.y (mgr.25277) 423 : cluster 0 pgmap v375: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 204/191274 objects degraded (0.107%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T05:02:52.651 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:52 smithi133 bash[105589]: cluster 2023-10-09T05:02:51.579965+0000 mgr.y (mgr.25277) 423 : cluster 0 pgmap v375: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 204/191274 objects degraded (0.107%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T05:02:52.729 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 48 16 67080 67064 87.3133 70.625 0.0209681 0.0114466 2023-10-09T05:02:53.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:52 smithi194 bash[92666]: cluster 2023-10-09T05:02:51.579965+0000 mgr.y (mgr.25277) 423 : cluster 0 pgmap v375: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 204/191274 objects degraded (0.107%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T05:02:53.729 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 49 16 68229 68213 86.9968 71.8125 0.00954886 0.0114879 2023-10-09T05:02:54.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:53 smithi133 bash[102858]: cluster 2023-10-09T05:02:53.096486+0000 mon.a (mon.0) 661 : cluster 3 Health check update: Degraded data redundancy: 204/191274 objects degraded (0.107%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:02:54.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:53 smithi133 bash[105589]: cluster 2023-10-09T05:02:53.096486+0000 mon.a (mon.0) 661 : cluster 3 Health check update: Degraded data redundancy: 204/191274 objects degraded (0.107%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:02:54.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:53 smithi194 bash[92666]: cluster 2023-10-09T05:02:53.096486+0000 mon.a (mon.0) 661 : cluster 3 Health check update: Degraded data redundancy: 204/191274 objects degraded (0.107%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:02:54.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:02:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:02:54] "GET /metrics HTTP/1.1" 200 36366 "" "Prometheus/2.43.0" 2023-10-09T05:02:54.729 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 50 16 68904 68888 86.1005 42.1875 0.00375669 0.0114967 2023-10-09T05:02:55.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:54 smithi133 bash[102858]: cluster 2023-10-09T05:02:53.580732+0000 mgr.y (mgr.25277) 424 : cluster 0 pgmap v376: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 204/193494 objects degraded (0.105%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T05:02:55.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:54 smithi133 bash[102858]: cluster 2023-10-09T05:02:54.119051+0000 mon.a (mon.0) 662 : cluster 0 osdmap e136: 8 total, 8 up, 8 in 2023-10-09T05:02:55.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:54 smithi133 bash[102858]: cluster 2023-10-09T05:02:54.129448+0000 osd.6 (osd.6) 57 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T05:02:55.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:54 smithi133 bash[105589]: cluster 2023-10-09T05:02:53.580732+0000 mgr.y (mgr.25277) 424 : cluster 0 pgmap v376: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 204/193494 objects degraded (0.105%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T05:02:55.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:54 smithi133 bash[105589]: cluster 2023-10-09T05:02:54.119051+0000 mon.a (mon.0) 662 : cluster 0 osdmap e136: 8 total, 8 up, 8 in 2023-10-09T05:02:55.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:54 smithi133 bash[105589]: cluster 2023-10-09T05:02:54.129448+0000 osd.6 (osd.6) 57 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T05:02:55.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:54 smithi194 bash[92666]: cluster 2023-10-09T05:02:53.580732+0000 mgr.y (mgr.25277) 424 : cluster 0 pgmap v376: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 204/193494 objects degraded (0.105%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T05:02:55.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:54 smithi194 bash[92666]: cluster 2023-10-09T05:02:54.119051+0000 mon.a (mon.0) 662 : cluster 0 osdmap e136: 8 total, 8 up, 8 in 2023-10-09T05:02:55.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:54 smithi194 bash[92666]: cluster 2023-10-09T05:02:54.129448+0000 osd.6 (osd.6) 57 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T05:02:55.729 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 51 16 69565 69549 85.2222 41.3125 0.0160295 0.0117277 2023-10-09T05:02:56.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:55 smithi133 bash[102858]: cluster 2023-10-09T05:02:54.126955+0000 osd.4 (osd.4) 67 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T05:02:56.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:55 smithi133 bash[102858]: cluster 2023-10-09T05:02:54.131166+0000 osd.5 (osd.5) 88 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T05:02:56.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:55 smithi133 bash[102858]: cluster 2023-10-09T05:02:54.133149+0000 osd.5 (osd.5) 89 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T05:02:56.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:55 smithi133 bash[102858]: cluster 2023-10-09T05:02:55.123025+0000 mon.a (mon.0) 663 : cluster 0 osdmap e137: 8 total, 8 up, 8 in 2023-10-09T05:02:56.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:55 smithi133 bash[102858]: cluster 2023-10-09T05:02:55.149975+0000 osd.6 (osd.6) 58 : cluster 3 failed to encode map e137 with expected crc 2023-10-09T05:02:56.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:55 smithi133 bash[105589]: cluster 2023-10-09T05:02:54.126955+0000 osd.4 (osd.4) 67 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T05:02:56.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:55 smithi133 bash[105589]: cluster 2023-10-09T05:02:54.131166+0000 osd.5 (osd.5) 88 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T05:02:56.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:55 smithi133 bash[105589]: cluster 2023-10-09T05:02:54.133149+0000 osd.5 (osd.5) 89 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T05:02:56.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:55 smithi133 bash[105589]: cluster 2023-10-09T05:02:55.123025+0000 mon.a (mon.0) 663 : cluster 0 osdmap e137: 8 total, 8 up, 8 in 2023-10-09T05:02:56.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:55 smithi133 bash[105589]: cluster 2023-10-09T05:02:55.149975+0000 osd.6 (osd.6) 58 : cluster 3 failed to encode map e137 with expected crc 2023-10-09T05:02:56.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:55 smithi194 bash[92666]: cluster 2023-10-09T05:02:54.126955+0000 osd.4 (osd.4) 67 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T05:02:56.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:55 smithi194 bash[92666]: cluster 2023-10-09T05:02:54.131166+0000 osd.5 (osd.5) 88 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T05:02:56.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:55 smithi194 bash[92666]: cluster 2023-10-09T05:02:54.133149+0000 osd.5 (osd.5) 89 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T05:02:56.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:55 smithi194 bash[92666]: cluster 2023-10-09T05:02:55.123025+0000 mon.a (mon.0) 663 : cluster 0 osdmap e137: 8 total, 8 up, 8 in 2023-10-09T05:02:56.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:55 smithi194 bash[92666]: cluster 2023-10-09T05:02:55.149975+0000 osd.6 (osd.6) 58 : cluster 3 failed to encode map e137 with expected crc 2023-10-09T05:02:56.729 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 52 16 70716 70700 84.9666 71.9375 0.0122224 0.0117639 2023-10-09T05:02:57.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:56 smithi133 bash[102858]: cluster 2023-10-09T05:02:55.119341+0000 osd.4 (osd.4) 68 : cluster 3 failed to encode map e137 with expected crc 2023-10-09T05:02:57.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:56 smithi133 bash[102858]: cluster 2023-10-09T05:02:55.127776+0000 osd.5 (osd.5) 90 : cluster 3 failed to encode map e137 with expected crc 2023-10-09T05:02:57.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:56 smithi133 bash[102858]: cluster 2023-10-09T05:02:55.581346+0000 mgr.y (mgr.25277) 425 : cluster 0 pgmap v379: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 204/201990 objects degraded (0.101%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T05:02:57.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:56 smithi133 bash[105589]: cluster 2023-10-09T05:02:55.119341+0000 osd.4 (osd.4) 68 : cluster 3 failed to encode map e137 with expected crc 2023-10-09T05:02:57.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:56 smithi133 bash[105589]: cluster 2023-10-09T05:02:55.127776+0000 osd.5 (osd.5) 90 : cluster 3 failed to encode map e137 with expected crc 2023-10-09T05:02:57.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:56 smithi133 bash[105589]: cluster 2023-10-09T05:02:55.581346+0000 mgr.y (mgr.25277) 425 : cluster 0 pgmap v379: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 204/201990 objects degraded (0.101%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T05:02:57.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:56 smithi194 bash[92666]: cluster 2023-10-09T05:02:55.119341+0000 osd.4 (osd.4) 68 : cluster 3 failed to encode map e137 with expected crc 2023-10-09T05:02:57.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:56 smithi194 bash[92666]: cluster 2023-10-09T05:02:55.127776+0000 osd.5 (osd.5) 90 : cluster 3 failed to encode map e137 with expected crc 2023-10-09T05:02:57.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:56 smithi194 bash[92666]: cluster 2023-10-09T05:02:55.581346+0000 mgr.y (mgr.25277) 425 : cluster 0 pgmap v379: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 204/201990 objects degraded (0.101%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T05:02:57.202 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:02:57.729 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 53 16 71866 71850 84.7195 71.875 0.0197424 0.0117981 2023-10-09T05:02:58.730 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 54 16 73022 73006 84.4884 72.25 0.0109168 0.0118302 2023-10-09T05:02:59.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:58 smithi133 bash[102858]: cluster 2023-10-09T05:02:57.583121+0000 mgr.y (mgr.25277) 426 : cluster 0 pgmap v380: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 204/205644 objects degraded (0.099%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T05:02:59.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:58 smithi133 bash[102858]: cluster 2023-10-09T05:02:58.097690+0000 mon.a (mon.0) 664 : cluster 3 Health check update: Degraded data redundancy: 204/205644 objects degraded (0.099%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:02:59.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:58 smithi133 bash[102858]: audit 2023-10-09T05:02:58.426049+0000 mon.a (mon.0) 665 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:59.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:58 smithi133 bash[105589]: cluster 2023-10-09T05:02:57.583121+0000 mgr.y (mgr.25277) 426 : cluster 0 pgmap v380: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 204/205644 objects degraded (0.099%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T05:02:59.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:58 smithi133 bash[105589]: cluster 2023-10-09T05:02:58.097690+0000 mon.a (mon.0) 664 : cluster 3 Health check update: Degraded data redundancy: 204/205644 objects degraded (0.099%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:02:59.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:58 smithi133 bash[105589]: audit 2023-10-09T05:02:58.426049+0000 mon.a (mon.0) 665 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:59.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:58 smithi194 bash[92666]: cluster 2023-10-09T05:02:57.583121+0000 mgr.y (mgr.25277) 426 : cluster 0 pgmap v380: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 204/205644 objects degraded (0.099%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T05:02:59.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:58 smithi194 bash[92666]: cluster 2023-10-09T05:02:58.097690+0000 mon.a (mon.0) 664 : cluster 3 Health check update: Degraded data redundancy: 204/205644 objects degraded (0.099%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:02:59.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:58 smithi194 bash[92666]: audit 2023-10-09T05:02:58.426049+0000 mon.a (mon.0) 665 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:59.730 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 55 15 74155 74140 84.2408 70.875 0.00678999 0.011865 2023-10-09T05:02:59.971 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:59 smithi133 bash[102858]: cluster 2023-10-09T05:02:59.661622+0000 mon.a (mon.0) 666 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 204/205644 objects degraded (0.099%), 1 pg degraded) 2023-10-09T05:02:59.971 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:02:59 smithi133 bash[102858]: cluster 2023-10-09T05:02:59.661664+0000 mon.a (mon.0) 667 : cluster 1 Cluster is now healthy 2023-10-09T05:02:59.971 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:59 smithi133 bash[105589]: cluster 2023-10-09T05:02:59.661622+0000 mon.a (mon.0) 666 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 204/205644 objects degraded (0.099%), 1 pg degraded) 2023-10-09T05:02:59.971 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:02:59 smithi133 bash[105589]: cluster 2023-10-09T05:02:59.661664+0000 mon.a (mon.0) 667 : cluster 1 Cluster is now healthy 2023-10-09T05:03:00.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:59 smithi194 bash[92666]: cluster 2023-10-09T05:02:59.661622+0000 mon.a (mon.0) 666 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 204/205644 objects degraded (0.099%), 1 pg degraded) 2023-10-09T05:03:00.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:02:59 smithi194 bash[92666]: cluster 2023-10-09T05:02:59.661664+0000 mon.a (mon.0) 667 : cluster 1 Cluster is now healthy 2023-10-09T05:03:00.730 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 56 16 75222 75206 83.9261 66.625 0.0202139 0.0119093 2023-10-09T05:03:01.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:00 smithi133 bash[102858]: cluster 2023-10-09T05:02:59.585265+0000 mgr.y (mgr.25277) 427 : cluster 0 pgmap v381: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 46 MiB/s wr, 732 op/s; 1.6 MiB/s, 25 objects/s recovering 2023-10-09T05:03:01.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:00 smithi133 bash[105589]: cluster 2023-10-09T05:02:59.585265+0000 mgr.y (mgr.25277) 427 : cluster 0 pgmap v381: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 46 MiB/s wr, 732 op/s; 1.6 MiB/s, 25 objects/s recovering 2023-10-09T05:03:01.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:00 smithi194 bash[92666]: cluster 2023-10-09T05:02:59.585265+0000 mgr.y (mgr.25277) 427 : cluster 0 pgmap v381: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 46 MiB/s wr, 732 op/s; 1.6 MiB/s, 25 objects/s recovering 2023-10-09T05:03:01.730 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 57 15 76289 76274 83.6247 66.75 0.00826474 0.0119512 2023-10-09T05:03:02.730 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 58 16 77324 77308 83.297 64.625 0.00954633 0.011998 2023-10-09T05:03:03.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:02 smithi133 bash[105589]: cluster 2023-10-09T05:03:01.587821+0000 mgr.y (mgr.25277) 428 : cluster 0 pgmap v382: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 1.6 MiB/s, 25 objects/s recovering 2023-10-09T05:03:03.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:02 smithi133 bash[102858]: cluster 2023-10-09T05:03:01.587821+0000 mgr.y (mgr.25277) 428 : cluster 0 pgmap v382: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 1.6 MiB/s, 25 objects/s recovering 2023-10-09T05:03:03.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:02 smithi194 bash[92666]: cluster 2023-10-09T05:03:01.587821+0000 mgr.y (mgr.25277) 428 : cluster 0 pgmap v382: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 1.6 MiB/s, 25 objects/s recovering 2023-10-09T05:03:03.247 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:03:03.247 INFO:teuthology.orchestra.run.smithi133.stdout: "mon": { 2023-10-09T05:03:03.247 INFO:teuthology.orchestra.run.smithi133.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 3 2023-10-09T05:03:03.247 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:03:03.247 INFO:teuthology.orchestra.run.smithi133.stdout: "mgr": { 2023-10-09T05:03:03.248 INFO:teuthology.orchestra.run.smithi133.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 2 2023-10-09T05:03:03.248 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:03:03.248 INFO:teuthology.orchestra.run.smithi133.stdout: "osd": { 2023-10-09T05:03:03.248 INFO:teuthology.orchestra.run.smithi133.stdout: "ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)": 4, 2023-10-09T05:03:03.248 INFO:teuthology.orchestra.run.smithi133.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 4 2023-10-09T05:03:03.248 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:03:03.248 INFO:teuthology.orchestra.run.smithi133.stdout: "mds": { 2023-10-09T05:03:03.248 INFO:teuthology.orchestra.run.smithi133.stdout: "ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)": 2 2023-10-09T05:03:03.248 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:03:03.248 INFO:teuthology.orchestra.run.smithi133.stdout: "overall": { 2023-10-09T05:03:03.248 INFO:teuthology.orchestra.run.smithi133.stdout: "ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)": 6, 2023-10-09T05:03:03.248 INFO:teuthology.orchestra.run.smithi133.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 9 2023-10-09T05:03:03.249 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:03:03.249 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:03:03.730 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 59 16 78350 78334 82.9719 64.125 0.00897995 0.0120459 2023-10-09T05:03:04.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:04 smithi194 bash[92666]: audit 2023-10-09T05:03:03.241020+0000 mon.a (mon.0) 668 : audit 0 from='client.? 172.21.15.133:0/208077550' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:03:04.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:04 smithi194 bash[92666]: cluster 2023-10-09T05:03:03.588532+0000 mgr.y (mgr.25277) 429 : cluster 0 pgmap v383: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 1.4 MiB/s, 21 objects/s recovering 2023-10-09T05:03:04.478 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:03:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:03:04] "GET /metrics HTTP/1.1" 200 36364 "" "Prometheus/2.43.0" 2023-10-09T05:03:04.478 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:04 smithi133 bash[105589]: audit 2023-10-09T05:03:03.241020+0000 mon.a (mon.0) 668 : audit 0 from='client.? 172.21.15.133:0/208077550' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:03:04.479 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:04 smithi133 bash[105589]: cluster 2023-10-09T05:03:03.588532+0000 mgr.y (mgr.25277) 429 : cluster 0 pgmap v383: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 1.4 MiB/s, 21 objects/s recovering 2023-10-09T05:03:04.479 DEBUG:teuthology.parallel:result is None 2023-10-09T05:03:04.479 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:04 smithi133 bash[102858]: audit 2023-10-09T05:03:03.241020+0000 mon.a (mon.0) 668 : audit 0 from='client.? 172.21.15.133:0/208077550' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:03:04.479 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:04 smithi133 bash[102858]: cluster 2023-10-09T05:03:03.588532+0000 mgr.y (mgr.25277) 429 : cluster 0 pgmap v383: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 1.4 MiB/s, 21 objects/s recovering 2023-10-09T05:03:04.730 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T05:03:04.729555+0000 min lat: 0.00115667 max lat: 2.19705 avg lat: 0.0120806 2023-10-09T05:03:04.730 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:03:04.731 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 60 16 79450 79434 82.7348 68.75 0.0100136 0.0120806 2023-10-09T05:03:05.730 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 61 16 80501 80485 82.4552 65.6875 0.0334051 0.012121 2023-10-09T05:03:06.730 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 62 16 81564 81548 82.1968 66.4375 0.0198085 0.0121603 2023-10-09T05:03:07.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:06 smithi133 bash[105589]: cluster 2023-10-09T05:03:05.589256+0000 mgr.y (mgr.25277) 430 : cluster 0 pgmap v384: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 1.2 MiB/s, 19 objects/s recovering 2023-10-09T05:03:07.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:06 smithi133 bash[102858]: cluster 2023-10-09T05:03:05.589256+0000 mgr.y (mgr.25277) 430 : cluster 0 pgmap v384: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 1.2 MiB/s, 19 objects/s recovering 2023-10-09T05:03:07.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:06 smithi194 bash[92666]: cluster 2023-10-09T05:03:05.589256+0000 mgr.y (mgr.25277) 430 : cluster 0 pgmap v384: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 1.2 MiB/s, 19 objects/s recovering 2023-10-09T05:03:07.730 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 63 16 82602 82586 81.9217 64.875 0.0114098 0.0122003 2023-10-09T05:03:08.731 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 64 16 83648 83632 81.6631 65.375 0.0180996 0.0122399 2023-10-09T05:03:09.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:08 smithi133 bash[105589]: cluster 2023-10-09T05:03:07.589777+0000 mgr.y (mgr.25277) 431 : cluster 0 pgmap v385: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 1.1 MiB/s, 17 objects/s recovering 2023-10-09T05:03:09.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:08 smithi133 bash[105589]: audit 2023-10-09T05:03:08.574310+0000 mon.a (mon.0) 669 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:03:09.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:08 smithi133 bash[102858]: cluster 2023-10-09T05:03:07.589777+0000 mgr.y (mgr.25277) 431 : cluster 0 pgmap v385: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 1.1 MiB/s, 17 objects/s recovering 2023-10-09T05:03:09.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:08 smithi133 bash[102858]: audit 2023-10-09T05:03:08.574310+0000 mon.a (mon.0) 669 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:03:09.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:08 smithi194 bash[92666]: cluster 2023-10-09T05:03:07.589777+0000 mgr.y (mgr.25277) 431 : cluster 0 pgmap v385: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 1.1 MiB/s, 17 objects/s recovering 2023-10-09T05:03:09.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:08 smithi194 bash[92666]: audit 2023-10-09T05:03:08.574310+0000 mon.a (mon.0) 669 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:03:09.731 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 65 16 84689 84673 81.4076 65.0625 0.0187568 0.0122784 2023-10-09T05:03:10.731 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 66 16 85723 85707 81.1532 64.625 0.010239 0.0123167 2023-10-09T05:03:11.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:10 smithi133 bash[102858]: cluster 2023-10-09T05:03:09.590275+0000 mgr.y (mgr.25277) 432 : cluster 0 pgmap v386: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s; 1.1 MiB/s, 17 objects/s recovering 2023-10-09T05:03:11.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:10 smithi133 bash[105589]: cluster 2023-10-09T05:03:09.590275+0000 mgr.y (mgr.25277) 432 : cluster 0 pgmap v386: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s; 1.1 MiB/s, 17 objects/s recovering 2023-10-09T05:03:11.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:10 smithi194 bash[92666]: cluster 2023-10-09T05:03:09.590275+0000 mgr.y (mgr.25277) 432 : cluster 0 pgmap v386: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s; 1.1 MiB/s, 17 objects/s recovering 2023-10-09T05:03:11.731 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 67 16 86738 86722 80.8887 63.4375 0.0120515 0.012357 2023-10-09T05:03:12.731 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 68 16 87755 87739 80.6338 63.5625 0.00924 0.0123959 2023-10-09T05:03:13.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:12 smithi133 bash[102858]: cluster 2023-10-09T05:03:11.590939+0000 mgr.y (mgr.25277) 433 : cluster 0 pgmap v387: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T05:03:13.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:12 smithi133 bash[105589]: cluster 2023-10-09T05:03:11.590939+0000 mgr.y (mgr.25277) 433 : cluster 0 pgmap v387: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T05:03:13.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:12 smithi194 bash[92666]: cluster 2023-10-09T05:03:11.590939+0000 mgr.y (mgr.25277) 433 : cluster 0 pgmap v387: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T05:03:13.731 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 69 16 88743 88727 80.3601 61.75 0.0106875 0.0124376 2023-10-09T05:03:14.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:14 smithi133 bash[102858]: audit 2023-10-09T05:03:13.435864+0000 mon.a (mon.0) 670 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:03:14.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:14 smithi133 bash[102858]: audit 2023-10-09T05:03:13.437375+0000 mon.a (mon.0) 671 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:14.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:14 smithi133 bash[102858]: cluster 2023-10-09T05:03:13.593602+0000 mgr.y (mgr.25277) 434 : cluster 0 pgmap v388: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 60 MiB/s wr, 964 op/s 2023-10-09T05:03:14.550 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:03:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:03:14] "GET /metrics HTTP/1.1" 200 36336 "" "Prometheus/2.43.0" 2023-10-09T05:03:14.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:14 smithi133 bash[105589]: audit 2023-10-09T05:03:13.435864+0000 mon.a (mon.0) 670 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:03:14.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:14 smithi133 bash[105589]: audit 2023-10-09T05:03:13.437375+0000 mon.a (mon.0) 671 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:14.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:14 smithi133 bash[105589]: cluster 2023-10-09T05:03:13.593602+0000 mgr.y (mgr.25277) 434 : cluster 0 pgmap v388: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 60 MiB/s wr, 964 op/s 2023-10-09T05:03:14.731 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 70 16 89746 89730 80.1075 62.6875 0.0197643 0.0124781 2023-10-09T05:03:14.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:14 smithi194 bash[92666]: audit 2023-10-09T05:03:13.435864+0000 mon.a (mon.0) 670 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:03:14.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:14 smithi194 bash[92666]: audit 2023-10-09T05:03:13.437375+0000 mon.a (mon.0) 671 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:14.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:14 smithi194 bash[92666]: cluster 2023-10-09T05:03:13.593602+0000 mgr.y (mgr.25277) 434 : cluster 0 pgmap v388: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 60 MiB/s wr, 964 op/s 2023-10-09T05:03:15.731 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 71 16 90723 90707 79.8392 61.0625 0.0222308 0.0125174 2023-10-09T05:03:16.731 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 72 16 91715 91699 79.5913 62 0.0135796 0.012559 2023-10-09T05:03:17.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:16 smithi133 bash[102858]: cluster 2023-10-09T05:03:15.594381+0000 mgr.y (mgr.25277) 435 : cluster 0 pgmap v389: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-09T05:03:17.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:16 smithi133 bash[105589]: cluster 2023-10-09T05:03:15.594381+0000 mgr.y (mgr.25277) 435 : cluster 0 pgmap v389: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-09T05:03:17.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:16 smithi194 bash[92666]: cluster 2023-10-09T05:03:15.594381+0000 mgr.y (mgr.25277) 435 : cluster 0 pgmap v389: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-09T05:03:17.731 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 73 16 92676 92660 79.3237 60.0625 0.0118159 0.0126009 2023-10-09T05:03:18.732 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 74 16 93644 93628 79.0693 60.5 0.0114584 0.0126414 2023-10-09T05:03:19.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:18 smithi133 bash[105589]: cluster 2023-10-09T05:03:17.594954+0000 mgr.y (mgr.25277) 436 : cluster 0 pgmap v390: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s 2023-10-09T05:03:19.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:18 smithi133 bash[102858]: cluster 2023-10-09T05:03:17.594954+0000 mgr.y (mgr.25277) 436 : cluster 0 pgmap v390: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s 2023-10-09T05:03:19.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:18 smithi194 bash[92666]: cluster 2023-10-09T05:03:17.594954+0000 mgr.y (mgr.25277) 436 : cluster 0 pgmap v390: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s 2023-10-09T05:03:19.732 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 75 16 94594 94578 78.8066 59.375 0.00700944 0.0126837 2023-10-09T05:03:20.732 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 76 16 95536 95520 78.5443 58.875 0.00781512 0.0127255 2023-10-09T05:03:21.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:20 smithi133 bash[105589]: cluster 2023-10-09T05:03:19.595389+0000 mgr.y (mgr.25277) 437 : cluster 0 pgmap v391: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s 2023-10-09T05:03:21.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:20 smithi133 bash[102858]: cluster 2023-10-09T05:03:19.595389+0000 mgr.y (mgr.25277) 437 : cluster 0 pgmap v391: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s 2023-10-09T05:03:21.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:20 smithi194 bash[92666]: cluster 2023-10-09T05:03:19.595389+0000 mgr.y (mgr.25277) 437 : cluster 0 pgmap v391: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s 2023-10-09T05:03:21.732 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 77 16 96480 96464 78.2904 59 0.00873042 0.012767 2023-10-09T05:03:22.732 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 78 16 97422 97406 78.0414 58.875 0.0118426 0.0128078 2023-10-09T05:03:23.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:22 smithi133 bash[105589]: cluster 2023-10-09T05:03:21.597090+0000 mgr.y (mgr.25277) 438 : cluster 0 pgmap v392: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-09T05:03:23.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:22 smithi133 bash[102858]: cluster 2023-10-09T05:03:21.597090+0000 mgr.y (mgr.25277) 438 : cluster 0 pgmap v392: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-09T05:03:23.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:22 smithi194 bash[92666]: cluster 2023-10-09T05:03:21.597090+0000 mgr.y (mgr.25277) 438 : cluster 0 pgmap v392: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-09T05:03:23.732 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 79 16 98358 98342 77.794 58.5 0.0161757 0.0128485 2023-10-09T05:03:24.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:24 smithi194 bash[92666]: cluster 2023-10-09T05:03:23.597635+0000 mgr.y (mgr.25277) 439 : cluster 0 pgmap v393: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s 2023-10-09T05:03:24.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:24 smithi133 bash[102858]: cluster 2023-10-09T05:03:23.597635+0000 mgr.y (mgr.25277) 439 : cluster 0 pgmap v393: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s 2023-10-09T05:03:24.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:03:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:03:24] "GET /metrics HTTP/1.1" 200 36336 "" "Prometheus/2.43.0" 2023-10-09T05:03:24.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:24 smithi133 bash[105589]: cluster 2023-10-09T05:03:23.597635+0000 mgr.y (mgr.25277) 439 : cluster 0 pgmap v393: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s 2023-10-09T05:03:24.732 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T05:03:24.731499+0000 min lat: 0.00115667 max lat: 2.19705 avg lat: 0.0128932 2023-10-09T05:03:24.732 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:03:24.732 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 80 16 99258 99242 77.5246 56.25 0.0135312 0.0128932 2023-10-09T05:03:25.732 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 81 16 100181 100165 77.2797 57.6875 0.027244 0.0129344 2023-10-09T05:03:26.732 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 82 16 101111 101095 77.046 58.125 0.0159613 0.0129738 2023-10-09T05:03:27.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:26 smithi133 bash[105589]: cluster 2023-10-09T05:03:25.598342+0000 mgr.y (mgr.25277) 440 : cluster 0 pgmap v394: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T05:03:27.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:26 smithi133 bash[102858]: cluster 2023-10-09T05:03:25.598342+0000 mgr.y (mgr.25277) 440 : cluster 0 pgmap v394: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T05:03:27.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:26 smithi194 bash[92666]: cluster 2023-10-09T05:03:25.598342+0000 mgr.y (mgr.25277) 440 : cluster 0 pgmap v394: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T05:03:27.732 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 83 16 102044 102028 76.8202 58.3125 0.0100361 0.0130114 2023-10-09T05:03:28.732 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 84 16 102940 102924 76.5723 56 0.0242411 0.013054 2023-10-09T05:03:29.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:28 smithi133 bash[102858]: cluster 2023-10-09T05:03:27.598897+0000 mgr.y (mgr.25277) 441 : cluster 0 pgmap v395: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 898 op/s 2023-10-09T05:03:29.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:28 smithi133 bash[102858]: audit 2023-10-09T05:03:28.427067+0000 mon.a (mon.0) 672 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:29.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:28 smithi133 bash[105589]: cluster 2023-10-09T05:03:27.598897+0000 mgr.y (mgr.25277) 441 : cluster 0 pgmap v395: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 898 op/s 2023-10-09T05:03:29.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:28 smithi133 bash[105589]: audit 2023-10-09T05:03:28.427067+0000 mon.a (mon.0) 672 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:29.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:28 smithi194 bash[92666]: cluster 2023-10-09T05:03:27.598897+0000 mgr.y (mgr.25277) 441 : cluster 0 pgmap v395: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 898 op/s 2023-10-09T05:03:29.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:28 smithi194 bash[92666]: audit 2023-10-09T05:03:28.427067+0000 mon.a (mon.0) 672 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:29.733 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 85 16 103835 103819 76.3295 55.9375 0.0383157 0.013095 2023-10-09T05:03:30.733 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 86 16 104737 104721 76.0974 56.375 0.0145489 0.0131349 2023-10-09T05:03:31.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:30 smithi133 bash[102858]: cluster 2023-10-09T05:03:29.601873+0000 mgr.y (mgr.25277) 442 : cluster 0 pgmap v396: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 869 op/s 2023-10-09T05:03:31.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:30 smithi133 bash[105589]: cluster 2023-10-09T05:03:29.601873+0000 mgr.y (mgr.25277) 442 : cluster 0 pgmap v396: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 869 op/s 2023-10-09T05:03:31.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:30 smithi194 bash[92666]: cluster 2023-10-09T05:03:29.601873+0000 mgr.y (mgr.25277) 442 : cluster 0 pgmap v396: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 869 op/s 2023-10-09T05:03:31.733 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 87 16 105617 105601 75.8548 55 0.0138817 0.0131757 2023-10-09T05:03:32.733 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 88 16 106496 106480 75.6171 54.9375 0.00892758 0.0132171 2023-10-09T05:03:33.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:32 smithi133 bash[102858]: cluster 2023-10-09T05:03:31.602618+0000 mgr.y (mgr.25277) 443 : cluster 0 pgmap v397: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T05:03:33.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:32 smithi133 bash[105589]: cluster 2023-10-09T05:03:31.602618+0000 mgr.y (mgr.25277) 443 : cluster 0 pgmap v397: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T05:03:33.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:32 smithi194 bash[92666]: cluster 2023-10-09T05:03:31.602618+0000 mgr.y (mgr.25277) 443 : cluster 0 pgmap v397: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T05:03:33.733 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 89 16 107386 107370 75.3924 55.625 0.0121668 0.0132585 2023-10-09T05:03:34.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:34 smithi194 bash[92666]: cluster 2023-10-09T05:03:33.607021+0000 mgr.y (mgr.25277) 444 : cluster 0 pgmap v398: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 839 op/s 2023-10-09T05:03:34.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:03:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:03:34] "GET /metrics HTTP/1.1" 200 36341 "" "Prometheus/2.43.0" 2023-10-09T05:03:34.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:34 smithi133 bash[105589]: cluster 2023-10-09T05:03:33.607021+0000 mgr.y (mgr.25277) 444 : cluster 0 pgmap v398: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 839 op/s 2023-10-09T05:03:34.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:34 smithi133 bash[102858]: cluster 2023-10-09T05:03:33.607021+0000 mgr.y (mgr.25277) 444 : cluster 0 pgmap v398: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 839 op/s 2023-10-09T05:03:34.733 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 90 7 108241 108234 75.1546 54 0.0141764 0.0133 2023-10-09T05:03:35.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:35 smithi133 bash[102858]: audit 2023-10-09T05:03:35.191980+0000 mon.a (mon.0) 673 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:03:35.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:35 smithi133 bash[105589]: audit 2023-10-09T05:03:35.191980+0000 mon.a (mon.0) 673 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:03:35.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:35 smithi194 bash[92666]: audit 2023-10-09T05:03:35.191980+0000 mon.a (mon.0) 673 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:03:35.734 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Total time run: 90.0128 2023-10-09T05:03:35.734 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Total writes made: 108241 2023-10-09T05:03:35.734 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Write size: 65536 2023-10-09T05:03:35.734 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Object size: 65536 2023-10-09T05:03:35.734 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Bandwidth (MB/sec): 75.1567 2023-10-09T05:03:35.734 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Stddev Bandwidth: 32.8922 2023-10-09T05:03:35.734 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Max bandwidth (MB/sec): 197.562 2023-10-09T05:03:35.734 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Min bandwidth (MB/sec): 0 2023-10-09T05:03:35.734 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Average IOPS: 1202 2023-10-09T05:03:35.734 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Stddev IOPS: 526.279 2023-10-09T05:03:35.735 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Max IOPS: 3161 2023-10-09T05:03:35.735 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Min IOPS: 0 2023-10-09T05:03:35.735 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Average Latency(s): 0.0133005 2023-10-09T05:03:35.735 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Stddev Latency(s): 0.038959 2023-10-09T05:03:35.735 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Max latency(s): 2.19705 2023-10-09T05:03:35.735 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Min latency(s): 0.00115667 2023-10-09T05:03:35.743 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T05:03:36.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:36 smithi133 bash[102858]: cluster 2023-10-09T05:03:35.608160+0000 mgr.y (mgr.25277) 445 : cluster 0 pgmap v399: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 963 op/s 2023-10-09T05:03:36.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:36 smithi133 bash[105589]: cluster 2023-10-09T05:03:35.608160+0000 mgr.y (mgr.25277) 445 : cluster 0 pgmap v399: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 963 op/s 2023-10-09T05:03:36.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:36 smithi194 bash[92666]: cluster 2023-10-09T05:03:35.608160+0000 mgr.y (mgr.25277) 445 : cluster 0 pgmap v399: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 963 op/s 2023-10-09T05:03:39.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:38 smithi133 bash[102858]: cluster 2023-10-09T05:03:37.608847+0000 mgr.y (mgr.25277) 446 : cluster 0 pgmap v400: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s 2023-10-09T05:03:39.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:38 smithi133 bash[105589]: cluster 2023-10-09T05:03:37.608847+0000 mgr.y (mgr.25277) 446 : cluster 0 pgmap v400: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s 2023-10-09T05:03:39.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:38 smithi194 bash[92666]: cluster 2023-10-09T05:03:37.608847+0000 mgr.y (mgr.25277) 446 : cluster 0 pgmap v400: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s 2023-10-09T05:03:41.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:40 smithi133 bash[102858]: cluster 2023-10-09T05:03:39.609416+0000 mgr.y (mgr.25277) 447 : cluster 0 pgmap v401: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 47 MiB/s wr, 781 op/s 2023-10-09T05:03:41.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:40 smithi133 bash[105589]: cluster 2023-10-09T05:03:39.609416+0000 mgr.y (mgr.25277) 447 : cluster 0 pgmap v401: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 47 MiB/s wr, 781 op/s 2023-10-09T05:03:41.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:40 smithi194 bash[92666]: cluster 2023-10-09T05:03:39.609416+0000 mgr.y (mgr.25277) 447 : cluster 0 pgmap v401: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 47 MiB/s wr, 781 op/s 2023-10-09T05:03:42.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:42 smithi133 bash[105589]: audit 2023-10-09T05:03:41.511238+0000 mon.a (mon.0) 674 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:03:42.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:42 smithi133 bash[105589]: audit 2023-10-09T05:03:41.522324+0000 mon.a (mon.0) 675 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:03:42.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:42 smithi133 bash[105589]: cluster 2023-10-09T05:03:41.610147+0000 mgr.y (mgr.25277) 448 : cluster 0 pgmap v402: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 969 op/s 2023-10-09T05:03:42.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:42 smithi133 bash[105589]: audit 2023-10-09T05:03:41.862038+0000 mon.a (mon.0) 676 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:03:42.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:42 smithi133 bash[105589]: audit 2023-10-09T05:03:41.881305+0000 mon.a (mon.0) 677 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:03:42.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:42 smithi133 bash[105589]: audit 2023-10-09T05:03:42.433384+0000 mon.a (mon.0) 678 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:03:42.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:42 smithi133 bash[105589]: audit 2023-10-09T05:03:42.434674+0000 mon.a (mon.0) 679 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:03:42.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:42 smithi133 bash[105589]: audit 2023-10-09T05:03:42.442581+0000 mon.a (mon.0) 680 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:03:42.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:42 smithi133 bash[102858]: audit 2023-10-09T05:03:41.511238+0000 mon.a (mon.0) 674 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:03:42.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:42 smithi133 bash[102858]: audit 2023-10-09T05:03:41.522324+0000 mon.a (mon.0) 675 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:03:42.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:42 smithi133 bash[102858]: cluster 2023-10-09T05:03:41.610147+0000 mgr.y (mgr.25277) 448 : cluster 0 pgmap v402: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 969 op/s 2023-10-09T05:03:42.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:42 smithi133 bash[102858]: audit 2023-10-09T05:03:41.862038+0000 mon.a (mon.0) 676 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:03:42.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:42 smithi133 bash[102858]: audit 2023-10-09T05:03:41.881305+0000 mon.a (mon.0) 677 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:03:42.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:42 smithi133 bash[102858]: audit 2023-10-09T05:03:42.433384+0000 mon.a (mon.0) 678 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:03:42.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:42 smithi133 bash[102858]: audit 2023-10-09T05:03:42.434674+0000 mon.a (mon.0) 679 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:03:42.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:42 smithi133 bash[102858]: audit 2023-10-09T05:03:42.442581+0000 mon.a (mon.0) 680 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:03:42.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:42 smithi194 bash[92666]: audit 2023-10-09T05:03:41.511238+0000 mon.a (mon.0) 674 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:03:42.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:42 smithi194 bash[92666]: audit 2023-10-09T05:03:41.522324+0000 mon.a (mon.0) 675 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:03:42.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:42 smithi194 bash[92666]: cluster 2023-10-09T05:03:41.610147+0000 mgr.y (mgr.25277) 448 : cluster 0 pgmap v402: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 969 op/s 2023-10-09T05:03:42.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:42 smithi194 bash[92666]: audit 2023-10-09T05:03:41.862038+0000 mon.a (mon.0) 676 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:03:42.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:42 smithi194 bash[92666]: audit 2023-10-09T05:03:41.881305+0000 mon.a (mon.0) 677 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:03:42.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:42 smithi194 bash[92666]: audit 2023-10-09T05:03:42.433384+0000 mon.a (mon.0) 678 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:03:42.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:42 smithi194 bash[92666]: audit 2023-10-09T05:03:42.434674+0000 mon.a (mon.0) 679 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:03:42.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:42 smithi194 bash[92666]: audit 2023-10-09T05:03:42.442581+0000 mon.a (mon.0) 680 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:03:44.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:44 smithi194 bash[92666]: audit 2023-10-09T05:03:43.427310+0000 mon.a (mon.0) 681 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:44.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:44 smithi194 bash[92666]: cluster 2023-10-09T05:03:43.610615+0000 mgr.y (mgr.25277) 449 : cluster 0 pgmap v403: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 23 MiB/s wr, 730 op/s 2023-10-09T05:03:44.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:44 smithi133 bash[102858]: audit 2023-10-09T05:03:43.427310+0000 mon.a (mon.0) 681 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:44.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:44 smithi133 bash[102858]: cluster 2023-10-09T05:03:43.610615+0000 mgr.y (mgr.25277) 449 : cluster 0 pgmap v403: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 23 MiB/s wr, 730 op/s 2023-10-09T05:03:44.550 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:03:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:03:44] "GET /metrics HTTP/1.1" 200 36351 "" "Prometheus/2.43.0" 2023-10-09T05:03:44.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:44 smithi133 bash[105589]: audit 2023-10-09T05:03:43.427310+0000 mon.a (mon.0) 681 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:44.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:44 smithi133 bash[105589]: cluster 2023-10-09T05:03:43.610615+0000 mgr.y (mgr.25277) 449 : cluster 0 pgmap v403: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 23 MiB/s wr, 730 op/s 2023-10-09T05:03:47.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:46 smithi133 bash[102858]: cluster 2023-10-09T05:03:45.611251+0000 mgr.y (mgr.25277) 450 : cluster 0 pgmap v404: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 842 op/s 2023-10-09T05:03:47.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:46 smithi133 bash[105589]: cluster 2023-10-09T05:03:45.611251+0000 mgr.y (mgr.25277) 450 : cluster 0 pgmap v404: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 842 op/s 2023-10-09T05:03:47.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:46 smithi194 bash[92666]: cluster 2023-10-09T05:03:45.611251+0000 mgr.y (mgr.25277) 450 : cluster 0 pgmap v404: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 842 op/s 2023-10-09T05:03:49.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:48 smithi133 bash[105589]: cluster 2023-10-09T05:03:47.611819+0000 mgr.y (mgr.25277) 451 : cluster 0 pgmap v405: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 6.4 MiB/s wr, 730 op/s 2023-10-09T05:03:49.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:48 smithi133 bash[102858]: cluster 2023-10-09T05:03:47.611819+0000 mgr.y (mgr.25277) 451 : cluster 0 pgmap v405: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 6.4 MiB/s wr, 730 op/s 2023-10-09T05:03:49.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:48 smithi194 bash[92666]: cluster 2023-10-09T05:03:47.611819+0000 mgr.y (mgr.25277) 451 : cluster 0 pgmap v405: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 6.4 MiB/s wr, 730 op/s 2023-10-09T05:03:51.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:50 smithi133 bash[102858]: cluster 2023-10-09T05:03:49.612351+0000 mgr.y (mgr.25277) 452 : cluster 0 pgmap v406: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 1.8 MiB/s wr, 726 op/s 2023-10-09T05:03:51.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:50 smithi133 bash[105589]: cluster 2023-10-09T05:03:49.612351+0000 mgr.y (mgr.25277) 452 : cluster 0 pgmap v406: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 1.8 MiB/s wr, 726 op/s 2023-10-09T05:03:51.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:50 smithi194 bash[92666]: cluster 2023-10-09T05:03:49.612351+0000 mgr.y (mgr.25277) 452 : cluster 0 pgmap v406: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 1.8 MiB/s wr, 726 op/s 2023-10-09T05:03:53.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:52 smithi133 bash[102858]: cluster 2023-10-09T05:03:51.613166+0000 mgr.y (mgr.25277) 453 : cluster 0 pgmap v407: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 64 KiB/s wr, 953 op/s 2023-10-09T05:03:53.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:52 smithi133 bash[105589]: cluster 2023-10-09T05:03:51.613166+0000 mgr.y (mgr.25277) 453 : cluster 0 pgmap v407: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 64 KiB/s wr, 953 op/s 2023-10-09T05:03:53.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:52 smithi194 bash[92666]: cluster 2023-10-09T05:03:51.613166+0000 mgr.y (mgr.25277) 453 : cluster 0 pgmap v407: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 64 KiB/s wr, 953 op/s 2023-10-09T05:03:54.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:54 smithi133 bash[102858]: cluster 2023-10-09T05:03:53.613700+0000 mgr.y (mgr.25277) 454 : cluster 0 pgmap v408: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2023-10-09T05:03:54.550 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:03:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:03:54] "GET /metrics HTTP/1.1" 200 36351 "" "Prometheus/2.43.0" 2023-10-09T05:03:54.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:54 smithi133 bash[105589]: cluster 2023-10-09T05:03:53.613700+0000 mgr.y (mgr.25277) 454 : cluster 0 pgmap v408: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2023-10-09T05:03:54.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:54 smithi194 bash[92666]: cluster 2023-10-09T05:03:53.613700+0000 mgr.y (mgr.25277) 454 : cluster 0 pgmap v408: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2023-10-09T05:03:57.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:56 smithi133 bash[102858]: cluster 2023-10-09T05:03:55.614230+0000 mgr.y (mgr.25277) 455 : cluster 0 pgmap v409: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 880 op/s 2023-10-09T05:03:57.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:56 smithi133 bash[105589]: cluster 2023-10-09T05:03:55.614230+0000 mgr.y (mgr.25277) 455 : cluster 0 pgmap v409: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 880 op/s 2023-10-09T05:03:57.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:56 smithi194 bash[92666]: cluster 2023-10-09T05:03:55.614230+0000 mgr.y (mgr.25277) 455 : cluster 0 pgmap v409: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 880 op/s 2023-10-09T05:03:59.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:58 smithi133 bash[102858]: cluster 2023-10-09T05:03:57.617622+0000 mgr.y (mgr.25277) 456 : cluster 0 pgmap v410: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2023-10-09T05:03:59.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:03:58 smithi133 bash[102858]: audit 2023-10-09T05:03:58.427325+0000 mon.a (mon.0) 682 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:59.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:58 smithi133 bash[105589]: cluster 2023-10-09T05:03:57.617622+0000 mgr.y (mgr.25277) 456 : cluster 0 pgmap v410: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2023-10-09T05:03:59.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:03:58 smithi133 bash[105589]: audit 2023-10-09T05:03:58.427325+0000 mon.a (mon.0) 682 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:59.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:58 smithi194 bash[92666]: cluster 2023-10-09T05:03:57.617622+0000 mgr.y (mgr.25277) 456 : cluster 0 pgmap v410: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2023-10-09T05:03:59.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:03:58 smithi194 bash[92666]: audit 2023-10-09T05:03:58.427325+0000 mon.a (mon.0) 682 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:04:01.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:00 smithi133 bash[105589]: cluster 2023-10-09T05:03:59.618132+0000 mgr.y (mgr.25277) 457 : cluster 0 pgmap v411: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 764 op/s 2023-10-09T05:04:01.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:00 smithi133 bash[102858]: cluster 2023-10-09T05:03:59.618132+0000 mgr.y (mgr.25277) 457 : cluster 0 pgmap v411: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 764 op/s 2023-10-09T05:04:01.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:00 smithi194 bash[92666]: cluster 2023-10-09T05:03:59.618132+0000 mgr.y (mgr.25277) 457 : cluster 0 pgmap v411: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 764 op/s 2023-10-09T05:04:03.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:02 smithi133 bash[105589]: cluster 2023-10-09T05:04:01.618951+0000 mgr.y (mgr.25277) 458 : cluster 0 pgmap v412: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 971 op/s 2023-10-09T05:04:04.185 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:02 smithi133 bash[102858]: cluster 2023-10-09T05:04:01.618951+0000 mgr.y (mgr.25277) 458 : cluster 0 pgmap v412: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 971 op/s 2023-10-09T05:04:04.185 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:02 smithi194 bash[92666]: cluster 2023-10-09T05:04:01.618951+0000 mgr.y (mgr.25277) 458 : cluster 0 pgmap v412: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 971 op/s 2023-10-09T05:04:04.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:04 smithi194 bash[92666]: cluster 2023-10-09T05:04:03.619469+0000 mgr.y (mgr.25277) 459 : cluster 0 pgmap v413: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 750 op/s 2023-10-09T05:04:04.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:04:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:04:04] "GET /metrics HTTP/1.1" 200 36349 "" "Prometheus/2.43.0" 2023-10-09T05:04:04.606 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:04 smithi133 bash[105589]: cluster 2023-10-09T05:04:03.619469+0000 mgr.y (mgr.25277) 459 : cluster 0 pgmap v413: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 750 op/s 2023-10-09T05:04:04.606 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:04 smithi133 bash[102858]: cluster 2023-10-09T05:04:03.619469+0000 mgr.y (mgr.25277) 459 : cluster 0 pgmap v413: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 750 op/s 2023-10-09T05:04:07.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:06 smithi133 bash[105589]: cluster 2023-10-09T05:04:05.620156+0000 mgr.y (mgr.25277) 460 : cluster 0 pgmap v414: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2023-10-09T05:04:07.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:06 smithi133 bash[102858]: cluster 2023-10-09T05:04:05.620156+0000 mgr.y (mgr.25277) 460 : cluster 0 pgmap v414: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2023-10-09T05:04:07.149 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:06 smithi194 bash[92666]: cluster 2023-10-09T05:04:05.620156+0000 mgr.y (mgr.25277) 460 : cluster 0 pgmap v414: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2023-10-09T05:04:09.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:08 smithi133 bash[105589]: cluster 2023-10-09T05:04:07.620888+0000 mgr.y (mgr.25277) 461 : cluster 0 pgmap v415: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 763 op/s 2023-10-09T05:04:09.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:08 smithi133 bash[102858]: cluster 2023-10-09T05:04:07.620888+0000 mgr.y (mgr.25277) 461 : cluster 0 pgmap v415: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 763 op/s 2023-10-09T05:04:09.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:08 smithi194 bash[92666]: cluster 2023-10-09T05:04:07.620888+0000 mgr.y (mgr.25277) 461 : cluster 0 pgmap v415: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 763 op/s 2023-10-09T05:04:11.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:10 smithi133 bash[105589]: cluster 2023-10-09T05:04:09.621487+0000 mgr.y (mgr.25277) 462 : cluster 0 pgmap v416: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2023-10-09T05:04:11.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:10 smithi133 bash[102858]: cluster 2023-10-09T05:04:09.621487+0000 mgr.y (mgr.25277) 462 : cluster 0 pgmap v416: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2023-10-09T05:04:11.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:10 smithi194 bash[92666]: cluster 2023-10-09T05:04:09.621487+0000 mgr.y (mgr.25277) 462 : cluster 0 pgmap v416: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2023-10-09T05:04:13.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:12 smithi133 bash[102858]: cluster 2023-10-09T05:04:11.622158+0000 mgr.y (mgr.25277) 463 : cluster 0 pgmap v417: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 942 op/s 2023-10-09T05:04:13.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:12 smithi133 bash[105589]: cluster 2023-10-09T05:04:11.622158+0000 mgr.y (mgr.25277) 463 : cluster 0 pgmap v417: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 942 op/s 2023-10-09T05:04:13.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:12 smithi194 bash[92666]: cluster 2023-10-09T05:04:11.622158+0000 mgr.y (mgr.25277) 463 : cluster 0 pgmap v417: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 942 op/s 2023-10-09T05:04:14.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:14 smithi133 bash[102858]: audit 2023-10-09T05:04:13.427813+0000 mon.a (mon.0) 683 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:04:14.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:14 smithi133 bash[102858]: cluster 2023-10-09T05:04:13.622734+0000 mgr.y (mgr.25277) 464 : cluster 0 pgmap v418: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-10-09T05:04:14.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:04:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:04:14] "GET /metrics HTTP/1.1" 200 36345 "" "Prometheus/2.43.0" 2023-10-09T05:04:14.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:14 smithi133 bash[105589]: audit 2023-10-09T05:04:13.427813+0000 mon.a (mon.0) 683 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:04:14.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:14 smithi133 bash[105589]: cluster 2023-10-09T05:04:13.622734+0000 mgr.y (mgr.25277) 464 : cluster 0 pgmap v418: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-10-09T05:04:14.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:14 smithi194 bash[92666]: audit 2023-10-09T05:04:13.427813+0000 mon.a (mon.0) 683 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:04:14.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:14 smithi194 bash[92666]: cluster 2023-10-09T05:04:13.622734+0000 mgr.y (mgr.25277) 464 : cluster 0 pgmap v418: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-10-09T05:04:17.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:16 smithi133 bash[102858]: cluster 2023-10-09T05:04:15.623515+0000 mgr.y (mgr.25277) 465 : cluster 0 pgmap v419: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 844 op/s 2023-10-09T05:04:17.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:16 smithi133 bash[105589]: cluster 2023-10-09T05:04:15.623515+0000 mgr.y (mgr.25277) 465 : cluster 0 pgmap v419: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 844 op/s 2023-10-09T05:04:17.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:16 smithi194 bash[92666]: cluster 2023-10-09T05:04:15.623515+0000 mgr.y (mgr.25277) 465 : cluster 0 pgmap v419: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 844 op/s 2023-10-09T05:04:19.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:18 smithi133 bash[102858]: cluster 2023-10-09T05:04:17.624072+0000 mgr.y (mgr.25277) 466 : cluster 0 pgmap v420: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2023-10-09T05:04:19.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:18 smithi133 bash[105589]: cluster 2023-10-09T05:04:17.624072+0000 mgr.y (mgr.25277) 466 : cluster 0 pgmap v420: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2023-10-09T05:04:19.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:18 smithi194 bash[92666]: cluster 2023-10-09T05:04:17.624072+0000 mgr.y (mgr.25277) 466 : cluster 0 pgmap v420: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2023-10-09T05:04:21.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:20 smithi133 bash[102858]: cluster 2023-10-09T05:04:19.624497+0000 mgr.y (mgr.25277) 467 : cluster 0 pgmap v421: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2023-10-09T05:04:21.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:20 smithi133 bash[105589]: cluster 2023-10-09T05:04:19.624497+0000 mgr.y (mgr.25277) 467 : cluster 0 pgmap v421: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2023-10-09T05:04:21.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:20 smithi194 bash[92666]: cluster 2023-10-09T05:04:19.624497+0000 mgr.y (mgr.25277) 467 : cluster 0 pgmap v421: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2023-10-09T05:04:23.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:22 smithi133 bash[102858]: cluster 2023-10-09T05:04:21.625548+0000 mgr.y (mgr.25277) 468 : cluster 0 pgmap v422: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 912 op/s 2023-10-09T05:04:23.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:22 smithi133 bash[105589]: cluster 2023-10-09T05:04:21.625548+0000 mgr.y (mgr.25277) 468 : cluster 0 pgmap v422: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 912 op/s 2023-10-09T05:04:23.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:22 smithi194 bash[92666]: cluster 2023-10-09T05:04:21.625548+0000 mgr.y (mgr.25277) 468 : cluster 0 pgmap v422: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 912 op/s 2023-10-09T05:04:24.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:04:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:04:24] "GET /metrics HTTP/1.1" 200 36345 "" "Prometheus/2.43.0" 2023-10-09T05:04:24.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:24 smithi133 bash[105589]: cluster 2023-10-09T05:04:23.626111+0000 mgr.y (mgr.25277) 469 : cluster 0 pgmap v423: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2023-10-09T05:04:24.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:24 smithi133 bash[102858]: cluster 2023-10-09T05:04:23.626111+0000 mgr.y (mgr.25277) 469 : cluster 0 pgmap v423: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2023-10-09T05:04:24.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:24 smithi194 bash[92666]: cluster 2023-10-09T05:04:23.626111+0000 mgr.y (mgr.25277) 469 : cluster 0 pgmap v423: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2023-10-09T05:04:27.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:26 smithi133 bash[102858]: cluster 2023-10-09T05:04:25.626777+0000 mgr.y (mgr.25277) 470 : cluster 0 pgmap v424: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 826 op/s 2023-10-09T05:04:27.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:26 smithi133 bash[105589]: cluster 2023-10-09T05:04:25.626777+0000 mgr.y (mgr.25277) 470 : cluster 0 pgmap v424: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 826 op/s 2023-10-09T05:04:27.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:26 smithi194 bash[92666]: cluster 2023-10-09T05:04:25.626777+0000 mgr.y (mgr.25277) 470 : cluster 0 pgmap v424: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 826 op/s 2023-10-09T05:04:29.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:28 smithi133 bash[102858]: cluster 2023-10-09T05:04:27.627337+0000 mgr.y (mgr.25277) 471 : cluster 0 pgmap v425: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-10-09T05:04:29.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:28 smithi133 bash[102858]: audit 2023-10-09T05:04:28.427989+0000 mon.a (mon.0) 684 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:04:29.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:28 smithi133 bash[105589]: cluster 2023-10-09T05:04:27.627337+0000 mgr.y (mgr.25277) 471 : cluster 0 pgmap v425: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-10-09T05:04:29.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:28 smithi133 bash[105589]: audit 2023-10-09T05:04:28.427989+0000 mon.a (mon.0) 684 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:04:29.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:28 smithi194 bash[92666]: cluster 2023-10-09T05:04:27.627337+0000 mgr.y (mgr.25277) 471 : cluster 0 pgmap v425: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-10-09T05:04:29.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:28 smithi194 bash[92666]: audit 2023-10-09T05:04:28.427989+0000 mon.a (mon.0) 684 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:04:31.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:30 smithi133 bash[102858]: cluster 2023-10-09T05:04:29.627784+0000 mgr.y (mgr.25277) 472 : cluster 0 pgmap v426: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2023-10-09T05:04:31.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:30 smithi133 bash[105589]: cluster 2023-10-09T05:04:29.627784+0000 mgr.y (mgr.25277) 472 : cluster 0 pgmap v426: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2023-10-09T05:04:31.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:30 smithi194 bash[92666]: cluster 2023-10-09T05:04:29.627784+0000 mgr.y (mgr.25277) 472 : cluster 0 pgmap v426: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2023-10-09T05:04:33.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:32 smithi133 bash[102858]: cluster 2023-10-09T05:04:31.628504+0000 mgr.y (mgr.25277) 473 : cluster 0 pgmap v427: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2023-10-09T05:04:33.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:32 smithi133 bash[105589]: cluster 2023-10-09T05:04:31.628504+0000 mgr.y (mgr.25277) 473 : cluster 0 pgmap v427: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2023-10-09T05:04:33.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:32 smithi194 bash[92666]: cluster 2023-10-09T05:04:31.628504+0000 mgr.y (mgr.25277) 473 : cluster 0 pgmap v427: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2023-10-09T05:04:34.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:34 smithi133 bash[102858]: cluster 2023-10-09T05:04:33.629020+0000 mgr.y (mgr.25277) 474 : cluster 0 pgmap v428: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-10-09T05:04:34.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:04:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:04:34] "GET /metrics HTTP/1.1" 200 36346 "" "Prometheus/2.43.0" 2023-10-09T05:04:34.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:34 smithi133 bash[105589]: cluster 2023-10-09T05:04:33.629020+0000 mgr.y (mgr.25277) 474 : cluster 0 pgmap v428: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-10-09T05:04:34.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:34 smithi194 bash[92666]: cluster 2023-10-09T05:04:33.629020+0000 mgr.y (mgr.25277) 474 : cluster 0 pgmap v428: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-10-09T05:04:37.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:36 smithi133 bash[102858]: cluster 2023-10-09T05:04:35.629666+0000 mgr.y (mgr.25277) 475 : cluster 0 pgmap v429: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 814 op/s 2023-10-09T05:04:37.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:36 smithi133 bash[105589]: cluster 2023-10-09T05:04:35.629666+0000 mgr.y (mgr.25277) 475 : cluster 0 pgmap v429: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 814 op/s 2023-10-09T05:04:37.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:36 smithi194 bash[92666]: cluster 2023-10-09T05:04:35.629666+0000 mgr.y (mgr.25277) 475 : cluster 0 pgmap v429: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 814 op/s 2023-10-09T05:04:39.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:38 smithi133 bash[102858]: cluster 2023-10-09T05:04:37.630198+0000 mgr.y (mgr.25277) 476 : cluster 0 pgmap v430: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2023-10-09T05:04:39.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:38 smithi133 bash[105589]: cluster 2023-10-09T05:04:37.630198+0000 mgr.y (mgr.25277) 476 : cluster 0 pgmap v430: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2023-10-09T05:04:39.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:38 smithi194 bash[92666]: cluster 2023-10-09T05:04:37.630198+0000 mgr.y (mgr.25277) 476 : cluster 0 pgmap v430: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2023-10-09T05:04:41.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:40 smithi133 bash[102858]: cluster 2023-10-09T05:04:39.630755+0000 mgr.y (mgr.25277) 477 : cluster 0 pgmap v431: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-10-09T05:04:41.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:40 smithi133 bash[105589]: cluster 2023-10-09T05:04:39.630755+0000 mgr.y (mgr.25277) 477 : cluster 0 pgmap v431: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-10-09T05:04:41.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:40 smithi194 bash[92666]: cluster 2023-10-09T05:04:39.630755+0000 mgr.y (mgr.25277) 477 : cluster 0 pgmap v431: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-10-09T05:04:43.061 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:42 smithi133 bash[105589]: cluster 2023-10-09T05:04:41.631501+0000 mgr.y (mgr.25277) 478 : cluster 0 pgmap v432: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-10-09T05:04:43.061 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:42 smithi133 bash[105589]: audit 2023-10-09T05:04:42.544098+0000 mon.a (mon.0) 685 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:04:43.061 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:42 smithi133 bash[102858]: cluster 2023-10-09T05:04:41.631501+0000 mgr.y (mgr.25277) 478 : cluster 0 pgmap v432: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-10-09T05:04:43.061 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:42 smithi133 bash[102858]: audit 2023-10-09T05:04:42.544098+0000 mon.a (mon.0) 685 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:04:43.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:42 smithi194 bash[92666]: cluster 2023-10-09T05:04:41.631501+0000 mgr.y (mgr.25277) 478 : cluster 0 pgmap v432: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-10-09T05:04:43.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:42 smithi194 bash[92666]: audit 2023-10-09T05:04:42.544098+0000 mon.a (mon.0) 685 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:04:44.287 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:44 smithi133 bash[105589]: audit 2023-10-09T05:04:43.028691+0000 mon.a (mon.0) 686 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:04:44.287 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:44 smithi133 bash[105589]: audit 2023-10-09T05:04:43.036025+0000 mon.a (mon.0) 687 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:04:44.287 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:44 smithi133 bash[105589]: audit 2023-10-09T05:04:43.428579+0000 mon.a (mon.0) 688 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:04:44.288 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:44 smithi133 bash[102858]: audit 2023-10-09T05:04:43.028691+0000 mon.a (mon.0) 686 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:04:44.288 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:44 smithi133 bash[102858]: audit 2023-10-09T05:04:43.036025+0000 mon.a (mon.0) 687 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:04:44.288 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:44 smithi133 bash[102858]: audit 2023-10-09T05:04:43.428579+0000 mon.a (mon.0) 688 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:04:44.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:44 smithi194 bash[92666]: audit 2023-10-09T05:04:43.028691+0000 mon.a (mon.0) 686 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:04:44.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:44 smithi194 bash[92666]: audit 2023-10-09T05:04:43.036025+0000 mon.a (mon.0) 687 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:04:44.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:44 smithi194 bash[92666]: audit 2023-10-09T05:04:43.428579+0000 mon.a (mon.0) 688 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:04:44.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:04:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:04:44] "GET /metrics HTTP/1.1" 200 36344 "" "Prometheus/2.43.0" 2023-10-09T05:04:45.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:45 smithi133 bash[105589]: cluster 2023-10-09T05:04:43.632023+0000 mgr.y (mgr.25277) 479 : cluster 0 pgmap v433: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-10-09T05:04:45.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:45 smithi133 bash[102858]: cluster 2023-10-09T05:04:43.632023+0000 mgr.y (mgr.25277) 479 : cluster 0 pgmap v433: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-10-09T05:04:45.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:45 smithi194 bash[92666]: cluster 2023-10-09T05:04:43.632023+0000 mgr.y (mgr.25277) 479 : cluster 0 pgmap v433: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-10-09T05:04:47.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:47 smithi133 bash[102858]: cluster 2023-10-09T05:04:45.632565+0000 mgr.y (mgr.25277) 480 : cluster 0 pgmap v434: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2023-10-09T05:04:47.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:47 smithi133 bash[105589]: cluster 2023-10-09T05:04:45.632565+0000 mgr.y (mgr.25277) 480 : cluster 0 pgmap v434: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2023-10-09T05:04:47.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:47 smithi194 bash[92666]: cluster 2023-10-09T05:04:45.632565+0000 mgr.y (mgr.25277) 480 : cluster 0 pgmap v434: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2023-10-09T05:04:49.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:49 smithi194 bash[92666]: cluster 2023-10-09T05:04:47.633164+0000 mgr.y (mgr.25277) 481 : cluster 0 pgmap v435: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-10-09T05:04:49.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:49 smithi133 bash[102858]: cluster 2023-10-09T05:04:47.633164+0000 mgr.y (mgr.25277) 481 : cluster 0 pgmap v435: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-10-09T05:04:49.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:49 smithi133 bash[105589]: cluster 2023-10-09T05:04:47.633164+0000 mgr.y (mgr.25277) 481 : cluster 0 pgmap v435: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-10-09T05:04:50.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:50 smithi194 bash[92666]: audit 2023-10-09T05:04:49.323516+0000 mon.a (mon.0) 689 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:04:50.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:50 smithi194 bash[92666]: audit 2023-10-09T05:04:49.336362+0000 mon.a (mon.0) 690 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:04:50.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:50 smithi194 bash[92666]: cluster 2023-10-09T05:04:49.633682+0000 mgr.y (mgr.25277) 482 : cluster 0 pgmap v436: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2023-10-09T05:04:50.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:50 smithi194 bash[92666]: audit 2023-10-09T05:04:49.819642+0000 mon.a (mon.0) 691 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:04:50.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:50 smithi194 bash[92666]: audit 2023-10-09T05:04:49.828768+0000 mon.a (mon.0) 692 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:04:50.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:50 smithi133 bash[102858]: audit 2023-10-09T05:04:49.323516+0000 mon.a (mon.0) 689 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:04:50.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:50 smithi133 bash[102858]: audit 2023-10-09T05:04:49.336362+0000 mon.a (mon.0) 690 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:04:50.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:50 smithi133 bash[102858]: cluster 2023-10-09T05:04:49.633682+0000 mgr.y (mgr.25277) 482 : cluster 0 pgmap v436: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2023-10-09T05:04:50.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:50 smithi133 bash[102858]: audit 2023-10-09T05:04:49.819642+0000 mon.a (mon.0) 691 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:04:50.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:50 smithi133 bash[102858]: audit 2023-10-09T05:04:49.828768+0000 mon.a (mon.0) 692 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:04:50.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:50 smithi133 bash[105589]: audit 2023-10-09T05:04:49.323516+0000 mon.a (mon.0) 689 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:04:50.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:50 smithi133 bash[105589]: audit 2023-10-09T05:04:49.336362+0000 mon.a (mon.0) 690 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:04:50.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:50 smithi133 bash[105589]: cluster 2023-10-09T05:04:49.633682+0000 mgr.y (mgr.25277) 482 : cluster 0 pgmap v436: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2023-10-09T05:04:50.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:50 smithi133 bash[105589]: audit 2023-10-09T05:04:49.819642+0000 mon.a (mon.0) 691 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:04:50.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:50 smithi133 bash[105589]: audit 2023-10-09T05:04:49.828768+0000 mon.a (mon.0) 692 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:04:51.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:51 smithi194 bash[92666]: audit 2023-10-09T05:04:50.739790+0000 mon.a (mon.0) 693 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:04:51.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:51 smithi194 bash[92666]: audit 2023-10-09T05:04:50.740984+0000 mon.a (mon.0) 694 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:04:51.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:51 smithi194 bash[92666]: audit 2023-10-09T05:04:50.750710+0000 mon.a (mon.0) 695 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:04:51.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:51 smithi133 bash[102858]: audit 2023-10-09T05:04:50.739790+0000 mon.a (mon.0) 693 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:04:51.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:51 smithi133 bash[102858]: audit 2023-10-09T05:04:50.740984+0000 mon.a (mon.0) 694 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:04:51.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:51 smithi133 bash[102858]: audit 2023-10-09T05:04:50.750710+0000 mon.a (mon.0) 695 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:04:51.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:51 smithi133 bash[105589]: audit 2023-10-09T05:04:50.739790+0000 mon.a (mon.0) 693 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:04:51.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:51 smithi133 bash[105589]: audit 2023-10-09T05:04:50.740984+0000 mon.a (mon.0) 694 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:04:51.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:51 smithi133 bash[105589]: audit 2023-10-09T05:04:50.750710+0000 mon.a (mon.0) 695 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:04:52.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:52 smithi194 bash[92666]: cluster 2023-10-09T05:04:51.634503+0000 mgr.y (mgr.25277) 483 : cluster 0 pgmap v437: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 841 op/s 2023-10-09T05:04:52.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:52 smithi133 bash[105589]: cluster 2023-10-09T05:04:51.634503+0000 mgr.y (mgr.25277) 483 : cluster 0 pgmap v437: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 841 op/s 2023-10-09T05:04:52.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:52 smithi133 bash[102858]: cluster 2023-10-09T05:04:51.634503+0000 mgr.y (mgr.25277) 483 : cluster 0 pgmap v437: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 841 op/s 2023-10-09T05:04:54.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:54 smithi133 bash[102858]: cluster 2023-10-09T05:04:53.634952+0000 mgr.y (mgr.25277) 484 : cluster 0 pgmap v438: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-10-09T05:04:54.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:04:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:04:54] "GET /metrics HTTP/1.1" 200 36344 "" "Prometheus/2.43.0" 2023-10-09T05:04:54.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:54 smithi133 bash[105589]: cluster 2023-10-09T05:04:53.634952+0000 mgr.y (mgr.25277) 484 : cluster 0 pgmap v438: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-10-09T05:04:54.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:54 smithi194 bash[92666]: cluster 2023-10-09T05:04:53.634952+0000 mgr.y (mgr.25277) 484 : cluster 0 pgmap v438: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-10-09T05:04:57.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:56 smithi133 bash[102858]: cluster 2023-10-09T05:04:55.640497+0000 mgr.y (mgr.25277) 485 : cluster 0 pgmap v439: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 759 op/s 2023-10-09T05:04:57.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:56 smithi133 bash[105589]: cluster 2023-10-09T05:04:55.640497+0000 mgr.y (mgr.25277) 485 : cluster 0 pgmap v439: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 759 op/s 2023-10-09T05:04:57.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:56 smithi194 bash[92666]: cluster 2023-10-09T05:04:55.640497+0000 mgr.y (mgr.25277) 485 : cluster 0 pgmap v439: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 759 op/s 2023-10-09T05:04:59.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:58 smithi133 bash[102858]: cluster 2023-10-09T05:04:57.640919+0000 mgr.y (mgr.25277) 486 : cluster 0 pgmap v440: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-10-09T05:04:59.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:04:58 smithi133 bash[102858]: audit 2023-10-09T05:04:58.428854+0000 mon.a (mon.0) 696 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:04:59.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:58 smithi133 bash[105589]: cluster 2023-10-09T05:04:57.640919+0000 mgr.y (mgr.25277) 486 : cluster 0 pgmap v440: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-10-09T05:04:59.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:04:58 smithi133 bash[105589]: audit 2023-10-09T05:04:58.428854+0000 mon.a (mon.0) 696 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:04:59.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:58 smithi194 bash[92666]: cluster 2023-10-09T05:04:57.640919+0000 mgr.y (mgr.25277) 486 : cluster 0 pgmap v440: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-10-09T05:04:59.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:04:58 smithi194 bash[92666]: audit 2023-10-09T05:04:58.428854+0000 mon.a (mon.0) 696 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:05:01.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:00 smithi133 bash[105589]: cluster 2023-10-09T05:04:59.641435+0000 mgr.y (mgr.25277) 487 : cluster 0 pgmap v441: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-10-09T05:05:01.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:00 smithi133 bash[102858]: cluster 2023-10-09T05:04:59.641435+0000 mgr.y (mgr.25277) 487 : cluster 0 pgmap v441: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-10-09T05:05:01.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:00 smithi194 bash[92666]: cluster 2023-10-09T05:04:59.641435+0000 mgr.y (mgr.25277) 487 : cluster 0 pgmap v441: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-10-09T05:05:03.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:02 smithi133 bash[105589]: cluster 2023-10-09T05:05:01.642164+0000 mgr.y (mgr.25277) 488 : cluster 0 pgmap v442: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2023-10-09T05:05:03.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:02 smithi133 bash[102858]: cluster 2023-10-09T05:05:01.642164+0000 mgr.y (mgr.25277) 488 : cluster 0 pgmap v442: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2023-10-09T05:05:03.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:02 smithi194 bash[92666]: cluster 2023-10-09T05:05:01.642164+0000 mgr.y (mgr.25277) 488 : cluster 0 pgmap v442: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2023-10-09T05:05:04.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:05:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:05:04] "GET /metrics HTTP/1.1" 200 36342 "" "Prometheus/2.43.0" 2023-10-09T05:05:04.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:04 smithi133 bash[105589]: cluster 2023-10-09T05:05:03.642690+0000 mgr.y (mgr.25277) 489 : cluster 0 pgmap v443: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-10-09T05:05:04.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:04 smithi133 bash[102858]: cluster 2023-10-09T05:05:03.642690+0000 mgr.y (mgr.25277) 489 : cluster 0 pgmap v443: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-10-09T05:05:04.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:04 smithi194 bash[92666]: cluster 2023-10-09T05:05:03.642690+0000 mgr.y (mgr.25277) 489 : cluster 0 pgmap v443: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-10-09T05:05:07.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:06 smithi133 bash[105589]: cluster 2023-10-09T05:05:05.643408+0000 mgr.y (mgr.25277) 490 : cluster 0 pgmap v444: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 776 op/s 2023-10-09T05:05:07.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:06 smithi133 bash[102858]: cluster 2023-10-09T05:05:05.643408+0000 mgr.y (mgr.25277) 490 : cluster 0 pgmap v444: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 776 op/s 2023-10-09T05:05:07.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:06 smithi194 bash[92666]: cluster 2023-10-09T05:05:05.643408+0000 mgr.y (mgr.25277) 490 : cluster 0 pgmap v444: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 776 op/s 2023-10-09T05:05:09.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:08 smithi133 bash[102858]: cluster 2023-10-09T05:05:07.643877+0000 mgr.y (mgr.25277) 491 : cluster 0 pgmap v445: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-10-09T05:05:09.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:08 smithi133 bash[105589]: cluster 2023-10-09T05:05:07.643877+0000 mgr.y (mgr.25277) 491 : cluster 0 pgmap v445: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-10-09T05:05:09.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:08 smithi194 bash[92666]: cluster 2023-10-09T05:05:07.643877+0000 mgr.y (mgr.25277) 491 : cluster 0 pgmap v445: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-10-09T05:05:11.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:10 smithi133 bash[105589]: cluster 2023-10-09T05:05:09.644300+0000 mgr.y (mgr.25277) 492 : cluster 0 pgmap v446: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-10-09T05:05:11.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:10 smithi133 bash[102858]: cluster 2023-10-09T05:05:09.644300+0000 mgr.y (mgr.25277) 492 : cluster 0 pgmap v446: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-10-09T05:05:11.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:10 smithi194 bash[92666]: cluster 2023-10-09T05:05:09.644300+0000 mgr.y (mgr.25277) 492 : cluster 0 pgmap v446: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-10-09T05:05:13.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:12 smithi133 bash[102858]: cluster 2023-10-09T05:05:11.645024+0000 mgr.y (mgr.25277) 493 : cluster 0 pgmap v447: 89 pgs: 89 active+clean; 2.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 862 op/s 2023-10-09T05:05:13.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:12 smithi133 bash[105589]: cluster 2023-10-09T05:05:11.645024+0000 mgr.y (mgr.25277) 493 : cluster 0 pgmap v447: 89 pgs: 89 active+clean; 2.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 862 op/s 2023-10-09T05:05:13.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:12 smithi194 bash[92666]: cluster 2023-10-09T05:05:11.645024+0000 mgr.y (mgr.25277) 493 : cluster 0 pgmap v447: 89 pgs: 89 active+clean; 2.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 862 op/s 2023-10-09T05:05:14.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:14 smithi133 bash[102858]: audit 2023-10-09T05:05:13.429274+0000 mon.a (mon.0) 697 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:05:14.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:14 smithi133 bash[102858]: cluster 2023-10-09T05:05:13.645503+0000 mgr.y (mgr.25277) 494 : cluster 0 pgmap v448: 89 pgs: 89 active+clean; 2.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2023-10-09T05:05:14.550 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:05:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:05:14] "GET /metrics HTTP/1.1" 200 36340 "" "Prometheus/2.43.0" 2023-10-09T05:05:14.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:14 smithi133 bash[105589]: audit 2023-10-09T05:05:13.429274+0000 mon.a (mon.0) 697 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:05:14.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:14 smithi133 bash[105589]: cluster 2023-10-09T05:05:13.645503+0000 mgr.y (mgr.25277) 494 : cluster 0 pgmap v448: 89 pgs: 89 active+clean; 2.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2023-10-09T05:05:14.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:14 smithi194 bash[92666]: audit 2023-10-09T05:05:13.429274+0000 mon.a (mon.0) 697 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:05:14.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:14 smithi194 bash[92666]: cluster 2023-10-09T05:05:13.645503+0000 mgr.y (mgr.25277) 494 : cluster 0 pgmap v448: 89 pgs: 89 active+clean; 2.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2023-10-09T05:05:17.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:16 smithi133 bash[102858]: cluster 2023-10-09T05:05:15.646148+0000 mgr.y (mgr.25277) 495 : cluster 0 pgmap v449: 89 pgs: 89 active+clean; 2.0 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 781 op/s 2023-10-09T05:05:17.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:16 smithi133 bash[105589]: cluster 2023-10-09T05:05:15.646148+0000 mgr.y (mgr.25277) 495 : cluster 0 pgmap v449: 89 pgs: 89 active+clean; 2.0 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 781 op/s 2023-10-09T05:05:17.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:16 smithi194 bash[92666]: cluster 2023-10-09T05:05:15.646148+0000 mgr.y (mgr.25277) 495 : cluster 0 pgmap v449: 89 pgs: 89 active+clean; 2.0 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 781 op/s 2023-10-09T05:05:19.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:18 smithi133 bash[102858]: cluster 2023-10-09T05:05:17.646965+0000 mgr.y (mgr.25277) 496 : cluster 0 pgmap v450: 89 pgs: 89 active+clean; 1.9 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2023-10-09T05:05:19.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:18 smithi133 bash[105589]: cluster 2023-10-09T05:05:17.646965+0000 mgr.y (mgr.25277) 496 : cluster 0 pgmap v450: 89 pgs: 89 active+clean; 1.9 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2023-10-09T05:05:19.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:18 smithi194 bash[92666]: cluster 2023-10-09T05:05:17.646965+0000 mgr.y (mgr.25277) 496 : cluster 0 pgmap v450: 89 pgs: 89 active+clean; 1.9 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2023-10-09T05:05:21.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:20 smithi133 bash[105589]: cluster 2023-10-09T05:05:19.647469+0000 mgr.y (mgr.25277) 497 : cluster 0 pgmap v451: 89 pgs: 89 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-10-09T05:05:21.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:20 smithi133 bash[102858]: cluster 2023-10-09T05:05:19.647469+0000 mgr.y (mgr.25277) 497 : cluster 0 pgmap v451: 89 pgs: 89 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-10-09T05:05:21.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:20 smithi194 bash[92666]: cluster 2023-10-09T05:05:19.647469+0000 mgr.y (mgr.25277) 497 : cluster 0 pgmap v451: 89 pgs: 89 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-10-09T05:05:23.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:22 smithi133 bash[105589]: cluster 2023-10-09T05:05:21.648291+0000 mgr.y (mgr.25277) 498 : cluster 0 pgmap v452: 89 pgs: 89 active+clean; 1.7 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 865 op/s 2023-10-09T05:05:23.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:22 smithi133 bash[102858]: cluster 2023-10-09T05:05:21.648291+0000 mgr.y (mgr.25277) 498 : cluster 0 pgmap v452: 89 pgs: 89 active+clean; 1.7 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 865 op/s 2023-10-09T05:05:23.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:22 smithi194 bash[92666]: cluster 2023-10-09T05:05:21.648291+0000 mgr.y (mgr.25277) 498 : cluster 0 pgmap v452: 89 pgs: 89 active+clean; 1.7 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 865 op/s 2023-10-09T05:05:24.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:24 smithi133 bash[102858]: cluster 2023-10-09T05:05:23.648812+0000 mgr.y (mgr.25277) 499 : cluster 0 pgmap v453: 89 pgs: 89 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-10-09T05:05:24.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:05:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:05:24] "GET /metrics HTTP/1.1" 200 36340 "" "Prometheus/2.43.0" 2023-10-09T05:05:24.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:24 smithi133 bash[105589]: cluster 2023-10-09T05:05:23.648812+0000 mgr.y (mgr.25277) 499 : cluster 0 pgmap v453: 89 pgs: 89 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-10-09T05:05:24.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:24 smithi194 bash[92666]: cluster 2023-10-09T05:05:23.648812+0000 mgr.y (mgr.25277) 499 : cluster 0 pgmap v453: 89 pgs: 89 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-10-09T05:05:26.636 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 05:05:26 smithi194 bash[80638]: logger=sqlstore.transactions t=2023-10-09T05:05:26.22844618Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-10-09T05:05:27.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:26 smithi133 bash[102858]: cluster 2023-10-09T05:05:25.649547+0000 mgr.y (mgr.25277) 500 : cluster 0 pgmap v454: 89 pgs: 89 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2023-10-09T05:05:27.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:26 smithi133 bash[105589]: cluster 2023-10-09T05:05:25.649547+0000 mgr.y (mgr.25277) 500 : cluster 0 pgmap v454: 89 pgs: 89 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2023-10-09T05:05:27.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:26 smithi194 bash[92666]: cluster 2023-10-09T05:05:25.649547+0000 mgr.y (mgr.25277) 500 : cluster 0 pgmap v454: 89 pgs: 89 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2023-10-09T05:05:29.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:28 smithi133 bash[102858]: cluster 2023-10-09T05:05:27.650066+0000 mgr.y (mgr.25277) 501 : cluster 0 pgmap v455: 89 pgs: 89 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-10-09T05:05:29.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:28 smithi133 bash[102858]: audit 2023-10-09T05:05:28.429603+0000 mon.a (mon.0) 698 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:05:29.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:28 smithi133 bash[105589]: cluster 2023-10-09T05:05:27.650066+0000 mgr.y (mgr.25277) 501 : cluster 0 pgmap v455: 89 pgs: 89 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-10-09T05:05:29.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:28 smithi133 bash[105589]: audit 2023-10-09T05:05:28.429603+0000 mon.a (mon.0) 698 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:05:29.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:28 smithi194 bash[92666]: cluster 2023-10-09T05:05:27.650066+0000 mgr.y (mgr.25277) 501 : cluster 0 pgmap v455: 89 pgs: 89 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-10-09T05:05:29.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:28 smithi194 bash[92666]: audit 2023-10-09T05:05:28.429603+0000 mon.a (mon.0) 698 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:05:31.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:30 smithi133 bash[105589]: cluster 2023-10-09T05:05:29.650497+0000 mgr.y (mgr.25277) 502 : cluster 0 pgmap v456: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-10-09T05:05:31.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:30 smithi133 bash[102858]: cluster 2023-10-09T05:05:29.650497+0000 mgr.y (mgr.25277) 502 : cluster 0 pgmap v456: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-10-09T05:05:31.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:30 smithi194 bash[92666]: cluster 2023-10-09T05:05:29.650497+0000 mgr.y (mgr.25277) 502 : cluster 0 pgmap v456: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-10-09T05:05:33.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:32 smithi133 bash[102858]: cluster 2023-10-09T05:05:31.651281+0000 mgr.y (mgr.25277) 503 : cluster 0 pgmap v457: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 860 op/s 2023-10-09T05:05:33.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:32 smithi133 bash[105589]: cluster 2023-10-09T05:05:31.651281+0000 mgr.y (mgr.25277) 503 : cluster 0 pgmap v457: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 860 op/s 2023-10-09T05:05:33.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:32 smithi194 bash[92666]: cluster 2023-10-09T05:05:31.651281+0000 mgr.y (mgr.25277) 503 : cluster 0 pgmap v457: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 860 op/s 2023-10-09T05:05:34.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:34 smithi133 bash[102858]: cluster 2023-10-09T05:05:33.651823+0000 mgr.y (mgr.25277) 504 : cluster 0 pgmap v458: 89 pgs: 89 active+clean; 1.2 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-10-09T05:05:34.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:05:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:05:34] "GET /metrics HTTP/1.1" 200 36345 "" "Prometheus/2.43.0" 2023-10-09T05:05:34.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:34 smithi133 bash[105589]: cluster 2023-10-09T05:05:33.651823+0000 mgr.y (mgr.25277) 504 : cluster 0 pgmap v458: 89 pgs: 89 active+clean; 1.2 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-10-09T05:05:34.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:34 smithi194 bash[92666]: cluster 2023-10-09T05:05:33.651823+0000 mgr.y (mgr.25277) 504 : cluster 0 pgmap v458: 89 pgs: 89 active+clean; 1.2 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-10-09T05:05:37.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:36 smithi133 bash[105589]: cluster 2023-10-09T05:05:35.652362+0000 mgr.y (mgr.25277) 505 : cluster 0 pgmap v459: 89 pgs: 89 active+clean; 1.1 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 782 op/s 2023-10-09T05:05:37.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:36 smithi133 bash[102858]: cluster 2023-10-09T05:05:35.652362+0000 mgr.y (mgr.25277) 505 : cluster 0 pgmap v459: 89 pgs: 89 active+clean; 1.1 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 782 op/s 2023-10-09T05:05:37.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:36 smithi194 bash[92666]: cluster 2023-10-09T05:05:35.652362+0000 mgr.y (mgr.25277) 505 : cluster 0 pgmap v459: 89 pgs: 89 active+clean; 1.1 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 782 op/s 2023-10-09T05:05:39.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:38 smithi133 bash[102858]: cluster 2023-10-09T05:05:37.652905+0000 mgr.y (mgr.25277) 506 : cluster 0 pgmap v460: 89 pgs: 89 active+clean; 1.0 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-10-09T05:05:39.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:38 smithi133 bash[105589]: cluster 2023-10-09T05:05:37.652905+0000 mgr.y (mgr.25277) 506 : cluster 0 pgmap v460: 89 pgs: 89 active+clean; 1.0 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-10-09T05:05:39.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:38 smithi194 bash[92666]: cluster 2023-10-09T05:05:37.652905+0000 mgr.y (mgr.25277) 506 : cluster 0 pgmap v460: 89 pgs: 89 active+clean; 1.0 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-10-09T05:05:41.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:40 smithi133 bash[105589]: cluster 2023-10-09T05:05:39.653478+0000 mgr.y (mgr.25277) 507 : cluster 0 pgmap v461: 89 pgs: 89 active+clean; 1.0 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2023-10-09T05:05:41.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:40 smithi133 bash[102858]: cluster 2023-10-09T05:05:39.653478+0000 mgr.y (mgr.25277) 507 : cluster 0 pgmap v461: 89 pgs: 89 active+clean; 1.0 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2023-10-09T05:05:41.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:40 smithi194 bash[92666]: cluster 2023-10-09T05:05:39.653478+0000 mgr.y (mgr.25277) 507 : cluster 0 pgmap v461: 89 pgs: 89 active+clean; 1.0 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2023-10-09T05:05:43.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:42 smithi133 bash[105589]: cluster 2023-10-09T05:05:41.654184+0000 mgr.y (mgr.25277) 508 : cluster 0 pgmap v462: 89 pgs: 89 active+clean; 844 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2023-10-09T05:05:43.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:42 smithi133 bash[102858]: cluster 2023-10-09T05:05:41.654184+0000 mgr.y (mgr.25277) 508 : cluster 0 pgmap v462: 89 pgs: 89 active+clean; 844 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2023-10-09T05:05:43.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:42 smithi194 bash[92666]: cluster 2023-10-09T05:05:41.654184+0000 mgr.y (mgr.25277) 508 : cluster 0 pgmap v462: 89 pgs: 89 active+clean; 844 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2023-10-09T05:05:44.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:44 smithi133 bash[102858]: audit 2023-10-09T05:05:43.429668+0000 mon.a (mon.0) 699 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:05:44.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:44 smithi133 bash[102858]: cluster 2023-10-09T05:05:43.654691+0000 mgr.y (mgr.25277) 509 : cluster 0 pgmap v463: 89 pgs: 89 active+clean; 800 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-10-09T05:05:44.550 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:05:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:05:44] "GET /metrics HTTP/1.1" 200 36345 "" "Prometheus/2.43.0" 2023-10-09T05:05:44.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:44 smithi133 bash[105589]: audit 2023-10-09T05:05:43.429668+0000 mon.a (mon.0) 699 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:05:44.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:44 smithi133 bash[105589]: cluster 2023-10-09T05:05:43.654691+0000 mgr.y (mgr.25277) 509 : cluster 0 pgmap v463: 89 pgs: 89 active+clean; 800 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-10-09T05:05:44.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:44 smithi194 bash[92666]: audit 2023-10-09T05:05:43.429668+0000 mon.a (mon.0) 699 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:05:44.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:44 smithi194 bash[92666]: cluster 2023-10-09T05:05:43.654691+0000 mgr.y (mgr.25277) 509 : cluster 0 pgmap v463: 89 pgs: 89 active+clean; 800 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-10-09T05:05:47.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:46 smithi133 bash[102858]: cluster 2023-10-09T05:05:45.655379+0000 mgr.y (mgr.25277) 510 : cluster 0 pgmap v464: 89 pgs: 89 active+clean; 673 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2023-10-09T05:05:47.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:46 smithi133 bash[105589]: cluster 2023-10-09T05:05:45.655379+0000 mgr.y (mgr.25277) 510 : cluster 0 pgmap v464: 89 pgs: 89 active+clean; 673 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2023-10-09T05:05:47.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:46 smithi194 bash[92666]: cluster 2023-10-09T05:05:45.655379+0000 mgr.y (mgr.25277) 510 : cluster 0 pgmap v464: 89 pgs: 89 active+clean; 673 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2023-10-09T05:05:49.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:48 smithi133 bash[105589]: cluster 2023-10-09T05:05:47.655896+0000 mgr.y (mgr.25277) 511 : cluster 0 pgmap v465: 89 pgs: 89 active+clean; 617 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-10-09T05:05:49.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:48 smithi133 bash[102858]: cluster 2023-10-09T05:05:47.655896+0000 mgr.y (mgr.25277) 511 : cluster 0 pgmap v465: 89 pgs: 89 active+clean; 617 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-10-09T05:05:49.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:48 smithi194 bash[92666]: cluster 2023-10-09T05:05:47.655896+0000 mgr.y (mgr.25277) 511 : cluster 0 pgmap v465: 89 pgs: 89 active+clean; 617 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-10-09T05:05:51.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:50 smithi133 bash[102858]: cluster 2023-10-09T05:05:49.656399+0000 mgr.y (mgr.25277) 512 : cluster 0 pgmap v466: 89 pgs: 89 active+clean; 573 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-10-09T05:05:51.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:50 smithi133 bash[105589]: cluster 2023-10-09T05:05:49.656399+0000 mgr.y (mgr.25277) 512 : cluster 0 pgmap v466: 89 pgs: 89 active+clean; 573 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-10-09T05:05:51.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:50 smithi194 bash[92666]: cluster 2023-10-09T05:05:49.656399+0000 mgr.y (mgr.25277) 512 : cluster 0 pgmap v466: 89 pgs: 89 active+clean; 573 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-10-09T05:05:52.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:51 smithi133 bash[105589]: audit 2023-10-09T05:05:50.867379+0000 mon.a (mon.0) 700 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:05:52.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:51 smithi133 bash[102858]: audit 2023-10-09T05:05:50.867379+0000 mon.a (mon.0) 700 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:05:52.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:51 smithi194 bash[92666]: audit 2023-10-09T05:05:50.867379+0000 mon.a (mon.0) 700 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:05:53.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:52 smithi133 bash[105589]: cluster 2023-10-09T05:05:51.657124+0000 mgr.y (mgr.25277) 513 : cluster 0 pgmap v467: 89 pgs: 89 active+clean; 390 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2023-10-09T05:05:53.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:52 smithi133 bash[102858]: cluster 2023-10-09T05:05:51.657124+0000 mgr.y (mgr.25277) 513 : cluster 0 pgmap v467: 89 pgs: 89 active+clean; 390 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2023-10-09T05:05:53.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:52 smithi194 bash[92666]: cluster 2023-10-09T05:05:51.657124+0000 mgr.y (mgr.25277) 513 : cluster 0 pgmap v467: 89 pgs: 89 active+clean; 390 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2023-10-09T05:05:54.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:05:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:05:54] "GET /metrics HTTP/1.1" 200 36345 "" "Prometheus/2.43.0" 2023-10-09T05:05:54.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:54 smithi133 bash[105589]: cluster 2023-10-09T05:05:53.657669+0000 mgr.y (mgr.25277) 514 : cluster 0 pgmap v468: 89 pgs: 89 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2023-10-09T05:05:54.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:54 smithi133 bash[102858]: cluster 2023-10-09T05:05:53.657669+0000 mgr.y (mgr.25277) 514 : cluster 0 pgmap v468: 89 pgs: 89 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2023-10-09T05:05:54.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:54 smithi194 bash[92666]: cluster 2023-10-09T05:05:53.657669+0000 mgr.y (mgr.25277) 514 : cluster 0 pgmap v468: 89 pgs: 89 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2023-10-09T05:05:57.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:56 smithi133 bash[102858]: cluster 2023-10-09T05:05:55.658421+0000 mgr.y (mgr.25277) 515 : cluster 0 pgmap v469: 89 pgs: 89 active+clean; 220 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 773 op/s 2023-10-09T05:05:57.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:56 smithi133 bash[105589]: cluster 2023-10-09T05:05:55.658421+0000 mgr.y (mgr.25277) 515 : cluster 0 pgmap v469: 89 pgs: 89 active+clean; 220 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 773 op/s 2023-10-09T05:05:57.140 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:56 smithi194 bash[92666]: cluster 2023-10-09T05:05:55.658421+0000 mgr.y (mgr.25277) 515 : cluster 0 pgmap v469: 89 pgs: 89 active+clean; 220 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 773 op/s 2023-10-09T05:05:58.882 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Removed 108241 objects 2023-10-09T05:05:58.882 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Clean up completed and total clean up time :143.139 2023-10-09T05:05:58.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:58 smithi194 bash[92666]: audit 2023-10-09T05:05:57.597969+0000 mon.a (mon.0) 701 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:05:58.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:58 smithi194 bash[92666]: audit 2023-10-09T05:05:57.611992+0000 mon.a (mon.0) 702 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:05:58.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:58 smithi194 bash[92666]: audit 2023-10-09T05:05:57.631071+0000 mon.a (mon.0) 703 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:05:58.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:58 smithi194 bash[92666]: audit 2023-10-09T05:05:57.639651+0000 mon.a (mon.0) 704 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:05:58.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:58 smithi194 bash[92666]: cluster 2023-10-09T05:05:57.658970+0000 mgr.y (mgr.25277) 516 : cluster 0 pgmap v470: 89 pgs: 89 active+clean; 163 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2023-10-09T05:05:58.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:58 smithi194 bash[92666]: audit 2023-10-09T05:05:58.357705+0000 mon.a (mon.0) 705 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:05:58.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:58 smithi194 bash[92666]: audit 2023-10-09T05:05:58.358916+0000 mon.a (mon.0) 706 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:05:58.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:58 smithi194 bash[92666]: audit 2023-10-09T05:05:58.368541+0000 mon.a (mon.0) 707 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:05:58.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:05:58 smithi194 bash[92666]: audit 2023-10-09T05:05:58.434748+0000 mon.a (mon.0) 708 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:05:59.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:58 smithi133 bash[102858]: audit 2023-10-09T05:05:57.597969+0000 mon.a (mon.0) 701 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:05:59.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:58 smithi133 bash[102858]: audit 2023-10-09T05:05:57.611992+0000 mon.a (mon.0) 702 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:05:59.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:58 smithi133 bash[102858]: audit 2023-10-09T05:05:57.631071+0000 mon.a (mon.0) 703 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:05:59.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:58 smithi133 bash[102858]: audit 2023-10-09T05:05:57.639651+0000 mon.a (mon.0) 704 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:05:59.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:58 smithi133 bash[102858]: cluster 2023-10-09T05:05:57.658970+0000 mgr.y (mgr.25277) 516 : cluster 0 pgmap v470: 89 pgs: 89 active+clean; 163 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2023-10-09T05:05:59.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:58 smithi133 bash[102858]: audit 2023-10-09T05:05:58.357705+0000 mon.a (mon.0) 705 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:05:59.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:58 smithi133 bash[102858]: audit 2023-10-09T05:05:58.358916+0000 mon.a (mon.0) 706 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:05:59.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:58 smithi133 bash[102858]: audit 2023-10-09T05:05:58.368541+0000 mon.a (mon.0) 707 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:05:59.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:05:58 smithi133 bash[102858]: audit 2023-10-09T05:05:58.434748+0000 mon.a (mon.0) 708 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:05:59.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:58 smithi133 bash[105589]: audit 2023-10-09T05:05:57.597969+0000 mon.a (mon.0) 701 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:05:59.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:58 smithi133 bash[105589]: audit 2023-10-09T05:05:57.611992+0000 mon.a (mon.0) 702 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:05:59.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:58 smithi133 bash[105589]: audit 2023-10-09T05:05:57.631071+0000 mon.a (mon.0) 703 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:05:59.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:58 smithi133 bash[105589]: audit 2023-10-09T05:05:57.639651+0000 mon.a (mon.0) 704 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:05:59.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:58 smithi133 bash[105589]: cluster 2023-10-09T05:05:57.658970+0000 mgr.y (mgr.25277) 516 : cluster 0 pgmap v470: 89 pgs: 89 active+clean; 163 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2023-10-09T05:05:59.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:58 smithi133 bash[105589]: audit 2023-10-09T05:05:58.357705+0000 mon.a (mon.0) 705 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:05:59.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:58 smithi133 bash[105589]: audit 2023-10-09T05:05:58.358916+0000 mon.a (mon.0) 706 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:05:59.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:58 smithi133 bash[105589]: audit 2023-10-09T05:05:58.368541+0000 mon.a (mon.0) 707 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:05:59.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:05:58 smithi133 bash[105589]: audit 2023-10-09T05:05:58.434748+0000 mon.a (mon.0) 708 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:06:01.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:00 smithi133 bash[105589]: cluster 2023-10-09T05:05:59.659741+0000 mgr.y (mgr.25277) 517 : cluster 0 pgmap v471: 89 pgs: 89 active+clean; 123 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2023-10-09T05:06:01.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:00 smithi133 bash[102858]: cluster 2023-10-09T05:05:59.659741+0000 mgr.y (mgr.25277) 517 : cluster 0 pgmap v471: 89 pgs: 89 active+clean; 123 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2023-10-09T05:06:01.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:00 smithi194 bash[92666]: cluster 2023-10-09T05:05:59.659741+0000 mgr.y (mgr.25277) 517 : cluster 0 pgmap v471: 89 pgs: 89 active+clean; 123 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2023-10-09T05:06:02.632 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2023-10-09T05:06:02.632 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2023-10-09T05:06:03.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:02 smithi133 bash[105589]: cluster 2023-10-09T05:06:01.660850+0000 mgr.y (mgr.25277) 518 : cluster 0 pgmap v472: 89 pgs: 89 active+clean; 6.3 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 755 op/s 2023-10-09T05:06:03.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:02 smithi133 bash[102858]: cluster 2023-10-09T05:06:01.660850+0000 mgr.y (mgr.25277) 518 : cluster 0 pgmap v472: 89 pgs: 89 active+clean; 6.3 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 755 op/s 2023-10-09T05:06:03.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:02 smithi194 bash[92666]: cluster 2023-10-09T05:06:01.660850+0000 mgr.y (mgr.25277) 518 : cluster 0 pgmap v472: 89 pgs: 89 active+clean; 6.3 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 755 op/s 2023-10-09T05:06:04.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:06:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:06:04] "GET /metrics HTTP/1.1" 200 36339 "" "Prometheus/2.43.0" 2023-10-09T05:06:04.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:04 smithi133 bash[102858]: cluster 2023-10-09T05:06:03.661617+0000 mgr.y (mgr.25277) 519 : cluster 0 pgmap v473: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 518 op/s 2023-10-09T05:06:04.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:04 smithi133 bash[105589]: cluster 2023-10-09T05:06:03.661617+0000 mgr.y (mgr.25277) 519 : cluster 0 pgmap v473: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 518 op/s 2023-10-09T05:06:04.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:04 smithi194 bash[92666]: cluster 2023-10-09T05:06:03.661617+0000 mgr.y (mgr.25277) 519 : cluster 0 pgmap v473: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 518 op/s 2023-10-09T05:06:07.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:06 smithi133 bash[102858]: cluster 2023-10-09T05:06:05.662599+0000 mgr.y (mgr.25277) 520 : cluster 0 pgmap v474: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 459 op/s 2023-10-09T05:06:07.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:06 smithi133 bash[105589]: cluster 2023-10-09T05:06:05.662599+0000 mgr.y (mgr.25277) 520 : cluster 0 pgmap v474: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 459 op/s 2023-10-09T05:06:07.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:06 smithi194 bash[92666]: cluster 2023-10-09T05:06:05.662599+0000 mgr.y (mgr.25277) 520 : cluster 0 pgmap v474: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 459 op/s 2023-10-09T05:06:08.138 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:06:09.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:08 smithi133 bash[102858]: cluster 2023-10-09T05:06:07.663331+0000 mgr.y (mgr.25277) 521 : cluster 0 pgmap v475: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 292 op/s 2023-10-09T05:06:09.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:08 smithi133 bash[105589]: cluster 2023-10-09T05:06:07.663331+0000 mgr.y (mgr.25277) 521 : cluster 0 pgmap v475: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 292 op/s 2023-10-09T05:06:09.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:08 smithi194 bash[92666]: cluster 2023-10-09T05:06:07.663331+0000 mgr.y (mgr.25277) 521 : cluster 0 pgmap v475: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 292 op/s 2023-10-09T05:06:11.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:10 smithi133 bash[105589]: cluster 2023-10-09T05:06:09.664002+0000 mgr.y (mgr.25277) 522 : cluster 0 pgmap v476: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 216 op/s 2023-10-09T05:06:11.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:10 smithi133 bash[102858]: cluster 2023-10-09T05:06:09.664002+0000 mgr.y (mgr.25277) 522 : cluster 0 pgmap v476: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 216 op/s 2023-10-09T05:06:11.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:10 smithi194 bash[92666]: cluster 2023-10-09T05:06:09.664002+0000 mgr.y (mgr.25277) 522 : cluster 0 pgmap v476: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 216 op/s 2023-10-09T05:06:11.770 INFO:teuthology.orchestra.run.smithi133.stderr:pool 'unique_pool_3' removed 2023-10-09T05:06:12.017 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:11 smithi133 bash[102858]: audit 2023-10-09T05:06:11.595648+0000 mon.c (mon.2) 48 : audit 1 from='client.? 172.21.15.133:0/601779973' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:06:12.018 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:11 smithi133 bash[102858]: audit 2023-10-09T05:06:11.596436+0000 mon.a (mon.0) 709 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:06:12.018 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:11 smithi133 bash[105589]: audit 2023-10-09T05:06:11.595648+0000 mon.c (mon.2) 48 : audit 1 from='client.? 172.21.15.133:0/601779973' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:06:12.018 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:11 smithi133 bash[105589]: audit 2023-10-09T05:06:11.596436+0000 mon.a (mon.0) 709 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:06:12.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:11 smithi194 bash[92666]: audit 2023-10-09T05:06:11.595648+0000 mon.c (mon.2) 48 : audit 1 from='client.? 172.21.15.133:0/601779973' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:06:12.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:11 smithi194 bash[92666]: audit 2023-10-09T05:06:11.596436+0000 mon.a (mon.0) 709 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:06:12.686 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T05:06:12.686 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-10-09T05:06:12.686 DEBUG:teuthology.parallel:result is None 2023-10-09T05:06:12.687 INFO:teuthology.run_tasks:Running task print... 2023-10-09T05:06:12.696 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2023-10-09T05:06:12.696 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-10-09T05:06:12.719 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T05:06:12.719 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T05:06:12.720 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T05:06:13.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:12 smithi133 bash[105589]: cluster 2023-10-09T05:06:11.665094+0000 mgr.y (mgr.25277) 523 : cluster 0 pgmap v477: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 162 op/s 2023-10-09T05:06:13.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:12 smithi133 bash[105589]: cluster 2023-10-09T05:06:11.742830+0000 osd.4 (osd.4) 69 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T05:06:13.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:12 smithi133 bash[105589]: audit 2023-10-09T05:06:11.746389+0000 mon.a (mon.0) 710 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:06:13.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:12 smithi133 bash[105589]: cluster 2023-10-09T05:06:11.746528+0000 mon.a (mon.0) 711 : cluster 0 osdmap e138: 8 total, 8 up, 8 in 2023-10-09T05:06:13.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:12 smithi133 bash[105589]: cluster 2023-10-09T05:06:11.912842+0000 osd.6 (osd.6) 59 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T05:06:13.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:12 smithi133 bash[102858]: cluster 2023-10-09T05:06:11.665094+0000 mgr.y (mgr.25277) 523 : cluster 0 pgmap v477: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 162 op/s 2023-10-09T05:06:13.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:12 smithi133 bash[102858]: cluster 2023-10-09T05:06:11.742830+0000 osd.4 (osd.4) 69 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T05:06:13.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:12 smithi133 bash[102858]: audit 2023-10-09T05:06:11.746389+0000 mon.a (mon.0) 710 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:06:13.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:12 smithi133 bash[102858]: cluster 2023-10-09T05:06:11.746528+0000 mon.a (mon.0) 711 : cluster 0 osdmap e138: 8 total, 8 up, 8 in 2023-10-09T05:06:13.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:12 smithi133 bash[102858]: cluster 2023-10-09T05:06:11.912842+0000 osd.6 (osd.6) 59 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T05:06:13.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:12 smithi194 bash[92666]: cluster 2023-10-09T05:06:11.665094+0000 mgr.y (mgr.25277) 523 : cluster 0 pgmap v477: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 162 op/s 2023-10-09T05:06:13.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:12 smithi194 bash[92666]: cluster 2023-10-09T05:06:11.742830+0000 osd.4 (osd.4) 69 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T05:06:13.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:12 smithi194 bash[92666]: audit 2023-10-09T05:06:11.746389+0000 mon.a (mon.0) 710 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:06:13.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:12 smithi194 bash[92666]: cluster 2023-10-09T05:06:11.746528+0000 mon.a (mon.0) 711 : cluster 0 osdmap e138: 8 total, 8 up, 8 in 2023-10-09T05:06:13.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:12 smithi194 bash[92666]: cluster 2023-10-09T05:06:11.912842+0000 osd.6 (osd.6) 59 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T05:06:14.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:13 smithi133 bash[102858]: cluster 2023-10-09T05:06:11.912766+0000 osd.5 (osd.5) 91 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T05:06:14.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:13 smithi133 bash[102858]: cluster 2023-10-09T05:06:11.913034+0000 osd.7 (osd.7) 49 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T05:06:14.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:13 smithi133 bash[102858]: audit 2023-10-09T05:06:13.435492+0000 mon.a (mon.0) 712 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:06:14.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:13 smithi133 bash[105589]: cluster 2023-10-09T05:06:11.912766+0000 osd.5 (osd.5) 91 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T05:06:14.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:13 smithi133 bash[105589]: cluster 2023-10-09T05:06:11.913034+0000 osd.7 (osd.7) 49 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T05:06:14.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:13 smithi133 bash[105589]: audit 2023-10-09T05:06:13.435492+0000 mon.a (mon.0) 712 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:06:14.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:13 smithi194 bash[92666]: cluster 2023-10-09T05:06:11.912766+0000 osd.5 (osd.5) 91 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T05:06:14.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:13 smithi194 bash[92666]: cluster 2023-10-09T05:06:11.913034+0000 osd.7 (osd.7) 49 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T05:06:14.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:13 smithi194 bash[92666]: audit 2023-10-09T05:06:13.435492+0000 mon.a (mon.0) 712 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:06:14.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:06:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:06:14] "GET /metrics HTTP/1.1" 200 33987 "" "Prometheus/2.43.0" 2023-10-09T05:06:15.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:14 smithi133 bash[105589]: cluster 2023-10-09T05:06:13.665825+0000 mgr.y (mgr.25277) 524 : cluster 0 pgmap v479: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:15.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:14 smithi133 bash[102858]: cluster 2023-10-09T05:06:13.665825+0000 mgr.y (mgr.25277) 524 : cluster 0 pgmap v479: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:15.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:14 smithi194 bash[92666]: cluster 2023-10-09T05:06:13.665825+0000 mgr.y (mgr.25277) 524 : cluster 0 pgmap v479: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:17.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:16 smithi133 bash[105589]: cluster 2023-10-09T05:06:15.666700+0000 mgr.y (mgr.25277) 525 : cluster 0 pgmap v480: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:17.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:16 smithi133 bash[102858]: cluster 2023-10-09T05:06:15.666700+0000 mgr.y (mgr.25277) 525 : cluster 0 pgmap v480: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:17.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:16 smithi194 bash[92666]: cluster 2023-10-09T05:06:15.666700+0000 mgr.y (mgr.25277) 525 : cluster 0 pgmap v480: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:17.503 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:06:18.959 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:18 smithi133 bash[102858]: cluster 2023-10-09T05:06:17.667164+0000 mgr.y (mgr.25277) 526 : cluster 0 pgmap v481: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:18.959 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:18 smithi133 bash[105589]: cluster 2023-10-09T05:06:17.667164+0000 mgr.y (mgr.25277) 526 : cluster 0 pgmap v481: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:19.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:18 smithi194 bash[92666]: cluster 2023-10-09T05:06:17.667164+0000 mgr.y (mgr.25277) 526 : cluster 0 pgmap v481: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:20.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:20 smithi133 bash[105589]: cluster 2023-10-09T05:06:19.667743+0000 mgr.y (mgr.25277) 527 : cluster 0 pgmap v482: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:20.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:20 smithi133 bash[102858]: cluster 2023-10-09T05:06:19.667743+0000 mgr.y (mgr.25277) 527 : cluster 0 pgmap v482: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:21.062 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:06:21.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:20 smithi194 bash[92666]: cluster 2023-10-09T05:06:19.667743+0000 mgr.y (mgr.25277) 527 : cluster 0 pgmap v482: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:21.923 INFO:teuthology.orchestra.run.smithi133.stdout:{"epoch":35,"active_gid":25277,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6800","nonce":2244255414},{"type":"v1","addr":"172.21.15.133:6801","nonce":2244255414}]},"active_addr":"172.21.15.133:6801/2244255414","active_change":"2023-10-09T04:52:13.302136+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":25066,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.133:8443/","prometheus":"http://172.21.15.133:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":82,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.133:0","nonce":4185964829}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.133:0","nonce":1984131381}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.133:0","nonce":203823393}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.133:0","nonce":143777049}]}]} 2023-10-09T05:06:21.928 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T05:06:21.928 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T05:06:21.928 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T05:06:22.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:21 smithi133 bash[105589]: audit 2023-10-09T05:06:21.053385+0000 mon.a (mon.0) 713 : audit 0 from='client.? 172.21.15.133:0/3194859719' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T05:06:22.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:21 smithi133 bash[102858]: audit 2023-10-09T05:06:21.053385+0000 mon.a (mon.0) 713 : audit 0 from='client.? 172.21.15.133:0/3194859719' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T05:06:22.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:21 smithi194 bash[92666]: audit 2023-10-09T05:06:21.053385+0000 mon.a (mon.0) 713 : audit 0 from='client.? 172.21.15.133:0/3194859719' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T05:06:23.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:22 smithi133 bash[102858]: cluster 2023-10-09T05:06:21.668911+0000 mgr.y (mgr.25277) 528 : cluster 0 pgmap v483: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:23.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:22 smithi133 bash[105589]: cluster 2023-10-09T05:06:21.668911+0000 mgr.y (mgr.25277) 528 : cluster 0 pgmap v483: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:23.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:22 smithi194 bash[92666]: cluster 2023-10-09T05:06:21.668911+0000 mgr.y (mgr.25277) 528 : cluster 0 pgmap v483: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:24.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:24 smithi133 bash[102858]: cluster 2023-10-09T05:06:23.669745+0000 mgr.y (mgr.25277) 529 : cluster 0 pgmap v484: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:24.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:06:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:06:24] "GET /metrics HTTP/1.1" 200 33987 "" "Prometheus/2.43.0" 2023-10-09T05:06:24.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:24 smithi133 bash[105589]: cluster 2023-10-09T05:06:23.669745+0000 mgr.y (mgr.25277) 529 : cluster 0 pgmap v484: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:24.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:24 smithi194 bash[92666]: cluster 2023-10-09T05:06:23.669745+0000 mgr.y (mgr.25277) 529 : cluster 0 pgmap v484: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:26.705 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:06:27.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:26 smithi133 bash[105589]: cluster 2023-10-09T05:06:25.670697+0000 mgr.y (mgr.25277) 530 : cluster 0 pgmap v485: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:27.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:26 smithi133 bash[102858]: cluster 2023-10-09T05:06:25.670697+0000 mgr.y (mgr.25277) 530 : cluster 0 pgmap v485: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:27.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:26 smithi194 bash[92666]: cluster 2023-10-09T05:06:25.670697+0000 mgr.y (mgr.25277) 530 : cluster 0 pgmap v485: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:29.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:28 smithi133 bash[102858]: cluster 2023-10-09T05:06:27.671427+0000 mgr.y (mgr.25277) 531 : cluster 0 pgmap v486: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:29.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:28 smithi133 bash[102858]: audit 2023-10-09T05:06:28.435356+0000 mon.a (mon.0) 714 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:06:29.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:28 smithi133 bash[105589]: cluster 2023-10-09T05:06:27.671427+0000 mgr.y (mgr.25277) 531 : cluster 0 pgmap v486: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:29.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:28 smithi133 bash[105589]: audit 2023-10-09T05:06:28.435356+0000 mon.a (mon.0) 714 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:06:29.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:28 smithi194 bash[92666]: cluster 2023-10-09T05:06:27.671427+0000 mgr.y (mgr.25277) 531 : cluster 0 pgmap v486: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:29.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:28 smithi194 bash[92666]: audit 2023-10-09T05:06:28.435356+0000 mon.a (mon.0) 714 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:06:30.205 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:06:30.205 INFO:teuthology.orchestra.run.smithi133.stdout:{"epoch":138,"fsid":"855b5206-665c-11ee-8db5-212e2dc638e7","created":"2023-10-09T04:31:08.633675+0000","modified":"2023-10-09T05:06:11.726456+0000","last_up_change":"2023-10-09T05:02:08.616931+0000","last_in_change":"2023-10-09T04:40:39.315748+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T04:36:44.260202+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T04:42:47.518485+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T04:44:51.685755+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T04:44:52.162649+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"32633ec2-9b8f-42d8-bdfb-5c9668a26071","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":83,"up_from":88,"up_thru":126,"down_at":84,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6802","nonce":2098404708},{"type":"v1","addr":"172.21.15.133:6803","nonce":2098404708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6804","nonce":2098404708},{"type":"v1","addr":"172.21.15.133:6805","nonce":2098404708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6808","nonce":2098404708},{"type":"v1","addr":"172.21.15.133:6809","nonce":2098404708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6806","nonce":2098404708},{"type":"v1","addr":"172.21.15.133:6807","nonce":2098404708}]},"public_addr":"172.21.15.133:6803/2098404708","cluster_addr":"172.21.15.133:6805/2098404708","heartbeat_back_addr":"172.21.15.133:6809/2098404708","heartbeat_front_addr":"172.21.15.133:6807/2098404708","state":["exists","up"]},{"osd":1,"uuid":"7f2a961d-a790-4337-91c7-1e45f900bbd4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":92,"up_from":96,"up_thru":135,"down_at":93,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6810","nonce":2668659948},{"type":"v1","addr":"172.21.15.133:6811","nonce":2668659948}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6812","nonce":2668659948},{"type":"v1","addr":"172.21.15.133:6813","nonce":2668659948}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6816","nonce":2668659948},{"type":"v1","addr":"172.21.15.133:6817","nonce":2668659948}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6814","nonce":2668659948},{"type":"v1","addr":"172.21.15.133:6815","nonce":2668659948}]},"public_addr":"172.21.15.133:6811/2668659948","cluster_addr":"172.21.15.133:6813/2668659948","heartbeat_back_addr":"172.21.15.133:6817/2668659948","heartbeat_front_addr":"172.21.15.133:6815/2668659948","state":["exists","up"]},{"osd":2,"uuid":"08751330-a327-4bf8-9c4d-9ff986b1c67f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":102,"up_from":106,"up_thru":123,"down_at":103,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6818","nonce":3595178639},{"type":"v1","addr":"172.21.15.133:6819","nonce":3595178639}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6820","nonce":3595178639},{"type":"v1","addr":"172.21.15.133:6821","nonce":3595178639}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6824","nonce":3595178639},{"type":"v1","addr":"172.21.15.133:6825","nonce":3595178639}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6822","nonce":3595178639},{"type":"v1","addr":"172.21.15.133:6823","nonce":3595178639}]},"public_addr":"172.21.15.133:6819/3595178639","cluster_addr":"172.21.15.133:6821/3595178639","heartbeat_back_addr":"172.21.15.133:6825/3595178639","heartbeat_front_addr":"172.21.15.133:6823/3595178639","state":["exists","up"]},{"osd":3,"uuid":"56bbf807-c8f7-4226-b46e-802b539274ca","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":117,"up_from":123,"up_thru":136,"down_at":118,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6826","nonce":1082233372},{"type":"v1","addr":"172.21.15.133:6827","nonce":1082233372}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6828","nonce":1082233372},{"type":"v1","addr":"172.21.15.133:6829","nonce":1082233372}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6832","nonce":1082233372},{"type":"v1","addr":"172.21.15.133:6833","nonce":1082233372}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6830","nonce":1082233372},{"type":"v1","addr":"172.21.15.133:6831","nonce":1082233372}]},"public_addr":"172.21.15.133:6827/1082233372","cluster_addr":"172.21.15.133:6829/1082233372","heartbeat_back_addr":"172.21.15.133:6833/1082233372","heartbeat_front_addr":"172.21.15.133:6831/1082233372","state":["exists","up"]},{"osd":4,"uuid":"509ec8cc-5042-428e-9a69-8e602b5afce8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":129,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6801","nonce":1874338403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6803","nonce":1874338403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6807","nonce":1874338403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6805","nonce":1874338403}]},"public_addr":"172.21.15.194:6801/1874338403","cluster_addr":"172.21.15.194:6803/1874338403","heartbeat_back_addr":"172.21.15.194:6807/1874338403","heartbeat_front_addr":"172.21.15.194:6805/1874338403","state":["exists","up"]},{"osd":5,"uuid":"bbdc1e67-fd5b-49d0-9e42-cf151d167943","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":132,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6809","nonce":3989986662}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6811","nonce":3989986662}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6815","nonce":3989986662}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6813","nonce":3989986662}]},"public_addr":"172.21.15.194:6809/3989986662","cluster_addr":"172.21.15.194:6811/3989986662","heartbeat_back_addr":"172.21.15.194:6815/3989986662","heartbeat_front_addr":"172.21.15.194:6813/3989986662","state":["exists","up"]},{"osd":6,"uuid":"7b69d014-f4c0-4e73-a14e-7d51b7ee3ea2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":118,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6817","nonce":2607555245}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6819","nonce":2607555245}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6823","nonce":2607555245}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6820","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6821","nonce":2607555245}]},"public_addr":"172.21.15.194:6817/2607555245","cluster_addr":"172.21.15.194:6819/2607555245","heartbeat_back_addr":"172.21.15.194:6823/2607555245","heartbeat_front_addr":"172.21.15.194:6821/2607555245","state":["exists","up"]},{"osd":7,"uuid":"02506268-102c-4654-9b5c-68d36c28f7f7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":123,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6824","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6825","nonce":1051622545}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6827","nonce":1051622545}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6831","nonce":1051622545}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6828","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6829","nonce":1051622545}]},"public_addr":"172.21.15.194:6825/1051622545","cluster_addr":"172.21.15.194:6827/1051622545","heartbeat_back_addr":"172.21.15.194:6831/1051622545","heartbeat_front_addr":"172.21.15.194:6829/1051622545","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:34:53.028753+0000","dead_epoch":83},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:35:47.001796+0000","dead_epoch":92},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:36:41.220545+0000","dead_epoch":102},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:37:36.237434+0000","dead_epoch":117},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:38:28.301119+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:39:20.099747+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:40:12.106681+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:41:04.212546+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.7","mappings":[{"from":1,"to":2}]},{"pgid":"4.a","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.133:0/4173745360":"2023-10-10T04:31:42.639571+0000","172.21.15.133:0/3141871970":"2023-10-10T04:52:13.301898+0000","172.21.15.133:0/1787141186":"2023-10-10T04:49:30.896539+0000","172.21.15.194:0/2434253169":"2023-10-10T04:50:55.051958+0000","172.21.15.133:0/3777348175":"2023-10-10T04:52:13.301898+0000","172.21.15.133:0/2479656939":"2023-10-10T04:52:13.301898+0000","172.21.15.133:0/4053521787":"2023-10-10T04:52:13.301898+0000","172.21.15.133:0/320185713":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/3636510019":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/2419271305":"2023-10-10T04:49:30.896539+0000","172.21.15.133:0/740467257":"2023-10-10T04:31:42.639571+0000","172.21.15.133:6800/275314455":"2023-10-10T04:49:30.896539+0000","172.21.15.133:6835/4227843513":"2023-10-10T04:45:00.021661+0000","172.21.15.133:0/55818837":"2023-10-10T04:42:05.063462+0000","172.21.15.194:0/2197565959":"2023-10-10T04:50:55.051958+0000","172.21.15.133:6800/788799879":"2023-10-10T04:32:17.611609+0000","172.21.15.133:6801/788799879":"2023-10-10T04:32:17.611609+0000","172.21.15.133:0/70449665":"2023-10-10T04:52:13.301898+0000","172.21.15.133:0/1793292086":"2023-10-10T04:32:17.611609+0000","172.21.15.133:6801/3027596458":"2023-10-10T04:52:13.301898+0000","172.21.15.133:6801/275314455":"2023-10-10T04:49:30.896539+0000","172.21.15.194:0/3848858405":"2023-10-10T04:50:55.051958+0000","172.21.15.194:0/430115207":"2023-10-10T04:50:55.051958+0000","172.21.15.133:0/2159982120":"2023-10-10T04:49:30.896539+0000","172.21.15.133:6800/768614786":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/1575170036":"2023-10-10T04:49:30.896539+0000","172.21.15.133:6801/768614786":"2023-10-10T04:42:05.063462+0000","172.21.15.133:6834/4227843513":"2023-10-10T04:45:00.021661+0000","172.21.15.133:0/3619387632":"2023-10-10T04:32:17.611609+0000","172.21.15.133:0/1411744140":"2023-10-10T04:31:42.639571+0000","172.21.15.194:0/1505438352":"2023-10-10T04:50:55.051958+0000","172.21.15.133:0/1309073801":"2023-10-10T04:42:05.063462+0000","172.21.15.194:0/152828820":"2023-10-10T04:50:55.051958+0000","172.21.15.133:0/499357090":"2023-10-10T04:52:13.301898+0000","172.21.15.133:6800/3027596458":"2023-10-10T04:52:13.301898+0000","172.21.15.133:0/4290202542":"2023-10-10T04:32:17.611609+0000","172.21.15.194:6834/2184279986":"2023-10-10T04:50:55.051958+0000","172.21.15.194:6835/2184279986":"2023-10-10T04:50:55.051958+0000","172.21.15.133:6800/1761810618":"2023-10-10T04:31:42.639571+0000","172.21.15.133:6801/1761810618":"2023-10-10T04:31:42.639571+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:06:31.018 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:30 smithi133 bash[102858]: cluster 2023-10-09T05:06:29.672250+0000 mgr.y (mgr.25277) 532 : cluster 0 pgmap v487: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:31.018 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:30 smithi133 bash[102858]: audit 2023-10-09T05:06:30.201353+0000 mon.c (mon.2) 49 : audit 0 from='client.? 172.21.15.133:0/1140513436' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:06:31.018 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:30 smithi133 bash[105589]: cluster 2023-10-09T05:06:29.672250+0000 mgr.y (mgr.25277) 532 : cluster 0 pgmap v487: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:31.019 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:30 smithi133 bash[105589]: audit 2023-10-09T05:06:30.201353+0000 mon.c (mon.2) 49 : audit 0 from='client.? 172.21.15.133:0/1140513436' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:06:31.020 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T05:06:31.020 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T05:06:31.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:30 smithi194 bash[92666]: cluster 2023-10-09T05:06:29.672250+0000 mgr.y (mgr.25277) 532 : cluster 0 pgmap v487: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:31.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:30 smithi194 bash[92666]: audit 2023-10-09T05:06:30.201353+0000 mon.c (mon.2) 49 : audit 0 from='client.? 172.21.15.133:0/1140513436' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:06:33.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:32 smithi133 bash[102858]: cluster 2023-10-09T05:06:31.673396+0000 mgr.y (mgr.25277) 533 : cluster 0 pgmap v488: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:33.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:32 smithi133 bash[105589]: cluster 2023-10-09T05:06:31.673396+0000 mgr.y (mgr.25277) 533 : cluster 0 pgmap v488: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:33.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:32 smithi194 bash[92666]: cluster 2023-10-09T05:06:31.673396+0000 mgr.y (mgr.25277) 533 : cluster 0 pgmap v488: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:34.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:34 smithi133 bash[102858]: cluster 2023-10-09T05:06:33.674003+0000 mgr.y (mgr.25277) 534 : cluster 0 pgmap v489: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:34.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:06:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:06:34] "GET /metrics HTTP/1.1" 200 33988 "" "Prometheus/2.43.0" 2023-10-09T05:06:34.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:34 smithi133 bash[105589]: cluster 2023-10-09T05:06:33.674003+0000 mgr.y (mgr.25277) 534 : cluster 0 pgmap v489: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:34.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:34 smithi194 bash[92666]: cluster 2023-10-09T05:06:33.674003+0000 mgr.y (mgr.25277) 534 : cluster 0 pgmap v489: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:35.795 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:06:36.987 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:36 smithi133 bash[105589]: cluster 2023-10-09T05:06:35.674969+0000 mgr.y (mgr.25277) 535 : cluster 0 pgmap v490: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:36.988 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:36 smithi133 bash[102858]: cluster 2023-10-09T05:06:35.674969+0000 mgr.y (mgr.25277) 535 : cluster 0 pgmap v490: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:37.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:36 smithi194 bash[92666]: cluster 2023-10-09T05:06:35.674969+0000 mgr.y (mgr.25277) 535 : cluster 0 pgmap v490: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:39.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:38 smithi133 bash[102858]: cluster 2023-10-09T05:06:37.675741+0000 mgr.y (mgr.25277) 536 : cluster 0 pgmap v491: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:39.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:38 smithi133 bash[105589]: cluster 2023-10-09T05:06:37.675741+0000 mgr.y (mgr.25277) 536 : cluster 0 pgmap v491: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:39.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:38 smithi194 bash[92666]: cluster 2023-10-09T05:06:37.675741+0000 mgr.y (mgr.25277) 536 : cluster 0 pgmap v491: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:39.261 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:06:39.261 INFO:teuthology.orchestra.run.smithi133.stdout:{"epoch":138,"fsid":"855b5206-665c-11ee-8db5-212e2dc638e7","created":"2023-10-09T04:31:08.633675+0000","modified":"2023-10-09T05:06:11.726456+0000","last_up_change":"2023-10-09T05:02:08.616931+0000","last_in_change":"2023-10-09T04:40:39.315748+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T04:36:44.260202+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T04:42:47.518485+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T04:44:51.685755+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T04:44:52.162649+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"32633ec2-9b8f-42d8-bdfb-5c9668a26071","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":83,"up_from":88,"up_thru":126,"down_at":84,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6802","nonce":2098404708},{"type":"v1","addr":"172.21.15.133:6803","nonce":2098404708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6804","nonce":2098404708},{"type":"v1","addr":"172.21.15.133:6805","nonce":2098404708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6808","nonce":2098404708},{"type":"v1","addr":"172.21.15.133:6809","nonce":2098404708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6806","nonce":2098404708},{"type":"v1","addr":"172.21.15.133:6807","nonce":2098404708}]},"public_addr":"172.21.15.133:6803/2098404708","cluster_addr":"172.21.15.133:6805/2098404708","heartbeat_back_addr":"172.21.15.133:6809/2098404708","heartbeat_front_addr":"172.21.15.133:6807/2098404708","state":["exists","up"]},{"osd":1,"uuid":"7f2a961d-a790-4337-91c7-1e45f900bbd4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":92,"up_from":96,"up_thru":135,"down_at":93,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6810","nonce":2668659948},{"type":"v1","addr":"172.21.15.133:6811","nonce":2668659948}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6812","nonce":2668659948},{"type":"v1","addr":"172.21.15.133:6813","nonce":2668659948}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6816","nonce":2668659948},{"type":"v1","addr":"172.21.15.133:6817","nonce":2668659948}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6814","nonce":2668659948},{"type":"v1","addr":"172.21.15.133:6815","nonce":2668659948}]},"public_addr":"172.21.15.133:6811/2668659948","cluster_addr":"172.21.15.133:6813/2668659948","heartbeat_back_addr":"172.21.15.133:6817/2668659948","heartbeat_front_addr":"172.21.15.133:6815/2668659948","state":["exists","up"]},{"osd":2,"uuid":"08751330-a327-4bf8-9c4d-9ff986b1c67f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":102,"up_from":106,"up_thru":123,"down_at":103,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6818","nonce":3595178639},{"type":"v1","addr":"172.21.15.133:6819","nonce":3595178639}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6820","nonce":3595178639},{"type":"v1","addr":"172.21.15.133:6821","nonce":3595178639}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6824","nonce":3595178639},{"type":"v1","addr":"172.21.15.133:6825","nonce":3595178639}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6822","nonce":3595178639},{"type":"v1","addr":"172.21.15.133:6823","nonce":3595178639}]},"public_addr":"172.21.15.133:6819/3595178639","cluster_addr":"172.21.15.133:6821/3595178639","heartbeat_back_addr":"172.21.15.133:6825/3595178639","heartbeat_front_addr":"172.21.15.133:6823/3595178639","state":["exists","up"]},{"osd":3,"uuid":"56bbf807-c8f7-4226-b46e-802b539274ca","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":117,"up_from":123,"up_thru":136,"down_at":118,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6826","nonce":1082233372},{"type":"v1","addr":"172.21.15.133:6827","nonce":1082233372}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6828","nonce":1082233372},{"type":"v1","addr":"172.21.15.133:6829","nonce":1082233372}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6832","nonce":1082233372},{"type":"v1","addr":"172.21.15.133:6833","nonce":1082233372}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6830","nonce":1082233372},{"type":"v1","addr":"172.21.15.133:6831","nonce":1082233372}]},"public_addr":"172.21.15.133:6827/1082233372","cluster_addr":"172.21.15.133:6829/1082233372","heartbeat_back_addr":"172.21.15.133:6833/1082233372","heartbeat_front_addr":"172.21.15.133:6831/1082233372","state":["exists","up"]},{"osd":4,"uuid":"509ec8cc-5042-428e-9a69-8e602b5afce8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":129,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6801","nonce":1874338403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6803","nonce":1874338403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6807","nonce":1874338403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6805","nonce":1874338403}]},"public_addr":"172.21.15.194:6801/1874338403","cluster_addr":"172.21.15.194:6803/1874338403","heartbeat_back_addr":"172.21.15.194:6807/1874338403","heartbeat_front_addr":"172.21.15.194:6805/1874338403","state":["exists","up"]},{"osd":5,"uuid":"bbdc1e67-fd5b-49d0-9e42-cf151d167943","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":132,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6809","nonce":3989986662}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6811","nonce":3989986662}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6815","nonce":3989986662}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6813","nonce":3989986662}]},"public_addr":"172.21.15.194:6809/3989986662","cluster_addr":"172.21.15.194:6811/3989986662","heartbeat_back_addr":"172.21.15.194:6815/3989986662","heartbeat_front_addr":"172.21.15.194:6813/3989986662","state":["exists","up"]},{"osd":6,"uuid":"7b69d014-f4c0-4e73-a14e-7d51b7ee3ea2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":118,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6817","nonce":2607555245}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6819","nonce":2607555245}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6823","nonce":2607555245}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6820","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6821","nonce":2607555245}]},"public_addr":"172.21.15.194:6817/2607555245","cluster_addr":"172.21.15.194:6819/2607555245","heartbeat_back_addr":"172.21.15.194:6823/2607555245","heartbeat_front_addr":"172.21.15.194:6821/2607555245","state":["exists","up"]},{"osd":7,"uuid":"02506268-102c-4654-9b5c-68d36c28f7f7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":123,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6824","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6825","nonce":1051622545}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6827","nonce":1051622545}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6831","nonce":1051622545}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6828","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6829","nonce":1051622545}]},"public_addr":"172.21.15.194:6825/1051622545","cluster_addr":"172.21.15.194:6827/1051622545","heartbeat_back_addr":"172.21.15.194:6831/1051622545","heartbeat_front_addr":"172.21.15.194:6829/1051622545","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:34:53.028753+0000","dead_epoch":83},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:35:47.001796+0000","dead_epoch":92},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:36:41.220545+0000","dead_epoch":102},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:37:36.237434+0000","dead_epoch":117},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:38:28.301119+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:39:20.099747+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:40:12.106681+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:41:04.212546+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.7","mappings":[{"from":1,"to":2}]},{"pgid":"4.a","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.133:0/4173745360":"2023-10-10T04:31:42.639571+0000","172.21.15.133:0/3141871970":"2023-10-10T04:52:13.301898+0000","172.21.15.133:0/1787141186":"2023-10-10T04:49:30.896539+0000","172.21.15.194:0/2434253169":"2023-10-10T04:50:55.051958+0000","172.21.15.133:0/3777348175":"2023-10-10T04:52:13.301898+0000","172.21.15.133:0/2479656939":"2023-10-10T04:52:13.301898+0000","172.21.15.133:0/4053521787":"2023-10-10T04:52:13.301898+0000","172.21.15.133:0/320185713":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/3636510019":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/2419271305":"2023-10-10T04:49:30.896539+0000","172.21.15.133:0/740467257":"2023-10-10T04:31:42.639571+0000","172.21.15.133:6800/275314455":"2023-10-10T04:49:30.896539+0000","172.21.15.133:6835/4227843513":"2023-10-10T04:45:00.021661+0000","172.21.15.133:0/55818837":"2023-10-10T04:42:05.063462+0000","172.21.15.194:0/2197565959":"2023-10-10T04:50:55.051958+0000","172.21.15.133:6800/788799879":"2023-10-10T04:32:17.611609+0000","172.21.15.133:6801/788799879":"2023-10-10T04:32:17.611609+0000","172.21.15.133:0/70449665":"2023-10-10T04:52:13.301898+0000","172.21.15.133:0/1793292086":"2023-10-10T04:32:17.611609+0000","172.21.15.133:6801/3027596458":"2023-10-10T04:52:13.301898+0000","172.21.15.133:6801/275314455":"2023-10-10T04:49:30.896539+0000","172.21.15.194:0/3848858405":"2023-10-10T04:50:55.051958+0000","172.21.15.194:0/430115207":"2023-10-10T04:50:55.051958+0000","172.21.15.133:0/2159982120":"2023-10-10T04:49:30.896539+0000","172.21.15.133:6800/768614786":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/1575170036":"2023-10-10T04:49:30.896539+0000","172.21.15.133:6801/768614786":"2023-10-10T04:42:05.063462+0000","172.21.15.133:6834/4227843513":"2023-10-10T04:45:00.021661+0000","172.21.15.133:0/3619387632":"2023-10-10T04:32:17.611609+0000","172.21.15.133:0/1411744140":"2023-10-10T04:31:42.639571+0000","172.21.15.194:0/1505438352":"2023-10-10T04:50:55.051958+0000","172.21.15.133:0/1309073801":"2023-10-10T04:42:05.063462+0000","172.21.15.194:0/152828820":"2023-10-10T04:50:55.051958+0000","172.21.15.133:0/499357090":"2023-10-10T04:52:13.301898+0000","172.21.15.133:6800/3027596458":"2023-10-10T04:52:13.301898+0000","172.21.15.133:0/4290202542":"2023-10-10T04:32:17.611609+0000","172.21.15.194:6834/2184279986":"2023-10-10T04:50:55.051958+0000","172.21.15.194:6835/2184279986":"2023-10-10T04:50:55.051958+0000","172.21.15.133:6800/1761810618":"2023-10-10T04:31:42.639571+0000","172.21.15.133:6801/1761810618":"2023-10-10T04:31:42.639571+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:06:39.745 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:39 smithi133 bash[102858]: audit 2023-10-09T05:06:39.256669+0000 mon.c (mon.2) 50 : audit 0 from='client.? 172.21.15.133:0/3926313247' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:06:39.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:39 smithi133 bash[105589]: audit 2023-10-09T05:06:39.256669+0000 mon.c (mon.2) 50 : audit 0 from='client.? 172.21.15.133:0/3926313247' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:06:40.129 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T05:06:40.130 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T05:06:40.130 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T05:06:40.131 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T05:06:40.131 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T05:06:40.132 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T05:06:40.132 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T05:06:40.132 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T05:06:40.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:39 smithi194 bash[92666]: audit 2023-10-09T05:06:39.256669+0000 mon.c (mon.2) 50 : audit 0 from='client.? 172.21.15.133:0/3926313247' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:06:41.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:40 smithi133 bash[102858]: cluster 2023-10-09T05:06:39.676515+0000 mgr.y (mgr.25277) 537 : cluster 0 pgmap v492: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:41.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:40 smithi133 bash[105589]: cluster 2023-10-09T05:06:39.676515+0000 mgr.y (mgr.25277) 537 : cluster 0 pgmap v492: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:41.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:40 smithi194 bash[92666]: cluster 2023-10-09T05:06:39.676515+0000 mgr.y (mgr.25277) 537 : cluster 0 pgmap v492: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:43.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:42 smithi133 bash[102858]: cluster 2023-10-09T05:06:41.677845+0000 mgr.y (mgr.25277) 538 : cluster 0 pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:43.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:42 smithi133 bash[105589]: cluster 2023-10-09T05:06:41.677845+0000 mgr.y (mgr.25277) 538 : cluster 0 pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:43.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:42 smithi194 bash[92666]: cluster 2023-10-09T05:06:41.677845+0000 mgr.y (mgr.25277) 538 : cluster 0 pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:44.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:06:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:06:44] "GET /metrics HTTP/1.1" 200 33990 "" "Prometheus/2.43.0" 2023-10-09T05:06:44.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:44 smithi133 bash[102858]: audit 2023-10-09T05:06:43.436469+0000 mon.a (mon.0) 715 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:06:44.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:44 smithi133 bash[102858]: cluster 2023-10-09T05:06:43.678639+0000 mgr.y (mgr.25277) 539 : cluster 0 pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:44.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:44 smithi133 bash[105589]: audit 2023-10-09T05:06:43.436469+0000 mon.a (mon.0) 715 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:06:44.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:44 smithi133 bash[105589]: cluster 2023-10-09T05:06:43.678639+0000 mgr.y (mgr.25277) 539 : cluster 0 pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:44.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:44 smithi194 bash[92666]: audit 2023-10-09T05:06:43.436469+0000 mon.a (mon.0) 715 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:06:44.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:44 smithi194 bash[92666]: cluster 2023-10-09T05:06:43.678639+0000 mgr.y (mgr.25277) 539 : cluster 0 pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:45.015 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:06:45.016 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:06:45.018 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:06:45.018 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:06:45.019 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:06:45.019 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:06:45.023 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:06:45.023 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:06:47.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:46 smithi133 bash[102858]: cluster 2023-10-09T05:06:45.679534+0000 mgr.y (mgr.25277) 540 : cluster 0 pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:47.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:46 smithi133 bash[105589]: cluster 2023-10-09T05:06:45.679534+0000 mgr.y (mgr.25277) 540 : cluster 0 pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:47.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:46 smithi194 bash[92666]: cluster 2023-10-09T05:06:45.679534+0000 mgr.y (mgr.25277) 540 : cluster 0 pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:49.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:48 smithi133 bash[102858]: cluster 2023-10-09T05:06:47.680321+0000 mgr.y (mgr.25277) 541 : cluster 0 pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:49.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:48 smithi133 bash[105589]: cluster 2023-10-09T05:06:47.680321+0000 mgr.y (mgr.25277) 541 : cluster 0 pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:49.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:48 smithi194 bash[92666]: cluster 2023-10-09T05:06:47.680321+0000 mgr.y (mgr.25277) 541 : cluster 0 pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:50.894 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:50 smithi133 bash[105589]: cluster 2023-10-09T05:06:49.680868+0000 mgr.y (mgr.25277) 542 : cluster 0 pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:50.894 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:50 smithi133 bash[102858]: cluster 2023-10-09T05:06:49.680868+0000 mgr.y (mgr.25277) 542 : cluster 0 pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:51.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:50 smithi194 bash[92666]: cluster 2023-10-09T05:06:49.680868+0000 mgr.y (mgr.25277) 542 : cluster 0 pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:53.046 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:52 smithi133 bash[105589]: cluster 2023-10-09T05:06:51.681946+0000 mgr.y (mgr.25277) 543 : cluster 0 pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:53.047 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:52 smithi133 bash[102858]: cluster 2023-10-09T05:06:51.681946+0000 mgr.y (mgr.25277) 543 : cluster 0 pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:53.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:52 smithi194 bash[92666]: cluster 2023-10-09T05:06:51.681946+0000 mgr.y (mgr.25277) 543 : cluster 0 pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:54.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:06:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:06:54] "GET /metrics HTTP/1.1" 200 33990 "" "Prometheus/2.43.0" 2023-10-09T05:06:54.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:54 smithi133 bash[105589]: cluster 2023-10-09T05:06:53.682420+0000 mgr.y (mgr.25277) 544 : cluster 0 pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:54.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:54 smithi133 bash[102858]: cluster 2023-10-09T05:06:53.682420+0000 mgr.y (mgr.25277) 544 : cluster 0 pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:54.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:54 smithi194 bash[92666]: cluster 2023-10-09T05:06:53.682420+0000 mgr.y (mgr.25277) 544 : cluster 0 pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:57.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:56 smithi133 bash[105589]: cluster 2023-10-09T05:06:55.683328+0000 mgr.y (mgr.25277) 545 : cluster 0 pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:57.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:56 smithi133 bash[102858]: cluster 2023-10-09T05:06:55.683328+0000 mgr.y (mgr.25277) 545 : cluster 0 pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:57.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:56 smithi194 bash[92666]: cluster 2023-10-09T05:06:55.683328+0000 mgr.y (mgr.25277) 545 : cluster 0 pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:59.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:58 smithi133 bash[105589]: cluster 2023-10-09T05:06:57.684035+0000 mgr.y (mgr.25277) 546 : cluster 0 pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:59.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:58 smithi133 bash[105589]: audit 2023-10-09T05:06:58.436717+0000 mon.a (mon.0) 716 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:06:59.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:06:58 smithi133 bash[105589]: audit 2023-10-09T05:06:58.476330+0000 mon.a (mon.0) 717 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:06:59.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:58 smithi133 bash[102858]: cluster 2023-10-09T05:06:57.684035+0000 mgr.y (mgr.25277) 546 : cluster 0 pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:59.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:58 smithi133 bash[102858]: audit 2023-10-09T05:06:58.436717+0000 mon.a (mon.0) 716 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:06:59.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:06:58 smithi133 bash[102858]: audit 2023-10-09T05:06:58.476330+0000 mon.a (mon.0) 717 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:06:59.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:58 smithi194 bash[92666]: cluster 2023-10-09T05:06:57.684035+0000 mgr.y (mgr.25277) 546 : cluster 0 pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:06:59.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:58 smithi194 bash[92666]: audit 2023-10-09T05:06:58.436717+0000 mon.a (mon.0) 716 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:06:59.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:06:58 smithi194 bash[92666]: audit 2023-10-09T05:06:58.476330+0000 mon.a (mon.0) 717 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:07:01.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:00 smithi133 bash[105589]: cluster 2023-10-09T05:06:59.684707+0000 mgr.y (mgr.25277) 547 : cluster 0 pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:01.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:00 smithi133 bash[102858]: cluster 2023-10-09T05:06:59.684707+0000 mgr.y (mgr.25277) 547 : cluster 0 pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:01.114 INFO:teuthology.orchestra.run.smithi133.stdout:455266533449 2023-10-09T05:07:01.115 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T05:07:01.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:00 smithi194 bash[92666]: cluster 2023-10-09T05:06:59.684707+0000 mgr.y (mgr.25277) 547 : cluster 0 pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:01.303 INFO:teuthology.orchestra.run.smithi133.stdout:231928234296 2023-10-09T05:07:01.303 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T05:07:03.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:02 smithi133 bash[102858]: cluster 2023-10-09T05:07:01.685873+0000 mgr.y (mgr.25277) 548 : cluster 0 pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:03.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:02 smithi133 bash[105589]: cluster 2023-10-09T05:07:01.685873+0000 mgr.y (mgr.25277) 548 : cluster 0 pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:03.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:02 smithi194 bash[92666]: cluster 2023-10-09T05:07:01.685873+0000 mgr.y (mgr.25277) 548 : cluster 0 pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:03.455 INFO:teuthology.orchestra.run.smithi133.stdout:197568495938 2023-10-09T05:07:03.455 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T05:07:03.510 INFO:teuthology.orchestra.run.smithi133.stdout:412316860529 2023-10-09T05:07:03.511 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T05:07:04.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:04 smithi133 bash[102858]: cluster 2023-10-09T05:07:03.686406+0000 mgr.y (mgr.25277) 549 : cluster 0 pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:04.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:04 smithi133 bash[105589]: cluster 2023-10-09T05:07:03.686406+0000 mgr.y (mgr.25277) 549 : cluster 0 pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:04.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:07:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:07:04] "GET /metrics HTTP/1.1" 200 33987 "" "Prometheus/2.43.0" 2023-10-09T05:07:04.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:04 smithi194 bash[92666]: cluster 2023-10-09T05:07:03.686406+0000 mgr.y (mgr.25277) 549 : cluster 0 pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:05.571 INFO:teuthology.orchestra.run.smithi133.stdout:163208757581 2023-10-09T05:07:05.571 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T05:07:05.691 INFO:teuthology.orchestra.run.smithi133.stdout:128849019225 2023-10-09T05:07:05.691 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T05:07:05.706 INFO:teuthology.orchestra.run.smithi133.stdout:528280977468 2023-10-09T05:07:05.706 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T05:07:05.758 INFO:teuthology.orchestra.run.smithi133.stdout:377957122171 2023-10-09T05:07:05.759 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T05:07:06.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:05 smithi133 bash[102858]: audit 2023-10-09T05:07:04.912183+0000 mon.a (mon.0) 718 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:07:06.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:05 smithi133 bash[102858]: audit 2023-10-09T05:07:04.920126+0000 mon.a (mon.0) 719 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:07:06.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:05 smithi133 bash[105589]: audit 2023-10-09T05:07:04.912183+0000 mon.a (mon.0) 718 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:07:06.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:05 smithi133 bash[105589]: audit 2023-10-09T05:07:04.920126+0000 mon.a (mon.0) 719 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:07:06.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:05 smithi194 bash[92666]: audit 2023-10-09T05:07:04.912183+0000 mon.a (mon.0) 718 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:07:06.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:05 smithi194 bash[92666]: audit 2023-10-09T05:07:04.920126+0000 mon.a (mon.0) 719 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:07:07.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:06 smithi133 bash[105589]: cluster 2023-10-09T05:07:05.686935+0000 mgr.y (mgr.25277) 550 : cluster 0 pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:07.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:06 smithi133 bash[102858]: cluster 2023-10-09T05:07:05.686935+0000 mgr.y (mgr.25277) 550 : cluster 0 pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:07.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:06 smithi194 bash[92666]: cluster 2023-10-09T05:07:05.686935+0000 mgr.y (mgr.25277) 550 : cluster 0 pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:08.399 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:07:08.399 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:07:08.403 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:07:08.403 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:07:08.926 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:08 smithi133 bash[102858]: cluster 2023-10-09T05:07:07.687652+0000 mgr.y (mgr.25277) 551 : cluster 0 pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:09.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:08 smithi133 bash[105589]: cluster 2023-10-09T05:07:07.687652+0000 mgr.y (mgr.25277) 551 : cluster 0 pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:09.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:08 smithi194 bash[92666]: cluster 2023-10-09T05:07:07.687652+0000 mgr.y (mgr.25277) 551 : cluster 0 pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:09.411 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:07:10.424 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:07:10.424 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:07:10.425 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:07:10.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:10 smithi133 bash[105589]: audit 2023-10-09T05:07:09.533398+0000 mon.a (mon.0) 720 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:07:10.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:10 smithi133 bash[105589]: audit 2023-10-09T05:07:09.539583+0000 mon.a (mon.0) 721 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:07:10.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:10 smithi133 bash[105589]: cluster 2023-10-09T05:07:09.688310+0000 mgr.y (mgr.25277) 552 : cluster 0 pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:10.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:10 smithi133 bash[105589]: audit 2023-10-09T05:07:09.956991+0000 mon.a (mon.0) 722 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:07:10.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:10 smithi133 bash[105589]: audit 2023-10-09T05:07:09.958008+0000 mon.a (mon.0) 723 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:07:10.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:10 smithi133 bash[105589]: audit 2023-10-09T05:07:09.965193+0000 mon.a (mon.0) 724 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:07:10.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:10 smithi133 bash[102858]: audit 2023-10-09T05:07:09.533398+0000 mon.a (mon.0) 720 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:07:10.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:10 smithi133 bash[102858]: audit 2023-10-09T05:07:09.539583+0000 mon.a (mon.0) 721 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:07:10.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:10 smithi133 bash[102858]: cluster 2023-10-09T05:07:09.688310+0000 mgr.y (mgr.25277) 552 : cluster 0 pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:10.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:10 smithi133 bash[102858]: audit 2023-10-09T05:07:09.956991+0000 mon.a (mon.0) 722 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:07:10.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:10 smithi133 bash[102858]: audit 2023-10-09T05:07:09.958008+0000 mon.a (mon.0) 723 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:07:10.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:10 smithi133 bash[102858]: audit 2023-10-09T05:07:09.965193+0000 mon.a (mon.0) 724 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:07:10.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:10 smithi194 bash[92666]: audit 2023-10-09T05:07:09.533398+0000 mon.a (mon.0) 720 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:07:10.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:10 smithi194 bash[92666]: audit 2023-10-09T05:07:09.539583+0000 mon.a (mon.0) 721 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:07:10.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:10 smithi194 bash[92666]: cluster 2023-10-09T05:07:09.688310+0000 mgr.y (mgr.25277) 552 : cluster 0 pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:10.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:10 smithi194 bash[92666]: audit 2023-10-09T05:07:09.956991+0000 mon.a (mon.0) 722 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:07:10.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:10 smithi194 bash[92666]: audit 2023-10-09T05:07:09.958008+0000 mon.a (mon.0) 723 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:07:10.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:10 smithi194 bash[92666]: audit 2023-10-09T05:07:09.965193+0000 mon.a (mon.0) 724 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:07:12.991 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:12 smithi133 bash[105589]: cluster 2023-10-09T05:07:11.689101+0000 mgr.y (mgr.25277) 553 : cluster 0 pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:12.991 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:12 smithi133 bash[102858]: cluster 2023-10-09T05:07:11.689101+0000 mgr.y (mgr.25277) 553 : cluster 0 pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:13.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:12 smithi194 bash[92666]: cluster 2023-10-09T05:07:11.689101+0000 mgr.y (mgr.25277) 553 : cluster 0 pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:14.282 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:14 smithi133 bash[105589]: audit 2023-10-09T05:07:13.436988+0000 mon.a (mon.0) 725 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:07:14.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:14 smithi133 bash[105589]: cluster 2023-10-09T05:07:13.689661+0000 mgr.y (mgr.25277) 554 : cluster 0 pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:14.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:14 smithi133 bash[102858]: audit 2023-10-09T05:07:13.436988+0000 mon.a (mon.0) 725 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:07:14.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:14 smithi133 bash[102858]: cluster 2023-10-09T05:07:13.689661+0000 mgr.y (mgr.25277) 554 : cluster 0 pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:14.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:07:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:07:14] "GET /metrics HTTP/1.1" 200 33990 "" "Prometheus/2.43.0" 2023-10-09T05:07:14.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:14 smithi194 bash[92666]: audit 2023-10-09T05:07:13.436988+0000 mon.a (mon.0) 725 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:07:14.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:14 smithi194 bash[92666]: cluster 2023-10-09T05:07:13.689661+0000 mgr.y (mgr.25277) 554 : cluster 0 pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:17.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:16 smithi133 bash[105589]: cluster 2023-10-09T05:07:15.690671+0000 mgr.y (mgr.25277) 555 : cluster 0 pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:17.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:16 smithi133 bash[102858]: cluster 2023-10-09T05:07:15.690671+0000 mgr.y (mgr.25277) 555 : cluster 0 pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:17.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:16 smithi194 bash[92666]: cluster 2023-10-09T05:07:15.690671+0000 mgr.y (mgr.25277) 555 : cluster 0 pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:19.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:18 smithi133 bash[105589]: cluster 2023-10-09T05:07:17.691187+0000 mgr.y (mgr.25277) 556 : cluster 0 pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:19.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:18 smithi133 bash[102858]: cluster 2023-10-09T05:07:17.691187+0000 mgr.y (mgr.25277) 556 : cluster 0 pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:19.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:18 smithi194 bash[92666]: cluster 2023-10-09T05:07:17.691187+0000 mgr.y (mgr.25277) 556 : cluster 0 pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:20.864 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:20 smithi133 bash[105589]: cluster 2023-10-09T05:07:19.691781+0000 mgr.y (mgr.25277) 557 : cluster 0 pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:20.864 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:20 smithi133 bash[102858]: cluster 2023-10-09T05:07:19.691781+0000 mgr.y (mgr.25277) 557 : cluster 0 pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:21.130 INFO:teuthology.orchestra.run.smithi133.stdout:412316860533 2023-10-09T05:07:21.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:20 smithi194 bash[92666]: cluster 2023-10-09T05:07:19.691781+0000 mgr.y (mgr.25277) 557 : cluster 0 pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:22.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:21 smithi133 bash[102858]: audit 2023-10-09T05:07:21.129542+0000 mon.c (mon.2) 51 : audit 0 from='client.? 172.21.15.133:0/138372676' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:07:22.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:21 smithi133 bash[105589]: audit 2023-10-09T05:07:21.129542+0000 mon.c (mon.2) 51 : audit 0 from='client.? 172.21.15.133:0/138372676' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:07:22.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:21 smithi194 bash[92666]: audit 2023-10-09T05:07:21.129542+0000 mon.c (mon.2) 51 : audit 0 from='client.? 172.21.15.133:0/138372676' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:07:22.875 INFO:teuthology.orchestra.run.smithi133.stdout:231928234301 2023-10-09T05:07:23.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:22 smithi133 bash[105589]: cluster 2023-10-09T05:07:21.692528+0000 mgr.y (mgr.25277) 558 : cluster 0 pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:23.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:22 smithi133 bash[102858]: cluster 2023-10-09T05:07:21.692528+0000 mgr.y (mgr.25277) 558 : cluster 0 pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:23.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:22 smithi194 bash[92666]: cluster 2023-10-09T05:07:21.692528+0000 mgr.y (mgr.25277) 558 : cluster 0 pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:23.492 INFO:teuthology.orchestra.run.smithi133.stdout:455266533454 2023-10-09T05:07:23.826 INFO:teuthology.orchestra.run.smithi133.stdout:197568495943 2023-10-09T05:07:24.021 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:23 smithi133 bash[105589]: audit 2023-10-09T05:07:22.874697+0000 mon.c (mon.2) 52 : audit 0 from='client.? 172.21.15.133:0/1995015641' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:07:24.021 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:23 smithi133 bash[105589]: audit 2023-10-09T05:07:23.492079+0000 mon.a (mon.0) 726 : audit 0 from='client.? 172.21.15.133:0/3230855034' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:07:24.022 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:23 smithi133 bash[102858]: audit 2023-10-09T05:07:22.874697+0000 mon.c (mon.2) 52 : audit 0 from='client.? 172.21.15.133:0/1995015641' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:07:24.022 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:23 smithi133 bash[102858]: audit 2023-10-09T05:07:23.492079+0000 mon.a (mon.0) 726 : audit 0 from='client.? 172.21.15.133:0/3230855034' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:07:24.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:23 smithi194 bash[92666]: audit 2023-10-09T05:07:22.874697+0000 mon.c (mon.2) 52 : audit 0 from='client.? 172.21.15.133:0/1995015641' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:07:24.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:23 smithi194 bash[92666]: audit 2023-10-09T05:07:23.492079+0000 mon.a (mon.0) 726 : audit 0 from='client.? 172.21.15.133:0/3230855034' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:07:24.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:07:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:07:24] "GET /metrics HTTP/1.1" 200 33990 "" "Prometheus/2.43.0" 2023-10-09T05:07:24.777 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:24 smithi133 bash[102858]: cluster 2023-10-09T05:07:23.693033+0000 mgr.y (mgr.25277) 559 : cluster 0 pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:24.777 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:24 smithi133 bash[102858]: audit 2023-10-09T05:07:23.826082+0000 mon.c (mon.2) 53 : audit 0 from='client.? 172.21.15.133:0/1861335790' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:07:24.778 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:24 smithi133 bash[105589]: cluster 2023-10-09T05:07:23.693033+0000 mgr.y (mgr.25277) 559 : cluster 0 pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:24.778 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:24 smithi133 bash[105589]: audit 2023-10-09T05:07:23.826082+0000 mon.c (mon.2) 53 : audit 0 from='client.? 172.21.15.133:0/1861335790' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:07:25.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:24 smithi194 bash[92666]: cluster 2023-10-09T05:07:23.693033+0000 mgr.y (mgr.25277) 559 : cluster 0 pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:25.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:24 smithi194 bash[92666]: audit 2023-10-09T05:07:23.826082+0000 mon.c (mon.2) 53 : audit 0 from='client.? 172.21.15.133:0/1861335790' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:07:25.730 INFO:teuthology.orchestra.run.smithi133.stdout:163208757585 2023-10-09T05:07:26.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:25 smithi133 bash[102858]: audit 2023-10-09T05:07:25.730190+0000 mon.c (mon.2) 54 : audit 0 from='client.? 172.21.15.133:0/2730098953' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:07:26.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:25 smithi133 bash[105589]: audit 2023-10-09T05:07:25.730190+0000 mon.c (mon.2) 54 : audit 0 from='client.? 172.21.15.133:0/2730098953' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:07:26.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:25 smithi194 bash[92666]: audit 2023-10-09T05:07:25.730190+0000 mon.c (mon.2) 54 : audit 0 from='client.? 172.21.15.133:0/2730098953' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:07:27.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:26 smithi133 bash[102858]: cluster 2023-10-09T05:07:25.693643+0000 mgr.y (mgr.25277) 560 : cluster 0 pgmap v515: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:27.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:26 smithi133 bash[105589]: cluster 2023-10-09T05:07:25.693643+0000 mgr.y (mgr.25277) 560 : cluster 0 pgmap v515: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:27.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:26 smithi194 bash[92666]: cluster 2023-10-09T05:07:25.693643+0000 mgr.y (mgr.25277) 560 : cluster 0 pgmap v515: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:27.345 INFO:tasks.cephadm.ceph_manager.ceph:need seq 412316860529 got 412316860533 for osd.1 2023-10-09T05:07:27.346 DEBUG:teuthology.parallel:result is None 2023-10-09T05:07:28.479 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495938 got 197568495943 for osd.6 2023-10-09T05:07:28.479 DEBUG:teuthology.parallel:result is None 2023-10-09T05:07:28.485 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234296 got 231928234301 for osd.7 2023-10-09T05:07:28.486 DEBUG:teuthology.parallel:result is None 2023-10-09T05:07:28.486 INFO:tasks.cephadm.ceph_manager.ceph:need seq 455266533449 got 455266533454 for osd.2 2023-10-09T05:07:28.486 DEBUG:teuthology.parallel:result is None 2023-10-09T05:07:28.875 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757581 got 163208757585 for osd.5 2023-10-09T05:07:28.876 DEBUG:teuthology.parallel:result is None 2023-10-09T05:07:29.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:28 smithi194 bash[92666]: cluster 2023-10-09T05:07:27.694346+0000 mgr.y (mgr.25277) 561 : cluster 0 pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:29.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:28 smithi194 bash[92666]: audit 2023-10-09T05:07:28.437673+0000 mon.a (mon.0) 727 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:07:29.139 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:28 smithi133 bash[102858]: cluster 2023-10-09T05:07:27.694346+0000 mgr.y (mgr.25277) 561 : cluster 0 pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:29.139 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:28 smithi133 bash[102858]: audit 2023-10-09T05:07:28.437673+0000 mon.a (mon.0) 727 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:07:29.139 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:28 smithi133 bash[105589]: cluster 2023-10-09T05:07:27.694346+0000 mgr.y (mgr.25277) 561 : cluster 0 pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:29.139 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:28 smithi133 bash[105589]: audit 2023-10-09T05:07:28.437673+0000 mon.a (mon.0) 727 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:07:29.594 INFO:teuthology.orchestra.run.smithi133.stdout:128849019230 2023-10-09T05:07:29.785 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:29 smithi133 bash[102858]: audit 2023-10-09T05:07:29.594139+0000 mon.a (mon.0) 728 : audit 0 from='client.? 172.21.15.133:0/4139441597' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:07:29.910 INFO:teuthology.orchestra.run.smithi133.stdout:528280977474 2023-10-09T05:07:29.934 INFO:teuthology.orchestra.run.smithi133.stdout:377957122176 2023-10-09T05:07:30.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:29 smithi133 bash[105589]: audit 2023-10-09T05:07:29.594139+0000 mon.a (mon.0) 728 : audit 0 from='client.? 172.21.15.133:0/4139441597' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:07:30.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:29 smithi194 bash[92666]: audit 2023-10-09T05:07:29.594139+0000 mon.a (mon.0) 728 : audit 0 from='client.? 172.21.15.133:0/4139441597' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:07:30.935 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019225 got 128849019230 for osd.4 2023-10-09T05:07:30.936 DEBUG:teuthology.parallel:result is None 2023-10-09T05:07:31.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:30 smithi133 bash[105589]: cluster 2023-10-09T05:07:29.694875+0000 mgr.y (mgr.25277) 562 : cluster 0 pgmap v517: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:31.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:30 smithi133 bash[105589]: audit 2023-10-09T05:07:29.909222+0000 mon.a (mon.0) 729 : audit 0 from='client.? 172.21.15.133:0/2626105780' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:07:31.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:30 smithi133 bash[105589]: audit 2023-10-09T05:07:29.933415+0000 mon.c (mon.2) 55 : audit 0 from='client.? 172.21.15.133:0/4188949530' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:07:31.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:30 smithi133 bash[102858]: cluster 2023-10-09T05:07:29.694875+0000 mgr.y (mgr.25277) 562 : cluster 0 pgmap v517: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:31.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:30 smithi133 bash[102858]: audit 2023-10-09T05:07:29.909222+0000 mon.a (mon.0) 729 : audit 0 from='client.? 172.21.15.133:0/2626105780' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:07:31.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:30 smithi133 bash[102858]: audit 2023-10-09T05:07:29.933415+0000 mon.c (mon.2) 55 : audit 0 from='client.? 172.21.15.133:0/4188949530' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:07:31.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:30 smithi194 bash[92666]: cluster 2023-10-09T05:07:29.694875+0000 mgr.y (mgr.25277) 562 : cluster 0 pgmap v517: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:31.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:30 smithi194 bash[92666]: audit 2023-10-09T05:07:29.909222+0000 mon.a (mon.0) 729 : audit 0 from='client.? 172.21.15.133:0/2626105780' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:07:31.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:30 smithi194 bash[92666]: audit 2023-10-09T05:07:29.933415+0000 mon.c (mon.2) 55 : audit 0 from='client.? 172.21.15.133:0/4188949530' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:07:31.362 INFO:tasks.cephadm.ceph_manager.ceph:need seq 528280977468 got 528280977474 for osd.3 2023-10-09T05:07:31.362 DEBUG:teuthology.parallel:result is None 2023-10-09T05:07:31.796 INFO:tasks.cephadm.ceph_manager.ceph:need seq 377957122171 got 377957122176 for osd.0 2023-10-09T05:07:31.797 DEBUG:teuthology.parallel:result is None 2023-10-09T05:07:31.797 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T05:07:31.797 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:07:33.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:32 smithi133 bash[105589]: cluster 2023-10-09T05:07:31.695551+0000 mgr.y (mgr.25277) 563 : cluster 0 pgmap v518: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:33.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:32 smithi133 bash[102858]: cluster 2023-10-09T05:07:31.695551+0000 mgr.y (mgr.25277) 563 : cluster 0 pgmap v518: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:33.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:32 smithi194 bash[92666]: cluster 2023-10-09T05:07:31.695551+0000 mgr.y (mgr.25277) 563 : cluster 0 pgmap v518: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:34.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:34 smithi133 bash[105589]: cluster 2023-10-09T05:07:33.696243+0000 mgr.y (mgr.25277) 564 : cluster 0 pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:34.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:34 smithi133 bash[102858]: cluster 2023-10-09T05:07:33.696243+0000 mgr.y (mgr.25277) 564 : cluster 0 pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:34.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:07:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:07:34] "GET /metrics HTTP/1.1" 200 33986 "" "Prometheus/2.43.0" 2023-10-09T05:07:34.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:34 smithi194 bash[92666]: cluster 2023-10-09T05:07:33.696243+0000 mgr.y (mgr.25277) 564 : cluster 0 pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:36.567 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:07:37.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:36 smithi133 bash[102858]: cluster 2023-10-09T05:07:35.697229+0000 mgr.y (mgr.25277) 565 : cluster 0 pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:37.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:36 smithi133 bash[105589]: cluster 2023-10-09T05:07:35.697229+0000 mgr.y (mgr.25277) 565 : cluster 0 pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:37.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:36 smithi194 bash[92666]: cluster 2023-10-09T05:07:35.697229+0000 mgr.y (mgr.25277) 565 : cluster 0 pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:39.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:38 smithi133 bash[105589]: cluster 2023-10-09T05:07:37.697845+0000 mgr.y (mgr.25277) 566 : cluster 0 pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:39.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:38 smithi133 bash[102858]: cluster 2023-10-09T05:07:37.697845+0000 mgr.y (mgr.25277) 566 : cluster 0 pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:39.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:38 smithi194 bash[92666]: cluster 2023-10-09T05:07:37.697845+0000 mgr.y (mgr.25277) 566 : cluster 0 pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:40.145 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:07:40.149 INFO:teuthology.orchestra.run.smithi133.stderr:dumped all 2023-10-09T05:07:40.981 INFO:teuthology.orchestra.run.smithi133.stdout:{"pg_ready":true,"pg_map":{"version":522,"stamp":"2023-10-09T05:07:39.698068+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":538,"num_read_kb":964,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":222,"ondisk_log_size":222,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2663104,"kb_used_data":13696,"kb_used_omap":20,"kb_used_meta":2649323,"kb_avail":747298112,"statfs":{"total":767960285184,"available":765233266688,"internally_reserved":0,"allocated":14024704,"data_stored":8813245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2712907406},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004182"},"pg_stats":[{"pgid":"3.1a","version":"69'2","reported_seq":118,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129055+0000","last_change":"2023-10-09T05:01:02.663550+0000","last_active":"2023-10-09T05:02:55.129055+0000","last_peered":"2023-10-09T05:02:55.129055+0000","last_clean":"2023-10-09T05:02:55.129055+0000","last_became_active":"2023-10-09T05:01:02.663202+0000","last_became_peered":"2023-10-09T05:01:02.663202+0000","last_unstale":"2023-10-09T05:02:55.129055+0000","last_undegraded":"2023-10-09T05:02:55.129055+0000","last_fullsized":"2023-10-09T05:02:55.129055+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:30.743395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":86,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.914359+0000","last_change":"2023-10-09T05:02:09.656135+0000","last_active":"2023-10-09T05:06:11.914359+0000","last_peered":"2023-10-09T05:06:11.914359+0000","last_clean":"2023-10-09T05:06:11.914359+0000","last_became_active":"2023-10-09T05:02:09.654594+0000","last_became_peered":"2023-10-09T05:02:09.654594+0000","last_unstale":"2023-10-09T05:06:11.914359+0000","last_undegraded":"2023-10-09T05:06:11.914359+0000","last_fullsized":"2023-10-09T05:06:11.914359+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:20.024818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":83,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.744393+0000","last_change":"2023-10-09T05:02:09.649701+0000","last_active":"2023-10-09T05:06:11.744393+0000","last_peered":"2023-10-09T05:06:11.744393+0000","last_clean":"2023-10-09T05:06:11.744393+0000","last_became_active":"2023-10-09T05:02:09.649370+0000","last_became_peered":"2023-10-09T05:02:09.649370+0000","last_unstale":"2023-10-09T05:06:11.744393+0000","last_undegraded":"2023-10-09T05:06:11.744393+0000","last_fullsized":"2023-10-09T05:06:11.744393+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:45.255492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":92,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.927998+0000","last_change":"2023-10-09T05:02:09.648283+0000","last_active":"2023-10-09T05:06:11.927998+0000","last_peered":"2023-10-09T05:06:11.927998+0000","last_clean":"2023-10-09T05:06:11.927998+0000","last_became_active":"2023-10-09T05:02:09.647778+0000","last_became_peered":"2023-10-09T05:02:09.647778+0000","last_unstale":"2023-10-09T05:06:11.927998+0000","last_undegraded":"2023-10-09T05:06:11.927998+0000","last_fullsized":"2023-10-09T05:06:11.927998+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:24.582392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":104,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.915863+0000","last_change":"2023-10-09T05:02:09.657157+0000","last_active":"2023-10-09T05:06:11.915863+0000","last_peered":"2023-10-09T05:06:11.915863+0000","last_clean":"2023-10-09T05:06:11.915863+0000","last_became_active":"2023-10-09T05:02:09.656354+0000","last_became_peered":"2023-10-09T05:02:09.656354+0000","last_unstale":"2023-10-09T05:06:11.915863+0000","last_undegraded":"2023-10-09T05:06:11.915863+0000","last_fullsized":"2023-10-09T05:06:11.915863+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:57.512482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.155272+0000","last_change":"2023-10-09T04:57:42.011855+0000","last_active":"2023-10-09T05:02:55.155272+0000","last_peered":"2023-10-09T05:02:55.155272+0000","last_clean":"2023-10-09T05:02:55.155272+0000","last_became_active":"2023-10-09T04:57:42.011233+0000","last_became_peered":"2023-10-09T04:57:42.011233+0000","last_unstale":"2023-10-09T05:02:55.155272+0000","last_undegraded":"2023-10-09T05:02:55.155272+0000","last_fullsized":"2023-10-09T05:02:55.155272+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:18.585929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":90,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.917521+0000","last_change":"2023-10-09T05:02:09.646765+0000","last_active":"2023-10-09T05:06:11.917521+0000","last_peered":"2023-10-09T05:06:11.917521+0000","last_clean":"2023-10-09T05:06:11.917521+0000","last_became_active":"2023-10-09T05:02:09.645141+0000","last_became_peered":"2023-10-09T05:02:09.645141+0000","last_unstale":"2023-10-09T05:06:11.917521+0000","last_undegraded":"2023-10-09T05:06:11.917521+0000","last_fullsized":"2023-10-09T05:06:11.917521+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:10.098578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":86,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.915378+0000","last_change":"2023-10-09T05:02:09.656872+0000","last_active":"2023-10-09T05:06:11.915378+0000","last_peered":"2023-10-09T05:06:11.915378+0000","last_clean":"2023-10-09T05:06:11.915378+0000","last_became_active":"2023-10-09T05:02:09.654713+0000","last_became_peered":"2023-10-09T05:02:09.654713+0000","last_unstale":"2023-10-09T05:06:11.915378+0000","last_undegraded":"2023-10-09T05:06:11.915378+0000","last_fullsized":"2023-10-09T05:06:11.915378+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:00:17.539155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":99,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.133911+0000","last_change":"2023-10-09T04:57:42.023522+0000","last_active":"2023-10-09T05:02:55.133911+0000","last_peered":"2023-10-09T05:02:55.133911+0000","last_clean":"2023-10-09T05:02:55.133911+0000","last_became_active":"2023-10-09T04:57:42.023010+0000","last_became_peered":"2023-10-09T04:57:42.023010+0000","last_unstale":"2023-10-09T05:02:55.133911+0000","last_undegraded":"2023-10-09T05:02:55.133911+0000","last_fullsized":"2023-10-09T05:02:55.133911+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:44.280677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":81,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.914646+0000","last_change":"2023-10-09T05:02:09.651167+0000","last_active":"2023-10-09T05:06:11.914646+0000","last_peered":"2023-10-09T05:06:11.914646+0000","last_clean":"2023-10-09T05:06:11.914646+0000","last_became_active":"2023-10-09T05:02:09.650419+0000","last_became_peered":"2023-10-09T05:02:09.650419+0000","last_unstale":"2023-10-09T05:06:11.914646+0000","last_undegraded":"2023-10-09T05:06:11.914646+0000","last_fullsized":"2023-10-09T05:06:11.914646+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:55:16.966938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":104,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.148854+0000","last_change":"2023-10-09T05:02:09.649700+0000","last_active":"2023-10-09T05:02:55.148854+0000","last_peered":"2023-10-09T05:02:55.148854+0000","last_clean":"2023-10-09T05:02:55.148854+0000","last_became_active":"2023-10-09T05:02:09.649317+0000","last_became_peered":"2023-10-09T05:02:09.649317+0000","last_unstale":"2023-10-09T05:02:55.148854+0000","last_undegraded":"2023-10-09T05:02:55.148854+0000","last_fullsized":"2023-10-09T05:02:55.148854+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:50.711767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":83,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.746974+0000","last_change":"2023-10-09T05:01:02.653598+0000","last_active":"2023-10-09T05:06:11.746974+0000","last_peered":"2023-10-09T05:06:11.746974+0000","last_clean":"2023-10-09T05:06:11.746974+0000","last_became_active":"2023-10-09T05:01:02.653351+0000","last_became_peered":"2023-10-09T05:01:02.653351+0000","last_unstale":"2023-10-09T05:06:11.746974+0000","last_undegraded":"2023-10-09T05:06:11.746974+0000","last_fullsized":"2023-10-09T05:06:11.746974+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:09.106185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'23","reported_seq":110,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129315+0000","last_change":"2023-10-09T04:44:52.781203+0000","last_active":"2023-10-09T05:02:55.129315+0000","last_peered":"2023-10-09T05:02:55.129315+0000","last_clean":"2023-10-09T05:02:55.129315+0000","last_became_active":"2023-10-09T04:44:52.780902+0000","last_became_peered":"2023-10-09T04:44:52.780902+0000","last_unstale":"2023-10-09T05:02:55.129315+0000","last_undegraded":"2023-10-09T05:02:55.129315+0000","last_fullsized":"2023-10-09T05:02:55.129315+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:56.332974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129435+0000","last_change":"2023-10-09T04:57:42.024993+0000","last_active":"2023-10-09T05:02:55.129435+0000","last_peered":"2023-10-09T05:02:55.129435+0000","last_clean":"2023-10-09T05:02:55.129435+0000","last_became_active":"2023-10-09T04:57:42.024227+0000","last_became_peered":"2023-10-09T04:57:42.024227+0000","last_unstale":"2023-10-09T05:02:55.129435+0000","last_undegraded":"2023-10-09T05:02:55.129435+0000","last_fullsized":"2023-10-09T05:02:55.129435+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:02.031798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":103,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.148581+0000","last_change":"2023-10-09T05:01:02.669186+0000","last_active":"2023-10-09T05:02:55.148581+0000","last_peered":"2023-10-09T05:02:55.148581+0000","last_clean":"2023-10-09T05:02:55.148581+0000","last_became_active":"2023-10-09T05:01:02.668739+0000","last_became_peered":"2023-10-09T05:01:02.668739+0000","last_unstale":"2023-10-09T05:02:55.148581+0000","last_undegraded":"2023-10-09T05:02:55.148581+0000","last_fullsized":"2023-10-09T05:02:55.148581+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:41.352624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":86,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.915825+0000","last_change":"2023-10-09T05:02:09.656245+0000","last_active":"2023-10-09T05:06:11.915825+0000","last_peered":"2023-10-09T05:06:11.915825+0000","last_clean":"2023-10-09T05:06:11.915825+0000","last_became_active":"2023-10-09T05:02:09.654672+0000","last_became_peered":"2023-10-09T05:02:09.654672+0000","last_unstale":"2023-10-09T05:06:11.915825+0000","last_undegraded":"2023-10-09T05:06:11.915825+0000","last_fullsized":"2023-10-09T05:06:11.915825+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:54.640349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":90,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.917907+0000","last_change":"2023-10-09T05:02:09.645017+0000","last_active":"2023-10-09T05:06:11.917907+0000","last_peered":"2023-10-09T05:06:11.917907+0000","last_clean":"2023-10-09T05:06:11.917907+0000","last_became_active":"2023-10-09T05:02:09.644705+0000","last_became_peered":"2023-10-09T05:02:09.644705+0000","last_unstale":"2023-10-09T05:06:11.917907+0000","last_undegraded":"2023-10-09T05:06:11.917907+0000","last_fullsized":"2023-10-09T05:06:11.917907+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:20.695961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.133929+0000","last_change":"2023-10-09T05:02:09.648362+0000","last_active":"2023-10-09T05:02:55.133929+0000","last_peered":"2023-10-09T05:02:55.133929+0000","last_clean":"2023-10-09T05:02:55.133929+0000","last_became_active":"2023-10-09T05:02:09.647127+0000","last_became_peered":"2023-10-09T05:02:09.647127+0000","last_unstale":"2023-10-09T05:02:55.133929+0000","last_undegraded":"2023-10-09T05:02:55.133929+0000","last_fullsized":"2023-10-09T05:02:55.133929+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:38:02.875613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":82,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.148137+0000","last_change":"2023-10-09T04:44:52.792557+0000","last_active":"2023-10-09T05:02:55.148137+0000","last_peered":"2023-10-09T05:02:55.148137+0000","last_clean":"2023-10-09T05:02:55.148137+0000","last_became_active":"2023-10-09T04:44:52.791890+0000","last_became_peered":"2023-10-09T04:44:52.791890+0000","last_unstale":"2023-10-09T05:02:55.148137+0000","last_undegraded":"2023-10-09T05:02:55.148137+0000","last_fullsized":"2023-10-09T05:02:55.148137+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:04:46.379852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":91,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.928065+0000","last_change":"2023-10-09T05:02:09.652158+0000","last_active":"2023-10-09T05:06:11.928065+0000","last_peered":"2023-10-09T05:06:11.928065+0000","last_clean":"2023-10-09T05:06:11.928065+0000","last_became_active":"2023-10-09T05:02:09.651811+0000","last_became_peered":"2023-10-09T05:02:09.651811+0000","last_unstale":"2023-10-09T05:06:11.928065+0000","last_undegraded":"2023-10-09T05:06:11.928065+0000","last_fullsized":"2023-10-09T05:06:11.928065+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:49:10.141237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":111,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.154425+0000","last_change":"2023-10-09T04:56:56.076177+0000","last_active":"2023-10-09T05:02:55.154425+0000","last_peered":"2023-10-09T05:02:55.154425+0000","last_clean":"2023-10-09T05:02:55.154425+0000","last_became_active":"2023-10-09T04:56:56.075564+0000","last_became_peered":"2023-10-09T04:56:56.075564+0000","last_unstale":"2023-10-09T05:02:55.154425+0000","last_undegraded":"2023-10-09T05:02:55.154425+0000","last_fullsized":"2023-10-09T05:02:55.154425+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:38.390793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":86,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.915451+0000","last_change":"2023-10-09T05:02:09.656286+0000","last_active":"2023-10-09T05:06:11.915451+0000","last_peered":"2023-10-09T05:06:11.915451+0000","last_clean":"2023-10-09T05:06:11.915451+0000","last_became_active":"2023-10-09T05:02:09.654429+0000","last_became_peered":"2023-10-09T05:02:09.654429+0000","last_unstale":"2023-10-09T05:06:11.915451+0000","last_undegraded":"2023-10-09T05:06:11.915451+0000","last_fullsized":"2023-10-09T05:06:11.915451+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:55:05.953948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":101,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.149120+0000","last_change":"2023-10-09T04:56:56.075307+0000","last_active":"2023-10-09T05:02:55.149120+0000","last_peered":"2023-10-09T05:02:55.149120+0000","last_clean":"2023-10-09T05:02:55.149120+0000","last_became_active":"2023-10-09T04:56:56.074831+0000","last_became_peered":"2023-10-09T04:56:56.074831+0000","last_unstale":"2023-10-09T05:02:55.149120+0000","last_undegraded":"2023-10-09T05:02:55.149120+0000","last_fullsized":"2023-10-09T05:02:55.149120+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:51:56.867759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":81,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.134283+0000","last_change":"2023-10-09T04:44:53.962437+0000","last_active":"2023-10-09T05:02:55.134283+0000","last_peered":"2023-10-09T05:02:55.134283+0000","last_clean":"2023-10-09T05:02:55.134283+0000","last_became_active":"2023-10-09T04:44:53.961810+0000","last_became_peered":"2023-10-09T04:44:53.961810+0000","last_unstale":"2023-10-09T05:02:55.134283+0000","last_undegraded":"2023-10-09T05:02:55.134283+0000","last_fullsized":"2023-10-09T05:02:55.134283+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:56.436933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":102,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.148994+0000","last_change":"2023-10-09T04:57:42.023336+0000","last_active":"2023-10-09T05:02:55.148994+0000","last_peered":"2023-10-09T05:02:55.148994+0000","last_clean":"2023-10-09T05:02:55.148994+0000","last_became_active":"2023-10-09T04:57:42.022861+0000","last_became_peered":"2023-10-09T04:57:42.022861+0000","last_unstale":"2023-10-09T05:02:55.148994+0000","last_undegraded":"2023-10-09T05:02:55.148994+0000","last_fullsized":"2023-10-09T05:02:55.148994+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:57.858993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":115,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.128925+0000","last_change":"2023-10-09T05:02:09.653562+0000","last_active":"2023-10-09T05:02:55.128925+0000","last_peered":"2023-10-09T05:02:55.128925+0000","last_clean":"2023-10-09T05:02:55.128925+0000","last_became_active":"2023-10-09T05:02:09.652830+0000","last_became_peered":"2023-10-09T05:02:09.652830+0000","last_unstale":"2023-10-09T05:02:55.128925+0000","last_undegraded":"2023-10-09T05:02:55.128925+0000","last_fullsized":"2023-10-09T05:02:55.128925+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:54.527011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":84,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.148487+0000","last_change":"2023-10-09T04:44:52.787469+0000","last_active":"2023-10-09T05:02:55.148487+0000","last_peered":"2023-10-09T05:02:55.148487+0000","last_clean":"2023-10-09T05:02:55.148487+0000","last_became_active":"2023-10-09T04:44:52.787148+0000","last_became_peered":"2023-10-09T04:44:52.787148+0000","last_unstale":"2023-10-09T05:02:55.148487+0000","last_undegraded":"2023-10-09T05:02:55.148487+0000","last_fullsized":"2023-10-09T05:02:55.148487+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:12.469065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":108,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.154612+0000","last_change":"2023-10-09T05:01:02.659380+0000","last_active":"2023-10-09T05:02:55.154612+0000","last_peered":"2023-10-09T05:02:55.154612+0000","last_clean":"2023-10-09T05:02:55.154612+0000","last_became_active":"2023-10-09T05:01:02.659064+0000","last_became_peered":"2023-10-09T05:01:02.659064+0000","last_unstale":"2023-10-09T05:02:55.154612+0000","last_undegraded":"2023-10-09T05:02:55.154612+0000","last_fullsized":"2023-10-09T05:02:55.154612+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:35.342503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":99,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.133928+0000","last_change":"2023-10-09T05:02:09.653932+0000","last_active":"2023-10-09T05:02:55.133928+0000","last_peered":"2023-10-09T05:02:55.133928+0000","last_clean":"2023-10-09T05:02:55.133928+0000","last_became_active":"2023-10-09T05:02:09.652923+0000","last_became_peered":"2023-10-09T05:02:09.652923+0000","last_unstale":"2023-10-09T05:02:55.133928+0000","last_undegraded":"2023-10-09T05:02:55.133928+0000","last_fullsized":"2023-10-09T05:02:55.133928+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:44:11.928237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":91,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:05:55.547270+0000","last_change":"2023-10-09T04:57:42.020009+0000","last_active":"2023-10-09T05:05:55.547270+0000","last_peered":"2023-10-09T05:05:55.547270+0000","last_clean":"2023-10-09T05:05:55.547270+0000","last_became_active":"2023-10-09T04:57:42.019678+0000","last_became_peered":"2023-10-09T04:57:42.019678+0000","last_unstale":"2023-10-09T05:05:55.547270+0000","last_undegraded":"2023-10-09T05:05:55.547270+0000","last_fullsized":"2023-10-09T05:05:55.547270+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:16.812128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'3","reported_seq":83,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.914244+0000","last_change":"2023-10-09T05:02:09.650579+0000","last_active":"2023-10-09T05:06:11.914244+0000","last_peered":"2023-10-09T05:06:11.914244+0000","last_clean":"2023-10-09T05:06:11.914244+0000","last_became_active":"2023-10-09T05:02:09.650160+0000","last_became_peered":"2023-10-09T05:02:09.650160+0000","last_unstale":"2023-10-09T05:06:11.914244+0000","last_undegraded":"2023-10-09T05:06:11.914244+0000","last_fullsized":"2023-10-09T05:06:11.914244+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:18.292670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129404+0000","last_change":"2023-10-09T05:02:09.652449+0000","last_active":"2023-10-09T05:02:55.129404+0000","last_peered":"2023-10-09T05:02:55.129404+0000","last_clean":"2023-10-09T05:02:55.129404+0000","last_became_active":"2023-10-09T05:02:09.651994+0000","last_became_peered":"2023-10-09T05:02:09.651994+0000","last_unstale":"2023-10-09T05:02:55.129404+0000","last_undegraded":"2023-10-09T05:02:55.129404+0000","last_fullsized":"2023-10-09T05:02:55.129404+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:14.480513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":99,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.154853+0000","last_change":"2023-10-09T04:57:42.016620+0000","last_active":"2023-10-09T05:02:55.154853+0000","last_peered":"2023-10-09T05:02:55.154853+0000","last_clean":"2023-10-09T05:02:55.154853+0000","last_became_active":"2023-10-09T04:57:42.016215+0000","last_became_peered":"2023-10-09T04:57:42.016215+0000","last_unstale":"2023-10-09T05:02:55.154853+0000","last_undegraded":"2023-10-09T05:02:55.154853+0000","last_fullsized":"2023-10-09T05:02:55.154853+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:21:24.361501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":115,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129567+0000","last_change":"2023-10-09T05:01:02.661970+0000","last_active":"2023-10-09T05:02:55.129567+0000","last_peered":"2023-10-09T05:02:55.129567+0000","last_clean":"2023-10-09T05:02:55.129567+0000","last_became_active":"2023-10-09T05:01:02.661646+0000","last_became_peered":"2023-10-09T05:01:02.661646+0000","last_unstale":"2023-10-09T05:02:55.129567+0000","last_undegraded":"2023-10-09T05:02:55.129567+0000","last_fullsized":"2023-10-09T05:02:55.129567+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:12.013587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":99,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.128906+0000","last_change":"2023-10-09T05:01:02.663309+0000","last_active":"2023-10-09T05:02:55.128906+0000","last_peered":"2023-10-09T05:02:55.128906+0000","last_clean":"2023-10-09T05:02:55.128906+0000","last_became_active":"2023-10-09T05:01:02.663038+0000","last_became_peered":"2023-10-09T05:01:02.663038+0000","last_unstale":"2023-10-09T05:02:55.128906+0000","last_undegraded":"2023-10-09T05:02:55.128906+0000","last_fullsized":"2023-10-09T05:02:55.128906+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:29:36.689304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129594+0000","last_change":"2023-10-09T04:57:42.028346+0000","last_active":"2023-10-09T05:02:55.129594+0000","last_peered":"2023-10-09T05:02:55.129594+0000","last_clean":"2023-10-09T05:02:55.129594+0000","last_became_active":"2023-10-09T04:57:42.027881+0000","last_became_peered":"2023-10-09T04:57:42.027881+0000","last_unstale":"2023-10-09T05:02:55.129594+0000","last_undegraded":"2023-10-09T05:02:55.129594+0000","last_fullsized":"2023-10-09T05:02:55.129594+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:00:43.197716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":87,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.916032+0000","last_change":"2023-10-09T05:02:09.656904+0000","last_active":"2023-10-09T05:06:11.916032+0000","last_peered":"2023-10-09T05:06:11.916032+0000","last_clean":"2023-10-09T05:06:11.916032+0000","last_became_active":"2023-10-09T05:02:09.654658+0000","last_became_peered":"2023-10-09T05:02:09.654658+0000","last_unstale":"2023-10-09T05:06:11.916032+0000","last_undegraded":"2023-10-09T05:06:11.916032+0000","last_fullsized":"2023-10-09T05:06:11.916032+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:51:26.666116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":83,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.745701+0000","last_change":"2023-10-09T05:02:09.652086+0000","last_active":"2023-10-09T05:06:11.745701+0000","last_peered":"2023-10-09T05:06:11.745701+0000","last_clean":"2023-10-09T05:06:11.745701+0000","last_became_active":"2023-10-09T05:02:09.651733+0000","last_became_peered":"2023-10-09T05:02:09.651733+0000","last_unstale":"2023-10-09T05:06:11.745701+0000","last_undegraded":"2023-10-09T05:06:11.745701+0000","last_fullsized":"2023-10-09T05:06:11.745701+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:32:59.894793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":112,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:06:01.201267+0000","last_change":"2023-10-09T04:57:42.019588+0000","last_active":"2023-10-09T05:06:01.201267+0000","last_peered":"2023-10-09T05:06:01.201267+0000","last_clean":"2023-10-09T05:06:01.201267+0000","last_became_active":"2023-10-09T04:57:42.019274+0000","last_became_peered":"2023-10-09T04:57:42.019274+0000","last_unstale":"2023-10-09T05:06:01.201267+0000","last_undegraded":"2023-10-09T05:06:01.201267+0000","last_fullsized":"2023-10-09T05:06:01.201267+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:21.969343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":116,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.134140+0000","last_change":"2023-10-09T05:02:09.649559+0000","last_active":"2023-10-09T05:02:55.134140+0000","last_peered":"2023-10-09T05:02:55.134140+0000","last_clean":"2023-10-09T05:02:55.134140+0000","last_became_active":"2023-10-09T05:02:09.649195+0000","last_became_peered":"2023-10-09T05:02:09.649195+0000","last_unstale":"2023-10-09T05:02:55.134140+0000","last_undegraded":"2023-10-09T05:02:55.134140+0000","last_fullsized":"2023-10-09T05:02:55.134140+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:56.864170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":68,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:06:01.201204+0000","last_change":"2023-10-09T04:57:42.022222+0000","last_active":"2023-10-09T05:06:01.201204+0000","last_peered":"2023-10-09T05:06:01.201204+0000","last_clean":"2023-10-09T05:06:01.201204+0000","last_became_active":"2023-10-09T04:57:42.021893+0000","last_became_peered":"2023-10-09T04:57:42.021893+0000","last_unstale":"2023-10-09T05:06:01.201204+0000","last_undegraded":"2023-10-09T05:06:01.201204+0000","last_fullsized":"2023-10-09T05:06:01.201204+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:39:20.734382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":103,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.133781+0000","last_change":"2023-10-09T04:57:42.010576+0000","last_active":"2023-10-09T05:02:55.133781+0000","last_peered":"2023-10-09T05:02:55.133781+0000","last_clean":"2023-10-09T05:02:55.133781+0000","last_became_active":"2023-10-09T04:57:42.010145+0000","last_became_peered":"2023-10-09T04:57:42.010145+0000","last_unstale":"2023-10-09T05:02:55.133781+0000","last_undegraded":"2023-10-09T05:02:55.133781+0000","last_fullsized":"2023-10-09T05:02:55.133781+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:18:17.196612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":99,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129297+0000","last_change":"2023-10-09T04:56:56.087320+0000","last_active":"2023-10-09T05:02:55.129297+0000","last_peered":"2023-10-09T05:02:55.129297+0000","last_clean":"2023-10-09T05:02:55.129297+0000","last_became_active":"2023-10-09T04:56:56.086552+0000","last_became_peered":"2023-10-09T04:56:56.086552+0000","last_unstale":"2023-10-09T05:02:55.129297+0000","last_undegraded":"2023-10-09T05:02:55.129297+0000","last_fullsized":"2023-10-09T05:02:55.129297+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:14.205288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":111,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.134027+0000","last_change":"2023-10-09T05:02:09.653655+0000","last_active":"2023-10-09T05:02:55.134027+0000","last_peered":"2023-10-09T05:02:55.134027+0000","last_clean":"2023-10-09T05:02:55.134027+0000","last_became_active":"2023-10-09T05:02:09.652682+0000","last_became_peered":"2023-10-09T05:02:09.652682+0000","last_unstale":"2023-10-09T05:02:55.134027+0000","last_undegraded":"2023-10-09T05:02:55.134027+0000","last_fullsized":"2023-10-09T05:02:55.134027+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:21:28.150022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":116,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.748439+0000","last_change":"2023-10-09T05:02:09.645835+0000","last_active":"2023-10-09T05:06:11.748439+0000","last_peered":"2023-10-09T05:06:11.748439+0000","last_clean":"2023-10-09T05:06:11.748439+0000","last_became_active":"2023-10-09T05:02:09.645491+0000","last_became_peered":"2023-10-09T05:02:09.645491+0000","last_unstale":"2023-10-09T05:06:11.748439+0000","last_undegraded":"2023-10-09T05:06:11.748439+0000","last_fullsized":"2023-10-09T05:06:11.748439+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:06:20.823201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":83,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:05:41.165289+0000","last_change":"2023-10-09T05:01:02.656091+0000","last_active":"2023-10-09T05:05:41.165289+0000","last_peered":"2023-10-09T05:05:41.165289+0000","last_clean":"2023-10-09T05:05:41.165289+0000","last_became_active":"2023-10-09T05:01:02.654755+0000","last_became_peered":"2023-10-09T05:01:02.654755+0000","last_unstale":"2023-10-09T05:05:41.165289+0000","last_undegraded":"2023-10-09T05:05:41.165289+0000","last_fullsized":"2023-10-09T05:05:41.165289+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:07.144598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":92,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.749112+0000","last_change":"2023-10-09T05:01:02.667904+0000","last_active":"2023-10-09T05:06:11.749112+0000","last_peered":"2023-10-09T05:06:11.749112+0000","last_clean":"2023-10-09T05:06:11.749112+0000","last_became_active":"2023-10-09T05:01:02.667574+0000","last_became_peered":"2023-10-09T05:01:02.667574+0000","last_unstale":"2023-10-09T05:06:11.749112+0000","last_undegraded":"2023-10-09T05:06:11.749112+0000","last_fullsized":"2023-10-09T05:06:11.749112+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:48:50.643600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":103,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.133545+0000","last_change":"2023-10-09T05:01:02.685038+0000","last_active":"2023-10-09T05:02:55.133545+0000","last_peered":"2023-10-09T05:02:55.133545+0000","last_clean":"2023-10-09T05:02:55.133545+0000","last_became_active":"2023-10-09T05:01:02.683622+0000","last_became_peered":"2023-10-09T05:01:02.683622+0000","last_unstale":"2023-10-09T05:02:55.133545+0000","last_undegraded":"2023-10-09T05:02:55.133545+0000","last_fullsized":"2023-10-09T05:02:55.133545+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:35.435592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":476,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.151016+0000","last_change":"2023-10-09T04:56:56.075762+0000","last_active":"2023-10-09T05:02:55.151016+0000","last_peered":"2023-10-09T05:02:55.151016+0000","last_clean":"2023-10-09T05:02:55.151016+0000","last_became_active":"2023-10-09T04:56:56.075263+0000","last_became_peered":"2023-10-09T04:56:56.075263+0000","last_unstale":"2023-10-09T05:02:55.151016+0000","last_undegraded":"2023-10-09T05:02:55.151016+0000","last_fullsized":"2023-10-09T05:02:55.151016+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_clean_scrub_stamp":"2023-10-09T04:36:44.261013+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:02.769164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":81,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.914252+0000","last_change":"2023-10-09T05:02:09.654237+0000","last_active":"2023-10-09T05:06:11.914252+0000","last_peered":"2023-10-09T05:06:11.914252+0000","last_clean":"2023-10-09T05:06:11.914252+0000","last_became_active":"2023-10-09T05:02:09.653909+0000","last_became_peered":"2023-10-09T05:02:09.653909+0000","last_unstale":"2023-10-09T05:06:11.914252+0000","last_undegraded":"2023-10-09T05:06:11.914252+0000","last_fullsized":"2023-10-09T05:06:11.914252+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:02.002366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":81,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.154930+0000","last_change":"2023-10-09T04:44:53.959705+0000","last_active":"2023-10-09T05:02:55.154930+0000","last_peered":"2023-10-09T05:02:55.154930+0000","last_clean":"2023-10-09T05:02:55.154930+0000","last_became_active":"2023-10-09T04:44:53.959092+0000","last_became_peered":"2023-10-09T04:44:53.959092+0000","last_unstale":"2023-10-09T05:02:55.154930+0000","last_undegraded":"2023-10-09T05:02:55.154930+0000","last_fullsized":"2023-10-09T05:02:55.154930+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:31:29.189897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":120,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.151119+0000","last_change":"2023-10-09T04:57:42.023353+0000","last_active":"2023-10-09T05:02:55.151119+0000","last_peered":"2023-10-09T05:02:55.151119+0000","last_clean":"2023-10-09T05:02:55.151119+0000","last_became_active":"2023-10-09T04:57:42.023007+0000","last_became_peered":"2023-10-09T04:57:42.023007+0000","last_unstale":"2023-10-09T05:02:55.151119+0000","last_undegraded":"2023-10-09T05:02:55.151119+0000","last_fullsized":"2023-10-09T05:02:55.151119+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:43.715250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":91,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.914248+0000","last_change":"2023-10-09T05:02:09.648781+0000","last_active":"2023-10-09T05:06:11.914248+0000","last_peered":"2023-10-09T05:06:11.914248+0000","last_clean":"2023-10-09T05:06:11.914248+0000","last_became_active":"2023-10-09T05:02:09.648078+0000","last_became_peered":"2023-10-09T05:02:09.648078+0000","last_unstale":"2023-10-09T05:06:11.914248+0000","last_undegraded":"2023-10-09T05:06:11.914248+0000","last_fullsized":"2023-10-09T05:06:11.914248+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:35.287091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":106,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:05:35.530424+0000","last_change":"2023-10-09T05:01:02.653869+0000","last_active":"2023-10-09T05:05:35.530424+0000","last_peered":"2023-10-09T05:05:35.530424+0000","last_clean":"2023-10-09T05:05:35.530424+0000","last_became_active":"2023-10-09T05:01:02.653493+0000","last_became_peered":"2023-10-09T05:01:02.653493+0000","last_unstale":"2023-10-09T05:05:35.530424+0000","last_undegraded":"2023-10-09T05:05:35.530424+0000","last_fullsized":"2023-10-09T05:05:35.530424+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:40:46.722386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":103,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.148680+0000","last_change":"2023-10-09T04:56:56.073473+0000","last_active":"2023-10-09T05:02:55.148680+0000","last_peered":"2023-10-09T05:02:55.148680+0000","last_clean":"2023-10-09T05:02:55.148680+0000","last_became_active":"2023-10-09T04:56:56.073146+0000","last_became_peered":"2023-10-09T04:56:56.073146+0000","last_unstale":"2023-10-09T05:02:55.148680+0000","last_undegraded":"2023-10-09T05:02:55.148680+0000","last_fullsized":"2023-10-09T05:02:55.148680+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:55:25.132822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":83,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:06:01.201293+0000","last_change":"2023-10-09T05:01:02.664643+0000","last_active":"2023-10-09T05:06:01.201293+0000","last_peered":"2023-10-09T05:06:01.201293+0000","last_clean":"2023-10-09T05:06:01.201293+0000","last_became_active":"2023-10-09T05:01:02.664288+0000","last_became_peered":"2023-10-09T05:01:02.664288+0000","last_unstale":"2023-10-09T05:06:01.201293+0000","last_undegraded":"2023-10-09T05:06:01.201293+0000","last_fullsized":"2023-10-09T05:06:01.201293+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:01.572876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":77,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.916855+0000","last_change":"2023-10-09T04:56:56.071763+0000","last_active":"2023-10-09T05:06:11.916855+0000","last_peered":"2023-10-09T05:06:11.916855+0000","last_clean":"2023-10-09T05:06:11.916855+0000","last_became_active":"2023-10-09T04:56:56.071429+0000","last_became_peered":"2023-10-09T04:56:56.071429+0000","last_unstale":"2023-10-09T05:06:11.916855+0000","last_undegraded":"2023-10-09T05:06:11.916855+0000","last_fullsized":"2023-10-09T05:06:11.916855+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:53:02.051460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":82,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.915549+0000","last_change":"2023-10-09T05:02:09.651149+0000","last_active":"2023-10-09T05:06:11.915549+0000","last_peered":"2023-10-09T05:06:11.915549+0000","last_clean":"2023-10-09T05:06:11.915549+0000","last_became_active":"2023-10-09T05:02:09.650151+0000","last_became_peered":"2023-10-09T05:02:09.650151+0000","last_unstale":"2023-10-09T05:06:11.915549+0000","last_undegraded":"2023-10-09T05:06:11.915549+0000","last_fullsized":"2023-10-09T05:06:11.915549+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:49.100209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":67,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.744947+0000","last_change":"2023-10-09T04:57:42.023683+0000","last_active":"2023-10-09T05:06:11.744947+0000","last_peered":"2023-10-09T05:06:11.744947+0000","last_clean":"2023-10-09T05:06:11.744947+0000","last_became_active":"2023-10-09T04:57:42.023371+0000","last_became_peered":"2023-10-09T04:57:42.023371+0000","last_unstale":"2023-10-09T05:06:11.744947+0000","last_undegraded":"2023-10-09T05:06:11.744947+0000","last_fullsized":"2023-10-09T05:06:11.744947+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:17:29.936222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":78,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.094920+0000","last_change":"2023-10-09T05:02:09.650750+0000","last_active":"2023-10-09T05:06:08.094920+0000","last_peered":"2023-10-09T05:06:08.094920+0000","last_clean":"2023-10-09T05:06:08.094920+0000","last_became_active":"2023-10-09T05:02:09.650351+0000","last_became_peered":"2023-10-09T05:02:09.650351+0000","last_unstale":"2023-10-09T05:06:08.094920+0000","last_undegraded":"2023-10-09T05:06:08.094920+0000","last_fullsized":"2023-10-09T05:06:08.094920+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:14.826132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":103,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.155202+0000","last_change":"2023-10-09T05:01:02.660775+0000","last_active":"2023-10-09T05:02:55.155202+0000","last_peered":"2023-10-09T05:02:55.155202+0000","last_clean":"2023-10-09T05:02:55.155202+0000","last_became_active":"2023-10-09T05:01:02.660420+0000","last_became_peered":"2023-10-09T05:01:02.660420+0000","last_unstale":"2023-10-09T05:02:55.155202+0000","last_undegraded":"2023-10-09T05:02:55.155202+0000","last_fullsized":"2023-10-09T05:02:55.155202+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:54.031869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":95,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:05:35.530446+0000","last_change":"2023-10-09T04:57:42.017497+0000","last_active":"2023-10-09T05:05:35.530446+0000","last_peered":"2023-10-09T05:05:35.530446+0000","last_clean":"2023-10-09T05:05:35.530446+0000","last_became_active":"2023-10-09T04:57:42.017165+0000","last_became_peered":"2023-10-09T04:57:42.017165+0000","last_unstale":"2023-10-09T05:05:35.530446+0000","last_undegraded":"2023-10-09T05:05:35.530446+0000","last_fullsized":"2023-10-09T05:05:35.530446+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:43.258960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":81,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.128729+0000","last_change":"2023-10-09T04:44:53.792412+0000","last_active":"2023-10-09T05:02:55.128729+0000","last_peered":"2023-10-09T05:02:55.128729+0000","last_clean":"2023-10-09T05:02:55.128729+0000","last_became_active":"2023-10-09T04:44:53.792009+0000","last_became_peered":"2023-10-09T04:44:53.792009+0000","last_unstale":"2023-10-09T05:02:55.128729+0000","last_undegraded":"2023-10-09T05:02:55.128729+0000","last_fullsized":"2023-10-09T05:02:55.128729+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:39.423339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'2","reported_seq":81,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.917224+0000","last_change":"2023-10-09T04:56:56.079768+0000","last_active":"2023-10-09T05:06:11.917224+0000","last_peered":"2023-10-09T05:06:11.917224+0000","last_clean":"2023-10-09T05:06:11.917224+0000","last_became_active":"2023-10-09T04:56:56.077721+0000","last_became_peered":"2023-10-09T04:56:56.077721+0000","last_unstale":"2023-10-09T05:06:11.917224+0000","last_undegraded":"2023-10-09T05:06:11.917224+0000","last_fullsized":"2023-10-09T05:06:11.917224+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:36.384357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":118,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:06:00.911094+0000","last_change":"2023-10-09T05:02:09.644679+0000","last_active":"2023-10-09T05:06:00.911094+0000","last_peered":"2023-10-09T05:06:00.911094+0000","last_clean":"2023-10-09T05:06:00.911094+0000","last_became_active":"2023-10-09T05:02:09.644335+0000","last_became_peered":"2023-10-09T05:02:09.644335+0000","last_unstale":"2023-10-09T05:06:00.911094+0000","last_undegraded":"2023-10-09T05:06:00.911094+0000","last_fullsized":"2023-10-09T05:06:00.911094+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:06.142246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":102,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.134135+0000","last_change":"2023-10-09T04:57:42.018276+0000","last_active":"2023-10-09T05:02:55.134135+0000","last_peered":"2023-10-09T05:02:55.134135+0000","last_clean":"2023-10-09T05:02:55.134135+0000","last_became_active":"2023-10-09T04:57:42.017975+0000","last_became_peered":"2023-10-09T04:57:42.017975+0000","last_unstale":"2023-10-09T05:02:55.134135+0000","last_undegraded":"2023-10-09T05:02:55.134135+0000","last_fullsized":"2023-10-09T05:02:55.134135+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:48.867813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":115,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129845+0000","last_change":"2023-10-09T05:02:09.653528+0000","last_active":"2023-10-09T05:02:55.129845+0000","last_peered":"2023-10-09T05:02:55.129845+0000","last_clean":"2023-10-09T05:02:55.129845+0000","last_became_active":"2023-10-09T05:02:09.653171+0000","last_became_peered":"2023-10-09T05:02:09.653171+0000","last_unstale":"2023-10-09T05:02:55.129845+0000","last_undegraded":"2023-10-09T05:02:55.129845+0000","last_fullsized":"2023-10-09T05:02:55.129845+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:15.978289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":85,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.134213+0000","last_change":"2023-10-09T04:44:52.793709+0000","last_active":"2023-10-09T05:02:55.134213+0000","last_peered":"2023-10-09T05:02:55.134213+0000","last_clean":"2023-10-09T05:02:55.134213+0000","last_became_active":"2023-10-09T04:44:52.792973+0000","last_became_peered":"2023-10-09T04:44:52.792973+0000","last_unstale":"2023-10-09T05:02:55.134213+0000","last_undegraded":"2023-10-09T05:02:55.134213+0000","last_fullsized":"2023-10-09T05:02:55.134213+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:12.091089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":115,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129621+0000","last_change":"2023-10-09T05:02:09.649748+0000","last_active":"2023-10-09T05:02:55.129621+0000","last_peered":"2023-10-09T05:02:55.129621+0000","last_clean":"2023-10-09T05:02:55.129621+0000","last_became_active":"2023-10-09T05:02:09.649444+0000","last_became_peered":"2023-10-09T05:02:09.649444+0000","last_unstale":"2023-10-09T05:02:55.129621+0000","last_undegraded":"2023-10-09T05:02:55.129621+0000","last_fullsized":"2023-10-09T05:02:55.129621+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:07.436820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'2","reported_seq":99,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.095102+0000","last_change":"2023-10-09T05:02:09.647382+0000","last_active":"2023-10-09T05:06:08.095102+0000","last_peered":"2023-10-09T05:06:08.095102+0000","last_clean":"2023-10-09T05:06:08.095102+0000","last_became_active":"2023-10-09T05:02:09.646803+0000","last_became_peered":"2023-10-09T05:02:09.646803+0000","last_unstale":"2023-10-09T05:06:08.095102+0000","last_undegraded":"2023-10-09T05:06:08.095102+0000","last_fullsized":"2023-10-09T05:06:08.095102+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:23:00.161736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":113,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.914782+0000","last_change":"2023-10-09T05:02:09.649674+0000","last_active":"2023-10-09T05:06:11.914782+0000","last_peered":"2023-10-09T05:06:11.914782+0000","last_clean":"2023-10-09T05:06:11.914782+0000","last_became_active":"2023-10-09T05:02:09.649041+0000","last_became_peered":"2023-10-09T05:02:09.649041+0000","last_unstale":"2023-10-09T05:06:11.914782+0000","last_undegraded":"2023-10-09T05:06:11.914782+0000","last_fullsized":"2023-10-09T05:06:11.914782+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:18.524071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'5","reported_seq":101,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.917631+0000","last_change":"2023-10-09T05:01:02.672596+0000","last_active":"2023-10-09T05:06:11.917631+0000","last_peered":"2023-10-09T05:06:11.917631+0000","last_clean":"2023-10-09T05:06:11.917631+0000","last_became_active":"2023-10-09T05:01:02.672128+0000","last_became_peered":"2023-10-09T05:01:02.672128+0000","last_unstale":"2023-10-09T05:06:11.917631+0000","last_undegraded":"2023-10-09T05:06:11.917631+0000","last_fullsized":"2023-10-09T05:06:11.917631+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:17:05.887502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.128756+0000","last_change":"2023-10-09T04:57:42.027187+0000","last_active":"2023-10-09T05:02:55.128756+0000","last_peered":"2023-10-09T05:02:55.128756+0000","last_clean":"2023-10-09T05:02:55.128756+0000","last_became_active":"2023-10-09T04:57:42.024247+0000","last_became_peered":"2023-10-09T04:57:42.024247+0000","last_unstale":"2023-10-09T05:02:55.128756+0000","last_undegraded":"2023-10-09T05:02:55.128756+0000","last_fullsized":"2023-10-09T05:02:55.128756+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:42.410792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":38,"num_read_kb":40,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234305,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303396,"kb_used_data":2012,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93441756,"statfs":{"total":95995035648,"available":95684358144,"internally_reserved":0,"allocated":2060288,"data_stored":1463213,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":1.5109999999999999,"15min":1.4710000000000001},"min":{"1min":0.67600000000000005,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":1.208,"5min":11.093999999999999,"15min":11.093999999999999},"last":1.208},{"interface":"front","average":{"1min":1.0489999999999999,"5min":1.6100000000000001,"15min":1.6910000000000001},"min":{"1min":0.59699999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.2689999999999999,"5min":6.3730000000000002,"15min":9.5090000000000003},"last":1.149}]},{"osd":1,"last update":"Mon Oct 9 05:07:04 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":3.2010000000000001,"15min":2.125},"min":{"1min":0.71299999999999997,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.0880000000000001,"5min":135.322,"15min":135.322},"last":1.0880000000000001},{"interface":"front","average":{"1min":1.05,"5min":3.2149999999999999,"15min":2.2749999999999999},"min":{"1min":0.69199999999999995,"5min":0.33500000000000002,"15min":0.30499999999999999},"max":{"1min":1.294,"5min":135.09,"15min":135.09},"last":0.90700000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":1.633,"15min":1.369},"min":{"1min":0.625,"5min":0.315,"15min":0.315},"max":{"1min":1.2290000000000001,"5min":11.391,"15min":11.391},"last":1.0189999999999999},{"interface":"front","average":{"1min":0.89800000000000002,"5min":1.512,"15min":1.274},"min":{"1min":0.59899999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.262,"5min":6.6820000000000004,"15min":6.6820000000000004},"last":1.0429999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":1.4850000000000001,"15min":1.2569999999999999},"min":{"1min":0.58699999999999997,"5min":0.317,"15min":0.317},"max":{"1min":1.2649999999999999,"5min":6.548,"15min":6.548},"last":0.95999999999999996},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.823,"15min":1.7050000000000001},"min":{"1min":0.63700000000000001,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.474,"5min":9.1129999999999995,"15min":9.1129999999999995},"last":1.3049999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:07:04 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":1.349,"15min":1.1839999999999999},"min":{"1min":0.61399999999999999,"5min":0.33700000000000002,"15min":0.28699999999999998},"max":{"1min":1.319,"5min":11.409000000000001,"15min":11.409000000000001},"last":1.411},{"interface":"front","average":{"1min":1.123,"5min":1.5940000000000001,"15min":1.375},"min":{"1min":0.66800000000000004,"5min":0.36899999999999999,"15min":0.316},"max":{"1min":1.4350000000000001,"5min":15.763,"15min":15.763},"last":1.2470000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:07:04 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.677,"15min":1.3560000000000001},"min":{"1min":0.73799999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.389,"5min":8.3569999999999993,"15min":11.244},"last":1.129},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.573,"15min":1.403},"min":{"1min":0.89700000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.3029999999999999,"5min":7.1020000000000003,"15min":9.6010000000000009},"last":1.2649999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:07:04 2023","interfaces":[{"interface":"back","average":{"1min":1.139,"5min":1.5069999999999999,"15min":1.2629999999999999},"min":{"1min":0.93500000000000005,"5min":0.376,"15min":0.318},"max":{"1min":1.4470000000000001,"5min":6.9630000000000001,"15min":8.484},"last":1.3440000000000001},{"interface":"front","average":{"1min":1.2310000000000001,"5min":1.6739999999999999,"15min":1.355},"min":{"1min":0.82799999999999996,"5min":0.35499999999999998,"15min":0.35399999999999998},"max":{"1min":1.5029999999999999,"5min":6.468,"15min":6.468},"last":1.3839999999999999}]}]},{"osd":6,"up_from":46,"seq":197568495946,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299292,"kb_used_data":2004,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93445860,"statfs":{"total":95995035648,"available":95688560640,"internally_reserved":0,"allocated":2052096,"data_stored":1463390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.089,"5min":1.8100000000000001,"15min":1.5309999999999999},"min":{"1min":0.61199999999999999,"5min":0.24399999999999999,"15min":0.214},"max":{"1min":1.4119999999999999,"5min":11.449,"15min":19.18},"last":1.177},{"interface":"front","average":{"1min":1.1200000000000001,"5min":1.9450000000000001,"15min":1.6559999999999999},"min":{"1min":0.752,"5min":0.30399999999999999,"15min":0.27200000000000002},"max":{"1min":1.417,"5min":11.491,"15min":19.358000000000001},"last":1.27}]},{"osd":1,"last update":"Mon Oct 9 05:07:01 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":1.71,"15min":1.6919999999999999},"min":{"1min":0.66600000000000004,"5min":0.28000000000000003,"15min":0.25800000000000001},"max":{"1min":1.21,"5min":10.118,"15min":10.118},"last":0.83199999999999996},{"interface":"front","average":{"1min":0.94199999999999995,"5min":1.9339999999999999,"15min":1.4379999999999999},"min":{"1min":0.61199999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.3580000000000001,"5min":9.6630000000000003,"15min":12.417},"last":0.752}]},{"osd":2,"last update":"Mon Oct 9 05:07:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0800000000000001,"5min":1.857,"15min":1.5409999999999999},"min":{"1min":0.74199999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.4750000000000001,"5min":8.9730000000000008,"15min":8.9730000000000008},"last":0.97299999999999998},{"interface":"front","average":{"1min":0.88200000000000001,"5min":1.5029999999999999,"15min":1.365},"min":{"1min":0.58599999999999997,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.091,"5min":8.3810000000000002,"15min":8.3810000000000002},"last":1.1319999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":2.105,"15min":1.657},"min":{"1min":0.66400000000000003,"5min":0.315,"15min":0.315},"max":{"1min":1.1899999999999999,"5min":10.201000000000001,"15min":10.201000000000001},"last":1.0389999999999999},{"interface":"front","average":{"1min":1.121,"5min":1.841,"15min":1.4790000000000001},"min":{"1min":0.81399999999999995,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.411,"5min":9.7720000000000002,"15min":9.7720000000000002},"last":1.0700000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":1.6279999999999999,"15min":1.153},"min":{"1min":0.70899999999999996,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.325,"5min":16.119,"15min":16.119},"last":0.95199999999999996},{"interface":"front","average":{"1min":1.1419999999999999,"5min":1.5569999999999999,"15min":1.274},"min":{"1min":0.79500000000000004,"5min":0.32300000000000001,"15min":0.247},"max":{"1min":1.5329999999999999,"5min":11.272,"15min":11.272},"last":1.385}]},{"osd":5,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1100000000000001,"5min":1.746,"15min":1.319},"min":{"1min":0.94899999999999995,"5min":0.312,"15min":0.29299999999999998},"max":{"1min":1.444,"5min":7.7149999999999999,"15min":7.7149999999999999},"last":1.353},{"interface":"front","average":{"1min":1.2769999999999999,"5min":2.016,"15min":1.516},"min":{"1min":0.63800000000000001,"5min":0.41599999999999998,"15min":0.33100000000000002},"max":{"1min":1.5780000000000001,"5min":10.537000000000001,"15min":10.537000000000001},"last":1.45}]},{"osd":7,"last update":"Mon Oct 9 05:06:50 2023","interfaces":[{"interface":"back","average":{"1min":1.3120000000000001,"5min":1.53,"15min":1.329},"min":{"1min":0.40200000000000002,"5min":0.39700000000000002,"15min":0.30599999999999999},"max":{"1min":5.835,"5min":9.3780000000000001,"15min":9.3780000000000001},"last":1.236},{"interface":"front","average":{"1min":1.4450000000000001,"5min":1.6140000000000001,"15min":1.417},"min":{"1min":0.434,"5min":0.434,"15min":0.375},"max":{"1min":5.8550000000000004,"5min":9.4060000000000006,"15min":9.4060000000000006},"last":1.3149999999999999}]}]},{"osd":1,"up_from":96,"seq":412316860537,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299076,"kb_used_data":1596,"kb_used_omap":4,"kb_used_meta":297467,"kb_avail":93446076,"statfs":{"total":95995035648,"available":95688781824,"internally_reserved":0,"allocated":1634304,"data_stored":894480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":304607118},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:06:58 2023","interfaces":[{"interface":"back","average":{"1min":0.69299999999999995,"5min":2.016,"15min":1.923},"min":{"1min":0.47399999999999998,"5min":0.20899999999999999,"15min":0.19400000000000001},"max":{"1min":1.2230000000000001,"5min":22.382999999999999,"15min":22.382999999999999},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.70599999999999996,"5min":2.1829999999999998,"15min":2.4399999999999999},"min":{"1min":0.433,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.96899999999999997,"5min":10.159000000000001,"15min":10.268000000000001},"last":0.48399999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":1.7609999999999999,"15min":1.5820000000000001},"min":{"1min":0.40400000000000003,"5min":0.219,"15min":0.219},"max":{"1min":1.0029999999999999,"5min":10.115,"15min":10.115},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.78200000000000003,"5min":1.831,"15min":1.5449999999999999},"min":{"1min":0.45800000000000002,"5min":0.252,"15min":0.252},"max":{"1min":1.151,"5min":10.428000000000001,"15min":11.917},"last":0.64400000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:07:20 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":2.1339999999999999,"15min":2.141},"min":{"1min":0.51800000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.0489999999999999,"5min":9.6039999999999992,"15min":9.6039999999999992},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.80700000000000005,"5min":1.6899999999999999,"15min":1.4750000000000001},"min":{"1min":0.52800000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.21,"5min":9.9039999999999999,"15min":9.9039999999999999},"last":0.88800000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:06:58 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":1.5880000000000001,"15min":1.6379999999999999},"min":{"1min":0.56599999999999995,"5min":0.35599999999999998,"15min":0.34200000000000003},"max":{"1min":1.2450000000000001,"5min":9.9749999999999996,"15min":9.9749999999999996},"last":1.1060000000000001},{"interface":"front","average":{"1min":0.94699999999999995,"5min":1.917,"15min":1.5329999999999999},"min":{"1min":0.61399999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.5009999999999999,"5min":22.350000000000001,"15min":22.350000000000001},"last":0.77800000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:06:58 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":2.2589999999999999,"15min":1.694},"min":{"1min":0.71799999999999997,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.347,"5min":12.454000000000001,"15min":12.454000000000001},"last":1.0820000000000001},{"interface":"front","average":{"1min":1.1120000000000001,"5min":2.1579999999999999,"15min":1.9450000000000001},"min":{"1min":0.68899999999999995,"5min":0.34999999999999998,"15min":0.316},"max":{"1min":1.911,"5min":10.035,"15min":10.035},"last":1.339}]},{"osd":6,"last update":"Mon Oct 9 05:06:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":2.4249999999999998,"15min":1.9339999999999999},"min":{"1min":0.81599999999999995,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.4750000000000001,"5min":10.409000000000001,"15min":10.409000000000001},"last":0.86799999999999999},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.923,"15min":1.772},"min":{"1min":0.59899999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.407,"5min":10.007,"15min":10.007},"last":1.0109999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:06:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1950000000000001,"5min":1.7470000000000001,"15min":1.833},"min":{"1min":0.67300000000000004,"5min":0.41899999999999998,"15min":0.38800000000000001},"max":{"1min":1.4319999999999999,"5min":10.061,"15min":10.061},"last":1.1839999999999999},{"interface":"front","average":{"1min":1.121,"5min":2.028,"15min":1.728},"min":{"1min":0.67900000000000005,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.4610000000000001,"5min":22.175999999999998,"15min":22.175999999999998},"last":0.92300000000000004}]}]},{"osd":0,"up_from":88,"seq":377957122178,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":334284,"kb_used_data":2180,"kb_used_omap":5,"kb_used_meta":332090,"kb_avail":93410868,"statfs":{"total":95995035648,"available":95652728832,"internally_reserved":0,"allocated":2232320,"data_stored":1483977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":340060794},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:06:56 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":1.7629999999999999,"15min":2.2989999999999999},"min":{"1min":0.32100000000000001,"5min":0.217,"15min":0.217},"max":{"1min":1.2809999999999999,"5min":11.561999999999999,"15min":13.084},"last":0.85399999999999998},{"interface":"front","average":{"1min":0.79000000000000004,"5min":1.8899999999999999,"15min":2.085},"min":{"1min":0.495,"5min":0.26700000000000002,"15min":0.218},"max":{"1min":1.1120000000000001,"5min":10.746,"15min":11.161},"last":0.73199999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:07:10 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":1.7210000000000001,"15min":1.9550000000000001},"min":{"1min":0.434,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.99099999999999999,"5min":8.8030000000000008,"15min":9.1300000000000008},"last":0.92200000000000004},{"interface":"front","average":{"1min":0.72699999999999998,"5min":1.3100000000000001,"15min":1.7110000000000001},"min":{"1min":0.34300000000000003,"5min":0.24299999999999999,"15min":0.23300000000000001},"max":{"1min":1.2450000000000001,"5min":8.3599999999999994,"15min":10.423},"last":0.68899999999999995}]},{"osd":3,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":1.5489999999999999,"15min":1.4810000000000001},"min":{"1min":0.35799999999999998,"5min":0.253,"15min":0.253},"max":{"1min":1.389,"5min":11.433999999999999,"15min":11.433999999999999},"last":0.80300000000000005},{"interface":"front","average":{"1min":0.79500000000000004,"5min":1.575,"15min":1.3089999999999999},"min":{"1min":0.39700000000000002,"5min":0.318,"15min":0.318},"max":{"1min":1.411,"5min":10.041,"15min":10.041},"last":1.028}]},{"osd":4,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.3919999999999999,"15min":1.4830000000000001},"min":{"1min":0.53300000000000003,"5min":0.33500000000000002,"15min":0.30099999999999999},"max":{"1min":1.3320000000000001,"5min":8.7449999999999992,"15min":8.7449999999999992},"last":1.1399999999999999},{"interface":"front","average":{"1min":0.98599999999999999,"5min":1.331,"15min":1.4450000000000001},"min":{"1min":0.52300000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.3540000000000001,"5min":5.7320000000000002,"15min":7.202},"last":1.1519999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":1.49,"15min":1.5649999999999999},"min":{"1min":0.55400000000000005,"5min":0.34100000000000003,"15min":0.30399999999999999},"max":{"1min":1.6890000000000001,"5min":6.8959999999999999,"15min":12.526999999999999},"last":1.101},{"interface":"front","average":{"1min":1.022,"5min":1.7709999999999999,"15min":1.6359999999999999},"min":{"1min":0.50800000000000001,"5min":0.35299999999999998,"15min":0.316},"max":{"1min":1.6020000000000001,"5min":10.148,"15min":10.148},"last":1.0860000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.615,"15min":1.587},"min":{"1min":0.55600000000000005,"5min":0.371,"15min":0.371},"max":{"1min":1.3200000000000001,"5min":8.8710000000000004,"15min":8.8710000000000004},"last":1.0569999999999999},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.4399999999999999,"15min":1.5},"min":{"1min":0.67800000000000005,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.458,"5min":6.9370000000000003,"15min":10.429},"last":1.006}]},{"osd":7,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.3660000000000001,"15min":1.5049999999999999},"min":{"1min":0.64400000000000002,"5min":0.375,"15min":0.36599999999999999},"max":{"1min":1.631,"5min":6.9569999999999999,"15min":10.314},"last":1.194},{"interface":"front","average":{"1min":1.075,"5min":1.71,"15min":1.879},"min":{"1min":0.64400000000000002,"5min":0.436,"15min":0.39500000000000002},"max":{"1min":1.4039999999999999,"5min":8.6110000000000007,"15min":10.738},"last":1.22}]}]},{"osd":2,"up_from":106,"seq":455266533457,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":252696,"kb_used_data":1552,"kb_used_omap":7,"kb_used_meta":251128,"kb_avail":93492456,"statfs":{"total":95995035648,"available":95736274944,"internally_reserved":0,"allocated":1589248,"data_stored":886182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":257155153},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:07 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":1.619,"15min":1.294},"min":{"1min":0.40500000000000003,"5min":0.215,"15min":0.215},"max":{"1min":0.89500000000000002,"5min":23.010999999999999,"15min":23.010999999999999},"last":0.96699999999999997},{"interface":"front","average":{"1min":0.67500000000000004,"5min":1.9870000000000001,"15min":1.323},"min":{"1min":0.28799999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":1.0649999999999999,"5min":20.501999999999999,"15min":20.501999999999999},"last":0.89000000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:07:07 2023","interfaces":[{"interface":"back","average":{"1min":0.74299999999999999,"5min":1.619,"15min":1.2649999999999999},"min":{"1min":0.373,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.2070000000000001,"5min":20.539999999999999,"15min":20.539999999999999},"last":0.997},{"interface":"front","average":{"1min":0.65300000000000002,"5min":2.048,"15min":1.502},"min":{"1min":0.33600000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.036,"5min":24.058,"15min":24.058},"last":0.72999999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:07:20 2023","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":1.827,"15min":2.1429999999999998},"min":{"1min":0.41799999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.2250000000000001,"5min":20.907,"15min":20.907},"last":0.80600000000000005},{"interface":"front","average":{"1min":0.67600000000000005,"5min":1.859,"15min":1.778},"min":{"1min":0.39500000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.046,"5min":23.574999999999999,"15min":23.574999999999999},"last":0.90700000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:07:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":1.766,"15min":1.458},"min":{"1min":0.69299999999999995,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.3160000000000001,"5min":20.712,"15min":20.712},"last":1.0800000000000001},{"interface":"front","average":{"1min":0.94299999999999995,"5min":1.98,"15min":1.5069999999999999},"min":{"1min":0.61299999999999999,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.2789999999999999,"5min":23.547000000000001,"15min":23.547000000000001},"last":1.05}]},{"osd":5,"last update":"Mon Oct 9 05:07:07 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":2.1970000000000001,"15min":1.5329999999999999},"min":{"1min":0.63600000000000001,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.2170000000000001,"5min":20.782,"15min":20.782},"last":0.86199999999999999},{"interface":"front","average":{"1min":1.0229999999999999,"5min":2.2160000000000002,"15min":1.958},"min":{"1min":0.65400000000000003,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.48,"5min":20.792000000000002,"15min":20.792000000000002},"last":1.2070000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:07:07 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":1.9370000000000001,"15min":1.593},"min":{"1min":0.59099999999999997,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.226,"5min":23.510999999999999,"15min":23.510999999999999},"last":1.131},{"interface":"front","average":{"1min":1.0549999999999999,"5min":1.8160000000000001,"15min":1.54},"min":{"1min":0.754,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":1.3,"5min":20.684999999999999,"15min":20.684999999999999},"last":1.2789999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:07:07 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":1.748,"15min":1.53},"min":{"1min":0.67300000000000004,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":1.3460000000000001,"5min":20.887,"15min":20.887},"last":1.113},{"interface":"front","average":{"1min":1.113,"5min":1.8979999999999999,"15min":1.4690000000000001},"min":{"1min":0.83899999999999997,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.3959999999999999,"5min":20.863,"15min":20.863},"last":1.3169999999999999}]}]},{"osd":3,"up_from":123,"seq":528280977476,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":272160,"kb_used_data":1496,"kb_used_omap":2,"kb_used_meta":270653,"kb_avail":93472992,"statfs":{"total":95995035648,"available":95716343808,"internally_reserved":0,"allocated":1531904,"data_stored":879533,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":277148725},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":1.6279999999999999,"15min":2.0550000000000002},"min":{"1min":0.434,"5min":0.187,"15min":0.187},"max":{"1min":0.86599999999999999,"5min":14.141,"15min":14.141},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.72199999999999998,"5min":1.972,"15min":2.605},"min":{"1min":0.44700000000000001,"5min":0.184,"15min":0.184},"max":{"1min":1.0920000000000001,"5min":14.045999999999999,"15min":14.045999999999999},"last":0.71199999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":1.7589999999999999,"15min":2.4119999999999999},"min":{"1min":0.36199999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.147,"5min":14.193,"15min":14.193},"last":0.93000000000000005},{"interface":"front","average":{"1min":0.67900000000000005,"5min":1.669,"15min":2.548},"min":{"1min":0.45000000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":1.0740000000000001,"5min":17.498999999999999,"15min":17.498999999999999},"last":0.60099999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":1.855,"15min":2.0779999999999998},"min":{"1min":0.39800000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.91600000000000004,"5min":9.6129999999999995,"15min":9.6129999999999995},"last":0.84599999999999997},{"interface":"front","average":{"1min":0.77700000000000002,"5min":1.873,"15min":1.867},"min":{"1min":0.49299999999999999,"5min":0.245,"15min":0.245},"max":{"1min":1.236,"5min":8.1600000000000001,"15min":8.1600000000000001},"last":0.82299999999999995}]},{"osd":4,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":1.5780000000000001,"15min":1.73},"min":{"1min":0.66900000000000004,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.45,"5min":5.8070000000000004,"15min":5.8070000000000004},"last":1.016},{"interface":"front","average":{"1min":0.94699999999999995,"5min":1.3500000000000001,"15min":1.294},"min":{"1min":0.58399999999999996,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.3180000000000001,"5min":9.4749999999999996,"15min":9.4749999999999996},"last":0.76600000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":1.613,"15min":1.335},"min":{"1min":0.71299999999999997,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.27,"5min":9.5079999999999991,"15min":9.5079999999999991},"last":1.044},{"interface":"front","average":{"1min":1.024,"5min":2.0049999999999999,"15min":2.0550000000000002},"min":{"1min":0.67100000000000004,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.413,"5min":8.9380000000000006,"15min":8.9380000000000006},"last":0.98399999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":1.7789999999999999,"15min":1.875},"min":{"1min":0.72399999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.4339999999999999,"5min":9.6590000000000007,"15min":9.6590000000000007},"last":1.0780000000000001},{"interface":"front","average":{"1min":1.05,"5min":1.9199999999999999,"15min":1.9810000000000001},"min":{"1min":0.73999999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.405,"5min":6.3289999999999997,"15min":6.3289999999999997},"last":1.107}]},{"osd":7,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.9970000000000001,"15min":2.1699999999999999},"min":{"1min":0.72799999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.468,"5min":8.3209999999999997,"15min":8.3209999999999997},"last":0.88500000000000001},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.663,"15min":1.5589999999999999},"min":{"1min":0.81299999999999994,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.369,"5min":9.5359999999999996,"15min":9.5359999999999996},"last":1.127}]}]},{"osd":4,"up_from":30,"seq":128849019233,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298724,"kb_used_data":1436,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446428,"statfs":{"total":95995035648,"available":95689142272,"internally_reserved":0,"allocated":1470464,"data_stored":873363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:11 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":1.389,"15min":4.6070000000000002},"min":{"1min":0.55200000000000005,"5min":0.26600000000000001,"15min":0.22700000000000001},"max":{"1min":1.389,"5min":10.109999999999999,"15min":131.30699999999999},"last":1.224},{"interface":"front","average":{"1min":0.97999999999999998,"5min":1.244,"15min":4.5469999999999997},"min":{"1min":0.57999999999999996,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.3999999999999999,"5min":6.3470000000000004,"15min":131.36600000000001},"last":1.1910000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:07:01 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":2.5800000000000001,"15min":1.8720000000000001},"min":{"1min":0.746,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.2350000000000001,"5min":59.408000000000001,"15min":59.408000000000001},"last":0.98899999999999999},{"interface":"front","average":{"1min":0.98399999999999999,"5min":2.363,"15min":2.0169999999999999},"min":{"1min":0.76000000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.2270000000000001,"5min":60.207000000000001,"15min":60.207000000000001},"last":0.95099999999999996}]},{"osd":2,"last update":"Mon Oct 9 05:07:11 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.742,"15min":1.1870000000000001},"min":{"1min":0.59499999999999997,"5min":0.307,"15min":0.307},"max":{"1min":1.2949999999999999,"5min":10.196,"15min":10.196},"last":1.0469999999999999},{"interface":"front","average":{"1min":1.0009999999999999,"5min":1.387,"15min":1.107},"min":{"1min":0.628,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":1.294,"5min":6.3810000000000002,"15min":6.3810000000000002},"last":1.252}]},{"osd":3,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":1.869,"15min":1.734},"min":{"1min":0.68999999999999995,"5min":0.309,"15min":0.309},"max":{"1min":1.343,"5min":10.555999999999999,"15min":10.555999999999999},"last":1.1040000000000001},{"interface":"front","average":{"1min":1.022,"5min":1.851,"15min":1.657},"min":{"1min":0.83099999999999996,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.262,"5min":21.036999999999999,"15min":21.036999999999999},"last":1.071}]},{"osd":5,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1100000000000001,"5min":1.5289999999999999,"15min":1.2869999999999999},"min":{"1min":0.64500000000000002,"5min":0.32500000000000001,"15min":0.31},"max":{"1min":1.3899999999999999,"5min":10.214,"15min":10.214},"last":1.1679999999999999},{"interface":"front","average":{"1min":1.1870000000000001,"5min":1.6200000000000001,"15min":1.3260000000000001},"min":{"1min":0.83699999999999997,"5min":0.34200000000000003,"15min":0.30399999999999999},"max":{"1min":1.629,"5min":6.8310000000000004,"15min":10.890000000000001},"last":0.69799999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:07:01 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":1.839,"15min":1.359},"min":{"1min":0.72099999999999997,"5min":0.32800000000000001,"15min":0.27800000000000002},"max":{"1min":1.5580000000000001,"5min":11.557,"15min":11.557},"last":0.80600000000000005},{"interface":"front","average":{"1min":1.194,"5min":1.6659999999999999,"15min":1.371},"min":{"1min":0.79800000000000004,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.425,"5min":10.689,"15min":10.689},"last":1.286}]},{"osd":7,"last update":"Mon Oct 9 05:07:01 2023","interfaces":[{"interface":"back","average":{"1min":1.1539999999999999,"5min":1.3009999999999999,"15min":1.252},"min":{"1min":0.75600000000000001,"5min":0.377,"15min":0.35999999999999999},"max":{"1min":1.4750000000000001,"5min":6.4820000000000002,"15min":8.8859999999999992},"last":0.90300000000000002},{"interface":"front","average":{"1min":1.151,"5min":1.6120000000000001,"15min":1.3859999999999999},"min":{"1min":0.92800000000000005,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.4370000000000001,"5min":6.1699999999999999,"15min":7.7880000000000003},"last":1.139}]}]},{"osd":5,"up_from":38,"seq":163208757589,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":603476,"kb_used_data":1420,"kb_used_omap":0,"kb_used_meta":602048,"kb_avail":93141676,"statfs":{"total":95995035648,"available":95377076224,"internally_reserved":0,"allocated":1454080,"data_stored":869107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":616497152},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:12 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":3.0049999999999999,"15min":1.8480000000000001},"min":{"1min":0.52900000000000003,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.1950000000000001,"5min":77.745999999999995,"15min":77.745999999999995},"last":0.86099999999999999},{"interface":"front","average":{"1min":1.083,"5min":3.093,"15min":1.958},"min":{"1min":0.67000000000000004,"5min":0.37,"15min":0.35299999999999998},"max":{"1min":1.5620000000000001,"5min":77.820999999999998,"15min":77.820999999999998},"last":0.92700000000000005}]},{"osd":1,"last update":"Mon Oct 9 05:07:03 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":2.218,"15min":1.7110000000000001},"min":{"1min":0.58799999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.5109999999999999,"5min":13.401999999999999,"15min":13.401999999999999},"last":0.68700000000000006},{"interface":"front","average":{"1min":1.016,"5min":1.9690000000000001,"15min":1.788},"min":{"1min":0.69399999999999995,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.2450000000000001,"5min":13.398,"15min":13.398},"last":1.125}]},{"osd":2,"last update":"Mon Oct 9 05:07:17 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":1.7230000000000001,"15min":1.111},"min":{"1min":0.57199999999999995,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.3180000000000001,"5min":13.436,"15min":13.436},"last":1.1859999999999999},{"interface":"front","average":{"1min":1.014,"5min":1.8899999999999999,"15min":1.252},"min":{"1min":0.65200000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.2929999999999999,"5min":9.4949999999999992,"15min":9.4949999999999992},"last":1.004}]},{"osd":3,"last update":"Mon Oct 9 05:07:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.869,"15min":1.966},"min":{"1min":0.745,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.349,"5min":9.1010000000000009,"15min":9.1010000000000009},"last":0.95699999999999996},{"interface":"front","average":{"1min":1.1200000000000001,"5min":2.121,"15min":1.7490000000000001},"min":{"1min":0.54900000000000004,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":3.1739999999999999,"5min":11.859999999999999,"15min":11.859999999999999},"last":0.89500000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:07:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":1.6060000000000001,"15min":1.28},"min":{"1min":0.68600000000000005,"5min":0.25,"15min":0.25},"max":{"1min":1.4399999999999999,"5min":14.781000000000001,"15min":14.781000000000001},"last":1.0529999999999999},{"interface":"front","average":{"1min":1.056,"5min":1.921,"15min":1.4239999999999999},"min":{"1min":0.69799999999999995,"5min":0.34599999999999997,"15min":0.313},"max":{"1min":1.2909999999999999,"5min":9.625,"15min":9.625},"last":1.081}]},{"osd":6,"last update":"Mon Oct 9 05:07:03 2023","interfaces":[{"interface":"back","average":{"1min":1.129,"5min":2.0310000000000001,"15min":1.544},"min":{"1min":0.60899999999999999,"5min":0.38300000000000001,"15min":0.33900000000000002},"max":{"1min":1.397,"5min":11.654999999999999,"15min":11.654999999999999},"last":0.753},{"interface":"front","average":{"1min":1.149,"5min":1.8100000000000001,"15min":1.4630000000000001},"min":{"1min":0.70299999999999996,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.4419999999999999,"5min":8.4670000000000005,"15min":10.423999999999999},"last":0.79000000000000004}]},{"osd":7,"last update":"Mon Oct 9 05:06:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7370000000000001,"5min":1.7649999999999999,"15min":1.512},"min":{"1min":0.84199999999999997,"5min":0.36899999999999999,"15min":0.34799999999999998},"max":{"1min":5.1879999999999997,"5min":8.7720000000000002,"15min":9.6639999999999997},"last":0.629},{"interface":"front","average":{"1min":1.5269999999999999,"5min":1.8029999999999999,"15min":1.4810000000000001},"min":{"1min":0.54300000000000004,"5min":0.34699999999999998,"15min":0.313},"max":{"1min":9.0500000000000007,"5min":10.122999999999999,"15min":12.843999999999999},"last":1.232}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:07:40.987 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:07:41.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:40 smithi133 bash[102858]: cluster 2023-10-09T05:07:39.698466+0000 mgr.y (mgr.25277) 567 : cluster 0 pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:41.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:40 smithi133 bash[102858]: audit 2023-10-09T05:07:40.134151+0000 mgr.y (mgr.25277) 568 : audit 0 from='client.34761 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:07:41.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:40 smithi133 bash[105589]: cluster 2023-10-09T05:07:39.698466+0000 mgr.y (mgr.25277) 567 : cluster 0 pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:41.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:40 smithi133 bash[105589]: audit 2023-10-09T05:07:40.134151+0000 mgr.y (mgr.25277) 568 : audit 0 from='client.34761 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:07:41.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:40 smithi194 bash[92666]: cluster 2023-10-09T05:07:39.698466+0000 mgr.y (mgr.25277) 567 : cluster 0 pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:41.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:40 smithi194 bash[92666]: audit 2023-10-09T05:07:40.134151+0000 mgr.y (mgr.25277) 568 : audit 0 from='client.34761 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:07:43.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:42 smithi133 bash[102858]: cluster 2023-10-09T05:07:41.699346+0000 mgr.y (mgr.25277) 569 : cluster 0 pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:43.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:42 smithi133 bash[105589]: cluster 2023-10-09T05:07:41.699346+0000 mgr.y (mgr.25277) 569 : cluster 0 pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:43.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:42 smithi194 bash[92666]: cluster 2023-10-09T05:07:41.699346+0000 mgr.y (mgr.25277) 569 : cluster 0 pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:44.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:44 smithi133 bash[105589]: audit 2023-10-09T05:07:43.437988+0000 mon.a (mon.0) 730 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:07:44.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:44 smithi133 bash[105589]: cluster 2023-10-09T05:07:43.700088+0000 mgr.y (mgr.25277) 570 : cluster 0 pgmap v524: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:44.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:44 smithi133 bash[102858]: audit 2023-10-09T05:07:43.437988+0000 mon.a (mon.0) 730 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:07:44.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:44 smithi133 bash[102858]: cluster 2023-10-09T05:07:43.700088+0000 mgr.y (mgr.25277) 570 : cluster 0 pgmap v524: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:44.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:07:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:07:44] "GET /metrics HTTP/1.1" 200 33985 "" "Prometheus/2.43.0" 2023-10-09T05:07:44.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:44 smithi194 bash[92666]: audit 2023-10-09T05:07:43.437988+0000 mon.a (mon.0) 730 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:07:44.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:44 smithi194 bash[92666]: cluster 2023-10-09T05:07:43.700088+0000 mgr.y (mgr.25277) 570 : cluster 0 pgmap v524: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:45.757 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:07:47.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:46 smithi133 bash[102858]: cluster 2023-10-09T05:07:45.700899+0000 mgr.y (mgr.25277) 571 : cluster 0 pgmap v525: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:47.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:46 smithi133 bash[105589]: cluster 2023-10-09T05:07:45.700899+0000 mgr.y (mgr.25277) 571 : cluster 0 pgmap v525: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:47.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:46 smithi194 bash[92666]: cluster 2023-10-09T05:07:45.700899+0000 mgr.y (mgr.25277) 571 : cluster 0 pgmap v525: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:49.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:48 smithi133 bash[102858]: cluster 2023-10-09T05:07:47.701424+0000 mgr.y (mgr.25277) 572 : cluster 0 pgmap v526: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:49.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:48 smithi133 bash[105589]: cluster 2023-10-09T05:07:47.701424+0000 mgr.y (mgr.25277) 572 : cluster 0 pgmap v526: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:49.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:48 smithi194 bash[92666]: cluster 2023-10-09T05:07:47.701424+0000 mgr.y (mgr.25277) 572 : cluster 0 pgmap v526: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:49.264 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:07:49.266 INFO:teuthology.orchestra.run.smithi133.stderr:dumped all 2023-10-09T05:07:50.189 INFO:teuthology.orchestra.run.smithi133.stdout:{"pg_ready":true,"pg_map":{"version":526,"stamp":"2023-10-09T05:07:47.701144+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":538,"num_read_kb":964,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":222,"ondisk_log_size":222,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2663104,"kb_used_data":13696,"kb_used_omap":20,"kb_used_meta":2649323,"kb_avail":747298112,"statfs":{"total":767960285184,"available":765233266688,"internally_reserved":0,"allocated":14024704,"data_stored":8813245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2712907406},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004581"},"pg_stats":[{"pgid":"3.1a","version":"69'2","reported_seq":118,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129055+0000","last_change":"2023-10-09T05:01:02.663550+0000","last_active":"2023-10-09T05:02:55.129055+0000","last_peered":"2023-10-09T05:02:55.129055+0000","last_clean":"2023-10-09T05:02:55.129055+0000","last_became_active":"2023-10-09T05:01:02.663202+0000","last_became_peered":"2023-10-09T05:01:02.663202+0000","last_unstale":"2023-10-09T05:02:55.129055+0000","last_undegraded":"2023-10-09T05:02:55.129055+0000","last_fullsized":"2023-10-09T05:02:55.129055+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:30.743395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":86,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.914359+0000","last_change":"2023-10-09T05:02:09.656135+0000","last_active":"2023-10-09T05:06:11.914359+0000","last_peered":"2023-10-09T05:06:11.914359+0000","last_clean":"2023-10-09T05:06:11.914359+0000","last_became_active":"2023-10-09T05:02:09.654594+0000","last_became_peered":"2023-10-09T05:02:09.654594+0000","last_unstale":"2023-10-09T05:06:11.914359+0000","last_undegraded":"2023-10-09T05:06:11.914359+0000","last_fullsized":"2023-10-09T05:06:11.914359+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:20.024818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":83,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.744393+0000","last_change":"2023-10-09T05:02:09.649701+0000","last_active":"2023-10-09T05:06:11.744393+0000","last_peered":"2023-10-09T05:06:11.744393+0000","last_clean":"2023-10-09T05:06:11.744393+0000","last_became_active":"2023-10-09T05:02:09.649370+0000","last_became_peered":"2023-10-09T05:02:09.649370+0000","last_unstale":"2023-10-09T05:06:11.744393+0000","last_undegraded":"2023-10-09T05:06:11.744393+0000","last_fullsized":"2023-10-09T05:06:11.744393+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:45.255492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":92,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.927998+0000","last_change":"2023-10-09T05:02:09.648283+0000","last_active":"2023-10-09T05:06:11.927998+0000","last_peered":"2023-10-09T05:06:11.927998+0000","last_clean":"2023-10-09T05:06:11.927998+0000","last_became_active":"2023-10-09T05:02:09.647778+0000","last_became_peered":"2023-10-09T05:02:09.647778+0000","last_unstale":"2023-10-09T05:06:11.927998+0000","last_undegraded":"2023-10-09T05:06:11.927998+0000","last_fullsized":"2023-10-09T05:06:11.927998+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:24.582392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":104,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.915863+0000","last_change":"2023-10-09T05:02:09.657157+0000","last_active":"2023-10-09T05:06:11.915863+0000","last_peered":"2023-10-09T05:06:11.915863+0000","last_clean":"2023-10-09T05:06:11.915863+0000","last_became_active":"2023-10-09T05:02:09.656354+0000","last_became_peered":"2023-10-09T05:02:09.656354+0000","last_unstale":"2023-10-09T05:06:11.915863+0000","last_undegraded":"2023-10-09T05:06:11.915863+0000","last_fullsized":"2023-10-09T05:06:11.915863+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:57.512482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.155272+0000","last_change":"2023-10-09T04:57:42.011855+0000","last_active":"2023-10-09T05:02:55.155272+0000","last_peered":"2023-10-09T05:02:55.155272+0000","last_clean":"2023-10-09T05:02:55.155272+0000","last_became_active":"2023-10-09T04:57:42.011233+0000","last_became_peered":"2023-10-09T04:57:42.011233+0000","last_unstale":"2023-10-09T05:02:55.155272+0000","last_undegraded":"2023-10-09T05:02:55.155272+0000","last_fullsized":"2023-10-09T05:02:55.155272+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:18.585929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":90,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.917521+0000","last_change":"2023-10-09T05:02:09.646765+0000","last_active":"2023-10-09T05:06:11.917521+0000","last_peered":"2023-10-09T05:06:11.917521+0000","last_clean":"2023-10-09T05:06:11.917521+0000","last_became_active":"2023-10-09T05:02:09.645141+0000","last_became_peered":"2023-10-09T05:02:09.645141+0000","last_unstale":"2023-10-09T05:06:11.917521+0000","last_undegraded":"2023-10-09T05:06:11.917521+0000","last_fullsized":"2023-10-09T05:06:11.917521+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:10.098578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":86,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.915378+0000","last_change":"2023-10-09T05:02:09.656872+0000","last_active":"2023-10-09T05:06:11.915378+0000","last_peered":"2023-10-09T05:06:11.915378+0000","last_clean":"2023-10-09T05:06:11.915378+0000","last_became_active":"2023-10-09T05:02:09.654713+0000","last_became_peered":"2023-10-09T05:02:09.654713+0000","last_unstale":"2023-10-09T05:06:11.915378+0000","last_undegraded":"2023-10-09T05:06:11.915378+0000","last_fullsized":"2023-10-09T05:06:11.915378+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:00:17.539155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":99,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.133911+0000","last_change":"2023-10-09T04:57:42.023522+0000","last_active":"2023-10-09T05:02:55.133911+0000","last_peered":"2023-10-09T05:02:55.133911+0000","last_clean":"2023-10-09T05:02:55.133911+0000","last_became_active":"2023-10-09T04:57:42.023010+0000","last_became_peered":"2023-10-09T04:57:42.023010+0000","last_unstale":"2023-10-09T05:02:55.133911+0000","last_undegraded":"2023-10-09T05:02:55.133911+0000","last_fullsized":"2023-10-09T05:02:55.133911+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:44.280677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":81,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.914646+0000","last_change":"2023-10-09T05:02:09.651167+0000","last_active":"2023-10-09T05:06:11.914646+0000","last_peered":"2023-10-09T05:06:11.914646+0000","last_clean":"2023-10-09T05:06:11.914646+0000","last_became_active":"2023-10-09T05:02:09.650419+0000","last_became_peered":"2023-10-09T05:02:09.650419+0000","last_unstale":"2023-10-09T05:06:11.914646+0000","last_undegraded":"2023-10-09T05:06:11.914646+0000","last_fullsized":"2023-10-09T05:06:11.914646+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:55:16.966938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":104,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.148854+0000","last_change":"2023-10-09T05:02:09.649700+0000","last_active":"2023-10-09T05:02:55.148854+0000","last_peered":"2023-10-09T05:02:55.148854+0000","last_clean":"2023-10-09T05:02:55.148854+0000","last_became_active":"2023-10-09T05:02:09.649317+0000","last_became_peered":"2023-10-09T05:02:09.649317+0000","last_unstale":"2023-10-09T05:02:55.148854+0000","last_undegraded":"2023-10-09T05:02:55.148854+0000","last_fullsized":"2023-10-09T05:02:55.148854+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:50.711767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":83,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.746974+0000","last_change":"2023-10-09T05:01:02.653598+0000","last_active":"2023-10-09T05:06:11.746974+0000","last_peered":"2023-10-09T05:06:11.746974+0000","last_clean":"2023-10-09T05:06:11.746974+0000","last_became_active":"2023-10-09T05:01:02.653351+0000","last_became_peered":"2023-10-09T05:01:02.653351+0000","last_unstale":"2023-10-09T05:06:11.746974+0000","last_undegraded":"2023-10-09T05:06:11.746974+0000","last_fullsized":"2023-10-09T05:06:11.746974+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:09.106185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'23","reported_seq":110,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129315+0000","last_change":"2023-10-09T04:44:52.781203+0000","last_active":"2023-10-09T05:02:55.129315+0000","last_peered":"2023-10-09T05:02:55.129315+0000","last_clean":"2023-10-09T05:02:55.129315+0000","last_became_active":"2023-10-09T04:44:52.780902+0000","last_became_peered":"2023-10-09T04:44:52.780902+0000","last_unstale":"2023-10-09T05:02:55.129315+0000","last_undegraded":"2023-10-09T05:02:55.129315+0000","last_fullsized":"2023-10-09T05:02:55.129315+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:56.332974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129435+0000","last_change":"2023-10-09T04:57:42.024993+0000","last_active":"2023-10-09T05:02:55.129435+0000","last_peered":"2023-10-09T05:02:55.129435+0000","last_clean":"2023-10-09T05:02:55.129435+0000","last_became_active":"2023-10-09T04:57:42.024227+0000","last_became_peered":"2023-10-09T04:57:42.024227+0000","last_unstale":"2023-10-09T05:02:55.129435+0000","last_undegraded":"2023-10-09T05:02:55.129435+0000","last_fullsized":"2023-10-09T05:02:55.129435+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:02.031798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":103,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.148581+0000","last_change":"2023-10-09T05:01:02.669186+0000","last_active":"2023-10-09T05:02:55.148581+0000","last_peered":"2023-10-09T05:02:55.148581+0000","last_clean":"2023-10-09T05:02:55.148581+0000","last_became_active":"2023-10-09T05:01:02.668739+0000","last_became_peered":"2023-10-09T05:01:02.668739+0000","last_unstale":"2023-10-09T05:02:55.148581+0000","last_undegraded":"2023-10-09T05:02:55.148581+0000","last_fullsized":"2023-10-09T05:02:55.148581+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:41.352624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":86,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.915825+0000","last_change":"2023-10-09T05:02:09.656245+0000","last_active":"2023-10-09T05:06:11.915825+0000","last_peered":"2023-10-09T05:06:11.915825+0000","last_clean":"2023-10-09T05:06:11.915825+0000","last_became_active":"2023-10-09T05:02:09.654672+0000","last_became_peered":"2023-10-09T05:02:09.654672+0000","last_unstale":"2023-10-09T05:06:11.915825+0000","last_undegraded":"2023-10-09T05:06:11.915825+0000","last_fullsized":"2023-10-09T05:06:11.915825+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:54.640349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":90,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.917907+0000","last_change":"2023-10-09T05:02:09.645017+0000","last_active":"2023-10-09T05:06:11.917907+0000","last_peered":"2023-10-09T05:06:11.917907+0000","last_clean":"2023-10-09T05:06:11.917907+0000","last_became_active":"2023-10-09T05:02:09.644705+0000","last_became_peered":"2023-10-09T05:02:09.644705+0000","last_unstale":"2023-10-09T05:06:11.917907+0000","last_undegraded":"2023-10-09T05:06:11.917907+0000","last_fullsized":"2023-10-09T05:06:11.917907+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:20.695961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.133929+0000","last_change":"2023-10-09T05:02:09.648362+0000","last_active":"2023-10-09T05:02:55.133929+0000","last_peered":"2023-10-09T05:02:55.133929+0000","last_clean":"2023-10-09T05:02:55.133929+0000","last_became_active":"2023-10-09T05:02:09.647127+0000","last_became_peered":"2023-10-09T05:02:09.647127+0000","last_unstale":"2023-10-09T05:02:55.133929+0000","last_undegraded":"2023-10-09T05:02:55.133929+0000","last_fullsized":"2023-10-09T05:02:55.133929+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:38:02.875613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":82,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.148137+0000","last_change":"2023-10-09T04:44:52.792557+0000","last_active":"2023-10-09T05:02:55.148137+0000","last_peered":"2023-10-09T05:02:55.148137+0000","last_clean":"2023-10-09T05:02:55.148137+0000","last_became_active":"2023-10-09T04:44:52.791890+0000","last_became_peered":"2023-10-09T04:44:52.791890+0000","last_unstale":"2023-10-09T05:02:55.148137+0000","last_undegraded":"2023-10-09T05:02:55.148137+0000","last_fullsized":"2023-10-09T05:02:55.148137+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:04:46.379852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":91,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.928065+0000","last_change":"2023-10-09T05:02:09.652158+0000","last_active":"2023-10-09T05:06:11.928065+0000","last_peered":"2023-10-09T05:06:11.928065+0000","last_clean":"2023-10-09T05:06:11.928065+0000","last_became_active":"2023-10-09T05:02:09.651811+0000","last_became_peered":"2023-10-09T05:02:09.651811+0000","last_unstale":"2023-10-09T05:06:11.928065+0000","last_undegraded":"2023-10-09T05:06:11.928065+0000","last_fullsized":"2023-10-09T05:06:11.928065+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:49:10.141237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":111,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.154425+0000","last_change":"2023-10-09T04:56:56.076177+0000","last_active":"2023-10-09T05:02:55.154425+0000","last_peered":"2023-10-09T05:02:55.154425+0000","last_clean":"2023-10-09T05:02:55.154425+0000","last_became_active":"2023-10-09T04:56:56.075564+0000","last_became_peered":"2023-10-09T04:56:56.075564+0000","last_unstale":"2023-10-09T05:02:55.154425+0000","last_undegraded":"2023-10-09T05:02:55.154425+0000","last_fullsized":"2023-10-09T05:02:55.154425+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:38.390793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":86,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.915451+0000","last_change":"2023-10-09T05:02:09.656286+0000","last_active":"2023-10-09T05:06:11.915451+0000","last_peered":"2023-10-09T05:06:11.915451+0000","last_clean":"2023-10-09T05:06:11.915451+0000","last_became_active":"2023-10-09T05:02:09.654429+0000","last_became_peered":"2023-10-09T05:02:09.654429+0000","last_unstale":"2023-10-09T05:06:11.915451+0000","last_undegraded":"2023-10-09T05:06:11.915451+0000","last_fullsized":"2023-10-09T05:06:11.915451+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:55:05.953948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":101,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.149120+0000","last_change":"2023-10-09T04:56:56.075307+0000","last_active":"2023-10-09T05:02:55.149120+0000","last_peered":"2023-10-09T05:02:55.149120+0000","last_clean":"2023-10-09T05:02:55.149120+0000","last_became_active":"2023-10-09T04:56:56.074831+0000","last_became_peered":"2023-10-09T04:56:56.074831+0000","last_unstale":"2023-10-09T05:02:55.149120+0000","last_undegraded":"2023-10-09T05:02:55.149120+0000","last_fullsized":"2023-10-09T05:02:55.149120+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:51:56.867759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":81,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.134283+0000","last_change":"2023-10-09T04:44:53.962437+0000","last_active":"2023-10-09T05:02:55.134283+0000","last_peered":"2023-10-09T05:02:55.134283+0000","last_clean":"2023-10-09T05:02:55.134283+0000","last_became_active":"2023-10-09T04:44:53.961810+0000","last_became_peered":"2023-10-09T04:44:53.961810+0000","last_unstale":"2023-10-09T05:02:55.134283+0000","last_undegraded":"2023-10-09T05:02:55.134283+0000","last_fullsized":"2023-10-09T05:02:55.134283+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:56.436933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":102,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.148994+0000","last_change":"2023-10-09T04:57:42.023336+0000","last_active":"2023-10-09T05:02:55.148994+0000","last_peered":"2023-10-09T05:02:55.148994+0000","last_clean":"2023-10-09T05:02:55.148994+0000","last_became_active":"2023-10-09T04:57:42.022861+0000","last_became_peered":"2023-10-09T04:57:42.022861+0000","last_unstale":"2023-10-09T05:02:55.148994+0000","last_undegraded":"2023-10-09T05:02:55.148994+0000","last_fullsized":"2023-10-09T05:02:55.148994+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:57.858993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":115,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.128925+0000","last_change":"2023-10-09T05:02:09.653562+0000","last_active":"2023-10-09T05:02:55.128925+0000","last_peered":"2023-10-09T05:02:55.128925+0000","last_clean":"2023-10-09T05:02:55.128925+0000","last_became_active":"2023-10-09T05:02:09.652830+0000","last_became_peered":"2023-10-09T05:02:09.652830+0000","last_unstale":"2023-10-09T05:02:55.128925+0000","last_undegraded":"2023-10-09T05:02:55.128925+0000","last_fullsized":"2023-10-09T05:02:55.128925+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:54.527011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":84,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.148487+0000","last_change":"2023-10-09T04:44:52.787469+0000","last_active":"2023-10-09T05:02:55.148487+0000","last_peered":"2023-10-09T05:02:55.148487+0000","last_clean":"2023-10-09T05:02:55.148487+0000","last_became_active":"2023-10-09T04:44:52.787148+0000","last_became_peered":"2023-10-09T04:44:52.787148+0000","last_unstale":"2023-10-09T05:02:55.148487+0000","last_undegraded":"2023-10-09T05:02:55.148487+0000","last_fullsized":"2023-10-09T05:02:55.148487+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:12.469065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":108,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.154612+0000","last_change":"2023-10-09T05:01:02.659380+0000","last_active":"2023-10-09T05:02:55.154612+0000","last_peered":"2023-10-09T05:02:55.154612+0000","last_clean":"2023-10-09T05:02:55.154612+0000","last_became_active":"2023-10-09T05:01:02.659064+0000","last_became_peered":"2023-10-09T05:01:02.659064+0000","last_unstale":"2023-10-09T05:02:55.154612+0000","last_undegraded":"2023-10-09T05:02:55.154612+0000","last_fullsized":"2023-10-09T05:02:55.154612+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:35.342503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":99,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.133928+0000","last_change":"2023-10-09T05:02:09.653932+0000","last_active":"2023-10-09T05:02:55.133928+0000","last_peered":"2023-10-09T05:02:55.133928+0000","last_clean":"2023-10-09T05:02:55.133928+0000","last_became_active":"2023-10-09T05:02:09.652923+0000","last_became_peered":"2023-10-09T05:02:09.652923+0000","last_unstale":"2023-10-09T05:02:55.133928+0000","last_undegraded":"2023-10-09T05:02:55.133928+0000","last_fullsized":"2023-10-09T05:02:55.133928+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:44:11.928237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":91,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:05:55.547270+0000","last_change":"2023-10-09T04:57:42.020009+0000","last_active":"2023-10-09T05:05:55.547270+0000","last_peered":"2023-10-09T05:05:55.547270+0000","last_clean":"2023-10-09T05:05:55.547270+0000","last_became_active":"2023-10-09T04:57:42.019678+0000","last_became_peered":"2023-10-09T04:57:42.019678+0000","last_unstale":"2023-10-09T05:05:55.547270+0000","last_undegraded":"2023-10-09T05:05:55.547270+0000","last_fullsized":"2023-10-09T05:05:55.547270+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:16.812128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'3","reported_seq":83,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.914244+0000","last_change":"2023-10-09T05:02:09.650579+0000","last_active":"2023-10-09T05:06:11.914244+0000","last_peered":"2023-10-09T05:06:11.914244+0000","last_clean":"2023-10-09T05:06:11.914244+0000","last_became_active":"2023-10-09T05:02:09.650160+0000","last_became_peered":"2023-10-09T05:02:09.650160+0000","last_unstale":"2023-10-09T05:06:11.914244+0000","last_undegraded":"2023-10-09T05:06:11.914244+0000","last_fullsized":"2023-10-09T05:06:11.914244+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:18.292670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129404+0000","last_change":"2023-10-09T05:02:09.652449+0000","last_active":"2023-10-09T05:02:55.129404+0000","last_peered":"2023-10-09T05:02:55.129404+0000","last_clean":"2023-10-09T05:02:55.129404+0000","last_became_active":"2023-10-09T05:02:09.651994+0000","last_became_peered":"2023-10-09T05:02:09.651994+0000","last_unstale":"2023-10-09T05:02:55.129404+0000","last_undegraded":"2023-10-09T05:02:55.129404+0000","last_fullsized":"2023-10-09T05:02:55.129404+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:14.480513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":99,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.154853+0000","last_change":"2023-10-09T04:57:42.016620+0000","last_active":"2023-10-09T05:02:55.154853+0000","last_peered":"2023-10-09T05:02:55.154853+0000","last_clean":"2023-10-09T05:02:55.154853+0000","last_became_active":"2023-10-09T04:57:42.016215+0000","last_became_peered":"2023-10-09T04:57:42.016215+0000","last_unstale":"2023-10-09T05:02:55.154853+0000","last_undegraded":"2023-10-09T05:02:55.154853+0000","last_fullsized":"2023-10-09T05:02:55.154853+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:21:24.361501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":115,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129567+0000","last_change":"2023-10-09T05:01:02.661970+0000","last_active":"2023-10-09T05:02:55.129567+0000","last_peered":"2023-10-09T05:02:55.129567+0000","last_clean":"2023-10-09T05:02:55.129567+0000","last_became_active":"2023-10-09T05:01:02.661646+0000","last_became_peered":"2023-10-09T05:01:02.661646+0000","last_unstale":"2023-10-09T05:02:55.129567+0000","last_undegraded":"2023-10-09T05:02:55.129567+0000","last_fullsized":"2023-10-09T05:02:55.129567+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:12.013587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":99,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.128906+0000","last_change":"2023-10-09T05:01:02.663309+0000","last_active":"2023-10-09T05:02:55.128906+0000","last_peered":"2023-10-09T05:02:55.128906+0000","last_clean":"2023-10-09T05:02:55.128906+0000","last_became_active":"2023-10-09T05:01:02.663038+0000","last_became_peered":"2023-10-09T05:01:02.663038+0000","last_unstale":"2023-10-09T05:02:55.128906+0000","last_undegraded":"2023-10-09T05:02:55.128906+0000","last_fullsized":"2023-10-09T05:02:55.128906+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:29:36.689304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129594+0000","last_change":"2023-10-09T04:57:42.028346+0000","last_active":"2023-10-09T05:02:55.129594+0000","last_peered":"2023-10-09T05:02:55.129594+0000","last_clean":"2023-10-09T05:02:55.129594+0000","last_became_active":"2023-10-09T04:57:42.027881+0000","last_became_peered":"2023-10-09T04:57:42.027881+0000","last_unstale":"2023-10-09T05:02:55.129594+0000","last_undegraded":"2023-10-09T05:02:55.129594+0000","last_fullsized":"2023-10-09T05:02:55.129594+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:00:43.197716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":87,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.916032+0000","last_change":"2023-10-09T05:02:09.656904+0000","last_active":"2023-10-09T05:06:11.916032+0000","last_peered":"2023-10-09T05:06:11.916032+0000","last_clean":"2023-10-09T05:06:11.916032+0000","last_became_active":"2023-10-09T05:02:09.654658+0000","last_became_peered":"2023-10-09T05:02:09.654658+0000","last_unstale":"2023-10-09T05:06:11.916032+0000","last_undegraded":"2023-10-09T05:06:11.916032+0000","last_fullsized":"2023-10-09T05:06:11.916032+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:51:26.666116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":83,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.745701+0000","last_change":"2023-10-09T05:02:09.652086+0000","last_active":"2023-10-09T05:06:11.745701+0000","last_peered":"2023-10-09T05:06:11.745701+0000","last_clean":"2023-10-09T05:06:11.745701+0000","last_became_active":"2023-10-09T05:02:09.651733+0000","last_became_peered":"2023-10-09T05:02:09.651733+0000","last_unstale":"2023-10-09T05:06:11.745701+0000","last_undegraded":"2023-10-09T05:06:11.745701+0000","last_fullsized":"2023-10-09T05:06:11.745701+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:32:59.894793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":112,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:06:01.201267+0000","last_change":"2023-10-09T04:57:42.019588+0000","last_active":"2023-10-09T05:06:01.201267+0000","last_peered":"2023-10-09T05:06:01.201267+0000","last_clean":"2023-10-09T05:06:01.201267+0000","last_became_active":"2023-10-09T04:57:42.019274+0000","last_became_peered":"2023-10-09T04:57:42.019274+0000","last_unstale":"2023-10-09T05:06:01.201267+0000","last_undegraded":"2023-10-09T05:06:01.201267+0000","last_fullsized":"2023-10-09T05:06:01.201267+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:21.969343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":116,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.134140+0000","last_change":"2023-10-09T05:02:09.649559+0000","last_active":"2023-10-09T05:02:55.134140+0000","last_peered":"2023-10-09T05:02:55.134140+0000","last_clean":"2023-10-09T05:02:55.134140+0000","last_became_active":"2023-10-09T05:02:09.649195+0000","last_became_peered":"2023-10-09T05:02:09.649195+0000","last_unstale":"2023-10-09T05:02:55.134140+0000","last_undegraded":"2023-10-09T05:02:55.134140+0000","last_fullsized":"2023-10-09T05:02:55.134140+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:56.864170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":68,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:06:01.201204+0000","last_change":"2023-10-09T04:57:42.022222+0000","last_active":"2023-10-09T05:06:01.201204+0000","last_peered":"2023-10-09T05:06:01.201204+0000","last_clean":"2023-10-09T05:06:01.201204+0000","last_became_active":"2023-10-09T04:57:42.021893+0000","last_became_peered":"2023-10-09T04:57:42.021893+0000","last_unstale":"2023-10-09T05:06:01.201204+0000","last_undegraded":"2023-10-09T05:06:01.201204+0000","last_fullsized":"2023-10-09T05:06:01.201204+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:39:20.734382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":103,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.133781+0000","last_change":"2023-10-09T04:57:42.010576+0000","last_active":"2023-10-09T05:02:55.133781+0000","last_peered":"2023-10-09T05:02:55.133781+0000","last_clean":"2023-10-09T05:02:55.133781+0000","last_became_active":"2023-10-09T04:57:42.010145+0000","last_became_peered":"2023-10-09T04:57:42.010145+0000","last_unstale":"2023-10-09T05:02:55.133781+0000","last_undegraded":"2023-10-09T05:02:55.133781+0000","last_fullsized":"2023-10-09T05:02:55.133781+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:18:17.196612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":99,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129297+0000","last_change":"2023-10-09T04:56:56.087320+0000","last_active":"2023-10-09T05:02:55.129297+0000","last_peered":"2023-10-09T05:02:55.129297+0000","last_clean":"2023-10-09T05:02:55.129297+0000","last_became_active":"2023-10-09T04:56:56.086552+0000","last_became_peered":"2023-10-09T04:56:56.086552+0000","last_unstale":"2023-10-09T05:02:55.129297+0000","last_undegraded":"2023-10-09T05:02:55.129297+0000","last_fullsized":"2023-10-09T05:02:55.129297+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:14.205288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":111,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.134027+0000","last_change":"2023-10-09T05:02:09.653655+0000","last_active":"2023-10-09T05:02:55.134027+0000","last_peered":"2023-10-09T05:02:55.134027+0000","last_clean":"2023-10-09T05:02:55.134027+0000","last_became_active":"2023-10-09T05:02:09.652682+0000","last_became_peered":"2023-10-09T05:02:09.652682+0000","last_unstale":"2023-10-09T05:02:55.134027+0000","last_undegraded":"2023-10-09T05:02:55.134027+0000","last_fullsized":"2023-10-09T05:02:55.134027+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:21:28.150022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":116,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.748439+0000","last_change":"2023-10-09T05:02:09.645835+0000","last_active":"2023-10-09T05:06:11.748439+0000","last_peered":"2023-10-09T05:06:11.748439+0000","last_clean":"2023-10-09T05:06:11.748439+0000","last_became_active":"2023-10-09T05:02:09.645491+0000","last_became_peered":"2023-10-09T05:02:09.645491+0000","last_unstale":"2023-10-09T05:06:11.748439+0000","last_undegraded":"2023-10-09T05:06:11.748439+0000","last_fullsized":"2023-10-09T05:06:11.748439+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:06:20.823201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":84,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.746192+0000","last_change":"2023-10-09T05:01:02.656091+0000","last_active":"2023-10-09T05:06:11.746192+0000","last_peered":"2023-10-09T05:06:11.746192+0000","last_clean":"2023-10-09T05:06:11.746192+0000","last_became_active":"2023-10-09T05:01:02.654755+0000","last_became_peered":"2023-10-09T05:01:02.654755+0000","last_unstale":"2023-10-09T05:06:11.746192+0000","last_undegraded":"2023-10-09T05:06:11.746192+0000","last_fullsized":"2023-10-09T05:06:11.746192+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:07.144598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":92,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.749112+0000","last_change":"2023-10-09T05:01:02.667904+0000","last_active":"2023-10-09T05:06:11.749112+0000","last_peered":"2023-10-09T05:06:11.749112+0000","last_clean":"2023-10-09T05:06:11.749112+0000","last_became_active":"2023-10-09T05:01:02.667574+0000","last_became_peered":"2023-10-09T05:01:02.667574+0000","last_unstale":"2023-10-09T05:06:11.749112+0000","last_undegraded":"2023-10-09T05:06:11.749112+0000","last_fullsized":"2023-10-09T05:06:11.749112+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:48:50.643600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":103,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.133545+0000","last_change":"2023-10-09T05:01:02.685038+0000","last_active":"2023-10-09T05:02:55.133545+0000","last_peered":"2023-10-09T05:02:55.133545+0000","last_clean":"2023-10-09T05:02:55.133545+0000","last_became_active":"2023-10-09T05:01:02.683622+0000","last_became_peered":"2023-10-09T05:01:02.683622+0000","last_unstale":"2023-10-09T05:02:55.133545+0000","last_undegraded":"2023-10-09T05:02:55.133545+0000","last_fullsized":"2023-10-09T05:02:55.133545+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:35.435592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":476,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.151016+0000","last_change":"2023-10-09T04:56:56.075762+0000","last_active":"2023-10-09T05:02:55.151016+0000","last_peered":"2023-10-09T05:02:55.151016+0000","last_clean":"2023-10-09T05:02:55.151016+0000","last_became_active":"2023-10-09T04:56:56.075263+0000","last_became_peered":"2023-10-09T04:56:56.075263+0000","last_unstale":"2023-10-09T05:02:55.151016+0000","last_undegraded":"2023-10-09T05:02:55.151016+0000","last_fullsized":"2023-10-09T05:02:55.151016+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_clean_scrub_stamp":"2023-10-09T04:36:44.261013+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:02.769164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":81,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.914252+0000","last_change":"2023-10-09T05:02:09.654237+0000","last_active":"2023-10-09T05:06:11.914252+0000","last_peered":"2023-10-09T05:06:11.914252+0000","last_clean":"2023-10-09T05:06:11.914252+0000","last_became_active":"2023-10-09T05:02:09.653909+0000","last_became_peered":"2023-10-09T05:02:09.653909+0000","last_unstale":"2023-10-09T05:06:11.914252+0000","last_undegraded":"2023-10-09T05:06:11.914252+0000","last_fullsized":"2023-10-09T05:06:11.914252+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:02.002366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":81,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.154930+0000","last_change":"2023-10-09T04:44:53.959705+0000","last_active":"2023-10-09T05:02:55.154930+0000","last_peered":"2023-10-09T05:02:55.154930+0000","last_clean":"2023-10-09T05:02:55.154930+0000","last_became_active":"2023-10-09T04:44:53.959092+0000","last_became_peered":"2023-10-09T04:44:53.959092+0000","last_unstale":"2023-10-09T05:02:55.154930+0000","last_undegraded":"2023-10-09T05:02:55.154930+0000","last_fullsized":"2023-10-09T05:02:55.154930+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:31:29.189897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":120,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.151119+0000","last_change":"2023-10-09T04:57:42.023353+0000","last_active":"2023-10-09T05:02:55.151119+0000","last_peered":"2023-10-09T05:02:55.151119+0000","last_clean":"2023-10-09T05:02:55.151119+0000","last_became_active":"2023-10-09T04:57:42.023007+0000","last_became_peered":"2023-10-09T04:57:42.023007+0000","last_unstale":"2023-10-09T05:02:55.151119+0000","last_undegraded":"2023-10-09T05:02:55.151119+0000","last_fullsized":"2023-10-09T05:02:55.151119+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:43.715250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":91,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.914248+0000","last_change":"2023-10-09T05:02:09.648781+0000","last_active":"2023-10-09T05:06:11.914248+0000","last_peered":"2023-10-09T05:06:11.914248+0000","last_clean":"2023-10-09T05:06:11.914248+0000","last_became_active":"2023-10-09T05:02:09.648078+0000","last_became_peered":"2023-10-09T05:02:09.648078+0000","last_unstale":"2023-10-09T05:06:11.914248+0000","last_undegraded":"2023-10-09T05:06:11.914248+0000","last_fullsized":"2023-10-09T05:06:11.914248+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:35.287091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":107,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.927800+0000","last_change":"2023-10-09T05:01:02.653869+0000","last_active":"2023-10-09T05:06:11.927800+0000","last_peered":"2023-10-09T05:06:11.927800+0000","last_clean":"2023-10-09T05:06:11.927800+0000","last_became_active":"2023-10-09T05:01:02.653493+0000","last_became_peered":"2023-10-09T05:01:02.653493+0000","last_unstale":"2023-10-09T05:06:11.927800+0000","last_undegraded":"2023-10-09T05:06:11.927800+0000","last_fullsized":"2023-10-09T05:06:11.927800+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:40:46.722386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":103,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.148680+0000","last_change":"2023-10-09T04:56:56.073473+0000","last_active":"2023-10-09T05:02:55.148680+0000","last_peered":"2023-10-09T05:02:55.148680+0000","last_clean":"2023-10-09T05:02:55.148680+0000","last_became_active":"2023-10-09T04:56:56.073146+0000","last_became_peered":"2023-10-09T04:56:56.073146+0000","last_unstale":"2023-10-09T05:02:55.148680+0000","last_undegraded":"2023-10-09T05:02:55.148680+0000","last_fullsized":"2023-10-09T05:02:55.148680+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:55:25.132822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":83,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:06:01.201293+0000","last_change":"2023-10-09T05:01:02.664643+0000","last_active":"2023-10-09T05:06:01.201293+0000","last_peered":"2023-10-09T05:06:01.201293+0000","last_clean":"2023-10-09T05:06:01.201293+0000","last_became_active":"2023-10-09T05:01:02.664288+0000","last_became_peered":"2023-10-09T05:01:02.664288+0000","last_unstale":"2023-10-09T05:06:01.201293+0000","last_undegraded":"2023-10-09T05:06:01.201293+0000","last_fullsized":"2023-10-09T05:06:01.201293+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:01.572876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":77,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.916855+0000","last_change":"2023-10-09T04:56:56.071763+0000","last_active":"2023-10-09T05:06:11.916855+0000","last_peered":"2023-10-09T05:06:11.916855+0000","last_clean":"2023-10-09T05:06:11.916855+0000","last_became_active":"2023-10-09T04:56:56.071429+0000","last_became_peered":"2023-10-09T04:56:56.071429+0000","last_unstale":"2023-10-09T05:06:11.916855+0000","last_undegraded":"2023-10-09T05:06:11.916855+0000","last_fullsized":"2023-10-09T05:06:11.916855+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:53:02.051460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":82,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.915549+0000","last_change":"2023-10-09T05:02:09.651149+0000","last_active":"2023-10-09T05:06:11.915549+0000","last_peered":"2023-10-09T05:06:11.915549+0000","last_clean":"2023-10-09T05:06:11.915549+0000","last_became_active":"2023-10-09T05:02:09.650151+0000","last_became_peered":"2023-10-09T05:02:09.650151+0000","last_unstale":"2023-10-09T05:06:11.915549+0000","last_undegraded":"2023-10-09T05:06:11.915549+0000","last_fullsized":"2023-10-09T05:06:11.915549+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:49.100209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":67,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.744947+0000","last_change":"2023-10-09T04:57:42.023683+0000","last_active":"2023-10-09T05:06:11.744947+0000","last_peered":"2023-10-09T05:06:11.744947+0000","last_clean":"2023-10-09T05:06:11.744947+0000","last_became_active":"2023-10-09T04:57:42.023371+0000","last_became_peered":"2023-10-09T04:57:42.023371+0000","last_unstale":"2023-10-09T05:06:11.744947+0000","last_undegraded":"2023-10-09T05:06:11.744947+0000","last_fullsized":"2023-10-09T05:06:11.744947+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:17:29.936222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":78,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.094920+0000","last_change":"2023-10-09T05:02:09.650750+0000","last_active":"2023-10-09T05:06:08.094920+0000","last_peered":"2023-10-09T05:06:08.094920+0000","last_clean":"2023-10-09T05:06:08.094920+0000","last_became_active":"2023-10-09T05:02:09.650351+0000","last_became_peered":"2023-10-09T05:02:09.650351+0000","last_unstale":"2023-10-09T05:06:08.094920+0000","last_undegraded":"2023-10-09T05:06:08.094920+0000","last_fullsized":"2023-10-09T05:06:08.094920+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:14.826132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":103,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.155202+0000","last_change":"2023-10-09T05:01:02.660775+0000","last_active":"2023-10-09T05:02:55.155202+0000","last_peered":"2023-10-09T05:02:55.155202+0000","last_clean":"2023-10-09T05:02:55.155202+0000","last_became_active":"2023-10-09T05:01:02.660420+0000","last_became_peered":"2023-10-09T05:01:02.660420+0000","last_unstale":"2023-10-09T05:02:55.155202+0000","last_undegraded":"2023-10-09T05:02:55.155202+0000","last_fullsized":"2023-10-09T05:02:55.155202+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:54.031869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":96,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.927896+0000","last_change":"2023-10-09T04:57:42.017497+0000","last_active":"2023-10-09T05:06:11.927896+0000","last_peered":"2023-10-09T05:06:11.927896+0000","last_clean":"2023-10-09T05:06:11.927896+0000","last_became_active":"2023-10-09T04:57:42.017165+0000","last_became_peered":"2023-10-09T04:57:42.017165+0000","last_unstale":"2023-10-09T05:06:11.927896+0000","last_undegraded":"2023-10-09T05:06:11.927896+0000","last_fullsized":"2023-10-09T05:06:11.927896+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:43.258960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":81,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.128729+0000","last_change":"2023-10-09T04:44:53.792412+0000","last_active":"2023-10-09T05:02:55.128729+0000","last_peered":"2023-10-09T05:02:55.128729+0000","last_clean":"2023-10-09T05:02:55.128729+0000","last_became_active":"2023-10-09T04:44:53.792009+0000","last_became_peered":"2023-10-09T04:44:53.792009+0000","last_unstale":"2023-10-09T05:02:55.128729+0000","last_undegraded":"2023-10-09T05:02:55.128729+0000","last_fullsized":"2023-10-09T05:02:55.128729+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:39.423339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'2","reported_seq":81,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.917224+0000","last_change":"2023-10-09T04:56:56.079768+0000","last_active":"2023-10-09T05:06:11.917224+0000","last_peered":"2023-10-09T05:06:11.917224+0000","last_clean":"2023-10-09T05:06:11.917224+0000","last_became_active":"2023-10-09T04:56:56.077721+0000","last_became_peered":"2023-10-09T04:56:56.077721+0000","last_unstale":"2023-10-09T05:06:11.917224+0000","last_undegraded":"2023-10-09T05:06:11.917224+0000","last_fullsized":"2023-10-09T05:06:11.917224+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:36.384357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":118,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:06:00.911094+0000","last_change":"2023-10-09T05:02:09.644679+0000","last_active":"2023-10-09T05:06:00.911094+0000","last_peered":"2023-10-09T05:06:00.911094+0000","last_clean":"2023-10-09T05:06:00.911094+0000","last_became_active":"2023-10-09T05:02:09.644335+0000","last_became_peered":"2023-10-09T05:02:09.644335+0000","last_unstale":"2023-10-09T05:06:00.911094+0000","last_undegraded":"2023-10-09T05:06:00.911094+0000","last_fullsized":"2023-10-09T05:06:00.911094+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:06.142246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":102,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.134135+0000","last_change":"2023-10-09T04:57:42.018276+0000","last_active":"2023-10-09T05:02:55.134135+0000","last_peered":"2023-10-09T05:02:55.134135+0000","last_clean":"2023-10-09T05:02:55.134135+0000","last_became_active":"2023-10-09T04:57:42.017975+0000","last_became_peered":"2023-10-09T04:57:42.017975+0000","last_unstale":"2023-10-09T05:02:55.134135+0000","last_undegraded":"2023-10-09T05:02:55.134135+0000","last_fullsized":"2023-10-09T05:02:55.134135+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:48.867813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":115,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129845+0000","last_change":"2023-10-09T05:02:09.653528+0000","last_active":"2023-10-09T05:02:55.129845+0000","last_peered":"2023-10-09T05:02:55.129845+0000","last_clean":"2023-10-09T05:02:55.129845+0000","last_became_active":"2023-10-09T05:02:09.653171+0000","last_became_peered":"2023-10-09T05:02:09.653171+0000","last_unstale":"2023-10-09T05:02:55.129845+0000","last_undegraded":"2023-10-09T05:02:55.129845+0000","last_fullsized":"2023-10-09T05:02:55.129845+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:15.978289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":85,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.134213+0000","last_change":"2023-10-09T04:44:52.793709+0000","last_active":"2023-10-09T05:02:55.134213+0000","last_peered":"2023-10-09T05:02:55.134213+0000","last_clean":"2023-10-09T05:02:55.134213+0000","last_became_active":"2023-10-09T04:44:52.792973+0000","last_became_peered":"2023-10-09T04:44:52.792973+0000","last_unstale":"2023-10-09T05:02:55.134213+0000","last_undegraded":"2023-10-09T05:02:55.134213+0000","last_fullsized":"2023-10-09T05:02:55.134213+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:12.091089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":115,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129621+0000","last_change":"2023-10-09T05:02:09.649748+0000","last_active":"2023-10-09T05:02:55.129621+0000","last_peered":"2023-10-09T05:02:55.129621+0000","last_clean":"2023-10-09T05:02:55.129621+0000","last_became_active":"2023-10-09T05:02:09.649444+0000","last_became_peered":"2023-10-09T05:02:09.649444+0000","last_unstale":"2023-10-09T05:02:55.129621+0000","last_undegraded":"2023-10-09T05:02:55.129621+0000","last_fullsized":"2023-10-09T05:02:55.129621+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:07.436820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'2","reported_seq":99,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.095102+0000","last_change":"2023-10-09T05:02:09.647382+0000","last_active":"2023-10-09T05:06:08.095102+0000","last_peered":"2023-10-09T05:06:08.095102+0000","last_clean":"2023-10-09T05:06:08.095102+0000","last_became_active":"2023-10-09T05:02:09.646803+0000","last_became_peered":"2023-10-09T05:02:09.646803+0000","last_unstale":"2023-10-09T05:06:08.095102+0000","last_undegraded":"2023-10-09T05:06:08.095102+0000","last_fullsized":"2023-10-09T05:06:08.095102+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:23:00.161736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":113,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.914782+0000","last_change":"2023-10-09T05:02:09.649674+0000","last_active":"2023-10-09T05:06:11.914782+0000","last_peered":"2023-10-09T05:06:11.914782+0000","last_clean":"2023-10-09T05:06:11.914782+0000","last_became_active":"2023-10-09T05:02:09.649041+0000","last_became_peered":"2023-10-09T05:02:09.649041+0000","last_unstale":"2023-10-09T05:06:11.914782+0000","last_undegraded":"2023-10-09T05:06:11.914782+0000","last_fullsized":"2023-10-09T05:06:11.914782+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:18.524071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'5","reported_seq":101,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.917631+0000","last_change":"2023-10-09T05:01:02.672596+0000","last_active":"2023-10-09T05:06:11.917631+0000","last_peered":"2023-10-09T05:06:11.917631+0000","last_clean":"2023-10-09T05:06:11.917631+0000","last_became_active":"2023-10-09T05:01:02.672128+0000","last_became_peered":"2023-10-09T05:01:02.672128+0000","last_unstale":"2023-10-09T05:06:11.917631+0000","last_undegraded":"2023-10-09T05:06:11.917631+0000","last_fullsized":"2023-10-09T05:06:11.917631+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:17:05.887502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.128756+0000","last_change":"2023-10-09T04:57:42.027187+0000","last_active":"2023-10-09T05:02:55.128756+0000","last_peered":"2023-10-09T05:02:55.128756+0000","last_clean":"2023-10-09T05:02:55.128756+0000","last_became_active":"2023-10-09T04:57:42.024247+0000","last_became_peered":"2023-10-09T04:57:42.024247+0000","last_unstale":"2023-10-09T05:02:55.128756+0000","last_undegraded":"2023-10-09T05:02:55.128756+0000","last_fullsized":"2023-10-09T05:02:55.128756+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:42.410792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":38,"num_read_kb":40,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234306,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303396,"kb_used_data":2012,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93441756,"statfs":{"total":95995035648,"available":95684358144,"internally_reserved":0,"allocated":2060288,"data_stored":1463213,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":1.5109999999999999,"15min":1.4710000000000001},"min":{"1min":0.67600000000000005,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":1.208,"5min":11.093999999999999,"15min":11.093999999999999},"last":1.0609999999999999},{"interface":"front","average":{"1min":1.0489999999999999,"5min":1.6100000000000001,"15min":1.6910000000000001},"min":{"1min":0.59699999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.2689999999999999,"5min":6.3730000000000002,"15min":9.5090000000000003},"last":0.58099999999999996}]},{"osd":1,"last update":"Mon Oct 9 05:07:04 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":3.2010000000000001,"15min":2.125},"min":{"1min":0.71299999999999997,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.0880000000000001,"5min":135.322,"15min":135.322},"last":0.90700000000000003},{"interface":"front","average":{"1min":1.05,"5min":3.2149999999999999,"15min":2.2749999999999999},"min":{"1min":0.69199999999999995,"5min":0.33500000000000002,"15min":0.30499999999999999},"max":{"1min":1.294,"5min":135.09,"15min":135.09},"last":1.3009999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":1.633,"15min":1.369},"min":{"1min":0.625,"5min":0.315,"15min":0.315},"max":{"1min":1.2290000000000001,"5min":11.391,"15min":11.391},"last":1.1000000000000001},{"interface":"front","average":{"1min":0.89800000000000002,"5min":1.512,"15min":1.274},"min":{"1min":0.59899999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.262,"5min":6.6820000000000004,"15min":6.6820000000000004},"last":1.02}]},{"osd":3,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":1.4850000000000001,"15min":1.2569999999999999},"min":{"1min":0.58699999999999997,"5min":0.317,"15min":0.317},"max":{"1min":1.2649999999999999,"5min":6.548,"15min":6.548},"last":1.171},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.823,"15min":1.7050000000000001},"min":{"1min":0.63700000000000001,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.474,"5min":9.1129999999999995,"15min":9.1129999999999995},"last":1.196}]},{"osd":4,"last update":"Mon Oct 9 05:07:04 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":1.349,"15min":1.1839999999999999},"min":{"1min":0.61399999999999999,"5min":0.33700000000000002,"15min":0.28699999999999998},"max":{"1min":1.319,"5min":11.409000000000001,"15min":11.409000000000001},"last":0.98299999999999998},{"interface":"front","average":{"1min":1.123,"5min":1.5940000000000001,"15min":1.375},"min":{"1min":0.66800000000000004,"5min":0.36899999999999999,"15min":0.316},"max":{"1min":1.4350000000000001,"5min":15.763,"15min":15.763},"last":1.3640000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:07:04 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.677,"15min":1.3560000000000001},"min":{"1min":0.73799999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.389,"5min":8.3569999999999993,"15min":11.244},"last":1.2669999999999999},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.573,"15min":1.403},"min":{"1min":0.89700000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.3029999999999999,"5min":7.1020000000000003,"15min":9.6010000000000009},"last":1.1439999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:07:04 2023","interfaces":[{"interface":"back","average":{"1min":1.139,"5min":1.5069999999999999,"15min":1.2629999999999999},"min":{"1min":0.93500000000000005,"5min":0.376,"15min":0.318},"max":{"1min":1.4470000000000001,"5min":6.9630000000000001,"15min":8.484},"last":1.234},{"interface":"front","average":{"1min":1.2310000000000001,"5min":1.6739999999999999,"15min":1.355},"min":{"1min":0.82799999999999996,"5min":0.35499999999999998,"15min":0.35399999999999998},"max":{"1min":1.5029999999999999,"5min":6.468,"15min":6.468},"last":1.419}]}]},{"osd":6,"up_from":46,"seq":197568495948,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299292,"kb_used_data":2004,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93445860,"statfs":{"total":95995035648,"available":95688560640,"internally_reserved":0,"allocated":2052096,"data_stored":1463390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.089,"5min":1.8100000000000001,"15min":1.5309999999999999},"min":{"1min":0.61199999999999999,"5min":0.24399999999999999,"15min":0.214},"max":{"1min":1.4119999999999999,"5min":11.449,"15min":19.18},"last":1.0469999999999999},{"interface":"front","average":{"1min":1.1200000000000001,"5min":1.9450000000000001,"15min":1.6559999999999999},"min":{"1min":0.752,"5min":0.30399999999999999,"15min":0.27200000000000002},"max":{"1min":1.417,"5min":11.491,"15min":19.358000000000001},"last":1.325}]},{"osd":1,"last update":"Mon Oct 9 05:07:01 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":1.71,"15min":1.6919999999999999},"min":{"1min":0.66600000000000004,"5min":0.28000000000000003,"15min":0.25800000000000001},"max":{"1min":1.21,"5min":10.118,"15min":10.118},"last":1.1240000000000001},{"interface":"front","average":{"1min":0.94199999999999995,"5min":1.9339999999999999,"15min":1.4379999999999999},"min":{"1min":0.61199999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.3580000000000001,"5min":9.6630000000000003,"15min":12.417},"last":0.83099999999999996}]},{"osd":2,"last update":"Mon Oct 9 05:07:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0800000000000001,"5min":1.857,"15min":1.5409999999999999},"min":{"1min":0.74199999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.4750000000000001,"5min":8.9730000000000008,"15min":8.9730000000000008},"last":0.92600000000000005},{"interface":"front","average":{"1min":0.88200000000000001,"5min":1.5029999999999999,"15min":1.365},"min":{"1min":0.58599999999999997,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.091,"5min":8.3810000000000002,"15min":8.3810000000000002},"last":0.98699999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":2.105,"15min":1.657},"min":{"1min":0.66400000000000003,"5min":0.315,"15min":0.315},"max":{"1min":1.1899999999999999,"5min":10.201000000000001,"15min":10.201000000000001},"last":1.0249999999999999},{"interface":"front","average":{"1min":1.121,"5min":1.841,"15min":1.4790000000000001},"min":{"1min":0.81399999999999995,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.411,"5min":9.7720000000000002,"15min":9.7720000000000002},"last":1.2769999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":1.6279999999999999,"15min":1.153},"min":{"1min":0.70899999999999996,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.325,"5min":16.119,"15min":16.119},"last":0.95999999999999996},{"interface":"front","average":{"1min":1.1419999999999999,"5min":1.5569999999999999,"15min":1.274},"min":{"1min":0.79500000000000004,"5min":0.32300000000000001,"15min":0.247},"max":{"1min":1.5329999999999999,"5min":11.272,"15min":11.272},"last":1.226}]},{"osd":5,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1100000000000001,"5min":1.746,"15min":1.319},"min":{"1min":0.94899999999999995,"5min":0.312,"15min":0.29299999999999998},"max":{"1min":1.444,"5min":7.7149999999999999,"15min":7.7149999999999999},"last":0.79900000000000004},{"interface":"front","average":{"1min":1.2769999999999999,"5min":2.016,"15min":1.516},"min":{"1min":0.63800000000000001,"5min":0.41599999999999998,"15min":0.33100000000000002},"max":{"1min":1.5780000000000001,"5min":10.537000000000001,"15min":10.537000000000001},"last":1.157}]},{"osd":7,"last update":"Mon Oct 9 05:06:50 2023","interfaces":[{"interface":"back","average":{"1min":1.3120000000000001,"5min":1.53,"15min":1.329},"min":{"1min":0.40200000000000002,"5min":0.39700000000000002,"15min":0.30599999999999999},"max":{"1min":5.835,"5min":9.3780000000000001,"15min":9.3780000000000001},"last":0.88800000000000001},{"interface":"front","average":{"1min":1.4450000000000001,"5min":1.6140000000000001,"15min":1.417},"min":{"1min":0.434,"5min":0.434,"15min":0.375},"max":{"1min":5.8550000000000004,"5min":9.4060000000000006,"15min":9.4060000000000006},"last":1.089}]}]},{"osd":1,"up_from":96,"seq":412316860539,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299076,"kb_used_data":1596,"kb_used_omap":4,"kb_used_meta":297467,"kb_avail":93446076,"statfs":{"total":95995035648,"available":95688781824,"internally_reserved":0,"allocated":1634304,"data_stored":894480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":304607118},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:06:58 2023","interfaces":[{"interface":"back","average":{"1min":0.69299999999999995,"5min":2.016,"15min":1.923},"min":{"1min":0.47399999999999998,"5min":0.20899999999999999,"15min":0.19400000000000001},"max":{"1min":1.2230000000000001,"5min":22.382999999999999,"15min":22.382999999999999},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.70599999999999996,"5min":2.1829999999999998,"15min":2.4399999999999999},"min":{"1min":0.433,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.96899999999999997,"5min":10.159000000000001,"15min":10.268000000000001},"last":0.79600000000000004}]},{"osd":2,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":1.7609999999999999,"15min":1.5820000000000001},"min":{"1min":0.40400000000000003,"5min":0.219,"15min":0.219},"max":{"1min":1.0029999999999999,"5min":10.115,"15min":10.115},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.78200000000000003,"5min":1.831,"15min":1.5449999999999999},"min":{"1min":0.45800000000000002,"5min":0.252,"15min":0.252},"max":{"1min":1.151,"5min":10.428000000000001,"15min":11.917},"last":0.84199999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:07:20 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":2.1339999999999999,"15min":2.141},"min":{"1min":0.51800000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.0489999999999999,"5min":9.6039999999999992,"15min":9.6039999999999992},"last":0.91900000000000004},{"interface":"front","average":{"1min":0.80700000000000005,"5min":1.6899999999999999,"15min":1.4750000000000001},"min":{"1min":0.52800000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.21,"5min":9.9039999999999999,"15min":9.9039999999999999},"last":0.89100000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:06:58 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":1.5880000000000001,"15min":1.6379999999999999},"min":{"1min":0.56599999999999995,"5min":0.35599999999999998,"15min":0.34200000000000003},"max":{"1min":1.2450000000000001,"5min":9.9749999999999996,"15min":9.9749999999999996},"last":1.1890000000000001},{"interface":"front","average":{"1min":0.94699999999999995,"5min":1.917,"15min":1.5329999999999999},"min":{"1min":0.61399999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.5009999999999999,"5min":22.350000000000001,"15min":22.350000000000001},"last":1.133}]},{"osd":5,"last update":"Mon Oct 9 05:06:58 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":2.2589999999999999,"15min":1.694},"min":{"1min":0.71799999999999997,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.347,"5min":12.454000000000001,"15min":12.454000000000001},"last":1.095},{"interface":"front","average":{"1min":1.1120000000000001,"5min":2.1579999999999999,"15min":1.9450000000000001},"min":{"1min":0.68899999999999995,"5min":0.34999999999999998,"15min":0.316},"max":{"1min":1.911,"5min":10.035,"15min":10.035},"last":1.3100000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:06:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":2.4249999999999998,"15min":1.9339999999999999},"min":{"1min":0.81599999999999995,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.4750000000000001,"5min":10.409000000000001,"15min":10.409000000000001},"last":1.1639999999999999},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.923,"15min":1.772},"min":{"1min":0.59899999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.407,"5min":10.007,"15min":10.007},"last":1.371}]},{"osd":7,"last update":"Mon Oct 9 05:06:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1950000000000001,"5min":1.7470000000000001,"15min":1.833},"min":{"1min":0.67300000000000004,"5min":0.41899999999999998,"15min":0.38800000000000001},"max":{"1min":1.4319999999999999,"5min":10.061,"15min":10.061},"last":1.4259999999999999},{"interface":"front","average":{"1min":1.121,"5min":2.028,"15min":1.728},"min":{"1min":0.67900000000000005,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.4610000000000001,"5min":22.175999999999998,"15min":22.175999999999998},"last":1.2370000000000001}]}]},{"osd":0,"up_from":88,"seq":377957122180,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":334284,"kb_used_data":2180,"kb_used_omap":5,"kb_used_meta":332090,"kb_avail":93410868,"statfs":{"total":95995035648,"available":95652728832,"internally_reserved":0,"allocated":2232320,"data_stored":1483977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":340060794},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:06:56 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":1.7629999999999999,"15min":2.2989999999999999},"min":{"1min":0.32100000000000001,"5min":0.217,"15min":0.217},"max":{"1min":1.2809999999999999,"5min":11.561999999999999,"15min":13.084},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.79000000000000004,"5min":1.8899999999999999,"15min":2.085},"min":{"1min":0.495,"5min":0.26700000000000002,"15min":0.218},"max":{"1min":1.1120000000000001,"5min":10.746,"15min":11.161},"last":0.81200000000000006}]},{"osd":2,"last update":"Mon Oct 9 05:07:10 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":1.7210000000000001,"15min":1.9550000000000001},"min":{"1min":0.434,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.99099999999999999,"5min":8.8030000000000008,"15min":9.1300000000000008},"last":1.21},{"interface":"front","average":{"1min":0.72699999999999998,"5min":1.3100000000000001,"15min":1.7110000000000001},"min":{"1min":0.34300000000000003,"5min":0.24299999999999999,"15min":0.23300000000000001},"max":{"1min":1.2450000000000001,"5min":8.3599999999999994,"15min":10.423},"last":0.875}]},{"osd":3,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":1.5489999999999999,"15min":1.4810000000000001},"min":{"1min":0.35799999999999998,"5min":0.253,"15min":0.253},"max":{"1min":1.389,"5min":11.433999999999999,"15min":11.433999999999999},"last":1.054},{"interface":"front","average":{"1min":0.79500000000000004,"5min":1.575,"15min":1.3089999999999999},"min":{"1min":0.39700000000000002,"5min":0.318,"15min":0.318},"max":{"1min":1.411,"5min":10.041,"15min":10.041},"last":1}]},{"osd":4,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.3919999999999999,"15min":1.4830000000000001},"min":{"1min":0.53300000000000003,"5min":0.33500000000000002,"15min":0.30099999999999999},"max":{"1min":1.3320000000000001,"5min":8.7449999999999992,"15min":8.7449999999999992},"last":1.2989999999999999},{"interface":"front","average":{"1min":0.98599999999999999,"5min":1.331,"15min":1.4450000000000001},"min":{"1min":0.52300000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.3540000000000001,"5min":5.7320000000000002,"15min":7.202},"last":1.099}]},{"osd":5,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":1.49,"15min":1.5649999999999999},"min":{"1min":0.55400000000000005,"5min":0.34100000000000003,"15min":0.30399999999999999},"max":{"1min":1.6890000000000001,"5min":6.8959999999999999,"15min":12.526999999999999},"last":0.96999999999999997},{"interface":"front","average":{"1min":1.022,"5min":1.7709999999999999,"15min":1.6359999999999999},"min":{"1min":0.50800000000000001,"5min":0.35299999999999998,"15min":0.316},"max":{"1min":1.6020000000000001,"5min":10.148,"15min":10.148},"last":1.446}]},{"osd":6,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.615,"15min":1.587},"min":{"1min":0.55600000000000005,"5min":0.371,"15min":0.371},"max":{"1min":1.3200000000000001,"5min":8.8710000000000004,"15min":8.8710000000000004},"last":1.248},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.4399999999999999,"15min":1.5},"min":{"1min":0.67800000000000005,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.458,"5min":6.9370000000000003,"15min":10.429},"last":1.153}]},{"osd":7,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.3660000000000001,"15min":1.5049999999999999},"min":{"1min":0.64400000000000002,"5min":0.375,"15min":0.36599999999999999},"max":{"1min":1.631,"5min":6.9569999999999999,"15min":10.314},"last":1.196},{"interface":"front","average":{"1min":1.075,"5min":1.71,"15min":1.879},"min":{"1min":0.64400000000000002,"5min":0.436,"15min":0.39500000000000002},"max":{"1min":1.4039999999999999,"5min":8.6110000000000007,"15min":10.738},"last":1.5089999999999999}]}]},{"osd":2,"up_from":106,"seq":455266533459,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":252696,"kb_used_data":1552,"kb_used_omap":7,"kb_used_meta":251128,"kb_avail":93492456,"statfs":{"total":95995035648,"available":95736274944,"internally_reserved":0,"allocated":1589248,"data_stored":886182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":257155153},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:07 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":1.619,"15min":1.294},"min":{"1min":0.40500000000000003,"5min":0.215,"15min":0.215},"max":{"1min":0.89500000000000002,"5min":23.010999999999999,"15min":23.010999999999999},"last":0.94499999999999995},{"interface":"front","average":{"1min":0.67500000000000004,"5min":1.9870000000000001,"15min":1.323},"min":{"1min":0.28799999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":1.0649999999999999,"5min":20.501999999999999,"15min":20.501999999999999},"last":1.046}]},{"osd":1,"last update":"Mon Oct 9 05:07:07 2023","interfaces":[{"interface":"back","average":{"1min":0.74299999999999999,"5min":1.619,"15min":1.2649999999999999},"min":{"1min":0.373,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.2070000000000001,"5min":20.539999999999999,"15min":20.539999999999999},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.65300000000000002,"5min":2.048,"15min":1.502},"min":{"1min":0.33600000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.036,"5min":24.058,"15min":24.058},"last":0.80200000000000005}]},{"osd":3,"last update":"Mon Oct 9 05:07:20 2023","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":1.827,"15min":2.1429999999999998},"min":{"1min":0.41799999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.2250000000000001,"5min":20.907,"15min":20.907},"last":0.85799999999999998},{"interface":"front","average":{"1min":0.67600000000000005,"5min":1.859,"15min":1.778},"min":{"1min":0.39500000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.046,"5min":23.574999999999999,"15min":23.574999999999999},"last":0.90800000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:07:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":1.766,"15min":1.458},"min":{"1min":0.69299999999999995,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.3160000000000001,"5min":20.712,"15min":20.712},"last":0.97799999999999998},{"interface":"front","average":{"1min":0.94299999999999995,"5min":1.98,"15min":1.5069999999999999},"min":{"1min":0.61299999999999999,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.2789999999999999,"5min":23.547000000000001,"15min":23.547000000000001},"last":1.01}]},{"osd":5,"last update":"Mon Oct 9 05:07:07 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":2.1970000000000001,"15min":1.5329999999999999},"min":{"1min":0.63600000000000001,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.2170000000000001,"5min":20.782,"15min":20.782},"last":1.236},{"interface":"front","average":{"1min":1.0229999999999999,"5min":2.2160000000000002,"15min":1.958},"min":{"1min":0.65400000000000003,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.48,"5min":20.792000000000002,"15min":20.792000000000002},"last":1.0920000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:07:07 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":1.9370000000000001,"15min":1.593},"min":{"1min":0.59099999999999997,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.226,"5min":23.510999999999999,"15min":23.510999999999999},"last":1.1060000000000001},{"interface":"front","average":{"1min":1.0549999999999999,"5min":1.8160000000000001,"15min":1.54},"min":{"1min":0.754,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":1.3,"5min":20.684999999999999,"15min":20.684999999999999},"last":1.165}]},{"osd":7,"last update":"Mon Oct 9 05:07:07 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":1.748,"15min":1.53},"min":{"1min":0.67300000000000004,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":1.3460000000000001,"5min":20.887,"15min":20.887},"last":1.2849999999999999},{"interface":"front","average":{"1min":1.113,"5min":1.8979999999999999,"15min":1.4690000000000001},"min":{"1min":0.83899999999999997,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.3959999999999999,"5min":20.863,"15min":20.863},"last":1.137}]}]},{"osd":3,"up_from":123,"seq":528280977477,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":272160,"kb_used_data":1496,"kb_used_omap":2,"kb_used_meta":270653,"kb_avail":93472992,"statfs":{"total":95995035648,"available":95716343808,"internally_reserved":0,"allocated":1531904,"data_stored":879533,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":277148725},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":1.6279999999999999,"15min":2.0550000000000002},"min":{"1min":0.434,"5min":0.187,"15min":0.187},"max":{"1min":0.86599999999999999,"5min":14.141,"15min":14.141},"last":0.81899999999999995},{"interface":"front","average":{"1min":0.72199999999999998,"5min":1.972,"15min":2.605},"min":{"1min":0.44700000000000001,"5min":0.184,"15min":0.184},"max":{"1min":1.0920000000000001,"5min":14.045999999999999,"15min":14.045999999999999},"last":0.874}]},{"osd":1,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":1.7589999999999999,"15min":2.4119999999999999},"min":{"1min":0.36199999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.147,"5min":14.193,"15min":14.193},"last":0.78300000000000003},{"interface":"front","average":{"1min":0.67900000000000005,"5min":1.669,"15min":2.548},"min":{"1min":0.45000000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":1.0740000000000001,"5min":17.498999999999999,"15min":17.498999999999999},"last":1.0429999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":1.855,"15min":2.0779999999999998},"min":{"1min":0.39800000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.91600000000000004,"5min":9.6129999999999995,"15min":9.6129999999999995},"last":1.0840000000000001},{"interface":"front","average":{"1min":0.77700000000000002,"5min":1.873,"15min":1.867},"min":{"1min":0.49299999999999999,"5min":0.245,"15min":0.245},"max":{"1min":1.236,"5min":8.1600000000000001,"15min":8.1600000000000001},"last":0.91700000000000004}]},{"osd":4,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":1.5780000000000001,"15min":1.73},"min":{"1min":0.66900000000000004,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.45,"5min":5.8070000000000004,"15min":5.8070000000000004},"last":1.454},{"interface":"front","average":{"1min":0.94699999999999995,"5min":1.3500000000000001,"15min":1.294},"min":{"1min":0.58399999999999996,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.3180000000000001,"5min":9.4749999999999996,"15min":9.4749999999999996},"last":1.3420000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":1.613,"15min":1.335},"min":{"1min":0.71299999999999997,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.27,"5min":9.5079999999999991,"15min":9.5079999999999991},"last":1.4099999999999999},{"interface":"front","average":{"1min":1.024,"5min":2.0049999999999999,"15min":2.0550000000000002},"min":{"1min":0.67100000000000004,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.413,"5min":8.9380000000000006,"15min":8.9380000000000006},"last":1.621}]},{"osd":6,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":1.7789999999999999,"15min":1.875},"min":{"1min":0.72399999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.4339999999999999,"5min":9.6590000000000007,"15min":9.6590000000000007},"last":1.482},{"interface":"front","average":{"1min":1.05,"5min":1.9199999999999999,"15min":1.9810000000000001},"min":{"1min":0.73999999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.405,"5min":6.3289999999999997,"15min":6.3289999999999997},"last":1.5940000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.9970000000000001,"15min":2.1699999999999999},"min":{"1min":0.72799999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.468,"5min":8.3209999999999997,"15min":8.3209999999999997},"last":1.383},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.663,"15min":1.5589999999999999},"min":{"1min":0.81299999999999994,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.369,"5min":9.5359999999999996,"15min":9.5359999999999996},"last":1.5680000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019234,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298724,"kb_used_data":1436,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446428,"statfs":{"total":95995035648,"available":95689142272,"internally_reserved":0,"allocated":1470464,"data_stored":873363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:11 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":1.389,"15min":4.6070000000000002},"min":{"1min":0.55200000000000005,"5min":0.26600000000000001,"15min":0.22700000000000001},"max":{"1min":1.389,"5min":10.109999999999999,"15min":131.30699999999999},"last":1.0960000000000001},{"interface":"front","average":{"1min":0.97999999999999998,"5min":1.244,"15min":4.5469999999999997},"min":{"1min":0.57999999999999996,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.3999999999999999,"5min":6.3470000000000004,"15min":131.36600000000001},"last":1.0680000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:07:01 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":2.5800000000000001,"15min":1.8720000000000001},"min":{"1min":0.746,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.2350000000000001,"5min":59.408000000000001,"15min":59.408000000000001},"last":0.96799999999999997},{"interface":"front","average":{"1min":0.98399999999999999,"5min":2.363,"15min":2.0169999999999999},"min":{"1min":0.76000000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.2270000000000001,"5min":60.207000000000001,"15min":60.207000000000001},"last":1.2}]},{"osd":2,"last update":"Mon Oct 9 05:07:11 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.742,"15min":1.1870000000000001},"min":{"1min":0.59499999999999997,"5min":0.307,"15min":0.307},"max":{"1min":1.2949999999999999,"5min":10.196,"15min":10.196},"last":0.93799999999999994},{"interface":"front","average":{"1min":1.0009999999999999,"5min":1.387,"15min":1.107},"min":{"1min":0.628,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":1.294,"5min":6.3810000000000002,"15min":6.3810000000000002},"last":0.98599999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":1.869,"15min":1.734},"min":{"1min":0.68999999999999995,"5min":0.309,"15min":0.309},"max":{"1min":1.343,"5min":10.555999999999999,"15min":10.555999999999999},"last":1.026},{"interface":"front","average":{"1min":1.022,"5min":1.851,"15min":1.657},"min":{"1min":0.83099999999999996,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.262,"5min":21.036999999999999,"15min":21.036999999999999},"last":0.877}]},{"osd":5,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1100000000000001,"5min":1.5289999999999999,"15min":1.2869999999999999},"min":{"1min":0.64500000000000002,"5min":0.32500000000000001,"15min":0.31},"max":{"1min":1.3899999999999999,"5min":10.214,"15min":10.214},"last":0.80400000000000005},{"interface":"front","average":{"1min":1.1870000000000001,"5min":1.6200000000000001,"15min":1.3260000000000001},"min":{"1min":0.83699999999999997,"5min":0.34200000000000003,"15min":0.30399999999999999},"max":{"1min":1.629,"5min":6.8310000000000004,"15min":10.890000000000001},"last":0.55100000000000005}]},{"osd":6,"last update":"Mon Oct 9 05:07:01 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":1.839,"15min":1.359},"min":{"1min":0.72099999999999997,"5min":0.32800000000000001,"15min":0.27800000000000002},"max":{"1min":1.5580000000000001,"5min":11.557,"15min":11.557},"last":0.60499999999999998},{"interface":"front","average":{"1min":1.194,"5min":1.6659999999999999,"15min":1.371},"min":{"1min":0.79800000000000004,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.425,"5min":10.689,"15min":10.689},"last":1.1479999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:07:01 2023","interfaces":[{"interface":"back","average":{"1min":1.1539999999999999,"5min":1.3009999999999999,"15min":1.252},"min":{"1min":0.75600000000000001,"5min":0.377,"15min":0.35999999999999999},"max":{"1min":1.4750000000000001,"5min":6.4820000000000002,"15min":8.8859999999999992},"last":0.68100000000000005},{"interface":"front","average":{"1min":1.151,"5min":1.6120000000000001,"15min":1.3859999999999999},"min":{"1min":0.92800000000000005,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.4370000000000001,"5min":6.1699999999999999,"15min":7.7880000000000003},"last":0.77100000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757590,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":603476,"kb_used_data":1420,"kb_used_omap":0,"kb_used_meta":602048,"kb_avail":93141676,"statfs":{"total":95995035648,"available":95377076224,"internally_reserved":0,"allocated":1454080,"data_stored":869107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":616497152},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:12 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":3.0049999999999999,"15min":1.8480000000000001},"min":{"1min":0.52900000000000003,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.1950000000000001,"5min":77.745999999999995,"15min":77.745999999999995},"last":0.63400000000000001},{"interface":"front","average":{"1min":1.083,"5min":3.093,"15min":1.958},"min":{"1min":0.67000000000000004,"5min":0.37,"15min":0.35299999999999998},"max":{"1min":1.5620000000000001,"5min":77.820999999999998,"15min":77.820999999999998},"last":0.96499999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:07:03 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":2.218,"15min":1.7110000000000001},"min":{"1min":0.58799999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.5109999999999999,"5min":13.401999999999999,"15min":13.401999999999999},"last":1.1120000000000001},{"interface":"front","average":{"1min":1.016,"5min":1.9690000000000001,"15min":1.788},"min":{"1min":0.69399999999999995,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.2450000000000001,"5min":13.398,"15min":13.398},"last":0.72699999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:07:17 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":1.7230000000000001,"15min":1.111},"min":{"1min":0.57199999999999995,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.3180000000000001,"5min":13.436,"15min":13.436},"last":0.83699999999999997},{"interface":"front","average":{"1min":1.014,"5min":1.8899999999999999,"15min":1.252},"min":{"1min":0.65200000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.2929999999999999,"5min":9.4949999999999992,"15min":9.4949999999999992},"last":1.1699999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:07:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.869,"15min":1.966},"min":{"1min":0.745,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.349,"5min":9.1010000000000009,"15min":9.1010000000000009},"last":0.92500000000000004},{"interface":"front","average":{"1min":1.1200000000000001,"5min":2.121,"15min":1.7490000000000001},"min":{"1min":0.54900000000000004,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":3.1739999999999999,"5min":11.859999999999999,"15min":11.859999999999999},"last":1.304}]},{"osd":4,"last update":"Mon Oct 9 05:07:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":1.6060000000000001,"15min":1.28},"min":{"1min":0.68600000000000005,"5min":0.25,"15min":0.25},"max":{"1min":1.4399999999999999,"5min":14.781000000000001,"15min":14.781000000000001},"last":1.0069999999999999},{"interface":"front","average":{"1min":1.056,"5min":1.921,"15min":1.4239999999999999},"min":{"1min":0.69799999999999995,"5min":0.34599999999999997,"15min":0.313},"max":{"1min":1.2909999999999999,"5min":9.625,"15min":9.625},"last":1.363}]},{"osd":6,"last update":"Mon Oct 9 05:07:03 2023","interfaces":[{"interface":"back","average":{"1min":1.129,"5min":2.0310000000000001,"15min":1.544},"min":{"1min":0.60899999999999999,"5min":0.38300000000000001,"15min":0.33900000000000002},"max":{"1min":1.397,"5min":11.654999999999999,"15min":11.654999999999999},"last":1.0269999999999999},{"interface":"front","average":{"1min":1.149,"5min":1.8100000000000001,"15min":1.4630000000000001},"min":{"1min":0.70299999999999996,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.4419999999999999,"5min":8.4670000000000005,"15min":10.423999999999999},"last":1.27}]},{"osd":7,"last update":"Mon Oct 9 05:06:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7370000000000001,"5min":1.7649999999999999,"15min":1.512},"min":{"1min":0.84199999999999997,"5min":0.36899999999999999,"15min":0.34799999999999998},"max":{"1min":5.1879999999999997,"5min":8.7720000000000002,"15min":9.6639999999999997},"last":1.331},{"interface":"front","average":{"1min":1.5269999999999999,"5min":1.8029999999999999,"15min":1.4810000000000001},"min":{"1min":0.54300000000000004,"5min":0.34699999999999998,"15min":0.313},"max":{"1min":9.0500000000000007,"5min":10.122999999999999,"15min":12.843999999999999},"last":1.0760000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:07:50.195 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T05:07:50.195 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T05:07:50.195 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T05:07:50.195 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T05:07:51.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:50 smithi133 bash[102858]: audit 2023-10-09T05:07:49.251823+0000 mgr.y (mgr.25277) 573 : audit 0 from='client.44720 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:07:51.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:50 smithi133 bash[102858]: cluster 2023-10-09T05:07:49.702311+0000 mgr.y (mgr.25277) 574 : cluster 0 pgmap v527: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:51.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:50 smithi133 bash[105589]: audit 2023-10-09T05:07:49.251823+0000 mgr.y (mgr.25277) 573 : audit 0 from='client.44720 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:07:51.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:50 smithi133 bash[105589]: cluster 2023-10-09T05:07:49.702311+0000 mgr.y (mgr.25277) 574 : cluster 0 pgmap v527: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:51.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:50 smithi194 bash[92666]: audit 2023-10-09T05:07:49.251823+0000 mgr.y (mgr.25277) 573 : audit 0 from='client.44720 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:07:51.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:50 smithi194 bash[92666]: cluster 2023-10-09T05:07:49.702311+0000 mgr.y (mgr.25277) 574 : cluster 0 pgmap v527: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:53.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:52 smithi133 bash[102858]: cluster 2023-10-09T05:07:51.703200+0000 mgr.y (mgr.25277) 575 : cluster 0 pgmap v528: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:53.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:52 smithi133 bash[105589]: cluster 2023-10-09T05:07:51.703200+0000 mgr.y (mgr.25277) 575 : cluster 0 pgmap v528: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:53.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:52 smithi194 bash[92666]: cluster 2023-10-09T05:07:51.703200+0000 mgr.y (mgr.25277) 575 : cluster 0 pgmap v528: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:54.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:54 smithi133 bash[102858]: cluster 2023-10-09T05:07:53.704056+0000 mgr.y (mgr.25277) 576 : cluster 0 pgmap v529: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:54.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:07:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:07:54] "GET /metrics HTTP/1.1" 200 33985 "" "Prometheus/2.43.0" 2023-10-09T05:07:54.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:54 smithi133 bash[105589]: cluster 2023-10-09T05:07:53.704056+0000 mgr.y (mgr.25277) 576 : cluster 0 pgmap v529: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:54.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:54 smithi194 bash[92666]: cluster 2023-10-09T05:07:53.704056+0000 mgr.y (mgr.25277) 576 : cluster 0 pgmap v529: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:54.975 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:07:57.017 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:56 smithi133 bash[105589]: cluster 2023-10-09T05:07:55.705046+0000 mgr.y (mgr.25277) 577 : cluster 0 pgmap v530: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:57.017 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:56 smithi133 bash[102858]: cluster 2023-10-09T05:07:55.705046+0000 mgr.y (mgr.25277) 577 : cluster 0 pgmap v530: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:57.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:56 smithi194 bash[92666]: cluster 2023-10-09T05:07:55.705046+0000 mgr.y (mgr.25277) 577 : cluster 0 pgmap v530: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:58.546 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:07:58.546 INFO:teuthology.orchestra.run.smithi133.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T05:07:59.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:58 smithi133 bash[102858]: cluster 2023-10-09T05:07:57.705844+0000 mgr.y (mgr.25277) 578 : cluster 0 pgmap v531: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:59.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:58 smithi133 bash[102858]: audit 2023-10-09T05:07:58.438127+0000 mon.a (mon.0) 731 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:07:59.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:07:58 smithi133 bash[102858]: audit 2023-10-09T05:07:58.545558+0000 mon.c (mon.2) 56 : audit 0 from='client.? 172.21.15.133:0/3639741298' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T05:07:59.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:58 smithi133 bash[105589]: cluster 2023-10-09T05:07:57.705844+0000 mgr.y (mgr.25277) 578 : cluster 0 pgmap v531: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:59.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:58 smithi133 bash[105589]: audit 2023-10-09T05:07:58.438127+0000 mon.a (mon.0) 731 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:07:59.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:07:58 smithi133 bash[105589]: audit 2023-10-09T05:07:58.545558+0000 mon.c (mon.2) 56 : audit 0 from='client.? 172.21.15.133:0/3639741298' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T05:07:59.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:58 smithi194 bash[92666]: cluster 2023-10-09T05:07:57.705844+0000 mgr.y (mgr.25277) 578 : cluster 0 pgmap v531: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:07:59.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:58 smithi194 bash[92666]: audit 2023-10-09T05:07:58.438127+0000 mon.a (mon.0) 731 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:07:59.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:07:58 smithi194 bash[92666]: audit 2023-10-09T05:07:58.545558+0000 mon.c (mon.2) 56 : audit 0 from='client.? 172.21.15.133:0/3639741298' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T05:07:59.416 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T05:07:59.417 INFO:teuthology.run_tasks:Running task print... 2023-10-09T05:07:59.426 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2023-10-09T05:07:59.426 INFO:teuthology.run_tasks:Running task parallel... 2023-10-09T05:07:59.447 INFO:teuthology.task.parallel:starting parallel... 2023-10-09T05:07:59.448 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T05:07:59.448 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2023-10-09T05:07:59.450 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-10-09T05:07:59.450 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'reef'}, 'ceph': {'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}}, 'kernel': {'hwe': True}, 'workunit': {'branch': 'reef', 'sha1': 'a340fb5463aa3715e93e7472d5f60dfdcf184ab3'}} 2023-10-09T05:07:59.450 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-10-09T05:07:59.450 INFO:tasks.thrashosds:Beginning thrashosds... 2023-10-09T05:07:59.450 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T05:07:59.450 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:08:01.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:00 smithi133 bash[102858]: cluster 2023-10-09T05:07:59.706737+0000 mgr.y (mgr.25277) 579 : cluster 0 pgmap v532: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:01.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:00 smithi133 bash[105589]: cluster 2023-10-09T05:07:59.706737+0000 mgr.y (mgr.25277) 579 : cluster 0 pgmap v532: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:01.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:00 smithi194 bash[92666]: cluster 2023-10-09T05:07:59.706737+0000 mgr.y (mgr.25277) 579 : cluster 0 pgmap v532: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:03.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:02 smithi133 bash[102858]: cluster 2023-10-09T05:08:01.707518+0000 mgr.y (mgr.25277) 580 : cluster 0 pgmap v533: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:03.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:02 smithi133 bash[105589]: cluster 2023-10-09T05:08:01.707518+0000 mgr.y (mgr.25277) 580 : cluster 0 pgmap v533: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:03.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:02 smithi194 bash[92666]: cluster 2023-10-09T05:08:01.707518+0000 mgr.y (mgr.25277) 580 : cluster 0 pgmap v533: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:04.232 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:08:04.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:04 smithi133 bash[102858]: cluster 2023-10-09T05:08:03.708354+0000 mgr.y (mgr.25277) 581 : cluster 0 pgmap v534: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:04.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:08:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:08:04] "GET /metrics HTTP/1.1" 200 33985 "" "Prometheus/2.43.0" 2023-10-09T05:08:04.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:04 smithi133 bash[105589]: cluster 2023-10-09T05:08:03.708354+0000 mgr.y (mgr.25277) 581 : cluster 0 pgmap v534: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:04.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:04 smithi194 bash[92666]: cluster 2023-10-09T05:08:03.708354+0000 mgr.y (mgr.25277) 581 : cluster 0 pgmap v534: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:07.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:06 smithi133 bash[102858]: cluster 2023-10-09T05:08:05.709198+0000 mgr.y (mgr.25277) 582 : cluster 0 pgmap v535: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:07.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:06 smithi133 bash[105589]: cluster 2023-10-09T05:08:05.709198+0000 mgr.y (mgr.25277) 582 : cluster 0 pgmap v535: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:07.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:06 smithi194 bash[92666]: cluster 2023-10-09T05:08:05.709198+0000 mgr.y (mgr.25277) 582 : cluster 0 pgmap v535: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:07.688 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:08:07.692 INFO:teuthology.orchestra.run.smithi133.stderr:dumped all 2023-10-09T05:08:08.587 INFO:teuthology.orchestra.run.smithi133.stdout:{"pg_ready":true,"pg_map":{"version":535,"stamp":"2023-10-09T05:08:05.708609+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":538,"num_read_kb":964,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":222,"ondisk_log_size":222,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2663104,"kb_used_data":13696,"kb_used_omap":20,"kb_used_meta":2649323,"kb_avail":747298112,"statfs":{"total":767960285184,"available":765233266688,"internally_reserved":0,"allocated":14024704,"data_stored":8813245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2712907406},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005094"},"pg_stats":[{"pgid":"3.1a","version":"69'2","reported_seq":118,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129055+0000","last_change":"2023-10-09T05:01:02.663550+0000","last_active":"2023-10-09T05:02:55.129055+0000","last_peered":"2023-10-09T05:02:55.129055+0000","last_clean":"2023-10-09T05:02:55.129055+0000","last_became_active":"2023-10-09T05:01:02.663202+0000","last_became_peered":"2023-10-09T05:01:02.663202+0000","last_unstale":"2023-10-09T05:02:55.129055+0000","last_undegraded":"2023-10-09T05:02:55.129055+0000","last_fullsized":"2023-10-09T05:02:55.129055+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:30.743395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":86,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.914359+0000","last_change":"2023-10-09T05:02:09.656135+0000","last_active":"2023-10-09T05:06:11.914359+0000","last_peered":"2023-10-09T05:06:11.914359+0000","last_clean":"2023-10-09T05:06:11.914359+0000","last_became_active":"2023-10-09T05:02:09.654594+0000","last_became_peered":"2023-10-09T05:02:09.654594+0000","last_unstale":"2023-10-09T05:06:11.914359+0000","last_undegraded":"2023-10-09T05:06:11.914359+0000","last_fullsized":"2023-10-09T05:06:11.914359+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:20.024818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":83,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.744393+0000","last_change":"2023-10-09T05:02:09.649701+0000","last_active":"2023-10-09T05:06:11.744393+0000","last_peered":"2023-10-09T05:06:11.744393+0000","last_clean":"2023-10-09T05:06:11.744393+0000","last_became_active":"2023-10-09T05:02:09.649370+0000","last_became_peered":"2023-10-09T05:02:09.649370+0000","last_unstale":"2023-10-09T05:06:11.744393+0000","last_undegraded":"2023-10-09T05:06:11.744393+0000","last_fullsized":"2023-10-09T05:06:11.744393+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:45.255492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":92,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.927998+0000","last_change":"2023-10-09T05:02:09.648283+0000","last_active":"2023-10-09T05:06:11.927998+0000","last_peered":"2023-10-09T05:06:11.927998+0000","last_clean":"2023-10-09T05:06:11.927998+0000","last_became_active":"2023-10-09T05:02:09.647778+0000","last_became_peered":"2023-10-09T05:02:09.647778+0000","last_unstale":"2023-10-09T05:06:11.927998+0000","last_undegraded":"2023-10-09T05:06:11.927998+0000","last_fullsized":"2023-10-09T05:06:11.927998+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:24.582392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":104,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.915863+0000","last_change":"2023-10-09T05:02:09.657157+0000","last_active":"2023-10-09T05:06:11.915863+0000","last_peered":"2023-10-09T05:06:11.915863+0000","last_clean":"2023-10-09T05:06:11.915863+0000","last_became_active":"2023-10-09T05:02:09.656354+0000","last_became_peered":"2023-10-09T05:02:09.656354+0000","last_unstale":"2023-10-09T05:06:11.915863+0000","last_undegraded":"2023-10-09T05:06:11.915863+0000","last_fullsized":"2023-10-09T05:06:11.915863+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:57.512482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.155272+0000","last_change":"2023-10-09T04:57:42.011855+0000","last_active":"2023-10-09T05:02:55.155272+0000","last_peered":"2023-10-09T05:02:55.155272+0000","last_clean":"2023-10-09T05:02:55.155272+0000","last_became_active":"2023-10-09T04:57:42.011233+0000","last_became_peered":"2023-10-09T04:57:42.011233+0000","last_unstale":"2023-10-09T05:02:55.155272+0000","last_undegraded":"2023-10-09T05:02:55.155272+0000","last_fullsized":"2023-10-09T05:02:55.155272+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:18.585929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":90,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.917521+0000","last_change":"2023-10-09T05:02:09.646765+0000","last_active":"2023-10-09T05:06:11.917521+0000","last_peered":"2023-10-09T05:06:11.917521+0000","last_clean":"2023-10-09T05:06:11.917521+0000","last_became_active":"2023-10-09T05:02:09.645141+0000","last_became_peered":"2023-10-09T05:02:09.645141+0000","last_unstale":"2023-10-09T05:06:11.917521+0000","last_undegraded":"2023-10-09T05:06:11.917521+0000","last_fullsized":"2023-10-09T05:06:11.917521+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:10.098578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":86,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.915378+0000","last_change":"2023-10-09T05:02:09.656872+0000","last_active":"2023-10-09T05:06:11.915378+0000","last_peered":"2023-10-09T05:06:11.915378+0000","last_clean":"2023-10-09T05:06:11.915378+0000","last_became_active":"2023-10-09T05:02:09.654713+0000","last_became_peered":"2023-10-09T05:02:09.654713+0000","last_unstale":"2023-10-09T05:06:11.915378+0000","last_undegraded":"2023-10-09T05:06:11.915378+0000","last_fullsized":"2023-10-09T05:06:11.915378+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:00:17.539155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":99,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.133911+0000","last_change":"2023-10-09T04:57:42.023522+0000","last_active":"2023-10-09T05:02:55.133911+0000","last_peered":"2023-10-09T05:02:55.133911+0000","last_clean":"2023-10-09T05:02:55.133911+0000","last_became_active":"2023-10-09T04:57:42.023010+0000","last_became_peered":"2023-10-09T04:57:42.023010+0000","last_unstale":"2023-10-09T05:02:55.133911+0000","last_undegraded":"2023-10-09T05:02:55.133911+0000","last_fullsized":"2023-10-09T05:02:55.133911+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:44.280677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":81,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.914646+0000","last_change":"2023-10-09T05:02:09.651167+0000","last_active":"2023-10-09T05:06:11.914646+0000","last_peered":"2023-10-09T05:06:11.914646+0000","last_clean":"2023-10-09T05:06:11.914646+0000","last_became_active":"2023-10-09T05:02:09.650419+0000","last_became_peered":"2023-10-09T05:02:09.650419+0000","last_unstale":"2023-10-09T05:06:11.914646+0000","last_undegraded":"2023-10-09T05:06:11.914646+0000","last_fullsized":"2023-10-09T05:06:11.914646+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:55:16.966938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":104,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.148854+0000","last_change":"2023-10-09T05:02:09.649700+0000","last_active":"2023-10-09T05:02:55.148854+0000","last_peered":"2023-10-09T05:02:55.148854+0000","last_clean":"2023-10-09T05:02:55.148854+0000","last_became_active":"2023-10-09T05:02:09.649317+0000","last_became_peered":"2023-10-09T05:02:09.649317+0000","last_unstale":"2023-10-09T05:02:55.148854+0000","last_undegraded":"2023-10-09T05:02:55.148854+0000","last_fullsized":"2023-10-09T05:02:55.148854+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:50.711767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":83,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.746974+0000","last_change":"2023-10-09T05:01:02.653598+0000","last_active":"2023-10-09T05:06:11.746974+0000","last_peered":"2023-10-09T05:06:11.746974+0000","last_clean":"2023-10-09T05:06:11.746974+0000","last_became_active":"2023-10-09T05:01:02.653351+0000","last_became_peered":"2023-10-09T05:01:02.653351+0000","last_unstale":"2023-10-09T05:06:11.746974+0000","last_undegraded":"2023-10-09T05:06:11.746974+0000","last_fullsized":"2023-10-09T05:06:11.746974+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:09.106185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'23","reported_seq":110,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129315+0000","last_change":"2023-10-09T04:44:52.781203+0000","last_active":"2023-10-09T05:02:55.129315+0000","last_peered":"2023-10-09T05:02:55.129315+0000","last_clean":"2023-10-09T05:02:55.129315+0000","last_became_active":"2023-10-09T04:44:52.780902+0000","last_became_peered":"2023-10-09T04:44:52.780902+0000","last_unstale":"2023-10-09T05:02:55.129315+0000","last_undegraded":"2023-10-09T05:02:55.129315+0000","last_fullsized":"2023-10-09T05:02:55.129315+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:56.332974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129435+0000","last_change":"2023-10-09T04:57:42.024993+0000","last_active":"2023-10-09T05:02:55.129435+0000","last_peered":"2023-10-09T05:02:55.129435+0000","last_clean":"2023-10-09T05:02:55.129435+0000","last_became_active":"2023-10-09T04:57:42.024227+0000","last_became_peered":"2023-10-09T04:57:42.024227+0000","last_unstale":"2023-10-09T05:02:55.129435+0000","last_undegraded":"2023-10-09T05:02:55.129435+0000","last_fullsized":"2023-10-09T05:02:55.129435+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:02.031798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":103,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.148581+0000","last_change":"2023-10-09T05:01:02.669186+0000","last_active":"2023-10-09T05:02:55.148581+0000","last_peered":"2023-10-09T05:02:55.148581+0000","last_clean":"2023-10-09T05:02:55.148581+0000","last_became_active":"2023-10-09T05:01:02.668739+0000","last_became_peered":"2023-10-09T05:01:02.668739+0000","last_unstale":"2023-10-09T05:02:55.148581+0000","last_undegraded":"2023-10-09T05:02:55.148581+0000","last_fullsized":"2023-10-09T05:02:55.148581+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:41.352624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":86,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.915825+0000","last_change":"2023-10-09T05:02:09.656245+0000","last_active":"2023-10-09T05:06:11.915825+0000","last_peered":"2023-10-09T05:06:11.915825+0000","last_clean":"2023-10-09T05:06:11.915825+0000","last_became_active":"2023-10-09T05:02:09.654672+0000","last_became_peered":"2023-10-09T05:02:09.654672+0000","last_unstale":"2023-10-09T05:06:11.915825+0000","last_undegraded":"2023-10-09T05:06:11.915825+0000","last_fullsized":"2023-10-09T05:06:11.915825+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:54.640349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":90,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.917907+0000","last_change":"2023-10-09T05:02:09.645017+0000","last_active":"2023-10-09T05:06:11.917907+0000","last_peered":"2023-10-09T05:06:11.917907+0000","last_clean":"2023-10-09T05:06:11.917907+0000","last_became_active":"2023-10-09T05:02:09.644705+0000","last_became_peered":"2023-10-09T05:02:09.644705+0000","last_unstale":"2023-10-09T05:06:11.917907+0000","last_undegraded":"2023-10-09T05:06:11.917907+0000","last_fullsized":"2023-10-09T05:06:11.917907+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:20.695961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.133929+0000","last_change":"2023-10-09T05:02:09.648362+0000","last_active":"2023-10-09T05:02:55.133929+0000","last_peered":"2023-10-09T05:02:55.133929+0000","last_clean":"2023-10-09T05:02:55.133929+0000","last_became_active":"2023-10-09T05:02:09.647127+0000","last_became_peered":"2023-10-09T05:02:09.647127+0000","last_unstale":"2023-10-09T05:02:55.133929+0000","last_undegraded":"2023-10-09T05:02:55.133929+0000","last_fullsized":"2023-10-09T05:02:55.133929+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:38:02.875613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":82,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.148137+0000","last_change":"2023-10-09T04:44:52.792557+0000","last_active":"2023-10-09T05:02:55.148137+0000","last_peered":"2023-10-09T05:02:55.148137+0000","last_clean":"2023-10-09T05:02:55.148137+0000","last_became_active":"2023-10-09T04:44:52.791890+0000","last_became_peered":"2023-10-09T04:44:52.791890+0000","last_unstale":"2023-10-09T05:02:55.148137+0000","last_undegraded":"2023-10-09T05:02:55.148137+0000","last_fullsized":"2023-10-09T05:02:55.148137+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:04:46.379852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":91,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.928065+0000","last_change":"2023-10-09T05:02:09.652158+0000","last_active":"2023-10-09T05:06:11.928065+0000","last_peered":"2023-10-09T05:06:11.928065+0000","last_clean":"2023-10-09T05:06:11.928065+0000","last_became_active":"2023-10-09T05:02:09.651811+0000","last_became_peered":"2023-10-09T05:02:09.651811+0000","last_unstale":"2023-10-09T05:06:11.928065+0000","last_undegraded":"2023-10-09T05:06:11.928065+0000","last_fullsized":"2023-10-09T05:06:11.928065+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:49:10.141237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":111,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.154425+0000","last_change":"2023-10-09T04:56:56.076177+0000","last_active":"2023-10-09T05:02:55.154425+0000","last_peered":"2023-10-09T05:02:55.154425+0000","last_clean":"2023-10-09T05:02:55.154425+0000","last_became_active":"2023-10-09T04:56:56.075564+0000","last_became_peered":"2023-10-09T04:56:56.075564+0000","last_unstale":"2023-10-09T05:02:55.154425+0000","last_undegraded":"2023-10-09T05:02:55.154425+0000","last_fullsized":"2023-10-09T05:02:55.154425+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:38.390793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":86,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.915451+0000","last_change":"2023-10-09T05:02:09.656286+0000","last_active":"2023-10-09T05:06:11.915451+0000","last_peered":"2023-10-09T05:06:11.915451+0000","last_clean":"2023-10-09T05:06:11.915451+0000","last_became_active":"2023-10-09T05:02:09.654429+0000","last_became_peered":"2023-10-09T05:02:09.654429+0000","last_unstale":"2023-10-09T05:06:11.915451+0000","last_undegraded":"2023-10-09T05:06:11.915451+0000","last_fullsized":"2023-10-09T05:06:11.915451+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:55:05.953948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":101,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.149120+0000","last_change":"2023-10-09T04:56:56.075307+0000","last_active":"2023-10-09T05:02:55.149120+0000","last_peered":"2023-10-09T05:02:55.149120+0000","last_clean":"2023-10-09T05:02:55.149120+0000","last_became_active":"2023-10-09T04:56:56.074831+0000","last_became_peered":"2023-10-09T04:56:56.074831+0000","last_unstale":"2023-10-09T05:02:55.149120+0000","last_undegraded":"2023-10-09T05:02:55.149120+0000","last_fullsized":"2023-10-09T05:02:55.149120+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:51:56.867759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":81,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.134283+0000","last_change":"2023-10-09T04:44:53.962437+0000","last_active":"2023-10-09T05:02:55.134283+0000","last_peered":"2023-10-09T05:02:55.134283+0000","last_clean":"2023-10-09T05:02:55.134283+0000","last_became_active":"2023-10-09T04:44:53.961810+0000","last_became_peered":"2023-10-09T04:44:53.961810+0000","last_unstale":"2023-10-09T05:02:55.134283+0000","last_undegraded":"2023-10-09T05:02:55.134283+0000","last_fullsized":"2023-10-09T05:02:55.134283+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:56.436933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":102,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.148994+0000","last_change":"2023-10-09T04:57:42.023336+0000","last_active":"2023-10-09T05:02:55.148994+0000","last_peered":"2023-10-09T05:02:55.148994+0000","last_clean":"2023-10-09T05:02:55.148994+0000","last_became_active":"2023-10-09T04:57:42.022861+0000","last_became_peered":"2023-10-09T04:57:42.022861+0000","last_unstale":"2023-10-09T05:02:55.148994+0000","last_undegraded":"2023-10-09T05:02:55.148994+0000","last_fullsized":"2023-10-09T05:02:55.148994+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:57.858993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":115,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.128925+0000","last_change":"2023-10-09T05:02:09.653562+0000","last_active":"2023-10-09T05:02:55.128925+0000","last_peered":"2023-10-09T05:02:55.128925+0000","last_clean":"2023-10-09T05:02:55.128925+0000","last_became_active":"2023-10-09T05:02:09.652830+0000","last_became_peered":"2023-10-09T05:02:09.652830+0000","last_unstale":"2023-10-09T05:02:55.128925+0000","last_undegraded":"2023-10-09T05:02:55.128925+0000","last_fullsized":"2023-10-09T05:02:55.128925+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:54.527011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":84,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.148487+0000","last_change":"2023-10-09T04:44:52.787469+0000","last_active":"2023-10-09T05:02:55.148487+0000","last_peered":"2023-10-09T05:02:55.148487+0000","last_clean":"2023-10-09T05:02:55.148487+0000","last_became_active":"2023-10-09T04:44:52.787148+0000","last_became_peered":"2023-10-09T04:44:52.787148+0000","last_unstale":"2023-10-09T05:02:55.148487+0000","last_undegraded":"2023-10-09T05:02:55.148487+0000","last_fullsized":"2023-10-09T05:02:55.148487+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:12.469065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":108,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.154612+0000","last_change":"2023-10-09T05:01:02.659380+0000","last_active":"2023-10-09T05:02:55.154612+0000","last_peered":"2023-10-09T05:02:55.154612+0000","last_clean":"2023-10-09T05:02:55.154612+0000","last_became_active":"2023-10-09T05:01:02.659064+0000","last_became_peered":"2023-10-09T05:01:02.659064+0000","last_unstale":"2023-10-09T05:02:55.154612+0000","last_undegraded":"2023-10-09T05:02:55.154612+0000","last_fullsized":"2023-10-09T05:02:55.154612+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:35.342503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":99,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.133928+0000","last_change":"2023-10-09T05:02:09.653932+0000","last_active":"2023-10-09T05:02:55.133928+0000","last_peered":"2023-10-09T05:02:55.133928+0000","last_clean":"2023-10-09T05:02:55.133928+0000","last_became_active":"2023-10-09T05:02:09.652923+0000","last_became_peered":"2023-10-09T05:02:09.652923+0000","last_unstale":"2023-10-09T05:02:55.133928+0000","last_undegraded":"2023-10-09T05:02:55.133928+0000","last_fullsized":"2023-10-09T05:02:55.133928+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:44:11.928237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":91,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:05:55.547270+0000","last_change":"2023-10-09T04:57:42.020009+0000","last_active":"2023-10-09T05:05:55.547270+0000","last_peered":"2023-10-09T05:05:55.547270+0000","last_clean":"2023-10-09T05:05:55.547270+0000","last_became_active":"2023-10-09T04:57:42.019678+0000","last_became_peered":"2023-10-09T04:57:42.019678+0000","last_unstale":"2023-10-09T05:05:55.547270+0000","last_undegraded":"2023-10-09T05:05:55.547270+0000","last_fullsized":"2023-10-09T05:05:55.547270+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:16.812128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'3","reported_seq":83,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.914244+0000","last_change":"2023-10-09T05:02:09.650579+0000","last_active":"2023-10-09T05:06:11.914244+0000","last_peered":"2023-10-09T05:06:11.914244+0000","last_clean":"2023-10-09T05:06:11.914244+0000","last_became_active":"2023-10-09T05:02:09.650160+0000","last_became_peered":"2023-10-09T05:02:09.650160+0000","last_unstale":"2023-10-09T05:06:11.914244+0000","last_undegraded":"2023-10-09T05:06:11.914244+0000","last_fullsized":"2023-10-09T05:06:11.914244+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:18.292670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129404+0000","last_change":"2023-10-09T05:02:09.652449+0000","last_active":"2023-10-09T05:02:55.129404+0000","last_peered":"2023-10-09T05:02:55.129404+0000","last_clean":"2023-10-09T05:02:55.129404+0000","last_became_active":"2023-10-09T05:02:09.651994+0000","last_became_peered":"2023-10-09T05:02:09.651994+0000","last_unstale":"2023-10-09T05:02:55.129404+0000","last_undegraded":"2023-10-09T05:02:55.129404+0000","last_fullsized":"2023-10-09T05:02:55.129404+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:14.480513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":99,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.154853+0000","last_change":"2023-10-09T04:57:42.016620+0000","last_active":"2023-10-09T05:02:55.154853+0000","last_peered":"2023-10-09T05:02:55.154853+0000","last_clean":"2023-10-09T05:02:55.154853+0000","last_became_active":"2023-10-09T04:57:42.016215+0000","last_became_peered":"2023-10-09T04:57:42.016215+0000","last_unstale":"2023-10-09T05:02:55.154853+0000","last_undegraded":"2023-10-09T05:02:55.154853+0000","last_fullsized":"2023-10-09T05:02:55.154853+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:21:24.361501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":115,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129567+0000","last_change":"2023-10-09T05:01:02.661970+0000","last_active":"2023-10-09T05:02:55.129567+0000","last_peered":"2023-10-09T05:02:55.129567+0000","last_clean":"2023-10-09T05:02:55.129567+0000","last_became_active":"2023-10-09T05:01:02.661646+0000","last_became_peered":"2023-10-09T05:01:02.661646+0000","last_unstale":"2023-10-09T05:02:55.129567+0000","last_undegraded":"2023-10-09T05:02:55.129567+0000","last_fullsized":"2023-10-09T05:02:55.129567+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:12.013587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":99,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.128906+0000","last_change":"2023-10-09T05:01:02.663309+0000","last_active":"2023-10-09T05:02:55.128906+0000","last_peered":"2023-10-09T05:02:55.128906+0000","last_clean":"2023-10-09T05:02:55.128906+0000","last_became_active":"2023-10-09T05:01:02.663038+0000","last_became_peered":"2023-10-09T05:01:02.663038+0000","last_unstale":"2023-10-09T05:02:55.128906+0000","last_undegraded":"2023-10-09T05:02:55.128906+0000","last_fullsized":"2023-10-09T05:02:55.128906+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:29:36.689304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129594+0000","last_change":"2023-10-09T04:57:42.028346+0000","last_active":"2023-10-09T05:02:55.129594+0000","last_peered":"2023-10-09T05:02:55.129594+0000","last_clean":"2023-10-09T05:02:55.129594+0000","last_became_active":"2023-10-09T04:57:42.027881+0000","last_became_peered":"2023-10-09T04:57:42.027881+0000","last_unstale":"2023-10-09T05:02:55.129594+0000","last_undegraded":"2023-10-09T05:02:55.129594+0000","last_fullsized":"2023-10-09T05:02:55.129594+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:00:43.197716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":87,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.916032+0000","last_change":"2023-10-09T05:02:09.656904+0000","last_active":"2023-10-09T05:06:11.916032+0000","last_peered":"2023-10-09T05:06:11.916032+0000","last_clean":"2023-10-09T05:06:11.916032+0000","last_became_active":"2023-10-09T05:02:09.654658+0000","last_became_peered":"2023-10-09T05:02:09.654658+0000","last_unstale":"2023-10-09T05:06:11.916032+0000","last_undegraded":"2023-10-09T05:06:11.916032+0000","last_fullsized":"2023-10-09T05:06:11.916032+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:51:26.666116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":83,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.745701+0000","last_change":"2023-10-09T05:02:09.652086+0000","last_active":"2023-10-09T05:06:11.745701+0000","last_peered":"2023-10-09T05:06:11.745701+0000","last_clean":"2023-10-09T05:06:11.745701+0000","last_became_active":"2023-10-09T05:02:09.651733+0000","last_became_peered":"2023-10-09T05:02:09.651733+0000","last_unstale":"2023-10-09T05:06:11.745701+0000","last_undegraded":"2023-10-09T05:06:11.745701+0000","last_fullsized":"2023-10-09T05:06:11.745701+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:32:59.894793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":112,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:06:01.201267+0000","last_change":"2023-10-09T04:57:42.019588+0000","last_active":"2023-10-09T05:06:01.201267+0000","last_peered":"2023-10-09T05:06:01.201267+0000","last_clean":"2023-10-09T05:06:01.201267+0000","last_became_active":"2023-10-09T04:57:42.019274+0000","last_became_peered":"2023-10-09T04:57:42.019274+0000","last_unstale":"2023-10-09T05:06:01.201267+0000","last_undegraded":"2023-10-09T05:06:01.201267+0000","last_fullsized":"2023-10-09T05:06:01.201267+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:21.969343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":116,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.134140+0000","last_change":"2023-10-09T05:02:09.649559+0000","last_active":"2023-10-09T05:02:55.134140+0000","last_peered":"2023-10-09T05:02:55.134140+0000","last_clean":"2023-10-09T05:02:55.134140+0000","last_became_active":"2023-10-09T05:02:09.649195+0000","last_became_peered":"2023-10-09T05:02:09.649195+0000","last_unstale":"2023-10-09T05:02:55.134140+0000","last_undegraded":"2023-10-09T05:02:55.134140+0000","last_fullsized":"2023-10-09T05:02:55.134140+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:56.864170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":68,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:06:01.201204+0000","last_change":"2023-10-09T04:57:42.022222+0000","last_active":"2023-10-09T05:06:01.201204+0000","last_peered":"2023-10-09T05:06:01.201204+0000","last_clean":"2023-10-09T05:06:01.201204+0000","last_became_active":"2023-10-09T04:57:42.021893+0000","last_became_peered":"2023-10-09T04:57:42.021893+0000","last_unstale":"2023-10-09T05:06:01.201204+0000","last_undegraded":"2023-10-09T05:06:01.201204+0000","last_fullsized":"2023-10-09T05:06:01.201204+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:39:20.734382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":103,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.133781+0000","last_change":"2023-10-09T04:57:42.010576+0000","last_active":"2023-10-09T05:02:55.133781+0000","last_peered":"2023-10-09T05:02:55.133781+0000","last_clean":"2023-10-09T05:02:55.133781+0000","last_became_active":"2023-10-09T04:57:42.010145+0000","last_became_peered":"2023-10-09T04:57:42.010145+0000","last_unstale":"2023-10-09T05:02:55.133781+0000","last_undegraded":"2023-10-09T05:02:55.133781+0000","last_fullsized":"2023-10-09T05:02:55.133781+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:18:17.196612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":99,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129297+0000","last_change":"2023-10-09T04:56:56.087320+0000","last_active":"2023-10-09T05:02:55.129297+0000","last_peered":"2023-10-09T05:02:55.129297+0000","last_clean":"2023-10-09T05:02:55.129297+0000","last_became_active":"2023-10-09T04:56:56.086552+0000","last_became_peered":"2023-10-09T04:56:56.086552+0000","last_unstale":"2023-10-09T05:02:55.129297+0000","last_undegraded":"2023-10-09T05:02:55.129297+0000","last_fullsized":"2023-10-09T05:02:55.129297+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:14.205288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":111,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.134027+0000","last_change":"2023-10-09T05:02:09.653655+0000","last_active":"2023-10-09T05:02:55.134027+0000","last_peered":"2023-10-09T05:02:55.134027+0000","last_clean":"2023-10-09T05:02:55.134027+0000","last_became_active":"2023-10-09T05:02:09.652682+0000","last_became_peered":"2023-10-09T05:02:09.652682+0000","last_unstale":"2023-10-09T05:02:55.134027+0000","last_undegraded":"2023-10-09T05:02:55.134027+0000","last_fullsized":"2023-10-09T05:02:55.134027+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:21:28.150022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":116,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.748439+0000","last_change":"2023-10-09T05:02:09.645835+0000","last_active":"2023-10-09T05:06:11.748439+0000","last_peered":"2023-10-09T05:06:11.748439+0000","last_clean":"2023-10-09T05:06:11.748439+0000","last_became_active":"2023-10-09T05:02:09.645491+0000","last_became_peered":"2023-10-09T05:02:09.645491+0000","last_unstale":"2023-10-09T05:06:11.748439+0000","last_undegraded":"2023-10-09T05:06:11.748439+0000","last_fullsized":"2023-10-09T05:06:11.748439+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:06:20.823201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":84,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.746192+0000","last_change":"2023-10-09T05:01:02.656091+0000","last_active":"2023-10-09T05:06:11.746192+0000","last_peered":"2023-10-09T05:06:11.746192+0000","last_clean":"2023-10-09T05:06:11.746192+0000","last_became_active":"2023-10-09T05:01:02.654755+0000","last_became_peered":"2023-10-09T05:01:02.654755+0000","last_unstale":"2023-10-09T05:06:11.746192+0000","last_undegraded":"2023-10-09T05:06:11.746192+0000","last_fullsized":"2023-10-09T05:06:11.746192+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:07.144598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":92,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.749112+0000","last_change":"2023-10-09T05:01:02.667904+0000","last_active":"2023-10-09T05:06:11.749112+0000","last_peered":"2023-10-09T05:06:11.749112+0000","last_clean":"2023-10-09T05:06:11.749112+0000","last_became_active":"2023-10-09T05:01:02.667574+0000","last_became_peered":"2023-10-09T05:01:02.667574+0000","last_unstale":"2023-10-09T05:06:11.749112+0000","last_undegraded":"2023-10-09T05:06:11.749112+0000","last_fullsized":"2023-10-09T05:06:11.749112+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:48:50.643600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":103,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.133545+0000","last_change":"2023-10-09T05:01:02.685038+0000","last_active":"2023-10-09T05:02:55.133545+0000","last_peered":"2023-10-09T05:02:55.133545+0000","last_clean":"2023-10-09T05:02:55.133545+0000","last_became_active":"2023-10-09T05:01:02.683622+0000","last_became_peered":"2023-10-09T05:01:02.683622+0000","last_unstale":"2023-10-09T05:02:55.133545+0000","last_undegraded":"2023-10-09T05:02:55.133545+0000","last_fullsized":"2023-10-09T05:02:55.133545+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:35.435592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":476,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.151016+0000","last_change":"2023-10-09T04:56:56.075762+0000","last_active":"2023-10-09T05:02:55.151016+0000","last_peered":"2023-10-09T05:02:55.151016+0000","last_clean":"2023-10-09T05:02:55.151016+0000","last_became_active":"2023-10-09T04:56:56.075263+0000","last_became_peered":"2023-10-09T04:56:56.075263+0000","last_unstale":"2023-10-09T05:02:55.151016+0000","last_undegraded":"2023-10-09T05:02:55.151016+0000","last_fullsized":"2023-10-09T05:02:55.151016+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_clean_scrub_stamp":"2023-10-09T04:36:44.261013+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:02.769164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":81,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.914252+0000","last_change":"2023-10-09T05:02:09.654237+0000","last_active":"2023-10-09T05:06:11.914252+0000","last_peered":"2023-10-09T05:06:11.914252+0000","last_clean":"2023-10-09T05:06:11.914252+0000","last_became_active":"2023-10-09T05:02:09.653909+0000","last_became_peered":"2023-10-09T05:02:09.653909+0000","last_unstale":"2023-10-09T05:06:11.914252+0000","last_undegraded":"2023-10-09T05:06:11.914252+0000","last_fullsized":"2023-10-09T05:06:11.914252+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:02.002366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":81,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.154930+0000","last_change":"2023-10-09T04:44:53.959705+0000","last_active":"2023-10-09T05:02:55.154930+0000","last_peered":"2023-10-09T05:02:55.154930+0000","last_clean":"2023-10-09T05:02:55.154930+0000","last_became_active":"2023-10-09T04:44:53.959092+0000","last_became_peered":"2023-10-09T04:44:53.959092+0000","last_unstale":"2023-10-09T05:02:55.154930+0000","last_undegraded":"2023-10-09T05:02:55.154930+0000","last_fullsized":"2023-10-09T05:02:55.154930+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:31:29.189897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":120,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.151119+0000","last_change":"2023-10-09T04:57:42.023353+0000","last_active":"2023-10-09T05:02:55.151119+0000","last_peered":"2023-10-09T05:02:55.151119+0000","last_clean":"2023-10-09T05:02:55.151119+0000","last_became_active":"2023-10-09T04:57:42.023007+0000","last_became_peered":"2023-10-09T04:57:42.023007+0000","last_unstale":"2023-10-09T05:02:55.151119+0000","last_undegraded":"2023-10-09T05:02:55.151119+0000","last_fullsized":"2023-10-09T05:02:55.151119+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:43.715250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":91,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.914248+0000","last_change":"2023-10-09T05:02:09.648781+0000","last_active":"2023-10-09T05:06:11.914248+0000","last_peered":"2023-10-09T05:06:11.914248+0000","last_clean":"2023-10-09T05:06:11.914248+0000","last_became_active":"2023-10-09T05:02:09.648078+0000","last_became_peered":"2023-10-09T05:02:09.648078+0000","last_unstale":"2023-10-09T05:06:11.914248+0000","last_undegraded":"2023-10-09T05:06:11.914248+0000","last_fullsized":"2023-10-09T05:06:11.914248+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:35.287091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":107,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.927800+0000","last_change":"2023-10-09T05:01:02.653869+0000","last_active":"2023-10-09T05:06:11.927800+0000","last_peered":"2023-10-09T05:06:11.927800+0000","last_clean":"2023-10-09T05:06:11.927800+0000","last_became_active":"2023-10-09T05:01:02.653493+0000","last_became_peered":"2023-10-09T05:01:02.653493+0000","last_unstale":"2023-10-09T05:06:11.927800+0000","last_undegraded":"2023-10-09T05:06:11.927800+0000","last_fullsized":"2023-10-09T05:06:11.927800+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:40:46.722386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":103,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.148680+0000","last_change":"2023-10-09T04:56:56.073473+0000","last_active":"2023-10-09T05:02:55.148680+0000","last_peered":"2023-10-09T05:02:55.148680+0000","last_clean":"2023-10-09T05:02:55.148680+0000","last_became_active":"2023-10-09T04:56:56.073146+0000","last_became_peered":"2023-10-09T04:56:56.073146+0000","last_unstale":"2023-10-09T05:02:55.148680+0000","last_undegraded":"2023-10-09T05:02:55.148680+0000","last_fullsized":"2023-10-09T05:02:55.148680+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:55:25.132822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":83,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:06:01.201293+0000","last_change":"2023-10-09T05:01:02.664643+0000","last_active":"2023-10-09T05:06:01.201293+0000","last_peered":"2023-10-09T05:06:01.201293+0000","last_clean":"2023-10-09T05:06:01.201293+0000","last_became_active":"2023-10-09T05:01:02.664288+0000","last_became_peered":"2023-10-09T05:01:02.664288+0000","last_unstale":"2023-10-09T05:06:01.201293+0000","last_undegraded":"2023-10-09T05:06:01.201293+0000","last_fullsized":"2023-10-09T05:06:01.201293+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:01.572876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":77,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.916855+0000","last_change":"2023-10-09T04:56:56.071763+0000","last_active":"2023-10-09T05:06:11.916855+0000","last_peered":"2023-10-09T05:06:11.916855+0000","last_clean":"2023-10-09T05:06:11.916855+0000","last_became_active":"2023-10-09T04:56:56.071429+0000","last_became_peered":"2023-10-09T04:56:56.071429+0000","last_unstale":"2023-10-09T05:06:11.916855+0000","last_undegraded":"2023-10-09T05:06:11.916855+0000","last_fullsized":"2023-10-09T05:06:11.916855+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:53:02.051460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":82,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.915549+0000","last_change":"2023-10-09T05:02:09.651149+0000","last_active":"2023-10-09T05:06:11.915549+0000","last_peered":"2023-10-09T05:06:11.915549+0000","last_clean":"2023-10-09T05:06:11.915549+0000","last_became_active":"2023-10-09T05:02:09.650151+0000","last_became_peered":"2023-10-09T05:02:09.650151+0000","last_unstale":"2023-10-09T05:06:11.915549+0000","last_undegraded":"2023-10-09T05:06:11.915549+0000","last_fullsized":"2023-10-09T05:06:11.915549+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:49.100209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":67,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.744947+0000","last_change":"2023-10-09T04:57:42.023683+0000","last_active":"2023-10-09T05:06:11.744947+0000","last_peered":"2023-10-09T05:06:11.744947+0000","last_clean":"2023-10-09T05:06:11.744947+0000","last_became_active":"2023-10-09T04:57:42.023371+0000","last_became_peered":"2023-10-09T04:57:42.023371+0000","last_unstale":"2023-10-09T05:06:11.744947+0000","last_undegraded":"2023-10-09T05:06:11.744947+0000","last_fullsized":"2023-10-09T05:06:11.744947+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:17:29.936222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":78,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.094920+0000","last_change":"2023-10-09T05:02:09.650750+0000","last_active":"2023-10-09T05:06:08.094920+0000","last_peered":"2023-10-09T05:06:08.094920+0000","last_clean":"2023-10-09T05:06:08.094920+0000","last_became_active":"2023-10-09T05:02:09.650351+0000","last_became_peered":"2023-10-09T05:02:09.650351+0000","last_unstale":"2023-10-09T05:06:08.094920+0000","last_undegraded":"2023-10-09T05:06:08.094920+0000","last_fullsized":"2023-10-09T05:06:08.094920+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:14.826132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":103,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.155202+0000","last_change":"2023-10-09T05:01:02.660775+0000","last_active":"2023-10-09T05:02:55.155202+0000","last_peered":"2023-10-09T05:02:55.155202+0000","last_clean":"2023-10-09T05:02:55.155202+0000","last_became_active":"2023-10-09T05:01:02.660420+0000","last_became_peered":"2023-10-09T05:01:02.660420+0000","last_unstale":"2023-10-09T05:02:55.155202+0000","last_undegraded":"2023-10-09T05:02:55.155202+0000","last_fullsized":"2023-10-09T05:02:55.155202+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:54.031869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":96,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.927896+0000","last_change":"2023-10-09T04:57:42.017497+0000","last_active":"2023-10-09T05:06:11.927896+0000","last_peered":"2023-10-09T05:06:11.927896+0000","last_clean":"2023-10-09T05:06:11.927896+0000","last_became_active":"2023-10-09T04:57:42.017165+0000","last_became_peered":"2023-10-09T04:57:42.017165+0000","last_unstale":"2023-10-09T05:06:11.927896+0000","last_undegraded":"2023-10-09T05:06:11.927896+0000","last_fullsized":"2023-10-09T05:06:11.927896+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:43.258960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":81,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.128729+0000","last_change":"2023-10-09T04:44:53.792412+0000","last_active":"2023-10-09T05:02:55.128729+0000","last_peered":"2023-10-09T05:02:55.128729+0000","last_clean":"2023-10-09T05:02:55.128729+0000","last_became_active":"2023-10-09T04:44:53.792009+0000","last_became_peered":"2023-10-09T04:44:53.792009+0000","last_unstale":"2023-10-09T05:02:55.128729+0000","last_undegraded":"2023-10-09T05:02:55.128729+0000","last_fullsized":"2023-10-09T05:02:55.128729+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:39.423339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'2","reported_seq":81,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.917224+0000","last_change":"2023-10-09T04:56:56.079768+0000","last_active":"2023-10-09T05:06:11.917224+0000","last_peered":"2023-10-09T05:06:11.917224+0000","last_clean":"2023-10-09T05:06:11.917224+0000","last_became_active":"2023-10-09T04:56:56.077721+0000","last_became_peered":"2023-10-09T04:56:56.077721+0000","last_unstale":"2023-10-09T05:06:11.917224+0000","last_undegraded":"2023-10-09T05:06:11.917224+0000","last_fullsized":"2023-10-09T05:06:11.917224+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:36.384357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":118,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:06:00.911094+0000","last_change":"2023-10-09T05:02:09.644679+0000","last_active":"2023-10-09T05:06:00.911094+0000","last_peered":"2023-10-09T05:06:00.911094+0000","last_clean":"2023-10-09T05:06:00.911094+0000","last_became_active":"2023-10-09T05:02:09.644335+0000","last_became_peered":"2023-10-09T05:02:09.644335+0000","last_unstale":"2023-10-09T05:06:00.911094+0000","last_undegraded":"2023-10-09T05:06:00.911094+0000","last_fullsized":"2023-10-09T05:06:00.911094+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:06.142246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":102,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.134135+0000","last_change":"2023-10-09T04:57:42.018276+0000","last_active":"2023-10-09T05:02:55.134135+0000","last_peered":"2023-10-09T05:02:55.134135+0000","last_clean":"2023-10-09T05:02:55.134135+0000","last_became_active":"2023-10-09T04:57:42.017975+0000","last_became_peered":"2023-10-09T04:57:42.017975+0000","last_unstale":"2023-10-09T05:02:55.134135+0000","last_undegraded":"2023-10-09T05:02:55.134135+0000","last_fullsized":"2023-10-09T05:02:55.134135+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:48.867813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":115,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129845+0000","last_change":"2023-10-09T05:02:09.653528+0000","last_active":"2023-10-09T05:02:55.129845+0000","last_peered":"2023-10-09T05:02:55.129845+0000","last_clean":"2023-10-09T05:02:55.129845+0000","last_became_active":"2023-10-09T05:02:09.653171+0000","last_became_peered":"2023-10-09T05:02:09.653171+0000","last_unstale":"2023-10-09T05:02:55.129845+0000","last_undegraded":"2023-10-09T05:02:55.129845+0000","last_fullsized":"2023-10-09T05:02:55.129845+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:15.978289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":85,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.134213+0000","last_change":"2023-10-09T04:44:52.793709+0000","last_active":"2023-10-09T05:02:55.134213+0000","last_peered":"2023-10-09T05:02:55.134213+0000","last_clean":"2023-10-09T05:02:55.134213+0000","last_became_active":"2023-10-09T04:44:52.792973+0000","last_became_peered":"2023-10-09T04:44:52.792973+0000","last_unstale":"2023-10-09T05:02:55.134213+0000","last_undegraded":"2023-10-09T05:02:55.134213+0000","last_fullsized":"2023-10-09T05:02:55.134213+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:12.091089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":115,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129621+0000","last_change":"2023-10-09T05:02:09.649748+0000","last_active":"2023-10-09T05:02:55.129621+0000","last_peered":"2023-10-09T05:02:55.129621+0000","last_clean":"2023-10-09T05:02:55.129621+0000","last_became_active":"2023-10-09T05:02:09.649444+0000","last_became_peered":"2023-10-09T05:02:09.649444+0000","last_unstale":"2023-10-09T05:02:55.129621+0000","last_undegraded":"2023-10-09T05:02:55.129621+0000","last_fullsized":"2023-10-09T05:02:55.129621+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:07.436820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'2","reported_seq":99,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.095102+0000","last_change":"2023-10-09T05:02:09.647382+0000","last_active":"2023-10-09T05:06:08.095102+0000","last_peered":"2023-10-09T05:06:08.095102+0000","last_clean":"2023-10-09T05:06:08.095102+0000","last_became_active":"2023-10-09T05:02:09.646803+0000","last_became_peered":"2023-10-09T05:02:09.646803+0000","last_unstale":"2023-10-09T05:06:08.095102+0000","last_undegraded":"2023-10-09T05:06:08.095102+0000","last_fullsized":"2023-10-09T05:06:08.095102+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:23:00.161736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":113,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.914782+0000","last_change":"2023-10-09T05:02:09.649674+0000","last_active":"2023-10-09T05:06:11.914782+0000","last_peered":"2023-10-09T05:06:11.914782+0000","last_clean":"2023-10-09T05:06:11.914782+0000","last_became_active":"2023-10-09T05:02:09.649041+0000","last_became_peered":"2023-10-09T05:02:09.649041+0000","last_unstale":"2023-10-09T05:06:11.914782+0000","last_undegraded":"2023-10-09T05:06:11.914782+0000","last_fullsized":"2023-10-09T05:06:11.914782+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:18.524071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'5","reported_seq":101,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.917631+0000","last_change":"2023-10-09T05:01:02.672596+0000","last_active":"2023-10-09T05:06:11.917631+0000","last_peered":"2023-10-09T05:06:11.917631+0000","last_clean":"2023-10-09T05:06:11.917631+0000","last_became_active":"2023-10-09T05:01:02.672128+0000","last_became_peered":"2023-10-09T05:01:02.672128+0000","last_unstale":"2023-10-09T05:06:11.917631+0000","last_undegraded":"2023-10-09T05:06:11.917631+0000","last_fullsized":"2023-10-09T05:06:11.917631+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:17:05.887502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.128756+0000","last_change":"2023-10-09T04:57:42.027187+0000","last_active":"2023-10-09T05:02:55.128756+0000","last_peered":"2023-10-09T05:02:55.128756+0000","last_clean":"2023-10-09T05:02:55.128756+0000","last_became_active":"2023-10-09T04:57:42.024247+0000","last_became_peered":"2023-10-09T04:57:42.024247+0000","last_unstale":"2023-10-09T05:02:55.128756+0000","last_undegraded":"2023-10-09T05:02:55.128756+0000","last_fullsized":"2023-10-09T05:02:55.128756+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:42.410792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":38,"num_read_kb":40,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234310,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303396,"kb_used_data":2012,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93441756,"statfs":{"total":95995035648,"available":95684358144,"internally_reserved":0,"allocated":2060288,"data_stored":1463213,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":1.5109999999999999,"15min":1.4710000000000001},"min":{"1min":0.67600000000000005,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":1.208,"5min":11.093999999999999,"15min":11.093999999999999},"last":0.82399999999999995},{"interface":"front","average":{"1min":1.0489999999999999,"5min":1.6100000000000001,"15min":1.6910000000000001},"min":{"1min":0.59699999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.2689999999999999,"5min":6.3730000000000002,"15min":9.5090000000000003},"last":1.044}]},{"osd":1,"last update":"Mon Oct 9 05:07:04 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":3.2010000000000001,"15min":2.125},"min":{"1min":0.71299999999999997,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.0880000000000001,"5min":135.322,"15min":135.322},"last":0.95099999999999996},{"interface":"front","average":{"1min":1.05,"5min":3.2149999999999999,"15min":2.2749999999999999},"min":{"1min":0.69199999999999995,"5min":0.33500000000000002,"15min":0.30499999999999999},"max":{"1min":1.294,"5min":135.09,"15min":135.09},"last":0.77100000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":1.633,"15min":1.369},"min":{"1min":0.625,"5min":0.315,"15min":0.315},"max":{"1min":1.2290000000000001,"5min":11.391,"15min":11.391},"last":0.99199999999999999},{"interface":"front","average":{"1min":0.89800000000000002,"5min":1.512,"15min":1.274},"min":{"1min":0.59899999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.262,"5min":6.6820000000000004,"15min":6.6820000000000004},"last":0.86799999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":1.4850000000000001,"15min":1.2569999999999999},"min":{"1min":0.58699999999999997,"5min":0.317,"15min":0.317},"max":{"1min":1.2649999999999999,"5min":6.548,"15min":6.548},"last":1.117},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.823,"15min":1.7050000000000001},"min":{"1min":0.63700000000000001,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.474,"5min":9.1129999999999995,"15min":9.1129999999999995},"last":1.0920000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:07:04 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":1.349,"15min":1.1839999999999999},"min":{"1min":0.61399999999999999,"5min":0.33700000000000002,"15min":0.28699999999999998},"max":{"1min":1.319,"5min":11.409000000000001,"15min":11.409000000000001},"last":0.71499999999999997},{"interface":"front","average":{"1min":1.123,"5min":1.5940000000000001,"15min":1.375},"min":{"1min":0.66800000000000004,"5min":0.36899999999999999,"15min":0.316},"max":{"1min":1.4350000000000001,"5min":15.763,"15min":15.763},"last":0.91000000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:07:04 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.677,"15min":1.3560000000000001},"min":{"1min":0.73799999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.389,"5min":8.3569999999999993,"15min":11.244},"last":1.0169999999999999},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.573,"15min":1.403},"min":{"1min":0.89700000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.3029999999999999,"5min":7.1020000000000003,"15min":9.6010000000000009},"last":1.0609999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:07:04 2023","interfaces":[{"interface":"back","average":{"1min":1.139,"5min":1.5069999999999999,"15min":1.2629999999999999},"min":{"1min":0.93500000000000005,"5min":0.376,"15min":0.318},"max":{"1min":1.4470000000000001,"5min":6.9630000000000001,"15min":8.484},"last":1.151},{"interface":"front","average":{"1min":1.2310000000000001,"5min":1.6739999999999999,"15min":1.355},"min":{"1min":0.82799999999999996,"5min":0.35499999999999998,"15min":0.35399999999999998},"max":{"1min":1.5029999999999999,"5min":6.468,"15min":6.468},"last":1.1819999999999999}]}]},{"osd":6,"up_from":46,"seq":197568495952,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299292,"kb_used_data":2004,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93445860,"statfs":{"total":95995035648,"available":95688560640,"internally_reserved":0,"allocated":2052096,"data_stored":1463390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.089,"5min":1.8100000000000001,"15min":1.5309999999999999},"min":{"1min":0.61199999999999999,"5min":0.24399999999999999,"15min":0.214},"max":{"1min":1.4119999999999999,"5min":11.449,"15min":19.18},"last":0.89900000000000002},{"interface":"front","average":{"1min":1.1200000000000001,"5min":1.9450000000000001,"15min":1.6559999999999999},"min":{"1min":0.752,"5min":0.30399999999999999,"15min":0.27200000000000002},"max":{"1min":1.417,"5min":11.491,"15min":19.358000000000001},"last":1.0620000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:07:01 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":1.71,"15min":1.6919999999999999},"min":{"1min":0.66600000000000004,"5min":0.28000000000000003,"15min":0.25800000000000001},"max":{"1min":1.21,"5min":10.118,"15min":10.118},"last":0.95699999999999996},{"interface":"front","average":{"1min":0.94199999999999995,"5min":1.9339999999999999,"15min":1.4379999999999999},"min":{"1min":0.61199999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.3580000000000001,"5min":9.6630000000000003,"15min":12.417},"last":0.72199999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:07:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0800000000000001,"5min":1.857,"15min":1.5409999999999999},"min":{"1min":0.74199999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.4750000000000001,"5min":8.9730000000000008,"15min":8.9730000000000008},"last":0.81699999999999995},{"interface":"front","average":{"1min":0.88200000000000001,"5min":1.5029999999999999,"15min":1.365},"min":{"1min":0.58599999999999997,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.091,"5min":8.3810000000000002,"15min":8.3810000000000002},"last":1.002}]},{"osd":3,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":2.105,"15min":1.657},"min":{"1min":0.66400000000000003,"5min":0.315,"15min":0.315},"max":{"1min":1.1899999999999999,"5min":10.201000000000001,"15min":10.201000000000001},"last":1.0409999999999999},{"interface":"front","average":{"1min":1.121,"5min":1.841,"15min":1.4790000000000001},"min":{"1min":0.81399999999999995,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.411,"5min":9.7720000000000002,"15min":9.7720000000000002},"last":1.242}]},{"osd":4,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":1.6279999999999999,"15min":1.153},"min":{"1min":0.70899999999999996,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.325,"5min":16.119,"15min":16.119},"last":1.0880000000000001},{"interface":"front","average":{"1min":1.1419999999999999,"5min":1.5569999999999999,"15min":1.274},"min":{"1min":0.79500000000000004,"5min":0.32300000000000001,"15min":0.247},"max":{"1min":1.5329999999999999,"5min":11.272,"15min":11.272},"last":1.3260000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1100000000000001,"5min":1.746,"15min":1.319},"min":{"1min":0.94899999999999995,"5min":0.312,"15min":0.29299999999999998},"max":{"1min":1.444,"5min":7.7149999999999999,"15min":7.7149999999999999},"last":1.1910000000000001},{"interface":"front","average":{"1min":1.2769999999999999,"5min":2.016,"15min":1.516},"min":{"1min":0.63800000000000001,"5min":0.41599999999999998,"15min":0.33100000000000002},"max":{"1min":1.5780000000000001,"5min":10.537000000000001,"15min":10.537000000000001},"last":1.155}]},{"osd":7,"last update":"Mon Oct 9 05:07:50 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.472,"15min":1.3240000000000001},"min":{"1min":0.85699999999999998,"5min":0.40200000000000002,"15min":0.30599999999999999},"max":{"1min":1.3879999999999999,"5min":9.3780000000000001,"15min":9.3780000000000001},"last":1.119},{"interface":"front","average":{"1min":1.23,"5min":1.569,"15min":1.415},"min":{"1min":1.0249999999999999,"5min":0.434,"15min":0.375},"max":{"1min":1.5329999999999999,"5min":9.4060000000000006,"15min":9.4060000000000006},"last":1.294}]}]},{"osd":1,"up_from":96,"seq":412316860542,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299076,"kb_used_data":1596,"kb_used_omap":4,"kb_used_meta":297467,"kb_avail":93446076,"statfs":{"total":95995035648,"available":95688781824,"internally_reserved":0,"allocated":1634304,"data_stored":894480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":304607118},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:59 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":1.8959999999999999,"15min":1.845},"min":{"1min":0.38400000000000001,"5min":0.20899999999999999,"15min":0.19400000000000001},"max":{"1min":0.81799999999999995,"5min":22.382999999999999,"15min":22.382999999999999},"last":0.745},{"interface":"front","average":{"1min":0.65700000000000003,"5min":2.0089999999999999,"15min":2.306},"min":{"1min":0.42199999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.91800000000000004,"5min":10.159000000000001,"15min":10.268000000000001},"last":0.92000000000000004}]},{"osd":2,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":1.7609999999999999,"15min":1.5820000000000001},"min":{"1min":0.40400000000000003,"5min":0.219,"15min":0.219},"max":{"1min":1.0029999999999999,"5min":10.115,"15min":10.115},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.78200000000000003,"5min":1.831,"15min":1.5449999999999999},"min":{"1min":0.45800000000000002,"5min":0.252,"15min":0.252},"max":{"1min":1.151,"5min":10.428000000000001,"15min":11.917},"last":0.79100000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:07:20 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":2.1339999999999999,"15min":2.141},"min":{"1min":0.51800000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.0489999999999999,"5min":9.6039999999999992,"15min":9.6039999999999992},"last":0.83599999999999997},{"interface":"front","average":{"1min":0.80700000000000005,"5min":1.6899999999999999,"15min":1.4750000000000001},"min":{"1min":0.52800000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.21,"5min":9.9039999999999999,"15min":9.9039999999999999},"last":1.028}]},{"osd":4,"last update":"Mon Oct 9 05:07:59 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.5529999999999999,"15min":1.6000000000000001},"min":{"1min":0.82199999999999995,"5min":0.35599999999999998,"15min":0.34200000000000003},"max":{"1min":1.339,"5min":9.9749999999999996,"15min":9.9749999999999996},"last":1.159},{"interface":"front","average":{"1min":0.98499999999999999,"5min":1.8180000000000001,"15min":1.5129999999999999},"min":{"1min":0.72599999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.3,"5min":22.350000000000001,"15min":22.350000000000001},"last":0.995}]},{"osd":5,"last update":"Mon Oct 9 05:07:59 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":2.028,"15min":1.6859999999999999},"min":{"1min":0.63500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.218,"5min":12.454000000000001,"15min":12.454000000000001},"last":1.0609999999999999},{"interface":"front","average":{"1min":1.1100000000000001,"5min":2.1179999999999999,"15min":1.8959999999999999},"min":{"1min":0.73799999999999999,"5min":0.34999999999999998,"15min":0.316},"max":{"1min":1.3700000000000001,"5min":10.035,"15min":10.035},"last":1.093}]},{"osd":6,"last update":"Mon Oct 9 05:07:59 2023","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":2.222,"15min":1.901},"min":{"1min":0.78300000000000003,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.2450000000000001,"5min":10.409000000000001,"15min":10.409000000000001},"last":1.117},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.9359999999999999,"15min":1.7330000000000001},"min":{"1min":0.61499999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.3879999999999999,"5min":10.007,"15min":10.007},"last":1.2410000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:07:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.708,"15min":1.786},"min":{"1min":0.83599999999999997,"5min":0.41899999999999998,"15min":0.38800000000000001},"max":{"1min":1.4259999999999999,"5min":10.061,"15min":10.061},"last":1.3200000000000001},{"interface":"front","average":{"1min":1.0069999999999999,"5min":1.948,"15min":1.698},"min":{"1min":0.626,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.2370000000000001,"5min":22.175999999999998,"15min":22.175999999999998},"last":1.1910000000000001}]}]},{"osd":0,"up_from":88,"seq":377957122184,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":334284,"kb_used_data":2180,"kb_used_omap":5,"kb_used_meta":332090,"kb_avail":93410868,"statfs":{"total":95995035648,"available":95652728832,"internally_reserved":0,"allocated":2232320,"data_stored":1483977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":340060794},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:07:59 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":1.6459999999999999,"15min":2.173},"min":{"1min":0.47399999999999998,"5min":0.217,"15min":0.217},"max":{"1min":0.97999999999999998,"5min":11.561999999999999,"15min":13.084},"last":0.88300000000000001},{"interface":"front","average":{"1min":0.60999999999999999,"5min":1.585,"15min":1.9890000000000001},"min":{"1min":0.33400000000000002,"5min":0.30099999999999999,"15min":0.218},"max":{"1min":0.81299999999999994,"5min":10.746,"15min":11.161},"last":1.1220000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:07:10 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":1.7210000000000001,"15min":1.9550000000000001},"min":{"1min":0.434,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.99099999999999999,"5min":8.8030000000000008,"15min":9.1300000000000008},"last":1.0229999999999999},{"interface":"front","average":{"1min":0.72699999999999998,"5min":1.3100000000000001,"15min":1.7110000000000001},"min":{"1min":0.34300000000000003,"5min":0.24299999999999999,"15min":0.23300000000000001},"max":{"1min":1.2450000000000001,"5min":8.3599999999999994,"15min":10.423},"last":1.0469999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":1.5489999999999999,"15min":1.4810000000000001},"min":{"1min":0.35799999999999998,"5min":0.253,"15min":0.253},"max":{"1min":1.389,"5min":11.433999999999999,"15min":11.433999999999999},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.79500000000000004,"5min":1.575,"15min":1.3089999999999999},"min":{"1min":0.39700000000000002,"5min":0.318,"15min":0.318},"max":{"1min":1.411,"5min":10.041,"15min":10.041},"last":0.78300000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.3919999999999999,"15min":1.4830000000000001},"min":{"1min":0.53300000000000003,"5min":0.33500000000000002,"15min":0.30099999999999999},"max":{"1min":1.3320000000000001,"5min":8.7449999999999992,"15min":8.7449999999999992},"last":1.2070000000000001},{"interface":"front","average":{"1min":0.98599999999999999,"5min":1.331,"15min":1.4450000000000001},"min":{"1min":0.52300000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.3540000000000001,"5min":5.7320000000000002,"15min":7.202},"last":1.159}]},{"osd":5,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":1.49,"15min":1.5649999999999999},"min":{"1min":0.55400000000000005,"5min":0.34100000000000003,"15min":0.30399999999999999},"max":{"1min":1.6890000000000001,"5min":6.8959999999999999,"15min":12.526999999999999},"last":0.97099999999999997},{"interface":"front","average":{"1min":1.022,"5min":1.7709999999999999,"15min":1.6359999999999999},"min":{"1min":0.50800000000000001,"5min":0.35299999999999998,"15min":0.316},"max":{"1min":1.6020000000000001,"5min":10.148,"15min":10.148},"last":0.93600000000000005}]},{"osd":6,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.615,"15min":1.587},"min":{"1min":0.55600000000000005,"5min":0.371,"15min":0.371},"max":{"1min":1.3200000000000001,"5min":8.8710000000000004,"15min":8.8710000000000004},"last":1.2390000000000001},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.4399999999999999,"15min":1.5},"min":{"1min":0.67800000000000005,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.458,"5min":6.9370000000000003,"15min":10.429},"last":1.081}]},{"osd":7,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.3660000000000001,"15min":1.5049999999999999},"min":{"1min":0.64400000000000002,"5min":0.375,"15min":0.36599999999999999},"max":{"1min":1.631,"5min":6.9569999999999999,"15min":10.314},"last":1.1910000000000001},{"interface":"front","average":{"1min":1.075,"5min":1.71,"15min":1.879},"min":{"1min":0.64400000000000002,"5min":0.436,"15min":0.39500000000000002},"max":{"1min":1.4039999999999999,"5min":8.6110000000000007,"15min":10.738},"last":1.2849999999999999}]}]},{"osd":2,"up_from":106,"seq":455266533462,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":252696,"kb_used_data":1552,"kb_used_omap":7,"kb_used_meta":251128,"kb_avail":93492456,"statfs":{"total":95995035648,"available":95736274944,"internally_reserved":0,"allocated":1589248,"data_stored":886182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":257155153},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:07 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":1.619,"15min":1.294},"min":{"1min":0.40500000000000003,"5min":0.215,"15min":0.215},"max":{"1min":0.89500000000000002,"5min":23.010999999999999,"15min":23.010999999999999},"last":0.80900000000000005},{"interface":"front","average":{"1min":0.67500000000000004,"5min":1.9870000000000001,"15min":1.323},"min":{"1min":0.28799999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":1.0649999999999999,"5min":20.501999999999999,"15min":20.501999999999999},"last":0.78500000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:07:07 2023","interfaces":[{"interface":"back","average":{"1min":0.74299999999999999,"5min":1.619,"15min":1.2649999999999999},"min":{"1min":0.373,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.2070000000000001,"5min":20.539999999999999,"15min":20.539999999999999},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.65300000000000002,"5min":2.048,"15min":1.502},"min":{"1min":0.33600000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.036,"5min":24.058,"15min":24.058},"last":0.63400000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:07:20 2023","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":1.827,"15min":2.1429999999999998},"min":{"1min":0.41799999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.2250000000000001,"5min":20.907,"15min":20.907},"last":0.90300000000000002},{"interface":"front","average":{"1min":0.67600000000000005,"5min":1.859,"15min":1.778},"min":{"1min":0.39500000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.046,"5min":23.574999999999999,"15min":23.574999999999999},"last":0.75800000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:07:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":1.766,"15min":1.458},"min":{"1min":0.69299999999999995,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.3160000000000001,"5min":20.712,"15min":20.712},"last":1.0800000000000001},{"interface":"front","average":{"1min":0.94299999999999995,"5min":1.98,"15min":1.5069999999999999},"min":{"1min":0.61299999999999999,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.2789999999999999,"5min":23.547000000000001,"15min":23.547000000000001},"last":0.95999999999999996}]},{"osd":5,"last update":"Mon Oct 9 05:07:07 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":2.1970000000000001,"15min":1.5329999999999999},"min":{"1min":0.63600000000000001,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.2170000000000001,"5min":20.782,"15min":20.782},"last":1.03},{"interface":"front","average":{"1min":1.0229999999999999,"5min":2.2160000000000002,"15min":1.958},"min":{"1min":0.65400000000000003,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.48,"5min":20.792000000000002,"15min":20.792000000000002},"last":0.997}]},{"osd":6,"last update":"Mon Oct 9 05:07:07 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":1.9370000000000001,"15min":1.593},"min":{"1min":0.59099999999999997,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.226,"5min":23.510999999999999,"15min":23.510999999999999},"last":0.85799999999999998},{"interface":"front","average":{"1min":1.0549999999999999,"5min":1.8160000000000001,"15min":1.54},"min":{"1min":0.754,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":1.3,"5min":20.684999999999999,"15min":20.684999999999999},"last":1.2529999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:07:07 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":1.748,"15min":1.53},"min":{"1min":0.67300000000000004,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":1.3460000000000001,"5min":20.887,"15min":20.887},"last":1.3069999999999999},{"interface":"front","average":{"1min":1.113,"5min":1.8979999999999999,"15min":1.4690000000000001},"min":{"1min":0.83899999999999997,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.3959999999999999,"5min":20.863,"15min":20.863},"last":1.1379999999999999}]}]},{"osd":3,"up_from":123,"seq":528280977481,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":272160,"kb_used_data":1496,"kb_used_omap":2,"kb_used_meta":270653,"kb_avail":93472992,"statfs":{"total":95995035648,"available":95716343808,"internally_reserved":0,"allocated":1531904,"data_stored":879533,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":277148725},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":1.6279999999999999,"15min":2.0550000000000002},"min":{"1min":0.434,"5min":0.187,"15min":0.187},"max":{"1min":0.86599999999999999,"5min":14.141,"15min":14.141},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.72199999999999998,"5min":1.972,"15min":2.605},"min":{"1min":0.44700000000000001,"5min":0.184,"15min":0.184},"max":{"1min":1.0920000000000001,"5min":14.045999999999999,"15min":14.045999999999999},"last":0.54400000000000004}]},{"osd":1,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":1.7589999999999999,"15min":2.4119999999999999},"min":{"1min":0.36199999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.147,"5min":14.193,"15min":14.193},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.67900000000000005,"5min":1.669,"15min":2.548},"min":{"1min":0.45000000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":1.0740000000000001,"5min":17.498999999999999,"15min":17.498999999999999},"last":0.58199999999999996}]},{"osd":2,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":1.855,"15min":2.0779999999999998},"min":{"1min":0.39800000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.91600000000000004,"5min":9.6129999999999995,"15min":9.6129999999999995},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.77700000000000002,"5min":1.873,"15min":1.867},"min":{"1min":0.49299999999999999,"5min":0.245,"15min":0.245},"max":{"1min":1.236,"5min":8.1600000000000001,"15min":8.1600000000000001},"last":0.621}]},{"osd":4,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":1.5780000000000001,"15min":1.73},"min":{"1min":0.66900000000000004,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.45,"5min":5.8070000000000004,"15min":5.8070000000000004},"last":0.95099999999999996},{"interface":"front","average":{"1min":0.94699999999999995,"5min":1.3500000000000001,"15min":1.294},"min":{"1min":0.58399999999999996,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.3180000000000001,"5min":9.4749999999999996,"15min":9.4749999999999996},"last":0.92000000000000004}]},{"osd":5,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":1.613,"15min":1.335},"min":{"1min":0.71299999999999997,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.27,"5min":9.5079999999999991,"15min":9.5079999999999991},"last":1.006},{"interface":"front","average":{"1min":1.024,"5min":2.0049999999999999,"15min":2.0550000000000002},"min":{"1min":0.67100000000000004,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.413,"5min":8.9380000000000006,"15min":8.9380000000000006},"last":0.90200000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":1.7789999999999999,"15min":1.875},"min":{"1min":0.72399999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.4339999999999999,"5min":9.6590000000000007,"15min":9.6590000000000007},"last":0.73799999999999999},{"interface":"front","average":{"1min":1.05,"5min":1.9199999999999999,"15min":1.9810000000000001},"min":{"1min":0.73999999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.405,"5min":6.3289999999999997,"15min":6.3289999999999997},"last":0.97999999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.9970000000000001,"15min":2.1699999999999999},"min":{"1min":0.72799999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.468,"5min":8.3209999999999997,"15min":8.3209999999999997},"last":1.048},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.663,"15min":1.5589999999999999},"min":{"1min":0.81299999999999994,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.369,"5min":9.5359999999999996,"15min":9.5359999999999996},"last":1.083}]}]},{"osd":4,"up_from":30,"seq":128849019238,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298724,"kb_used_data":1436,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446428,"statfs":{"total":95995035648,"available":95689142272,"internally_reserved":0,"allocated":1470464,"data_stored":873363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:11 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":1.389,"15min":4.6070000000000002},"min":{"1min":0.55200000000000005,"5min":0.26600000000000001,"15min":0.22700000000000001},"max":{"1min":1.389,"5min":10.109999999999999,"15min":131.30699999999999},"last":1.1279999999999999},{"interface":"front","average":{"1min":0.97999999999999998,"5min":1.244,"15min":4.5469999999999997},"min":{"1min":0.57999999999999996,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.3999999999999999,"5min":6.3470000000000004,"15min":131.36600000000001},"last":1.355}]},{"osd":1,"last update":"Mon Oct 9 05:08:03 2023","interfaces":[{"interface":"back","average":{"1min":0.86099999999999999,"5min":2.3119999999999998,"15min":1.833},"min":{"1min":0.61199999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.075,"5min":59.408000000000001,"15min":59.408000000000001},"last":1.0720000000000001},{"interface":"front","average":{"1min":1.028,"5min":2.1779999999999999,"15min":1.9610000000000001},"min":{"1min":0.69399999999999995,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.353,"5min":60.207000000000001,"15min":60.207000000000001},"last":0.80600000000000005}]},{"osd":2,"last update":"Mon Oct 9 05:07:11 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.742,"15min":1.1870000000000001},"min":{"1min":0.59499999999999997,"5min":0.307,"15min":0.307},"max":{"1min":1.2949999999999999,"5min":10.196,"15min":10.196},"last":1.258},{"interface":"front","average":{"1min":1.0009999999999999,"5min":1.387,"15min":1.107},"min":{"1min":0.628,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":1.294,"5min":6.3810000000000002,"15min":6.3810000000000002},"last":1.038}]},{"osd":3,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":1.869,"15min":1.734},"min":{"1min":0.68999999999999995,"5min":0.309,"15min":0.309},"max":{"1min":1.343,"5min":10.555999999999999,"15min":10.555999999999999},"last":0.91600000000000004},{"interface":"front","average":{"1min":1.022,"5min":1.851,"15min":1.657},"min":{"1min":0.83099999999999996,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.262,"5min":21.036999999999999,"15min":21.036999999999999},"last":1.1679999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1100000000000001,"5min":1.5289999999999999,"15min":1.2869999999999999},"min":{"1min":0.64500000000000002,"5min":0.32500000000000001,"15min":0.31},"max":{"1min":1.3899999999999999,"5min":10.214,"15min":10.214},"last":0.998},{"interface":"front","average":{"1min":1.1870000000000001,"5min":1.6200000000000001,"15min":1.3260000000000001},"min":{"1min":0.83699999999999997,"5min":0.34200000000000003,"15min":0.30399999999999999},"max":{"1min":1.629,"5min":6.8310000000000004,"15min":10.890000000000001},"last":0.95099999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:08:03 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":1.748,"15min":1.3560000000000001},"min":{"1min":0.60499999999999998,"5min":0.32800000000000001,"15min":0.27800000000000002},"max":{"1min":1.2310000000000001,"5min":11.557,"15min":11.557},"last":1.2310000000000001},{"interface":"front","average":{"1min":1.214,"5min":1.5920000000000001,"15min":1.383},"min":{"1min":0.86699999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.504,"5min":10.689,"15min":10.689},"last":1.327}]},{"osd":7,"last update":"Mon Oct 9 05:08:03 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":1.329,"15min":1.2509999999999999},"min":{"1min":0.68100000000000005,"5min":0.377,"15min":0.35999999999999999},"max":{"1min":1.5209999999999999,"5min":6.4820000000000002,"15min":8.8859999999999992},"last":1.2849999999999999},{"interface":"front","average":{"1min":1.0409999999999999,"5min":1.5649999999999999,"15min":1.3819999999999999},"min":{"1min":0.77100000000000002,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.3,"5min":6.1699999999999999,"15min":7.7880000000000003},"last":1.3}]}]},{"osd":5,"up_from":38,"seq":163208757594,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":603476,"kb_used_data":1420,"kb_used_omap":0,"kb_used_meta":602048,"kb_avail":93141676,"statfs":{"total":95995035648,"available":95377076224,"internally_reserved":0,"allocated":1454080,"data_stored":869107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":616497152},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:12 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":3.0049999999999999,"15min":1.8480000000000001},"min":{"1min":0.52900000000000003,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.1950000000000001,"5min":77.745999999999995,"15min":77.745999999999995},"last":1.3160000000000001},{"interface":"front","average":{"1min":1.083,"5min":3.093,"15min":1.958},"min":{"1min":0.67000000000000004,"5min":0.37,"15min":0.35299999999999998},"max":{"1min":1.5620000000000001,"5min":77.820999999999998,"15min":77.820999999999998},"last":1.397}]},{"osd":1,"last update":"Mon Oct 9 05:07:03 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":2.218,"15min":1.7110000000000001},"min":{"1min":0.58799999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.5109999999999999,"5min":13.401999999999999,"15min":13.401999999999999},"last":0.97199999999999998},{"interface":"front","average":{"1min":1.016,"5min":1.9690000000000001,"15min":1.788},"min":{"1min":0.69399999999999995,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.2450000000000001,"5min":13.398,"15min":13.398},"last":0.91900000000000004}]},{"osd":2,"last update":"Mon Oct 9 05:07:17 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":1.7230000000000001,"15min":1.111},"min":{"1min":0.57199999999999995,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.3180000000000001,"5min":13.436,"15min":13.436},"last":1.2150000000000001},{"interface":"front","average":{"1min":1.014,"5min":1.8899999999999999,"15min":1.252},"min":{"1min":0.65200000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.2929999999999999,"5min":9.4949999999999992,"15min":9.4949999999999992},"last":0.85499999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:07:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.869,"15min":1.966},"min":{"1min":0.745,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.349,"5min":9.1010000000000009,"15min":9.1010000000000009},"last":1.0980000000000001},{"interface":"front","average":{"1min":1.1200000000000001,"5min":2.121,"15min":1.7490000000000001},"min":{"1min":0.54900000000000004,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":3.1739999999999999,"5min":11.859999999999999,"15min":11.859999999999999},"last":1.0680000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:07:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":1.6060000000000001,"15min":1.28},"min":{"1min":0.68600000000000005,"5min":0.25,"15min":0.25},"max":{"1min":1.4399999999999999,"5min":14.781000000000001,"15min":14.781000000000001},"last":1.359},{"interface":"front","average":{"1min":1.056,"5min":1.921,"15min":1.4239999999999999},"min":{"1min":0.69799999999999995,"5min":0.34599999999999997,"15min":0.313},"max":{"1min":1.2909999999999999,"5min":9.625,"15min":9.625},"last":1.2569999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:07:03 2023","interfaces":[{"interface":"back","average":{"1min":1.129,"5min":2.0310000000000001,"15min":1.544},"min":{"1min":0.60899999999999999,"5min":0.38300000000000001,"15min":0.33900000000000002},"max":{"1min":1.397,"5min":11.654999999999999,"15min":11.654999999999999},"last":1.1739999999999999},{"interface":"front","average":{"1min":1.149,"5min":1.8100000000000001,"15min":1.4630000000000001},"min":{"1min":0.70299999999999996,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.4419999999999999,"5min":8.4670000000000005,"15min":10.423999999999999},"last":1.4590000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:07:46 2023","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":1.7549999999999999,"15min":1.4550000000000001},"min":{"1min":0.629,"5min":0.36899999999999999,"15min":0.34799999999999998},"max":{"1min":1.508,"5min":8.7720000000000002,"15min":9.6639999999999997},"last":1.51},{"interface":"front","average":{"1min":1.238,"5min":1.823,"15min":1.492},"min":{"1min":0.90200000000000002,"5min":0.34699999999999998,"15min":0.313},"max":{"1min":1.577,"5min":10.122999999999999,"15min":12.843999999999999},"last":1.446}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:08:08.593 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:08:09.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:08 smithi133 bash[105589]: audit 2023-10-09T05:08:07.668394+0000 mgr.y (mgr.25277) 583 : audit 0 from='client.44732 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:08:09.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:08 smithi133 bash[105589]: cluster 2023-10-09T05:08:07.709692+0000 mgr.y (mgr.25277) 584 : cluster 0 pgmap v536: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:09.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:08 smithi133 bash[102858]: audit 2023-10-09T05:08:07.668394+0000 mgr.y (mgr.25277) 583 : audit 0 from='client.44732 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:08:09.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:08 smithi133 bash[102858]: cluster 2023-10-09T05:08:07.709692+0000 mgr.y (mgr.25277) 584 : cluster 0 pgmap v536: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:09.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:08 smithi194 bash[92666]: audit 2023-10-09T05:08:07.668394+0000 mgr.y (mgr.25277) 583 : audit 0 from='client.44732 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:08:09.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:08 smithi194 bash[92666]: cluster 2023-10-09T05:08:07.709692+0000 mgr.y (mgr.25277) 584 : cluster 0 pgmap v536: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:11.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:10 smithi133 bash[102858]: cluster 2023-10-09T05:08:09.710499+0000 mgr.y (mgr.25277) 585 : cluster 0 pgmap v537: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:11.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:10 smithi133 bash[102858]: audit 2023-10-09T05:08:10.039967+0000 mon.a (mon.0) 732 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:08:11.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:10 smithi133 bash[105589]: cluster 2023-10-09T05:08:09.710499+0000 mgr.y (mgr.25277) 585 : cluster 0 pgmap v537: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:11.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:10 smithi133 bash[105589]: audit 2023-10-09T05:08:10.039967+0000 mon.a (mon.0) 732 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:08:11.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:10 smithi194 bash[92666]: cluster 2023-10-09T05:08:09.710499+0000 mgr.y (mgr.25277) 585 : cluster 0 pgmap v537: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:11.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:10 smithi194 bash[92666]: audit 2023-10-09T05:08:10.039967+0000 mon.a (mon.0) 732 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:08:13.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:12 smithi133 bash[105589]: cluster 2023-10-09T05:08:11.711360+0000 mgr.y (mgr.25277) 586 : cluster 0 pgmap v538: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:13.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:12 smithi133 bash[102858]: cluster 2023-10-09T05:08:11.711360+0000 mgr.y (mgr.25277) 586 : cluster 0 pgmap v538: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:13.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:12 smithi194 bash[92666]: cluster 2023-10-09T05:08:11.711360+0000 mgr.y (mgr.25277) 586 : cluster 0 pgmap v538: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:13.414 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:08:14.481 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:08:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:08:14] "GET /metrics HTTP/1.1" 200 33988 "" "Prometheus/2.43.0" 2023-10-09T05:08:14.482 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:14 smithi133 bash[105589]: audit 2023-10-09T05:08:13.438816+0000 mon.a (mon.0) 733 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:08:14.482 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:14 smithi133 bash[105589]: cluster 2023-10-09T05:08:13.712119+0000 mgr.y (mgr.25277) 587 : cluster 0 pgmap v539: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:14.482 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:14 smithi133 bash[102858]: audit 2023-10-09T05:08:13.438816+0000 mon.a (mon.0) 733 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:08:14.483 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:14 smithi133 bash[102858]: cluster 2023-10-09T05:08:13.712119+0000 mgr.y (mgr.25277) 587 : cluster 0 pgmap v539: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:14.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:14 smithi194 bash[92666]: audit 2023-10-09T05:08:13.438816+0000 mon.a (mon.0) 733 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:08:14.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:14 smithi194 bash[92666]: cluster 2023-10-09T05:08:13.712119+0000 mgr.y (mgr.25277) 587 : cluster 0 pgmap v539: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:16.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:16 smithi133 bash[102858]: audit 2023-10-09T05:08:15.531537+0000 mon.a (mon.0) 734 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:08:16.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:16 smithi133 bash[102858]: audit 2023-10-09T05:08:15.539290+0000 mon.a (mon.0) 735 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:08:16.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:16 smithi133 bash[102858]: cluster 2023-10-09T05:08:15.712812+0000 mgr.y (mgr.25277) 588 : cluster 0 pgmap v540: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:16.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:16 smithi133 bash[102858]: audit 2023-10-09T05:08:16.442261+0000 mon.a (mon.0) 736 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:08:16.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:16 smithi133 bash[102858]: audit 2023-10-09T05:08:16.451399+0000 mon.a (mon.0) 737 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:08:16.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:16 smithi133 bash[105589]: audit 2023-10-09T05:08:15.531537+0000 mon.a (mon.0) 734 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:08:16.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:16 smithi133 bash[105589]: audit 2023-10-09T05:08:15.539290+0000 mon.a (mon.0) 735 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:08:16.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:16 smithi133 bash[105589]: cluster 2023-10-09T05:08:15.712812+0000 mgr.y (mgr.25277) 588 : cluster 0 pgmap v540: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:16.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:16 smithi133 bash[105589]: audit 2023-10-09T05:08:16.442261+0000 mon.a (mon.0) 736 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:08:16.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:16 smithi133 bash[105589]: audit 2023-10-09T05:08:16.451399+0000 mon.a (mon.0) 737 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:08:16.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:16 smithi194 bash[92666]: audit 2023-10-09T05:08:15.531537+0000 mon.a (mon.0) 734 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:08:16.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:16 smithi194 bash[92666]: audit 2023-10-09T05:08:15.539290+0000 mon.a (mon.0) 735 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:08:16.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:16 smithi194 bash[92666]: cluster 2023-10-09T05:08:15.712812+0000 mgr.y (mgr.25277) 588 : cluster 0 pgmap v540: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:16.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:16 smithi194 bash[92666]: audit 2023-10-09T05:08:16.442261+0000 mon.a (mon.0) 736 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:08:16.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:16 smithi194 bash[92666]: audit 2023-10-09T05:08:16.451399+0000 mon.a (mon.0) 737 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:08:16.920 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:08:16.922 INFO:teuthology.orchestra.run.smithi133.stderr:dumped all 2023-10-09T05:08:17.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:17 smithi133 bash[105589]: audit 2023-10-09T05:08:16.901844+0000 mgr.y (mgr.25277) 589 : audit 0 from='client.44738 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:08:17.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:17 smithi133 bash[105589]: audit 2023-10-09T05:08:16.972223+0000 mon.a (mon.0) 738 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:08:17.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:17 smithi133 bash[105589]: audit 2023-10-09T05:08:16.973578+0000 mon.a (mon.0) 739 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:08:17.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:17 smithi133 bash[105589]: audit 2023-10-09T05:08:16.985927+0000 mon.a (mon.0) 740 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:08:17.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:17 smithi133 bash[102858]: audit 2023-10-09T05:08:16.901844+0000 mgr.y (mgr.25277) 589 : audit 0 from='client.44738 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:08:17.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:17 smithi133 bash[102858]: audit 2023-10-09T05:08:16.972223+0000 mon.a (mon.0) 738 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:08:17.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:17 smithi133 bash[102858]: audit 2023-10-09T05:08:16.973578+0000 mon.a (mon.0) 739 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:08:17.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:17 smithi133 bash[102858]: audit 2023-10-09T05:08:16.985927+0000 mon.a (mon.0) 740 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:08:17.824 INFO:teuthology.orchestra.run.smithi133.stdout:{"pg_ready":true,"pg_map":{"version":540,"stamp":"2023-10-09T05:08:15.712348+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":538,"num_read_kb":964,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":222,"ondisk_log_size":222,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2663104,"kb_used_data":13696,"kb_used_omap":20,"kb_used_meta":2649323,"kb_avail":747298112,"statfs":{"total":767960285184,"available":765233266688,"internally_reserved":0,"allocated":14024704,"data_stored":8813245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2712907406},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004443"},"pg_stats":[{"pgid":"3.1a","version":"69'2","reported_seq":118,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129055+0000","last_change":"2023-10-09T05:01:02.663550+0000","last_active":"2023-10-09T05:02:55.129055+0000","last_peered":"2023-10-09T05:02:55.129055+0000","last_clean":"2023-10-09T05:02:55.129055+0000","last_became_active":"2023-10-09T05:01:02.663202+0000","last_became_peered":"2023-10-09T05:01:02.663202+0000","last_unstale":"2023-10-09T05:02:55.129055+0000","last_undegraded":"2023-10-09T05:02:55.129055+0000","last_fullsized":"2023-10-09T05:02:55.129055+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:30.743395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":86,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.914359+0000","last_change":"2023-10-09T05:02:09.656135+0000","last_active":"2023-10-09T05:06:11.914359+0000","last_peered":"2023-10-09T05:06:11.914359+0000","last_clean":"2023-10-09T05:06:11.914359+0000","last_became_active":"2023-10-09T05:02:09.654594+0000","last_became_peered":"2023-10-09T05:02:09.654594+0000","last_unstale":"2023-10-09T05:06:11.914359+0000","last_undegraded":"2023-10-09T05:06:11.914359+0000","last_fullsized":"2023-10-09T05:06:11.914359+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:20.024818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":83,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.744393+0000","last_change":"2023-10-09T05:02:09.649701+0000","last_active":"2023-10-09T05:06:11.744393+0000","last_peered":"2023-10-09T05:06:11.744393+0000","last_clean":"2023-10-09T05:06:11.744393+0000","last_became_active":"2023-10-09T05:02:09.649370+0000","last_became_peered":"2023-10-09T05:02:09.649370+0000","last_unstale":"2023-10-09T05:06:11.744393+0000","last_undegraded":"2023-10-09T05:06:11.744393+0000","last_fullsized":"2023-10-09T05:06:11.744393+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:45.255492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":92,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.927998+0000","last_change":"2023-10-09T05:02:09.648283+0000","last_active":"2023-10-09T05:06:11.927998+0000","last_peered":"2023-10-09T05:06:11.927998+0000","last_clean":"2023-10-09T05:06:11.927998+0000","last_became_active":"2023-10-09T05:02:09.647778+0000","last_became_peered":"2023-10-09T05:02:09.647778+0000","last_unstale":"2023-10-09T05:06:11.927998+0000","last_undegraded":"2023-10-09T05:06:11.927998+0000","last_fullsized":"2023-10-09T05:06:11.927998+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:24.582392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":104,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.915863+0000","last_change":"2023-10-09T05:02:09.657157+0000","last_active":"2023-10-09T05:06:11.915863+0000","last_peered":"2023-10-09T05:06:11.915863+0000","last_clean":"2023-10-09T05:06:11.915863+0000","last_became_active":"2023-10-09T05:02:09.656354+0000","last_became_peered":"2023-10-09T05:02:09.656354+0000","last_unstale":"2023-10-09T05:06:11.915863+0000","last_undegraded":"2023-10-09T05:06:11.915863+0000","last_fullsized":"2023-10-09T05:06:11.915863+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:57.512482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.155272+0000","last_change":"2023-10-09T04:57:42.011855+0000","last_active":"2023-10-09T05:02:55.155272+0000","last_peered":"2023-10-09T05:02:55.155272+0000","last_clean":"2023-10-09T05:02:55.155272+0000","last_became_active":"2023-10-09T04:57:42.011233+0000","last_became_peered":"2023-10-09T04:57:42.011233+0000","last_unstale":"2023-10-09T05:02:55.155272+0000","last_undegraded":"2023-10-09T05:02:55.155272+0000","last_fullsized":"2023-10-09T05:02:55.155272+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:18.585929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":90,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.917521+0000","last_change":"2023-10-09T05:02:09.646765+0000","last_active":"2023-10-09T05:06:11.917521+0000","last_peered":"2023-10-09T05:06:11.917521+0000","last_clean":"2023-10-09T05:06:11.917521+0000","last_became_active":"2023-10-09T05:02:09.645141+0000","last_became_peered":"2023-10-09T05:02:09.645141+0000","last_unstale":"2023-10-09T05:06:11.917521+0000","last_undegraded":"2023-10-09T05:06:11.917521+0000","last_fullsized":"2023-10-09T05:06:11.917521+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:10.098578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":86,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.915378+0000","last_change":"2023-10-09T05:02:09.656872+0000","last_active":"2023-10-09T05:06:11.915378+0000","last_peered":"2023-10-09T05:06:11.915378+0000","last_clean":"2023-10-09T05:06:11.915378+0000","last_became_active":"2023-10-09T05:02:09.654713+0000","last_became_peered":"2023-10-09T05:02:09.654713+0000","last_unstale":"2023-10-09T05:06:11.915378+0000","last_undegraded":"2023-10-09T05:06:11.915378+0000","last_fullsized":"2023-10-09T05:06:11.915378+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:00:17.539155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":99,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.133911+0000","last_change":"2023-10-09T04:57:42.023522+0000","last_active":"2023-10-09T05:02:55.133911+0000","last_peered":"2023-10-09T05:02:55.133911+0000","last_clean":"2023-10-09T05:02:55.133911+0000","last_became_active":"2023-10-09T04:57:42.023010+0000","last_became_peered":"2023-10-09T04:57:42.023010+0000","last_unstale":"2023-10-09T05:02:55.133911+0000","last_undegraded":"2023-10-09T05:02:55.133911+0000","last_fullsized":"2023-10-09T05:02:55.133911+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:44.280677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":81,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.914646+0000","last_change":"2023-10-09T05:02:09.651167+0000","last_active":"2023-10-09T05:06:11.914646+0000","last_peered":"2023-10-09T05:06:11.914646+0000","last_clean":"2023-10-09T05:06:11.914646+0000","last_became_active":"2023-10-09T05:02:09.650419+0000","last_became_peered":"2023-10-09T05:02:09.650419+0000","last_unstale":"2023-10-09T05:06:11.914646+0000","last_undegraded":"2023-10-09T05:06:11.914646+0000","last_fullsized":"2023-10-09T05:06:11.914646+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:55:16.966938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":104,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.148854+0000","last_change":"2023-10-09T05:02:09.649700+0000","last_active":"2023-10-09T05:02:55.148854+0000","last_peered":"2023-10-09T05:02:55.148854+0000","last_clean":"2023-10-09T05:02:55.148854+0000","last_became_active":"2023-10-09T05:02:09.649317+0000","last_became_peered":"2023-10-09T05:02:09.649317+0000","last_unstale":"2023-10-09T05:02:55.148854+0000","last_undegraded":"2023-10-09T05:02:55.148854+0000","last_fullsized":"2023-10-09T05:02:55.148854+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:50.711767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":83,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.746974+0000","last_change":"2023-10-09T05:01:02.653598+0000","last_active":"2023-10-09T05:06:11.746974+0000","last_peered":"2023-10-09T05:06:11.746974+0000","last_clean":"2023-10-09T05:06:11.746974+0000","last_became_active":"2023-10-09T05:01:02.653351+0000","last_became_peered":"2023-10-09T05:01:02.653351+0000","last_unstale":"2023-10-09T05:06:11.746974+0000","last_undegraded":"2023-10-09T05:06:11.746974+0000","last_fullsized":"2023-10-09T05:06:11.746974+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:09.106185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'23","reported_seq":110,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129315+0000","last_change":"2023-10-09T04:44:52.781203+0000","last_active":"2023-10-09T05:02:55.129315+0000","last_peered":"2023-10-09T05:02:55.129315+0000","last_clean":"2023-10-09T05:02:55.129315+0000","last_became_active":"2023-10-09T04:44:52.780902+0000","last_became_peered":"2023-10-09T04:44:52.780902+0000","last_unstale":"2023-10-09T05:02:55.129315+0000","last_undegraded":"2023-10-09T05:02:55.129315+0000","last_fullsized":"2023-10-09T05:02:55.129315+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:56.332974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129435+0000","last_change":"2023-10-09T04:57:42.024993+0000","last_active":"2023-10-09T05:02:55.129435+0000","last_peered":"2023-10-09T05:02:55.129435+0000","last_clean":"2023-10-09T05:02:55.129435+0000","last_became_active":"2023-10-09T04:57:42.024227+0000","last_became_peered":"2023-10-09T04:57:42.024227+0000","last_unstale":"2023-10-09T05:02:55.129435+0000","last_undegraded":"2023-10-09T05:02:55.129435+0000","last_fullsized":"2023-10-09T05:02:55.129435+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:02.031798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":103,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.148581+0000","last_change":"2023-10-09T05:01:02.669186+0000","last_active":"2023-10-09T05:02:55.148581+0000","last_peered":"2023-10-09T05:02:55.148581+0000","last_clean":"2023-10-09T05:02:55.148581+0000","last_became_active":"2023-10-09T05:01:02.668739+0000","last_became_peered":"2023-10-09T05:01:02.668739+0000","last_unstale":"2023-10-09T05:02:55.148581+0000","last_undegraded":"2023-10-09T05:02:55.148581+0000","last_fullsized":"2023-10-09T05:02:55.148581+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:41.352624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":86,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.915825+0000","last_change":"2023-10-09T05:02:09.656245+0000","last_active":"2023-10-09T05:06:11.915825+0000","last_peered":"2023-10-09T05:06:11.915825+0000","last_clean":"2023-10-09T05:06:11.915825+0000","last_became_active":"2023-10-09T05:02:09.654672+0000","last_became_peered":"2023-10-09T05:02:09.654672+0000","last_unstale":"2023-10-09T05:06:11.915825+0000","last_undegraded":"2023-10-09T05:06:11.915825+0000","last_fullsized":"2023-10-09T05:06:11.915825+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:54.640349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":90,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.917907+0000","last_change":"2023-10-09T05:02:09.645017+0000","last_active":"2023-10-09T05:06:11.917907+0000","last_peered":"2023-10-09T05:06:11.917907+0000","last_clean":"2023-10-09T05:06:11.917907+0000","last_became_active":"2023-10-09T05:02:09.644705+0000","last_became_peered":"2023-10-09T05:02:09.644705+0000","last_unstale":"2023-10-09T05:06:11.917907+0000","last_undegraded":"2023-10-09T05:06:11.917907+0000","last_fullsized":"2023-10-09T05:06:11.917907+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:20.695961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.133929+0000","last_change":"2023-10-09T05:02:09.648362+0000","last_active":"2023-10-09T05:02:55.133929+0000","last_peered":"2023-10-09T05:02:55.133929+0000","last_clean":"2023-10-09T05:02:55.133929+0000","last_became_active":"2023-10-09T05:02:09.647127+0000","last_became_peered":"2023-10-09T05:02:09.647127+0000","last_unstale":"2023-10-09T05:02:55.133929+0000","last_undegraded":"2023-10-09T05:02:55.133929+0000","last_fullsized":"2023-10-09T05:02:55.133929+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:38:02.875613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":82,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.148137+0000","last_change":"2023-10-09T04:44:52.792557+0000","last_active":"2023-10-09T05:02:55.148137+0000","last_peered":"2023-10-09T05:02:55.148137+0000","last_clean":"2023-10-09T05:02:55.148137+0000","last_became_active":"2023-10-09T04:44:52.791890+0000","last_became_peered":"2023-10-09T04:44:52.791890+0000","last_unstale":"2023-10-09T05:02:55.148137+0000","last_undegraded":"2023-10-09T05:02:55.148137+0000","last_fullsized":"2023-10-09T05:02:55.148137+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:04:46.379852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":91,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.928065+0000","last_change":"2023-10-09T05:02:09.652158+0000","last_active":"2023-10-09T05:06:11.928065+0000","last_peered":"2023-10-09T05:06:11.928065+0000","last_clean":"2023-10-09T05:06:11.928065+0000","last_became_active":"2023-10-09T05:02:09.651811+0000","last_became_peered":"2023-10-09T05:02:09.651811+0000","last_unstale":"2023-10-09T05:06:11.928065+0000","last_undegraded":"2023-10-09T05:06:11.928065+0000","last_fullsized":"2023-10-09T05:06:11.928065+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:49:10.141237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":111,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.154425+0000","last_change":"2023-10-09T04:56:56.076177+0000","last_active":"2023-10-09T05:02:55.154425+0000","last_peered":"2023-10-09T05:02:55.154425+0000","last_clean":"2023-10-09T05:02:55.154425+0000","last_became_active":"2023-10-09T04:56:56.075564+0000","last_became_peered":"2023-10-09T04:56:56.075564+0000","last_unstale":"2023-10-09T05:02:55.154425+0000","last_undegraded":"2023-10-09T05:02:55.154425+0000","last_fullsized":"2023-10-09T05:02:55.154425+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:38.390793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":86,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.915451+0000","last_change":"2023-10-09T05:02:09.656286+0000","last_active":"2023-10-09T05:06:11.915451+0000","last_peered":"2023-10-09T05:06:11.915451+0000","last_clean":"2023-10-09T05:06:11.915451+0000","last_became_active":"2023-10-09T05:02:09.654429+0000","last_became_peered":"2023-10-09T05:02:09.654429+0000","last_unstale":"2023-10-09T05:06:11.915451+0000","last_undegraded":"2023-10-09T05:06:11.915451+0000","last_fullsized":"2023-10-09T05:06:11.915451+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:55:05.953948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":101,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.149120+0000","last_change":"2023-10-09T04:56:56.075307+0000","last_active":"2023-10-09T05:02:55.149120+0000","last_peered":"2023-10-09T05:02:55.149120+0000","last_clean":"2023-10-09T05:02:55.149120+0000","last_became_active":"2023-10-09T04:56:56.074831+0000","last_became_peered":"2023-10-09T04:56:56.074831+0000","last_unstale":"2023-10-09T05:02:55.149120+0000","last_undegraded":"2023-10-09T05:02:55.149120+0000","last_fullsized":"2023-10-09T05:02:55.149120+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:51:56.867759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":81,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.134283+0000","last_change":"2023-10-09T04:44:53.962437+0000","last_active":"2023-10-09T05:02:55.134283+0000","last_peered":"2023-10-09T05:02:55.134283+0000","last_clean":"2023-10-09T05:02:55.134283+0000","last_became_active":"2023-10-09T04:44:53.961810+0000","last_became_peered":"2023-10-09T04:44:53.961810+0000","last_unstale":"2023-10-09T05:02:55.134283+0000","last_undegraded":"2023-10-09T05:02:55.134283+0000","last_fullsized":"2023-10-09T05:02:55.134283+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:56.436933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":102,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.148994+0000","last_change":"2023-10-09T04:57:42.023336+0000","last_active":"2023-10-09T05:02:55.148994+0000","last_peered":"2023-10-09T05:02:55.148994+0000","last_clean":"2023-10-09T05:02:55.148994+0000","last_became_active":"2023-10-09T04:57:42.022861+0000","last_became_peered":"2023-10-09T04:57:42.022861+0000","last_unstale":"2023-10-09T05:02:55.148994+0000","last_undegraded":"2023-10-09T05:02:55.148994+0000","last_fullsized":"2023-10-09T05:02:55.148994+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:57.858993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":115,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.128925+0000","last_change":"2023-10-09T05:02:09.653562+0000","last_active":"2023-10-09T05:02:55.128925+0000","last_peered":"2023-10-09T05:02:55.128925+0000","last_clean":"2023-10-09T05:02:55.128925+0000","last_became_active":"2023-10-09T05:02:09.652830+0000","last_became_peered":"2023-10-09T05:02:09.652830+0000","last_unstale":"2023-10-09T05:02:55.128925+0000","last_undegraded":"2023-10-09T05:02:55.128925+0000","last_fullsized":"2023-10-09T05:02:55.128925+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:54.527011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":84,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.148487+0000","last_change":"2023-10-09T04:44:52.787469+0000","last_active":"2023-10-09T05:02:55.148487+0000","last_peered":"2023-10-09T05:02:55.148487+0000","last_clean":"2023-10-09T05:02:55.148487+0000","last_became_active":"2023-10-09T04:44:52.787148+0000","last_became_peered":"2023-10-09T04:44:52.787148+0000","last_unstale":"2023-10-09T05:02:55.148487+0000","last_undegraded":"2023-10-09T05:02:55.148487+0000","last_fullsized":"2023-10-09T05:02:55.148487+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:12.469065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":108,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.154612+0000","last_change":"2023-10-09T05:01:02.659380+0000","last_active":"2023-10-09T05:02:55.154612+0000","last_peered":"2023-10-09T05:02:55.154612+0000","last_clean":"2023-10-09T05:02:55.154612+0000","last_became_active":"2023-10-09T05:01:02.659064+0000","last_became_peered":"2023-10-09T05:01:02.659064+0000","last_unstale":"2023-10-09T05:02:55.154612+0000","last_undegraded":"2023-10-09T05:02:55.154612+0000","last_fullsized":"2023-10-09T05:02:55.154612+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:35.342503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":99,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.133928+0000","last_change":"2023-10-09T05:02:09.653932+0000","last_active":"2023-10-09T05:02:55.133928+0000","last_peered":"2023-10-09T05:02:55.133928+0000","last_clean":"2023-10-09T05:02:55.133928+0000","last_became_active":"2023-10-09T05:02:09.652923+0000","last_became_peered":"2023-10-09T05:02:09.652923+0000","last_unstale":"2023-10-09T05:02:55.133928+0000","last_undegraded":"2023-10-09T05:02:55.133928+0000","last_fullsized":"2023-10-09T05:02:55.133928+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:44:11.928237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":92,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.917044+0000","last_change":"2023-10-09T04:57:42.020009+0000","last_active":"2023-10-09T05:06:11.917044+0000","last_peered":"2023-10-09T05:06:11.917044+0000","last_clean":"2023-10-09T05:06:11.917044+0000","last_became_active":"2023-10-09T04:57:42.019678+0000","last_became_peered":"2023-10-09T04:57:42.019678+0000","last_unstale":"2023-10-09T05:06:11.917044+0000","last_undegraded":"2023-10-09T05:06:11.917044+0000","last_fullsized":"2023-10-09T05:06:11.917044+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:16.812128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'3","reported_seq":83,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.914244+0000","last_change":"2023-10-09T05:02:09.650579+0000","last_active":"2023-10-09T05:06:11.914244+0000","last_peered":"2023-10-09T05:06:11.914244+0000","last_clean":"2023-10-09T05:06:11.914244+0000","last_became_active":"2023-10-09T05:02:09.650160+0000","last_became_peered":"2023-10-09T05:02:09.650160+0000","last_unstale":"2023-10-09T05:06:11.914244+0000","last_undegraded":"2023-10-09T05:06:11.914244+0000","last_fullsized":"2023-10-09T05:06:11.914244+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:18.292670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129404+0000","last_change":"2023-10-09T05:02:09.652449+0000","last_active":"2023-10-09T05:02:55.129404+0000","last_peered":"2023-10-09T05:02:55.129404+0000","last_clean":"2023-10-09T05:02:55.129404+0000","last_became_active":"2023-10-09T05:02:09.651994+0000","last_became_peered":"2023-10-09T05:02:09.651994+0000","last_unstale":"2023-10-09T05:02:55.129404+0000","last_undegraded":"2023-10-09T05:02:55.129404+0000","last_fullsized":"2023-10-09T05:02:55.129404+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:14.480513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":99,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.154853+0000","last_change":"2023-10-09T04:57:42.016620+0000","last_active":"2023-10-09T05:02:55.154853+0000","last_peered":"2023-10-09T05:02:55.154853+0000","last_clean":"2023-10-09T05:02:55.154853+0000","last_became_active":"2023-10-09T04:57:42.016215+0000","last_became_peered":"2023-10-09T04:57:42.016215+0000","last_unstale":"2023-10-09T05:02:55.154853+0000","last_undegraded":"2023-10-09T05:02:55.154853+0000","last_fullsized":"2023-10-09T05:02:55.154853+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:21:24.361501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":115,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129567+0000","last_change":"2023-10-09T05:01:02.661970+0000","last_active":"2023-10-09T05:02:55.129567+0000","last_peered":"2023-10-09T05:02:55.129567+0000","last_clean":"2023-10-09T05:02:55.129567+0000","last_became_active":"2023-10-09T05:01:02.661646+0000","last_became_peered":"2023-10-09T05:01:02.661646+0000","last_unstale":"2023-10-09T05:02:55.129567+0000","last_undegraded":"2023-10-09T05:02:55.129567+0000","last_fullsized":"2023-10-09T05:02:55.129567+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:12.013587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":99,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.128906+0000","last_change":"2023-10-09T05:01:02.663309+0000","last_active":"2023-10-09T05:02:55.128906+0000","last_peered":"2023-10-09T05:02:55.128906+0000","last_clean":"2023-10-09T05:02:55.128906+0000","last_became_active":"2023-10-09T05:01:02.663038+0000","last_became_peered":"2023-10-09T05:01:02.663038+0000","last_unstale":"2023-10-09T05:02:55.128906+0000","last_undegraded":"2023-10-09T05:02:55.128906+0000","last_fullsized":"2023-10-09T05:02:55.128906+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:29:36.689304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129594+0000","last_change":"2023-10-09T04:57:42.028346+0000","last_active":"2023-10-09T05:02:55.129594+0000","last_peered":"2023-10-09T05:02:55.129594+0000","last_clean":"2023-10-09T05:02:55.129594+0000","last_became_active":"2023-10-09T04:57:42.027881+0000","last_became_peered":"2023-10-09T04:57:42.027881+0000","last_unstale":"2023-10-09T05:02:55.129594+0000","last_undegraded":"2023-10-09T05:02:55.129594+0000","last_fullsized":"2023-10-09T05:02:55.129594+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:00:43.197716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":87,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.916032+0000","last_change":"2023-10-09T05:02:09.656904+0000","last_active":"2023-10-09T05:06:11.916032+0000","last_peered":"2023-10-09T05:06:11.916032+0000","last_clean":"2023-10-09T05:06:11.916032+0000","last_became_active":"2023-10-09T05:02:09.654658+0000","last_became_peered":"2023-10-09T05:02:09.654658+0000","last_unstale":"2023-10-09T05:06:11.916032+0000","last_undegraded":"2023-10-09T05:06:11.916032+0000","last_fullsized":"2023-10-09T05:06:11.916032+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:51:26.666116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":83,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.745701+0000","last_change":"2023-10-09T05:02:09.652086+0000","last_active":"2023-10-09T05:06:11.745701+0000","last_peered":"2023-10-09T05:06:11.745701+0000","last_clean":"2023-10-09T05:06:11.745701+0000","last_became_active":"2023-10-09T05:02:09.651733+0000","last_became_peered":"2023-10-09T05:02:09.651733+0000","last_unstale":"2023-10-09T05:06:11.745701+0000","last_undegraded":"2023-10-09T05:06:11.745701+0000","last_fullsized":"2023-10-09T05:06:11.745701+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:32:59.894793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":113,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.744561+0000","last_change":"2023-10-09T04:57:42.019588+0000","last_active":"2023-10-09T05:06:11.744561+0000","last_peered":"2023-10-09T05:06:11.744561+0000","last_clean":"2023-10-09T05:06:11.744561+0000","last_became_active":"2023-10-09T04:57:42.019274+0000","last_became_peered":"2023-10-09T04:57:42.019274+0000","last_unstale":"2023-10-09T05:06:11.744561+0000","last_undegraded":"2023-10-09T05:06:11.744561+0000","last_fullsized":"2023-10-09T05:06:11.744561+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:21.969343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":116,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.134140+0000","last_change":"2023-10-09T05:02:09.649559+0000","last_active":"2023-10-09T05:02:55.134140+0000","last_peered":"2023-10-09T05:02:55.134140+0000","last_clean":"2023-10-09T05:02:55.134140+0000","last_became_active":"2023-10-09T05:02:09.649195+0000","last_became_peered":"2023-10-09T05:02:09.649195+0000","last_unstale":"2023-10-09T05:02:55.134140+0000","last_undegraded":"2023-10-09T05:02:55.134140+0000","last_fullsized":"2023-10-09T05:02:55.134140+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:56.864170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":69,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.746089+0000","last_change":"2023-10-09T04:57:42.022222+0000","last_active":"2023-10-09T05:06:11.746089+0000","last_peered":"2023-10-09T05:06:11.746089+0000","last_clean":"2023-10-09T05:06:11.746089+0000","last_became_active":"2023-10-09T04:57:42.021893+0000","last_became_peered":"2023-10-09T04:57:42.021893+0000","last_unstale":"2023-10-09T05:06:11.746089+0000","last_undegraded":"2023-10-09T05:06:11.746089+0000","last_fullsized":"2023-10-09T05:06:11.746089+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:39:20.734382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":103,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.133781+0000","last_change":"2023-10-09T04:57:42.010576+0000","last_active":"2023-10-09T05:02:55.133781+0000","last_peered":"2023-10-09T05:02:55.133781+0000","last_clean":"2023-10-09T05:02:55.133781+0000","last_became_active":"2023-10-09T04:57:42.010145+0000","last_became_peered":"2023-10-09T04:57:42.010145+0000","last_unstale":"2023-10-09T05:02:55.133781+0000","last_undegraded":"2023-10-09T05:02:55.133781+0000","last_fullsized":"2023-10-09T05:02:55.133781+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:18:17.196612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":99,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129297+0000","last_change":"2023-10-09T04:56:56.087320+0000","last_active":"2023-10-09T05:02:55.129297+0000","last_peered":"2023-10-09T05:02:55.129297+0000","last_clean":"2023-10-09T05:02:55.129297+0000","last_became_active":"2023-10-09T04:56:56.086552+0000","last_became_peered":"2023-10-09T04:56:56.086552+0000","last_unstale":"2023-10-09T05:02:55.129297+0000","last_undegraded":"2023-10-09T05:02:55.129297+0000","last_fullsized":"2023-10-09T05:02:55.129297+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:14.205288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":111,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.134027+0000","last_change":"2023-10-09T05:02:09.653655+0000","last_active":"2023-10-09T05:02:55.134027+0000","last_peered":"2023-10-09T05:02:55.134027+0000","last_clean":"2023-10-09T05:02:55.134027+0000","last_became_active":"2023-10-09T05:02:09.652682+0000","last_became_peered":"2023-10-09T05:02:09.652682+0000","last_unstale":"2023-10-09T05:02:55.134027+0000","last_undegraded":"2023-10-09T05:02:55.134027+0000","last_fullsized":"2023-10-09T05:02:55.134027+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:21:28.150022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":116,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.748439+0000","last_change":"2023-10-09T05:02:09.645835+0000","last_active":"2023-10-09T05:06:11.748439+0000","last_peered":"2023-10-09T05:06:11.748439+0000","last_clean":"2023-10-09T05:06:11.748439+0000","last_became_active":"2023-10-09T05:02:09.645491+0000","last_became_peered":"2023-10-09T05:02:09.645491+0000","last_unstale":"2023-10-09T05:06:11.748439+0000","last_undegraded":"2023-10-09T05:06:11.748439+0000","last_fullsized":"2023-10-09T05:06:11.748439+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:06:20.823201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":84,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.746192+0000","last_change":"2023-10-09T05:01:02.656091+0000","last_active":"2023-10-09T05:06:11.746192+0000","last_peered":"2023-10-09T05:06:11.746192+0000","last_clean":"2023-10-09T05:06:11.746192+0000","last_became_active":"2023-10-09T05:01:02.654755+0000","last_became_peered":"2023-10-09T05:01:02.654755+0000","last_unstale":"2023-10-09T05:06:11.746192+0000","last_undegraded":"2023-10-09T05:06:11.746192+0000","last_fullsized":"2023-10-09T05:06:11.746192+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:07.144598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":92,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.749112+0000","last_change":"2023-10-09T05:01:02.667904+0000","last_active":"2023-10-09T05:06:11.749112+0000","last_peered":"2023-10-09T05:06:11.749112+0000","last_clean":"2023-10-09T05:06:11.749112+0000","last_became_active":"2023-10-09T05:01:02.667574+0000","last_became_peered":"2023-10-09T05:01:02.667574+0000","last_unstale":"2023-10-09T05:06:11.749112+0000","last_undegraded":"2023-10-09T05:06:11.749112+0000","last_fullsized":"2023-10-09T05:06:11.749112+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:48:50.643600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":103,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.133545+0000","last_change":"2023-10-09T05:01:02.685038+0000","last_active":"2023-10-09T05:02:55.133545+0000","last_peered":"2023-10-09T05:02:55.133545+0000","last_clean":"2023-10-09T05:02:55.133545+0000","last_became_active":"2023-10-09T05:01:02.683622+0000","last_became_peered":"2023-10-09T05:01:02.683622+0000","last_unstale":"2023-10-09T05:02:55.133545+0000","last_undegraded":"2023-10-09T05:02:55.133545+0000","last_fullsized":"2023-10-09T05:02:55.133545+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:35.435592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":476,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.151016+0000","last_change":"2023-10-09T04:56:56.075762+0000","last_active":"2023-10-09T05:02:55.151016+0000","last_peered":"2023-10-09T05:02:55.151016+0000","last_clean":"2023-10-09T05:02:55.151016+0000","last_became_active":"2023-10-09T04:56:56.075263+0000","last_became_peered":"2023-10-09T04:56:56.075263+0000","last_unstale":"2023-10-09T05:02:55.151016+0000","last_undegraded":"2023-10-09T05:02:55.151016+0000","last_fullsized":"2023-10-09T05:02:55.151016+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_clean_scrub_stamp":"2023-10-09T04:36:44.261013+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:02.769164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":81,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.914252+0000","last_change":"2023-10-09T05:02:09.654237+0000","last_active":"2023-10-09T05:06:11.914252+0000","last_peered":"2023-10-09T05:06:11.914252+0000","last_clean":"2023-10-09T05:06:11.914252+0000","last_became_active":"2023-10-09T05:02:09.653909+0000","last_became_peered":"2023-10-09T05:02:09.653909+0000","last_unstale":"2023-10-09T05:06:11.914252+0000","last_undegraded":"2023-10-09T05:06:11.914252+0000","last_fullsized":"2023-10-09T05:06:11.914252+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:02.002366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":81,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.154930+0000","last_change":"2023-10-09T04:44:53.959705+0000","last_active":"2023-10-09T05:02:55.154930+0000","last_peered":"2023-10-09T05:02:55.154930+0000","last_clean":"2023-10-09T05:02:55.154930+0000","last_became_active":"2023-10-09T04:44:53.959092+0000","last_became_peered":"2023-10-09T04:44:53.959092+0000","last_unstale":"2023-10-09T05:02:55.154930+0000","last_undegraded":"2023-10-09T05:02:55.154930+0000","last_fullsized":"2023-10-09T05:02:55.154930+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:31:29.189897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":120,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.151119+0000","last_change":"2023-10-09T04:57:42.023353+0000","last_active":"2023-10-09T05:02:55.151119+0000","last_peered":"2023-10-09T05:02:55.151119+0000","last_clean":"2023-10-09T05:02:55.151119+0000","last_became_active":"2023-10-09T04:57:42.023007+0000","last_became_peered":"2023-10-09T04:57:42.023007+0000","last_unstale":"2023-10-09T05:02:55.151119+0000","last_undegraded":"2023-10-09T05:02:55.151119+0000","last_fullsized":"2023-10-09T05:02:55.151119+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:43.715250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":91,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.914248+0000","last_change":"2023-10-09T05:02:09.648781+0000","last_active":"2023-10-09T05:06:11.914248+0000","last_peered":"2023-10-09T05:06:11.914248+0000","last_clean":"2023-10-09T05:06:11.914248+0000","last_became_active":"2023-10-09T05:02:09.648078+0000","last_became_peered":"2023-10-09T05:02:09.648078+0000","last_unstale":"2023-10-09T05:06:11.914248+0000","last_undegraded":"2023-10-09T05:06:11.914248+0000","last_fullsized":"2023-10-09T05:06:11.914248+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:35.287091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":107,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.927800+0000","last_change":"2023-10-09T05:01:02.653869+0000","last_active":"2023-10-09T05:06:11.927800+0000","last_peered":"2023-10-09T05:06:11.927800+0000","last_clean":"2023-10-09T05:06:11.927800+0000","last_became_active":"2023-10-09T05:01:02.653493+0000","last_became_peered":"2023-10-09T05:01:02.653493+0000","last_unstale":"2023-10-09T05:06:11.927800+0000","last_undegraded":"2023-10-09T05:06:11.927800+0000","last_fullsized":"2023-10-09T05:06:11.927800+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:40:46.722386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":103,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.148680+0000","last_change":"2023-10-09T04:56:56.073473+0000","last_active":"2023-10-09T05:02:55.148680+0000","last_peered":"2023-10-09T05:02:55.148680+0000","last_clean":"2023-10-09T05:02:55.148680+0000","last_became_active":"2023-10-09T04:56:56.073146+0000","last_became_peered":"2023-10-09T04:56:56.073146+0000","last_unstale":"2023-10-09T05:02:55.148680+0000","last_undegraded":"2023-10-09T05:02:55.148680+0000","last_fullsized":"2023-10-09T05:02:55.148680+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:55:25.132822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":84,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.744812+0000","last_change":"2023-10-09T05:01:02.664643+0000","last_active":"2023-10-09T05:06:11.744812+0000","last_peered":"2023-10-09T05:06:11.744812+0000","last_clean":"2023-10-09T05:06:11.744812+0000","last_became_active":"2023-10-09T05:01:02.664288+0000","last_became_peered":"2023-10-09T05:01:02.664288+0000","last_unstale":"2023-10-09T05:06:11.744812+0000","last_undegraded":"2023-10-09T05:06:11.744812+0000","last_fullsized":"2023-10-09T05:06:11.744812+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:01.572876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":77,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.916855+0000","last_change":"2023-10-09T04:56:56.071763+0000","last_active":"2023-10-09T05:06:11.916855+0000","last_peered":"2023-10-09T05:06:11.916855+0000","last_clean":"2023-10-09T05:06:11.916855+0000","last_became_active":"2023-10-09T04:56:56.071429+0000","last_became_peered":"2023-10-09T04:56:56.071429+0000","last_unstale":"2023-10-09T05:06:11.916855+0000","last_undegraded":"2023-10-09T05:06:11.916855+0000","last_fullsized":"2023-10-09T05:06:11.916855+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:53:02.051460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":82,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.915549+0000","last_change":"2023-10-09T05:02:09.651149+0000","last_active":"2023-10-09T05:06:11.915549+0000","last_peered":"2023-10-09T05:06:11.915549+0000","last_clean":"2023-10-09T05:06:11.915549+0000","last_became_active":"2023-10-09T05:02:09.650151+0000","last_became_peered":"2023-10-09T05:02:09.650151+0000","last_unstale":"2023-10-09T05:06:11.915549+0000","last_undegraded":"2023-10-09T05:06:11.915549+0000","last_fullsized":"2023-10-09T05:06:11.915549+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:49.100209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":67,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.744947+0000","last_change":"2023-10-09T04:57:42.023683+0000","last_active":"2023-10-09T05:06:11.744947+0000","last_peered":"2023-10-09T05:06:11.744947+0000","last_clean":"2023-10-09T05:06:11.744947+0000","last_became_active":"2023-10-09T04:57:42.023371+0000","last_became_peered":"2023-10-09T04:57:42.023371+0000","last_unstale":"2023-10-09T05:06:11.744947+0000","last_undegraded":"2023-10-09T05:06:11.744947+0000","last_fullsized":"2023-10-09T05:06:11.744947+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:17:29.936222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":79,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.915934+0000","last_change":"2023-10-09T05:02:09.650750+0000","last_active":"2023-10-09T05:06:11.915934+0000","last_peered":"2023-10-09T05:06:11.915934+0000","last_clean":"2023-10-09T05:06:11.915934+0000","last_became_active":"2023-10-09T05:02:09.650351+0000","last_became_peered":"2023-10-09T05:02:09.650351+0000","last_unstale":"2023-10-09T05:06:11.915934+0000","last_undegraded":"2023-10-09T05:06:11.915934+0000","last_fullsized":"2023-10-09T05:06:11.915934+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:14.826132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":103,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.155202+0000","last_change":"2023-10-09T05:01:02.660775+0000","last_active":"2023-10-09T05:02:55.155202+0000","last_peered":"2023-10-09T05:02:55.155202+0000","last_clean":"2023-10-09T05:02:55.155202+0000","last_became_active":"2023-10-09T05:01:02.660420+0000","last_became_peered":"2023-10-09T05:01:02.660420+0000","last_unstale":"2023-10-09T05:02:55.155202+0000","last_undegraded":"2023-10-09T05:02:55.155202+0000","last_fullsized":"2023-10-09T05:02:55.155202+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:54.031869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":96,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.927896+0000","last_change":"2023-10-09T04:57:42.017497+0000","last_active":"2023-10-09T05:06:11.927896+0000","last_peered":"2023-10-09T05:06:11.927896+0000","last_clean":"2023-10-09T05:06:11.927896+0000","last_became_active":"2023-10-09T04:57:42.017165+0000","last_became_peered":"2023-10-09T04:57:42.017165+0000","last_unstale":"2023-10-09T05:06:11.927896+0000","last_undegraded":"2023-10-09T05:06:11.927896+0000","last_fullsized":"2023-10-09T05:06:11.927896+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:43.258960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":81,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.128729+0000","last_change":"2023-10-09T04:44:53.792412+0000","last_active":"2023-10-09T05:02:55.128729+0000","last_peered":"2023-10-09T05:02:55.128729+0000","last_clean":"2023-10-09T05:02:55.128729+0000","last_became_active":"2023-10-09T04:44:53.792009+0000","last_became_peered":"2023-10-09T04:44:53.792009+0000","last_unstale":"2023-10-09T05:02:55.128729+0000","last_undegraded":"2023-10-09T05:02:55.128729+0000","last_fullsized":"2023-10-09T05:02:55.128729+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:39.423339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'2","reported_seq":81,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.917224+0000","last_change":"2023-10-09T04:56:56.079768+0000","last_active":"2023-10-09T05:06:11.917224+0000","last_peered":"2023-10-09T05:06:11.917224+0000","last_clean":"2023-10-09T05:06:11.917224+0000","last_became_active":"2023-10-09T04:56:56.077721+0000","last_became_peered":"2023-10-09T04:56:56.077721+0000","last_unstale":"2023-10-09T05:06:11.917224+0000","last_undegraded":"2023-10-09T05:06:11.917224+0000","last_fullsized":"2023-10-09T05:06:11.917224+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:36.384357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":118,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:06:00.911094+0000","last_change":"2023-10-09T05:02:09.644679+0000","last_active":"2023-10-09T05:06:00.911094+0000","last_peered":"2023-10-09T05:06:00.911094+0000","last_clean":"2023-10-09T05:06:00.911094+0000","last_became_active":"2023-10-09T05:02:09.644335+0000","last_became_peered":"2023-10-09T05:02:09.644335+0000","last_unstale":"2023-10-09T05:06:00.911094+0000","last_undegraded":"2023-10-09T05:06:00.911094+0000","last_fullsized":"2023-10-09T05:06:00.911094+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:06.142246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":102,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.134135+0000","last_change":"2023-10-09T04:57:42.018276+0000","last_active":"2023-10-09T05:02:55.134135+0000","last_peered":"2023-10-09T05:02:55.134135+0000","last_clean":"2023-10-09T05:02:55.134135+0000","last_became_active":"2023-10-09T04:57:42.017975+0000","last_became_peered":"2023-10-09T04:57:42.017975+0000","last_unstale":"2023-10-09T05:02:55.134135+0000","last_undegraded":"2023-10-09T05:02:55.134135+0000","last_fullsized":"2023-10-09T05:02:55.134135+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:48.867813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":115,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129845+0000","last_change":"2023-10-09T05:02:09.653528+0000","last_active":"2023-10-09T05:02:55.129845+0000","last_peered":"2023-10-09T05:02:55.129845+0000","last_clean":"2023-10-09T05:02:55.129845+0000","last_became_active":"2023-10-09T05:02:09.653171+0000","last_became_peered":"2023-10-09T05:02:09.653171+0000","last_unstale":"2023-10-09T05:02:55.129845+0000","last_undegraded":"2023-10-09T05:02:55.129845+0000","last_fullsized":"2023-10-09T05:02:55.129845+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:15.978289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":85,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.134213+0000","last_change":"2023-10-09T04:44:52.793709+0000","last_active":"2023-10-09T05:02:55.134213+0000","last_peered":"2023-10-09T05:02:55.134213+0000","last_clean":"2023-10-09T05:02:55.134213+0000","last_became_active":"2023-10-09T04:44:52.792973+0000","last_became_peered":"2023-10-09T04:44:52.792973+0000","last_unstale":"2023-10-09T05:02:55.134213+0000","last_undegraded":"2023-10-09T05:02:55.134213+0000","last_fullsized":"2023-10-09T05:02:55.134213+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:12.091089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":115,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.129621+0000","last_change":"2023-10-09T05:02:09.649748+0000","last_active":"2023-10-09T05:02:55.129621+0000","last_peered":"2023-10-09T05:02:55.129621+0000","last_clean":"2023-10-09T05:02:55.129621+0000","last_became_active":"2023-10-09T05:02:09.649444+0000","last_became_peered":"2023-10-09T05:02:09.649444+0000","last_unstale":"2023-10-09T05:02:55.129621+0000","last_undegraded":"2023-10-09T05:02:55.129621+0000","last_fullsized":"2023-10-09T05:02:55.129621+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:07.436820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'2","reported_seq":100,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.914367+0000","last_change":"2023-10-09T05:02:09.647382+0000","last_active":"2023-10-09T05:06:11.914367+0000","last_peered":"2023-10-09T05:06:11.914367+0000","last_clean":"2023-10-09T05:06:11.914367+0000","last_became_active":"2023-10-09T05:02:09.646803+0000","last_became_peered":"2023-10-09T05:02:09.646803+0000","last_unstale":"2023-10-09T05:06:11.914367+0000","last_undegraded":"2023-10-09T05:06:11.914367+0000","last_fullsized":"2023-10-09T05:06:11.914367+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:23:00.161736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":113,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.914782+0000","last_change":"2023-10-09T05:02:09.649674+0000","last_active":"2023-10-09T05:06:11.914782+0000","last_peered":"2023-10-09T05:06:11.914782+0000","last_clean":"2023-10-09T05:06:11.914782+0000","last_became_active":"2023-10-09T05:02:09.649041+0000","last_became_peered":"2023-10-09T05:02:09.649041+0000","last_unstale":"2023-10-09T05:06:11.914782+0000","last_undegraded":"2023-10-09T05:06:11.914782+0000","last_fullsized":"2023-10-09T05:06:11.914782+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:18.524071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'5","reported_seq":101,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:06:11.917631+0000","last_change":"2023-10-09T05:01:02.672596+0000","last_active":"2023-10-09T05:06:11.917631+0000","last_peered":"2023-10-09T05:06:11.917631+0000","last_clean":"2023-10-09T05:06:11.917631+0000","last_became_active":"2023-10-09T05:01:02.672128+0000","last_became_peered":"2023-10-09T05:01:02.672128+0000","last_unstale":"2023-10-09T05:06:11.917631+0000","last_undegraded":"2023-10-09T05:06:11.917631+0000","last_fullsized":"2023-10-09T05:06:11.917631+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:17:05.887502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:02:55.128756+0000","last_change":"2023-10-09T04:57:42.027187+0000","last_active":"2023-10-09T05:02:55.128756+0000","last_peered":"2023-10-09T05:02:55.128756+0000","last_clean":"2023-10-09T05:02:55.128756+0000","last_became_active":"2023-10-09T04:57:42.024247+0000","last_became_peered":"2023-10-09T04:57:42.024247+0000","last_unstale":"2023-10-09T05:02:55.128756+0000","last_undegraded":"2023-10-09T05:02:55.128756+0000","last_fullsized":"2023-10-09T05:02:55.128756+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:42.410792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":38,"num_read_kb":40,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234312,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303396,"kb_used_data":2012,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93441756,"statfs":{"total":95995035648,"available":95684358144,"internally_reserved":0,"allocated":2060288,"data_stored":1463213,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":1.5109999999999999,"15min":1.4710000000000001},"min":{"1min":0.67600000000000005,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":1.208,"5min":11.093999999999999,"15min":11.093999999999999},"last":0.96199999999999997},{"interface":"front","average":{"1min":1.0489999999999999,"5min":1.6100000000000001,"15min":1.6910000000000001},"min":{"1min":0.59699999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.2689999999999999,"5min":6.3730000000000002,"15min":9.5090000000000003},"last":1.147}]},{"osd":1,"last update":"Mon Oct 9 05:08:04 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":1.472,"15min":2.0680000000000001},"min":{"1min":0.60699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.151,"5min":9.7210000000000001,"15min":135.322},"last":1.099},{"interface":"front","average":{"1min":0.96299999999999997,"5min":1.534,"15min":2.2160000000000002},"min":{"1min":0.65600000000000003,"5min":0.33500000000000002,"15min":0.30499999999999999},"max":{"1min":1.3009999999999999,"5min":9.0660000000000007,"15min":135.09},"last":1.119}]},{"osd":2,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":1.633,"15min":1.369},"min":{"1min":0.625,"5min":0.315,"15min":0.315},"max":{"1min":1.2290000000000001,"5min":11.391,"15min":11.391},"last":1.069},{"interface":"front","average":{"1min":0.89800000000000002,"5min":1.512,"15min":1.274},"min":{"1min":0.59899999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.262,"5min":6.6820000000000004,"15min":6.6820000000000004},"last":0.998}]},{"osd":3,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":1.4850000000000001,"15min":1.2569999999999999},"min":{"1min":0.58699999999999997,"5min":0.317,"15min":0.317},"max":{"1min":1.2649999999999999,"5min":6.548,"15min":6.548},"last":0.83799999999999997},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.823,"15min":1.7050000000000001},"min":{"1min":0.63700000000000001,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.474,"5min":9.1129999999999995,"15min":9.1129999999999995},"last":0.878}]},{"osd":4,"last update":"Mon Oct 9 05:08:04 2023","interfaces":[{"interface":"back","average":{"1min":1,"5min":1.345,"15min":1.1559999999999999},"min":{"1min":0.71499999999999997,"5min":0.33700000000000002,"15min":0.28699999999999998},"max":{"1min":1.583,"5min":11.409000000000001,"15min":11.409000000000001},"last":0.75900000000000001},{"interface":"front","average":{"1min":1.179,"5min":1.5920000000000001,"15min":1.3859999999999999},"min":{"1min":0.81200000000000006,"5min":0.36899999999999999,"15min":0.316},"max":{"1min":1.4430000000000001,"5min":15.763,"15min":15.763},"last":1.032}]},{"osd":5,"last update":"Mon Oct 9 05:08:04 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.702,"15min":1.327},"min":{"1min":0.90800000000000003,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.5369999999999999,"5min":8.3569999999999993,"15min":8.3569999999999993},"last":1.0800000000000001},{"interface":"front","average":{"1min":1.155,"5min":1.6100000000000001,"15min":1.3460000000000001},"min":{"1min":0.878,"5min":0.35999999999999999,"15min":0.32900000000000001},"max":{"1min":1.468,"5min":7.1020000000000003,"15min":7.6109999999999998},"last":1.222}]},{"osd":6,"last update":"Mon Oct 9 05:08:04 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":1.5089999999999999,"15min":1.2529999999999999},"min":{"1min":0.91400000000000003,"5min":0.39700000000000002,"15min":0.318},"max":{"1min":1.651,"5min":6.9630000000000001,"15min":7.9939999999999998},"last":1.397},{"interface":"front","average":{"1min":1.3120000000000001,"5min":1.714,"15min":1.3700000000000001},"min":{"1min":1.123,"5min":0.41499999999999998,"15min":0.35399999999999998},"max":{"1min":1.504,"5min":6.468,"15min":6.468},"last":1.175}]}]},{"osd":6,"up_from":46,"seq":197568495954,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299292,"kb_used_data":2004,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93445860,"statfs":{"total":95995035648,"available":95688560640,"internally_reserved":0,"allocated":2052096,"data_stored":1463390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.089,"5min":1.8100000000000001,"15min":1.5309999999999999},"min":{"1min":0.61199999999999999,"5min":0.24399999999999999,"15min":0.214},"max":{"1min":1.4119999999999999,"5min":11.449,"15min":19.18},"last":0.94799999999999995},{"interface":"front","average":{"1min":1.1200000000000001,"5min":1.9450000000000001,"15min":1.6559999999999999},"min":{"1min":0.752,"5min":0.30399999999999999,"15min":0.27200000000000002},"max":{"1min":1.417,"5min":11.491,"15min":19.358000000000001},"last":1.2490000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:08:06 2023","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":1.601,"15min":1.623},"min":{"1min":0.70199999999999996,"5min":0.28000000000000003,"15min":0.25800000000000001},"max":{"1min":1.1240000000000001,"5min":10.118,"15min":10.118},"last":1.048},{"interface":"front","average":{"1min":0.96299999999999997,"5min":1.859,"15min":1.4330000000000001},"min":{"1min":0.57099999999999995,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":1.5049999999999999,"5min":9.6630000000000003,"15min":12.417},"last":0.68899999999999995}]},{"osd":2,"last update":"Mon Oct 9 05:08:12 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.7809999999999999,"15min":1.518},"min":{"1min":0.81699999999999995,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.3959999999999999,"5min":8.9730000000000008,"15min":8.9730000000000008},"last":1.3089999999999999},{"interface":"front","average":{"1min":0.95299999999999996,"5min":1.3899999999999999,"15min":1.3420000000000001},"min":{"1min":0.69599999999999995,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.1319999999999999,"5min":8.3810000000000002,"15min":8.3810000000000002},"last":1.1160000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":2.105,"15min":1.657},"min":{"1min":0.66400000000000003,"5min":0.315,"15min":0.315},"max":{"1min":1.1899999999999999,"5min":10.201000000000001,"15min":10.201000000000001},"last":0.91300000000000003},{"interface":"front","average":{"1min":1.121,"5min":1.841,"15min":1.4790000000000001},"min":{"1min":0.81399999999999995,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.411,"5min":9.7720000000000002,"15min":9.7720000000000002},"last":1.018}]},{"osd":4,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":1.6279999999999999,"15min":1.153},"min":{"1min":0.70899999999999996,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.325,"5min":16.119,"15min":16.119},"last":0.73399999999999999},{"interface":"front","average":{"1min":1.1419999999999999,"5min":1.5569999999999999,"15min":1.274},"min":{"1min":0.79500000000000004,"5min":0.32300000000000001,"15min":0.247},"max":{"1min":1.5329999999999999,"5min":11.272,"15min":11.272},"last":0.80300000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1100000000000001,"5min":1.746,"15min":1.319},"min":{"1min":0.94899999999999995,"5min":0.312,"15min":0.29299999999999998},"max":{"1min":1.444,"5min":7.7149999999999999,"15min":7.7149999999999999},"last":0.63900000000000001},{"interface":"front","average":{"1min":1.2769999999999999,"5min":2.016,"15min":1.516},"min":{"1min":0.63800000000000001,"5min":0.41599999999999998,"15min":0.33100000000000002},"max":{"1min":1.5780000000000001,"5min":10.537000000000001,"15min":10.537000000000001},"last":1.1519999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:07:50 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.472,"15min":1.3240000000000001},"min":{"1min":0.85699999999999998,"5min":0.40200000000000002,"15min":0.30599999999999999},"max":{"1min":1.3879999999999999,"5min":9.3780000000000001,"15min":9.3780000000000001},"last":0.875},{"interface":"front","average":{"1min":1.23,"5min":1.569,"15min":1.415},"min":{"1min":1.0249999999999999,"5min":0.434,"15min":0.375},"max":{"1min":1.5329999999999999,"5min":9.4060000000000006,"15min":9.4060000000000006},"last":0.97999999999999998}]}]},{"osd":1,"up_from":96,"seq":412316860544,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299076,"kb_used_data":1596,"kb_used_omap":4,"kb_used_meta":297467,"kb_avail":93446076,"statfs":{"total":95995035648,"available":95688781824,"internally_reserved":0,"allocated":1634304,"data_stored":894480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":304607118},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:59 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":1.8959999999999999,"15min":1.845},"min":{"1min":0.38400000000000001,"5min":0.20899999999999999,"15min":0.19400000000000001},"max":{"1min":0.81799999999999995,"5min":22.382999999999999,"15min":22.382999999999999},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.65700000000000003,"5min":2.0089999999999999,"15min":2.306},"min":{"1min":0.42199999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.91800000000000004,"5min":10.159000000000001,"15min":10.268000000000001},"last":0.96699999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":1.7609999999999999,"15min":1.5820000000000001},"min":{"1min":0.40400000000000003,"5min":0.219,"15min":0.219},"max":{"1min":1.0029999999999999,"5min":10.115,"15min":10.115},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.78200000000000003,"5min":1.831,"15min":1.5449999999999999},"min":{"1min":0.45800000000000002,"5min":0.252,"15min":0.252},"max":{"1min":1.151,"5min":10.428000000000001,"15min":11.917},"last":0.92400000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:07:20 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":2.1339999999999999,"15min":2.141},"min":{"1min":0.51800000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.0489999999999999,"5min":9.6039999999999992,"15min":9.6039999999999992},"last":0.81200000000000006},{"interface":"front","average":{"1min":0.80700000000000005,"5min":1.6899999999999999,"15min":1.4750000000000001},"min":{"1min":0.52800000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.21,"5min":9.9039999999999999,"15min":9.9039999999999999},"last":0.77900000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:07:59 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.5529999999999999,"15min":1.6000000000000001},"min":{"1min":0.82199999999999995,"5min":0.35599999999999998,"15min":0.34200000000000003},"max":{"1min":1.339,"5min":9.9749999999999996,"15min":9.9749999999999996},"last":1.153},{"interface":"front","average":{"1min":0.98499999999999999,"5min":1.8180000000000001,"15min":1.5129999999999999},"min":{"1min":0.72599999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.3,"5min":22.350000000000001,"15min":22.350000000000001},"last":1}]},{"osd":5,"last update":"Mon Oct 9 05:07:59 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":2.028,"15min":1.6859999999999999},"min":{"1min":0.63500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.218,"5min":12.454000000000001,"15min":12.454000000000001},"last":1.448},{"interface":"front","average":{"1min":1.1100000000000001,"5min":2.1179999999999999,"15min":1.8959999999999999},"min":{"1min":0.73799999999999999,"5min":0.34999999999999998,"15min":0.316},"max":{"1min":1.3700000000000001,"5min":10.035,"15min":10.035},"last":1.367}]},{"osd":6,"last update":"Mon Oct 9 05:07:59 2023","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":2.222,"15min":1.901},"min":{"1min":0.78300000000000003,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.2450000000000001,"5min":10.409000000000001,"15min":10.409000000000001},"last":1.4950000000000001},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.9359999999999999,"15min":1.7330000000000001},"min":{"1min":0.61499999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.3879999999999999,"5min":10.007,"15min":10.007},"last":1.476}]},{"osd":7,"last update":"Mon Oct 9 05:07:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.708,"15min":1.786},"min":{"1min":0.83599999999999997,"5min":0.41899999999999998,"15min":0.38800000000000001},"max":{"1min":1.4259999999999999,"5min":10.061,"15min":10.061},"last":1.288},{"interface":"front","average":{"1min":1.0069999999999999,"5min":1.948,"15min":1.698},"min":{"1min":0.626,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.2370000000000001,"5min":22.175999999999998,"15min":22.175999999999998},"last":1.0569999999999999}]}]},{"osd":0,"up_from":88,"seq":377957122186,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":334284,"kb_used_data":2180,"kb_used_omap":5,"kb_used_meta":332090,"kb_avail":93410868,"statfs":{"total":95995035648,"available":95652728832,"internally_reserved":0,"allocated":2232320,"data_stored":1483977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":340060794},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:07:59 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":1.6459999999999999,"15min":2.173},"min":{"1min":0.47399999999999998,"5min":0.217,"15min":0.217},"max":{"1min":0.97999999999999998,"5min":11.561999999999999,"15min":13.084},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.60999999999999999,"5min":1.585,"15min":1.9890000000000001},"min":{"1min":0.33400000000000002,"5min":0.30099999999999999,"15min":0.218},"max":{"1min":0.81299999999999994,"5min":10.746,"15min":11.161},"last":0.34499999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:08:12 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":1.681,"15min":1.8680000000000001},"min":{"1min":0.45400000000000001,"5min":0.33600000000000002,"15min":0.27200000000000002},"max":{"1min":1.21,"5min":8.8030000000000008,"15min":9.1300000000000008},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.72899999999999998,"5min":1.1879999999999999,"15min":1.6319999999999999},"min":{"1min":0.53400000000000003,"5min":0.245,"15min":0.23300000000000001},"max":{"1min":1.0469999999999999,"5min":8.3599999999999994,"15min":10.423},"last":0.49299999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":1.5489999999999999,"15min":1.4810000000000001},"min":{"1min":0.35799999999999998,"5min":0.253,"15min":0.253},"max":{"1min":1.389,"5min":11.433999999999999,"15min":11.433999999999999},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.79500000000000004,"5min":1.575,"15min":1.3089999999999999},"min":{"1min":0.39700000000000002,"5min":0.318,"15min":0.318},"max":{"1min":1.411,"5min":10.041,"15min":10.041},"last":0.41299999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.3919999999999999,"15min":1.4830000000000001},"min":{"1min":0.53300000000000003,"5min":0.33500000000000002,"15min":0.30099999999999999},"max":{"1min":1.3320000000000001,"5min":8.7449999999999992,"15min":8.7449999999999992},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.98599999999999999,"5min":1.331,"15min":1.4450000000000001},"min":{"1min":0.52300000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.3540000000000001,"5min":5.7320000000000002,"15min":7.202},"last":0.72599999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":1.49,"15min":1.5649999999999999},"min":{"1min":0.55400000000000005,"5min":0.34100000000000003,"15min":0.30399999999999999},"max":{"1min":1.6890000000000001,"5min":6.8959999999999999,"15min":12.526999999999999},"last":0.79900000000000004},{"interface":"front","average":{"1min":1.022,"5min":1.7709999999999999,"15min":1.6359999999999999},"min":{"1min":0.50800000000000001,"5min":0.35299999999999998,"15min":0.316},"max":{"1min":1.6020000000000001,"5min":10.148,"15min":10.148},"last":0.82499999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.615,"15min":1.587},"min":{"1min":0.55600000000000005,"5min":0.371,"15min":0.371},"max":{"1min":1.3200000000000001,"5min":8.8710000000000004,"15min":8.8710000000000004},"last":0.78000000000000003},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.4399999999999999,"15min":1.5},"min":{"1min":0.67800000000000005,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.458,"5min":6.9370000000000003,"15min":10.429},"last":0.84099999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.3660000000000001,"15min":1.5049999999999999},"min":{"1min":0.64400000000000002,"5min":0.375,"15min":0.36599999999999999},"max":{"1min":1.631,"5min":6.9569999999999999,"15min":10.314},"last":0.70099999999999996},{"interface":"front","average":{"1min":1.075,"5min":1.71,"15min":1.879},"min":{"1min":0.64400000000000002,"5min":0.436,"15min":0.39500000000000002},"max":{"1min":1.4039999999999999,"5min":8.6110000000000007,"15min":10.738},"last":0.86299999999999999}]}]},{"osd":2,"up_from":106,"seq":455266533464,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":252696,"kb_used_data":1552,"kb_used_omap":7,"kb_used_meta":251128,"kb_avail":93492456,"statfs":{"total":95995035648,"available":95736274944,"internally_reserved":0,"allocated":1589248,"data_stored":886182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":257155153},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:08:07 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":1.5289999999999999,"15min":1.264},"min":{"1min":0.33700000000000002,"5min":0.215,"15min":0.215},"max":{"1min":0.96699999999999997,"5min":23.010999999999999,"15min":23.010999999999999},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.72099999999999997,"5min":1.6739999999999999,"15min":1.3049999999999999},"min":{"1min":0.31,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":1.23,"5min":20.501999999999999,"15min":20.501999999999999},"last":0.89200000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:08:07 2023","interfaces":[{"interface":"back","average":{"1min":0.74399999999999999,"5min":1.4399999999999999,"15min":1.242},"min":{"1min":0.373,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.1559999999999999,"5min":20.539999999999999,"15min":20.539999999999999},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.65400000000000003,"5min":1.8600000000000001,"15min":1.464},"min":{"1min":0.35899999999999999,"5min":0.23899999999999999,"15min":0.23100000000000001},"max":{"1min":0.996,"5min":24.058,"15min":24.058},"last":0.61499999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:07:20 2023","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":1.827,"15min":2.1429999999999998},"min":{"1min":0.41799999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.2250000000000001,"5min":20.907,"15min":20.907},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.67600000000000005,"5min":1.859,"15min":1.778},"min":{"1min":0.39500000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.046,"5min":23.574999999999999,"15min":23.574999999999999},"last":0.67400000000000004}]},{"osd":4,"last update":"Mon Oct 9 05:08:07 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":1.54,"15min":1.4370000000000001},"min":{"1min":0.58799999999999997,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.419,"5min":20.712,"15min":20.712},"last":0.78600000000000003},{"interface":"front","average":{"1min":0.92100000000000004,"5min":1.845,"15min":1.484},"min":{"1min":0.61799999999999999,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.169,"5min":23.547000000000001,"15min":23.547000000000001},"last":0.81399999999999995}]},{"osd":5,"last update":"Mon Oct 9 05:08:07 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":2.0369999999999999,"15min":1.514},"min":{"1min":0.57999999999999996,"5min":0.443,"15min":0.39600000000000002},"max":{"1min":1.236,"5min":20.782,"15min":20.782},"last":1.0029999999999999},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.8460000000000001,"15min":1.905},"min":{"1min":0.69699999999999995,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.4139999999999999,"5min":20.792000000000002,"15min":20.792000000000002},"last":0.94699999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:08:07 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":1.7569999999999999,"15min":1.5649999999999999},"min":{"1min":0.746,"5min":0.35799999999999998,"15min":0.34999999999999998},"max":{"1min":1.268,"5min":23.510999999999999,"15min":23.510999999999999},"last":0.73799999999999999},{"interface":"front","average":{"1min":1.129,"5min":1.7589999999999999,"15min":1.522},"min":{"1min":0.71799999999999997,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":1.3759999999999999,"5min":20.684999999999999,"15min":20.684999999999999},"last":0.86299999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:08:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":1.6850000000000001,"15min":1.506},"min":{"1min":0.60899999999999999,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":1.4530000000000001,"5min":20.887,"15min":20.887},"last":1.052},{"interface":"front","average":{"1min":1.1359999999999999,"5min":1.6970000000000001,"15min":1.4610000000000001},"min":{"1min":0.72299999999999998,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.498,"5min":20.863,"15min":20.863},"last":0.97999999999999998}]}]},{"osd":3,"up_from":123,"seq":528280977483,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":272160,"kb_used_data":1496,"kb_used_omap":2,"kb_used_meta":270653,"kb_avail":93472992,"statfs":{"total":95995035648,"available":95716343808,"internally_reserved":0,"allocated":1531904,"data_stored":879533,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":277148725},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":1.6279999999999999,"15min":2.0550000000000002},"min":{"1min":0.434,"5min":0.187,"15min":0.187},"max":{"1min":0.86599999999999999,"5min":14.141,"15min":14.141},"last":0.82299999999999995},{"interface":"front","average":{"1min":0.72199999999999998,"5min":1.972,"15min":2.605},"min":{"1min":0.44700000000000001,"5min":0.184,"15min":0.184},"max":{"1min":1.0920000000000001,"5min":14.045999999999999,"15min":14.045999999999999},"last":0.51000000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":1.7589999999999999,"15min":2.4119999999999999},"min":{"1min":0.36199999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.147,"5min":14.193,"15min":14.193},"last":0.753},{"interface":"front","average":{"1min":0.67900000000000005,"5min":1.669,"15min":2.548},"min":{"1min":0.45000000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":1.0740000000000001,"5min":17.498999999999999,"15min":17.498999999999999},"last":0.67800000000000005}]},{"osd":2,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":1.855,"15min":2.0779999999999998},"min":{"1min":0.39800000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.91600000000000004,"5min":9.6129999999999995,"15min":9.6129999999999995},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.77700000000000002,"5min":1.873,"15min":1.867},"min":{"1min":0.49299999999999999,"5min":0.245,"15min":0.245},"max":{"1min":1.236,"5min":8.1600000000000001,"15min":8.1600000000000001},"last":0.59799999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":1.5780000000000001,"15min":1.73},"min":{"1min":0.66900000000000004,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.45,"5min":5.8070000000000004,"15min":5.8070000000000004},"last":1.0600000000000001},{"interface":"front","average":{"1min":0.94699999999999995,"5min":1.3500000000000001,"15min":1.294},"min":{"1min":0.58399999999999996,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.3180000000000001,"5min":9.4749999999999996,"15min":9.4749999999999996},"last":0.92900000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":1.613,"15min":1.335},"min":{"1min":0.71299999999999997,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.27,"5min":9.5079999999999991,"15min":9.5079999999999991},"last":0.995},{"interface":"front","average":{"1min":1.024,"5min":2.0049999999999999,"15min":2.0550000000000002},"min":{"1min":0.67100000000000004,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.413,"5min":8.9380000000000006,"15min":8.9380000000000006},"last":0.87}]},{"osd":6,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":1.7789999999999999,"15min":1.875},"min":{"1min":0.72399999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.4339999999999999,"5min":9.6590000000000007,"15min":9.6590000000000007},"last":1.099},{"interface":"front","average":{"1min":1.05,"5min":1.9199999999999999,"15min":1.9810000000000001},"min":{"1min":0.73999999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.405,"5min":6.3289999999999997,"15min":6.3289999999999997},"last":0.90400000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.9970000000000001,"15min":2.1699999999999999},"min":{"1min":0.72799999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.468,"5min":8.3209999999999997,"15min":8.3209999999999997},"last":0.97599999999999998},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.663,"15min":1.5589999999999999},"min":{"1min":0.81299999999999994,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.369,"5min":9.5359999999999996,"15min":9.5359999999999996},"last":1.131}]}]},{"osd":4,"up_from":30,"seq":128849019240,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298724,"kb_used_data":1436,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446428,"statfs":{"total":95995035648,"available":95689142272,"internally_reserved":0,"allocated":1470464,"data_stored":873363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:11 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":1.389,"15min":4.6070000000000002},"min":{"1min":0.55200000000000005,"5min":0.26600000000000001,"15min":0.22700000000000001},"max":{"1min":1.389,"5min":10.109999999999999,"15min":131.30699999999999},"last":1.2569999999999999},{"interface":"front","average":{"1min":0.97999999999999998,"5min":1.244,"15min":4.5469999999999997},"min":{"1min":0.57999999999999996,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.3999999999999999,"5min":6.3470000000000004,"15min":131.36600000000001},"last":1.0169999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:08:03 2023","interfaces":[{"interface":"back","average":{"1min":0.86099999999999999,"5min":2.3119999999999998,"15min":1.833},"min":{"1min":0.61199999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.075,"5min":59.408000000000001,"15min":59.408000000000001},"last":0.80400000000000005},{"interface":"front","average":{"1min":1.028,"5min":2.1779999999999999,"15min":1.9610000000000001},"min":{"1min":0.69399999999999995,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.353,"5min":60.207000000000001,"15min":60.207000000000001},"last":1.0629999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:07:11 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.742,"15min":1.1870000000000001},"min":{"1min":0.59499999999999997,"5min":0.307,"15min":0.307},"max":{"1min":1.2949999999999999,"5min":10.196,"15min":10.196},"last":0.95699999999999996},{"interface":"front","average":{"1min":1.0009999999999999,"5min":1.387,"15min":1.107},"min":{"1min":0.628,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":1.294,"5min":6.3810000000000002,"15min":6.3810000000000002},"last":1.1140000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":1.869,"15min":1.734},"min":{"1min":0.68999999999999995,"5min":0.309,"15min":0.309},"max":{"1min":1.343,"5min":10.555999999999999,"15min":10.555999999999999},"last":1.1459999999999999},{"interface":"front","average":{"1min":1.022,"5min":1.851,"15min":1.657},"min":{"1min":0.83099999999999996,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.262,"5min":21.036999999999999,"15min":21.036999999999999},"last":0.875}]},{"osd":5,"last update":"Mon Oct 9 05:07:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1100000000000001,"5min":1.5289999999999999,"15min":1.2869999999999999},"min":{"1min":0.64500000000000002,"5min":0.32500000000000001,"15min":0.31},"max":{"1min":1.3899999999999999,"5min":10.214,"15min":10.214},"last":1.3029999999999999},{"interface":"front","average":{"1min":1.1870000000000001,"5min":1.6200000000000001,"15min":1.3260000000000001},"min":{"1min":0.83699999999999997,"5min":0.34200000000000003,"15min":0.30399999999999999},"max":{"1min":1.629,"5min":6.8310000000000004,"15min":10.890000000000001},"last":0.64000000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:08:03 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":1.748,"15min":1.3560000000000001},"min":{"1min":0.60499999999999998,"5min":0.32800000000000001,"15min":0.27800000000000002},"max":{"1min":1.2310000000000001,"5min":11.557,"15min":11.557},"last":0.91600000000000004},{"interface":"front","average":{"1min":1.214,"5min":1.5920000000000001,"15min":1.383},"min":{"1min":0.86699999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.504,"5min":10.689,"15min":10.689},"last":1.204}]},{"osd":7,"last update":"Mon Oct 9 05:08:03 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":1.329,"15min":1.2509999999999999},"min":{"1min":0.68100000000000005,"5min":0.377,"15min":0.35999999999999999},"max":{"1min":1.5209999999999999,"5min":6.4820000000000002,"15min":8.8859999999999992},"last":1.1870000000000001},{"interface":"front","average":{"1min":1.0409999999999999,"5min":1.5649999999999999,"15min":1.3819999999999999},"min":{"1min":0.77100000000000002,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.3,"5min":6.1699999999999999,"15min":7.7880000000000003},"last":0.995}]}]},{"osd":5,"up_from":38,"seq":163208757596,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":603476,"kb_used_data":1420,"kb_used_omap":0,"kb_used_meta":602048,"kb_avail":93141676,"statfs":{"total":95995035648,"available":95377076224,"internally_reserved":0,"allocated":1454080,"data_stored":869107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":616497152},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:08:13 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":1.7569999999999999,"15min":1.8360000000000001},"min":{"1min":0.58899999999999997,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.3160000000000001,"5min":8.7449999999999992,"15min":77.745999999999995},"last":1.0800000000000001},{"interface":"front","average":{"1min":1.117,"5min":1.847,"15min":1.9450000000000001},"min":{"1min":0.83799999999999997,"5min":0.37,"15min":0.35299999999999998},"max":{"1min":1.397,"5min":8.7200000000000006,"15min":77.820999999999998},"last":1.1899999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:08:06 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":1.9390000000000001,"15min":1.6839999999999999},"min":{"1min":0.68700000000000006,"5min":0.28499999999999998,"15min":0.26900000000000002},"max":{"1min":1.397,"5min":13.401999999999999,"15min":13.401999999999999},"last":0.86199999999999999},{"interface":"front","average":{"1min":0.98999999999999999,"5min":1.756,"15min":1.7410000000000001},"min":{"1min":0.72699999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.2929999999999999,"5min":13.398,"15min":13.398},"last":1.107}]},{"osd":2,"last update":"Mon Oct 9 05:07:17 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":1.7230000000000001,"15min":1.111},"min":{"1min":0.57199999999999995,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.3180000000000001,"5min":13.436,"15min":13.436},"last":1.248},{"interface":"front","average":{"1min":1.014,"5min":1.8899999999999999,"15min":1.252},"min":{"1min":0.65200000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.2929999999999999,"5min":9.4949999999999992,"15min":9.4949999999999992},"last":0.96999999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:07:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.869,"15min":1.966},"min":{"1min":0.745,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.349,"5min":9.1010000000000009,"15min":9.1010000000000009},"last":0.92500000000000004},{"interface":"front","average":{"1min":1.1200000000000001,"5min":2.121,"15min":1.7490000000000001},"min":{"1min":0.54900000000000004,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":3.1739999999999999,"5min":11.859999999999999,"15min":11.859999999999999},"last":1.2929999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:08:13 2023","interfaces":[{"interface":"back","average":{"1min":1.1559999999999999,"5min":1.589,"15min":1.298},"min":{"1min":0.68500000000000005,"5min":0.25,"15min":0.25},"max":{"1min":1.554,"5min":14.781000000000001,"15min":14.781000000000001},"last":1.3400000000000001},{"interface":"front","average":{"1min":1.127,"5min":1.831,"15min":1.4299999999999999},"min":{"1min":0.65600000000000003,"5min":0.34599999999999997,"15min":0.313},"max":{"1min":1.5409999999999999,"5min":9.625,"15min":9.625},"last":1.155}]},{"osd":6,"last update":"Mon Oct 9 05:08:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":1.9970000000000001,"15min":1.5569999999999999},"min":{"1min":0.70499999999999996,"5min":0.38300000000000001,"15min":0.33900000000000002},"max":{"1min":1.45,"5min":11.654999999999999,"15min":11.654999999999999},"last":1.4570000000000001},{"interface":"front","average":{"1min":1.1719999999999999,"5min":1.7330000000000001,"15min":1.4630000000000001},"min":{"1min":0.58099999999999996,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.464,"5min":8.4670000000000005,"15min":10.423999999999999},"last":1.371}]},{"osd":7,"last update":"Mon Oct 9 05:07:46 2023","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":1.7549999999999999,"15min":1.4550000000000001},"min":{"1min":0.629,"5min":0.36899999999999999,"15min":0.34799999999999998},"max":{"1min":1.508,"5min":8.7720000000000002,"15min":9.6639999999999997},"last":1.5289999999999999},{"interface":"front","average":{"1min":1.238,"5min":1.823,"15min":1.492},"min":{"1min":0.90200000000000002,"5min":0.34699999999999998,"15min":0.313},"max":{"1min":1.577,"5min":10.122999999999999,"15min":12.843999999999999},"last":1.498}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:08:17.831 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T05:08:17.831 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd dump 2023-10-09T05:08:17.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:17 smithi194 bash[92666]: audit 2023-10-09T05:08:16.901844+0000 mgr.y (mgr.25277) 589 : audit 0 from='client.44738 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:08:17.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:17 smithi194 bash[92666]: audit 2023-10-09T05:08:16.972223+0000 mon.a (mon.0) 738 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:08:17.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:17 smithi194 bash[92666]: audit 2023-10-09T05:08:16.973578+0000 mon.a (mon.0) 739 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:08:17.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:17 smithi194 bash[92666]: audit 2023-10-09T05:08:16.985927+0000 mon.a (mon.0) 740 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:08:18.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:18 smithi194 bash[92666]: cluster 2023-10-09T05:08:17.713482+0000 mgr.y (mgr.25277) 590 : cluster 0 pgmap v541: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:19.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:18 smithi133 bash[102858]: cluster 2023-10-09T05:08:17.713482+0000 mgr.y (mgr.25277) 590 : cluster 0 pgmap v541: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:19.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:18 smithi133 bash[105589]: cluster 2023-10-09T05:08:17.713482+0000 mgr.y (mgr.25277) 590 : cluster 0 pgmap v541: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:21.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:20 smithi133 bash[105589]: cluster 2023-10-09T05:08:19.714428+0000 mgr.y (mgr.25277) 591 : cluster 0 pgmap v542: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:21.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:20 smithi133 bash[102858]: cluster 2023-10-09T05:08:19.714428+0000 mgr.y (mgr.25277) 591 : cluster 0 pgmap v542: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:21.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:20 smithi194 bash[92666]: cluster 2023-10-09T05:08:19.714428+0000 mgr.y (mgr.25277) 591 : cluster 0 pgmap v542: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:22.642 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:08:23.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:22 smithi133 bash[102858]: cluster 2023-10-09T05:08:21.715311+0000 mgr.y (mgr.25277) 592 : cluster 0 pgmap v543: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:23.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:22 smithi133 bash[105589]: cluster 2023-10-09T05:08:21.715311+0000 mgr.y (mgr.25277) 592 : cluster 0 pgmap v543: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:23.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:22 smithi194 bash[92666]: cluster 2023-10-09T05:08:21.715311+0000 mgr.y (mgr.25277) 592 : cluster 0 pgmap v543: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:24.550 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:08:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:08:24] "GET /metrics HTTP/1.1" 200 33988 "" "Prometheus/2.43.0" 2023-10-09T05:08:24.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:24 smithi133 bash[105589]: cluster 2023-10-09T05:08:23.716150+0000 mgr.y (mgr.25277) 593 : cluster 0 pgmap v544: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:24.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:24 smithi133 bash[102858]: cluster 2023-10-09T05:08:23.716150+0000 mgr.y (mgr.25277) 593 : cluster 0 pgmap v544: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:24.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:24 smithi194 bash[92666]: cluster 2023-10-09T05:08:23.716150+0000 mgr.y (mgr.25277) 593 : cluster 0 pgmap v544: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:26.318 INFO:teuthology.orchestra.run.smithi133.stdout:epoch 138 2023-10-09T05:08:26.318 INFO:teuthology.orchestra.run.smithi133.stdout:fsid 855b5206-665c-11ee-8db5-212e2dc638e7 2023-10-09T05:08:26.318 INFO:teuthology.orchestra.run.smithi133.stdout:created 2023-10-09T04:31:08.633675+0000 2023-10-09T05:08:26.318 INFO:teuthology.orchestra.run.smithi133.stdout:modified 2023-10-09T05:06:11.726456+0000 2023-10-09T05:08:26.318 INFO:teuthology.orchestra.run.smithi133.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2023-10-09T05:08:26.318 INFO:teuthology.orchestra.run.smithi133.stdout:crush_version 22 2023-10-09T05:08:26.318 INFO:teuthology.orchestra.run.smithi133.stdout:full_ratio 0.95 2023-10-09T05:08:26.319 INFO:teuthology.orchestra.run.smithi133.stdout:backfillfull_ratio 0.9 2023-10-09T05:08:26.319 INFO:teuthology.orchestra.run.smithi133.stdout:nearfull_ratio 0.85 2023-10-09T05:08:26.319 INFO:teuthology.orchestra.run.smithi133.stdout:require_min_compat_client luminous 2023-10-09T05:08:26.319 INFO:teuthology.orchestra.run.smithi133.stdout:min_compat_client luminous 2023-10-09T05:08:26.319 INFO:teuthology.orchestra.run.smithi133.stdout:require_osd_release quincy 2023-10-09T05:08:26.319 INFO:teuthology.orchestra.run.smithi133.stdout:stretch_mode_enabled false 2023-10-09T05:08:26.319 INFO:teuthology.orchestra.run.smithi133.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 21 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89 2023-10-09T05:08:26.319 INFO:teuthology.orchestra.run.smithi133.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 63 flags hashpspool stripe_width 0 application rbd read_balance_score 2.00 2023-10-09T05:08:26.319 INFO:teuthology.orchestra.run.smithi133.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 67 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2023-10-09T05:08:26.319 INFO:teuthology.orchestra.run.smithi133.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 68 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2023-10-09T05:08:26.319 INFO:teuthology.orchestra.run.smithi133.stdout:max_osd 8 2023-10-09T05:08:26.319 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0 up in weight 1 up_from 88 up_thru 126 down_at 84 last_clean_interval [8,83) [v2:172.21.15.133:6802/2098404708,v1:172.21.15.133:6803/2098404708] [v2:172.21.15.133:6804/2098404708,v1:172.21.15.133:6805/2098404708] exists,up 32633ec2-9b8f-42d8-bdfb-5c9668a26071 2023-10-09T05:08:26.320 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1 up in weight 1 up_from 96 up_thru 135 down_at 93 last_clean_interval [13,92) [v2:172.21.15.133:6810/2668659948,v1:172.21.15.133:6811/2668659948] [v2:172.21.15.133:6812/2668659948,v1:172.21.15.133:6813/2668659948] exists,up 7f2a961d-a790-4337-91c7-1e45f900bbd4 2023-10-09T05:08:26.320 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2 up in weight 1 up_from 106 up_thru 123 down_at 103 last_clean_interval [18,102) [v2:172.21.15.133:6818/3595178639,v1:172.21.15.133:6819/3595178639] [v2:172.21.15.133:6820/3595178639,v1:172.21.15.133:6821/3595178639] exists,up 08751330-a327-4bf8-9c4d-9ff986b1c67f 2023-10-09T05:08:26.320 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3 up in weight 1 up_from 123 up_thru 136 down_at 118 last_clean_interval [25,117) [v2:172.21.15.133:6826/1082233372,v1:172.21.15.133:6827/1082233372] [v2:172.21.15.133:6828/1082233372,v1:172.21.15.133:6829/1082233372] exists,up 56bbf807-c8f7-4226-b46e-802b539274ca 2023-10-09T05:08:26.320 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4 up in weight 1 up_from 30 up_thru 129 down_at 0 last_clean_interval [0,0) [v2:172.21.15.194:6800/1874338403,v1:172.21.15.194:6801/1874338403] [v2:172.21.15.194:6802/1874338403,v1:172.21.15.194:6803/1874338403] exists,up 509ec8cc-5042-428e-9a69-8e602b5afce8 2023-10-09T05:08:26.320 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5 up in weight 1 up_from 38 up_thru 132 down_at 0 last_clean_interval [0,0) [v2:172.21.15.194:6808/3989986662,v1:172.21.15.194:6809/3989986662] [v2:172.21.15.194:6810/3989986662,v1:172.21.15.194:6811/3989986662] exists,up bbdc1e67-fd5b-49d0-9e42-cf151d167943 2023-10-09T05:08:26.320 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6 up in weight 1 up_from 46 up_thru 118 down_at 0 last_clean_interval [0,0) [v2:172.21.15.194:6816/2607555245,v1:172.21.15.194:6817/2607555245] [v2:172.21.15.194:6818/2607555245,v1:172.21.15.194:6819/2607555245] exists,up 7b69d014-f4c0-4e73-a14e-7d51b7ee3ea2 2023-10-09T05:08:26.320 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7 up in weight 1 up_from 54 up_thru 123 down_at 0 last_clean_interval [0,0) [v2:172.21.15.194:6824/1051622545,v1:172.21.15.194:6825/1051622545] [v2:172.21.15.194:6826/1051622545,v1:172.21.15.194:6827/1051622545] exists,up 02506268-102c-4654-9b5c-68d36c28f7f7 2023-10-09T05:08:26.320 INFO:teuthology.orchestra.run.smithi133.stdout:pg_upmap_items 4.4 [1,5] 2023-10-09T05:08:26.320 INFO:teuthology.orchestra.run.smithi133.stdout:pg_upmap_items 4.7 [1,2] 2023-10-09T05:08:26.320 INFO:teuthology.orchestra.run.smithi133.stdout:pg_upmap_items 4.a [1,2] 2023-10-09T05:08:26.320 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:0/4173745360 expires 2023-10-10T04:31:42.639571+0000 2023-10-09T05:08:26.321 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:0/3141871970 expires 2023-10-10T04:52:13.301898+0000 2023-10-09T05:08:26.321 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:0/1787141186 expires 2023-10-10T04:49:30.896539+0000 2023-10-09T05:08:26.321 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.194:0/2434253169 expires 2023-10-10T04:50:55.051958+0000 2023-10-09T05:08:26.321 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:0/3777348175 expires 2023-10-10T04:52:13.301898+0000 2023-10-09T05:08:26.321 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:0/2479656939 expires 2023-10-10T04:52:13.301898+0000 2023-10-09T05:08:26.321 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:0/4053521787 expires 2023-10-10T04:52:13.301898+0000 2023-10-09T05:08:26.321 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:0/320185713 expires 2023-10-10T04:42:05.063462+0000 2023-10-09T05:08:26.321 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:0/3636510019 expires 2023-10-10T04:42:05.063462+0000 2023-10-09T05:08:26.321 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:0/2419271305 expires 2023-10-10T04:49:30.896539+0000 2023-10-09T05:08:26.321 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:0/740467257 expires 2023-10-10T04:31:42.639571+0000 2023-10-09T05:08:26.321 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:6800/275314455 expires 2023-10-10T04:49:30.896539+0000 2023-10-09T05:08:26.322 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:6835/4227843513 expires 2023-10-10T04:45:00.021661+0000 2023-10-09T05:08:26.322 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:0/55818837 expires 2023-10-10T04:42:05.063462+0000 2023-10-09T05:08:26.322 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.194:0/2197565959 expires 2023-10-10T04:50:55.051958+0000 2023-10-09T05:08:26.322 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:6800/788799879 expires 2023-10-10T04:32:17.611609+0000 2023-10-09T05:08:26.322 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:6801/788799879 expires 2023-10-10T04:32:17.611609+0000 2023-10-09T05:08:26.322 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:0/70449665 expires 2023-10-10T04:52:13.301898+0000 2023-10-09T05:08:26.322 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:0/1793292086 expires 2023-10-10T04:32:17.611609+0000 2023-10-09T05:08:26.322 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:6801/3027596458 expires 2023-10-10T04:52:13.301898+0000 2023-10-09T05:08:26.322 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:6801/275314455 expires 2023-10-10T04:49:30.896539+0000 2023-10-09T05:08:26.322 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.194:0/3848858405 expires 2023-10-10T04:50:55.051958+0000 2023-10-09T05:08:26.322 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.194:0/430115207 expires 2023-10-10T04:50:55.051958+0000 2023-10-09T05:08:26.322 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:0/2159982120 expires 2023-10-10T04:49:30.896539+0000 2023-10-09T05:08:26.323 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:6800/768614786 expires 2023-10-10T04:42:05.063462+0000 2023-10-09T05:08:26.323 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:0/1575170036 expires 2023-10-10T04:49:30.896539+0000 2023-10-09T05:08:26.323 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:6801/768614786 expires 2023-10-10T04:42:05.063462+0000 2023-10-09T05:08:26.323 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:6834/4227843513 expires 2023-10-10T04:45:00.021661+0000 2023-10-09T05:08:26.323 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:0/3619387632 expires 2023-10-10T04:32:17.611609+0000 2023-10-09T05:08:26.323 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:0/1411744140 expires 2023-10-10T04:31:42.639571+0000 2023-10-09T05:08:26.323 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.194:0/1505438352 expires 2023-10-10T04:50:55.051958+0000 2023-10-09T05:08:26.323 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:0/1309073801 expires 2023-10-10T04:42:05.063462+0000 2023-10-09T05:08:26.323 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.194:0/152828820 expires 2023-10-10T04:50:55.051958+0000 2023-10-09T05:08:26.323 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:0/499357090 expires 2023-10-10T04:52:13.301898+0000 2023-10-09T05:08:26.323 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:6800/3027596458 expires 2023-10-10T04:52:13.301898+0000 2023-10-09T05:08:26.323 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:0/4290202542 expires 2023-10-10T04:32:17.611609+0000 2023-10-09T05:08:26.323 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.194:6834/2184279986 expires 2023-10-10T04:50:55.051958+0000 2023-10-09T05:08:26.324 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.194:6835/2184279986 expires 2023-10-10T04:50:55.051958+0000 2023-10-09T05:08:26.324 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:6800/1761810618 expires 2023-10-10T04:31:42.639571+0000 2023-10-09T05:08:26.324 INFO:teuthology.orchestra.run.smithi133.stdout:blocklist 172.21.15.133:6801/1761810618 expires 2023-10-10T04:31:42.639571+0000 2023-10-09T05:08:26.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:26 smithi133 bash[105589]: cluster 2023-10-09T05:08:25.717162+0000 mgr.y (mgr.25277) 594 : cluster 0 pgmap v545: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:26.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:26 smithi133 bash[105589]: audit 2023-10-09T05:08:26.315850+0000 mon.c (mon.2) 57 : audit 0 from='client.? 172.21.15.133:0/965331684' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T05:08:26.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:26 smithi133 bash[102858]: cluster 2023-10-09T05:08:25.717162+0000 mgr.y (mgr.25277) 594 : cluster 0 pgmap v545: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:26.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:26 smithi133 bash[102858]: audit 2023-10-09T05:08:26.315850+0000 mon.c (mon.2) 57 : audit 0 from='client.? 172.21.15.133:0/965331684' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T05:08:27.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:26 smithi194 bash[92666]: cluster 2023-10-09T05:08:25.717162+0000 mgr.y (mgr.25277) 594 : cluster 0 pgmap v545: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:27.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:26 smithi194 bash[92666]: audit 2023-10-09T05:08:26.315850+0000 mon.c (mon.2) 57 : audit 0 from='client.? 172.21.15.133:0/965331684' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T05:08:27.140 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 88 up_thru 126 down_at 84 last_clean_interval [8,83) [v2:172.21.15.133:6802/2098404708,v1:172.21.15.133:6803/2098404708] [v2:172.21.15.133:6804/2098404708,v1:172.21.15.133:6805/2098404708] exists,up 32633ec2-9b8f-42d8-bdfb-5c9668a26071', 'osd.1 up in weight 1 up_from 96 up_thru 135 down_at 93 last_clean_interval [13,92) [v2:172.21.15.133:6810/2668659948,v1:172.21.15.133:6811/2668659948] [v2:172.21.15.133:6812/2668659948,v1:172.21.15.133:6813/2668659948] exists,up 7f2a961d-a790-4337-91c7-1e45f900bbd4', 'osd.2 up in weight 1 up_from 106 up_thru 123 down_at 103 last_clean_interval [18,102) [v2:172.21.15.133:6818/3595178639,v1:172.21.15.133:6819/3595178639] [v2:172.21.15.133:6820/3595178639,v1:172.21.15.133:6821/3595178639] exists,up 08751330-a327-4bf8-9c4d-9ff986b1c67f', 'osd.3 up in weight 1 up_from 123 up_thru 136 down_at 118 last_clean_interval [25,117) [v2:172.21.15.133:6826/1082233372,v1:172.21.15.133:6827/1082233372] [v2:172.21.15.133:6828/1082233372,v1:172.21.15.133:6829/1082233372] exists,up 56bbf807-c8f7-4226-b46e-802b539274ca', 'osd.4 up in weight 1 up_from 30 up_thru 129 down_at 0 last_clean_interval [0,0) [v2:172.21.15.194:6800/1874338403,v1:172.21.15.194:6801/1874338403] [v2:172.21.15.194:6802/1874338403,v1:172.21.15.194:6803/1874338403] exists,up 509ec8cc-5042-428e-9a69-8e602b5afce8', 'osd.5 up in weight 1 up_from 38 up_thru 132 down_at 0 last_clean_interval [0,0) [v2:172.21.15.194:6808/3989986662,v1:172.21.15.194:6809/3989986662] [v2:172.21.15.194:6810/3989986662,v1:172.21.15.194:6811/3989986662] exists,up bbdc1e67-fd5b-49d0-9e42-cf151d167943', 'osd.6 up in weight 1 up_from 46 up_thru 118 down_at 0 last_clean_interval [0,0) [v2:172.21.15.194:6816/2607555245,v1:172.21.15.194:6817/2607555245] [v2:172.21.15.194:6818/2607555245,v1:172.21.15.194:6819/2607555245] exists,up 7b69d014-f4c0-4e73-a14e-7d51b7ee3ea2', 'osd.7 up in weight 1 up_from 54 up_thru 123 down_at 0 last_clean_interval [0,0) [v2:172.21.15.194:6824/1051622545,v1:172.21.15.194:6825/1051622545] [v2:172.21.15.194:6826/1051622545,v1:172.21.15.194:6827/1051622545] exists,up 02506268-102c-4654-9b5c-68d36c28f7f7'] 2023-10-09T05:08:27.141 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon mon.a config show 2023-10-09T05:08:29.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:28 smithi133 bash[105589]: cluster 2023-10-09T05:08:27.717925+0000 mgr.y (mgr.25277) 595 : cluster 0 pgmap v546: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:29.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:28 smithi133 bash[105589]: audit 2023-10-09T05:08:28.439135+0000 mon.a (mon.0) 741 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:08:29.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:28 smithi133 bash[102858]: cluster 2023-10-09T05:08:27.717925+0000 mgr.y (mgr.25277) 595 : cluster 0 pgmap v546: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:29.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:28 smithi133 bash[102858]: audit 2023-10-09T05:08:28.439135+0000 mon.a (mon.0) 741 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:08:29.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:28 smithi194 bash[92666]: cluster 2023-10-09T05:08:27.717925+0000 mgr.y (mgr.25277) 595 : cluster 0 pgmap v546: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:29.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:28 smithi194 bash[92666]: audit 2023-10-09T05:08:28.439135+0000 mon.a (mon.0) 741 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:08:31.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:30 smithi133 bash[102858]: cluster 2023-10-09T05:08:29.718843+0000 mgr.y (mgr.25277) 596 : cluster 0 pgmap v547: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:31.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:30 smithi133 bash[105589]: cluster 2023-10-09T05:08:29.718843+0000 mgr.y (mgr.25277) 596 : cluster 0 pgmap v547: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:31.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:30 smithi194 bash[92666]: cluster 2023-10-09T05:08:29.718843+0000 mgr.y (mgr.25277) 596 : cluster 0 pgmap v547: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:31.919 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:08:33.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:32 smithi133 bash[102858]: cluster 2023-10-09T05:08:31.719716+0000 mgr.y (mgr.25277) 597 : cluster 0 pgmap v548: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:33.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:32 smithi133 bash[105589]: cluster 2023-10-09T05:08:31.719716+0000 mgr.y (mgr.25277) 597 : cluster 0 pgmap v548: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:33.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:32 smithi194 bash[92666]: cluster 2023-10-09T05:08:31.719716+0000 mgr.y (mgr.25277) 597 : cluster 0 pgmap v548: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:34.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:08:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:08:34] "GET /metrics HTTP/1.1" 200 33984 "" "Prometheus/2.43.0" 2023-10-09T05:08:34.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:34 smithi133 bash[105589]: cluster 2023-10-09T05:08:33.720500+0000 mgr.y (mgr.25277) 598 : cluster 0 pgmap v549: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:34.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:34 smithi133 bash[102858]: cluster 2023-10-09T05:08:33.720500+0000 mgr.y (mgr.25277) 598 : cluster 0 pgmap v549: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:34.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:34 smithi194 bash[92666]: cluster 2023-10-09T05:08:33.720500+0000 mgr.y (mgr.25277) 598 : cluster 0 pgmap v549: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:35.383 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:08:35.383 INFO:teuthology.orchestra.run.smithi133.stdout: "name": "mon.a", 2023-10-09T05:08:35.383 INFO:teuthology.orchestra.run.smithi133.stdout: "cluster": "ceph", 2023-10-09T05:08:35.383 INFO:teuthology.orchestra.run.smithi133.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2023-10-09T05:08:35.384 INFO:teuthology.orchestra.run.smithi133.stdout: "admin_socket_mode": "", 2023-10-09T05:08:35.384 INFO:teuthology.orchestra.run.smithi133.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2023-10-09T05:08:35.384 INFO:teuthology.orchestra.run.smithi133.stdout: "auth_client_required": "cephx, none", 2023-10-09T05:08:35.384 INFO:teuthology.orchestra.run.smithi133.stdout: "auth_cluster_required": "cephx", 2023-10-09T05:08:35.384 INFO:teuthology.orchestra.run.smithi133.stdout: "auth_debug": "true", 2023-10-09T05:08:35.384 INFO:teuthology.orchestra.run.smithi133.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2023-10-09T05:08:35.384 INFO:teuthology.orchestra.run.smithi133.stdout: "auth_mon_ticket_ttl": "660.000000", 2023-10-09T05:08:35.384 INFO:teuthology.orchestra.run.smithi133.stdout: "auth_service_required": "cephx", 2023-10-09T05:08:35.384 INFO:teuthology.orchestra.run.smithi133.stdout: "auth_service_ticket_ttl": "240.000000", 2023-10-09T05:08:35.384 INFO:teuthology.orchestra.run.smithi133.stdout: "auth_supported": "", 2023-10-09T05:08:35.384 INFO:teuthology.orchestra.run.smithi133.stdout: "bdev_aio": "true", 2023-10-09T05:08:35.385 INFO:teuthology.orchestra.run.smithi133.stdout: "bdev_aio_max_queue_depth": "1024", 2023-10-09T05:08:35.385 INFO:teuthology.orchestra.run.smithi133.stdout: "bdev_aio_poll_ms": "250", 2023-10-09T05:08:35.385 INFO:teuthology.orchestra.run.smithi133.stdout: "bdev_aio_reap_max": "16", 2023-10-09T05:08:35.385 INFO:teuthology.orchestra.run.smithi133.stdout: "bdev_async_discard": "false", 2023-10-09T05:08:35.385 INFO:teuthology.orchestra.run.smithi133.stdout: "bdev_block_size": "4096", 2023-10-09T05:08:35.385 INFO:teuthology.orchestra.run.smithi133.stdout: "bdev_debug_aio": "false", 2023-10-09T05:08:35.385 INFO:teuthology.orchestra.run.smithi133.stdout: "bdev_debug_aio_log_age": "5.000000", 2023-10-09T05:08:35.385 INFO:teuthology.orchestra.run.smithi133.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2023-10-09T05:08:35.385 INFO:teuthology.orchestra.run.smithi133.stdout: "bdev_debug_inflight_ios": "false", 2023-10-09T05:08:35.385 INFO:teuthology.orchestra.run.smithi133.stdout: "bdev_enable_discard": "false", 2023-10-09T05:08:35.385 INFO:teuthology.orchestra.run.smithi133.stdout: "bdev_flock_retry": "3", 2023-10-09T05:08:35.386 INFO:teuthology.orchestra.run.smithi133.stdout: "bdev_flock_retry_interval": "0.100000", 2023-10-09T05:08:35.386 INFO:teuthology.orchestra.run.smithi133.stdout: "bdev_inject_crash": "0", 2023-10-09T05:08:35.386 INFO:teuthology.orchestra.run.smithi133.stdout: "bdev_inject_crash_flush_delay": "2", 2023-10-09T05:08:35.386 INFO:teuthology.orchestra.run.smithi133.stdout: "bdev_ioring": "false", 2023-10-09T05:08:35.386 INFO:teuthology.orchestra.run.smithi133.stdout: "bdev_ioring_hipri": "false", 2023-10-09T05:08:35.386 INFO:teuthology.orchestra.run.smithi133.stdout: "bdev_ioring_sqthread_poll": "false", 2023-10-09T05:08:35.386 INFO:teuthology.orchestra.run.smithi133.stdout: "bdev_nvme_unbind_from_kernel": "false", 2023-10-09T05:08:35.386 INFO:teuthology.orchestra.run.smithi133.stdout: "bdev_read_buffer_alignment": "4096", 2023-10-09T05:08:35.387 INFO:teuthology.orchestra.run.smithi133.stdout: "bdev_read_preallocated_huge_buffers": "", 2023-10-09T05:08:35.387 INFO:teuthology.orchestra.run.smithi133.stdout: "bdev_type": "", 2023-10-09T05:08:35.387 INFO:teuthology.orchestra.run.smithi133.stdout: "bluefs_alloc_size": "1048576", 2023-10-09T05:08:35.387 INFO:teuthology.orchestra.run.smithi133.stdout: "bluefs_allocator": "hybrid", 2023-10-09T05:08:35.387 INFO:teuthology.orchestra.run.smithi133.stdout: "bluefs_buffered_io": "true", 2023-10-09T05:08:35.387 INFO:teuthology.orchestra.run.smithi133.stdout: "bluefs_check_for_zeros": "false", 2023-10-09T05:08:35.387 INFO:teuthology.orchestra.run.smithi133.stdout: "bluefs_check_volume_selector_often": "false", 2023-10-09T05:08:35.387 INFO:teuthology.orchestra.run.smithi133.stdout: "bluefs_check_volume_selector_on_umount": "false", 2023-10-09T05:08:35.388 INFO:teuthology.orchestra.run.smithi133.stdout: "bluefs_compact_log_sync": "false", 2023-10-09T05:08:35.388 INFO:teuthology.orchestra.run.smithi133.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2023-10-09T05:08:35.388 INFO:teuthology.orchestra.run.smithi133.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2023-10-09T05:08:35.388 INFO:teuthology.orchestra.run.smithi133.stdout: "bluefs_log_compact_min_size": "16777216", 2023-10-09T05:08:35.388 INFO:teuthology.orchestra.run.smithi133.stdout: "bluefs_log_replay_check_allocations": "true", 2023-10-09T05:08:35.388 INFO:teuthology.orchestra.run.smithi133.stdout: "bluefs_max_log_runway": "4194304", 2023-10-09T05:08:35.388 INFO:teuthology.orchestra.run.smithi133.stdout: "bluefs_max_prefetch": "1048576", 2023-10-09T05:08:35.388 INFO:teuthology.orchestra.run.smithi133.stdout: "bluefs_min_flush_size": "524288", 2023-10-09T05:08:35.388 INFO:teuthology.orchestra.run.smithi133.stdout: "bluefs_min_log_runway": "1048576", 2023-10-09T05:08:35.388 INFO:teuthology.orchestra.run.smithi133.stdout: "bluefs_replay_recovery": "false", 2023-10-09T05:08:35.388 INFO:teuthology.orchestra.run.smithi133.stdout: "bluefs_replay_recovery_disable_compact": "false", 2023-10-09T05:08:35.388 INFO:teuthology.orchestra.run.smithi133.stdout: "bluefs_shared_alloc_size": "65536", 2023-10-09T05:08:35.389 INFO:teuthology.orchestra.run.smithi133.stdout: "bluefs_sync_write": "false", 2023-10-09T05:08:35.389 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2023-10-09T05:08:35.389 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2023-10-09T05:08:35.389 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2023-10-09T05:08:35.389 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_allocation_from_file": "true", 2023-10-09T05:08:35.389 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_allocator": "hybrid", 2023-10-09T05:08:35.389 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2023-10-09T05:08:35.389 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2023-10-09T05:08:35.389 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2023-10-09T05:08:35.390 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2023-10-09T05:08:35.390 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2023-10-09T05:08:35.390 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_bitmapallocator_span_size": "1024", 2023-10-09T05:08:35.390 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_blobid_prealloc": "10240", 2023-10-09T05:08:35.390 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_block_create": "true", 2023-10-09T05:08:35.390 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_block_db_create": "false", 2023-10-09T05:08:35.390 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_block_db_path": "", 2023-10-09T05:08:35.390 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_block_db_size": "0", 2023-10-09T05:08:35.390 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_block_path": "", 2023-10-09T05:08:35.390 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_block_preallocate_file": "false", 2023-10-09T05:08:35.390 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_block_size": "107374182400", 2023-10-09T05:08:35.391 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_block_wal_create": "false", 2023-10-09T05:08:35.391 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_block_wal_path": "", 2023-10-09T05:08:35.391 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_block_wal_size": "100663296", 2023-10-09T05:08:35.391 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_bluefs": "true", 2023-10-09T05:08:35.391 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2023-10-09T05:08:35.391 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_bluefs_env_mirror": "false", 2023-10-09T05:08:35.391 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_bluefs_max_free": "10737418240", 2023-10-09T05:08:35.391 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2023-10-09T05:08:35.391 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T05:08:35.391 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T05:08:35.391 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2023-10-09T05:08:35.392 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T05:08:35.392 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_cache_autotune": "true", 2023-10-09T05:08:35.392 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_cache_autotune_interval": "5.000000", 2023-10-09T05:08:35.392 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2023-10-09T05:08:35.392 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_cache_kv_ratio": "0.450000", 2023-10-09T05:08:35.392 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_cache_meta_ratio": "0.450000", 2023-10-09T05:08:35.392 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_cache_size": "0", 2023-10-09T05:08:35.392 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_cache_size_hdd": "1073741824", 2023-10-09T05:08:35.392 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_cache_size_ssd": "3221225472", 2023-10-09T05:08:35.392 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_cache_trim_interval": "0.050000", 2023-10-09T05:08:35.392 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2023-10-09T05:08:35.392 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_cache_type": "2q", 2023-10-09T05:08:35.393 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2023-10-09T05:08:35.393 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_clone_cow": "true", 2023-10-09T05:08:35.393 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_compression_algorithm": "snappy", 2023-10-09T05:08:35.393 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_compression_max_blob_size": "0", 2023-10-09T05:08:35.393 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2023-10-09T05:08:35.393 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2023-10-09T05:08:35.393 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_compression_min_blob_size": "0", 2023-10-09T05:08:35.393 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2023-10-09T05:08:35.393 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2023-10-09T05:08:35.393 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_compression_mode": "none", 2023-10-09T05:08:35.393 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_compression_required_ratio": "0.875000", 2023-10-09T05:08:35.393 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_csum_type": "crc32c", 2023-10-09T05:08:35.393 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_debug_enforce_settings": "default", 2023-10-09T05:08:35.394 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_debug_freelist": "false", 2023-10-09T05:08:35.394 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_debug_fsck_abort": "false", 2023-10-09T05:08:35.394 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2023-10-09T05:08:35.394 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_debug_inject_bug21040": "false", 2023-10-09T05:08:35.394 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2023-10-09T05:08:35.394 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_debug_inject_read_err": "false", 2023-10-09T05:08:35.394 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_debug_legacy_omap": "false", 2023-10-09T05:08:35.394 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_debug_no_reuse_blocks": "false", 2023-10-09T05:08:35.394 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_debug_omit_block_device_write": "false", 2023-10-09T05:08:35.394 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_debug_omit_kv_commit": "false", 2023-10-09T05:08:35.394 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2023-10-09T05:08:35.394 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_debug_prefill": "0.000000", 2023-10-09T05:08:35.395 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_debug_prefragment_max": "1048576", 2023-10-09T05:08:35.395 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_debug_random_read_err": "0.000000", 2023-10-09T05:08:35.395 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2023-10-09T05:08:35.395 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_debug_small_allocations": "0", 2023-10-09T05:08:35.395 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2023-10-09T05:08:35.395 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_default_buffered_read": "true", 2023-10-09T05:08:35.395 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_default_buffered_write": "false", 2023-10-09T05:08:35.395 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_deferred_batch_ops": "0", 2023-10-09T05:08:35.395 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2023-10-09T05:08:35.395 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2023-10-09T05:08:35.396 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2023-10-09T05:08:35.396 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_extent_map_shard_max_size": "1200", 2023-10-09T05:08:35.396 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_extent_map_shard_min_size": "150", 2023-10-09T05:08:35.396 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_extent_map_shard_target_size": "500", 2023-10-09T05:08:35.396 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2023-10-09T05:08:35.396 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_fail_eio": "false", 2023-10-09T05:08:35.396 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_freelist_blocks_per_key": "128", 2023-10-09T05:08:35.396 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2023-10-09T05:08:35.396 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2023-10-09T05:08:35.396 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2023-10-09T05:08:35.396 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_fsck_on_mkfs": "true", 2023-10-09T05:08:35.396 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2023-10-09T05:08:35.397 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_fsck_on_mount": "false", 2023-10-09T05:08:35.397 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_fsck_on_mount_deep": "false", 2023-10-09T05:08:35.397 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_fsck_on_umount": "false", 2023-10-09T05:08:35.397 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_fsck_on_umount_deep": "false", 2023-10-09T05:08:35.397 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2023-10-09T05:08:35.397 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_fsck_quick_fix_threads": "2", 2023-10-09T05:08:35.397 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2023-10-09T05:08:35.397 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2023-10-09T05:08:35.397 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_gc_enable_blob_threshold": "0", 2023-10-09T05:08:35.397 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_gc_enable_total_threshold": "0", 2023-10-09T05:08:35.397 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2023-10-09T05:08:35.397 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_ignore_data_csum": "false", 2023-10-09T05:08:35.398 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2023-10-09T05:08:35.398 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_kvbackend": "rocksdb", 2023-10-09T05:08:35.398 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_log_collection_list_age": "60.000000", 2023-10-09T05:08:35.398 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2023-10-09T05:08:35.398 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_log_op_age": "5.000000", 2023-10-09T05:08:35.398 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_max_alloc_size": "0", 2023-10-09T05:08:35.398 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_max_blob_size": "0", 2023-10-09T05:08:35.398 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_max_blob_size_hdd": "65536", 2023-10-09T05:08:35.398 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_max_blob_size_ssd": "65536", 2023-10-09T05:08:35.398 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_max_defer_interval": "3.000000", 2023-10-09T05:08:35.398 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_max_deferred_txc": "32", 2023-10-09T05:08:35.399 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_min_alloc_size": "0", 2023-10-09T05:08:35.399 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_min_alloc_size_hdd": "4096", 2023-10-09T05:08:35.399 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_min_alloc_size_ssd": "4096", 2023-10-09T05:08:35.399 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_nid_prealloc": "1024", 2023-10-09T05:08:35.399 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_prefer_deferred_size": "0", 2023-10-09T05:08:35.399 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2023-10-09T05:08:35.399 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2023-10-09T05:08:35.399 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_qfsck_on_mount": "true", 2023-10-09T05:08:35.399 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_retry_disk_reads": "3", 2023-10-09T05:08:35.399 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_rocksdb_cf": "true", 2023-10-09T05:08:35.399 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2023-10-09T05:08:35.400 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_rocksdb_options": "compression=kNoCompression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2023-10-09T05:08:35.400 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_rocksdb_options_annex": "", 2023-10-09T05:08:35.400 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_spdk_coremask": "0x1", 2023-10-09T05:08:35.400 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_spdk_io_sleep": "5", 2023-10-09T05:08:35.400 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_spdk_max_io_completion": "0", 2023-10-09T05:08:35.400 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_spdk_mem": "512", 2023-10-09T05:08:35.400 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_sync_submit_transaction": "false", 2023-10-09T05:08:35.400 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_throttle_bytes": "67108864", 2023-10-09T05:08:35.400 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_throttle_cost_per_io": "0", 2023-10-09T05:08:35.400 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2023-10-09T05:08:35.400 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2023-10-09T05:08:35.401 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2023-10-09T05:08:35.401 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_throttle_trace_rate": "0.000000", 2023-10-09T05:08:35.401 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_tracing": "false", 2023-10-09T05:08:35.401 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2023-10-09T05:08:35.401 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2023-10-09T05:08:35.401 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_volume_selection_reserved": "0", 2023-10-09T05:08:35.401 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2023-10-09T05:08:35.401 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2023-10-09T05:08:35.401 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_warn_on_legacy_statfs": "true", 2023-10-09T05:08:35.401 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2023-10-09T05:08:35.401 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2023-10-09T05:08:35.401 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2023-10-09T05:08:35.402 INFO:teuthology.orchestra.run.smithi133.stdout: "bluestore_zero_block_detection": "false", 2023-10-09T05:08:35.402 INFO:teuthology.orchestra.run.smithi133.stdout: "cephadm_path": "/usr/sbin/cephadm", 2023-10-09T05:08:35.402 INFO:teuthology.orchestra.run.smithi133.stdout: "cephfs_mirror_action_update_interval": "2", 2023-10-09T05:08:35.402 INFO:teuthology.orchestra.run.smithi133.stdout: "cephfs_mirror_directory_scan_interval": "10", 2023-10-09T05:08:35.402 INFO:teuthology.orchestra.run.smithi133.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2023-10-09T05:08:35.402 INFO:teuthology.orchestra.run.smithi133.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2023-10-09T05:08:35.402 INFO:teuthology.orchestra.run.smithi133.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2023-10-09T05:08:35.402 INFO:teuthology.orchestra.run.smithi133.stdout: "cephfs_mirror_mount_timeout": "10", 2023-10-09T05:08:35.402 INFO:teuthology.orchestra.run.smithi133.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2023-10-09T05:08:35.402 INFO:teuthology.orchestra.run.smithi133.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2023-10-09T05:08:35.402 INFO:teuthology.orchestra.run.smithi133.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2023-10-09T05:08:35.403 INFO:teuthology.orchestra.run.smithi133.stdout: "cephsqlite_blocklist_dead_locker": "true", 2023-10-09T05:08:35.403 INFO:teuthology.orchestra.run.smithi133.stdout: "cephsqlite_lock_renewal_interval": "2000", 2023-10-09T05:08:35.403 INFO:teuthology.orchestra.run.smithi133.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2023-10-09T05:08:35.403 INFO:teuthology.orchestra.run.smithi133.stdout: "cephx_cluster_require_signatures": "false", 2023-10-09T05:08:35.403 INFO:teuthology.orchestra.run.smithi133.stdout: "cephx_cluster_require_version": "2", 2023-10-09T05:08:35.403 INFO:teuthology.orchestra.run.smithi133.stdout: "cephx_require_signatures": "false", 2023-10-09T05:08:35.403 INFO:teuthology.orchestra.run.smithi133.stdout: "cephx_require_version": "2", 2023-10-09T05:08:35.403 INFO:teuthology.orchestra.run.smithi133.stdout: "cephx_service_require_signatures": "false", 2023-10-09T05:08:35.403 INFO:teuthology.orchestra.run.smithi133.stdout: "cephx_service_require_version": "2", 2023-10-09T05:08:35.403 INFO:teuthology.orchestra.run.smithi133.stdout: "cephx_sign_messages": "true", 2023-10-09T05:08:35.403 INFO:teuthology.orchestra.run.smithi133.stdout: "chdir": "", 2023-10-09T05:08:35.403 INFO:teuthology.orchestra.run.smithi133.stdout: "client_acl_type": "", 2023-10-09T05:08:35.404 INFO:teuthology.orchestra.run.smithi133.stdout: "client_asio_thread_count": "2", 2023-10-09T05:08:35.404 INFO:teuthology.orchestra.run.smithi133.stdout: "client_cache_mid": "0.750000", 2023-10-09T05:08:35.404 INFO:teuthology.orchestra.run.smithi133.stdout: "client_cache_size": "16384", 2023-10-09T05:08:35.404 INFO:teuthology.orchestra.run.smithi133.stdout: "client_caps_release_delay": "5", 2023-10-09T05:08:35.404 INFO:teuthology.orchestra.run.smithi133.stdout: "client_check_pool_perm": "true", 2023-10-09T05:08:35.404 INFO:teuthology.orchestra.run.smithi133.stdout: "client_collect_and_send_global_metrics": "false", 2023-10-09T05:08:35.404 INFO:teuthology.orchestra.run.smithi133.stdout: "client_debug_force_sync_read": "false", 2023-10-09T05:08:35.404 INFO:teuthology.orchestra.run.smithi133.stdout: "client_debug_getattr_caps": "false", 2023-10-09T05:08:35.404 INFO:teuthology.orchestra.run.smithi133.stdout: "client_debug_inject_tick_delay": "0", 2023-10-09T05:08:35.404 INFO:teuthology.orchestra.run.smithi133.stdout: "client_die_on_failed_dentry_invalidate": "true", 2023-10-09T05:08:35.404 INFO:teuthology.orchestra.run.smithi133.stdout: "client_die_on_failed_remount": "false", 2023-10-09T05:08:35.404 INFO:teuthology.orchestra.run.smithi133.stdout: "client_dirsize_rbytes": "true", 2023-10-09T05:08:35.405 INFO:teuthology.orchestra.run.smithi133.stdout: "client_force_lazyio": "false", 2023-10-09T05:08:35.405 INFO:teuthology.orchestra.run.smithi133.stdout: "client_fs": "", 2023-10-09T05:08:35.405 INFO:teuthology.orchestra.run.smithi133.stdout: "client_inject_fixed_oldest_tid": "false", 2023-10-09T05:08:35.405 INFO:teuthology.orchestra.run.smithi133.stdout: "client_inject_release_failure": "false", 2023-10-09T05:08:35.405 INFO:teuthology.orchestra.run.smithi133.stdout: "client_max_inline_size": "4096", 2023-10-09T05:08:35.405 INFO:teuthology.orchestra.run.smithi133.stdout: "client_max_retries_on_remount_failure": "5", 2023-10-09T05:08:35.405 INFO:teuthology.orchestra.run.smithi133.stdout: "client_mds_namespace": "", 2023-10-09T05:08:35.405 INFO:teuthology.orchestra.run.smithi133.stdout: "client_metadata": "", 2023-10-09T05:08:35.405 INFO:teuthology.orchestra.run.smithi133.stdout: "client_mount_gid": "-1", 2023-10-09T05:08:35.405 INFO:teuthology.orchestra.run.smithi133.stdout: "client_mount_timeout": "300", 2023-10-09T05:08:35.405 INFO:teuthology.orchestra.run.smithi133.stdout: "client_mount_uid": "-1", 2023-10-09T05:08:35.406 INFO:teuthology.orchestra.run.smithi133.stdout: "client_mountpoint": "/", 2023-10-09T05:08:35.406 INFO:teuthology.orchestra.run.smithi133.stdout: "client_notify_timeout": "10", 2023-10-09T05:08:35.406 INFO:teuthology.orchestra.run.smithi133.stdout: "client_oc": "true", 2023-10-09T05:08:35.406 INFO:teuthology.orchestra.run.smithi133.stdout: "client_oc_max_dirty": "104857600", 2023-10-09T05:08:35.406 INFO:teuthology.orchestra.run.smithi133.stdout: "client_oc_max_dirty_age": "5.000000", 2023-10-09T05:08:35.406 INFO:teuthology.orchestra.run.smithi133.stdout: "client_oc_max_objects": "1000", 2023-10-09T05:08:35.406 INFO:teuthology.orchestra.run.smithi133.stdout: "client_oc_size": "209715200", 2023-10-09T05:08:35.406 INFO:teuthology.orchestra.run.smithi133.stdout: "client_oc_target_dirty": "8388608", 2023-10-09T05:08:35.406 INFO:teuthology.orchestra.run.smithi133.stdout: "client_permissions": "true", 2023-10-09T05:08:35.406 INFO:teuthology.orchestra.run.smithi133.stdout: "client_quota": "true", 2023-10-09T05:08:35.406 INFO:teuthology.orchestra.run.smithi133.stdout: "client_quota_df": "true", 2023-10-09T05:08:35.407 INFO:teuthology.orchestra.run.smithi133.stdout: "client_readahead_max_bytes": "0", 2023-10-09T05:08:35.407 INFO:teuthology.orchestra.run.smithi133.stdout: "client_readahead_max_periods": "4", 2023-10-09T05:08:35.407 INFO:teuthology.orchestra.run.smithi133.stdout: "client_readahead_min": "131072", 2023-10-09T05:08:35.407 INFO:teuthology.orchestra.run.smithi133.stdout: "client_reconnect_stale": "false", 2023-10-09T05:08:35.407 INFO:teuthology.orchestra.run.smithi133.stdout: "client_shutdown_timeout": "30", 2023-10-09T05:08:35.407 INFO:teuthology.orchestra.run.smithi133.stdout: "client_snapdir": ".snap", 2023-10-09T05:08:35.407 INFO:teuthology.orchestra.run.smithi133.stdout: "client_tick_interval": "1", 2023-10-09T05:08:35.407 INFO:teuthology.orchestra.run.smithi133.stdout: "client_trace": "", 2023-10-09T05:08:35.407 INFO:teuthology.orchestra.run.smithi133.stdout: "client_try_dentry_invalidate": "false", 2023-10-09T05:08:35.407 INFO:teuthology.orchestra.run.smithi133.stdout: "client_use_faked_inos": "false", 2023-10-09T05:08:35.407 INFO:teuthology.orchestra.run.smithi133.stdout: "client_use_random_mds": "false", 2023-10-09T05:08:35.408 INFO:teuthology.orchestra.run.smithi133.stdout: "clog_to_graylog": "false", 2023-10-09T05:08:35.408 INFO:teuthology.orchestra.run.smithi133.stdout: "clog_to_graylog_host": "127.0.0.1", 2023-10-09T05:08:35.408 INFO:teuthology.orchestra.run.smithi133.stdout: "clog_to_graylog_port": "12201", 2023-10-09T05:08:35.408 INFO:teuthology.orchestra.run.smithi133.stdout: "clog_to_monitors": "default=true", 2023-10-09T05:08:35.408 INFO:teuthology.orchestra.run.smithi133.stdout: "clog_to_syslog": "false", 2023-10-09T05:08:35.408 INFO:teuthology.orchestra.run.smithi133.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2023-10-09T05:08:35.408 INFO:teuthology.orchestra.run.smithi133.stdout: "clog_to_syslog_level": "info", 2023-10-09T05:08:35.408 INFO:teuthology.orchestra.run.smithi133.stdout: "cluster_addr": "-", 2023-10-09T05:08:35.408 INFO:teuthology.orchestra.run.smithi133.stdout: "cluster_network": "", 2023-10-09T05:08:35.408 INFO:teuthology.orchestra.run.smithi133.stdout: "cluster_network_interface": "", 2023-10-09T05:08:35.408 INFO:teuthology.orchestra.run.smithi133.stdout: "compressor_zlib_isal": "false", 2023-10-09T05:08:35.408 INFO:teuthology.orchestra.run.smithi133.stdout: "compressor_zlib_level": "5", 2023-10-09T05:08:35.409 INFO:teuthology.orchestra.run.smithi133.stdout: "compressor_zlib_winsize": "-15", 2023-10-09T05:08:35.409 INFO:teuthology.orchestra.run.smithi133.stdout: "compressor_zstd_level": "1", 2023-10-09T05:08:35.409 INFO:teuthology.orchestra.run.smithi133.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c", 2023-10-09T05:08:35.409 INFO:teuthology.orchestra.run.smithi133.stdout: "crash_dir": "/var/lib/ceph/crash", 2023-10-09T05:08:35.409 INFO:teuthology.orchestra.run.smithi133.stdout: "crimson_alien_op_num_threads": "6", 2023-10-09T05:08:35.409 INFO:teuthology.orchestra.run.smithi133.stdout: "crimson_alien_thread_cpu_cores": "", 2023-10-09T05:08:35.409 INFO:teuthology.orchestra.run.smithi133.stdout: "crimson_osd_obc_lru_size": "10", 2023-10-09T05:08:35.409 INFO:teuthology.orchestra.run.smithi133.stdout: "crimson_osd_scheduler_concurrency": "0", 2023-10-09T05:08:35.409 INFO:teuthology.orchestra.run.smithi133.stdout: "crimson_seastar_smp": "1", 2023-10-09T05:08:35.409 INFO:teuthology.orchestra.run.smithi133.stdout: "crush_location": "", 2023-10-09T05:08:35.409 INFO:teuthology.orchestra.run.smithi133.stdout: "crush_location_hook": "", 2023-10-09T05:08:35.410 INFO:teuthology.orchestra.run.smithi133.stdout: "crush_location_hook_timeout": "10", 2023-10-09T05:08:35.410 INFO:teuthology.orchestra.run.smithi133.stdout: "daemonize": "false", 2023-10-09T05:08:35.410 INFO:teuthology.orchestra.run.smithi133.stdout: "daos_pool": "tank", 2023-10-09T05:08:35.410 INFO:teuthology.orchestra.run.smithi133.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2023-10-09T05:08:35.410 INFO:teuthology.orchestra.run.smithi133.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2023-10-09T05:08:35.410 INFO:teuthology.orchestra.run.smithi133.stdout: "dbstore_db_name_prefix": "dbstore", 2023-10-09T05:08:35.410 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_alienstore": "0/5", 2023-10-09T05:08:35.410 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_allow_any_pool_priority": "false", 2023-10-09T05:08:35.410 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_asok": "1/5", 2023-10-09T05:08:35.410 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_asok_assert_abort": "false", 2023-10-09T05:08:35.410 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_asserts_on_shutdown": "true", 2023-10-09T05:08:35.411 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_auth": "1/5", 2023-10-09T05:08:35.411 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_bdev": "1/3", 2023-10-09T05:08:35.411 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_bluefs": "1/5", 2023-10-09T05:08:35.411 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_bluestore": "1/5", 2023-10-09T05:08:35.411 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_buffer": "0/1", 2023-10-09T05:08:35.411 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_ceph_exporter": "1/5", 2023-10-09T05:08:35.411 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_cephfs_mirror": "0/5", 2023-10-09T05:08:35.411 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_cephsqlite": "0/5", 2023-10-09T05:08:35.411 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_client": "0/5", 2023-10-09T05:08:35.411 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_compressor": "1/5", 2023-10-09T05:08:35.411 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_context": "0/1", 2023-10-09T05:08:35.411 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_crush": "1/1", 2023-10-09T05:08:35.412 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_crypto": "1/5", 2023-10-09T05:08:35.412 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_cyanstore": "0/5", 2023-10-09T05:08:35.412 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_deliberately_leak_memory": "false", 2023-10-09T05:08:35.412 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_disable_randomized_ping": "false", 2023-10-09T05:08:35.412 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_dpdk": "1/5", 2023-10-09T05:08:35.412 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_eventtrace": "1/5", 2023-10-09T05:08:35.412 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_filer": "0/1", 2023-10-09T05:08:35.412 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_filestore": "1/3", 2023-10-09T05:08:35.412 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_finisher": "1/1", 2023-10-09T05:08:35.412 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_fuse": "1/5", 2023-10-09T05:08:35.412 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_heartbeat_testing_span": "0", 2023-10-09T05:08:35.412 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_heartbeatmap": "1/5", 2023-10-09T05:08:35.413 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_immutable_obj_cache": "0/5", 2023-10-09T05:08:35.413 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_javaclient": "1/5", 2023-10-09T05:08:35.413 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_journal": "1/3", 2023-10-09T05:08:35.413 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_journaler": "0/5", 2023-10-09T05:08:35.413 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_kstore": "1/5", 2023-10-09T05:08:35.413 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_leveldb": "4/5", 2023-10-09T05:08:35.413 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_lockdep": "0/1", 2023-10-09T05:08:35.413 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_mclock": "1/5", 2023-10-09T05:08:35.413 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_mds": "1/5", 2023-10-09T05:08:35.413 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_mds_balancer": "1/5", 2023-10-09T05:08:35.413 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_mds_locker": "1/5", 2023-10-09T05:08:35.413 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_mds_log": "1/5", 2023-10-09T05:08:35.414 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_mds_log_expire": "1/5", 2023-10-09T05:08:35.414 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_mds_migrator": "1/5", 2023-10-09T05:08:35.414 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_memstore": "1/5", 2023-10-09T05:08:35.414 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_mgr": "2/5", 2023-10-09T05:08:35.414 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_mgrc": "1/5", 2023-10-09T05:08:35.414 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_mon": "20/20", 2023-10-09T05:08:35.414 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_monc": "0/10", 2023-10-09T05:08:35.414 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_ms": "1/1", 2023-10-09T05:08:35.414 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_none": "0/5", 2023-10-09T05:08:35.414 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_objclass": "0/5", 2023-10-09T05:08:35.414 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_objectcacher": "0/5", 2023-10-09T05:08:35.415 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_objecter": "0/1", 2023-10-09T05:08:35.415 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_optracker": "0/5", 2023-10-09T05:08:35.415 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_osd": "1/5", 2023-10-09T05:08:35.415 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_paxos": "20/20", 2023-10-09T05:08:35.415 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_perfcounter": "1/5", 2023-10-09T05:08:35.415 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_prioritycache": "1/5", 2023-10-09T05:08:35.415 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_rados": "0/5", 2023-10-09T05:08:35.415 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_rbd": "0/5", 2023-10-09T05:08:35.415 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_rbd_mirror": "0/5", 2023-10-09T05:08:35.415 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_rbd_pwl": "0/5", 2023-10-09T05:08:35.415 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_rbd_replay": "0/5", 2023-10-09T05:08:35.416 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_refs": "0/0", 2023-10-09T05:08:35.416 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_reserver": "1/1", 2023-10-09T05:08:35.416 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_rgw": "1/5", 2023-10-09T05:08:35.416 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_rgw_access": "1/5", 2023-10-09T05:08:35.416 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_rgw_datacache": "1/5", 2023-10-09T05:08:35.416 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_rgw_dbstore": "1/5", 2023-10-09T05:08:35.416 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_rgw_flight": "1/5", 2023-10-09T05:08:35.416 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_rgw_sync": "1/5", 2023-10-09T05:08:35.416 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_rocksdb": "4/5", 2023-10-09T05:08:35.416 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_seastore": "0/5", 2023-10-09T05:08:35.416 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_seastore_backref": "0/5", 2023-10-09T05:08:35.416 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_seastore_cache": "0/5", 2023-10-09T05:08:35.416 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_seastore_cleaner": "0/5", 2023-10-09T05:08:35.417 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_seastore_device": "0/5", 2023-10-09T05:08:35.417 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_seastore_epm": "0/5", 2023-10-09T05:08:35.417 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_seastore_fixedkv_tree": "0/5", 2023-10-09T05:08:35.417 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_seastore_journal": "0/5", 2023-10-09T05:08:35.417 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_seastore_lba": "0/5", 2023-10-09T05:08:35.417 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_seastore_odata": "0/5", 2023-10-09T05:08:35.417 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_seastore_omap": "0/5", 2023-10-09T05:08:35.417 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_seastore_onode": "0/5", 2023-10-09T05:08:35.417 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_seastore_t": "0/5", 2023-10-09T05:08:35.417 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_seastore_tm": "0/5", 2023-10-09T05:08:35.417 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_striper": "0/1", 2023-10-09T05:08:35.417 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_test": "0/5", 2023-10-09T05:08:35.418 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_throttle": "1/1", 2023-10-09T05:08:35.418 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_timer": "0/1", 2023-10-09T05:08:35.418 INFO:teuthology.orchestra.run.smithi133.stdout: "debug_tp": "0/5", 2023-10-09T05:08:35.418 INFO:teuthology.orchestra.run.smithi133.stdout: "device_failure_prediction_mode": "none", 2023-10-09T05:08:35.418 INFO:teuthology.orchestra.run.smithi133.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2023-10-09T05:08:35.418 INFO:teuthology.orchestra.run.smithi133.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2023-10-09T05:08:35.418 INFO:teuthology.orchestra.run.smithi133.stdout: "err_to_graylog": "false", 2023-10-09T05:08:35.418 INFO:teuthology.orchestra.run.smithi133.stdout: "err_to_journald": "false", 2023-10-09T05:08:35.418 INFO:teuthology.orchestra.run.smithi133.stdout: "err_to_stderr": "true", 2023-10-09T05:08:35.418 INFO:teuthology.orchestra.run.smithi133.stdout: "err_to_syslog": "false", 2023-10-09T05:08:35.418 INFO:teuthology.orchestra.run.smithi133.stdout: "event_tracing": "false", 2023-10-09T05:08:35.419 INFO:teuthology.orchestra.run.smithi133.stdout: "exporter_addr": "0.0.0.0", 2023-10-09T05:08:35.419 INFO:teuthology.orchestra.run.smithi133.stdout: "exporter_http_port": "9926", 2023-10-09T05:08:35.419 INFO:teuthology.orchestra.run.smithi133.stdout: "exporter_prio_limit": "5", 2023-10-09T05:08:35.419 INFO:teuthology.orchestra.run.smithi133.stdout: "exporter_sock_dir": "/var/run/ceph/", 2023-10-09T05:08:35.419 INFO:teuthology.orchestra.run.smithi133.stdout: "exporter_sort_metrics": "true", 2023-10-09T05:08:35.419 INFO:teuthology.orchestra.run.smithi133.stdout: "exporter_stats_period": "5", 2023-10-09T05:08:35.419 INFO:teuthology.orchestra.run.smithi133.stdout: "fake_statfs_for_testing": "0", 2023-10-09T05:08:35.419 INFO:teuthology.orchestra.run.smithi133.stdout: "fatal_signal_handlers": "true", 2023-10-09T05:08:35.419 INFO:teuthology.orchestra.run.smithi133.stdout: "filer_max_purge_ops": "10", 2023-10-09T05:08:35.419 INFO:teuthology.orchestra.run.smithi133.stdout: "filer_max_truncate_ops": "128", 2023-10-09T05:08:35.419 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_apply_finisher_threads": "1", 2023-10-09T05:08:35.419 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_blackhole": "false", 2023-10-09T05:08:35.420 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_btrfs_clone_range": "true", 2023-10-09T05:08:35.420 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_btrfs_snap": "true", 2023-10-09T05:08:35.420 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_caller_concurrency": "10", 2023-10-09T05:08:35.420 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_collect_device_partition_information": "true", 2023-10-09T05:08:35.420 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_commit_timeout": "600.000000", 2023-10-09T05:08:35.420 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_debug_inject_read_err": "false", 2023-10-09T05:08:35.420 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_debug_omap_check": "false", 2023-10-09T05:08:35.420 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_debug_random_read_err": "0.000000", 2023-10-09T05:08:35.420 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_debug_verify_split": "false", 2023-10-09T05:08:35.420 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_dump_file": "", 2023-10-09T05:08:35.420 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2023-10-09T05:08:35.421 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_expected_throughput_ops": "200.000000", 2023-10-09T05:08:35.421 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_fadvise": "true", 2023-10-09T05:08:35.421 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_fail_eio": "true", 2023-10-09T05:08:35.421 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_fd_cache_shards": "16", 2023-10-09T05:08:35.421 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_fd_cache_size": "128", 2023-10-09T05:08:35.421 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_fiemap": "false", 2023-10-09T05:08:35.421 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_fiemap_threshold": "4096", 2023-10-09T05:08:35.421 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_fsync_flushes_journal_data": "false", 2023-10-09T05:08:35.421 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_index_retry_probability": "0.000000", 2023-10-09T05:08:35.421 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_inject_stall": "0", 2023-10-09T05:08:35.421 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_journal_parallel": "false", 2023-10-09T05:08:35.421 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_journal_trailing": "false", 2023-10-09T05:08:35.422 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_journal_writeahead": "false", 2023-10-09T05:08:35.422 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_kill_at": "0", 2023-10-09T05:08:35.422 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_max_alloc_hint_size": "1048576", 2023-10-09T05:08:35.422 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_max_inline_xattr_size": "0", 2023-10-09T05:08:35.422 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2023-10-09T05:08:35.422 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_max_inline_xattr_size_other": "512", 2023-10-09T05:08:35.422 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2023-10-09T05:08:35.422 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_max_inline_xattrs": "0", 2023-10-09T05:08:35.422 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2023-10-09T05:08:35.422 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_max_inline_xattrs_other": "2", 2023-10-09T05:08:35.423 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_max_inline_xattrs_xfs": "10", 2023-10-09T05:08:35.423 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_max_sync_interval": "5.000000", 2023-10-09T05:08:35.423 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_max_xattr_value_size": "0", 2023-10-09T05:08:35.423 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2023-10-09T05:08:35.423 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_max_xattr_value_size_other": "1024", 2023-10-09T05:08:35.423 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2023-10-09T05:08:35.423 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_merge_threshold": "-10", 2023-10-09T05:08:35.423 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_min_sync_interval": "0.010000", 2023-10-09T05:08:35.423 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_odsync_write": "false", 2023-10-09T05:08:35.423 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_omap_backend": "rocksdb", 2023-10-09T05:08:35.423 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_omap_backend_path": "", 2023-10-09T05:08:35.423 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_omap_header_cache_size": "1024", 2023-10-09T05:08:35.424 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_ondisk_finisher_threads": "1", 2023-10-09T05:08:35.424 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_op_thread_suicide_timeout": "180", 2023-10-09T05:08:35.424 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_op_thread_timeout": "60", 2023-10-09T05:08:35.424 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_op_threads": "2", 2023-10-09T05:08:35.424 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_punch_hole": "false", 2023-10-09T05:08:35.424 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2023-10-09T05:08:35.424 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2023-10-09T05:08:35.424 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2023-10-09T05:08:35.424 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_queue_high_threshhold": "0.900000", 2023-10-09T05:08:35.424 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_queue_low_threshhold": "0.300000", 2023-10-09T05:08:35.424 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_queue_max_bytes": "104857600", 2023-10-09T05:08:35.425 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2023-10-09T05:08:35.425 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2023-10-09T05:08:35.425 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2023-10-09T05:08:35.425 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_queue_max_ops": "50", 2023-10-09T05:08:35.425 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2023-10-09T05:08:35.425 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_seek_data_hole": "false", 2023-10-09T05:08:35.425 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_sloppy_crc": "false", 2023-10-09T05:08:35.425 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_sloppy_crc_block_size": "65536", 2023-10-09T05:08:35.425 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_splice": "false", 2023-10-09T05:08:35.425 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_split_multiple": "2", 2023-10-09T05:08:35.425 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_split_rand_factor": "20", 2023-10-09T05:08:35.426 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_update_to": "1000", 2023-10-09T05:08:35.426 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2023-10-09T05:08:35.426 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2023-10-09T05:08:35.426 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2023-10-09T05:08:35.426 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2023-10-09T05:08:35.426 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2023-10-09T05:08:35.426 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2023-10-09T05:08:35.426 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_wbthrottle_enable": "true", 2023-10-09T05:08:35.426 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2023-10-09T05:08:35.426 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2023-10-09T05:08:35.426 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2023-10-09T05:08:35.427 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2023-10-09T05:08:35.427 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2023-10-09T05:08:35.427 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2023-10-09T05:08:35.427 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_xfs_extsize": "false", 2023-10-09T05:08:35.427 INFO:teuthology.orchestra.run.smithi133.stdout: "filestore_zfs_snap": "false", 2023-10-09T05:08:35.427 INFO:teuthology.orchestra.run.smithi133.stdout: "fio_dir": "/tmp/fio", 2023-10-09T05:08:35.427 INFO:teuthology.orchestra.run.smithi133.stdout: "fsid": "855b5206-665c-11ee-8db5-212e2dc638e7", 2023-10-09T05:08:35.427 INFO:teuthology.orchestra.run.smithi133.stdout: "fuse_allow_other": "true", 2023-10-09T05:08:35.427 INFO:teuthology.orchestra.run.smithi133.stdout: "fuse_atomic_o_trunc": "true", 2023-10-09T05:08:35.427 INFO:teuthology.orchestra.run.smithi133.stdout: "fuse_big_writes": "true", 2023-10-09T05:08:35.427 INFO:teuthology.orchestra.run.smithi133.stdout: "fuse_debug": "false", 2023-10-09T05:08:35.428 INFO:teuthology.orchestra.run.smithi133.stdout: "fuse_default_permissions": "false", 2023-10-09T05:08:35.428 INFO:teuthology.orchestra.run.smithi133.stdout: "fuse_disable_pagecache": "false", 2023-10-09T05:08:35.428 INFO:teuthology.orchestra.run.smithi133.stdout: "fuse_max_write": "0", 2023-10-09T05:08:35.428 INFO:teuthology.orchestra.run.smithi133.stdout: "fuse_multithreaded": "true", 2023-10-09T05:08:35.428 INFO:teuthology.orchestra.run.smithi133.stdout: "fuse_require_active_mds": "true", 2023-10-09T05:08:35.428 INFO:teuthology.orchestra.run.smithi133.stdout: "fuse_set_user_groups": "true", 2023-10-09T05:08:35.428 INFO:teuthology.orchestra.run.smithi133.stdout: "fuse_splice_move": "true", 2023-10-09T05:08:35.428 INFO:teuthology.orchestra.run.smithi133.stdout: "fuse_splice_read": "true", 2023-10-09T05:08:35.428 INFO:teuthology.orchestra.run.smithi133.stdout: "fuse_splice_write": "true", 2023-10-09T05:08:35.428 INFO:teuthology.orchestra.run.smithi133.stdout: "fuse_syncfs_on_mksnap": "true", 2023-10-09T05:08:35.428 INFO:teuthology.orchestra.run.smithi133.stdout: "fuse_use_invalidate_cb": "true", 2023-10-09T05:08:35.428 INFO:teuthology.orchestra.run.smithi133.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2023-10-09T05:08:35.429 INFO:teuthology.orchestra.run.smithi133.stdout: "gss_target_name": "ceph", 2023-10-09T05:08:35.429 INFO:teuthology.orchestra.run.smithi133.stdout: "heartbeat_file": "", 2023-10-09T05:08:35.429 INFO:teuthology.orchestra.run.smithi133.stdout: "heartbeat_inject_failure": "0", 2023-10-09T05:08:35.429 INFO:teuthology.orchestra.run.smithi133.stdout: "heartbeat_interval": "5", 2023-10-09T05:08:35.429 INFO:teuthology.orchestra.run.smithi133.stdout: "host": "smithi133", 2023-10-09T05:08:35.429 INFO:teuthology.orchestra.run.smithi133.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2023-10-09T05:08:35.429 INFO:teuthology.orchestra.run.smithi133.stdout: "immutable_object_cache_max_inflight_ops": "128", 2023-10-09T05:08:35.429 INFO:teuthology.orchestra.run.smithi133.stdout: "immutable_object_cache_max_size": "1073741824", 2023-10-09T05:08:35.429 INFO:teuthology.orchestra.run.smithi133.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2023-10-09T05:08:35.429 INFO:teuthology.orchestra.run.smithi133.stdout: "immutable_object_cache_qos_bps_burst": "0", 2023-10-09T05:08:35.429 INFO:teuthology.orchestra.run.smithi133.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2023-10-09T05:08:35.429 INFO:teuthology.orchestra.run.smithi133.stdout: "immutable_object_cache_qos_bps_limit": "0", 2023-10-09T05:08:35.430 INFO:teuthology.orchestra.run.smithi133.stdout: "immutable_object_cache_qos_iops_burst": "0", 2023-10-09T05:08:35.430 INFO:teuthology.orchestra.run.smithi133.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2023-10-09T05:08:35.430 INFO:teuthology.orchestra.run.smithi133.stdout: "immutable_object_cache_qos_iops_limit": "0", 2023-10-09T05:08:35.430 INFO:teuthology.orchestra.run.smithi133.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2023-10-09T05:08:35.430 INFO:teuthology.orchestra.run.smithi133.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2023-10-09T05:08:35.430 INFO:teuthology.orchestra.run.smithi133.stdout: "immutable_object_cache_watermark": "0.900000", 2023-10-09T05:08:35.430 INFO:teuthology.orchestra.run.smithi133.stdout: "inject_early_sigterm": "false", 2023-10-09T05:08:35.430 INFO:teuthology.orchestra.run.smithi133.stdout: "jaeger_agent_port": "6799", 2023-10-09T05:08:35.430 INFO:teuthology.orchestra.run.smithi133.stdout: "jaeger_tracing_enable": "false", 2023-10-09T05:08:35.430 INFO:teuthology.orchestra.run.smithi133.stdout: "journal_aio": "true", 2023-10-09T05:08:35.430 INFO:teuthology.orchestra.run.smithi133.stdout: "journal_align_min_size": "65536", 2023-10-09T05:08:35.431 INFO:teuthology.orchestra.run.smithi133.stdout: "journal_block_align": "true", 2023-10-09T05:08:35.431 INFO:teuthology.orchestra.run.smithi133.stdout: "journal_block_size": "4096", 2023-10-09T05:08:35.431 INFO:teuthology.orchestra.run.smithi133.stdout: "journal_dio": "true", 2023-10-09T05:08:35.431 INFO:teuthology.orchestra.run.smithi133.stdout: "journal_discard": "false", 2023-10-09T05:08:35.431 INFO:teuthology.orchestra.run.smithi133.stdout: "journal_force_aio": "false", 2023-10-09T05:08:35.431 INFO:teuthology.orchestra.run.smithi133.stdout: "journal_ignore_corruption": "false", 2023-10-09T05:08:35.431 INFO:teuthology.orchestra.run.smithi133.stdout: "journal_max_write_bytes": "10485760", 2023-10-09T05:08:35.431 INFO:teuthology.orchestra.run.smithi133.stdout: "journal_max_write_entries": "100", 2023-10-09T05:08:35.431 INFO:teuthology.orchestra.run.smithi133.stdout: "journal_replay_from": "0", 2023-10-09T05:08:35.431 INFO:teuthology.orchestra.run.smithi133.stdout: "journal_throttle_high_multiple": "0.000000", 2023-10-09T05:08:35.431 INFO:teuthology.orchestra.run.smithi133.stdout: "journal_throttle_high_threshhold": "0.900000", 2023-10-09T05:08:35.432 INFO:teuthology.orchestra.run.smithi133.stdout: "journal_throttle_low_threshhold": "0.600000", 2023-10-09T05:08:35.432 INFO:teuthology.orchestra.run.smithi133.stdout: "journal_throttle_max_multiple": "0.000000", 2023-10-09T05:08:35.432 INFO:teuthology.orchestra.run.smithi133.stdout: "journal_write_header_frequency": "0", 2023-10-09T05:08:35.432 INFO:teuthology.orchestra.run.smithi133.stdout: "journal_zero_on_create": "false", 2023-10-09T05:08:35.432 INFO:teuthology.orchestra.run.smithi133.stdout: "journaler_prefetch_periods": "10", 2023-10-09T05:08:35.432 INFO:teuthology.orchestra.run.smithi133.stdout: "journaler_prezero_periods": "5", 2023-10-09T05:08:35.432 INFO:teuthology.orchestra.run.smithi133.stdout: "journaler_write_head_interval": "15", 2023-10-09T05:08:35.432 INFO:teuthology.orchestra.run.smithi133.stdout: "key": "", 2023-10-09T05:08:35.432 INFO:teuthology.orchestra.run.smithi133.stdout: "keyfile": "", 2023-10-09T05:08:35.432 INFO:teuthology.orchestra.run.smithi133.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2023-10-09T05:08:35.432 INFO:teuthology.orchestra.run.smithi133.stdout: "kstore_backend": "rocksdb", 2023-10-09T05:08:35.432 INFO:teuthology.orchestra.run.smithi133.stdout: "kstore_default_stripe_size": "65536", 2023-10-09T05:08:35.433 INFO:teuthology.orchestra.run.smithi133.stdout: "kstore_fsck_on_mount": "false", 2023-10-09T05:08:35.433 INFO:teuthology.orchestra.run.smithi133.stdout: "kstore_fsck_on_mount_deep": "true", 2023-10-09T05:08:35.433 INFO:teuthology.orchestra.run.smithi133.stdout: "kstore_max_bytes": "67108864", 2023-10-09T05:08:35.433 INFO:teuthology.orchestra.run.smithi133.stdout: "kstore_max_ops": "512", 2023-10-09T05:08:35.433 INFO:teuthology.orchestra.run.smithi133.stdout: "kstore_nid_prealloc": "1024", 2023-10-09T05:08:35.433 INFO:teuthology.orchestra.run.smithi133.stdout: "kstore_onode_map_size": "1024", 2023-10-09T05:08:35.433 INFO:teuthology.orchestra.run.smithi133.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2023-10-09T05:08:35.433 INFO:teuthology.orchestra.run.smithi133.stdout: "kstore_sync_submit_transaction": "false", 2023-10-09T05:08:35.433 INFO:teuthology.orchestra.run.smithi133.stdout: "kstore_sync_transaction": "false", 2023-10-09T05:08:35.433 INFO:teuthology.orchestra.run.smithi133.stdout: "leveldb_block_size": "65536", 2023-10-09T05:08:35.433 INFO:teuthology.orchestra.run.smithi133.stdout: "leveldb_bloom_size": "0", 2023-10-09T05:08:35.434 INFO:teuthology.orchestra.run.smithi133.stdout: "leveldb_cache_size": "536870912", 2023-10-09T05:08:35.434 INFO:teuthology.orchestra.run.smithi133.stdout: "leveldb_compact_on_mount": "false", 2023-10-09T05:08:35.434 INFO:teuthology.orchestra.run.smithi133.stdout: "leveldb_compression": "false", 2023-10-09T05:08:35.434 INFO:teuthology.orchestra.run.smithi133.stdout: "leveldb_log": "", 2023-10-09T05:08:35.434 INFO:teuthology.orchestra.run.smithi133.stdout: "leveldb_log_to_ceph_log": "true", 2023-10-09T05:08:35.434 INFO:teuthology.orchestra.run.smithi133.stdout: "leveldb_max_open_files": "0", 2023-10-09T05:08:35.434 INFO:teuthology.orchestra.run.smithi133.stdout: "leveldb_paranoid": "false", 2023-10-09T05:08:35.434 INFO:teuthology.orchestra.run.smithi133.stdout: "leveldb_write_buffer_size": "33554432", 2023-10-09T05:08:35.434 INFO:teuthology.orchestra.run.smithi133.stdout: "librados_thread_count": "2", 2023-10-09T05:08:35.434 INFO:teuthology.orchestra.run.smithi133.stdout: "lockdep": "false", 2023-10-09T05:08:35.434 INFO:teuthology.orchestra.run.smithi133.stdout: "lockdep_force_backtrace": "false", 2023-10-09T05:08:35.435 INFO:teuthology.orchestra.run.smithi133.stdout: "log_coarse_timestamps": "true", 2023-10-09T05:08:35.435 INFO:teuthology.orchestra.run.smithi133.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2023-10-09T05:08:35.435 INFO:teuthology.orchestra.run.smithi133.stdout: "log_flush_on_exit": "false", 2023-10-09T05:08:35.435 INFO:teuthology.orchestra.run.smithi133.stdout: "log_graylog_host": "127.0.0.1", 2023-10-09T05:08:35.435 INFO:teuthology.orchestra.run.smithi133.stdout: "log_graylog_port": "12201", 2023-10-09T05:08:35.435 INFO:teuthology.orchestra.run.smithi133.stdout: "log_max_new": "1000", 2023-10-09T05:08:35.435 INFO:teuthology.orchestra.run.smithi133.stdout: "log_max_recent": "10000", 2023-10-09T05:08:35.435 INFO:teuthology.orchestra.run.smithi133.stdout: "log_stderr_prefix": "debug ", 2023-10-09T05:08:35.435 INFO:teuthology.orchestra.run.smithi133.stdout: "log_stop_at_utilization": "0.970000", 2023-10-09T05:08:35.435 INFO:teuthology.orchestra.run.smithi133.stdout: "log_to_file": "true", 2023-10-09T05:08:35.436 INFO:teuthology.orchestra.run.smithi133.stdout: "log_to_graylog": "false", 2023-10-09T05:08:35.436 INFO:teuthology.orchestra.run.smithi133.stdout: "log_to_journald": "false", 2023-10-09T05:08:35.436 INFO:teuthology.orchestra.run.smithi133.stdout: "log_to_stderr": "false", 2023-10-09T05:08:35.436 INFO:teuthology.orchestra.run.smithi133.stdout: "log_to_syslog": "false", 2023-10-09T05:08:35.436 INFO:teuthology.orchestra.run.smithi133.stdout: "max_rotating_auth_attempts": "10", 2023-10-09T05:08:35.436 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2023-10-09T05:08:35.436 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_action_on_write_error": "1", 2023-10-09T05:08:35.436 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_alternate_name_max": "8192", 2023-10-09T05:08:35.436 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_asio_thread_count": "2", 2023-10-09T05:08:35.436 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_bal_export_pin": "true", 2023-10-09T05:08:35.436 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_bal_fragment_dirs": "true", 2023-10-09T05:08:35.437 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2023-10-09T05:08:35.437 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_bal_fragment_interval": "5", 2023-10-09T05:08:35.437 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_bal_fragment_size_max": "100000", 2023-10-09T05:08:35.437 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_bal_idle_threshold": "0.000000", 2023-10-09T05:08:35.437 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_bal_interval": "10", 2023-10-09T05:08:35.437 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_bal_max": "-1", 2023-10-09T05:08:35.437 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_bal_max_until": "-1", 2023-10-09T05:08:35.437 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_bal_merge_size": "50", 2023-10-09T05:08:35.437 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_bal_midchunk": "0.300000", 2023-10-09T05:08:35.437 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_bal_min_rebalance": "0.100000", 2023-10-09T05:08:35.437 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_bal_min_start": "0.200000", 2023-10-09T05:08:35.437 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_bal_minchunk": "0.001000", 2023-10-09T05:08:35.438 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_bal_mode": "0", 2023-10-09T05:08:35.438 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_bal_need_max": "1.200000", 2023-10-09T05:08:35.438 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_bal_need_min": "0.800000", 2023-10-09T05:08:35.438 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_bal_replicate_threshold": "8000.000000", 2023-10-09T05:08:35.438 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_bal_sample_interval": "3.000000", 2023-10-09T05:08:35.438 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_bal_split_bits": "3", 2023-10-09T05:08:35.438 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_bal_split_rd": "25000.000000", 2023-10-09T05:08:35.438 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_bal_split_size": "10000", 2023-10-09T05:08:35.438 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_bal_split_wr": "10000.000000", 2023-10-09T05:08:35.438 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_bal_target_decay": "10.000000", 2023-10-09T05:08:35.438 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2023-10-09T05:08:35.439 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_beacon_grace": "15.000000", 2023-10-09T05:08:35.439 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_beacon_interval": "4.000000", 2023-10-09T05:08:35.439 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_beacon_mon_down_grace": "60", 2023-10-09T05:08:35.439 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_cache_memory_limit": "4294967296", 2023-10-09T05:08:35.439 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_cache_mid": "0.700000", 2023-10-09T05:08:35.439 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_cache_release_free_interval": "10", 2023-10-09T05:08:35.439 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_cache_reservation": "0.050000", 2023-10-09T05:08:35.439 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_cache_trim_decay_rate": "1.000000", 2023-10-09T05:08:35.439 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_cache_trim_interval": "1", 2023-10-09T05:08:35.439 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_cache_trim_threshold": "262144", 2023-10-09T05:08:35.439 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2023-10-09T05:08:35.439 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2023-10-09T05:08:35.440 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_client_delegate_inos_pct": "50", 2023-10-09T05:08:35.440 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_client_prealloc_inos": "1000", 2023-10-09T05:08:35.440 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2023-10-09T05:08:35.440 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_connect_bootstrapping": "false", 2023-10-09T05:08:35.440 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_damage_table_max_entries": "10000", 2023-10-09T05:08:35.440 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2023-10-09T05:08:35.440 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_debug_auth_pins": "false", 2023-10-09T05:08:35.440 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_debug_frag": "false", 2023-10-09T05:08:35.440 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_debug_scatterstat": "false", 2023-10-09T05:08:35.440 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_debug_subtrees": "false", 2023-10-09T05:08:35.440 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_decay_halflife": "5.000000", 2023-10-09T05:08:35.441 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_default_dir_hash": "2", 2023-10-09T05:08:35.441 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_defer_session_stale": "true", 2023-10-09T05:08:35.441 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_deny_all_reconnect": "false", 2023-10-09T05:08:35.441 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_dir_keys_per_op": "16384", 2023-10-09T05:08:35.441 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_dir_max_commit_size": "10", 2023-10-09T05:08:35.441 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_dir_max_entries": "0", 2023-10-09T05:08:35.441 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_dir_prefetch": "true", 2023-10-09T05:08:35.441 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_dirstat_min_interval": "1.000000", 2023-10-09T05:08:35.441 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_dump_cache_after_rejoin": "false", 2023-10-09T05:08:35.441 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_dump_cache_on_map": "false", 2023-10-09T05:08:35.441 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_dump_cache_threshold_file": "0", 2023-10-09T05:08:35.441 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2023-10-09T05:08:35.442 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_early_reply": "true", 2023-10-09T05:08:35.442 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_enable_op_tracker": "true", 2023-10-09T05:08:35.442 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_enforce_unique_name": "true", 2023-10-09T05:08:35.442 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_export_ephemeral_distributed": "true", 2023-10-09T05:08:35.442 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2023-10-09T05:08:35.442 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_export_ephemeral_random": "true", 2023-10-09T05:08:35.442 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_export_ephemeral_random_max": "0.010000", 2023-10-09T05:08:35.442 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_extraordinary_events_dump_interval": "0", 2023-10-09T05:08:35.442 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_forward_all_requests_to_auth": "false", 2023-10-09T05:08:35.442 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_freeze_tree_timeout": "30.000000", 2023-10-09T05:08:35.442 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_fscrypt_last_block_max_size": "4096", 2023-10-09T05:08:35.443 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_go_bad_corrupt_dentry": "true", 2023-10-09T05:08:35.443 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2023-10-09T05:08:35.443 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_health_cache_threshold": "1.500000", 2023-10-09T05:08:35.443 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_health_summarize_threshold": "10", 2023-10-09T05:08:35.443 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_heartbeat_grace": "15.000000", 2023-10-09T05:08:35.443 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_heartbeat_reset_grace": "1000", 2023-10-09T05:08:35.443 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_inject_health_dummy": "false", 2023-10-09T05:08:35.443 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2023-10-09T05:08:35.443 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_inject_migrator_session_race": "false", 2023-10-09T05:08:35.443 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2023-10-09T05:08:35.443 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_inject_skip_replaying_inotable": "false", 2023-10-09T05:08:35.443 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2023-10-09T05:08:35.444 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_join_fs": "", 2023-10-09T05:08:35.444 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_journal_format": "1", 2023-10-09T05:08:35.444 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_kill_create_at": "0", 2023-10-09T05:08:35.444 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_kill_export_at": "0", 2023-10-09T05:08:35.444 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_kill_import_at": "0", 2023-10-09T05:08:35.444 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_kill_journal_at": "0", 2023-10-09T05:08:35.444 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_kill_journal_expire_at": "0", 2023-10-09T05:08:35.444 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_kill_journal_replay_at": "0", 2023-10-09T05:08:35.444 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_kill_link_at": "0", 2023-10-09T05:08:35.444 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_kill_mdstable_at": "0", 2023-10-09T05:08:35.444 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_kill_openc_at": "0", 2023-10-09T05:08:35.445 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_kill_rename_at": "0", 2023-10-09T05:08:35.445 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_kill_skip_replaying_inotable": "false", 2023-10-09T05:08:35.445 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_log_events_per_segment": "1024", 2023-10-09T05:08:35.445 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_log_max_events": "-1", 2023-10-09T05:08:35.445 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_log_max_segments": "128", 2023-10-09T05:08:35.445 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_log_pause": "false", 2023-10-09T05:08:35.445 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_log_segment_size": "0", 2023-10-09T05:08:35.445 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_log_skip_corrupt_events": "false", 2023-10-09T05:08:35.445 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_log_warn_factor": "2.000000", 2023-10-09T05:08:35.445 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_max_caps_per_client": "1048576", 2023-10-09T05:08:35.445 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_max_completed_flushes": "100000", 2023-10-09T05:08:35.445 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_max_completed_requests": "100000", 2023-10-09T05:08:35.446 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_max_export_size": "20971520", 2023-10-09T05:08:35.446 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_max_file_recover": "32", 2023-10-09T05:08:35.446 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_max_purge_files": "64", 2023-10-09T05:08:35.446 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_max_purge_ops": "8192", 2023-10-09T05:08:35.446 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2023-10-09T05:08:35.446 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_max_scrub_ops_in_progress": "5", 2023-10-09T05:08:35.446 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_max_snaps_per_dir": "100", 2023-10-09T05:08:35.446 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_max_xattr_pairs_size": "65536", 2023-10-09T05:08:35.446 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_metrics_update_interval": "2", 2023-10-09T05:08:35.446 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_min_caps_per_client": "100", 2023-10-09T05:08:35.446 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_min_caps_working_set": "10000", 2023-10-09T05:08:35.446 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_mon_shutdown_timeout": "5.000000", 2023-10-09T05:08:35.447 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_numa_node": "-1", 2023-10-09T05:08:35.447 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_oft_prefetch_dirfrags": "false", 2023-10-09T05:08:35.447 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_op_complaint_time": "30.000000", 2023-10-09T05:08:35.447 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_op_history_duration": "600", 2023-10-09T05:08:35.447 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_op_history_size": "20", 2023-10-09T05:08:35.447 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_op_log_threshold": "5", 2023-10-09T05:08:35.447 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_ping_grace": "15", 2023-10-09T05:08:35.447 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_ping_interval": "5", 2023-10-09T05:08:35.447 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2023-10-09T05:08:35.447 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_recall_global_max_decay_threshold": "131072", 2023-10-09T05:08:35.447 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_recall_max_caps": "30000", 2023-10-09T05:08:35.447 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_recall_max_decay_rate": "1.500000", 2023-10-09T05:08:35.447 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_recall_max_decay_threshold": "131072", 2023-10-09T05:08:35.449 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_recall_warning_decay_rate": "60.000000", 2023-10-09T05:08:35.449 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_recall_warning_threshold": "262144", 2023-10-09T05:08:35.449 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_reconnect_timeout": "45.000000", 2023-10-09T05:08:35.449 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_replay_interval": "1.000000", 2023-10-09T05:08:35.449 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_replay_unsafe_with_closed_session": "false", 2023-10-09T05:08:35.449 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_request_load_average_decay_rate": "60.000000", 2023-10-09T05:08:35.449 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_root_ino_gid": "0", 2023-10-09T05:08:35.449 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_root_ino_uid": "0", 2023-10-09T05:08:35.450 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_scatter_nudge_interval": "5.000000", 2023-10-09T05:08:35.450 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_session_blocklist_on_evict": "true", 2023-10-09T05:08:35.450 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_session_blocklist_on_timeout": "true", 2023-10-09T05:08:35.450 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2023-10-09T05:08:35.450 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_session_cache_liveness_magnitude": "10", 2023-10-09T05:08:35.450 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2023-10-09T05:08:35.450 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_session_cap_acquisition_throttle": "100000", 2023-10-09T05:08:35.450 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2023-10-09T05:08:35.450 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_sessionmap_keys_per_op": "1024", 2023-10-09T05:08:35.450 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_shutdown_check": "0", 2023-10-09T05:08:35.450 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_skip_ino": "0", 2023-10-09T05:08:35.451 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_sleep_rank_change": "0.000000", 2023-10-09T05:08:35.451 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_snap_max_uid": "4294967294", 2023-10-09T05:08:35.451 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_snap_min_uid": "0", 2023-10-09T05:08:35.451 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_snap_rstat": "false", 2023-10-09T05:08:35.451 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_standby_replay_damaged": "false", 2023-10-09T05:08:35.451 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_symlink_recovery": "true", 2023-10-09T05:08:35.451 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_task_status_update_interval": "2.000000", 2023-10-09T05:08:35.451 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_thrash_exports": "0", 2023-10-09T05:08:35.451 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_thrash_fragments": "0", 2023-10-09T05:08:35.451 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_tick_interval": "5.000000", 2023-10-09T05:08:35.451 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_valgrind_exit": "false", 2023-10-09T05:08:35.451 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_verify_backtrace": "1", 2023-10-09T05:08:35.452 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_verify_scatter": "false", 2023-10-09T05:08:35.452 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_wipe_ino_prealloc": "false", 2023-10-09T05:08:35.452 INFO:teuthology.orchestra.run.smithi133.stdout: "mds_wipe_sessions": "false", 2023-10-09T05:08:35.452 INFO:teuthology.orchestra.run.smithi133.stdout: "mempool_debug": "false", 2023-10-09T05:08:35.452 INFO:teuthology.orchestra.run.smithi133.stdout: "memstore_debug_omit_block_device_write": "false", 2023-10-09T05:08:35.452 INFO:teuthology.orchestra.run.smithi133.stdout: "memstore_device_bytes": "1073741824", 2023-10-09T05:08:35.452 INFO:teuthology.orchestra.run.smithi133.stdout: "memstore_page_set": "false", 2023-10-09T05:08:35.452 INFO:teuthology.orchestra.run.smithi133.stdout: "memstore_page_size": "65536", 2023-10-09T05:08:35.452 INFO:teuthology.orchestra.run.smithi133.stdout: "mgr_client_bytes": "134217728", 2023-10-09T05:08:35.452 INFO:teuthology.orchestra.run.smithi133.stdout: "mgr_client_messages": "512", 2023-10-09T05:08:35.452 INFO:teuthology.orchestra.run.smithi133.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2023-10-09T05:08:35.453 INFO:teuthology.orchestra.run.smithi133.stdout: "mgr_connect_retry_interval": "1.000000", 2023-10-09T05:08:35.453 INFO:teuthology.orchestra.run.smithi133.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2023-10-09T05:08:35.453 INFO:teuthology.orchestra.run.smithi133.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2023-10-09T05:08:35.453 INFO:teuthology.orchestra.run.smithi133.stdout: "mgr_disabled_modules": "", 2023-10-09T05:08:35.453 INFO:teuthology.orchestra.run.smithi133.stdout: "mgr_initial_modules": "restful iostat nfs", 2023-10-09T05:08:35.453 INFO:teuthology.orchestra.run.smithi133.stdout: "mgr_max_pg_num_change": "128", 2023-10-09T05:08:35.453 INFO:teuthology.orchestra.run.smithi133.stdout: "mgr_mds_bytes": "134217728", 2023-10-09T05:08:35.453 INFO:teuthology.orchestra.run.smithi133.stdout: "mgr_mds_messages": "128", 2023-10-09T05:08:35.453 INFO:teuthology.orchestra.run.smithi133.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2023-10-09T05:08:35.453 INFO:teuthology.orchestra.run.smithi133.stdout: "mgr_mon_bytes": "134217728", 2023-10-09T05:08:35.453 INFO:teuthology.orchestra.run.smithi133.stdout: "mgr_mon_messages": "128", 2023-10-09T05:08:35.453 INFO:teuthology.orchestra.run.smithi133.stdout: "mgr_osd_bytes": "536870912", 2023-10-09T05:08:35.454 INFO:teuthology.orchestra.run.smithi133.stdout: "mgr_osd_messages": "8192", 2023-10-09T05:08:35.454 INFO:teuthology.orchestra.run.smithi133.stdout: "mgr_pool": "true", 2023-10-09T05:08:35.454 INFO:teuthology.orchestra.run.smithi133.stdout: "mgr_service_beacon_grace": "60.000000", 2023-10-09T05:08:35.454 INFO:teuthology.orchestra.run.smithi133.stdout: "mgr_standby_modules": "true", 2023-10-09T05:08:35.454 INFO:teuthology.orchestra.run.smithi133.stdout: "mgr_stats_period": "5", 2023-10-09T05:08:35.454 INFO:teuthology.orchestra.run.smithi133.stdout: "mgr_stats_threshold": "5", 2023-10-09T05:08:35.454 INFO:teuthology.orchestra.run.smithi133.stdout: "mgr_tick_period": "2", 2023-10-09T05:08:35.454 INFO:teuthology.orchestra.run.smithi133.stdout: "mgr_ttl_cache_expire_seconds": "0", 2023-10-09T05:08:35.454 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_accept_timeout_factor": "2.000000", 2023-10-09T05:08:35.454 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_allow_pool_delete": "true", 2023-10-09T05:08:35.454 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_allow_pool_size_one": "false", 2023-10-09T05:08:35.455 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_auth_validate_all_caps": "true", 2023-10-09T05:08:35.455 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2023-10-09T05:08:35.455 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2023-10-09T05:08:35.455 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_client_bytes": "104857600", 2023-10-09T05:08:35.455 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_client_directed_command_retry": "2", 2023-10-09T05:08:35.455 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_client_hunt_interval": "3.000000", 2023-10-09T05:08:35.455 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2023-10-09T05:08:35.455 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2023-10-09T05:08:35.455 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2023-10-09T05:08:35.455 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_client_hunt_parallel": "3", 2023-10-09T05:08:35.455 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_client_log_interval": "1.000000", 2023-10-09T05:08:35.456 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_client_max_log_entries_per_message": "1000", 2023-10-09T05:08:35.456 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_client_ping_interval": "10.000000", 2023-10-09T05:08:35.456 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_client_ping_timeout": "30.000000", 2023-10-09T05:08:35.456 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_clock_drift_allowed": "1.000000", 2023-10-09T05:08:35.456 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2023-10-09T05:08:35.456 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2023-10-09T05:08:35.456 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_cluster_log_file_level": "debug", 2023-10-09T05:08:35.456 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_cluster_log_to_file": "false", 2023-10-09T05:08:35.456 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_cluster_log_to_graylog": "false", 2023-10-09T05:08:35.456 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2023-10-09T05:08:35.456 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_cluster_log_to_graylog_port": "12201", 2023-10-09T05:08:35.456 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_cluster_log_to_journald": "false", 2023-10-09T05:08:35.457 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_cluster_log_to_stderr": "true", 2023-10-09T05:08:35.457 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_cluster_log_to_syslog": "default=false", 2023-10-09T05:08:35.457 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2023-10-09T05:08:35.457 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_cluster_log_to_syslog_level": "info", 2023-10-09T05:08:35.457 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_compact_on_bootstrap": "false", 2023-10-09T05:08:35.457 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_compact_on_start": "false", 2023-10-09T05:08:35.457 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_compact_on_trim": "true", 2023-10-09T05:08:35.457 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_con_tracker_persist_interval": "10", 2023-10-09T05:08:35.457 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_con_tracker_score_halflife": "43200", 2023-10-09T05:08:35.457 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_config_key_max_entry_size": "65536", 2023-10-09T05:08:35.457 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_cpu_threads": "4", 2023-10-09T05:08:35.457 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_crush_min_required_version": "hammer", 2023-10-09T05:08:35.458 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_daemon_bytes": "419430400", 2023-10-09T05:08:35.458 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2023-10-09T05:08:35.458 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_data_avail_crit": "5", 2023-10-09T05:08:35.458 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_data_avail_warn": "5", 2023-10-09T05:08:35.458 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_data_size_warn": "16106127360", 2023-10-09T05:08:35.458 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_debug_block_osdmap_trim": "false", 2023-10-09T05:08:35.458 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_debug_deprecated_as_obsolete": "false", 2023-10-09T05:08:35.458 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_debug_dump_json": "false", 2023-10-09T05:08:35.458 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2023-10-09T05:08:35.458 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_debug_dump_transactions": "false", 2023-10-09T05:08:35.458 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_debug_extra_checks": "false", 2023-10-09T05:08:35.459 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_debug_no_initial_persistent_features": "false", 2023-10-09T05:08:35.459 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2023-10-09T05:08:35.459 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_debug_no_require_quincy": "false", 2023-10-09T05:08:35.459 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_debug_no_require_reef": "false", 2023-10-09T05:08:35.459 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2023-10-09T05:08:35.459 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_delta_reset_interval": "10.000000", 2023-10-09T05:08:35.459 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_dns_srv_name": "ceph-mon", 2023-10-09T05:08:35.459 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_down_mkfs_grace": "60", 2023-10-09T05:08:35.459 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_election_default_strategy": "1", 2023-10-09T05:08:35.459 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_election_timeout": "5.000000", 2023-10-09T05:08:35.459 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2023-10-09T05:08:35.459 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_elector_ping_divisor": "2", 2023-10-09T05:08:35.460 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_elector_ping_timeout": "2.000000", 2023-10-09T05:08:35.460 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_enable_op_tracker": "true", 2023-10-09T05:08:35.460 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_fake_pool_delete": "false", 2023-10-09T05:08:35.460 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_force_quorum_join": "false", 2023-10-09T05:08:35.460 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_globalid_prealloc": "10000", 2023-10-09T05:08:35.460 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_health_detail_to_clog": "true", 2023-10-09T05:08:35.460 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_health_log_update_period": "5", 2023-10-09T05:08:35.460 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_health_max_detail": "50", 2023-10-09T05:08:35.460 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_health_to_clog": "true", 2023-10-09T05:08:35.460 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_health_to_clog_interval": "600", 2023-10-09T05:08:35.460 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2023-10-09T05:08:35.460 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_host": "[v2:172.21.15.133:3300/0,v1:172.21.15.133:6789/0] [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] [v2:172.21.15.133:3301/0,v1:172.21.15.133:6790/0]", 2023-10-09T05:08:35.461 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_host_override": "", 2023-10-09T05:08:35.461 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_initial_members": "", 2023-10-09T05:08:35.461 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2023-10-09T05:08:35.461 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2023-10-09T05:08:35.461 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_inject_transaction_delay_max": "10.000000", 2023-10-09T05:08:35.461 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2023-10-09T05:08:35.461 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_keyvaluedb": "rocksdb", 2023-10-09T05:08:35.461 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_lease": "5.000000", 2023-10-09T05:08:35.461 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2023-10-09T05:08:35.461 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_lease_renew_interval_factor": "0.600000", 2023-10-09T05:08:35.461 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_log_full_interval": "50", 2023-10-09T05:08:35.461 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_log_max": "10000", 2023-10-09T05:08:35.462 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_log_max_summary": "50", 2023-10-09T05:08:35.462 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_max_log_entries_per_event": "4096", 2023-10-09T05:08:35.462 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_max_log_epochs": "500", 2023-10-09T05:08:35.462 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_max_mdsmap_epochs": "500", 2023-10-09T05:08:35.462 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_max_mgrmap_epochs": "500", 2023-10-09T05:08:35.462 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_max_osd": "10000", 2023-10-09T05:08:35.462 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_max_pg_per_osd": "10000", 2023-10-09T05:08:35.462 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_max_pool_pg_num": "65536", 2023-10-09T05:08:35.462 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_max_snap_prune_per_epoch": "100", 2023-10-09T05:08:35.462 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_mds_blocklist_interval": "86400.000000", 2023-10-09T05:08:35.462 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_mds_force_trim_to": "0", 2023-10-09T05:08:35.462 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_mds_skip_sanity": "true", 2023-10-09T05:08:35.463 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_memory_autotune": "true", 2023-10-09T05:08:35.463 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_memory_target": "2147483648", 2023-10-09T05:08:35.463 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_mgr_beacon_grace": "30", 2023-10-09T05:08:35.463 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2023-10-09T05:08:35.463 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_mgr_digest_period": "5", 2023-10-09T05:08:35.463 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_mgr_inactive_grace": "60", 2023-10-09T05:08:35.463 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_mgr_mkfs_grace": "240", 2023-10-09T05:08:35.463 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2023-10-09T05:08:35.463 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_min_osdmap_epochs": "500", 2023-10-09T05:08:35.463 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_op_complaint_time": "30", 2023-10-09T05:08:35.463 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_op_history_duration": "600", 2023-10-09T05:08:35.464 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_op_history_size": "20", 2023-10-09T05:08:35.464 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_op_history_slow_op_size": "20", 2023-10-09T05:08:35.464 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_op_history_slow_op_threshold": "10", 2023-10-09T05:08:35.464 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_op_log_threshold": "5", 2023-10-09T05:08:35.464 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_adjust_down_out_interval": "true", 2023-10-09T05:08:35.464 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2023-10-09T05:08:35.464 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2023-10-09T05:08:35.464 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_auto_mark_in": "false", 2023-10-09T05:08:35.464 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_auto_mark_new_in": "true", 2023-10-09T05:08:35.464 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2023-10-09T05:08:35.464 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2023-10-09T05:08:35.464 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_cache_size": "500", 2023-10-09T05:08:35.464 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_cache_size_min": "134217728", 2023-10-09T05:08:35.465 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_crush_smoke_test": "true", 2023-10-09T05:08:35.465 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_destroyed_out_interval": "600", 2023-10-09T05:08:35.465 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_down_out_interval": "600", 2023-10-09T05:08:35.465 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_down_out_subtree_limit": "rack", 2023-10-09T05:08:35.465 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2023-10-09T05:08:35.465 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_force_trim_to": "0", 2023-10-09T05:08:35.465 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_full_ratio": "0.950000", 2023-10-09T05:08:35.465 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2023-10-09T05:08:35.465 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_laggy_halflife": "3600", 2023-10-09T05:08:35.465 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_laggy_max_interval": "300", 2023-10-09T05:08:35.465 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_laggy_weight": "0.300000", 2023-10-09T05:08:35.466 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2023-10-09T05:08:35.466 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_max_creating_pgs": "1024", 2023-10-09T05:08:35.466 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_max_initial_pgs": "1024", 2023-10-09T05:08:35.466 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_min_down_reporters": "2", 2023-10-09T05:08:35.466 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_min_in_ratio": "0.750000", 2023-10-09T05:08:35.466 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_min_up_ratio": "0.300000", 2023-10-09T05:08:35.466 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_nearfull_ratio": "0.850000", 2023-10-09T05:08:35.466 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_prime_pg_temp": "true", 2023-10-09T05:08:35.466 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2023-10-09T05:08:35.466 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2023-10-09T05:08:35.466 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_report_timeout": "900", 2023-10-09T05:08:35.467 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_reporter_subtree_level": "osd", 2023-10-09T05:08:35.467 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2023-10-09T05:08:35.467 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_warn_num_repaired": "10", 2023-10-09T05:08:35.467 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osd_warn_op_age": "32.000000", 2023-10-09T05:08:35.467 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osdmap_full_prune_enabled": "true", 2023-10-09T05:08:35.467 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osdmap_full_prune_interval": "10", 2023-10-09T05:08:35.467 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osdmap_full_prune_min": "10000", 2023-10-09T05:08:35.467 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_osdmap_full_prune_txsize": "100", 2023-10-09T05:08:35.467 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2023-10-09T05:08:35.467 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_pg_stuck_threshold": "60", 2023-10-09T05:08:35.467 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2023-10-09T05:08:35.468 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_pg_warn_min_objects": "10000", 2023-10-09T05:08:35.468 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_pg_warn_min_per_osd": "0", 2023-10-09T05:08:35.468 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_pg_warn_min_pool_objects": "1000", 2023-10-09T05:08:35.468 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_pool_quota_crit_threshold": "0", 2023-10-09T05:08:35.468 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_pool_quota_warn_threshold": "0", 2023-10-09T05:08:35.468 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_probe_timeout": "2.000000", 2023-10-09T05:08:35.468 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_reweight_max_change": "0.050000", 2023-10-09T05:08:35.468 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_reweight_max_osds": "4", 2023-10-09T05:08:35.468 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_reweight_min_bytes_per_osd": "10", 2023-10-09T05:08:35.468 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_reweight_min_pgs_per_osd": "4", 2023-10-09T05:08:35.468 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2023-10-09T05:08:35.468 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2023-10-09T05:08:35.469 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2023-10-09T05:08:35.469 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_scrub_interval": "86400", 2023-10-09T05:08:35.469 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_scrub_max_keys": "100", 2023-10-09T05:08:35.469 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_scrub_timeout": "300", 2023-10-09T05:08:35.469 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_session_timeout": "300", 2023-10-09T05:08:35.469 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_smart_report_timeout": "5", 2023-10-09T05:08:35.469 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_stat_smooth_intervals": "6", 2023-10-09T05:08:35.469 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2023-10-09T05:08:35.469 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_stretch_pool_min_size": "2", 2023-10-09T05:08:35.469 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_stretch_pool_size": "4", 2023-10-09T05:08:35.469 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2023-10-09T05:08:35.469 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_subscribe_interval": "86400.000000", 2023-10-09T05:08:35.470 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_sync_debug": "false", 2023-10-09T05:08:35.470 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_sync_max_payload_keys": "2000", 2023-10-09T05:08:35.470 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_sync_max_payload_size": "1048576", 2023-10-09T05:08:35.470 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_sync_provider_kill_at": "0", 2023-10-09T05:08:35.470 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_sync_requester_kill_at": "0", 2023-10-09T05:08:35.470 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_sync_timeout": "60.000000", 2023-10-09T05:08:35.470 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_target_pg_per_osd": "100", 2023-10-09T05:08:35.470 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_tick_interval": "5", 2023-10-09T05:08:35.470 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_timecheck_interval": "300.000000", 2023-10-09T05:08:35.470 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_timecheck_skew_interval": "30.000000", 2023-10-09T05:08:35.470 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_use_min_delay_socket": "false", 2023-10-09T05:08:35.470 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_warn_older_version_delay": "604800", 2023-10-09T05:08:35.471 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2023-10-09T05:08:35.471 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2023-10-09T05:08:35.471 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2023-10-09T05:08:35.471 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_warn_on_filestore_osds": "true", 2023-10-09T05:08:35.471 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2023-10-09T05:08:35.471 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2023-10-09T05:08:35.471 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2023-10-09T05:08:35.471 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_warn_on_misplaced": "false", 2023-10-09T05:08:35.471 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2023-10-09T05:08:35.471 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_warn_on_older_version": "true", 2023-10-09T05:08:35.471 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2023-10-09T05:08:35.471 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_warn_on_pool_no_app": "true", 2023-10-09T05:08:35.471 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_warn_on_pool_no_redundancy": "true", 2023-10-09T05:08:35.472 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2023-10-09T05:08:35.472 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2023-10-09T05:08:35.472 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2023-10-09T05:08:35.472 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_warn_on_too_few_osds": "false", 2023-10-09T05:08:35.472 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2023-10-09T05:08:35.472 INFO:teuthology.orchestra.run.smithi133.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2023-10-09T05:08:35.472 INFO:teuthology.orchestra.run.smithi133.stdout: "monmap": "", 2023-10-09T05:08:35.472 INFO:teuthology.orchestra.run.smithi133.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-10-09T05:08:35.472 INFO:teuthology.orchestra.run.smithi133.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2023-10-09T05:08:35.472 INFO:teuthology.orchestra.run.smithi133.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2023-10-09T05:08:35.472 INFO:teuthology.orchestra.run.smithi133.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-10-09T05:08:35.472 INFO:teuthology.orchestra.run.smithi133.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2023-10-09T05:08:35.473 INFO:teuthology.orchestra.run.smithi133.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2023-10-09T05:08:35.473 INFO:teuthology.orchestra.run.smithi133.stdout: "motr_tracing_enabled": "false", 2023-10-09T05:08:35.473 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_async_op_threads": "3", 2023-10-09T05:08:35.473 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_async_rdma_buffer_size": "131072", 2023-10-09T05:08:35.473 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_async_rdma_cm": "false", 2023-10-09T05:08:35.473 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_async_rdma_device_name": "", 2023-10-09T05:08:35.473 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_async_rdma_dscp": "96", 2023-10-09T05:08:35.473 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_async_rdma_enable_hugepage": "false", 2023-10-09T05:08:35.473 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_async_rdma_gid_idx": "0", 2023-10-09T05:08:35.473 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_async_rdma_local_gid": "", 2023-10-09T05:08:35.473 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_async_rdma_polling_us": "1000", 2023-10-09T05:08:35.474 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_async_rdma_port_num": "1", 2023-10-09T05:08:35.474 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_async_rdma_receive_buffers": "32768", 2023-10-09T05:08:35.474 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_async_rdma_receive_queue_len": "4096", 2023-10-09T05:08:35.474 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_async_rdma_roce_ver": "1", 2023-10-09T05:08:35.474 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_async_rdma_send_buffers": "1024", 2023-10-09T05:08:35.474 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_async_rdma_sl": "3", 2023-10-09T05:08:35.474 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_async_rdma_support_srq": "true", 2023-10-09T05:08:35.474 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_async_rdma_type": "ib", 2023-10-09T05:08:35.474 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_async_reap_threshold": "5", 2023-10-09T05:08:35.474 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_bind_before_connect": "false", 2023-10-09T05:08:35.474 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_bind_ipv4": "true", 2023-10-09T05:08:35.474 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_bind_ipv6": "false", 2023-10-09T05:08:35.475 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_bind_msgr1": "true", 2023-10-09T05:08:35.475 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_bind_msgr2": "true", 2023-10-09T05:08:35.475 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_bind_port_max": "7568", 2023-10-09T05:08:35.475 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_bind_port_min": "6800", 2023-10-09T05:08:35.475 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_bind_prefer_ipv4": "false", 2023-10-09T05:08:35.475 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_bind_retry_count": "3", 2023-10-09T05:08:35.475 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_bind_retry_delay": "5", 2023-10-09T05:08:35.475 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_blackhole_client": "false", 2023-10-09T05:08:35.475 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_blackhole_mds": "false", 2023-10-09T05:08:35.475 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_blackhole_mgr": "false", 2023-10-09T05:08:35.475 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_blackhole_mon": "false", 2023-10-09T05:08:35.476 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_blackhole_osd": "false", 2023-10-09T05:08:35.476 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_client_mode": "crc secure", 2023-10-09T05:08:35.476 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_cluster_mode": "crc secure", 2023-10-09T05:08:35.476 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_cluster_type": "", 2023-10-09T05:08:35.476 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_compress_secure": "false", 2023-10-09T05:08:35.476 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_connection_idle_timeout": "900", 2023-10-09T05:08:35.476 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_connection_ready_timeout": "10", 2023-10-09T05:08:35.476 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_crc_data": "true", 2023-10-09T05:08:35.476 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_crc_header": "true", 2023-10-09T05:08:35.476 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_die_on_bad_msg": "false", 2023-10-09T05:08:35.476 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_die_on_bug": "true", 2023-10-09T05:08:35.476 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_die_on_old_message": "true", 2023-10-09T05:08:35.476 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_die_on_skipped_message": "false", 2023-10-09T05:08:35.477 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_die_on_unhandled_msg": "false", 2023-10-09T05:08:35.477 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_dispatch_throttle_bytes": "104857600", 2023-10-09T05:08:35.477 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_dpdk_coremask": "0xF", 2023-10-09T05:08:35.477 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_dpdk_debug_allow_loopback": "false", 2023-10-09T05:08:35.477 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_dpdk_devs_allowlist": "", 2023-10-09T05:08:35.477 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_dpdk_enable_tso": "true", 2023-10-09T05:08:35.477 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2023-10-09T05:08:35.477 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_dpdk_host_ipv4_addr": "", 2023-10-09T05:08:35.477 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_dpdk_hugepages": "", 2023-10-09T05:08:35.477 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_dpdk_hw_flow_control": "true", 2023-10-09T05:08:35.477 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2023-10-09T05:08:35.477 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_dpdk_lro": "true", 2023-10-09T05:08:35.478 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_dpdk_memory_channel": "4", 2023-10-09T05:08:35.478 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2023-10-09T05:08:35.478 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_dpdk_pmd": "", 2023-10-09T05:08:35.478 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_dpdk_port_id": "0", 2023-10-09T05:08:35.478 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2023-10-09T05:08:35.478 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_dump_corrupt_message_level": "1", 2023-10-09T05:08:35.478 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_dump_on_send": "false", 2023-10-09T05:08:35.478 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_initial_backoff": "0.200000", 2023-10-09T05:08:35.478 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_inject_delay_max": "1.000000", 2023-10-09T05:08:35.478 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_inject_delay_probability": "0.000000", 2023-10-09T05:08:35.478 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_inject_delay_type": "", 2023-10-09T05:08:35.478 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_inject_internal_delays": "0.000000", 2023-10-09T05:08:35.479 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_inject_network_congestion": "0", 2023-10-09T05:08:35.479 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_inject_socket_failures": "0", 2023-10-09T05:08:35.479 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_learn_addr_from_peer": "true", 2023-10-09T05:08:35.479 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_max_accept_failures": "4", 2023-10-09T05:08:35.479 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_max_backoff": "15.000000", 2023-10-09T05:08:35.479 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_mon_client_mode": "secure crc", 2023-10-09T05:08:35.479 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_mon_cluster_mode": "secure crc", 2023-10-09T05:08:35.479 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_mon_service_mode": "secure crc", 2023-10-09T05:08:35.479 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_osd_compress_min_size": "1024", 2023-10-09T05:08:35.479 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_osd_compress_mode": "none", 2023-10-09T05:08:35.479 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_osd_compression_algorithm": "snappy", 2023-10-09T05:08:35.479 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2023-10-09T05:08:35.480 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_pq_min_cost": "65536", 2023-10-09T05:08:35.480 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_public_type": "", 2023-10-09T05:08:35.480 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_service_mode": "crc secure", 2023-10-09T05:08:35.480 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_tcp_listen_backlog": "512", 2023-10-09T05:08:35.480 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_tcp_nodelay": "true", 2023-10-09T05:08:35.480 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_tcp_prefetch_max_size": "4096", 2023-10-09T05:08:35.480 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_tcp_rcvbuf": "0", 2023-10-09T05:08:35.480 INFO:teuthology.orchestra.run.smithi133.stdout: "ms_type": "async+posix", 2023-10-09T05:08:35.480 INFO:teuthology.orchestra.run.smithi133.stdout: "no_config_file": "false", 2023-10-09T05:08:35.480 INFO:teuthology.orchestra.run.smithi133.stdout: "objecter_completion_locks_per_session": "32", 2023-10-09T05:08:35.480 INFO:teuthology.orchestra.run.smithi133.stdout: "objecter_debug_inject_relock_delay": "false", 2023-10-09T05:08:35.480 INFO:teuthology.orchestra.run.smithi133.stdout: "objecter_inflight_op_bytes": "104857600", 2023-10-09T05:08:35.481 INFO:teuthology.orchestra.run.smithi133.stdout: "objecter_inflight_ops": "1024", 2023-10-09T05:08:35.481 INFO:teuthology.orchestra.run.smithi133.stdout: "objecter_inject_no_watch_ping": "false", 2023-10-09T05:08:35.481 INFO:teuthology.orchestra.run.smithi133.stdout: "objecter_retry_writes_after_first_reply": "false", 2023-10-09T05:08:35.481 INFO:teuthology.orchestra.run.smithi133.stdout: "objecter_tick_interval": "5.000000", 2023-10-09T05:08:35.481 INFO:teuthology.orchestra.run.smithi133.stdout: "objecter_timeout": "10.000000", 2023-10-09T05:08:35.481 INFO:teuthology.orchestra.run.smithi133.stdout: "objectstore_blackhole": "false", 2023-10-09T05:08:35.481 INFO:teuthology.orchestra.run.smithi133.stdout: "openssl_engine_opts": "", 2023-10-09T05:08:35.481 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_agent_delay_time": "5.000000", 2023-10-09T05:08:35.481 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_agent_hist_halflife": "1000", 2023-10-09T05:08:35.481 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_agent_max_low_ops": "2", 2023-10-09T05:08:35.481 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_agent_max_ops": "4", 2023-10-09T05:08:35.481 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_agent_min_evict_effort": "0.100000", 2023-10-09T05:08:35.482 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_agent_quantize_effort": "0.100000", 2023-10-09T05:08:35.482 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_agent_slop": "0.020000", 2023-10-09T05:08:35.482 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_aggregated_slow_ops_logging": "true", 2023-10-09T05:08:35.482 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_allow_recovery_below_min_size": "true", 2023-10-09T05:08:35.482 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_asio_thread_count": "2", 2023-10-09T05:08:35.482 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_async_recovery_min_cost": "100", 2023-10-09T05:08:35.482 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_auto_mark_unfound_lost": "false", 2023-10-09T05:08:35.482 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_backfill_retry_interval": "30.000000", 2023-10-09T05:08:35.482 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_backfill_scan_max": "512", 2023-10-09T05:08:35.482 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_backfill_scan_min": "64", 2023-10-09T05:08:35.482 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_backoff_on_degraded": "false", 2023-10-09T05:08:35.482 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_backoff_on_peering": "false", 2023-10-09T05:08:35.483 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_backoff_on_unfound": "true", 2023-10-09T05:08:35.483 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_beacon_report_interval": "300", 2023-10-09T05:08:35.483 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_bench_duration": "30", 2023-10-09T05:08:35.483 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_bench_large_size_max_throughput": "104857600", 2023-10-09T05:08:35.483 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_bench_max_block_size": "67108864", 2023-10-09T05:08:35.483 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_bench_small_size_max_iops": "100", 2023-10-09T05:08:35.483 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_blkin_trace_all": "false", 2023-10-09T05:08:35.483 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_blocked_scrub_grace_period": "120", 2023-10-09T05:08:35.483 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2023-10-09T05:08:35.483 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2023-10-09T05:08:35.483 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2023-10-09T05:08:35.483 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_check_for_log_corruption": "false", 2023-10-09T05:08:35.484 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_check_max_object_name_len_on_startup": "true", 2023-10-09T05:08:35.484 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-10-09T05:08:35.484 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2023-10-09T05:08:35.484 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-10-09T05:08:35.484 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_class_update_on_start": "true", 2023-10-09T05:08:35.484 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_client_message_cap": "256", 2023-10-09T05:08:35.484 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_client_message_size_cap": "524288000", 2023-10-09T05:08:35.484 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_client_op_priority": "63", 2023-10-09T05:08:35.484 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_client_watch_timeout": "30", 2023-10-09T05:08:35.484 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_command_max_records": "256", 2023-10-09T05:08:35.484 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_compact_on_start": "false", 2023-10-09T05:08:35.484 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_copyfrom_max_chunk": "8388608", 2023-10-09T05:08:35.485 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_crush_chooseleaf_type": "1", 2023-10-09T05:08:35.485 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_crush_initial_weight": "-1.000000", 2023-10-09T05:08:35.485 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_crush_update_on_start": "true", 2023-10-09T05:08:35.485 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_crush_update_weight_set": "true", 2023-10-09T05:08:35.485 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2023-10-09T05:08:35.485 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2023-10-09T05:08:35.485 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2023-10-09T05:08:35.485 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_debug_drop_ping_duration": "0", 2023-10-09T05:08:35.485 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_debug_drop_ping_probability": "0.000000", 2023-10-09T05:08:35.485 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_debug_feed_pullee": "-1", 2023-10-09T05:08:35.485 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_debug_inject_copyfrom_error": "false", 2023-10-09T05:08:35.485 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2023-10-09T05:08:35.485 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2023-10-09T05:08:35.486 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_debug_misdirected_ops": "false", 2023-10-09T05:08:35.486 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_debug_no_acting_change": "false", 2023-10-09T05:08:35.486 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_debug_no_purge_strays": "false", 2023-10-09T05:08:35.486 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_debug_op_order": "false", 2023-10-09T05:08:35.486 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_debug_pg_log_writeout": "false", 2023-10-09T05:08:35.486 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_debug_pretend_recovery_active": "false", 2023-10-09T05:08:35.486 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_debug_random_push_read_error": "0.000000", 2023-10-09T05:08:35.486 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2023-10-09T05:08:35.486 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_debug_shutdown": "false", 2023-10-09T05:08:35.486 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2023-10-09T05:08:35.486 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2023-10-09T05:08:35.487 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_debug_verify_cached_snaps": "false", 2023-10-09T05:08:35.487 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_debug_verify_missing_on_start": "false", 2023-10-09T05:08:35.487 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_debug_verify_snaps": "false", 2023-10-09T05:08:35.487 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_debug_verify_stray_on_activate": "false", 2023-10-09T05:08:35.487 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_deep_scrub_interval": "604800.000000", 2023-10-09T05:08:35.487 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_deep_scrub_keys": "1024", 2023-10-09T05:08:35.487 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2023-10-09T05:08:35.487 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2023-10-09T05:08:35.487 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2023-10-09T05:08:35.487 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_deep_scrub_stride": "524288", 2023-10-09T05:08:35.487 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2023-10-09T05:08:35.487 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_default_data_pool_replay_window": "45", 2023-10-09T05:08:35.488 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_default_notify_timeout": "30", 2023-10-09T05:08:35.488 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_delete_sleep": "0.000000", 2023-10-09T05:08:35.488 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_delete_sleep_hdd": "5.000000", 2023-10-09T05:08:35.488 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_delete_sleep_hybrid": "1.000000", 2023-10-09T05:08:35.488 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_delete_sleep_ssd": "1.000000", 2023-10-09T05:08:35.488 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_discard_disconnected_ops": "true", 2023-10-09T05:08:35.488 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_enable_op_tracker": "true", 2023-10-09T05:08:35.488 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2023-10-09T05:08:35.488 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_extblkdev_plugins": "vdo", 2023-10-09T05:08:35.488 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_failsafe_full_ratio": "0.970000", 2023-10-09T05:08:35.488 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_fast_fail_on_connection_refused": "true", 2023-10-09T05:08:35.488 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_fast_info": "true", 2023-10-09T05:08:35.489 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_fast_shutdown": "true", 2023-10-09T05:08:35.489 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_fast_shutdown_notify_mon": "true", 2023-10-09T05:08:35.489 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_fast_shutdown_timeout": "15", 2023-10-09T05:08:35.489 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_find_best_info_ignore_history_les": "false", 2023-10-09T05:08:35.489 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_force_auth_primary_missing_objects": "100", 2023-10-09T05:08:35.489 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2023-10-09T05:08:35.489 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_function_tracing": "false", 2023-10-09T05:08:35.489 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_heartbeat_grace": "20", 2023-10-09T05:08:35.489 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_heartbeat_interval": "6", 2023-10-09T05:08:35.489 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2023-10-09T05:08:35.489 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_heartbeat_min_peers": "10", 2023-10-09T05:08:35.489 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_heartbeat_min_size": "2000", 2023-10-09T05:08:35.489 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_heartbeat_stale": "600", 2023-10-09T05:08:35.490 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2023-10-09T05:08:35.490 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_hit_set_max_size": "100000", 2023-10-09T05:08:35.490 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_hit_set_min_size": "1000", 2023-10-09T05:08:35.490 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_hit_set_namespace": ".ceph-internal", 2023-10-09T05:08:35.490 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_ignore_stale_divergent_priors": "false", 2023-10-09T05:08:35.490 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2023-10-09T05:08:35.490 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_inject_failure_on_pg_removal": "false", 2023-10-09T05:08:35.490 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2023-10-09T05:08:35.490 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_journal_flush_on_shutdown": "true", 2023-10-09T05:08:35.490 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_journal_size": "5120", 2023-10-09T05:08:35.490 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_kill_backfill_at": "0", 2023-10-09T05:08:35.491 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_loop_before_reset_tphandle": "64", 2023-10-09T05:08:35.491 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_map_cache_size": "50", 2023-10-09T05:08:35.491 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_map_dedup": "true", 2023-10-09T05:08:35.491 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_map_message_max": "40", 2023-10-09T05:08:35.491 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_map_message_max_bytes": "10485760", 2023-10-09T05:08:35.491 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_map_share_max_epochs": "40", 2023-10-09T05:08:35.491 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_max_attr_name_len": "100", 2023-10-09T05:08:35.491 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_max_attr_size": "0", 2023-10-09T05:08:35.491 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_max_backfills": "1", 2023-10-09T05:08:35.491 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_max_markdown_count": "5", 2023-10-09T05:08:35.491 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_max_markdown_period": "600", 2023-10-09T05:08:35.491 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_max_object_name_len": "2048", 2023-10-09T05:08:35.492 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_max_object_namespace_len": "256", 2023-10-09T05:08:35.492 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_max_object_size": "134217728", 2023-10-09T05:08:35.492 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2023-10-09T05:08:35.492 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_max_omap_entries_per_request": "1024", 2023-10-09T05:08:35.492 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_max_pg_blocked_by": "16", 2023-10-09T05:08:35.492 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_max_pg_log_entries": "10000", 2023-10-09T05:08:35.492 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2023-10-09T05:08:35.492 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_max_pgls": "1024", 2023-10-09T05:08:35.492 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_max_push_cost": "8388608", 2023-10-09T05:08:35.492 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_max_push_objects": "10", 2023-10-09T05:08:35.492 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_max_scrubs": "1", 2023-10-09T05:08:35.492 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2023-10-09T05:08:35.492 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_max_trimming_pgs": "2", 2023-10-09T05:08:35.493 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_max_write_op_reply_len": "64", 2023-10-09T05:08:35.493 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_max_write_size": "90", 2023-10-09T05:08:35.493 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2023-10-09T05:08:35.493 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2023-10-09T05:08:35.493 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2023-10-09T05:08:35.493 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2023-10-09T05:08:35.493 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2023-10-09T05:08:35.493 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2023-10-09T05:08:35.493 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2023-10-09T05:08:35.493 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_mclock_override_recovery_settings": "false", 2023-10-09T05:08:35.493 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_mclock_profile": "balanced", 2023-10-09T05:08:35.494 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2023-10-09T05:08:35.494 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2023-10-09T05:08:35.494 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2023-10-09T05:08:35.494 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2023-10-09T05:08:35.494 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2023-10-09T05:08:35.494 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2023-10-09T05:08:35.494 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2023-10-09T05:08:35.494 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2023-10-09T05:08:35.494 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2023-10-09T05:08:35.494 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_mclock_scheduler_client_wgt": "1", 2023-10-09T05:08:35.494 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_mclock_skip_benchmark": "false", 2023-10-09T05:08:35.494 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_memory_base": "805306368", 2023-10-09T05:08:35.495 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_memory_cache_min": "134217728", 2023-10-09T05:08:35.495 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_memory_cache_resize_interval": "1.000000", 2023-10-09T05:08:35.495 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_memory_expected_fragmentation": "0.150000", 2023-10-09T05:08:35.495 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_memory_target": "4294967296", 2023-10-09T05:08:35.495 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_memory_target_autotune": "false", 2023-10-09T05:08:35.495 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2023-10-09T05:08:35.495 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_min_pg_log_entries": "250", 2023-10-09T05:08:35.495 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_min_recovery_priority": "0", 2023-10-09T05:08:35.495 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_mon_heartbeat_interval": "30", 2023-10-09T05:08:35.495 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2023-10-09T05:08:35.495 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_mon_report_interval": "5", 2023-10-09T05:08:35.495 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_mon_report_max_in_flight": "2", 2023-10-09T05:08:35.496 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_mon_shutdown_timeout": "5.000000", 2023-10-09T05:08:35.496 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_num_cache_shards": "32", 2023-10-09T05:08:35.496 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_num_op_tracker_shard": "32", 2023-10-09T05:08:35.496 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_numa_auto_affinity": "true", 2023-10-09T05:08:35.496 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_numa_node": "-1", 2023-10-09T05:08:35.496 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_numa_prefer_iface": "true", 2023-10-09T05:08:35.496 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_object_clean_region_max_num_intervals": "10", 2023-10-09T05:08:35.496 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_objecter_finishers": "1", 2023-10-09T05:08:35.496 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_objectstore": "bluestore", 2023-10-09T05:08:35.496 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_objectstore_fuse": "false", 2023-10-09T05:08:35.496 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_objectstore_tracing": "false", 2023-10-09T05:08:35.496 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_op_complaint_time": "30.000000", 2023-10-09T05:08:35.496 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_op_history_duration": "600", 2023-10-09T05:08:35.497 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_op_history_size": "20", 2023-10-09T05:08:35.497 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_op_history_slow_op_size": "20", 2023-10-09T05:08:35.497 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2023-10-09T05:08:35.497 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_op_log_threshold": "5", 2023-10-09T05:08:35.497 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_op_num_shards": "0", 2023-10-09T05:08:35.497 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_op_num_shards_hdd": "5", 2023-10-09T05:08:35.497 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_op_num_shards_ssd": "8", 2023-10-09T05:08:35.497 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_op_num_threads_per_shard": "0", 2023-10-09T05:08:35.497 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2023-10-09T05:08:35.497 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2023-10-09T05:08:35.497 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2023-10-09T05:08:35.498 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_op_pq_min_cost": "65536", 2023-10-09T05:08:35.498 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_op_queue": "mclock_scheduler", 2023-10-09T05:08:35.498 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_op_queue_cut_off": "high", 2023-10-09T05:08:35.498 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_op_thread_suicide_timeout": "150", 2023-10-09T05:08:35.498 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_op_thread_timeout": "15", 2023-10-09T05:08:35.498 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_open_classes_on_start": "true", 2023-10-09T05:08:35.498 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_os_flags": "0", 2023-10-09T05:08:35.498 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_peering_op_priority": "255", 2023-10-09T05:08:35.498 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pg_delete_cost": "1048576", 2023-10-09T05:08:35.498 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pg_delete_priority": "5", 2023-10-09T05:08:35.498 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2023-10-09T05:08:35.498 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2023-10-09T05:08:35.499 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pg_log_dups_tracked": "3000", 2023-10-09T05:08:35.499 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pg_log_trim_max": "10000", 2023-10-09T05:08:35.499 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pg_log_trim_min": "100", 2023-10-09T05:08:35.499 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2023-10-09T05:08:35.499 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pg_object_context_cache_count": "64", 2023-10-09T05:08:35.499 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pg_stat_report_interval_max": "500", 2023-10-09T05:08:35.499 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2023-10-09T05:08:35.499 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pool_default_cache_min_evict_age": "0", 2023-10-09T05:08:35.499 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pool_default_cache_min_flush_age": "0", 2023-10-09T05:08:35.499 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2023-10-09T05:08:35.499 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2023-10-09T05:08:35.499 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2023-10-09T05:08:35.500 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pool_default_crimson": "false", 2023-10-09T05:08:35.500 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pool_default_crush_rule": "-1", 2023-10-09T05:08:35.500 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pool_default_ec_fast_read": "false", 2023-10-09T05:08:35.500 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2023-10-09T05:08:35.500 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pool_default_flag_bulk": "false", 2023-10-09T05:08:35.500 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pool_default_flag_hashpspool": "true", 2023-10-09T05:08:35.500 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pool_default_flag_nodelete": "false", 2023-10-09T05:08:35.500 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pool_default_flag_nopgchange": "false", 2023-10-09T05:08:35.500 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pool_default_flag_nosizechange": "false", 2023-10-09T05:08:35.500 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pool_default_flags": "0", 2023-10-09T05:08:35.500 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2023-10-09T05:08:35.500 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pool_default_min_size": "0", 2023-10-09T05:08:35.501 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2023-10-09T05:08:35.501 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pool_default_pg_num": "32", 2023-10-09T05:08:35.501 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pool_default_pgp_num": "0", 2023-10-09T05:08:35.501 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2023-10-09T05:08:35.501 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pool_default_size": "3", 2023-10-09T05:08:35.501 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pool_default_type": "replicated", 2023-10-09T05:08:35.501 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2023-10-09T05:08:35.501 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_pool_use_gmt_hitset": "true", 2023-10-09T05:08:35.501 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_push_per_object_cost": "1000", 2023-10-09T05:08:35.501 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_read_ec_check_for_errors": "false", 2023-10-09T05:08:35.501 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_recover_clone_overlap": "true", 2023-10-09T05:08:35.501 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_recover_clone_overlap_limit": "10", 2023-10-09T05:08:35.502 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_recovery_cost": "20971520", 2023-10-09T05:08:35.502 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_recovery_delay_start": "0.000000", 2023-10-09T05:08:35.502 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_recovery_max_active": "0", 2023-10-09T05:08:35.502 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_recovery_max_active_hdd": "3", 2023-10-09T05:08:35.502 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_recovery_max_active_ssd": "10", 2023-10-09T05:08:35.502 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_recovery_max_chunk": "8388608", 2023-10-09T05:08:35.502 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2023-10-09T05:08:35.502 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_recovery_max_single_start": "1", 2023-10-09T05:08:35.502 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_recovery_op_priority": "3", 2023-10-09T05:08:35.502 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_recovery_op_warn_multiple": "16", 2023-10-09T05:08:35.502 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_recovery_priority": "5", 2023-10-09T05:08:35.502 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_recovery_retry_interval": "30.000000", 2023-10-09T05:08:35.503 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_recovery_sleep": "0.000000", 2023-10-09T05:08:35.503 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_recovery_sleep_hdd": "0.100000", 2023-10-09T05:08:35.503 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2023-10-09T05:08:35.503 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_recovery_sleep_ssd": "0.000000", 2023-10-09T05:08:35.503 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_repair_during_recovery": "false", 2023-10-09T05:08:35.503 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_requested_scrub_priority": "120", 2023-10-09T05:08:35.503 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2023-10-09T05:08:35.503 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_rollback_to_cluster_snap": "", 2023-10-09T05:08:35.503 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_scrub_auto_repair": "false", 2023-10-09T05:08:35.503 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_scrub_auto_repair_num_errors": "5", 2023-10-09T05:08:35.503 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_scrub_backoff_ratio": "0.660000", 2023-10-09T05:08:35.504 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_scrub_begin_hour": "0", 2023-10-09T05:08:35.504 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_scrub_begin_week_day": "0", 2023-10-09T05:08:35.504 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_scrub_chunk_max": "25", 2023-10-09T05:08:35.504 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_scrub_chunk_min": "5", 2023-10-09T05:08:35.504 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_scrub_cost": "52428800", 2023-10-09T05:08:35.504 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_scrub_during_recovery": "false", 2023-10-09T05:08:35.504 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_scrub_end_hour": "0", 2023-10-09T05:08:35.504 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_scrub_end_week_day": "0", 2023-10-09T05:08:35.504 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_scrub_event_cost": "4096", 2023-10-09T05:08:35.504 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_scrub_extended_sleep": "0.000000", 2023-10-09T05:08:35.504 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2023-10-09T05:08:35.504 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_scrub_invalid_stats": "true", 2023-10-09T05:08:35.505 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_scrub_load_threshold": "0.500000", 2023-10-09T05:08:35.505 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_scrub_max_interval": "604800.000000", 2023-10-09T05:08:35.505 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_scrub_max_preemptions": "5", 2023-10-09T05:08:35.505 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_scrub_min_interval": "86400.000000", 2023-10-09T05:08:35.505 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_scrub_priority": "5", 2023-10-09T05:08:35.505 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_scrub_reservation_timeout": "5000", 2023-10-09T05:08:35.505 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_scrub_sleep": "0.000000", 2023-10-09T05:08:35.505 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_scrub_slow_reservation_response": "2200", 2023-10-09T05:08:35.505 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_shallow_scrub_chunk_max": "100", 2023-10-09T05:08:35.505 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_shallow_scrub_chunk_min": "50", 2023-10-09T05:08:35.505 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_shutdown_pgref_assert": "false", 2023-10-09T05:08:35.505 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_skip_data_digest": "false", 2023-10-09T05:08:35.506 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_smart_report_timeout": "5", 2023-10-09T05:08:35.506 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_snap_trim_cost": "1048576", 2023-10-09T05:08:35.506 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_snap_trim_priority": "5", 2023-10-09T05:08:35.506 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_snap_trim_sleep": "0.000000", 2023-10-09T05:08:35.506 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2023-10-09T05:08:35.506 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2023-10-09T05:08:35.506 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2023-10-09T05:08:35.506 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_stats_update_period_not_scrubbing": "120", 2023-10-09T05:08:35.506 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_stats_update_period_scrubbing": "15", 2023-10-09T05:08:35.506 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2023-10-09T05:08:35.506 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_target_transaction_size": "30", 2023-10-09T05:08:35.507 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_tier_default_cache_hit_set_count": "4", 2023-10-09T05:08:35.507 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2023-10-09T05:08:35.507 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2023-10-09T05:08:35.507 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2023-10-09T05:08:35.507 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2023-10-09T05:08:35.507 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2023-10-09T05:08:35.507 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2023-10-09T05:08:35.507 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_tier_default_cache_mode": "writeback", 2023-10-09T05:08:35.507 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2023-10-09T05:08:35.507 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_tier_promote_max_objects_sec": "25", 2023-10-09T05:08:35.507 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_tracing": "false", 2023-10-09T05:08:35.507 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_use_stale_snap": "false", 2023-10-09T05:08:35.508 INFO:teuthology.orchestra.run.smithi133.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2023-10-09T05:08:35.508 INFO:teuthology.orchestra.run.smithi133.stdout: "osdc_blkin_trace_all": "false", 2023-10-09T05:08:35.508 INFO:teuthology.orchestra.run.smithi133.stdout: "paxos_kill_at": "0", 2023-10-09T05:08:35.508 INFO:teuthology.orchestra.run.smithi133.stdout: "paxos_max_join_drift": "10", 2023-10-09T05:08:35.508 INFO:teuthology.orchestra.run.smithi133.stdout: "paxos_min": "500", 2023-10-09T05:08:35.508 INFO:teuthology.orchestra.run.smithi133.stdout: "paxos_min_wait": "0.050000", 2023-10-09T05:08:35.508 INFO:teuthology.orchestra.run.smithi133.stdout: "paxos_propose_interval": "1.000000", 2023-10-09T05:08:35.508 INFO:teuthology.orchestra.run.smithi133.stdout: "paxos_service_trim_max": "500", 2023-10-09T05:08:35.508 INFO:teuthology.orchestra.run.smithi133.stdout: "paxos_service_trim_max_multiplier": "20", 2023-10-09T05:08:35.508 INFO:teuthology.orchestra.run.smithi133.stdout: "paxos_service_trim_min": "250", 2023-10-09T05:08:35.508 INFO:teuthology.orchestra.run.smithi133.stdout: "paxos_stash_full_interval": "25", 2023-10-09T05:08:35.509 INFO:teuthology.orchestra.run.smithi133.stdout: "paxos_trim_max": "500", 2023-10-09T05:08:35.509 INFO:teuthology.orchestra.run.smithi133.stdout: "paxos_trim_min": "250", 2023-10-09T05:08:35.509 INFO:teuthology.orchestra.run.smithi133.stdout: "perf": "true", 2023-10-09T05:08:35.509 INFO:teuthology.orchestra.run.smithi133.stdout: "pid_file": "", 2023-10-09T05:08:35.509 INFO:teuthology.orchestra.run.smithi133.stdout: "plugin_crypto_accelerator": "crypto_isal", 2023-10-09T05:08:35.509 INFO:teuthology.orchestra.run.smithi133.stdout: "plugin_dir": "/usr/lib64/ceph", 2023-10-09T05:08:35.509 INFO:teuthology.orchestra.run.smithi133.stdout: "public_addr": "-", 2023-10-09T05:08:35.509 INFO:teuthology.orchestra.run.smithi133.stdout: "public_addrv": "", 2023-10-09T05:08:35.509 INFO:teuthology.orchestra.run.smithi133.stdout: "public_bind_addr": "-", 2023-10-09T05:08:35.509 INFO:teuthology.orchestra.run.smithi133.stdout: "public_network": "172.21.15.254/32,172.21.0.0/20", 2023-10-09T05:08:35.510 INFO:teuthology.orchestra.run.smithi133.stdout: "public_network_interface": "", 2023-10-09T05:08:35.510 INFO:teuthology.orchestra.run.smithi133.stdout: "qat_compressor_enabled": "false", 2023-10-09T05:08:35.510 INFO:teuthology.orchestra.run.smithi133.stdout: "qat_compressor_session_max_number": "256", 2023-10-09T05:08:35.510 INFO:teuthology.orchestra.run.smithi133.stdout: "rados_mon_op_timeout": "0", 2023-10-09T05:08:35.510 INFO:teuthology.orchestra.run.smithi133.stdout: "rados_osd_op_timeout": "0", 2023-10-09T05:08:35.510 INFO:teuthology.orchestra.run.smithi133.stdout: "rados_tracing": "false", 2023-10-09T05:08:35.510 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_atime_update_interval": "60", 2023-10-09T05:08:35.510 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2023-10-09T05:08:35.510 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_balance_parent_reads": "false", 2023-10-09T05:08:35.510 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_balance_snap_reads": "false", 2023-10-09T05:08:35.510 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_blkin_trace_all": "false", 2023-10-09T05:08:35.510 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_blocklist_expire_seconds": "0", 2023-10-09T05:08:35.511 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_blocklist_on_break_lock": "true", 2023-10-09T05:08:35.511 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_cache": "true", 2023-10-09T05:08:35.511 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_cache_block_writes_upfront": "false", 2023-10-09T05:08:35.511 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_cache_max_dirty": "25165824", 2023-10-09T05:08:35.511 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_cache_max_dirty_age": "1.000000", 2023-10-09T05:08:35.511 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_cache_max_dirty_object": "0", 2023-10-09T05:08:35.511 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_cache_policy": "writearound", 2023-10-09T05:08:35.511 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_cache_size": "33554432", 2023-10-09T05:08:35.511 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_cache_target_dirty": "16777216", 2023-10-09T05:08:35.511 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_cache_writethrough_until_flush": "true", 2023-10-09T05:08:35.511 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_clone_copy_on_read": "false", 2023-10-09T05:08:35.512 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_compression_hint": "none", 2023-10-09T05:08:35.512 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_concurrent_management_ops": "10", 2023-10-09T05:08:35.512 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_config_pool_override_update_timestamp": "0", 2023-10-09T05:08:35.512 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_default_clone_format": "auto", 2023-10-09T05:08:35.512 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_default_data_pool": "", 2023-10-09T05:08:35.512 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_default_features": "61", 2023-10-09T05:08:35.512 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_default_format": "2", 2023-10-09T05:08:35.512 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_default_map_options": "", 2023-10-09T05:08:35.512 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_default_order": "22", 2023-10-09T05:08:35.512 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_default_pool": "rbd", 2023-10-09T05:08:35.512 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2023-10-09T05:08:35.513 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_default_stripe_count": "0", 2023-10-09T05:08:35.513 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_default_stripe_unit": "0", 2023-10-09T05:08:35.513 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_disable_zero_copy_writes": "true", 2023-10-09T05:08:35.513 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_discard_granularity_bytes": "65536", 2023-10-09T05:08:35.513 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_discard_on_zeroed_write_same": "true", 2023-10-09T05:08:35.513 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_enable_alloc_hint": "true", 2023-10-09T05:08:35.513 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2023-10-09T05:08:35.513 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_io_scheduler": "simple", 2023-10-09T05:08:35.513 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2023-10-09T05:08:35.513 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_journal_commit_age": "5.000000", 2023-10-09T05:08:35.513 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2023-10-09T05:08:35.514 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_journal_max_payload_bytes": "16384", 2023-10-09T05:08:35.514 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_journal_object_flush_age": "0.000000", 2023-10-09T05:08:35.514 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_journal_object_flush_bytes": "1048576", 2023-10-09T05:08:35.514 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_journal_object_flush_interval": "0", 2023-10-09T05:08:35.514 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2023-10-09T05:08:35.514 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2023-10-09T05:08:35.514 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_journal_order": "24", 2023-10-09T05:08:35.514 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_journal_pool": "", 2023-10-09T05:08:35.514 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_journal_splay_width": "4", 2023-10-09T05:08:35.514 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_localize_parent_reads": "false", 2023-10-09T05:08:35.514 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_localize_snap_reads": "false", 2023-10-09T05:08:35.514 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2023-10-09T05:08:35.515 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2023-10-09T05:08:35.515 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2023-10-09T05:08:35.515 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2023-10-09T05:08:35.515 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2023-10-09T05:08:35.515 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2023-10-09T05:08:35.515 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_mirror_image_policy_type": "simple", 2023-10-09T05:08:35.515 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2023-10-09T05:08:35.515 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_mirror_image_state_check_interval": "30", 2023-10-09T05:08:35.515 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2023-10-09T05:08:35.515 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2023-10-09T05:08:35.516 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2023-10-09T05:08:35.516 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2023-10-09T05:08:35.516 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2023-10-09T05:08:35.516 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_mirror_memory_autotune": "true", 2023-10-09T05:08:35.516 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_mirror_memory_base": "805306368", 2023-10-09T05:08:35.516 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2023-10-09T05:08:35.516 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_mirror_memory_cache_min": "134217728", 2023-10-09T05:08:35.516 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2023-10-09T05:08:35.516 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2023-10-09T05:08:35.516 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_mirror_memory_target": "4294967296", 2023-10-09T05:08:35.516 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_mirror_perf_stats_prio": "5", 2023-10-09T05:08:35.517 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2023-10-09T05:08:35.517 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2023-10-09T05:08:35.517 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_mirroring_delete_delay": "0", 2023-10-09T05:08:35.517 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2023-10-09T05:08:35.517 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_mirroring_replay_delay": "0", 2023-10-09T05:08:35.517 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2023-10-09T05:08:35.517 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2023-10-09T05:08:35.517 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_move_to_trash_on_remove": "false", 2023-10-09T05:08:35.517 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2023-10-09T05:08:35.517 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_mtime_update_interval": "60", 2023-10-09T05:08:35.517 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_non_blocking_aio": "true", 2023-10-09T05:08:35.518 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_op_thread_timeout": "60", 2023-10-09T05:08:35.518 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_op_threads": "1", 2023-10-09T05:08:35.518 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_parent_cache_enabled": "false", 2023-10-09T05:08:35.518 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_persistent_cache_mode": "disabled", 2023-10-09T05:08:35.518 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_persistent_cache_path": "/tmp", 2023-10-09T05:08:35.518 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_persistent_cache_size": "1073741824", 2023-10-09T05:08:35.518 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_plugins": "", 2023-10-09T05:08:35.518 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_qos_bps_burst": "0", 2023-10-09T05:08:35.518 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_qos_bps_burst_seconds": "1", 2023-10-09T05:08:35.518 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_qos_bps_limit": "0", 2023-10-09T05:08:35.518 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_qos_exclude_ops": "0", 2023-10-09T05:08:35.519 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_qos_iops_burst": "0", 2023-10-09T05:08:35.519 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_qos_iops_burst_seconds": "1", 2023-10-09T05:08:35.519 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_qos_iops_limit": "0", 2023-10-09T05:08:35.519 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_qos_read_bps_burst": "0", 2023-10-09T05:08:35.519 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2023-10-09T05:08:35.519 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_qos_read_bps_limit": "0", 2023-10-09T05:08:35.519 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_qos_read_iops_burst": "0", 2023-10-09T05:08:35.519 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2023-10-09T05:08:35.519 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_qos_read_iops_limit": "0", 2023-10-09T05:08:35.519 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_qos_schedule_tick_min": "50", 2023-10-09T05:08:35.519 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_qos_write_bps_burst": "0", 2023-10-09T05:08:35.520 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2023-10-09T05:08:35.520 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_qos_write_bps_limit": "0", 2023-10-09T05:08:35.520 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_qos_write_iops_burst": "0", 2023-10-09T05:08:35.520 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2023-10-09T05:08:35.520 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_qos_write_iops_limit": "0", 2023-10-09T05:08:35.520 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_quiesce_notification_attempts": "10", 2023-10-09T05:08:35.520 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_read_from_replica_policy": "default", 2023-10-09T05:08:35.520 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2023-10-09T05:08:35.520 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_readahead_max_bytes": "524288", 2023-10-09T05:08:35.520 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_readahead_trigger_requests": "10", 2023-10-09T05:08:35.520 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_request_timed_out_seconds": "30", 2023-10-09T05:08:35.521 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_skip_partial_discard": "true", 2023-10-09T05:08:35.521 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2023-10-09T05:08:35.521 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_tracing": "false", 2023-10-09T05:08:35.521 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_validate_names": "true", 2023-10-09T05:08:35.521 INFO:teuthology.orchestra.run.smithi133.stdout: "rbd_validate_pool": "true", 2023-10-09T05:08:35.521 INFO:teuthology.orchestra.run.smithi133.stdout: "restapi_base_url": "", 2023-10-09T05:08:35.521 INFO:teuthology.orchestra.run.smithi133.stdout: "restapi_log_level": "", 2023-10-09T05:08:35.521 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_acl_grants_max_num": "100", 2023-10-09T05:08:35.521 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_admin_entry": "admin", 2023-10-09T05:08:35.521 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2023-10-09T05:08:35.521 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_backend_store": "rados", 2023-10-09T05:08:35.521 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_barbican_url": "", 2023-10-09T05:08:35.522 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_beast_enable_async": "true", 2023-10-09T05:08:35.522 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2023-10-09T05:08:35.522 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_bucket_default_quota_max_size": "-1", 2023-10-09T05:08:35.522 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_bucket_index_max_aio": "128", 2023-10-09T05:08:35.522 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2023-10-09T05:08:35.522 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_bucket_quota_cache_size": "10000", 2023-10-09T05:08:35.522 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_bucket_quota_ttl": "600", 2023-10-09T05:08:35.522 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_bucket_sync_spawn_window": "20", 2023-10-09T05:08:35.522 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_cache_enabled": "true", 2023-10-09T05:08:35.522 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_cache_expiry_interval": "900", 2023-10-09T05:08:35.522 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_cache_lru_size": "10000", 2023-10-09T05:08:35.523 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_config_store": "rados", 2023-10-09T05:08:35.523 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_content_length_compat": "false", 2023-10-09T05:08:35.523 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_copy_obj_progress": "true", 2023-10-09T05:08:35.523 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2023-10-09T05:08:35.523 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_cors_rules_max_num": "100", 2023-10-09T05:08:35.523 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_cross_domain_policy": "", 2023-10-09T05:08:35.523 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_default_encryption_key": "", 2023-10-09T05:08:35.523 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_kmip_addr": "", 2023-10-09T05:08:35.523 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_kmip_ca_path": "", 2023-10-09T05:08:35.523 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_kmip_client_cert": "", 2023-10-09T05:08:35.523 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_kmip_client_key": "", 2023-10-09T05:08:35.523 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_kmip_kms_key_template": "", 2023-10-09T05:08:35.524 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_kmip_password": "", 2023-10-09T05:08:35.524 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2023-10-09T05:08:35.524 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_kmip_username": "", 2023-10-09T05:08:35.524 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_require_ssl": "true", 2023-10-09T05:08:35.524 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2023-10-09T05:08:35.524 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2023-10-09T05:08:35.524 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_sse_s3_backend": "vault", 2023-10-09T05:08:35.524 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2023-10-09T05:08:35.524 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2023-10-09T05:08:35.524 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2023-10-09T05:08:35.524 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2023-10-09T05:08:35.524 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2023-10-09T05:08:35.525 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2023-10-09T05:08:35.525 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2023-10-09T05:08:35.525 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2023-10-09T05:08:35.525 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2023-10-09T05:08:35.525 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2023-10-09T05:08:35.525 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2023-10-09T05:08:35.525 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_suppress_logs": "true", 2023-10-09T05:08:35.525 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_vault_addr": "", 2023-10-09T05:08:35.525 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_vault_auth": "token", 2023-10-09T05:08:35.525 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_vault_namespace": "", 2023-10-09T05:08:35.525 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_vault_prefix": "", 2023-10-09T05:08:35.526 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_vault_secret_engine": "transit", 2023-10-09T05:08:35.526 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_vault_ssl_cacert": "", 2023-10-09T05:08:35.526 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2023-10-09T05:08:35.526 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2023-10-09T05:08:35.526 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_vault_token_file": "", 2023-10-09T05:08:35.526 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_crypt_vault_verify_ssl": "true", 2023-10-09T05:08:35.526 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_curl_buffersize": "524288", 2023-10-09T05:08:35.526 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_curl_low_speed_limit": "1024", 2023-10-09T05:08:35.526 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_curl_low_speed_time": "300", 2023-10-09T05:08:35.526 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_curl_tcp_keepalive": "0", 2023-10-09T05:08:35.526 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_curl_wait_timeout_ms": "1000", 2023-10-09T05:08:35.527 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2023-10-09T05:08:35.527 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2023-10-09T05:08:35.527 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2023-10-09T05:08:35.527 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2023-10-09T05:08:35.527 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_d3n_l1_fadvise": "4", 2023-10-09T05:08:35.527 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2023-10-09T05:08:35.527 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_d3n_libaio_aio_num": "64", 2023-10-09T05:08:35.527 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_d3n_libaio_aio_threads": "20", 2023-10-09T05:08:35.527 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2023-10-09T05:08:35.527 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_data_log_changes_size": "1000", 2023-10-09T05:08:35.527 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_data_log_num_shards": "128", 2023-10-09T05:08:35.528 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_data_log_obj_prefix": "data_log", 2023-10-09T05:08:35.528 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_data_log_window": "30", 2023-10-09T05:08:35.528 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_data_notify_interval_msec": "0", 2023-10-09T05:08:35.528 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_data_sync_poll_interval": "20", 2023-10-09T05:08:35.528 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_data_sync_spawn_window": "20", 2023-10-09T05:08:35.528 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2023-10-09T05:08:35.528 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_debug_inject_set_olh_err": "0", 2023-10-09T05:08:35.528 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_default_data_log_backing": "fifo", 2023-10-09T05:08:35.528 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_default_realm_info_oid": "default.realm", 2023-10-09T05:08:35.528 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_default_region_info_oid": "default.region", 2023-10-09T05:08:35.528 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_default_zone_info_oid": "default.zone", 2023-10-09T05:08:35.528 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2023-10-09T05:08:35.529 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_defer_to_bucket_acls": "", 2023-10-09T05:08:35.529 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_delete_multi_obj_max_num": "1000", 2023-10-09T05:08:35.529 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_dmclock_admin_lim": "0.000000", 2023-10-09T05:08:35.529 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_dmclock_admin_res": "100.000000", 2023-10-09T05:08:35.529 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2023-10-09T05:08:35.529 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_dmclock_auth_lim": "0.000000", 2023-10-09T05:08:35.529 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_dmclock_auth_res": "200.000000", 2023-10-09T05:08:35.529 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2023-10-09T05:08:35.529 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_dmclock_data_lim": "0.000000", 2023-10-09T05:08:35.529 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_dmclock_data_res": "500.000000", 2023-10-09T05:08:35.529 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_dmclock_data_wgt": "500.000000", 2023-10-09T05:08:35.529 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2023-10-09T05:08:35.530 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_dmclock_metadata_res": "500.000000", 2023-10-09T05:08:35.530 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2023-10-09T05:08:35.530 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_dns_name": "", 2023-10-09T05:08:35.530 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_dns_s3website_name": "", 2023-10-09T05:08:35.530 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_dynamic_resharding": "true", 2023-10-09T05:08:35.530 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2023-10-09T05:08:35.530 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_enable_gc_threads": "true", 2023-10-09T05:08:35.530 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_enable_lc_threads": "true", 2023-10-09T05:08:35.530 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_enable_ops_log": "false", 2023-10-09T05:08:35.530 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_enable_quota_threads": "true", 2023-10-09T05:08:35.531 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_enable_static_website": "false", 2023-10-09T05:08:35.531 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_enable_usage_log": "false", 2023-10-09T05:08:35.531 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_enforce_swift_acls": "true", 2023-10-09T05:08:35.531 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_exit_timeout_secs": "120", 2023-10-09T05:08:35.531 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_expose_bucket": "false", 2023-10-09T05:08:35.531 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_extended_http_attrs": "", 2023-10-09T05:08:35.531 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_filter": "none", 2023-10-09T05:08:35.531 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2023-10-09T05:08:35.531 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_frontends": "beast port=7480", 2023-10-09T05:08:35.531 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_gc_max_concurrent_io": "10", 2023-10-09T05:08:35.531 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_gc_max_deferred": "50", 2023-10-09T05:08:35.531 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2023-10-09T05:08:35.531 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_gc_max_objs": "32", 2023-10-09T05:08:35.532 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_gc_max_queue_size": "134213632", 2023-10-09T05:08:35.532 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_gc_max_trim_chunk": "16", 2023-10-09T05:08:35.532 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_gc_obj_min_wait": "7200", 2023-10-09T05:08:35.532 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_gc_processor_max_time": "3600", 2023-10-09T05:08:35.532 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_gc_processor_period": "3600", 2023-10-09T05:08:35.532 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_get_obj_max_req_size": "4194304", 2023-10-09T05:08:35.532 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_get_obj_window_size": "16777216", 2023-10-09T05:08:35.532 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_healthcheck_disabling_path": "", 2023-10-09T05:08:35.532 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_ignore_get_invalid_range": "false", 2023-10-09T05:08:35.532 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_init_timeout": "300", 2023-10-09T05:08:35.532 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2023-10-09T05:08:35.532 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2023-10-09T05:08:35.533 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_keystone_accepted_admin_roles": "", 2023-10-09T05:08:35.533 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2023-10-09T05:08:35.533 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_keystone_admin_domain": "", 2023-10-09T05:08:35.533 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_keystone_admin_password": "", 2023-10-09T05:08:35.533 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_keystone_admin_password_path": "", 2023-10-09T05:08:35.533 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_keystone_admin_project": "", 2023-10-09T05:08:35.533 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_keystone_admin_tenant": "", 2023-10-09T05:08:35.533 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_keystone_admin_token": "", 2023-10-09T05:08:35.533 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_keystone_admin_token_path": "", 2023-10-09T05:08:35.533 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_keystone_admin_user": "", 2023-10-09T05:08:35.533 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_keystone_api_version": "2", 2023-10-09T05:08:35.534 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_keystone_barbican_domain": "", 2023-10-09T05:08:35.534 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_keystone_barbican_password": "", 2023-10-09T05:08:35.534 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_keystone_barbican_project": "", 2023-10-09T05:08:35.534 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_keystone_barbican_tenant": "", 2023-10-09T05:08:35.534 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_keystone_barbican_user": "", 2023-10-09T05:08:35.534 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2023-10-09T05:08:35.534 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_keystone_implicit_tenants": "false", 2023-10-09T05:08:35.534 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2023-10-09T05:08:35.534 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_keystone_service_token_enabled": "false", 2023-10-09T05:08:35.534 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_keystone_token_cache_size": "10000", 2023-10-09T05:08:35.534 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_keystone_url": "", 2023-10-09T05:08:35.535 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_keystone_verify_ssl": "true", 2023-10-09T05:08:35.535 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_lc_debug_interval": "-1", 2023-10-09T05:08:35.535 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_lc_lock_max_time": "90", 2023-10-09T05:08:35.535 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_lc_max_objs": "32", 2023-10-09T05:08:35.535 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_lc_max_rules": "1000", 2023-10-09T05:08:35.535 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_lc_max_worker": "3", 2023-10-09T05:08:35.535 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_lc_max_wp_worker": "3", 2023-10-09T05:08:35.535 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_lc_thread_delay": "0", 2023-10-09T05:08:35.535 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2023-10-09T05:08:35.535 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_ldap_dnattr": "uid", 2023-10-09T05:08:35.536 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2023-10-09T05:08:35.536 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_ldap_searchfilter": "", 2023-10-09T05:08:35.536 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2023-10-09T05:08:35.536 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_ldap_uri": "ldaps://", 2023-10-09T05:08:35.536 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2023-10-09T05:08:35.536 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_list_bucket_min_readahead": "1000", 2023-10-09T05:08:35.536 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_list_buckets_max_chunk": "1000", 2023-10-09T05:08:35.536 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_log_http_headers": "", 2023-10-09T05:08:35.536 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_log_nonexistent_bucket": "false", 2023-10-09T05:08:35.536 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2023-10-09T05:08:35.536 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_log_object_name_utc": "false", 2023-10-09T05:08:35.537 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_luarocks_location": "/tmp/luarocks", 2023-10-09T05:08:35.537 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_max_attr_name_len": "0", 2023-10-09T05:08:35.537 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_max_attr_size": "0", 2023-10-09T05:08:35.537 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_max_attrs_num_in_req": "0", 2023-10-09T05:08:35.537 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_max_chunk_size": "4194304", 2023-10-09T05:08:35.537 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_max_concurrent_requests": "1024", 2023-10-09T05:08:35.537 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2023-10-09T05:08:35.537 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_max_dynamic_shards": "1999", 2023-10-09T05:08:35.537 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_max_listing_results": "1000", 2023-10-09T05:08:35.537 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_max_notify_retries": "10", 2023-10-09T05:08:35.538 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_max_objs_per_shard": "100000", 2023-10-09T05:08:35.538 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_max_put_param_size": "1048576", 2023-10-09T05:08:35.538 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_max_put_size": "5368709120", 2023-10-09T05:08:35.538 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_max_slo_entries": "1000", 2023-10-09T05:08:35.538 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_md_log_max_shards": "64", 2023-10-09T05:08:35.538 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_md_notify_interval_msec": "200", 2023-10-09T05:08:35.538 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_meta_sync_poll_interval": "20", 2023-10-09T05:08:35.538 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_meta_sync_spawn_window": "20", 2023-10-09T05:08:35.538 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_mime_types_file": "/etc/mime.types", 2023-10-09T05:08:35.538 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_mp_lock_max_time": "600", 2023-10-09T05:08:35.538 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_multi_obj_del_max_aio": "16", 2023-10-09T05:08:35.539 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_multipart_min_part_size": "5242880", 2023-10-09T05:08:35.539 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_multipart_part_upload_limit": "10000", 2023-10-09T05:08:35.539 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_nfs_fhcache_partitions": "3", 2023-10-09T05:08:35.539 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_nfs_fhcache_size": "2017", 2023-10-09T05:08:35.539 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_nfs_frontends": "rgw-nfs", 2023-10-09T05:08:35.539 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2023-10-09T05:08:35.539 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_nfs_lru_lanes": "5", 2023-10-09T05:08:35.539 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_nfs_max_gc": "300", 2023-10-09T05:08:35.539 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_nfs_namespace_expire_secs": "300", 2023-10-09T05:08:35.539 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_nfs_run_gc_threads": "false", 2023-10-09T05:08:35.539 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_nfs_run_lc_threads": "false", 2023-10-09T05:08:35.539 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_nfs_run_quota_threads": "false", 2023-10-09T05:08:35.540 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_nfs_run_sync_thread": "false", 2023-10-09T05:08:35.540 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_nfs_s3_fast_attrs": "false", 2023-10-09T05:08:35.540 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_nfs_write_completion_interval_s": "10", 2023-10-09T05:08:35.540 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_num_async_rados_threads": "32", 2023-10-09T05:08:35.540 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_num_control_oids": "8", 2023-10-09T05:08:35.540 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_numa_node": "-1", 2023-10-09T05:08:35.540 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_obj_stripe_size": "4194304", 2023-10-09T05:08:35.540 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_obj_tombstone_cache_size": "1000", 2023-10-09T05:08:35.540 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_objexp_chunk_size": "100", 2023-10-09T05:08:35.540 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_objexp_gc_interval": "600", 2023-10-09T05:08:35.540 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_objexp_hints_num_shards": "127", 2023-10-09T05:08:35.541 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_olh_pending_timeout_sec": "3600", 2023-10-09T05:08:35.541 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_op_thread_suicide_timeout": "0", 2023-10-09T05:08:35.541 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_op_thread_timeout": "600", 2023-10-09T05:08:35.541 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_op_tracing": "false", 2023-10-09T05:08:35.541 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_opa_token": "", 2023-10-09T05:08:35.541 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_opa_url": "", 2023-10-09T05:08:35.541 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_opa_verify_ssl": "true", 2023-10-09T05:08:35.541 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_ops_log_data_backlog": "5242880", 2023-10-09T05:08:35.541 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2023-10-09T05:08:35.541 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_ops_log_rados": "false", 2023-10-09T05:08:35.541 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_ops_log_socket_path": "", 2023-10-09T05:08:35.541 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_override_bucket_index_max_shards": "0", 2023-10-09T05:08:35.542 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2023-10-09T05:08:35.542 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2023-10-09T05:08:35.542 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_period_push_interval": "2.000000", 2023-10-09T05:08:35.542 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_period_push_interval_max": "30.000000", 2023-10-09T05:08:35.542 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_period_root_pool": ".rgw.root", 2023-10-09T05:08:35.542 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_policy_reject_invalid_principals": "true", 2023-10-09T05:08:35.542 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_print_continue": "true", 2023-10-09T05:08:35.542 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_print_prohibited_content_length": "false", 2023-10-09T05:08:35.542 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_put_obj_max_window_size": "67108864", 2023-10-09T05:08:35.542 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_put_obj_min_window_size": "16777216", 2023-10-09T05:08:35.542 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2023-10-09T05:08:35.543 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_rados_pool_recovery_priority": "5", 2023-10-09T05:08:35.543 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_rados_tracing": "false", 2023-10-09T05:08:35.543 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_realm": "", 2023-10-09T05:08:35.543 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_realm_id": "", 2023-10-09T05:08:35.543 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_realm_root_pool": ".rgw.root", 2023-10-09T05:08:35.543 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_region": "", 2023-10-09T05:08:35.543 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_region_root_pool": ".rgw.root", 2023-10-09T05:08:35.543 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_relaxed_region_enforcement": "false", 2023-10-09T05:08:35.543 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_relaxed_s3_bucket_names": "false", 2023-10-09T05:08:35.543 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2023-10-09T05:08:35.543 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_request_uri": "", 2023-10-09T05:08:35.543 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_reshard_batch_size": "64", 2023-10-09T05:08:35.544 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_reshard_bucket_lock_duration": "360", 2023-10-09T05:08:35.544 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_reshard_max_aio": "128", 2023-10-09T05:08:35.544 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_reshard_num_logs": "16", 2023-10-09T05:08:35.544 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_reshard_thread_interval": "600", 2023-10-09T05:08:35.544 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_resolve_cname": "false", 2023-10-09T05:08:35.544 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_rest_getusage_op_compat": "false", 2023-10-09T05:08:35.544 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_run_sync_thread": "true", 2023-10-09T05:08:35.544 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_s3_auth_order": "sts, external, local", 2023-10-09T05:08:35.544 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_s3_auth_use_keystone": "false", 2023-10-09T05:08:35.544 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_s3_auth_use_ldap": "false", 2023-10-09T05:08:35.544 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_s3_auth_use_rados": "true", 2023-10-09T05:08:35.544 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_s3_auth_use_sts": "false", 2023-10-09T05:08:35.545 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_s3_client_max_sig_ver": "-1", 2023-10-09T05:08:35.545 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_s3_success_create_obj_status": "0", 2023-10-09T05:08:35.545 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_safe_max_objects_per_shard": "102400", 2023-10-09T05:08:35.545 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_scheduler_type": "throttler", 2023-10-09T05:08:35.545 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_script_uri": "", 2023-10-09T05:08:35.545 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_service_provider_name": "", 2023-10-09T05:08:35.545 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_shard_warning_threshold": "90.000000", 2023-10-09T05:08:35.545 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_sts_client_id": "", 2023-10-09T05:08:35.545 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_sts_client_secret": "", 2023-10-09T05:08:35.545 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_sts_entry": "sts", 2023-10-09T05:08:35.545 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_sts_key": "sts", 2023-10-09T05:08:35.545 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_sts_max_session_duration": "43200", 2023-10-09T05:08:35.545 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_sts_min_session_duration": "900", 2023-10-09T05:08:35.546 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_sts_token_introspection_url": "", 2023-10-09T05:08:35.546 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_swift_account_in_url": "false", 2023-10-09T05:08:35.546 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_swift_auth_entry": "auth", 2023-10-09T05:08:35.546 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_swift_auth_url": "", 2023-10-09T05:08:35.546 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_swift_custom_header": "", 2023-10-09T05:08:35.546 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_swift_enforce_content_length": "false", 2023-10-09T05:08:35.546 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_swift_need_stats": "true", 2023-10-09T05:08:35.546 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_swift_tenant_name": "", 2023-10-09T05:08:35.546 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_swift_token_expiration": "86400", 2023-10-09T05:08:35.546 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_swift_url": "", 2023-10-09T05:08:35.546 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_swift_url_prefix": "swift", 2023-10-09T05:08:35.546 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_swift_versioning_enabled": "false", 2023-10-09T05:08:35.547 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2023-10-09T05:08:35.547 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2023-10-09T05:08:35.547 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_sync_lease_period": "120", 2023-10-09T05:08:35.547 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2023-10-09T05:08:35.547 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_sync_log_trim_interval": "1200", 2023-10-09T05:08:35.547 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_sync_log_trim_max_buckets": "16", 2023-10-09T05:08:35.547 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2023-10-09T05:08:35.547 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2023-10-09T05:08:35.547 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_sync_obj_etag_verify": "false", 2023-10-09T05:08:35.547 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_sync_trace_history_size": "4096", 2023-10-09T05:08:35.547 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_sync_trace_per_node_log_size": "32", 2023-10-09T05:08:35.548 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2023-10-09T05:08:35.548 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_thread_pool_size": "512", 2023-10-09T05:08:35.548 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_torrent_comment": "", 2023-10-09T05:08:35.548 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_torrent_createby": "", 2023-10-09T05:08:35.548 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_torrent_encoding": "", 2023-10-09T05:08:35.548 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_torrent_flag": "false", 2023-10-09T05:08:35.548 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_torrent_origin": "", 2023-10-09T05:08:35.548 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_torrent_sha_unit": "524288", 2023-10-09T05:08:35.548 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_torrent_tracker": "", 2023-10-09T05:08:35.548 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_trust_forwarded_https": "false", 2023-10-09T05:08:35.548 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_usage_log_flush_threshold": "1024", 2023-10-09T05:08:35.548 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_usage_log_tick_interval": "30", 2023-10-09T05:08:35.549 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_usage_max_shards": "32", 2023-10-09T05:08:35.549 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_usage_max_user_shards": "1", 2023-10-09T05:08:35.549 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_use_opa_authz": "false", 2023-10-09T05:08:35.549 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_user_default_quota_max_objects": "-1", 2023-10-09T05:08:35.549 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_user_default_quota_max_size": "-1", 2023-10-09T05:08:35.549 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_user_max_buckets": "1000", 2023-10-09T05:08:35.549 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_user_policies_max_num": "100", 2023-10-09T05:08:35.549 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2023-10-09T05:08:35.549 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_user_quota_sync_idle_users": "false", 2023-10-09T05:08:35.549 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_user_quota_sync_interval": "86400", 2023-10-09T05:08:35.549 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_user_quota_sync_wait_time": "86400", 2023-10-09T05:08:35.549 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_user_unique_email": "true", 2023-10-09T05:08:35.550 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_verify_ssl": "true", 2023-10-09T05:08:35.550 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_website_routing_rules_max_num": "50", 2023-10-09T05:08:35.550 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_zone": "", 2023-10-09T05:08:35.550 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_zone_id": "", 2023-10-09T05:08:35.550 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_zone_root_pool": ".rgw.root", 2023-10-09T05:08:35.550 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_zonegroup": "", 2023-10-09T05:08:35.550 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_zonegroup_id": "", 2023-10-09T05:08:35.550 INFO:teuthology.orchestra.run.smithi133.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2023-10-09T05:08:35.550 INFO:teuthology.orchestra.run.smithi133.stdout: "rgwlc_auto_session_clear": "true", 2023-10-09T05:08:35.550 INFO:teuthology.orchestra.run.smithi133.stdout: "rgwlc_skip_bucket_step": "false", 2023-10-09T05:08:35.551 INFO:teuthology.orchestra.run.smithi133.stdout: "rocksdb_block_size": "4096", 2023-10-09T05:08:35.551 INFO:teuthology.orchestra.run.smithi133.stdout: "rocksdb_bloom_bits_per_key": "20", 2023-10-09T05:08:35.551 INFO:teuthology.orchestra.run.smithi133.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2023-10-09T05:08:35.551 INFO:teuthology.orchestra.run.smithi133.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2023-10-09T05:08:35.551 INFO:teuthology.orchestra.run.smithi133.stdout: "rocksdb_cache_row_ratio": "0.000000", 2023-10-09T05:08:35.551 INFO:teuthology.orchestra.run.smithi133.stdout: "rocksdb_cache_shard_bits": "4", 2023-10-09T05:08:35.551 INFO:teuthology.orchestra.run.smithi133.stdout: "rocksdb_cache_size": "536870912", 2023-10-09T05:08:35.551 INFO:teuthology.orchestra.run.smithi133.stdout: "rocksdb_cache_type": "binned_lru", 2023-10-09T05:08:35.551 INFO:teuthology.orchestra.run.smithi133.stdout: "rocksdb_cf_compact_on_deletion": "true", 2023-10-09T05:08:35.551 INFO:teuthology.orchestra.run.smithi133.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2023-10-09T05:08:35.552 INFO:teuthology.orchestra.run.smithi133.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2023-10-09T05:08:35.552 INFO:teuthology.orchestra.run.smithi133.stdout: "rocksdb_collect_compaction_stats": "false", 2023-10-09T05:08:35.552 INFO:teuthology.orchestra.run.smithi133.stdout: "rocksdb_collect_extended_stats": "false", 2023-10-09T05:08:35.552 INFO:teuthology.orchestra.run.smithi133.stdout: "rocksdb_collect_memory_stats": "false", 2023-10-09T05:08:35.552 INFO:teuthology.orchestra.run.smithi133.stdout: "rocksdb_delete_range_threshold": "1048576", 2023-10-09T05:08:35.552 INFO:teuthology.orchestra.run.smithi133.stdout: "rocksdb_index_type": "binary_search", 2023-10-09T05:08:35.552 INFO:teuthology.orchestra.run.smithi133.stdout: "rocksdb_log_to_ceph_log": "true", 2023-10-09T05:08:35.552 INFO:teuthology.orchestra.run.smithi133.stdout: "rocksdb_metadata_block_size": "4096", 2023-10-09T05:08:35.552 INFO:teuthology.orchestra.run.smithi133.stdout: "rocksdb_partition_filters": "false", 2023-10-09T05:08:35.552 INFO:teuthology.orchestra.run.smithi133.stdout: "rocksdb_perf": "false", 2023-10-09T05:08:35.553 INFO:teuthology.orchestra.run.smithi133.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2023-10-09T05:08:35.553 INFO:teuthology.orchestra.run.smithi133.stdout: "rotating_keys_bootstrap_timeout": "30", 2023-10-09T05:08:35.553 INFO:teuthology.orchestra.run.smithi133.stdout: "rotating_keys_renewal_timeout": "10", 2023-10-09T05:08:35.553 INFO:teuthology.orchestra.run.smithi133.stdout: "run_dir": "/var/run/ceph", 2023-10-09T05:08:35.553 INFO:teuthology.orchestra.run.smithi133.stdout: "seastore_block_create": "true", 2023-10-09T05:08:35.553 INFO:teuthology.orchestra.run.smithi133.stdout: "seastore_cache_lru_size": "67108864", 2023-10-09T05:08:35.553 INFO:teuthology.orchestra.run.smithi133.stdout: "seastore_cbjournal_size": "5368709120", 2023-10-09T05:08:35.553 INFO:teuthology.orchestra.run.smithi133.stdout: "seastore_default_max_object_size": "16777216", 2023-10-09T05:08:35.553 INFO:teuthology.orchestra.run.smithi133.stdout: "seastore_default_object_metadata_reservation": "16777216", 2023-10-09T05:08:35.553 INFO:teuthology.orchestra.run.smithi133.stdout: "seastore_device_size": "53687091200", 2023-10-09T05:08:35.553 INFO:teuthology.orchestra.run.smithi133.stdout: "seastore_journal_batch_capacity": "16", 2023-10-09T05:08:35.554 INFO:teuthology.orchestra.run.smithi133.stdout: "seastore_journal_batch_flush_size": "16777216", 2023-10-09T05:08:35.554 INFO:teuthology.orchestra.run.smithi133.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2023-10-09T05:08:35.554 INFO:teuthology.orchestra.run.smithi133.stdout: "seastore_journal_iodepth_limit": "5", 2023-10-09T05:08:35.554 INFO:teuthology.orchestra.run.smithi133.stdout: "seastore_main_device_type": "SSD", 2023-10-09T05:08:35.554 INFO:teuthology.orchestra.run.smithi133.stdout: "seastore_max_concurrent_transactions": "8", 2023-10-09T05:08:35.554 INFO:teuthology.orchestra.run.smithi133.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2023-10-09T05:08:35.554 INFO:teuthology.orchestra.run.smithi133.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2023-10-09T05:08:35.554 INFO:teuthology.orchestra.run.smithi133.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2023-10-09T05:08:35.554 INFO:teuthology.orchestra.run.smithi133.stdout: "seastore_obj_data_write_amplification": "1.250000", 2023-10-09T05:08:35.554 INFO:teuthology.orchestra.run.smithi133.stdout: "seastore_segment_size": "67108864", 2023-10-09T05:08:35.555 INFO:teuthology.orchestra.run.smithi133.stdout: "set_keepcaps": "false", 2023-10-09T05:08:35.555 INFO:teuthology.orchestra.run.smithi133.stdout: "setgroup": "ceph", 2023-10-09T05:08:35.555 INFO:teuthology.orchestra.run.smithi133.stdout: "setuser": "ceph", 2023-10-09T05:08:35.555 INFO:teuthology.orchestra.run.smithi133.stdout: "setuser_match_path": "", 2023-10-09T05:08:35.555 INFO:teuthology.orchestra.run.smithi133.stdout: "target_max_misplaced_ratio": "0.050000", 2023-10-09T05:08:35.555 INFO:teuthology.orchestra.run.smithi133.stdout: "thp": "false", 2023-10-09T05:08:35.555 INFO:teuthology.orchestra.run.smithi133.stdout: "threadpool_default_timeout": "60", 2023-10-09T05:08:35.555 INFO:teuthology.orchestra.run.smithi133.stdout: "threadpool_empty_queue_max_wait": "2", 2023-10-09T05:08:35.555 INFO:teuthology.orchestra.run.smithi133.stdout: "throttler_perf_counter": "true" 2023-10-09T05:08:35.555 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:08:36.258 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2023-10-09T05:08:37.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:36 smithi133 bash[102858]: cluster 2023-10-09T05:08:35.721115+0000 mgr.y (mgr.25277) 599 : cluster 0 pgmap v550: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:37.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:36 smithi133 bash[105589]: cluster 2023-10-09T05:08:35.721115+0000 mgr.y (mgr.25277) 599 : cluster 0 pgmap v550: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:37.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:36 smithi194 bash[92666]: cluster 2023-10-09T05:08:35.721115+0000 mgr.y (mgr.25277) 599 : cluster 0 pgmap v550: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:39.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:38 smithi133 bash[102858]: cluster 2023-10-09T05:08:37.721842+0000 mgr.y (mgr.25277) 600 : cluster 0 pgmap v551: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:39.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:38 smithi133 bash[105589]: cluster 2023-10-09T05:08:37.721842+0000 mgr.y (mgr.25277) 600 : cluster 0 pgmap v551: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:39.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:38 smithi194 bash[92666]: cluster 2023-10-09T05:08:37.721842+0000 mgr.y (mgr.25277) 600 : cluster 0 pgmap v551: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:41.031 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:08:41.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:40 smithi133 bash[102858]: cluster 2023-10-09T05:08:39.722814+0000 mgr.y (mgr.25277) 601 : cluster 0 pgmap v552: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:41.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:40 smithi133 bash[105589]: cluster 2023-10-09T05:08:39.722814+0000 mgr.y (mgr.25277) 601 : cluster 0 pgmap v552: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:41.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:40 smithi194 bash[92666]: cluster 2023-10-09T05:08:39.722814+0000 mgr.y (mgr.25277) 601 : cluster 0 pgmap v552: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:43.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:42 smithi133 bash[102858]: cluster 2023-10-09T05:08:41.723842+0000 mgr.y (mgr.25277) 602 : cluster 0 pgmap v553: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:43.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:42 smithi133 bash[105589]: cluster 2023-10-09T05:08:41.723842+0000 mgr.y (mgr.25277) 602 : cluster 0 pgmap v553: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:43.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:42 smithi194 bash[92666]: cluster 2023-10-09T05:08:41.723842+0000 mgr.y (mgr.25277) 602 : cluster 0 pgmap v553: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:44.401 INFO:teuthology.orchestra.run.smithi133.stdout:mon.a: {} 2023-10-09T05:08:44.422 INFO:teuthology.orchestra.run.smithi133.stdout:mon.b: {} 2023-10-09T05:08:44.440 INFO:teuthology.orchestra.run.smithi133.stdout:mon.c: {} 2023-10-09T05:08:44.442 INFO:teuthology.orchestra.run.smithi133.stderr:mon.a: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T05:08:44.442 INFO:teuthology.orchestra.run.smithi133.stderr:mon.b: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T05:08:44.442 INFO:teuthology.orchestra.run.smithi133.stderr:mon.c: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T05:08:44.494 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:08:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:08:44] "GET /metrics HTTP/1.1" 200 33986 "" "Prometheus/2.43.0" 2023-10-09T05:08:44.495 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:44 smithi133 bash[105589]: audit 2023-10-09T05:08:43.439440+0000 mon.a (mon.0) 742 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:08:44.495 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:44 smithi133 bash[105589]: cluster 2023-10-09T05:08:43.724629+0000 mgr.y (mgr.25277) 603 : cluster 0 pgmap v554: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:44.495 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:44 smithi133 bash[102858]: audit 2023-10-09T05:08:43.439440+0000 mon.a (mon.0) 742 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:08:44.495 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:44 smithi133 bash[102858]: cluster 2023-10-09T05:08:43.724629+0000 mgr.y (mgr.25277) 603 : cluster 0 pgmap v554: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:44.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:44 smithi194 bash[92666]: audit 2023-10-09T05:08:43.439440+0000 mon.a (mon.0) 742 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:08:44.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:44 smithi194 bash[92666]: cluster 2023-10-09T05:08:43.724629+0000 mgr.y (mgr.25277) 603 : cluster 0 pgmap v554: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:45.319 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2023-10-09T05:08:45.319 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-10-09T05:08:45.319 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T05:08:45.319 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T05:08:45.320 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2023-10-09T05:08:45.320 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_4 16 2023-10-09T05:08:45.320 INFO:tasks.thrashosds.thrasher:starting do_thrash 2023-10-09T05:08:45.321 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:08:45.321 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:08:45.321 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2023-10-09T05:08:45.321 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T05:08:45.321 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T05:08:45.321 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 config set filestore_inject_stall 3 2023-10-09T05:08:45.322 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2023-10-09T05:08:45.322 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:45.322 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2023-10-09T05:08:45.322 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:08:45.323 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2023-10-09T05:08:45.323 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T05:08:45.323 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2023-10-09T05:08:45.323 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:08:45.358 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:45.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:45.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:08:45.461 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:45.495 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:45.497 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:45 smithi194 bash[92666]: audit 2023-10-09T05:08:44.383308+0000 mon.c (mon.2) 58 : audit 0 from='client.? 172.21.15.133:0/3242730128' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:08:45.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:45.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:08:45.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:45 smithi133 bash[102858]: audit 2023-10-09T05:08:44.383308+0000 mon.c (mon.2) 58 : audit 0 from='client.? 172.21.15.133:0/3242730128' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:08:45.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:45 smithi133 bash[105589]: audit 2023-10-09T05:08:44.383308+0000 mon.c (mon.2) 58 : audit 0 from='client.? 172.21.15.133:0/3242730128' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:08:45.599 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:45.631 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:45.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:45.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:08:45.734 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:45.767 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:45.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:45.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:08:45.870 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:45.905 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:45.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:45.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:08:46.008 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:08:46.038 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:08:46.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:46.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:08:46.142 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:46.177 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:46.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:46.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:08:46.281 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:46.292 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:46 smithi133 bash[105589]: cluster 2023-10-09T05:08:45.725736+0000 mgr.y (mgr.25277) 604 : cluster 0 pgmap v555: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:46.293 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:46 smithi133 bash[102858]: cluster 2023-10-09T05:08:45.725736+0000 mgr.y (mgr.25277) 604 : cluster 0 pgmap v555: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:46.310 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:46.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:46.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:08:46.414 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:46.445 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:46.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:46.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:08:46.549 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:46.563 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:46 smithi194 bash[92666]: cluster 2023-10-09T05:08:45.725736+0000 mgr.y (mgr.25277) 604 : cluster 0 pgmap v555: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:46.583 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:46.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:46.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:08:46.687 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:46.716 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:46.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:46.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:08:46.820 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:46.852 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:46.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:46.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:08:46.956 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:46.988 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:46.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:46.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:08:47.091 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:47.126 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:47.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:47.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:08:47.230 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:47.262 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:47.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:47.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:08:47.366 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:47.399 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:47.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:47.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:08:47.503 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:47.537 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:47.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:47.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:08:47.641 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:47.673 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:47.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:47.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:08:47.777 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:08:47.806 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:08:47.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:47.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:08:47.909 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:47.940 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:47.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:47.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:08:48.044 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:48.077 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:48.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:48.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:08:48.180 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:48.213 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:48.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:48.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:08:48.317 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:48.351 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:48.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:48.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:08:48.454 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:48.490 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:48.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:48.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:08:48.593 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:48.624 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:48.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:48.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:08:48.728 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:48.758 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:48.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:48.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:08:48.862 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:48.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:48 smithi194 bash[92666]: cluster 2023-10-09T05:08:47.726566+0000 mgr.y (mgr.25277) 605 : cluster 0 pgmap v556: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:48.896 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:48.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:48.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:08:49.000 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:49.035 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:49.036 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:48 smithi133 bash[105589]: cluster 2023-10-09T05:08:47.726566+0000 mgr.y (mgr.25277) 605 : cluster 0 pgmap v556: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:49.036 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:48 smithi133 bash[102858]: cluster 2023-10-09T05:08:47.726566+0000 mgr.y (mgr.25277) 605 : cluster 0 pgmap v556: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:49.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:49.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:08:49.138 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:49.170 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:49.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:49.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:08:49.274 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:49.496 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:49.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:49.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:08:49.876 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:49.908 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:49.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:49.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:08:50.028 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:50.059 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:50.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:50.136 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:08:50.137 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:08:50.137 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:08:50.137 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:08:50.138 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:08:50.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:08:50.239 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:08:50.339 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:08:50.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:50.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:08:50.443 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:50.478 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:50.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:50.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:08:50.582 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:50.609 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:50.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:50.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:08:50.713 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:50.745 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:50.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:50.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:08:50.849 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:08:50.862 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:50 smithi133 bash[102858]: cluster 2023-10-09T05:08:49.727461+0000 mgr.y (mgr.25277) 606 : cluster 0 pgmap v557: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:50.862 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:50 smithi133 bash[105589]: cluster 2023-10-09T05:08:49.727461+0000 mgr.y (mgr.25277) 606 : cluster 0 pgmap v557: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:50.878 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:08:50.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:50.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:08:50.981 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:51.013 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:51.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:51.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:08:51.116 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:51.127 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:50 smithi194 bash[92666]: cluster 2023-10-09T05:08:49.727461+0000 mgr.y (mgr.25277) 606 : cluster 0 pgmap v557: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:51.146 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:51.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:51.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:08:51.249 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:51.276 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:51.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:51.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:08:51.380 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:51.416 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:51.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:51.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:08:51.521 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:51.552 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:51.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:51.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:08:51.655 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:51.685 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:51.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:51.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:08:51.789 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:51.820 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:51.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:51.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:08:51.924 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:51.960 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:51.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:51.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:08:52.063 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:52.090 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:52.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:52.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:08:52.194 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:52.227 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:52.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:52.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:08:52.330 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:52.357 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:52.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:52.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:08:52.460 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:52.491 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:52.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:52.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:08:52.595 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:08:52.622 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:08:52.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:52.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:08:52.726 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:52.759 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:52.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:52.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:08:52.864 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:52.897 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:52.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:52.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:08:53.001 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:53.009 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:52 smithi133 bash[105589]: cluster 2023-10-09T05:08:51.728078+0000 mgr.y (mgr.25277) 607 : cluster 0 pgmap v558: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:53.009 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:52 smithi133 bash[102858]: cluster 2023-10-09T05:08:51.728078+0000 mgr.y (mgr.25277) 607 : cluster 0 pgmap v558: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:53.023 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:53.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:53.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:08:53.126 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:53.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:52 smithi194 bash[92666]: cluster 2023-10-09T05:08:51.728078+0000 mgr.y (mgr.25277) 607 : cluster 0 pgmap v558: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:53.158 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:53.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:53.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:08:53.262 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:53.287 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:53.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:53.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:08:53.391 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:53.423 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:53.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:53.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:08:53.527 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:53.555 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:53.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:53.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:08:53.660 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:53.688 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:53.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:53.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:08:53.792 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:53.821 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:53.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:53.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:08:53.925 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:53.960 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:53.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:53.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:08:54.064 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:54.094 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:54.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:54.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:08:54.198 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:54.223 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:54.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:54.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:08:54.326 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:54.354 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:54.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:54.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:08:54.458 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:54.489 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:54.492 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:54 smithi194 bash[92666]: cluster 2023-10-09T05:08:53.728774+0000 mgr.y (mgr.25277) 608 : cluster 0 pgmap v559: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:54.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:54.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:08:54.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:54 smithi133 bash[102858]: cluster 2023-10-09T05:08:53.728774+0000 mgr.y (mgr.25277) 608 : cluster 0 pgmap v559: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:54.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:54 smithi133 bash[105589]: cluster 2023-10-09T05:08:53.728774+0000 mgr.y (mgr.25277) 608 : cluster 0 pgmap v559: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:54.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:08:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:08:54] "GET /metrics HTTP/1.1" 200 33986 "" "Prometheus/2.43.0" 2023-10-09T05:08:54.594 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:54.626 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:54.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:54.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:08:54.730 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:08:54.758 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:08:54.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:54.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:08:54.861 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:54.893 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:54.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:54.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:08:54.997 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:55.020 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:55.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:55.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:08:55.124 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:55.151 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:55.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:55.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:08:55.254 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:55.288 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:55.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:55.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:08:55.392 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:08:55.419 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:08:55.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:55.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:08:55.522 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:08:55.546 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:08:55.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:55.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:08:55.649 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:55.674 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:55.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:55.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:08:55.777 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:55.814 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:55.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:55.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:08:55.918 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:55.943 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:55.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:55.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:08:56.045 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:56.079 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:56.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:56.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:08:56.183 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:56.205 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:56.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:56.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:08:56.308 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:56.340 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:56.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:56.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:08:56.444 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:56.478 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:56.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:56.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:08:56.582 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:56.609 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:56.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:56.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:08:56.712 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:56.742 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:56.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:56.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:08:56.846 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:56.877 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:56.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:56.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:08:56.981 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:57.009 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:57.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:57.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:08:57.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:56 smithi133 bash[102858]: cluster 2023-10-09T05:08:55.729704+0000 mgr.y (mgr.25277) 609 : cluster 0 pgmap v560: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:57.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:56 smithi133 bash[105589]: cluster 2023-10-09T05:08:55.729704+0000 mgr.y (mgr.25277) 609 : cluster 0 pgmap v560: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:57.112 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:57.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:56 smithi194 bash[92666]: cluster 2023-10-09T05:08:55.729704+0000 mgr.y (mgr.25277) 609 : cluster 0 pgmap v560: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:57.139 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:57.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:57.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:08:57.243 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:08:57.271 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:08:57.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:57.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:08:57.375 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:57.404 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:57.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:57.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:08:57.508 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:57.536 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:57.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:57.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:08:57.640 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:57.669 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:57.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:57.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:08:57.772 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:57.801 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:57.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:57.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:08:57.905 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:08:57.929 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:08:57.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:57.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:08:58.032 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:58.058 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:58.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:58.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:08:58.162 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:58.193 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:58.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:58.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:08:58.296 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:58.326 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:58.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:58.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:08:58.430 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:58.465 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:58.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:58.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:08:58.569 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:58.604 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:58.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:58.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:08:58.708 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:58.740 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:58.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:58.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:08:58.844 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:58.869 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:58.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:58.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:08:58.972 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:58.984 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:58 smithi194 bash[92666]: cluster 2023-10-09T05:08:57.730398+0000 mgr.y (mgr.25277) 610 : cluster 0 pgmap v561: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:58.985 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:08:58 smithi194 bash[92666]: audit 2023-10-09T05:08:58.439179+0000 mon.a (mon.0) 743 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:08:59.005 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:59.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:59.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:08:59.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:58 smithi133 bash[105589]: cluster 2023-10-09T05:08:57.730398+0000 mgr.y (mgr.25277) 610 : cluster 0 pgmap v561: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:59.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:08:58 smithi133 bash[105589]: audit 2023-10-09T05:08:58.439179+0000 mon.a (mon.0) 743 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:08:59.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:58 smithi133 bash[102858]: cluster 2023-10-09T05:08:57.730398+0000 mgr.y (mgr.25277) 610 : cluster 0 pgmap v561: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:08:59.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:08:58 smithi133 bash[102858]: audit 2023-10-09T05:08:58.439179+0000 mon.a (mon.0) 743 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:08:59.108 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:59.134 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:59.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:59.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:08:59.281 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:59.313 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:59.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:59.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:08:59.417 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:59.453 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:59.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:59.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:08:59.557 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:59.588 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:59.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:59.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:08:59.692 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:59.721 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:59.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:59.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:08:59.824 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:59.852 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:59.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:59.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:08:59.956 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:59.991 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:59.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:59.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:00.095 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:00.125 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:00.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:00.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:00.229 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:00.264 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:00.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:00.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:00.367 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:00.399 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:00.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:00.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:00.503 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:00.534 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:00.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:00.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:00.639 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:00.672 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:00.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:00.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:00.775 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:00.809 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:00.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:00.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:00.914 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:00.926 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:00 smithi133 bash[102858]: cluster 2023-10-09T05:08:59.731095+0000 mgr.y (mgr.25277) 611 : cluster 0 pgmap v562: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:00.927 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:00 smithi133 bash[105589]: cluster 2023-10-09T05:08:59.731095+0000 mgr.y (mgr.25277) 611 : cluster 0 pgmap v562: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:00.947 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:00.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:00.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:01.050 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:01.076 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:01.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:01.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:01.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:00 smithi194 bash[92666]: cluster 2023-10-09T05:08:59.731095+0000 mgr.y (mgr.25277) 611 : cluster 0 pgmap v562: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:01.180 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:01.211 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:01.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:01.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:01.314 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:01.337 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:01.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:01.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:01.440 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:01.471 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:01.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:01.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:01.575 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:01.601 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:01.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:01.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:01.704 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:01.721 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:09:01.721 INFO:teuthology.orchestra.run.smithi133.stdout: "ops": [], 2023-10-09T05:09:01.721 INFO:teuthology.orchestra.run.smithi133.stdout: "num_ops": 0 2023-10-09T05:09:01.722 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:09:01.739 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:01.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:01.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:01.772 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:09:01.797 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:09:01.818 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:09:01.842 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:09:01.843 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:01.868 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:09:01.876 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:01.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:01.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:01.894 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:09:01.925 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:09:01.952 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:09:01.954 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:09:01.954 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:09:01.954 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:09:01.954 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:09:01.954 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:09:01.954 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:09:01.954 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:09:01.954 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:09:01.980 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:01.992 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:01 smithi194 bash[92666]: audit 2023-10-09T05:09:01.746378+0000 mon.a (mon.0) 744 : audit 0 from='client.? 172.21.15.133:0/3920346693' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:09:02.007 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:01 smithi133 bash[105589]: audit 2023-10-09T05:09:01.746378+0000 mon.a (mon.0) 744 : audit 0 from='client.? 172.21.15.133:0/3920346693' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:09:02.008 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:01 smithi133 bash[102858]: audit 2023-10-09T05:09:01.746378+0000 mon.a (mon.0) 744 : audit 0 from='client.? 172.21.15.133:0/3920346693' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:09:02.012 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:02.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:02.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:02.116 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:02.142 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:02.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:02.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:02.175 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:09:02.176 INFO:teuthology.orchestra.run.smithi133.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T05:09:02.176 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:09:02.246 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:02.281 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:02.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:02.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:02.385 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:02.407 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:02.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:02.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:02.510 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:02.533 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:02.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:02.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:02.636 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:02.670 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:02.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:02.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:02.774 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:02.808 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:02.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:02.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:02.819 INFO:teuthology.orchestra.run.smithi133.stderr:pool 'unique_pool_4' created 2023-10-09T05:09:02.826 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is set 2023-10-09T05:09:02.912 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:02.923 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:02 smithi194 bash[92666]: cluster 2023-10-09T05:09:01.731601+0000 mgr.y (mgr.25277) 612 : cluster 0 pgmap v563: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:02.923 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:02 smithi194 bash[92666]: audit 2023-10-09T05:09:02.009361+0000 mon.a (mon.0) 745 : audit 1 from='client.? 172.21.15.133:0/3303484405' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T05:09:02.923 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:02 smithi194 bash[92666]: audit 2023-10-09T05:09:02.317347+0000 mon.c (mon.2) 59 : audit 1 from='client.? 172.21.15.133:0/407499562' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:09:02.923 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:02 smithi194 bash[92666]: audit 2023-10-09T05:09:02.458837+0000 mon.a (mon.0) 746 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:09:02.946 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:02.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:02.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:03.050 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:03.081 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:03.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:03.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:03.088 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:02 smithi133 bash[102858]: cluster 2023-10-09T05:09:01.731601+0000 mgr.y (mgr.25277) 612 : cluster 0 pgmap v563: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:03.088 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:02 smithi133 bash[102858]: audit 2023-10-09T05:09:02.009361+0000 mon.a (mon.0) 745 : audit 1 from='client.? 172.21.15.133:0/3303484405' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T05:09:03.088 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:02 smithi133 bash[102858]: audit 2023-10-09T05:09:02.317347+0000 mon.c (mon.2) 59 : audit 1 from='client.? 172.21.15.133:0/407499562' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:09:03.088 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:02 smithi133 bash[102858]: audit 2023-10-09T05:09:02.458837+0000 mon.a (mon.0) 746 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:09:03.088 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:02 smithi133 bash[105589]: cluster 2023-10-09T05:09:01.731601+0000 mgr.y (mgr.25277) 612 : cluster 0 pgmap v563: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:03.088 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:02 smithi133 bash[105589]: audit 2023-10-09T05:09:02.009361+0000 mon.a (mon.0) 745 : audit 1 from='client.? 172.21.15.133:0/3303484405' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T05:09:03.088 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:02 smithi133 bash[105589]: audit 2023-10-09T05:09:02.317347+0000 mon.c (mon.2) 59 : audit 1 from='client.? 172.21.15.133:0/407499562' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:09:03.088 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:02 smithi133 bash[105589]: audit 2023-10-09T05:09:02.458837+0000 mon.a (mon.0) 746 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:09:03.185 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:03.218 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:03.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:03.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:03.322 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:03.356 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:03.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:03.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:03.459 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:03.488 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:03.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:03.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:03.592 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:03.623 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:03.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:03.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:03.726 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:03.760 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:03.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:03.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:03.863 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:03.893 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:03.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:03.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:03.996 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:04.020 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:04.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:04.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:04.123 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:04.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:03 smithi194 bash[92666]: cluster 2023-10-09T05:09:02.801865+0000 mon.a (mon.0) 747 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:09:04.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:03 smithi194 bash[92666]: audit 2023-10-09T05:09:02.812310+0000 mon.a (mon.0) 748 : audit 1 from='client.? 172.21.15.133:0/3303484405' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T05:09:04.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:03 smithi194 bash[92666]: audit 2023-10-09T05:09:02.812369+0000 mon.a (mon.0) 749 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:09:04.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:03 smithi194 bash[92666]: cluster 2023-10-09T05:09:02.812461+0000 mon.a (mon.0) 750 : cluster 0 osdmap e139: 8 total, 8 up, 8 in 2023-10-09T05:09:04.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:03 smithi194 bash[92666]: cluster 2023-10-09T05:09:02.813139+0000 osd.5 (osd.5) 92 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T05:09:04.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:03 smithi194 bash[92666]: cluster 2023-10-09T05:09:02.817305+0000 osd.6 (osd.6) 60 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T05:09:04.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:03 smithi194 bash[92666]: cluster 2023-10-09T05:09:02.817659+0000 osd.4 (osd.4) 70 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T05:09:04.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:03 smithi194 bash[92666]: cluster 2023-10-09T05:09:02.818995+0000 osd.7 (osd.7) 50 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T05:09:04.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:03 smithi194 bash[92666]: cluster 2023-10-09T05:09:02.820926+0000 osd.4 (osd.4) 71 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T05:09:04.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:03 smithi194 bash[92666]: cluster 2023-10-09T05:09:02.821932+0000 osd.6 (osd.6) 61 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T05:09:04.153 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:04.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:04.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:04.256 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:04.281 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:03 smithi133 bash[105589]: cluster 2023-10-09T05:09:02.801865+0000 mon.a (mon.0) 747 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:09:04.282 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:03 smithi133 bash[105589]: audit 2023-10-09T05:09:02.812310+0000 mon.a (mon.0) 748 : audit 1 from='client.? 172.21.15.133:0/3303484405' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T05:09:04.282 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:03 smithi133 bash[105589]: audit 2023-10-09T05:09:02.812369+0000 mon.a (mon.0) 749 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:09:04.282 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:03 smithi133 bash[105589]: cluster 2023-10-09T05:09:02.812461+0000 mon.a (mon.0) 750 : cluster 0 osdmap e139: 8 total, 8 up, 8 in 2023-10-09T05:09:04.282 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:03 smithi133 bash[105589]: cluster 2023-10-09T05:09:02.813139+0000 osd.5 (osd.5) 92 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T05:09:04.282 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:03 smithi133 bash[105589]: cluster 2023-10-09T05:09:02.817305+0000 osd.6 (osd.6) 60 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T05:09:04.282 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:03 smithi133 bash[105589]: cluster 2023-10-09T05:09:02.817659+0000 osd.4 (osd.4) 70 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T05:09:04.282 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:03 smithi133 bash[105589]: cluster 2023-10-09T05:09:02.818995+0000 osd.7 (osd.7) 50 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T05:09:04.282 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:03 smithi133 bash[105589]: cluster 2023-10-09T05:09:02.820926+0000 osd.4 (osd.4) 71 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T05:09:04.282 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:03 smithi133 bash[105589]: cluster 2023-10-09T05:09:02.821932+0000 osd.6 (osd.6) 61 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T05:09:04.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:03 smithi133 bash[102858]: cluster 2023-10-09T05:09:02.801865+0000 mon.a (mon.0) 747 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:09:04.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:03 smithi133 bash[102858]: audit 2023-10-09T05:09:02.812310+0000 mon.a (mon.0) 748 : audit 1 from='client.? 172.21.15.133:0/3303484405' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T05:09:04.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:03 smithi133 bash[102858]: audit 2023-10-09T05:09:02.812369+0000 mon.a (mon.0) 749 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:09:04.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:03 smithi133 bash[102858]: cluster 2023-10-09T05:09:02.812461+0000 mon.a (mon.0) 750 : cluster 0 osdmap e139: 8 total, 8 up, 8 in 2023-10-09T05:09:04.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:03 smithi133 bash[102858]: cluster 2023-10-09T05:09:02.813139+0000 osd.5 (osd.5) 92 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T05:09:04.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:03 smithi133 bash[102858]: cluster 2023-10-09T05:09:02.817305+0000 osd.6 (osd.6) 60 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T05:09:04.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:03 smithi133 bash[102858]: cluster 2023-10-09T05:09:02.817659+0000 osd.4 (osd.4) 70 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T05:09:04.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:03 smithi133 bash[102858]: cluster 2023-10-09T05:09:02.818995+0000 osd.7 (osd.7) 50 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T05:09:04.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:03 smithi133 bash[102858]: cluster 2023-10-09T05:09:02.820926+0000 osd.4 (osd.4) 71 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T05:09:04.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:03 smithi133 bash[102858]: cluster 2023-10-09T05:09:02.821932+0000 osd.6 (osd.6) 61 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T05:09:04.289 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:04.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:04.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:04.394 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:04.427 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:04.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:04.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:04.531 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:04.544 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:09:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:09:04] "GET /metrics HTTP/1.1" 200 33988 "" "Prometheus/2.43.0" 2023-10-09T05:09:04.561 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:04.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:04.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:04.664 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:04.697 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:04.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:04.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:04.801 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:04.831 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:04.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:04.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:04.935 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:04.960 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T05:09:04.968 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:04.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:04.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:05.071 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:05.072 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:04 smithi133 bash[102858]: cluster 2023-10-09T05:09:03.732142+0000 mgr.y (mgr.25277) 613 : cluster 0 pgmap v565: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:05.073 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:04 smithi133 bash[102858]: cluster 2023-10-09T05:09:03.814452+0000 osd.6 (osd.6) 62 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T05:09:05.073 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:04 smithi133 bash[102858]: cluster 2023-10-09T05:09:03.817636+0000 mon.a (mon.0) 751 : cluster 0 osdmap e140: 8 total, 8 up, 8 in 2023-10-09T05:09:05.073 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:04 smithi133 bash[102858]: cluster 2023-10-09T05:09:03.818179+0000 osd.7 (osd.7) 51 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T05:09:05.073 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:04 smithi133 bash[102858]: cluster 2023-10-09T05:09:03.818262+0000 osd.5 (osd.5) 93 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T05:09:05.073 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:04 smithi133 bash[105589]: cluster 2023-10-09T05:09:03.732142+0000 mgr.y (mgr.25277) 613 : cluster 0 pgmap v565: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:05.073 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:04 smithi133 bash[105589]: cluster 2023-10-09T05:09:03.814452+0000 osd.6 (osd.6) 62 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T05:09:05.073 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:04 smithi133 bash[105589]: cluster 2023-10-09T05:09:03.817636+0000 mon.a (mon.0) 751 : cluster 0 osdmap e140: 8 total, 8 up, 8 in 2023-10-09T05:09:05.073 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:04 smithi133 bash[105589]: cluster 2023-10-09T05:09:03.818179+0000 osd.7 (osd.7) 51 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T05:09:05.074 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:04 smithi133 bash[105589]: cluster 2023-10-09T05:09:03.818262+0000 osd.5 (osd.5) 93 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T05:09:05.087 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:04 smithi194 bash[92666]: cluster 2023-10-09T05:09:03.732142+0000 mgr.y (mgr.25277) 613 : cluster 0 pgmap v565: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:05.087 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:04 smithi194 bash[92666]: cluster 2023-10-09T05:09:03.814452+0000 osd.6 (osd.6) 62 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T05:09:05.087 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:04 smithi194 bash[92666]: cluster 2023-10-09T05:09:03.817636+0000 mon.a (mon.0) 751 : cluster 0 osdmap e140: 8 total, 8 up, 8 in 2023-10-09T05:09:05.087 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:04 smithi194 bash[92666]: cluster 2023-10-09T05:09:03.818179+0000 osd.7 (osd.7) 51 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T05:09:05.087 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:04 smithi194 bash[92666]: cluster 2023-10-09T05:09:03.818262+0000 osd.5 (osd.5) 93 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T05:09:05.108 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:05.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:05.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:05.212 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:05.244 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:05.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:05.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:05.348 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:05.377 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:05.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:05.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:05.481 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:05.513 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:05.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:05.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:05.617 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:05.645 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:05.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:05.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:05.748 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:05.783 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:05.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:05.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:05.834 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2023-10-09T05:09:05.887 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:05.912 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:05 smithi133 bash[102858]: cluster 2023-10-09T05:09:04.046338+0000 osd.4 (osd.4) 72 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T05:09:05.913 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:05 smithi133 bash[102858]: cluster 2023-10-09T05:09:04.828854+0000 mon.a (mon.0) 752 : cluster 0 osdmap e141: 8 total, 8 up, 8 in 2023-10-09T05:09:05.913 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:05 smithi133 bash[105589]: cluster 2023-10-09T05:09:04.046338+0000 osd.4 (osd.4) 72 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T05:09:05.913 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:05 smithi133 bash[105589]: cluster 2023-10-09T05:09:04.828854+0000 mon.a (mon.0) 752 : cluster 0 osdmap e141: 8 total, 8 up, 8 in 2023-10-09T05:09:05.926 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:05.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:05.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:06.030 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:06.057 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:06.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:06.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:06.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:05 smithi194 bash[92666]: cluster 2023-10-09T05:09:04.046338+0000 osd.4 (osd.4) 72 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T05:09:06.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:05 smithi194 bash[92666]: cluster 2023-10-09T05:09:04.828854+0000 mon.a (mon.0) 752 : cluster 0 osdmap e141: 8 total, 8 up, 8 in 2023-10-09T05:09:06.162 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:06.192 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:06.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:06.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:06.295 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:06.322 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:06.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:06.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:06.425 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:06.453 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:06.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:06.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:06.556 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:06.588 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:06.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:06.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:06.692 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:06.729 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:06.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:06.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:06.833 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:06.864 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:06.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:06.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:06.881 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:09:06.968 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:06.999 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:07.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:07.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:07.103 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:07.114 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:06 smithi194 bash[92666]: cluster 2023-10-09T05:09:05.732863+0000 mgr.y (mgr.25277) 614 : cluster 0 pgmap v568: 89 pgs: 4 creating+peering, 2 unknown, 83 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:07.114 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:06 smithi194 bash[92666]: cluster 2023-10-09T05:09:05.890159+0000 osd.7 (osd.7) 52 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T05:09:07.115 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:06 smithi194 bash[92666]: cluster 2023-10-09T05:09:06.367641+0000 osd.6 (osd.6) 63 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T05:09:07.134 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:07.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:07.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:07.238 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:07.272 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:07.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:07.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:07.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:06 smithi133 bash[105589]: cluster 2023-10-09T05:09:05.732863+0000 mgr.y (mgr.25277) 614 : cluster 0 pgmap v568: 89 pgs: 4 creating+peering, 2 unknown, 83 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:07.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:06 smithi133 bash[105589]: cluster 2023-10-09T05:09:05.890159+0000 osd.7 (osd.7) 52 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T05:09:07.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:06 smithi133 bash[105589]: cluster 2023-10-09T05:09:06.367641+0000 osd.6 (osd.6) 63 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T05:09:07.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:06 smithi133 bash[102858]: cluster 2023-10-09T05:09:05.732863+0000 mgr.y (mgr.25277) 614 : cluster 0 pgmap v568: 89 pgs: 4 creating+peering, 2 unknown, 83 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:07.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:06 smithi133 bash[102858]: cluster 2023-10-09T05:09:05.890159+0000 osd.7 (osd.7) 52 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T05:09:07.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:06 smithi133 bash[102858]: cluster 2023-10-09T05:09:06.367641+0000 osd.6 (osd.6) 63 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T05:09:07.376 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:07.413 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:07.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:07.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:07.517 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:07.552 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:07.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:07.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:07.656 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:07.690 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:07.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:07.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:07.793 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:07.826 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:07.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:07.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:07.855 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:09:07.931 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:07.963 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:07.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:07.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:08.067 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:08.074 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:07 smithi133 bash[105589]: cluster 2023-10-09T05:09:06.106710+0000 osd.5 (osd.5) 94 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T05:09:08.075 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:07 smithi133 bash[105589]: cluster 2023-10-09T05:09:06.109585+0000 osd.5 (osd.5) 95 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T05:09:08.075 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:07 smithi133 bash[105589]: cluster 2023-10-09T05:09:06.111874+0000 osd.5 (osd.5) 96 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T05:09:08.075 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:07 smithi133 bash[105589]: cluster 2023-10-09T05:09:06.113425+0000 osd.5 (osd.5) 97 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T05:09:08.075 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:07 smithi133 bash[105589]: cluster 2023-10-09T05:09:06.323482+0000 osd.4 (osd.4) 73 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T05:09:08.075 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:07 smithi133 bash[102858]: cluster 2023-10-09T05:09:06.106710+0000 osd.5 (osd.5) 94 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T05:09:08.075 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:07 smithi133 bash[102858]: cluster 2023-10-09T05:09:06.109585+0000 osd.5 (osd.5) 95 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T05:09:08.075 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:07 smithi133 bash[102858]: cluster 2023-10-09T05:09:06.111874+0000 osd.5 (osd.5) 96 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T05:09:08.076 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:07 smithi133 bash[102858]: cluster 2023-10-09T05:09:06.113425+0000 osd.5 (osd.5) 97 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T05:09:08.076 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:07 smithi133 bash[102858]: cluster 2023-10-09T05:09:06.323482+0000 osd.4 (osd.4) 73 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T05:09:08.090 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:08.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:08.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:08.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:07 smithi194 bash[92666]: cluster 2023-10-09T05:09:06.106710+0000 osd.5 (osd.5) 94 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T05:09:08.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:07 smithi194 bash[92666]: cluster 2023-10-09T05:09:06.109585+0000 osd.5 (osd.5) 95 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T05:09:08.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:07 smithi194 bash[92666]: cluster 2023-10-09T05:09:06.111874+0000 osd.5 (osd.5) 96 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T05:09:08.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:07 smithi194 bash[92666]: cluster 2023-10-09T05:09:06.113425+0000 osd.5 (osd.5) 97 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T05:09:08.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:07 smithi194 bash[92666]: cluster 2023-10-09T05:09:06.323482+0000 osd.4 (osd.4) 73 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T05:09:08.193 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:08.227 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:08.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:08.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:08.330 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:08.362 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:08.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:08.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:08.466 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:08.498 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:08.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:08.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:08.603 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:08.639 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:08.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:08.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:08.742 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:08.777 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:08.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:08.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:08.882 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:08.911 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:08.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:08.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:09.014 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:09.027 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:08 smithi194 bash[92666]: cluster 2023-10-09T05:09:07.733496+0000 mgr.y (mgr.25277) 615 : cluster 0 pgmap v569: 89 pgs: 4 creating+peering, 85 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:09.046 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:09.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:08 smithi133 bash[105589]: cluster 2023-10-09T05:09:07.733496+0000 mgr.y (mgr.25277) 615 : cluster 0 pgmap v569: 89 pgs: 4 creating+peering, 85 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:09.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:08 smithi133 bash[102858]: cluster 2023-10-09T05:09:07.733496+0000 mgr.y (mgr.25277) 615 : cluster 0 pgmap v569: 89 pgs: 4 creating+peering, 85 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:09.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:09.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:09.150 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:09.183 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:09.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:09.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:09.286 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:09.321 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:09.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:09.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:09.424 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:09.458 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:09.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:09.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:09.563 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:09.596 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:09.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:09.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:09.700 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:09.733 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:09.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:09.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:09.738 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:09:09.739 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:09:09.838 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:09.871 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:09.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:09.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:09.974 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:10.000 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:10.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:10.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:10.103 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:10.138 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:10.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:10.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:10.242 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:10.265 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:10.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:10.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:10.344 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:09:10.345 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:09:10.345 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:09:10.345 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd out 1 2023-10-09T05:09:10.368 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:10.400 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:10.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:10.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:10.504 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:10.534 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:10.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:10.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:10.638 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:10.668 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:10.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:10.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:10.752 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:09:10.771 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:10.796 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:10.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:10.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:10.899 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:10.930 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:10.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:10.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:11.034 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:11.046 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:10 smithi133 bash[102858]: cluster 2023-10-09T05:09:09.734624+0000 mgr.y (mgr.25277) 616 : cluster 0 pgmap v570: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:11.046 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:10 smithi133 bash[105589]: cluster 2023-10-09T05:09:09.734624+0000 mgr.y (mgr.25277) 616 : cluster 0 pgmap v570: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:11.060 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:11.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:11.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:11.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:10 smithi194 bash[92666]: cluster 2023-10-09T05:09:09.734624+0000 mgr.y (mgr.25277) 616 : cluster 0 pgmap v570: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:11.164 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:11.193 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:11.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:11.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:11.296 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:11.328 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:11.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:11.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:11.431 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:11.463 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:11.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:11.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:11.566 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:11.590 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:11.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:11.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:11.693 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:11.726 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:11.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:11.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:11.766 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:09:11.830 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:11.851 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:11.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:11.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:11.956 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:11.977 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:11.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:11.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:12.081 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:12.114 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:12.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:12.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:12.218 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:12.242 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:12.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:12.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:12.345 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:12.373 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:12.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:12.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:12.477 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:12.511 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:12.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:12.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:12.614 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:12.643 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:12.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:12.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:12.747 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:12.779 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:12.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:12.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:12.884 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:12.894 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:12 smithi133 bash[105589]: cluster 2023-10-09T05:09:11.735488+0000 mgr.y (mgr.25277) 617 : cluster 0 pgmap v571: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:12.894 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:12 smithi133 bash[102858]: cluster 2023-10-09T05:09:11.735488+0000 mgr.y (mgr.25277) 617 : cluster 0 pgmap v571: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:12.909 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:12.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:12.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:13.013 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:13.042 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:13.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:13.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:13.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:12 smithi194 bash[92666]: cluster 2023-10-09T05:09:11.735488+0000 mgr.y (mgr.25277) 617 : cluster 0 pgmap v571: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:13.146 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:13.177 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:13.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:13.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:13.281 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:13.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:13.319 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:13.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:13.420 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:13.454 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:13.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:13.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:13.558 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:13.590 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:13.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:13.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:13.695 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:13.717 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:13.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:13.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:13.820 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:13.843 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:13.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:13.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:13.946 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:13.976 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:13.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:13.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:14.080 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:14.105 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:14.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:14.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:14.208 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:14.236 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:14.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:14.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:14.340 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:14.347 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:09:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:09:14] "GET /metrics HTTP/1.1" 200 36343 "" "Prometheus/2.43.0" 2023-10-09T05:09:14.360 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:14.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:14.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:14.463 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:14.484 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:14.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:14.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:14.587 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:14.608 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:14.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:14.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:14.685 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:14 smithi133 bash[102858]: audit 2023-10-09T05:09:13.447155+0000 mon.a (mon.0) 753 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:09:14.686 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:14 smithi133 bash[102858]: audit 2023-10-09T05:09:13.448291+0000 mon.a (mon.0) 754 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:09:14.686 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:14 smithi133 bash[102858]: cluster 2023-10-09T05:09:13.736158+0000 mgr.y (mgr.25277) 618 : cluster 0 pgmap v572: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:14.686 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:14 smithi133 bash[102858]: audit 2023-10-09T05:09:13.758257+0000 mon.a (mon.0) 755 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:09:14.686 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:14 smithi133 bash[105589]: audit 2023-10-09T05:09:13.447155+0000 mon.a (mon.0) 753 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:09:14.686 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:14 smithi133 bash[105589]: audit 2023-10-09T05:09:13.448291+0000 mon.a (mon.0) 754 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:09:14.686 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:14 smithi133 bash[105589]: cluster 2023-10-09T05:09:13.736158+0000 mgr.y (mgr.25277) 618 : cluster 0 pgmap v572: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:14.686 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:14 smithi133 bash[105589]: audit 2023-10-09T05:09:13.758257+0000 mon.a (mon.0) 755 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:09:14.712 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:14.723 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:14 smithi194 bash[92666]: audit 2023-10-09T05:09:13.447155+0000 mon.a (mon.0) 753 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:09:14.723 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:14 smithi194 bash[92666]: audit 2023-10-09T05:09:13.448291+0000 mon.a (mon.0) 754 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:09:14.723 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:14 smithi194 bash[92666]: cluster 2023-10-09T05:09:13.736158+0000 mgr.y (mgr.25277) 618 : cluster 0 pgmap v572: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:14.723 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:14 smithi194 bash[92666]: audit 2023-10-09T05:09:13.758257+0000 mon.a (mon.0) 755 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:09:14.742 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:14.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:14.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:14.846 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:14.868 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:14.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:14.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:14.972 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:15.003 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:15.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:15.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:15.106 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:15.134 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:15.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:15.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:15.237 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:15.258 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:15.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:15.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:15.362 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:15.384 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:15.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:15.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:15.487 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:15.513 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:15.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:15.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:15.617 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:15.649 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:15.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:15.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:15.754 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:15.780 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:15.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:15.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:15.884 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:15.905 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:15.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:15.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:16.008 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:16.043 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:16.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:16.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:16.147 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:16.177 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:16.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:16.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:16.281 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:16.307 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:16.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:16.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:16.409 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:16.441 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:16.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:16.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:16.545 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:16.566 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:16.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:16.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:16.668 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:16.704 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:16.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:16.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:16.808 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:16.831 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:16.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:16.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:16.934 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:16.960 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:16.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:16.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:17.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:16 smithi133 bash[102858]: cluster 2023-10-09T05:09:15.737196+0000 mgr.y (mgr.25277) 619 : cluster 0 pgmap v573: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:17.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:16 smithi133 bash[105589]: cluster 2023-10-09T05:09:15.737196+0000 mgr.y (mgr.25277) 619 : cluster 0 pgmap v573: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:17.064 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:17.096 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:17.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:17.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:17.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:16 smithi194 bash[92666]: cluster 2023-10-09T05:09:15.737196+0000 mgr.y (mgr.25277) 619 : cluster 0 pgmap v573: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:17.199 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:17.221 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:17.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:17.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:17.324 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:17.355 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:17.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:17.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:17.458 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:17.492 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:17.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:17.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:17.596 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:17.622 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:17.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:17.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:17.725 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:17.748 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:17.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:17.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:17.852 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:17.876 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:17.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:17.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:17.980 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:18.015 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:18.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:18.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:18.118 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:18.130 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:17 smithi194 bash[92666]: audit 2023-10-09T05:09:17.067419+0000 mon.a (mon.0) 756 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:09:18.150 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:18.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:18.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:18.255 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:18.289 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:18.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:18.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:18.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:17 smithi133 bash[102858]: audit 2023-10-09T05:09:17.067419+0000 mon.a (mon.0) 756 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:09:18.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:17 smithi133 bash[105589]: audit 2023-10-09T05:09:17.067419+0000 mon.a (mon.0) 756 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:09:18.393 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:18.426 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:18.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:18.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:18.529 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:18.554 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:18.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:18.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:18.657 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:18.688 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:18.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:18.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:18.793 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:18.822 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:18.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:18.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:18.925 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:18.954 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:18.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:18.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:19.057 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:19.070 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:18 smithi194 bash[92666]: cluster 2023-10-09T05:09:17.737714+0000 mgr.y (mgr.25277) 620 : cluster 0 pgmap v574: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:19.089 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:19.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:19.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:19.193 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:19.228 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:19.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:19.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:19.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:18 smithi133 bash[102858]: cluster 2023-10-09T05:09:17.737714+0000 mgr.y (mgr.25277) 620 : cluster 0 pgmap v574: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:19.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:18 smithi133 bash[105589]: cluster 2023-10-09T05:09:17.737714+0000 mgr.y (mgr.25277) 620 : cluster 0 pgmap v574: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:19.332 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:19.357 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:19.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:19.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:19.460 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:19.487 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:19.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:19.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:19.591 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:19.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:19.721 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:19.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:19.822 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:19.851 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:19.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:19.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:19.857 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:09:19.954 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:19.975 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:19.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:19.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:20.079 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:20.102 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:20.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:20.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:20.205 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:20.231 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:20.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:20.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:20.334 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:20.356 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:20.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:20.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:20.460 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:20.494 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:20.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:20.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:20.598 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:20.620 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:20.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:20.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:20.723 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:20.753 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:20.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:20.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:20.861 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:20.873 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:20 smithi133 bash[102858]: cluster 2023-10-09T05:09:19.738289+0000 mgr.y (mgr.25277) 621 : cluster 0 pgmap v575: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:20.873 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:20 smithi133 bash[105589]: cluster 2023-10-09T05:09:19.738289+0000 mgr.y (mgr.25277) 621 : cluster 0 pgmap v575: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:20.888 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:20.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:20.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:20.992 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:21.004 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:20 smithi194 bash[92666]: cluster 2023-10-09T05:09:19.738289+0000 mgr.y (mgr.25277) 621 : cluster 0 pgmap v575: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:21.026 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:21.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:21.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:21.129 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:21.152 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:21.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:21.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:21.256 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:21.288 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:21.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:21.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:21.347 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:09:21.347 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:09:21.391 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:21.426 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:21.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:21.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:21.531 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:21.554 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:21.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:21.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:21.656 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:21.676 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:21.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:21.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:21.782 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:21.803 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:21.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:21.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:21.906 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:21.941 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:21.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:21.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:22.045 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:22.101 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:22.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:22.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:22.204 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:22.228 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:22.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:22.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:22.331 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:22.354 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:22.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:22.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:22.457 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:22.486 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:22.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:22.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:22.590 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:22.618 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:22.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:22.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:22.721 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:22.783 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:22.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:22.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:22.887 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:22.905 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:22.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:22.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:23.008 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:23.015 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:22 smithi133 bash[105589]: cluster 2023-10-09T05:09:21.738738+0000 mgr.y (mgr.25277) 622 : cluster 0 pgmap v576: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:23.016 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:22 smithi133 bash[102858]: cluster 2023-10-09T05:09:21.738738+0000 mgr.y (mgr.25277) 622 : cluster 0 pgmap v576: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:23.027 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:23.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:23.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:23.131 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:23.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:22 smithi194 bash[92666]: cluster 2023-10-09T05:09:21.738738+0000 mgr.y (mgr.25277) 622 : cluster 0 pgmap v576: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:23.161 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:23.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:23.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:23.265 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:23.294 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:23.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:23.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:23.398 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:23.421 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:23.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:23.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:23.523 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:23.554 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:23.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:23.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:23.657 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:23.686 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:23.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:23.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:23.790 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:23.817 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:23.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:23.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:23.920 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:23.951 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:23.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:23.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:24.054 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:24.078 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:24.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:24.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:24.111 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T05:09:24.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:24 smithi194 bash[92666]: audit 2023-10-09T05:09:23.003528+0000 mon.a (mon.0) 757 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:09:24.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:24 smithi194 bash[92666]: audit 2023-10-09T05:09:23.010339+0000 mon.a (mon.0) 758 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:09:24.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:24 smithi194 bash[92666]: audit 2023-10-09T05:09:23.507355+0000 mon.a (mon.0) 759 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:09:24.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:24 smithi194 bash[92666]: audit 2023-10-09T05:09:23.516095+0000 mon.a (mon.0) 760 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:09:24.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:24 smithi194 bash[92666]: audit 2023-10-09T05:09:23.987734+0000 mon.a (mon.0) 761 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:09:24.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:24 smithi194 bash[92666]: audit 2023-10-09T05:09:23.989152+0000 mon.a (mon.0) 762 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:09:24.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:24 smithi194 bash[92666]: audit 2023-10-09T05:09:23.996539+0000 mon.a (mon.0) 763 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:09:24.176 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:09:24.180 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:24.202 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:09:24.204 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:24.206 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:24 smithi133 bash[105589]: audit 2023-10-09T05:09:23.003528+0000 mon.a (mon.0) 757 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:09:24.206 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:24 smithi133 bash[105589]: audit 2023-10-09T05:09:23.010339+0000 mon.a (mon.0) 758 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:09:24.206 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:24 smithi133 bash[105589]: audit 2023-10-09T05:09:23.507355+0000 mon.a (mon.0) 759 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:09:24.206 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:24 smithi133 bash[105589]: audit 2023-10-09T05:09:23.516095+0000 mon.a (mon.0) 760 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:09:24.206 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:24 smithi133 bash[105589]: audit 2023-10-09T05:09:23.987734+0000 mon.a (mon.0) 761 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:09:24.206 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:24 smithi133 bash[105589]: audit 2023-10-09T05:09:23.989152+0000 mon.a (mon.0) 762 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:09:24.206 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:24 smithi133 bash[105589]: audit 2023-10-09T05:09:23.996539+0000 mon.a (mon.0) 763 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:09:24.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:24.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:24.208 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:24 smithi133 bash[102858]: audit 2023-10-09T05:09:23.003528+0000 mon.a (mon.0) 757 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:09:24.208 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:24 smithi133 bash[102858]: audit 2023-10-09T05:09:23.010339+0000 mon.a (mon.0) 758 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:09:24.208 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:24 smithi133 bash[102858]: audit 2023-10-09T05:09:23.507355+0000 mon.a (mon.0) 759 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:09:24.208 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:24 smithi133 bash[102858]: audit 2023-10-09T05:09:23.516095+0000 mon.a (mon.0) 760 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:09:24.208 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:24 smithi133 bash[102858]: audit 2023-10-09T05:09:23.987734+0000 mon.a (mon.0) 761 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:09:24.208 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:24 smithi133 bash[102858]: audit 2023-10-09T05:09:23.989152+0000 mon.a (mon.0) 762 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:09:24.208 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:24 smithi133 bash[102858]: audit 2023-10-09T05:09:23.996539+0000 mon.a (mon.0) 763 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:09:24.231 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:09:24.255 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:09:24.280 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:09:24.309 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:24.320 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:09:24.340 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:24.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:24.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:24.355 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:09:24.383 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:09:24.386 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:09:24.386 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:09:24.386 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:09:24.386 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:09:24.386 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:09:24.386 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:09:24.386 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:09:24.386 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:09:24.443 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:24.472 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:24.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:24.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:24.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:09:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:09:24] "GET /metrics HTTP/1.1" 200 36343 "" "Prometheus/2.43.0" 2023-10-09T05:09:24.576 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:24.612 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:24.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:24.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:24.715 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:24.746 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:24.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:24.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:24.854 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:24.887 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:24.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:24.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:24.991 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:25.019 INFO:teuthology.orchestra.run.smithi133.stderr:enabled application 'rados' on pool 'unique_pool_4' 2023-10-09T05:09:25.020 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is set 2023-10-09T05:09:25.024 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:25.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:25.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:25.128 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:25.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:25 smithi194 bash[92666]: cluster 2023-10-09T05:09:23.739228+0000 mgr.y (mgr.25277) 623 : cluster 0 pgmap v577: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:25.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:25 smithi194 bash[92666]: audit 2023-10-09T05:09:24.153108+0000 mon.a (mon.0) 764 : audit 0 from='client.? 172.21.15.133:0/704673806' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:09:25.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:25 smithi194 bash[92666]: audit 2023-10-09T05:09:24.416373+0000 mon.c (mon.2) 60 : audit 1 from='client.? 172.21.15.133:0/2800992996' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:09:25.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:25 smithi194 bash[92666]: audit 2023-10-09T05:09:24.416682+0000 mon.a (mon.0) 765 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:09:25.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:25 smithi194 bash[92666]: audit 2023-10-09T05:09:24.661701+0000 mon.c (mon.2) 61 : audit 1 from='client.? 172.21.15.133:0/1084339731' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:09:25.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:25 smithi194 bash[92666]: audit 2023-10-09T05:09:24.662197+0000 mon.a (mon.0) 766 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:09:25.160 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:25.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:25.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:25.263 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:25.272 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:25 smithi133 bash[105589]: cluster 2023-10-09T05:09:23.739228+0000 mgr.y (mgr.25277) 623 : cluster 0 pgmap v577: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:25.272 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:25 smithi133 bash[105589]: audit 2023-10-09T05:09:24.153108+0000 mon.a (mon.0) 764 : audit 0 from='client.? 172.21.15.133:0/704673806' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:09:25.272 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:25 smithi133 bash[105589]: audit 2023-10-09T05:09:24.416373+0000 mon.c (mon.2) 60 : audit 1 from='client.? 172.21.15.133:0/2800992996' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:09:25.272 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:25 smithi133 bash[105589]: audit 2023-10-09T05:09:24.416682+0000 mon.a (mon.0) 765 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:09:25.272 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:25 smithi133 bash[105589]: audit 2023-10-09T05:09:24.661701+0000 mon.c (mon.2) 61 : audit 1 from='client.? 172.21.15.133:0/1084339731' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:09:25.272 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:25 smithi133 bash[105589]: audit 2023-10-09T05:09:24.662197+0000 mon.a (mon.0) 766 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:09:25.272 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:25 smithi133 bash[102858]: cluster 2023-10-09T05:09:23.739228+0000 mgr.y (mgr.25277) 623 : cluster 0 pgmap v577: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:25.273 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:25 smithi133 bash[102858]: audit 2023-10-09T05:09:24.153108+0000 mon.a (mon.0) 764 : audit 0 from='client.? 172.21.15.133:0/704673806' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:09:25.273 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:25 smithi133 bash[102858]: audit 2023-10-09T05:09:24.416373+0000 mon.c (mon.2) 60 : audit 1 from='client.? 172.21.15.133:0/2800992996' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:09:25.273 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:25 smithi133 bash[102858]: audit 2023-10-09T05:09:24.416682+0000 mon.a (mon.0) 765 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:09:25.273 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:25 smithi133 bash[102858]: audit 2023-10-09T05:09:24.661701+0000 mon.c (mon.2) 61 : audit 1 from='client.? 172.21.15.133:0/1084339731' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:09:25.273 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:25 smithi133 bash[102858]: audit 2023-10-09T05:09:24.662197+0000 mon.a (mon.0) 766 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:09:25.289 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:25.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:25.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:25.393 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:25.427 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:25.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:25.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:25.530 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:25.551 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:25.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:25.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:25.655 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:25.682 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:25.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:25.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:25.785 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:25.820 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:25.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:25.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:25.923 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:25.952 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:25.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:25.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:26.056 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:26.067 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:26 smithi194 bash[92666]: cluster 2023-10-09T05:09:25.002294+0000 mon.a (mon.0) 767 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:09:26.067 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:26 smithi194 bash[92666]: cluster 2023-10-09T05:09:25.008333+0000 osd.4 (osd.4) 74 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T05:09:26.067 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:26 smithi194 bash[92666]: cluster 2023-10-09T05:09:25.008645+0000 osd.5 (osd.5) 98 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T05:09:26.067 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:26 smithi194 bash[92666]: audit 2023-10-09T05:09:25.012227+0000 mon.a (mon.0) 768 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:09:26.067 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:26 smithi194 bash[92666]: audit 2023-10-09T05:09:25.012345+0000 mon.a (mon.0) 769 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:09:26.067 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:26 smithi194 bash[92666]: cluster 2023-10-09T05:09:25.012459+0000 mon.a (mon.0) 770 : cluster 0 osdmap e142: 8 total, 8 up, 8 in 2023-10-09T05:09:26.088 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:26.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:26.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:26.192 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:26.200 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:26 smithi133 bash[102858]: cluster 2023-10-09T05:09:25.002294+0000 mon.a (mon.0) 767 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:09:26.200 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:26 smithi133 bash[102858]: cluster 2023-10-09T05:09:25.008333+0000 osd.4 (osd.4) 74 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T05:09:26.200 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:26 smithi133 bash[102858]: cluster 2023-10-09T05:09:25.008645+0000 osd.5 (osd.5) 98 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T05:09:26.200 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:26 smithi133 bash[102858]: audit 2023-10-09T05:09:25.012227+0000 mon.a (mon.0) 768 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:09:26.200 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:26 smithi133 bash[102858]: audit 2023-10-09T05:09:25.012345+0000 mon.a (mon.0) 769 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:09:26.200 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:26 smithi133 bash[102858]: cluster 2023-10-09T05:09:25.012459+0000 mon.a (mon.0) 770 : cluster 0 osdmap e142: 8 total, 8 up, 8 in 2023-10-09T05:09:26.201 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:26 smithi133 bash[105589]: cluster 2023-10-09T05:09:25.002294+0000 mon.a (mon.0) 767 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:09:26.201 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:26 smithi133 bash[105589]: cluster 2023-10-09T05:09:25.008333+0000 osd.4 (osd.4) 74 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T05:09:26.201 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:26 smithi133 bash[105589]: cluster 2023-10-09T05:09:25.008645+0000 osd.5 (osd.5) 98 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T05:09:26.201 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:26 smithi133 bash[105589]: audit 2023-10-09T05:09:25.012227+0000 mon.a (mon.0) 768 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:09:26.201 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:26 smithi133 bash[105589]: audit 2023-10-09T05:09:25.012345+0000 mon.a (mon.0) 769 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:09:26.201 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:26 smithi133 bash[105589]: cluster 2023-10-09T05:09:25.012459+0000 mon.a (mon.0) 770 : cluster 0 osdmap e142: 8 total, 8 up, 8 in 2023-10-09T05:09:26.217 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:26.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:26.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:26.320 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:26.353 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:26.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:26.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:26.456 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:26.487 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:26.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:26.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:26.591 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:26.612 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:26.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:26.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:26.715 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:26.745 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:26.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:26.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:26.850 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:26.882 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:26.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:26.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:26.986 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:27.008 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:27.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:27.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:27.114 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:27.146 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:27.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:27.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:27.250 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:27.265 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:27 smithi133 bash[105589]: cluster 2023-10-09T05:09:25.739989+0000 mgr.y (mgr.25277) 624 : cluster 0 pgmap v579: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:27.265 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:27 smithi133 bash[102858]: cluster 2023-10-09T05:09:25.739989+0000 mgr.y (mgr.25277) 624 : cluster 0 pgmap v579: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:27.282 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:27.284 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:27 smithi194 bash[92666]: cluster 2023-10-09T05:09:25.739989+0000 mgr.y (mgr.25277) 624 : cluster 0 pgmap v579: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:27.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:27.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:27.386 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:27.418 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:27.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:27.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:27.522 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:27.554 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:27.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:27.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:27.657 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:27.684 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:27.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:27.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:27.787 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:27.813 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:27.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:27.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:27.917 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:27.951 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:27.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:27.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:28.055 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:28.084 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:28.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:28.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:28.187 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:28.217 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:28.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:28.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:28.321 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:28.352 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:28.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:28.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:28.456 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:28.488 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:28.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:28.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:28.930 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:28.964 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:28.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:28.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:29.038 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:09:29.068 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:29.097 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:29.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:29.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:29.202 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:29.210 DEBUG:teuthology.orchestra.run.smithi133:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2023-10-09T05:09:29.211 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:29 smithi133 bash[105589]: cluster 2023-10-09T05:09:27.740766+0000 mgr.y (mgr.25277) 625 : cluster 0 pgmap v580: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:29.211 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:29 smithi133 bash[105589]: audit 2023-10-09T05:09:28.440649+0000 mon.a (mon.0) 771 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:09:29.211 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:29 smithi133 bash[102858]: cluster 2023-10-09T05:09:27.740766+0000 mgr.y (mgr.25277) 625 : cluster 0 pgmap v580: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:29.212 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:29 smithi133 bash[102858]: audit 2023-10-09T05:09:28.440649+0000 mon.a (mon.0) 771 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:09:29.212 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T05:09:29.212 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T05:09:29.229 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:29.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:29.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:29.253 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:hints = 1 2023-10-09T05:09:29.253 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T05:09:29.253 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Object prefix: benchmark_data_smithi133_153303 2023-10-09T05:09:29.254 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:09:29.254 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 0 0 0 0 0 0 - 0 2023-10-09T05:09:29.333 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:29.343 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:29 smithi194 bash[92666]: cluster 2023-10-09T05:09:27.740766+0000 mgr.y (mgr.25277) 625 : cluster 0 pgmap v580: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:09:29.343 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:29 smithi194 bash[92666]: audit 2023-10-09T05:09:28.440649+0000 mon.a (mon.0) 771 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:09:29.361 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:29.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:29.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:29.465 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:29.491 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:29.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:29.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:29.595 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:29.619 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:29.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:29.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:29.723 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:29.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:29.755 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:29.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:29.856 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:29.881 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:29.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:29.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:29.986 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:30.014 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:30.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:30.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:30.044 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:30 smithi194 bash[92666]: audit 2023-10-09T05:09:29.909883+0000 mon.b (mon.1) 11 : audit 1 from='client.? 172.21.15.133:0/2246985041' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T05:09:30.044 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:30 smithi194 bash[92666]: audit 2023-10-09T05:09:29.910375+0000 mon.a (mon.0) 772 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T05:09:30.044 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:30 smithi194 bash[92666]: cluster 2023-10-09T05:09:29.910512+0000 mon.a (mon.0) 773 : cluster 1 Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T05:09:30.066 INFO:teuthology.orchestra.run.smithi133.stderr:marked out osd.1. 2023-10-09T05:09:30.119 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:30.153 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:30.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:30.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:30.254 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 1 16 2212 2196 137.236 137.25 0.00171058 0.0058602 2023-10-09T05:09:30.256 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:30.270 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:30 smithi133 bash[105589]: audit 2023-10-09T05:09:29.909883+0000 mon.b (mon.1) 11 : audit 1 from='client.? 172.21.15.133:0/2246985041' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T05:09:30.270 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:30 smithi133 bash[105589]: audit 2023-10-09T05:09:29.910375+0000 mon.a (mon.0) 772 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T05:09:30.270 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:30 smithi133 bash[105589]: cluster 2023-10-09T05:09:29.910512+0000 mon.a (mon.0) 773 : cluster 1 Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T05:09:30.270 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:29 smithi133 bash[102858]: debug 2023-10-09T05:09:29.908+0000 7f5eb4d1d700 -1 mon.a@0(leader).osd e142 definitely_dead 0 2023-10-09T05:09:30.271 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:30 smithi133 bash[102858]: audit 2023-10-09T05:09:29.909883+0000 mon.b (mon.1) 11 : audit 1 from='client.? 172.21.15.133:0/2246985041' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T05:09:30.271 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:30 smithi133 bash[102858]: audit 2023-10-09T05:09:29.910375+0000 mon.a (mon.0) 772 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T05:09:30.271 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:30 smithi133 bash[102858]: cluster 2023-10-09T05:09:29.910512+0000 mon.a (mon.0) 773 : cluster 1 Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T05:09:30.290 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:30.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:30.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:30.393 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:30.424 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:30.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:30.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:30.528 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:30.561 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:30.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:30.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:30.665 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:30.695 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:30.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:30.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:30.801 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:30.833 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:30.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:30.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:30.937 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:30.949 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:09:30.971 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:30.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:30.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:31.075 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:31.112 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:31.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:31.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:31.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:31 smithi194 bash[92666]: cluster 2023-10-09T05:09:29.741549+0000 mgr.y (mgr.25277) 626 : cluster 0 pgmap v581: 89 pgs: 89 active+clean; 6.9 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 646 KiB/s wr, 10 op/s 2023-10-09T05:09:31.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:31 smithi194 bash[92666]: cluster 2023-10-09T05:09:30.050765+0000 osd.7 (osd.7) 53 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:31 smithi194 bash[92666]: cluster 2023-10-09T05:09:30.050868+0000 osd.4 (osd.4) 75 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:31 smithi194 bash[92666]: cluster 2023-10-09T05:09:30.052333+0000 osd.5 (osd.5) 99 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:31 smithi194 bash[92666]: cluster 2023-10-09T05:09:30.055443+0000 osd.6 (osd.6) 64 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:31 smithi194 bash[92666]: cluster 2023-10-09T05:09:30.055992+0000 osd.4 (osd.4) 76 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:31 smithi194 bash[92666]: cluster 2023-10-09T05:09:30.056854+0000 osd.5 (osd.5) 100 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:31 smithi194 bash[92666]: cluster 2023-10-09T05:09:30.056984+0000 osd.7 (osd.7) 54 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:31 smithi194 bash[92666]: cluster 2023-10-09T05:09:30.057342+0000 osd.6 (osd.6) 65 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:31 smithi194 bash[92666]: cluster 2023-10-09T05:09:30.057827+0000 osd.4 (osd.4) 77 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:31 smithi194 bash[92666]: cluster 2023-10-09T05:09:30.058694+0000 osd.6 (osd.6) 66 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:31 smithi194 bash[92666]: audit 2023-10-09T05:09:30.058870+0000 mon.a (mon.0) 774 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T05:09:31.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:31 smithi194 bash[92666]: cluster 2023-10-09T05:09:30.058960+0000 mon.a (mon.0) 775 : cluster 0 osdmap e143: 8 total, 8 up, 7 in 2023-10-09T05:09:31.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:31 smithi194 bash[92666]: cluster 2023-10-09T05:09:30.059643+0000 osd.5 (osd.5) 101 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:31 smithi194 bash[92666]: cluster 2023-10-09T05:09:30.060259+0000 osd.6 (osd.6) 67 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.217 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:31.226 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:31 smithi133 bash[102858]: cluster 2023-10-09T05:09:29.741549+0000 mgr.y (mgr.25277) 626 : cluster 0 pgmap v581: 89 pgs: 89 active+clean; 6.9 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 646 KiB/s wr, 10 op/s 2023-10-09T05:09:31.226 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:31 smithi133 bash[102858]: cluster 2023-10-09T05:09:30.050765+0000 osd.7 (osd.7) 53 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.226 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:31 smithi133 bash[102858]: cluster 2023-10-09T05:09:30.050868+0000 osd.4 (osd.4) 75 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.226 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:31 smithi133 bash[102858]: cluster 2023-10-09T05:09:30.052333+0000 osd.5 (osd.5) 99 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.227 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:31 smithi133 bash[102858]: cluster 2023-10-09T05:09:30.055443+0000 osd.6 (osd.6) 64 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.227 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:31 smithi133 bash[102858]: cluster 2023-10-09T05:09:30.055992+0000 osd.4 (osd.4) 76 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.227 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:31 smithi133 bash[102858]: cluster 2023-10-09T05:09:30.056854+0000 osd.5 (osd.5) 100 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.227 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:31 smithi133 bash[102858]: cluster 2023-10-09T05:09:30.056984+0000 osd.7 (osd.7) 54 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.227 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:31 smithi133 bash[102858]: cluster 2023-10-09T05:09:30.057342+0000 osd.6 (osd.6) 65 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.227 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:31 smithi133 bash[102858]: cluster 2023-10-09T05:09:30.057827+0000 osd.4 (osd.4) 77 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.227 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:31 smithi133 bash[102858]: cluster 2023-10-09T05:09:30.058694+0000 osd.6 (osd.6) 66 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.227 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:31 smithi133 bash[102858]: audit 2023-10-09T05:09:30.058870+0000 mon.a (mon.0) 774 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T05:09:31.227 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:31 smithi133 bash[102858]: cluster 2023-10-09T05:09:30.058960+0000 mon.a (mon.0) 775 : cluster 0 osdmap e143: 8 total, 8 up, 7 in 2023-10-09T05:09:31.227 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:31 smithi133 bash[102858]: cluster 2023-10-09T05:09:30.059643+0000 osd.5 (osd.5) 101 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.227 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:31 smithi133 bash[102858]: cluster 2023-10-09T05:09:30.060259+0000 osd.6 (osd.6) 67 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.228 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:31 smithi133 bash[105589]: cluster 2023-10-09T05:09:29.741549+0000 mgr.y (mgr.25277) 626 : cluster 0 pgmap v581: 89 pgs: 89 active+clean; 6.9 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 646 KiB/s wr, 10 op/s 2023-10-09T05:09:31.228 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:31 smithi133 bash[105589]: cluster 2023-10-09T05:09:30.050765+0000 osd.7 (osd.7) 53 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.228 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:31 smithi133 bash[105589]: cluster 2023-10-09T05:09:30.050868+0000 osd.4 (osd.4) 75 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.228 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:31 smithi133 bash[105589]: cluster 2023-10-09T05:09:30.052333+0000 osd.5 (osd.5) 99 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.228 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:31 smithi133 bash[105589]: cluster 2023-10-09T05:09:30.055443+0000 osd.6 (osd.6) 64 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.228 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:31 smithi133 bash[105589]: cluster 2023-10-09T05:09:30.055992+0000 osd.4 (osd.4) 76 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.228 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:31 smithi133 bash[105589]: cluster 2023-10-09T05:09:30.056854+0000 osd.5 (osd.5) 100 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.228 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:31 smithi133 bash[105589]: cluster 2023-10-09T05:09:30.056984+0000 osd.7 (osd.7) 54 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.228 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:31 smithi133 bash[105589]: cluster 2023-10-09T05:09:30.057342+0000 osd.6 (osd.6) 65 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.229 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:31 smithi133 bash[105589]: cluster 2023-10-09T05:09:30.057827+0000 osd.4 (osd.4) 77 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.229 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:31 smithi133 bash[105589]: cluster 2023-10-09T05:09:30.058694+0000 osd.6 (osd.6) 66 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.229 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:31 smithi133 bash[105589]: audit 2023-10-09T05:09:30.058870+0000 mon.a (mon.0) 774 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T05:09:31.229 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:31 smithi133 bash[105589]: cluster 2023-10-09T05:09:30.058960+0000 mon.a (mon.0) 775 : cluster 0 osdmap e143: 8 total, 8 up, 7 in 2023-10-09T05:09:31.229 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:31 smithi133 bash[105589]: cluster 2023-10-09T05:09:30.059643+0000 osd.5 (osd.5) 101 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.229 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:31 smithi133 bash[105589]: cluster 2023-10-09T05:09:30.060259+0000 osd.6 (osd.6) 67 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T05:09:31.241 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:31.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:31.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:31.254 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 2 16 2212 2196 68.6176 0 - 0.0058602 2023-10-09T05:09:31.345 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:31.369 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:31.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:31.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:31.474 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:31.505 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:31.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:31.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:31.609 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:31.642 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:31.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:31.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:31.746 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:31.779 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:31.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:31.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:31.883 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:31.911 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:31.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:31.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:32.015 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:32.052 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:32.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:32.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:32.155 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:32.190 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:32.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:32.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:32.254 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 3 16 2614 2598 54.1193 12.5625 0.00735648 0.0184551 2023-10-09T05:09:32.296 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:32.305 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:32 smithi194 bash[92666]: cluster 2023-10-09T05:09:31.046314+0000 osd.5 (osd.5) 102 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T05:09:32.305 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:32 smithi194 bash[92666]: cluster 2023-10-09T05:09:31.046648+0000 mon.a (mon.0) 776 : cluster 0 osdmap e144: 8 total, 8 up, 7 in 2023-10-09T05:09:32.305 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:32 smithi194 bash[92666]: cluster 2023-10-09T05:09:31.047234+0000 osd.6 (osd.6) 68 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T05:09:32.305 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:32 smithi194 bash[92666]: cluster 2023-10-09T05:09:31.047371+0000 osd.4 (osd.4) 78 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T05:09:32.305 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:32 smithi194 bash[92666]: cluster 2023-10-09T05:09:31.047400+0000 osd.7 (osd.7) 55 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T05:09:32.305 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:32 smithi194 bash[92666]: cluster 2023-10-09T05:09:31.051462+0000 osd.5 (osd.5) 103 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T05:09:32.306 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:32 smithi194 bash[92666]: cluster 2023-10-09T05:09:31.051928+0000 osd.4 (osd.4) 79 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T05:09:32.324 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:32.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:32.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:32.430 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:32.458 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:32.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:32.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:32.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:32 smithi133 bash[105589]: cluster 2023-10-09T05:09:31.046314+0000 osd.5 (osd.5) 102 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T05:09:32.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:32 smithi133 bash[105589]: cluster 2023-10-09T05:09:31.046648+0000 mon.a (mon.0) 776 : cluster 0 osdmap e144: 8 total, 8 up, 7 in 2023-10-09T05:09:32.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:32 smithi133 bash[105589]: cluster 2023-10-09T05:09:31.047234+0000 osd.6 (osd.6) 68 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T05:09:32.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:32 smithi133 bash[105589]: cluster 2023-10-09T05:09:31.047371+0000 osd.4 (osd.4) 78 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T05:09:32.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:32 smithi133 bash[105589]: cluster 2023-10-09T05:09:31.047400+0000 osd.7 (osd.7) 55 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T05:09:32.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:32 smithi133 bash[105589]: cluster 2023-10-09T05:09:31.051462+0000 osd.5 (osd.5) 103 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T05:09:32.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:32 smithi133 bash[105589]: cluster 2023-10-09T05:09:31.051928+0000 osd.4 (osd.4) 79 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T05:09:32.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:32 smithi133 bash[102858]: cluster 2023-10-09T05:09:31.046314+0000 osd.5 (osd.5) 102 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T05:09:32.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:32 smithi133 bash[102858]: cluster 2023-10-09T05:09:31.046648+0000 mon.a (mon.0) 776 : cluster 0 osdmap e144: 8 total, 8 up, 7 in 2023-10-09T05:09:32.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:32 smithi133 bash[102858]: cluster 2023-10-09T05:09:31.047234+0000 osd.6 (osd.6) 68 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T05:09:32.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:32 smithi133 bash[102858]: cluster 2023-10-09T05:09:31.047371+0000 osd.4 (osd.4) 78 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T05:09:32.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:32 smithi133 bash[102858]: cluster 2023-10-09T05:09:31.047400+0000 osd.7 (osd.7) 55 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T05:09:32.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:32 smithi133 bash[102858]: cluster 2023-10-09T05:09:31.051462+0000 osd.5 (osd.5) 103 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T05:09:32.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:32 smithi133 bash[102858]: cluster 2023-10-09T05:09:31.051928+0000 osd.4 (osd.4) 79 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T05:09:32.561 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:32.586 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:32.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:32.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:32.689 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:32.716 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:32.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:32.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:32.822 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:32.864 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:32.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:32.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:32.968 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:32.999 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:33.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:33.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:33.103 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:33.113 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:33 smithi133 bash[105589]: cluster 2023-10-09T05:09:31.742328+0000 mgr.y (mgr.25277) 627 : cluster 0 pgmap v584: 89 pgs: 8 peering, 81 active+clean; 33 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 4.8 MiB/s wr, 77 op/s 2023-10-09T05:09:33.113 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:33 smithi133 bash[105589]: cluster 2023-10-09T05:09:32.055208+0000 mon.a (mon.0) 777 : cluster 3 Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T05:09:33.113 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:33 smithi133 bash[105589]: cluster 2023-10-09T05:09:32.067775+0000 osd.5 (osd.5) 104 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.113 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:33 smithi133 bash[105589]: cluster 2023-10-09T05:09:32.067775+0000 osd.6 (osd.6) 69 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.113 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:33 smithi133 bash[105589]: cluster 2023-10-09T05:09:32.070151+0000 osd.7 (osd.7) 56 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.113 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:33 smithi133 bash[105589]: cluster 2023-10-09T05:09:32.070593+0000 osd.5 (osd.5) 105 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.113 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:33 smithi133 bash[105589]: cluster 2023-10-09T05:09:32.070648+0000 osd.4 (osd.4) 80 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.113 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:33 smithi133 bash[105589]: cluster 2023-10-09T05:09:32.070725+0000 osd.6 (osd.6) 70 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.114 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:33 smithi133 bash[105589]: cluster 2023-10-09T05:09:32.071751+0000 mon.a (mon.0) 778 : cluster 0 osdmap e145: 8 total, 8 up, 7 in 2023-10-09T05:09:33.114 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:33 smithi133 bash[105589]: cluster 2023-10-09T05:09:32.072459+0000 osd.7 (osd.7) 57 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.114 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:33 smithi133 bash[105589]: cluster 2023-10-09T05:09:32.073482+0000 osd.6 (osd.6) 71 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.114 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:33 smithi133 bash[105589]: cluster 2023-10-09T05:09:32.074007+0000 osd.7 (osd.7) 58 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.114 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:33 smithi133 bash[105589]: cluster 2023-10-09T05:09:32.074799+0000 osd.4 (osd.4) 81 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.114 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:33 smithi133 bash[105589]: cluster 2023-10-09T05:09:32.077324+0000 osd.4 (osd.4) 82 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.114 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:33 smithi133 bash[102858]: cluster 2023-10-09T05:09:31.742328+0000 mgr.y (mgr.25277) 627 : cluster 0 pgmap v584: 89 pgs: 8 peering, 81 active+clean; 33 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 4.8 MiB/s wr, 77 op/s 2023-10-09T05:09:33.115 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:33 smithi133 bash[102858]: cluster 2023-10-09T05:09:32.055208+0000 mon.a (mon.0) 777 : cluster 3 Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T05:09:33.115 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:33 smithi133 bash[102858]: cluster 2023-10-09T05:09:32.067775+0000 osd.5 (osd.5) 104 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.115 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:33 smithi133 bash[102858]: cluster 2023-10-09T05:09:32.067775+0000 osd.6 (osd.6) 69 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.115 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:33 smithi133 bash[102858]: cluster 2023-10-09T05:09:32.070151+0000 osd.7 (osd.7) 56 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.115 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:33 smithi133 bash[102858]: cluster 2023-10-09T05:09:32.070593+0000 osd.5 (osd.5) 105 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.115 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:33 smithi133 bash[102858]: cluster 2023-10-09T05:09:32.070648+0000 osd.4 (osd.4) 80 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.115 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:33 smithi133 bash[102858]: cluster 2023-10-09T05:09:32.070725+0000 osd.6 (osd.6) 70 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.115 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:33 smithi133 bash[102858]: cluster 2023-10-09T05:09:32.071751+0000 mon.a (mon.0) 778 : cluster 0 osdmap e145: 8 total, 8 up, 7 in 2023-10-09T05:09:33.115 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:33 smithi133 bash[102858]: cluster 2023-10-09T05:09:32.072459+0000 osd.7 (osd.7) 57 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.115 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:33 smithi133 bash[102858]: cluster 2023-10-09T05:09:32.073482+0000 osd.6 (osd.6) 71 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.115 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:33 smithi133 bash[102858]: cluster 2023-10-09T05:09:32.074007+0000 osd.7 (osd.7) 58 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.116 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:33 smithi133 bash[102858]: cluster 2023-10-09T05:09:32.074799+0000 osd.4 (osd.4) 81 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.116 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:33 smithi133 bash[102858]: cluster 2023-10-09T05:09:32.077324+0000 osd.4 (osd.4) 82 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.130 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:33.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:33.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:33.233 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:33.254 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 4 16 4784 4768 74.4922 135.625 0.00309147 0.0129348 2023-10-09T05:09:33.267 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:33.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:33.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:33.374 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:33.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:33 smithi194 bash[92666]: cluster 2023-10-09T05:09:31.742328+0000 mgr.y (mgr.25277) 627 : cluster 0 pgmap v584: 89 pgs: 8 peering, 81 active+clean; 33 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 4.8 MiB/s wr, 77 op/s 2023-10-09T05:09:33.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:33 smithi194 bash[92666]: cluster 2023-10-09T05:09:32.055208+0000 mon.a (mon.0) 777 : cluster 3 Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T05:09:33.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:33 smithi194 bash[92666]: cluster 2023-10-09T05:09:32.067775+0000 osd.5 (osd.5) 104 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:33 smithi194 bash[92666]: cluster 2023-10-09T05:09:32.067775+0000 osd.6 (osd.6) 69 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:33 smithi194 bash[92666]: cluster 2023-10-09T05:09:32.070151+0000 osd.7 (osd.7) 56 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:33 smithi194 bash[92666]: cluster 2023-10-09T05:09:32.070593+0000 osd.5 (osd.5) 105 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:33 smithi194 bash[92666]: cluster 2023-10-09T05:09:32.070648+0000 osd.4 (osd.4) 80 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:33 smithi194 bash[92666]: cluster 2023-10-09T05:09:32.070725+0000 osd.6 (osd.6) 70 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:33 smithi194 bash[92666]: cluster 2023-10-09T05:09:32.071751+0000 mon.a (mon.0) 778 : cluster 0 osdmap e145: 8 total, 8 up, 7 in 2023-10-09T05:09:33.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:33 smithi194 bash[92666]: cluster 2023-10-09T05:09:32.072459+0000 osd.7 (osd.7) 57 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:33 smithi194 bash[92666]: cluster 2023-10-09T05:09:32.073482+0000 osd.6 (osd.6) 71 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:33 smithi194 bash[92666]: cluster 2023-10-09T05:09:32.074007+0000 osd.7 (osd.7) 58 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:33 smithi194 bash[92666]: cluster 2023-10-09T05:09:32.074799+0000 osd.4 (osd.4) 81 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:33 smithi194 bash[92666]: cluster 2023-10-09T05:09:32.077324+0000 osd.4 (osd.4) 82 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T05:09:33.407 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:33.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:33.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:33.511 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:33.543 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:33.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:33.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:33.647 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:33.678 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:33.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:33.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:33.783 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:33.812 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:33.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:33.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:33.915 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:33.946 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:33.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:33.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:33.955 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:09:33.955 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:09:34.051 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:34.081 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:34.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:34.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:34.184 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:34.192 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:34 smithi133 bash[105589]: cluster 2023-10-09T05:09:33.075514+0000 osd.4 (osd.4) 83 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T05:09:34.193 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:34 smithi133 bash[105589]: cluster 2023-10-09T05:09:33.077004+0000 osd.7 (osd.7) 59 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T05:09:34.193 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:34 smithi133 bash[105589]: cluster 2023-10-09T05:09:33.077793+0000 osd.5 (osd.5) 106 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T05:09:34.193 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:34 smithi133 bash[105589]: cluster 2023-10-09T05:09:33.078923+0000 osd.6 (osd.6) 72 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T05:09:34.193 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:34 smithi133 bash[105589]: cluster 2023-10-09T05:09:33.080663+0000 mon.a (mon.0) 779 : cluster 0 osdmap e146: 8 total, 8 up, 7 in 2023-10-09T05:09:34.193 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:34 smithi133 bash[105589]: cluster 2023-10-09T05:09:33.081064+0000 osd.5 (osd.5) 107 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T05:09:34.193 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:34 smithi133 bash[105589]: cluster 2023-10-09T05:09:33.084942+0000 osd.7 (osd.7) 60 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T05:09:34.193 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:34 smithi133 bash[105589]: cluster 2023-10-09T05:09:33.087941+0000 osd.5 (osd.5) 108 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T05:09:34.193 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:34 smithi133 bash[102858]: cluster 2023-10-09T05:09:33.075514+0000 osd.4 (osd.4) 83 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T05:09:34.194 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:34 smithi133 bash[102858]: cluster 2023-10-09T05:09:33.077004+0000 osd.7 (osd.7) 59 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T05:09:34.194 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:34 smithi133 bash[102858]: cluster 2023-10-09T05:09:33.077793+0000 osd.5 (osd.5) 106 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T05:09:34.194 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:34 smithi133 bash[102858]: cluster 2023-10-09T05:09:33.078923+0000 osd.6 (osd.6) 72 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T05:09:34.194 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:34 smithi133 bash[102858]: cluster 2023-10-09T05:09:33.080663+0000 mon.a (mon.0) 779 : cluster 0 osdmap e146: 8 total, 8 up, 7 in 2023-10-09T05:09:34.194 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:34 smithi133 bash[102858]: cluster 2023-10-09T05:09:33.081064+0000 osd.5 (osd.5) 107 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T05:09:34.194 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:34 smithi133 bash[102858]: cluster 2023-10-09T05:09:33.084942+0000 osd.7 (osd.7) 60 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T05:09:34.194 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:34 smithi133 bash[102858]: cluster 2023-10-09T05:09:33.087941+0000 osd.5 (osd.5) 108 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T05:09:34.207 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:34.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:34.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:34.254 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 5 16 4865 4849 60.606 5.0625 0.0018894 0.0160558 2023-10-09T05:09:34.310 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:34.321 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:34 smithi194 bash[92666]: cluster 2023-10-09T05:09:33.075514+0000 osd.4 (osd.4) 83 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T05:09:34.321 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:34 smithi194 bash[92666]: cluster 2023-10-09T05:09:33.077004+0000 osd.7 (osd.7) 59 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T05:09:34.321 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:34 smithi194 bash[92666]: cluster 2023-10-09T05:09:33.077793+0000 osd.5 (osd.5) 106 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T05:09:34.322 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:34 smithi194 bash[92666]: cluster 2023-10-09T05:09:33.078923+0000 osd.6 (osd.6) 72 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T05:09:34.322 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:34 smithi194 bash[92666]: cluster 2023-10-09T05:09:33.080663+0000 mon.a (mon.0) 779 : cluster 0 osdmap e146: 8 total, 8 up, 7 in 2023-10-09T05:09:34.322 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:34 smithi194 bash[92666]: cluster 2023-10-09T05:09:33.081064+0000 osd.5 (osd.5) 107 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T05:09:34.322 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:34 smithi194 bash[92666]: cluster 2023-10-09T05:09:33.084942+0000 osd.7 (osd.7) 60 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T05:09:34.322 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:34 smithi194 bash[92666]: cluster 2023-10-09T05:09:33.087941+0000 osd.5 (osd.5) 108 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T05:09:34.339 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:34.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:34.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:34.443 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:34.472 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:34.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:34.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:34.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:09:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:09:34] "GET /metrics HTTP/1.1" 200 36341 "" "Prometheus/2.43.0" 2023-10-09T05:09:34.575 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:34.605 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:34.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:34.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:34.709 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:34.736 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:34.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:34.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:34.839 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:34.871 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:34.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:34.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:34.975 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:35.008 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:35.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:35.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:35.112 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:35.136 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:35.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:35 smithi194 bash[92666]: cluster 2023-10-09T05:09:33.743114+0000 mgr.y (mgr.25277) 628 : cluster 0 pgmap v587: 89 pgs: 1 active+recovering+undersized+remapped, 8 peering, 80 active+clean; 108 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 25 MiB/s wr, 404 op/s; 92/5226 objects misplaced (1.760%); 672 KiB/s, 10 objects/s recovering 2023-10-09T05:09:35.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:35 smithi194 bash[92666]: cluster 2023-10-09T05:09:34.080961+0000 osd.7 (osd.7) 61 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T05:09:35.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:35 smithi194 bash[92666]: cluster 2023-10-09T05:09:34.081765+0000 osd.5 (osd.5) 109 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T05:09:35.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:35 smithi194 bash[92666]: cluster 2023-10-09T05:09:34.083196+0000 mon.a (mon.0) 780 : cluster 0 osdmap e147: 8 total, 8 up, 7 in 2023-10-09T05:09:35.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:35 smithi194 bash[92666]: cluster 2023-10-09T05:09:34.083431+0000 osd.7 (osd.7) 62 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T05:09:35.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:35 smithi194 bash[92666]: cluster 2023-10-09T05:09:34.083709+0000 osd.6 (osd.6) 73 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T05:09:35.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:35 smithi194 bash[92666]: cluster 2023-10-09T05:09:34.084222+0000 osd.5 (osd.5) 110 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T05:09:35.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:35 smithi194 bash[92666]: cluster 2023-10-09T05:09:34.087178+0000 osd.5 (osd.5) 111 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T05:09:35.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:35 smithi194 bash[92666]: cluster 2023-10-09T05:09:34.088161+0000 osd.6 (osd.6) 74 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T05:09:35.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:35.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:35.239 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:35.254 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 6 16 4914 4898 51.0155 3.0625 0.00219142 0.0191509 2023-10-09T05:09:35.263 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:35.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:35.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:35.366 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:35.396 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:35.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:35.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:35.445 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:35 smithi133 bash[105589]: cluster 2023-10-09T05:09:33.743114+0000 mgr.y (mgr.25277) 628 : cluster 0 pgmap v587: 89 pgs: 1 active+recovering+undersized+remapped, 8 peering, 80 active+clean; 108 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 25 MiB/s wr, 404 op/s; 92/5226 objects misplaced (1.760%); 672 KiB/s, 10 objects/s recovering 2023-10-09T05:09:35.445 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:35 smithi133 bash[105589]: cluster 2023-10-09T05:09:34.080961+0000 osd.7 (osd.7) 61 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T05:09:35.445 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:35 smithi133 bash[105589]: cluster 2023-10-09T05:09:34.081765+0000 osd.5 (osd.5) 109 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T05:09:35.445 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:35 smithi133 bash[105589]: cluster 2023-10-09T05:09:34.083196+0000 mon.a (mon.0) 780 : cluster 0 osdmap e147: 8 total, 8 up, 7 in 2023-10-09T05:09:35.445 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:35 smithi133 bash[105589]: cluster 2023-10-09T05:09:34.083431+0000 osd.7 (osd.7) 62 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T05:09:35.445 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:35 smithi133 bash[105589]: cluster 2023-10-09T05:09:34.083709+0000 osd.6 (osd.6) 73 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T05:09:35.445 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:35 smithi133 bash[105589]: cluster 2023-10-09T05:09:34.084222+0000 osd.5 (osd.5) 110 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T05:09:35.445 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:35 smithi133 bash[105589]: cluster 2023-10-09T05:09:34.087178+0000 osd.5 (osd.5) 111 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T05:09:35.446 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:35 smithi133 bash[105589]: cluster 2023-10-09T05:09:34.088161+0000 osd.6 (osd.6) 74 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T05:09:35.446 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:35 smithi133 bash[102858]: cluster 2023-10-09T05:09:33.743114+0000 mgr.y (mgr.25277) 628 : cluster 0 pgmap v587: 89 pgs: 1 active+recovering+undersized+remapped, 8 peering, 80 active+clean; 108 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 25 MiB/s wr, 404 op/s; 92/5226 objects misplaced (1.760%); 672 KiB/s, 10 objects/s recovering 2023-10-09T05:09:35.446 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:35 smithi133 bash[102858]: cluster 2023-10-09T05:09:34.080961+0000 osd.7 (osd.7) 61 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T05:09:35.446 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:35 smithi133 bash[102858]: cluster 2023-10-09T05:09:34.081765+0000 osd.5 (osd.5) 109 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T05:09:35.446 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:35 smithi133 bash[102858]: cluster 2023-10-09T05:09:34.083196+0000 mon.a (mon.0) 780 : cluster 0 osdmap e147: 8 total, 8 up, 7 in 2023-10-09T05:09:35.446 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:35 smithi133 bash[102858]: cluster 2023-10-09T05:09:34.083431+0000 osd.7 (osd.7) 62 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T05:09:35.446 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:35 smithi133 bash[102858]: cluster 2023-10-09T05:09:34.083709+0000 osd.6 (osd.6) 73 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T05:09:35.446 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:35 smithi133 bash[102858]: cluster 2023-10-09T05:09:34.084222+0000 osd.5 (osd.5) 110 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T05:09:35.446 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:35 smithi133 bash[102858]: cluster 2023-10-09T05:09:34.087178+0000 osd.5 (osd.5) 111 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T05:09:35.446 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:35 smithi133 bash[102858]: cluster 2023-10-09T05:09:34.088161+0000 osd.6 (osd.6) 74 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T05:09:35.499 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:35.527 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:35.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:35.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:35.630 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:35.662 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:35.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:35.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:35.765 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:35.789 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:35.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:35.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:35.892 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:35.921 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:35.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:35.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:35.984 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:09:36.024 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:36.053 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:36.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:36.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:36.157 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:36.166 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:36 smithi194 bash[92666]: cluster 2023-10-09T05:09:35.086782+0000 mon.a (mon.0) 781 : cluster 0 osdmap e148: 8 total, 8 up, 7 in 2023-10-09T05:09:36.166 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:36 smithi194 bash[92666]: cluster 2023-10-09T05:09:35.086848+0000 osd.6 (osd.6) 75 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.166 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:36 smithi194 bash[92666]: cluster 2023-10-09T05:09:35.087432+0000 osd.7 (osd.7) 63 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.166 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:36 smithi194 bash[92666]: cluster 2023-10-09T05:09:35.088229+0000 osd.4 (osd.4) 84 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.166 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:36 smithi194 bash[92666]: cluster 2023-10-09T05:09:35.089615+0000 osd.5 (osd.5) 112 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.166 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:36 smithi194 bash[92666]: cluster 2023-10-09T05:09:35.091699+0000 osd.6 (osd.6) 76 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.166 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:36 smithi194 bash[92666]: cluster 2023-10-09T05:09:35.092287+0000 osd.4 (osd.4) 85 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.166 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:36 smithi194 bash[92666]: cluster 2023-10-09T05:09:35.092566+0000 osd.7 (osd.7) 64 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.167 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:36 smithi194 bash[92666]: cluster 2023-10-09T05:09:35.093013+0000 osd.6 (osd.6) 77 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.167 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:36 smithi194 bash[92666]: cluster 2023-10-09T05:09:35.093525+0000 osd.5 (osd.5) 113 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.167 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:36 smithi194 bash[92666]: cluster 2023-10-09T05:09:35.095050+0000 osd.4 (osd.4) 86 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.167 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:36 smithi194 bash[92666]: cluster 2023-10-09T05:09:35.097199+0000 osd.4 (osd.4) 87 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.167 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:36 smithi194 bash[92666]: cluster 2023-10-09T05:09:35.097910+0000 osd.5 (osd.5) 114 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.167 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:36 smithi194 bash[92666]: cluster 2023-10-09T05:09:35.100334+0000 osd.5 (osd.5) 115 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.188 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:36.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:36.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:36.254 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 7 16 5261 5245 46.8255 21.6875 0.00176556 0.0213327 2023-10-09T05:09:36.292 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:36.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:36 smithi133 bash[105589]: cluster 2023-10-09T05:09:35.086782+0000 mon.a (mon.0) 781 : cluster 0 osdmap e148: 8 total, 8 up, 7 in 2023-10-09T05:09:36.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:36 smithi133 bash[105589]: cluster 2023-10-09T05:09:35.086848+0000 osd.6 (osd.6) 75 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:36 smithi133 bash[105589]: cluster 2023-10-09T05:09:35.087432+0000 osd.7 (osd.7) 63 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:36 smithi133 bash[105589]: cluster 2023-10-09T05:09:35.088229+0000 osd.4 (osd.4) 84 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:36 smithi133 bash[105589]: cluster 2023-10-09T05:09:35.089615+0000 osd.5 (osd.5) 112 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:36 smithi133 bash[105589]: cluster 2023-10-09T05:09:35.091699+0000 osd.6 (osd.6) 76 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:36 smithi133 bash[105589]: cluster 2023-10-09T05:09:35.092287+0000 osd.4 (osd.4) 85 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:36 smithi133 bash[105589]: cluster 2023-10-09T05:09:35.092566+0000 osd.7 (osd.7) 64 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:36 smithi133 bash[105589]: cluster 2023-10-09T05:09:35.093013+0000 osd.6 (osd.6) 77 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:36 smithi133 bash[105589]: cluster 2023-10-09T05:09:35.093525+0000 osd.5 (osd.5) 113 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:36 smithi133 bash[105589]: cluster 2023-10-09T05:09:35.095050+0000 osd.4 (osd.4) 86 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:36 smithi133 bash[105589]: cluster 2023-10-09T05:09:35.097199+0000 osd.4 (osd.4) 87 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:36 smithi133 bash[105589]: cluster 2023-10-09T05:09:35.097910+0000 osd.5 (osd.5) 114 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:36 smithi133 bash[105589]: cluster 2023-10-09T05:09:35.100334+0000 osd.5 (osd.5) 115 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:36 smithi133 bash[102858]: cluster 2023-10-09T05:09:35.086782+0000 mon.a (mon.0) 781 : cluster 0 osdmap e148: 8 total, 8 up, 7 in 2023-10-09T05:09:36.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:36 smithi133 bash[102858]: cluster 2023-10-09T05:09:35.086848+0000 osd.6 (osd.6) 75 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:36 smithi133 bash[102858]: cluster 2023-10-09T05:09:35.087432+0000 osd.7 (osd.7) 63 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:36 smithi133 bash[102858]: cluster 2023-10-09T05:09:35.088229+0000 osd.4 (osd.4) 84 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:36 smithi133 bash[102858]: cluster 2023-10-09T05:09:35.089615+0000 osd.5 (osd.5) 112 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:36 smithi133 bash[102858]: cluster 2023-10-09T05:09:35.091699+0000 osd.6 (osd.6) 76 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:36 smithi133 bash[102858]: cluster 2023-10-09T05:09:35.092287+0000 osd.4 (osd.4) 85 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:36 smithi133 bash[102858]: cluster 2023-10-09T05:09:35.092566+0000 osd.7 (osd.7) 64 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:36 smithi133 bash[102858]: cluster 2023-10-09T05:09:35.093013+0000 osd.6 (osd.6) 77 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:36 smithi133 bash[102858]: cluster 2023-10-09T05:09:35.093525+0000 osd.5 (osd.5) 113 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:36 smithi133 bash[102858]: cluster 2023-10-09T05:09:35.095050+0000 osd.4 (osd.4) 86 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:36 smithi133 bash[102858]: cluster 2023-10-09T05:09:35.097199+0000 osd.4 (osd.4) 87 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:36 smithi133 bash[102858]: cluster 2023-10-09T05:09:35.097910+0000 osd.5 (osd.5) 114 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.301 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:36 smithi133 bash[102858]: cluster 2023-10-09T05:09:35.100334+0000 osd.5 (osd.5) 115 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T05:09:36.326 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:36.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:36.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:36.432 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:36.443 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:09:36.443 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:09:36.444 INFO:tasks.thrashosds.thrasher:Adding osd 1 2023-10-09T05:09:36.444 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd in 1 2023-10-09T05:09:36.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:36.461 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:36.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:36.563 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:36.592 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:36.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:36.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:36.696 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:36.724 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:36.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:36.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:36.827 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:36.855 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:36.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:36.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:36.961 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:36.993 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:36.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:36.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:37.096 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:37.099 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:37 smithi194 bash[92666]: cluster 2023-10-09T05:09:35.744117+0000 mgr.y (mgr.25277) 629 : cluster 0 pgmap v590: 89 pgs: 2 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+remapped, 2 active+recovering+undersized+remapped, 6 peering, 76 active+clean; 303 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s; 1/14577 objects degraded (0.007%); 214/14577 objects misplaced (1.468%); 16 MiB/s, 255 objects/s recovering 2023-10-09T05:09:37.099 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:37 smithi194 bash[92666]: cluster 2023-10-09T05:09:36.093935+0000 mon.a (mon.0) 782 : cluster 3 Health check failed: Degraded data redundancy: 1/14577 objects degraded (0.007%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:09:37.099 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:37 smithi194 bash[92666]: cluster 2023-10-09T05:09:36.105847+0000 osd.5 (osd.5) 116 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T05:09:37.099 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:37 smithi194 bash[92666]: cluster 2023-10-09T05:09:36.106106+0000 osd.4 (osd.4) 88 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T05:09:37.099 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:37 smithi194 bash[92666]: cluster 2023-10-09T05:09:36.106126+0000 osd.7 (osd.7) 65 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T05:09:37.099 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:37 smithi194 bash[92666]: cluster 2023-10-09T05:09:36.108165+0000 mon.a (mon.0) 783 : cluster 0 osdmap e149: 8 total, 8 up, 7 in 2023-10-09T05:09:37.099 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:37 smithi194 bash[92666]: cluster 2023-10-09T05:09:36.108251+0000 osd.5 (osd.5) 117 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T05:09:37.099 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:37 smithi194 bash[92666]: cluster 2023-10-09T05:09:36.108619+0000 osd.4 (osd.4) 89 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T05:09:37.099 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:37 smithi194 bash[92666]: cluster 2023-10-09T05:09:36.108689+0000 osd.7 (osd.7) 66 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T05:09:37.100 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:37 smithi194 bash[92666]: cluster 2023-10-09T05:09:36.110671+0000 osd.5 (osd.5) 118 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T05:09:37.100 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:37 smithi194 bash[92666]: cluster 2023-10-09T05:09:36.110690+0000 osd.4 (osd.4) 90 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T05:09:37.117 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:37.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:37.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:37.220 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:37.230 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:37 smithi133 bash[102858]: cluster 2023-10-09T05:09:35.744117+0000 mgr.y (mgr.25277) 629 : cluster 0 pgmap v590: 89 pgs: 2 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+remapped, 2 active+recovering+undersized+remapped, 6 peering, 76 active+clean; 303 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s; 1/14577 objects degraded (0.007%); 214/14577 objects misplaced (1.468%); 16 MiB/s, 255 objects/s recovering 2023-10-09T05:09:37.230 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:37 smithi133 bash[102858]: cluster 2023-10-09T05:09:36.093935+0000 mon.a (mon.0) 782 : cluster 3 Health check failed: Degraded data redundancy: 1/14577 objects degraded (0.007%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:09:37.231 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:37 smithi133 bash[102858]: cluster 2023-10-09T05:09:36.105847+0000 osd.5 (osd.5) 116 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T05:09:37.231 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:37 smithi133 bash[102858]: cluster 2023-10-09T05:09:36.106106+0000 osd.4 (osd.4) 88 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T05:09:37.231 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:37 smithi133 bash[102858]: cluster 2023-10-09T05:09:36.106126+0000 osd.7 (osd.7) 65 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T05:09:37.231 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:37 smithi133 bash[102858]: cluster 2023-10-09T05:09:36.108165+0000 mon.a (mon.0) 783 : cluster 0 osdmap e149: 8 total, 8 up, 7 in 2023-10-09T05:09:37.254 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 8 16 7612 7596 59.3377 146.938 0.00748288 0.016843 2023-10-09T05:09:37.255 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:37.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:37.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:37.360 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:37.369 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:37 smithi194 bash[92666]: cluster 2023-10-09T05:09:36.113589+0000 osd.5 (osd.5) 119 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T05:09:37.388 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:37.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:37.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:37.493 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:37.502 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:37 smithi133 bash[102858]: cluster 2023-10-09T05:09:36.108251+0000 osd.5 (osd.5) 117 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T05:09:37.502 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:37 smithi133 bash[102858]: cluster 2023-10-09T05:09:36.108619+0000 osd.4 (osd.4) 89 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T05:09:37.502 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:37 smithi133 bash[102858]: cluster 2023-10-09T05:09:36.108689+0000 osd.7 (osd.7) 66 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T05:09:37.503 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:37 smithi133 bash[102858]: cluster 2023-10-09T05:09:36.110671+0000 osd.5 (osd.5) 118 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T05:09:37.503 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:37 smithi133 bash[102858]: cluster 2023-10-09T05:09:36.110690+0000 osd.4 (osd.4) 90 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T05:09:37.503 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:37 smithi133 bash[102858]: cluster 2023-10-09T05:09:36.113589+0000 osd.5 (osd.5) 119 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T05:09:37.503 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:37 smithi133 bash[105589]: cluster 2023-10-09T05:09:35.744117+0000 mgr.y (mgr.25277) 629 : cluster 0 pgmap v590: 89 pgs: 2 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+remapped, 2 active+recovering+undersized+remapped, 6 peering, 76 active+clean; 303 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s; 1/14577 objects degraded (0.007%); 214/14577 objects misplaced (1.468%); 16 MiB/s, 255 objects/s recovering 2023-10-09T05:09:37.503 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:37 smithi133 bash[105589]: cluster 2023-10-09T05:09:36.093935+0000 mon.a (mon.0) 782 : cluster 3 Health check failed: Degraded data redundancy: 1/14577 objects degraded (0.007%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:09:37.503 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:37 smithi133 bash[105589]: cluster 2023-10-09T05:09:36.105847+0000 osd.5 (osd.5) 116 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T05:09:37.503 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:37 smithi133 bash[105589]: cluster 2023-10-09T05:09:36.106106+0000 osd.4 (osd.4) 88 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T05:09:37.503 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:37 smithi133 bash[105589]: cluster 2023-10-09T05:09:36.106126+0000 osd.7 (osd.7) 65 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T05:09:37.503 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:37 smithi133 bash[105589]: cluster 2023-10-09T05:09:36.108165+0000 mon.a (mon.0) 783 : cluster 0 osdmap e149: 8 total, 8 up, 7 in 2023-10-09T05:09:37.503 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:37 smithi133 bash[105589]: cluster 2023-10-09T05:09:36.108251+0000 osd.5 (osd.5) 117 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T05:09:37.504 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:37 smithi133 bash[105589]: cluster 2023-10-09T05:09:36.108619+0000 osd.4 (osd.4) 89 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T05:09:37.504 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:37 smithi133 bash[105589]: cluster 2023-10-09T05:09:36.108689+0000 osd.7 (osd.7) 66 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T05:09:37.504 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:37 smithi133 bash[105589]: cluster 2023-10-09T05:09:36.110671+0000 osd.5 (osd.5) 118 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T05:09:37.504 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:37 smithi133 bash[105589]: cluster 2023-10-09T05:09:36.110690+0000 osd.4 (osd.4) 90 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T05:09:37.504 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:37 smithi133 bash[105589]: cluster 2023-10-09T05:09:36.113589+0000 osd.5 (osd.5) 119 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T05:09:37.518 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:37.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:37.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:37.621 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:37.650 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:37.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:37.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:37.755 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:37.786 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:37.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:37.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:37.889 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:37.917 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:37.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:37.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:38.019 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:38.050 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:38.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:38.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:38.152 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:38.179 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:38.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:38.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:38.254 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 9 15 10179 10164 70.5762 160.5 0.00740266 0.0141613 2023-10-09T05:09:38.282 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:38.321 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:38.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:38.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:38.426 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:38.435 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:38 smithi194 bash[92666]: cluster 2023-10-09T05:09:38.102438+0000 mon.a (mon.0) 784 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T05:09:38.454 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:38.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:38.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:38.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:38 smithi133 bash[105589]: cluster 2023-10-09T05:09:38.102438+0000 mon.a (mon.0) 784 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T05:09:38.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:38 smithi133 bash[102858]: cluster 2023-10-09T05:09:38.102438+0000 mon.a (mon.0) 784 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T05:09:38.557 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:38.585 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:38.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:38.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:38.689 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:38.716 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:38.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:38.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:38.820 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:38.853 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:38.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:38.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:38.957 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:38.985 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:38.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:38.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:39.087 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:39.116 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:39.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:39.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:39.220 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:39.252 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:39.254 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 10 16 12486 12470 77.9297 144.125 0.0095416 0.0128262 2023-10-09T05:09:39.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:39.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:39.356 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:39.364 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:39 smithi194 bash[92666]: cluster 2023-10-09T05:09:37.744604+0000 mgr.y (mgr.25277) 630 : cluster 0 pgmap v592: 89 pgs: 2 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+remapped, 2 active+recovering+undersized+remapped, 2 peering, 80 active+clean; 303 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 58 MiB/s wr, 924 op/s; 1/14577 objects degraded (0.007%); 214/14577 objects misplaced (1.468%); 14 MiB/s, 219 objects/s recovering 2023-10-09T05:09:39.386 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:39.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:39.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:39.446 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:39 smithi133 bash[105589]: cluster 2023-10-09T05:09:37.744604+0000 mgr.y (mgr.25277) 630 : cluster 0 pgmap v592: 89 pgs: 2 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+remapped, 2 active+recovering+undersized+remapped, 2 peering, 80 active+clean; 303 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 58 MiB/s wr, 924 op/s; 1/14577 objects degraded (0.007%); 214/14577 objects misplaced (1.468%); 14 MiB/s, 219 objects/s recovering 2023-10-09T05:09:39.446 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:39 smithi133 bash[102858]: cluster 2023-10-09T05:09:37.744604+0000 mgr.y (mgr.25277) 630 : cluster 0 pgmap v592: 89 pgs: 2 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+remapped, 2 active+recovering+undersized+remapped, 2 peering, 80 active+clean; 303 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 58 MiB/s wr, 924 op/s; 1/14577 objects degraded (0.007%); 214/14577 objects misplaced (1.468%); 14 MiB/s, 219 objects/s recovering 2023-10-09T05:09:39.491 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:39.521 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:39.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:39.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:39.628 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:39.655 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:39.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:39.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:39.759 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:39.786 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:39.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:39.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:39.889 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:39.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:39.922 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:39.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:40.023 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:40.050 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:40.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:40.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:40.153 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:40.179 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:40.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:40.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:40.213 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:40 smithi194 bash[92666]: cluster 2023-10-09T05:09:39.745618+0000 mgr.y (mgr.25277) 631 : cluster 0 pgmap v593: 89 pgs: 1 peering, 88 active+clean; 630 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s; 12 MiB/s, 199 objects/s recovering 2023-10-09T05:09:40.213 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:40 smithi194 bash[92666]: cluster 2023-10-09T05:09:40.158008+0000 mon.a (mon.0) 785 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/14577 objects degraded (0.007%), 1 pg degraded) 2023-10-09T05:09:40.255 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 11 15 14648 14633 83.1337 135.188 0.00520248 0.0120209 2023-10-09T05:09:40.282 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:40.292 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:40 smithi133 bash[102858]: cluster 2023-10-09T05:09:39.745618+0000 mgr.y (mgr.25277) 631 : cluster 0 pgmap v593: 89 pgs: 1 peering, 88 active+clean; 630 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s; 12 MiB/s, 199 objects/s recovering 2023-10-09T05:09:40.292 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:40 smithi133 bash[102858]: cluster 2023-10-09T05:09:40.158008+0000 mon.a (mon.0) 785 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/14577 objects degraded (0.007%), 1 pg degraded) 2023-10-09T05:09:40.292 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:40 smithi133 bash[105589]: cluster 2023-10-09T05:09:39.745618+0000 mgr.y (mgr.25277) 631 : cluster 0 pgmap v593: 89 pgs: 1 peering, 88 active+clean; 630 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s; 12 MiB/s, 199 objects/s recovering 2023-10-09T05:09:40.292 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:40 smithi133 bash[105589]: cluster 2023-10-09T05:09:40.158008+0000 mon.a (mon.0) 785 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/14577 objects degraded (0.007%), 1 pg degraded) 2023-10-09T05:09:40.308 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:40.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:40.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:40.411 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:40.442 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:40.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:40.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:40.546 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:40.579 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:40.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:40.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:40.684 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:40.711 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:40.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:40.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:40.816 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:40.848 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:40.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:40.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:40.951 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:40.972 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:40.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:40.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:41.075 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:41.103 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:41.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:41.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:41.207 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:41.237 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:41.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:41.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:41.255 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 12 16 16748 16732 87.1372 131.188 0.00545619 0.0114676 2023-10-09T05:09:41.341 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:41.366 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:41.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:41.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:41.474 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:41.504 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:41.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:41.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:41.607 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:41.631 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:41.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:41.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:41.736 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:41.769 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:41.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:41.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:41.873 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:41.899 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:41.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:41.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:42.004 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:42.034 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:42.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:42.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:42.137 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:42.160 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:42.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:42.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:42.255 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 13 16 18663 18647 89.6398 119.688 0.00483466 0.0110607 2023-10-09T05:09:42.264 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:42.296 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:42.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:42.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:42.399 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:42.430 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:42.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:42.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:42.534 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:42.565 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:42.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:42.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:42.672 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:42.698 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:42.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:42.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:42.802 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:42.803 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:42 smithi194 bash[92666]: cluster 2023-10-09T05:09:41.746215+0000 mgr.y (mgr.25277) 632 : cluster 0 pgmap v594: 89 pgs: 89 active+clean; 788 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s; 9.7 MiB/s, 156 objects/s recovering 2023-10-09T05:09:42.828 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:42.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:42.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:42.933 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:42.959 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:42.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:42.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:43.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:42 smithi133 bash[102858]: cluster 2023-10-09T05:09:41.746215+0000 mgr.y (mgr.25277) 632 : cluster 0 pgmap v594: 89 pgs: 89 active+clean; 788 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s; 9.7 MiB/s, 156 objects/s recovering 2023-10-09T05:09:43.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:42 smithi133 bash[105589]: cluster 2023-10-09T05:09:41.746215+0000 mgr.y (mgr.25277) 632 : cluster 0 pgmap v594: 89 pgs: 89 active+clean; 788 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s; 9.7 MiB/s, 156 objects/s recovering 2023-10-09T05:09:43.061 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:43.087 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:43.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:43.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:43.190 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:43.225 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:43.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:43.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:43.255 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 14 16 20250 20234 90.321 99.1875 0.0063933 0.0110643 2023-10-09T05:09:43.329 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:43.353 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:43.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:43.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:43.456 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:43.485 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:43.486 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:09:43.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:43.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:43.593 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:43.621 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:43.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:43.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:43.726 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:43.745 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:43.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:43.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:43.847 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:43.877 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:43.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:43.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:43.984 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:44.013 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:44.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:44.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:44.119 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:44.149 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:44.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:44.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:44.242 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:44 smithi133 bash[105589]: audit 2023-10-09T05:09:43.439916+0000 mon.a (mon.0) 786 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:09:44.242 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:44 smithi133 bash[105589]: cluster 2023-10-09T05:09:43.746626+0000 mgr.y (mgr.25277) 633 : cluster 0 pgmap v595: 89 pgs: 89 active+clean; 946 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 1.6 MiB/s, 24 objects/s recovering 2023-10-09T05:09:44.242 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:44 smithi133 bash[102858]: audit 2023-10-09T05:09:43.439916+0000 mon.a (mon.0) 786 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:09:44.242 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:44 smithi133 bash[102858]: cluster 2023-10-09T05:09:43.746626+0000 mgr.y (mgr.25277) 633 : cluster 0 pgmap v595: 89 pgs: 89 active+clean; 946 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 1.6 MiB/s, 24 objects/s recovering 2023-10-09T05:09:44.253 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:44.256 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 15 16 21973 21957 91.4781 107.688 0.00748925 0.0109243 2023-10-09T05:09:44.285 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:44.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:44.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:44.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:44 smithi194 bash[92666]: audit 2023-10-09T05:09:43.439916+0000 mon.a (mon.0) 786 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:09:44.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:44 smithi194 bash[92666]: cluster 2023-10-09T05:09:43.746626+0000 mgr.y (mgr.25277) 633 : cluster 0 pgmap v595: 89 pgs: 89 active+clean; 946 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 1.6 MiB/s, 24 objects/s recovering 2023-10-09T05:09:44.388 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:44.415 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:44.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:44.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:44.519 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:44.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:09:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:09:44] "GET /metrics HTTP/1.1" 200 36416 "" "Prometheus/2.43.0" 2023-10-09T05:09:44.552 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:44.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:44.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:44.656 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:44.683 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:44.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:44.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:44.787 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:44.812 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:44.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:44.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:44.915 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:44.941 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:44.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:44.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:45.045 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:45.076 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:45.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:45.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:45.181 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:45.205 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:45.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:45.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:45.255 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 16 16 23965 23949 93.5413 124.5 0.00591335 0.0106823 2023-10-09T05:09:45.309 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:45.335 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:45.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:45.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:45.437 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:45.464 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:45.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:45.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:45.568 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:45.593 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:45.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:45.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:45.697 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:45.726 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:45.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:45.729 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:45.830 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:45.857 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:45.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:45.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:45.961 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:45.986 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:45.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:45.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:46.091 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:46.127 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:46.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:46.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:46.234 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:46.255 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 17 16 25805 25789 94.8026 115 0.0124526 0.0105384 2023-10-09T05:09:46.269 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:46.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:46.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:46.372 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:46.399 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:46.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:46.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:46.502 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:46.530 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:46.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:46.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:46.636 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:46.667 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:46.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:46.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:46.770 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:46.797 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:46.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:46.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:46.902 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:46.933 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:46.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:46.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:47.039 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:47.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:46 smithi133 bash[102858]: cluster 2023-10-09T05:09:45.747367+0000 mgr.y (mgr.25277) 634 : cluster 0 pgmap v596: 89 pgs: 89 active+clean; 1.4 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 108 MiB/s wr, 1.73k op/s; 1.3 MiB/s, 21 objects/s recovering 2023-10-09T05:09:47.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:46 smithi133 bash[105589]: cluster 2023-10-09T05:09:45.747367+0000 mgr.y (mgr.25277) 634 : cluster 0 pgmap v596: 89 pgs: 89 active+clean; 1.4 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 108 MiB/s wr, 1.73k op/s; 1.3 MiB/s, 21 objects/s recovering 2023-10-09T05:09:47.068 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:47.069 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:46 smithi194 bash[92666]: cluster 2023-10-09T05:09:45.747367+0000 mgr.y (mgr.25277) 634 : cluster 0 pgmap v596: 89 pgs: 89 active+clean; 1.4 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 108 MiB/s wr, 1.73k op/s; 1.3 MiB/s, 21 objects/s recovering 2023-10-09T05:09:47.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:47.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:47.171 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:47.194 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:47.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:47.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:47.255 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 18 16 27332 27316 94.837 95.4375 0.00298553 0.0104996 2023-10-09T05:09:47.297 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:47.334 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:47.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:47.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:47.437 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:47.463 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:47.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:47.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:47.569 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:47.596 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:47.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:47.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:47.700 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:47.735 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:47.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:47.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:47.840 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:47.865 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:47.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:47.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:47.970 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:47.993 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:47.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:47.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:48.097 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:48.128 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:48.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:48.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:48.234 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:48.256 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 19 16 29036 29020 95.4475 106.5 0.00386954 0.0104679 2023-10-09T05:09:48.260 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:48.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:48.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:48.364 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:48.392 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:48.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:48.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:48.500 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:48.525 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:48.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:48.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:48.630 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:48.655 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:48.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:48.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:48.759 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:48.801 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:48.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:48.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:48.905 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:48.933 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:48.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:48.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:49.037 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:49.046 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:48 smithi194 bash[92666]: cluster 2023-10-09T05:09:47.747897+0000 mgr.y (mgr.25277) 635 : cluster 0 pgmap v597: 89 pgs: 89 active+clean; 1.4 GiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s; 1.2 MiB/s, 18 objects/s recovering 2023-10-09T05:09:49.047 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:48 smithi194 bash[92666]: audit 2023-10-09T05:09:48.588757+0000 mon.c (mon.2) 62 : audit 1 from='client.? 172.21.15.133:0/2555864877' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:09:49.047 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:48 smithi194 bash[92666]: audit 2023-10-09T05:09:48.589700+0000 mon.a (mon.0) 787 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:09:49.047 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:48 smithi194 bash[92666]: audit 2023-10-09T05:09:48.777544+0000 mon.a (mon.0) 788 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:09:49.063 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:49.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:49.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:49.169 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:49.201 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:49.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:49.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:49.247 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is unset 2023-10-09T05:09:49.256 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T05:09:49.256459+0000 min lat: 0.00144352 max lat: 2.06455 avg lat: 0.0104351 2023-10-09T05:09:49.256 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:09:49.256 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 20 16 30660 30644 95.7496 101.5 0.0117717 0.0104351 2023-10-09T05:09:49.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:48 smithi133 bash[102858]: cluster 2023-10-09T05:09:47.747897+0000 mgr.y (mgr.25277) 635 : cluster 0 pgmap v597: 89 pgs: 89 active+clean; 1.4 GiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s; 1.2 MiB/s, 18 objects/s recovering 2023-10-09T05:09:49.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:48 smithi133 bash[102858]: audit 2023-10-09T05:09:48.588757+0000 mon.c (mon.2) 62 : audit 1 from='client.? 172.21.15.133:0/2555864877' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:09:49.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:48 smithi133 bash[102858]: audit 2023-10-09T05:09:48.589700+0000 mon.a (mon.0) 787 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:09:49.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:48 smithi133 bash[102858]: audit 2023-10-09T05:09:48.777544+0000 mon.a (mon.0) 788 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:09:49.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:48 smithi133 bash[105589]: cluster 2023-10-09T05:09:47.747897+0000 mgr.y (mgr.25277) 635 : cluster 0 pgmap v597: 89 pgs: 89 active+clean; 1.4 GiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s; 1.2 MiB/s, 18 objects/s recovering 2023-10-09T05:09:49.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:48 smithi133 bash[105589]: audit 2023-10-09T05:09:48.588757+0000 mon.c (mon.2) 62 : audit 1 from='client.? 172.21.15.133:0/2555864877' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:09:49.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:48 smithi133 bash[105589]: audit 2023-10-09T05:09:48.589700+0000 mon.a (mon.0) 787 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:09:49.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:48 smithi133 bash[105589]: audit 2023-10-09T05:09:48.777544+0000 mon.a (mon.0) 788 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:09:49.307 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:49.331 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:49.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:49.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:49.434 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:49.465 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:49.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:49.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:49.571 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:49.597 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:49.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:49.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:49.641 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:09:49.697 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:09:49.700 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:49.706 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:09:49.707 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:09:49.731 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:49.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:49.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:49.736 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:09:49.784 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:09:49.819 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:09:49.835 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:49.862 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:09:49.866 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:49.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:49.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:49.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:49 smithi194 bash[92666]: cluster 2023-10-09T05:09:49.233321+0000 mon.a (mon.0) 789 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:09:49.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:49 smithi194 bash[92666]: cluster 2023-10-09T05:09:49.235744+0000 osd.6 (osd.6) 78 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T05:09:49.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:49 smithi194 bash[92666]: audit 2023-10-09T05:09:49.238589+0000 mon.a (mon.0) 790 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:09:49.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:49 smithi194 bash[92666]: cluster 2023-10-09T05:09:49.238691+0000 mon.a (mon.0) 791 : cluster 0 osdmap e150: 8 total, 8 up, 7 in 2023-10-09T05:09:49.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:49 smithi194 bash[92666]: cluster 2023-10-09T05:09:49.247144+0000 osd.7 (osd.7) 67 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T05:09:49.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:49 smithi194 bash[92666]: audit 2023-10-09T05:09:49.588456+0000 mon.a (mon.0) 792 : audit 0 from='client.? 172.21.15.133:0/2151437733' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:09:49.898 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:09:49.934 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:09:49.940 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:09:49.940 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:09:49.940 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:09:49.940 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:09:49.940 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:09:49.940 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:09:49.940 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:09:49.941 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:09:49.969 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:49.995 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:50.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:50.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:50.023 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:49 smithi133 bash[105589]: cluster 2023-10-09T05:09:49.233321+0000 mon.a (mon.0) 789 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:09:50.023 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:49 smithi133 bash[105589]: cluster 2023-10-09T05:09:49.235744+0000 osd.6 (osd.6) 78 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T05:09:50.023 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:49 smithi133 bash[105589]: audit 2023-10-09T05:09:49.238589+0000 mon.a (mon.0) 790 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:09:50.024 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:49 smithi133 bash[105589]: cluster 2023-10-09T05:09:49.238691+0000 mon.a (mon.0) 791 : cluster 0 osdmap e150: 8 total, 8 up, 7 in 2023-10-09T05:09:50.024 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:49 smithi133 bash[105589]: cluster 2023-10-09T05:09:49.247144+0000 osd.7 (osd.7) 67 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T05:09:50.024 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:49 smithi133 bash[105589]: audit 2023-10-09T05:09:49.588456+0000 mon.a (mon.0) 792 : audit 0 from='client.? 172.21.15.133:0/2151437733' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:09:50.024 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:49 smithi133 bash[102858]: cluster 2023-10-09T05:09:49.233321+0000 mon.a (mon.0) 789 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:09:50.025 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:49 smithi133 bash[102858]: cluster 2023-10-09T05:09:49.235744+0000 osd.6 (osd.6) 78 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T05:09:50.025 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:49 smithi133 bash[102858]: audit 2023-10-09T05:09:49.238589+0000 mon.a (mon.0) 790 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:09:50.025 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:49 smithi133 bash[102858]: cluster 2023-10-09T05:09:49.238691+0000 mon.a (mon.0) 791 : cluster 0 osdmap e150: 8 total, 8 up, 7 in 2023-10-09T05:09:50.025 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:49 smithi133 bash[102858]: cluster 2023-10-09T05:09:49.247144+0000 osd.7 (osd.7) 67 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T05:09:50.025 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:49 smithi133 bash[102858]: audit 2023-10-09T05:09:49.588456+0000 mon.a (mon.0) 792 : audit 0 from='client.? 172.21.15.133:0/2151437733' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:09:50.101 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:50.129 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:50.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:50.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:50.234 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:50.256 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 21 16 32177 32161 95.7045 94.8125 0.0105781 0.0104418 2023-10-09T05:09:50.263 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:50.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:50.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:50.368 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:50.401 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:50.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:50.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:50.504 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:50.531 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:50.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:50.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:50.639 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:50.667 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:50.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:50.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:50.773 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:50.800 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:50.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:50.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:50.904 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:50.929 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:50.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:50.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:51.032 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:51.042 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:50 smithi133 bash[105589]: cluster 2023-10-09T05:09:49.236322+0000 osd.5 (osd.5) 120 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T05:09:51.042 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:50 smithi133 bash[105589]: cluster 2023-10-09T05:09:49.748418+0000 mgr.y (mgr.25277) 636 : cluster 0 pgmap v599: 89 pgs: 89 active+clean; 1.8 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-10-09T05:09:51.042 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:50 smithi133 bash[102858]: cluster 2023-10-09T05:09:49.236322+0000 osd.5 (osd.5) 120 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T05:09:51.042 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:50 smithi133 bash[102858]: cluster 2023-10-09T05:09:49.748418+0000 mgr.y (mgr.25277) 636 : cluster 0 pgmap v599: 89 pgs: 89 active+clean; 1.8 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-10-09T05:09:51.059 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:51.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:51.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:51.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:50 smithi194 bash[92666]: cluster 2023-10-09T05:09:49.236322+0000 osd.5 (osd.5) 120 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T05:09:51.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:50 smithi194 bash[92666]: cluster 2023-10-09T05:09:49.748418+0000 mgr.y (mgr.25277) 636 : cluster 0 pgmap v599: 89 pgs: 89 active+clean; 1.8 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-10-09T05:09:51.163 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:51.196 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:51.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:51.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:51.257 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 22 16 33856 33840 96.1237 104.938 0.0126886 0.010397 2023-10-09T05:09:51.300 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:51.323 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:51.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:51.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:51.427 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:51.454 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:51.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:51.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:51.558 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:51.585 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:51.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:51.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:51.689 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:51.716 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:51.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:51.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:51.819 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:51.846 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:51.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:51.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:51.951 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:51.985 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:51.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:51.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:52.090 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:52.121 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:52.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:52.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:52.225 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:52.257 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 23 16 35481 35465 96.3598 101.562 0.00914659 0.0103708 2023-10-09T05:09:52.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:52.274 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:52.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:52.375 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:52.413 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:52.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:52.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:52.517 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:52.550 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:52.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:52.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:52.652 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:52.682 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:52.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:52.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:52.785 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:52.823 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:52.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:52.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:52.926 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:52.948 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:52.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:52.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:53.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:52 smithi133 bash[102858]: cluster 2023-10-09T05:09:51.749012+0000 mgr.y (mgr.25277) 637 : cluster 0 pgmap v600: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-10-09T05:09:53.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:52 smithi133 bash[105589]: cluster 2023-10-09T05:09:51.749012+0000 mgr.y (mgr.25277) 637 : cluster 0 pgmap v600: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-10-09T05:09:53.051 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:53.059 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:52 smithi194 bash[92666]: cluster 2023-10-09T05:09:51.749012+0000 mgr.y (mgr.25277) 637 : cluster 0 pgmap v600: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-10-09T05:09:53.082 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:53.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:53.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:53.185 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:53.213 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:53.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:53.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:53.257 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 24 16 37082 37066 96.5137 100.062 0.00883816 0.0103555 2023-10-09T05:09:53.315 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:53.343 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:53.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:53.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:53.448 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:53.465 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T05:09:53.487 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:53.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:53.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:53.592 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:53.618 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:53.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:53.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:53.725 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:53.754 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:53.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:53.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:53.860 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:53.894 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:53.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:53.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:53.998 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:54.027 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:54.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:54.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:54.132 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:54.172 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:54.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:54.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:54.257 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 25 16 38566 38550 96.3628 92.75 0.00762271 0.010371 2023-10-09T05:09:54.278 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:54.302 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:54.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:54.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:54.406 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:54.433 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:54.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:54.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:54.537 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:54.546 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:54 smithi133 bash[105589]: cluster 2023-10-09T05:09:53.749632+0000 mgr.y (mgr.25277) 638 : cluster 0 pgmap v601: 89 pgs: 89 active+clean; 2.0 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-10-09T05:09:54.546 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:54 smithi133 bash[102858]: cluster 2023-10-09T05:09:53.749632+0000 mgr.y (mgr.25277) 638 : cluster 0 pgmap v601: 89 pgs: 89 active+clean; 2.0 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-10-09T05:09:54.546 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:09:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:09:54] "GET /metrics HTTP/1.1" 200 36416 "" "Prometheus/2.43.0" 2023-10-09T05:09:54.561 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:54.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:54.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:54.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:54 smithi194 bash[92666]: cluster 2023-10-09T05:09:53.749632+0000 mgr.y (mgr.25277) 638 : cluster 0 pgmap v601: 89 pgs: 89 active+clean; 2.0 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-10-09T05:09:54.665 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:54.691 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:54.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:54.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:54.796 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:54.820 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:54.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:54.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:54.923 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:54.956 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:54.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:54.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:55.062 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:55.090 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:55.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:55.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:55.175 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:09:55.194 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:55.224 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:55.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:55.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:55.257 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 26 16 40065 40049 96.2596 93.6875 0.00938034 0.0103821 2023-10-09T05:09:55.328 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:55.365 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:55.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:55.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:55.468 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:55.472 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:09:55.495 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:55.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:55.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:55.601 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:55.627 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:55.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:55.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:55.731 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:55.777 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:55.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:55.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:55.881 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:55.914 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:55.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:55.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:56.016 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:56.045 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:56.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:56.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:56.151 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:56.175 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:56.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:56.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:56.257 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 27 16 41453 41437 95.9071 86.75 0.00764177 0.0104198 2023-10-09T05:09:56.279 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:56.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:56 smithi133 bash[102858]: debug 2023-10-09T05:09:56.028+0000 7f5eb4d1d700 -1 mon.a@0(leader).osd e150 definitely_dead 0 2023-10-09T05:09:56.312 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:56.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:56.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:56.417 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:56.443 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:56.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:56.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:56.548 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:56.583 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:56.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:56.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:56.688 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:56.713 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:09:56.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:56.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:09:56.819 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:56.833 INFO:teuthology.orchestra.run.smithi133.stderr:marked in osd.1. 2023-10-09T05:09:56.853 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:56.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:56.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:56.955 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:56.983 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:56.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:56.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:57.020 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:56 smithi133 bash[105589]: cluster 2023-10-09T05:09:55.751266+0000 mgr.y (mgr.25277) 639 : cluster 0 pgmap v602: 89 pgs: 89 active+clean; 2.4 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-09T05:09:57.021 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:56 smithi133 bash[105589]: audit 2023-10-09T05:09:56.030579+0000 mon.c (mon.2) 63 : audit 0 from='client.? 172.21.15.133:0/4260307332' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T05:09:57.021 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:56 smithi133 bash[105589]: audit 2023-10-09T05:09:56.030977+0000 mon.a (mon.0) 793 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T05:09:57.021 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:56 smithi133 bash[102858]: cluster 2023-10-09T05:09:55.751266+0000 mgr.y (mgr.25277) 639 : cluster 0 pgmap v602: 89 pgs: 89 active+clean; 2.4 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-09T05:09:57.021 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:56 smithi133 bash[102858]: audit 2023-10-09T05:09:56.030579+0000 mon.c (mon.2) 63 : audit 0 from='client.? 172.21.15.133:0/4260307332' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T05:09:57.021 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:56 smithi133 bash[102858]: audit 2023-10-09T05:09:56.030977+0000 mon.a (mon.0) 793 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T05:09:57.087 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:57.099 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:56 smithi194 bash[92666]: cluster 2023-10-09T05:09:55.751266+0000 mgr.y (mgr.25277) 639 : cluster 0 pgmap v602: 89 pgs: 89 active+clean; 2.4 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-09T05:09:57.099 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:56 smithi194 bash[92666]: audit 2023-10-09T05:09:56.030579+0000 mon.c (mon.2) 63 : audit 0 from='client.? 172.21.15.133:0/4260307332' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T05:09:57.099 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:56 smithi194 bash[92666]: audit 2023-10-09T05:09:56.030977+0000 mon.a (mon.0) 793 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T05:09:57.112 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:57.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:57.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:57.214 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:57.247 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:57.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:57.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:57.257 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 28 16 42303 42287 94.379 53.125 0.0292452 0.0104293 2023-10-09T05:09:57.350 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:57.378 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:57.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:57.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:57.481 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:57.516 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:57.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:57.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:57.619 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:57.649 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:57.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:57.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:57.753 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:57.785 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:57.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:57.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:57.889 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:57.896 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:57 smithi133 bash[105589]: cluster 2023-10-09T05:09:56.818439+0000 osd.5 (osd.5) 121 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T05:09:57.896 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:57 smithi133 bash[105589]: audit 2023-10-09T05:09:56.820348+0000 mon.a (mon.0) 794 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T05:09:57.897 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:57 smithi133 bash[105589]: cluster 2023-10-09T05:09:56.820551+0000 mon.a (mon.0) 795 : cluster 0 osdmap e151: 8 total, 8 up, 8 in 2023-10-09T05:09:57.897 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:57 smithi133 bash[105589]: cluster 2023-10-09T05:09:56.822892+0000 osd.4 (osd.4) 91 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T05:09:57.897 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:57 smithi133 bash[105589]: cluster 2023-10-09T05:09:56.828005+0000 osd.7 (osd.7) 68 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T05:09:57.897 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:57 smithi133 bash[102858]: cluster 2023-10-09T05:09:56.818439+0000 osd.5 (osd.5) 121 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T05:09:57.897 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:57 smithi133 bash[102858]: audit 2023-10-09T05:09:56.820348+0000 mon.a (mon.0) 794 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T05:09:57.897 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:57 smithi133 bash[102858]: cluster 2023-10-09T05:09:56.820551+0000 mon.a (mon.0) 795 : cluster 0 osdmap e151: 8 total, 8 up, 8 in 2023-10-09T05:09:57.897 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:57 smithi133 bash[102858]: cluster 2023-10-09T05:09:56.822892+0000 osd.4 (osd.4) 91 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T05:09:57.897 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:57 smithi133 bash[102858]: cluster 2023-10-09T05:09:56.828005+0000 osd.7 (osd.7) 68 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T05:09:57.916 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:57.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:57.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:57.940 INFO:tasks.thrashosds.thrasher:Added osd 1 2023-10-09T05:09:58.020 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:58.048 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:58.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:58.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:58.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:57 smithi194 bash[92666]: cluster 2023-10-09T05:09:56.818439+0000 osd.5 (osd.5) 121 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T05:09:58.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:57 smithi194 bash[92666]: audit 2023-10-09T05:09:56.820348+0000 mon.a (mon.0) 794 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T05:09:58.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:57 smithi194 bash[92666]: cluster 2023-10-09T05:09:56.820551+0000 mon.a (mon.0) 795 : cluster 0 osdmap e151: 8 total, 8 up, 8 in 2023-10-09T05:09:58.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:57 smithi194 bash[92666]: cluster 2023-10-09T05:09:56.822892+0000 osd.4 (osd.4) 91 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T05:09:58.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:57 smithi194 bash[92666]: cluster 2023-10-09T05:09:56.828005+0000 osd.7 (osd.7) 68 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T05:09:58.152 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:58.182 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:58.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:58.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:58.258 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 29 16 42303 42287 91.1243 0 - 0.0104293 2023-10-09T05:09:58.287 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:58.320 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:58.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:58.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:58.424 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:58.452 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:09:58.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:58.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:09:58.555 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:58.585 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:58.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:58.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:58.688 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:58.724 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:58.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:58.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:58.828 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:58.868 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:09:58.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:58.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:09:58.975 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:59.011 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:59.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:59.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:59.114 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:59.122 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:58 smithi194 bash[92666]: cluster 2023-10-09T05:09:57.751689+0000 mgr.y (mgr.25277) 640 : cluster 0 pgmap v604: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-09T05:09:59.122 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:58 smithi194 bash[92666]: cluster 2023-10-09T05:09:57.830328+0000 mon.a (mon.0) 796 : cluster 0 osdmap e152: 8 total, 8 up, 8 in 2023-10-09T05:09:59.122 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:58 smithi194 bash[92666]: cluster 2023-10-09T05:09:57.830340+0000 osd.4 (osd.4) 92 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T05:09:59.122 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:58 smithi194 bash[92666]: cluster 2023-10-09T05:09:57.830973+0000 osd.6 (osd.6) 79 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T05:09:59.123 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:58 smithi194 bash[92666]: cluster 2023-10-09T05:09:57.831041+0000 osd.7 (osd.7) 69 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T05:09:59.123 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:58 smithi194 bash[92666]: cluster 2023-10-09T05:09:57.832001+0000 osd.5 (osd.5) 122 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T05:09:59.123 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:58 smithi194 bash[92666]: cluster 2023-10-09T05:09:57.833549+0000 osd.4 (osd.4) 93 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T05:09:59.123 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:58 smithi194 bash[92666]: cluster 2023-10-09T05:09:57.835052+0000 osd.4 (osd.4) 94 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T05:09:59.123 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:58 smithi194 bash[92666]: cluster 2023-10-09T05:09:57.836574+0000 osd.5 (osd.5) 123 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T05:09:59.123 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:58 smithi194 bash[92666]: cluster 2023-10-09T05:09:57.836671+0000 osd.7 (osd.7) 70 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T05:09:59.123 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:58 smithi194 bash[92666]: cluster 2023-10-09T05:09:57.838429+0000 osd.5 (osd.5) 124 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T05:09:59.123 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:58 smithi194 bash[92666]: audit 2023-10-09T05:09:58.440452+0000 mon.a (mon.0) 797 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:09:59.141 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:59.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:59.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:59.248 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:59.257 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 30 16 42444 42428 88.3806 4.40625 0.00798833 0.0113062 2023-10-09T05:09:59.276 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:09:59.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:59.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:09:59.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:58 smithi133 bash[105589]: cluster 2023-10-09T05:09:57.751689+0000 mgr.y (mgr.25277) 640 : cluster 0 pgmap v604: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-09T05:09:59.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:58 smithi133 bash[105589]: cluster 2023-10-09T05:09:57.830328+0000 mon.a (mon.0) 796 : cluster 0 osdmap e152: 8 total, 8 up, 8 in 2023-10-09T05:09:59.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:58 smithi133 bash[105589]: cluster 2023-10-09T05:09:57.830340+0000 osd.4 (osd.4) 92 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T05:09:59.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:58 smithi133 bash[105589]: cluster 2023-10-09T05:09:57.830973+0000 osd.6 (osd.6) 79 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T05:09:59.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:58 smithi133 bash[105589]: cluster 2023-10-09T05:09:57.831041+0000 osd.7 (osd.7) 69 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T05:09:59.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:58 smithi133 bash[105589]: cluster 2023-10-09T05:09:57.832001+0000 osd.5 (osd.5) 122 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T05:09:59.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:58 smithi133 bash[105589]: cluster 2023-10-09T05:09:57.833549+0000 osd.4 (osd.4) 93 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T05:09:59.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:58 smithi133 bash[105589]: cluster 2023-10-09T05:09:57.835052+0000 osd.4 (osd.4) 94 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T05:09:59.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:58 smithi133 bash[105589]: cluster 2023-10-09T05:09:57.836574+0000 osd.5 (osd.5) 123 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T05:09:59.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:58 smithi133 bash[105589]: cluster 2023-10-09T05:09:57.836671+0000 osd.7 (osd.7) 70 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T05:09:59.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:58 smithi133 bash[105589]: cluster 2023-10-09T05:09:57.838429+0000 osd.5 (osd.5) 124 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T05:09:59.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:58 smithi133 bash[105589]: audit 2023-10-09T05:09:58.440452+0000 mon.a (mon.0) 797 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:09:59.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:58 smithi133 bash[102858]: cluster 2023-10-09T05:09:57.751689+0000 mgr.y (mgr.25277) 640 : cluster 0 pgmap v604: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-09T05:09:59.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:58 smithi133 bash[102858]: cluster 2023-10-09T05:09:57.830328+0000 mon.a (mon.0) 796 : cluster 0 osdmap e152: 8 total, 8 up, 8 in 2023-10-09T05:09:59.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:58 smithi133 bash[102858]: cluster 2023-10-09T05:09:57.830340+0000 osd.4 (osd.4) 92 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T05:09:59.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:58 smithi133 bash[102858]: cluster 2023-10-09T05:09:57.830973+0000 osd.6 (osd.6) 79 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T05:09:59.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:58 smithi133 bash[102858]: cluster 2023-10-09T05:09:57.831041+0000 osd.7 (osd.7) 69 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T05:09:59.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:58 smithi133 bash[102858]: cluster 2023-10-09T05:09:57.832001+0000 osd.5 (osd.5) 122 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T05:09:59.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:58 smithi133 bash[102858]: cluster 2023-10-09T05:09:57.833549+0000 osd.4 (osd.4) 93 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T05:09:59.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:58 smithi133 bash[102858]: cluster 2023-10-09T05:09:57.835052+0000 osd.4 (osd.4) 94 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T05:09:59.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:58 smithi133 bash[102858]: cluster 2023-10-09T05:09:57.836574+0000 osd.5 (osd.5) 123 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T05:09:59.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:58 smithi133 bash[102858]: cluster 2023-10-09T05:09:57.836671+0000 osd.7 (osd.7) 70 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T05:09:59.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:58 smithi133 bash[102858]: cluster 2023-10-09T05:09:57.838429+0000 osd.5 (osd.5) 124 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T05:09:59.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:58 smithi133 bash[102858]: audit 2023-10-09T05:09:58.440452+0000 mon.a (mon.0) 797 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:09:59.386 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:59.416 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:09:59.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:59.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:09:59.521 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:59.551 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:09:59.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:59.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:09:59.657 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:59.688 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:59.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:59.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:09:59.792 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:59.817 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:09:59.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:59.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:09:59.920 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:59.946 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:09:59.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:09:59.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:00.049 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:00.058 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:59 smithi194 bash[92666]: cluster 2023-10-09T05:09:58.831211+0000 osd.5 (osd.5) 125 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T05:10:00.058 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:59 smithi194 bash[92666]: cluster 2023-10-09T05:09:58.835244+0000 mon.a (mon.0) 798 : cluster 0 osdmap e153: 8 total, 8 up, 8 in 2023-10-09T05:10:00.059 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:59 smithi194 bash[92666]: cluster 2023-10-09T05:09:58.835377+0000 osd.7 (osd.7) 71 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T05:10:00.059 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:59 smithi194 bash[92666]: cluster 2023-10-09T05:09:58.836298+0000 osd.5 (osd.5) 126 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T05:10:00.059 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:59 smithi194 bash[92666]: cluster 2023-10-09T05:09:58.837915+0000 osd.5 (osd.5) 127 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T05:10:00.059 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:59 smithi194 bash[92666]: cluster 2023-10-09T05:09:58.838763+0000 osd.7 (osd.7) 72 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T05:10:00.059 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:09:59 smithi194 bash[92666]: cluster 2023-10-09T05:09:58.841633+0000 osd.7 (osd.7) 73 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T05:10:00.078 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:00.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:00.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:00.183 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:00.191 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:59 smithi133 bash[102858]: cluster 2023-10-09T05:09:58.831211+0000 osd.5 (osd.5) 125 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T05:10:00.191 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:59 smithi133 bash[102858]: cluster 2023-10-09T05:09:58.835244+0000 mon.a (mon.0) 798 : cluster 0 osdmap e153: 8 total, 8 up, 8 in 2023-10-09T05:10:00.192 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:59 smithi133 bash[102858]: cluster 2023-10-09T05:09:58.835377+0000 osd.7 (osd.7) 71 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T05:10:00.192 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:59 smithi133 bash[102858]: cluster 2023-10-09T05:09:58.836298+0000 osd.5 (osd.5) 126 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T05:10:00.192 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:59 smithi133 bash[102858]: cluster 2023-10-09T05:09:58.837915+0000 osd.5 (osd.5) 127 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T05:10:00.192 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:59 smithi133 bash[102858]: cluster 2023-10-09T05:09:58.838763+0000 osd.7 (osd.7) 72 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T05:10:00.192 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:09:59 smithi133 bash[102858]: cluster 2023-10-09T05:09:58.841633+0000 osd.7 (osd.7) 73 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T05:10:00.192 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:59 smithi133 bash[105589]: cluster 2023-10-09T05:09:58.831211+0000 osd.5 (osd.5) 125 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T05:10:00.193 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:59 smithi133 bash[105589]: cluster 2023-10-09T05:09:58.835244+0000 mon.a (mon.0) 798 : cluster 0 osdmap e153: 8 total, 8 up, 8 in 2023-10-09T05:10:00.193 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:59 smithi133 bash[105589]: cluster 2023-10-09T05:09:58.835377+0000 osd.7 (osd.7) 71 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T05:10:00.193 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:59 smithi133 bash[105589]: cluster 2023-10-09T05:09:58.836298+0000 osd.5 (osd.5) 126 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T05:10:00.193 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:59 smithi133 bash[105589]: cluster 2023-10-09T05:09:58.837915+0000 osd.5 (osd.5) 127 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T05:10:00.193 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:59 smithi133 bash[105589]: cluster 2023-10-09T05:09:58.838763+0000 osd.7 (osd.7) 72 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T05:10:00.193 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:09:59 smithi133 bash[105589]: cluster 2023-10-09T05:09:58.841633+0000 osd.7 (osd.7) 73 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T05:10:00.212 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:00.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:00.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:00.257 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 31 16 43374 43358 87.4045 58.125 0.0104736 0.0114336 2023-10-09T05:10:00.317 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:00.343 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:00.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:00.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:00.447 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:00.453 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:00.459 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:00.459 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:00.481 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:00.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:00.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:00.584 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:00.611 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:00.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:00.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:00.718 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:00.747 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:00.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:00.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:00.857 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:00.885 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:00.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:00.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:00.989 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:01.132 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:00 smithi194 bash[92666]: cluster 2023-10-09T05:09:59.754039+0000 mgr.y (mgr.25277) 641 : cluster 0 pgmap v607: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 82 active+clean; 2.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s; 15893/125079 objects degraded (12.706%); 11 KiB/s, 0 objects/s recovering 2023-10-09T05:10:01.133 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:00 smithi194 bash[92666]: cluster 2023-10-09T05:09:59.837779+0000 mon.a (mon.0) 799 : cluster 3 Health check failed: Degraded data redundancy: 15893/125079 objects degraded (12.706%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:01.133 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:00 smithi194 bash[92666]: cluster 2023-10-09T05:10:00.000125+0000 mon.a (mon.0) 800 : cluster 3 Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 15893/125079 objects degraded (12.706%), 6 pgs degraded 2023-10-09T05:10:01.133 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:00 smithi194 bash[92666]: cluster 2023-10-09T05:10:00.000166+0000 mon.a (mon.0) 801 : cluster 3 [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-10-09T05:10:01.133 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:00 smithi194 bash[92666]: cluster 2023-10-09T05:10:00.000184+0000 mon.a (mon.0) 802 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 15893/125079 objects degraded (12.706%), 6 pgs degraded 2023-10-09T05:10:01.133 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:00 smithi194 bash[92666]: cluster 2023-10-09T05:10:00.000199+0000 mon.a (mon.0) 803 : cluster 3 pg 9.3 is active+recovery_wait+undersized+degraded+remapped, acting [7,3] 2023-10-09T05:10:01.134 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:00 smithi194 bash[92666]: cluster 2023-10-09T05:10:00.000214+0000 mon.a (mon.0) 804 : cluster 3 pg 9.6 is active+recovery_wait+undersized+degraded+remapped, acting [5,2] 2023-10-09T05:10:01.134 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:00 smithi194 bash[92666]: cluster 2023-10-09T05:10:00.000290+0000 mon.a (mon.0) 805 : cluster 3 pg 9.a is active+recovery_wait+undersized+degraded+remapped, acting [5,7] 2023-10-09T05:10:01.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:00 smithi194 bash[92666]: cluster 2023-10-09T05:10:00.000306+0000 mon.a (mon.0) 806 : cluster 3 pg 9.d is active+recovery_wait+undersized+degraded+remapped, acting [7,5] 2023-10-09T05:10:01.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:00 smithi194 bash[92666]: cluster 2023-10-09T05:10:00.000334+0000 mon.a (mon.0) 807 : cluster 3 pg 9.e is active+recovery_wait+undersized+degraded+remapped, acting [7,4] 2023-10-09T05:10:01.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:00 smithi194 bash[92666]: cluster 2023-10-09T05:10:00.000349+0000 mon.a (mon.0) 808 : cluster 3 pg 9.f is active+recovery_wait+undersized+degraded+remapped, acting [5,3] 2023-10-09T05:10:01.152 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:01.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:01.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:01.259 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 32 16 44336 44320 86.5518 60.125 0.0132385 0.0115468 2023-10-09T05:10:01.260 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:01.286 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:01.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:01.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:01.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:00 smithi133 bash[105589]: cluster 2023-10-09T05:09:59.754039+0000 mgr.y (mgr.25277) 641 : cluster 0 pgmap v607: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 82 active+clean; 2.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s; 15893/125079 objects degraded (12.706%); 11 KiB/s, 0 objects/s recovering 2023-10-09T05:10:01.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:00 smithi133 bash[105589]: cluster 2023-10-09T05:09:59.837779+0000 mon.a (mon.0) 799 : cluster 3 Health check failed: Degraded data redundancy: 15893/125079 objects degraded (12.706%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:01.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:00 smithi133 bash[105589]: cluster 2023-10-09T05:10:00.000125+0000 mon.a (mon.0) 800 : cluster 3 Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 15893/125079 objects degraded (12.706%), 6 pgs degraded 2023-10-09T05:10:01.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:00 smithi133 bash[105589]: cluster 2023-10-09T05:10:00.000166+0000 mon.a (mon.0) 801 : cluster 3 [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-10-09T05:10:01.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:00 smithi133 bash[105589]: cluster 2023-10-09T05:10:00.000184+0000 mon.a (mon.0) 802 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 15893/125079 objects degraded (12.706%), 6 pgs degraded 2023-10-09T05:10:01.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:00 smithi133 bash[105589]: cluster 2023-10-09T05:10:00.000199+0000 mon.a (mon.0) 803 : cluster 3 pg 9.3 is active+recovery_wait+undersized+degraded+remapped, acting [7,3] 2023-10-09T05:10:01.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:00 smithi133 bash[105589]: cluster 2023-10-09T05:10:00.000214+0000 mon.a (mon.0) 804 : cluster 3 pg 9.6 is active+recovery_wait+undersized+degraded+remapped, acting [5,2] 2023-10-09T05:10:01.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:00 smithi133 bash[105589]: cluster 2023-10-09T05:10:00.000290+0000 mon.a (mon.0) 805 : cluster 3 pg 9.a is active+recovery_wait+undersized+degraded+remapped, acting [5,7] 2023-10-09T05:10:01.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:00 smithi133 bash[105589]: cluster 2023-10-09T05:10:00.000306+0000 mon.a (mon.0) 806 : cluster 3 pg 9.d is active+recovery_wait+undersized+degraded+remapped, acting [7,5] 2023-10-09T05:10:01.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:00 smithi133 bash[105589]: cluster 2023-10-09T05:10:00.000334+0000 mon.a (mon.0) 807 : cluster 3 pg 9.e is active+recovery_wait+undersized+degraded+remapped, acting [7,4] 2023-10-09T05:10:01.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:00 smithi133 bash[105589]: cluster 2023-10-09T05:10:00.000349+0000 mon.a (mon.0) 808 : cluster 3 pg 9.f is active+recovery_wait+undersized+degraded+remapped, acting [5,3] 2023-10-09T05:10:01.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:00 smithi133 bash[102858]: cluster 2023-10-09T05:09:59.754039+0000 mgr.y (mgr.25277) 641 : cluster 0 pgmap v607: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 82 active+clean; 2.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s; 15893/125079 objects degraded (12.706%); 11 KiB/s, 0 objects/s recovering 2023-10-09T05:10:01.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:00 smithi133 bash[102858]: cluster 2023-10-09T05:09:59.837779+0000 mon.a (mon.0) 799 : cluster 3 Health check failed: Degraded data redundancy: 15893/125079 objects degraded (12.706%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:01.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:00 smithi133 bash[102858]: cluster 2023-10-09T05:10:00.000125+0000 mon.a (mon.0) 800 : cluster 3 Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 15893/125079 objects degraded (12.706%), 6 pgs degraded 2023-10-09T05:10:01.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:00 smithi133 bash[102858]: cluster 2023-10-09T05:10:00.000166+0000 mon.a (mon.0) 801 : cluster 3 [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-10-09T05:10:01.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:00 smithi133 bash[102858]: cluster 2023-10-09T05:10:00.000184+0000 mon.a (mon.0) 802 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 15893/125079 objects degraded (12.706%), 6 pgs degraded 2023-10-09T05:10:01.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:00 smithi133 bash[102858]: cluster 2023-10-09T05:10:00.000199+0000 mon.a (mon.0) 803 : cluster 3 pg 9.3 is active+recovery_wait+undersized+degraded+remapped, acting [7,3] 2023-10-09T05:10:01.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:00 smithi133 bash[102858]: cluster 2023-10-09T05:10:00.000214+0000 mon.a (mon.0) 804 : cluster 3 pg 9.6 is active+recovery_wait+undersized+degraded+remapped, acting [5,2] 2023-10-09T05:10:01.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:00 smithi133 bash[102858]: cluster 2023-10-09T05:10:00.000290+0000 mon.a (mon.0) 805 : cluster 3 pg 9.a is active+recovery_wait+undersized+degraded+remapped, acting [5,7] 2023-10-09T05:10:01.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:00 smithi133 bash[102858]: cluster 2023-10-09T05:10:00.000306+0000 mon.a (mon.0) 806 : cluster 3 pg 9.d is active+recovery_wait+undersized+degraded+remapped, acting [7,5] 2023-10-09T05:10:01.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:00 smithi133 bash[102858]: cluster 2023-10-09T05:10:00.000334+0000 mon.a (mon.0) 807 : cluster 3 pg 9.e is active+recovery_wait+undersized+degraded+remapped, acting [7,4] 2023-10-09T05:10:01.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:00 smithi133 bash[102858]: cluster 2023-10-09T05:10:00.000349+0000 mon.a (mon.0) 808 : cluster 3 pg 9.f is active+recovery_wait+undersized+degraded+remapped, acting [5,3] 2023-10-09T05:10:01.394 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:01.426 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:01.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:01.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:01.532 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:01.564 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:01.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:01.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:01.670 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:01.691 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:01.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:01.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:01.796 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:01.827 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:01.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:01.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:01.932 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:01.970 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:01.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:01.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:02.073 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:02.129 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:02.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:02.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:02.232 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:02.258 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 33 16 45289 45273 85.7338 59.5625 0.0278498 0.0116562 2023-10-09T05:10:02.267 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:02.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:02.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:02.371 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:02.400 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:02.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:02.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:02.503 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:02.531 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:02.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:02.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:02.636 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:02.667 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:02.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:02.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:02.773 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:02.802 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:02.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:02.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:02.858 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:02 smithi133 bash[102858]: cluster 2023-10-09T05:10:01.754724+0000 mgr.y (mgr.25277) 642 : cluster 0 pgmap v608: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 80 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 43 MiB/s wr, 690 op/s; 15893/128457 objects degraded (12.372%); 5313/128457 objects misplaced (4.136%); 586 KiB/s, 9 objects/s recovering 2023-10-09T05:10:02.905 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:02.932 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:02.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:02.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:02.941 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:10:02.941 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:10:02.941 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-10-09T05:10:02.941 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:10:03.035 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:03.045 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:02 smithi194 bash[92666]: cluster 2023-10-09T05:10:01.754724+0000 mgr.y (mgr.25277) 642 : cluster 0 pgmap v608: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 80 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 43 MiB/s wr, 690 op/s; 15893/128457 objects degraded (12.372%); 5313/128457 objects misplaced (4.136%); 586 KiB/s, 9 objects/s recovering 2023-10-09T05:10:03.060 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:03.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:03.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:03.163 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:03.173 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:02 smithi133 bash[105589]: cluster 2023-10-09T05:10:01.754724+0000 mgr.y (mgr.25277) 642 : cluster 0 pgmap v608: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 80 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 43 MiB/s wr, 690 op/s; 15893/128457 objects degraded (12.372%); 5313/128457 objects misplaced (4.136%); 586 KiB/s, 9 objects/s recovering 2023-10-09T05:10:03.188 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:03.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:03.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:03.258 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 34 16 46052 46036 84.6147 47.6875 0.0204816 0.0118119 2023-10-09T05:10:03.311 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:03.336 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:03.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:03.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:03.439 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:03.464 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:03.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:03.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:03.568 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:03.603 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:03.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:03.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:03.818 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:03.856 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:03.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:03.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:03.960 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:03.993 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:03.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:03.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:04.098 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:04.132 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:04.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:04.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:04.235 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:04.258 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 35 16 46973 46957 83.8416 57.5625 0.0104929 0.01192 2023-10-09T05:10:04.267 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:04.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:04.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:04.372 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:04.381 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:04 smithi133 bash[102858]: cluster 2023-10-09T05:10:03.755388+0000 mgr.y (mgr.25277) 643 : cluster 0 pgmap v609: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 45 MiB/s wr, 726 op/s; 18400/131145 objects degraded (14.030%); 5313/131145 objects misplaced (4.051%); 507 KiB/s, 8 objects/s recovering 2023-10-09T05:10:04.381 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:10:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:10:04] "GET /metrics HTTP/1.1" 200 36401 "" "Prometheus/2.43.0" 2023-10-09T05:10:04.381 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:04 smithi133 bash[105589]: cluster 2023-10-09T05:10:03.755388+0000 mgr.y (mgr.25277) 643 : cluster 0 pgmap v609: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 45 MiB/s wr, 726 op/s; 18400/131145 objects degraded (14.030%); 5313/131145 objects misplaced (4.051%); 507 KiB/s, 8 objects/s recovering 2023-10-09T05:10:04.401 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:04.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:04.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:04.506 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:04.515 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:04 smithi194 bash[92666]: cluster 2023-10-09T05:10:03.755388+0000 mgr.y (mgr.25277) 643 : cluster 0 pgmap v609: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 45 MiB/s wr, 726 op/s; 18400/131145 objects degraded (14.030%); 5313/131145 objects misplaced (4.051%); 507 KiB/s, 8 objects/s recovering 2023-10-09T05:10:04.533 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:04.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:04.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:04.639 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:04.664 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:04.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:04.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:04.769 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:04.793 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:04.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:04.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:04.899 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:04.924 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:04.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:04.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:05.029 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:05.057 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:05.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:05.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:05.163 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:05.204 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:05.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:05.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:05.258 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 36 16 47954 47938 83.2157 61.3125 0.0101918 0.012009 2023-10-09T05:10:05.311 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:05.359 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:05.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:05.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:05.464 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:05.491 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:05.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:05.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:05.595 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:05.621 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:05.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:05.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:05.725 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:05.758 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:05.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:05.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:05.862 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:05.887 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:05.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:05.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:05.993 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:06.002 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:05 smithi194 bash[92666]: cluster 2023-10-09T05:10:05.808500+0000 mon.a (mon.0) 809 : cluster 3 Health check update: Degraded data redundancy: 18400/141528 objects degraded (13.001%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:06.019 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:06.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:06.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:06.122 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:06.179 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:06.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:06.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:06.259 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 37 16 48790 48774 82.3787 52.25 0.0193193 0.0121306 2023-10-09T05:10:06.283 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:06.286 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:05 smithi133 bash[105589]: cluster 2023-10-09T05:10:05.808500+0000 mon.a (mon.0) 809 : cluster 3 Health check update: Degraded data redundancy: 18400/141528 objects degraded (13.001%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:06.286 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:05 smithi133 bash[102858]: cluster 2023-10-09T05:10:05.808500+0000 mon.a (mon.0) 809 : cluster 3 Health check update: Degraded data redundancy: 18400/141528 objects degraded (13.001%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:06.313 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:06.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:06.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:06.417 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:06.454 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:06.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:06.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:06.557 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:06.596 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:06.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:06.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:06.701 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:06.732 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:06.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:06.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:06.837 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:06.869 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:06.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:06.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:06.974 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:07.000 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:07.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:07.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:07.104 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:07.116 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:06 smithi194 bash[92666]: cluster 2023-10-09T05:10:05.756160+0000 mgr.y (mgr.25277) 644 : cluster 0 pgmap v610: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 18400/141528 objects degraded (13.001%); 5125/141528 objects misplaced (3.621%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:10:07.140 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:07.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:07.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:07.245 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:07.257 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:06 smithi133 bash[105589]: cluster 2023-10-09T05:10:05.756160+0000 mgr.y (mgr.25277) 644 : cluster 0 pgmap v610: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 18400/141528 objects degraded (13.001%); 5125/141528 objects misplaced (3.621%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:10:07.257 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:06 smithi133 bash[102858]: cluster 2023-10-09T05:10:05.756160+0000 mgr.y (mgr.25277) 644 : cluster 0 pgmap v610: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 18400/141528 objects degraded (13.001%); 5125/141528 objects misplaced (3.621%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:10:07.259 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 38 16 49776 49760 81.8324 61.625 0.014844 0.0122143 2023-10-09T05:10:07.273 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:07.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:07.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:07.379 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:07.407 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:07.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:07.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:07.511 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:07.542 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:07.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:07.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:07.647 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:07.679 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:07.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:07.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:07.785 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:07.817 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:07.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:07.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:07.919 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:07.961 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:07.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:07.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:08.065 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:08.099 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:08.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:08.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:08.203 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:08.229 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:08.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:08.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:08.258 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 39 16 50687 50671 81.194 56.9375 0.0424612 0.012308 2023-10-09T05:10:08.334 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:08.362 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:08.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:08.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:08.465 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:08.497 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:08.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:08.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:08.600 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:08.637 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:08.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:08.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:08.744 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:08.772 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:08.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:08.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:08.877 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:08.907 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:08.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:08.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:09.011 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:09.046 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:09.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:09.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:09.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:08 smithi194 bash[92666]: cluster 2023-10-09T05:10:07.756848+0000 mgr.y (mgr.25277) 645 : cluster 0 pgmap v611: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s; 18400/141528 objects degraded (13.001%); 5125/141528 objects misplaced (3.621%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T05:10:09.150 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:09.177 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:09.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:09.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:09.258 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T05:10:09.258467+0000 min lat: 0.00144352 max lat: 2.39474 avg lat: 0.0123914 2023-10-09T05:10:09.258 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:10:09.259 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 40 16 51637 51621 80.6483 59.375 0.0409152 0.0123914 2023-10-09T05:10:09.280 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:09.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:08 smithi133 bash[105589]: cluster 2023-10-09T05:10:07.756848+0000 mgr.y (mgr.25277) 645 : cluster 0 pgmap v611: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s; 18400/141528 objects degraded (13.001%); 5125/141528 objects misplaced (3.621%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T05:10:09.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:08 smithi133 bash[102858]: cluster 2023-10-09T05:10:07.756848+0000 mgr.y (mgr.25277) 645 : cluster 0 pgmap v611: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s; 18400/141528 objects degraded (13.001%); 5125/141528 objects misplaced (3.621%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T05:10:09.309 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:09.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:09.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:09.413 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:09.441 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:09.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:09.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:09.545 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:09.588 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:09.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:09.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:09.690 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:09.714 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:09.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:09.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:09.817 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:09.857 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:09.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:09.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:09.963 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:09.996 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:10.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:10.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:10.101 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:10.132 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:10.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:10.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:10.239 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:10.258 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 41 16 52543 52527 80.0623 56.625 0.0251807 0.0124825 2023-10-09T05:10:10.277 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:10.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:10.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:10.381 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:10.415 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:10.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:10.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:10.519 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:10.543 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:10.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:10.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:10.647 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:10.675 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:10.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:10.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:10.699 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:10.779 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:10.806 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:10.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:10.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:10.909 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:10.919 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:10 smithi194 bash[92666]: cluster 2023-10-09T05:10:09.757944+0000 mgr.y (mgr.25277) 646 : cluster 0 pgmap v612: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 59 MiB/s wr, 951 op/s; 18400/151299 objects degraded (12.161%); 5125/151299 objects misplaced (3.387%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T05:10:10.935 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:10.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:10.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:11.041 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:11.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:10 smithi133 bash[102858]: cluster 2023-10-09T05:10:09.757944+0000 mgr.y (mgr.25277) 646 : cluster 0 pgmap v612: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 59 MiB/s wr, 951 op/s; 18400/151299 objects degraded (12.161%); 5125/151299 objects misplaced (3.387%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T05:10:11.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:10 smithi133 bash[105589]: cluster 2023-10-09T05:10:09.757944+0000 mgr.y (mgr.25277) 646 : cluster 0 pgmap v612: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 59 MiB/s wr, 951 op/s; 18400/151299 objects degraded (12.161%); 5125/151299 objects misplaced (3.387%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T05:10:11.078 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:11.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:11.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:11.182 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:11.222 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:11.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:11.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:11.258 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 42 16 53424 53408 79.4669 55.0625 0.0319015 0.0125776 2023-10-09T05:10:11.326 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:11.353 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:11.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:11.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:11.462 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:11.489 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:11.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:11.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:11.594 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:11.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:11.623 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:11.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:11.725 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:11.768 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:11.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:11.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:11.874 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:11.902 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:11.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:11.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:12.006 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:12.036 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:12.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:12.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:12.140 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:12.151 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:11 smithi133 bash[102858]: cluster 2023-10-09T05:10:11.875242+0000 mon.a (mon.0) 810 : cluster 3 Health check update: Degraded data redundancy: 18400/155577 objects degraded (11.827%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:12.151 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:11 smithi133 bash[105589]: cluster 2023-10-09T05:10:11.875242+0000 mon.a (mon.0) 810 : cluster 3 Health check update: Degraded data redundancy: 18400/155577 objects degraded (11.827%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:12.175 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:12.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:12.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:12.259 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 43 16 54256 54240 78.828 52 0.016742 0.0126742 2023-10-09T05:10:12.281 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:12.305 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:11 smithi194 bash[92666]: cluster 2023-10-09T05:10:11.875242+0000 mon.a (mon.0) 810 : cluster 3 Health check update: Degraded data redundancy: 18400/155577 objects degraded (11.827%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:12.322 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:12.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:12.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:12.430 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:12.456 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:12.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:12.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:12.559 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:12.600 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:12.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:12.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:12.703 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:12.737 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:12.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:12.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:12.840 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:12.871 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:12.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:12.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:12.971 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:12 smithi133 bash[102858]: cluster 2023-10-09T05:10:11.758645+0000 mgr.y (mgr.25277) 647 : cluster 0 pgmap v613: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s; 18400/155577 objects degraded (11.827%); 4935/155577 objects misplaced (3.172%); 2.2 MiB/s, 36 objects/s recovering 2023-10-09T05:10:12.974 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:13.010 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:13.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:13.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:13.113 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:13.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:12 smithi194 bash[92666]: cluster 2023-10-09T05:10:11.758645+0000 mgr.y (mgr.25277) 647 : cluster 0 pgmap v613: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s; 18400/155577 objects degraded (11.827%); 4935/155577 objects misplaced (3.172%); 2.2 MiB/s, 36 objects/s recovering 2023-10-09T05:10:13.145 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:13.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:13.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:13.252 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:13.259 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 44 16 55017 55001 78.1174 47.5625 0.0142066 0.0127933 2023-10-09T05:10:13.260 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:12 smithi133 bash[105589]: cluster 2023-10-09T05:10:11.758645+0000 mgr.y (mgr.25277) 647 : cluster 0 pgmap v613: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s; 18400/155577 objects degraded (11.827%); 4935/155577 objects misplaced (3.172%); 2.2 MiB/s, 36 objects/s recovering 2023-10-09T05:10:13.282 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:13.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:13.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:13.387 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:13.413 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:13.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:13.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:13.517 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:13.550 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:13.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:13.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:13.655 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:13.682 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:13.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:13.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:13.789 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:13.817 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:13.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:13.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:13.924 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:13.945 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:13.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:13.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:14.054 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:14.081 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:14.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:14.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:14.185 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:14.211 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:14.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:14.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:14.259 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 45 16 55880 55864 77.5799 53.9375 0.0121336 0.0128809 2023-10-09T05:10:14.316 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:14.344 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:14.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:14.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:14.448 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:14.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:10:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:10:14] "GET /metrics HTTP/1.1" 200 36430 "" "Prometheus/2.43.0" 2023-10-09T05:10:14.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:14 smithi133 bash[105589]: audit 2023-10-09T05:10:13.448524+0000 mon.a (mon.0) 811 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:10:14.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:14 smithi133 bash[105589]: audit 2023-10-09T05:10:13.449917+0000 mon.a (mon.0) 812 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:10:14.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:14 smithi133 bash[105589]: cluster 2023-10-09T05:10:13.759296+0000 mgr.y (mgr.25277) 648 : cluster 0 pgmap v614: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s; 18400/158661 objects degraded (11.597%); 4935/158661 objects misplaced (3.110%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T05:10:14.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:14 smithi133 bash[102858]: audit 2023-10-09T05:10:13.448524+0000 mon.a (mon.0) 811 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:10:14.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:14 smithi133 bash[102858]: audit 2023-10-09T05:10:13.449917+0000 mon.a (mon.0) 812 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:10:14.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:14 smithi133 bash[102858]: cluster 2023-10-09T05:10:13.759296+0000 mgr.y (mgr.25277) 648 : cluster 0 pgmap v614: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s; 18400/158661 objects degraded (11.597%); 4935/158661 objects misplaced (3.110%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T05:10:14.577 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:14.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:14.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:14.683 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:14.710 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:14.712 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:14 smithi194 bash[92666]: audit 2023-10-09T05:10:13.448524+0000 mon.a (mon.0) 811 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:10:14.713 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:14 smithi194 bash[92666]: audit 2023-10-09T05:10:13.449917+0000 mon.a (mon.0) 812 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:10:14.713 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:14 smithi194 bash[92666]: cluster 2023-10-09T05:10:13.759296+0000 mgr.y (mgr.25277) 648 : cluster 0 pgmap v614: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s; 18400/158661 objects degraded (11.597%); 4935/158661 objects misplaced (3.110%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T05:10:14.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:14.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:14.814 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:14.858 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:14.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:14.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:14.962 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:14.993 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:14.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:14.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:15.098 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:15.116 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:15.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:15.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:15.219 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:15.252 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:15.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:15.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:15.259 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 46 16 56694 56678 76.9993 50.875 0.0199966 0.0129785 2023-10-09T05:10:15.356 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:15.388 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:15.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:15.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:15.489 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is unset 2023-10-09T05:10:15.491 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:15.774 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:15 smithi133 bash[102858]: audit 2023-10-09T05:10:15.380359+0000 mon.c (mon.2) 64 : audit 1 from='client.? 172.21.15.133:0/4063712517' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:10:15.774 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:15 smithi133 bash[102858]: audit 2023-10-09T05:10:15.381049+0000 mon.a (mon.0) 813 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:10:15.774 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:15 smithi133 bash[105589]: audit 2023-10-09T05:10:15.380359+0000 mon.c (mon.2) 64 : audit 1 from='client.? 172.21.15.133:0/4063712517' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:10:15.774 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:15 smithi133 bash[105589]: audit 2023-10-09T05:10:15.381049+0000 mon.a (mon.0) 813 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:10:15.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:15 smithi194 bash[92666]: audit 2023-10-09T05:10:15.380359+0000 mon.c (mon.2) 64 : audit 1 from='client.? 172.21.15.133:0/4063712517' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:10:15.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:15 smithi194 bash[92666]: audit 2023-10-09T05:10:15.381049+0000 mon.a (mon.0) 813 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:10:16.116 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:16.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:16.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:16.221 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:16.252 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:16.259 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 47 16 57602 57586 76.5683 56.75 0.0124245 0.0130525 2023-10-09T05:10:16.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:16.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:16.361 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:16.387 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:16.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:16.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:16.493 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:16.518 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:16.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:16.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:16.522 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:16 smithi133 bash[102858]: cluster 2023-10-09T05:10:15.455996+0000 mon.a (mon.0) 814 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:10:16.522 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:16 smithi133 bash[102858]: cluster 2023-10-09T05:10:15.469211+0000 osd.4 (osd.4) 95 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T05:10:16.522 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:16 smithi133 bash[102858]: audit 2023-10-09T05:10:15.471701+0000 mon.a (mon.0) 815 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:10:16.522 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:16 smithi133 bash[102858]: cluster 2023-10-09T05:10:15.471861+0000 mon.a (mon.0) 816 : cluster 0 osdmap e154: 8 total, 8 up, 8 in 2023-10-09T05:10:16.522 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:16 smithi133 bash[102858]: cluster 2023-10-09T05:10:15.474755+0000 osd.7 (osd.7) 74 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T05:10:16.522 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:16 smithi133 bash[102858]: cluster 2023-10-09T05:10:15.477897+0000 osd.5 (osd.5) 128 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T05:10:16.522 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:16 smithi133 bash[102858]: cluster 2023-10-09T05:10:15.488216+0000 osd.6 (osd.6) 80 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T05:10:16.522 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:16 smithi133 bash[102858]: cluster 2023-10-09T05:10:15.759900+0000 mgr.y (mgr.25277) 649 : cluster 0 pgmap v616: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 56 MiB/s wr, 889 op/s; 18400/168213 objects degraded (10.939%); 4737/168213 objects misplaced (2.816%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:10:16.523 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:16 smithi133 bash[105589]: cluster 2023-10-09T05:10:15.455996+0000 mon.a (mon.0) 814 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:10:16.523 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:16 smithi133 bash[105589]: cluster 2023-10-09T05:10:15.469211+0000 osd.4 (osd.4) 95 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T05:10:16.621 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:16.648 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:16.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:16.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:16.723 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:10:16.723 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:10:16.753 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:16.778 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:16.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:16.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:16.784 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:16 smithi133 bash[105589]: audit 2023-10-09T05:10:15.471701+0000 mon.a (mon.0) 815 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:10:16.784 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:16 smithi133 bash[105589]: cluster 2023-10-09T05:10:15.471861+0000 mon.a (mon.0) 816 : cluster 0 osdmap e154: 8 total, 8 up, 8 in 2023-10-09T05:10:16.784 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:16 smithi133 bash[105589]: cluster 2023-10-09T05:10:15.474755+0000 osd.7 (osd.7) 74 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T05:10:16.784 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:16 smithi133 bash[105589]: cluster 2023-10-09T05:10:15.477897+0000 osd.5 (osd.5) 128 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T05:10:16.784 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:16 smithi133 bash[105589]: cluster 2023-10-09T05:10:15.488216+0000 osd.6 (osd.6) 80 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T05:10:16.784 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:16 smithi133 bash[105589]: cluster 2023-10-09T05:10:15.759900+0000 mgr.y (mgr.25277) 649 : cluster 0 pgmap v616: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 56 MiB/s wr, 889 op/s; 18400/168213 objects degraded (10.939%); 4737/168213 objects misplaced (2.816%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:10:16.884 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:16.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:16 smithi194 bash[92666]: cluster 2023-10-09T05:10:15.455996+0000 mon.a (mon.0) 814 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:10:16.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:16 smithi194 bash[92666]: cluster 2023-10-09T05:10:15.469211+0000 osd.4 (osd.4) 95 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T05:10:16.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:16 smithi194 bash[92666]: audit 2023-10-09T05:10:15.471701+0000 mon.a (mon.0) 815 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:10:16.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:16 smithi194 bash[92666]: cluster 2023-10-09T05:10:15.471861+0000 mon.a (mon.0) 816 : cluster 0 osdmap e154: 8 total, 8 up, 8 in 2023-10-09T05:10:16.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:16 smithi194 bash[92666]: cluster 2023-10-09T05:10:15.474755+0000 osd.7 (osd.7) 74 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T05:10:16.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:16 smithi194 bash[92666]: cluster 2023-10-09T05:10:15.477897+0000 osd.5 (osd.5) 128 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T05:10:16.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:16 smithi194 bash[92666]: cluster 2023-10-09T05:10:15.488216+0000 osd.6 (osd.6) 80 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T05:10:16.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:16 smithi194 bash[92666]: cluster 2023-10-09T05:10:15.759900+0000 mgr.y (mgr.25277) 649 : cluster 0 pgmap v616: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 56 MiB/s wr, 889 op/s; 18400/168213 objects degraded (10.939%); 4737/168213 objects misplaced (2.816%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:10:16.919 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:16.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:16.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:17.024 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:17.049 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:17.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:17.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:17.152 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:17.205 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:17.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:17.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:17.259 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 48 16 58395 58379 76.0056 49.5625 0.00345358 0.0131426 2023-10-09T05:10:17.310 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:17.343 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:17.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:17.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:17.447 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:17.479 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:17.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:17.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:17.584 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:17.615 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:17.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:17.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:17.718 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:17.740 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:17.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:17.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:17.846 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:17.879 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:17.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:17.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:17.982 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:18.012 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:18.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:18.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:18.120 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:18.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:18.158 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:18.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:18.259 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 49 16 59220 59204 75.5067 51.5625 0.018171 0.0132376 2023-10-09T05:10:18.260 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:18.285 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:18.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:18.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:18.391 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:18.425 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:18.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:18.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:18.528 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:18.578 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:18.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:18.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:18.685 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:18.712 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:18.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:18.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:18.817 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:18.840 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:18.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:18.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:18.946 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:18.954 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:18 smithi194 bash[92666]: cluster 2023-10-09T05:10:17.760520+0000 mgr.y (mgr.25277) 650 : cluster 0 pgmap v617: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 56 MiB/s wr, 889 op/s; 18400/168213 objects degraded (10.939%); 4737/168213 objects misplaced (2.816%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:10:18.954 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:18 smithi194 bash[92666]: cluster 2023-10-09T05:10:18.238529+0000 mon.a (mon.0) 817 : cluster 3 Health check update: Degraded data redundancy: 18400/168213 objects degraded (10.939%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:18.970 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:18.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:18.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:19.057 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:18 smithi133 bash[102858]: cluster 2023-10-09T05:10:17.760520+0000 mgr.y (mgr.25277) 650 : cluster 0 pgmap v617: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 56 MiB/s wr, 889 op/s; 18400/168213 objects degraded (10.939%); 4737/168213 objects misplaced (2.816%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:10:19.058 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:18 smithi133 bash[102858]: cluster 2023-10-09T05:10:18.238529+0000 mon.a (mon.0) 817 : cluster 3 Health check update: Degraded data redundancy: 18400/168213 objects degraded (10.939%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:19.058 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:18 smithi133 bash[105589]: cluster 2023-10-09T05:10:17.760520+0000 mgr.y (mgr.25277) 650 : cluster 0 pgmap v617: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 56 MiB/s wr, 889 op/s; 18400/168213 objects degraded (10.939%); 4737/168213 objects misplaced (2.816%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:10:19.058 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:18 smithi133 bash[105589]: cluster 2023-10-09T05:10:18.238529+0000 mon.a (mon.0) 817 : cluster 3 Health check update: Degraded data redundancy: 18400/168213 objects degraded (10.939%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:19.066 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:10:19.074 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:19.090 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:10:19.096 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:19.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:19.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:19.109 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:10:19.199 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:19.201 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:10:19.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:19.243 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:19.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:19.249 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:10:19.259 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 50 16 60010 59994 74.984 49.375 0.0186582 0.0133298 2023-10-09T05:10:19.285 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:10:19.324 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:10:19.344 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:19.367 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:10:19.371 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:19.374 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:10:19.374 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:10:19.374 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:10:19.375 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:10:19.375 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:10:19.375 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:10:19.375 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:10:19.375 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:10:19.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:19.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:19.477 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:19.560 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:19.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:19.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:19.665 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:19.689 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:19.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:19.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:19.792 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:19.821 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:19.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:19.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:19.924 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:19.951 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:19.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:19.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:20.058 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:20.068 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:19 smithi194 bash[92666]: audit 2023-10-09T05:10:19.028477+0000 mon.a (mon.0) 818 : audit 0 from='client.? 172.21.15.133:0/895026981' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:10:20.084 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:20.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:20.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:20.169 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:19 smithi133 bash[102858]: audit 2023-10-09T05:10:19.028477+0000 mon.a (mon.0) 818 : audit 0 from='client.? 172.21.15.133:0/895026981' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:10:20.169 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:19 smithi133 bash[105589]: audit 2023-10-09T05:10:19.028477+0000 mon.a (mon.0) 818 : audit 0 from='client.? 172.21.15.133:0/895026981' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:10:20.193 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:20.218 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:20.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:20.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:20.259 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 51 16 60877 60861 74.5761 54.1875 0.0175228 0.0134013 2023-10-09T05:10:20.321 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:20.359 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:20.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:20.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:20.465 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:20.496 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:20.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:20.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:20.599 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:20.626 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:20.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:20.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:20.729 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:20.747 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:20.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:20.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:20.849 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:20.877 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:20.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:20.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:20.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:20 smithi194 bash[92666]: cluster 2023-10-09T05:10:19.761361+0000 mgr.y (mgr.25277) 651 : cluster 0 pgmap v618: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 18400/176733 objects degraded (10.411%); 4737/176733 objects misplaced (2.680%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:10:20.981 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:21.006 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:21.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:21.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:21.060 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:20 smithi133 bash[102858]: cluster 2023-10-09T05:10:19.761361+0000 mgr.y (mgr.25277) 651 : cluster 0 pgmap v618: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 18400/176733 objects degraded (10.411%); 4737/176733 objects misplaced (2.680%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:10:21.060 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:20 smithi133 bash[105589]: cluster 2023-10-09T05:10:19.761361+0000 mgr.y (mgr.25277) 651 : cluster 0 pgmap v618: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 18400/176733 objects degraded (10.411%); 4737/176733 objects misplaced (2.680%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:10:21.111 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:21.135 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:21.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:21.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:21.241 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:21.259 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 52 16 61648 61632 74.0685 48.1875 0.0188668 0.0134933 2023-10-09T05:10:21.273 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:21.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:21.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:21.378 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:21.405 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:21.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:21.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:21.508 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:21.535 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:21.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:21.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:21.640 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:21.676 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:21.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:21.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:21.782 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:21.807 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:21.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:21.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:21.911 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:21.944 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:21.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:21.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:22.050 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:22.082 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:22.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:22.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:22.222 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:22.259 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:22.259 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 53 16 62493 62477 73.6673 52.8125 0.00909233 0.0135665 2023-10-09T05:10:22.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:22.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:22.309 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T05:10:22.362 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:22.390 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:22.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:22.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:22.495 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:22.526 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:22.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:22.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:22.633 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:22.667 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:22.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:22.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:22.771 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:22.800 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:22.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:22.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:22.830 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:22 smithi133 bash[102858]: cluster 2023-10-09T05:10:21.762027+0000 mgr.y (mgr.25277) 652 : cluster 0 pgmap v619: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s; 18400/180627 objects degraded (10.187%); 4526/180627 objects misplaced (2.506%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T05:10:22.908 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:22.934 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:22.935 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:10:22.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:22.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:23.041 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:23.073 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:23.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:23.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:23.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:22 smithi194 bash[92666]: cluster 2023-10-09T05:10:21.762027+0000 mgr.y (mgr.25277) 652 : cluster 0 pgmap v619: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s; 18400/180627 objects degraded (10.187%); 4526/180627 objects misplaced (2.506%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T05:10:23.178 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:23.206 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:23.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:23.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:23.260 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 54 16 63279 63263 73.2126 49.125 0.0423098 0.0136518 2023-10-09T05:10:23.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:22 smithi133 bash[105589]: cluster 2023-10-09T05:10:21.762027+0000 mgr.y (mgr.25277) 652 : cluster 0 pgmap v619: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s; 18400/180627 objects degraded (10.187%); 4526/180627 objects misplaced (2.506%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T05:10:23.313 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:23.350 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:23.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:23.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:23.453 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:23.480 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:23.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:23.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:23.584 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:23.616 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:23.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:23.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:23.722 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:23.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:23.750 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:23.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:23.834 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:23 smithi194 bash[92666]: cluster 2023-10-09T05:10:23.239322+0000 mon.a (mon.0) 819 : cluster 3 Health check update: Degraded data redundancy: 18400/180627 objects degraded (10.187%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:23.851 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:23.858 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:23 smithi133 bash[102858]: cluster 2023-10-09T05:10:23.239322+0000 mon.a (mon.0) 819 : cluster 3 Health check update: Degraded data redundancy: 18400/180627 objects degraded (10.187%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:23.859 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:23 smithi133 bash[105589]: cluster 2023-10-09T05:10:23.239322+0000 mon.a (mon.0) 819 : cluster 3 Health check update: Degraded data redundancy: 18400/180627 objects degraded (10.187%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:23.879 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:23.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:23.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:23.985 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:24.011 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:24.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:24.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:24.115 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:24.144 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:24.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:24.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:24.254 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:24.260 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 55 16 64134 64118 72.853 53.4375 0.010203 0.0137191 2023-10-09T05:10:24.282 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:24.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:24.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:24.385 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:24.417 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:24.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:24.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:24.521 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:24.531 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:10:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:10:24] "GET /metrics HTTP/1.1" 200 36430 "" "Prometheus/2.43.0" 2023-10-09T05:10:24.552 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:24.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:24.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:24.580 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:10:24.656 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:24.691 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:24.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:24.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:24.796 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:24.831 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:24.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:24.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:24.935 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:24.958 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:24.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:24.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:24.995 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:10:24.997 INFO:teuthology.orchestra.run.smithi133.stderr:dumped all 2023-10-09T05:10:25.065 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:25.089 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:25.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:25.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:25.102 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:24 smithi133 bash[102858]: cluster 2023-10-09T05:10:23.762670+0000 mgr.y (mgr.25277) 653 : cluster 0 pgmap v620: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s; 18400/183696 objects degraded (10.017%); 4526/183696 objects misplaced (2.464%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T05:10:25.102 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:24 smithi133 bash[102858]: audit 2023-10-09T05:10:24.073004+0000 mon.a (mon.0) 820 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:10:25.103 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:24 smithi133 bash[105589]: cluster 2023-10-09T05:10:23.762670+0000 mgr.y (mgr.25277) 653 : cluster 0 pgmap v620: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s; 18400/183696 objects degraded (10.017%); 4526/183696 objects misplaced (2.464%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T05:10:25.103 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:24 smithi133 bash[105589]: audit 2023-10-09T05:10:24.073004+0000 mon.a (mon.0) 820 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:10:25.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:24 smithi194 bash[92666]: cluster 2023-10-09T05:10:23.762670+0000 mgr.y (mgr.25277) 653 : cluster 0 pgmap v620: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s; 18400/183696 objects degraded (10.017%); 4526/183696 objects misplaced (2.464%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T05:10:25.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:24 smithi194 bash[92666]: audit 2023-10-09T05:10:24.073004+0000 mon.a (mon.0) 820 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:10:25.193 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:25.221 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:25.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:25.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:25.260 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 56 16 64848 64832 72.3489 44.625 0.007642 0.0138133 2023-10-09T05:10:25.324 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:25.355 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:25.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:25.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:25.459 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:25.496 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:25.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:25.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:25.600 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:25.632 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:25.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:25.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:25.736 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:25.760 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:25.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:25.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:25.863 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:25.889 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:25.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:25.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:25.996 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:26.006 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:25 smithi194 bash[92666]: audit 2023-10-09T05:10:24.971191+0000 mgr.y (mgr.25277) 654 : audit 0 from='client.34884 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:10:26.021 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:26.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:26.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:26.125 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:26.151 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:26.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:26.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:26.255 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:26.260 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 57 16 65709 65693 72.0236 53.8125 0.0102747 0.0138766 2023-10-09T05:10:26.264 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:25 smithi133 bash[105589]: audit 2023-10-09T05:10:24.971191+0000 mgr.y (mgr.25277) 654 : audit 0 from='client.34884 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:10:26.264 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:25 smithi133 bash[102858]: audit 2023-10-09T05:10:24.971191+0000 mgr.y (mgr.25277) 654 : audit 0 from='client.34884 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:10:26.288 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:26.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:26.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:26.391 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:26.418 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:26.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:26.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:26.490 INFO:teuthology.orchestra.run.smithi133.stdout:{"pg_ready":true,"pg_map":{"version":620,"stamp":"2023-10-09T05:10:23.762240+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4011923360,"num_objects":61232,"num_object_clones":0,"num_object_copies":183696,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18400,"num_objects_misplaced":4526,"num_objects_unfound":0,"num_objects_dirty":61232,"num_whiteouts":0,"num_read":538,"num_read_kb":964,"num_write":61508,"num_write_kb":3919757,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2089,"num_bytes_recovered":137889376,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":61430,"ondisk_log_size":61430,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":276,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14997724,"kb_used_data":11854876,"kb_used_omap":20,"kb_used_meta":3142763,"kb_avail":734963492,"statfs":{"total":767960285184,"available":752602615808,"internally_reserved":0,"allocated":12139393024,"data_stored":12133789894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":3218189966},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,7,7,7,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":546897920,"num_objects":8345,"num_object_clones":0,"num_object_copies":25035,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-409,"num_objects_unfound":0,"num_objects_dirty":8345,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8345,"num_write_kb":534080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":409,"num_bytes_recovered":26804224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003381"},"pg_stats":[{"pgid":"3.1a","version":"69'2","reported_seq":154,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.875719+0000","last_change":"2023-10-09T05:09:57.873320+0000","last_active":"2023-10-09T05:09:58.875719+0000","last_peered":"2023-10-09T05:09:58.875719+0000","last_clean":"2023-10-09T05:09:58.875719+0000","last_became_active":"2023-10-09T05:09:57.868376+0000","last_became_peered":"2023-10-09T05:09:57.868376+0000","last_unstale":"2023-10-09T05:09:58.875719+0000","last_undegraded":"2023-10-09T05:09:58.875719+0000","last_fullsized":"2023-10-09T05:09:58.875719+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:27.186610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":121,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.832846+0000","last_change":"2023-10-09T05:09:57.854324+0000","last_active":"2023-10-09T05:09:58.832846+0000","last_peered":"2023-10-09T05:09:58.832846+0000","last_clean":"2023-10-09T05:09:58.832846+0000","last_became_active":"2023-10-09T05:09:57.853925+0000","last_became_peered":"2023-10-09T05:09:57.853925+0000","last_unstale":"2023-10-09T05:09:58.832846+0000","last_undegraded":"2023-10-09T05:09:58.832846+0000","last_fullsized":"2023-10-09T05:09:58.832846+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:11.255018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:10:01.321880+0000","last_change":"2023-10-09T05:09:57.857511+0000","last_active":"2023-10-09T05:10:01.321880+0000","last_peered":"2023-10-09T05:10:01.321880+0000","last_clean":"2023-10-09T05:10:01.321880+0000","last_became_active":"2023-10-09T05:09:57.857193+0000","last_became_peered":"2023-10-09T05:09:57.857193+0000","last_unstale":"2023-10-09T05:10:01.321880+0000","last_undegraded":"2023-10-09T05:10:01.321880+0000","last_fullsized":"2023-10-09T05:10:01.321880+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:32.842980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":109,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.838845+0000","last_change":"2023-10-09T05:02:09.648283+0000","last_active":"2023-10-09T05:09:58.838845+0000","last_peered":"2023-10-09T05:09:58.838845+0000","last_clean":"2023-10-09T05:09:58.838845+0000","last_became_active":"2023-10-09T05:02:09.647778+0000","last_became_peered":"2023-10-09T05:02:09.647778+0000","last_unstale":"2023-10-09T05:09:58.838845+0000","last_undegraded":"2023-10-09T05:09:58.838845+0000","last_fullsized":"2023-10-09T05:09:58.838845+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:24.582392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":139,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.832533+0000","last_change":"2023-10-09T05:09:57.849878+0000","last_active":"2023-10-09T05:09:58.832533+0000","last_peered":"2023-10-09T05:09:58.832533+0000","last_clean":"2023-10-09T05:09:58.832533+0000","last_became_active":"2023-10-09T05:09:57.849541+0000","last_became_peered":"2023-10-09T05:09:57.849541+0000","last_unstale":"2023-10-09T05:09:58.832533+0000","last_undegraded":"2023-10-09T05:09:58.832533+0000","last_fullsized":"2023-10-09T05:09:58.832533+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:49:03.330682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":134,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.882631+0000","last_change":"2023-10-09T05:09:57.852119+0000","last_active":"2023-10-09T05:09:58.882631+0000","last_peered":"2023-10-09T05:09:58.882631+0000","last_clean":"2023-10-09T05:09:58.882631+0000","last_became_active":"2023-10-09T05:09:57.851752+0000","last_became_peered":"2023-10-09T05:09:57.851752+0000","last_unstale":"2023-10-09T05:09:58.882631+0000","last_undegraded":"2023-10-09T05:09:58.882631+0000","last_fullsized":"2023-10-09T05:09:58.882631+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:58:09.571362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":107,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.839136+0000","last_change":"2023-10-09T05:02:09.646765+0000","last_active":"2023-10-09T05:09:58.839136+0000","last_peered":"2023-10-09T05:09:58.839136+0000","last_clean":"2023-10-09T05:09:58.839136+0000","last_became_active":"2023-10-09T05:02:09.645141+0000","last_became_peered":"2023-10-09T05:02:09.645141+0000","last_unstale":"2023-10-09T05:09:58.839136+0000","last_undegraded":"2023-10-09T05:09:58.839136+0000","last_fullsized":"2023-10-09T05:09:58.839136+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:10.098578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":121,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.832170+0000","last_change":"2023-10-09T05:09:57.849695+0000","last_active":"2023-10-09T05:09:58.832170+0000","last_peered":"2023-10-09T05:09:58.832170+0000","last_clean":"2023-10-09T05:09:58.832170+0000","last_became_active":"2023-10-09T05:09:57.848478+0000","last_became_peered":"2023-10-09T05:09:57.848478+0000","last_unstale":"2023-10-09T05:09:58.832170+0000","last_undegraded":"2023-10-09T05:09:58.832170+0000","last_fullsized":"2023-10-09T05:09:58.832170+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:26.115312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":135,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.842144+0000","last_change":"2023-10-09T05:09:57.867139+0000","last_active":"2023-10-09T05:09:58.842144+0000","last_peered":"2023-10-09T05:09:58.842144+0000","last_clean":"2023-10-09T05:09:58.842144+0000","last_became_active":"2023-10-09T05:09:57.866134+0000","last_became_peered":"2023-10-09T05:09:57.866134+0000","last_unstale":"2023-10-09T05:09:58.842144+0000","last_undegraded":"2023-10-09T05:09:58.842144+0000","last_fullsized":"2023-10-09T05:09:58.842144+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:45.375784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":98,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.837047+0000","last_change":"2023-10-09T05:02:09.651167+0000","last_active":"2023-10-09T05:09:58.837047+0000","last_peered":"2023-10-09T05:09:58.837047+0000","last_clean":"2023-10-09T05:09:58.837047+0000","last_became_active":"2023-10-09T05:02:09.650419+0000","last_became_peered":"2023-10-09T05:02:09.650419+0000","last_unstale":"2023-10-09T05:09:58.837047+0000","last_undegraded":"2023-10-09T05:09:58.837047+0000","last_fullsized":"2023-10-09T05:09:58.837047+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:55:16.966938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":120,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.874671+0000","last_change":"2023-10-09T05:02:09.649700+0000","last_active":"2023-10-09T05:09:58.874671+0000","last_peered":"2023-10-09T05:09:58.874671+0000","last_clean":"2023-10-09T05:09:58.874671+0000","last_became_active":"2023-10-09T05:02:09.649317+0000","last_became_peered":"2023-10-09T05:02:09.649317+0000","last_unstale":"2023-10-09T05:09:58.874671+0000","last_undegraded":"2023-10-09T05:09:58.874671+0000","last_fullsized":"2023-10-09T05:09:58.874671+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:50.711767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:10:01.321897+0000","last_change":"2023-10-09T05:09:57.848297+0000","last_active":"2023-10-09T05:10:01.321897+0000","last_peered":"2023-10-09T05:10:01.321897+0000","last_clean":"2023-10-09T05:10:01.321897+0000","last_became_active":"2023-10-09T05:09:57.847987+0000","last_became_peered":"2023-10-09T05:09:57.847987+0000","last_unstale":"2023-10-09T05:10:01.321897+0000","last_undegraded":"2023-10-09T05:10:01.321897+0000","last_fullsized":"2023-10-09T05:10:01.321897+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:16.385601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'23","reported_seq":126,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.876135+0000","last_change":"2023-10-09T04:44:52.781203+0000","last_active":"2023-10-09T05:09:58.876135+0000","last_peered":"2023-10-09T05:09:58.876135+0000","last_clean":"2023-10-09T05:09:58.876135+0000","last_became_active":"2023-10-09T04:44:52.780902+0000","last_became_peered":"2023-10-09T04:44:52.780902+0000","last_unstale":"2023-10-09T05:09:58.876135+0000","last_undegraded":"2023-10-09T05:09:58.876135+0000","last_fullsized":"2023-10-09T05:09:58.876135+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:56.332974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":134,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.876238+0000","last_change":"2023-10-09T05:09:57.868761+0000","last_active":"2023-10-09T05:09:58.876238+0000","last_peered":"2023-10-09T05:09:58.876238+0000","last_clean":"2023-10-09T05:09:58.876238+0000","last_became_active":"2023-10-09T05:09:57.868121+0000","last_became_peered":"2023-10-09T05:09:57.868121+0000","last_unstale":"2023-10-09T05:09:58.876238+0000","last_undegraded":"2023-10-09T05:09:58.876238+0000","last_fullsized":"2023-10-09T05:09:58.876238+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:43.412235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":119,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.930531+0000","last_change":"2023-10-09T05:01:02.669186+0000","last_active":"2023-10-09T05:09:58.930531+0000","last_peered":"2023-10-09T05:09:58.930531+0000","last_clean":"2023-10-09T05:09:58.930531+0000","last_became_active":"2023-10-09T05:01:02.668739+0000","last_became_peered":"2023-10-09T05:01:02.668739+0000","last_unstale":"2023-10-09T05:09:58.930531+0000","last_undegraded":"2023-10-09T05:09:58.930531+0000","last_fullsized":"2023-10-09T05:09:58.930531+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:41.352624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":121,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.832650+0000","last_change":"2023-10-09T05:09:57.854619+0000","last_active":"2023-10-09T05:09:58.832650+0000","last_peered":"2023-10-09T05:09:58.832650+0000","last_clean":"2023-10-09T05:09:58.832650+0000","last_became_active":"2023-10-09T05:09:57.852039+0000","last_became_peered":"2023-10-09T05:09:57.852039+0000","last_unstale":"2023-10-09T05:09:58.832650+0000","last_undegraded":"2023-10-09T05:09:58.832650+0000","last_fullsized":"2023-10-09T05:09:58.832650+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:46:02.986058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":107,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.837725+0000","last_change":"2023-10-09T05:02:09.645017+0000","last_active":"2023-10-09T05:09:58.837725+0000","last_peered":"2023-10-09T05:09:58.837725+0000","last_clean":"2023-10-09T05:09:58.837725+0000","last_became_active":"2023-10-09T05:02:09.644705+0000","last_became_peered":"2023-10-09T05:02:09.644705+0000","last_unstale":"2023-10-09T05:09:58.837725+0000","last_undegraded":"2023-10-09T05:09:58.837725+0000","last_fullsized":"2023-10-09T05:09:58.837725+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:20.695961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":114,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.842806+0000","last_change":"2023-10-09T05:02:09.648362+0000","last_active":"2023-10-09T05:09:58.842806+0000","last_peered":"2023-10-09T05:09:58.842806+0000","last_clean":"2023-10-09T05:09:58.842806+0000","last_became_active":"2023-10-09T05:02:09.647127+0000","last_became_peered":"2023-10-09T05:02:09.647127+0000","last_unstale":"2023-10-09T05:09:58.842806+0000","last_undegraded":"2023-10-09T05:09:58.842806+0000","last_fullsized":"2023-10-09T05:09:58.842806+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:38:02.875613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":98,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.846953+0000","last_change":"2023-10-09T04:44:52.792557+0000","last_active":"2023-10-09T05:09:58.846953+0000","last_peered":"2023-10-09T05:09:58.846953+0000","last_clean":"2023-10-09T05:09:58.846953+0000","last_became_active":"2023-10-09T04:44:52.791890+0000","last_became_peered":"2023-10-09T04:44:52.791890+0000","last_unstale":"2023-10-09T05:09:58.846953+0000","last_undegraded":"2023-10-09T05:09:58.846953+0000","last_fullsized":"2023-10-09T05:09:58.846953+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:04:46.379852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":108,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.838961+0000","last_change":"2023-10-09T05:02:09.652158+0000","last_active":"2023-10-09T05:09:58.838961+0000","last_peered":"2023-10-09T05:09:58.838961+0000","last_clean":"2023-10-09T05:09:58.838961+0000","last_became_active":"2023-10-09T05:02:09.651811+0000","last_became_peered":"2023-10-09T05:02:09.651811+0000","last_unstale":"2023-10-09T05:09:58.838961+0000","last_undegraded":"2023-10-09T05:09:58.838961+0000","last_fullsized":"2023-10-09T05:09:58.838961+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:49:10.141237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":127,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.881853+0000","last_change":"2023-10-09T04:56:56.076177+0000","last_active":"2023-10-09T05:09:58.881853+0000","last_peered":"2023-10-09T05:09:58.881853+0000","last_clean":"2023-10-09T05:09:58.881853+0000","last_became_active":"2023-10-09T04:56:56.075564+0000","last_became_peered":"2023-10-09T04:56:56.075564+0000","last_unstale":"2023-10-09T05:09:58.881853+0000","last_undegraded":"2023-10-09T05:09:58.881853+0000","last_fullsized":"2023-10-09T05:09:58.881853+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:38.390793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":121,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.836881+0000","last_change":"2023-10-09T05:09:57.854475+0000","last_active":"2023-10-09T05:09:58.836881+0000","last_peered":"2023-10-09T05:09:58.836881+0000","last_clean":"2023-10-09T05:09:58.836881+0000","last_became_active":"2023-10-09T05:09:57.851782+0000","last_became_peered":"2023-10-09T05:09:57.851782+0000","last_unstale":"2023-10-09T05:09:58.836881+0000","last_undegraded":"2023-10-09T05:09:58.836881+0000","last_fullsized":"2023-10-09T05:09:58.836881+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:20:41.445868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"154'3782","reported_seq":3806,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-09T05:10:20.592606+0000","last_change":"2023-10-09T05:09:03.829573+0000","last_active":"2023-10-09T05:10:20.592606+0000","last_peered":"2023-10-09T05:10:20.592606+0000","last_clean":"2023-10-09T05:10:20.592606+0000","last_became_active":"2023-10-09T05:09:03.829205+0000","last_became_peered":"2023-10-09T05:09:03.829205+0000","last_unstale":"2023-10-09T05:10:20.592606+0000","last_undegraded":"2023-10-09T05:10:20.592606+0000","last_fullsized":"2023-10-09T05:10:20.592606+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":3782,"log_dups_size":0,"ondisk_log_size":3782,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:51.920206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247857152,"num_objects":3782,"num_object_clones":0,"num_object_copies":11346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3782,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3782,"num_write_kb":242048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":117,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.847367+0000","last_change":"2023-10-09T04:56:56.075307+0000","last_active":"2023-10-09T05:09:58.847367+0000","last_peered":"2023-10-09T05:09:58.847367+0000","last_clean":"2023-10-09T05:09:58.847367+0000","last_became_active":"2023-10-09T04:56:56.074831+0000","last_became_peered":"2023-10-09T04:56:56.074831+0000","last_unstale":"2023-10-09T05:09:58.847367+0000","last_undegraded":"2023-10-09T05:09:58.847367+0000","last_fullsized":"2023-10-09T05:09:58.847367+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:51:56.867759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":97,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.841803+0000","last_change":"2023-10-09T04:44:53.962437+0000","last_active":"2023-10-09T05:09:58.841803+0000","last_peered":"2023-10-09T05:09:58.841803+0000","last_clean":"2023-10-09T05:09:58.841803+0000","last_became_active":"2023-10-09T04:44:53.961810+0000","last_became_peered":"2023-10-09T04:44:53.961810+0000","last_unstale":"2023-10-09T05:09:58.841803+0000","last_undegraded":"2023-10-09T05:09:58.841803+0000","last_fullsized":"2023-10-09T05:09:58.841803+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:56.436933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"154'3839","reported_seq":5817,"reported_epoch":154,"state":"active+recovering+remapped","last_fresh":"2023-10-09T05:10:20.640610+0000","last_change":"2023-10-09T05:09:59.078160+0000","last_active":"2023-10-09T05:10:20.640610+0000","last_peered":"2023-10-09T05:10:20.640610+0000","last_clean":"2023-10-09T05:09:56.804034+0000","last_became_active":"2023-10-09T05:09:59.055939+0000","last_became_peered":"2023-10-09T05:09:59.055939+0000","last_unstale":"2023-10-09T05:10:20.640610+0000","last_undegraded":"2023-10-09T05:10:20.640610+0000","last_fullsized":"2023-10-09T05:10:20.640610+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":3839,"log_dups_size":0,"ondisk_log_size":3839,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:31:10.513696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251592704,"num_objects":3839,"num_object_clones":0,"num_object_copies":11517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1805,"num_objects_unfound":0,"num_objects_dirty":3839,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3839,"num_write_kb":245696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":963,"num_bytes_recovered":63111168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[0,4,3],"avail_no_missing":["0","3","4"],"object_location_counts":[{"shards":"0,3,4","objects":2034}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":146,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.848056+0000","last_change":"2023-10-09T05:09:57.898403+0000","last_active":"2023-10-09T05:09:58.848056+0000","last_peered":"2023-10-09T05:09:58.848056+0000","last_clean":"2023-10-09T05:09:58.848056+0000","last_became_active":"2023-10-09T05:09:57.866053+0000","last_became_peered":"2023-10-09T05:09:57.866053+0000","last_unstale":"2023-10-09T05:09:58.848056+0000","last_undegraded":"2023-10-09T05:09:58.848056+0000","last_fullsized":"2023-10-09T05:09:58.848056+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:31.760924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":151,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.875905+0000","last_change":"2023-10-09T05:09:57.868031+0000","last_active":"2023-10-09T05:09:58.875905+0000","last_peered":"2023-10-09T05:09:58.875905+0000","last_clean":"2023-10-09T05:09:58.875905+0000","last_became_active":"2023-10-09T05:09:57.866477+0000","last_became_peered":"2023-10-09T05:09:57.866477+0000","last_unstale":"2023-10-09T05:09:58.875905+0000","last_undegraded":"2023-10-09T05:09:58.875905+0000","last_fullsized":"2023-10-09T05:09:58.875905+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:27.102583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"154'3823","reported_seq":3846,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-09T05:10:19.368916+0000","last_change":"2023-10-09T05:09:03.830963+0000","last_active":"2023-10-09T05:10:19.368916+0000","last_peered":"2023-10-09T05:10:19.368916+0000","last_clean":"2023-10-09T05:10:19.368916+0000","last_became_active":"2023-10-09T05:09:03.830610+0000","last_became_peered":"2023-10-09T05:09:03.830610+0000","last_unstale":"2023-10-09T05:10:19.368916+0000","last_undegraded":"2023-10-09T05:10:19.368916+0000","last_fullsized":"2023-10-09T05:10:19.368916+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":3823,"log_dups_size":0,"ondisk_log_size":3823,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:48:37.895856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250544128,"num_objects":3823,"num_object_clones":0,"num_object_copies":11469,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3823,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3823,"num_write_kb":244672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":100,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.874669+0000","last_change":"2023-10-09T04:44:52.787469+0000","last_active":"2023-10-09T05:09:58.874669+0000","last_peered":"2023-10-09T05:09:58.874669+0000","last_clean":"2023-10-09T05:09:58.874669+0000","last_became_active":"2023-10-09T04:44:52.787148+0000","last_became_peered":"2023-10-09T04:44:52.787148+0000","last_unstale":"2023-10-09T05:09:58.874669+0000","last_undegraded":"2023-10-09T05:09:58.874669+0000","last_fullsized":"2023-10-09T05:09:58.874669+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:12.469065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":144,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.882277+0000","last_change":"2023-10-09T05:09:57.855670+0000","last_active":"2023-10-09T05:09:58.882277+0000","last_peered":"2023-10-09T05:09:58.882277+0000","last_clean":"2023-10-09T05:09:58.882277+0000","last_became_active":"2023-10-09T05:09:57.855293+0000","last_became_peered":"2023-10-09T05:09:57.855293+0000","last_unstale":"2023-10-09T05:09:58.882277+0000","last_undegraded":"2023-10-09T05:09:58.882277+0000","last_fullsized":"2023-10-09T05:09:58.882277+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:19:56.843974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"154'3753","reported_seq":4121,"reported_epoch":154,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:10:19.370858+0000","last_change":"2023-10-09T05:09:59.220535+0000","last_active":"2023-10-09T05:10:19.370858+0000","last_peered":"2023-10-09T05:10:19.370858+0000","last_clean":"2023-10-09T05:09:56.825417+0000","last_became_active":"2023-10-09T05:09:59.206075+0000","last_became_peered":"2023-10-09T05:09:59.206075+0000","last_unstale":"2023-10-09T05:10:19.370858+0000","last_undegraded":"2023-10-09T05:09:58.879029+0000","last_fullsized":"2023-10-09T05:09:58.842133+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":3753,"log_dups_size":0,"ondisk_log_size":3753,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:15:01.681332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245956608,"num_objects":3753,"num_object_clones":0,"num_object_copies":11259,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2631,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3753,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3753,"num_write_kb":240192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":157,"num_bytes_recovered":10289152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2],"avail_no_missing":["5","2","6"],"object_location_counts":[{"shards":"2,5","objects":1122},{"shards":"2,5,6","objects":2631}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":115,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.842224+0000","last_change":"2023-10-09T05:02:09.653932+0000","last_active":"2023-10-09T05:09:58.842224+0000","last_peered":"2023-10-09T05:09:58.842224+0000","last_clean":"2023-10-09T05:09:58.842224+0000","last_became_active":"2023-10-09T05:02:09.652923+0000","last_became_peered":"2023-10-09T05:02:09.652923+0000","last_unstale":"2023-10-09T05:09:58.842224+0000","last_undegraded":"2023-10-09T05:09:58.842224+0000","last_fullsized":"2023-10-09T05:09:58.842224+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:44:11.928237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":126,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.838574+0000","last_change":"2023-10-09T05:09:57.852914+0000","last_active":"2023-10-09T05:09:58.838574+0000","last_peered":"2023-10-09T05:09:58.838574+0000","last_clean":"2023-10-09T05:09:58.838574+0000","last_became_active":"2023-10-09T05:09:57.852522+0000","last_became_peered":"2023-10-09T05:09:57.852522+0000","last_unstale":"2023-10-09T05:09:58.838574+0000","last_undegraded":"2023-10-09T05:09:58.838574+0000","last_fullsized":"2023-10-09T05:09:58.838574+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:24.848494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"154'3890","reported_seq":3914,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-09T05:10:23.082088+0000","last_change":"2023-10-09T05:09:03.828954+0000","last_active":"2023-10-09T05:10:23.082088+0000","last_peered":"2023-10-09T05:10:23.082088+0000","last_clean":"2023-10-09T05:10:23.082088+0000","last_became_active":"2023-10-09T05:09:03.828532+0000","last_became_peered":"2023-10-09T05:09:03.828532+0000","last_unstale":"2023-10-09T05:10:23.082088+0000","last_undegraded":"2023-10-09T05:10:23.082088+0000","last_fullsized":"2023-10-09T05:10:23.082088+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":3890,"log_dups_size":0,"ondisk_log_size":3890,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:23.822369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254935040,"num_objects":3890,"num_object_clones":0,"num_object_copies":11670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3890,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3890,"num_write_kb":248960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"69'3","reported_seq":100,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.834515+0000","last_change":"2023-10-09T05:02:09.650579+0000","last_active":"2023-10-09T05:09:58.834515+0000","last_peered":"2023-10-09T05:09:58.834515+0000","last_clean":"2023-10-09T05:09:58.834515+0000","last_became_active":"2023-10-09T05:02:09.650160+0000","last_became_peered":"2023-10-09T05:02:09.650160+0000","last_unstale":"2023-10-09T05:09:58.834515+0000","last_undegraded":"2023-10-09T05:09:58.834515+0000","last_fullsized":"2023-10-09T05:09:58.834515+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:18.292670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":114,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.876122+0000","last_change":"2023-10-09T05:02:09.652449+0000","last_active":"2023-10-09T05:09:58.876122+0000","last_peered":"2023-10-09T05:09:58.876122+0000","last_clean":"2023-10-09T05:09:58.876122+0000","last_became_active":"2023-10-09T05:02:09.651994+0000","last_became_peered":"2023-10-09T05:02:09.651994+0000","last_unstale":"2023-10-09T05:09:58.876122+0000","last_undegraded":"2023-10-09T05:09:58.876122+0000","last_fullsized":"2023-10-09T05:09:58.876122+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:14.480513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"154'3866","reported_seq":4211,"reported_epoch":154,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-09T05:10:20.632059+0000","last_change":"2023-10-09T05:09:59.127832+0000","last_active":"2023-10-09T05:10:20.632059+0000","last_peered":"2023-10-09T05:10:20.632059+0000","last_clean":"2023-10-09T05:09:56.827891+0000","last_became_active":"2023-10-09T05:09:59.100161+0000","last_became_peered":"2023-10-09T05:09:59.100161+0000","last_unstale":"2023-10-09T05:10:20.632059+0000","last_undegraded":"2023-10-09T05:10:20.632059+0000","last_fullsized":"2023-10-09T05:10:20.632059+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":3866,"log_dups_size":0,"ondisk_log_size":3866,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:00.212739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253362176,"num_objects":3866,"num_object_clones":0,"num_object_copies":11598,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2721,"num_objects_unfound":0,"num_objects_dirty":3866,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3866,"num_write_kb":247424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":147,"num_bytes_recovered":9633792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[0,6,2],"avail_no_missing":["0","2","6"],"object_location_counts":[{"shards":"0,2,6","objects":1145}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":135,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.882360+0000","last_change":"2023-10-09T05:09:57.849784+0000","last_active":"2023-10-09T05:09:58.882360+0000","last_peered":"2023-10-09T05:09:58.882360+0000","last_clean":"2023-10-09T05:09:58.882360+0000","last_became_active":"2023-10-09T05:09:57.849216+0000","last_became_peered":"2023-10-09T05:09:57.849216+0000","last_unstale":"2023-10-09T05:09:58.882360+0000","last_undegraded":"2023-10-09T05:09:58.882360+0000","last_fullsized":"2023-10-09T05:09:58.882360+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:29.764343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":151,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.880555+0000","last_change":"2023-10-09T05:09:57.868658+0000","last_active":"2023-10-09T05:09:58.880555+0000","last_peered":"2023-10-09T05:09:58.880555+0000","last_clean":"2023-10-09T05:09:58.880555+0000","last_became_active":"2023-10-09T05:09:57.868221+0000","last_became_peered":"2023-10-09T05:09:57.868221+0000","last_unstale":"2023-10-09T05:09:58.880555+0000","last_undegraded":"2023-10-09T05:09:58.880555+0000","last_fullsized":"2023-10-09T05:09:58.880555+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:12:15.536917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"154'3621","reported_seq":3918,"reported_epoch":154,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:10:19.357488+0000","last_change":"2023-10-09T05:09:59.082468+0000","last_active":"2023-10-09T05:10:19.357488+0000","last_peered":"2023-10-09T05:10:19.357488+0000","last_clean":"2023-10-09T05:09:56.807871+0000","last_became_active":"2023-10-09T05:09:59.074831+0000","last_became_peered":"2023-10-09T05:09:59.074831+0000","last_unstale":"2023-10-09T05:10:19.357488+0000","last_undegraded":"2023-10-09T05:09:58.929479+0000","last_fullsized":"2023-10-09T05:09:58.845580+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":3621,"log_dups_size":0,"ondisk_log_size":3621,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:03.517929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237305856,"num_objects":3621,"num_object_clones":0,"num_object_copies":10863,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2546,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3621,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3621,"num_write_kb":231744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":122,"num_bytes_recovered":7995392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3],"avail_no_missing":["7","3","5"],"object_location_counts":[{"shards":"3,5,7","objects":2546},{"shards":"3,7","objects":1075}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":115,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.875826+0000","last_change":"2023-10-09T05:01:02.663309+0000","last_active":"2023-10-09T05:09:58.875826+0000","last_peered":"2023-10-09T05:09:58.875826+0000","last_clean":"2023-10-09T05:09:58.875826+0000","last_became_active":"2023-10-09T05:01:02.663038+0000","last_became_peered":"2023-10-09T05:01:02.663038+0000","last_unstale":"2023-10-09T05:09:58.875826+0000","last_undegraded":"2023-10-09T05:09:58.875826+0000","last_fullsized":"2023-10-09T05:09:58.875826+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:29:36.689304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":134,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.876324+0000","last_change":"2023-10-09T05:09:57.868378+0000","last_active":"2023-10-09T05:09:58.876324+0000","last_peered":"2023-10-09T05:09:58.876324+0000","last_clean":"2023-10-09T05:09:58.876324+0000","last_became_active":"2023-10-09T05:09:57.867507+0000","last_became_peered":"2023-10-09T05:09:57.867507+0000","last_unstale":"2023-10-09T05:09:58.876324+0000","last_undegraded":"2023-10-09T05:09:58.876324+0000","last_fullsized":"2023-10-09T05:09:58.876324+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:14:35.199631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"154'3819","reported_seq":4137,"reported_epoch":154,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:10:23.142388+0000","last_change":"2023-10-09T05:09:59.104712+0000","last_active":"2023-10-09T05:10:23.142388+0000","last_peered":"2023-10-09T05:10:23.142388+0000","last_clean":"2023-10-09T05:09:56.749891+0000","last_became_active":"2023-10-09T05:09:59.097963+0000","last_became_peered":"2023-10-09T05:09:59.097963+0000","last_unstale":"2023-10-09T05:10:23.142388+0000","last_undegraded":"2023-10-09T05:09:58.854922+0000","last_fullsized":"2023-10-09T05:09:58.832139+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":3819,"log_dups_size":0,"ondisk_log_size":3819,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:40.209073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250281984,"num_objects":3819,"num_object_clones":0,"num_object_copies":11457,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2507,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3819,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3819,"num_write_kb":244416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":134,"num_bytes_recovered":8781824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,2],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3","objects":1312},{"shards":"2,3,7","objects":2507}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":122,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.831898+0000","last_change":"2023-10-09T05:09:57.852151+0000","last_active":"2023-10-09T05:09:58.831898+0000","last_peered":"2023-10-09T05:09:58.831898+0000","last_clean":"2023-10-09T05:09:58.831898+0000","last_became_active":"2023-10-09T05:09:57.851499+0000","last_became_peered":"2023-10-09T05:09:57.851499+0000","last_unstale":"2023-10-09T05:09:58.831898+0000","last_undegraded":"2023-10-09T05:09:58.831898+0000","last_fullsized":"2023-10-09T05:09:58.831898+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:25:21.153305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:10:01.322011+0000","last_change":"2023-10-09T05:09:57.856764+0000","last_active":"2023-10-09T05:10:01.322011+0000","last_peered":"2023-10-09T05:10:01.322011+0000","last_clean":"2023-10-09T05:10:01.322011+0000","last_became_active":"2023-10-09T05:09:57.856461+0000","last_became_peered":"2023-10-09T05:09:57.856461+0000","last_unstale":"2023-10-09T05:10:01.322011+0000","last_undegraded":"2023-10-09T05:10:01.322011+0000","last_fullsized":"2023-10-09T05:10:01.322011+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:06.848082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"154'3821","reported_seq":4137,"reported_epoch":154,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:10:19.381277+0000","last_change":"2023-10-09T05:09:59.115413+0000","last_active":"2023-10-09T05:10:19.381277+0000","last_peered":"2023-10-09T05:10:19.381277+0000","last_clean":"2023-10-09T05:09:56.775881+0000","last_became_active":"2023-10-09T05:09:59.102861+0000","last_became_peered":"2023-10-09T05:09:59.102861+0000","last_unstale":"2023-10-09T05:10:19.381277+0000","last_undegraded":"2023-10-09T05:09:58.877226+0000","last_fullsized":"2023-10-09T05:09:58.842223+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":3821,"log_dups_size":0,"ondisk_log_size":3821,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:24:23.156903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250413056,"num_objects":3821,"num_object_clones":0,"num_object_copies":11463,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2696,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3821,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3821,"num_write_kb":244544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":131,"num_bytes_recovered":8585216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,3],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":2696},{"shards":"3,5","objects":1125}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":13,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:10:01.321929+0000","last_change":"2023-10-09T05:09:57.849066+0000","last_active":"2023-10-09T05:10:01.321929+0000","last_peered":"2023-10-09T05:10:01.321929+0000","last_clean":"2023-10-09T05:10:01.321929+0000","last_became_active":"2023-10-09T05:09:57.848717+0000","last_became_peered":"2023-10-09T05:09:57.848717+0000","last_unstale":"2023-10-09T05:10:01.321929+0000","last_undegraded":"2023-10-09T05:10:01.321929+0000","last_fullsized":"2023-10-09T05:10:01.321929+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:31.267804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":132,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.842960+0000","last_change":"2023-10-09T05:02:09.649559+0000","last_active":"2023-10-09T05:09:58.842960+0000","last_peered":"2023-10-09T05:09:58.842960+0000","last_clean":"2023-10-09T05:09:58.842960+0000","last_became_active":"2023-10-09T05:02:09.649195+0000","last_became_peered":"2023-10-09T05:02:09.649195+0000","last_unstale":"2023-10-09T05:09:58.842960+0000","last_undegraded":"2023-10-09T05:09:58.842960+0000","last_fullsized":"2023-10-09T05:09:58.842960+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:56.864170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:10:01.321913+0000","last_change":"2023-10-09T05:09:57.865132+0000","last_active":"2023-10-09T05:10:01.321913+0000","last_peered":"2023-10-09T05:10:01.321913+0000","last_clean":"2023-10-09T05:10:01.321913+0000","last_became_active":"2023-10-09T05:09:57.864773+0000","last_became_peered":"2023-10-09T05:09:57.864773+0000","last_unstale":"2023-10-09T05:10:01.321913+0000","last_undegraded":"2023-10-09T05:10:01.321913+0000","last_fullsized":"2023-10-09T05:10:01.321913+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:04.194629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"154'3834","reported_seq":3858,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-09T05:10:20.619855+0000","last_change":"2023-10-09T05:09:03.830161+0000","last_active":"2023-10-09T05:10:20.619855+0000","last_peered":"2023-10-09T05:10:20.619855+0000","last_clean":"2023-10-09T05:10:20.619855+0000","last_became_active":"2023-10-09T05:09:03.829750+0000","last_became_peered":"2023-10-09T05:09:03.829750+0000","last_unstale":"2023-10-09T05:10:20.619855+0000","last_undegraded":"2023-10-09T05:10:20.619855+0000","last_fullsized":"2023-10-09T05:10:20.619855+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":3834,"log_dups_size":0,"ondisk_log_size":3834,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:28.876874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251265024,"num_objects":3834,"num_object_clones":0,"num_object_copies":11502,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3834,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3834,"num_write_kb":245376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":139,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.874864+0000","last_change":"2023-10-09T05:09:57.871040+0000","last_active":"2023-10-09T05:09:58.874864+0000","last_peered":"2023-10-09T05:09:58.874864+0000","last_clean":"2023-10-09T05:09:58.874864+0000","last_became_active":"2023-10-09T05:09:57.866399+0000","last_became_peered":"2023-10-09T05:09:57.866399+0000","last_unstale":"2023-10-09T05:09:58.874864+0000","last_undegraded":"2023-10-09T05:09:58.874864+0000","last_fullsized":"2023-10-09T05:09:58.874864+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:42.545057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":115,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.876190+0000","last_change":"2023-10-09T04:56:56.087320+0000","last_active":"2023-10-09T05:09:58.876190+0000","last_peered":"2023-10-09T05:09:58.876190+0000","last_clean":"2023-10-09T05:09:58.876190+0000","last_became_active":"2023-10-09T04:56:56.086552+0000","last_became_peered":"2023-10-09T04:56:56.086552+0000","last_unstale":"2023-10-09T05:09:58.876190+0000","last_undegraded":"2023-10-09T05:09:58.876190+0000","last_fullsized":"2023-10-09T05:09:58.876190+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:14.205288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:10:01.321945+0000","last_change":"2023-10-09T05:09:57.843174+0000","last_active":"2023-10-09T05:10:01.321945+0000","last_peered":"2023-10-09T05:10:01.321945+0000","last_clean":"2023-10-09T05:10:01.321945+0000","last_became_active":"2023-10-09T05:09:57.842818+0000","last_became_peered":"2023-10-09T05:09:57.842818+0000","last_unstale":"2023-10-09T05:10:01.321945+0000","last_undegraded":"2023-10-09T05:10:01.321945+0000","last_fullsized":"2023-10-09T05:10:01.321945+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:10.285164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"154'3835","reported_seq":4187,"reported_epoch":154,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:10:19.373451+0000","last_change":"2023-10-09T05:09:59.108285+0000","last_active":"2023-10-09T05:10:19.373451+0000","last_peered":"2023-10-09T05:10:19.373451+0000","last_clean":"2023-10-09T05:09:56.806054+0000","last_became_active":"2023-10-09T05:09:59.097985+0000","last_became_peered":"2023-10-09T05:09:59.097985+0000","last_unstale":"2023-10-09T05:10:19.373451+0000","last_undegraded":"2023-10-09T05:09:58.873107+0000","last_fullsized":"2023-10-09T05:09:58.841848+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":3835,"log_dups_size":0,"ondisk_log_size":3835,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:44:00.189725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251330560,"num_objects":3835,"num_object_clones":0,"num_object_copies":11505,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2719,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3835,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3835,"num_write_kb":245440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":149,"num_bytes_recovered":9764864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,7],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":2719},{"shards":"5,7","objects":1116}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":133,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.858416+0000","last_change":"2023-10-09T05:02:09.645835+0000","last_active":"2023-10-09T05:09:58.858416+0000","last_peered":"2023-10-09T05:09:58.858416+0000","last_clean":"2023-10-09T05:09:58.858416+0000","last_became_active":"2023-10-09T05:02:09.645491+0000","last_became_peered":"2023-10-09T05:02:09.645491+0000","last_unstale":"2023-10-09T05:09:58.858416+0000","last_undegraded":"2023-10-09T05:09:58.858416+0000","last_fullsized":"2023-10-09T05:09:58.858416+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:06:20.823201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:10:01.321854+0000","last_change":"2023-10-09T05:09:57.854064+0000","last_active":"2023-10-09T05:10:01.321854+0000","last_peered":"2023-10-09T05:10:01.321854+0000","last_clean":"2023-10-09T05:10:01.321854+0000","last_became_active":"2023-10-09T05:09:57.847953+0000","last_became_peered":"2023-10-09T05:09:57.847953+0000","last_unstale":"2023-10-09T05:10:01.321854+0000","last_undegraded":"2023-10-09T05:10:01.321854+0000","last_fullsized":"2023-10-09T05:10:01.321854+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:45.914976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:10:01.321996+0000","last_change":"2023-10-09T05:09:57.865256+0000","last_active":"2023-10-09T05:10:01.321996+0000","last_peered":"2023-10-09T05:10:01.321996+0000","last_clean":"2023-10-09T05:10:01.321996+0000","last_became_active":"2023-10-09T05:09:57.864756+0000","last_became_peered":"2023-10-09T05:09:57.864756+0000","last_unstale":"2023-10-09T05:10:01.321996+0000","last_undegraded":"2023-10-09T05:10:01.321996+0000","last_fullsized":"2023-10-09T05:10:01.321996+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:14.071437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"154'3899","reported_seq":3923,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-09T05:10:23.084709+0000","last_change":"2023-10-09T05:09:03.827371+0000","last_active":"2023-10-09T05:10:23.084709+0000","last_peered":"2023-10-09T05:10:23.084709+0000","last_clean":"2023-10-09T05:10:23.084709+0000","last_became_active":"2023-10-09T05:09:03.827089+0000","last_became_peered":"2023-10-09T05:09:03.827089+0000","last_unstale":"2023-10-09T05:10:23.084709+0000","last_undegraded":"2023-10-09T05:10:23.084709+0000","last_fullsized":"2023-10-09T05:10:23.084709+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":3899,"log_dups_size":0,"ondisk_log_size":3899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:04:02.318218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255524864,"num_objects":3899,"num_object_clones":0,"num_object_copies":11697,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3899,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3899,"num_write_kb":249536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":119,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.842311+0000","last_change":"2023-10-09T05:01:02.685038+0000","last_active":"2023-10-09T05:09:58.842311+0000","last_peered":"2023-10-09T05:09:58.842311+0000","last_clean":"2023-10-09T05:09:58.842311+0000","last_became_active":"2023-10-09T05:01:02.683622+0000","last_became_peered":"2023-10-09T05:01:02.683622+0000","last_unstale":"2023-10-09T05:09:58.842311+0000","last_undegraded":"2023-10-09T05:09:58.842311+0000","last_fullsized":"2023-10-09T05:09:58.842311+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:35.435592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":492,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.846705+0000","last_change":"2023-10-09T04:56:56.075762+0000","last_active":"2023-10-09T05:09:58.846705+0000","last_peered":"2023-10-09T05:09:58.846705+0000","last_clean":"2023-10-09T05:09:58.846705+0000","last_became_active":"2023-10-09T04:56:56.075263+0000","last_became_peered":"2023-10-09T04:56:56.075263+0000","last_unstale":"2023-10-09T05:09:58.846705+0000","last_undegraded":"2023-10-09T05:09:58.846705+0000","last_fullsized":"2023-10-09T05:09:58.846705+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_clean_scrub_stamp":"2023-10-09T04:36:44.261013+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:02.769164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":98,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.833569+0000","last_change":"2023-10-09T05:02:09.654237+0000","last_active":"2023-10-09T05:09:58.833569+0000","last_peered":"2023-10-09T05:09:58.833569+0000","last_clean":"2023-10-09T05:09:58.833569+0000","last_became_active":"2023-10-09T05:02:09.653909+0000","last_became_peered":"2023-10-09T05:02:09.653909+0000","last_unstale":"2023-10-09T05:09:58.833569+0000","last_undegraded":"2023-10-09T05:09:58.833569+0000","last_fullsized":"2023-10-09T05:09:58.833569+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:02.002366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":97,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.882649+0000","last_change":"2023-10-09T04:44:53.959705+0000","last_active":"2023-10-09T05:09:58.882649+0000","last_peered":"2023-10-09T05:09:58.882649+0000","last_clean":"2023-10-09T05:09:58.882649+0000","last_became_active":"2023-10-09T04:44:53.959092+0000","last_became_peered":"2023-10-09T04:44:53.959092+0000","last_unstale":"2023-10-09T05:09:58.882649+0000","last_undegraded":"2023-10-09T05:09:58.882649+0000","last_fullsized":"2023-10-09T05:09:58.882649+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:31:29.189897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"154'3846","reported_seq":3870,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-09T05:10:20.616859+0000","last_change":"2023-10-09T05:09:03.826421+0000","last_active":"2023-10-09T05:10:20.616859+0000","last_peered":"2023-10-09T05:10:20.616859+0000","last_clean":"2023-10-09T05:10:20.616859+0000","last_became_active":"2023-10-09T05:09:03.826107+0000","last_became_peered":"2023-10-09T05:09:03.826107+0000","last_unstale":"2023-10-09T05:10:20.616859+0000","last_undegraded":"2023-10-09T05:10:20.616859+0000","last_fullsized":"2023-10-09T05:10:20.616859+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":3846,"log_dups_size":0,"ondisk_log_size":3846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:53:42.881604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252051456,"num_objects":3846,"num_object_clones":0,"num_object_copies":11538,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3846,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3846,"num_write_kb":246144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":156,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.846582+0000","last_change":"2023-10-09T05:09:57.855069+0000","last_active":"2023-10-09T05:09:58.846582+0000","last_peered":"2023-10-09T05:09:58.846582+0000","last_clean":"2023-10-09T05:09:58.846582+0000","last_became_active":"2023-10-09T05:09:57.854693+0000","last_became_peered":"2023-10-09T05:09:57.854693+0000","last_unstale":"2023-10-09T05:09:58.846582+0000","last_undegraded":"2023-10-09T05:09:58.846582+0000","last_fullsized":"2023-10-09T05:09:58.846582+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:21:27.305940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":108,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.837295+0000","last_change":"2023-10-09T05:02:09.648781+0000","last_active":"2023-10-09T05:09:58.837295+0000","last_peered":"2023-10-09T05:09:58.837295+0000","last_clean":"2023-10-09T05:09:58.837295+0000","last_became_active":"2023-10-09T05:02:09.648078+0000","last_became_peered":"2023-10-09T05:02:09.648078+0000","last_unstale":"2023-10-09T05:09:58.837295+0000","last_undegraded":"2023-10-09T05:09:58.837295+0000","last_fullsized":"2023-10-09T05:09:58.837295+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:35.287091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":142,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.838593+0000","last_change":"2023-10-09T05:09:57.847731+0000","last_active":"2023-10-09T05:09:58.838593+0000","last_peered":"2023-10-09T05:09:58.838593+0000","last_clean":"2023-10-09T05:09:58.838593+0000","last_became_active":"2023-10-09T05:09:57.847359+0000","last_became_peered":"2023-10-09T05:09:57.847359+0000","last_unstale":"2023-10-09T05:09:58.838593+0000","last_undegraded":"2023-10-09T05:09:58.838593+0000","last_fullsized":"2023-10-09T05:09:58.838593+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:43:30.260279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"154'3833","reported_seq":4152,"reported_epoch":154,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:10:19.246497+0000","last_change":"2023-10-09T05:09:59.091678+0000","last_active":"2023-10-09T05:10:19.246497+0000","last_peered":"2023-10-09T05:10:19.246497+0000","last_clean":"2023-10-09T05:09:56.834673+0000","last_became_active":"2023-10-09T05:09:59.086420+0000","last_became_peered":"2023-10-09T05:09:59.086420+0000","last_unstale":"2023-10-09T05:10:19.246497+0000","last_undegraded":"2023-10-09T05:09:58.874557+0000","last_fullsized":"2023-10-09T05:09:58.845617+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":3833,"log_dups_size":0,"ondisk_log_size":3833,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:13.864729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251199488,"num_objects":3833,"num_object_clones":0,"num_object_copies":11499,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2673,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3833,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3833,"num_write_kb":245312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":133,"num_bytes_recovered":8716288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4],"avail_no_missing":["7","0","4"],"object_location_counts":[{"shards":"0,4,7","objects":2673},{"shards":"4,7","objects":1160}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":119,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.874557+0000","last_change":"2023-10-09T04:56:56.073473+0000","last_active":"2023-10-09T05:09:58.874557+0000","last_peered":"2023-10-09T05:09:58.874557+0000","last_clean":"2023-10-09T05:09:58.874557+0000","last_became_active":"2023-10-09T04:56:56.073146+0000","last_became_peered":"2023-10-09T04:56:56.073146+0000","last_unstale":"2023-10-09T05:09:58.874557+0000","last_undegraded":"2023-10-09T05:09:58.874557+0000","last_fullsized":"2023-10-09T05:09:58.874557+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:55:25.132822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:10:01.321963+0000","last_change":"2023-10-09T05:09:57.864289+0000","last_active":"2023-10-09T05:10:01.321963+0000","last_peered":"2023-10-09T05:10:01.321963+0000","last_clean":"2023-10-09T05:10:01.321963+0000","last_became_active":"2023-10-09T05:09:57.863963+0000","last_became_peered":"2023-10-09T05:09:57.863963+0000","last_unstale":"2023-10-09T05:10:01.321963+0000","last_undegraded":"2023-10-09T05:10:01.321963+0000","last_fullsized":"2023-10-09T05:10:01.321963+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:45:02.500193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":94,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.838556+0000","last_change":"2023-10-09T04:56:56.071763+0000","last_active":"2023-10-09T05:09:58.838556+0000","last_peered":"2023-10-09T05:09:58.838556+0000","last_clean":"2023-10-09T05:09:58.838556+0000","last_became_active":"2023-10-09T04:56:56.071429+0000","last_became_peered":"2023-10-09T04:56:56.071429+0000","last_unstale":"2023-10-09T05:09:58.838556+0000","last_undegraded":"2023-10-09T05:09:58.838556+0000","last_fullsized":"2023-10-09T05:09:58.838556+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:53:02.051460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"154'3715","reported_seq":4050,"reported_epoch":154,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:10:19.332637+0000","last_change":"2023-10-09T05:09:59.076651+0000","last_active":"2023-10-09T05:10:19.332637+0000","last_peered":"2023-10-09T05:10:19.332637+0000","last_clean":"2023-10-09T05:09:56.818963+0000","last_became_active":"2023-10-09T05:09:59.069993+0000","last_became_peered":"2023-10-09T05:09:59.069993+0000","last_unstale":"2023-10-09T05:10:19.332637+0000","last_undegraded":"2023-10-09T05:09:58.873311+0000","last_fullsized":"2023-10-09T05:09:58.845731+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":3715,"log_dups_size":0,"ondisk_log_size":3715,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:12.318397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243466240,"num_objects":3715,"num_object_clones":0,"num_object_copies":11145,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2628,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3715,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3715,"num_write_kb":237760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":141,"num_bytes_recovered":9240576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,5],"avail_no_missing":["7","4","5"],"object_location_counts":[{"shards":"4,5,7","objects":2628},{"shards":"5,7","objects":1087}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":99,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.837287+0000","last_change":"2023-10-09T05:02:09.651149+0000","last_active":"2023-10-09T05:09:58.837287+0000","last_peered":"2023-10-09T05:09:58.837287+0000","last_clean":"2023-10-09T05:09:58.837287+0000","last_became_active":"2023-10-09T05:02:09.650151+0000","last_became_peered":"2023-10-09T05:02:09.650151+0000","last_unstale":"2023-10-09T05:09:58.837287+0000","last_undegraded":"2023-10-09T05:09:58.837287+0000","last_fullsized":"2023-10-09T05:09:58.837287+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:49.100209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":13,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:10:01.321979+0000","last_change":"2023-10-09T05:09:57.858617+0000","last_active":"2023-10-09T05:10:01.321979+0000","last_peered":"2023-10-09T05:10:01.321979+0000","last_clean":"2023-10-09T05:10:01.321979+0000","last_became_active":"2023-10-09T05:09:57.858128+0000","last_became_peered":"2023-10-09T05:09:57.858128+0000","last_unstale":"2023-10-09T05:10:01.321979+0000","last_undegraded":"2023-10-09T05:10:01.321979+0000","last_fullsized":"2023-10-09T05:10:01.321979+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:02.502985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":96,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:10:08.145954+0000","last_change":"2023-10-09T05:02:09.650750+0000","last_active":"2023-10-09T05:10:08.145954+0000","last_peered":"2023-10-09T05:10:08.145954+0000","last_clean":"2023-10-09T05:10:08.145954+0000","last_became_active":"2023-10-09T05:02:09.650351+0000","last_became_peered":"2023-10-09T05:02:09.650351+0000","last_unstale":"2023-10-09T05:10:08.145954+0000","last_undegraded":"2023-10-09T05:10:08.145954+0000","last_fullsized":"2023-10-09T05:10:08.145954+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:14.826132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"154'4032","reported_seq":4056,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-09T05:10:23.104376+0000","last_change":"2023-10-09T05:09:04.054690+0000","last_active":"2023-10-09T05:10:23.104376+0000","last_peered":"2023-10-09T05:10:23.104376+0000","last_clean":"2023-10-09T05:10:23.104376+0000","last_became_active":"2023-10-09T05:09:04.054340+0000","last_became_peered":"2023-10-09T05:09:04.054340+0000","last_unstale":"2023-10-09T05:10:23.104376+0000","last_undegraded":"2023-10-09T05:10:23.104376+0000","last_fullsized":"2023-10-09T05:10:23.104376+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":4032,"log_dups_size":0,"ondisk_log_size":4032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:19:34.276593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264241152,"num_objects":4032,"num_object_clones":0,"num_object_copies":12096,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4032,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4032,"num_write_kb":258048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":119,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.882650+0000","last_change":"2023-10-09T05:01:02.660775+0000","last_active":"2023-10-09T05:09:58.882650+0000","last_peered":"2023-10-09T05:09:58.882650+0000","last_clean":"2023-10-09T05:09:58.882650+0000","last_became_active":"2023-10-09T05:01:02.660420+0000","last_became_peered":"2023-10-09T05:01:02.660420+0000","last_unstale":"2023-10-09T05:09:58.882650+0000","last_undegraded":"2023-10-09T05:09:58.882650+0000","last_fullsized":"2023-10-09T05:09:58.882650+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:54.031869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":139,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.838728+0000","last_change":"2023-10-09T05:09:57.878002+0000","last_active":"2023-10-09T05:09:58.838728+0000","last_peered":"2023-10-09T05:09:58.838728+0000","last_clean":"2023-10-09T05:09:58.838728+0000","last_became_active":"2023-10-09T05:09:57.852792+0000","last_became_peered":"2023-10-09T05:09:57.852792+0000","last_unstale":"2023-10-09T05:09:58.838728+0000","last_undegraded":"2023-10-09T05:09:58.838728+0000","last_fullsized":"2023-10-09T05:09:58.838728+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:18.763275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":97,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.875718+0000","last_change":"2023-10-09T04:44:53.792412+0000","last_active":"2023-10-09T05:09:58.875718+0000","last_peered":"2023-10-09T05:09:58.875718+0000","last_clean":"2023-10-09T05:09:58.875718+0000","last_became_active":"2023-10-09T04:44:53.792009+0000","last_became_peered":"2023-10-09T04:44:53.792009+0000","last_unstale":"2023-10-09T05:09:58.875718+0000","last_undegraded":"2023-10-09T05:09:58.875718+0000","last_fullsized":"2023-10-09T05:09:58.875718+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:39.423339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'2","reported_seq":98,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.838755+0000","last_change":"2023-10-09T04:56:56.079768+0000","last_active":"2023-10-09T05:09:58.838755+0000","last_peered":"2023-10-09T05:09:58.838755+0000","last_clean":"2023-10-09T05:09:58.838755+0000","last_became_active":"2023-10-09T04:56:56.077721+0000","last_became_peered":"2023-10-09T04:56:56.077721+0000","last_unstale":"2023-10-09T05:09:58.838755+0000","last_undegraded":"2023-10-09T05:09:58.838755+0000","last_fullsized":"2023-10-09T05:09:58.838755+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:36.384357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":153,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.860002+0000","last_change":"2023-10-09T05:09:57.842586+0000","last_active":"2023-10-09T05:09:58.860002+0000","last_peered":"2023-10-09T05:09:58.860002+0000","last_clean":"2023-10-09T05:09:58.860002+0000","last_became_active":"2023-10-09T05:09:57.842214+0000","last_became_peered":"2023-10-09T05:09:57.842214+0000","last_unstale":"2023-10-09T05:09:58.860002+0000","last_undegraded":"2023-10-09T05:09:58.860002+0000","last_fullsized":"2023-10-09T05:09:58.860002+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:26.074875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":146,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.842393+0000","last_change":"2023-10-09T05:09:57.901164+0000","last_active":"2023-10-09T05:09:58.842393+0000","last_peered":"2023-10-09T05:09:58.842393+0000","last_clean":"2023-10-09T05:09:58.842393+0000","last_became_active":"2023-10-09T05:09:57.867734+0000","last_became_peered":"2023-10-09T05:09:57.867734+0000","last_unstale":"2023-10-09T05:09:58.842393+0000","last_undegraded":"2023-10-09T05:09:58.842393+0000","last_fullsized":"2023-10-09T05:09:58.842393+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:46:30.541419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":151,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.876374+0000","last_change":"2023-10-09T05:09:57.867865+0000","last_active":"2023-10-09T05:09:58.876374+0000","last_peered":"2023-10-09T05:09:58.876374+0000","last_clean":"2023-10-09T05:09:58.876374+0000","last_became_active":"2023-10-09T05:09:57.866723+0000","last_became_peered":"2023-10-09T05:09:57.866723+0000","last_unstale":"2023-10-09T05:09:58.876374+0000","last_undegraded":"2023-10-09T05:09:58.876374+0000","last_fullsized":"2023-10-09T05:09:58.876374+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:58.946358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":101,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.843171+0000","last_change":"2023-10-09T04:44:52.793709+0000","last_active":"2023-10-09T05:09:58.843171+0000","last_peered":"2023-10-09T05:09:58.843171+0000","last_clean":"2023-10-09T05:09:58.843171+0000","last_became_active":"2023-10-09T04:44:52.792973+0000","last_became_peered":"2023-10-09T04:44:52.792973+0000","last_unstale":"2023-10-09T05:09:58.843171+0000","last_undegraded":"2023-10-09T05:09:58.843171+0000","last_fullsized":"2023-10-09T05:09:58.843171+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:12.091089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":131,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.875807+0000","last_change":"2023-10-09T05:02:09.649748+0000","last_active":"2023-10-09T05:09:58.875807+0000","last_peered":"2023-10-09T05:09:58.875807+0000","last_clean":"2023-10-09T05:09:58.875807+0000","last_became_active":"2023-10-09T05:02:09.649444+0000","last_became_peered":"2023-10-09T05:02:09.649444+0000","last_unstale":"2023-10-09T05:09:58.875807+0000","last_undegraded":"2023-10-09T05:09:58.875807+0000","last_fullsized":"2023-10-09T05:09:58.875807+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:07.436820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'2","reported_seq":117,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:10:08.153554+0000","last_change":"2023-10-09T05:02:09.647382+0000","last_active":"2023-10-09T05:10:08.153554+0000","last_peered":"2023-10-09T05:10:08.153554+0000","last_clean":"2023-10-09T05:10:08.153554+0000","last_became_active":"2023-10-09T05:02:09.646803+0000","last_became_peered":"2023-10-09T05:02:09.646803+0000","last_unstale":"2023-10-09T05:10:08.153554+0000","last_undegraded":"2023-10-09T05:10:08.153554+0000","last_fullsized":"2023-10-09T05:10:08.153554+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:23:00.161736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":130,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.836897+0000","last_change":"2023-10-09T05:02:09.649674+0000","last_active":"2023-10-09T05:09:58.836897+0000","last_peered":"2023-10-09T05:09:58.836897+0000","last_clean":"2023-10-09T05:09:58.836897+0000","last_became_active":"2023-10-09T05:02:09.649041+0000","last_became_peered":"2023-10-09T05:02:09.649041+0000","last_unstale":"2023-10-09T05:09:58.836897+0000","last_undegraded":"2023-10-09T05:09:58.836897+0000","last_fullsized":"2023-10-09T05:09:58.836897+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:18.524071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'5","reported_seq":118,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.839227+0000","last_change":"2023-10-09T05:01:02.672596+0000","last_active":"2023-10-09T05:09:58.839227+0000","last_peered":"2023-10-09T05:09:58.839227+0000","last_clean":"2023-10-09T05:09:58.839227+0000","last_became_active":"2023-10-09T05:01:02.672128+0000","last_became_peered":"2023-10-09T05:01:02.672128+0000","last_unstale":"2023-10-09T05:09:58.839227+0000","last_undegraded":"2023-10-09T05:09:58.839227+0000","last_fullsized":"2023-10-09T05:09:58.839227+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:17:05.887502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":134,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:09:58.875597+0000","last_change":"2023-10-09T05:09:57.866732+0000","last_active":"2023-10-09T05:09:58.875597+0000","last_peered":"2023-10-09T05:09:58.875597+0000","last_clean":"2023-10-09T05:09:58.875597+0000","last_became_active":"2023-10-09T05:09:57.866117+0000","last_became_peered":"2023-10-09T05:09:57.866117+0000","last_unstale":"2023-10-09T05:09:58.875597+0000","last_undegraded":"2023-10-09T05:09:58.875597+0000","last_fullsized":"2023-10-09T05:09:58.875597+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:49:14.815342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":38,"num_read_kb":40,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":4011327488,"num_objects":61208,"num_object_clones":0,"num_object_copies":183624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18400,"num_objects_misplaced":4526,"num_objects_unfound":0,"num_objects_dirty":61208,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61208,"num_write_kb":3917312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2077,"num_bytes_recovered":136118272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12230197248,"data_stored":12230197248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":61208,"ondisk_log_size":61208,"up":48,"acting":41,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234337,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2617340,"kb_used_data":2311860,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":91127812,"statfs":{"total":95995035648,"available":93314879488,"internally_reserved":0,"allocated":2367344640,"data_stored":2366722998,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:09:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":1.385,"15min":1.393},"min":{"1min":0.69299999999999995,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":1.361,"5min":11.093999999999999,"15min":11.093999999999999},"last":0.36399999999999999},{"interface":"front","average":{"1min":1.0329999999999999,"5min":1.4730000000000001,"15min":1.617},"min":{"1min":0.60999999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.423,"5min":6.3730000000000002,"15min":9.5090000000000003},"last":0.496}]},{"osd":1,"last update":"Mon Oct 9 05:10:09 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":1.1519999999999999,"15min":1.9550000000000001},"min":{"1min":0.38400000000000001,"5min":0.307,"15min":0.28699999999999998},"max":{"1min":2.1579999999999999,"5min":6.7210000000000001,"15min":135.322},"last":0.29699999999999999},{"interface":"front","average":{"1min":2.1120000000000001,"5min":1.393,"15min":2.1739999999999999},"min":{"1min":0.45200000000000001,"5min":0.35599999999999998,"15min":0.30499999999999999},"max":{"1min":9.8870000000000005,"5min":9.8870000000000005,"15min":135.09},"last":0.67900000000000005}]},{"osd":2,"last update":"Mon Oct 9 05:09:22 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":1.4399999999999999,"15min":1.3280000000000001},"min":{"1min":0.53700000000000003,"5min":0.315,"15min":0.315},"max":{"1min":1.143,"5min":11.391,"15min":11.391},"last":1.8069999999999999},{"interface":"front","average":{"1min":0.90900000000000003,"5min":1.4490000000000001,"15min":1.244},"min":{"1min":0.623,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.522,"5min":6.6820000000000004,"15min":6.6820000000000004},"last":2.3359999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:09:22 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":1.3320000000000001,"15min":1.23},"min":{"1min":0.55900000000000005,"5min":0.317,"15min":0.317},"max":{"1min":1.157,"5min":6.548,"15min":6.548},"last":5.3440000000000003},{"interface":"front","average":{"1min":1.107,"5min":1.619,"15min":1.633},"min":{"1min":0.76900000000000002,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.387,"5min":9.1129999999999995,"15min":9.1129999999999995},"last":0.55900000000000005}]},{"osd":4,"last update":"Mon Oct 9 05:10:09 2023","interfaces":[{"interface":"back","average":{"1min":0.75700000000000001,"5min":1.147,"15min":1.0740000000000001},"min":{"1min":0.33900000000000002,"5min":0.33700000000000002,"15min":0.28699999999999998},"max":{"1min":2.0920000000000001,"5min":6.9880000000000004,"15min":11.409000000000001},"last":0.40699999999999997},{"interface":"front","average":{"1min":2.2210000000000001,"5min":1.4770000000000001,"15min":1.4470000000000001},"min":{"1min":0.41199999999999998,"5min":0.38800000000000001,"15min":0.318},"max":{"1min":9.8620000000000001,"5min":9.8620000000000001,"15min":15.763},"last":0.64000000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:10:09 2023","interfaces":[{"interface":"back","average":{"1min":1.5109999999999999,"5min":1.536,"15min":1.351},"min":{"1min":0.42999999999999999,"5min":0.38200000000000001,"15min":0.34399999999999997},"max":{"1min":3.1139999999999999,"5min":7.6180000000000003,"15min":8.3569999999999993},"last":2.3780000000000001},{"interface":"front","average":{"1min":1.6279999999999999,"5min":1.464,"15min":1.3049999999999999},"min":{"1min":0.54200000000000004,"5min":0.35999999999999999,"15min":0.32900000000000001},"max":{"1min":7.1580000000000004,"5min":7.1580000000000004,"15min":7.1580000000000004},"last":0.44700000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:10:09 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.3779999999999999,"15min":1.256},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.318},"max":{"1min":2.7599999999999998,"5min":6.9630000000000001,"15min":7.9939999999999998},"last":0.47699999999999998},{"interface":"front","average":{"1min":2.4830000000000001,"5min":1.6899999999999999,"15min":1.486},"min":{"1min":0.55600000000000005,"5min":0.41499999999999998,"15min":0.35499999999999998},"max":{"1min":10.106,"5min":10.106,"15min":10.106},"last":0.72799999999999998}]}]},{"osd":6,"up_from":46,"seq":197568495979,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1198516,"kb_used_data":893036,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":92546636,"statfs":{"total":95995035648,"available":94767755264,"internally_reserved":0,"allocated":914468864,"data_stored":913855591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:10:18 2023","interfaces":[{"interface":"back","average":{"1min":4.6500000000000004,"5min":2.0350000000000001,"15min":1.7709999999999999},"min":{"1min":0.28199999999999997,"5min":0.24399999999999999,"15min":0.214},"max":{"1min":98.275000000000006,"5min":98.275000000000006,"15min":98.275000000000006},"last":2.282},{"interface":"front","average":{"1min":4.7800000000000002,"5min":2.1349999999999998,"15min":1.8939999999999999},"min":{"1min":0.373,"5min":0.315,"15min":0.27200000000000002},"max":{"1min":98.334999999999994,"5min":98.334999999999994,"15min":98.334999999999994},"last":2.3109999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:10:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.202,"15min":1.498},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.25800000000000001},"max":{"1min":8.8360000000000003,"5min":8.8360000000000003,"15min":10.118},"last":1.923},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.448,"15min":1.4410000000000001},"min":{"1min":0.34200000000000003,"5min":0.32500000000000001,"15min":0.27900000000000003},"max":{"1min":11.103,"5min":11.103,"15min":12.417},"last":1.9419999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:10:16 2023","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":1.29,"15min":1.454},"min":{"1min":0.38200000000000001,"5min":0.318,"15min":0.28999999999999998},"max":{"1min":1.782,"5min":7.3040000000000003,"15min":8.9730000000000008},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.77900000000000003,"5min":0.98899999999999999,"15min":1.2769999999999999},"min":{"1min":0.36199999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.8660000000000001,"5min":5.8959999999999999,"15min":8.3810000000000002},"last":2.125}]},{"osd":3,"last update":"Mon Oct 9 05:10:18 2023","interfaces":[{"interface":"back","average":{"1min":1.3540000000000001,"5min":1.2869999999999999,"15min":1.5840000000000001},"min":{"1min":0.40699999999999997,"5min":0.33400000000000002,"15min":0.315},"max":{"1min":10.544,"5min":10.544,"15min":10.544},"last":2.105},{"interface":"front","average":{"1min":1.4339999999999999,"5min":1.3600000000000001,"15min":1.4630000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":6.4109999999999996,"5min":9.7720000000000002,"15min":9.7720000000000002},"last":0.53600000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:10:18 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":1.0509999999999999,"15min":1.127},"min":{"1min":0.36099999999999999,"5min":0.33600000000000002,"15min":0.28199999999999997},"max":{"1min":2.3140000000000001,"5min":5.5490000000000004,"15min":16.119},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.83499999999999996,"5min":1.1699999999999999,"15min":1.252},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.247},"max":{"1min":2.0059999999999998,"5min":6.3920000000000003,"15min":11.272},"last":0.42499999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:10:18 2023","interfaces":[{"interface":"back","average":{"1min":1.284,"5min":1.2629999999999999,"15min":1.3149999999999999},"min":{"1min":0.437,"5min":0.38100000000000001,"15min":0.29299999999999998},"max":{"1min":6.2199999999999998,"5min":6.2199999999999998,"15min":7.7149999999999999},"last":2.484},{"interface":"front","average":{"1min":0.99199999999999999,"5min":1.397,"15min":1.474},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.33100000000000002},"max":{"1min":2.8170000000000002,"5min":9.5730000000000004,"15min":10.537000000000001},"last":2.8170000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:09:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3340000000000001,"5min":1.3089999999999999,"15min":1.339},"min":{"1min":0.45100000000000001,"5min":0.40200000000000002,"15min":0.30599999999999999},"max":{"1min":4.4779999999999998,"5min":8.0899999999999999,"15min":9.3780000000000001},"last":0.51500000000000001},{"interface":"front","average":{"1min":1.4339999999999999,"5min":1.429,"15min":1.4379999999999999},"min":{"1min":0.49399999999999999,"5min":0.434,"15min":0.375},"max":{"1min":4.5119999999999996,"5min":8.1080000000000005,"15min":9.4060000000000006},"last":0.60099999999999998}]}]},{"osd":1,"up_from":96,"seq":412316860570,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1119964,"kb_used_data":742100,"kb_used_omap":4,"kb_used_meta":377851,"kb_avail":92625188,"statfs":{"total":95995035648,"available":94848192512,"internally_reserved":0,"allocated":759910400,"data_stored":759088822,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":386920334},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:10:06 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":1.2869999999999999,"15min":1.696},"min":{"1min":0.22500000000000001,"5min":0.20899999999999999,"15min":0.19400000000000001},"max":{"1min":2.6989999999999998,"5min":22.382999999999999,"15min":22.382999999999999},"last":1.5880000000000001},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.1759999999999999,"15min":2.0710000000000002},"min":{"1min":0.32500000000000001,"5min":0.26600000000000001,"15min":0.21099999999999999},"max":{"1min":3.9689999999999999,"5min":9.6259999999999994,"15min":10.268000000000001},"last":1.0149999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:09:20 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":1.286,"15min":1.472},"min":{"1min":0.49299999999999999,"5min":0.35799999999999998,"15min":0.219},"max":{"1min":0.84699999999999998,"5min":8.1310000000000002,"15min":10.115},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.82299999999999995,"5min":1.4139999999999999,"15min":1.468},"min":{"1min":0.55500000000000005,"5min":0.38400000000000001,"15min":0.252},"max":{"1min":1.1839999999999999,"5min":10.428000000000001,"15min":11.917},"last":0.54100000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:09:23 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":1.512,"15min":1.948},"min":{"1min":0.51000000000000001,"5min":0.23400000000000001,"15min":0.23100000000000001},"max":{"1min":1.012,"5min":9.6039999999999992,"15min":9.6039999999999992},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.81499999999999995,"5min":1.3009999999999999,"15min":1.405},"min":{"1min":0.56699999999999995,"5min":0.28100000000000003,"15min":0.24399999999999999},"max":{"1min":1.073,"5min":7.1859999999999999,"15min":9.9039999999999999},"last":0.43099999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:10:06 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":1.256,"15min":1.5089999999999999},"min":{"1min":0.40000000000000002,"5min":0.36899999999999999,"15min":0.34200000000000003},"max":{"1min":1.962,"5min":6.7869999999999999,"15min":9.9749999999999996},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.88100000000000001,"5min":1.4750000000000001,"15min":1.4690000000000001},"min":{"1min":0.34699999999999998,"5min":0.33400000000000002,"15min":0.30099999999999999},"max":{"1min":2.1840000000000002,"5min":22.350000000000001,"15min":22.350000000000001},"last":0.47799999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:10:06 2023","interfaces":[{"interface":"back","average":{"1min":1.4139999999999999,"5min":1.4650000000000001,"15min":1.7090000000000001},"min":{"1min":0.57599999999999996,"5min":0.34699999999999998,"15min":0.32500000000000001},"max":{"1min":5.5609999999999999,"5min":12.454000000000001,"15min":12.454000000000001},"last":1.7629999999999999},{"interface":"front","average":{"1min":0.88300000000000001,"5min":1.4750000000000001,"15min":1.7849999999999999},"min":{"1min":0.51000000000000001,"5min":0.41399999999999998,"15min":0.316},"max":{"1min":2.5430000000000001,"5min":7.2370000000000001,"15min":10.035},"last":0.60099999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:10:06 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.403,"15min":1.8380000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.35299999999999998},"max":{"1min":3.492,"5min":8.1940000000000008,"15min":10.409000000000001},"last":0.98699999999999999},{"interface":"front","average":{"1min":1.01,"5min":1.5049999999999999,"15min":1.653},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.35099999999999998},"max":{"1min":1.998,"5min":6.7530000000000001,"15min":10.007},"last":0.623}]},{"osd":7,"last update":"Mon Oct 9 05:10:06 2023","interfaces":[{"interface":"back","average":{"1min":1.4279999999999999,"5min":1.5469999999999999,"15min":1.718},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.38800000000000001},"max":{"1min":3.238,"5min":7.1260000000000003,"15min":10.061},"last":4.1920000000000002},{"interface":"front","average":{"1min":1.3740000000000001,"5min":1.768,"15min":1.6679999999999999},"min":{"1min":0.52400000000000002,"5min":0.41199999999999998,"15min":0.36399999999999999},"max":{"1min":3.2530000000000001,"5min":22.175999999999998,"15min":22.175999999999998},"last":4.141}]}]},{"osd":0,"up_from":88,"seq":377957122211,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1999188,"kb_used_data":1635084,"kb_used_omap":5,"kb_used_meta":364090,"kb_avail":91745964,"statfs":{"total":95995035648,"available":93947867136,"internally_reserved":0,"allocated":1674326016,"data_stored":1673511150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":372828794},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,6,1,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:10:01 2023","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":0.92800000000000005,"15min":1.931},"min":{"1min":0.24399999999999999,"5min":0.217,"15min":0.217},"max":{"1min":2.6120000000000001,"5min":7.4379999999999997,"15min":13.084},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.98499999999999999,"15min":1.806},"min":{"1min":0.253,"5min":0.253,"15min":0.218},"max":{"1min":1.244,"5min":8.3879999999999999,"15min":11.161},"last":2.71}]},{"osd":2,"last update":"Mon Oct 9 05:10:16 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":0.998,"15min":1.6919999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":3.0390000000000001,"5min":8.3420000000000005,"15min":9.1300000000000008},"last":2.6880000000000002},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.89000000000000001,"15min":1.4710000000000001},"min":{"1min":0.29099999999999998,"5min":0.25,"15min":0.23300000000000001},"max":{"1min":2.665,"5min":8.3599999999999994,"15min":10.423},"last":4.7220000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:09:20 2023","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":1.054,"15min":1.383},"min":{"1min":0.33800000000000002,"5min":0.253,"15min":0.253},"max":{"1min":1.1220000000000001,"5min":11.433999999999999,"15min":11.433999999999999},"last":3.3999999999999999},{"interface":"front","average":{"1min":0.80800000000000005,"5min":1.2809999999999999,"15min":1.26},"min":{"1min":0.495,"5min":0.318,"15min":0.318},"max":{"1min":1.153,"5min":10.041,"15min":10.041},"last":2.7789999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:09:20 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":1.248,"15min":1.4239999999999999},"min":{"1min":0.65900000000000003,"5min":0.39400000000000002,"15min":0.30099999999999999},"max":{"1min":1.2729999999999999,"5min":8.7449999999999992,"15min":8.7449999999999992},"last":2.8359999999999999},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.1579999999999999,"15min":1.409},"min":{"1min":0.71299999999999997,"5min":0.34100000000000003,"15min":0.33700000000000002},"max":{"1min":1.393,"5min":5.6459999999999999,"15min":7.202},"last":4.6879999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:09:20 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.2470000000000001,"15min":1.5269999999999999},"min":{"1min":0.72799999999999998,"5min":0.34499999999999997,"15min":0.30399999999999999},"max":{"1min":1.242,"5min":6.8959999999999999,"15min":12.526999999999999},"last":2.5139999999999998},{"interface":"front","average":{"1min":1.036,"5min":1.569,"15min":1.6000000000000001},"min":{"1min":0.77100000000000002,"5min":0.35299999999999998,"15min":0.316},"max":{"1min":1.3149999999999999,"5min":10.148,"15min":10.148},"last":3.7429999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:09:20 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.3160000000000001,"15min":1.5589999999999999},"min":{"1min":0.69499999999999995,"5min":0.371,"15min":0.371},"max":{"1min":1.4830000000000001,"5min":8.8710000000000004,"15min":8.8710000000000004},"last":4.7480000000000002},{"interface":"front","average":{"1min":1.044,"5min":1.27,"15min":1.4610000000000001},"min":{"1min":0.75700000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.5629999999999999,"5min":6.9370000000000003,"15min":10.429},"last":2.548}]},{"osd":7,"last update":"Mon Oct 9 05:09:20 2023","interfaces":[{"interface":"back","average":{"1min":1.081,"5min":1.242,"15min":1.454},"min":{"1min":0.73899999999999999,"5min":0.42699999999999999,"15min":0.36599999999999999},"max":{"1min":1.3720000000000001,"5min":6.9569999999999999,"15min":10.314},"last":2.5710000000000002},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.464,"15min":1.7729999999999999},"min":{"1min":0.69899999999999995,"5min":0.436,"15min":0.39500000000000002},"max":{"1min":1.508,"5min":8.6110000000000007,"15min":10.738},"last":2.8599999999999999}]}]},{"osd":2,"up_from":106,"seq":455266533490,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1157028,"kb_used_data":901788,"kb_used_omap":7,"kb_used_meta":255224,"kb_avail":92588124,"statfs":{"total":95995035648,"available":94810238976,"internally_reserved":0,"allocated":923430912,"data_stored":922657197,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":261349457},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:10:10 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":1.0609999999999999,"15min":1.1930000000000001},"min":{"1min":0.30599999999999999,"5min":0.27400000000000002,"15min":0.215},"max":{"1min":1.3859999999999999,"5min":23.010999999999999,"15min":23.010999999999999},"last":4.7009999999999996},{"interface":"front","average":{"1min":0.58699999999999997,"5min":1.1870000000000001,"15min":1.2629999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.25700000000000001},"max":{"1min":1.762,"5min":20.501999999999999,"15min":20.501999999999999},"last":2.1240000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:10:10 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":1.0960000000000001,"15min":1.1899999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.29299999999999998},"max":{"1min":1.819,"5min":20.539999999999999,"15min":20.539999999999999},"last":2.5640000000000001},{"interface":"front","average":{"1min":0.86099999999999999,"5min":1.2230000000000001,"15min":1.403},"min":{"1min":0.33500000000000002,"5min":0.23899999999999999,"15min":0.23100000000000001},"max":{"1min":6.0099999999999998,"5min":24.058,"15min":24.058},"last":3.415}]},{"osd":3,"last update":"Mon Oct 9 05:09:24 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":1.3600000000000001,"15min":1.9419999999999999},"min":{"1min":0.442,"5min":0.29899999999999999,"15min":0.27300000000000002},"max":{"1min":1.264,"5min":20.907,"15min":20.907},"last":2.0539999999999998},{"interface":"front","average":{"1min":0.68000000000000005,"5min":1.2569999999999999,"15min":1.643},"min":{"1min":0.47999999999999998,"5min":0.28899999999999998,"15min":0.28299999999999997},"max":{"1min":1.038,"5min":23.574999999999999,"15min":23.574999999999999},"last":1.1120000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:10:10 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":1.3169999999999999,"15min":1.387},"min":{"1min":0.36899999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.8560000000000001,"5min":20.712,"15min":20.712},"last":2.0259999999999998},{"interface":"front","average":{"1min":0.74399999999999999,"5min":1.3779999999999999,"15min":1.4319999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.653,"5min":23.547000000000001,"15min":23.547000000000001},"last":0.35899999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:10:10 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":1.504,"15min":1.4750000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.3899999999999999,"5min":20.782,"15min":20.782},"last":0.495},{"interface":"front","average":{"1min":1.1240000000000001,"5min":1.365,"15min":1.8080000000000001},"min":{"1min":0.55600000000000005,"5min":0.36699999999999999,"15min":0.36299999999999999},"max":{"1min":3.9700000000000002,"5min":20.792000000000002,"15min":20.792000000000002},"last":1.952}]},{"osd":6,"last update":"Mon Oct 9 05:10:10 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":1.351,"15min":1.5},"min":{"1min":0.48599999999999999,"5min":0.40899999999999997,"15min":0.34999999999999998},"max":{"1min":2.528,"5min":23.510999999999999,"15min":23.510999999999999},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.73799999999999999,"5min":1.45,"15min":1.4570000000000001},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.2629999999999999,"5min":20.684999999999999,"15min":20.684999999999999},"last":0.41699999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:10:10 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":1.411,"15min":1.4490000000000001},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":2.8759999999999999,"5min":20.887,"15min":20.887},"last":0.46300000000000002},{"interface":"front","average":{"1min":1.0449999999999999,"5min":1.381,"15min":1.4419999999999999},"min":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.45400000000000001},"max":{"1min":2.8889999999999998,"5min":20.863,"15min":20.863},"last":1.996}]}]},{"osd":3,"up_from":123,"seq":528280977509,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2507184,"kb_used_data":2164328,"kb_used_omap":2,"kb_used_meta":342845,"kb_avail":91237968,"statfs":{"total":95995035648,"available":93427679232,"internally_reserved":0,"allocated":2216271872,"data_stored":2215545024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":351073333},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2,2,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:09:21 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":1.0369999999999999,"15min":1.847},"min":{"1min":0.34699999999999998,"5min":0.187,"15min":0.187},"max":{"1min":2.3029999999999999,"5min":6.8310000000000004,"15min":14.141},"last":4.29},{"interface":"front","average":{"1min":0.874,"5min":1.4359999999999999,"15min":2.3220000000000001},"min":{"1min":0.53600000000000003,"5min":0.184,"15min":0.184},"max":{"1min":2.2210000000000001,"5min":9.6780000000000008,"15min":14.045999999999999},"last":3.173}]},{"osd":1,"last update":"Mon Oct 9 05:09:21 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":1.161,"15min":2.145},"min":{"1min":0.41699999999999998,"5min":0.308,"15min":0.28000000000000003},"max":{"1min":2.048,"5min":8.1199999999999992,"15min":14.193},"last":14.808},{"interface":"front","average":{"1min":0.76900000000000002,"5min":0.93799999999999994,"15min":2.2490000000000001},"min":{"1min":0.502,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":2.1259999999999999,"5min":6.1950000000000003,"15min":17.498999999999999},"last":1.131}]},{"osd":2,"last update":"Mon Oct 9 05:09:21 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":1.238,"15min":1.8919999999999999},"min":{"1min":0.47199999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":2.3250000000000002,"5min":9.6129999999999995,"15min":9.6129999999999995},"last":3.1019999999999999},{"interface":"front","average":{"1min":0.83199999999999996,"5min":1.3380000000000001,"15min":1.7210000000000001},"min":{"1min":0.54000000000000004,"5min":0.33300000000000002,"15min":0.245},"max":{"1min":2.1850000000000001,"5min":8.1600000000000001,"15min":8.1600000000000001},"last":14.754}]},{"osd":4,"last update":"Mon Oct 9 05:09:21 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":1.2949999999999999,"15min":1.6259999999999999},"min":{"1min":0.77500000000000002,"5min":0.36399999999999999,"15min":0.34399999999999997},"max":{"1min":2.5649999999999999,"5min":5.6689999999999996,"15min":5.8070000000000004},"last":14.833},{"interface":"front","average":{"1min":1.0760000000000001,"5min":1.0780000000000001,"15min":1.264},"min":{"1min":0.71899999999999997,"5min":0.36299999999999999,"15min":0.34999999999999998},"max":{"1min":2.355,"5min":4.5010000000000003,"15min":9.4749999999999996},"last":1.1870000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:09:21 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":1.304,"15min":1.321},"min":{"1min":0.751,"5min":0.41999999999999998,"15min":0.38300000000000001},"max":{"1min":2.4060000000000001,"5min":6.4420000000000002,"15min":9.5079999999999991},"last":1.1639999999999999},{"interface":"front","average":{"1min":1.1419999999999999,"5min":1.512,"15min":1.9239999999999999},"min":{"1min":0.82499999999999996,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":2.5870000000000002,"5min":8.1699999999999999,"15min":8.9380000000000006},"last":3.2010000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:09:21 2023","interfaces":[{"interface":"back","average":{"1min":1.181,"5min":1.4870000000000001,"15min":1.7689999999999999},"min":{"1min":0.79100000000000004,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":2.665,"5min":9.6590000000000007,"15min":9.6590000000000007},"last":3.141},{"interface":"front","average":{"1min":1.139,"5min":1.472,"15min":1.8600000000000001},"min":{"1min":0.76200000000000001,"5min":0.38400000000000001,"15min":0.34699999999999998},"max":{"1min":2.3940000000000001,"5min":5.7119999999999997,"15min":6.3289999999999997},"last":14.881}]},{"osd":7,"last update":"Mon Oct 9 05:09:21 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.4570000000000001,"15min":2.02},"min":{"1min":0.81100000000000005,"5min":0.42499999999999999,"15min":0.40999999999999998},"max":{"1min":2.633,"5min":5.6020000000000003,"15min":8.3209999999999997},"last":14.856},{"interface":"front","average":{"1min":1.141,"5min":1.357,"15min":1.512},"min":{"1min":0.873,"5min":0.38900000000000001,"15min":0.38800000000000001},"max":{"1min":2.4340000000000002,"5min":8.2300000000000004,"15min":9.5359999999999996},"last":4.327}]}]},{"osd":4,"up_from":30,"seq":128849019265,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1482172,"kb_used_data":892404,"kb_used_omap":0,"kb_used_meta":589760,"kb_avail":92262980,"statfs":{"total":95995035648,"available":94477291520,"internally_reserved":0,"allocated":913821696,"data_stored":913200028,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603914240},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:10:17 2023","interfaces":[{"interface":"back","average":{"1min":8.2929999999999993,"5min":2.6200000000000001,"15min":3.4119999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.22700000000000001},"max":{"1min":102.669,"5min":102.669,"15min":131.30699999999999},"last":0.35799999999999998},{"interface":"front","average":{"1min":8.3460000000000001,"5min":2.581,"15min":3.3500000000000001},"min":{"1min":0.36699999999999999,"5min":0.34000000000000002,"15min":0.26900000000000002},"max":{"1min":102.72499999999999,"5min":102.72499999999999,"15min":131.36600000000001},"last":0.46899999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:10:05 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.1499999999999999,"15min":1.77},"min":{"1min":0.46899999999999997,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.6829999999999998,"5min":9.6649999999999991,"15min":59.408000000000001},"last":0.501},{"interface":"front","average":{"1min":0.86799999999999999,"5min":1.1779999999999999,"15min":1.8280000000000001},"min":{"1min":0.34599999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.1320000000000001,"5min":10.163,"15min":60.207000000000001},"last":2.3220000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:10:17 2023","interfaces":[{"interface":"back","average":{"1min":0.74099999999999999,"5min":1.099,"15min":1.1850000000000001},"min":{"1min":0.36199999999999999,"5min":0.33200000000000002,"15min":0.307},"max":{"1min":2.407,"5min":10.138,"15min":10.196},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.77200000000000002,"5min":1.042,"15min":1.1020000000000001},"min":{"1min":0.35899999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.3050000000000002,"5min":6.3810000000000002,"15min":6.3810000000000002},"last":0.41899999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:09:18 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":1.607,"15min":1.655},"min":{"1min":0.69599999999999995,"5min":0.371,"15min":0.309},"max":{"1min":1.474,"5min":10.555999999999999,"15min":10.555999999999999},"last":1.6200000000000001},{"interface":"front","average":{"1min":1.101,"5min":1.5129999999999999,"15min":1.589},"min":{"1min":0.67200000000000004,"5min":0.40200000000000002,"15min":0.35399999999999998},"max":{"1min":1.619,"5min":7.8449999999999998,"15min":21.036999999999999},"last":0.373}]},{"osd":5,"last update":"Mon Oct 9 05:09:18 2023","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":1.3280000000000001,"15min":1.256},"min":{"1min":0.71399999999999997,"5min":0.32500000000000001,"15min":0.314},"max":{"1min":1.4890000000000001,"5min":10.214,"15min":10.214},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.155,"5min":1.351,"15min":1.298},"min":{"1min":0.745,"5min":0.34200000000000003,"15min":0.30399999999999999},"max":{"1min":1.6599999999999999,"5min":6.8310000000000004,"15min":10.890000000000001},"last":2.1179999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:10:05 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":1.1240000000000001,"15min":1.3500000000000001},"min":{"1min":0.438,"5min":0.32800000000000001,"15min":0.27800000000000002},"max":{"1min":1.9990000000000001,"5min":5.2530000000000001,"15min":11.557},"last":0.73599999999999999},{"interface":"front","average":{"1min":0.96299999999999997,"5min":1.391,"15min":1.3280000000000001},"min":{"1min":0.41799999999999998,"5min":0.38500000000000001,"15min":0.34699999999999998},"max":{"1min":2.2919999999999998,"5min":10.689,"15min":10.689},"last":0.48799999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:10:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":1.224,"15min":1.204},"min":{"1min":0.47499999999999998,"5min":0.377,"15min":0.35999999999999999},"max":{"1min":3.6989999999999998,"5min":6.4820000000000002,"15min":8.8859999999999992},"last":0.66900000000000004},{"interface":"front","average":{"1min":1.0009999999999999,"5min":1.2130000000000001,"15min":1.3400000000000001},"min":{"1min":0.44900000000000001,"5min":0.41899999999999998,"15min":0.38600000000000001},"max":{"1min":1.962,"5min":5.4359999999999999,"15min":7.7880000000000003},"last":0.69599999999999995}]}]},{"osd":5,"up_from":38,"seq":163208757621,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2916332,"kb_used_data":2314276,"kb_used_omap":0,"kb_used_meta":602048,"kb_avail":90828820,"statfs":{"total":95995035648,"available":93008711680,"internally_reserved":0,"allocated":2369818624,"data_stored":2369209084,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":616497152},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:10:14 2023","interfaces":[{"interface":"back","average":{"1min":2.8570000000000002,"5min":1.5820000000000001,"15min":1.9339999999999999},"min":{"1min":0.373,"5min":0.373,"15min":0.24399999999999999},"max":{"1min":40.097999999999999,"5min":40.097999999999999,"15min":77.745999999999995},"last":1.752},{"interface":"front","average":{"1min":2.9729999999999999,"5min":1.7050000000000001,"15min":2.0419999999999998},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.35299999999999998},"max":{"1min":40.161000000000001,"5min":40.161000000000001,"15min":77.820999999999998},"last":1.79}]},{"osd":1,"last update":"Mon Oct 9 05:10:12 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":1.2509999999999999,"15min":1.643},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.26900000000000002},"max":{"1min":5.5030000000000001,"5min":5.7229999999999999,"15min":13.401999999999999},"last":0.32600000000000001},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.1200000000000001,"15min":1.6519999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.26800000000000002},"max":{"1min":2.3769999999999998,"5min":5.6959999999999997,"15min":13.398},"last":4.923}]},{"osd":2,"last update":"Mon Oct 9 05:09:20 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.498,"15min":1.1399999999999999},"min":{"1min":0.60399999999999998,"5min":0.36099999999999999,"15min":0.28499999999999998},"max":{"1min":1.252,"5min":13.436,"15min":13.436},"last":0.45600000000000002},{"interface":"front","average":{"1min":1.028,"5min":1.5720000000000001,"15min":1.266},"min":{"1min":0.68799999999999994,"5min":0.36899999999999999,"15min":0.32000000000000001},"max":{"1min":1.4470000000000001,"5min":9.4949999999999992,"15min":9.4949999999999992},"last":0.58599999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:09:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":1.5429999999999999,"15min":1.841},"min":{"1min":0.80800000000000005,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.4379999999999999,"5min":9.1010000000000009,"15min":9.1010000000000009},"last":5.093},{"interface":"front","average":{"1min":1.0780000000000001,"5min":1.841,"15min":1.6839999999999999},"min":{"1min":0.77500000000000002,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.4179999999999999,"5min":11.859999999999999,"15min":11.859999999999999},"last":0.61599999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:10:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":1.0940000000000001,"15min":1.288},"min":{"1min":0.33100000000000002,"5min":0.316,"15min":0.25},"max":{"1min":3.3130000000000002,"5min":5.1879999999999997,"15min":14.781000000000001},"last":0.38600000000000001},{"interface":"front","average":{"1min":1.323,"5min":1.3149999999999999,"15min":1.419},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.34499999999999997},"max":{"1min":5.1580000000000004,"5min":9.625,"15min":9.625},"last":0.54300000000000004}]},{"osd":6,"last update":"Mon Oct 9 05:10:12 2023","interfaces":[{"interface":"back","average":{"1min":1.393,"5min":1.423,"15min":1.532},"min":{"1min":0.45900000000000002,"5min":0.44800000000000001,"15min":0.34699999999999998},"max":{"1min":5.218,"5min":9.8330000000000002,"15min":11.654999999999999},"last":0.64200000000000002},{"interface":"front","average":{"1min":1.4279999999999999,"5min":1.353,"15min":1.4830000000000001},"min":{"1min":0.44,"5min":0.44,"15min":0.33500000000000002},"max":{"1min":5.4409999999999998,"5min":5.4409999999999998,"15min":10.423999999999999},"last":0.52000000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:09:51 2023","interfaces":[{"interface":"back","average":{"1min":1.5720000000000001,"5min":1.5980000000000001,"15min":1.484},"min":{"1min":0.5,"5min":0.38900000000000001,"15min":0.36899999999999999},"max":{"1min":6.9640000000000004,"5min":8.7720000000000002,"15min":9.6639999999999997},"last":0.56599999999999995},{"interface":"front","average":{"1min":1.4379999999999999,"5min":1.615,"15min":1.482},"min":{"1min":0.40000000000000002,"5min":0.378,"15min":0.313},"max":{"1min":3.3860000000000001,"5min":10.122999999999999,"15min":12.843999999999999},"last":2.7730000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1678245888,"data_stored":1678245888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":774176768,"data_stored":774176768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":924123136,"data_stored":924123136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2239954944,"data_stored":2239954944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":921960448,"data_stored":921960448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2389377024,"data_stored":2389377024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":917766144,"data_stored":917766144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2384592896,"data_stored":2384592896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:10:26.500 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 1 2023-10-09T05:10:26.523 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:26.564 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:26.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:26.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:26.671 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:26.703 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:26.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:26.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:26.806 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:26.833 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:26.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:26.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:26.939 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:26.965 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:26.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:26.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:27.069 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:27.101 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:27.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:27.104 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:26 smithi133 bash[102858]: cluster 2023-10-09T05:10:25.763276+0000 mgr.y (mgr.25277) 655 : cluster 0 pgmap v621: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s; 18400/192789 objects degraded (9.544%); 4297/192789 objects misplaced (2.229%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T05:10:27.104 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:26 smithi133 bash[105589]: cluster 2023-10-09T05:10:25.763276+0000 mgr.y (mgr.25277) 655 : cluster 0 pgmap v621: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s; 18400/192789 objects degraded (9.544%); 4297/192789 objects misplaced (2.229%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T05:10:27.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:27.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:26 smithi194 bash[92666]: cluster 2023-10-09T05:10:25.763276+0000 mgr.y (mgr.25277) 655 : cluster 0 pgmap v621: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s; 18400/192789 objects degraded (9.544%); 4297/192789 objects misplaced (2.229%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T05:10:27.205 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:27.239 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:27.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:27.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:27.260 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 58 16 66499 66483 71.6331 49.375 0.01236 0.0139522 2023-10-09T05:10:27.342 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:27.390 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:27.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:27.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:27.494 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:27.541 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:27.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:27.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:27.648 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:27.674 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:27.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:27.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:27.777 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:27.804 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:27.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:27.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:27.908 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:27.941 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:27.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:27.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:28.045 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:28.076 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:28.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:28.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:28.136 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 05:10:27 smithi194 bash[80638]: logger=cleanup t=2023-10-09T05:10:27.976347628Z level=info msg="Completed cleanup jobs" duration=69.921959ms 2023-10-09T05:10:28.185 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:28.216 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:28.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:28.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:28.260 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 59 16 67286 67270 71.2525 49.1875 0.0226885 0.0140269 2023-10-09T05:10:28.320 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:28.352 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:28.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:28.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:28.458 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:28.502 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:28.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:28.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:28.607 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:28.632 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:28.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:28.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:28.738 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:28.778 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:28.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:28.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:28.881 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:28.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:28 smithi194 bash[92666]: cluster 2023-10-09T05:10:27.765980+0000 mgr.y (mgr.25277) 656 : cluster 0 pgmap v622: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 43 MiB/s wr, 682 op/s; 18400/192789 objects degraded (9.544%); 4297/192789 objects misplaced (2.229%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:10:28.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:28 smithi194 bash[92666]: cluster 2023-10-09T05:10:28.240833+0000 mon.a (mon.0) 821 : cluster 3 Health check update: Degraded data redundancy: 18400/192789 objects degraded (9.544%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:28.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:28 smithi194 bash[92666]: audit 2023-10-09T05:10:28.449980+0000 mon.a (mon.0) 822 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:10:28.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:28 smithi194 bash[92666]: audit 2023-10-09T05:10:28.451628+0000 mon.a (mon.0) 823 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:10:28.908 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:28.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:28.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:29.016 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:29.017 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:29.018 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:29.031 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:28 smithi133 bash[105589]: cluster 2023-10-09T05:10:27.765980+0000 mgr.y (mgr.25277) 656 : cluster 0 pgmap v622: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 43 MiB/s wr, 682 op/s; 18400/192789 objects degraded (9.544%); 4297/192789 objects misplaced (2.229%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:10:29.031 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:28 smithi133 bash[105589]: cluster 2023-10-09T05:10:28.240833+0000 mon.a (mon.0) 821 : cluster 3 Health check update: Degraded data redundancy: 18400/192789 objects degraded (9.544%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:29.031 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:28 smithi133 bash[105589]: audit 2023-10-09T05:10:28.449980+0000 mon.a (mon.0) 822 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:10:29.031 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:28 smithi133 bash[105589]: audit 2023-10-09T05:10:28.451628+0000 mon.a (mon.0) 823 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:10:29.032 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:28 smithi133 bash[102858]: cluster 2023-10-09T05:10:27.765980+0000 mgr.y (mgr.25277) 656 : cluster 0 pgmap v622: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 43 MiB/s wr, 682 op/s; 18400/192789 objects degraded (9.544%); 4297/192789 objects misplaced (2.229%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:10:29.032 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:28 smithi133 bash[102858]: cluster 2023-10-09T05:10:28.240833+0000 mon.a (mon.0) 821 : cluster 3 Health check update: Degraded data redundancy: 18400/192789 objects degraded (9.544%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:29.032 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:28 smithi133 bash[102858]: audit 2023-10-09T05:10:28.449980+0000 mon.a (mon.0) 822 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:10:29.032 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:28 smithi133 bash[102858]: audit 2023-10-09T05:10:28.451628+0000 mon.a (mon.0) 823 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:10:29.065 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:29.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:29.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:29.175 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:29.244 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:29.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:29.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:29.260 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T05:10:29.260532+0000 min lat: 0.00144352 max lat: 2.39474 avg lat: 0.0141035 2023-10-09T05:10:29.261 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:10:29.261 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 60 16 68051 68035 70.8618 47.8125 0.0221066 0.0141035 2023-10-09T05:10:29.346 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:29.375 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:29.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:29.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:29.480 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:29.506 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:29.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:29.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:29.610 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:29.642 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:29.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:29.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:29.746 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:29.789 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:29.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:29.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:29.896 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:29.926 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:29.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:29.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:30.030 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:30.043 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:30.065 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:30.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:30.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:30.171 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:30.225 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:30.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:30.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:30.260 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 61 16 68735 68719 70.4009 42.75 0.0104716 0.0141962 2023-10-09T05:10:30.330 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:30.366 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:30.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:30.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:30.475 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:30.507 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:30.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:30.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:30.611 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:30.637 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:30.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:30.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:30.741 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:30.774 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:30.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:30.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:30.884 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:30.917 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:30.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:30.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:31.025 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:31.057 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:31.059 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:31.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:31.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:31.080 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:31 smithi133 bash[105589]: cluster 2023-10-09T05:10:29.768125+0000 mgr.y (mgr.25277) 657 : cluster 0 pgmap v623: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 57 MiB/s wr, 909 op/s; 18400/200991 objects degraded (9.155%); 4297/200991 objects misplaced (2.138%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:10:31.080 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:30 smithi133 bash[102858]: cluster 2023-10-09T05:10:29.768125+0000 mgr.y (mgr.25277) 657 : cluster 0 pgmap v623: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 57 MiB/s wr, 909 op/s; 18400/200991 objects degraded (9.155%); 4297/200991 objects misplaced (2.138%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:10:31.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:30 smithi194 bash[92666]: cluster 2023-10-09T05:10:29.768125+0000 mgr.y (mgr.25277) 657 : cluster 0 pgmap v623: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 57 MiB/s wr, 909 op/s; 18400/200991 objects degraded (9.155%); 4297/200991 objects misplaced (2.138%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:10:31.162 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:31.193 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:31.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:31.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:31.261 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 62 16 69451 69435 69.9871 44.75 0.0210372 0.0142808 2023-10-09T05:10:31.296 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:31.332 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:31.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:31.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:31.437 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:31.484 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:31.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:31.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:31.590 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:31.629 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:31.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:31.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:31.739 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:31.765 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:31.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:31.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:31.874 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:31.904 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:31.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:31.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:32.009 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:32.038 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:32.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:32.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:32.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:31 smithi194 bash[92666]: audit 2023-10-09T05:10:30.884150+0000 mon.a (mon.0) 824 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:10:32.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:31 smithi194 bash[92666]: audit 2023-10-09T05:10:30.894017+0000 mon.a (mon.0) 825 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:10:32.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:31 smithi194 bash[92666]: audit 2023-10-09T05:10:31.151444+0000 mon.a (mon.0) 826 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:10:32.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:31 smithi194 bash[92666]: audit 2023-10-09T05:10:31.164012+0000 mon.a (mon.0) 827 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:10:32.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:31 smithi194 bash[92666]: audit 2023-10-09T05:10:31.770408+0000 mon.a (mon.0) 828 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:10:32.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:31 smithi194 bash[92666]: audit 2023-10-09T05:10:31.771795+0000 mon.a (mon.0) 829 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:10:32.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:31 smithi194 bash[92666]: audit 2023-10-09T05:10:31.780233+0000 mon.a (mon.0) 830 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:10:32.145 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:32.147 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:31 smithi133 bash[105589]: audit 2023-10-09T05:10:30.884150+0000 mon.a (mon.0) 824 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:10:32.147 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:31 smithi133 bash[105589]: audit 2023-10-09T05:10:30.894017+0000 mon.a (mon.0) 825 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:10:32.147 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:31 smithi133 bash[105589]: audit 2023-10-09T05:10:31.151444+0000 mon.a (mon.0) 826 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:10:32.147 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:31 smithi133 bash[105589]: audit 2023-10-09T05:10:31.164012+0000 mon.a (mon.0) 827 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:10:32.147 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:31 smithi133 bash[105589]: audit 2023-10-09T05:10:31.770408+0000 mon.a (mon.0) 828 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:10:32.147 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:31 smithi133 bash[105589]: audit 2023-10-09T05:10:31.771795+0000 mon.a (mon.0) 829 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:10:32.148 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:31 smithi133 bash[105589]: audit 2023-10-09T05:10:31.780233+0000 mon.a (mon.0) 830 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:10:32.148 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:31 smithi133 bash[102858]: audit 2023-10-09T05:10:30.884150+0000 mon.a (mon.0) 824 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:10:32.148 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:31 smithi133 bash[102858]: audit 2023-10-09T05:10:30.894017+0000 mon.a (mon.0) 825 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:10:32.148 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:31 smithi133 bash[102858]: audit 2023-10-09T05:10:31.151444+0000 mon.a (mon.0) 826 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:10:32.148 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:31 smithi133 bash[102858]: audit 2023-10-09T05:10:31.164012+0000 mon.a (mon.0) 827 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:10:32.148 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:31 smithi133 bash[102858]: audit 2023-10-09T05:10:31.770408+0000 mon.a (mon.0) 828 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:10:32.148 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:31 smithi133 bash[102858]: audit 2023-10-09T05:10:31.771795+0000 mon.a (mon.0) 829 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:10:32.148 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:31 smithi133 bash[102858]: audit 2023-10-09T05:10:31.780233+0000 mon.a (mon.0) 830 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:10:32.182 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:32.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:32.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:32.261 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 63 16 70219 70203 69.638 48 0.00545881 0.0143523 2023-10-09T05:10:32.288 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:32.323 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:32.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:32.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:32.427 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:32.475 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:32.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:32.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:32.580 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:32.617 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:32.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:32.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:32.723 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:32.749 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:32.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:32.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:32.856 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:32.886 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:32.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:32.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:32.989 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:33.024 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:33.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:33.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:33.130 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:33.161 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:33.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:33.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:33.261 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 64 16 70933 70917 69.2471 44.625 0.00317972 0.0143997 2023-10-09T05:10:33.265 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:33.274 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:33 smithi133 bash[105589]: cluster 2023-10-09T05:10:31.768820+0000 mgr.y (mgr.25277) 658 : cluster 0 pgmap v624: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s; 18400/204627 objects degraded (8.992%); 4048/204627 objects misplaced (1.978%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T05:10:33.275 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:33 smithi133 bash[102858]: cluster 2023-10-09T05:10:31.768820+0000 mgr.y (mgr.25277) 658 : cluster 0 pgmap v624: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s; 18400/204627 objects degraded (8.992%); 4048/204627 objects misplaced (1.978%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T05:10:33.293 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:33.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:33.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:33.397 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:33.405 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:33 smithi194 bash[92666]: cluster 2023-10-09T05:10:31.768820+0000 mgr.y (mgr.25277) 658 : cluster 0 pgmap v624: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s; 18400/204627 objects degraded (8.992%); 4048/204627 objects misplaced (1.978%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T05:10:33.427 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:33.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:33.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:33.531 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:33.560 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:33.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:33.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:33.666 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:33.695 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:33.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:33.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:33.800 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:33.835 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:33.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:33.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:33.940 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:33.966 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:33.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:33.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:34.069 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:34.107 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:34.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:34.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:34.212 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:34.224 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:34 smithi194 bash[92666]: cluster 2023-10-09T05:10:33.242047+0000 mon.a (mon.0) 831 : cluster 3 Health check update: Degraded data redundancy: 18400/204627 objects degraded (8.992%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:34.244 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:34.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:34.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:34.261 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 65 16 71727 71711 68.9451 49.625 0.0351227 0.0144976 2023-10-09T05:10:34.353 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:34.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:34 smithi133 bash[102858]: cluster 2023-10-09T05:10:33.242047+0000 mon.a (mon.0) 831 : cluster 3 Health check update: Degraded data redundancy: 18400/204627 objects degraded (8.992%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:34.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:10:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:10:34] "GET /metrics HTTP/1.1" 200 36433 "" "Prometheus/2.43.0" 2023-10-09T05:10:34.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:34 smithi133 bash[105589]: cluster 2023-10-09T05:10:33.242047+0000 mon.a (mon.0) 831 : cluster 3 Health check update: Degraded data redundancy: 18400/204627 objects degraded (8.992%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:34.678 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:34.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:34.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:34.782 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:34.807 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:34.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:34.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:34.912 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:34.938 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:34.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:34.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:35.041 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:35.073 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:35.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:35.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:35.179 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:35.208 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:35.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:35.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:35.261 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 66 16 72503 72487 68.6353 48.5 0.030819 0.0145614 2023-10-09T05:10:35.317 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:35.350 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:35.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:35.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:35.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:35 smithi194 bash[92666]: cluster 2023-10-09T05:10:33.769510+0000 mgr.y (mgr.25277) 659 : cluster 0 pgmap v625: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s; 18400/207258 objects degraded (8.878%); 4048/207258 objects misplaced (1.953%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T05:10:35.454 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:35.464 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:35 smithi133 bash[102858]: cluster 2023-10-09T05:10:33.769510+0000 mgr.y (mgr.25277) 659 : cluster 0 pgmap v625: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s; 18400/207258 objects degraded (8.878%); 4048/207258 objects misplaced (1.953%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T05:10:35.464 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:35 smithi133 bash[105589]: cluster 2023-10-09T05:10:33.769510+0000 mgr.y (mgr.25277) 659 : cluster 0 pgmap v625: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s; 18400/207258 objects degraded (8.878%); 4048/207258 objects misplaced (1.953%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T05:10:35.481 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:35.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:35.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:35.585 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:35.611 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:35.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:35.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:35.717 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:35.752 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:35.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:35.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:35.857 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:35.888 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:35.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:35.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:35.992 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:36.021 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:36.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:36.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:36.126 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:36.154 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:36.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:36.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:36.257 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:36.261 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 67 16 73301 73285 68.3552 49.875 0.0108361 0.0146227 2023-10-09T05:10:36.266 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:36 smithi194 bash[92666]: cluster 2023-10-09T05:10:35.770378+0000 mgr.y (mgr.25277) 660 : cluster 0 pgmap v626: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 56 MiB/s wr, 888 op/s; 18400/215700 objects degraded (8.530%); 3805/215700 objects misplaced (1.764%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T05:10:36.288 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:36.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:36.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:36.393 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:36.403 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:36 smithi133 bash[105589]: cluster 2023-10-09T05:10:35.770378+0000 mgr.y (mgr.25277) 660 : cluster 0 pgmap v626: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 56 MiB/s wr, 888 op/s; 18400/215700 objects degraded (8.530%); 3805/215700 objects misplaced (1.764%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T05:10:36.403 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:36 smithi133 bash[102858]: cluster 2023-10-09T05:10:35.770378+0000 mgr.y (mgr.25277) 660 : cluster 0 pgmap v626: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 56 MiB/s wr, 888 op/s; 18400/215700 objects degraded (8.530%); 3805/215700 objects misplaced (1.764%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T05:10:36.431 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:36.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:36.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:36.537 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:36.563 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:36.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:36.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:36.672 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:36.708 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:36.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:36.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:36.811 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:36.840 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:36.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:36.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:36.944 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:36.977 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:36.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:36.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:37.085 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:37.118 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:37.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:37.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:37.223 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:37.261 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 68 16 74091 74075 68.0761 49.375 0.0136324 0.0146826 2023-10-09T05:10:37.265 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:37.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:37.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:37.369 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:37.404 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:37.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:37.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:37.510 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:37.530 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:37.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:37.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:37.633 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:37.658 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:37.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:37.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:37.761 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:37.795 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:37.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:37.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:37.903 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:37.933 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:37.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:37.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:38.036 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:38.064 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:38.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:38.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:38.172 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:38.200 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:38.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:38.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:38.261 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 69 16 74823 74807 67.7524 45.75 0.0113343 0.014752 2023-10-09T05:10:38.304 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:38.347 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:38.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:38.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:38.452 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:38.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:38.494 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:38.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:38.595 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:38.637 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:38.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:38.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:38.739 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:38.767 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:38.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:38.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:38.873 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:38.882 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:38 smithi133 bash[102858]: cluster 2023-10-09T05:10:37.772110+0000 mgr.y (mgr.25277) 661 : cluster 0 pgmap v627: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 40 MiB/s wr, 635 op/s; 18400/215700 objects degraded (8.530%); 3805/215700 objects misplaced (1.764%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T05:10:38.882 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:38 smithi133 bash[102858]: cluster 2023-10-09T05:10:38.245363+0000 mon.a (mon.0) 832 : cluster 3 Health check update: Degraded data redundancy: 18400/215700 objects degraded (8.530%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:38.883 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:38 smithi133 bash[105589]: cluster 2023-10-09T05:10:37.772110+0000 mgr.y (mgr.25277) 661 : cluster 0 pgmap v627: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 40 MiB/s wr, 635 op/s; 18400/215700 objects degraded (8.530%); 3805/215700 objects misplaced (1.764%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T05:10:38.883 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:38 smithi133 bash[105589]: cluster 2023-10-09T05:10:38.245363+0000 mon.a (mon.0) 832 : cluster 3 Health check update: Degraded data redundancy: 18400/215700 objects degraded (8.530%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:38.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:38 smithi194 bash[92666]: cluster 2023-10-09T05:10:37.772110+0000 mgr.y (mgr.25277) 661 : cluster 0 pgmap v627: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 40 MiB/s wr, 635 op/s; 18400/215700 objects degraded (8.530%); 3805/215700 objects misplaced (1.764%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T05:10:38.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:38 smithi194 bash[92666]: cluster 2023-10-09T05:10:38.245363+0000 mon.a (mon.0) 832 : cluster 3 Health check update: Degraded data redundancy: 18400/215700 objects degraded (8.530%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:38.900 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:38.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:38.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:39.006 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:39.031 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:39.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:39.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:39.136 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:39.162 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:39.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:39.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:39.261 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 70 16 75623 75607 67.4988 50 0.0166163 0.0148083 2023-10-09T05:10:39.268 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:39.297 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:39.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:39.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:39.406 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:39.435 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:39.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:39.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:39.540 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:39.572 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:39.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:39.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:39.676 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:39.705 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:39.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:39.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:39.808 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:39.842 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:39.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:39.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:39.948 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:39.971 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:39.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:39.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:40.075 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:40.106 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:40.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:40.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:40.214 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:40.241 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:40.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:40.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:40.261 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 71 16 76412 76396 67.2426 49.3125 0.0179788 0.0148644 2023-10-09T05:10:40.346 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:40.376 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:40.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:40.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:40.480 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:40.504 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:40.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:40.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:40.609 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:40.651 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:40.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:40.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:40.756 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:40.782 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:40.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:40.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:40.836 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:40 smithi194 bash[92666]: cluster 2023-10-09T05:10:39.772783+0000 mgr.y (mgr.25277) 662 : cluster 0 pgmap v628: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 861 op/s; 18400/223812 objects degraded (8.221%); 3805/223812 objects misplaced (1.700%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T05:10:40.888 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:40.898 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:40 smithi133 bash[102858]: cluster 2023-10-09T05:10:39.772783+0000 mgr.y (mgr.25277) 662 : cluster 0 pgmap v628: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 861 op/s; 18400/223812 objects degraded (8.221%); 3805/223812 objects misplaced (1.700%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T05:10:40.898 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:40 smithi133 bash[105589]: cluster 2023-10-09T05:10:39.772783+0000 mgr.y (mgr.25277) 662 : cluster 0 pgmap v628: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 861 op/s; 18400/223812 objects degraded (8.221%); 3805/223812 objects misplaced (1.700%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T05:10:40.914 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:40.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:40.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:41.018 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:41.042 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:41.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:41.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:41.145 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:41.181 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:41.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:41.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:41.262 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 72 16 77185 77169 66.9796 48.3125 0.0120798 0.0149214 2023-10-09T05:10:41.285 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:41.318 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:41.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:41.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:41.424 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:41.456 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:41.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:41.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:41.559 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:41.583 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:41.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:41.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:41.687 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:41.719 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:41.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:41.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:41.828 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:41.852 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:41.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:41.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:41.956 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:41.982 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:41.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:41.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:42.085 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:42.118 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:42.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:42.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:42.222 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:42.256 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:42.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:42.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:42.262 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 73 16 77884 77868 66.6605 43.6875 0.0133129 0.0149924 2023-10-09T05:10:42.362 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:42.398 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:42.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:42.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:42.501 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:42.532 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:42.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:42.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:42.636 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:42.666 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:42.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:42.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:42.769 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:42.801 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:42.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:42.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:42.909 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:42.932 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:42.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:42.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:43.035 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:43.049 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:42 smithi194 bash[92666]: cluster 2023-10-09T05:10:41.773536+0000 mgr.y (mgr.25277) 663 : cluster 0 pgmap v629: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 46 MiB/s wr, 734 op/s; 18400/227451 objects degraded (8.090%); 3521/227451 objects misplaced (1.548%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T05:10:43.071 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:43.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:43.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:43.176 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:43.185 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:42 smithi133 bash[102858]: cluster 2023-10-09T05:10:41.773536+0000 mgr.y (mgr.25277) 663 : cluster 0 pgmap v629: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 46 MiB/s wr, 734 op/s; 18400/227451 objects degraded (8.090%); 3521/227451 objects misplaced (1.548%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T05:10:43.186 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:42 smithi133 bash[105589]: cluster 2023-10-09T05:10:41.773536+0000 mgr.y (mgr.25277) 663 : cluster 0 pgmap v629: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 46 MiB/s wr, 734 op/s; 18400/227451 objects degraded (8.090%); 3521/227451 objects misplaced (1.548%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T05:10:43.211 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:43.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:43.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:43.262 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 74 16 78611 78595 66.3736 45.4375 0.0207683 0.0150596 2023-10-09T05:10:43.314 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:43.347 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:43.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:43.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:43.451 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:43.474 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:43.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:43.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:43.577 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:43.624 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:43.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:43.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:43.728 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:43.755 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:43.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:43.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:43.841 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:43 smithi133 bash[105589]: cluster 2023-10-09T05:10:43.247848+0000 mon.a (mon.0) 833 : cluster 3 Health check update: Degraded data redundancy: 18400/227451 objects degraded (8.090%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:43.841 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:43 smithi133 bash[105589]: audit 2023-10-09T05:10:43.441285+0000 mon.a (mon.0) 834 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:10:43.841 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:43 smithi133 bash[102858]: cluster 2023-10-09T05:10:43.247848+0000 mon.a (mon.0) 833 : cluster 3 Health check update: Degraded data redundancy: 18400/227451 objects degraded (8.090%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:43.842 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:43 smithi133 bash[102858]: audit 2023-10-09T05:10:43.441285+0000 mon.a (mon.0) 834 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:10:43.859 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:43.895 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:43.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:43.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:43.999 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:44.008 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:43 smithi194 bash[92666]: cluster 2023-10-09T05:10:43.247848+0000 mon.a (mon.0) 833 : cluster 3 Health check update: Degraded data redundancy: 18400/227451 objects degraded (8.090%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:44.008 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:43 smithi194 bash[92666]: audit 2023-10-09T05:10:43.441285+0000 mon.a (mon.0) 834 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:10:44.024 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:44.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:44.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:44.132 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:44.172 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:44.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:44.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:44.262 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 75 15 79365 79350 66.1177 47.1875 0.0225252 0.015118 2023-10-09T05:10:44.277 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:44.300 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:44.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:44.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:44.403 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:44.412 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:10:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:10:44] "GET /metrics HTTP/1.1" 200 36428 "" "Prometheus/2.43.0" 2023-10-09T05:10:44.433 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:44.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:44.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:44.491 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:10:44.491 INFO:teuthology.orchestra.run.smithi133.stdout: "ops": [], 2023-10-09T05:10:44.491 INFO:teuthology.orchestra.run.smithi133.stdout: "complaint_time": 30, 2023-10-09T05:10:44.491 INFO:teuthology.orchestra.run.smithi133.stdout: "num_blocked_ops": 0 2023-10-09T05:10:44.491 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:10:44.538 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:44.564 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:44.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:44.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:44.669 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:44.701 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:44.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:44.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:44.808 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:44.837 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:44.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:44.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:44.945 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:44.955 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:44 smithi133 bash[102858]: cluster 2023-10-09T05:10:43.774197+0000 mgr.y (mgr.25277) 664 : cluster 0 pgmap v630: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 44 MiB/s wr, 711 op/s; 18400/230247 objects degraded (7.991%); 3521/230247 objects misplaced (1.529%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:10:44.956 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:44 smithi133 bash[105589]: cluster 2023-10-09T05:10:43.774197+0000 mgr.y (mgr.25277) 664 : cluster 0 pgmap v630: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 44 MiB/s wr, 711 op/s; 18400/230247 objects degraded (7.991%); 3521/230247 objects misplaced (1.529%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:10:44.972 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:44.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:44.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:45.079 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:45.105 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:45.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:45.108 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:44 smithi194 bash[92666]: cluster 2023-10-09T05:10:43.774197+0000 mgr.y (mgr.25277) 664 : cluster 0 pgmap v630: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 44 MiB/s wr, 711 op/s; 18400/230247 objects degraded (7.991%); 3521/230247 objects misplaced (1.529%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:10:45.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:45.209 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:45.250 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:45.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:45.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:45.262 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 76 16 80074 80058 65.8299 44.25 0.00564088 0.0151828 2023-10-09T05:10:45.354 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:45.379 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:45.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:45.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:45.485 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:45.514 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:45.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:45.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:45.621 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:45.663 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:45.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:45.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:45.768 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:45.800 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:45.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:45.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:45.866 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is set 2023-10-09T05:10:45.868 INFO:teuthology.orchestra.run.smithi133.stderr:set pool 1 pgp_num to 1 2023-10-09T05:10:45.905 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:45.910 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:45 smithi133 bash[105589]: audit 2023-10-09T05:10:45.361637+0000 mon.a (mon.0) 835 : audit 1 from='client.? 172.21.15.133:0/830878041' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:10:45.911 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:45 smithi133 bash[105589]: audit 2023-10-09T05:10:45.784033+0000 mon.c (mon.2) 65 : audit 1 from='client.? 172.21.15.133:0/1301415187' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T05:10:45.911 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:45 smithi133 bash[105589]: audit 2023-10-09T05:10:45.784978+0000 mon.a (mon.0) 836 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T05:10:45.911 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:45 smithi133 bash[102858]: audit 2023-10-09T05:10:45.361637+0000 mon.a (mon.0) 835 : audit 1 from='client.? 172.21.15.133:0/830878041' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:10:45.911 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:45 smithi133 bash[102858]: audit 2023-10-09T05:10:45.784033+0000 mon.c (mon.2) 65 : audit 1 from='client.? 172.21.15.133:0/1301415187' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T05:10:45.911 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:45 smithi133 bash[102858]: audit 2023-10-09T05:10:45.784978+0000 mon.a (mon.0) 836 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T05:10:45.937 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:45.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:45.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:46.040 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:46.073 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:10:46.076 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:46.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:46.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:46.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:45 smithi194 bash[92666]: audit 2023-10-09T05:10:45.361637+0000 mon.a (mon.0) 835 : audit 1 from='client.? 172.21.15.133:0/830878041' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:10:46.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:45 smithi194 bash[92666]: audit 2023-10-09T05:10:45.784033+0000 mon.c (mon.2) 65 : audit 1 from='client.? 172.21.15.133:0/1301415187' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T05:10:46.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:45 smithi194 bash[92666]: audit 2023-10-09T05:10:45.784978+0000 mon.a (mon.0) 836 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T05:10:46.183 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:46.190 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:10:46.210 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:46.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:46.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:46.237 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:10:46.262 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 77 16 80693 80677 65.4774 38.6875 0.0181214 0.0152657 2023-10-09T05:10:46.274 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:10:46.308 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:10:46.316 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:46.344 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:46.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:46.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:46.348 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:10:46.384 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:10:46.414 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:10:46.421 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:10:46.421 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:10:46.421 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:10:46.421 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:10:46.422 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:10:46.422 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:10:46.422 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:10:46.422 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:10:46.447 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:46.471 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:46.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:46.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:46.574 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:46.609 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:46.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:46.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:46.713 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:46.740 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:46.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:46.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:46.846 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:46.875 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:46.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:46.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:46.979 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:46.998 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T05:10:47.004 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:47.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:47.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:47.111 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:47.126 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:46 smithi133 bash[102858]: cluster 2023-10-09T05:10:45.775051+0000 mgr.y (mgr.25277) 665 : cluster 0 pgmap v631: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 870 op/s; 18400/238605 objects degraded (7.711%); 3232/238605 objects misplaced (1.355%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T05:10:47.126 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:46 smithi133 bash[102858]: cluster 2023-10-09T05:10:45.842170+0000 mon.a (mon.0) 837 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:10:47.126 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:46 smithi133 bash[102858]: cluster 2023-10-09T05:10:45.851338+0000 osd.5 (osd.5) 129 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T05:10:47.127 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:46 smithi133 bash[102858]: cluster 2023-10-09T05:10:45.851856+0000 osd.7 (osd.7) 75 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T05:10:47.127 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:46 smithi133 bash[102858]: audit 2023-10-09T05:10:45.856917+0000 mon.a (mon.0) 838 : audit 1 from='client.? 172.21.15.133:0/830878041' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:10:47.127 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:46 smithi133 bash[102858]: audit 2023-10-09T05:10:45.857182+0000 mon.a (mon.0) 839 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-10-09T05:10:47.127 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:46 smithi133 bash[102858]: cluster 2023-10-09T05:10:45.857388+0000 mon.a (mon.0) 840 : cluster 0 osdmap e155: 8 total, 8 up, 8 in 2023-10-09T05:10:47.127 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:46 smithi133 bash[102858]: cluster 2023-10-09T05:10:45.884292+0000 osd.6 (osd.6) 81 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T05:10:47.127 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:46 smithi133 bash[102858]: audit 2023-10-09T05:10:46.037010+0000 mon.a (mon.0) 841 : audit 0 from='client.? 172.21.15.133:0/3216303690' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:10:47.128 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:46 smithi133 bash[105589]: cluster 2023-10-09T05:10:45.775051+0000 mgr.y (mgr.25277) 665 : cluster 0 pgmap v631: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 870 op/s; 18400/238605 objects degraded (7.711%); 3232/238605 objects misplaced (1.355%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T05:10:47.128 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:46 smithi133 bash[105589]: cluster 2023-10-09T05:10:45.842170+0000 mon.a (mon.0) 837 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:10:47.128 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:46 smithi133 bash[105589]: cluster 2023-10-09T05:10:45.851338+0000 osd.5 (osd.5) 129 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T05:10:47.128 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:46 smithi133 bash[105589]: cluster 2023-10-09T05:10:45.851856+0000 osd.7 (osd.7) 75 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T05:10:47.128 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:46 smithi133 bash[105589]: audit 2023-10-09T05:10:45.856917+0000 mon.a (mon.0) 838 : audit 1 from='client.? 172.21.15.133:0/830878041' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:10:47.129 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:46 smithi133 bash[105589]: audit 2023-10-09T05:10:45.857182+0000 mon.a (mon.0) 839 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-10-09T05:10:47.129 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:46 smithi133 bash[105589]: cluster 2023-10-09T05:10:45.857388+0000 mon.a (mon.0) 840 : cluster 0 osdmap e155: 8 total, 8 up, 8 in 2023-10-09T05:10:47.129 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:46 smithi133 bash[105589]: cluster 2023-10-09T05:10:45.884292+0000 osd.6 (osd.6) 81 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T05:10:47.129 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:46 smithi133 bash[105589]: audit 2023-10-09T05:10:46.037010+0000 mon.a (mon.0) 841 : audit 0 from='client.? 172.21.15.133:0/3216303690' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:10:47.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:46 smithi194 bash[92666]: cluster 2023-10-09T05:10:45.775051+0000 mgr.y (mgr.25277) 665 : cluster 0 pgmap v631: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 870 op/s; 18400/238605 objects degraded (7.711%); 3232/238605 objects misplaced (1.355%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T05:10:47.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:46 smithi194 bash[92666]: cluster 2023-10-09T05:10:45.842170+0000 mon.a (mon.0) 837 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:10:47.138 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:46 smithi194 bash[92666]: cluster 2023-10-09T05:10:45.851338+0000 osd.5 (osd.5) 129 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T05:10:47.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:46 smithi194 bash[92666]: cluster 2023-10-09T05:10:45.851856+0000 osd.7 (osd.7) 75 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T05:10:47.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:46 smithi194 bash[92666]: audit 2023-10-09T05:10:45.856917+0000 mon.a (mon.0) 838 : audit 1 from='client.? 172.21.15.133:0/830878041' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:10:47.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:46 smithi194 bash[92666]: audit 2023-10-09T05:10:45.857182+0000 mon.a (mon.0) 839 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-10-09T05:10:47.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:46 smithi194 bash[92666]: cluster 2023-10-09T05:10:45.857388+0000 mon.a (mon.0) 840 : cluster 0 osdmap e155: 8 total, 8 up, 8 in 2023-10-09T05:10:47.139 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:46 smithi194 bash[92666]: cluster 2023-10-09T05:10:45.884292+0000 osd.6 (osd.6) 81 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T05:10:47.140 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:46 smithi194 bash[92666]: audit 2023-10-09T05:10:46.037010+0000 mon.a (mon.0) 841 : audit 0 from='client.? 172.21.15.133:0/3216303690' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:10:47.160 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:47.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:47.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:47.262 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 78 16 81411 81395 65.2132 44.875 0.021996 0.015327 2023-10-09T05:10:47.265 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:47.309 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:47.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:47.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:47.413 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:47.441 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:47.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:47.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:47.546 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:47.582 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:47.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:47.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:47.686 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:47.731 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:47.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:47.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:47.837 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:47.868 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:47.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:47.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:47.971 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:48.005 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:48.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:48.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:48.109 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:48.141 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:48.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:48.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:48.248 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:48.262 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 79 16 82199 82183 65.0111 49.25 0.0241365 0.0153753 2023-10-09T05:10:48.294 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:48.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:48.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:48.402 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:48.436 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:48.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:48.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:48.544 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:48.574 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:48.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:48.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:48.681 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:48.733 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:48.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:48.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:48.837 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:48.873 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:48.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:48.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:48.977 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:49.001 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:49.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:49.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:49.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:48 smithi133 bash[102858]: cluster 2023-10-09T05:10:47.775620+0000 mgr.y (mgr.25277) 666 : cluster 0 pgmap v633: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 48 MiB/s wr, 763 op/s; 18400/238605 objects degraded (7.711%); 3232/238605 objects misplaced (1.355%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T05:10:49.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:48 smithi133 bash[102858]: cluster 2023-10-09T05:10:48.249304+0000 mon.a (mon.0) 842 : cluster 3 Health check update: Degraded data redundancy: 18400/238605 objects degraded (7.711%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:49.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:48 smithi133 bash[105589]: cluster 2023-10-09T05:10:47.775620+0000 mgr.y (mgr.25277) 666 : cluster 0 pgmap v633: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 48 MiB/s wr, 763 op/s; 18400/238605 objects degraded (7.711%); 3232/238605 objects misplaced (1.355%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T05:10:49.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:48 smithi133 bash[105589]: cluster 2023-10-09T05:10:48.249304+0000 mon.a (mon.0) 842 : cluster 3 Health check update: Degraded data redundancy: 18400/238605 objects degraded (7.711%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:49.105 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:49.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:48 smithi194 bash[92666]: cluster 2023-10-09T05:10:47.775620+0000 mgr.y (mgr.25277) 666 : cluster 0 pgmap v633: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 48 MiB/s wr, 763 op/s; 18400/238605 objects degraded (7.711%); 3232/238605 objects misplaced (1.355%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T05:10:49.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:48 smithi194 bash[92666]: cluster 2023-10-09T05:10:48.249304+0000 mon.a (mon.0) 842 : cluster 3 Health check update: Degraded data redundancy: 18400/238605 objects degraded (7.711%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:49.137 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:49.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:49.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:49.240 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:49.262 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T05:10:49.262495+0000 min lat: 0.00144352 max lat: 2.39474 avg lat: 0.015423 2023-10-09T05:10:49.262 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:10:49.263 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 80 16 82985 82969 64.8125 49.125 0.0163398 0.015423 2023-10-09T05:10:49.267 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:49.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:49.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:49.371 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:49.398 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:49.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:49.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:49.504 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:49.530 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:49.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:49.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:49.638 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:49.668 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:49.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:49.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:49.774 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:49.799 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:49.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:49.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:49.906 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:49.941 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:49.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:49.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:50.048 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:50.076 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:50.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:50.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:50.182 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:50.215 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:50.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:50.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:50.263 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 81 16 83775 83759 64.6218 49.375 0.00647098 0.0154658 2023-10-09T05:10:50.321 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:50.348 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:50.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:50.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:50.452 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:50.490 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:50.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:50.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:50.596 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:50.624 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:50.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:50.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:50.731 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:50.762 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:50.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:50.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:50.870 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:50.898 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:50.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:50.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:51.001 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:51.039 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:51.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:51.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:51.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:50 smithi194 bash[92666]: cluster 2023-10-09T05:10:49.778779+0000 mgr.y (mgr.25277) 667 : cluster 0 pgmap v634: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 742 op/s; 18400/246099 objects degraded (7.477%); 3232/246099 objects misplaced (1.313%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T05:10:51.142 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:51.183 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:51.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:51.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:51.263 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 82 16 84540 84524 64.4168 47.8125 0.0175287 0.0155177 2023-10-09T05:10:51.287 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:51.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:50 smithi133 bash[102858]: cluster 2023-10-09T05:10:49.778779+0000 mgr.y (mgr.25277) 667 : cluster 0 pgmap v634: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 742 op/s; 18400/246099 objects degraded (7.477%); 3232/246099 objects misplaced (1.313%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T05:10:51.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:50 smithi133 bash[105589]: cluster 2023-10-09T05:10:49.778779+0000 mgr.y (mgr.25277) 667 : cluster 0 pgmap v634: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 742 op/s; 18400/246099 objects degraded (7.477%); 3232/246099 objects misplaced (1.313%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T05:10:51.316 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:51.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:51.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:51.419 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:51.449 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:51.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:51.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:51.552 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:10:51.553 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:51.589 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:10:51.603 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:51.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:51.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:51.711 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:51.739 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:51.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:51.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:51.840 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:51.849 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:51.893 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:51.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:51.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:51.998 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:52.025 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:52.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:52.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:52.131 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:52.175 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:52.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:52.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:52.263 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 83 16 85229 85213 64.1594 43.0625 0.0200915 0.0155795 2023-10-09T05:10:52.279 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:52.311 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:52.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:52.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:52.414 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:52.440 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:52.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:52.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:52.548 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:52.575 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:52.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:52.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:52.681 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:52.707 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:52.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:52.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:52.811 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:52.840 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:52.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:52.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:52.944 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:52.952 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:52 smithi133 bash[102858]: cluster 2023-10-09T05:10:51.779504+0000 mgr.y (mgr.25277) 668 : cluster 0 pgmap v635: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 736 op/s; 18400/249552 objects degraded (7.373%); 2887/249552 objects misplaced (1.157%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T05:10:52.953 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:52 smithi133 bash[105589]: cluster 2023-10-09T05:10:51.779504+0000 mgr.y (mgr.25277) 668 : cluster 0 pgmap v635: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 736 op/s; 18400/249552 objects degraded (7.373%); 2887/249552 objects misplaced (1.157%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T05:10:52.971 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:52.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:52.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:53.074 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:53.101 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:53.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:53.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:53.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:52 smithi194 bash[92666]: cluster 2023-10-09T05:10:51.779504+0000 mgr.y (mgr.25277) 668 : cluster 0 pgmap v635: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 736 op/s; 18400/249552 objects degraded (7.373%); 2887/249552 objects misplaced (1.157%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T05:10:53.206 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:53.240 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:53.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:53.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:53.263 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 84 16 85993 85977 63.964 47.75 0.0139019 0.0156278 2023-10-09T05:10:53.346 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:53.383 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:53.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:53.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:53.487 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:53.513 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:53.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:53.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:53.617 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:53.643 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:53.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:53.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:53.746 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:53.771 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:53.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:53.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:53.874 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:53.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:53 smithi194 bash[92666]: cluster 2023-10-09T05:10:53.250606+0000 mon.a (mon.0) 843 : cluster 3 Health check update: Degraded data redundancy: 18400/249552 objects degraded (7.373%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:53.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:53 smithi194 bash[92666]: cluster 2023-10-09T05:10:53.254399+0000 osd.6 (osd.6) 82 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T05:10:53.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:53 smithi194 bash[92666]: cluster 2023-10-09T05:10:53.256123+0000 mon.a (mon.0) 844 : cluster 0 osdmap e156: 8 total, 8 up, 8 in 2023-10-09T05:10:53.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:53 smithi194 bash[92666]: cluster 2023-10-09T05:10:53.279192+0000 osd.7 (osd.7) 76 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T05:10:53.909 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:53.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:53.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:54.013 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:54.043 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:54.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:54.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:54.136 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:53 smithi133 bash[102858]: cluster 2023-10-09T05:10:53.250606+0000 mon.a (mon.0) 843 : cluster 3 Health check update: Degraded data redundancy: 18400/249552 objects degraded (7.373%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:54.137 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:53 smithi133 bash[102858]: cluster 2023-10-09T05:10:53.254399+0000 osd.6 (osd.6) 82 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T05:10:54.137 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:53 smithi133 bash[102858]: cluster 2023-10-09T05:10:53.256123+0000 mon.a (mon.0) 844 : cluster 0 osdmap e156: 8 total, 8 up, 8 in 2023-10-09T05:10:54.137 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:53 smithi133 bash[102858]: cluster 2023-10-09T05:10:53.279192+0000 osd.7 (osd.7) 76 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T05:10:54.137 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:53 smithi133 bash[105589]: cluster 2023-10-09T05:10:53.250606+0000 mon.a (mon.0) 843 : cluster 3 Health check update: Degraded data redundancy: 18400/249552 objects degraded (7.373%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:54.137 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:53 smithi133 bash[105589]: cluster 2023-10-09T05:10:53.254399+0000 osd.6 (osd.6) 82 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T05:10:54.137 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:53 smithi133 bash[105589]: cluster 2023-10-09T05:10:53.256123+0000 mon.a (mon.0) 844 : cluster 0 osdmap e156: 8 total, 8 up, 8 in 2023-10-09T05:10:54.137 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:53 smithi133 bash[105589]: cluster 2023-10-09T05:10:53.279192+0000 osd.7 (osd.7) 76 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T05:10:54.147 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:54.178 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:54.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:54.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:54.263 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 85 16 86225 86209 63.3821 14.5 0.00497503 0.0156275 2023-10-09T05:10:54.282 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:54.311 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:54.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:54.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:54.415 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:54.442 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:54.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:54.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:54.544 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:54.565 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:54.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:54.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:54.571 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:10:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:10:54] "GET /metrics HTTP/1.1" 200 36428 "" "Prometheus/2.43.0" 2023-10-09T05:10:54.606 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:10:54.606 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:10:54.606 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_4 2023-10-09T05:10:54.606 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:10:54.669 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:54.710 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:54.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:54.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:54.818 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:54.842 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:54.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:54.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:54.875 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:54 smithi133 bash[102858]: cluster 2023-10-09T05:10:53.261858+0000 osd.5 (osd.5) 130 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T05:10:54.875 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:54 smithi133 bash[102858]: cluster 2023-10-09T05:10:53.283648+0000 osd.4 (osd.4) 96 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T05:10:54.876 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:54 smithi133 bash[102858]: cluster 2023-10-09T05:10:53.780445+0000 mgr.y (mgr.25277) 669 : cluster 0 pgmap v637: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 36 MiB/s wr, 582 op/s; 18400/252600 objects degraded (7.284%); 2887/252600 objects misplaced (1.143%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:10:54.876 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:54 smithi194 bash[92666]: cluster 2023-10-09T05:10:53.261858+0000 osd.5 (osd.5) 130 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T05:10:54.876 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:54 smithi194 bash[92666]: cluster 2023-10-09T05:10:53.283648+0000 osd.4 (osd.4) 96 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T05:10:54.876 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:54 smithi194 bash[92666]: cluster 2023-10-09T05:10:53.780445+0000 mgr.y (mgr.25277) 669 : cluster 0 pgmap v637: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 36 MiB/s wr, 582 op/s; 18400/252600 objects degraded (7.284%); 2887/252600 objects misplaced (1.143%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:10:54.876 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:54 smithi194 bash[92666]: cluster 2023-10-09T05:10:54.277351+0000 mon.a (mon.0) 845 : cluster 0 osdmap e157: 8 total, 8 up, 8 in 2023-10-09T05:10:54.876 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:54 smithi194 bash[92666]: cluster 2023-10-09T05:10:54.286821+0000 osd.7 (osd.7) 77 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T05:10:54.876 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:54 smithi194 bash[92666]: cluster 2023-10-09T05:10:54.318889+0000 osd.6 (osd.6) 83 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T05:10:54.949 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:55.248 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:55.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:55.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:55.263 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 86 16 86897 86881 63.1334 42 0.0200988 0.0158323 2023-10-09T05:10:55.265 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:54 smithi133 bash[105589]: cluster 2023-10-09T05:10:53.261858+0000 osd.5 (osd.5) 130 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T05:10:55.265 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:54 smithi133 bash[105589]: cluster 2023-10-09T05:10:53.283648+0000 osd.4 (osd.4) 96 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T05:10:55.265 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:54 smithi133 bash[105589]: cluster 2023-10-09T05:10:53.780445+0000 mgr.y (mgr.25277) 669 : cluster 0 pgmap v637: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 36 MiB/s wr, 582 op/s; 18400/252600 objects degraded (7.284%); 2887/252600 objects misplaced (1.143%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:10:55.265 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:54 smithi133 bash[105589]: cluster 2023-10-09T05:10:54.277351+0000 mon.a (mon.0) 845 : cluster 0 osdmap e157: 8 total, 8 up, 8 in 2023-10-09T05:10:55.265 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:54 smithi133 bash[105589]: cluster 2023-10-09T05:10:54.286821+0000 osd.7 (osd.7) 77 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T05:10:55.265 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:54 smithi133 bash[105589]: cluster 2023-10-09T05:10:54.318889+0000 osd.6 (osd.6) 83 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T05:10:55.265 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:54 smithi133 bash[102858]: cluster 2023-10-09T05:10:54.277351+0000 mon.a (mon.0) 845 : cluster 0 osdmap e157: 8 total, 8 up, 8 in 2023-10-09T05:10:55.265 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:54 smithi133 bash[102858]: cluster 2023-10-09T05:10:54.286821+0000 osd.7 (osd.7) 77 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T05:10:55.266 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:54 smithi133 bash[102858]: cluster 2023-10-09T05:10:54.318889+0000 osd.6 (osd.6) 83 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T05:10:55.356 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:55.395 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:55.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:55.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:55.499 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:55.527 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:55.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:55.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:55.631 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:55.664 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:55.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:55.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:55.767 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:55.804 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:55.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:55.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:55.911 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:55.917 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:55.917 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:55.940 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:55.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:55.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:56.051 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:56.060 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:55 smithi194 bash[92666]: cluster 2023-10-09T05:10:54.276021+0000 osd.5 (osd.5) 131 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T05:10:56.061 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:55 smithi194 bash[92666]: cluster 2023-10-09T05:10:54.284759+0000 osd.4 (osd.4) 97 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T05:10:56.079 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:56.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:56.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:56.188 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:56.198 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:55 smithi133 bash[102858]: cluster 2023-10-09T05:10:54.276021+0000 osd.5 (osd.5) 131 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T05:10:56.198 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:55 smithi133 bash[102858]: cluster 2023-10-09T05:10:54.284759+0000 osd.4 (osd.4) 97 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T05:10:56.198 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:55 smithi133 bash[105589]: cluster 2023-10-09T05:10:54.276021+0000 osd.5 (osd.5) 131 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T05:10:56.198 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:55 smithi133 bash[105589]: cluster 2023-10-09T05:10:54.284759+0000 osd.4 (osd.4) 97 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T05:10:56.223 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:56.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:56.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:56.263 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 87 16 87670 87654 62.963 48.3125 0.0127443 0.0158761 2023-10-09T05:10:56.329 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:56.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:56.360 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:56.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:56.460 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:56.485 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:56.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:56.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:56.593 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:56.621 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:56.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:56.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:56.724 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:56.753 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:56.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:56.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:56.859 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:56.886 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:56.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:56.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:56.990 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:57.026 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:57.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:57.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:57.129 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:57.157 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:57.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:57.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:57.216 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:56 smithi133 bash[105589]: cluster 2023-10-09T05:10:55.781338+0000 mgr.y (mgr.25277) 670 : cluster 0 pgmap v639: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s; 15772/259068 objects degraded (6.088%); 5436/259068 objects misplaced (2.098%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T05:10:57.216 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:56 smithi133 bash[102858]: cluster 2023-10-09T05:10:55.781338+0000 mgr.y (mgr.25277) 670 : cluster 0 pgmap v639: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s; 15772/259068 objects degraded (6.088%); 5436/259068 objects misplaced (2.098%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T05:10:57.261 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:57.263 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 88 16 88451 88435 62.8021 48.8125 0.0071199 0.0159164 2023-10-09T05:10:57.269 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:56 smithi194 bash[92666]: cluster 2023-10-09T05:10:55.781338+0000 mgr.y (mgr.25277) 670 : cluster 0 pgmap v639: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s; 15772/259068 objects degraded (6.088%); 5436/259068 objects misplaced (2.098%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T05:10:57.300 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:57.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:57.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:57.409 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:57.460 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:57.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:57.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:57.565 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:57.603 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:57.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:57.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:57.709 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:57.735 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:57.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:57.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:57.839 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:57.873 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:57.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:57.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:10:57.979 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:58.002 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:58.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:58.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:58.107 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:58.130 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:58.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:58.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:58.241 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:58.263 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 89 16 89230 89214 62.6435 48.6875 0.0213463 0.0159566 2023-10-09T05:10:58.272 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:58.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:58.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:58.379 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:58.403 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:10:58.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:58.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:10:58.509 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:58.543 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:58.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:58.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:58.652 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:58.681 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:58.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:58.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:58.786 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:58.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:58.837 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:58.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:58.938 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:58.956 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:58 smithi133 bash[102858]: cluster 2023-10-09T05:10:57.782095+0000 mgr.y (mgr.25277) 671 : cluster 0 pgmap v640: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 36 MiB/s wr, 568 op/s; 15772/259755 objects degraded (6.072%); 5270/259755 objects misplaced (2.029%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T05:10:58.956 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:58 smithi133 bash[102858]: cluster 2023-10-09T05:10:58.252004+0000 mon.a (mon.0) 846 : cluster 3 Health check update: Degraded data redundancy: 15772/259755 objects degraded (6.072%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:58.956 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:58 smithi133 bash[102858]: audit 2023-10-09T05:10:58.447233+0000 mon.a (mon.0) 847 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:10:58.957 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:10:58 smithi133 bash[102858]: audit 2023-10-09T05:10:58.451240+0000 mon.a (mon.0) 848 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:10:58.957 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:58 smithi133 bash[105589]: cluster 2023-10-09T05:10:57.782095+0000 mgr.y (mgr.25277) 671 : cluster 0 pgmap v640: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 36 MiB/s wr, 568 op/s; 15772/259755 objects degraded (6.072%); 5270/259755 objects misplaced (2.029%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T05:10:58.957 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:58 smithi133 bash[105589]: cluster 2023-10-09T05:10:58.252004+0000 mon.a (mon.0) 846 : cluster 3 Health check update: Degraded data redundancy: 15772/259755 objects degraded (6.072%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:58.957 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:58 smithi133 bash[105589]: audit 2023-10-09T05:10:58.447233+0000 mon.a (mon.0) 847 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:10:58.957 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:10:58 smithi133 bash[105589]: audit 2023-10-09T05:10:58.451240+0000 mon.a (mon.0) 848 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:10:58.974 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:58.981 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:58.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:58.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:59.090 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:59.122 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:10:59.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:59.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:10:59.231 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:59.264 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 90 9 89966 89957 62.4634 46.4375 0.011704 0.0160031 2023-10-09T05:10:59.304 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:58 smithi194 bash[92666]: cluster 2023-10-09T05:10:57.782095+0000 mgr.y (mgr.25277) 671 : cluster 0 pgmap v640: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 36 MiB/s wr, 568 op/s; 15772/259755 objects degraded (6.072%); 5270/259755 objects misplaced (2.029%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T05:10:59.304 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:58 smithi194 bash[92666]: cluster 2023-10-09T05:10:58.252004+0000 mon.a (mon.0) 846 : cluster 3 Health check update: Degraded data redundancy: 15772/259755 objects degraded (6.072%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T05:10:59.304 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:58 smithi194 bash[92666]: audit 2023-10-09T05:10:58.447233+0000 mon.a (mon.0) 847 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:10:59.304 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:10:58 smithi194 bash[92666]: audit 2023-10-09T05:10:58.451240+0000 mon.a (mon.0) 848 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:10:59.320 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:10:59.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:59.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:10:59.421 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:10:59.421 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:10:59.425 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:59.459 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:10:59.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:59.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:10:59.563 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:59.589 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:10:59.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:59.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:10:59.692 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:59.718 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:10:59.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:59.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:10:59.822 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:59.850 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:10:59.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:59.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:10:59.952 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:59.981 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:10:59.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:10:59.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:00.085 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:00.108 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:00.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:00.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:00.211 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:00.239 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:00.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:00.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:00.264 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Total time run: 90.0148 2023-10-09T05:11:00.265 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Total writes made: 89966 2023-10-09T05:11:00.265 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Write size: 65536 2023-10-09T05:11:00.265 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Object size: 65536 2023-10-09T05:11:00.266 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Bandwidth (MB/sec): 62.4661 2023-10-09T05:11:00.266 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Stddev Bandwidth: 34.433 2023-10-09T05:11:00.267 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Max bandwidth (MB/sec): 160.5 2023-10-09T05:11:00.267 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Min bandwidth (MB/sec): 0 2023-10-09T05:11:00.267 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Average IOPS: 999 2023-10-09T05:11:00.267 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Stddev IOPS: 550.937 2023-10-09T05:11:00.267 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Max IOPS: 2568 2023-10-09T05:11:00.268 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Min IOPS: 0 2023-10-09T05:11:00.268 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Average Latency(s): 0.0160033 2023-10-09T05:11:00.268 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Stddev Latency(s): 0.0496901 2023-10-09T05:11:00.268 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Max latency(s): 2.39474 2023-10-09T05:11:00.268 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Min latency(s): 0.00144352 2023-10-09T05:11:00.283 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T05:11:00.343 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:00.378 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:00.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:00.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:00.487 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:00.513 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:00.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:00.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:00.621 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:00.654 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:00.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:00.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:00.759 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:00.787 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:00.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:00.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:00.894 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:00.900 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:00 smithi133 bash[102858]: cluster 2023-10-09T05:10:59.783330+0000 mgr.y (mgr.25277) 672 : cluster 0 pgmap v641: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 727 op/s; 15772/267030 objects degraded (5.906%); 5103/267030 objects misplaced (1.911%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T05:11:00.901 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:00 smithi133 bash[105589]: cluster 2023-10-09T05:10:59.783330+0000 mgr.y (mgr.25277) 672 : cluster 0 pgmap v641: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 727 op/s; 15772/267030 objects degraded (5.906%); 5103/267030 objects misplaced (1.911%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T05:11:00.920 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:00.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:00.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:01.028 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:01.037 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:00 smithi194 bash[92666]: cluster 2023-10-09T05:10:59.783330+0000 mgr.y (mgr.25277) 672 : cluster 0 pgmap v641: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 727 op/s; 15772/267030 objects degraded (5.906%); 5103/267030 objects misplaced (1.911%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T05:11:01.061 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:01.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:01.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:01.167 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:01.200 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:01.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:01.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:01.304 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:01.325 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:01.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:01.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:01.428 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:01.461 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:01.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:01.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:01.564 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:01.600 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:01.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:01.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:01.707 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:01.726 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:01.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:01.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:01.828 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:01.869 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:01.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:01.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:01.972 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:02.005 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:02.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:02.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:02.109 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:02.136 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:02.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:02.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:02.242 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:02.275 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:02.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:02.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:02.381 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:02.415 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:02.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:02.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:02.519 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:02.560 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:02.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:02.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:02.670 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:02.696 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:02.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:02.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:02.800 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:02.841 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:02.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:02.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:02.944 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:02.979 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:02.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:02.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:03.085 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:03.086 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:02 smithi133 bash[102858]: cluster 2023-10-09T05:11:01.783962+0000 mgr.y (mgr.25277) 673 : cluster 0 pgmap v642: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 41 MiB/s wr, 673 op/s; 15772/268959 objects degraded (5.864%); 5103/268959 objects misplaced (1.897%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T05:11:03.086 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:02 smithi133 bash[105589]: cluster 2023-10-09T05:11:01.783962+0000 mgr.y (mgr.25277) 673 : cluster 0 pgmap v642: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 41 MiB/s wr, 673 op/s; 15772/268959 objects degraded (5.864%); 5103/268959 objects misplaced (1.897%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T05:11:03.094 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:02 smithi194 bash[92666]: cluster 2023-10-09T05:11:01.783962+0000 mgr.y (mgr.25277) 673 : cluster 0 pgmap v642: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 41 MiB/s wr, 673 op/s; 15772/268959 objects degraded (5.864%); 5103/268959 objects misplaced (1.897%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T05:11:03.112 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:03.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:03.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:03.217 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:03.245 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:03.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:03.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:03.348 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:03.372 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:03.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:03.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:03.474 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:03.503 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:03.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:03.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:03.613 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:03.650 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:03.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:03.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:03.754 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:03.778 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:03.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:03.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:03.883 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:03.908 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:03.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:03.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:03.918 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T05:11:04.013 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:04.016 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:03 smithi133 bash[102858]: cluster 2023-10-09T05:11:03.255689+0000 mon.a (mon.0) 849 : cluster 3 Health check update: Degraded data redundancy: 15772/268959 objects degraded (5.864%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:04.016 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:03 smithi133 bash[105589]: cluster 2023-10-09T05:11:03.255689+0000 mon.a (mon.0) 849 : cluster 3 Health check update: Degraded data redundancy: 15772/268959 objects degraded (5.864%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:04.046 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:04.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:04.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:04.149 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:04.159 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:03 smithi194 bash[92666]: cluster 2023-10-09T05:11:03.255689+0000 mon.a (mon.0) 849 : cluster 3 Health check update: Degraded data redundancy: 15772/268959 objects degraded (5.864%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:04.179 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:04.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:04.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:04.285 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:04.314 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:04.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:04.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:04.424 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:04.451 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:04.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:04.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:04.559 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:04.601 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:04.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:04.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:04.704 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:04.713 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:11:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:11:04] "GET /metrics HTTP/1.1" 200 36431 "" "Prometheus/2.43.0" 2023-10-09T05:11:04.725 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:04.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:04.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:04.830 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:04.870 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:04.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:04.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:04.976 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:05.004 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:05.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:05.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:05.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:04 smithi133 bash[105589]: cluster 2023-10-09T05:11:03.784643+0000 mgr.y (mgr.25277) 674 : cluster 0 pgmap v643: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 36 MiB/s wr, 649 op/s; 15772/267837 objects degraded (5.889%); 5103/267837 objects misplaced (1.905%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:11:05.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:04 smithi133 bash[102858]: cluster 2023-10-09T05:11:03.784643+0000 mgr.y (mgr.25277) 674 : cluster 0 pgmap v643: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 36 MiB/s wr, 649 op/s; 15772/267837 objects degraded (5.889%); 5103/267837 objects misplaced (1.905%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:11:05.108 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:05.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:04 smithi194 bash[92666]: cluster 2023-10-09T05:11:03.784643+0000 mgr.y (mgr.25277) 674 : cluster 0 pgmap v643: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 36 MiB/s wr, 649 op/s; 15772/267837 objects degraded (5.889%); 5103/267837 objects misplaced (1.905%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:11:05.140 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:05.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:05.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:05.245 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:05.272 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:05.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:05.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:05.381 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:05.417 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:05.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:05.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:05.521 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:05.549 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:05.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:05.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:05.656 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:05.680 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:05.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:05.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:05.783 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:05.809 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:05.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:05.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:05.913 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:05.945 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:05.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:05.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:06.048 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:06.081 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:06.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:06.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:06.185 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:06.214 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:06.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:06.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:06.321 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:06.355 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:06.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:06.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:06.459 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:06.491 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:06.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:06.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:06.598 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:06.634 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:06.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:06.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:06.741 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:06.772 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:06.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:06.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:06.880 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:06.922 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:06 smithi133 bash[105589]: cluster 2023-10-09T05:11:05.785472+0000 mgr.y (mgr.25277) 675 : cluster 0 pgmap v644: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 88 B/s rd, 31 MiB/s wr, 683 op/s; 15772/263763 objects degraded (5.980%); 4867/263763 objects misplaced (1.845%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T05:11:06.923 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:06 smithi133 bash[102858]: cluster 2023-10-09T05:11:05.785472+0000 mgr.y (mgr.25277) 675 : cluster 0 pgmap v644: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 88 B/s rd, 31 MiB/s wr, 683 op/s; 15772/263763 objects degraded (5.980%); 4867/263763 objects misplaced (1.845%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T05:11:06.923 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:06.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:06.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:07.028 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:07.066 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:07.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:07.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:07.170 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:07.202 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:07.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:07.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:07.306 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:07.315 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:06 smithi194 bash[92666]: cluster 2023-10-09T05:11:05.785472+0000 mgr.y (mgr.25277) 675 : cluster 0 pgmap v644: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 88 B/s rd, 31 MiB/s wr, 683 op/s; 15772/263763 objects degraded (5.980%); 4867/263763 objects misplaced (1.845%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T05:11:07.337 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:07.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:07.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:07.440 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:07.488 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:07.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:07.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:07.593 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:07.629 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:07.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:07.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:07.736 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:07.771 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:07.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:07.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:07.874 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:07.906 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:07.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:07.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:08.010 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:08.036 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:08.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:08.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:08.145 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:08.170 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:08.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:08.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:08.274 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:08.305 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:08.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:08.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:08.411 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:08.448 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:08.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:08.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:08.554 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:08.579 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:08.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:08.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:08.684 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:08.721 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:08.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:08.729 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:08.830 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:08.867 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:08.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:08.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:08.972 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:09.000 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:09.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:09.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:09.104 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:09.112 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:08 smithi194 bash[92666]: cluster 2023-10-09T05:11:07.786156+0000 mgr.y (mgr.25277) 676 : cluster 0 pgmap v645: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 576 op/s; 15772/260109 objects degraded (6.064%); 4867/260109 objects misplaced (1.871%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T05:11:09.113 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:08 smithi194 bash[92666]: cluster 2023-10-09T05:11:08.256982+0000 mon.a (mon.0) 850 : cluster 3 Health check update: Degraded data redundancy: 15772/260109 objects degraded (6.064%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:09.134 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:09.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:09.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:09.238 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:09.247 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:08 smithi133 bash[105589]: cluster 2023-10-09T05:11:07.786156+0000 mgr.y (mgr.25277) 676 : cluster 0 pgmap v645: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 576 op/s; 15772/260109 objects degraded (6.064%); 4867/260109 objects misplaced (1.871%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T05:11:09.247 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:08 smithi133 bash[105589]: cluster 2023-10-09T05:11:08.256982+0000 mon.a (mon.0) 850 : cluster 3 Health check update: Degraded data redundancy: 15772/260109 objects degraded (6.064%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:09.247 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:08 smithi133 bash[102858]: cluster 2023-10-09T05:11:07.786156+0000 mgr.y (mgr.25277) 676 : cluster 0 pgmap v645: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 576 op/s; 15772/260109 objects degraded (6.064%); 4867/260109 objects misplaced (1.871%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T05:11:09.248 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:08 smithi133 bash[102858]: cluster 2023-10-09T05:11:08.256982+0000 mon.a (mon.0) 850 : cluster 3 Health check update: Degraded data redundancy: 15772/260109 objects degraded (6.064%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:09.272 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:09.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:09.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:09.381 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:09.407 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:09.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:09.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:09.512 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:09.544 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:09.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:09.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:09.649 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:09.673 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:09.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:09.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:09.780 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:09.805 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:09.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:09.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:09.908 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:09.938 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:09.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:09.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:10.044 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:10.071 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:10.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:10.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:10.173 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:10.204 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:10.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:10.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:10.308 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:10.338 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:10.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:10.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:10.446 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:10.501 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:10.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:10.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:10.605 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:10.629 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:10.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:10.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:10.729 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:11:10.733 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:10.774 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:10.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:10.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:10.883 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:10.912 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:10.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:10.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:11.016 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:11.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:10 smithi133 bash[102858]: cluster 2023-10-09T05:11:09.786964+0000 mgr.y (mgr.25277) 677 : cluster 0 pgmap v646: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 782 op/s; 15772/252000 objects degraded (6.259%); 4675/252000 objects misplaced (1.855%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T05:11:11.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:10 smithi133 bash[105589]: cluster 2023-10-09T05:11:09.786964+0000 mgr.y (mgr.25277) 677 : cluster 0 pgmap v646: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 782 op/s; 15772/252000 objects degraded (6.259%); 4675/252000 objects misplaced (1.855%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T05:11:11.049 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:11.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:11.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:11.152 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:11.161 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:10 smithi194 bash[92666]: cluster 2023-10-09T05:11:09.786964+0000 mgr.y (mgr.25277) 677 : cluster 0 pgmap v646: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 782 op/s; 15772/252000 objects degraded (6.259%); 4675/252000 objects misplaced (1.855%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T05:11:11.180 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:11.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:11.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:11.283 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:11.320 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:11.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:11.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:11.427 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:11.457 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:11.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:11.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:11.560 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:11.589 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:11.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:11.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:11.692 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:11.727 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:11.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:11.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:11.834 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:11.863 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:11.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:11.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:11.968 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:12.013 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:12.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:12.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:12.120 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:12.152 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:12.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:12.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:12.258 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:12.286 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:12.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:12.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:12.391 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:12.425 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:12.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:12.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:12.533 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:12.557 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:12.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:12.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:12.661 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:12.682 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:12.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:12.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:12.788 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:12.825 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:12.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:12.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:12.931 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:12.974 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:12.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:12.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:13.078 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:13.125 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:13.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:13.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:13.206 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:12 smithi133 bash[102858]: cluster 2023-10-09T05:11:11.788995+0000 mgr.y (mgr.25277) 678 : cluster 0 pgmap v647: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 5.1 MiB/s wr, 682 op/s; 15772/248340 objects degraded (6.351%); 4675/248340 objects misplaced (1.882%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T05:11:13.207 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:12 smithi133 bash[105589]: cluster 2023-10-09T05:11:11.788995+0000 mgr.y (mgr.25277) 678 : cluster 0 pgmap v647: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 5.1 MiB/s wr, 682 op/s; 15772/248340 objects degraded (6.351%); 4675/248340 objects misplaced (1.882%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T05:11:13.233 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:13.240 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:12 smithi194 bash[92666]: cluster 2023-10-09T05:11:11.788995+0000 mgr.y (mgr.25277) 678 : cluster 0 pgmap v647: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 5.1 MiB/s wr, 682 op/s; 15772/248340 objects degraded (6.351%); 4675/248340 objects misplaced (1.882%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T05:11:13.259 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:13.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:13.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:13.264 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:11:13.314 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:11:13.362 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:13.363 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:11:13.389 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:13.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:13.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:13.406 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:11:13.452 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:11:13.494 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:13.502 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:11:13.517 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:13.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:13.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:13.542 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:11:13.586 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:11:13.590 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:11:13.590 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:11:13.590 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:11:13.590 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:11:13.590 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:11:13.590 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:11:13.590 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:11:13.590 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:11:13.623 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:13.652 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:11:13.655 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:13.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:13.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:13.659 INFO:teuthology.orchestra.run.smithi133.stderr:dumped all 2023-10-09T05:11:13.758 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:13.784 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:13.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:13.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:13.890 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:13.920 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:13.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:13.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:14.027 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:14.053 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:14.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:14.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:14.159 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:14.169 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:13 smithi133 bash[105589]: audit 2023-10-09T05:11:13.200027+0000 mon.c (mon.2) 66 : audit 0 from='client.? 172.21.15.133:0/3761185709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:11:14.169 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:13 smithi133 bash[105589]: cluster 2023-10-09T05:11:13.258371+0000 mon.a (mon.0) 851 : cluster 3 Health check update: Degraded data redundancy: 15772/248340 objects degraded (6.351%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:14.169 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:13 smithi133 bash[105589]: audit 2023-10-09T05:11:13.442361+0000 mon.a (mon.0) 852 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:11:14.169 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:13 smithi133 bash[105589]: audit 2023-10-09T05:11:13.772295+0000 mon.a (mon.0) 853 : audit 1 from='client.? 172.21.15.133:0/2928855975' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:11:14.169 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:13 smithi133 bash[102858]: audit 2023-10-09T05:11:13.200027+0000 mon.c (mon.2) 66 : audit 0 from='client.? 172.21.15.133:0/3761185709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:11:14.169 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:13 smithi133 bash[102858]: cluster 2023-10-09T05:11:13.258371+0000 mon.a (mon.0) 851 : cluster 3 Health check update: Degraded data redundancy: 15772/248340 objects degraded (6.351%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:14.169 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:13 smithi133 bash[102858]: audit 2023-10-09T05:11:13.442361+0000 mon.a (mon.0) 852 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:11:14.169 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:13 smithi133 bash[102858]: audit 2023-10-09T05:11:13.772295+0000 mon.a (mon.0) 853 : audit 1 from='client.? 172.21.15.133:0/2928855975' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:11:14.191 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:14.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:14.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:14.295 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:14.303 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is set 2023-10-09T05:11:14.318 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:14.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:14.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:14.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:13 smithi194 bash[92666]: audit 2023-10-09T05:11:13.200027+0000 mon.c (mon.2) 66 : audit 0 from='client.? 172.21.15.133:0/3761185709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:11:14.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:13 smithi194 bash[92666]: cluster 2023-10-09T05:11:13.258371+0000 mon.a (mon.0) 851 : cluster 3 Health check update: Degraded data redundancy: 15772/248340 objects degraded (6.351%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:14.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:13 smithi194 bash[92666]: audit 2023-10-09T05:11:13.442361+0000 mon.a (mon.0) 852 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:11:14.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:13 smithi194 bash[92666]: audit 2023-10-09T05:11:13.772295+0000 mon.a (mon.0) 853 : audit 1 from='client.? 172.21.15.133:0/2928855975' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:11:14.422 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:14.463 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:14.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:14.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:14.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:11:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:11:14] "GET /metrics HTTP/1.1" 200 36432 "" "Prometheus/2.43.0" 2023-10-09T05:11:14.565 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:14.684 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:14.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:14.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:14.787 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:14.815 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:14.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:14.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:14.920 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:14.944 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:14.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:14.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:14.953 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:14 smithi194 bash[92666]: audit 2023-10-09T05:11:13.627480+0000 mgr.y (mgr.25277) 679 : audit 0 from='client.34905 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:11:14.953 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:14 smithi194 bash[92666]: cluster 2023-10-09T05:11:13.789691+0000 mgr.y (mgr.25277) 680 : cluster 0 pgmap v648: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 1.0 MiB/s wr, 678 op/s; 15772/245679 objects degraded (6.420%); 4675/245679 objects misplaced (1.903%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T05:11:14.953 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:14 smithi194 bash[92666]: cluster 2023-10-09T05:11:14.277383+0000 mon.a (mon.0) 854 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:11:14.953 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:14 smithi194 bash[92666]: cluster 2023-10-09T05:11:14.292644+0000 osd.6 (osd.6) 84 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T05:11:14.953 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:14 smithi194 bash[92666]: audit 2023-10-09T05:11:14.295747+0000 mon.a (mon.0) 855 : audit 1 from='client.? 172.21.15.133:0/2928855975' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:11:14.954 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:14 smithi194 bash[92666]: cluster 2023-10-09T05:11:14.295978+0000 mon.a (mon.0) 856 : cluster 0 osdmap e158: 8 total, 8 up, 8 in 2023-10-09T05:11:14.954 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:14 smithi194 bash[92666]: cluster 2023-10-09T05:11:14.300624+0000 osd.7 (osd.7) 78 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T05:11:14.954 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:14 smithi194 bash[92666]: cluster 2023-10-09T05:11:14.302594+0000 osd.5 (osd.5) 132 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T05:11:15.047 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:15.071 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:15.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:15.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:15.174 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:15.205 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:15.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:15.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:15.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:14 smithi133 bash[105589]: audit 2023-10-09T05:11:13.627480+0000 mgr.y (mgr.25277) 679 : audit 0 from='client.34905 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:11:15.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:14 smithi133 bash[105589]: cluster 2023-10-09T05:11:13.789691+0000 mgr.y (mgr.25277) 680 : cluster 0 pgmap v648: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 1.0 MiB/s wr, 678 op/s; 15772/245679 objects degraded (6.420%); 4675/245679 objects misplaced (1.903%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T05:11:15.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:14 smithi133 bash[105589]: cluster 2023-10-09T05:11:14.277383+0000 mon.a (mon.0) 854 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:11:15.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:14 smithi133 bash[105589]: cluster 2023-10-09T05:11:14.292644+0000 osd.6 (osd.6) 84 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T05:11:15.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:14 smithi133 bash[105589]: audit 2023-10-09T05:11:14.295747+0000 mon.a (mon.0) 855 : audit 1 from='client.? 172.21.15.133:0/2928855975' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:11:15.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:14 smithi133 bash[105589]: cluster 2023-10-09T05:11:14.295978+0000 mon.a (mon.0) 856 : cluster 0 osdmap e158: 8 total, 8 up, 8 in 2023-10-09T05:11:15.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:14 smithi133 bash[105589]: cluster 2023-10-09T05:11:14.300624+0000 osd.7 (osd.7) 78 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T05:11:15.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:14 smithi133 bash[105589]: cluster 2023-10-09T05:11:14.302594+0000 osd.5 (osd.5) 132 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T05:11:15.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:14 smithi133 bash[102858]: audit 2023-10-09T05:11:13.627480+0000 mgr.y (mgr.25277) 679 : audit 0 from='client.34905 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:11:15.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:14 smithi133 bash[102858]: cluster 2023-10-09T05:11:13.789691+0000 mgr.y (mgr.25277) 680 : cluster 0 pgmap v648: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 1.0 MiB/s wr, 678 op/s; 15772/245679 objects degraded (6.420%); 4675/245679 objects misplaced (1.903%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T05:11:15.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:14 smithi133 bash[102858]: cluster 2023-10-09T05:11:14.277383+0000 mon.a (mon.0) 854 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:11:15.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:14 smithi133 bash[102858]: cluster 2023-10-09T05:11:14.292644+0000 osd.6 (osd.6) 84 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T05:11:15.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:14 smithi133 bash[102858]: audit 2023-10-09T05:11:14.295747+0000 mon.a (mon.0) 855 : audit 1 from='client.? 172.21.15.133:0/2928855975' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:11:15.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:14 smithi133 bash[102858]: cluster 2023-10-09T05:11:14.295978+0000 mon.a (mon.0) 856 : cluster 0 osdmap e158: 8 total, 8 up, 8 in 2023-10-09T05:11:15.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:14 smithi133 bash[102858]: cluster 2023-10-09T05:11:14.300624+0000 osd.7 (osd.7) 78 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T05:11:15.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:14 smithi133 bash[102858]: cluster 2023-10-09T05:11:14.302594+0000 osd.5 (osd.5) 132 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T05:11:15.311 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:15.353 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:15.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:15.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:15.457 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:15.485 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:15.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:15.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:15.588 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:15.614 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:15.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:15.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:15.719 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:15.749 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:15.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:15.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:15.853 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:15.880 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:15.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:15.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:15.985 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:16.011 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:16.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:16.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:16.114 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:16.147 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:16.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:16.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:16.252 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:16.281 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:16.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:16.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:16.384 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:16.411 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:16.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:16.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:16.516 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:16.539 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:16.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:16.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:16.641 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:16.673 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:16.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:16.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:16.779 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:16.805 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:16.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:16.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:16.910 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:16.944 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:16.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:16.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:17.048 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:17.078 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:17.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:17.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:17.184 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:17.214 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:17.217 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:16 smithi133 bash[105589]: cluster 2023-10-09T05:11:15.790280+0000 mgr.y (mgr.25277) 681 : cluster 0 pgmap v650: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 15772/240783 objects degraded (6.550%); 4505/240783 objects misplaced (1.871%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:11:17.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:17.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:17.218 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:16 smithi133 bash[102858]: cluster 2023-10-09T05:11:15.790280+0000 mgr.y (mgr.25277) 681 : cluster 0 pgmap v650: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 15772/240783 objects degraded (6.550%); 4505/240783 objects misplaced (1.871%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:11:17.319 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:17.347 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:17.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:17.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:17.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:16 smithi194 bash[92666]: cluster 2023-10-09T05:11:15.790280+0000 mgr.y (mgr.25277) 681 : cluster 0 pgmap v650: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 15772/240783 objects degraded (6.550%); 4505/240783 objects misplaced (1.871%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:11:17.451 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:17.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:17.495 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:17.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:17.596 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:17.630 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:17.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:17.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:17.734 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:17.766 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:17.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:17.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:17.868 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:17.899 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:17.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:17.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:18.007 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:18.031 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:18.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:18.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:18.135 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:18.174 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:18.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:18.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:18.276 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:18.312 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:18.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:18.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:18.416 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:18.446 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:18.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:18.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:18.553 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:18.575 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:18.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:18.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:18.681 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:18.716 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:18.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:18.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:18.825 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:18.868 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:18.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:18.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:18.948 INFO:teuthology.orchestra.run.smithi133.stdout:{"pg_ready":true,"pg_map":{"version":647,"stamp":"2023-10-09T05:11:11.788283+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5424027576,"num_objects":82780,"num_object_clones":0,"num_object_copies":248340,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15772,"num_objects_misplaced":4675,"num_objects_unfound":0,"num_objects_dirty":82780,"num_whiteouts":0,"num_read":539,"num_read_kb":965,"num_write":97478,"num_write_kb":5760270,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4568,"num_bytes_recovered":300353120,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97400,"ondisk_log_size":97400,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":276,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":20607212,"kb_used_data":16719020,"kb_used_omap":20,"kb_used_meta":3887979,"kb_avail":729354004,"statfs":{"total":767960285184,"available":746858500096,"internally_reserved":0,"allocated":17120276480,"data_stored":17114532700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":3981291150},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-408354792,"num_objects":-6230,"num_object_clones":0,"num_object_copies":-18690,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-428,"num_objects_unfound":0,"num_objects_dirty":-6230,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8192,"num_write_kb":62721,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":428,"num_bytes_recovered":28049408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005883"},"pg_stats":[{"pgid":"3.1a","version":"69'2","reported_seq":157,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.287222+0000","last_change":"2023-10-09T05:09:57.873320+0000","last_active":"2023-10-09T05:10:53.287222+0000","last_peered":"2023-10-09T05:10:53.287222+0000","last_clean":"2023-10-09T05:10:53.287222+0000","last_became_active":"2023-10-09T05:09:57.868376+0000","last_became_peered":"2023-10-09T05:09:57.868376+0000","last_unstale":"2023-10-09T05:10:53.287222+0000","last_undegraded":"2023-10-09T05:10:53.287222+0000","last_fullsized":"2023-10-09T05:10:53.287222+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:27.186610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":124,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.277098+0000","last_change":"2023-10-09T05:09:57.854324+0000","last_active":"2023-10-09T05:10:53.277098+0000","last_peered":"2023-10-09T05:10:53.277098+0000","last_clean":"2023-10-09T05:10:53.277098+0000","last_became_active":"2023-10-09T05:09:57.853925+0000","last_became_peered":"2023-10-09T05:09:57.853925+0000","last_unstale":"2023-10-09T05:10:53.277098+0000","last_undegraded":"2023-10-09T05:10:53.277098+0000","last_fullsized":"2023-10-09T05:10:53.277098+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:11.255018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":16,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.261169+0000","last_change":"2023-10-09T05:09:57.857511+0000","last_active":"2023-10-09T05:10:53.261169+0000","last_peered":"2023-10-09T05:10:53.261169+0000","last_clean":"2023-10-09T05:10:53.261169+0000","last_became_active":"2023-10-09T05:09:57.857193+0000","last_became_peered":"2023-10-09T05:09:57.857193+0000","last_unstale":"2023-10-09T05:10:53.261169+0000","last_undegraded":"2023-10-09T05:10:53.261169+0000","last_fullsized":"2023-10-09T05:10:53.261169+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:32.842980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":113,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-10-09T05:10:54.315964+0000","last_change":"2023-10-09T05:02:09.648283+0000","last_active":"2023-10-09T05:10:54.315964+0000","last_peered":"2023-10-09T05:10:54.315964+0000","last_clean":"2023-10-09T05:10:54.315964+0000","last_became_active":"2023-10-09T05:02:09.647778+0000","last_became_peered":"2023-10-09T05:02:09.647778+0000","last_unstale":"2023-10-09T05:10:54.315964+0000","last_undegraded":"2023-10-09T05:10:54.315964+0000","last_fullsized":"2023-10-09T05:10:54.315964+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:24.582392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":142,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.274424+0000","last_change":"2023-10-09T05:09:57.849878+0000","last_active":"2023-10-09T05:10:53.274424+0000","last_peered":"2023-10-09T05:10:53.274424+0000","last_clean":"2023-10-09T05:10:53.274424+0000","last_became_active":"2023-10-09T05:09:57.849541+0000","last_became_peered":"2023-10-09T05:09:57.849541+0000","last_unstale":"2023-10-09T05:10:53.274424+0000","last_undegraded":"2023-10-09T05:10:53.274424+0000","last_fullsized":"2023-10-09T05:10:53.274424+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:49:03.330682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":137,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.259944+0000","last_change":"2023-10-09T05:09:57.852119+0000","last_active":"2023-10-09T05:10:53.259944+0000","last_peered":"2023-10-09T05:10:53.259944+0000","last_clean":"2023-10-09T05:10:53.259944+0000","last_became_active":"2023-10-09T05:09:57.851752+0000","last_became_peered":"2023-10-09T05:09:57.851752+0000","last_unstale":"2023-10-09T05:10:53.259944+0000","last_undegraded":"2023-10-09T05:10:53.259944+0000","last_fullsized":"2023-10-09T05:10:53.259944+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:58:09.571362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":110,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.267472+0000","last_change":"2023-10-09T05:02:09.646765+0000","last_active":"2023-10-09T05:10:53.267472+0000","last_peered":"2023-10-09T05:10:53.267472+0000","last_clean":"2023-10-09T05:10:53.267472+0000","last_became_active":"2023-10-09T05:02:09.645141+0000","last_became_peered":"2023-10-09T05:02:09.645141+0000","last_unstale":"2023-10-09T05:10:53.267472+0000","last_undegraded":"2023-10-09T05:10:53.267472+0000","last_fullsized":"2023-10-09T05:10:53.267472+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:10.098578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":124,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.274488+0000","last_change":"2023-10-09T05:09:57.849695+0000","last_active":"2023-10-09T05:10:53.274488+0000","last_peered":"2023-10-09T05:10:53.274488+0000","last_clean":"2023-10-09T05:10:53.274488+0000","last_became_active":"2023-10-09T05:09:57.848478+0000","last_became_peered":"2023-10-09T05:09:57.848478+0000","last_unstale":"2023-10-09T05:10:53.274488+0000","last_undegraded":"2023-10-09T05:10:53.274488+0000","last_fullsized":"2023-10-09T05:10:53.274488+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:26.115312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":138,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.275986+0000","last_change":"2023-10-09T05:09:57.867139+0000","last_active":"2023-10-09T05:10:53.275986+0000","last_peered":"2023-10-09T05:10:53.275986+0000","last_clean":"2023-10-09T05:10:53.275986+0000","last_became_active":"2023-10-09T05:09:57.866134+0000","last_became_peered":"2023-10-09T05:09:57.866134+0000","last_unstale":"2023-10-09T05:10:53.275986+0000","last_undegraded":"2023-10-09T05:10:53.275986+0000","last_fullsized":"2023-10-09T05:10:53.275986+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:45.375784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":101,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.277149+0000","last_change":"2023-10-09T05:02:09.651167+0000","last_active":"2023-10-09T05:10:53.277149+0000","last_peered":"2023-10-09T05:10:53.277149+0000","last_clean":"2023-10-09T05:10:53.277149+0000","last_became_active":"2023-10-09T05:02:09.650419+0000","last_became_peered":"2023-10-09T05:02:09.650419+0000","last_unstale":"2023-10-09T05:10:53.277149+0000","last_undegraded":"2023-10-09T05:10:53.277149+0000","last_fullsized":"2023-10-09T05:10:53.277149+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:55:16.966938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":123,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.290801+0000","last_change":"2023-10-09T05:02:09.649700+0000","last_active":"2023-10-09T05:10:53.290801+0000","last_peered":"2023-10-09T05:10:53.290801+0000","last_clean":"2023-10-09T05:10:53.290801+0000","last_became_active":"2023-10-09T05:02:09.649317+0000","last_became_peered":"2023-10-09T05:02:09.649317+0000","last_unstale":"2023-10-09T05:10:53.290801+0000","last_undegraded":"2023-10-09T05:10:53.290801+0000","last_fullsized":"2023-10-09T05:10:53.290801+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:50.711767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":16,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.270771+0000","last_change":"2023-10-09T05:09:57.848297+0000","last_active":"2023-10-09T05:10:53.270771+0000","last_peered":"2023-10-09T05:10:53.270771+0000","last_clean":"2023-10-09T05:10:53.270771+0000","last_became_active":"2023-10-09T05:09:57.847987+0000","last_became_peered":"2023-10-09T05:09:57.847987+0000","last_unstale":"2023-10-09T05:10:53.270771+0000","last_undegraded":"2023-10-09T05:10:53.270771+0000","last_fullsized":"2023-10-09T05:10:53.270771+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:16.385601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'23","reported_seq":129,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.288132+0000","last_change":"2023-10-09T04:44:52.781203+0000","last_active":"2023-10-09T05:10:53.288132+0000","last_peered":"2023-10-09T05:10:53.288132+0000","last_clean":"2023-10-09T05:10:53.288132+0000","last_became_active":"2023-10-09T04:44:52.780902+0000","last_became_peered":"2023-10-09T04:44:52.780902+0000","last_unstale":"2023-10-09T05:10:53.288132+0000","last_undegraded":"2023-10-09T05:10:53.288132+0000","last_fullsized":"2023-10-09T05:10:53.288132+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:56.332974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":137,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.287569+0000","last_change":"2023-10-09T05:09:57.868761+0000","last_active":"2023-10-09T05:10:53.287569+0000","last_peered":"2023-10-09T05:10:53.287569+0000","last_clean":"2023-10-09T05:10:53.287569+0000","last_became_active":"2023-10-09T05:09:57.868121+0000","last_became_peered":"2023-10-09T05:09:57.868121+0000","last_unstale":"2023-10-09T05:10:53.287569+0000","last_undegraded":"2023-10-09T05:10:53.287569+0000","last_fullsized":"2023-10-09T05:10:53.287569+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:43.412235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":122,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.286740+0000","last_change":"2023-10-09T05:01:02.669186+0000","last_active":"2023-10-09T05:10:53.286740+0000","last_peered":"2023-10-09T05:10:53.286740+0000","last_clean":"2023-10-09T05:10:53.286740+0000","last_became_active":"2023-10-09T05:01:02.668739+0000","last_became_peered":"2023-10-09T05:01:02.668739+0000","last_unstale":"2023-10-09T05:10:53.286740+0000","last_undegraded":"2023-10-09T05:10:53.286740+0000","last_fullsized":"2023-10-09T05:10:53.286740+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:41.352624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":124,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.283553+0000","last_change":"2023-10-09T05:09:57.854619+0000","last_active":"2023-10-09T05:10:53.283553+0000","last_peered":"2023-10-09T05:10:53.283553+0000","last_clean":"2023-10-09T05:10:53.283553+0000","last_became_active":"2023-10-09T05:09:57.852039+0000","last_became_peered":"2023-10-09T05:09:57.852039+0000","last_unstale":"2023-10-09T05:10:53.283553+0000","last_undegraded":"2023-10-09T05:10:53.283553+0000","last_fullsized":"2023-10-09T05:10:53.283553+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:46:02.986058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":111,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-10-09T05:10:54.314166+0000","last_change":"2023-10-09T05:02:09.645017+0000","last_active":"2023-10-09T05:10:54.314166+0000","last_peered":"2023-10-09T05:10:54.314166+0000","last_clean":"2023-10-09T05:10:54.314166+0000","last_became_active":"2023-10-09T05:02:09.644705+0000","last_became_peered":"2023-10-09T05:02:09.644705+0000","last_unstale":"2023-10-09T05:10:54.314166+0000","last_undegraded":"2023-10-09T05:10:54.314166+0000","last_fullsized":"2023-10-09T05:10:54.314166+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:20.695961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":117,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.275708+0000","last_change":"2023-10-09T05:02:09.648362+0000","last_active":"2023-10-09T05:10:53.275708+0000","last_peered":"2023-10-09T05:10:53.275708+0000","last_clean":"2023-10-09T05:10:53.275708+0000","last_became_active":"2023-10-09T05:02:09.647127+0000","last_became_peered":"2023-10-09T05:02:09.647127+0000","last_unstale":"2023-10-09T05:10:53.275708+0000","last_undegraded":"2023-10-09T05:10:53.275708+0000","last_fullsized":"2023-10-09T05:10:53.275708+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:38:02.875613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":101,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.283672+0000","last_change":"2023-10-09T04:44:52.792557+0000","last_active":"2023-10-09T05:10:53.283672+0000","last_peered":"2023-10-09T05:10:53.283672+0000","last_clean":"2023-10-09T05:10:53.283672+0000","last_became_active":"2023-10-09T04:44:52.791890+0000","last_became_peered":"2023-10-09T04:44:52.791890+0000","last_unstale":"2023-10-09T05:10:53.283672+0000","last_undegraded":"2023-10-09T05:10:53.283672+0000","last_fullsized":"2023-10-09T05:10:53.283672+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:04:46.379852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":111,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.267400+0000","last_change":"2023-10-09T05:02:09.652158+0000","last_active":"2023-10-09T05:10:53.267400+0000","last_peered":"2023-10-09T05:10:53.267400+0000","last_clean":"2023-10-09T05:10:53.267400+0000","last_became_active":"2023-10-09T05:02:09.651811+0000","last_became_peered":"2023-10-09T05:02:09.651811+0000","last_unstale":"2023-10-09T05:10:53.267400+0000","last_undegraded":"2023-10-09T05:10:53.267400+0000","last_fullsized":"2023-10-09T05:10:53.267400+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:49:10.141237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":130,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.259215+0000","last_change":"2023-10-09T04:56:56.076177+0000","last_active":"2023-10-09T05:10:53.259215+0000","last_peered":"2023-10-09T05:10:53.259215+0000","last_clean":"2023-10-09T05:10:53.259215+0000","last_became_active":"2023-10-09T04:56:56.075564+0000","last_became_peered":"2023-10-09T04:56:56.075564+0000","last_unstale":"2023-10-09T05:10:53.259215+0000","last_undegraded":"2023-10-09T05:10:53.259215+0000","last_fullsized":"2023-10-09T05:10:53.259215+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:38.390793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":124,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.277191+0000","last_change":"2023-10-09T05:09:57.854475+0000","last_active":"2023-10-09T05:10:53.277191+0000","last_peered":"2023-10-09T05:10:53.277191+0000","last_clean":"2023-10-09T05:10:53.277191+0000","last_became_active":"2023-10-09T05:09:57.851782+0000","last_became_peered":"2023-10-09T05:09:57.851782+0000","last_unstale":"2023-10-09T05:10:53.277191+0000","last_undegraded":"2023-10-09T05:10:53.277191+0000","last_fullsized":"2023-10-09T05:10:53.277191+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:20:41.445868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"157'6131","reported_seq":6159,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-10-09T05:11:10.776942+0000","last_change":"2023-10-09T05:09:03.829573+0000","last_active":"2023-10-09T05:11:10.776942+0000","last_peered":"2023-10-09T05:11:10.776942+0000","last_clean":"2023-10-09T05:11:10.776942+0000","last_became_active":"2023-10-09T05:09:03.829205+0000","last_became_peered":"2023-10-09T05:09:03.829205+0000","last_unstale":"2023-10-09T05:11:10.776942+0000","last_undegraded":"2023-10-09T05:11:10.776942+0000","last_fullsized":"2023-10-09T05:11:10.776942+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":6131,"log_dups_size":0,"ondisk_log_size":6131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:51.920206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334299136,"num_objects":5101,"num_object_clones":0,"num_object_copies":15303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6131,"num_write_kb":359424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":120,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.282919+0000","last_change":"2023-10-09T04:56:56.075307+0000","last_active":"2023-10-09T05:10:53.282919+0000","last_peered":"2023-10-09T05:10:53.282919+0000","last_clean":"2023-10-09T05:10:53.282919+0000","last_became_active":"2023-10-09T04:56:56.074831+0000","last_became_peered":"2023-10-09T04:56:56.074831+0000","last_unstale":"2023-10-09T05:10:53.282919+0000","last_undegraded":"2023-10-09T05:10:53.282919+0000","last_fullsized":"2023-10-09T05:10:53.282919+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:51:56.867759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":100,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.275072+0000","last_change":"2023-10-09T04:44:53.962437+0000","last_active":"2023-10-09T05:10:53.275072+0000","last_peered":"2023-10-09T05:10:53.275072+0000","last_clean":"2023-10-09T05:10:53.275072+0000","last_became_active":"2023-10-09T04:44:53.961810+0000","last_became_peered":"2023-10-09T04:44:53.961810+0000","last_unstale":"2023-10-09T05:10:53.275072+0000","last_undegraded":"2023-10-09T05:10:53.275072+0000","last_fullsized":"2023-10-09T05:10:53.275072+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:56.436933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"157'6180","reported_seq":11782,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-10-09T05:11:11.541416+0000","last_change":"2023-10-09T05:10:54.294336+0000","last_active":"2023-10-09T05:11:11.541416+0000","last_peered":"2023-10-09T05:11:11.541416+0000","last_clean":"2023-10-09T05:11:11.541416+0000","last_became_active":"2023-10-09T05:10:54.289152+0000","last_became_peered":"2023-10-09T05:10:54.289152+0000","last_unstale":"2023-10-09T05:11:11.541416+0000","last_undegraded":"2023-10-09T05:11:11.541416+0000","last_fullsized":"2023-10-09T05:11:11.541416+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":6180,"log_dups_size":0,"ondisk_log_size":6180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:43:23.604211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336330752,"num_objects":5132,"num_object_clones":0,"num_object_copies":15396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5132,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6180,"num_write_kb":361984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2768,"num_bytes_recovered":181403648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":149,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.284717+0000","last_change":"2023-10-09T05:09:57.898403+0000","last_active":"2023-10-09T05:10:53.284717+0000","last_peered":"2023-10-09T05:10:53.284717+0000","last_clean":"2023-10-09T05:10:53.284717+0000","last_became_active":"2023-10-09T05:09:57.866053+0000","last_became_peered":"2023-10-09T05:09:57.866053+0000","last_unstale":"2023-10-09T05:10:53.284717+0000","last_undegraded":"2023-10-09T05:10:53.284717+0000","last_fullsized":"2023-10-09T05:10:53.284717+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:31.760924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":154,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.287008+0000","last_change":"2023-10-09T05:09:57.868031+0000","last_active":"2023-10-09T05:10:53.287008+0000","last_peered":"2023-10-09T05:10:53.287008+0000","last_clean":"2023-10-09T05:10:53.287008+0000","last_became_active":"2023-10-09T05:09:57.866477+0000","last_became_peered":"2023-10-09T05:09:57.866477+0000","last_unstale":"2023-10-09T05:10:53.287008+0000","last_undegraded":"2023-10-09T05:10:53.287008+0000","last_fullsized":"2023-10-09T05:10:53.287008+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:27.102583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"157'6158","reported_seq":6184,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-10-09T05:11:09.388116+0000","last_change":"2023-10-09T05:09:03.830963+0000","last_active":"2023-10-09T05:11:09.388116+0000","last_peered":"2023-10-09T05:11:09.388116+0000","last_clean":"2023-10-09T05:11:09.388116+0000","last_became_active":"2023-10-09T05:09:03.830610+0000","last_became_peered":"2023-10-09T05:09:03.830610+0000","last_unstale":"2023-10-09T05:11:09.388116+0000","last_undegraded":"2023-10-09T05:11:09.388116+0000","last_fullsized":"2023-10-09T05:11:09.388116+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":6158,"log_dups_size":0,"ondisk_log_size":6158,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:48:37.895856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345505792,"num_objects":5272,"num_object_clones":0,"num_object_copies":15816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6158,"num_write_kb":365760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":103,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.290634+0000","last_change":"2023-10-09T04:44:52.787469+0000","last_active":"2023-10-09T05:10:53.290634+0000","last_peered":"2023-10-09T05:10:53.290634+0000","last_clean":"2023-10-09T05:10:53.290634+0000","last_became_active":"2023-10-09T04:44:52.787148+0000","last_became_peered":"2023-10-09T04:44:52.787148+0000","last_unstale":"2023-10-09T05:10:53.290634+0000","last_undegraded":"2023-10-09T05:10:53.290634+0000","last_fullsized":"2023-10-09T05:10:53.290634+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:12.469065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":147,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.259186+0000","last_change":"2023-10-09T05:09:57.855670+0000","last_active":"2023-10-09T05:10:53.259186+0000","last_peered":"2023-10-09T05:10:53.259186+0000","last_clean":"2023-10-09T05:10:53.259186+0000","last_became_active":"2023-10-09T05:09:57.855293+0000","last_became_peered":"2023-10-09T05:09:57.855293+0000","last_unstale":"2023-10-09T05:10:53.259186+0000","last_undegraded":"2023-10-09T05:10:53.259186+0000","last_fullsized":"2023-10-09T05:10:53.259186+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:19:56.843974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"157'6052","reported_seq":6421,"reported_epoch":157,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:11:09.417576+0000","last_change":"2023-10-09T05:09:59.220535+0000","last_active":"2023-10-09T05:11:09.417576+0000","last_peered":"2023-10-09T05:11:09.417576+0000","last_clean":"2023-10-09T05:09:56.825417+0000","last_became_active":"2023-10-09T05:09:59.206075+0000","last_became_peered":"2023-10-09T05:09:59.206075+0000","last_unstale":"2023-10-09T05:11:09.417576+0000","last_undegraded":"2023-10-09T05:09:58.879029+0000","last_fullsized":"2023-10-09T05:09:58.842133+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":6052,"log_dups_size":0,"ondisk_log_size":6052,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:15:01.681332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334626816,"num_objects":5106,"num_object_clones":0,"num_object_copies":15318,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2631,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5106,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6052,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":157,"num_bytes_recovered":10289152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2],"avail_no_missing":["5","2","6"],"object_location_counts":[{"shards":"2,5","objects":2475},{"shards":"2,5,6","objects":2158}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":118,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.275306+0000","last_change":"2023-10-09T05:02:09.653932+0000","last_active":"2023-10-09T05:10:53.275306+0000","last_peered":"2023-10-09T05:10:53.275306+0000","last_clean":"2023-10-09T05:10:53.275306+0000","last_became_active":"2023-10-09T05:02:09.652923+0000","last_became_peered":"2023-10-09T05:02:09.652923+0000","last_unstale":"2023-10-09T05:10:53.275306+0000","last_undegraded":"2023-10-09T05:10:53.275306+0000","last_fullsized":"2023-10-09T05:10:53.275306+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:44:11.928237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":130,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.277078+0000","last_change":"2023-10-09T05:09:57.852914+0000","last_active":"2023-10-09T05:10:53.277078+0000","last_peered":"2023-10-09T05:10:53.277078+0000","last_clean":"2023-10-09T05:10:53.277078+0000","last_became_active":"2023-10-09T05:09:57.852522+0000","last_became_peered":"2023-10-09T05:09:57.852522+0000","last_unstale":"2023-10-09T05:10:53.277078+0000","last_undegraded":"2023-10-09T05:10:53.277078+0000","last_fullsized":"2023-10-09T05:10:53.277078+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:24.848494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"157'5997","reported_seq":6024,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-10-09T05:11:08.257127+0000","last_change":"2023-10-09T05:09:03.828954+0000","last_active":"2023-10-09T05:11:08.257127+0000","last_peered":"2023-10-09T05:11:08.257127+0000","last_clean":"2023-10-09T05:11:08.257127+0000","last_became_active":"2023-10-09T05:09:03.828532+0000","last_became_peered":"2023-10-09T05:09:03.828532+0000","last_unstale":"2023-10-09T05:11:08.257127+0000","last_undegraded":"2023-10-09T05:11:08.257127+0000","last_fullsized":"2023-10-09T05:11:08.257127+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":5997,"log_dups_size":0,"ondisk_log_size":5997,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:23.822369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340459520,"num_objects":5195,"num_object_clones":0,"num_object_copies":15585,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5195,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5997,"num_write_kb":358144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"69'3","reported_seq":104,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-10-09T05:10:54.281817+0000","last_change":"2023-10-09T05:02:09.650579+0000","last_active":"2023-10-09T05:10:54.281817+0000","last_peered":"2023-10-09T05:10:54.281817+0000","last_clean":"2023-10-09T05:10:54.281817+0000","last_became_active":"2023-10-09T05:02:09.650160+0000","last_became_peered":"2023-10-09T05:02:09.650160+0000","last_unstale":"2023-10-09T05:10:54.281817+0000","last_undegraded":"2023-10-09T05:10:54.281817+0000","last_fullsized":"2023-10-09T05:10:54.281817+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:18.292670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":117,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.288213+0000","last_change":"2023-10-09T05:02:09.652449+0000","last_active":"2023-10-09T05:10:53.288213+0000","last_peered":"2023-10-09T05:10:53.288213+0000","last_clean":"2023-10-09T05:10:53.288213+0000","last_became_active":"2023-10-09T05:02:09.651994+0000","last_became_peered":"2023-10-09T05:02:09.651994+0000","last_unstale":"2023-10-09T05:10:53.288213+0000","last_undegraded":"2023-10-09T05:10:53.288213+0000","last_fullsized":"2023-10-09T05:10:53.288213+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:14.480513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"157'6147","reported_seq":6495,"reported_epoch":157,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-09T05:11:10.750836+0000","last_change":"2023-10-09T05:09:59.127832+0000","last_active":"2023-10-09T05:11:10.750836+0000","last_peered":"2023-10-09T05:11:10.750836+0000","last_clean":"2023-10-09T05:09:56.827891+0000","last_became_active":"2023-10-09T05:09:59.100161+0000","last_became_peered":"2023-10-09T05:09:59.100161+0000","last_unstale":"2023-10-09T05:11:10.750836+0000","last_undegraded":"2023-10-09T05:11:10.750836+0000","last_fullsized":"2023-10-09T05:11:10.750836+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":6147,"log_dups_size":0,"ondisk_log_size":6147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:00.212739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335478784,"num_objects":5119,"num_object_clones":0,"num_object_copies":15357,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2721,"num_objects_unfound":0,"num_objects_dirty":5119,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6147,"num_write_kb":360512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":147,"num_bytes_recovered":9633792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[0,6,2],"avail_no_missing":["0","2","6"],"object_location_counts":[{"shards":"0,2,6","objects":2398}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":138,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.259360+0000","last_change":"2023-10-09T05:09:57.849784+0000","last_active":"2023-10-09T05:10:53.259360+0000","last_peered":"2023-10-09T05:10:53.259360+0000","last_clean":"2023-10-09T05:10:53.259360+0000","last_became_active":"2023-10-09T05:09:57.849216+0000","last_became_peered":"2023-10-09T05:09:57.849216+0000","last_unstale":"2023-10-09T05:10:53.259360+0000","last_undegraded":"2023-10-09T05:10:53.259360+0000","last_fullsized":"2023-10-09T05:10:53.259360+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:29.764343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":154,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.288105+0000","last_change":"2023-10-09T05:09:57.868658+0000","last_active":"2023-10-09T05:10:53.288105+0000","last_peered":"2023-10-09T05:10:53.288105+0000","last_clean":"2023-10-09T05:10:53.288105+0000","last_became_active":"2023-10-09T05:09:57.868221+0000","last_became_peered":"2023-10-09T05:09:57.868221+0000","last_unstale":"2023-10-09T05:10:53.288105+0000","last_undegraded":"2023-10-09T05:10:53.288105+0000","last_fullsized":"2023-10-09T05:10:53.288105+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:12:15.536917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"157'5862","reported_seq":6165,"reported_epoch":157,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:11:09.343764+0000","last_change":"2023-10-09T05:09:59.082468+0000","last_active":"2023-10-09T05:11:09.343764+0000","last_peered":"2023-10-09T05:11:09.343764+0000","last_clean":"2023-10-09T05:09:56.807871+0000","last_became_active":"2023-10-09T05:09:59.074831+0000","last_became_peered":"2023-10-09T05:09:59.074831+0000","last_unstale":"2023-10-09T05:11:09.343764+0000","last_undegraded":"2023-10-09T05:09:58.929479+0000","last_fullsized":"2023-10-09T05:09:58.845580+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":5862,"log_dups_size":0,"ondisk_log_size":5862,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:03.517929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331218944,"num_objects":5054,"num_object_clones":0,"num_object_copies":15162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2546,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5054,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5862,"num_write_kb":349312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":122,"num_bytes_recovered":7995392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3],"avail_no_missing":["7","3","5"],"object_location_counts":[{"shards":"3,5,7","objects":2142},{"shards":"3,7","objects":2508}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":118,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.287004+0000","last_change":"2023-10-09T05:01:02.663309+0000","last_active":"2023-10-09T05:10:53.287004+0000","last_peered":"2023-10-09T05:10:53.287004+0000","last_clean":"2023-10-09T05:10:53.287004+0000","last_became_active":"2023-10-09T05:01:02.663038+0000","last_became_peered":"2023-10-09T05:01:02.663038+0000","last_unstale":"2023-10-09T05:10:53.287004+0000","last_undegraded":"2023-10-09T05:10:53.287004+0000","last_fullsized":"2023-10-09T05:10:53.287004+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:29:36.689304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":137,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.288682+0000","last_change":"2023-10-09T05:09:57.868378+0000","last_active":"2023-10-09T05:10:53.288682+0000","last_peered":"2023-10-09T05:10:53.288682+0000","last_clean":"2023-10-09T05:10:53.288682+0000","last_became_active":"2023-10-09T05:09:57.867507+0000","last_became_peered":"2023-10-09T05:09:57.867507+0000","last_unstale":"2023-10-09T05:10:53.288682+0000","last_undegraded":"2023-10-09T05:10:53.288682+0000","last_fullsized":"2023-10-09T05:10:53.288682+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:14:35.199631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"157'5879","reported_seq":6201,"reported_epoch":157,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:11:08.232679+0000","last_change":"2023-10-09T05:09:59.104712+0000","last_active":"2023-10-09T05:11:08.232679+0000","last_peered":"2023-10-09T05:11:08.232679+0000","last_clean":"2023-10-09T05:09:56.749891+0000","last_became_active":"2023-10-09T05:09:59.097963+0000","last_became_peered":"2023-10-09T05:09:59.097963+0000","last_unstale":"2023-10-09T05:11:08.232679+0000","last_undegraded":"2023-10-09T05:09:58.854922+0000","last_fullsized":"2023-10-09T05:09:58.832139+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":5879,"log_dups_size":0,"ondisk_log_size":5879,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:40.209073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336658432,"num_objects":5137,"num_object_clones":0,"num_object_copies":15411,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2507,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5137,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5879,"num_write_kb":352512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":134,"num_bytes_recovered":8781824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,2],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3","objects":2630},{"shards":"2,3,7","objects":2136}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":126,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.274280+0000","last_change":"2023-10-09T05:09:57.852151+0000","last_active":"2023-10-09T05:10:53.274280+0000","last_peered":"2023-10-09T05:10:53.274280+0000","last_clean":"2023-10-09T05:10:53.274280+0000","last_became_active":"2023-10-09T05:09:57.851499+0000","last_became_peered":"2023-10-09T05:09:57.851499+0000","last_unstale":"2023-10-09T05:10:53.274280+0000","last_undegraded":"2023-10-09T05:10:53.274280+0000","last_fullsized":"2023-10-09T05:10:53.274280+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:25:21.153305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":16,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.284599+0000","last_change":"2023-10-09T05:09:57.856764+0000","last_active":"2023-10-09T05:10:53.284599+0000","last_peered":"2023-10-09T05:10:53.284599+0000","last_clean":"2023-10-09T05:10:53.284599+0000","last_became_active":"2023-10-09T05:09:57.856461+0000","last_became_peered":"2023-10-09T05:09:57.856461+0000","last_unstale":"2023-10-09T05:10:53.284599+0000","last_undegraded":"2023-10-09T05:10:53.284599+0000","last_fullsized":"2023-10-09T05:10:53.284599+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:06.848082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"157'6170","reported_seq":6489,"reported_epoch":157,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:11:09.370992+0000","last_change":"2023-10-09T05:09:59.115413+0000","last_active":"2023-10-09T05:11:09.370992+0000","last_peered":"2023-10-09T05:11:09.370992+0000","last_clean":"2023-10-09T05:09:56.775881+0000","last_became_active":"2023-10-09T05:09:59.102861+0000","last_became_peered":"2023-10-09T05:09:59.102861+0000","last_unstale":"2023-10-09T05:11:09.370992+0000","last_undegraded":"2023-10-09T05:09:58.877226+0000","last_fullsized":"2023-10-09T05:09:58.842223+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":6170,"log_dups_size":0,"ondisk_log_size":6170,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:24:23.156903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348651520,"num_objects":5320,"num_object_clones":0,"num_object_copies":15960,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2696,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5320,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6170,"num_write_kb":367680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":131,"num_bytes_recovered":8585216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,3],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":2271},{"shards":"3,5","objects":2624}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.275123+0000","last_change":"2023-10-09T05:09:57.849066+0000","last_active":"2023-10-09T05:10:53.275123+0000","last_peered":"2023-10-09T05:10:53.275123+0000","last_clean":"2023-10-09T05:10:53.275123+0000","last_became_active":"2023-10-09T05:09:57.848717+0000","last_became_peered":"2023-10-09T05:09:57.848717+0000","last_unstale":"2023-10-09T05:10:53.275123+0000","last_undegraded":"2023-10-09T05:10:53.275123+0000","last_fullsized":"2023-10-09T05:10:53.275123+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:31.267804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":135,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.275806+0000","last_change":"2023-10-09T05:02:09.649559+0000","last_active":"2023-10-09T05:10:53.275806+0000","last_peered":"2023-10-09T05:10:53.275806+0000","last_clean":"2023-10-09T05:10:53.275806+0000","last_became_active":"2023-10-09T05:02:09.649195+0000","last_became_peered":"2023-10-09T05:02:09.649195+0000","last_unstale":"2023-10-09T05:10:53.275806+0000","last_undegraded":"2023-10-09T05:10:53.275806+0000","last_fullsized":"2023-10-09T05:10:53.275806+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:56.864170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":16,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.270853+0000","last_change":"2023-10-09T05:09:57.865132+0000","last_active":"2023-10-09T05:10:53.270853+0000","last_peered":"2023-10-09T05:10:53.270853+0000","last_clean":"2023-10-09T05:10:53.270853+0000","last_became_active":"2023-10-09T05:09:57.864773+0000","last_became_peered":"2023-10-09T05:09:57.864773+0000","last_unstale":"2023-10-09T05:10:53.270853+0000","last_undegraded":"2023-10-09T05:10:53.270853+0000","last_fullsized":"2023-10-09T05:10:53.270853+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:04.194629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"157'6052","reported_seq":6079,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-10-09T05:11:10.660776+0000","last_change":"2023-10-09T05:09:03.830161+0000","last_active":"2023-10-09T05:11:10.660776+0000","last_peered":"2023-10-09T05:11:10.660776+0000","last_clean":"2023-10-09T05:11:10.660776+0000","last_became_active":"2023-10-09T05:09:03.829750+0000","last_became_peered":"2023-10-09T05:09:03.829750+0000","last_unstale":"2023-10-09T05:11:10.660776+0000","last_undegraded":"2023-10-09T05:11:10.660776+0000","last_fullsized":"2023-10-09T05:11:10.660776+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":6052,"log_dups_size":0,"ondisk_log_size":6052,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:28.876874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331087872,"num_objects":5052,"num_object_clones":0,"num_object_copies":15156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5052,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6052,"num_write_kb":355328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":142,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.278418+0000","last_change":"2023-10-09T05:09:57.871040+0000","last_active":"2023-10-09T05:10:53.278418+0000","last_peered":"2023-10-09T05:10:53.278418+0000","last_clean":"2023-10-09T05:10:53.278418+0000","last_became_active":"2023-10-09T05:09:57.866399+0000","last_became_peered":"2023-10-09T05:09:57.866399+0000","last_unstale":"2023-10-09T05:10:53.278418+0000","last_undegraded":"2023-10-09T05:10:53.278418+0000","last_fullsized":"2023-10-09T05:10:53.278418+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:42.545057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":118,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.287501+0000","last_change":"2023-10-09T04:56:56.087320+0000","last_active":"2023-10-09T05:10:53.287501+0000","last_peered":"2023-10-09T05:10:53.287501+0000","last_clean":"2023-10-09T05:10:53.287501+0000","last_became_active":"2023-10-09T04:56:56.086552+0000","last_became_peered":"2023-10-09T04:56:56.086552+0000","last_unstale":"2023-10-09T05:10:53.287501+0000","last_undegraded":"2023-10-09T05:10:53.287501+0000","last_fullsized":"2023-10-09T05:10:53.287501+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:14.205288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":16,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.278250+0000","last_change":"2023-10-09T05:09:57.843174+0000","last_active":"2023-10-09T05:10:53.278250+0000","last_peered":"2023-10-09T05:10:53.278250+0000","last_clean":"2023-10-09T05:10:53.278250+0000","last_became_active":"2023-10-09T05:09:57.842818+0000","last_became_peered":"2023-10-09T05:09:57.842818+0000","last_unstale":"2023-10-09T05:10:53.278250+0000","last_undegraded":"2023-10-09T05:10:53.278250+0000","last_fullsized":"2023-10-09T05:10:53.278250+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:10.285164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"157'6153","reported_seq":6508,"reported_epoch":157,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:11:09.377030+0000","last_change":"2023-10-09T05:09:59.108285+0000","last_active":"2023-10-09T05:11:09.377030+0000","last_peered":"2023-10-09T05:11:09.377030+0000","last_clean":"2023-10-09T05:09:56.806054+0000","last_became_active":"2023-10-09T05:09:59.097985+0000","last_became_peered":"2023-10-09T05:09:59.097985+0000","last_unstale":"2023-10-09T05:11:09.377030+0000","last_undegraded":"2023-10-09T05:09:58.873107+0000","last_fullsized":"2023-10-09T05:09:58.841848+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":6153,"log_dups_size":0,"ondisk_log_size":6153,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:44:00.189725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343998464,"num_objects":5249,"num_object_clones":0,"num_object_copies":15747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2719,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5249,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6153,"num_write_kb":364864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":149,"num_bytes_recovered":9764864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,7],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":2267},{"shards":"5,7","objects":2530}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":136,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.259910+0000","last_change":"2023-10-09T05:02:09.645835+0000","last_active":"2023-10-09T05:10:53.259910+0000","last_peered":"2023-10-09T05:10:53.259910+0000","last_clean":"2023-10-09T05:10:53.259910+0000","last_became_active":"2023-10-09T05:02:09.645491+0000","last_became_peered":"2023-10-09T05:02:09.645491+0000","last_unstale":"2023-10-09T05:10:53.259910+0000","last_undegraded":"2023-10-09T05:10:53.259910+0000","last_fullsized":"2023-10-09T05:10:53.259910+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:06:20.823201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":16,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.264335+0000","last_change":"2023-10-09T05:09:57.854064+0000","last_active":"2023-10-09T05:10:53.264335+0000","last_peered":"2023-10-09T05:10:53.264335+0000","last_clean":"2023-10-09T05:10:53.264335+0000","last_became_active":"2023-10-09T05:09:57.847953+0000","last_became_peered":"2023-10-09T05:09:57.847953+0000","last_unstale":"2023-10-09T05:10:53.264335+0000","last_undegraded":"2023-10-09T05:10:53.264335+0000","last_fullsized":"2023-10-09T05:10:53.264335+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:45.914976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":16,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.284528+0000","last_change":"2023-10-09T05:09:57.865256+0000","last_active":"2023-10-09T05:10:53.284528+0000","last_peered":"2023-10-09T05:10:53.284528+0000","last_clean":"2023-10-09T05:10:53.284528+0000","last_became_active":"2023-10-09T05:09:57.864756+0000","last_became_peered":"2023-10-09T05:09:57.864756+0000","last_unstale":"2023-10-09T05:10:53.284528+0000","last_undegraded":"2023-10-09T05:10:53.284528+0000","last_fullsized":"2023-10-09T05:10:53.284528+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:14.071437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"157'5890","reported_seq":5917,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-10-09T05:11:08.244018+0000","last_change":"2023-10-09T05:09:03.827371+0000","last_active":"2023-10-09T05:11:08.244018+0000","last_peered":"2023-10-09T05:11:08.244018+0000","last_clean":"2023-10-09T05:11:08.244018+0000","last_became_active":"2023-10-09T05:09:03.827089+0000","last_became_peered":"2023-10-09T05:09:03.827089+0000","last_unstale":"2023-10-09T05:11:08.244018+0000","last_undegraded":"2023-10-09T05:11:08.244018+0000","last_fullsized":"2023-10-09T05:11:08.244018+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":5890,"log_dups_size":0,"ondisk_log_size":5890,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:04:02.318218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337903616,"num_objects":5156,"num_object_clones":0,"num_object_copies":15468,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5156,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5890,"num_write_kb":353472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":122,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.275461+0000","last_change":"2023-10-09T05:01:02.685038+0000","last_active":"2023-10-09T05:10:53.275461+0000","last_peered":"2023-10-09T05:10:53.275461+0000","last_clean":"2023-10-09T05:10:53.275461+0000","last_became_active":"2023-10-09T05:01:02.683622+0000","last_became_peered":"2023-10-09T05:01:02.683622+0000","last_unstale":"2023-10-09T05:10:53.275461+0000","last_undegraded":"2023-10-09T05:10:53.275461+0000","last_fullsized":"2023-10-09T05:10:53.275461+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:35.435592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":495,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.282876+0000","last_change":"2023-10-09T04:56:56.075762+0000","last_active":"2023-10-09T05:10:53.282876+0000","last_peered":"2023-10-09T05:10:53.282876+0000","last_clean":"2023-10-09T05:10:53.282876+0000","last_became_active":"2023-10-09T04:56:56.075263+0000","last_became_peered":"2023-10-09T04:56:56.075263+0000","last_unstale":"2023-10-09T05:10:53.282876+0000","last_undegraded":"2023-10-09T05:10:53.282876+0000","last_fullsized":"2023-10-09T05:10:53.282876+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_clean_scrub_stamp":"2023-10-09T04:36:44.261013+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:02.769164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":102,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.276907+0000","last_change":"2023-10-09T05:02:09.654237+0000","last_active":"2023-10-09T05:10:53.276907+0000","last_peered":"2023-10-09T05:10:53.276907+0000","last_clean":"2023-10-09T05:10:53.276907+0000","last_became_active":"2023-10-09T05:02:09.653909+0000","last_became_peered":"2023-10-09T05:02:09.653909+0000","last_unstale":"2023-10-09T05:10:53.276907+0000","last_undegraded":"2023-10-09T05:10:53.276907+0000","last_fullsized":"2023-10-09T05:10:53.276907+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:02.002366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":100,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.259784+0000","last_change":"2023-10-09T04:44:53.959705+0000","last_active":"2023-10-09T05:10:53.259784+0000","last_peered":"2023-10-09T05:10:53.259784+0000","last_clean":"2023-10-09T05:10:53.259784+0000","last_became_active":"2023-10-09T04:44:53.959092+0000","last_became_peered":"2023-10-09T04:44:53.959092+0000","last_unstale":"2023-10-09T05:10:53.259784+0000","last_undegraded":"2023-10-09T05:10:53.259784+0000","last_fullsized":"2023-10-09T05:10:53.259784+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:31:29.189897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"157'6156","reported_seq":6183,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-10-09T05:11:10.789514+0000","last_change":"2023-10-09T05:09:03.826421+0000","last_active":"2023-10-09T05:11:10.789514+0000","last_peered":"2023-10-09T05:11:10.789514+0000","last_clean":"2023-10-09T05:11:10.789514+0000","last_became_active":"2023-10-09T05:09:03.826107+0000","last_became_peered":"2023-10-09T05:09:03.826107+0000","last_unstale":"2023-10-09T05:11:10.789514+0000","last_undegraded":"2023-10-09T05:11:10.789514+0000","last_fullsized":"2023-10-09T05:11:10.789514+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":6156,"log_dups_size":0,"ondisk_log_size":6156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:53:42.881604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334233600,"num_objects":5100,"num_object_clones":0,"num_object_copies":15300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6156,"num_write_kb":360192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":159,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.282969+0000","last_change":"2023-10-09T05:09:57.855069+0000","last_active":"2023-10-09T05:10:53.282969+0000","last_peered":"2023-10-09T05:10:53.282969+0000","last_clean":"2023-10-09T05:10:53.282969+0000","last_became_active":"2023-10-09T05:09:57.854693+0000","last_became_peered":"2023-10-09T05:09:57.854693+0000","last_unstale":"2023-10-09T05:10:53.282969+0000","last_undegraded":"2023-10-09T05:10:53.282969+0000","last_fullsized":"2023-10-09T05:10:53.282969+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:21:27.305940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":111,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.267034+0000","last_change":"2023-10-09T05:02:09.648781+0000","last_active":"2023-10-09T05:10:53.267034+0000","last_peered":"2023-10-09T05:10:53.267034+0000","last_clean":"2023-10-09T05:10:53.267034+0000","last_became_active":"2023-10-09T05:02:09.648078+0000","last_became_peered":"2023-10-09T05:02:09.648078+0000","last_unstale":"2023-10-09T05:10:53.267034+0000","last_undegraded":"2023-10-09T05:10:53.267034+0000","last_fullsized":"2023-10-09T05:10:53.267034+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:35.287091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":145,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.267261+0000","last_change":"2023-10-09T05:09:57.847731+0000","last_active":"2023-10-09T05:10:53.267261+0000","last_peered":"2023-10-09T05:10:53.267261+0000","last_clean":"2023-10-09T05:10:53.267261+0000","last_became_active":"2023-10-09T05:09:57.847359+0000","last_became_peered":"2023-10-09T05:09:57.847359+0000","last_unstale":"2023-10-09T05:10:53.267261+0000","last_undegraded":"2023-10-09T05:10:53.267261+0000","last_fullsized":"2023-10-09T05:10:53.267261+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:43:30.260279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"157'6141","reported_seq":6463,"reported_epoch":157,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:11:09.360914+0000","last_change":"2023-10-09T05:09:59.091678+0000","last_active":"2023-10-09T05:11:09.360914+0000","last_peered":"2023-10-09T05:11:09.360914+0000","last_clean":"2023-10-09T05:09:56.834673+0000","last_became_active":"2023-10-09T05:09:59.086420+0000","last_became_peered":"2023-10-09T05:09:59.086420+0000","last_unstale":"2023-10-09T05:11:09.360914+0000","last_undegraded":"2023-10-09T05:09:58.874557+0000","last_fullsized":"2023-10-09T05:09:58.845617+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":6141,"log_dups_size":0,"ondisk_log_size":6141,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:13.864729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344915968,"num_objects":5263,"num_object_clones":0,"num_object_copies":15789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2673,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5263,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6141,"num_write_kb":364928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":133,"num_bytes_recovered":8716288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4],"avail_no_missing":["7","0","4"],"object_location_counts":[{"shards":"0,4,7","objects":2234},{"shards":"4,7","objects":2590}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":122,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.290718+0000","last_change":"2023-10-09T04:56:56.073473+0000","last_active":"2023-10-09T05:10:53.290718+0000","last_peered":"2023-10-09T05:10:53.290718+0000","last_clean":"2023-10-09T05:10:53.290718+0000","last_became_active":"2023-10-09T04:56:56.073146+0000","last_became_peered":"2023-10-09T04:56:56.073146+0000","last_unstale":"2023-10-09T05:10:53.290718+0000","last_undegraded":"2023-10-09T05:10:53.290718+0000","last_fullsized":"2023-10-09T05:10:53.290718+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:55:25.132822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":16,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.278339+0000","last_change":"2023-10-09T05:09:57.864289+0000","last_active":"2023-10-09T05:10:53.278339+0000","last_peered":"2023-10-09T05:10:53.278339+0000","last_clean":"2023-10-09T05:10:53.278339+0000","last_became_active":"2023-10-09T05:09:57.863963+0000","last_became_peered":"2023-10-09T05:09:57.863963+0000","last_unstale":"2023-10-09T05:10:53.278339+0000","last_undegraded":"2023-10-09T05:10:53.278339+0000","last_fullsized":"2023-10-09T05:10:53.278339+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:45:02.500193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":97,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.277419+0000","last_change":"2023-10-09T04:56:56.071763+0000","last_active":"2023-10-09T05:10:53.277419+0000","last_peered":"2023-10-09T05:10:53.277419+0000","last_clean":"2023-10-09T05:10:53.277419+0000","last_became_active":"2023-10-09T04:56:56.071429+0000","last_became_peered":"2023-10-09T04:56:56.071429+0000","last_unstale":"2023-10-09T05:10:53.277419+0000","last_undegraded":"2023-10-09T05:10:53.277419+0000","last_fullsized":"2023-10-09T05:10:53.277419+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:53:02.051460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"157'6051","reported_seq":7738,"reported_epoch":157,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T05:11:09.382678+0000","last_change":"2023-10-09T05:10:53.176239+0000","last_active":"2023-10-09T05:11:09.382678+0000","last_peered":"2023-10-09T05:11:09.382678+0000","last_clean":"2023-10-09T05:09:56.818963+0000","last_became_active":"2023-10-09T05:09:59.069993+0000","last_became_peered":"2023-10-09T05:09:59.069993+0000","last_unstale":"2023-10-09T05:11:09.382678+0000","last_undegraded":"2023-10-09T05:11:09.382678+0000","last_fullsized":"2023-10-09T05:09:58.845731+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":6051,"log_dups_size":0,"ondisk_log_size":6051,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:12.318397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339476504,"num_objects":5181,"num_object_clones":0,"num_object_copies":15543,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1954,"num_objects_unfound":0,"num_objects_dirty":5181,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6051,"num_write_kb":359361,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":815,"num_bytes_recovered":53411840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,5],"avail_no_missing":["7","4","5"],"object_location_counts":[{"shards":"4,5,7","objects":1954},{"shards":"5,7","objects":3227}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":103,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.277285+0000","last_change":"2023-10-09T05:02:09.651149+0000","last_active":"2023-10-09T05:10:53.277285+0000","last_peered":"2023-10-09T05:10:53.277285+0000","last_clean":"2023-10-09T05:10:53.277285+0000","last_became_active":"2023-10-09T05:02:09.650151+0000","last_became_peered":"2023-10-09T05:02:09.650151+0000","last_unstale":"2023-10-09T05:10:53.277285+0000","last_undegraded":"2023-10-09T05:10:53.277285+0000","last_fullsized":"2023-10-09T05:10:53.277285+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:49.100209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":16,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.274793+0000","last_change":"2023-10-09T05:09:57.858617+0000","last_active":"2023-10-09T05:10:53.274793+0000","last_peered":"2023-10-09T05:10:53.274793+0000","last_clean":"2023-10-09T05:10:53.274793+0000","last_became_active":"2023-10-09T05:09:57.858128+0000","last_became_peered":"2023-10-09T05:09:57.858128+0000","last_unstale":"2023-10-09T05:10:53.274793+0000","last_undegraded":"2023-10-09T05:10:53.274793+0000","last_fullsized":"2023-10-09T05:10:53.274793+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:02.502985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":99,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.274353+0000","last_change":"2023-10-09T05:02:09.650750+0000","last_active":"2023-10-09T05:10:53.274353+0000","last_peered":"2023-10-09T05:10:53.274353+0000","last_clean":"2023-10-09T05:10:53.274353+0000","last_became_active":"2023-10-09T05:02:09.650351+0000","last_became_peered":"2023-10-09T05:02:09.650351+0000","last_unstale":"2023-10-09T05:10:53.274353+0000","last_undegraded":"2023-10-09T05:10:53.274353+0000","last_fullsized":"2023-10-09T05:10:53.274353+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:14.826132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"157'6159","reported_seq":6186,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-10-09T05:11:08.237594+0000","last_change":"2023-10-09T05:09:04.054690+0000","last_active":"2023-10-09T05:11:08.237594+0000","last_peered":"2023-10-09T05:11:08.237594+0000","last_clean":"2023-10-09T05:11:08.237594+0000","last_became_active":"2023-10-09T05:09:04.054340+0000","last_became_peered":"2023-10-09T05:09:04.054340+0000","last_unstale":"2023-10-09T05:11:08.237594+0000","last_undegraded":"2023-10-09T05:11:08.237594+0000","last_fullsized":"2023-10-09T05:11:08.237594+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":6159,"log_dups_size":0,"ondisk_log_size":6159,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:19:34.276593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348585984,"num_objects":5319,"num_object_clones":0,"num_object_copies":15957,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5319,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6159,"num_write_kb":367296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":122,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.259876+0000","last_change":"2023-10-09T05:01:02.660775+0000","last_active":"2023-10-09T05:10:53.259876+0000","last_peered":"2023-10-09T05:10:53.259876+0000","last_clean":"2023-10-09T05:10:53.259876+0000","last_became_active":"2023-10-09T05:01:02.660420+0000","last_became_peered":"2023-10-09T05:01:02.660420+0000","last_unstale":"2023-10-09T05:10:53.259876+0000","last_undegraded":"2023-10-09T05:10:53.259876+0000","last_fullsized":"2023-10-09T05:10:53.259876+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:54.031869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":142,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.267315+0000","last_change":"2023-10-09T05:09:57.878002+0000","last_active":"2023-10-09T05:10:53.267315+0000","last_peered":"2023-10-09T05:10:53.267315+0000","last_clean":"2023-10-09T05:10:53.267315+0000","last_became_active":"2023-10-09T05:09:57.852792+0000","last_became_peered":"2023-10-09T05:09:57.852792+0000","last_unstale":"2023-10-09T05:10:53.267315+0000","last_undegraded":"2023-10-09T05:10:53.267315+0000","last_fullsized":"2023-10-09T05:10:53.267315+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:18.763275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":100,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.286866+0000","last_change":"2023-10-09T04:44:53.792412+0000","last_active":"2023-10-09T05:10:53.286866+0000","last_peered":"2023-10-09T05:10:53.286866+0000","last_clean":"2023-10-09T05:10:53.286866+0000","last_became_active":"2023-10-09T04:44:53.792009+0000","last_became_peered":"2023-10-09T04:44:53.792009+0000","last_unstale":"2023-10-09T05:10:53.286866+0000","last_undegraded":"2023-10-09T05:10:53.286866+0000","last_fullsized":"2023-10-09T05:10:53.286866+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:39.423339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'2","reported_seq":101,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.277165+0000","last_change":"2023-10-09T04:56:56.079768+0000","last_active":"2023-10-09T05:10:53.277165+0000","last_peered":"2023-10-09T05:10:53.277165+0000","last_clean":"2023-10-09T05:10:53.277165+0000","last_became_active":"2023-10-09T04:56:56.077721+0000","last_became_peered":"2023-10-09T04:56:56.077721+0000","last_unstale":"2023-10-09T05:10:53.277165+0000","last_undegraded":"2023-10-09T05:10:53.277165+0000","last_fullsized":"2023-10-09T05:10:53.277165+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:36.384357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":157,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-10-09T05:10:54.306827+0000","last_change":"2023-10-09T05:09:57.842586+0000","last_active":"2023-10-09T05:10:54.306827+0000","last_peered":"2023-10-09T05:10:54.306827+0000","last_clean":"2023-10-09T05:10:54.306827+0000","last_became_active":"2023-10-09T05:09:57.842214+0000","last_became_peered":"2023-10-09T05:09:57.842214+0000","last_unstale":"2023-10-09T05:10:54.306827+0000","last_undegraded":"2023-10-09T05:10:54.306827+0000","last_fullsized":"2023-10-09T05:10:54.306827+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:26.074875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":149,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.275388+0000","last_change":"2023-10-09T05:09:57.901164+0000","last_active":"2023-10-09T05:10:53.275388+0000","last_peered":"2023-10-09T05:10:53.275388+0000","last_clean":"2023-10-09T05:10:53.275388+0000","last_became_active":"2023-10-09T05:09:57.867734+0000","last_became_peered":"2023-10-09T05:09:57.867734+0000","last_unstale":"2023-10-09T05:10:53.275388+0000","last_undegraded":"2023-10-09T05:10:53.275388+0000","last_fullsized":"2023-10-09T05:10:53.275388+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:46:30.541419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":154,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.287817+0000","last_change":"2023-10-09T05:09:57.867865+0000","last_active":"2023-10-09T05:10:53.287817+0000","last_peered":"2023-10-09T05:10:53.287817+0000","last_clean":"2023-10-09T05:10:53.287817+0000","last_became_active":"2023-10-09T05:09:57.866723+0000","last_became_peered":"2023-10-09T05:09:57.866723+0000","last_unstale":"2023-10-09T05:10:53.287817+0000","last_undegraded":"2023-10-09T05:10:53.287817+0000","last_fullsized":"2023-10-09T05:10:53.287817+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:58.946358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":104,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.275894+0000","last_change":"2023-10-09T04:44:52.793709+0000","last_active":"2023-10-09T05:10:53.275894+0000","last_peered":"2023-10-09T05:10:53.275894+0000","last_clean":"2023-10-09T05:10:53.275894+0000","last_became_active":"2023-10-09T04:44:52.792973+0000","last_became_peered":"2023-10-09T04:44:52.792973+0000","last_unstale":"2023-10-09T05:10:53.275894+0000","last_undegraded":"2023-10-09T05:10:53.275894+0000","last_fullsized":"2023-10-09T05:10:53.275894+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:12.091089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":134,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.287349+0000","last_change":"2023-10-09T05:02:09.649748+0000","last_active":"2023-10-09T05:10:53.287349+0000","last_peered":"2023-10-09T05:10:53.287349+0000","last_clean":"2023-10-09T05:10:53.287349+0000","last_became_active":"2023-10-09T05:02:09.649444+0000","last_became_peered":"2023-10-09T05:02:09.649444+0000","last_unstale":"2023-10-09T05:10:53.287349+0000","last_undegraded":"2023-10-09T05:10:53.287349+0000","last_fullsized":"2023-10-09T05:10:53.287349+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:07.436820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'2","reported_seq":120,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.273994+0000","last_change":"2023-10-09T05:02:09.647382+0000","last_active":"2023-10-09T05:10:53.273994+0000","last_peered":"2023-10-09T05:10:53.273994+0000","last_clean":"2023-10-09T05:10:53.273994+0000","last_became_active":"2023-10-09T05:02:09.646803+0000","last_became_peered":"2023-10-09T05:02:09.646803+0000","last_unstale":"2023-10-09T05:10:53.273994+0000","last_undegraded":"2023-10-09T05:10:53.273994+0000","last_fullsized":"2023-10-09T05:10:53.273994+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:23:00.161736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":133,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.277066+0000","last_change":"2023-10-09T05:02:09.649674+0000","last_active":"2023-10-09T05:10:53.277066+0000","last_peered":"2023-10-09T05:10:53.277066+0000","last_clean":"2023-10-09T05:10:53.277066+0000","last_became_active":"2023-10-09T05:02:09.649041+0000","last_became_peered":"2023-10-09T05:02:09.649041+0000","last_unstale":"2023-10-09T05:10:53.277066+0000","last_undegraded":"2023-10-09T05:10:53.277066+0000","last_fullsized":"2023-10-09T05:10:53.277066+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:18.524071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'5","reported_seq":121,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.267504+0000","last_change":"2023-10-09T05:01:02.672596+0000","last_active":"2023-10-09T05:10:53.267504+0000","last_peered":"2023-10-09T05:10:53.267504+0000","last_clean":"2023-10-09T05:10:53.267504+0000","last_became_active":"2023-10-09T05:01:02.672128+0000","last_became_peered":"2023-10-09T05:01:02.672128+0000","last_unstale":"2023-10-09T05:10:53.267504+0000","last_undegraded":"2023-10-09T05:10:53.267504+0000","last_fullsized":"2023-10-09T05:10:53.267504+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:17:05.887502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":137,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T05:10:53.286849+0000","last_change":"2023-10-09T05:09:57.866732+0000","last_active":"2023-10-09T05:10:53.286849+0000","last_peered":"2023-10-09T05:10:53.286849+0000","last_clean":"2023-10-09T05:10:53.286849+0000","last_became_active":"2023-10-09T05:09:57.866117+0000","last_became_peered":"2023-10-09T05:09:57.866117+0000","last_unstale":"2023-10-09T05:10:53.286849+0000","last_undegraded":"2023-10-09T05:10:53.286849+0000","last_fullsized":"2023-10-09T05:10:53.286849+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:49:14.815342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":38,"num_read_kb":40,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":5423431704,"num_objects":82756,"num_object_clones":0,"num_object_copies":248268,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15772,"num_objects_misplaced":4675,"num_objects_unfound":0,"num_objects_dirty":82756,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":97178,"num_write_kb":5757825,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4556,"num_bytes_recovered":298582016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17039044608,"data_stored":17039032392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97178,"ondisk_log_size":97178,"up":48,"acting":41,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234347,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3744676,"kb_used_data":3159004,"kb_used_omap":0,"kb_used_meta":585664,"kb_avail":90000476,"statfs":{"total":95995035648,"available":92160487424,"internally_reserved":0,"allocated":3234820096,"data_stored":3234182466,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599719936},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:10:26 2023","interfaces":[{"interface":"back","average":{"1min":1.663,"5min":1.282,"15min":1.4019999999999999},"min":{"1min":0.30199999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":19.995999999999999,"5min":19.995999999999999,"15min":19.995999999999999},"last":0.309},{"interface":"front","average":{"1min":2.9079999999999999,"5min":1.5740000000000001,"15min":1.704},"min":{"1min":0.317,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":20.032,"5min":20.032,"15min":20.032},"last":0.70199999999999996}]},{"osd":1,"last update":"Mon Oct 9 05:10:09 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":1.1519999999999999,"15min":1.9550000000000001},"min":{"1min":0.38400000000000001,"5min":0.307,"15min":0.28699999999999998},"max":{"1min":2.1579999999999999,"5min":6.7210000000000001,"15min":135.322},"last":0.35299999999999998},{"interface":"front","average":{"1min":2.1120000000000001,"5min":1.393,"15min":2.1739999999999999},"min":{"1min":0.45200000000000001,"5min":0.35599999999999998,"15min":0.30499999999999999},"max":{"1min":9.8870000000000005,"5min":9.8870000000000005,"15min":135.09},"last":0.745}]},{"osd":2,"last update":"Mon Oct 9 05:10:26 2023","interfaces":[{"interface":"back","average":{"1min":2.044,"5min":1.3380000000000001,"15min":1.3819999999999999},"min":{"1min":0.42299999999999999,"5min":0.315,"15min":0.315},"max":{"1min":9.9299999999999997,"5min":11.391,"15min":11.391},"last":0.77700000000000002},{"interface":"front","average":{"1min":1.3400000000000001,"5min":1.3660000000000001,"15min":1.2569999999999999},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.32600000000000001},"max":{"1min":3.1560000000000001,"5min":6.6820000000000004,"15min":6.6820000000000004},"last":3.8319999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:10:26 2023","interfaces":[{"interface":"back","average":{"1min":3.141,"5min":1.6639999999999999,"15min":1.363},"min":{"1min":0.53000000000000003,"5min":0.52000000000000002,"15min":0.317},"max":{"1min":21.050999999999998,"5min":21.050999999999998,"15min":21.050999999999998},"last":3.7919999999999998},{"interface":"front","average":{"1min":2.9329999999999998,"5min":1.617,"15min":1.718},"min":{"1min":0.44800000000000001,"5min":0.42099999999999999,"15min":0.40799999999999997},"max":{"1min":19.622,"5min":19.622,"15min":19.622},"last":0.84399999999999997}]},{"osd":4,"last update":"Mon Oct 9 05:10:09 2023","interfaces":[{"interface":"back","average":{"1min":0.75700000000000001,"5min":1.147,"15min":1.0740000000000001},"min":{"1min":0.33900000000000002,"5min":0.33700000000000002,"15min":0.28699999999999998},"max":{"1min":2.0920000000000001,"5min":6.9880000000000004,"15min":11.409000000000001},"last":1.198},{"interface":"front","average":{"1min":2.2210000000000001,"5min":1.4770000000000001,"15min":1.4470000000000001},"min":{"1min":0.41199999999999998,"5min":0.38800000000000001,"15min":0.318},"max":{"1min":9.8620000000000001,"5min":9.8620000000000001,"15min":15.763},"last":0.80900000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:10:09 2023","interfaces":[{"interface":"back","average":{"1min":1.5109999999999999,"5min":1.536,"15min":1.351},"min":{"1min":0.42999999999999999,"5min":0.38200000000000001,"15min":0.34399999999999997},"max":{"1min":3.1139999999999999,"5min":7.6180000000000003,"15min":8.3569999999999993},"last":3.7370000000000001},{"interface":"front","average":{"1min":1.6279999999999999,"5min":1.464,"15min":1.3049999999999999},"min":{"1min":0.54200000000000004,"5min":0.35999999999999999,"15min":0.32900000000000001},"max":{"1min":7.1580000000000004,"5min":7.1580000000000004,"15min":7.1580000000000004},"last":0.88600000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:10:09 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.3779999999999999,"15min":1.256},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.318},"max":{"1min":2.7599999999999998,"5min":6.9630000000000001,"15min":7.9939999999999998},"last":0.64600000000000002},{"interface":"front","average":{"1min":2.4830000000000001,"5min":1.6899999999999999,"15min":1.486},"min":{"1min":0.55600000000000005,"5min":0.41499999999999998,"15min":0.35499999999999998},"max":{"1min":10.106,"5min":10.106,"15min":10.106},"last":3.4580000000000002}]}]},{"osd":6,"up_from":46,"seq":197568495989,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1459416,"kb_used_data":1162128,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":92285736,"statfs":{"total":95995035648,"available":94500593664,"internally_reserved":0,"allocated":1190019072,"data_stored":1189393883,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:10:18 2023","interfaces":[{"interface":"back","average":{"1min":4.6500000000000004,"5min":2.0350000000000001,"15min":1.7709999999999999},"min":{"1min":0.28199999999999997,"5min":0.24399999999999999,"15min":0.214},"max":{"1min":98.275000000000006,"5min":98.275000000000006,"15min":98.275000000000006},"last":1.853},{"interface":"front","average":{"1min":4.7800000000000002,"5min":2.1349999999999998,"15min":1.8939999999999999},"min":{"1min":0.373,"5min":0.315,"15min":0.27200000000000002},"max":{"1min":98.334999999999994,"5min":98.334999999999994,"15min":98.334999999999994},"last":1.897}]},{"osd":1,"last update":"Mon Oct 9 05:10:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.202,"15min":1.498},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.25800000000000001},"max":{"1min":8.8360000000000003,"5min":8.8360000000000003,"15min":10.118},"last":0.52400000000000002},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.448,"15min":1.4410000000000001},"min":{"1min":0.34200000000000003,"5min":0.32500000000000001,"15min":0.27900000000000003},"max":{"1min":11.103,"5min":11.103,"15min":12.417},"last":0.39100000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:10:16 2023","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":1.29,"15min":1.454},"min":{"1min":0.38200000000000001,"5min":0.318,"15min":0.28999999999999998},"max":{"1min":1.782,"5min":7.3040000000000003,"15min":8.9730000000000008},"last":0.44},{"interface":"front","average":{"1min":0.77900000000000003,"5min":0.98899999999999999,"15min":1.2769999999999999},"min":{"1min":0.36199999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.8660000000000001,"5min":5.8959999999999999,"15min":8.3810000000000002},"last":0.375}]},{"osd":3,"last update":"Mon Oct 9 05:10:18 2023","interfaces":[{"interface":"back","average":{"1min":1.3540000000000001,"5min":1.2869999999999999,"15min":1.5840000000000001},"min":{"1min":0.40699999999999997,"5min":0.33400000000000002,"15min":0.315},"max":{"1min":10.544,"5min":10.544,"15min":10.544},"last":1.252},{"interface":"front","average":{"1min":1.4339999999999999,"5min":1.3600000000000001,"15min":1.4630000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":6.4109999999999996,"5min":9.7720000000000002,"15min":9.7720000000000002},"last":0.54400000000000004}]},{"osd":4,"last update":"Mon Oct 9 05:10:18 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":1.0509999999999999,"15min":1.127},"min":{"1min":0.36099999999999999,"5min":0.33600000000000002,"15min":0.28199999999999997},"max":{"1min":2.3140000000000001,"5min":5.5490000000000004,"15min":16.119},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.83499999999999996,"5min":1.1699999999999999,"15min":1.252},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.247},"max":{"1min":2.0059999999999998,"5min":6.3920000000000003,"15min":11.272},"last":3.3940000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:10:18 2023","interfaces":[{"interface":"back","average":{"1min":1.284,"5min":1.2629999999999999,"15min":1.3149999999999999},"min":{"1min":0.437,"5min":0.38100000000000001,"15min":0.29299999999999998},"max":{"1min":6.2199999999999998,"5min":6.2199999999999998,"15min":7.7149999999999999},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.99199999999999999,"5min":1.397,"15min":1.474},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.33100000000000002},"max":{"1min":2.8170000000000002,"5min":9.5730000000000004,"15min":10.537000000000001},"last":0.504}]},{"osd":7,"last update":"Mon Oct 9 05:10:56 2023","interfaces":[{"interface":"back","average":{"1min":1.4490000000000001,"5min":1.23,"15min":1.347},"min":{"1min":0.42499999999999999,"5min":0.40200000000000002,"15min":0.30599999999999999},"max":{"1min":8.1120000000000001,"5min":8.1120000000000001,"15min":9.3780000000000001},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.5249999999999999,"5min":1.355,"15min":1.4350000000000001},"min":{"1min":0.47999999999999998,"5min":0.434,"15min":0.375},"max":{"1min":8.1859999999999999,"5min":8.1859999999999999,"15min":9.4060000000000006},"last":0.47999999999999998}]}]},{"osd":1,"up_from":96,"seq":412316860580,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2318800,"kb_used_data":1870280,"kb_used_omap":4,"kb_used_meta":448507,"kb_avail":91426352,"statfs":{"total":95995035648,"available":93620584448,"internally_reserved":0,"allocated":1915166720,"data_stored":1914318005,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":459272078},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:11:07 2023","interfaces":[{"interface":"back","average":{"1min":1.6379999999999999,"5min":0.85199999999999998,"15min":1.6870000000000001},"min":{"1min":0.24299999999999999,"5min":0.22500000000000001,"15min":0.19400000000000001},"max":{"1min":6.2699999999999996,"5min":6.2699999999999996,"15min":22.382999999999999},"last":2.6859999999999999},{"interface":"front","average":{"1min":1.9379999999999999,"5min":1.016,"15min":2.0230000000000001},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":7.0259999999999998,"5min":7.0259999999999998,"15min":10.268000000000001},"last":2.5630000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:10:23 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":1.0509999999999999,"15min":1.4379999999999999},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.219},"max":{"1min":6.3259999999999996,"5min":8.1310000000000002,"15min":10.115},"last":0.69399999999999995},{"interface":"front","average":{"1min":1.0089999999999999,"5min":1.1259999999999999,"15min":1.4419999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.252},"max":{"1min":6.4050000000000002,"5min":10.428000000000001,"15min":11.917},"last":0.72399999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:10:23 2023","interfaces":[{"interface":"back","average":{"1min":3.3940000000000001,"5min":1.4850000000000001,"15min":2.0310000000000001},"min":{"1min":0.27500000000000002,"5min":0.23400000000000001,"15min":0.23100000000000001},"max":{"1min":40.613999999999997,"5min":40.613999999999997,"15min":40.613999999999997},"last":61.280000000000001},{"interface":"front","average":{"1min":4.0519999999999996,"5min":1.6559999999999999,"15min":1.5840000000000001},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.24399999999999999},"max":{"1min":38.479999999999997,"5min":38.479999999999997,"15min":38.479999999999997},"last":65.046000000000006}]},{"osd":4,"last update":"Mon Oct 9 05:11:07 2023","interfaces":[{"interface":"back","average":{"1min":1.984,"5min":1.214,"15min":1.53},"min":{"1min":0.5,"5min":0.40000000000000002,"15min":0.34200000000000003},"max":{"1min":6.3520000000000003,"5min":6.3520000000000003,"15min":9.9749999999999996},"last":0.626},{"interface":"front","average":{"1min":1.698,"5min":1.103,"15min":1.498},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.30099999999999999},"max":{"1min":6.5339999999999998,"5min":6.5339999999999998,"15min":22.350000000000001},"last":2.7759999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:11:07 2023","interfaces":[{"interface":"back","average":{"1min":2.75,"5min":1.4350000000000001,"15min":1.821},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.32500000000000001},"max":{"1min":9.0530000000000008,"5min":9.0530000000000008,"15min":12.454000000000001},"last":2.5259999999999998},{"interface":"front","average":{"1min":2.4569999999999999,"5min":1.345,"15min":1.8260000000000001},"min":{"1min":0.59099999999999997,"5min":0.51000000000000001,"15min":0.316},"max":{"1min":6.6660000000000004,"5min":6.6660000000000004,"15min":10.035},"last":4.5839999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:11:07 2023","interfaces":[{"interface":"back","average":{"1min":1.385,"5min":1.1140000000000001,"15min":1.8300000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.7050000000000001,"5min":4.7050000000000001,"15min":10.409000000000001},"last":2.5880000000000001},{"interface":"front","average":{"1min":2.1389999999999998,"5min":1.3089999999999999,"15min":1.6819999999999999},"min":{"1min":0.54700000000000004,"5min":0.44800000000000001,"15min":0.35099999999999998},"max":{"1min":6.4690000000000003,"5min":6.4690000000000003,"15min":10.007},"last":0.59199999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:11:07 2023","interfaces":[{"interface":"back","average":{"1min":2.323,"5min":1.468,"15min":1.75},"min":{"1min":0.64000000000000001,"5min":0.46899999999999997,"15min":0.38800000000000001},"max":{"1min":6.4969999999999999,"5min":6.4969999999999999,"15min":10.061},"last":4.093},{"interface":"front","average":{"1min":1.8620000000000001,"5min":1.2969999999999999,"15min":1.6950000000000001},"min":{"1min":0.44,"5min":0.44,"15min":0.36399999999999999},"max":{"1min":6.3209999999999997,"5min":6.3209999999999997,"15min":22.175999999999998},"last":4.1139999999999999}]}]},{"osd":0,"up_from":88,"seq":377957122221,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2538368,"kb_used_data":2160312,"kb_used_omap":5,"kb_used_meta":378042,"kb_avail":91206784,"statfs":{"total":95995035648,"available":93395746816,"internally_reserved":0,"allocated":2212159488,"data_stored":2211325380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":387115642},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:11:04 2023","interfaces":[{"interface":"back","average":{"1min":7.8019999999999996,"5min":2.1219999999999999,"15min":2.282},"min":{"1min":0.34300000000000003,"5min":0.24399999999999999,"15min":0.217},"max":{"1min":109.88,"5min":109.88,"15min":109.88},"last":3.3159999999999998},{"interface":"front","average":{"1min":7.641,"5min":2.0800000000000001,"15min":2.1789999999999998},"min":{"1min":0.25,"5min":0.25,"15min":0.218},"max":{"1min":109.435,"5min":109.435,"15min":109.435},"last":0.55300000000000005}]},{"osd":2,"last update":"Mon Oct 9 05:10:16 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":0.998,"15min":1.6919999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":3.0390000000000001,"5min":8.3420000000000005,"15min":9.1300000000000008},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.89000000000000001,"15min":1.4710000000000001},"min":{"1min":0.29099999999999998,"5min":0.25,"15min":0.23300000000000001},"max":{"1min":2.665,"5min":8.3599999999999994,"15min":10.423},"last":2.7810000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:10:23 2023","interfaces":[{"interface":"back","average":{"1min":3.218,"5min":1.3360000000000001,"15min":1.5009999999999999},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":34.292000000000002,"5min":34.292000000000002,"15min":34.292000000000002},"last":7.625},{"interface":"front","average":{"1min":3.198,"5min":1.4399999999999999,"15min":1.395},"min":{"1min":0.41999999999999998,"5min":0.318,"15min":0.318},"max":{"1min":34.674999999999997,"5min":34.674999999999997,"15min":34.674999999999997},"last":0.45300000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:10:23 2023","interfaces":[{"interface":"back","average":{"1min":1.2969999999999999,"5min":1.1919999999999999,"15min":1.415},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.30099999999999999},"max":{"1min":3.1899999999999999,"5min":8.7449999999999992,"15min":8.7449999999999992},"last":0.52600000000000002},{"interface":"front","average":{"1min":1.2190000000000001,"5min":1.0589999999999999,"15min":1.4059999999999999},"min":{"1min":0.41899999999999998,"5min":0.35099999999999998,"15min":0.33700000000000002},"max":{"1min":4.6879999999999997,"5min":4.6879999999999997,"15min":7.202},"last":2.948}]},{"osd":5,"last update":"Mon Oct 9 05:10:23 2023","interfaces":[{"interface":"back","average":{"1min":1.3380000000000001,"5min":1.145,"15min":1.5309999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.2229999999999999,"5min":6.8959999999999999,"15min":12.526999999999999},"last":5.9279999999999999},{"interface":"front","average":{"1min":1.8180000000000001,"5min":1.4370000000000001,"15min":1.6319999999999999},"min":{"1min":0.376,"5min":0.35299999999999998,"15min":0.316},"max":{"1min":4.5549999999999997,"5min":8.4169999999999998,"15min":10.148},"last":0.47899999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:10:23 2023","interfaces":[{"interface":"back","average":{"1min":1.204,"5min":1.1240000000000001,"15min":1.556},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.371},"max":{"1min":4.7480000000000002,"5min":4.7480000000000002,"15min":8.8710000000000004},"last":2.9980000000000002},{"interface":"front","average":{"1min":1.1539999999999999,"5min":1.127,"15min":1.4490000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":2.7829999999999999,"5min":6.9370000000000003,"15min":10.429},"last":7.282}]},{"osd":7,"last update":"Mon Oct 9 05:10:23 2023","interfaces":[{"interface":"back","average":{"1min":1.7050000000000001,"5min":1.2529999999999999,"15min":1.4710000000000001},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.36599999999999999},"max":{"1min":5.0469999999999997,"5min":6.9569999999999999,"15min":10.314},"last":3.5699999999999998},{"interface":"front","average":{"1min":1.5469999999999999,"5min":1.3560000000000001,"15min":1.7490000000000001},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.39500000000000002},"max":{"1min":3.3370000000000002,"5min":8.6110000000000007,"15min":10.738},"last":0.503}]}]},{"osd":2,"up_from":106,"seq":455266533500,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1435024,"kb_used_data":1147784,"kb_used_omap":7,"kb_used_meta":287224,"kb_avail":92310128,"statfs":{"total":95995035648,"available":94525571072,"internally_reserved":0,"allocated":1175330816,"data_stored":1174537880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":294117457},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:10:10 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":1.0609999999999999,"15min":1.1930000000000001},"min":{"1min":0.30599999999999999,"5min":0.27400000000000002,"15min":0.215},"max":{"1min":1.3859999999999999,"5min":23.010999999999999,"15min":23.010999999999999},"last":0.26300000000000001},{"interface":"front","average":{"1min":0.58699999999999997,"5min":1.1870000000000001,"15min":1.2629999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.25700000000000001},"max":{"1min":1.762,"5min":20.501999999999999,"15min":20.501999999999999},"last":0.34599999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:10:10 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":1.0960000000000001,"15min":1.1899999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.29299999999999998},"max":{"1min":1.819,"5min":20.539999999999999,"15min":20.539999999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.86099999999999999,"5min":1.2230000000000001,"15min":1.403},"min":{"1min":0.33500000000000002,"5min":0.23899999999999999,"15min":0.23100000000000001},"max":{"1min":6.0099999999999998,"5min":24.058,"15min":24.058},"last":0.42299999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:10:26 2023","interfaces":[{"interface":"back","average":{"1min":1.087,"5min":1.0649999999999999,"15min":1.861},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.27300000000000002},"max":{"1min":4.4480000000000004,"5min":6.0579999999999998,"15min":20.907},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.92100000000000004,"15min":1.591},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.242,"5min":6.1550000000000002,"15min":23.574999999999999},"last":0.53200000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:10:10 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":1.3169999999999999,"15min":1.387},"min":{"1min":0.36899999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.8560000000000001,"5min":20.712,"15min":20.712},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.74399999999999999,"5min":1.3779999999999999,"15min":1.4319999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.653,"5min":23.547000000000001,"15min":23.547000000000001},"last":0.46300000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:10:10 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":1.504,"15min":1.4750000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.3899999999999999,"5min":20.782,"15min":20.782},"last":0.66200000000000003},{"interface":"front","average":{"1min":1.1240000000000001,"5min":1.365,"15min":1.8080000000000001},"min":{"1min":0.55600000000000005,"5min":0.36699999999999999,"15min":0.36299999999999999},"max":{"1min":3.9700000000000002,"5min":20.792000000000002,"15min":20.792000000000002},"last":0.56100000000000005}]},{"osd":6,"last update":"Mon Oct 9 05:10:10 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":1.351,"15min":1.5},"min":{"1min":0.48599999999999999,"5min":0.40899999999999997,"15min":0.34999999999999998},"max":{"1min":2.528,"5min":23.510999999999999,"15min":23.510999999999999},"last":0.5},{"interface":"front","average":{"1min":0.73799999999999999,"5min":1.45,"15min":1.4570000000000001},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.2629999999999999,"5min":20.684999999999999,"15min":20.684999999999999},"last":0.629}]},{"osd":7,"last update":"Mon Oct 9 05:10:10 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":1.411,"15min":1.4490000000000001},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":2.8759999999999999,"5min":20.887,"15min":20.887},"last":0.57799999999999996},{"interface":"front","average":{"1min":1.0449999999999999,"5min":1.381,"15min":1.4419999999999999},"min":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.45400000000000001},"max":{"1min":2.8889999999999998,"5min":20.863,"15min":20.863},"last":0.60999999999999999}]}]},{"osd":3,"up_from":123,"seq":528280977518,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3267484,"kb_used_data":2860564,"kb_used_omap":2,"kb_used_meta":406845,"kb_avail":90477668,"statfs":{"total":95995035648,"available":92649132032,"internally_reserved":0,"allocated":2929217536,"data_stored":2928471446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":416609333},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:10:23 2023","interfaces":[{"interface":"back","average":{"1min":1.988,"5min":1.173,"15min":1.8280000000000001},"min":{"1min":0.35999999999999999,"5min":0.34699999999999998,"15min":0.187},"max":{"1min":6.5720000000000001,"5min":6.5720000000000001,"15min":14.141},"last":0.749},{"interface":"front","average":{"1min":1.4930000000000001,"5min":1.1459999999999999,"15min":2.226},"min":{"1min":0.374,"5min":0.253,"15min":0.184},"max":{"1min":6.1769999999999996,"5min":7.8630000000000004,"15min":14.045999999999999},"last":2.2669999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:10:23 2023","interfaces":[{"interface":"back","average":{"1min":2.1070000000000002,"5min":1.2529999999999999,"15min":2.1019999999999999},"min":{"1min":0.48599999999999999,"5min":0.308,"15min":0.28000000000000003},"max":{"1min":14.808,"5min":14.808,"15min":14.808},"last":2.3559999999999999},{"interface":"front","average":{"1min":1.0800000000000001,"5min":0.95599999999999996,"15min":2.1219999999999999},"min":{"1min":0.32400000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":2.6299999999999999,"5min":6.1950000000000003,"15min":17.498999999999999},"last":0.57899999999999996}]},{"osd":2,"last update":"Mon Oct 9 05:10:23 2023","interfaces":[{"interface":"back","average":{"1min":1.3380000000000001,"5min":1.004,"15min":1.835},"min":{"1min":0.35399999999999998,"5min":0.28899999999999998,"15min":0.23699999999999999},"max":{"1min":5.556,"5min":8.4909999999999997,"15min":9.6129999999999995},"last":0.41899999999999998},{"interface":"front","average":{"1min":2.0169999999999999,"5min":1.4019999999999999,"15min":1.732},"min":{"1min":0.315,"5min":0.315,"15min":0.245},"max":{"1min":14.754,"5min":14.754,"15min":14.754},"last":0.24099999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:10:23 2023","interfaces":[{"interface":"back","average":{"1min":2.1549999999999998,"5min":1.3799999999999999,"15min":1.649},"min":{"1min":0.45100000000000001,"5min":0.44900000000000001,"15min":0.34399999999999997},"max":{"1min":14.833,"5min":14.833,"15min":14.833},"last":0.32100000000000001},{"interface":"front","average":{"1min":1.4390000000000001,"5min":1.099,"15min":1.276},"min":{"1min":0.38,"5min":0.38,"15min":0.34999999999999998},"max":{"1min":3.6829999999999998,"5min":4.5010000000000003,"15min":9.4749999999999996},"last":0.59299999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:10:23 2023","interfaces":[{"interface":"back","average":{"1min":1.381,"5min":1.2,"15min":1.333},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.38300000000000001},"max":{"1min":2.6829999999999998,"5min":5.2859999999999996,"15min":9.5079999999999991},"last":0.61099999999999999},{"interface":"front","average":{"1min":1.968,"5min":1.413,"15min":1.9159999999999999},"min":{"1min":0.52700000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":5.843,"5min":8.1699999999999999,"15min":8.9380000000000006},"last":2.2389999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:10:23 2023","interfaces":[{"interface":"back","average":{"1min":1.7709999999999999,"5min":1.288,"15min":1.7589999999999999},"min":{"1min":0.47799999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":7.351,"5min":7.351,"15min":9.6590000000000007},"last":2.2000000000000002},{"interface":"front","average":{"1min":2.3740000000000001,"5min":1.5640000000000001,"15min":1.8839999999999999},"min":{"1min":0.53700000000000003,"5min":0.38400000000000001,"15min":0.34699999999999998},"max":{"1min":14.881,"5min":14.881,"15min":14.881},"last":0.38}]},{"osd":7,"last update":"Mon Oct 9 05:10:23 2023","interfaces":[{"interface":"back","average":{"1min":2.8479999999999999,"5min":1.6359999999999999,"15min":2.0590000000000002},"min":{"1min":0.82599999999999996,"5min":0.42499999999999999,"15min":0.40999999999999998},"max":{"1min":14.856,"5min":14.856,"15min":14.856},"last":0.39800000000000002},{"interface":"front","average":{"1min":2.085,"5min":1.4179999999999999,"15min":1.55},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.38800000000000001},"max":{"1min":6.4379999999999997,"5min":6.4379999999999997,"15min":9.5359999999999996},"last":0.55900000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019275,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1771872,"kb_used_data":1186200,"kb_used_omap":0,"kb_used_meta":585664,"kb_avail":91973280,"statfs":{"total":95995035648,"available":94180638720,"internally_reserved":0,"allocated":1214668800,"data_stored":1214035216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599719936},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:10:17 2023","interfaces":[{"interface":"back","average":{"1min":8.2929999999999993,"5min":2.6200000000000001,"15min":3.4119999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.22700000000000001},"max":{"1min":102.669,"5min":102.669,"15min":131.30699999999999},"last":0.753},{"interface":"front","average":{"1min":8.3460000000000001,"5min":2.581,"15min":3.3500000000000001},"min":{"1min":0.36699999999999999,"5min":0.34000000000000002,"15min":0.26900000000000002},"max":{"1min":102.72499999999999,"5min":102.72499999999999,"15min":131.36600000000001},"last":0.79000000000000004}]},{"osd":1,"last update":"Mon Oct 9 05:11:08 2023","interfaces":[{"interface":"back","average":{"1min":1.351,"5min":1.016,"15min":1.764},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.28999999999999998},"max":{"1min":4.9770000000000003,"5min":4.9770000000000003,"15min":59.408000000000001},"last":0.34499999999999997},{"interface":"front","average":{"1min":1.2130000000000001,"5min":0.999,"15min":1.784},"min":{"1min":0.378,"5min":0.34599999999999997,"15min":0.28199999999999997},"max":{"1min":4.8700000000000001,"5min":4.8700000000000001,"15min":60.207000000000001},"last":0.58899999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:10:17 2023","interfaces":[{"interface":"back","average":{"1min":0.74099999999999999,"5min":1.099,"15min":1.1850000000000001},"min":{"1min":0.36199999999999999,"5min":0.33200000000000002,"15min":0.307},"max":{"1min":2.407,"5min":10.138,"15min":10.196},"last":0.72499999999999998},{"interface":"front","average":{"1min":0.77200000000000002,"5min":1.042,"15min":1.1020000000000001},"min":{"1min":0.35899999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.3050000000000002,"5min":6.3810000000000002,"15min":6.3810000000000002},"last":0.41599999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:10:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":1.327,"15min":1.6240000000000001},"min":{"1min":0.38400000000000001,"5min":0.371,"15min":0.309},"max":{"1min":4.3239999999999998,"5min":10.555999999999999,"15min":10.555999999999999},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.89900000000000002,"5min":1.1140000000000001,"15min":1.5449999999999999},"min":{"1min":0.373,"5min":0.373,"15min":0.35399999999999998},"max":{"1min":2.431,"5min":6.4160000000000004,"15min":21.036999999999999},"last":0.54200000000000004}]},{"osd":5,"last update":"Mon Oct 9 05:10:20 2023","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":1.222,"15min":1.234},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.314},"max":{"1min":2.3319999999999999,"5min":10.214,"15min":10.214},"last":0.69199999999999995},{"interface":"front","average":{"1min":1.4259999999999999,"5min":1.2609999999999999,"15min":1.278},"min":{"1min":0.437,"5min":0.36499999999999999,"15min":0.31900000000000001},"max":{"1min":3.7360000000000002,"5min":6.4459999999999997,"15min":10.890000000000001},"last":0.47699999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:11:08 2023","interfaces":[{"interface":"back","average":{"1min":1.3,"5min":1.0680000000000001,"15min":1.323},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.32800000000000001},"max":{"1min":5.2249999999999996,"5min":5.2249999999999996,"15min":11.557},"last":3.2010000000000001},{"interface":"front","average":{"1min":0.76400000000000001,"5min":1.077,"15min":1.2629999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.34699999999999998},"max":{"1min":3.5609999999999999,"5min":3.5609999999999999,"15min":10.689},"last":0.66400000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:11:08 2023","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":1.0840000000000001,"15min":1.117},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.36299999999999999},"max":{"1min":2.0459999999999998,"5min":3.6989999999999998,"15min":8.6449999999999996},"last":0.81699999999999995},{"interface":"front","average":{"1min":1.9179999999999999,"5min":1.2509999999999999,"15min":1.333},"min":{"1min":0.443,"5min":0.443,"15min":0.38600000000000001},"max":{"1min":8.1270000000000007,"5min":8.1270000000000007,"15min":8.1270000000000007},"last":3.254}]}]},{"osd":5,"up_from":38,"seq":163208757631,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4071572,"kb_used_data":3172748,"kb_used_omap":0,"kb_used_meta":898752,"kb_avail":89673580,"statfs":{"total":95995035648,"available":91825745920,"internally_reserved":0,"allocated":3248893952,"data_stored":3248268424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920322048},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:10:14 2023","interfaces":[{"interface":"back","average":{"1min":2.8570000000000002,"5min":1.5820000000000001,"15min":1.9339999999999999},"min":{"1min":0.373,"5min":0.373,"15min":0.24399999999999999},"max":{"1min":40.097999999999999,"5min":40.097999999999999,"15min":77.745999999999995},"last":4.1799999999999997},{"interface":"front","average":{"1min":2.9729999999999999,"5min":1.7050000000000001,"15min":2.0419999999999998},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.35299999999999998},"max":{"1min":40.161000000000001,"5min":40.161000000000001,"15min":77.820999999999998},"last":4.2140000000000004}]},{"osd":1,"last update":"Mon Oct 9 05:10:12 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":1.2509999999999999,"15min":1.643},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.26900000000000002},"max":{"1min":5.5030000000000001,"5min":5.7229999999999999,"15min":13.401999999999999},"last":4.2539999999999996},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.1200000000000001,"15min":1.6519999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.26800000000000002},"max":{"1min":2.3769999999999998,"5min":5.6959999999999997,"15min":13.398},"last":0.61899999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":1.052,"15min":1.1459999999999999},"min":{"1min":0.374,"5min":0.36699999999999999,"15min":0.28499999999999998},"max":{"1min":3.3730000000000002,"5min":5.1459999999999999,"15min":13.436},"last":0.65400000000000003},{"interface":"front","average":{"1min":1.448,"5min":1.264,"15min":1.3},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.32000000000000001},"max":{"1min":6.1059999999999999,"5min":9.4949999999999992,"15min":9.4949999999999992},"last":0.71199999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:10:25 2023","interfaces":[{"interface":"back","average":{"1min":1.401,"5min":1.353,"15min":1.8},"min":{"1min":0.46600000000000003,"5min":0.39000000000000001,"15min":0.32500000000000001},"max":{"1min":5.093,"5min":9.1010000000000009,"15min":9.1010000000000009},"last":4.1520000000000001},{"interface":"front","average":{"1min":1.778,"5min":1.514,"15min":1.6990000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.36799999999999999},"max":{"1min":5.2590000000000003,"5min":9.5500000000000007,"15min":11.859999999999999},"last":0.745}]},{"osd":4,"last update":"Mon Oct 9 05:10:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":1.0940000000000001,"15min":1.288},"min":{"1min":0.33100000000000002,"5min":0.316,"15min":0.25},"max":{"1min":3.3130000000000002,"5min":5.1879999999999997,"15min":14.781000000000001},"last":0.68400000000000005},{"interface":"front","average":{"1min":1.323,"5min":1.3149999999999999,"15min":1.419},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.34499999999999997},"max":{"1min":5.1580000000000004,"5min":9.625,"15min":9.625},"last":0.56599999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:10:12 2023","interfaces":[{"interface":"back","average":{"1min":1.393,"5min":1.423,"15min":1.532},"min":{"1min":0.45900000000000002,"5min":0.44800000000000001,"15min":0.34699999999999998},"max":{"1min":5.218,"5min":9.8330000000000002,"15min":11.654999999999999},"last":0.77100000000000002},{"interface":"front","average":{"1min":1.4279999999999999,"5min":1.353,"15min":1.4830000000000001},"min":{"1min":0.44,"5min":0.44,"15min":0.33500000000000002},"max":{"1min":5.4409999999999998,"5min":5.4409999999999998,"15min":10.423999999999999},"last":3.7549999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:10:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3300000000000001,"5min":1.4059999999999999,"15min":1.4350000000000001},"min":{"1min":0.443,"5min":0.443,"15min":0.36899999999999999},"max":{"1min":5.3390000000000004,"5min":6.9640000000000004,"15min":9.6639999999999997},"last":4.133},{"interface":"front","average":{"1min":1.4359999999999999,"5min":1.3799999999999999,"15min":1.49},"min":{"1min":0.42999999999999999,"5min":0.40000000000000002,"15min":0.313},"max":{"1min":5.2199999999999998,"5min":9.0500000000000007,"15min":12.843999999999999},"last":4.4660000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2194735104,"data_stored":2194735104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1911164928,"data_stored":1911160856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1170014208,"data_stored":1170014208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2924544000,"data_stored":2924544000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1205665792,"data_stored":1205665792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3225686016,"data_stored":3225681944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1183055872,"data_stored":1183055872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3224178688,"data_stored":3224174616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:11:18.955 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T05:11:18.955 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_4 pg_num 26 2023-10-09T05:11:18.973 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:19.016 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:19.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:19.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:19.122 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:19.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:18 smithi194 bash[92666]: cluster 2023-10-09T05:11:17.794954+0000 mgr.y (mgr.25277) 682 : cluster 0 pgmap v651: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 15772/237540 objects degraded (6.640%); 4505/237540 objects misplaced (1.897%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:11:19.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:18 smithi194 bash[92666]: cluster 2023-10-09T05:11:18.260737+0000 mon.a (mon.0) 857 : cluster 3 Health check update: Degraded data redundancy: 15772/237540 objects degraded (6.640%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:19.149 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:19.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:19.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:19.153 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:11:19.207 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:18 smithi133 bash[102858]: cluster 2023-10-09T05:11:17.794954+0000 mgr.y (mgr.25277) 682 : cluster 0 pgmap v651: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 15772/237540 objects degraded (6.640%); 4505/237540 objects misplaced (1.897%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:11:19.207 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:18 smithi133 bash[102858]: cluster 2023-10-09T05:11:18.260737+0000 mon.a (mon.0) 857 : cluster 3 Health check update: Degraded data redundancy: 15772/237540 objects degraded (6.640%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:19.208 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:18 smithi133 bash[105589]: cluster 2023-10-09T05:11:17.794954+0000 mgr.y (mgr.25277) 682 : cluster 0 pgmap v651: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 15772/237540 objects degraded (6.640%); 4505/237540 objects misplaced (1.897%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:11:19.208 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:18 smithi133 bash[105589]: cluster 2023-10-09T05:11:18.260737+0000 mon.a (mon.0) 857 : cluster 3 Health check update: Degraded data redundancy: 15772/237540 objects degraded (6.640%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:19.251 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:19.303 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:19.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:19.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:19.408 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:19.458 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:19.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:19.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:19.563 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:19.606 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:11:19.606 INFO:teuthology.orchestra.run.smithi133.stdout: "ops": [], 2023-10-09T05:11:19.606 INFO:teuthology.orchestra.run.smithi133.stdout: "num_ops": 0 2023-10-09T05:11:19.606 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:11:19.608 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:19.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:19.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:19.713 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:19.954 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:19.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:19.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:20.059 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:20.078 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:20.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:20.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:20.181 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:20.211 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:20.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:20.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:20.315 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:20.350 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:20.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:20.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:20.461 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:20.495 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:20.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:20.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:20.601 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:20.632 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:20.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:20.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:20.739 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:20.771 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:20.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:20.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:20.875 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:20.908 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:20.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:20.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:20.924 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:11:21.013 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:21.038 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:21.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:21.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:21.094 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T05:11:21.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:20 smithi194 bash[92666]: cluster 2023-10-09T05:11:19.795650+0000 mgr.y (mgr.25277) 683 : cluster 0 pgmap v652: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 15772/230562 objects degraded (6.841%); 4288/230562 objects misplaced (1.860%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:11:21.141 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:21.179 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:21.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:21.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:21.284 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:21.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:20 smithi133 bash[102858]: cluster 2023-10-09T05:11:19.795650+0000 mgr.y (mgr.25277) 683 : cluster 0 pgmap v652: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 15772/230562 objects degraded (6.841%); 4288/230562 objects misplaced (1.860%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:11:21.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:20 smithi133 bash[105589]: cluster 2023-10-09T05:11:19.795650+0000 mgr.y (mgr.25277) 683 : cluster 0 pgmap v652: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 15772/230562 objects degraded (6.841%); 4288/230562 objects misplaced (1.860%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:11:21.315 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:21.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:21.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:21.424 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:21.446 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:21.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:21.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:21.552 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:21.580 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:21.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:21.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:21.684 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:21.719 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:21.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:21.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:21.824 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:21.852 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:21.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:21.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:21.956 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:21.995 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:21.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:21.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:22.100 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:22.129 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:22.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:22.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:22.236 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:22.266 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:22.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:22.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:22.372 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:22.406 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:22.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:22.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:22.511 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:22.536 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:22.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:22.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:22.643 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:22.678 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:22.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:22.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:22.781 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:22.814 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:22.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:22.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:22.920 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:22.971 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:22.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:22.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:23.074 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:23.083 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:22 smithi194 bash[92666]: cluster 2023-10-09T05:11:21.796432+0000 mgr.y (mgr.25277) 684 : cluster 0 pgmap v653: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 15772/227256 objects degraded (6.940%); 4288/227256 objects misplaced (1.887%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:11:23.108 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:23.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:23.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:23.212 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:23.241 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:23.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:23.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:23.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:22 smithi133 bash[105589]: cluster 2023-10-09T05:11:21.796432+0000 mgr.y (mgr.25277) 684 : cluster 0 pgmap v653: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 15772/227256 objects degraded (6.940%); 4288/227256 objects misplaced (1.887%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:11:23.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:22 smithi133 bash[102858]: cluster 2023-10-09T05:11:21.796432+0000 mgr.y (mgr.25277) 684 : cluster 0 pgmap v653: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 15772/227256 objects degraded (6.940%); 4288/227256 objects misplaced (1.887%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:11:23.344 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:23.377 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:23.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:23.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:23.482 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:23.514 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:23.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:23.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:23.619 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:23.673 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:23.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:23.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:23.778 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:23.800 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:23.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:23.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:23.903 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:23.933 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:23.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:23.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:23.976 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:23 smithi133 bash[102858]: cluster 2023-10-09T05:11:23.261858+0000 mon.a (mon.0) 858 : cluster 3 Health check update: Degraded data redundancy: 15772/227256 objects degraded (6.940%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:24.042 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:24.050 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:23 smithi194 bash[92666]: cluster 2023-10-09T05:11:23.261858+0000 mon.a (mon.0) 858 : cluster 3 Health check update: Degraded data redundancy: 15772/227256 objects degraded (6.940%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:24.070 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:24.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:24.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:24.178 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:24.205 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:24.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:24.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:24.282 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:23 smithi133 bash[105589]: cluster 2023-10-09T05:11:23.261858+0000 mon.a (mon.0) 858 : cluster 3 Health check update: Degraded data redundancy: 15772/227256 objects degraded (6.940%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:24.314 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:24.347 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:24.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:24.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:24.453 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:24.483 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:24.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:24.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:24.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:11:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:11:24] "GET /metrics HTTP/1.1" 200 36432 "" "Prometheus/2.43.0" 2023-10-09T05:11:24.680 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:24.681 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:11:24.681 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:11:24.739 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:24.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:24.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:24.846 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:24.871 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:24.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:24.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:24.974 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:24.977 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:24 smithi133 bash[102858]: cluster 2023-10-09T05:11:23.797147+0000 mgr.y (mgr.25277) 685 : cluster 0 pgmap v654: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 15772/224655 objects degraded (7.021%); 4288/224655 objects misplaced (1.909%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:11:25.001 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:25.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:25.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:25.103 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:25.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:24 smithi194 bash[92666]: cluster 2023-10-09T05:11:23.797147+0000 mgr.y (mgr.25277) 685 : cluster 0 pgmap v654: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 15772/224655 objects degraded (7.021%); 4288/224655 objects misplaced (1.909%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:11:25.143 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:25.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:25.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:25.250 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:25.294 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:25.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:24 smithi133 bash[105589]: cluster 2023-10-09T05:11:23.797147+0000 mgr.y (mgr.25277) 685 : cluster 0 pgmap v654: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 15772/224655 objects degraded (7.021%); 4288/224655 objects misplaced (1.909%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:11:25.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:25.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:25.402 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:25.437 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:25.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:25.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:25.541 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:25.569 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:25.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:25.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:25.658 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:11:25.658 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:11:25.673 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:25.702 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:25.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:25.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:25.807 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:25.846 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:25.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:25.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:25.949 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:25.979 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:25.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:25.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:26.084 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:26.110 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:26.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:26.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:26.213 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:26.256 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:26.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:26.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:26.365 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:26.395 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:26.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:26.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:26.501 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:26.554 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:26.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:26.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:26.660 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:26.696 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:26.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:26.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:26.800 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:26.832 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:26.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:26.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:26.936 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:26.967 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:26.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:26.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:26.984 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:26 smithi133 bash[102858]: cluster 2023-10-09T05:11:25.797800+0000 mgr.y (mgr.25277) 686 : cluster 0 pgmap v655: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 15772/220083 objects degraded (7.166%); 4080/220083 objects misplaced (1.854%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T05:11:27.077 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:27.118 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:27.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:27.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:27.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:26 smithi194 bash[92666]: cluster 2023-10-09T05:11:25.797800+0000 mgr.y (mgr.25277) 686 : cluster 0 pgmap v655: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 15772/220083 objects degraded (7.166%); 4080/220083 objects misplaced (1.854%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T05:11:27.224 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:27.253 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:27.255 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:26 smithi133 bash[105589]: cluster 2023-10-09T05:11:25.797800+0000 mgr.y (mgr.25277) 686 : cluster 0 pgmap v655: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 15772/220083 objects degraded (7.166%); 4080/220083 objects misplaced (1.854%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T05:11:27.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:27.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:27.360 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:27.387 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:27.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:27.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:27.494 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:27.538 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:27.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:27.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:27.642 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:27.669 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:27.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:27.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:27.777 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:27.804 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:27.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:27.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:27.909 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:27.942 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:27.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:27.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:28.048 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:28.095 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:28.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:28.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:28.198 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:28.230 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:28.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:28.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:28.333 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:28.368 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:28.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:28.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:28.476 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:28.497 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:28.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:28.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:28.599 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:28.617 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:28.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:28.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:28.719 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:28.767 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:28.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:28.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:28.873 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:28.920 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:28.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:28.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:29.024 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:29.070 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:29.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:29.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:29.175 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:29.200 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:29.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:29.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:29.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:28 smithi133 bash[102858]: cluster 2023-10-09T05:11:27.798374+0000 mgr.y (mgr.25277) 687 : cluster 0 pgmap v656: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 15772/216831 objects degraded (7.274%); 4080/216831 objects misplaced (1.882%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T05:11:29.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:28 smithi133 bash[102858]: cluster 2023-10-09T05:11:28.263183+0000 mon.a (mon.0) 859 : cluster 3 Health check update: Degraded data redundancy: 15772/216831 objects degraded (7.274%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:29.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:28 smithi133 bash[102858]: audit 2023-10-09T05:11:28.442658+0000 mon.a (mon.0) 860 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:11:29.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:28 smithi133 bash[105589]: cluster 2023-10-09T05:11:27.798374+0000 mgr.y (mgr.25277) 687 : cluster 0 pgmap v656: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 15772/216831 objects degraded (7.274%); 4080/216831 objects misplaced (1.882%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T05:11:29.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:28 smithi133 bash[105589]: cluster 2023-10-09T05:11:28.263183+0000 mon.a (mon.0) 859 : cluster 3 Health check update: Degraded data redundancy: 15772/216831 objects degraded (7.274%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:29.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:28 smithi133 bash[105589]: audit 2023-10-09T05:11:28.442658+0000 mon.a (mon.0) 860 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:11:29.304 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:29.313 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:28 smithi194 bash[92666]: cluster 2023-10-09T05:11:27.798374+0000 mgr.y (mgr.25277) 687 : cluster 0 pgmap v656: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 15772/216831 objects degraded (7.274%); 4080/216831 objects misplaced (1.882%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T05:11:29.313 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:28 smithi194 bash[92666]: cluster 2023-10-09T05:11:28.263183+0000 mon.a (mon.0) 859 : cluster 3 Health check update: Degraded data redundancy: 15772/216831 objects degraded (7.274%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:29.313 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:28 smithi194 bash[92666]: audit 2023-10-09T05:11:28.442658+0000 mon.a (mon.0) 860 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:11:29.334 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:29.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:29.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:29.439 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:29.475 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:29.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:29.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:29.583 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:29.617 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:29.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:29.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:29.723 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:29.751 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:29.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:29.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:29.857 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:29.890 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:29.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:29.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:29.996 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:30.027 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:30.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:30.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:30.131 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:30.165 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:30.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:30.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:30.271 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:30.293 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:30.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:30.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:30.396 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:30.429 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:30.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:30.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:30.533 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:30.568 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:30.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:30.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:30.671 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:30.706 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:30.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:30.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:30.812 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:30.846 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:30.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:30.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:30.950 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:30.974 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:30.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:30.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:31.077 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:31.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:31.114 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:31.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:31.215 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:31.224 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:30 smithi133 bash[102858]: cluster 2023-10-09T05:11:29.799179+0000 mgr.y (mgr.25277) 688 : cluster 0 pgmap v657: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 15772/209373 objects degraded (7.533%); 3868/209373 objects misplaced (1.847%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T05:11:31.224 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:30 smithi133 bash[105589]: cluster 2023-10-09T05:11:29.799179+0000 mgr.y (mgr.25277) 688 : cluster 0 pgmap v657: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 15772/209373 objects degraded (7.533%); 3868/209373 objects misplaced (1.847%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T05:11:31.242 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:31.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:31.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:31.346 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:31.355 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:30 smithi194 bash[92666]: cluster 2023-10-09T05:11:29.799179+0000 mgr.y (mgr.25277) 688 : cluster 0 pgmap v657: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 15772/209373 objects degraded (7.533%); 3868/209373 objects misplaced (1.847%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T05:11:31.381 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:31.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:31.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:31.484 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:31.516 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:31.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:31.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:31.620 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:31.660 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:31.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:31.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:31.766 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:31.794 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:31.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:31.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:31.898 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:31.930 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:31.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:31.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:32.033 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:32.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:31 smithi133 bash[102858]: audit 2023-10-09T05:11:31.893751+0000 mon.a (mon.0) 861 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:11:32.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:32 smithi133 bash[105589]: audit 2023-10-09T05:11:31.893751+0000 mon.a (mon.0) 861 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:11:32.073 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:32.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:32.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:32.177 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:32.208 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:32.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:32.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:32.312 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:32.329 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:32.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:32.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:32.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:32 smithi194 bash[92666]: audit 2023-10-09T05:11:31.893751+0000 mon.a (mon.0) 861 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:11:32.431 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:32.461 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:32.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:32.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:32.564 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:32.590 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:32.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:32.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:32.697 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:32.720 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:32.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:32.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:32.823 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:32.861 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:32.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:32.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:32.964 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:32.993 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:32.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:32.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:33.098 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:33.128 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:33.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:33.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:33.233 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:33.258 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:33.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:33.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:33.313 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:33 smithi133 bash[102858]: cluster 2023-10-09T05:11:31.799859+0000 mgr.y (mgr.25277) 689 : cluster 0 pgmap v658: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 15772/206112 objects degraded (7.652%); 3868/206112 objects misplaced (1.877%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T05:11:33.313 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:33 smithi133 bash[105589]: cluster 2023-10-09T05:11:31.799859+0000 mgr.y (mgr.25277) 689 : cluster 0 pgmap v658: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 15772/206112 objects degraded (7.652%); 3868/206112 objects misplaced (1.877%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T05:11:33.363 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:33.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:33 smithi194 bash[92666]: cluster 2023-10-09T05:11:31.799859+0000 mgr.y (mgr.25277) 689 : cluster 0 pgmap v658: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 15772/206112 objects degraded (7.652%); 3868/206112 objects misplaced (1.877%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T05:11:33.386 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:33.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:33.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:33.489 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:33.521 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:33.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:33.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:33.625 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:33.653 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:33.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:33.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:33.759 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:33.784 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:33.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:33.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:33.889 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:34.146 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:34 smithi194 bash[92666]: cluster 2023-10-09T05:11:33.264147+0000 mon.a (mon.0) 862 : cluster 3 Health check update: Degraded data redundancy: 15772/206112 objects degraded (7.652%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:34.147 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:34 smithi133 bash[105589]: cluster 2023-10-09T05:11:33.264147+0000 mon.a (mon.0) 862 : cluster 3 Health check update: Degraded data redundancy: 15772/206112 objects degraded (7.652%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:34.147 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:34 smithi133 bash[102858]: cluster 2023-10-09T05:11:33.264147+0000 mon.a (mon.0) 862 : cluster 3 Health check update: Degraded data redundancy: 15772/206112 objects degraded (7.652%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:34.176 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:34.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:34.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:34.282 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:34.312 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:34.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:34.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:34.414 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:34.447 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:34.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:34.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:34.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:11:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:11:34] "GET /metrics HTTP/1.1" 200 36432 "" "Prometheus/2.43.0" 2023-10-09T05:11:34.550 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:34.579 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:34.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:34.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:34.685 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:34.709 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:34.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:34.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:34.813 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:34.840 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:34.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:34.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:34.944 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:34.974 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:34.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:34.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:35.081 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:35.094 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:35 smithi133 bash[105589]: cluster 2023-10-09T05:11:33.800493+0000 mgr.y (mgr.25277) 690 : cluster 0 pgmap v659: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 15772/203592 objects degraded (7.747%); 3868/203592 objects misplaced (1.900%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T05:11:35.095 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:35 smithi133 bash[102858]: cluster 2023-10-09T05:11:33.800493+0000 mgr.y (mgr.25277) 690 : cluster 0 pgmap v659: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 15772/203592 objects degraded (7.747%); 3868/203592 objects misplaced (1.900%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T05:11:35.113 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:35.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:35.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:35.218 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:35.228 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:35 smithi194 bash[92666]: cluster 2023-10-09T05:11:33.800493+0000 mgr.y (mgr.25277) 690 : cluster 0 pgmap v659: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 15772/203592 objects degraded (7.747%); 3868/203592 objects misplaced (1.900%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T05:11:35.253 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:35.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:35.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:35.357 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:35.390 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:35.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:35.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:35.497 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:35.523 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:35.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:35.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:35.626 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:35.667 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:35.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:35.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:35.769 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:35.804 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:35.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:35.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:35.915 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:35.941 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:35.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:35.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:36.047 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:36.076 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:36.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:36.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:36.181 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:36.214 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:36.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:36.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:36.320 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:36.355 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:36.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:36.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:36.461 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:36.491 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:36.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:36.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:36.598 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:36.633 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:36.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:36.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:36.735 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:36.770 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:36.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:36.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:36.874 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:36.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:36.922 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:36.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:37.023 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:37.050 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:37.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:37.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:37.155 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:37.165 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:37 smithi133 bash[102858]: cluster 2023-10-09T05:11:35.802890+0000 mgr.y (mgr.25277) 691 : cluster 0 pgmap v660: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 15772/199008 objects degraded (7.925%); 3645/199008 objects misplaced (1.832%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T05:11:37.166 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:37 smithi133 bash[105589]: cluster 2023-10-09T05:11:35.802890+0000 mgr.y (mgr.25277) 691 : cluster 0 pgmap v660: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 15772/199008 objects degraded (7.925%); 3645/199008 objects misplaced (1.832%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T05:11:37.185 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:37.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:37.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:37.291 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:37.325 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:37 smithi194 bash[92666]: cluster 2023-10-09T05:11:35.802890+0000 mgr.y (mgr.25277) 691 : cluster 0 pgmap v660: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 15772/199008 objects degraded (7.925%); 3645/199008 objects misplaced (1.832%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T05:11:37.343 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:37.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:37.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:37.446 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:37.482 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:37.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:37.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:37.589 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:37.613 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:37.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:37.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:37.721 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:37.751 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:37.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:37.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:37.854 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:37.881 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:37.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:37.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:37.990 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:38.023 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:38.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:38.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:38.127 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:38.152 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:38.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:38.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:38.255 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:38.286 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:38.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:38.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:38.392 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:38.431 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:38.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:38.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:38.534 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:38.559 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:38.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:38.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:38.664 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:38.696 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:38.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:38.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:38.808 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:38.837 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:38.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:38.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:38.941 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:39.003 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:39.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:39.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:39.107 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:39.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:39 smithi194 bash[92666]: cluster 2023-10-09T05:11:37.803659+0000 mgr.y (mgr.25277) 692 : cluster 0 pgmap v661: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 15772/195747 objects degraded (8.057%); 3645/195747 objects misplaced (1.862%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:11:39.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:39 smithi194 bash[92666]: cluster 2023-10-09T05:11:38.265372+0000 mon.a (mon.0) 863 : cluster 3 Health check update: Degraded data redundancy: 15772/195747 objects degraded (8.057%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:39.151 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:39.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:39.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:39.256 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:39.265 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:39 smithi133 bash[102858]: cluster 2023-10-09T05:11:37.803659+0000 mgr.y (mgr.25277) 692 : cluster 0 pgmap v661: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 15772/195747 objects degraded (8.057%); 3645/195747 objects misplaced (1.862%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:11:39.266 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:39 smithi133 bash[102858]: cluster 2023-10-09T05:11:38.265372+0000 mon.a (mon.0) 863 : cluster 3 Health check update: Degraded data redundancy: 15772/195747 objects degraded (8.057%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:39.266 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:39 smithi133 bash[105589]: cluster 2023-10-09T05:11:37.803659+0000 mgr.y (mgr.25277) 692 : cluster 0 pgmap v661: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 15772/195747 objects degraded (8.057%); 3645/195747 objects misplaced (1.862%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:11:39.266 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:39 smithi133 bash[105589]: cluster 2023-10-09T05:11:38.265372+0000 mon.a (mon.0) 863 : cluster 3 Health check update: Degraded data redundancy: 15772/195747 objects degraded (8.057%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:39.295 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:39.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:39.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:39.407 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:39.431 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:39.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:39.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:39.540 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:39.585 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:39.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:39.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:39.689 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:39.715 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:39.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:39.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:39.824 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:39.876 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:39.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:39.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:39.998 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:40.029 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:40.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:40.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:40.120 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:40 smithi133 bash[102858]: audit 2023-10-09T05:11:39.116431+0000 mon.a (mon.0) 864 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:11:40.137 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:40.186 INFO:teuthology.orchestra.run.smithi133.stderr:set pool 9 pg_num to 26 2023-10-09T05:11:40.202 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:40.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:40.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:40.245 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:11:40.311 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:11:40.311 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:40.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:40.343 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:40.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:40.368 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:11:40.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:40 smithi194 bash[92666]: audit 2023-10-09T05:11:39.116431+0000 mon.a (mon.0) 864 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:11:40.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:40 smithi194 bash[92666]: audit 2023-10-09T05:11:39.134548+0000 mon.a (mon.0) 865 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:11:40.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:40 smithi194 bash[92666]: audit 2023-10-09T05:11:39.674367+0000 mon.b (mon.1) 12 : audit 1 from='client.? 172.21.15.133:0/2588781071' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T05:11:40.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:40 smithi194 bash[92666]: audit 2023-10-09T05:11:39.674748+0000 mon.a (mon.0) 866 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T05:11:40.424 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:11:40.444 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:40.454 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:40 smithi133 bash[102858]: audit 2023-10-09T05:11:39.134548+0000 mon.a (mon.0) 865 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:11:40.454 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:40 smithi133 bash[102858]: audit 2023-10-09T05:11:39.674367+0000 mon.b (mon.1) 12 : audit 1 from='client.? 172.21.15.133:0/2588781071' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T05:11:40.454 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:40 smithi133 bash[102858]: audit 2023-10-09T05:11:39.674748+0000 mon.a (mon.0) 866 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T05:11:40.455 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:40 smithi133 bash[105589]: audit 2023-10-09T05:11:39.116431+0000 mon.a (mon.0) 864 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:11:40.455 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:40 smithi133 bash[105589]: audit 2023-10-09T05:11:39.134548+0000 mon.a (mon.0) 865 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:11:40.455 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:40 smithi133 bash[105589]: audit 2023-10-09T05:11:39.674367+0000 mon.b (mon.1) 12 : audit 1 from='client.? 172.21.15.133:0/2588781071' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T05:11:40.455 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:40 smithi133 bash[105589]: audit 2023-10-09T05:11:39.674748+0000 mon.a (mon.0) 866 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T05:11:40.458 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:11:40.482 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:40.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:40.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:40.519 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:11:40.577 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:11:40.587 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:40.630 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:40.634 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:11:40.635 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:11:40.636 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:11:40.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:40.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:40.638 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:11:40.638 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:11:40.638 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:11:40.639 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:11:40.639 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:11:40.639 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:11:40.639 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:11:40.639 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:11:40.738 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:40.780 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:40.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:40.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:40.884 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:40.917 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:40.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:40.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:41.021 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:41.052 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:41.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:41.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:41.158 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:41.182 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:41.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:41.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:41.289 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:41.298 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:41 smithi194 bash[92666]: cluster 2023-10-09T05:11:39.804450+0000 mgr.y (mgr.25277) 693 : cluster 0 pgmap v662: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 15772/188820 objects degraded (8.353%); 3424/188820 objects misplaced (1.813%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T05:11:41.298 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:41 smithi194 bash[92666]: cluster 2023-10-09T05:11:40.139517+0000 osd.7 (osd.7) 79 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T05:11:41.298 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:41 smithi194 bash[92666]: cluster 2023-10-09T05:11:40.147128+0000 osd.5 (osd.5) 133 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T05:11:41.298 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:41 smithi194 bash[92666]: audit 2023-10-09T05:11:40.154980+0000 mon.a (mon.0) 867 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-10-09T05:11:41.298 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:41 smithi194 bash[92666]: cluster 2023-10-09T05:11:40.155090+0000 mon.a (mon.0) 868 : cluster 0 osdmap e159: 8 total, 8 up, 8 in 2023-10-09T05:11:41.298 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:41 smithi194 bash[92666]: cluster 2023-10-09T05:11:40.181447+0000 osd.4 (osd.4) 98 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T05:11:41.298 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:41 smithi194 bash[92666]: audit 2023-10-09T05:11:40.181953+0000 mon.b (mon.1) 13 : audit 0 from='client.? 172.21.15.133:0/1631753000' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:11:41.298 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:41 smithi194 bash[92666]: cluster 2023-10-09T05:11:40.184167+0000 osd.4 (osd.4) 99 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T05:11:41.298 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:41 smithi194 bash[92666]: audit 2023-10-09T05:11:40.241634+0000 mon.a (mon.0) 869 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:11:41.299 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:41 smithi194 bash[92666]: audit 2023-10-09T05:11:40.248709+0000 mon.a (mon.0) 870 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:11:41.299 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:41 smithi194 bash[92666]: audit 2023-10-09T05:11:40.885496+0000 mon.a (mon.0) 871 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:11:41.299 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:41 smithi194 bash[92666]: audit 2023-10-09T05:11:40.886973+0000 mon.a (mon.0) 872 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:11:41.299 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:41 smithi194 bash[92666]: audit 2023-10-09T05:11:40.903280+0000 mon.a (mon.0) 873 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:11:41.325 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:41.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:41.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:41.428 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:41.453 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:41.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:41.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:41.464 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:41 smithi133 bash[105589]: cluster 2023-10-09T05:11:39.804450+0000 mgr.y (mgr.25277) 693 : cluster 0 pgmap v662: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 15772/188820 objects degraded (8.353%); 3424/188820 objects misplaced (1.813%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T05:11:41.464 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:41 smithi133 bash[105589]: cluster 2023-10-09T05:11:40.139517+0000 osd.7 (osd.7) 79 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T05:11:41.464 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:41 smithi133 bash[105589]: cluster 2023-10-09T05:11:40.147128+0000 osd.5 (osd.5) 133 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T05:11:41.464 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:41 smithi133 bash[105589]: audit 2023-10-09T05:11:40.154980+0000 mon.a (mon.0) 867 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-10-09T05:11:41.464 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:41 smithi133 bash[105589]: cluster 2023-10-09T05:11:40.155090+0000 mon.a (mon.0) 868 : cluster 0 osdmap e159: 8 total, 8 up, 8 in 2023-10-09T05:11:41.464 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:41 smithi133 bash[105589]: cluster 2023-10-09T05:11:40.181447+0000 osd.4 (osd.4) 98 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T05:11:41.464 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:41 smithi133 bash[105589]: audit 2023-10-09T05:11:40.181953+0000 mon.b (mon.1) 13 : audit 0 from='client.? 172.21.15.133:0/1631753000' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:11:41.464 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:41 smithi133 bash[105589]: cluster 2023-10-09T05:11:40.184167+0000 osd.4 (osd.4) 99 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T05:11:41.464 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:41 smithi133 bash[105589]: audit 2023-10-09T05:11:40.241634+0000 mon.a (mon.0) 869 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:11:41.464 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:41 smithi133 bash[105589]: audit 2023-10-09T05:11:40.248709+0000 mon.a (mon.0) 870 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:11:41.465 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:41 smithi133 bash[105589]: audit 2023-10-09T05:11:40.885496+0000 mon.a (mon.0) 871 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:11:41.465 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:41 smithi133 bash[105589]: audit 2023-10-09T05:11:40.886973+0000 mon.a (mon.0) 872 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:11:41.465 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:41 smithi133 bash[105589]: audit 2023-10-09T05:11:40.903280+0000 mon.a (mon.0) 873 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:11:41.465 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:41 smithi133 bash[102858]: cluster 2023-10-09T05:11:39.804450+0000 mgr.y (mgr.25277) 693 : cluster 0 pgmap v662: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 15772/188820 objects degraded (8.353%); 3424/188820 objects misplaced (1.813%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T05:11:41.465 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:41 smithi133 bash[102858]: cluster 2023-10-09T05:11:40.139517+0000 osd.7 (osd.7) 79 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T05:11:41.465 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:41 smithi133 bash[102858]: cluster 2023-10-09T05:11:40.147128+0000 osd.5 (osd.5) 133 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T05:11:41.465 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:41 smithi133 bash[102858]: audit 2023-10-09T05:11:40.154980+0000 mon.a (mon.0) 867 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-10-09T05:11:41.465 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:41 smithi133 bash[102858]: cluster 2023-10-09T05:11:40.155090+0000 mon.a (mon.0) 868 : cluster 0 osdmap e159: 8 total, 8 up, 8 in 2023-10-09T05:11:41.465 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:41 smithi133 bash[102858]: cluster 2023-10-09T05:11:40.181447+0000 osd.4 (osd.4) 98 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T05:11:41.466 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:41 smithi133 bash[102858]: audit 2023-10-09T05:11:40.181953+0000 mon.b (mon.1) 13 : audit 0 from='client.? 172.21.15.133:0/1631753000' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:11:41.466 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:41 smithi133 bash[102858]: cluster 2023-10-09T05:11:40.184167+0000 osd.4 (osd.4) 99 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T05:11:41.466 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:41 smithi133 bash[102858]: audit 2023-10-09T05:11:40.241634+0000 mon.a (mon.0) 869 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:11:41.466 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:41 smithi133 bash[102858]: audit 2023-10-09T05:11:40.248709+0000 mon.a (mon.0) 870 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:11:41.466 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:41 smithi133 bash[102858]: audit 2023-10-09T05:11:40.885496+0000 mon.a (mon.0) 871 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:11:41.466 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:41 smithi133 bash[102858]: audit 2023-10-09T05:11:40.886973+0000 mon.a (mon.0) 872 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:11:41.466 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:41 smithi133 bash[102858]: audit 2023-10-09T05:11:40.903280+0000 mon.a (mon.0) 873 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:11:41.558 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:41.585 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:41.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:41.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:41.689 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:41.715 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:41.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:41.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:41.818 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:41.862 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:41.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:41.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:41.966 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:41.991 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:41.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:41.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:42.094 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:42.122 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:42.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:42.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:42.183 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is unset 2023-10-09T05:11:42.233 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:42.244 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:42 smithi194 bash[92666]: audit 2023-10-09T05:11:41.231583+0000 mon.b (mon.1) 14 : audit 1 from='client.? 172.21.15.133:0/2590377449' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:11:42.244 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:42 smithi194 bash[92666]: audit 2023-10-09T05:11:41.231827+0000 mon.a (mon.0) 874 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:11:42.244 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:42 smithi194 bash[92666]: audit 2023-10-09T05:11:41.806460+0000 mon.a (mon.0) 875 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T05:11:42.286 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:42.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:42.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:42.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:42 smithi133 bash[102858]: audit 2023-10-09T05:11:41.231583+0000 mon.b (mon.1) 14 : audit 1 from='client.? 172.21.15.133:0/2590377449' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:11:42.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:42 smithi133 bash[102858]: audit 2023-10-09T05:11:41.231827+0000 mon.a (mon.0) 874 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:11:42.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:42 smithi133 bash[102858]: audit 2023-10-09T05:11:41.806460+0000 mon.a (mon.0) 875 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T05:11:42.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:42 smithi133 bash[105589]: audit 2023-10-09T05:11:41.231583+0000 mon.b (mon.1) 14 : audit 1 from='client.? 172.21.15.133:0/2590377449' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:11:42.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:42 smithi133 bash[105589]: audit 2023-10-09T05:11:41.231827+0000 mon.a (mon.0) 874 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:11:42.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:42 smithi133 bash[105589]: audit 2023-10-09T05:11:41.806460+0000 mon.a (mon.0) 875 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T05:11:42.397 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:42.439 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:42.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:42.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:42.546 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:42.581 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:42.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:42.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:42.690 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:42.719 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:42.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:42.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:42.821 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:42.842 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:42.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:42.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:42.946 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:42.969 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:42.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:42.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:43.072 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:43.097 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:43.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:43.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:43.200 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:43.226 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:43.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:43.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:43.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:43 smithi133 bash[105589]: cluster 2023-10-09T05:11:41.805200+0000 mgr.y (mgr.25277) 694 : cluster 0 pgmap v664: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 15772/185976 objects degraded (8.481%); 3424/185976 objects misplaced (1.841%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T05:11:43.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:43 smithi133 bash[105589]: cluster 2023-10-09T05:11:42.140535+0000 mon.a (mon.0) 876 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:11:43.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:43 smithi133 bash[105589]: audit 2023-10-09T05:11:42.163500+0000 mon.a (mon.0) 877 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:11:43.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:43 smithi133 bash[105589]: audit 2023-10-09T05:11:42.163591+0000 mon.a (mon.0) 878 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T05:11:43.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:43 smithi133 bash[105589]: cluster 2023-10-09T05:11:42.163645+0000 mon.a (mon.0) 879 : cluster 0 osdmap e160: 8 total, 8 up, 8 in 2023-10-09T05:11:43.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:43 smithi133 bash[105589]: cluster 2023-10-09T05:11:42.174636+0000 osd.5 (osd.5) 134 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T05:11:43.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:43 smithi133 bash[105589]: cluster 2023-10-09T05:11:42.178780+0000 osd.4 (osd.4) 100 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T05:11:43.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:43 smithi133 bash[105589]: cluster 2023-10-09T05:11:42.179345+0000 osd.7 (osd.7) 80 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T05:11:43.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:43 smithi133 bash[102858]: cluster 2023-10-09T05:11:41.805200+0000 mgr.y (mgr.25277) 694 : cluster 0 pgmap v664: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 15772/185976 objects degraded (8.481%); 3424/185976 objects misplaced (1.841%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T05:11:43.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:43 smithi133 bash[102858]: cluster 2023-10-09T05:11:42.140535+0000 mon.a (mon.0) 876 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:11:43.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:43 smithi133 bash[102858]: audit 2023-10-09T05:11:42.163500+0000 mon.a (mon.0) 877 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:11:43.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:43 smithi133 bash[102858]: audit 2023-10-09T05:11:42.163591+0000 mon.a (mon.0) 878 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T05:11:43.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:43 smithi133 bash[102858]: cluster 2023-10-09T05:11:42.163645+0000 mon.a (mon.0) 879 : cluster 0 osdmap e160: 8 total, 8 up, 8 in 2023-10-09T05:11:43.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:43 smithi133 bash[102858]: cluster 2023-10-09T05:11:42.174636+0000 osd.5 (osd.5) 134 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T05:11:43.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:43 smithi133 bash[102858]: cluster 2023-10-09T05:11:42.178780+0000 osd.4 (osd.4) 100 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T05:11:43.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:43 smithi133 bash[102858]: cluster 2023-10-09T05:11:42.179345+0000 osd.7 (osd.7) 80 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T05:11:43.329 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:43.355 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:43.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:43.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:43.461 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:43.486 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:43.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:43.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:43.588 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:43.595 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:43 smithi194 bash[92666]: cluster 2023-10-09T05:11:41.805200+0000 mgr.y (mgr.25277) 694 : cluster 0 pgmap v664: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 15772/185976 objects degraded (8.481%); 3424/185976 objects misplaced (1.841%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T05:11:43.595 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:43 smithi194 bash[92666]: cluster 2023-10-09T05:11:42.140535+0000 mon.a (mon.0) 876 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:11:43.595 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:43 smithi194 bash[92666]: audit 2023-10-09T05:11:42.163500+0000 mon.a (mon.0) 877 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:11:43.595 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:43 smithi194 bash[92666]: audit 2023-10-09T05:11:42.163591+0000 mon.a (mon.0) 878 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T05:11:43.595 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:43 smithi194 bash[92666]: cluster 2023-10-09T05:11:42.163645+0000 mon.a (mon.0) 879 : cluster 0 osdmap e160: 8 total, 8 up, 8 in 2023-10-09T05:11:43.595 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:43 smithi194 bash[92666]: cluster 2023-10-09T05:11:42.174636+0000 osd.5 (osd.5) 134 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T05:11:43.595 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:43 smithi194 bash[92666]: cluster 2023-10-09T05:11:42.178780+0000 osd.4 (osd.4) 100 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T05:11:43.596 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:43 smithi194 bash[92666]: cluster 2023-10-09T05:11:42.179345+0000 osd.7 (osd.7) 80 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T05:11:43.613 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:43.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:43.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:43.717 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:43.745 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:43.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:43.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:43.848 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:43.877 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:43.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:43.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:43.981 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:44.006 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:44.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:44.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:44.110 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:44.140 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:44.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:44.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:44.244 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:44.271 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:44.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:44.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:44.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:44 smithi133 bash[102858]: cluster 2023-10-09T05:11:43.159812+0000 osd.4 (osd.4) 101 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T05:11:44.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:44 smithi133 bash[102858]: cluster 2023-10-09T05:11:43.161087+0000 mon.a (mon.0) 880 : cluster 0 osdmap e161: 8 total, 8 up, 8 in 2023-10-09T05:11:44.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:44 smithi133 bash[105589]: cluster 2023-10-09T05:11:43.159812+0000 osd.4 (osd.4) 101 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T05:11:44.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:44 smithi133 bash[105589]: cluster 2023-10-09T05:11:43.161087+0000 mon.a (mon.0) 880 : cluster 0 osdmap e161: 8 total, 8 up, 8 in 2023-10-09T05:11:44.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:44 smithi133 bash[105589]: cluster 2023-10-09T05:11:43.161724+0000 osd.7 (osd.7) 81 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T05:11:44.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:44 smithi133 bash[105589]: cluster 2023-10-09T05:11:43.164231+0000 osd.5 (osd.5) 135 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T05:11:44.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:44 smithi133 bash[105589]: cluster 2023-10-09T05:11:43.164328+0000 osd.6 (osd.6) 85 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T05:11:44.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:44 smithi133 bash[105589]: cluster 2023-10-09T05:11:43.166049+0000 osd.5 (osd.5) 136 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T05:11:44.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:44 smithi133 bash[105589]: cluster 2023-10-09T05:11:43.167342+0000 osd.7 (osd.7) 82 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T05:11:44.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:44 smithi133 bash[105589]: cluster 2023-10-09T05:11:43.168042+0000 osd.5 (osd.5) 137 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T05:11:44.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:44 smithi133 bash[105589]: cluster 2023-10-09T05:11:43.169147+0000 osd.7 (osd.7) 83 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T05:11:44.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:44 smithi133 bash[105589]: cluster 2023-10-09T05:11:43.266322+0000 mon.a (mon.0) 881 : cluster 3 Health check update: Degraded data redundancy: 15772/185976 objects degraded (8.481%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:44.304 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:44 smithi133 bash[105589]: audit 2023-10-09T05:11:43.442621+0000 mon.a (mon.0) 882 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:11:44.381 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:44.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:44 smithi194 bash[92666]: cluster 2023-10-09T05:11:43.159812+0000 osd.4 (osd.4) 101 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T05:11:44.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:44 smithi194 bash[92666]: cluster 2023-10-09T05:11:43.161087+0000 mon.a (mon.0) 880 : cluster 0 osdmap e161: 8 total, 8 up, 8 in 2023-10-09T05:11:44.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:44 smithi194 bash[92666]: cluster 2023-10-09T05:11:43.161724+0000 osd.7 (osd.7) 81 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T05:11:44.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:44 smithi194 bash[92666]: cluster 2023-10-09T05:11:43.164231+0000 osd.5 (osd.5) 135 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T05:11:44.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:44 smithi194 bash[92666]: cluster 2023-10-09T05:11:43.164328+0000 osd.6 (osd.6) 85 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T05:11:44.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:44 smithi194 bash[92666]: cluster 2023-10-09T05:11:43.166049+0000 osd.5 (osd.5) 136 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T05:11:44.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:44 smithi194 bash[92666]: cluster 2023-10-09T05:11:43.167342+0000 osd.7 (osd.7) 82 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T05:11:44.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:44 smithi194 bash[92666]: cluster 2023-10-09T05:11:43.168042+0000 osd.5 (osd.5) 137 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T05:11:44.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:44 smithi194 bash[92666]: cluster 2023-10-09T05:11:43.169147+0000 osd.7 (osd.7) 83 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T05:11:44.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:44 smithi194 bash[92666]: cluster 2023-10-09T05:11:43.266322+0000 mon.a (mon.0) 881 : cluster 3 Health check update: Degraded data redundancy: 15772/185976 objects degraded (8.481%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:44.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:44 smithi194 bash[92666]: audit 2023-10-09T05:11:43.442621+0000 mon.a (mon.0) 882 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:11:44.403 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:44.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:44.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:44.507 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:44.543 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:44.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:44.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:44.651 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:44.660 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:44 smithi133 bash[102858]: cluster 2023-10-09T05:11:43.161724+0000 osd.7 (osd.7) 81 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T05:11:44.660 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:44 smithi133 bash[102858]: cluster 2023-10-09T05:11:43.164231+0000 osd.5 (osd.5) 135 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T05:11:44.660 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:44 smithi133 bash[102858]: cluster 2023-10-09T05:11:43.164328+0000 osd.6 (osd.6) 85 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T05:11:44.660 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:44 smithi133 bash[102858]: cluster 2023-10-09T05:11:43.166049+0000 osd.5 (osd.5) 136 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T05:11:44.660 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:44 smithi133 bash[102858]: cluster 2023-10-09T05:11:43.167342+0000 osd.7 (osd.7) 82 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T05:11:44.660 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:44 smithi133 bash[102858]: cluster 2023-10-09T05:11:43.168042+0000 osd.5 (osd.5) 137 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T05:11:44.660 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:44 smithi133 bash[102858]: cluster 2023-10-09T05:11:43.169147+0000 osd.7 (osd.7) 83 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T05:11:44.660 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:44 smithi133 bash[102858]: cluster 2023-10-09T05:11:43.266322+0000 mon.a (mon.0) 881 : cluster 3 Health check update: Degraded data redundancy: 15772/185976 objects degraded (8.481%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:44.660 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:44 smithi133 bash[102858]: audit 2023-10-09T05:11:43.442621+0000 mon.a (mon.0) 882 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:11:44.661 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:11:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:11:44] "GET /metrics HTTP/1.1" 200 36434 "" "Prometheus/2.43.0" 2023-10-09T05:11:44.675 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:44.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:44.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:44.778 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:44.803 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:44.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:44.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:44.907 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:44.937 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:44.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:44.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:45.043 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:45.075 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:45.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:45.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:45.170 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:45 smithi133 bash[102858]: cluster 2023-10-09T05:11:43.805843+0000 mgr.y (mgr.25277) 695 : cluster 0 pgmap v667: 99 pgs: 1 peering, 7 unknown, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 14506/184305 objects degraded (7.871%); 3424/184305 objects misplaced (1.858%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T05:11:45.170 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:45 smithi133 bash[102858]: cluster 2023-10-09T05:11:44.168095+0000 osd.5 (osd.5) 138 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T05:11:45.177 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T05:11:45.179 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:45.204 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:45.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:45.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:45.308 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:45.338 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:45.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:45.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:45.448 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:45.456 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:45 smithi194 bash[92666]: cluster 2023-10-09T05:11:43.805843+0000 mgr.y (mgr.25277) 695 : cluster 0 pgmap v667: 99 pgs: 1 peering, 7 unknown, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 14506/184305 objects degraded (7.871%); 3424/184305 objects misplaced (1.858%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T05:11:45.456 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:45 smithi194 bash[92666]: cluster 2023-10-09T05:11:44.168095+0000 osd.5 (osd.5) 138 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T05:11:45.456 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:45 smithi194 bash[92666]: cluster 2023-10-09T05:11:44.169537+0000 osd.7 (osd.7) 84 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T05:11:45.456 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:45 smithi194 bash[92666]: cluster 2023-10-09T05:11:44.170049+0000 mon.a (mon.0) 883 : cluster 0 osdmap e162: 8 total, 8 up, 8 in 2023-10-09T05:11:45.456 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:45 smithi194 bash[92666]: cluster 2023-10-09T05:11:44.172131+0000 osd.5 (osd.5) 139 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T05:11:45.456 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:45 smithi194 bash[92666]: cluster 2023-10-09T05:11:44.172164+0000 osd.7 (osd.7) 85 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T05:11:45.456 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:45 smithi194 bash[92666]: cluster 2023-10-09T05:11:44.173741+0000 osd.5 (osd.5) 140 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T05:11:45.456 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:45 smithi194 bash[92666]: cluster 2023-10-09T05:11:44.174509+0000 osd.7 (osd.7) 86 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T05:11:45.456 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:45 smithi194 bash[92666]: cluster 2023-10-09T05:11:44.174915+0000 osd.5 (osd.5) 141 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T05:11:45.456 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:45 smithi194 bash[92666]: cluster 2023-10-09T05:11:44.202523+0000 osd.4 (osd.4) 102 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T05:11:45.470 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:45.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:45.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:45.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:45 smithi133 bash[102858]: cluster 2023-10-09T05:11:44.169537+0000 osd.7 (osd.7) 84 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T05:11:45.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:45 smithi133 bash[102858]: cluster 2023-10-09T05:11:44.170049+0000 mon.a (mon.0) 883 : cluster 0 osdmap e162: 8 total, 8 up, 8 in 2023-10-09T05:11:45.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:45 smithi133 bash[102858]: cluster 2023-10-09T05:11:44.172131+0000 osd.5 (osd.5) 139 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T05:11:45.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:45 smithi133 bash[102858]: cluster 2023-10-09T05:11:44.172164+0000 osd.7 (osd.7) 85 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T05:11:45.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:45 smithi133 bash[102858]: cluster 2023-10-09T05:11:44.173741+0000 osd.5 (osd.5) 140 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T05:11:45.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:45 smithi133 bash[102858]: cluster 2023-10-09T05:11:44.174509+0000 osd.7 (osd.7) 86 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T05:11:45.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:45 smithi133 bash[102858]: cluster 2023-10-09T05:11:44.174915+0000 osd.5 (osd.5) 141 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T05:11:45.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:45 smithi133 bash[102858]: cluster 2023-10-09T05:11:44.202523+0000 osd.4 (osd.4) 102 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T05:11:45.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:45 smithi133 bash[105589]: cluster 2023-10-09T05:11:43.805843+0000 mgr.y (mgr.25277) 695 : cluster 0 pgmap v667: 99 pgs: 1 peering, 7 unknown, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 14506/184305 objects degraded (7.871%); 3424/184305 objects misplaced (1.858%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T05:11:45.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:45 smithi133 bash[105589]: cluster 2023-10-09T05:11:44.168095+0000 osd.5 (osd.5) 138 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T05:11:45.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:45 smithi133 bash[105589]: cluster 2023-10-09T05:11:44.169537+0000 osd.7 (osd.7) 84 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T05:11:45.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:45 smithi133 bash[105589]: cluster 2023-10-09T05:11:44.170049+0000 mon.a (mon.0) 883 : cluster 0 osdmap e162: 8 total, 8 up, 8 in 2023-10-09T05:11:45.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:45 smithi133 bash[105589]: cluster 2023-10-09T05:11:44.172131+0000 osd.5 (osd.5) 139 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T05:11:45.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:45 smithi133 bash[105589]: cluster 2023-10-09T05:11:44.172164+0000 osd.7 (osd.7) 85 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T05:11:45.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:45 smithi133 bash[105589]: cluster 2023-10-09T05:11:44.173741+0000 osd.5 (osd.5) 140 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T05:11:45.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:45 smithi133 bash[105589]: cluster 2023-10-09T05:11:44.174509+0000 osd.7 (osd.7) 86 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T05:11:45.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:45 smithi133 bash[105589]: cluster 2023-10-09T05:11:44.174915+0000 osd.5 (osd.5) 141 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T05:11:45.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:45 smithi133 bash[105589]: cluster 2023-10-09T05:11:44.202523+0000 osd.4 (osd.4) 102 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T05:11:45.573 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:45.602 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:45.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:45.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:45.706 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:45.736 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:45.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:45.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:45.844 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:45.875 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:45.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:45.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:45.980 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:46.009 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:46.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:46.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:46.113 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:46.136 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:46.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:46.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:46.239 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:46.268 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:46.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:46.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:46.371 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:46.417 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:46.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:46.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:46.520 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:46.546 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:46.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:46.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:46.656 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:46.686 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:46.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:46.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:46.793 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:46.819 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:46.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:46.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:46.911 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:11:46.924 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:46.995 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:46.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:46.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:47.100 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:47.125 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:47.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:47.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:47.176 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:47 smithi133 bash[102858]: cluster 2023-10-09T05:11:45.807598+0000 mgr.y (mgr.25277) 696 : cluster 0 pgmap v669: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 4 unknown, 5 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 412 op/s; 14479/181800 objects degraded (7.964%); 2990/181800 objects misplaced (1.645%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T05:11:47.177 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:47 smithi133 bash[105589]: cluster 2023-10-09T05:11:45.807598+0000 mgr.y (mgr.25277) 696 : cluster 0 pgmap v669: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 4 unknown, 5 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 412 op/s; 14479/181800 objects degraded (7.964%); 2990/181800 objects misplaced (1.645%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T05:11:47.232 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:47.259 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:11:47.260 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:47.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:47.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:47.364 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:47.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:47 smithi194 bash[92666]: cluster 2023-10-09T05:11:45.807598+0000 mgr.y (mgr.25277) 696 : cluster 0 pgmap v669: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 4 unknown, 5 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 412 op/s; 14479/181800 objects degraded (7.964%); 2990/181800 objects misplaced (1.645%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T05:11:47.394 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:47.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:47.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:47.498 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:47.527 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:47.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:47.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:47.630 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:47.663 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:47.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:47.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:47.768 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:47.802 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:47.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:47.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:47.907 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:47.932 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:47.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:47.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:48.038 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:48.066 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:48.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:48.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:48.172 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:48.213 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:48.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:48.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:48.319 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:48.350 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:48.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:48.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:48.455 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:48.482 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:48.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:48.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:48.586 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:48.613 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:48.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:48.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:48.717 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:48.746 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:48.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:48.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:48.849 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:48.876 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:48.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:48.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:48.979 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:49.010 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:49.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:49.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:49.029 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:11:49.030 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:11:49.030 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_4 2023-10-09T05:11:49.031 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:11:49.114 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:49.148 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:49.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:49.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:49.251 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:49.279 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:49.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:49.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:49.383 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:49.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:49 smithi194 bash[92666]: cluster 2023-10-09T05:11:47.808677+0000 mgr.y (mgr.25277) 697 : cluster 0 pgmap v670: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 6 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 356 op/s; 15765/179559 objects degraded (8.780%); 547/179559 objects misplaced (0.305%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T05:11:49.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:49 smithi194 bash[92666]: cluster 2023-10-09T05:11:48.187359+0000 osd.6 (osd.6) 86 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T05:11:49.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:49 smithi194 bash[92666]: cluster 2023-10-09T05:11:48.190054+0000 mon.a (mon.0) 884 : cluster 0 osdmap e163: 8 total, 8 up, 8 in 2023-10-09T05:11:49.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:49 smithi194 bash[92666]: cluster 2023-10-09T05:11:48.190587+0000 osd.7 (osd.7) 87 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T05:11:49.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:49 smithi194 bash[92666]: cluster 2023-10-09T05:11:48.193234+0000 osd.5 (osd.5) 142 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T05:11:49.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:49 smithi194 bash[92666]: cluster 2023-10-09T05:11:48.199441+0000 osd.4 (osd.4) 103 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T05:11:49.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:49 smithi194 bash[92666]: cluster 2023-10-09T05:11:48.267477+0000 mon.a (mon.0) 885 : cluster 3 Health check update: Degraded data redundancy: 15765/179559 objects degraded (8.780%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:49.416 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:49.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:49.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:49.522 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:49.546 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:49.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:49 smithi133 bash[105589]: cluster 2023-10-09T05:11:47.808677+0000 mgr.y (mgr.25277) 697 : cluster 0 pgmap v670: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 6 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 356 op/s; 15765/179559 objects degraded (8.780%); 547/179559 objects misplaced (0.305%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T05:11:49.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:49 smithi133 bash[105589]: cluster 2023-10-09T05:11:48.187359+0000 osd.6 (osd.6) 86 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T05:11:49.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:49 smithi133 bash[105589]: cluster 2023-10-09T05:11:48.190054+0000 mon.a (mon.0) 884 : cluster 0 osdmap e163: 8 total, 8 up, 8 in 2023-10-09T05:11:49.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:49 smithi133 bash[105589]: cluster 2023-10-09T05:11:48.190587+0000 osd.7 (osd.7) 87 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T05:11:49.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:49 smithi133 bash[105589]: cluster 2023-10-09T05:11:48.193234+0000 osd.5 (osd.5) 142 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T05:11:49.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:49 smithi133 bash[105589]: cluster 2023-10-09T05:11:48.199441+0000 osd.4 (osd.4) 103 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T05:11:49.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:49 smithi133 bash[105589]: cluster 2023-10-09T05:11:48.267477+0000 mon.a (mon.0) 885 : cluster 3 Health check update: Degraded data redundancy: 15765/179559 objects degraded (8.780%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:49.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:49 smithi133 bash[102858]: cluster 2023-10-09T05:11:47.808677+0000 mgr.y (mgr.25277) 697 : cluster 0 pgmap v670: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 6 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 356 op/s; 15765/179559 objects degraded (8.780%); 547/179559 objects misplaced (0.305%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T05:11:49.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:49 smithi133 bash[102858]: cluster 2023-10-09T05:11:48.187359+0000 osd.6 (osd.6) 86 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T05:11:49.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:49 smithi133 bash[102858]: cluster 2023-10-09T05:11:48.190054+0000 mon.a (mon.0) 884 : cluster 0 osdmap e163: 8 total, 8 up, 8 in 2023-10-09T05:11:49.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:49 smithi133 bash[102858]: cluster 2023-10-09T05:11:48.190587+0000 osd.7 (osd.7) 87 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T05:11:49.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:49 smithi133 bash[102858]: cluster 2023-10-09T05:11:48.193234+0000 osd.5 (osd.5) 142 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T05:11:49.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:49 smithi133 bash[102858]: cluster 2023-10-09T05:11:48.199441+0000 osd.4 (osd.4) 103 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T05:11:49.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:49 smithi133 bash[102858]: cluster 2023-10-09T05:11:48.267477+0000 mon.a (mon.0) 885 : cluster 3 Health check update: Degraded data redundancy: 15765/179559 objects degraded (8.780%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:49.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:49.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:49.653 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:49.684 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:49.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:49.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:49.790 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:49.824 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:49.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:49.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:49.930 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:49.961 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:49.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:49.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:50.036 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:11:50.064 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:50.095 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:50.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:50.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:50.201 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:50.222 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:50.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:50.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:50.327 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:50.336 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:50 smithi133 bash[102858]: cluster 2023-10-09T05:11:49.199147+0000 osd.7 (osd.7) 88 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T05:11:50.336 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:50 smithi133 bash[102858]: cluster 2023-10-09T05:11:49.201088+0000 mon.a (mon.0) 886 : cluster 0 osdmap e164: 8 total, 8 up, 8 in 2023-10-09T05:11:50.337 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:50 smithi133 bash[102858]: cluster 2023-10-09T05:11:49.203923+0000 osd.7 (osd.7) 89 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T05:11:50.337 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:50 smithi133 bash[102858]: cluster 2023-10-09T05:11:49.205900+0000 osd.7 (osd.7) 90 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T05:11:50.337 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:50 smithi133 bash[102858]: cluster 2023-10-09T05:11:49.209007+0000 osd.5 (osd.5) 143 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T05:11:50.337 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:50 smithi133 bash[102858]: cluster 2023-10-09T05:11:49.230847+0000 osd.6 (osd.6) 87 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T05:11:50.337 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:50 smithi133 bash[102858]: cluster 2023-10-09T05:11:49.809554+0000 mgr.y (mgr.25277) 698 : cluster 0 pgmap v673: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 15849/173181 objects degraded (9.152%); 797/173181 objects misplaced (0.460%); 17 MiB/s, 264 objects/s recovering 2023-10-09T05:11:50.337 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:50 smithi133 bash[105589]: cluster 2023-10-09T05:11:49.199147+0000 osd.7 (osd.7) 88 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T05:11:50.337 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:50 smithi133 bash[105589]: cluster 2023-10-09T05:11:49.201088+0000 mon.a (mon.0) 886 : cluster 0 osdmap e164: 8 total, 8 up, 8 in 2023-10-09T05:11:50.338 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:50 smithi133 bash[105589]: cluster 2023-10-09T05:11:49.203923+0000 osd.7 (osd.7) 89 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T05:11:50.338 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:50 smithi133 bash[105589]: cluster 2023-10-09T05:11:49.205900+0000 osd.7 (osd.7) 90 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T05:11:50.338 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:50 smithi133 bash[105589]: cluster 2023-10-09T05:11:49.209007+0000 osd.5 (osd.5) 143 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T05:11:50.338 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:50 smithi133 bash[105589]: cluster 2023-10-09T05:11:49.230847+0000 osd.6 (osd.6) 87 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T05:11:50.338 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:50 smithi133 bash[105589]: cluster 2023-10-09T05:11:49.809554+0000 mgr.y (mgr.25277) 698 : cluster 0 pgmap v673: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 15849/173181 objects degraded (9.152%); 797/173181 objects misplaced (0.460%); 17 MiB/s, 264 objects/s recovering 2023-10-09T05:11:50.356 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:50.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:50.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:50.464 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:50.497 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:50.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:50.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:50.601 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:50.611 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:50 smithi194 bash[92666]: cluster 2023-10-09T05:11:49.199147+0000 osd.7 (osd.7) 88 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T05:11:50.611 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:50 smithi194 bash[92666]: cluster 2023-10-09T05:11:49.201088+0000 mon.a (mon.0) 886 : cluster 0 osdmap e164: 8 total, 8 up, 8 in 2023-10-09T05:11:50.611 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:50 smithi194 bash[92666]: cluster 2023-10-09T05:11:49.203923+0000 osd.7 (osd.7) 89 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T05:11:50.611 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:50 smithi194 bash[92666]: cluster 2023-10-09T05:11:49.205900+0000 osd.7 (osd.7) 90 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T05:11:50.611 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:50 smithi194 bash[92666]: cluster 2023-10-09T05:11:49.209007+0000 osd.5 (osd.5) 143 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T05:11:50.611 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:50 smithi194 bash[92666]: cluster 2023-10-09T05:11:49.230847+0000 osd.6 (osd.6) 87 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T05:11:50.611 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:50 smithi194 bash[92666]: cluster 2023-10-09T05:11:49.809554+0000 mgr.y (mgr.25277) 698 : cluster 0 pgmap v673: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 15849/173181 objects degraded (9.152%); 797/173181 objects misplaced (0.460%); 17 MiB/s, 264 objects/s recovering 2023-10-09T05:11:50.631 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:50.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:50.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:50.735 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:50.760 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:50.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:50.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:50.865 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:50.892 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:50.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:50.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:50.996 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:51.022 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:51.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:51.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:51.058 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:11:51.127 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:51.164 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:51.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:51.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:51.268 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:51.299 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:51.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:51.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:51.405 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:51.434 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:51.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:51.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:51.538 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:51.564 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:51.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:51.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:51.667 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:51.692 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:51.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:51.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:51.796 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:51.818 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:51.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:51.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:51.922 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:51.948 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:51.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:51.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:52.052 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:52.080 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:11:52.088 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:52.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:52.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:52.196 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:52.232 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:52.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:52.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:52.338 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:52.366 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:52.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:52.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:52.473 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:52.502 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:52.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:52.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:52.609 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:52.638 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:52.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:52.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:52.746 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:52.770 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:52.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:52.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:52.874 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:52.902 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:52.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:52.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:53.007 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:53.033 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:53.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:53.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:53.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:52 smithi133 bash[105589]: cluster 2023-10-09T05:11:51.810263+0000 mgr.y (mgr.25277) 699 : cluster 0 pgmap v674: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 15722/170148 objects degraded (9.240%); 564/170148 objects misplaced (0.331%); 16 MiB/s, 255 objects/s recovering 2023-10-09T05:11:53.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:52 smithi133 bash[102858]: cluster 2023-10-09T05:11:51.810263+0000 mgr.y (mgr.25277) 699 : cluster 0 pgmap v674: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 15722/170148 objects degraded (9.240%); 564/170148 objects misplaced (0.331%); 16 MiB/s, 255 objects/s recovering 2023-10-09T05:11:53.093 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:11:53.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:52 smithi194 bash[92666]: cluster 2023-10-09T05:11:51.810263+0000 mgr.y (mgr.25277) 699 : cluster 0 pgmap v674: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 15722/170148 objects degraded (9.240%); 564/170148 objects misplaced (0.331%); 16 MiB/s, 255 objects/s recovering 2023-10-09T05:11:53.141 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:53.178 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:53.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:53.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:53.285 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:53.313 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:53.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:53.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:53.417 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:53.448 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:53.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:53.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:53.553 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:53.578 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:53.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:53.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:53.681 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:53.713 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:53.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:53.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:53.816 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:53.844 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:53.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:53.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:53.948 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:53.955 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:53 smithi194 bash[92666]: cluster 2023-10-09T05:11:53.268937+0000 mon.a (mon.0) 887 : cluster 3 Health check update: Degraded data redundancy: 15722/170148 objects degraded (9.240%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:53.971 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:53.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:53.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:54.074 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:54.083 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:53 smithi133 bash[105589]: cluster 2023-10-09T05:11:53.268937+0000 mon.a (mon.0) 887 : cluster 3 Health check update: Degraded data redundancy: 15722/170148 objects degraded (9.240%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:54.083 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:53 smithi133 bash[102858]: cluster 2023-10-09T05:11:53.268937+0000 mon.a (mon.0) 887 : cluster 3 Health check update: Degraded data redundancy: 15722/170148 objects degraded (9.240%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:54.107 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:54.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:54.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:54.211 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:54.240 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:54.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:54.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:54.344 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:54.353 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:11:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:11:54] "GET /metrics HTTP/1.1" 200 36434 "" "Prometheus/2.43.0" 2023-10-09T05:11:54.370 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:54.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:54.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:54.473 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:54.502 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:54.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:54.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:54.605 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:54.632 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:54.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:54.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:54.735 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:54.766 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:54.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:54.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:54.872 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:54.889 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:54.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:54.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:54.992 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:55.023 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:55.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:55.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:55.027 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:54 smithi133 bash[102858]: cluster 2023-10-09T05:11:53.810905+0000 mgr.y (mgr.25277) 700 : cluster 0 pgmap v675: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 15514/167517 objects degraded (9.261%); 564/167517 objects misplaced (0.337%); 13 MiB/s, 212 objects/s recovering 2023-10-09T05:11:55.028 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:54 smithi133 bash[105589]: cluster 2023-10-09T05:11:53.810905+0000 mgr.y (mgr.25277) 700 : cluster 0 pgmap v675: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 15514/167517 objects degraded (9.261%); 564/167517 objects misplaced (0.337%); 13 MiB/s, 212 objects/s recovering 2023-10-09T05:11:55.128 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:55.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:54 smithi194 bash[92666]: cluster 2023-10-09T05:11:53.810905+0000 mgr.y (mgr.25277) 700 : cluster 0 pgmap v675: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 15514/167517 objects degraded (9.261%); 564/167517 objects misplaced (0.337%); 13 MiB/s, 212 objects/s recovering 2023-10-09T05:11:55.157 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:55.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:55.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:55.261 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:55.288 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:55.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:55.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:55.393 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:55.420 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:55.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:55.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:55.526 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:55.550 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:55.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:55.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:55.653 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:55.687 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:55.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:55.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:55.794 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:55.831 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:55.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:55.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:55.935 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:55.974 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:55.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:55.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:56.077 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:56.102 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:56.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:56.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:56.205 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:56.242 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:56.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:56.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:56.351 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:56.389 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:56.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:56.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:56.495 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:56.540 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:11:56.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:56.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:11:56.647 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:56.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:56.678 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:56.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:56.779 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:56.809 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:56.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:56.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:56.878 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:56 smithi194 bash[92666]: cluster 2023-10-09T05:11:55.811722+0000 mgr.y (mgr.25277) 701 : cluster 0 pgmap v676: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 15019/160758 objects degraded (9.343%); 391/160758 objects misplaced (0.243%); 17 MiB/s, 271 objects/s recovering 2023-10-09T05:11:56.914 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:56.944 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:56.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:56.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:57.048 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:57.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:56 smithi133 bash[105589]: cluster 2023-10-09T05:11:55.811722+0000 mgr.y (mgr.25277) 701 : cluster 0 pgmap v676: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 15019/160758 objects degraded (9.343%); 391/160758 objects misplaced (0.243%); 17 MiB/s, 271 objects/s recovering 2023-10-09T05:11:57.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:56 smithi133 bash[102858]: cluster 2023-10-09T05:11:55.811722+0000 mgr.y (mgr.25277) 701 : cluster 0 pgmap v676: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 15019/160758 objects degraded (9.343%); 391/160758 objects misplaced (0.243%); 17 MiB/s, 271 objects/s recovering 2023-10-09T05:11:57.077 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:57.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:57.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:57.182 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:57.209 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:57.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:57.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:57.315 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:57.340 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:57.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:57.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:57.448 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:57.476 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:57.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:57.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:57.580 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:57.616 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:57.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:57.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:57.721 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:57.757 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:57.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:57.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:57.863 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:57.889 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:57.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:57.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:57.992 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:58.021 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:58.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:58.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:58.124 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:58.149 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:58.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:58.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:58.256 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:58.278 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:11:58.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:58.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:11:58.386 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:58.422 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:58.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:58.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:58.531 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:58.559 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:58.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:58.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:58.668 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:58.702 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:58.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:58.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:58.805 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:58.840 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:11:58.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:58.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:11:58.945 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:58.972 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:58.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:58.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:11:59.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:58 smithi133 bash[102858]: cluster 2023-10-09T05:11:57.812422+0000 mgr.y (mgr.25277) 702 : cluster 0 pgmap v677: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 14889/159243 objects degraded (9.350%); 346/159243 objects misplaced (0.217%); 15 MiB/s, 289 objects/s recovering 2023-10-09T05:11:59.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:58 smithi133 bash[102858]: cluster 2023-10-09T05:11:58.270059+0000 mon.a (mon.0) 888 : cluster 3 Health check update: Degraded data redundancy: 14889/159243 objects degraded (9.350%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:59.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:11:58 smithi133 bash[102858]: audit 2023-10-09T05:11:58.443126+0000 mon.a (mon.0) 889 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:11:59.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:58 smithi133 bash[105589]: cluster 2023-10-09T05:11:57.812422+0000 mgr.y (mgr.25277) 702 : cluster 0 pgmap v677: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 14889/159243 objects degraded (9.350%); 346/159243 objects misplaced (0.217%); 15 MiB/s, 289 objects/s recovering 2023-10-09T05:11:59.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:58 smithi133 bash[105589]: cluster 2023-10-09T05:11:58.270059+0000 mon.a (mon.0) 888 : cluster 3 Health check update: Degraded data redundancy: 14889/159243 objects degraded (9.350%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:59.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:11:58 smithi133 bash[105589]: audit 2023-10-09T05:11:58.443126+0000 mon.a (mon.0) 889 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:11:59.078 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:59.112 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:59.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:59.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:59.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:58 smithi194 bash[92666]: cluster 2023-10-09T05:11:57.812422+0000 mgr.y (mgr.25277) 702 : cluster 0 pgmap v677: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 14889/159243 objects degraded (9.350%); 346/159243 objects misplaced (0.217%); 15 MiB/s, 289 objects/s recovering 2023-10-09T05:11:59.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:58 smithi194 bash[92666]: cluster 2023-10-09T05:11:58.270059+0000 mon.a (mon.0) 888 : cluster 3 Health check update: Degraded data redundancy: 14889/159243 objects degraded (9.350%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:11:59.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:11:58 smithi194 bash[92666]: audit 2023-10-09T05:11:58.443126+0000 mon.a (mon.0) 889 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:11:59.216 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:59.241 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:59.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:59.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:59.346 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:59.372 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:11:59.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:59.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:11:59.480 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:59.509 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:11:59.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:59.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:11:59.617 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:59.648 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:11:59.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:59.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:11:59.751 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:59.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:59.781 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:11:59.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:11:59.882 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:59.908 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:11:59.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:11:59.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:00.011 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:00.037 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:00.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:00.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:00.144 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:00.186 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:00.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:00.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:00.291 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:00.319 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:00.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:00.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:00.423 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:00.458 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:00.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:00.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:00.565 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:00.595 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:00.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:00.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:00.699 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:00.721 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:00.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:00.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:00.825 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:00.856 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:00.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:00.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:00.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:00 smithi194 bash[92666]: cluster 2023-10-09T05:11:59.814087+0000 mgr.y (mgr.25277) 703 : cluster 0 pgmap v678: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 14297/149460 objects degraded (9.566%); 114/149460 objects misplaced (0.076%); 18 MiB/s, 322 objects/s recovering 2023-10-09T05:12:00.960 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:00.986 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:00.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:00.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:01.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:00 smithi133 bash[102858]: cluster 2023-10-09T05:11:59.814087+0000 mgr.y (mgr.25277) 703 : cluster 0 pgmap v678: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 14297/149460 objects degraded (9.566%); 114/149460 objects misplaced (0.076%); 18 MiB/s, 322 objects/s recovering 2023-10-09T05:12:01.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:00 smithi133 bash[105589]: cluster 2023-10-09T05:11:59.814087+0000 mgr.y (mgr.25277) 703 : cluster 0 pgmap v678: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 14297/149460 objects degraded (9.566%); 114/149460 objects misplaced (0.076%); 18 MiB/s, 322 objects/s recovering 2023-10-09T05:12:01.090 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:01.113 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:01.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:01.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:01.219 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:01.248 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:01.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:01.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:01.352 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:01.388 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:01.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:01.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:01.491 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:01.510 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:01.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:01.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:01.613 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:01.649 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:01.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:01.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:01.756 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:01.785 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:01.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:01.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:01.894 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:01.928 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:01.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:01.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:02.031 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:02.063 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:02.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:02.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:02.166 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:02.187 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:02.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:02.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:02.293 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:02.316 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:02.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:02.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:02.424 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:02.456 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:02.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:02.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:02.559 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:02.592 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:02.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:02.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:02.698 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:02.727 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:02.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:02.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:02.829 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:02.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:02.861 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:02.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:02.907 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:02 smithi133 bash[105589]: cluster 2023-10-09T05:12:01.815828+0000 mgr.y (mgr.25277) 704 : cluster 0 pgmap v679: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 13268/148014 objects degraded (8.964%); 114/148014 objects misplaced (0.077%); 12 MiB/s, 296 objects/s recovering 2023-10-09T05:12:02.907 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:02 smithi133 bash[102858]: cluster 2023-10-09T05:12:01.815828+0000 mgr.y (mgr.25277) 704 : cluster 0 pgmap v679: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 13268/148014 objects degraded (8.964%); 114/148014 objects misplaced (0.077%); 12 MiB/s, 296 objects/s recovering 2023-10-09T05:12:02.962 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:02.986 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:02.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:02.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:03.090 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:03.097 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:02 smithi194 bash[92666]: cluster 2023-10-09T05:12:01.815828+0000 mgr.y (mgr.25277) 704 : cluster 0 pgmap v679: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 13268/148014 objects degraded (8.964%); 114/148014 objects misplaced (0.077%); 12 MiB/s, 296 objects/s recovering 2023-10-09T05:12:03.118 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:03.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:03.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:03.224 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:03.256 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:03.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:03.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:03.362 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:03.400 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:03.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:03.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:03.508 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:03.536 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:03.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:03.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:03.644 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:03.666 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:03.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:03.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:03.772 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:03.795 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:03.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:03.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:03.899 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:03.909 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:03 smithi194 bash[92666]: cluster 2023-10-09T05:12:03.270852+0000 mon.a (mon.0) 890 : cluster 3 Health check update: Degraded data redundancy: 13268/148014 objects degraded (8.964%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:12:03.939 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:03.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:03.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:04.046 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:04.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:03 smithi133 bash[102858]: cluster 2023-10-09T05:12:03.270852+0000 mon.a (mon.0) 890 : cluster 3 Health check update: Degraded data redundancy: 13268/148014 objects degraded (8.964%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:12:04.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:03 smithi133 bash[105589]: cluster 2023-10-09T05:12:03.270852+0000 mon.a (mon.0) 890 : cluster 3 Health check update: Degraded data redundancy: 13268/148014 objects degraded (8.964%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:12:04.070 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:04.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:04.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:04.174 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:04.203 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:04.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:04.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:04.308 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:04.317 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:12:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:12:04] "GET /metrics HTTP/1.1" 200 36443 "" "Prometheus/2.43.0" 2023-10-09T05:12:04.334 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:04.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:04.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:04.440 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:04.478 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:04.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:04.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:04.583 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:04.616 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:04.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:04.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:04.721 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:04.745 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:04.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:04.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:04.849 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:04.869 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:04.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:04.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:04.972 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:04.998 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:05.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:05.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:05.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:04 smithi133 bash[102858]: cluster 2023-10-09T05:12:03.817699+0000 mgr.y (mgr.25277) 705 : cluster 0 pgmap v680: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 13111/144939 objects degraded (9.046%); 114/144939 objects misplaced (0.079%); 11 MiB/s, 278 objects/s recovering 2023-10-09T05:12:05.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:04 smithi133 bash[105589]: cluster 2023-10-09T05:12:03.817699+0000 mgr.y (mgr.25277) 705 : cluster 0 pgmap v680: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 13111/144939 objects degraded (9.046%); 114/144939 objects misplaced (0.079%); 11 MiB/s, 278 objects/s recovering 2023-10-09T05:12:05.102 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:05.136 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:05.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:05.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:05.241 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:05.251 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:04 smithi194 bash[92666]: cluster 2023-10-09T05:12:03.817699+0000 mgr.y (mgr.25277) 705 : cluster 0 pgmap v680: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 13111/144939 objects degraded (9.046%); 114/144939 objects misplaced (0.079%); 11 MiB/s, 278 objects/s recovering 2023-10-09T05:12:05.278 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:05.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:05.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:05.382 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:05.401 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:05.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:05.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:05.504 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:05.529 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:05.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:05.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:05.632 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:05.661 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:05.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:05.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:05.768 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:05.794 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:05.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:05.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:05.902 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:05.943 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:05.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:05.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:06.049 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:06.079 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:06.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:06.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:06.182 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:06.206 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:06.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:06.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:06.309 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:06.334 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:06.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:06.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:06.440 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:06.471 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:06.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:06.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:06.575 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:06.605 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:06.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:06.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:06.709 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:06.739 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:06.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:06.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:06.846 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:06.877 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:06.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:06.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:06.980 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:07.009 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:07.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:07.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:07.112 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:07.121 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:06 smithi194 bash[92666]: cluster 2023-10-09T05:12:05.820279+0000 mgr.y (mgr.25277) 706 : cluster 0 pgmap v681: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 12883/137739 objects degraded (9.353%); 11 MiB/s, 289 objects/s recovering 2023-10-09T05:12:07.143 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:07.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:07.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:07.250 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:07.284 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:07.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:07.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:07.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:06 smithi133 bash[105589]: cluster 2023-10-09T05:12:05.820279+0000 mgr.y (mgr.25277) 706 : cluster 0 pgmap v681: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 12883/137739 objects degraded (9.353%); 11 MiB/s, 289 objects/s recovering 2023-10-09T05:12:07.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:06 smithi133 bash[102858]: cluster 2023-10-09T05:12:05.820279+0000 mgr.y (mgr.25277) 706 : cluster 0 pgmap v681: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 12883/137739 objects degraded (9.353%); 11 MiB/s, 289 objects/s recovering 2023-10-09T05:12:07.387 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:07.416 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:07.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:07.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:07.520 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:07.556 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:07.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:07.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:07.661 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:07.695 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:07.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:07.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:07.802 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:07.827 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:07.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:07.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:07.930 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:07.960 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:07.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:07.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:08.065 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:08.121 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:08.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:08.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:08.225 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:08.252 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:08.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:08.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:08.359 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:08.391 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:08.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:08.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:08.497 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:08.529 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:08.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:08.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:08.634 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:08.654 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:08.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:08.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:08.757 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:08.785 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:08.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:08.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:08.891 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:08.933 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:08.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:08.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:08.950 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:12:08.951 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:12:09.036 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:09.064 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:09.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:09.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:09.168 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:09.179 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:08 smithi133 bash[102858]: cluster 2023-10-09T05:12:07.821931+0000 mgr.y (mgr.25277) 707 : cluster 0 pgmap v682: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 12883/136257 objects degraded (9.455%); 7.9 MiB/s, 289 objects/s recovering 2023-10-09T05:12:09.180 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:08 smithi133 bash[102858]: cluster 2023-10-09T05:12:08.272020+0000 mon.a (mon.0) 891 : cluster 3 Health check update: Degraded data redundancy: 12883/136257 objects degraded (9.455%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:12:09.180 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:08 smithi133 bash[105589]: cluster 2023-10-09T05:12:07.821931+0000 mgr.y (mgr.25277) 707 : cluster 0 pgmap v682: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 12883/136257 objects degraded (9.455%); 7.9 MiB/s, 289 objects/s recovering 2023-10-09T05:12:09.180 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:08 smithi133 bash[105589]: cluster 2023-10-09T05:12:08.272020+0000 mon.a (mon.0) 891 : cluster 3 Health check update: Degraded data redundancy: 12883/136257 objects degraded (9.455%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:12:09.199 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:09.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:09.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:09.305 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:09.334 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:09.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:09.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:09.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:08 smithi194 bash[92666]: cluster 2023-10-09T05:12:07.821931+0000 mgr.y (mgr.25277) 707 : cluster 0 pgmap v682: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 12883/136257 objects degraded (9.455%); 7.9 MiB/s, 289 objects/s recovering 2023-10-09T05:12:09.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:08 smithi194 bash[92666]: cluster 2023-10-09T05:12:08.272020+0000 mon.a (mon.0) 891 : cluster 3 Health check update: Degraded data redundancy: 12883/136257 objects degraded (9.455%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:12:09.442 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:09.473 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:09.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:09.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:09.580 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:09.604 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:09.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:09.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:09.709 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:09.738 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:09.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:09.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:09.841 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:09.892 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:09.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:09.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:09.936 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:12:09.967 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:12:09.998 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:10.014 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:12:10.042 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:10.046 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:12:10.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:10.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:10.095 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:12:10.124 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:12:10.142 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:12:10.144 INFO:teuthology.orchestra.run.smithi133.stderr:dumped all 2023-10-09T05:12:10.148 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:10.158 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:12:10.178 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:09 smithi133 bash[105589]: audit 2023-10-09T05:12:09.893852+0000 mon.a (mon.0) 892 : audit 0 from='client.? 172.21.15.133:0/3582229359' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:12:10.178 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:09 smithi133 bash[102858]: audit 2023-10-09T05:12:09.893852+0000 mon.a (mon.0) 892 : audit 0 from='client.? 172.21.15.133:0/3582229359' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:12:10.182 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:10.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:10.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:10.198 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:12:10.202 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:12:10.202 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:12:10.203 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:12:10.203 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:12:10.203 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:12:10.203 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:12:10.203 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:12:10.203 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:12:10.286 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:10.295 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:09 smithi194 bash[92666]: audit 2023-10-09T05:12:09.893852+0000 mon.a (mon.0) 892 : audit 0 from='client.? 172.21.15.133:0/3582229359' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:12:10.323 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:10.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:10.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:10.429 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:10.456 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:10.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:10.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:10.559 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:10.576 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:10.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:10.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:10.679 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:10.712 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:10.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:10.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:10.817 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:10.841 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:10.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:10.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:10.944 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:10.950 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:10 smithi194 bash[92666]: cluster 2023-10-09T05:12:09.822875+0000 mgr.y (mgr.25277) 708 : cluster 0 pgmap v683: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 939 op/s; 12708/125382 objects degraded (10.135%); 7.0 MiB/s, 252 objects/s recovering 2023-10-09T05:12:10.950 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:10 smithi194 bash[92666]: audit 2023-10-09T05:12:10.122095+0000 mgr.y (mgr.25277) 709 : audit 0 from='client.44867 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:12:10.950 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:10 smithi194 bash[92666]: audit 2023-10-09T05:12:10.879385+0000 mon.a (mon.0) 893 : audit 1 from='client.? 172.21.15.133:0/924652578' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:12:10.971 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is unset 2023-10-09T05:12:10.975 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:10.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:10.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:11.078 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:11.114 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:11.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:11.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:11.221 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:11.254 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:11.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:11.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:11.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:10 smithi133 bash[102858]: cluster 2023-10-09T05:12:09.822875+0000 mgr.y (mgr.25277) 708 : cluster 0 pgmap v683: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 939 op/s; 12708/125382 objects degraded (10.135%); 7.0 MiB/s, 252 objects/s recovering 2023-10-09T05:12:11.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:10 smithi133 bash[102858]: audit 2023-10-09T05:12:10.122095+0000 mgr.y (mgr.25277) 709 : audit 0 from='client.44867 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:12:11.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:10 smithi133 bash[102858]: audit 2023-10-09T05:12:10.879385+0000 mon.a (mon.0) 893 : audit 1 from='client.? 172.21.15.133:0/924652578' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:12:11.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:10 smithi133 bash[105589]: cluster 2023-10-09T05:12:09.822875+0000 mgr.y (mgr.25277) 708 : cluster 0 pgmap v683: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 939 op/s; 12708/125382 objects degraded (10.135%); 7.0 MiB/s, 252 objects/s recovering 2023-10-09T05:12:11.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:10 smithi133 bash[105589]: audit 2023-10-09T05:12:10.122095+0000 mgr.y (mgr.25277) 709 : audit 0 from='client.44867 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:12:11.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:10 smithi133 bash[105589]: audit 2023-10-09T05:12:10.879385+0000 mon.a (mon.0) 893 : audit 1 from='client.? 172.21.15.133:0/924652578' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:12:11.359 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:11.390 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:11.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:11.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:11.493 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:11.518 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:11.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:11.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:11.621 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:11.647 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:11.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:11.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:11.750 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:11.780 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:11.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:11.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:11.886 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:11.909 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:11.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:11.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:12.012 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:12.049 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:12.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:12.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:12.066 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T05:12:12.153 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:12.162 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:11 smithi194 bash[92666]: cluster 2023-10-09T05:12:10.946269+0000 mon.a (mon.0) 894 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:12:12.162 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:11 smithi194 bash[92666]: audit 2023-10-09T05:12:10.961666+0000 mon.a (mon.0) 895 : audit 1 from='client.? 172.21.15.133:0/924652578' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:12:12.162 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:11 smithi194 bash[92666]: cluster 2023-10-09T05:12:10.961971+0000 mon.a (mon.0) 896 : cluster 0 osdmap e165: 8 total, 8 up, 8 in 2023-10-09T05:12:12.162 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:11 smithi194 bash[92666]: cluster 2023-10-09T05:12:10.966355+0000 osd.7 (osd.7) 91 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T05:12:12.162 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:11 smithi194 bash[92666]: cluster 2023-10-09T05:12:10.971140+0000 osd.6 (osd.6) 88 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T05:12:12.181 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:12.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:12.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:12.285 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:12.292 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:11 smithi133 bash[105589]: cluster 2023-10-09T05:12:10.946269+0000 mon.a (mon.0) 894 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:12:12.293 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:11 smithi133 bash[105589]: audit 2023-10-09T05:12:10.961666+0000 mon.a (mon.0) 895 : audit 1 from='client.? 172.21.15.133:0/924652578' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:12:12.293 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:11 smithi133 bash[105589]: cluster 2023-10-09T05:12:10.961971+0000 mon.a (mon.0) 896 : cluster 0 osdmap e165: 8 total, 8 up, 8 in 2023-10-09T05:12:12.293 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:11 smithi133 bash[105589]: cluster 2023-10-09T05:12:10.966355+0000 osd.7 (osd.7) 91 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T05:12:12.293 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:11 smithi133 bash[105589]: cluster 2023-10-09T05:12:10.971140+0000 osd.6 (osd.6) 88 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T05:12:12.293 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:11 smithi133 bash[102858]: cluster 2023-10-09T05:12:10.946269+0000 mon.a (mon.0) 894 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:12:12.293 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:11 smithi133 bash[102858]: audit 2023-10-09T05:12:10.961666+0000 mon.a (mon.0) 895 : audit 1 from='client.? 172.21.15.133:0/924652578' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:12:12.293 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:11 smithi133 bash[102858]: cluster 2023-10-09T05:12:10.961971+0000 mon.a (mon.0) 896 : cluster 0 osdmap e165: 8 total, 8 up, 8 in 2023-10-09T05:12:12.293 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:11 smithi133 bash[102858]: cluster 2023-10-09T05:12:10.966355+0000 osd.7 (osd.7) 91 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T05:12:12.293 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:11 smithi133 bash[102858]: cluster 2023-10-09T05:12:10.971140+0000 osd.6 (osd.6) 88 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T05:12:12.322 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:12.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:12.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:12.430 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:12.465 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:12.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:12.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:12.568 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:12.593 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:12.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:12.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:12.697 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:12.742 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:12.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:12.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:12.845 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:12.881 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:12.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:12.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:12.989 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:13.021 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:13.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:13.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:13.082 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:12 smithi133 bash[105589]: cluster 2023-10-09T05:12:10.969584+0000 osd.5 (osd.5) 144 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T05:12:13.082 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:12 smithi133 bash[105589]: cluster 2023-10-09T05:12:10.973088+0000 osd.5 (osd.5) 145 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T05:12:13.082 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:12 smithi133 bash[105589]: cluster 2023-10-09T05:12:10.975170+0000 osd.5 (osd.5) 146 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T05:12:13.082 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:12 smithi133 bash[105589]: cluster 2023-10-09T05:12:10.977049+0000 osd.4 (osd.4) 104 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T05:12:13.082 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:12 smithi133 bash[105589]: cluster 2023-10-09T05:12:11.823461+0000 mgr.y (mgr.25277) 710 : cluster 0 pgmap v685: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 12708/125382 objects degraded (10.135%); 2.5 MiB/s, 134 objects/s recovering 2023-10-09T05:12:13.083 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:12 smithi133 bash[102858]: cluster 2023-10-09T05:12:10.969584+0000 osd.5 (osd.5) 144 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T05:12:13.083 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:12 smithi133 bash[102858]: cluster 2023-10-09T05:12:10.973088+0000 osd.5 (osd.5) 145 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T05:12:13.083 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:12 smithi133 bash[102858]: cluster 2023-10-09T05:12:10.975170+0000 osd.5 (osd.5) 146 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T05:12:13.083 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:12 smithi133 bash[102858]: cluster 2023-10-09T05:12:10.977049+0000 osd.4 (osd.4) 104 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T05:12:13.083 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:12 smithi133 bash[102858]: cluster 2023-10-09T05:12:11.823461+0000 mgr.y (mgr.25277) 710 : cluster 0 pgmap v685: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 12708/125382 objects degraded (10.135%); 2.5 MiB/s, 134 objects/s recovering 2023-10-09T05:12:13.125 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:13.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:12 smithi194 bash[92666]: cluster 2023-10-09T05:12:10.969584+0000 osd.5 (osd.5) 144 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T05:12:13.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:12 smithi194 bash[92666]: cluster 2023-10-09T05:12:10.973088+0000 osd.5 (osd.5) 145 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T05:12:13.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:12 smithi194 bash[92666]: cluster 2023-10-09T05:12:10.975170+0000 osd.5 (osd.5) 146 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T05:12:13.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:12 smithi194 bash[92666]: cluster 2023-10-09T05:12:10.977049+0000 osd.4 (osd.4) 104 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T05:12:13.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:12 smithi194 bash[92666]: cluster 2023-10-09T05:12:11.823461+0000 mgr.y (mgr.25277) 710 : cluster 0 pgmap v685: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 12708/125382 objects degraded (10.135%); 2.5 MiB/s, 134 objects/s recovering 2023-10-09T05:12:13.147 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:13.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:13.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:13.251 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:13.286 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:13.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:13.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:13.393 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:13.437 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:13.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:13.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:13.541 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:13.571 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:13.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:13.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:13.623 INFO:teuthology.orchestra.run.smithi133.stdout:{"pg_ready":true,"pg_map":{"version":683,"stamp":"2023-10-09T05:12:09.822177+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2737969080,"num_objects":41794,"num_object_clones":0,"num_object_copies":125382,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12708,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":41794,"num_whiteouts":0,"num_read":539,"num_read_kb":965,"num_write":138464,"num_write_kb":5760270,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12309,"num_bytes_recovered":667420256,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":203143,"ondisk_log_size":203143,"up":297,"acting":294,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":311,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":13882984,"kb_used_data":9920808,"kb_used_omap":20,"kb_used_meta":3962027,"kb_avail":736078232,"statfs":{"total":767960285184,"available":753744109568,"internally_reserved":0,"allocated":10158907392,"data_stored":10152981604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":4057116302},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-739704832,"num_objects":-11287,"num_object_clones":0,"num_object_copies":-33861,"num_objects_missing_on_primary":-1539,"num_objects_missing":-1539,"num_objects_degraded":-2181,"num_objects_misplaced":-346,"num_objects_unfound":0,"num_objects_dirty":-11287,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11287,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3037,"num_bytes_recovered":88211456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.010252"},"pg_stats":[{"pgid":"9.10","version":"164'8044","reported_seq":11143,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:12:06.705753+0000","last_change":"2023-10-09T05:12:00.173110+0000","last_active":"2023-10-09T05:12:06.705753+0000","last_peered":"2023-10-09T05:12:06.705753+0000","last_clean":"2023-10-09T05:12:06.705753+0000","last_became_active":"2023-10-09T05:11:43.418555+0000","last_became_peered":"2023-10-09T05:11:43.418555+0000","last_unstale":"2023-10-09T05:12:06.705753+0000","last_undegraded":"2023-10-09T05:12:06.705753+0000","last_fullsized":"2023-10-09T05:12:06.705753+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":160,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":8044,"log_dups_size":0,"ondisk_log_size":8044,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87654400,"num_objects":1338,"num_object_clones":0,"num_object_copies":4014,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1338,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4296,"num_write_kb":180256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1441,"num_bytes_recovered":31817728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"69'2","reported_seq":164,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.204609+0000","last_change":"2023-10-09T05:09:57.873320+0000","last_active":"2023-10-09T05:11:48.204609+0000","last_peered":"2023-10-09T05:11:48.204609+0000","last_clean":"2023-10-09T05:11:48.204609+0000","last_became_active":"2023-10-09T05:09:57.868376+0000","last_became_peered":"2023-10-09T05:09:57.868376+0000","last_unstale":"2023-10-09T05:11:48.204609+0000","last_undegraded":"2023-10-09T05:11:48.204609+0000","last_fullsized":"2023-10-09T05:11:48.204609+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:27.186610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":132,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:11:49.240027+0000","last_change":"2023-10-09T05:09:57.854324+0000","last_active":"2023-10-09T05:11:49.240027+0000","last_peered":"2023-10-09T05:11:49.240027+0000","last_clean":"2023-10-09T05:11:49.240027+0000","last_became_active":"2023-10-09T05:09:57.853925+0000","last_became_peered":"2023-10-09T05:09:57.853925+0000","last_unstale":"2023-10-09T05:11:49.240027+0000","last_undegraded":"2023-10-09T05:11:49.240027+0000","last_fullsized":"2023-10-09T05:11:49.240027+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:11.255018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.13","version":"164'7834","reported_seq":8933,"reported_epoch":164,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:12:09.391985+0000","last_change":"2023-10-09T05:11:43.302576+0000","last_active":"2023-10-09T05:12:09.391985+0000","last_peered":"2023-10-09T05:12:09.391985+0000","last_clean":"2023-10-09T05:09:56.807871+0000","last_became_active":"2023-10-09T05:11:43.287144+0000","last_became_peered":"2023-10-09T05:11:43.287144+0000","last_unstale":"2023-10-09T05:12:09.391985+0000","last_undegraded":"2023-10-09T05:11:43.263303+0000","last_fullsized":"2023-10-09T05:12:09.391985+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":160,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":7834,"log_dups_size":0,"ondisk_log_size":7834,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81494016,"num_objects":1244,"num_object_clones":0,"num_object_copies":3732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":887,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4215,"num_write_kb":174656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":452,"num_bytes_recovered":29622272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":["7","3","5"],"object_location_counts":[{"shards":"1,3,7","objects":1244}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":23,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.193540+0000","last_change":"2023-10-09T05:09:57.857511+0000","last_active":"2023-10-09T05:11:48.193540+0000","last_peered":"2023-10-09T05:11:48.193540+0000","last_clean":"2023-10-09T05:11:48.193540+0000","last_became_active":"2023-10-09T05:09:57.857193+0000","last_became_peered":"2023-10-09T05:09:57.857193+0000","last_unstale":"2023-10-09T05:11:48.193540+0000","last_undegraded":"2023-10-09T05:11:48.193540+0000","last_fullsized":"2023-10-09T05:11:48.193540+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:32.842980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":120,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.200035+0000","last_change":"2023-10-09T05:02:09.648283+0000","last_active":"2023-10-09T05:11:48.200035+0000","last_peered":"2023-10-09T05:11:48.200035+0000","last_clean":"2023-10-09T05:11:48.200035+0000","last_became_active":"2023-10-09T05:02:09.647778+0000","last_became_peered":"2023-10-09T05:02:09.647778+0000","last_unstale":"2023-10-09T05:11:48.200035+0000","last_undegraded":"2023-10-09T05:11:48.200035+0000","last_fullsized":"2023-10-09T05:11:48.200035+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:24.582392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.12","version":"164'7792","reported_seq":8879,"reported_epoch":164,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:12:08.347359+0000","last_change":"2023-10-09T05:11:43.296037+0000","last_active":"2023-10-09T05:12:08.347359+0000","last_peered":"2023-10-09T05:12:08.347359+0000","last_clean":"2023-10-09T05:09:56.749891+0000","last_became_active":"2023-10-09T05:11:43.277734+0000","last_became_peered":"2023-10-09T05:11:43.277734+0000","last_unstale":"2023-10-09T05:12:08.347359+0000","last_undegraded":"2023-10-09T05:11:43.272336+0000","last_fullsized":"2023-10-09T05:12:08.347359+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":160,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":7792,"log_dups_size":0,"ondisk_log_size":7792,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87916544,"num_objects":1342,"num_object_clones":0,"num_object_copies":4026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":869,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4167,"num_write_kb":176256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":442,"num_bytes_recovered":28966912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"1,2,3","objects":1342}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":150,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.375029+0000","last_change":"2023-10-09T05:09:57.849878+0000","last_active":"2023-10-09T05:11:48.375029+0000","last_peered":"2023-10-09T05:11:48.375029+0000","last_clean":"2023-10-09T05:11:48.375029+0000","last_became_active":"2023-10-09T05:09:57.849541+0000","last_became_peered":"2023-10-09T05:09:57.849541+0000","last_unstale":"2023-10-09T05:11:48.375029+0000","last_undegraded":"2023-10-09T05:11:48.375029+0000","last_fullsized":"2023-10-09T05:11:48.375029+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:49:03.330682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":144,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.193025+0000","last_change":"2023-10-09T05:09:57.852119+0000","last_active":"2023-10-09T05:11:48.193025+0000","last_peered":"2023-10-09T05:11:48.193025+0000","last_clean":"2023-10-09T05:11:48.193025+0000","last_became_active":"2023-10-09T05:09:57.851752+0000","last_became_peered":"2023-10-09T05:09:57.851752+0000","last_unstale":"2023-10-09T05:11:48.193025+0000","last_undegraded":"2023-10-09T05:11:48.193025+0000","last_fullsized":"2023-10-09T05:11:48.193025+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:58:09.571362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":118,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.199921+0000","last_change":"2023-10-09T05:02:09.646765+0000","last_active":"2023-10-09T05:11:48.199921+0000","last_peered":"2023-10-09T05:11:48.199921+0000","last_clean":"2023-10-09T05:11:48.199921+0000","last_became_active":"2023-10-09T05:02:09.645141+0000","last_became_peered":"2023-10-09T05:02:09.645141+0000","last_unstale":"2023-10-09T05:11:48.199921+0000","last_undegraded":"2023-10-09T05:11:48.199921+0000","last_fullsized":"2023-10-09T05:11:48.199921+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:10.098578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":132,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.196651+0000","last_change":"2023-10-09T05:09:57.849695+0000","last_active":"2023-10-09T05:11:48.196651+0000","last_peered":"2023-10-09T05:11:48.196651+0000","last_clean":"2023-10-09T05:11:48.196651+0000","last_became_active":"2023-10-09T05:09:57.848478+0000","last_became_peered":"2023-10-09T05:09:57.848478+0000","last_unstale":"2023-10-09T05:11:48.196651+0000","last_undegraded":"2023-10-09T05:11:48.196651+0000","last_fullsized":"2023-10-09T05:11:48.196651+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:26.115312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":145,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.202094+0000","last_change":"2023-10-09T05:09:57.867139+0000","last_active":"2023-10-09T05:11:48.202094+0000","last_peered":"2023-10-09T05:11:48.202094+0000","last_clean":"2023-10-09T05:11:48.202094+0000","last_became_active":"2023-10-09T05:09:57.866134+0000","last_became_peered":"2023-10-09T05:09:57.866134+0000","last_unstale":"2023-10-09T05:11:48.202094+0000","last_undegraded":"2023-10-09T05:11:48.202094+0000","last_fullsized":"2023-10-09T05:11:48.202094+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:45.375784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":109,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.198421+0000","last_change":"2023-10-09T05:02:09.651167+0000","last_active":"2023-10-09T05:11:48.198421+0000","last_peered":"2023-10-09T05:11:48.198421+0000","last_clean":"2023-10-09T05:11:48.198421+0000","last_became_active":"2023-10-09T05:02:09.650419+0000","last_became_peered":"2023-10-09T05:02:09.650419+0000","last_unstale":"2023-10-09T05:11:48.198421+0000","last_undegraded":"2023-10-09T05:11:48.198421+0000","last_fullsized":"2023-10-09T05:11:48.198421+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:55:16.966938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":130,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.197542+0000","last_change":"2023-10-09T05:02:09.649700+0000","last_active":"2023-10-09T05:11:48.197542+0000","last_peered":"2023-10-09T05:11:48.197542+0000","last_clean":"2023-10-09T05:11:48.197542+0000","last_became_active":"2023-10-09T05:02:09.649317+0000","last_became_peered":"2023-10-09T05:02:09.649317+0000","last_unstale":"2023-10-09T05:11:48.197542+0000","last_undegraded":"2023-10-09T05:11:48.197542+0000","last_fullsized":"2023-10-09T05:11:48.197542+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:50.711767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":23,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.204081+0000","last_change":"2023-10-09T05:09:57.848297+0000","last_active":"2023-10-09T05:11:48.204081+0000","last_peered":"2023-10-09T05:11:48.204081+0000","last_clean":"2023-10-09T05:11:48.204081+0000","last_became_active":"2023-10-09T05:09:57.847987+0000","last_became_peered":"2023-10-09T05:09:57.847987+0000","last_unstale":"2023-10-09T05:11:48.204081+0000","last_undegraded":"2023-10-09T05:11:48.204081+0000","last_fullsized":"2023-10-09T05:11:48.204081+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:16.385601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'23","reported_seq":136,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.205110+0000","last_change":"2023-10-09T04:44:52.781203+0000","last_active":"2023-10-09T05:11:48.205110+0000","last_peered":"2023-10-09T05:11:48.205110+0000","last_clean":"2023-10-09T05:11:48.205110+0000","last_became_active":"2023-10-09T04:44:52.780902+0000","last_became_peered":"2023-10-09T04:44:52.780902+0000","last_unstale":"2023-10-09T05:11:48.205110+0000","last_undegraded":"2023-10-09T05:11:48.205110+0000","last_fullsized":"2023-10-09T05:11:48.205110+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:56.332974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":144,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.204871+0000","last_change":"2023-10-09T05:09:57.868761+0000","last_active":"2023-10-09T05:11:48.204871+0000","last_peered":"2023-10-09T05:11:48.204871+0000","last_clean":"2023-10-09T05:11:48.204871+0000","last_became_active":"2023-10-09T05:09:57.868121+0000","last_became_peered":"2023-10-09T05:09:57.868121+0000","last_unstale":"2023-10-09T05:11:48.204871+0000","last_undegraded":"2023-10-09T05:11:48.204871+0000","last_fullsized":"2023-10-09T05:11:48.204871+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:43.412235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.19","version":"164'7990","reported_seq":8035,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:12:08.898802+0000","last_change":"2023-10-09T05:11:43.259778+0000","last_active":"2023-10-09T05:12:08.898802+0000","last_peered":"2023-10-09T05:12:08.898802+0000","last_clean":"2023-10-09T05:12:08.898802+0000","last_became_active":"2023-10-09T05:11:43.240778+0000","last_became_peered":"2023-10-09T05:11:43.240778+0000","last_unstale":"2023-10-09T05:12:08.898802+0000","last_undegraded":"2023-10-09T05:12:08.898802+0000","last_fullsized":"2023-10-09T05:12:08.898802+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":160,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":7990,"log_dups_size":0,"ondisk_log_size":7990,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83623936,"num_objects":1276,"num_object_clones":0,"num_object_copies":3828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4276,"num_write_kb":177664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":129,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.197338+0000","last_change":"2023-10-09T05:01:02.669186+0000","last_active":"2023-10-09T05:11:48.197338+0000","last_peered":"2023-10-09T05:11:48.197338+0000","last_clean":"2023-10-09T05:11:48.197338+0000","last_became_active":"2023-10-09T05:01:02.668739+0000","last_became_peered":"2023-10-09T05:01:02.668739+0000","last_unstale":"2023-10-09T05:11:48.197338+0000","last_undegraded":"2023-10-09T05:11:48.197338+0000","last_fullsized":"2023-10-09T05:11:48.197338+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:41.352624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":132,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.200376+0000","last_change":"2023-10-09T05:09:57.854619+0000","last_active":"2023-10-09T05:11:48.200376+0000","last_peered":"2023-10-09T05:11:48.200376+0000","last_clean":"2023-10-09T05:11:48.200376+0000","last_became_active":"2023-10-09T05:09:57.852039+0000","last_became_peered":"2023-10-09T05:09:57.852039+0000","last_unstale":"2023-10-09T05:11:48.200376+0000","last_undegraded":"2023-10-09T05:11:48.200376+0000","last_fullsized":"2023-10-09T05:11:48.200376+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:46:02.986058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.18","version":"164'7914","reported_seq":7958,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:12:08.292882+0000","last_change":"2023-10-09T05:11:43.261733+0000","last_active":"2023-10-09T05:12:08.292882+0000","last_peered":"2023-10-09T05:12:08.292882+0000","last_clean":"2023-10-09T05:12:08.292882+0000","last_became_active":"2023-10-09T05:11:43.241737+0000","last_became_peered":"2023-10-09T05:11:43.241737+0000","last_unstale":"2023-10-09T05:12:08.292882+0000","last_undegraded":"2023-10-09T05:12:08.292882+0000","last_fullsized":"2023-10-09T05:12:08.292882+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":160,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":7914,"log_dups_size":0,"ondisk_log_size":7914,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83918848,"num_objects":1281,"num_object_clones":0,"num_object_copies":3843,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1281,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4243,"num_write_kb":176736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":118,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.199561+0000","last_change":"2023-10-09T05:02:09.645017+0000","last_active":"2023-10-09T05:11:48.199561+0000","last_peered":"2023-10-09T05:11:48.199561+0000","last_clean":"2023-10-09T05:11:48.199561+0000","last_became_active":"2023-10-09T05:02:09.644705+0000","last_became_peered":"2023-10-09T05:02:09.644705+0000","last_unstale":"2023-10-09T05:11:48.199561+0000","last_undegraded":"2023-10-09T05:11:48.199561+0000","last_fullsized":"2023-10-09T05:11:48.199561+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:20.695961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":124,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.201030+0000","last_change":"2023-10-09T05:02:09.648362+0000","last_active":"2023-10-09T05:11:48.201030+0000","last_peered":"2023-10-09T05:11:48.201030+0000","last_clean":"2023-10-09T05:11:48.201030+0000","last_became_active":"2023-10-09T05:02:09.647127+0000","last_became_peered":"2023-10-09T05:02:09.647127+0000","last_unstale":"2023-10-09T05:11:48.201030+0000","last_undegraded":"2023-10-09T05:11:48.201030+0000","last_fullsized":"2023-10-09T05:11:48.201030+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:38:02.875613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":108,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.196249+0000","last_change":"2023-10-09T04:44:52.792557+0000","last_active":"2023-10-09T05:11:48.196249+0000","last_peered":"2023-10-09T05:11:48.196249+0000","last_clean":"2023-10-09T05:11:48.196249+0000","last_became_active":"2023-10-09T04:44:52.791890+0000","last_became_peered":"2023-10-09T04:44:52.791890+0000","last_unstale":"2023-10-09T05:11:48.196249+0000","last_undegraded":"2023-10-09T05:11:48.196249+0000","last_fullsized":"2023-10-09T05:11:48.196249+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:04:46.379852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":119,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.200108+0000","last_change":"2023-10-09T05:02:09.652158+0000","last_active":"2023-10-09T05:11:48.200108+0000","last_peered":"2023-10-09T05:11:48.200108+0000","last_clean":"2023-10-09T05:11:48.200108+0000","last_became_active":"2023-10-09T05:02:09.651811+0000","last_became_peered":"2023-10-09T05:02:09.651811+0000","last_unstale":"2023-10-09T05:11:48.200108+0000","last_undegraded":"2023-10-09T05:11:48.200108+0000","last_fullsized":"2023-10-09T05:11:48.200108+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:49:10.141237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":137,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.192446+0000","last_change":"2023-10-09T04:56:56.076177+0000","last_active":"2023-10-09T05:11:48.192446+0000","last_peered":"2023-10-09T05:11:48.192446+0000","last_clean":"2023-10-09T05:11:48.192446+0000","last_became_active":"2023-10-09T04:56:56.075564+0000","last_became_peered":"2023-10-09T04:56:56.075564+0000","last_unstale":"2023-10-09T05:11:48.192446+0000","last_undegraded":"2023-10-09T05:11:48.192446+0000","last_fullsized":"2023-10-09T05:11:48.192446+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:38.390793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":132,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.375218+0000","last_change":"2023-10-09T05:09:57.854475+0000","last_active":"2023-10-09T05:11:48.375218+0000","last_peered":"2023-10-09T05:11:48.375218+0000","last_clean":"2023-10-09T05:11:48.375218+0000","last_became_active":"2023-10-09T05:09:57.851782+0000","last_became_peered":"2023-10-09T05:09:57.851782+0000","last_unstale":"2023-10-09T05:11:48.375218+0000","last_undegraded":"2023-10-09T05:11:48.375218+0000","last_fullsized":"2023-10-09T05:11:48.375218+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:20:41.445868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"164'8023","reported_seq":8067,"reported_epoch":164,"state":"active+recovery_wait","last_fresh":"2023-10-09T05:12:08.929835+0000","last_change":"2023-10-09T05:11:43.245901+0000","last_active":"2023-10-09T05:12:08.929835+0000","last_peered":"2023-10-09T05:12:08.929835+0000","last_clean":"2023-10-09T05:11:42.185536+0000","last_became_active":"2023-10-09T05:11:43.182632+0000","last_became_peered":"2023-10-09T05:11:43.182632+0000","last_unstale":"2023-10-09T05:12:08.929835+0000","last_undegraded":"2023-10-09T05:12:08.929835+0000","last_fullsized":"2023-10-09T05:12:08.929835+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":8023,"log_dups_size":0,"ondisk_log_size":8023,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85884928,"num_objects":1310,"num_object_clones":0,"num_object_copies":3930,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1310,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4305,"num_write_kb":179712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5,7","objects":1310}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":127,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.196556+0000","last_change":"2023-10-09T04:56:56.075307+0000","last_active":"2023-10-09T05:11:48.196556+0000","last_peered":"2023-10-09T05:11:48.196556+0000","last_clean":"2023-10-09T05:11:48.196556+0000","last_became_active":"2023-10-09T04:56:56.074831+0000","last_became_peered":"2023-10-09T04:56:56.074831+0000","last_unstale":"2023-10-09T05:11:48.196556+0000","last_undegraded":"2023-10-09T05:11:48.196556+0000","last_fullsized":"2023-10-09T05:11:48.196556+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:51:56.867759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":107,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.202148+0000","last_change":"2023-10-09T04:44:53.962437+0000","last_active":"2023-10-09T05:11:48.202148+0000","last_peered":"2023-10-09T05:11:48.202148+0000","last_clean":"2023-10-09T05:11:48.202148+0000","last_became_active":"2023-10-09T04:44:53.961810+0000","last_became_peered":"2023-10-09T04:44:53.961810+0000","last_unstale":"2023-10-09T05:11:48.202148+0000","last_undegraded":"2023-10-09T05:11:48.202148+0000","last_fullsized":"2023-10-09T05:11:48.202148+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:56.436933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"164'8041","reported_seq":13660,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:12:06.778635+0000","last_change":"2023-10-09T05:11:43.260145+0000","last_active":"2023-10-09T05:12:06.778635+0000","last_peered":"2023-10-09T05:12:06.778635+0000","last_clean":"2023-10-09T05:12:06.778635+0000","last_became_active":"2023-10-09T05:11:43.219844+0000","last_became_peered":"2023-10-09T05:11:43.219844+0000","last_unstale":"2023-10-09T05:12:06.778635+0000","last_undegraded":"2023-10-09T05:12:06.778635+0000","last_fullsized":"2023-10-09T05:12:06.778635+0000","mapping_epoch":156,"log_start":"149'600","ondisk_log_start":"149'600","created":139,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":7441,"log_dups_size":0,"ondisk_log_size":7441,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89096192,"num_objects":1359,"num_object_clones":0,"num_object_copies":4077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4296,"num_write_kb":180992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1384,"num_bytes_recovered":90701824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":156,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.196427+0000","last_change":"2023-10-09T05:09:57.898403+0000","last_active":"2023-10-09T05:11:48.196427+0000","last_peered":"2023-10-09T05:11:48.196427+0000","last_clean":"2023-10-09T05:11:48.196427+0000","last_became_active":"2023-10-09T05:09:57.866053+0000","last_became_peered":"2023-10-09T05:09:57.866053+0000","last_unstale":"2023-10-09T05:11:48.196427+0000","last_undegraded":"2023-10-09T05:11:48.196427+0000","last_fullsized":"2023-10-09T05:11:48.196427+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:31.760924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":161,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.204490+0000","last_change":"2023-10-09T05:09:57.868031+0000","last_active":"2023-10-09T05:11:48.204490+0000","last_peered":"2023-10-09T05:11:48.204490+0000","last_clean":"2023-10-09T05:11:48.204490+0000","last_became_active":"2023-10-09T05:09:57.866477+0000","last_became_peered":"2023-10-09T05:09:57.866477+0000","last_unstale":"2023-10-09T05:11:48.204490+0000","last_undegraded":"2023-10-09T05:11:48.204490+0000","last_fullsized":"2023-10-09T05:11:48.204490+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:27.102583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"164'8172","reported_seq":8213,"reported_epoch":164,"state":"active+recovery_wait","last_fresh":"2023-10-09T05:12:09.494663+0000","last_change":"2023-10-09T05:11:43.305017+0000","last_active":"2023-10-09T05:12:09.494663+0000","last_peered":"2023-10-09T05:12:09.494663+0000","last_clean":"2023-10-09T05:11:42.110526+0000","last_became_active":"2023-10-09T05:11:43.231863+0000","last_became_peered":"2023-10-09T05:11:43.231863+0000","last_unstale":"2023-10-09T05:12:09.494663+0000","last_undegraded":"2023-10-09T05:12:09.494663+0000","last_fullsized":"2023-10-09T05:12:09.494663+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":8172,"log_dups_size":0,"ondisk_log_size":8172,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88178688,"num_objects":1345,"num_object_clones":0,"num_object_copies":4035,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1345,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4369,"num_write_kb":182880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1345}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":110,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.197332+0000","last_change":"2023-10-09T04:44:52.787469+0000","last_active":"2023-10-09T05:11:48.197332+0000","last_peered":"2023-10-09T05:11:48.197332+0000","last_clean":"2023-10-09T05:11:48.197332+0000","last_became_active":"2023-10-09T04:44:52.787148+0000","last_became_peered":"2023-10-09T04:44:52.787148+0000","last_unstale":"2023-10-09T05:11:48.197332+0000","last_undegraded":"2023-10-09T05:11:48.197332+0000","last_fullsized":"2023-10-09T05:11:48.197332+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:12.469065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":154,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.192688+0000","last_change":"2023-10-09T05:09:57.855670+0000","last_active":"2023-10-09T05:11:48.192688+0000","last_peered":"2023-10-09T05:11:48.192688+0000","last_clean":"2023-10-09T05:11:48.192688+0000","last_became_active":"2023-10-09T05:09:57.855293+0000","last_became_peered":"2023-10-09T05:09:57.855293+0000","last_unstale":"2023-10-09T05:11:48.192688+0000","last_undegraded":"2023-10-09T05:11:48.192688+0000","last_fullsized":"2023-10-09T05:11:48.192688+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:19:56.843974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"164'8007","reported_seq":9180,"reported_epoch":164,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:12:09.294858+0000","last_change":"2023-10-09T05:11:44.206029+0000","last_active":"2023-10-09T05:12:09.294858+0000","last_peered":"2023-10-09T05:12:09.294858+0000","last_clean":"2023-10-09T05:09:56.825417+0000","last_became_active":"2023-10-09T05:11:44.183373+0000","last_became_peered":"2023-10-09T05:11:44.183373+0000","last_unstale":"2023-10-09T05:12:09.294858+0000","last_undegraded":"2023-10-09T05:11:44.179980+0000","last_fullsized":"2023-10-09T05:12:09.294858+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":8007,"log_dups_size":0,"ondisk_log_size":8007,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83722240,"num_objects":1277,"num_object_clones":0,"num_object_copies":3831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":895,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4301,"num_write_kb":178528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":470,"num_bytes_recovered":30834688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":["5","2","6"],"object_location_counts":[{"shards":"1,2,5","objects":1277}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":125,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.200757+0000","last_change":"2023-10-09T05:02:09.653932+0000","last_active":"2023-10-09T05:11:48.200757+0000","last_peered":"2023-10-09T05:11:48.200757+0000","last_clean":"2023-10-09T05:11:48.200757+0000","last_became_active":"2023-10-09T05:02:09.652923+0000","last_became_peered":"2023-10-09T05:02:09.652923+0000","last_unstale":"2023-10-09T05:11:48.200757+0000","last_undegraded":"2023-10-09T05:11:48.200757+0000","last_fullsized":"2023-10-09T05:11:48.200757+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:44:11.928237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":137,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.199948+0000","last_change":"2023-10-09T05:09:57.852914+0000","last_active":"2023-10-09T05:11:48.199948+0000","last_peered":"2023-10-09T05:11:48.199948+0000","last_clean":"2023-10-09T05:11:48.199948+0000","last_became_active":"2023-10-09T05:09:57.852522+0000","last_became_peered":"2023-10-09T05:09:57.852522+0000","last_unstale":"2023-10-09T05:11:48.199948+0000","last_undegraded":"2023-10-09T05:11:48.199948+0000","last_fullsized":"2023-10-09T05:11:48.199948+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:24.848494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"164'8023","reported_seq":8068,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:12:08.366477+0000","last_change":"2023-10-09T05:11:43.219461+0000","last_active":"2023-10-09T05:12:08.366477+0000","last_peered":"2023-10-09T05:12:08.366477+0000","last_clean":"2023-10-09T05:12:08.366477+0000","last_became_active":"2023-10-09T05:11:43.193675+0000","last_became_peered":"2023-10-09T05:11:43.193675+0000","last_unstale":"2023-10-09T05:12:08.366477+0000","last_undegraded":"2023-10-09T05:12:08.366477+0000","last_fullsized":"2023-10-09T05:12:08.366477+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":8023,"log_dups_size":0,"ondisk_log_size":8023,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84934656,"num_objects":1296,"num_object_clones":0,"num_object_copies":3888,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1296,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4300,"num_write_kb":179072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"69'3","reported_seq":111,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.200878+0000","last_change":"2023-10-09T05:02:09.650579+0000","last_active":"2023-10-09T05:11:48.200878+0000","last_peered":"2023-10-09T05:11:48.200878+0000","last_clean":"2023-10-09T05:11:48.200878+0000","last_became_active":"2023-10-09T05:02:09.650160+0000","last_became_peered":"2023-10-09T05:02:09.650160+0000","last_unstale":"2023-10-09T05:11:48.200878+0000","last_undegraded":"2023-10-09T05:11:48.200878+0000","last_fullsized":"2023-10-09T05:11:48.200878+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:18.292670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":124,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.205245+0000","last_change":"2023-10-09T05:02:09.652449+0000","last_active":"2023-10-09T05:11:48.205245+0000","last_peered":"2023-10-09T05:11:48.205245+0000","last_clean":"2023-10-09T05:11:48.205245+0000","last_became_active":"2023-10-09T05:02:09.651994+0000","last_became_peered":"2023-10-09T05:02:09.651994+0000","last_unstale":"2023-10-09T05:11:48.205245+0000","last_undegraded":"2023-10-09T05:11:48.205245+0000","last_fullsized":"2023-10-09T05:11:48.205245+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:14.480513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"164'8049","reported_seq":11122,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:12:06.758691+0000","last_change":"2023-10-09T05:12:05.375491+0000","last_active":"2023-10-09T05:12:06.758691+0000","last_peered":"2023-10-09T05:12:06.758691+0000","last_clean":"2023-10-09T05:12:06.758691+0000","last_became_active":"2023-10-09T05:11:44.646717+0000","last_became_peered":"2023-10-09T05:11:44.646717+0000","last_unstale":"2023-10-09T05:12:06.758691+0000","last_undegraded":"2023-10-09T05:12:06.758691+0000","last_fullsized":"2023-10-09T05:12:06.758691+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":8049,"log_dups_size":0,"ondisk_log_size":8049,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87326720,"num_objects":1332,"num_object_clones":0,"num_object_copies":3996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4300,"num_write_kb":180256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1427,"num_bytes_recovered":33980416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":145,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.192897+0000","last_change":"2023-10-09T05:09:57.849784+0000","last_active":"2023-10-09T05:11:48.192897+0000","last_peered":"2023-10-09T05:11:48.192897+0000","last_clean":"2023-10-09T05:11:48.192897+0000","last_became_active":"2023-10-09T05:09:57.849216+0000","last_became_peered":"2023-10-09T05:09:57.849216+0000","last_unstale":"2023-10-09T05:11:48.192897+0000","last_undegraded":"2023-10-09T05:11:48.192897+0000","last_fullsized":"2023-10-09T05:11:48.192897+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:29.764343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":161,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.204961+0000","last_change":"2023-10-09T05:09:57.868658+0000","last_active":"2023-10-09T05:11:48.204961+0000","last_peered":"2023-10-09T05:11:48.204961+0000","last_clean":"2023-10-09T05:11:48.204961+0000","last_became_active":"2023-10-09T05:09:57.868221+0000","last_became_peered":"2023-10-09T05:09:57.868221+0000","last_unstale":"2023-10-09T05:11:48.204961+0000","last_undegraded":"2023-10-09T05:11:48.204961+0000","last_fullsized":"2023-10-09T05:11:48.204961+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:12:15.536917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"164'7813","reported_seq":8883,"reported_epoch":164,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:12:09.394497+0000","last_change":"2023-10-09T05:11:44.209540+0000","last_active":"2023-10-09T05:12:09.394497+0000","last_peered":"2023-10-09T05:12:09.394497+0000","last_clean":"2023-10-09T05:09:56.807871+0000","last_became_active":"2023-10-09T05:11:44.188419+0000","last_became_peered":"2023-10-09T05:11:44.188419+0000","last_unstale":"2023-10-09T05:12:09.394497+0000","last_undegraded":"2023-10-09T05:11:44.179934+0000","last_fullsized":"2023-10-09T05:12:09.394497+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":7813,"log_dups_size":0,"ondisk_log_size":7813,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82870272,"num_objects":1264,"num_object_clones":0,"num_object_copies":3792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":893,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4193,"num_write_kb":174656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":436,"num_bytes_recovered":28508160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":["7","3","5"],"object_location_counts":[{"shards":"1,3,7","objects":1264}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":125,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.204430+0000","last_change":"2023-10-09T05:01:02.663309+0000","last_active":"2023-10-09T05:11:48.204430+0000","last_peered":"2023-10-09T05:11:48.204430+0000","last_clean":"2023-10-09T05:11:48.204430+0000","last_became_active":"2023-10-09T05:01:02.663038+0000","last_became_peered":"2023-10-09T05:01:02.663038+0000","last_unstale":"2023-10-09T05:11:48.204430+0000","last_undegraded":"2023-10-09T05:11:48.204430+0000","last_fullsized":"2023-10-09T05:11:48.204430+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:29:36.689304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":144,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.205407+0000","last_change":"2023-10-09T05:09:57.868378+0000","last_active":"2023-10-09T05:11:48.205407+0000","last_peered":"2023-10-09T05:11:48.205407+0000","last_clean":"2023-10-09T05:11:48.205407+0000","last_became_active":"2023-10-09T05:09:57.867507+0000","last_became_peered":"2023-10-09T05:09:57.867507+0000","last_unstale":"2023-10-09T05:11:48.205407+0000","last_undegraded":"2023-10-09T05:11:48.205407+0000","last_fullsized":"2023-10-09T05:11:48.205407+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:14:35.199631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"164'7813","reported_seq":8929,"reported_epoch":164,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:12:08.295177+0000","last_change":"2023-10-09T05:11:44.186105+0000","last_active":"2023-10-09T05:12:08.295177+0000","last_peered":"2023-10-09T05:12:08.295177+0000","last_clean":"2023-10-09T05:09:56.749891+0000","last_became_active":"2023-10-09T05:11:44.175169+0000","last_became_peered":"2023-10-09T05:11:44.175169+0000","last_unstale":"2023-10-09T05:12:08.295177+0000","last_undegraded":"2023-10-09T05:11:44.171483+0000","last_fullsized":"2023-10-09T05:12:08.295177+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":7813,"log_dups_size":0,"ondisk_log_size":7813,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86540288,"num_objects":1320,"num_object_clones":0,"num_object_copies":3960,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":874,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1320,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4187,"num_write_kb":176256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":456,"num_bytes_recovered":29884416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"1,2,3","objects":1320}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":133,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.196240+0000","last_change":"2023-10-09T05:09:57.852151+0000","last_active":"2023-10-09T05:11:48.196240+0000","last_peered":"2023-10-09T05:11:48.196240+0000","last_clean":"2023-10-09T05:11:48.196240+0000","last_became_active":"2023-10-09T05:09:57.851499+0000","last_became_peered":"2023-10-09T05:09:57.851499+0000","last_unstale":"2023-10-09T05:11:48.196240+0000","last_undegraded":"2023-10-09T05:11:48.196240+0000","last_fullsized":"2023-10-09T05:11:48.196240+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:25:21.153305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":23,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.195300+0000","last_change":"2023-10-09T05:09:57.856764+0000","last_active":"2023-10-09T05:11:48.195300+0000","last_peered":"2023-10-09T05:11:48.195300+0000","last_clean":"2023-10-09T05:11:48.195300+0000","last_became_active":"2023-10-09T05:09:57.856461+0000","last_became_peered":"2023-10-09T05:09:57.856461+0000","last_unstale":"2023-10-09T05:11:48.195300+0000","last_undegraded":"2023-10-09T05:11:48.195300+0000","last_fullsized":"2023-10-09T05:11:48.195300+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:06.848082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"164'8872","reported_seq":9198,"reported_epoch":164,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:12:09.479707+0000","last_change":"2023-10-09T05:09:59.115413+0000","last_active":"2023-10-09T05:12:09.479707+0000","last_peered":"2023-10-09T05:12:09.479707+0000","last_clean":"2023-10-09T05:09:56.775881+0000","last_became_active":"2023-10-09T05:09:59.102861+0000","last_became_peered":"2023-10-09T05:09:59.102861+0000","last_unstale":"2023-10-09T05:12:09.479707+0000","last_undegraded":"2023-10-09T05:09:58.877226+0000","last_fullsized":"2023-10-09T05:09:58.842223+0000","mapping_epoch":152,"log_start":"149'500","ondisk_log_start":"149'500","created":139,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":8372,"log_dups_size":0,"ondisk_log_size":8372,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:24:23.156903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171573248,"num_objects":2618,"num_object_clones":0,"num_object_copies":7854,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2696,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2618,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8872,"num_write_kb":367680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":131,"num_bytes_recovered":8585216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,3],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"3,5","objects":-78}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":24,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:11:49.203079+0000","last_change":"2023-10-09T05:09:57.849066+0000","last_active":"2023-10-09T05:11:49.203079+0000","last_peered":"2023-10-09T05:11:49.203079+0000","last_clean":"2023-10-09T05:11:49.203079+0000","last_became_active":"2023-10-09T05:09:57.848717+0000","last_became_peered":"2023-10-09T05:09:57.848717+0000","last_unstale":"2023-10-09T05:11:49.203079+0000","last_undegraded":"2023-10-09T05:11:49.203079+0000","last_fullsized":"2023-10-09T05:11:49.203079+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:31.267804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":142,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.200961+0000","last_change":"2023-10-09T05:02:09.649559+0000","last_active":"2023-10-09T05:11:48.200961+0000","last_peered":"2023-10-09T05:11:48.200961+0000","last_clean":"2023-10-09T05:11:48.200961+0000","last_became_active":"2023-10-09T05:02:09.649195+0000","last_became_peered":"2023-10-09T05:02:09.649195+0000","last_unstale":"2023-10-09T05:11:48.200961+0000","last_undegraded":"2023-10-09T05:11:48.200961+0000","last_fullsized":"2023-10-09T05:11:48.200961+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:56.864170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":24,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:11:49.202863+0000","last_change":"2023-10-09T05:09:57.865132+0000","last_active":"2023-10-09T05:11:49.202863+0000","last_peered":"2023-10-09T05:11:49.202863+0000","last_clean":"2023-10-09T05:11:49.202863+0000","last_became_active":"2023-10-09T05:09:57.864773+0000","last_became_peered":"2023-10-09T05:09:57.864773+0000","last_unstale":"2023-10-09T05:11:49.202863+0000","last_undegraded":"2023-10-09T05:11:49.202863+0000","last_fullsized":"2023-10-09T05:11:49.202863+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:04.194629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"164'8018","reported_seq":8063,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:12:08.958618+0000","last_change":"2023-10-09T05:11:43.198382+0000","last_active":"2023-10-09T05:12:08.958618+0000","last_peered":"2023-10-09T05:12:08.958618+0000","last_clean":"2023-10-09T05:12:08.958618+0000","last_became_active":"2023-10-09T05:11:43.188578+0000","last_became_peered":"2023-10-09T05:11:43.188578+0000","last_unstale":"2023-10-09T05:12:08.958618+0000","last_undegraded":"2023-10-09T05:12:08.958618+0000","last_fullsized":"2023-10-09T05:12:08.958618+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":8018,"log_dups_size":0,"ondisk_log_size":8018,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81788928,"num_objects":1248,"num_object_clones":0,"num_object_copies":3744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4304,"num_write_kb":177664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":149,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.205432+0000","last_change":"2023-10-09T05:09:57.871040+0000","last_active":"2023-10-09T05:11:48.205432+0000","last_peered":"2023-10-09T05:11:48.205432+0000","last_clean":"2023-10-09T05:11:48.205432+0000","last_became_active":"2023-10-09T05:09:57.866399+0000","last_became_peered":"2023-10-09T05:09:57.866399+0000","last_unstale":"2023-10-09T05:11:48.205432+0000","last_undegraded":"2023-10-09T05:11:48.205432+0000","last_fullsized":"2023-10-09T05:11:48.205432+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:42.545057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":125,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.204818+0000","last_change":"2023-10-09T04:56:56.087320+0000","last_active":"2023-10-09T05:11:48.204818+0000","last_peered":"2023-10-09T05:11:48.204818+0000","last_clean":"2023-10-09T05:11:48.204818+0000","last_became_active":"2023-10-09T04:56:56.086552+0000","last_became_peered":"2023-10-09T04:56:56.086552+0000","last_unstale":"2023-10-09T05:11:48.204818+0000","last_undegraded":"2023-10-09T05:11:48.204818+0000","last_fullsized":"2023-10-09T05:11:48.204818+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:14.205288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":24,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:11:49.202808+0000","last_change":"2023-10-09T05:09:57.843174+0000","last_active":"2023-10-09T05:11:49.202808+0000","last_peered":"2023-10-09T05:11:49.202808+0000","last_clean":"2023-10-09T05:11:49.202808+0000","last_became_active":"2023-10-09T05:09:57.842818+0000","last_became_peered":"2023-10-09T05:09:57.842818+0000","last_unstale":"2023-10-09T05:11:49.202808+0000","last_undegraded":"2023-10-09T05:11:49.202808+0000","last_fullsized":"2023-10-09T05:11:49.202808+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:10.285164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"164'8843","reported_seq":10674,"reported_epoch":164,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:12:09.505935+0000","last_change":"2023-10-09T05:11:47.839849+0000","last_active":"2023-10-09T05:12:09.505935+0000","last_peered":"2023-10-09T05:12:09.505935+0000","last_clean":"2023-10-09T05:09:56.806054+0000","last_became_active":"2023-10-09T05:09:59.097985+0000","last_became_peered":"2023-10-09T05:09:59.097985+0000","last_unstale":"2023-10-09T05:12:09.505935+0000","last_undegraded":"2023-10-09T05:09:58.873107+0000","last_fullsized":"2023-10-09T05:09:58.841848+0000","mapping_epoch":152,"log_start":"149'500","ondisk_log_start":"149'500","created":139,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":8343,"log_dups_size":0,"ondisk_log_size":8343,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:44:00.189725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167706624,"num_objects":2559,"num_object_clones":0,"num_object_copies":7677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1984,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2559,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8843,"num_write_kb":364864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":884,"num_bytes_recovered":40042496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,7],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"5,7","objects":575}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":144,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.193883+0000","last_change":"2023-10-09T05:02:09.645835+0000","last_active":"2023-10-09T05:11:48.193883+0000","last_peered":"2023-10-09T05:11:48.193883+0000","last_clean":"2023-10-09T05:11:48.193883+0000","last_became_active":"2023-10-09T05:02:09.645491+0000","last_became_peered":"2023-10-09T05:02:09.645491+0000","last_unstale":"2023-10-09T05:11:48.193883+0000","last_undegraded":"2023-10-09T05:11:48.193883+0000","last_fullsized":"2023-10-09T05:11:48.193883+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:06:20.823201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":24,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:11:49.203867+0000","last_change":"2023-10-09T05:09:57.854064+0000","last_active":"2023-10-09T05:11:49.203867+0000","last_peered":"2023-10-09T05:11:49.203867+0000","last_clean":"2023-10-09T05:11:49.203867+0000","last_became_active":"2023-10-09T05:09:57.847953+0000","last_became_peered":"2023-10-09T05:09:57.847953+0000","last_unstale":"2023-10-09T05:11:49.203867+0000","last_undegraded":"2023-10-09T05:11:49.203867+0000","last_fullsized":"2023-10-09T05:11:49.203867+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:45.914976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":23,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.195208+0000","last_change":"2023-10-09T05:09:57.865256+0000","last_active":"2023-10-09T05:11:48.195208+0000","last_peered":"2023-10-09T05:11:48.195208+0000","last_clean":"2023-10-09T05:11:48.195208+0000","last_became_active":"2023-10-09T05:09:57.864756+0000","last_became_peered":"2023-10-09T05:09:57.864756+0000","last_unstale":"2023-10-09T05:11:48.195208+0000","last_undegraded":"2023-10-09T05:11:48.195208+0000","last_fullsized":"2023-10-09T05:11:48.195208+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:14.071437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"164'7916","reported_seq":7960,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:12:08.383721+0000","last_change":"2023-10-09T05:11:43.216966+0000","last_active":"2023-10-09T05:12:08.383721+0000","last_peered":"2023-10-09T05:12:08.383721+0000","last_clean":"2023-10-09T05:12:08.383721+0000","last_became_active":"2023-10-09T05:11:43.174565+0000","last_became_peered":"2023-10-09T05:11:43.174565+0000","last_unstale":"2023-10-09T05:12:08.383721+0000","last_undegraded":"2023-10-09T05:12:08.383721+0000","last_fullsized":"2023-10-09T05:12:08.383721+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":7916,"log_dups_size":0,"ondisk_log_size":7916,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83787776,"num_objects":1278,"num_object_clones":0,"num_object_copies":3834,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1278,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4244,"num_write_kb":176736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":129,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.200782+0000","last_change":"2023-10-09T05:01:02.685038+0000","last_active":"2023-10-09T05:11:48.200782+0000","last_peered":"2023-10-09T05:11:48.200782+0000","last_clean":"2023-10-09T05:11:48.200782+0000","last_became_active":"2023-10-09T05:01:02.683622+0000","last_became_peered":"2023-10-09T05:01:02.683622+0000","last_unstale":"2023-10-09T05:11:48.200782+0000","last_undegraded":"2023-10-09T05:11:48.200782+0000","last_fullsized":"2023-10-09T05:11:48.200782+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:35.435592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":502,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.196023+0000","last_change":"2023-10-09T04:56:56.075762+0000","last_active":"2023-10-09T05:11:48.196023+0000","last_peered":"2023-10-09T05:11:48.196023+0000","last_clean":"2023-10-09T05:11:48.196023+0000","last_became_active":"2023-10-09T04:56:56.075263+0000","last_became_peered":"2023-10-09T04:56:56.075263+0000","last_unstale":"2023-10-09T05:11:48.196023+0000","last_undegraded":"2023-10-09T05:11:48.196023+0000","last_fullsized":"2023-10-09T05:11:48.196023+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_clean_scrub_stamp":"2023-10-09T04:36:44.261013+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:02.769164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":109,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.200716+0000","last_change":"2023-10-09T05:02:09.654237+0000","last_active":"2023-10-09T05:11:48.200716+0000","last_peered":"2023-10-09T05:11:48.200716+0000","last_clean":"2023-10-09T05:11:48.200716+0000","last_became_active":"2023-10-09T05:02:09.653909+0000","last_became_peered":"2023-10-09T05:02:09.653909+0000","last_unstale":"2023-10-09T05:11:48.200716+0000","last_undegraded":"2023-10-09T05:11:48.200716+0000","last_fullsized":"2023-10-09T05:11:48.200716+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:02.002366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":107,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.192973+0000","last_change":"2023-10-09T04:44:53.959705+0000","last_active":"2023-10-09T05:11:48.192973+0000","last_peered":"2023-10-09T05:11:48.192973+0000","last_clean":"2023-10-09T05:11:48.192973+0000","last_became_active":"2023-10-09T04:44:53.959092+0000","last_became_peered":"2023-10-09T04:44:53.959092+0000","last_unstale":"2023-10-09T05:11:48.192973+0000","last_undegraded":"2023-10-09T05:11:48.192973+0000","last_fullsized":"2023-10-09T05:11:48.192973+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:31:29.189897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"164'8636","reported_seq":8671,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:12:09.007655+0000","last_change":"2023-10-09T05:09:03.826421+0000","last_active":"2023-10-09T05:12:09.007655+0000","last_peered":"2023-10-09T05:12:09.007655+0000","last_clean":"2023-10-09T05:12:09.007655+0000","last_became_active":"2023-10-09T05:09:03.826107+0000","last_became_peered":"2023-10-09T05:09:03.826107+0000","last_unstale":"2023-10-09T05:12:09.007655+0000","last_undegraded":"2023-10-09T05:12:09.007655+0000","last_fullsized":"2023-10-09T05:12:09.007655+0000","mapping_epoch":139,"log_start":"149'700","ondisk_log_start":"149'700","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":7936,"log_dups_size":0,"ondisk_log_size":7936,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:53:42.881604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171704320,"num_objects":2620,"num_object_clones":0,"num_object_copies":7860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8636,"num_write_kb":360192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":166,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.196009+0000","last_change":"2023-10-09T05:09:57.855069+0000","last_active":"2023-10-09T05:11:48.196009+0000","last_peered":"2023-10-09T05:11:48.196009+0000","last_clean":"2023-10-09T05:11:48.196009+0000","last_became_active":"2023-10-09T05:09:57.854693+0000","last_became_peered":"2023-10-09T05:09:57.854693+0000","last_unstale":"2023-10-09T05:11:48.196009+0000","last_undegraded":"2023-10-09T05:11:48.196009+0000","last_fullsized":"2023-10-09T05:11:48.196009+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:21:27.305940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":119,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.200992+0000","last_change":"2023-10-09T05:02:09.648781+0000","last_active":"2023-10-09T05:11:48.200992+0000","last_peered":"2023-10-09T05:11:48.200992+0000","last_clean":"2023-10-09T05:11:48.200992+0000","last_became_active":"2023-10-09T05:02:09.648078+0000","last_became_peered":"2023-10-09T05:02:09.648078+0000","last_unstale":"2023-10-09T05:11:48.200992+0000","last_undegraded":"2023-10-09T05:11:48.200992+0000","last_fullsized":"2023-10-09T05:11:48.200992+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:35.287091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":153,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:11:49.208087+0000","last_change":"2023-10-09T05:09:57.847731+0000","last_active":"2023-10-09T05:11:49.208087+0000","last_peered":"2023-10-09T05:11:49.208087+0000","last_clean":"2023-10-09T05:11:49.208087+0000","last_became_active":"2023-10-09T05:09:57.847359+0000","last_became_peered":"2023-10-09T05:09:57.847359+0000","last_unstale":"2023-10-09T05:11:49.208087+0000","last_undegraded":"2023-10-09T05:11:49.208087+0000","last_fullsized":"2023-10-09T05:11:49.208087+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:43:30.260279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"164'8790","reported_seq":9119,"reported_epoch":164,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:12:09.400264+0000","last_change":"2023-10-09T05:09:59.091678+0000","last_active":"2023-10-09T05:12:09.400264+0000","last_peered":"2023-10-09T05:12:09.400264+0000","last_clean":"2023-10-09T05:09:56.834673+0000","last_became_active":"2023-10-09T05:09:59.086420+0000","last_became_peered":"2023-10-09T05:09:59.086420+0000","last_unstale":"2023-10-09T05:12:09.400264+0000","last_undegraded":"2023-10-09T05:09:58.874557+0000","last_fullsized":"2023-10-09T05:09:58.845617+0000","mapping_epoch":152,"log_start":"150'2234","ondisk_log_start":"150'2234","created":139,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":6556,"log_dups_size":0,"ondisk_log_size":6556,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:13.864729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171311104,"num_objects":2614,"num_object_clones":0,"num_object_copies":7842,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2673,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2614,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8790,"num_write_kb":364928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":133,"num_bytes_recovered":8716288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4],"avail_no_missing":["7","0","4"],"object_location_counts":[{"shards":"4,7","objects":-59}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":129,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.197409+0000","last_change":"2023-10-09T04:56:56.073473+0000","last_active":"2023-10-09T05:11:48.197409+0000","last_peered":"2023-10-09T05:11:48.197409+0000","last_clean":"2023-10-09T05:11:48.197409+0000","last_became_active":"2023-10-09T04:56:56.073146+0000","last_became_peered":"2023-10-09T04:56:56.073146+0000","last_unstale":"2023-10-09T05:11:48.197409+0000","last_undegraded":"2023-10-09T05:11:48.197409+0000","last_fullsized":"2023-10-09T05:11:48.197409+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:55:25.132822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":24,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:11:49.202986+0000","last_change":"2023-10-09T05:09:57.864289+0000","last_active":"2023-10-09T05:11:49.202986+0000","last_peered":"2023-10-09T05:11:49.202986+0000","last_clean":"2023-10-09T05:11:49.202986+0000","last_became_active":"2023-10-09T05:09:57.863963+0000","last_became_peered":"2023-10-09T05:09:57.863963+0000","last_unstale":"2023-10-09T05:11:49.202986+0000","last_undegraded":"2023-10-09T05:11:49.202986+0000","last_fullsized":"2023-10-09T05:11:49.202986+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:45:02.500193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":105,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.199771+0000","last_change":"2023-10-09T04:56:56.071763+0000","last_active":"2023-10-09T05:11:48.199771+0000","last_peered":"2023-10-09T05:11:48.199771+0000","last_clean":"2023-10-09T05:11:48.199771+0000","last_became_active":"2023-10-09T04:56:56.071429+0000","last_became_peered":"2023-10-09T04:56:56.071429+0000","last_unstale":"2023-10-09T05:11:48.199771+0000","last_undegraded":"2023-10-09T05:11:48.199771+0000","last_fullsized":"2023-10-09T05:11:48.199771+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:53:02.051460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"164'8650","reported_seq":14264,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:12:09.403620+0000","last_change":"2023-10-09T05:11:49.237006+0000","last_active":"2023-10-09T05:12:09.403620+0000","last_peered":"2023-10-09T05:12:09.403620+0000","last_clean":"2023-10-09T05:12:09.403620+0000","last_became_active":"2023-10-09T05:11:49.214120+0000","last_became_peered":"2023-10-09T05:11:49.214120+0000","last_unstale":"2023-10-09T05:12:09.403620+0000","last_undegraded":"2023-10-09T05:12:09.403620+0000","last_fullsized":"2023-10-09T05:12:09.403620+0000","mapping_epoch":163,"log_start":"150'2059","ondisk_log_start":"150'2059","created":139,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":6591,"log_dups_size":0,"ondisk_log_size":6591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:31.708426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169148440,"num_objects":2582,"num_object_clones":0,"num_object_copies":7746,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2582,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8650,"num_write_kb":359361,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2769,"num_bytes_recovered":181469184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":110,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.200434+0000","last_change":"2023-10-09T05:02:09.651149+0000","last_active":"2023-10-09T05:11:48.200434+0000","last_peered":"2023-10-09T05:11:48.200434+0000","last_clean":"2023-10-09T05:11:48.200434+0000","last_became_active":"2023-10-09T05:02:09.650151+0000","last_became_peered":"2023-10-09T05:02:09.650151+0000","last_unstale":"2023-10-09T05:11:48.200434+0000","last_undegraded":"2023-10-09T05:11:48.200434+0000","last_fullsized":"2023-10-09T05:11:48.200434+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:49.100209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":23,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.198376+0000","last_change":"2023-10-09T05:09:57.858617+0000","last_active":"2023-10-09T05:11:48.198376+0000","last_peered":"2023-10-09T05:11:48.198376+0000","last_clean":"2023-10-09T05:11:48.198376+0000","last_became_active":"2023-10-09T05:09:57.858128+0000","last_became_peered":"2023-10-09T05:09:57.858128+0000","last_unstale":"2023-10-09T05:11:48.198376+0000","last_undegraded":"2023-10-09T05:11:48.198376+0000","last_fullsized":"2023-10-09T05:11:48.198376+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:02.502985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":107,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:11:49.239297+0000","last_change":"2023-10-09T05:02:09.650750+0000","last_active":"2023-10-09T05:11:49.239297+0000","last_peered":"2023-10-09T05:11:49.239297+0000","last_clean":"2023-10-09T05:11:49.239297+0000","last_became_active":"2023-10-09T05:02:09.650351+0000","last_became_peered":"2023-10-09T05:02:09.650351+0000","last_unstale":"2023-10-09T05:11:49.239297+0000","last_undegraded":"2023-10-09T05:11:49.239297+0000","last_fullsized":"2023-10-09T05:11:49.239297+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:14.826132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"164'8810","reported_seq":8845,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:12:08.378842+0000","last_change":"2023-10-09T05:09:04.054690+0000","last_active":"2023-10-09T05:12:08.378842+0000","last_peered":"2023-10-09T05:12:08.378842+0000","last_clean":"2023-10-09T05:12:08.378842+0000","last_became_active":"2023-10-09T05:09:04.054340+0000","last_became_peered":"2023-10-09T05:09:04.054340+0000","last_unstale":"2023-10-09T05:12:08.378842+0000","last_undegraded":"2023-10-09T05:12:08.378842+0000","last_fullsized":"2023-10-09T05:12:08.378842+0000","mapping_epoch":139,"log_start":"150'2194","ondisk_log_start":"150'2194","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":6616,"log_dups_size":0,"ondisk_log_size":6616,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:19:34.276593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174850048,"num_objects":2668,"num_object_clones":0,"num_object_copies":8004,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2668,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8810,"num_write_kb":367296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":129,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.193037+0000","last_change":"2023-10-09T05:01:02.660775+0000","last_active":"2023-10-09T05:11:48.193037+0000","last_peered":"2023-10-09T05:11:48.193037+0000","last_clean":"2023-10-09T05:11:48.193037+0000","last_became_active":"2023-10-09T05:01:02.660420+0000","last_became_peered":"2023-10-09T05:01:02.660420+0000","last_unstale":"2023-10-09T05:11:48.193037+0000","last_undegraded":"2023-10-09T05:11:48.193037+0000","last_fullsized":"2023-10-09T05:11:48.193037+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:54.031869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":150,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:11:49.208139+0000","last_change":"2023-10-09T05:09:57.878002+0000","last_active":"2023-10-09T05:11:49.208139+0000","last_peered":"2023-10-09T05:11:49.208139+0000","last_clean":"2023-10-09T05:11:49.208139+0000","last_became_active":"2023-10-09T05:09:57.852792+0000","last_became_peered":"2023-10-09T05:09:57.852792+0000","last_unstale":"2023-10-09T05:11:49.208139+0000","last_undegraded":"2023-10-09T05:11:49.208139+0000","last_fullsized":"2023-10-09T05:11:49.208139+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:18.763275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":107,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.204296+0000","last_change":"2023-10-09T04:44:53.792412+0000","last_active":"2023-10-09T05:11:48.204296+0000","last_peered":"2023-10-09T05:11:48.204296+0000","last_clean":"2023-10-09T05:11:48.204296+0000","last_became_active":"2023-10-09T04:44:53.792009+0000","last_became_peered":"2023-10-09T04:44:53.792009+0000","last_unstale":"2023-10-09T05:11:48.204296+0000","last_undegraded":"2023-10-09T05:11:48.204296+0000","last_fullsized":"2023-10-09T05:11:48.204296+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:39.423339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.11","version":"164'7996","reported_seq":8041,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:12:08.289471+0000","last_change":"2023-10-09T05:11:43.270711+0000","last_active":"2023-10-09T05:12:08.289471+0000","last_peered":"2023-10-09T05:12:08.289471+0000","last_clean":"2023-10-09T05:12:08.289471+0000","last_became_active":"2023-10-09T05:11:43.248111+0000","last_became_peered":"2023-10-09T05:11:43.248111+0000","last_unstale":"2023-10-09T05:12:08.289471+0000","last_undegraded":"2023-10-09T05:12:08.289471+0000","last_fullsized":"2023-10-09T05:12:08.289471+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":160,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":7996,"log_dups_size":0,"ondisk_log_size":7996,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86704128,"num_objects":1323,"num_object_clones":0,"num_object_copies":3969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4273,"num_write_kb":179072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"69'2","reported_seq":109,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:11:49.207793+0000","last_change":"2023-10-09T04:56:56.079768+0000","last_active":"2023-10-09T05:11:49.207793+0000","last_peered":"2023-10-09T05:11:49.207793+0000","last_clean":"2023-10-09T05:11:49.207793+0000","last_became_active":"2023-10-09T04:56:56.077721+0000","last_became_peered":"2023-10-09T04:56:56.077721+0000","last_unstale":"2023-10-09T05:11:49.207793+0000","last_undegraded":"2023-10-09T05:11:49.207793+0000","last_fullsized":"2023-10-09T05:11:49.207793+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:36.384357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":164,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.194150+0000","last_change":"2023-10-09T05:09:57.842586+0000","last_active":"2023-10-09T05:11:48.194150+0000","last_peered":"2023-10-09T05:11:48.194150+0000","last_clean":"2023-10-09T05:11:48.194150+0000","last_became_active":"2023-10-09T05:09:57.842214+0000","last_became_peered":"2023-10-09T05:09:57.842214+0000","last_unstale":"2023-10-09T05:11:48.194150+0000","last_undegraded":"2023-10-09T05:11:48.194150+0000","last_fullsized":"2023-10-09T05:11:48.194150+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:26.074875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.16","version":"164'8018","reported_seq":9217,"reported_epoch":164,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:12:09.505535+0000","last_change":"2023-10-09T05:11:43.296928+0000","last_active":"2023-10-09T05:12:09.505535+0000","last_peered":"2023-10-09T05:12:09.505535+0000","last_clean":"2023-10-09T05:09:56.825417+0000","last_became_active":"2023-10-09T05:11:43.284061+0000","last_became_peered":"2023-10-09T05:11:43.284061+0000","last_unstale":"2023-10-09T05:12:09.505535+0000","last_undegraded":"2023-10-09T05:11:43.261255+0000","last_fullsized":"2023-10-09T05:12:09.505535+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":160,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":8018,"log_dups_size":0,"ondisk_log_size":8018,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83001344,"num_objects":1267,"num_object_clones":0,"num_object_copies":3801,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":937,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1267,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4313,"num_write_kb":178528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":486,"num_bytes_recovered":31817728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":["5","2","6"],"object_location_counts":[{"shards":"1,2,5","objects":1267}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":156,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.200869+0000","last_change":"2023-10-09T05:09:57.901164+0000","last_active":"2023-10-09T05:11:48.200869+0000","last_peered":"2023-10-09T05:11:48.200869+0000","last_clean":"2023-10-09T05:11:48.200869+0000","last_became_active":"2023-10-09T05:09:57.867734+0000","last_became_peered":"2023-10-09T05:09:57.867734+0000","last_unstale":"2023-10-09T05:11:48.200869+0000","last_undegraded":"2023-10-09T05:11:48.200869+0000","last_fullsized":"2023-10-09T05:11:48.200869+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:46:30.541419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":161,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.204969+0000","last_change":"2023-10-09T05:09:57.867865+0000","last_active":"2023-10-09T05:11:48.204969+0000","last_peered":"2023-10-09T05:11:48.204969+0000","last_clean":"2023-10-09T05:11:48.204969+0000","last_became_active":"2023-10-09T05:09:57.866723+0000","last_became_peered":"2023-10-09T05:09:57.866723+0000","last_unstale":"2023-10-09T05:11:48.204969+0000","last_undegraded":"2023-10-09T05:11:48.204969+0000","last_fullsized":"2023-10-09T05:11:48.204969+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:58.946358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.17","version":"164'8223","reported_seq":8263,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:12:09.504073+0000","last_change":"2023-10-09T05:11:43.286661+0000","last_active":"2023-10-09T05:12:09.504073+0000","last_peered":"2023-10-09T05:12:09.504073+0000","last_clean":"2023-10-09T05:12:09.504073+0000","last_became_active":"2023-10-09T05:11:43.272136+0000","last_became_peered":"2023-10-09T05:11:43.272136+0000","last_unstale":"2023-10-09T05:12:09.504073+0000","last_undegraded":"2023-10-09T05:12:09.504073+0000","last_fullsized":"2023-10-09T05:12:09.504073+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":160,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":8223,"log_dups_size":0,"ondisk_log_size":8223,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84836352,"num_objects":1295,"num_object_clones":0,"num_object_copies":3885,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1295,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4421,"num_write_kb":182880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":111,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.200879+0000","last_change":"2023-10-09T04:44:52.793709+0000","last_active":"2023-10-09T05:11:48.200879+0000","last_peered":"2023-10-09T05:11:48.200879+0000","last_clean":"2023-10-09T05:11:48.200879+0000","last_became_active":"2023-10-09T04:44:52.792973+0000","last_became_peered":"2023-10-09T04:44:52.792973+0000","last_unstale":"2023-10-09T05:11:48.200879+0000","last_undegraded":"2023-10-09T05:11:48.200879+0000","last_fullsized":"2023-10-09T05:11:48.200879+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:12.091089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":141,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.204702+0000","last_change":"2023-10-09T05:02:09.649748+0000","last_active":"2023-10-09T05:11:48.204702+0000","last_peered":"2023-10-09T05:11:48.204702+0000","last_clean":"2023-10-09T05:11:48.204702+0000","last_became_active":"2023-10-09T05:02:09.649444+0000","last_became_peered":"2023-10-09T05:02:09.649444+0000","last_unstale":"2023-10-09T05:11:48.204702+0000","last_undegraded":"2023-10-09T05:11:48.204702+0000","last_fullsized":"2023-10-09T05:11:48.204702+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:07.436820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.14","version":"164'8010","reported_seq":13629,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:12:06.795468+0000","last_change":"2023-10-09T05:11:43.275071+0000","last_active":"2023-10-09T05:12:06.795468+0000","last_peered":"2023-10-09T05:12:06.795468+0000","last_clean":"2023-10-09T05:12:06.795468+0000","last_became_active":"2023-10-09T05:11:43.260392+0000","last_became_peered":"2023-10-09T05:11:43.260392+0000","last_unstale":"2023-10-09T05:12:06.795468+0000","last_undegraded":"2023-10-09T05:12:06.795468+0000","last_fullsized":"2023-10-09T05:12:06.795468+0000","mapping_epoch":160,"log_start":"149'600","ondisk_log_start":"149'600","created":160,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":7410,"log_dups_size":0,"ondisk_log_size":7410,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91127808,"num_objects":1391,"num_object_clones":0,"num_object_copies":4173,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1391,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4266,"num_write_kb":180992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1384,"num_bytes_recovered":90701824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"69'2","reported_seq":128,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:11:49.240021+0000","last_change":"2023-10-09T05:02:09.647382+0000","last_active":"2023-10-09T05:11:49.240021+0000","last_peered":"2023-10-09T05:11:49.240021+0000","last_clean":"2023-10-09T05:11:49.240021+0000","last_became_active":"2023-10-09T05:02:09.646803+0000","last_became_peered":"2023-10-09T05:02:09.646803+0000","last_unstale":"2023-10-09T05:11:49.240021+0000","last_undegraded":"2023-10-09T05:11:49.240021+0000","last_fullsized":"2023-10-09T05:11:49.240021+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:23:00.161736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":141,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.375106+0000","last_change":"2023-10-09T05:02:09.649674+0000","last_active":"2023-10-09T05:11:48.375106+0000","last_peered":"2023-10-09T05:11:48.375106+0000","last_clean":"2023-10-09T05:11:48.375106+0000","last_became_active":"2023-10-09T05:02:09.649041+0000","last_became_peered":"2023-10-09T05:02:09.649041+0000","last_unstale":"2023-10-09T05:11:48.375106+0000","last_undegraded":"2023-10-09T05:11:48.375106+0000","last_fullsized":"2023-10-09T05:11:48.375106+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:18.524071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.15","version":"164'8011","reported_seq":8055,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:12:08.770581+0000","last_change":"2023-10-09T05:11:43.264239+0000","last_active":"2023-10-09T05:12:08.770581+0000","last_peered":"2023-10-09T05:12:08.770581+0000","last_clean":"2023-10-09T05:12:08.770581+0000","last_became_active":"2023-10-09T05:11:43.246388+0000","last_became_peered":"2023-10-09T05:11:43.246388+0000","last_unstale":"2023-10-09T05:12:08.770581+0000","last_undegraded":"2023-10-09T05:12:08.770581+0000","last_fullsized":"2023-10-09T05:12:08.770581+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":160,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:09:02.801471+0000","last_clean_scrub_stamp":"2023-10-09T05:09:02.801471+0000","objects_scrubbed":0,"log_size":8011,"log_dups_size":0,"ondisk_log_size":8011,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86671360,"num_objects":1323,"num_object_clones":0,"num_object_copies":3969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4294,"num_write_kb":179712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"69'5","reported_seq":129,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.200165+0000","last_change":"2023-10-09T05:01:02.672596+0000","last_active":"2023-10-09T05:11:48.200165+0000","last_peered":"2023-10-09T05:11:48.200165+0000","last_clean":"2023-10-09T05:11:48.200165+0000","last_became_active":"2023-10-09T05:01:02.672128+0000","last_became_peered":"2023-10-09T05:01:02.672128+0000","last_unstale":"2023-10-09T05:11:48.200165+0000","last_undegraded":"2023-10-09T05:11:48.200165+0000","last_fullsized":"2023-10-09T05:11:48.200165+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:17:05.887502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":144,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T05:11:48.204306+0000","last_change":"2023-10-09T05:09:57.866732+0000","last_active":"2023-10-09T05:11:48.204306+0000","last_peered":"2023-10-09T05:11:48.204306+0000","last_clean":"2023-10-09T05:11:48.204306+0000","last_became_active":"2023-10-09T05:09:57.866117+0000","last_became_peered":"2023-10-09T05:09:57.866117+0000","last_unstale":"2023-10-09T05:11:48.204306+0000","last_undegraded":"2023-10-09T05:11:48.204306+0000","last_fullsized":"2023-10-09T05:11:48.204306+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:49:14.815342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":38,"num_read_kb":40,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":9,"num_pg":26,"stat_sum":{"num_bytes":2737373208,"num_objects":41770,"num_object_clones":0,"num_object_copies":125310,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12708,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":41770,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":138164,"num_write_kb":5757825,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12297,"num_bytes_recovered":665649152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10062409728,"data_stored":10062397512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":202921,"ondisk_log_size":202921,"up":78,"acting":75,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234359,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2249896,"kb_used_data":1660128,"kb_used_omap":0,"kb_used_meta":589760,"kb_avail":91495256,"statfs":{"total":95995035648,"available":93691142144,"internally_reserved":0,"allocated":1699971072,"data_stored":1699321415,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603914240},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:11:26 2023","interfaces":[{"interface":"back","average":{"1min":2.9950000000000001,"5min":1.49,"15min":1.4990000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.22900000000000001},"max":{"1min":22.988,"5min":22.988,"15min":22.988},"last":2.9769999999999999},{"interface":"front","average":{"1min":2.778,"5min":1.774,"15min":1.7809999999999999},"min":{"1min":0.39500000000000002,"5min":0.317,"15min":0.27800000000000002},"max":{"1min":22.835000000000001,"5min":22.835000000000001,"15min":22.835000000000001},"last":3.0390000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:11:12 2023","interfaces":[{"interface":"back","average":{"1min":3.6320000000000001,"5min":1.4059999999999999,"15min":2.0840000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.28699999999999998},"max":{"1min":35.134,"5min":35.134,"15min":135.322},"last":3.0299999999999998},{"interface":"front","average":{"1min":3.3519999999999999,"5min":1.6830000000000001,"15min":2.274},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.30499999999999999},"max":{"1min":35.116999999999997,"5min":35.116999999999997,"15min":135.09},"last":3.1739999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:11:26 2023","interfaces":[{"interface":"back","average":{"1min":1.3480000000000001,"5min":1.2350000000000001,"15min":1.389},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.315},"max":{"1min":4.4009999999999998,"5min":9.9299999999999997,"15min":11.391},"last":3.21},{"interface":"front","average":{"1min":1.0549999999999999,"5min":1.0309999999999999,"15min":1.25},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.8319999999999999,"5min":3.8319999999999999,"15min":6.6820000000000004},"last":0.442}]},{"osd":3,"last update":"Mon Oct 9 05:11:26 2023","interfaces":[{"interface":"back","average":{"1min":5.7030000000000003,"5min":2.327,"15min":1.667},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.317},"max":{"1min":38.125999999999998,"5min":38.125999999999998,"15min":38.125999999999998},"last":0.48599999999999999},{"interface":"front","average":{"1min":5.3079999999999998,"5min":2.298,"15min":1.9630000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":38.170000000000002,"5min":38.170000000000002,"15min":38.170000000000002},"last":3.29}]},{"osd":4,"last update":"Mon Oct 9 05:11:12 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":0.94699999999999995,"15min":1.03},"min":{"1min":0.40699999999999997,"5min":0.33900000000000002,"15min":0.28699999999999998},"max":{"1min":4.2880000000000003,"5min":4.2880000000000003,"15min":11.409000000000001},"last":2.931},{"interface":"front","average":{"1min":0.92600000000000005,"5min":1.329,"15min":1.415},"min":{"1min":0.46600000000000003,"5min":0.41199999999999998,"15min":0.318},"max":{"1min":3.887,"5min":9.8620000000000001,"15min":15.763},"last":3.0910000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:11:12 2023","interfaces":[{"interface":"back","average":{"1min":1.51,"5min":1.24,"15min":1.3759999999999999},"min":{"1min":0.44900000000000001,"5min":0.42999999999999999,"15min":0.34399999999999997},"max":{"1min":4.5380000000000003,"5min":4.5380000000000003,"15min":8.3569999999999993},"last":1.3660000000000001},{"interface":"front","average":{"1min":1.9770000000000001,"5min":1.4039999999999999,"15min":1.3169999999999999},"min":{"1min":0.37,"5min":0.37,"15min":0.32900000000000001},"max":{"1min":10.103,"5min":10.103,"15min":10.103},"last":2.8839999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:11:12 2023","interfaces":[{"interface":"back","average":{"1min":1.2010000000000001,"5min":1.163,"15min":1.236},"min":{"1min":0.39600000000000002,"5min":0.38500000000000001,"15min":0.318},"max":{"1min":8.4130000000000003,"5min":8.4130000000000003,"15min":8.4130000000000003},"last":3.0049999999999999},{"interface":"front","average":{"1min":1.1279999999999999,"5min":1.4870000000000001,"15min":1.4910000000000001},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.35499999999999998},"max":{"1min":4.0140000000000002,"5min":10.106,"15min":10.106},"last":3.1400000000000001}]}]},{"osd":6,"up_from":46,"seq":197568496000,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1019036,"kb_used_data":709460,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":92726116,"statfs":{"total":95995035648,"available":94951542784,"internally_reserved":0,"allocated":726487040,"data_stored":725849824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:11:19 2023","interfaces":[{"interface":"back","average":{"1min":1.696,"5min":1.8799999999999999,"15min":1.8180000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.214},"max":{"1min":13.983000000000001,"5min":98.275000000000006,"15min":98.275000000000006},"last":0.66700000000000004},{"interface":"front","average":{"1min":1.7849999999999999,"5min":1.984,"15min":1.9359999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.27200000000000002},"max":{"1min":14.028,"5min":98.334999999999994,"15min":98.334999999999994},"last":0.69299999999999995}]},{"osd":1,"last update":"Mon Oct 9 05:11:17 2023","interfaces":[{"interface":"back","average":{"1min":1.3480000000000001,"5min":1.0309999999999999,"15min":1.4610000000000001},"min":{"1min":0.377,"5min":0.30299999999999999,"15min":0.25800000000000001},"max":{"1min":5.8390000000000004,"5min":8.8360000000000003,"15min":10.118},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.272,"5min":1.0760000000000001,"15min":1.4570000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.27900000000000003},"max":{"1min":4.0679999999999996,"5min":11.103,"15min":12.417},"last":0.746}]},{"osd":2,"last update":"Mon Oct 9 05:11:17 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.92000000000000004,"15min":1.407},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.28999999999999998},"max":{"1min":2.7069999999999999,"5min":2.7069999999999999,"15min":8.9730000000000008},"last":0.621},{"interface":"front","average":{"1min":0.73799999999999999,"5min":0.83899999999999997,"15min":1.24},"min":{"1min":0.375,"5min":0.36199999999999999,"15min":0.30099999999999999},"max":{"1min":2.2170000000000001,"5min":2.2170000000000001,"15min":8.3810000000000002},"last":0.42799999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:11:19 2023","interfaces":[{"interface":"back","average":{"1min":1.649,"5min":1.171,"15min":1.5980000000000001},"min":{"1min":0.436,"5min":0.40699999999999997,"15min":0.315},"max":{"1min":6.899,"5min":10.544,"15min":10.544},"last":1.508},{"interface":"front","average":{"1min":1.7809999999999999,"5min":1.3100000000000001,"15min":1.4950000000000001},"min":{"1min":0.39000000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.8959999999999999,"5min":6.4109999999999996,"15min":9.7720000000000002},"last":0.71899999999999997}]},{"osd":4,"last update":"Mon Oct 9 05:11:19 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":0.89900000000000002,"15min":1.111},"min":{"1min":0.45100000000000001,"5min":0.36099999999999999,"15min":0.28199999999999997},"max":{"1min":3.988,"5min":3.988,"15min":16.119},"last":0.55900000000000005},{"interface":"front","average":{"1min":1.028,"5min":1.0649999999999999,"15min":1.2310000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.247},"max":{"1min":3.6230000000000002,"5min":3.6230000000000002,"15min":11.272},"last":0.53800000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:11:19 2023","interfaces":[{"interface":"back","average":{"1min":1.349,"5min":1.161,"15min":1.3100000000000001},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.29299999999999998},"max":{"1min":5.7400000000000002,"5min":6.2199999999999998,"15min":7.7149999999999999},"last":0.60999999999999999},{"interface":"front","average":{"1min":1.0549999999999999,"5min":1.1659999999999999,"15min":1.429},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":3.7349999999999999,"5min":3.7349999999999999,"15min":10.537000000000001},"last":0.83599999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:11:58 2023","interfaces":[{"interface":"back","average":{"1min":1.6859999999999999,"5min":1.3049999999999999,"15min":1.3440000000000001},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.30599999999999999},"max":{"1min":7.5590000000000002,"5min":8.1120000000000001,"15min":9.3780000000000001},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.9339999999999999,"5min":1.4530000000000001,"15min":1.4450000000000001},"min":{"1min":0.441,"5min":0.441,"15min":0.375},"max":{"1min":7.6040000000000001,"5min":8.1859999999999999,"15min":9.4060000000000006},"last":0.67600000000000005}]}]},{"osd":1,"up_from":96,"seq":412316860591,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2081200,"kb_used_data":1586728,"kb_used_omap":4,"kb_used_meta":494459,"kb_avail":91663952,"statfs":{"total":95995035648,"available":93863886848,"internally_reserved":0,"allocated":1624809472,"data_stored":1623922639,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":506326926},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:11:07 2023","interfaces":[{"interface":"back","average":{"1min":1.6379999999999999,"5min":0.85199999999999998,"15min":1.6870000000000001},"min":{"1min":0.24299999999999999,"5min":0.22500000000000001,"15min":0.19400000000000001},"max":{"1min":6.2699999999999996,"5min":6.2699999999999996,"15min":22.382999999999999},"last":3.4750000000000001},{"interface":"front","average":{"1min":1.9379999999999999,"5min":1.016,"15min":2.0230000000000001},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":7.0259999999999998,"5min":7.0259999999999998,"15min":10.268000000000001},"last":3.173}]},{"osd":2,"last update":"Mon Oct 9 05:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.579,"5min":0.91100000000000003,"15min":1.444},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.219},"max":{"1min":5.0279999999999996,"5min":6.3259999999999996,"15min":10.115},"last":2.274},{"interface":"front","average":{"1min":1.3979999999999999,"5min":0.96599999999999997,"15min":1.4410000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.252},"max":{"1min":5.0629999999999997,"5min":6.4050000000000002,"15min":11.917},"last":2.3140000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:11:25 2023","interfaces":[{"interface":"back","average":{"1min":4.6070000000000002,"5min":2.0219999999999998,"15min":2.1960000000000002},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.23100000000000001},"max":{"1min":61.280000000000001,"5min":61.280000000000001,"15min":61.280000000000001},"last":3.3130000000000002},{"interface":"front","average":{"1min":4.6929999999999996,"5min":2.2469999999999999,"15min":1.8049999999999999},"min":{"1min":0.52700000000000002,"5min":0.42599999999999999,"15min":0.24399999999999999},"max":{"1min":65.046000000000006,"5min":65.046000000000006,"15min":65.046000000000006},"last":2.3450000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:11:07 2023","interfaces":[{"interface":"back","average":{"1min":1.984,"5min":1.214,"15min":1.53},"min":{"1min":0.5,"5min":0.40000000000000002,"15min":0.34200000000000003},"max":{"1min":6.3520000000000003,"5min":6.3520000000000003,"15min":9.9749999999999996},"last":4.8810000000000002},{"interface":"front","average":{"1min":1.698,"5min":1.103,"15min":1.498},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.30099999999999999},"max":{"1min":6.5339999999999998,"5min":6.5339999999999998,"15min":22.350000000000001},"last":4.9699999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:11:07 2023","interfaces":[{"interface":"back","average":{"1min":2.75,"5min":1.4350000000000001,"15min":1.821},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.32500000000000001},"max":{"1min":9.0530000000000008,"5min":9.0530000000000008,"15min":12.454000000000001},"last":3.395},{"interface":"front","average":{"1min":2.4569999999999999,"5min":1.345,"15min":1.8260000000000001},"min":{"1min":0.59099999999999997,"5min":0.51000000000000001,"15min":0.316},"max":{"1min":6.6660000000000004,"5min":6.6660000000000004,"15min":10.035},"last":2.661}]},{"osd":6,"last update":"Mon Oct 9 05:11:07 2023","interfaces":[{"interface":"back","average":{"1min":1.385,"5min":1.1140000000000001,"15min":1.8300000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.7050000000000001,"5min":4.7050000000000001,"15min":10.409000000000001},"last":3.278},{"interface":"front","average":{"1min":2.1389999999999998,"5min":1.3089999999999999,"15min":1.6819999999999999},"min":{"1min":0.54700000000000004,"5min":0.44800000000000001,"15min":0.35099999999999998},"max":{"1min":6.4690000000000003,"5min":6.4690000000000003,"15min":10.007},"last":2.6349999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:11:07 2023","interfaces":[{"interface":"back","average":{"1min":2.323,"5min":1.468,"15min":1.75},"min":{"1min":0.64000000000000001,"5min":0.46899999999999997,"15min":0.38800000000000001},"max":{"1min":6.4969999999999999,"5min":6.4969999999999999,"15min":10.061},"last":2.6059999999999999},{"interface":"front","average":{"1min":1.8620000000000001,"5min":1.2969999999999999,"15min":1.6950000000000001},"min":{"1min":0.44,"5min":0.44,"15min":0.36399999999999999},"max":{"1min":6.3209999999999997,"5min":6.3209999999999997,"15min":22.175999999999998},"last":3.6739999999999999}]}]},{"osd":0,"up_from":88,"seq":377957122233,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1847964,"kb_used_data":1421524,"kb_used_omap":5,"kb_used_meta":426426,"kb_avail":91897188,"statfs":{"total":95995035648,"available":94102720512,"internally_reserved":0,"allocated":1455640576,"data_stored":1454772202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":436660858},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:11:04 2023","interfaces":[{"interface":"back","average":{"1min":7.8019999999999996,"5min":2.1219999999999999,"15min":2.282},"min":{"1min":0.34300000000000003,"5min":0.24399999999999999,"15min":0.217},"max":{"1min":109.88,"5min":109.88,"15min":109.88},"last":0.32700000000000001},{"interface":"front","average":{"1min":7.641,"5min":2.0800000000000001,"15min":2.1789999999999998},"min":{"1min":0.25,"5min":0.25,"15min":0.218},"max":{"1min":109.435,"5min":109.435,"15min":109.435},"last":0.87}]},{"osd":2,"last update":"Mon Oct 9 05:11:21 2023","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":0.76400000000000001,"15min":1.6120000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":2.7010000000000001,"5min":3.0390000000000001,"15min":9.1300000000000008},"last":0.81899999999999995},{"interface":"front","average":{"1min":1.379,"5min":0.84799999999999998,"15min":1.4359999999999999},"min":{"1min":0.43099999999999999,"5min":0.29099999999999998,"15min":0.23300000000000001},"max":{"1min":4.7220000000000004,"5min":4.7220000000000004,"15min":10.423},"last":0.40899999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:11:24 2023","interfaces":[{"interface":"back","average":{"1min":1.6659999999999999,"5min":1.403,"15min":1.5149999999999999},"min":{"1min":0.40699999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":7.625,"5min":34.292000000000002,"15min":34.292000000000002},"last":40.649999999999999},{"interface":"front","average":{"1min":1.292,"5min":1.381,"15min":1.4019999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":4.8220000000000001,"5min":34.674999999999997,"15min":34.674999999999997},"last":42.758000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:11:24 2023","interfaces":[{"interface":"back","average":{"1min":0.86199999999999999,"5min":1.03,"15min":1.3759999999999999},"min":{"1min":0.47599999999999998,"5min":0.34599999999999997,"15min":0.30099999999999999},"max":{"1min":2.778,"5min":3.1899999999999999,"15min":8.7449999999999992},"last":0.90200000000000002},{"interface":"front","average":{"1min":1.768,"5min":1.1899999999999999,"15min":1.4399999999999999},"min":{"1min":0.45900000000000002,"5min":0.41899999999999998,"15min":0.33700000000000002},"max":{"1min":6.4649999999999999,"5min":6.4649999999999999,"15min":7.202},"last":0.504}]},{"osd":5,"last update":"Mon Oct 9 05:11:24 2023","interfaces":[{"interface":"back","average":{"1min":1.669,"5min":1.1879999999999999,"15min":1.5580000000000001},"min":{"1min":0.34699999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.9279999999999999,"5min":5.9279999999999999,"15min":12.526999999999999},"last":0.58499999999999996},{"interface":"front","average":{"1min":1.5189999999999999,"5min":1.3,"15min":1.643},"min":{"1min":0.375,"5min":0.375,"15min":0.316},"max":{"1min":6.9349999999999996,"5min":6.9349999999999996,"15min":10.148},"last":0.93300000000000005}]},{"osd":6,"last update":"Mon Oct 9 05:11:24 2023","interfaces":[{"interface":"back","average":{"1min":1.7589999999999999,"5min":1.22,"15min":1.589},"min":{"1min":0.496,"5min":0.45900000000000002,"15min":0.371},"max":{"1min":6.79,"5min":6.79,"15min":8.8710000000000004},"last":0.77100000000000002},{"interface":"front","average":{"1min":1.599,"5min":1.1779999999999999,"15min":1.4670000000000001},"min":{"1min":0.45100000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":7.282,"5min":7.282,"15min":10.429},"last":0.78700000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:11:24 2023","interfaces":[{"interface":"back","average":{"1min":2.5800000000000001,"5min":1.494,"15min":1.5469999999999999},"min":{"1min":0.55300000000000005,"5min":0.41499999999999998,"15min":0.36599999999999999},"max":{"1min":7.7380000000000004,"5min":7.7380000000000004,"15min":10.314},"last":0.70799999999999996},{"interface":"front","average":{"1min":2.2629999999999999,"5min":1.417,"15min":1.7729999999999999},"min":{"1min":0.503,"5min":0.47099999999999997,"15min":0.39500000000000002},"max":{"1min":6.9960000000000004,"5min":6.9960000000000004,"15min":10.738},"last":1.3180000000000001}]}]},{"osd":2,"up_from":106,"seq":455266533511,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":962668,"kb_used_data":704548,"kb_used_omap":7,"kb_used_meta":258104,"kb_avail":92782484,"statfs":{"total":95995035648,"available":95009263616,"internally_reserved":0,"allocated":721457152,"data_stored":720629610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":264298577},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:11:12 2023","interfaces":[{"interface":"back","average":{"1min":3.8719999999999999,"5min":1.2709999999999999,"15min":1.3759999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.215},"max":{"1min":53.088000000000001,"5min":53.088000000000001,"15min":53.088000000000001},"last":0.27600000000000002},{"interface":"front","average":{"1min":3.3610000000000002,"5min":1.2230000000000001,"15min":1.421},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":46.762,"5min":46.762,"15min":46.762},"last":0.39900000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:11:12 2023","interfaces":[{"interface":"back","average":{"1min":1.5349999999999999,"5min":0.88500000000000001,"15min":1.22},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.29299999999999998},"max":{"1min":6.375,"5min":6.375,"15min":20.539999999999999},"last":3.0129999999999999},{"interface":"front","average":{"1min":1.6339999999999999,"5min":0.89700000000000002,"15min":1.4299999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.23100000000000001},"max":{"1min":5.0449999999999999,"5min":6.0099999999999998,"15min":24.058},"last":0.63500000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:11:29 2023","interfaces":[{"interface":"back","average":{"1min":1.1180000000000001,"5min":0.90300000000000002,"15min":1.782},"min":{"1min":0.34100000000000003,"5min":0.30599999999999999,"15min":0.27300000000000002},"max":{"1min":3.1549999999999998,"5min":4.4480000000000004,"15min":20.907},"last":0.67500000000000004},{"interface":"front","average":{"1min":1.181,"5min":0.85199999999999998,"15min":1.554},"min":{"1min":0.30099999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.6739999999999999,"5min":3.6739999999999999,"15min":23.574999999999999},"last":0.53400000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:11:12 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":0.95099999999999996,"15min":1.361},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.33300000000000002},"max":{"1min":3.2869999999999999,"5min":3.2869999999999999,"15min":20.712},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.69999999999999996,"5min":0.86699999999999999,"15min":1.3939999999999999},"min":{"1min":0.35899999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":3.1139999999999999,"5min":3.1139999999999999,"15min":23.547000000000001},"last":0.60899999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:11:12 2023","interfaces":[{"interface":"back","average":{"1min":1.3520000000000001,"5min":1.0149999999999999,"15min":1.4850000000000001},"min":{"1min":0.35799999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":5.532,"5min":5.532,"15min":20.782},"last":0.58799999999999997},{"interface":"front","average":{"1min":1.157,"5min":1.0820000000000001,"15min":1.7629999999999999},"min":{"1min":0.438,"5min":0.438,"15min":0.36299999999999999},"max":{"1min":3.4620000000000002,"5min":3.9700000000000002,"15min":20.792000000000002},"last":0.89900000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:11:12 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":0.995,"15min":1.4790000000000001},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.34999999999999998},"max":{"1min":4.5259999999999998,"5min":4.5259999999999998,"15min":23.510999999999999},"last":0.56999999999999995},{"interface":"front","average":{"1min":1.1930000000000001,"5min":1.0429999999999999,"15min":1.4430000000000001},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":4.4930000000000003,"5min":4.4930000000000003,"15min":20.684999999999999},"last":0.51800000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:11:12 2023","interfaces":[{"interface":"back","average":{"1min":1.224,"5min":1.0640000000000001,"15min":1.4359999999999999},"min":{"1min":0.46300000000000002,"5min":0.44,"15min":0.44},"max":{"1min":3.431,"5min":3.431,"15min":20.887},"last":0.55100000000000005},{"interface":"front","average":{"1min":1.3540000000000001,"5min":1.163,"15min":1.448},"min":{"1min":0.59199999999999997,"5min":0.53800000000000003,"15min":0.45400000000000001},"max":{"1min":3.3860000000000001,"5min":3.3860000000000001,"15min":20.863},"last":1.2290000000000001}]}]},{"osd":3,"up_from":123,"seq":528280977530,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1946608,"kb_used_data":1551400,"kb_used_omap":2,"kb_used_meta":395197,"kb_avail":91798544,"statfs":{"total":95995035648,"available":94001709056,"internally_reserved":0,"allocated":1588633600,"data_stored":1587860600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":404681781},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:11:25 2023","interfaces":[{"interface":"back","average":{"1min":3.3069999999999999,"5min":1.4650000000000001,"15min":1.897},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.187},"max":{"1min":9.2129999999999992,"5min":9.2129999999999992,"15min":14.141},"last":1.079},{"interface":"front","average":{"1min":1.867,"5min":1.135,"15min":2.1560000000000001},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.184},"max":{"1min":8.0150000000000006,"5min":8.0150000000000006,"15min":14.045999999999999},"last":1.734}]},{"osd":1,"last update":"Mon Oct 9 05:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.9750000000000001,"5min":1.2490000000000001,"15min":2.0510000000000002},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":7.2229999999999999,"5min":14.808,"15min":14.808},"last":1.3220000000000001},{"interface":"front","average":{"1min":2.8100000000000001,"5min":1.212,"15min":2.1099999999999999},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.20999999999999999},"max":{"1min":9.0510000000000002,"5min":9.0510000000000002,"15min":17.498999999999999},"last":1.1259999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.9199999999999999,"5min":1.1080000000000001,"15min":1.8169999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.23699999999999999},"max":{"1min":8.0619999999999994,"5min":8.0619999999999994,"15min":9.6129999999999995},"last":1.3440000000000001},{"interface":"front","average":{"1min":1.393,"5min":1.147,"15min":1.7},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":5.3559999999999999,"5min":14.754,"15min":14.754},"last":1.881}]},{"osd":4,"last update":"Mon Oct 9 05:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3899999999999999,"5min":1.3089999999999999,"15min":1.6220000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.9530000000000003,"5min":14.833,"15min":14.833},"last":1.9610000000000001},{"interface":"front","average":{"1min":1.968,"5min":1.2889999999999999,"15min":1.323},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.34999999999999998},"max":{"1min":6.6909999999999998,"5min":6.6909999999999998,"15min":9.4749999999999996},"last":1.542}]},{"osd":5,"last update":"Mon Oct 9 05:11:25 2023","interfaces":[{"interface":"back","average":{"1min":2.0750000000000002,"5min":1.3180000000000001,"15min":1.3919999999999999},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38300000000000001},"max":{"1min":6.7290000000000001,"5min":6.7290000000000001,"15min":9.5079999999999991},"last":3.2879999999999998},{"interface":"front","average":{"1min":3.0409999999999999,"5min":1.6459999999999999,"15min":1.98},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":10.456,"5min":10.456,"15min":10.456},"last":1.427}]},{"osd":6,"last update":"Mon Oct 9 05:11:25 2023","interfaces":[{"interface":"back","average":{"1min":2.153,"5min":1.4490000000000001,"15min":1.7749999999999999},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.33200000000000002},"max":{"1min":8.0899999999999999,"5min":8.0899999999999999,"15min":9.6590000000000007},"last":1.302},{"interface":"front","average":{"1min":1.671,"5min":1.4610000000000001,"15min":1.8620000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.34699999999999998},"max":{"1min":5.0270000000000001,"5min":14.881,"15min":14.881},"last":1.9139999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.492,"5min":1.5309999999999999,"15min":2.008},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":5.056,"5min":14.856,"15min":14.856},"last":1.9399999999999999},{"interface":"front","average":{"1min":2.3799999999999999,"5min":1.571,"15min":1.6080000000000001},"min":{"1min":0.40600000000000003,"5min":0.39700000000000002,"15min":0.38800000000000001},"max":{"1min":8.1370000000000005,"5min":8.1370000000000005,"15min":9.5359999999999996},"last":5.1890000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019287,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1224996,"kb_used_data":639324,"kb_used_omap":0,"kb_used_meta":585664,"kb_avail":92520156,"statfs":{"total":95995035648,"available":94740639744,"internally_reserved":0,"allocated":654667776,"data_stored":654022165,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599719936},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:11:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":2.4910000000000001,"15min":2.8490000000000002},"min":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.22700000000000001},"max":{"1min":2.984,"5min":102.669,"15min":131.30699999999999},"last":37.503999999999998},{"interface":"front","average":{"1min":1.1850000000000001,"5min":2.5350000000000001,"15min":2.7959999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.26900000000000002},"max":{"1min":3.004,"5min":102.72499999999999,"15min":131.36600000000001},"last":37.536000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:11:08 2023","interfaces":[{"interface":"back","average":{"1min":1.351,"5min":1.016,"15min":1.764},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.28999999999999998},"max":{"1min":4.9770000000000003,"5min":4.9770000000000003,"15min":59.408000000000001},"last":0.42999999999999999},{"interface":"front","average":{"1min":1.2130000000000001,"5min":0.999,"15min":1.784},"min":{"1min":0.378,"5min":0.34599999999999997,"15min":0.28199999999999997},"max":{"1min":4.8700000000000001,"5min":4.8700000000000001,"15min":60.207000000000001},"last":0.45600000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:11:18 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.85999999999999999,"15min":1.1659999999999999},"min":{"1min":0.37,"5min":0.36199999999999999,"15min":0.307},"max":{"1min":1.6779999999999999,"5min":2.407,"15min":10.196},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.83299999999999996,"5min":0.93100000000000005,"15min":1.093},"min":{"1min":0.41599999999999998,"5min":0.35899999999999999,"15min":0.32800000000000001},"max":{"1min":4.069,"5min":4.069,"15min":6.3810000000000002},"last":0.55500000000000005}]},{"osd":3,"last update":"Mon Oct 9 05:11:21 2023","interfaces":[{"interface":"back","average":{"1min":1.857,"5min":1.2450000000000001,"15min":1.637},"min":{"1min":0.48999999999999999,"5min":0.38400000000000001,"15min":0.309},"max":{"1min":8.3559999999999999,"5min":8.3559999999999999,"15min":10.555999999999999},"last":37.585999999999999},{"interface":"front","average":{"1min":1.4330000000000001,"5min":1.091,"15min":1.536},"min":{"1min":0.46899999999999997,"5min":0.373,"15min":0.35399999999999998},"max":{"1min":8.1379999999999999,"5min":8.1379999999999999,"15min":21.036999999999999},"last":37.658999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:11:21 2023","interfaces":[{"interface":"back","average":{"1min":0.83699999999999997,"5min":1.002,"15min":1.2130000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.314},"max":{"1min":2.3370000000000002,"5min":2.3370000000000002,"15min":10.214},"last":0.67400000000000004},{"interface":"front","average":{"1min":1.1890000000000001,"5min":1.194,"15min":1.23},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.31900000000000001},"max":{"1min":5.7350000000000003,"5min":5.7350000000000003,"15min":6.8310000000000004},"last":0.69899999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:11:08 2023","interfaces":[{"interface":"back","average":{"1min":1.3,"5min":1.0680000000000001,"15min":1.323},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.32800000000000001},"max":{"1min":5.2249999999999996,"5min":5.2249999999999996,"15min":11.557},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.76400000000000001,"5min":1.077,"15min":1.2629999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.34699999999999998},"max":{"1min":3.5609999999999999,"5min":3.5609999999999999,"15min":10.689},"last":0.72299999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:11:08 2023","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":1.0840000000000001,"15min":1.117},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.36299999999999999},"max":{"1min":2.0459999999999998,"5min":3.6989999999999998,"15min":8.6449999999999996},"last":0.78800000000000003},{"interface":"front","average":{"1min":1.9179999999999999,"5min":1.2509999999999999,"15min":1.333},"min":{"1min":0.443,"5min":0.443,"15min":0.38600000000000001},"max":{"1min":8.1270000000000007,"5min":8.1270000000000007,"15min":8.1270000000000007},"last":0.75700000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757643,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2550616,"kb_used_data":1647696,"kb_used_omap":0,"kb_used_meta":902848,"kb_avail":91194536,"statfs":{"total":95995035648,"available":93383204864,"internally_reserved":0,"allocated":1687240704,"data_stored":1686603149,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924516352},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:11:17 2023","interfaces":[{"interface":"back","average":{"1min":1.593,"5min":1.47,"15min":1.9630000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.24399999999999999},"max":{"1min":6.3840000000000003,"5min":40.097999999999999,"15min":77.745999999999995},"last":0.89900000000000002},{"interface":"front","average":{"1min":1.647,"5min":1.5860000000000001,"15min":2.0640000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.4459999999999997,"5min":40.161000000000001,"15min":77.820999999999998},"last":0.99099999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:11:17 2023","interfaces":[{"interface":"back","average":{"1min":1.492,"5min":1.1060000000000001,"15min":1.6519999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.26900000000000002},"max":{"1min":7.1500000000000004,"5min":7.1500000000000004,"15min":13.401999999999999},"last":0.93200000000000005},{"interface":"front","average":{"1min":1.7869999999999999,"5min":1.169,"15min":1.6579999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.26800000000000002},"max":{"1min":6.9699999999999998,"5min":6.9699999999999998,"15min":13.398},"last":0.79200000000000004}]},{"osd":2,"last update":"Mon Oct 9 05:11:31 2023","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":0.96399999999999997,"15min":1.1519999999999999},"min":{"1min":0.376,"5min":0.374,"15min":0.28499999999999998},"max":{"1min":4.9429999999999996,"5min":4.9429999999999996,"15min":13.436},"last":0.63500000000000001},{"interface":"front","average":{"1min":2.593,"5min":1.425,"15min":1.411},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.32000000000000001},"max":{"1min":8.6839999999999993,"5min":8.6839999999999993,"15min":9.4949999999999992},"last":0.67800000000000005}]},{"osd":3,"last update":"Mon Oct 9 05:11:31 2023","interfaces":[{"interface":"back","average":{"1min":1.718,"5min":1.256,"15min":1.78},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.32500000000000001},"max":{"1min":6.4930000000000003,"5min":6.4930000000000003,"15min":9.1010000000000009},"last":1.339},{"interface":"front","average":{"1min":2.875,"5min":1.585,"15min":1.786},"min":{"1min":0.46700000000000003,"5min":0.41099999999999998,"15min":0.36799999999999999},"max":{"1min":11.016999999999999,"5min":11.016999999999999,"15min":11.859999999999999},"last":0.60999999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:11:17 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.048,"15min":1.2230000000000001},"min":{"1min":0.33500000000000002,"5min":0.33100000000000002,"15min":0.25},"max":{"1min":4.7720000000000002,"5min":4.7720000000000002,"15min":14.781000000000001},"last":0.58499999999999996},{"interface":"front","average":{"1min":2.2789999999999999,"5min":1.381,"15min":1.4490000000000001},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.34499999999999997},"max":{"1min":8.7759999999999998,"5min":8.7759999999999998,"15min":9.625},"last":0.55900000000000005}]},{"osd":6,"last update":"Mon Oct 9 05:11:17 2023","interfaces":[{"interface":"back","average":{"1min":2.452,"5min":1.46,"15min":1.6160000000000001},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.34699999999999998},"max":{"1min":8.7550000000000008,"5min":8.7550000000000008,"15min":11.654999999999999},"last":0.71499999999999997},{"interface":"front","average":{"1min":1.0149999999999999,"5min":1.1799999999999999,"15min":1.4319999999999999},"min":{"1min":0.373,"5min":0.373,"15min":0.33500000000000002},"max":{"1min":4.085,"5min":5.4409999999999998,"15min":10.423999999999999},"last":0.95599999999999996}]},{"osd":7,"last update":"Mon Oct 9 05:11:57 2023","interfaces":[{"interface":"back","average":{"1min":2.0619999999999998,"5min":1.4710000000000001,"15min":1.4570000000000001},"min":{"1min":0.61899999999999999,"5min":0.443,"15min":0.36899999999999999},"max":{"1min":6.3689999999999998,"5min":6.9640000000000004,"15min":9.6639999999999997},"last":0.97399999999999998},{"interface":"front","average":{"1min":1.3919999999999999,"5min":1.353,"15min":1.427},"min":{"1min":0.48099999999999998,"5min":0.40000000000000002,"15min":0.34699999999999998},"max":{"1min":4.7510000000000003,"5min":5.2199999999999998,"15min":10.122999999999999},"last":2.6709999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1451491328,"data_stored":1451491328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1602621440,"data_stored":1602617368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":716374016,"data_stored":716374016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1560870912,"data_stored":1560870912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":643956736,"data_stored":643956736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1683230720,"data_stored":1683226648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":720502784,"data_stored":720502784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1683361792,"data_stored":1683357720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:12:13.634 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T05:12:13.635 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_4 pg_num 36 2023-10-09T05:12:13.677 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:13.708 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:13.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:13.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:13.815 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:13.846 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:13.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:13.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:13.950 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:13.985 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:13.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:13.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:14.045 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:14 smithi133 bash[102858]: cluster 2023-10-09T05:12:13.273020+0000 mon.a (mon.0) 897 : cluster 3 Health check update: Degraded data redundancy: 12708/125382 objects degraded (10.135%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:12:14.045 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:14 smithi133 bash[102858]: audit 2023-10-09T05:12:13.453653+0000 mon.a (mon.0) 898 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:12:14.045 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:14 smithi133 bash[102858]: audit 2023-10-09T05:12:13.455154+0000 mon.a (mon.0) 899 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:12:14.092 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:14.125 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:14.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:14.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:14.230 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:14.238 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:13 smithi194 bash[92666]: cluster 2023-10-09T05:12:13.273020+0000 mon.a (mon.0) 897 : cluster 3 Health check update: Degraded data redundancy: 12708/125382 objects degraded (10.135%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:12:14.239 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:13 smithi194 bash[92666]: audit 2023-10-09T05:12:13.453653+0000 mon.a (mon.0) 898 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:12:14.239 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:13 smithi194 bash[92666]: audit 2023-10-09T05:12:13.455154+0000 mon.a (mon.0) 899 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:12:14.264 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:14.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:14.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:14.298 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:12:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:12:14] "GET /metrics HTTP/1.1" 200 36432 "" "Prometheus/2.43.0" 2023-10-09T05:12:14.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:14 smithi133 bash[105589]: cluster 2023-10-09T05:12:13.273020+0000 mon.a (mon.0) 897 : cluster 3 Health check update: Degraded data redundancy: 12708/125382 objects degraded (10.135%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:12:14.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:14 smithi133 bash[105589]: audit 2023-10-09T05:12:13.453653+0000 mon.a (mon.0) 898 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:12:14.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:14 smithi133 bash[105589]: audit 2023-10-09T05:12:13.455154+0000 mon.a (mon.0) 899 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:12:14.369 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:14.398 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:14.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:14.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:14.505 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:14.605 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:14.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:14.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:14.709 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:14.740 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:14.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:14.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:14.848 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:14.876 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:14.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:14.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:14.981 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:15.005 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:15.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:15.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:15.111 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:15.214 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:15.219 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:14 smithi194 bash[92666]: cluster 2023-10-09T05:12:13.825933+0000 mgr.y (mgr.25277) 711 : cluster 0 pgmap v686: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 12708/120756 objects degraded (10.524%); 1.5 MiB/s, 118 objects/s recovering 2023-10-09T05:12:15.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:15.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:15.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:14 smithi133 bash[102858]: cluster 2023-10-09T05:12:13.825933+0000 mgr.y (mgr.25277) 711 : cluster 0 pgmap v686: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 12708/120756 objects degraded (10.524%); 1.5 MiB/s, 118 objects/s recovering 2023-10-09T05:12:15.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:14 smithi133 bash[105589]: cluster 2023-10-09T05:12:13.825933+0000 mgr.y (mgr.25277) 711 : cluster 0 pgmap v686: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 12708/120756 objects degraded (10.524%); 1.5 MiB/s, 118 objects/s recovering 2023-10-09T05:12:15.321 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:15.354 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:15.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:15.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:15.459 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:15.493 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:15.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:15.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:15.596 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:15.635 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:15.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:15.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:15.739 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:15.765 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:15.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:15.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:15.872 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:15.903 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:15.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:15.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:16.012 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:16.038 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:16.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:16.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:16.141 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:16.172 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:16.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:16.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:16.276 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:16.306 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:16.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:16.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:16.412 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:16.444 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:16.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:16.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:16.549 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:16.565 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:12:16.576 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:16.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:16.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:16.582 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:12:16.683 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:16.712 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:16.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:16.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:16.815 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:16.849 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:16.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:16.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:16.948 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:12:16.954 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:17.012 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:17.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:17.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:17.117 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:17.126 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:16 smithi133 bash[102858]: cluster 2023-10-09T05:12:15.826638+0000 mgr.y (mgr.25277) 712 : cluster 0 pgmap v687: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 12554/112806 objects degraded (11.129%); 0 B/s, 99 objects/s recovering 2023-10-09T05:12:17.126 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:16 smithi133 bash[105589]: cluster 2023-10-09T05:12:15.826638+0000 mgr.y (mgr.25277) 712 : cluster 0 pgmap v687: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 12554/112806 objects degraded (11.129%); 0 B/s, 99 objects/s recovering 2023-10-09T05:12:17.148 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:17.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:17.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:17.252 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:17.262 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:16 smithi194 bash[92666]: cluster 2023-10-09T05:12:15.826638+0000 mgr.y (mgr.25277) 712 : cluster 0 pgmap v687: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 12554/112806 objects degraded (11.129%); 0 B/s, 99 objects/s recovering 2023-10-09T05:12:17.278 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:17.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:17.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:17.381 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:17.413 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:17.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:17.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:17.516 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:17.542 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:17.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:17.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:17.649 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:17.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:17.700 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:17.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:17.801 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:17.827 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:17.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:17.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:17.930 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:17.973 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:17.975 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:12:17.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:17.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:18.077 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:18.106 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:18.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:18.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:18.210 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:18.238 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:18.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:18.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:18.348 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:18.379 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:18.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:18.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:18.484 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:18.527 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:18.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:18.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:18.631 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:18.658 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:18.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:18.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:18.764 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:18.794 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:18.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:18.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:18.898 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:18.933 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:18.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:18.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:19.043 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:19.053 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:18 smithi194 bash[92666]: cluster 2023-10-09T05:12:17.827249+0000 mgr.y (mgr.25277) 713 : cluster 0 pgmap v688: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 12554/111030 objects degraded (11.307%); 0 B/s, 32 objects/s recovering 2023-10-09T05:12:19.053 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:18 smithi194 bash[92666]: cluster 2023-10-09T05:12:18.274226+0000 mon.a (mon.0) 900 : cluster 3 Health check update: Degraded data redundancy: 12554/111030 objects degraded (11.307%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:12:19.074 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:19.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:19.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:19.178 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:19.211 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:19.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:19.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:19.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:18 smithi133 bash[102858]: cluster 2023-10-09T05:12:17.827249+0000 mgr.y (mgr.25277) 713 : cluster 0 pgmap v688: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 12554/111030 objects degraded (11.307%); 0 B/s, 32 objects/s recovering 2023-10-09T05:12:19.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:18 smithi133 bash[102858]: cluster 2023-10-09T05:12:18.274226+0000 mon.a (mon.0) 900 : cluster 3 Health check update: Degraded data redundancy: 12554/111030 objects degraded (11.307%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:12:19.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:18 smithi133 bash[105589]: cluster 2023-10-09T05:12:17.827249+0000 mgr.y (mgr.25277) 713 : cluster 0 pgmap v688: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 12554/111030 objects degraded (11.307%); 0 B/s, 32 objects/s recovering 2023-10-09T05:12:19.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:18 smithi133 bash[105589]: cluster 2023-10-09T05:12:18.274226+0000 mon.a (mon.0) 900 : cluster 3 Health check update: Degraded data redundancy: 12554/111030 objects degraded (11.307%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:12:19.316 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:19.351 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:19.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:19.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:19.457 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:19.500 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:19.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:19.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:19.605 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:19.637 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:19.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:19.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:19.741 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:19.772 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:19.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:19.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:19.879 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:19.909 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:19.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:19.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:20.018 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:20.041 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:20.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:20.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:20.145 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:20.173 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:20.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:20.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:20.279 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:20.305 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:20.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:20.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:20.408 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:20.432 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:20.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:20.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:20.538 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:20.580 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:20.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:20.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:20.686 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:20.713 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:20.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:20.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:20.817 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:20.842 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:20.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:20.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:20.948 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:20.978 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:20.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:20.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:21.047 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:12:21.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:20 smithi133 bash[105589]: cluster 2023-10-09T05:12:19.828084+0000 mgr.y (mgr.25277) 714 : cluster 0 pgmap v689: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 12397/99024 objects degraded (12.519%); 0 B/s, 31 objects/s recovering 2023-10-09T05:12:21.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:20 smithi133 bash[102858]: cluster 2023-10-09T05:12:19.828084+0000 mgr.y (mgr.25277) 714 : cluster 0 pgmap v689: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 12397/99024 objects degraded (12.519%); 0 B/s, 31 objects/s recovering 2023-10-09T05:12:21.050 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:12:21.081 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:21.111 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:21.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:21.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:21.215 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:21.224 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:20 smithi194 bash[92666]: cluster 2023-10-09T05:12:19.828084+0000 mgr.y (mgr.25277) 714 : cluster 0 pgmap v689: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 12397/99024 objects degraded (12.519%); 0 B/s, 31 objects/s recovering 2023-10-09T05:12:21.242 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:21.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:21.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:21.345 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:21.394 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:21.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:21.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:21.499 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:21.543 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:21.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:21.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:21.646 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:21.674 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:21.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:21.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:21.782 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:21.809 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:21.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:21.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:21.912 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:21.953 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:21.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:21.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:22.063 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:22.094 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:22.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:22.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:22.198 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:22.230 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:22.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:22.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:22.332 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:22.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:22.374 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:22.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:22.475 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:22.505 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:22.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:22.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:22.609 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:22.635 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:22.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:22.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:22.738 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:22.772 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:22.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:22.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:22.876 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:22.907 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:22.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:22.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:22.991 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:22 smithi133 bash[102858]: cluster 2023-10-09T05:12:21.828744+0000 mgr.y (mgr.25277) 715 : cluster 0 pgmap v690: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 12397/99024 objects degraded (12.519%); 0 B/s, 28 objects/s recovering 2023-10-09T05:12:22.991 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:22 smithi133 bash[105589]: cluster 2023-10-09T05:12:21.828744+0000 mgr.y (mgr.25277) 715 : cluster 0 pgmap v690: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 12397/99024 objects degraded (12.519%); 0 B/s, 28 objects/s recovering 2023-10-09T05:12:23.013 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:23.035 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:23.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:23.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:23.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:22 smithi194 bash[92666]: cluster 2023-10-09T05:12:21.828744+0000 mgr.y (mgr.25277) 715 : cluster 0 pgmap v690: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 12397/99024 objects degraded (12.519%); 0 B/s, 28 objects/s recovering 2023-10-09T05:12:23.143 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:23.181 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:23.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:23.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:23.288 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:23.315 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:23.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:23.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:23.421 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:23.452 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:23.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:23.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:23.557 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:23.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:23.600 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:23.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:23.701 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:23.727 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:23.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:23.729 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:23.830 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:23.867 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:23.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:23.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:23.974 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:24.009 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:24.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:24.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:24.097 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:23 smithi133 bash[102858]: cluster 2023-10-09T05:12:23.275273+0000 mon.a (mon.0) 901 : cluster 3 Health check update: Degraded data redundancy: 12397/99024 objects degraded (12.519%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:12:24.098 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:23 smithi133 bash[105589]: cluster 2023-10-09T05:12:23.275273+0000 mon.a (mon.0) 901 : cluster 3 Health check update: Degraded data redundancy: 12397/99024 objects degraded (12.519%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:12:24.118 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:24.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:23 smithi194 bash[92666]: cluster 2023-10-09T05:12:23.275273+0000 mon.a (mon.0) 901 : cluster 3 Health check update: Degraded data redundancy: 12397/99024 objects degraded (12.519%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:12:24.155 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:24.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:24.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:24.262 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:24.289 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:24.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:24.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:24.373 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:12:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:12:24] "GET /metrics HTTP/1.1" 200 36432 "" "Prometheus/2.43.0" 2023-10-09T05:12:24.395 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:24.432 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:24.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:24.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:24.536 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:24.566 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:24.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:24.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:24.673 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:24.705 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:24.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:24.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:24.809 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:24.841 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:24.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:24.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:24.945 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:24.978 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:24.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:24.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:25.086 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:25.120 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:25.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:25.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:25.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:24 smithi194 bash[92666]: cluster 2023-10-09T05:12:23.832311+0000 mgr.y (mgr.25277) 716 : cluster 0 pgmap v691: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 874 op/s; 12397/93861 objects degraded (13.208%); 0 B/s, 25 objects/s recovering 2023-10-09T05:12:25.225 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:25.257 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:25.258 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:24 smithi133 bash[105589]: cluster 2023-10-09T05:12:23.832311+0000 mgr.y (mgr.25277) 716 : cluster 0 pgmap v691: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 874 op/s; 12397/93861 objects degraded (13.208%); 0 B/s, 25 objects/s recovering 2023-10-09T05:12:25.259 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:24 smithi133 bash[102858]: cluster 2023-10-09T05:12:23.832311+0000 mgr.y (mgr.25277) 716 : cluster 0 pgmap v691: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 874 op/s; 12397/93861 objects degraded (13.208%); 0 B/s, 25 objects/s recovering 2023-10-09T05:12:25.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:25.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:25.360 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:25.392 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:25.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:25.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:25.495 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:25.519 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:25.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:25.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:25.625 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:25.652 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:25.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:25.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:25.757 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:25.777 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:25.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:25.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:25.880 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:25.904 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:25.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:25.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:26.009 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:26.040 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:26.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:26.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:26.145 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:26.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:26.182 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:26.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:26.283 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:26.313 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:26.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:26.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:26.416 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:26.444 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:26.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:26.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:26.546 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:26.574 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:26.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:26.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:26.679 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:26.706 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:26.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:26.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:26.810 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:26.850 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:26.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:26.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:26.953 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:26.994 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:26.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:26.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:27.099 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:27.107 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:27 smithi194 bash[92666]: cluster 2023-10-09T05:12:25.833224+0000 mgr.y (mgr.25277) 717 : cluster 0 pgmap v692: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 979 op/s; 12233/85455 objects degraded (14.315%); 0 B/s, 39 objects/s recovering 2023-10-09T05:12:27.120 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:27.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:27.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:27.223 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:27.240 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:27.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:27.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:27.343 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:27.344 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:27 smithi133 bash[105589]: cluster 2023-10-09T05:12:25.833224+0000 mgr.y (mgr.25277) 717 : cluster 0 pgmap v692: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 979 op/s; 12233/85455 objects degraded (14.315%); 0 B/s, 39 objects/s recovering 2023-10-09T05:12:27.344 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:27 smithi133 bash[102858]: cluster 2023-10-09T05:12:25.833224+0000 mgr.y (mgr.25277) 717 : cluster 0 pgmap v692: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 979 op/s; 12233/85455 objects degraded (14.315%); 0 B/s, 39 objects/s recovering 2023-10-09T05:12:27.371 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:27.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:27.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:27.478 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:27.518 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:27.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:27.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:27.626 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:27.653 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:27.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:27.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:27.759 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:27.789 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:27.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:27.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:27.892 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:27.933 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:27.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:27.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:28.039 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:28.064 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:28.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:28.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:28.167 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:28.195 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:28.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:28.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:28.298 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:28.344 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:28.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:28.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:28.449 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:28.475 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:28.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:28.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:28.583 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:28.607 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:28.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:28.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:28.713 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:28.749 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:28.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:28.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:28.853 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:28.881 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:28.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:28.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:28.985 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:29.016 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:29.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:29.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:29.119 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:29.131 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:29 smithi133 bash[105589]: cluster 2023-10-09T05:12:27.833923+0000 mgr.y (mgr.25277) 718 : cluster 0 pgmap v693: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 12233/83859 objects degraded (14.588%); 0 B/s, 26 objects/s recovering 2023-10-09T05:12:29.131 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:29 smithi133 bash[105589]: cluster 2023-10-09T05:12:28.276337+0000 mon.a (mon.0) 902 : cluster 3 Health check update: Degraded data redundancy: 12233/83859 objects degraded (14.588%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:12:29.131 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:29 smithi133 bash[105589]: audit 2023-10-09T05:12:28.443558+0000 mon.a (mon.0) 903 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:12:29.131 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:29 smithi133 bash[102858]: cluster 2023-10-09T05:12:27.833923+0000 mgr.y (mgr.25277) 718 : cluster 0 pgmap v693: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 12233/83859 objects degraded (14.588%); 0 B/s, 26 objects/s recovering 2023-10-09T05:12:29.131 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:29 smithi133 bash[102858]: cluster 2023-10-09T05:12:28.276337+0000 mon.a (mon.0) 902 : cluster 3 Health check update: Degraded data redundancy: 12233/83859 objects degraded (14.588%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:12:29.132 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:29 smithi133 bash[102858]: audit 2023-10-09T05:12:28.443558+0000 mon.a (mon.0) 903 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:12:29.165 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:29.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:29.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:29.270 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:29.286 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:29 smithi194 bash[92666]: cluster 2023-10-09T05:12:27.833923+0000 mgr.y (mgr.25277) 718 : cluster 0 pgmap v693: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 12233/83859 objects degraded (14.588%); 0 B/s, 26 objects/s recovering 2023-10-09T05:12:29.286 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:29 smithi194 bash[92666]: cluster 2023-10-09T05:12:28.276337+0000 mon.a (mon.0) 902 : cluster 3 Health check update: Degraded data redundancy: 12233/83859 objects degraded (14.588%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:12:29.287 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:29 smithi194 bash[92666]: audit 2023-10-09T05:12:28.443558+0000 mon.a (mon.0) 903 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:12:29.303 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:29.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:29.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:29.411 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:29.441 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:29.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:29.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:29.544 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:29.571 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:29.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:29.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:29.673 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:29.708 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:29.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:29.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:29.814 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:29.859 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:29.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:29.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:29.962 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:30.004 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:30.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:30.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:30.044 INFO:teuthology.orchestra.run.smithi133.stderr:set pool 9 pg_num to 36 2023-10-09T05:12:30.109 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:30.153 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:30.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:30.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:30.259 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:30.270 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:30 smithi194 bash[92666]: audit 2023-10-09T05:12:29.345634+0000 mon.c (mon.2) 67 : audit 1 from='client.? 172.21.15.133:0/4062889433' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:12:30.270 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:30 smithi194 bash[92666]: audit 2023-10-09T05:12:29.346142+0000 mon.a (mon.0) 904 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:12:30.298 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:30.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:30.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:30.405 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:30.428 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:30.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:30.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:30.435 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:30 smithi133 bash[102858]: audit 2023-10-09T05:12:29.345634+0000 mon.c (mon.2) 67 : audit 1 from='client.? 172.21.15.133:0/4062889433' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:12:30.435 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:30 smithi133 bash[102858]: audit 2023-10-09T05:12:29.346142+0000 mon.a (mon.0) 904 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:12:30.531 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:30.573 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:30.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:30.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:30.672 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:12:30.672 INFO:teuthology.orchestra.run.smithi133.stdout: "ops": [], 2023-10-09T05:12:30.672 INFO:teuthology.orchestra.run.smithi133.stdout: "num_ops": 0 2023-10-09T05:12:30.672 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:12:30.679 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:30.687 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:30 smithi133 bash[105589]: audit 2023-10-09T05:12:29.345634+0000 mon.c (mon.2) 67 : audit 1 from='client.? 172.21.15.133:0/4062889433' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:12:30.687 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:30 smithi133 bash[105589]: audit 2023-10-09T05:12:29.346142+0000 mon.a (mon.0) 904 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:12:30.712 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:30.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:30.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:30.815 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:30.840 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:30.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:30.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:30.943 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:30.976 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:30.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:30.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:31.026 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:31 smithi133 bash[105589]: cluster 2023-10-09T05:12:29.834845+0000 mgr.y (mgr.25277) 719 : cluster 0 pgmap v694: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 12048/73428 objects degraded (16.408%); 0 B/s, 42 objects/s recovering 2023-10-09T05:12:31.026 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:31 smithi133 bash[105589]: cluster 2023-10-09T05:12:30.022835+0000 osd.4 (osd.4) 105 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T05:12:31.026 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:31 smithi133 bash[105589]: audit 2023-10-09T05:12:30.026603+0000 mon.a (mon.0) 905 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "36"}]': finished 2023-10-09T05:12:31.026 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:31 smithi133 bash[105589]: cluster 2023-10-09T05:12:30.026783+0000 mon.a (mon.0) 906 : cluster 0 osdmap e166: 8 total, 8 up, 8 in 2023-10-09T05:12:31.026 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:31 smithi133 bash[105589]: cluster 2023-10-09T05:12:30.032626+0000 osd.5 (osd.5) 147 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T05:12:31.083 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:31.091 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:31 smithi194 bash[92666]: cluster 2023-10-09T05:12:29.834845+0000 mgr.y (mgr.25277) 719 : cluster 0 pgmap v694: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 12048/73428 objects degraded (16.408%); 0 B/s, 42 objects/s recovering 2023-10-09T05:12:31.092 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:31 smithi194 bash[92666]: cluster 2023-10-09T05:12:30.022835+0000 osd.4 (osd.4) 105 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T05:12:31.092 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:31 smithi194 bash[92666]: audit 2023-10-09T05:12:30.026603+0000 mon.a (mon.0) 905 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "36"}]': finished 2023-10-09T05:12:31.092 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:31 smithi194 bash[92666]: cluster 2023-10-09T05:12:30.026783+0000 mon.a (mon.0) 906 : cluster 0 osdmap e166: 8 total, 8 up, 8 in 2023-10-09T05:12:31.092 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:31 smithi194 bash[92666]: cluster 2023-10-09T05:12:30.032626+0000 osd.5 (osd.5) 147 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T05:12:31.110 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:31.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:31.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:31.214 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:31.241 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:31.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:31.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:31.285 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:31 smithi133 bash[102858]: cluster 2023-10-09T05:12:29.834845+0000 mgr.y (mgr.25277) 719 : cluster 0 pgmap v694: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 12048/73428 objects degraded (16.408%); 0 B/s, 42 objects/s recovering 2023-10-09T05:12:31.285 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:31 smithi133 bash[102858]: cluster 2023-10-09T05:12:30.022835+0000 osd.4 (osd.4) 105 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T05:12:31.285 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:31 smithi133 bash[102858]: audit 2023-10-09T05:12:30.026603+0000 mon.a (mon.0) 905 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "36"}]': finished 2023-10-09T05:12:31.285 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:31 smithi133 bash[102858]: cluster 2023-10-09T05:12:30.026783+0000 mon.a (mon.0) 906 : cluster 0 osdmap e166: 8 total, 8 up, 8 in 2023-10-09T05:12:31.285 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:31 smithi133 bash[102858]: cluster 2023-10-09T05:12:30.032626+0000 osd.5 (osd.5) 147 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T05:12:31.347 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:31.375 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:31.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:31.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:31.480 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:31.502 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:31.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:31.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:31.604 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:31.631 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:31.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:31.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:31.736 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:31.767 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:31.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:31.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:31.872 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:31.905 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:31.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:31.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:32.009 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:32.049 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:32.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:32.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:32.157 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:32.224 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:32.227 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:32 smithi133 bash[102858]: audit 2023-10-09T05:12:31.837031+0000 mon.a (mon.0) 907 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T05:12:32.227 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:32 smithi133 bash[105589]: audit 2023-10-09T05:12:31.837031+0000 mon.a (mon.0) 907 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T05:12:32.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:32.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:32.331 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:32.353 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:32.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:32.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:32.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:32 smithi194 bash[92666]: audit 2023-10-09T05:12:31.837031+0000 mon.a (mon.0) 907 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T05:12:32.456 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:32.482 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:32.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:32.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:32.587 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:32.620 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:32.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:32.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:32.723 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:32.746 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:32.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:32.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:32.848 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:32.872 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:32.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:32.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:32.975 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:33.004 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:33.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:33.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:33.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:33 smithi133 bash[105589]: cluster 2023-10-09T05:12:31.835763+0000 mgr.y (mgr.25277) 720 : cluster 0 pgmap v696: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 852 op/s; 12048/73428 objects degraded (16.408%); 0 B/s, 34 objects/s recovering 2023-10-09T05:12:33.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:33 smithi133 bash[105589]: audit 2023-10-09T05:12:32.038111+0000 mon.a (mon.0) 908 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T05:12:33.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:33 smithi133 bash[105589]: cluster 2023-10-09T05:12:32.038145+0000 osd.5 (osd.5) 148 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T05:12:33.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:33 smithi133 bash[105589]: cluster 2023-10-09T05:12:32.038258+0000 mon.a (mon.0) 909 : cluster 0 osdmap e167: 8 total, 8 up, 8 in 2023-10-09T05:12:33.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:33 smithi133 bash[105589]: cluster 2023-10-09T05:12:32.187607+0000 osd.6 (osd.6) 89 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T05:12:33.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:33 smithi133 bash[105589]: cluster 2023-10-09T05:12:32.235001+0000 osd.4 (osd.4) 106 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T05:12:33.107 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:33.132 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:33.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:33.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:33.236 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:33.260 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:33.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:33.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:33.365 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:33.372 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:33 smithi133 bash[102858]: cluster 2023-10-09T05:12:31.835763+0000 mgr.y (mgr.25277) 720 : cluster 0 pgmap v696: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 852 op/s; 12048/73428 objects degraded (16.408%); 0 B/s, 34 objects/s recovering 2023-10-09T05:12:33.372 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:33 smithi133 bash[102858]: audit 2023-10-09T05:12:32.038111+0000 mon.a (mon.0) 908 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T05:12:33.372 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:33 smithi133 bash[102858]: cluster 2023-10-09T05:12:32.038145+0000 osd.5 (osd.5) 148 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T05:12:33.372 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:33 smithi133 bash[102858]: cluster 2023-10-09T05:12:32.038258+0000 mon.a (mon.0) 909 : cluster 0 osdmap e167: 8 total, 8 up, 8 in 2023-10-09T05:12:33.372 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:33 smithi133 bash[102858]: cluster 2023-10-09T05:12:32.187607+0000 osd.6 (osd.6) 89 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T05:12:33.372 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:33 smithi133 bash[102858]: cluster 2023-10-09T05:12:32.235001+0000 osd.4 (osd.4) 106 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T05:12:33.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:33 smithi194 bash[92666]: cluster 2023-10-09T05:12:31.835763+0000 mgr.y (mgr.25277) 720 : cluster 0 pgmap v696: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 852 op/s; 12048/73428 objects degraded (16.408%); 0 B/s, 34 objects/s recovering 2023-10-09T05:12:33.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:33 smithi194 bash[92666]: audit 2023-10-09T05:12:32.038111+0000 mon.a (mon.0) 908 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T05:12:33.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:33 smithi194 bash[92666]: cluster 2023-10-09T05:12:32.038145+0000 osd.5 (osd.5) 148 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T05:12:33.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:33 smithi194 bash[92666]: cluster 2023-10-09T05:12:32.038258+0000 mon.a (mon.0) 909 : cluster 0 osdmap e167: 8 total, 8 up, 8 in 2023-10-09T05:12:33.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:33 smithi194 bash[92666]: cluster 2023-10-09T05:12:32.187607+0000 osd.6 (osd.6) 89 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T05:12:33.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:33 smithi194 bash[92666]: cluster 2023-10-09T05:12:32.235001+0000 osd.4 (osd.4) 106 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T05:12:33.390 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:33.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:33.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:33.496 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:33.526 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:33.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:33.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:33.628 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:33.650 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:33.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:33.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:33.754 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:33.774 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:33.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:33.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:33.877 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:33.910 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:33.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:33.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:34.014 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:34.047 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:34.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:34.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:34.151 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:34.173 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:34.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:34.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:34.276 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:34.281 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:34 smithi133 bash[102858]: cluster 2023-10-09T05:12:33.043848+0000 osd.4 (osd.4) 107 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T05:12:34.281 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:34 smithi133 bash[102858]: cluster 2023-10-09T05:12:33.045149+0000 mon.a (mon.0) 910 : cluster 0 osdmap e168: 8 total, 8 up, 8 in 2023-10-09T05:12:34.282 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:34 smithi133 bash[102858]: cluster 2023-10-09T05:12:33.045746+0000 osd.7 (osd.7) 92 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T05:12:34.282 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:34 smithi133 bash[102858]: cluster 2023-10-09T05:12:33.049022+0000 osd.5 (osd.5) 149 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T05:12:34.282 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:34 smithi133 bash[102858]: cluster 2023-10-09T05:12:33.050428+0000 osd.6 (osd.6) 90 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T05:12:34.282 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:34 smithi133 bash[102858]: cluster 2023-10-09T05:12:33.054556+0000 osd.6 (osd.6) 91 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T05:12:34.282 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:34 smithi133 bash[102858]: cluster 2023-10-09T05:12:33.277467+0000 mon.a (mon.0) 911 : cluster 3 Health check update: Degraded data redundancy: 12048/73428 objects degraded (16.408%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:12:34.282 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:34 smithi133 bash[102858]: cluster 2023-10-09T05:12:33.282604+0000 mon.a (mon.0) 912 : cluster 0 osdmap e169: 8 total, 8 up, 8 in 2023-10-09T05:12:34.282 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:34 smithi133 bash[102858]: cluster 2023-10-09T05:12:33.283134+0000 osd.7 (osd.7) 93 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T05:12:34.282 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:34 smithi133 bash[102858]: cluster 2023-10-09T05:12:33.284691+0000 osd.5 (osd.5) 150 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T05:12:34.282 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:34 smithi133 bash[102858]: cluster 2023-10-09T05:12:33.287310+0000 osd.7 (osd.7) 94 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T05:12:34.282 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:34 smithi133 bash[102858]: cluster 2023-10-09T05:12:33.288600+0000 osd.5 (osd.5) 151 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T05:12:34.282 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:34 smithi133 bash[102858]: cluster 2023-10-09T05:12:33.291314+0000 osd.5 (osd.5) 152 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T05:12:34.282 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:34 smithi133 bash[102858]: cluster 2023-10-09T05:12:33.328238+0000 osd.6 (osd.6) 92 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T05:12:34.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:34 smithi133 bash[105589]: cluster 2023-10-09T05:12:33.043848+0000 osd.4 (osd.4) 107 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T05:12:34.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:34 smithi133 bash[105589]: cluster 2023-10-09T05:12:33.045149+0000 mon.a (mon.0) 910 : cluster 0 osdmap e168: 8 total, 8 up, 8 in 2023-10-09T05:12:34.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:34 smithi133 bash[105589]: cluster 2023-10-09T05:12:33.045746+0000 osd.7 (osd.7) 92 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T05:12:34.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:34 smithi133 bash[105589]: cluster 2023-10-09T05:12:33.049022+0000 osd.5 (osd.5) 149 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T05:12:34.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:34 smithi133 bash[105589]: cluster 2023-10-09T05:12:33.050428+0000 osd.6 (osd.6) 90 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T05:12:34.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:34 smithi133 bash[105589]: cluster 2023-10-09T05:12:33.054556+0000 osd.6 (osd.6) 91 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T05:12:34.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:34 smithi133 bash[105589]: cluster 2023-10-09T05:12:33.277467+0000 mon.a (mon.0) 911 : cluster 3 Health check update: Degraded data redundancy: 12048/73428 objects degraded (16.408%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:12:34.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:34 smithi133 bash[105589]: cluster 2023-10-09T05:12:33.282604+0000 mon.a (mon.0) 912 : cluster 0 osdmap e169: 8 total, 8 up, 8 in 2023-10-09T05:12:34.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:34 smithi133 bash[105589]: cluster 2023-10-09T05:12:33.283134+0000 osd.7 (osd.7) 93 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T05:12:34.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:34 smithi133 bash[105589]: cluster 2023-10-09T05:12:33.284691+0000 osd.5 (osd.5) 150 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T05:12:34.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:34 smithi133 bash[105589]: cluster 2023-10-09T05:12:33.287310+0000 osd.7 (osd.7) 94 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T05:12:34.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:34 smithi133 bash[105589]: cluster 2023-10-09T05:12:33.288600+0000 osd.5 (osd.5) 151 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T05:12:34.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:34 smithi133 bash[105589]: cluster 2023-10-09T05:12:33.291314+0000 osd.5 (osd.5) 152 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T05:12:34.284 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:34 smithi133 bash[105589]: cluster 2023-10-09T05:12:33.328238+0000 osd.6 (osd.6) 92 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T05:12:34.302 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:34.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:34.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:34.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:34 smithi194 bash[92666]: cluster 2023-10-09T05:12:33.043848+0000 osd.4 (osd.4) 107 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T05:12:34.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:34 smithi194 bash[92666]: cluster 2023-10-09T05:12:33.045149+0000 mon.a (mon.0) 910 : cluster 0 osdmap e168: 8 total, 8 up, 8 in 2023-10-09T05:12:34.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:34 smithi194 bash[92666]: cluster 2023-10-09T05:12:33.045746+0000 osd.7 (osd.7) 92 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T05:12:34.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:34 smithi194 bash[92666]: cluster 2023-10-09T05:12:33.049022+0000 osd.5 (osd.5) 149 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T05:12:34.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:34 smithi194 bash[92666]: cluster 2023-10-09T05:12:33.050428+0000 osd.6 (osd.6) 90 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T05:12:34.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:34 smithi194 bash[92666]: cluster 2023-10-09T05:12:33.054556+0000 osd.6 (osd.6) 91 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T05:12:34.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:34 smithi194 bash[92666]: cluster 2023-10-09T05:12:33.277467+0000 mon.a (mon.0) 911 : cluster 3 Health check update: Degraded data redundancy: 12048/73428 objects degraded (16.408%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:12:34.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:34 smithi194 bash[92666]: cluster 2023-10-09T05:12:33.282604+0000 mon.a (mon.0) 912 : cluster 0 osdmap e169: 8 total, 8 up, 8 in 2023-10-09T05:12:34.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:34 smithi194 bash[92666]: cluster 2023-10-09T05:12:33.283134+0000 osd.7 (osd.7) 93 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T05:12:34.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:34 smithi194 bash[92666]: cluster 2023-10-09T05:12:33.284691+0000 osd.5 (osd.5) 150 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T05:12:34.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:34 smithi194 bash[92666]: cluster 2023-10-09T05:12:33.287310+0000 osd.7 (osd.7) 94 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T05:12:34.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:34 smithi194 bash[92666]: cluster 2023-10-09T05:12:33.288600+0000 osd.5 (osd.5) 151 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T05:12:34.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:34 smithi194 bash[92666]: cluster 2023-10-09T05:12:33.291314+0000 osd.5 (osd.5) 152 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T05:12:34.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:34 smithi194 bash[92666]: cluster 2023-10-09T05:12:33.328238+0000 osd.6 (osd.6) 92 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T05:12:34.406 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:34.433 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:34.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:34.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:34.537 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:34.565 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:34.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:34.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:34.582 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:12:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:12:34] "GET /metrics HTTP/1.1" 200 36419 "" "Prometheus/2.43.0" 2023-10-09T05:12:34.668 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:34.691 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:34.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:34.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:34.796 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:34.821 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:34.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:34.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:34.925 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:34.957 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:34.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:34.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:35.064 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:35.075 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:35 smithi194 bash[92666]: cluster 2023-10-09T05:12:33.836616+0000 mgr.y (mgr.25277) 721 : cluster 0 pgmap v700: 109 pgs: 7 unknown, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 1.4 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 326 op/s; 11179/69510 objects degraded (16.083%); 0 B/s, 79 objects/s recovering 2023-10-09T05:12:35.075 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:35 smithi194 bash[92666]: cluster 2023-10-09T05:12:34.297598+0000 osd.5 (osd.5) 153 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T05:12:35.075 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:35 smithi194 bash[92666]: cluster 2023-10-09T05:12:34.299192+0000 osd.5 (osd.5) 154 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T05:12:35.075 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:35 smithi194 bash[92666]: cluster 2023-10-09T05:12:34.299473+0000 mon.a (mon.0) 913 : cluster 0 osdmap e170: 8 total, 8 up, 8 in 2023-10-09T05:12:35.075 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:35 smithi194 bash[92666]: cluster 2023-10-09T05:12:34.300939+0000 osd.5 (osd.5) 155 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T05:12:35.075 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:35 smithi194 bash[92666]: cluster 2023-10-09T05:12:34.345073+0000 osd.6 (osd.6) 93 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T05:12:35.075 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:35 smithi194 bash[92666]: cluster 2023-10-09T05:12:34.348037+0000 osd.4 (osd.4) 108 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T05:12:35.106 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:35.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:35.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:35.213 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:35.249 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:35.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:35.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:35.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:35 smithi133 bash[102858]: cluster 2023-10-09T05:12:33.836616+0000 mgr.y (mgr.25277) 721 : cluster 0 pgmap v700: 109 pgs: 7 unknown, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 1.4 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 326 op/s; 11179/69510 objects degraded (16.083%); 0 B/s, 79 objects/s recovering 2023-10-09T05:12:35.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:35 smithi133 bash[102858]: cluster 2023-10-09T05:12:34.297598+0000 osd.5 (osd.5) 153 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T05:12:35.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:35 smithi133 bash[102858]: cluster 2023-10-09T05:12:34.299192+0000 osd.5 (osd.5) 154 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T05:12:35.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:35 smithi133 bash[102858]: cluster 2023-10-09T05:12:34.299473+0000 mon.a (mon.0) 913 : cluster 0 osdmap e170: 8 total, 8 up, 8 in 2023-10-09T05:12:35.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:35 smithi133 bash[102858]: cluster 2023-10-09T05:12:34.300939+0000 osd.5 (osd.5) 155 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T05:12:35.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:35 smithi133 bash[102858]: cluster 2023-10-09T05:12:34.345073+0000 osd.6 (osd.6) 93 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T05:12:35.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:35 smithi133 bash[102858]: cluster 2023-10-09T05:12:34.348037+0000 osd.4 (osd.4) 108 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T05:12:35.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:35 smithi133 bash[105589]: cluster 2023-10-09T05:12:33.836616+0000 mgr.y (mgr.25277) 721 : cluster 0 pgmap v700: 109 pgs: 7 unknown, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 1.4 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 326 op/s; 11179/69510 objects degraded (16.083%); 0 B/s, 79 objects/s recovering 2023-10-09T05:12:35.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:35 smithi133 bash[105589]: cluster 2023-10-09T05:12:34.297598+0000 osd.5 (osd.5) 153 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T05:12:35.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:35 smithi133 bash[105589]: cluster 2023-10-09T05:12:34.299192+0000 osd.5 (osd.5) 154 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T05:12:35.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:35 smithi133 bash[105589]: cluster 2023-10-09T05:12:34.299473+0000 mon.a (mon.0) 913 : cluster 0 osdmap e170: 8 total, 8 up, 8 in 2023-10-09T05:12:35.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:35 smithi133 bash[105589]: cluster 2023-10-09T05:12:34.300939+0000 osd.5 (osd.5) 155 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T05:12:35.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:35 smithi133 bash[105589]: cluster 2023-10-09T05:12:34.345073+0000 osd.6 (osd.6) 93 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T05:12:35.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:35 smithi133 bash[105589]: cluster 2023-10-09T05:12:34.348037+0000 osd.4 (osd.4) 108 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T05:12:35.353 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:35.380 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:35.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:35.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:35.484 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:35.507 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:35.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:35.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:35.611 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:35.644 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:35.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:35.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:35.755 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:35.780 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:35.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:35.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:35.887 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:35.914 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:35.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:35.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:36.018 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:36.047 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:36.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:36.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:36.154 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:36.178 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:36.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:36.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:36.282 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:36.308 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:36.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:36.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:36.411 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:36.436 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:36.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:36.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:36.539 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:36.569 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:36.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:36.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:36.673 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:36.699 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:36.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:36.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:36.805 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:36.811 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T05:12:36.829 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:36.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:36.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:36.933 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:36.961 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:36.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:36.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:37.065 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:37.075 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:37 smithi194 bash[92666]: cluster 2023-10-09T05:12:35.837261+0000 mgr.y (mgr.25277) 722 : cluster 0 pgmap v702: 109 pgs: 1 unknown, 11 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 1.3 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 11081/63963 objects degraded (17.324%); 0 B/s, 103 objects/s recovering 2023-10-09T05:12:37.075 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:37 smithi194 bash[92666]: cluster 2023-10-09T05:12:36.141711+0000 osd.0 (osd.0) 1 : cluster 0 9.5 scrub starts 2023-10-09T05:12:37.101 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:37.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:37.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:37.205 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:37.233 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:37.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:37.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:37.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:37 smithi133 bash[105589]: cluster 2023-10-09T05:12:35.837261+0000 mgr.y (mgr.25277) 722 : cluster 0 pgmap v702: 109 pgs: 1 unknown, 11 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 1.3 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 11081/63963 objects degraded (17.324%); 0 B/s, 103 objects/s recovering 2023-10-09T05:12:37.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:37 smithi133 bash[105589]: cluster 2023-10-09T05:12:36.141711+0000 osd.0 (osd.0) 1 : cluster 0 9.5 scrub starts 2023-10-09T05:12:37.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:37 smithi133 bash[102858]: cluster 2023-10-09T05:12:35.837261+0000 mgr.y (mgr.25277) 722 : cluster 0 pgmap v702: 109 pgs: 1 unknown, 11 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 1.3 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 11081/63963 objects degraded (17.324%); 0 B/s, 103 objects/s recovering 2023-10-09T05:12:37.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:37 smithi133 bash[102858]: cluster 2023-10-09T05:12:36.141711+0000 osd.0 (osd.0) 1 : cluster 0 9.5 scrub starts 2023-10-09T05:12:37.337 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:37.368 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:37.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:37.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:37.473 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:37.497 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:37.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:37.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:37.601 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:37.631 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:37.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:37.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:37.736 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:37.764 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:37.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:37.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:37.867 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:37.893 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:37.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:37.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:37.996 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:38.033 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:38.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:38.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:38.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:38 smithi194 bash[92666]: cluster 2023-10-09T05:12:37.159266+0000 osd.0 (osd.0) 2 : cluster 0 9.9 scrub starts 2023-10-09T05:12:38.136 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:38.166 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:38.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:38.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:38.178 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:12:38.232 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:12:38.267 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:12:38.269 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:38.300 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:38.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:38.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:38.307 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:12:38.353 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:12:38.391 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:12:38.404 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:38.432 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:38.434 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:12:38.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:38.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:38.474 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:12:38.482 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:12:38.482 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:12:38.483 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:12:38.483 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:12:38.483 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:12:38.483 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:12:38.483 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:12:38.483 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:12:38.525 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:38 smithi133 bash[102858]: cluster 2023-10-09T05:12:37.159266+0000 osd.0 (osd.0) 2 : cluster 0 9.9 scrub starts 2023-10-09T05:12:38.525 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:38 smithi133 bash[105589]: cluster 2023-10-09T05:12:37.159266+0000 osd.0 (osd.0) 2 : cluster 0 9.9 scrub starts 2023-10-09T05:12:38.537 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:38.567 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:38.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:38.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:38.671 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:38.695 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:38.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:38.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:38.801 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:38.832 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:38.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:38.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:38.937 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:38.963 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:38.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:38.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:39.025 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:12:39.026 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:12:39.026 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [0, 2, 3, 4, 5, 6, 7, 1] 2023-10-09T05:12:39.026 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd out 5 2023-10-09T05:12:39.070 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:39.102 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:39.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:39.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:39.205 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:39.235 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:39.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:39.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:39.296 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is set 2023-10-09T05:12:39.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:39 smithi133 bash[102858]: cluster 2023-10-09T05:12:37.837909+0000 mgr.y (mgr.25277) 723 : cluster 0 pgmap v703: 109 pgs: 11 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.3 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 11081/62493 objects degraded (17.732%); 0 B/s, 71 objects/s recovering 2023-10-09T05:12:39.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:39 smithi133 bash[102858]: audit 2023-10-09T05:12:38.127196+0000 mon.b (mon.1) 15 : audit 0 from='client.? 172.21.15.133:0/1944436503' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:12:39.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:39 smithi133 bash[102858]: cluster 2023-10-09T05:12:38.279117+0000 mon.a (mon.0) 914 : cluster 3 Health check update: Degraded data redundancy: 11081/62493 objects degraded (17.732%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T05:12:39.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:39 smithi133 bash[102858]: audit 2023-10-09T05:12:38.548452+0000 mon.b (mon.1) 16 : audit 1 from='client.? 172.21.15.133:0/3004528735' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:12:39.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:39 smithi133 bash[102858]: audit 2023-10-09T05:12:38.549057+0000 mon.a (mon.0) 915 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:12:39.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:39 smithi133 bash[105589]: cluster 2023-10-09T05:12:37.837909+0000 mgr.y (mgr.25277) 723 : cluster 0 pgmap v703: 109 pgs: 11 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.3 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 11081/62493 objects degraded (17.732%); 0 B/s, 71 objects/s recovering 2023-10-09T05:12:39.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:39 smithi133 bash[105589]: audit 2023-10-09T05:12:38.127196+0000 mon.b (mon.1) 15 : audit 0 from='client.? 172.21.15.133:0/1944436503' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:12:39.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:39 smithi133 bash[105589]: cluster 2023-10-09T05:12:38.279117+0000 mon.a (mon.0) 914 : cluster 3 Health check update: Degraded data redundancy: 11081/62493 objects degraded (17.732%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T05:12:39.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:39 smithi133 bash[105589]: audit 2023-10-09T05:12:38.548452+0000 mon.b (mon.1) 16 : audit 1 from='client.? 172.21.15.133:0/3004528735' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:12:39.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:39 smithi133 bash[105589]: audit 2023-10-09T05:12:38.549057+0000 mon.a (mon.0) 915 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:12:39.343 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:39.365 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:39.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:39.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:39.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:39 smithi194 bash[92666]: cluster 2023-10-09T05:12:37.837909+0000 mgr.y (mgr.25277) 723 : cluster 0 pgmap v703: 109 pgs: 11 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.3 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 11081/62493 objects degraded (17.732%); 0 B/s, 71 objects/s recovering 2023-10-09T05:12:39.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:39 smithi194 bash[92666]: audit 2023-10-09T05:12:38.127196+0000 mon.b (mon.1) 15 : audit 0 from='client.? 172.21.15.133:0/1944436503' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:12:39.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:39 smithi194 bash[92666]: cluster 2023-10-09T05:12:38.279117+0000 mon.a (mon.0) 914 : cluster 3 Health check update: Degraded data redundancy: 11081/62493 objects degraded (17.732%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T05:12:39.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:39 smithi194 bash[92666]: audit 2023-10-09T05:12:38.548452+0000 mon.b (mon.1) 16 : audit 1 from='client.? 172.21.15.133:0/3004528735' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:12:39.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:39 smithi194 bash[92666]: audit 2023-10-09T05:12:38.549057+0000 mon.a (mon.0) 915 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:12:39.469 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:39.504 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:39.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:39.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:39.607 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:39.639 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:39.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:39.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:39.745 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:39.774 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:39.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:39.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:39.880 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:39.909 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:39.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:39.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:40.013 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:40.048 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:40.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:40.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:40.068 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:40 smithi194 bash[92666]: cluster 2023-10-09T05:12:39.281578+0000 mon.a (mon.0) 916 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:12:40.068 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:40 smithi194 bash[92666]: audit 2023-10-09T05:12:39.288350+0000 mon.a (mon.0) 917 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:12:40.068 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:40 smithi194 bash[92666]: cluster 2023-10-09T05:12:39.288446+0000 mon.a (mon.0) 918 : cluster 0 osdmap e171: 8 total, 8 up, 8 in 2023-10-09T05:12:40.068 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:40 smithi194 bash[92666]: cluster 2023-10-09T05:12:39.291258+0000 osd.7 (osd.7) 95 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T05:12:40.069 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:40 smithi194 bash[92666]: cluster 2023-10-09T05:12:39.291453+0000 osd.5 (osd.5) 156 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T05:12:40.069 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:40 smithi194 bash[92666]: cluster 2023-10-09T05:12:39.293955+0000 osd.5 (osd.5) 157 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T05:12:40.152 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:40.186 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:40.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:40.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:40.284 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:40 smithi133 bash[102858]: cluster 2023-10-09T05:12:39.281578+0000 mon.a (mon.0) 916 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:12:40.284 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:40 smithi133 bash[102858]: audit 2023-10-09T05:12:39.288350+0000 mon.a (mon.0) 917 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:12:40.284 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:40 smithi133 bash[102858]: cluster 2023-10-09T05:12:39.288446+0000 mon.a (mon.0) 918 : cluster 0 osdmap e171: 8 total, 8 up, 8 in 2023-10-09T05:12:40.284 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:40 smithi133 bash[102858]: cluster 2023-10-09T05:12:39.291258+0000 osd.7 (osd.7) 95 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T05:12:40.285 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:40 smithi133 bash[102858]: cluster 2023-10-09T05:12:39.291453+0000 osd.5 (osd.5) 156 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T05:12:40.285 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:40 smithi133 bash[102858]: cluster 2023-10-09T05:12:39.293955+0000 osd.5 (osd.5) 157 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T05:12:40.285 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:40 smithi133 bash[105589]: cluster 2023-10-09T05:12:39.281578+0000 mon.a (mon.0) 916 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:12:40.285 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:40 smithi133 bash[105589]: audit 2023-10-09T05:12:39.288350+0000 mon.a (mon.0) 917 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:12:40.285 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:40 smithi133 bash[105589]: cluster 2023-10-09T05:12:39.288446+0000 mon.a (mon.0) 918 : cluster 0 osdmap e171: 8 total, 8 up, 8 in 2023-10-09T05:12:40.285 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:40 smithi133 bash[105589]: cluster 2023-10-09T05:12:39.291258+0000 osd.7 (osd.7) 95 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T05:12:40.285 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:40 smithi133 bash[105589]: cluster 2023-10-09T05:12:39.291453+0000 osd.5 (osd.5) 156 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T05:12:40.285 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:40 smithi133 bash[105589]: cluster 2023-10-09T05:12:39.293955+0000 osd.5 (osd.5) 157 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T05:12:40.292 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:40.317 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:40.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:40.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:40.425 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:40.464 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:40.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:40.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:40.569 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:40.595 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:40.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:40.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:40.698 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:40.724 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:40.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:40.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:40.828 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:40.857 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:40.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:40.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:40.962 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:40.994 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:40.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:40.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:41.098 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:41.130 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:41.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:41 smithi194 bash[92666]: cluster 2023-10-09T05:12:39.839340+0000 mgr.y (mgr.25277) 724 : cluster 0 pgmap v705: 109 pgs: 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1006 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.20k op/s; 10194/48348 objects degraded (21.085%); 0 B/s, 186 objects/s recovering 2023-10-09T05:12:41.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:41 smithi194 bash[92666]: audit 2023-10-09T05:12:40.996289+0000 mon.a (mon.0) 919 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:12:41.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:41.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:41.238 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:41.248 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:41 smithi133 bash[102858]: cluster 2023-10-09T05:12:39.839340+0000 mgr.y (mgr.25277) 724 : cluster 0 pgmap v705: 109 pgs: 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1006 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.20k op/s; 10194/48348 objects degraded (21.085%); 0 B/s, 186 objects/s recovering 2023-10-09T05:12:41.248 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:41 smithi133 bash[102858]: audit 2023-10-09T05:12:40.996289+0000 mon.a (mon.0) 919 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:12:41.248 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:41 smithi133 bash[105589]: cluster 2023-10-09T05:12:39.839340+0000 mgr.y (mgr.25277) 724 : cluster 0 pgmap v705: 109 pgs: 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1006 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.20k op/s; 10194/48348 objects degraded (21.085%); 0 B/s, 186 objects/s recovering 2023-10-09T05:12:41.248 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:41 smithi133 bash[105589]: audit 2023-10-09T05:12:40.996289+0000 mon.a (mon.0) 919 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:12:41.270 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:41.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:41.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:41.374 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:41.396 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:41.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:41.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:41.501 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:41.530 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:41.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:41.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:41.635 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:41.667 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:41.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:41.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:41.773 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:41.806 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:41.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:41.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:41.914 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:41.948 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:41.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:41.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:42.053 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:42.082 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:42.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:42.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:42.187 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:42.212 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:42.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:42.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:42.318 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:42.344 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:42.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:42.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:42.449 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:42.474 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:42.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:42.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:42.481 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:42 smithi133 bash[102858]: audit 2023-10-09T05:12:41.477294+0000 mon.a (mon.0) 920 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:12:42.481 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:42 smithi133 bash[102858]: audit 2023-10-09T05:12:41.488275+0000 mon.a (mon.0) 921 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:12:42.481 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:42 smithi133 bash[102858]: cluster 2023-10-09T05:12:41.840004+0000 mgr.y (mgr.25277) 725 : cluster 0 pgmap v706: 109 pgs: 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 983 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 927 op/s; 10194/47244 objects degraded (21.577%); 0 B/s, 113 objects/s recovering 2023-10-09T05:12:42.482 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:42 smithi133 bash[105589]: audit 2023-10-09T05:12:41.477294+0000 mon.a (mon.0) 920 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:12:42.482 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:42 smithi133 bash[105589]: audit 2023-10-09T05:12:41.488275+0000 mon.a (mon.0) 921 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:12:42.482 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:42 smithi133 bash[105589]: cluster 2023-10-09T05:12:41.840004+0000 mgr.y (mgr.25277) 725 : cluster 0 pgmap v706: 109 pgs: 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 983 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 927 op/s; 10194/47244 objects degraded (21.577%); 0 B/s, 113 objects/s recovering 2023-10-09T05:12:42.564 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:12:42.580 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:42.642 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:42.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:42.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:42.747 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:42.756 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:42 smithi194 bash[92666]: audit 2023-10-09T05:12:41.477294+0000 mon.a (mon.0) 920 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:12:42.756 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:42 smithi194 bash[92666]: audit 2023-10-09T05:12:41.488275+0000 mon.a (mon.0) 921 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:12:42.756 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:42 smithi194 bash[92666]: cluster 2023-10-09T05:12:41.840004+0000 mgr.y (mgr.25277) 725 : cluster 0 pgmap v706: 109 pgs: 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 983 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 927 op/s; 10194/47244 objects degraded (21.577%); 0 B/s, 113 objects/s recovering 2023-10-09T05:12:42.775 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:42.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:42.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:42.880 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:42.912 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:42.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:42.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:43.018 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:43.052 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:43.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:43.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:43.157 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:43.184 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:43.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:43.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:43.288 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:43.318 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:43.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:43.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:43.425 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:43.451 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:43.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:43.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:43.556 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:12:43.557 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:43.566 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:43 smithi194 bash[92666]: cluster 2023-10-09T05:12:43.280436+0000 mon.a (mon.0) 922 : cluster 3 Health check update: Degraded data redundancy: 10194/47244 objects degraded (21.577%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T05:12:43.566 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:43 smithi194 bash[92666]: audit 2023-10-09T05:12:43.449849+0000 mon.a (mon.0) 923 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:12:43.566 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:43 smithi194 bash[92666]: audit 2023-10-09T05:12:43.451328+0000 mon.a (mon.0) 924 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:12:43.587 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:43.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:43.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:43.694 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:43.703 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:43 smithi133 bash[105589]: cluster 2023-10-09T05:12:43.280436+0000 mon.a (mon.0) 922 : cluster 3 Health check update: Degraded data redundancy: 10194/47244 objects degraded (21.577%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T05:12:43.703 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:43 smithi133 bash[105589]: audit 2023-10-09T05:12:43.449849+0000 mon.a (mon.0) 923 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:12:43.703 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:43 smithi133 bash[105589]: audit 2023-10-09T05:12:43.451328+0000 mon.a (mon.0) 924 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:12:43.704 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:43 smithi133 bash[102858]: cluster 2023-10-09T05:12:43.280436+0000 mon.a (mon.0) 922 : cluster 3 Health check update: Degraded data redundancy: 10194/47244 objects degraded (21.577%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T05:12:43.704 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:43 smithi133 bash[102858]: audit 2023-10-09T05:12:43.449849+0000 mon.a (mon.0) 923 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:12:43.704 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:43 smithi133 bash[102858]: audit 2023-10-09T05:12:43.451328+0000 mon.a (mon.0) 924 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:12:43.732 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:43.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:43.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:43.836 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:43.862 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:43.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:43.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:43.967 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:44.000 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:44.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:44.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:44.058 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:12:44.104 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:44.132 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:44.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:44.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:44.237 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:44.263 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:44.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:44.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:44.367 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:44.392 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:44.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:44.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:44.495 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:44.501 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:12:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:12:44] "GET /metrics HTTP/1.1" 200 36411 "" "Prometheus/2.43.0" 2023-10-09T05:12:44.502 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:44 smithi133 bash[105589]: cluster 2023-10-09T05:12:43.840594+0000 mgr.y (mgr.25277) 726 : cluster 0 pgmap v707: 109 pgs: 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 900 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 857 op/s; 10194/43224 objects degraded (23.584%); 0 B/s, 94 objects/s recovering 2023-10-09T05:12:44.502 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:44 smithi133 bash[102858]: cluster 2023-10-09T05:12:43.840594+0000 mgr.y (mgr.25277) 726 : cluster 0 pgmap v707: 109 pgs: 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 900 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 857 op/s; 10194/43224 objects degraded (23.584%); 0 B/s, 94 objects/s recovering 2023-10-09T05:12:44.531 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:44.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:44.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:44.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:44 smithi194 bash[92666]: cluster 2023-10-09T05:12:43.840594+0000 mgr.y (mgr.25277) 726 : cluster 0 pgmap v707: 109 pgs: 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 900 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 857 op/s; 10194/43224 objects degraded (23.584%); 0 B/s, 94 objects/s recovering 2023-10-09T05:12:44.637 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:44.671 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:44.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:44.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:44.775 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:44.806 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:44.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:44.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:44.913 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:44.939 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:44.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:44.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:45.044 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:45.069 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:45.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:45.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:45.172 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:45.204 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:45.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:45.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:45.309 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:45.337 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:45.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:45.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:45.443 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:45.479 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:45.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:45.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:45.584 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:45.616 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:45.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:45.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:45.719 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:45.745 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:45.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:45.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:45.850 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:45.888 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:45.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:45.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:45.993 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:46.024 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:46.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:46.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:46.105 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:12:46.130 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:46.192 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:46.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:46.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:46.296 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:46.351 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:46.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:46.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:46.453 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:46.487 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:46.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:46.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:46.597 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:46.629 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:46.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:46.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:46.738 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:46.764 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:46.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:46.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:46.867 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:46.900 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:46.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:46.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:47.003 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:47.013 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:46 smithi194 bash[92666]: cluster 2023-10-09T05:12:45.841413+0000 mgr.y (mgr.25277) 727 : cluster 0 pgmap v708: 109 pgs: 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 639 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s; 10194/30726 objects degraded (33.177%); 0 B/s, 116 objects/s recovering 2023-10-09T05:12:47.033 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:47.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:47.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:47.120 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:12:47.136 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:47.171 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:47.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:47.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:47.277 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:47.298 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:47.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:47.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:47.325 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:46 smithi133 bash[105589]: cluster 2023-10-09T05:12:45.841413+0000 mgr.y (mgr.25277) 727 : cluster 0 pgmap v708: 109 pgs: 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 639 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s; 10194/30726 objects degraded (33.177%); 0 B/s, 116 objects/s recovering 2023-10-09T05:12:47.325 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:46 smithi133 bash[102858]: cluster 2023-10-09T05:12:45.841413+0000 mgr.y (mgr.25277) 727 : cluster 0 pgmap v708: 109 pgs: 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 639 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s; 10194/30726 objects degraded (33.177%); 0 B/s, 116 objects/s recovering 2023-10-09T05:12:47.404 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:47.435 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:47.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:47.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:47.539 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:47.565 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:47.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:47.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:47.671 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:47.705 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:47.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:47.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:47.810 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:47.838 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:47.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:47.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:47.941 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:47.972 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:47.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:47.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:48.076 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:48.105 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:48.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:48.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:48.138 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:12:48.209 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:48.234 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:48.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:48.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:48.341 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:48.375 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:48.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:48.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:48.481 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:48.536 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:48.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:48.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:48.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:48 smithi194 bash[92666]: audit 2023-10-09T05:12:47.461111+0000 mon.a (mon.0) 925 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:12:48.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:48 smithi194 bash[92666]: audit 2023-10-09T05:12:47.473248+0000 mon.a (mon.0) 926 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:12:48.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:48 smithi194 bash[92666]: cluster 2023-10-09T05:12:47.843928+0000 mgr.y (mgr.25277) 728 : cluster 0 pgmap v709: 109 pgs: 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 639 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 10194/30726 objects degraded (33.177%); 0 B/s, 116 objects/s recovering 2023-10-09T05:12:48.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:48 smithi194 bash[92666]: audit 2023-10-09T05:12:47.978015+0000 mon.a (mon.0) 927 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:12:48.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:48 smithi194 bash[92666]: audit 2023-10-09T05:12:47.988984+0000 mon.a (mon.0) 928 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:12:48.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:48 smithi194 bash[92666]: cluster 2023-10-09T05:12:48.281654+0000 mon.a (mon.0) 929 : cluster 3 Health check update: Degraded data redundancy: 10194/30726 objects degraded (33.177%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T05:12:48.639 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:48.668 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:48.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:48.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:48.772 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:48.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:48 smithi133 bash[105589]: audit 2023-10-09T05:12:47.461111+0000 mon.a (mon.0) 925 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:12:48.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:48 smithi133 bash[105589]: audit 2023-10-09T05:12:47.473248+0000 mon.a (mon.0) 926 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:12:48.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:48 smithi133 bash[105589]: cluster 2023-10-09T05:12:47.843928+0000 mgr.y (mgr.25277) 728 : cluster 0 pgmap v709: 109 pgs: 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 639 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 10194/30726 objects degraded (33.177%); 0 B/s, 116 objects/s recovering 2023-10-09T05:12:48.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:48 smithi133 bash[105589]: audit 2023-10-09T05:12:47.978015+0000 mon.a (mon.0) 927 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:12:48.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:48 smithi133 bash[105589]: audit 2023-10-09T05:12:47.988984+0000 mon.a (mon.0) 928 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:12:48.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:48 smithi133 bash[105589]: cluster 2023-10-09T05:12:48.281654+0000 mon.a (mon.0) 929 : cluster 3 Health check update: Degraded data redundancy: 10194/30726 objects degraded (33.177%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T05:12:48.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:48 smithi133 bash[102858]: audit 2023-10-09T05:12:47.461111+0000 mon.a (mon.0) 925 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:12:48.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:48 smithi133 bash[102858]: audit 2023-10-09T05:12:47.473248+0000 mon.a (mon.0) 926 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:12:48.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:48 smithi133 bash[102858]: cluster 2023-10-09T05:12:47.843928+0000 mgr.y (mgr.25277) 728 : cluster 0 pgmap v709: 109 pgs: 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 639 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 10194/30726 objects degraded (33.177%); 0 B/s, 116 objects/s recovering 2023-10-09T05:12:48.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:48 smithi133 bash[102858]: audit 2023-10-09T05:12:47.978015+0000 mon.a (mon.0) 927 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:12:48.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:48 smithi133 bash[102858]: audit 2023-10-09T05:12:47.988984+0000 mon.a (mon.0) 928 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:12:48.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:48 smithi133 bash[102858]: cluster 2023-10-09T05:12:48.281654+0000 mon.a (mon.0) 929 : cluster 3 Health check update: Degraded data redundancy: 10194/30726 objects degraded (33.177%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T05:12:48.800 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:48.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:48.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:48.909 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:48.932 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:48.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:48.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:49.036 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:49.070 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:49.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:49.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:49.174 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:49.196 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:49.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:49.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:49.298 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:49.323 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:49.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:49.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:49.429 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:49.460 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:49.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:49.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:49.564 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:49.573 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:49 smithi194 bash[92666]: audit 2023-10-09T05:12:48.650118+0000 mon.a (mon.0) 930 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:12:49.574 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:49 smithi194 bash[92666]: audit 2023-10-09T05:12:48.651221+0000 mon.a (mon.0) 931 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:12:49.574 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:49 smithi194 bash[92666]: audit 2023-10-09T05:12:48.666734+0000 mon.a (mon.0) 932 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:12:49.604 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:49.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:49.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:49.708 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:49.716 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:49 smithi133 bash[105589]: audit 2023-10-09T05:12:48.650118+0000 mon.a (mon.0) 930 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:12:49.716 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:49 smithi133 bash[105589]: audit 2023-10-09T05:12:48.651221+0000 mon.a (mon.0) 931 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:12:49.717 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:49 smithi133 bash[105589]: audit 2023-10-09T05:12:48.666734+0000 mon.a (mon.0) 932 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:12:49.717 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:49 smithi133 bash[102858]: audit 2023-10-09T05:12:48.650118+0000 mon.a (mon.0) 930 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:12:49.717 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:49 smithi133 bash[102858]: audit 2023-10-09T05:12:48.651221+0000 mon.a (mon.0) 931 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:12:49.717 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:49 smithi133 bash[102858]: audit 2023-10-09T05:12:48.666734+0000 mon.a (mon.0) 932 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:12:49.730 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:49.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:49.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:49.833 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:49.868 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:49.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:49.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:49.972 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:50.001 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:50.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:50.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:50.105 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:50.144 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:50.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:50.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:50.248 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:50.266 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:50.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:50.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:50.370 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:50.394 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:50.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:50.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:50.497 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:50.515 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:50.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:50.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:50.547 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:50 smithi133 bash[105589]: cluster 2023-10-09T05:12:49.844900+0000 mgr.y (mgr.25277) 729 : cluster 0 pgmap v710: 109 pgs: 9 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 360 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.22k op/s; 9257/17349 objects degraded (53.358%); 0 B/s, 96 objects/s recovering 2023-10-09T05:12:50.547 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:50 smithi133 bash[102858]: cluster 2023-10-09T05:12:49.844900+0000 mgr.y (mgr.25277) 729 : cluster 0 pgmap v710: 109 pgs: 9 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 360 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.22k op/s; 9257/17349 objects degraded (53.358%); 0 B/s, 96 objects/s recovering 2023-10-09T05:12:50.618 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:50.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:50 smithi194 bash[92666]: cluster 2023-10-09T05:12:49.844900+0000 mgr.y (mgr.25277) 729 : cluster 0 pgmap v710: 109 pgs: 9 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 360 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.22k op/s; 9257/17349 objects degraded (53.358%); 0 B/s, 96 objects/s recovering 2023-10-09T05:12:50.653 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:50.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:50.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:50.758 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:50.778 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:50.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:50.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:50.883 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:50.910 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:50.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:50.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:51.013 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:51.039 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:51.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:51.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:51.143 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:51.167 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:51.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:51.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:51.272 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:51.296 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:51.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:51.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:51.401 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:51.430 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:51.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:51.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:51.536 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:51.567 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:51.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:51.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:51.671 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:51.702 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:51.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:51.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:51.806 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:51.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:51.832 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:51.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:51.933 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:51.966 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:51.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:51.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:52.071 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:52.095 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:52.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:52.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:52.201 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:52.228 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:52.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:52.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:52.334 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:52.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:52.363 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:52.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:52.464 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:52.500 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:52.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:52.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:52.603 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:52.632 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:52.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:52.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:52.736 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:52.762 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:52.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:52.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:52.866 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:52.898 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:52.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:52.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:53.004 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:53.029 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:53.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:53.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:53.133 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:53.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:52 smithi194 bash[92666]: cluster 2023-10-09T05:12:51.845814+0000 mgr.y (mgr.25277) 730 : cluster 0 pgmap v711: 109 pgs: 9 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 321 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 913 op/s; 9257/15459 objects degraded (59.881%); 0 B/s, 63 objects/s recovering 2023-10-09T05:12:53.160 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:53.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:53.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:53.164 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:52 smithi133 bash[105589]: cluster 2023-10-09T05:12:51.845814+0000 mgr.y (mgr.25277) 730 : cluster 0 pgmap v711: 109 pgs: 9 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 321 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 913 op/s; 9257/15459 objects degraded (59.881%); 0 B/s, 63 objects/s recovering 2023-10-09T05:12:53.164 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:52 smithi133 bash[102858]: cluster 2023-10-09T05:12:51.845814+0000 mgr.y (mgr.25277) 730 : cluster 0 pgmap v711: 109 pgs: 9 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 321 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 913 op/s; 9257/15459 objects degraded (59.881%); 0 B/s, 63 objects/s recovering 2023-10-09T05:12:53.265 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:53.293 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:53.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:53.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:53.398 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:53.425 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:53.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:53.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:53.528 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:53.561 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:53.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:53.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:53.665 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:53.693 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:53.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:53.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:53.796 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:53.830 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:53.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:53.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:53.934 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:53.957 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:53.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:53.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:54.063 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:54.072 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:53 smithi133 bash[102858]: cluster 2023-10-09T05:12:53.282648+0000 mon.a (mon.0) 933 : cluster 3 Health check update: Degraded data redundancy: 9257/15459 objects degraded (59.881%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:12:54.072 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:53 smithi133 bash[105589]: cluster 2023-10-09T05:12:53.282648+0000 mon.a (mon.0) 933 : cluster 3 Health check update: Degraded data redundancy: 9257/15459 objects degraded (59.881%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:12:54.089 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:54.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:54.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:54.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:53 smithi194 bash[92666]: cluster 2023-10-09T05:12:53.282648+0000 mon.a (mon.0) 933 : cluster 3 Health check update: Degraded data redundancy: 9257/15459 objects degraded (59.881%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:12:54.192 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:54.222 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:54.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:54.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:54.252 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Removed 89966 objects 2023-10-09T05:12:54.252 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Clean up completed and total clean up time :113.969 2023-10-09T05:12:54.327 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:54.334 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:12:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:12:54] "GET /metrics HTTP/1.1" 200 36411 "" "Prometheus/2.43.0" 2023-10-09T05:12:54.350 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:54.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:54.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:54.453 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:54.482 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:54.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:54.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:54.585 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:54.607 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:54.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:54.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:54.710 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:54.739 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:54.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:54.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:54.842 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:54.866 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:54.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:54.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:54.969 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:54.994 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:54.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:54.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:55.097 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:55.108 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:54 smithi133 bash[105589]: cluster 2023-10-09T05:12:53.846399+0000 mgr.y (mgr.25277) 731 : cluster 0 pgmap v712: 109 pgs: 9 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 237 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 996 op/s; 9257/11409 objects degraded (81.138%); 0 B/s, 63 objects/s recovering 2023-10-09T05:12:55.109 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:54 smithi133 bash[102858]: cluster 2023-10-09T05:12:53.846399+0000 mgr.y (mgr.25277) 731 : cluster 0 pgmap v712: 109 pgs: 9 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 237 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 996 op/s; 9257/11409 objects degraded (81.138%); 0 B/s, 63 objects/s recovering 2023-10-09T05:12:55.125 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:55.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:55.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:55.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:54 smithi194 bash[92666]: cluster 2023-10-09T05:12:53.846399+0000 mgr.y (mgr.25277) 731 : cluster 0 pgmap v712: 109 pgs: 9 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 237 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 996 op/s; 9257/11409 objects degraded (81.138%); 0 B/s, 63 objects/s recovering 2023-10-09T05:12:55.229 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:55.263 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:55.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:55.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:55.372 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:55.413 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:55.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:55.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:55.520 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:55.547 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:55.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:55.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:55.651 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:55.684 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:55.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:55.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:55.788 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:55.822 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:55.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:55.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:55.926 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:55.954 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:55.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:55.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:56.059 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:56.092 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:56.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:56.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:56.196 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:56.230 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:56.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:56.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:56.334 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:56.373 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:56.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:56.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:56.477 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:56.498 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:56.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:56.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:56.601 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:56.633 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:56.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:56.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:56.658 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2023-10-09T05:12:56.658 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2023-10-09T05:12:56.737 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:56.757 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:56.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:56.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:56.860 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:56.892 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:56.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:56.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:56.995 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:57.026 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:12:57.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:57.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:12:57.068 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:56 smithi133 bash[105589]: cluster 2023-10-09T05:12:55.847428+0000 mgr.y (mgr.25277) 732 : cluster 0 pgmap v713: 109 pgs: 9 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 28 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s; 9257/1389 objects degraded (666.451%); 0 B/s, 97 objects/s recovering 2023-10-09T05:12:57.069 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:56 smithi133 bash[102858]: cluster 2023-10-09T05:12:55.847428+0000 mgr.y (mgr.25277) 732 : cluster 0 pgmap v713: 109 pgs: 9 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 28 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s; 9257/1389 objects degraded (666.451%); 0 B/s, 97 objects/s recovering 2023-10-09T05:12:57.129 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:57.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:56 smithi194 bash[92666]: cluster 2023-10-09T05:12:55.847428+0000 mgr.y (mgr.25277) 732 : cluster 0 pgmap v713: 109 pgs: 9 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 28 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s; 9257/1389 objects degraded (666.451%); 0 B/s, 97 objects/s recovering 2023-10-09T05:12:57.149 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:57.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:57.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:57.252 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:57.281 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:57.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:57.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:57.384 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:57.411 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:57.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:57.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:57.514 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:57.545 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:57.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:57.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:57.649 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:57.680 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:57.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:57.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:57.784 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:57.814 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:57.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:57.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:57.917 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:57.952 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:57.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:57.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:58.055 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:58.083 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:12:58.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:58.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:12:58.186 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:58.206 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:58.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:58.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:58.309 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:58.327 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:58.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:58.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:58.430 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:58.454 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:58.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:58.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:58.557 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:58.585 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:12:58.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:58.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:12:58.688 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:58.714 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:58.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:58.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:58.817 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:58.841 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:58.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:58.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:58.944 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:58.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:58.975 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:58.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:59.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:58 smithi133 bash[102858]: cluster 2023-10-09T05:12:57.848294+0000 mgr.y (mgr.25277) 733 : cluster 0 pgmap v714: 109 pgs: 9 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 28 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 9257/1389 objects degraded (666.451%); 0 B/s, 67 objects/s recovering 2023-10-09T05:12:59.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:58 smithi133 bash[102858]: cluster 2023-10-09T05:12:58.283810+0000 mon.a (mon.0) 934 : cluster 3 Health check update: Degraded data redundancy: 9257/1389 objects degraded (666.451%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:12:59.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:12:58 smithi133 bash[102858]: audit 2023-10-09T05:12:58.444334+0000 mon.a (mon.0) 935 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:12:59.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:58 smithi133 bash[105589]: cluster 2023-10-09T05:12:57.848294+0000 mgr.y (mgr.25277) 733 : cluster 0 pgmap v714: 109 pgs: 9 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 28 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 9257/1389 objects degraded (666.451%); 0 B/s, 67 objects/s recovering 2023-10-09T05:12:59.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:58 smithi133 bash[105589]: cluster 2023-10-09T05:12:58.283810+0000 mon.a (mon.0) 934 : cluster 3 Health check update: Degraded data redundancy: 9257/1389 objects degraded (666.451%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:12:59.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:12:58 smithi133 bash[105589]: audit 2023-10-09T05:12:58.444334+0000 mon.a (mon.0) 935 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:12:59.076 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:59.108 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:12:59.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:59.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:12:59.211 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:59.222 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:58 smithi194 bash[92666]: cluster 2023-10-09T05:12:57.848294+0000 mgr.y (mgr.25277) 733 : cluster 0 pgmap v714: 109 pgs: 9 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 28 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 9257/1389 objects degraded (666.451%); 0 B/s, 67 objects/s recovering 2023-10-09T05:12:59.222 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:58 smithi194 bash[92666]: cluster 2023-10-09T05:12:58.283810+0000 mon.a (mon.0) 934 : cluster 3 Health check update: Degraded data redundancy: 9257/1389 objects degraded (666.451%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:12:59.222 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:12:58 smithi194 bash[92666]: audit 2023-10-09T05:12:58.444334+0000 mon.a (mon.0) 935 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:12:59.237 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:59.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:59.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:59.343 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:59.365 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:59.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:59.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:59.470 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:59.498 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:12:59.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:59.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:12:59.601 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:59.623 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:12:59.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:59.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:12:59.726 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:59.748 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:59.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:59.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:12:59.851 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:59.873 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:12:59.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:12:59.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:12:59.976 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:59.996 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:12:59.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:00.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:00.042 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:13:00.043 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:13:00.101 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:00.124 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:00.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:00.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:00.227 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:00.258 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:00.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:00.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:00.361 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:00.388 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:00.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:00.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:00.492 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:00.515 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:00.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:00.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:00.619 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:00.651 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:00.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:00.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:00.755 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:00.778 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:00.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:00.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:00.882 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:00.904 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:00.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:00.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:01.008 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:01.039 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:01.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:01.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:01.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:00 smithi133 bash[102858]: cluster 2023-10-09T05:12:59.849261+0000 mgr.y (mgr.25277) 734 : cluster 0 pgmap v715: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 16 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 851 op/s; 7521/798 objects degraded (942.481%); 0 B/s, 206 objects/s recovering 2023-10-09T05:13:01.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:00 smithi133 bash[105589]: cluster 2023-10-09T05:12:59.849261+0000 mgr.y (mgr.25277) 734 : cluster 0 pgmap v715: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 16 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 851 op/s; 7521/798 objects degraded (942.481%); 0 B/s, 206 objects/s recovering 2023-10-09T05:13:01.143 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:01.164 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:01.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:01.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:01.267 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:01.280 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:00 smithi194 bash[92666]: cluster 2023-10-09T05:12:59.849261+0000 mgr.y (mgr.25277) 734 : cluster 0 pgmap v715: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 16 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 851 op/s; 7521/798 objects degraded (942.481%); 0 B/s, 206 objects/s recovering 2023-10-09T05:13:01.298 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:01.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:01.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:01.402 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:01.432 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:01.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:01.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:01.536 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:01.545 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:01 smithi133 bash[102858]: debug 2023-10-09T05:13:01.200+0000 7f5eb4d1d700 -1 mon.a@0(leader).osd e171 definitely_dead 0 2023-10-09T05:13:01.562 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:01.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:01.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:01.665 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:01.696 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:01.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:01.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:01.800 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:01.827 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:01.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:01.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:01.930 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:01.941 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:01 smithi194 bash[92666]: audit 2023-10-09T05:13:01.201053+0000 mon.c (mon.2) 68 : audit 1 from='client.? 172.21.15.133:0/233251225' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T05:13:01.941 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:01 smithi194 bash[92666]: audit 2023-10-09T05:13:01.201418+0000 mon.a (mon.0) 936 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T05:13:01.942 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:01 smithi194 bash[92666]: cluster 2023-10-09T05:13:01.201506+0000 mon.a (mon.0) 937 : cluster 1 Client client.admin marked osd.5 out, while it was still marked up 2023-10-09T05:13:01.944 INFO:teuthology.orchestra.run.smithi133.stderr:marked out osd.5. 2023-10-09T05:13:01.963 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:01.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:01.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:02.067 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:02.078 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:01 smithi133 bash[105589]: audit 2023-10-09T05:13:01.201053+0000 mon.c (mon.2) 68 : audit 1 from='client.? 172.21.15.133:0/233251225' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T05:13:02.078 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:01 smithi133 bash[105589]: audit 2023-10-09T05:13:01.201418+0000 mon.a (mon.0) 936 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T05:13:02.078 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:01 smithi133 bash[105589]: cluster 2023-10-09T05:13:01.201506+0000 mon.a (mon.0) 937 : cluster 1 Client client.admin marked osd.5 out, while it was still marked up 2023-10-09T05:13:02.079 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:01 smithi133 bash[102858]: audit 2023-10-09T05:13:01.201053+0000 mon.c (mon.2) 68 : audit 1 from='client.? 172.21.15.133:0/233251225' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T05:13:02.079 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:01 smithi133 bash[102858]: audit 2023-10-09T05:13:01.201418+0000 mon.a (mon.0) 936 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T05:13:02.079 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:01 smithi133 bash[102858]: cluster 2023-10-09T05:13:01.201506+0000 mon.a (mon.0) 937 : cluster 1 Client client.admin marked osd.5 out, while it was still marked up 2023-10-09T05:13:02.092 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:02.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:02.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:02.195 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:02.217 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:02.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:02.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:02.320 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:02.346 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:02.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:02.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:02.448 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:02.482 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:02.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:02.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:02.585 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:02.607 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:02.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:02.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:02.711 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:02.737 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:02.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:02.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:02.840 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:02.870 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:02.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:02.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:02.974 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:03.009 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:03.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:03.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:03.020 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[102858]: cluster 2023-10-09T05:13:01.849859+0000 mgr.y (mgr.25277) 735 : cluster 0 pgmap v716: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 16 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 7521/798 objects degraded (942.481%); 0 B/s, 172 objects/s recovering 2023-10-09T05:13:03.020 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[102858]: audit 2023-10-09T05:13:01.926572+0000 mon.a (mon.0) 938 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-09T05:13:03.020 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[102858]: cluster 2023-10-09T05:13:01.926683+0000 mon.a (mon.0) 939 : cluster 0 osdmap e172: 8 total, 8 up, 7 in 2023-10-09T05:13:03.020 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[102858]: cluster 2023-10-09T05:13:01.930054+0000 osd.5 (osd.5) 158 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T05:13:03.021 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[102858]: cluster 2023-10-09T05:13:01.930308+0000 osd.7 (osd.7) 96 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T05:13:03.021 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[102858]: cluster 2023-10-09T05:13:01.930437+0000 osd.4 (osd.4) 109 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T05:13:03.021 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[102858]: cluster 2023-10-09T05:13:01.931008+0000 osd.6 (osd.6) 94 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T05:13:03.021 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[102858]: cluster 2023-10-09T05:13:01.933585+0000 osd.7 (osd.7) 97 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T05:13:03.021 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[102858]: cluster 2023-10-09T05:13:01.935064+0000 osd.5 (osd.5) 159 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T05:13:03.021 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[102858]: cluster 2023-10-09T05:13:01.935478+0000 osd.6 (osd.6) 95 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T05:13:03.021 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[102858]: cluster 2023-10-09T05:13:01.938254+0000 osd.5 (osd.5) 160 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T05:13:03.021 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[102858]: cluster 2023-10-09T05:13:01.941289+0000 osd.5 (osd.5) 161 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T05:13:03.113 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:03.122 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:02 smithi194 bash[92666]: cluster 2023-10-09T05:13:01.849859+0000 mgr.y (mgr.25277) 735 : cluster 0 pgmap v716: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 16 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 7521/798 objects degraded (942.481%); 0 B/s, 172 objects/s recovering 2023-10-09T05:13:03.122 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:02 smithi194 bash[92666]: audit 2023-10-09T05:13:01.926572+0000 mon.a (mon.0) 938 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-09T05:13:03.122 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:02 smithi194 bash[92666]: cluster 2023-10-09T05:13:01.926683+0000 mon.a (mon.0) 939 : cluster 0 osdmap e172: 8 total, 8 up, 7 in 2023-10-09T05:13:03.123 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:02 smithi194 bash[92666]: cluster 2023-10-09T05:13:01.930054+0000 osd.5 (osd.5) 158 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T05:13:03.123 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:02 smithi194 bash[92666]: cluster 2023-10-09T05:13:01.930308+0000 osd.7 (osd.7) 96 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T05:13:03.123 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:02 smithi194 bash[92666]: cluster 2023-10-09T05:13:01.930437+0000 osd.4 (osd.4) 109 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T05:13:03.123 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:02 smithi194 bash[92666]: cluster 2023-10-09T05:13:01.931008+0000 osd.6 (osd.6) 94 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T05:13:03.123 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:02 smithi194 bash[92666]: cluster 2023-10-09T05:13:01.933585+0000 osd.7 (osd.7) 97 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T05:13:03.123 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:02 smithi194 bash[92666]: cluster 2023-10-09T05:13:01.935064+0000 osd.5 (osd.5) 159 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T05:13:03.123 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:02 smithi194 bash[92666]: cluster 2023-10-09T05:13:01.935478+0000 osd.6 (osd.6) 95 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T05:13:03.123 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:02 smithi194 bash[92666]: cluster 2023-10-09T05:13:01.938254+0000 osd.5 (osd.5) 160 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T05:13:03.123 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:02 smithi194 bash[92666]: cluster 2023-10-09T05:13:01.941289+0000 osd.5 (osd.5) 161 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T05:13:03.136 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:03.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:03.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:03.239 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:03.268 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:03.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:03.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:03.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[105589]: cluster 2023-10-09T05:13:01.849859+0000 mgr.y (mgr.25277) 735 : cluster 0 pgmap v716: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 16 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 7521/798 objects degraded (942.481%); 0 B/s, 172 objects/s recovering 2023-10-09T05:13:03.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[105589]: audit 2023-10-09T05:13:01.926572+0000 mon.a (mon.0) 938 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-09T05:13:03.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[105589]: cluster 2023-10-09T05:13:01.926683+0000 mon.a (mon.0) 939 : cluster 0 osdmap e172: 8 total, 8 up, 7 in 2023-10-09T05:13:03.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[105589]: cluster 2023-10-09T05:13:01.930054+0000 osd.5 (osd.5) 158 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T05:13:03.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[105589]: cluster 2023-10-09T05:13:01.930308+0000 osd.7 (osd.7) 96 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T05:13:03.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[105589]: cluster 2023-10-09T05:13:01.930437+0000 osd.4 (osd.4) 109 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T05:13:03.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[105589]: cluster 2023-10-09T05:13:01.931008+0000 osd.6 (osd.6) 94 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T05:13:03.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[105589]: cluster 2023-10-09T05:13:01.933585+0000 osd.7 (osd.7) 97 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T05:13:03.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[105589]: cluster 2023-10-09T05:13:01.935064+0000 osd.5 (osd.5) 159 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T05:13:03.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[105589]: cluster 2023-10-09T05:13:01.935478+0000 osd.6 (osd.6) 95 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T05:13:03.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[105589]: cluster 2023-10-09T05:13:01.938254+0000 osd.5 (osd.5) 160 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T05:13:03.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[105589]: cluster 2023-10-09T05:13:01.941289+0000 osd.5 (osd.5) 161 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T05:13:03.371 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:03.398 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:03.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:03.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:03.501 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:03.533 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:03.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:03.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:03.636 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:03.661 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:03.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:03.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:03.699 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T05:13:03.765 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:03.807 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:03.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:03.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:03.910 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:03.940 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[102858]: cluster 2023-10-09T05:13:02.928491+0000 osd.7 (osd.7) 98 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T05:13:03.940 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[102858]: cluster 2023-10-09T05:13:02.930703+0000 mon.a (mon.0) 940 : cluster 0 osdmap e173: 8 total, 8 up, 7 in 2023-10-09T05:13:03.940 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[102858]: cluster 2023-10-09T05:13:02.931367+0000 osd.6 (osd.6) 96 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T05:13:03.940 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[102858]: cluster 2023-10-09T05:13:02.931550+0000 osd.4 (osd.4) 110 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T05:13:03.940 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[102858]: cluster 2023-10-09T05:13:02.931975+0000 osd.5 (osd.5) 162 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T05:13:03.940 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[102858]: cluster 2023-10-09T05:13:02.932778+0000 osd.7 (osd.7) 99 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T05:13:03.940 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[102858]: cluster 2023-10-09T05:13:02.934346+0000 osd.5 (osd.5) 163 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T05:13:03.940 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[102858]: cluster 2023-10-09T05:13:02.935029+0000 osd.6 (osd.6) 97 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T05:13:03.941 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[102858]: cluster 2023-10-09T05:13:02.937864+0000 osd.5 (osd.5) 164 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T05:13:03.941 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[102858]: cluster 2023-10-09T05:13:03.284769+0000 mon.a (mon.0) 941 : cluster 3 Health check update: Degraded data redundancy: 7521/798 objects degraded (942.481%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:13:03.941 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[102858]: cluster 2023-10-09T05:13:03.930101+0000 mon.a (mon.0) 942 : cluster 0 osdmap e174: 8 total, 8 up, 7 in 2023-10-09T05:13:03.941 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:03.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:03.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:04.045 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:04.090 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:04.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:04.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:04.197 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:04.208 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[105589]: cluster 2023-10-09T05:13:02.928491+0000 osd.7 (osd.7) 98 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T05:13:04.208 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[105589]: cluster 2023-10-09T05:13:02.930703+0000 mon.a (mon.0) 940 : cluster 0 osdmap e173: 8 total, 8 up, 7 in 2023-10-09T05:13:04.208 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[105589]: cluster 2023-10-09T05:13:02.931367+0000 osd.6 (osd.6) 96 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T05:13:04.208 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[105589]: cluster 2023-10-09T05:13:02.931550+0000 osd.4 (osd.4) 110 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T05:13:04.208 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[105589]: cluster 2023-10-09T05:13:02.931975+0000 osd.5 (osd.5) 162 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T05:13:04.208 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[105589]: cluster 2023-10-09T05:13:02.932778+0000 osd.7 (osd.7) 99 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T05:13:04.208 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[105589]: cluster 2023-10-09T05:13:02.934346+0000 osd.5 (osd.5) 163 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T05:13:04.208 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[105589]: cluster 2023-10-09T05:13:02.935029+0000 osd.6 (osd.6) 97 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T05:13:04.208 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[105589]: cluster 2023-10-09T05:13:02.937864+0000 osd.5 (osd.5) 164 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T05:13:04.208 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[105589]: cluster 2023-10-09T05:13:03.284769+0000 mon.a (mon.0) 941 : cluster 3 Health check update: Degraded data redundancy: 7521/798 objects degraded (942.481%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:13:04.209 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:03 smithi133 bash[105589]: cluster 2023-10-09T05:13:03.930101+0000 mon.a (mon.0) 942 : cluster 0 osdmap e174: 8 total, 8 up, 7 in 2023-10-09T05:13:04.253 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:04.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:04.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:04.361 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:04.374 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:03 smithi194 bash[92666]: cluster 2023-10-09T05:13:02.928491+0000 osd.7 (osd.7) 98 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T05:13:04.374 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:03 smithi194 bash[92666]: cluster 2023-10-09T05:13:02.930703+0000 mon.a (mon.0) 940 : cluster 0 osdmap e173: 8 total, 8 up, 7 in 2023-10-09T05:13:04.374 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:03 smithi194 bash[92666]: cluster 2023-10-09T05:13:02.931367+0000 osd.6 (osd.6) 96 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T05:13:04.374 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:03 smithi194 bash[92666]: cluster 2023-10-09T05:13:02.931550+0000 osd.4 (osd.4) 110 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T05:13:04.374 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:03 smithi194 bash[92666]: cluster 2023-10-09T05:13:02.931975+0000 osd.5 (osd.5) 162 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T05:13:04.374 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:03 smithi194 bash[92666]: cluster 2023-10-09T05:13:02.932778+0000 osd.7 (osd.7) 99 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T05:13:04.374 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:03 smithi194 bash[92666]: cluster 2023-10-09T05:13:02.934346+0000 osd.5 (osd.5) 163 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T05:13:04.375 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:03 smithi194 bash[92666]: cluster 2023-10-09T05:13:02.935029+0000 osd.6 (osd.6) 97 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T05:13:04.375 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:03 smithi194 bash[92666]: cluster 2023-10-09T05:13:02.937864+0000 osd.5 (osd.5) 164 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T05:13:04.375 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:03 smithi194 bash[92666]: cluster 2023-10-09T05:13:03.284769+0000 mon.a (mon.0) 941 : cluster 3 Health check update: Degraded data redundancy: 7521/798 objects degraded (942.481%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:13:04.375 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:03 smithi194 bash[92666]: cluster 2023-10-09T05:13:03.930101+0000 mon.a (mon.0) 942 : cluster 0 osdmap e174: 8 total, 8 up, 7 in 2023-10-09T05:13:04.392 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:04.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:04.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:04.492 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:13:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:13:04] "GET /metrics HTTP/1.1" 200 36427 "" "Prometheus/2.43.0" 2023-10-09T05:13:04.496 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:04.522 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:04.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:04.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:04.626 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:04.655 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:04.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:04.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:04.760 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:04.785 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:04.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:04.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:04.888 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:04.918 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:04.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:04.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:04.977 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:13:05.020 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:13:05.022 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:05.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:04 smithi133 bash[102858]: cluster 2023-10-09T05:13:03.850374+0000 mgr.y (mgr.25277) 736 : cluster 0 pgmap v719: 109 pgs: 2 remapped+peering, 5 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 16 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 33 op/s; 7052/798 objects degraded (883.709%); 0 B/s, 267 objects/s recovering 2023-10-09T05:13:05.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:04 smithi133 bash[102858]: cluster 2023-10-09T05:13:03.927748+0000 osd.4 (osd.4) 111 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T05:13:05.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:04 smithi133 bash[102858]: cluster 2023-10-09T05:13:03.928921+0000 osd.2 (osd.2) 1 : cluster 0 9.16 starting backfill to osd.6 from (0'0,150'2627] MIN to 171'9287 2023-10-09T05:13:05.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:04 smithi133 bash[102858]: cluster 2023-10-09T05:13:03.929244+0000 osd.2 (osd.2) 2 : cluster 0 9.6 starting backfill to osd.6 from (0'0,150'2631] MIN to 171'9282 2023-10-09T05:13:05.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:04 smithi133 bash[102858]: cluster 2023-10-09T05:13:03.930981+0000 osd.7 (osd.7) 100 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T05:13:05.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:04 smithi133 bash[102858]: cluster 2023-10-09T05:13:03.933998+0000 osd.7 (osd.7) 101 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T05:13:05.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:04 smithi133 bash[102858]: cluster 2023-10-09T05:13:03.934861+0000 osd.0 (osd.0) 3 : cluster 0 9.1b starting backfill to osd.2 from (0'0,0'0] MAX to 171'10582 2023-10-09T05:13:05.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:04 smithi133 bash[102858]: cluster 2023-10-09T05:13:03.935734+0000 osd.7 (osd.7) 102 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T05:13:05.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:04 smithi133 bash[102858]: cluster 2023-10-09T05:13:03.938766+0000 osd.7 (osd.7) 103 : cluster 0 9.1d starting backfill to osd.4 from (0'0,150'2628] MIN to 171'10551 2023-10-09T05:13:05.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:04 smithi133 bash[102858]: cluster 2023-10-09T05:13:03.949560+0000 osd.0 (osd.0) 4 : cluster 0 9.b starting backfill to osd.2 from (0'0,0'0] MAX to 171'10587 2023-10-09T05:13:05.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:04 smithi133 bash[102858]: cluster 2023-10-09T05:13:03.957280+0000 osd.7 (osd.7) 104 : cluster 0 9.d starting backfill to osd.4 from (0'0,0'0] MAX to 171'10544 2023-10-09T05:13:05.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:04 smithi133 bash[102858]: cluster 2023-10-09T05:13:04.036703+0000 osd.6 (osd.6) 98 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T05:13:05.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:04 smithi133 bash[105589]: cluster 2023-10-09T05:13:03.850374+0000 mgr.y (mgr.25277) 736 : cluster 0 pgmap v719: 109 pgs: 2 remapped+peering, 5 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 16 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 33 op/s; 7052/798 objects degraded (883.709%); 0 B/s, 267 objects/s recovering 2023-10-09T05:13:05.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:04 smithi133 bash[105589]: cluster 2023-10-09T05:13:03.927748+0000 osd.4 (osd.4) 111 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T05:13:05.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:04 smithi133 bash[105589]: cluster 2023-10-09T05:13:03.928921+0000 osd.2 (osd.2) 1 : cluster 0 9.16 starting backfill to osd.6 from (0'0,150'2627] MIN to 171'9287 2023-10-09T05:13:05.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:04 smithi133 bash[105589]: cluster 2023-10-09T05:13:03.929244+0000 osd.2 (osd.2) 2 : cluster 0 9.6 starting backfill to osd.6 from (0'0,150'2631] MIN to 171'9282 2023-10-09T05:13:05.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:04 smithi133 bash[105589]: cluster 2023-10-09T05:13:03.930981+0000 osd.7 (osd.7) 100 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T05:13:05.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:04 smithi133 bash[105589]: cluster 2023-10-09T05:13:03.933998+0000 osd.7 (osd.7) 101 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T05:13:05.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:04 smithi133 bash[105589]: cluster 2023-10-09T05:13:03.934861+0000 osd.0 (osd.0) 3 : cluster 0 9.1b starting backfill to osd.2 from (0'0,0'0] MAX to 171'10582 2023-10-09T05:13:05.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:04 smithi133 bash[105589]: cluster 2023-10-09T05:13:03.935734+0000 osd.7 (osd.7) 102 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T05:13:05.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:04 smithi133 bash[105589]: cluster 2023-10-09T05:13:03.938766+0000 osd.7 (osd.7) 103 : cluster 0 9.1d starting backfill to osd.4 from (0'0,150'2628] MIN to 171'10551 2023-10-09T05:13:05.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:04 smithi133 bash[105589]: cluster 2023-10-09T05:13:03.949560+0000 osd.0 (osd.0) 4 : cluster 0 9.b starting backfill to osd.2 from (0'0,0'0] MAX to 171'10587 2023-10-09T05:13:05.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:04 smithi133 bash[105589]: cluster 2023-10-09T05:13:03.957280+0000 osd.7 (osd.7) 104 : cluster 0 9.d starting backfill to osd.4 from (0'0,0'0] MAX to 171'10544 2023-10-09T05:13:05.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:04 smithi133 bash[105589]: cluster 2023-10-09T05:13:04.036703+0000 osd.6 (osd.6) 98 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T05:13:05.051 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:05.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:05.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:05.066 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:13:05.104 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:13:05.126 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:13:05.153 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:05.160 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:13:05.166 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:04 smithi194 bash[92666]: cluster 2023-10-09T05:13:03.850374+0000 mgr.y (mgr.25277) 736 : cluster 0 pgmap v719: 109 pgs: 2 remapped+peering, 5 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 16 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 33 op/s; 7052/798 objects degraded (883.709%); 0 B/s, 267 objects/s recovering 2023-10-09T05:13:05.167 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:04 smithi194 bash[92666]: cluster 2023-10-09T05:13:03.927748+0000 osd.4 (osd.4) 111 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T05:13:05.167 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:04 smithi194 bash[92666]: cluster 2023-10-09T05:13:03.928921+0000 osd.2 (osd.2) 1 : cluster 0 9.16 starting backfill to osd.6 from (0'0,150'2627] MIN to 171'9287 2023-10-09T05:13:05.167 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:04 smithi194 bash[92666]: cluster 2023-10-09T05:13:03.929244+0000 osd.2 (osd.2) 2 : cluster 0 9.6 starting backfill to osd.6 from (0'0,150'2631] MIN to 171'9282 2023-10-09T05:13:05.167 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:04 smithi194 bash[92666]: cluster 2023-10-09T05:13:03.930981+0000 osd.7 (osd.7) 100 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T05:13:05.167 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:04 smithi194 bash[92666]: cluster 2023-10-09T05:13:03.933998+0000 osd.7 (osd.7) 101 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T05:13:05.167 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:04 smithi194 bash[92666]: cluster 2023-10-09T05:13:03.934861+0000 osd.0 (osd.0) 3 : cluster 0 9.1b starting backfill to osd.2 from (0'0,0'0] MAX to 171'10582 2023-10-09T05:13:05.167 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:04 smithi194 bash[92666]: cluster 2023-10-09T05:13:03.935734+0000 osd.7 (osd.7) 102 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T05:13:05.167 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:04 smithi194 bash[92666]: cluster 2023-10-09T05:13:03.938766+0000 osd.7 (osd.7) 103 : cluster 0 9.1d starting backfill to osd.4 from (0'0,150'2628] MIN to 171'10551 2023-10-09T05:13:05.167 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:04 smithi194 bash[92666]: cluster 2023-10-09T05:13:03.949560+0000 osd.0 (osd.0) 4 : cluster 0 9.b starting backfill to osd.2 from (0'0,0'0] MAX to 171'10587 2023-10-09T05:13:05.168 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:04 smithi194 bash[92666]: cluster 2023-10-09T05:13:03.957280+0000 osd.7 (osd.7) 104 : cluster 0 9.d starting backfill to osd.4 from (0'0,0'0] MAX to 171'10544 2023-10-09T05:13:05.168 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:04 smithi194 bash[92666]: cluster 2023-10-09T05:13:04.036703+0000 osd.6 (osd.6) 98 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T05:13:05.186 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:05.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:05.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:05.195 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:13:05.226 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:13:05.228 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:13:05.228 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:13:05.228 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:13:05.229 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:13:05.229 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:13:05.229 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:13:05.229 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:13:05.229 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:13:05.289 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:05.322 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:05.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:05.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:05.352 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:13:05.425 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:05.452 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:05.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:05.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:05.555 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:05.580 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:05.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:05.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:05.682 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:05.710 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:05.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:05.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:05.813 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:05.844 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:05.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:05.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:05.948 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:05.971 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:05.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:05.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:06.074 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:06.095 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:06.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:06.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:06.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:05 smithi194 bash[92666]: audit 2023-10-09T05:13:04.946928+0000 mon.a (mon.0) 943 : audit 0 from='client.? 172.21.15.133:0/3711082133' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:13:06.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:05 smithi194 bash[92666]: cluster 2023-10-09T05:13:05.299971+0000 mon.a (mon.0) 944 : cluster 0 osdmap e175: 8 total, 8 up, 7 in 2023-10-09T05:13:06.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:05 smithi194 bash[92666]: cluster 2023-10-09T05:13:05.301581+0000 osd.6 (osd.6) 99 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T05:13:06.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:05 smithi194 bash[92666]: cluster 2023-10-09T05:13:05.304303+0000 osd.6 (osd.6) 100 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T05:13:06.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:05 smithi194 bash[92666]: cluster 2023-10-09T05:13:05.305224+0000 osd.7 (osd.7) 105 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T05:13:06.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:05 smithi194 bash[92666]: audit 2023-10-09T05:13:05.902648+0000 mon.c (mon.2) 69 : audit 1 from='client.? 172.21.15.133:0/1759056279' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:13:06.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:05 smithi194 bash[92666]: audit 2023-10-09T05:13:05.902995+0000 mon.a (mon.0) 945 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:13:06.200 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:06.230 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:06.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:06.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:06.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:05 smithi133 bash[105589]: audit 2023-10-09T05:13:04.946928+0000 mon.a (mon.0) 943 : audit 0 from='client.? 172.21.15.133:0/3711082133' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:13:06.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:05 smithi133 bash[105589]: cluster 2023-10-09T05:13:05.299971+0000 mon.a (mon.0) 944 : cluster 0 osdmap e175: 8 total, 8 up, 7 in 2023-10-09T05:13:06.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:05 smithi133 bash[105589]: cluster 2023-10-09T05:13:05.301581+0000 osd.6 (osd.6) 99 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T05:13:06.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:05 smithi133 bash[105589]: cluster 2023-10-09T05:13:05.304303+0000 osd.6 (osd.6) 100 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T05:13:06.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:05 smithi133 bash[105589]: cluster 2023-10-09T05:13:05.305224+0000 osd.7 (osd.7) 105 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T05:13:06.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:05 smithi133 bash[105589]: audit 2023-10-09T05:13:05.902648+0000 mon.c (mon.2) 69 : audit 1 from='client.? 172.21.15.133:0/1759056279' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:13:06.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:05 smithi133 bash[105589]: audit 2023-10-09T05:13:05.902995+0000 mon.a (mon.0) 945 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:13:06.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:05 smithi133 bash[102858]: audit 2023-10-09T05:13:04.946928+0000 mon.a (mon.0) 943 : audit 0 from='client.? 172.21.15.133:0/3711082133' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:13:06.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:05 smithi133 bash[102858]: cluster 2023-10-09T05:13:05.299971+0000 mon.a (mon.0) 944 : cluster 0 osdmap e175: 8 total, 8 up, 7 in 2023-10-09T05:13:06.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:05 smithi133 bash[102858]: cluster 2023-10-09T05:13:05.301581+0000 osd.6 (osd.6) 99 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T05:13:06.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:05 smithi133 bash[102858]: cluster 2023-10-09T05:13:05.304303+0000 osd.6 (osd.6) 100 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T05:13:06.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:05 smithi133 bash[102858]: cluster 2023-10-09T05:13:05.305224+0000 osd.7 (osd.7) 105 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T05:13:06.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:05 smithi133 bash[102858]: audit 2023-10-09T05:13:05.902648+0000 mon.c (mon.2) 69 : audit 1 from='client.? 172.21.15.133:0/1759056279' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:13:06.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:05 smithi133 bash[102858]: audit 2023-10-09T05:13:05.902995+0000 mon.a (mon.0) 945 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:13:06.319 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is set 2023-10-09T05:13:06.334 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:06.360 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:06.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:06.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:06.463 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:06.498 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:06.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:06.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:06.602 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:06.624 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:06.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:06.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:06.731 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:06.756 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:06.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:06.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:06.860 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:06.884 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:06.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:06.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:06.987 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:07.010 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:07.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:07.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:07.113 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:07.123 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:06 smithi133 bash[102858]: cluster 2023-10-09T05:13:05.297804+0000 osd.4 (osd.4) 112 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T05:13:07.123 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:06 smithi133 bash[102858]: cluster 2023-10-09T05:13:05.301495+0000 osd.5 (osd.5) 165 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T05:13:07.123 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:06 smithi133 bash[102858]: cluster 2023-10-09T05:13:05.304246+0000 osd.5 (osd.5) 166 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T05:13:07.123 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:06 smithi133 bash[102858]: cluster 2023-10-09T05:13:05.850942+0000 mgr.y (mgr.25277) 737 : cluster 0 pgmap v722: 109 pgs: 1 active+recovering+undersized+remapped, 2 activating+remapped, 8 activating+undersized+degraded+remapped, 2 remapped+peering, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 16 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 29540/798 objects degraded (3701.754%); 6/798 objects misplaced (0.752%); 155 B/s, 248 objects/s recovering 2023-10-09T05:13:07.123 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:06 smithi133 bash[102858]: cluster 2023-10-09T05:13:06.295178+0000 mon.a (mon.0) 946 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:13:07.124 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:06 smithi133 bash[102858]: audit 2023-10-09T05:13:06.306887+0000 mon.a (mon.0) 947 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:13:07.124 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:06 smithi133 bash[102858]: cluster 2023-10-09T05:13:06.306938+0000 mon.a (mon.0) 948 : cluster 0 osdmap e176: 8 total, 8 up, 7 in 2023-10-09T05:13:07.124 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:06 smithi133 bash[102858]: cluster 2023-10-09T05:13:06.307683+0000 osd.6 (osd.6) 101 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T05:13:07.124 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:06 smithi133 bash[102858]: cluster 2023-10-09T05:13:06.310473+0000 osd.6 (osd.6) 102 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T05:13:07.124 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:06 smithi133 bash[102858]: cluster 2023-10-09T05:13:06.311551+0000 osd.7 (osd.7) 106 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T05:13:07.124 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:06 smithi133 bash[105589]: cluster 2023-10-09T05:13:05.297804+0000 osd.4 (osd.4) 112 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T05:13:07.124 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:06 smithi133 bash[105589]: cluster 2023-10-09T05:13:05.301495+0000 osd.5 (osd.5) 165 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T05:13:07.124 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:06 smithi133 bash[105589]: cluster 2023-10-09T05:13:05.304246+0000 osd.5 (osd.5) 166 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T05:13:07.124 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:06 smithi133 bash[105589]: cluster 2023-10-09T05:13:05.850942+0000 mgr.y (mgr.25277) 737 : cluster 0 pgmap v722: 109 pgs: 1 active+recovering+undersized+remapped, 2 activating+remapped, 8 activating+undersized+degraded+remapped, 2 remapped+peering, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 16 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 29540/798 objects degraded (3701.754%); 6/798 objects misplaced (0.752%); 155 B/s, 248 objects/s recovering 2023-10-09T05:13:07.124 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:06 smithi133 bash[105589]: cluster 2023-10-09T05:13:06.295178+0000 mon.a (mon.0) 946 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:13:07.125 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:06 smithi133 bash[105589]: audit 2023-10-09T05:13:06.306887+0000 mon.a (mon.0) 947 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:13:07.125 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:06 smithi133 bash[105589]: cluster 2023-10-09T05:13:06.306938+0000 mon.a (mon.0) 948 : cluster 0 osdmap e176: 8 total, 8 up, 7 in 2023-10-09T05:13:07.125 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:06 smithi133 bash[105589]: cluster 2023-10-09T05:13:06.307683+0000 osd.6 (osd.6) 101 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T05:13:07.125 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:06 smithi133 bash[105589]: cluster 2023-10-09T05:13:06.310473+0000 osd.6 (osd.6) 102 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T05:13:07.125 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:06 smithi133 bash[105589]: cluster 2023-10-09T05:13:06.311551+0000 osd.7 (osd.7) 106 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T05:13:07.138 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:07.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:07.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:07.242 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:07.267 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:07.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:07.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:07.370 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:07.382 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:06 smithi194 bash[92666]: cluster 2023-10-09T05:13:05.297804+0000 osd.4 (osd.4) 112 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T05:13:07.382 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:06 smithi194 bash[92666]: cluster 2023-10-09T05:13:05.301495+0000 osd.5 (osd.5) 165 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T05:13:07.382 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:06 smithi194 bash[92666]: cluster 2023-10-09T05:13:05.304246+0000 osd.5 (osd.5) 166 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T05:13:07.382 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:06 smithi194 bash[92666]: cluster 2023-10-09T05:13:05.850942+0000 mgr.y (mgr.25277) 737 : cluster 0 pgmap v722: 109 pgs: 1 active+recovering+undersized+remapped, 2 activating+remapped, 8 activating+undersized+degraded+remapped, 2 remapped+peering, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 16 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 29540/798 objects degraded (3701.754%); 6/798 objects misplaced (0.752%); 155 B/s, 248 objects/s recovering 2023-10-09T05:13:07.383 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:06 smithi194 bash[92666]: cluster 2023-10-09T05:13:06.295178+0000 mon.a (mon.0) 946 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:13:07.383 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:06 smithi194 bash[92666]: audit 2023-10-09T05:13:06.306887+0000 mon.a (mon.0) 947 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:13:07.383 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:06 smithi194 bash[92666]: cluster 2023-10-09T05:13:06.306938+0000 mon.a (mon.0) 948 : cluster 0 osdmap e176: 8 total, 8 up, 7 in 2023-10-09T05:13:07.383 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:06 smithi194 bash[92666]: cluster 2023-10-09T05:13:06.307683+0000 osd.6 (osd.6) 101 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T05:13:07.383 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:06 smithi194 bash[92666]: cluster 2023-10-09T05:13:06.310473+0000 osd.6 (osd.6) 102 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T05:13:07.383 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:06 smithi194 bash[92666]: cluster 2023-10-09T05:13:06.311551+0000 osd.7 (osd.7) 106 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T05:13:07.398 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:07.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:07.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:07.500 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:07.523 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:07.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:07.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:07.628 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:07.653 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:07.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:07.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:07.757 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:07.781 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:07.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:07.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:07.884 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:07.912 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:07.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:07.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:07.951 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:07 smithi133 bash[102858]: cluster 2023-10-09T05:13:06.304675+0000 osd.4 (osd.4) 113 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T05:13:07.951 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:07 smithi133 bash[102858]: cluster 2023-10-09T05:13:06.473314+0000 osd.5 (osd.5) 167 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T05:13:08.016 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:08.045 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:08.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:08.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:08.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:07 smithi133 bash[105589]: cluster 2023-10-09T05:13:06.304675+0000 osd.4 (osd.4) 113 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T05:13:08.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:07 smithi133 bash[105589]: cluster 2023-10-09T05:13:06.473314+0000 osd.5 (osd.5) 167 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T05:13:08.149 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:08.171 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:08.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:08.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:08.275 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:08.297 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:08.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:08.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:08.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:07 smithi194 bash[92666]: cluster 2023-10-09T05:13:06.304675+0000 osd.4 (osd.4) 113 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T05:13:08.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:07 smithi194 bash[92666]: cluster 2023-10-09T05:13:06.473314+0000 osd.5 (osd.5) 167 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T05:13:08.400 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:08.428 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:08.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:08.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:08.532 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:08.562 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:08.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:08.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:08.666 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:08.689 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:08.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:08.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:08.792 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:08.816 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:08.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:08.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:08.919 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:08.949 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:08.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:08.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:08.953 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:08 smithi133 bash[105589]: cluster 2023-10-09T05:13:07.851424+0000 mgr.y (mgr.25277) 738 : cluster 0 pgmap v724: 109 pgs: 2 peering, 1 active+recovering+undersized+remapped, 2 activating+remapped, 8 activating+undersized+degraded+remapped, 2 remapped+peering, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 16 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 29540/798 objects degraded (3701.754%); 6/798 objects misplaced (0.752%); 126 B/s, 12 objects/s recovering 2023-10-09T05:13:08.953 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:08 smithi133 bash[105589]: cluster 2023-10-09T05:13:08.285981+0000 mon.a (mon.0) 949 : cluster 3 Health check update: Degraded data redundancy: 29540/798 objects degraded (3701.754%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:13:09.052 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:09.082 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:09.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:09.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:09.186 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:09.210 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:09.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:09.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:09.244 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:08 smithi133 bash[102858]: cluster 2023-10-09T05:13:07.851424+0000 mgr.y (mgr.25277) 738 : cluster 0 pgmap v724: 109 pgs: 2 peering, 1 active+recovering+undersized+remapped, 2 activating+remapped, 8 activating+undersized+degraded+remapped, 2 remapped+peering, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 16 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 29540/798 objects degraded (3701.754%); 6/798 objects misplaced (0.752%); 126 B/s, 12 objects/s recovering 2023-10-09T05:13:09.244 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:08 smithi133 bash[102858]: cluster 2023-10-09T05:13:08.285981+0000 mon.a (mon.0) 949 : cluster 3 Health check update: Degraded data redundancy: 29540/798 objects degraded (3701.754%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:13:09.314 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:09.327 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:08 smithi194 bash[92666]: cluster 2023-10-09T05:13:07.851424+0000 mgr.y (mgr.25277) 738 : cluster 0 pgmap v724: 109 pgs: 2 peering, 1 active+recovering+undersized+remapped, 2 activating+remapped, 8 activating+undersized+degraded+remapped, 2 remapped+peering, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 16 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 29540/798 objects degraded (3701.754%); 6/798 objects misplaced (0.752%); 126 B/s, 12 objects/s recovering 2023-10-09T05:13:09.327 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:08 smithi194 bash[92666]: cluster 2023-10-09T05:13:08.285981+0000 mon.a (mon.0) 949 : cluster 3 Health check update: Degraded data redundancy: 29540/798 objects degraded (3701.754%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:13:09.342 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:09.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:09.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:09.444 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:09.478 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:09.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:09.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:09.581 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:09.602 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:09.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:09.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:09.705 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:09.729 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:09.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:09.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:09.832 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:09.861 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:09.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:09.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:09.965 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:09.997 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:09.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:10.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:10.100 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:10.125 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:10.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:10.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:10.228 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:10.252 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:10.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:10.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:10.278 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 6, 7, 1] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:13:10.278 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:13:10.278 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-10-09T05:13:10.279 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T05:13:10.279 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T05:13:10.279 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-10-09T05:13:10.355 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:10.390 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:10.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:10.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:10.493 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:10.517 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:10.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:10.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:10.619 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:10.639 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:10.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:10.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:10.743 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:10.769 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:10.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:10.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:10.873 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:10.898 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:10.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:10.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:11.002 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:11.027 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:11.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:11.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:11.131 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:11.140 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:10 smithi133 bash[105589]: cluster 2023-10-09T05:13:09.852609+0000 mgr.y (mgr.25277) 739 : cluster 0 pgmap v725: 109 pgs: 2 peering, 2 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 16 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 32319/798 objects degraded (4050.000%); 3/798 objects misplaced (0.376%); 103 B/s, 209 objects/s recovering 2023-10-09T05:13:11.140 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:10 smithi133 bash[102858]: cluster 2023-10-09T05:13:09.852609+0000 mgr.y (mgr.25277) 739 : cluster 0 pgmap v725: 109 pgs: 2 peering, 2 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 16 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 32319/798 objects degraded (4050.000%); 3/798 objects misplaced (0.376%); 103 B/s, 209 objects/s recovering 2023-10-09T05:13:11.154 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:11.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:11.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:11.257 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:11.283 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:11.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:11.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:11.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:10 smithi194 bash[92666]: cluster 2023-10-09T05:13:09.852609+0000 mgr.y (mgr.25277) 739 : cluster 0 pgmap v725: 109 pgs: 2 peering, 2 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 16 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 32319/798 objects degraded (4050.000%); 3/798 objects misplaced (0.376%); 103 B/s, 209 objects/s recovering 2023-10-09T05:13:11.386 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:11.416 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:11.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:11.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:11.520 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:11.548 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:11.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:11.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:11.651 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:11.675 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:11.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:11.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:11.780 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:11.812 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:11.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:11.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:11.920 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:11.951 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:11.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:11.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:12.055 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:12.079 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:12.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:12.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:12.182 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:12.205 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:12.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:12.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:12.308 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:12.333 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:12.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:12.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:12.436 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:12.473 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:12.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:12.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:12.576 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:12.604 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:12.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:12.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:12.708 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:12.733 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:12.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:12.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:12.836 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:12.869 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:12.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:12.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:12.973 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:13.004 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:13.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:13.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:13.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:12 smithi133 bash[102858]: cluster 2023-10-09T05:13:11.853248+0000 mgr.y (mgr.25277) 740 : cluster 0 pgmap v726: 109 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 16 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 32319/798 objects degraded (4050.000%); 3/798 objects misplaced (0.376%); 78 B/s, 227 objects/s recovering 2023-10-09T05:13:13.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:12 smithi133 bash[105589]: cluster 2023-10-09T05:13:11.853248+0000 mgr.y (mgr.25277) 740 : cluster 0 pgmap v726: 109 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 16 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 32319/798 objects degraded (4050.000%); 3/798 objects misplaced (0.376%); 78 B/s, 227 objects/s recovering 2023-10-09T05:13:13.062 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:13:13.107 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:13.130 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:13.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:13.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:13.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:12 smithi194 bash[92666]: cluster 2023-10-09T05:13:11.853248+0000 mgr.y (mgr.25277) 740 : cluster 0 pgmap v726: 109 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 16 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 32319/798 objects degraded (4050.000%); 3/798 objects misplaced (0.376%); 78 B/s, 227 objects/s recovering 2023-10-09T05:13:13.150 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:13:13.234 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:13.257 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:13.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:13.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:13.360 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:13.393 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:13.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:13.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:13.496 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:13.523 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:13.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:13.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:13.627 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:13.653 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:13.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:13.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:13.759 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:13.794 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:13.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:13.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:13.897 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:13.922 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:13.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:13.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:13.961 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:13 smithi133 bash[102858]: cluster 2023-10-09T05:13:13.287235+0000 mon.a (mon.0) 950 : cluster 3 Health check update: Degraded data redundancy: 32319/798 objects degraded (4050.000%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T05:13:13.961 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:13 smithi133 bash[102858]: audit 2023-10-09T05:13:13.444649+0000 mon.a (mon.0) 951 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:13:14.014 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:13:14.025 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:14.032 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:13 smithi194 bash[92666]: cluster 2023-10-09T05:13:13.287235+0000 mon.a (mon.0) 950 : cluster 3 Health check update: Degraded data redundancy: 32319/798 objects degraded (4050.000%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T05:13:14.032 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:13 smithi194 bash[92666]: audit 2023-10-09T05:13:13.444649+0000 mon.a (mon.0) 951 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:13:14.048 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:14.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:14.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:14.151 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:14.183 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:14.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:14.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:14.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:13 smithi133 bash[105589]: cluster 2023-10-09T05:13:13.287235+0000 mon.a (mon.0) 950 : cluster 3 Health check update: Degraded data redundancy: 32319/798 objects degraded (4050.000%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T05:13:14.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:13 smithi133 bash[105589]: audit 2023-10-09T05:13:13.444649+0000 mon.a (mon.0) 951 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:13:14.287 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:14.298 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:13:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:13:14] "GET /metrics HTTP/1.1" 200 36437 "" "Prometheus/2.43.0" 2023-10-09T05:13:14.316 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:14.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:14.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:14.419 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:14.444 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:14.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:14.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:14.547 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:14.571 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:14.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:14.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:14.674 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:14.699 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:14.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:14.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:14.802 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:14.833 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:14.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:14.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:14.936 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:14.971 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:14.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:14.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:15.074 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:15.108 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:15.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:15.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:15.212 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:15.220 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:14 smithi133 bash[105589]: cluster 2023-10-09T05:13:13.853793+0000 mgr.y (mgr.25277) 741 : cluster 0 pgmap v727: 109 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 16 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 32319/798 objects degraded (4050.000%); 3/798 objects misplaced (0.376%); 0 B/s, 247 objects/s recovering 2023-10-09T05:13:15.221 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:14 smithi133 bash[102858]: cluster 2023-10-09T05:13:13.853793+0000 mgr.y (mgr.25277) 741 : cluster 0 pgmap v727: 109 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 16 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 32319/798 objects degraded (4050.000%); 3/798 objects misplaced (0.376%); 0 B/s, 247 objects/s recovering 2023-10-09T05:13:15.238 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:15.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:15.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:15.341 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:15.369 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:15.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:15.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:15.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:14 smithi194 bash[92666]: cluster 2023-10-09T05:13:13.853793+0000 mgr.y (mgr.25277) 741 : cluster 0 pgmap v727: 109 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 16 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 32319/798 objects degraded (4050.000%); 3/798 objects misplaced (0.376%); 0 B/s, 247 objects/s recovering 2023-10-09T05:13:15.473 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:15.495 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:15.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:15.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:15.595 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:13:15.599 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:15.629 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:15.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:15.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:15.736 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:15.766 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:15.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:15.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:15.869 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:15.892 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:15.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:15.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:15.995 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:16.018 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:16.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:16.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:16.121 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:16.154 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:16.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:16.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:16.257 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:16.287 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:16.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:16.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:16.390 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:16.420 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:16.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:16.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:16.523 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:16.549 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:16.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:16.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:16.654 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:16.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:16.691 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:16.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:16.792 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:16.823 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:16.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:16.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:16.928 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:16.968 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:16.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:16.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:17.071 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:17.082 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:16 smithi194 bash[92666]: cluster 2023-10-09T05:13:15.854753+0000 mgr.y (mgr.25277) 742 : cluster 0 pgmap v728: 109 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 16 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 32319/798 objects degraded (4050.000%); 3/798 objects misplaced (0.376%); 0 B/s, 268 objects/s recovering 2023-10-09T05:13:17.100 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:17.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:17.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:17.204 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:17.225 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:17.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:17.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:17.251 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:16 smithi133 bash[102858]: cluster 2023-10-09T05:13:15.854753+0000 mgr.y (mgr.25277) 742 : cluster 0 pgmap v728: 109 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 16 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 32319/798 objects degraded (4050.000%); 3/798 objects misplaced (0.376%); 0 B/s, 268 objects/s recovering 2023-10-09T05:13:17.252 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:16 smithi133 bash[105589]: cluster 2023-10-09T05:13:15.854753+0000 mgr.y (mgr.25277) 742 : cluster 0 pgmap v728: 109 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 16 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 32319/798 objects degraded (4050.000%); 3/798 objects misplaced (0.376%); 0 B/s, 268 objects/s recovering 2023-10-09T05:13:17.329 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:17.360 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:17.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:17.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:17.464 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:17.486 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:17.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:17.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:17.590 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:17.612 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:17.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:17.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:17.620 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:13:17.620 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:13:17.716 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:17.748 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:17.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:17.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:17.852 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:17.873 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:17.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:17.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:17.976 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:18.009 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:18.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:18.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:18.112 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:18.147 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:18.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:18.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:18.251 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:18.278 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:18.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:18.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:18.381 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:18.405 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:18.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:18.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:18.509 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:18.531 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:18.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:18.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:18.634 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:18.668 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:18.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:18.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:18.772 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:18.793 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:18.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:18.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:18.897 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:18.919 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:18.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:18.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:19.023 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:19.031 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:18 smithi133 bash[102858]: cluster 2023-10-09T05:13:17.855254+0000 mgr.y (mgr.25277) 743 : cluster 0 pgmap v729: 109 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 16 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 32319/798 objects degraded (4050.000%); 3/798 objects misplaced (0.376%); 0 B/s, 312 objects/s recovering 2023-10-09T05:13:19.032 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:18 smithi133 bash[102858]: audit 2023-10-09T05:13:18.899432+0000 mon.c (mon.2) 70 : audit 1 from='client.? 172.21.15.133:0/405202441' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:13:19.032 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:18 smithi133 bash[102858]: audit 2023-10-09T05:13:18.899910+0000 mon.a (mon.0) 952 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:13:19.032 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:18 smithi133 bash[105589]: cluster 2023-10-09T05:13:17.855254+0000 mgr.y (mgr.25277) 743 : cluster 0 pgmap v729: 109 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 16 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 32319/798 objects degraded (4050.000%); 3/798 objects misplaced (0.376%); 0 B/s, 312 objects/s recovering 2023-10-09T05:13:19.032 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:18 smithi133 bash[105589]: audit 2023-10-09T05:13:18.899432+0000 mon.c (mon.2) 70 : audit 1 from='client.? 172.21.15.133:0/405202441' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:13:19.032 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:18 smithi133 bash[105589]: audit 2023-10-09T05:13:18.899910+0000 mon.a (mon.0) 952 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:13:19.046 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:19.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:19.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:19.149 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:19.184 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:19.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:19.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:19.288 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:19.301 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:18 smithi194 bash[92666]: cluster 2023-10-09T05:13:17.855254+0000 mgr.y (mgr.25277) 743 : cluster 0 pgmap v729: 109 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 16 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 32319/798 objects degraded (4050.000%); 3/798 objects misplaced (0.376%); 0 B/s, 312 objects/s recovering 2023-10-09T05:13:19.301 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:18 smithi194 bash[92666]: audit 2023-10-09T05:13:18.899432+0000 mon.c (mon.2) 70 : audit 1 from='client.? 172.21.15.133:0/405202441' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:13:19.302 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:18 smithi194 bash[92666]: audit 2023-10-09T05:13:18.899910+0000 mon.a (mon.0) 952 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:13:19.316 INFO:teuthology.orchestra.run.smithi133.stderr:pool 'unique_pool_4' removed 2023-10-09T05:13:19.318 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:19.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:19.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:19.422 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:19.449 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:19.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:19.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:19.553 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:19.583 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:19.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:19.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:19.688 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:19.726 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:19.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:19.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:19.829 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:19.860 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:19.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:19.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:19.964 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:19.998 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:20.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:20.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:20.102 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:20.135 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:20.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:20.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:20.239 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:20.272 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:20.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:20.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:20.376 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:20.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:20 smithi194 bash[92666]: audit 2023-10-09T05:13:19.296838+0000 mon.a (mon.0) 953 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:13:20.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:20 smithi194 bash[92666]: cluster 2023-10-09T05:13:19.296921+0000 mon.a (mon.0) 954 : cluster 0 osdmap e177: 8 total, 8 up, 7 in 2023-10-09T05:13:20.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:20 smithi194 bash[92666]: cluster 2023-10-09T05:13:19.855962+0000 mgr.y (mgr.25277) 744 : cluster 0 pgmap v731: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:20.398 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:20.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:20.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:20.504 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:20.528 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:20.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:20.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:20.631 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:20.663 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:20.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:20.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:20.767 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:20.800 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:20.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:20.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:20.848 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:20 smithi133 bash[102858]: audit 2023-10-09T05:13:19.296838+0000 mon.a (mon.0) 953 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:13:20.849 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:20 smithi133 bash[102858]: cluster 2023-10-09T05:13:19.296921+0000 mon.a (mon.0) 954 : cluster 0 osdmap e177: 8 total, 8 up, 7 in 2023-10-09T05:13:20.849 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:20 smithi133 bash[102858]: cluster 2023-10-09T05:13:19.855962+0000 mgr.y (mgr.25277) 744 : cluster 0 pgmap v731: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:20.849 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:20 smithi133 bash[105589]: audit 2023-10-09T05:13:19.296838+0000 mon.a (mon.0) 953 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:13:20.849 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:20 smithi133 bash[105589]: cluster 2023-10-09T05:13:19.296921+0000 mon.a (mon.0) 954 : cluster 0 osdmap e177: 8 total, 8 up, 7 in 2023-10-09T05:13:20.849 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:20 smithi133 bash[105589]: cluster 2023-10-09T05:13:19.855962+0000 mgr.y (mgr.25277) 744 : cluster 0 pgmap v731: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:20.903 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:20.933 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:20.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:20.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:21.037 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:21.074 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:21.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:21.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:21.179 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:21.210 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:21.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:21.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:21.314 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:21.340 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:21.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:21.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:21.443 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:21.476 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:21.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:21.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:21.581 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:21.595 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:21 smithi133 bash[102858]: cluster 2023-10-09T05:13:20.285954+0000 osd.7 (osd.7) 107 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T05:13:21.595 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:21 smithi133 bash[102858]: cluster 2023-10-09T05:13:20.286040+0000 osd.6 (osd.6) 103 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T05:13:21.595 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:21 smithi133 bash[102858]: cluster 2023-10-09T05:13:20.286177+0000 osd.4 (osd.4) 114 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T05:13:21.595 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:21 smithi133 bash[102858]: cluster 2023-10-09T05:13:20.286180+0000 osd.5 (osd.5) 168 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T05:13:21.595 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:21 smithi133 bash[102858]: cluster 2023-10-09T05:13:20.291370+0000 mon.a (mon.0) 955 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 32319/798 objects degraded (4050.000%), 12 pgs degraded) 2023-10-09T05:13:21.595 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:21 smithi133 bash[105589]: cluster 2023-10-09T05:13:20.285954+0000 osd.7 (osd.7) 107 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T05:13:21.596 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:21 smithi133 bash[105589]: cluster 2023-10-09T05:13:20.286040+0000 osd.6 (osd.6) 103 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T05:13:21.596 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:21 smithi133 bash[105589]: cluster 2023-10-09T05:13:20.286177+0000 osd.4 (osd.4) 114 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T05:13:21.596 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:21 smithi133 bash[105589]: cluster 2023-10-09T05:13:20.286180+0000 osd.5 (osd.5) 168 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T05:13:21.596 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:21 smithi133 bash[105589]: cluster 2023-10-09T05:13:20.291370+0000 mon.a (mon.0) 955 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 32319/798 objects degraded (4050.000%), 12 pgs degraded) 2023-10-09T05:13:21.611 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:21.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:21.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:21.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:21 smithi194 bash[92666]: cluster 2023-10-09T05:13:20.285954+0000 osd.7 (osd.7) 107 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T05:13:21.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:21 smithi194 bash[92666]: cluster 2023-10-09T05:13:20.286040+0000 osd.6 (osd.6) 103 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T05:13:21.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:21 smithi194 bash[92666]: cluster 2023-10-09T05:13:20.286177+0000 osd.4 (osd.4) 114 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T05:13:21.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:21 smithi194 bash[92666]: cluster 2023-10-09T05:13:20.286180+0000 osd.5 (osd.5) 168 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T05:13:21.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:21 smithi194 bash[92666]: cluster 2023-10-09T05:13:20.291370+0000 mon.a (mon.0) 955 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 32319/798 objects degraded (4050.000%), 12 pgs degraded) 2023-10-09T05:13:21.714 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:21.742 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:21.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:21.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:21.845 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:21.874 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:21.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:21.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:21.978 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:22.009 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:22.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:22.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:22.113 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:22.147 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:22.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:22.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:22.250 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:22.283 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:22.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:22.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:22.387 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:22.409 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:22.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:22.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:22.442 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T05:13:22.443 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T05:13:22.443 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2023-10-09T05:13:22.443 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_5 16 2023-10-09T05:13:22.512 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:22.527 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:22 smithi194 bash[92666]: cluster 2023-10-09T05:13:21.856706+0000 mgr.y (mgr.25277) 745 : cluster 0 pgmap v732: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:22.544 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:22.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:22.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:22.605 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:22 smithi133 bash[105589]: cluster 2023-10-09T05:13:21.856706+0000 mgr.y (mgr.25277) 745 : cluster 0 pgmap v732: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:22.605 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:22 smithi133 bash[102858]: cluster 2023-10-09T05:13:21.856706+0000 mgr.y (mgr.25277) 745 : cluster 0 pgmap v732: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:22.647 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:22.666 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:22.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:22.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:22.769 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:22.801 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:22.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:22.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:22.905 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:22.936 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:22.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:22.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:23.039 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:23.063 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:23.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:23.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:23.167 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:23.199 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:23.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:23.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:23.303 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:23.326 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:23.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:23.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:23.429 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:23.459 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:23.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:23.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:23.563 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:23.593 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:23.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:23.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:23.697 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:23.731 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:23.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:23.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:23.835 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:23.859 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:23.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:23.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:23.962 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:23.989 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:23.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:23.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:24.092 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:24.121 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:24.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:24.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:24.225 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:24.257 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:24.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:24.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:24.361 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:24.373 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:24 smithi194 bash[92666]: cluster 2023-10-09T05:13:23.857473+0000 mgr.y (mgr.25277) 746 : cluster 0 pgmap v733: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:24.390 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:24.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:24.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:24.494 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:24.503 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:24 smithi133 bash[102858]: cluster 2023-10-09T05:13:23.857473+0000 mgr.y (mgr.25277) 746 : cluster 0 pgmap v733: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:24.503 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:13:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:13:24] "GET /metrics HTTP/1.1" 200 36437 "" "Prometheus/2.43.0" 2023-10-09T05:13:24.503 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:24 smithi133 bash[105589]: cluster 2023-10-09T05:13:23.857473+0000 mgr.y (mgr.25277) 746 : cluster 0 pgmap v733: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:24.523 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:24.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:24.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:24.627 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:24.662 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:24.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:24.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:24.766 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:24.797 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:24.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:24.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:24.901 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:24.935 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:24.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:24.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:25.038 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:25.060 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:25.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:25.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:25.163 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:25.188 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:25.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:25.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:25.291 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:25.320 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:25.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:25.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:25.424 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:25.447 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:25.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:25.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:25.550 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:25.576 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:25.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:25.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:25.679 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:25.704 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:25.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:25.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:25.807 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:25.839 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:25.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:25.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:25.943 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:25.966 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:25.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:25.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:26.069 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:26.100 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:26.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:26.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:26.203 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:26.228 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:26.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:26.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:26.332 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:26.359 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:26.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:26.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:26.464 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:26.494 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:26.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:26.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:26.598 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:26.630 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:26.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:26.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:26.734 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:26.756 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:26.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:26.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:26.859 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:26.890 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:26.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:26.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:26.994 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:26.998 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:13:26.998 INFO:teuthology.orchestra.run.smithi133.stdout: "size": 20, 2023-10-09T05:13:26.998 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 600, 2023-10-09T05:13:26.998 INFO:teuthology.orchestra.run.smithi133.stdout: "ops": [ 2023-10-09T05:13:26.998 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:26.999 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.34845.0:2199 9.2 9:44a6f507:::benchmark_data_smithi133_153303_object2198:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-10-09T05:13:26.999 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:09:30.056574+0000", 2023-10-09T05:13:26.999 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 236.93690559300001, 2023-10-09T05:13:26.999 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.037409915, 2023-10-09T05:13:26.999 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:13:26.999 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:26.999 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:13:26.999 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.34845", 2023-10-09T05:13:26.999 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:13:26.999 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 2199 2023-10-09T05:13:26.999 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:26.999 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:13:27.000 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.000 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:13:27.000 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:30.056574+0000", 2023-10-09T05:13:27.000 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.000 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.000 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.000 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:13:27.000 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:30.056574+0000", 2023-10-09T05:13:27.000 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.000 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.000 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.000 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:13:27.001 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:30.056572+0000", 2023-10-09T05:13:27.001 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:13:27.001 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.001 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.001 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:13:27.001 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:30.056624+0000", 2023-10-09T05:13:27.001 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.2373999999999997e-05 2023-10-09T05:13:27.001 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.001 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.001 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:13:27.002 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:30.056639+0000", 2023-10-09T05:13:27.002 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.4688999999999999e-05 2023-10-09T05:13:27.002 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.002 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.002 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:13:27.002 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:30.056658+0000", 2023-10-09T05:13:27.002 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.9125000000000001e-05 2023-10-09T05:13:27.002 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.002 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.002 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:13:27.002 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:30.063012+0000", 2023-10-09T05:13:27.003 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0063541559999999997 2023-10-09T05:13:27.003 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.003 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.003 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:13:27.003 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:30.063026+0000", 2023-10-09T05:13:27.003 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.3410999999999999e-05 2023-10-09T05:13:27.003 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.003 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.003 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:13:27.003 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:31.049053+0000", 2023-10-09T05:13:27.004 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.98602728500000003 2023-10-09T05:13:27.004 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.004 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.004 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:13:27.004 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:31.049067+0000", 2023-10-09T05:13:27.004 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.3604999999999999e-05 2023-10-09T05:13:27.004 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.004 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.004 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:13:27.004 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:32.089413+0000", 2023-10-09T05:13:27.004 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.0403466299999999 2023-10-09T05:13:27.004 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.004 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.005 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:13:27.005 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:32.089638+0000", 2023-10-09T05:13:27.005 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00022449300000000001 2023-10-09T05:13:27.005 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.005 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.005 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,7", 2023-10-09T05:13:27.005 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:32.089879+0000", 2023-10-09T05:13:27.005 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000241022 2023-10-09T05:13:27.005 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.005 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.005 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:13:27.005 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:32.092427+0000", 2023-10-09T05:13:27.006 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0025479000000000001 2023-10-09T05:13:27.006 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.006 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.006 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:27.006 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:32.092505+0000", 2023-10-09T05:13:27.006 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.8317000000000002e-05 2023-10-09T05:13:27.006 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.006 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.006 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:27.006 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:32.093746+0000", 2023-10-09T05:13:27.006 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.001241395 2023-10-09T05:13:27.006 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.007 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.007 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:13:27.007 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:32.093798+0000", 2023-10-09T05:13:27.007 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.1190000000000003e-05 2023-10-09T05:13:27.007 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.007 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.007 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:13:27.007 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:32.093984+0000", 2023-10-09T05:13:27.007 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00018598200000000001 2023-10-09T05:13:27.007 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.007 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:13:27.007 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.008 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.008 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.008 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.34845.0:2202 9.2 9:4e2aa47b:::benchmark_data_smithi133_153303_object2201:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-10-09T05:13:27.008 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:09:30.065087+0000", 2023-10-09T05:13:27.008 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 236.92839239099999, 2023-10-09T05:13:27.008 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.0292245580000001, 2023-10-09T05:13:27.008 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:13:27.008 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:27.008 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:13:27.008 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.34845", 2023-10-09T05:13:27.008 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:13:27.008 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 2202 2023-10-09T05:13:27.009 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.009 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:13:27.009 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.009 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:13:27.009 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:30.065087+0000", 2023-10-09T05:13:27.009 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.009 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.009 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.009 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:13:27.009 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:30.065087+0000", 2023-10-09T05:13:27.009 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.009 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.009 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.010 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:13:27.010 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:30.065085+0000", 2023-10-09T05:13:27.010 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999981 2023-10-09T05:13:27.010 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.011 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.011 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:13:27.011 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:30.065115+0000", 2023-10-09T05:13:27.011 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.9539999999999998e-05 2023-10-09T05:13:27.011 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.011 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.011 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:13:27.011 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:30.065138+0000", 2023-10-09T05:13:27.011 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.3022000000000001e-05 2023-10-09T05:13:27.012 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.012 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.012 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:13:27.012 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:30.065146+0000", 2023-10-09T05:13:27.012 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.6600000000000001e-06 2023-10-09T05:13:27.012 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.012 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.012 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:13:27.012 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:30.065233+0000", 2023-10-09T05:13:27.012 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.6623000000000001e-05 2023-10-09T05:13:27.013 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.013 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.013 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:13:27.013 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:30.065243+0000", 2023-10-09T05:13:27.013 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.0016e-05 2023-10-09T05:13:27.013 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.013 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.013 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:13:27.013 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:31.049093+0000", 2023-10-09T05:13:27.013 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.98385040899999998 2023-10-09T05:13:27.013 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.014 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.014 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:13:27.014 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:31.049101+0000", 2023-10-09T05:13:27.014 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.1179999999999992e-06 2023-10-09T05:13:27.014 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.014 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.014 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:13:27.014 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:32.090934+0000", 2023-10-09T05:13:27.014 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.041832198 2023-10-09T05:13:27.014 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.015 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.015 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:13:27.015 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:32.091079+0000", 2023-10-09T05:13:27.015 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000145518 2023-10-09T05:13:27.015 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.015 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.015 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,7", 2023-10-09T05:13:27.015 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:32.091217+0000", 2023-10-09T05:13:27.015 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000137565 2023-10-09T05:13:27.015 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.015 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.016 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:13:27.016 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:32.092437+0000", 2023-10-09T05:13:27.016 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0012202280000000001 2023-10-09T05:13:27.016 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.016 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.016 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:27.016 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:32.092792+0000", 2023-10-09T05:13:27.016 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00035505399999999999 2023-10-09T05:13:27.016 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.016 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.016 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:27.016 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:32.094063+0000", 2023-10-09T05:13:27.017 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0012711980000000001 2023-10-09T05:13:27.017 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.017 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.017 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:13:27.017 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:32.094114+0000", 2023-10-09T05:13:27.017 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.0557999999999999e-05 2023-10-09T05:13:27.017 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.017 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.017 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:13:27.017 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:32.094311+0000", 2023-10-09T05:13:27.017 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00019764399999999999 2023-10-09T05:13:27.017 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.018 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:13:27.018 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.018 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.018 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.018 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.34845.0:2203 9.2 9:44abe55f:::benchmark_data_smithi133_153303_object2202:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-10-09T05:13:27.018 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:09:30.066749+0000", 2023-10-09T05:13:27.018 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 236.926730607, 2023-10-09T05:13:27.018 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.0278962859999998, 2023-10-09T05:13:27.018 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:13:27.018 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:27.018 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:13:27.019 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.34845", 2023-10-09T05:13:27.019 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:13:27.019 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 2203 2023-10-09T05:13:27.019 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.019 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:13:27.019 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.019 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:13:27.019 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:30.066749+0000", 2023-10-09T05:13:27.019 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.019 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.019 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.019 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:13:27.020 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:30.066749+0000", 2023-10-09T05:13:27.020 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.020 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.020 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.020 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:13:27.020 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:30.066748+0000", 2023-10-09T05:13:27.020 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:13:27.020 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.020 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.020 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:13:27.020 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:30.066788+0000", 2023-10-09T05:13:27.021 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.0034000000000003e-05 2023-10-09T05:13:27.021 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.021 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.021 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:13:27.021 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:30.068684+0000", 2023-10-09T05:13:27.021 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0018959960000000001 2023-10-09T05:13:27.021 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.021 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.021 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:13:27.021 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:30.068699+0000", 2023-10-09T05:13:27.021 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.5588999999999999e-05 2023-10-09T05:13:27.021 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.022 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.022 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:13:27.022 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:30.069056+0000", 2023-10-09T05:13:27.022 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00035706699999999997 2023-10-09T05:13:27.022 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.022 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.022 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:13:27.022 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:30.069071+0000", 2023-10-09T05:13:27.022 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.4527e-05 2023-10-09T05:13:27.022 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.022 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.022 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:13:27.023 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:31.049124+0000", 2023-10-09T05:13:27.023 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.98005310800000001 2023-10-09T05:13:27.023 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.023 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.023 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:13:27.023 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:31.049132+0000", 2023-10-09T05:13:27.023 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.9810000000000003e-06 2023-10-09T05:13:27.023 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.023 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.023 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:13:27.023 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:32.091674+0000", 2023-10-09T05:13:27.023 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.04254168 2023-10-09T05:13:27.024 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.024 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.024 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:13:27.024 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:32.091811+0000", 2023-10-09T05:13:27.024 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000137689 2023-10-09T05:13:27.024 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.024 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.024 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,7", 2023-10-09T05:13:27.024 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:32.091991+0000", 2023-10-09T05:13:27.024 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000179801 2023-10-09T05:13:27.024 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.024 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.024 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:13:27.025 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:32.092658+0000", 2023-10-09T05:13:27.025 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00066703499999999996 2023-10-09T05:13:27.025 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.025 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.025 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:27.025 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:32.092874+0000", 2023-10-09T05:13:27.025 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000215682 2023-10-09T05:13:27.025 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.025 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.025 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:27.025 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:32.094426+0000", 2023-10-09T05:13:27.025 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0015519259999999999 2023-10-09T05:13:27.026 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.026 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.026 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:13:27.026 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:32.094481+0000", 2023-10-09T05:13:27.026 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.5529000000000003e-05 2023-10-09T05:13:27.026 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.026 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.026 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:13:27.026 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:32.094645+0000", 2023-10-09T05:13:27.026 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00016374400000000001 2023-10-09T05:13:27.026 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.026 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:13:27.027 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.027 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.027 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.027 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.34845.0:4794 9.2 9:42994825:::benchmark_data_smithi133_153303_object4793:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-10-09T05:13:27.027 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:09:34.107910+0000", 2023-10-09T05:13:27.027 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 232.885568901, 2023-10-09T05:13:27.027 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.0015794259999999, 2023-10-09T05:13:27.027 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:13:27.027 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:27.027 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:13:27.027 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.34845", 2023-10-09T05:13:27.027 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:13:27.027 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 4794 2023-10-09T05:13:27.028 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.028 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:13:27.028 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.028 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:13:27.028 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:34.107910+0000", 2023-10-09T05:13:27.028 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.028 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.028 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.028 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:13:27.028 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:34.107910+0000", 2023-10-09T05:13:27.028 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.028 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.028 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.029 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:13:27.029 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:34.107910+0000", 2023-10-09T05:13:27.029 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:13:27.029 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.029 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.029 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:13:27.029 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:34.107972+0000", 2023-10-09T05:13:27.029 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.2823999999999993e-05 2023-10-09T05:13:27.029 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.029 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.029 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:13:27.029 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:34.107981+0000", 2023-10-09T05:13:27.029 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.4610000000000008e-06 2023-10-09T05:13:27.030 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.030 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.030 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:13:27.030 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:34.107989+0000", 2023-10-09T05:13:27.031 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.3639999999999999e-06 2023-10-09T05:13:27.031 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.031 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.031 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:13:27.031 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:34.108024+0000", 2023-10-09T05:13:27.031 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.4833999999999999e-05 2023-10-09T05:13:27.031 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.031 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.031 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:13:27.031 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:34.108029+0000", 2023-10-09T05:13:27.031 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.2569999999999998e-06 2023-10-09T05:13:27.032 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.032 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.032 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:13:27.032 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:35.103892+0000", 2023-10-09T05:13:27.032 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.99586284400000002 2023-10-09T05:13:27.032 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.032 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.032 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:13:27.032 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:35.104057+0000", 2023-10-09T05:13:27.033 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00016450699999999999 2023-10-09T05:13:27.033 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.034 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.034 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,7", 2023-10-09T05:13:27.034 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:35.104214+0000", 2023-10-09T05:13:27.034 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00015772200000000001 2023-10-09T05:13:27.034 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.034 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.035 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:13:27.035 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:35.105944+0000", 2023-10-09T05:13:27.035 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.001729754 2023-10-09T05:13:27.035 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.035 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.035 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:27.036 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:35.107541+0000", 2023-10-09T05:13:27.036 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.001596383 2023-10-09T05:13:27.036 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.036 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.036 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:27.036 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:35.109401+0000", 2023-10-09T05:13:27.036 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0018602099999999999 2023-10-09T05:13:27.036 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.036 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.036 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:13:27.036 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:35.109448+0000", 2023-10-09T05:13:27.037 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.6794e-05 2023-10-09T05:13:27.037 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.037 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.037 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:13:27.037 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:09:35.109490+0000", 2023-10-09T05:13:27.037 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.2212999999999997e-05 2023-10-09T05:13:27.037 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.037 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:13:27.037 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.037 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.037 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.037 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.34845.0:153128 9.2 9:46e5d0e9:::benchmark_data_smithi133_153303_object63159:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-10-09T05:13:27.037 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:12:25.695183+0000", 2023-10-09T05:13:27.038 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 61.298295754999998, 2023-10-09T05:13:27.038 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.18122089999999999, 2023-10-09T05:13:27.038 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:13:27.038 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:27.038 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:13:27.038 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.34845", 2023-10-09T05:13:27.038 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:13:27.038 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 153128 2023-10-09T05:13:27.038 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.039 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:13:27.039 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.039 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:13:27.039 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:25.695183+0000", 2023-10-09T05:13:27.039 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.039 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.039 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.039 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:13:27.040 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:25.695183+0000", 2023-10-09T05:13:27.040 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.040 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.040 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.040 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:13:27.040 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:25.695182+0000", 2023-10-09T05:13:27.040 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:13:27.040 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.040 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.040 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:13:27.040 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:25.695189+0000", 2023-10-09T05:13:27.040 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.267e-06 2023-10-09T05:13:27.041 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.041 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.041 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:13:27.041 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:25.695201+0000", 2023-10-09T05:13:27.041 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.1182000000000001e-05 2023-10-09T05:13:27.041 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.041 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.041 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:13:27.042 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:25.695207+0000", 2023-10-09T05:13:27.042 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.7129999999999999e-06 2023-10-09T05:13:27.042 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.042 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.042 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:13:27.042 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:25.695247+0000", 2023-10-09T05:13:27.042 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.9425000000000003e-05 2023-10-09T05:13:27.042 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.042 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.042 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:13:27.043 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:25.841994+0000", 2023-10-09T05:13:27.043 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.14674732099999999 2023-10-09T05:13:27.043 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.043 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.043 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:13:27.043 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:25.842901+0000", 2023-10-09T05:13:27.043 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00090675900000000004 2023-10-09T05:13:27.044 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.044 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.044 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:13:27.044 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:25.846559+0000", 2023-10-09T05:13:27.044 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.003658512 2023-10-09T05:13:27.044 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.044 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.044 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:27.045 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:25.852018+0000", 2023-10-09T05:13:27.045 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0054587539999999997 2023-10-09T05:13:27.045 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.045 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.045 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:27.045 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:25.876125+0000", 2023-10-09T05:13:27.045 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.024106993 2023-10-09T05:13:27.046 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.046 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.046 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:13:27.046 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:25.876173+0000", 2023-10-09T05:13:27.046 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.7862999999999999e-05 2023-10-09T05:13:27.047 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.047 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.047 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:13:27.047 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:25.876404+0000", 2023-10-09T05:13:27.047 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00023139600000000001 2023-10-09T05:13:27.047 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.048 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:13:27.048 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.048 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.048 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.048 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.34845.0:154179 9.c 9:314aad15:::benchmark_data_smithi133_153303_object64210:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-10-09T05:13:27.048 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:12:27.172176+0000", 2023-10-09T05:13:27.048 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 59.821302803000002, 2023-10-09T05:13:27.048 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.17151222599999999, 2023-10-09T05:13:27.049 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:13:27.049 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:27.049 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:13:27.049 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.34845", 2023-10-09T05:13:27.049 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:13:27.049 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 154179 2023-10-09T05:13:27.050 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.050 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:13:27.050 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.050 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:13:27.050 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.172176+0000", 2023-10-09T05:13:27.050 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.050 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.051 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.051 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:13:27.051 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.172176+0000", 2023-10-09T05:13:27.051 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.051 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.051 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.051 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:13:27.051 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.172176+0000", 2023-10-09T05:13:27.051 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999995 2023-10-09T05:13:27.051 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.051 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.051 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:13:27.052 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.172180+0000", 2023-10-09T05:13:27.052 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.1479999999999999e-06 2023-10-09T05:13:27.052 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.052 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.052 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:13:27.052 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.172188+0000", 2023-10-09T05:13:27.052 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.7999999999999999e-06 2023-10-09T05:13:27.052 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.052 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.052 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:13:27.052 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.172195+0000", 2023-10-09T05:13:27.052 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.379e-06 2023-10-09T05:13:27.052 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.053 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.053 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:13:27.053 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.172240+0000", 2023-10-09T05:13:27.053 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.4943e-05 2023-10-09T05:13:27.053 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.053 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.053 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:13:27.053 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.172355+0000", 2023-10-09T05:13:27.053 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000115086 2023-10-09T05:13:27.053 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.053 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.053 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 4,7", 2023-10-09T05:13:27.053 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.172463+0000", 2023-10-09T05:13:27.054 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000108146 2023-10-09T05:13:27.054 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.054 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.054 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:13:27.054 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.175201+0000", 2023-10-09T05:13:27.054 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0027374650000000001 2023-10-09T05:13:27.054 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.054 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.054 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:27.054 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.176694+0000", 2023-10-09T05:13:27.054 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0014937469999999999 2023-10-09T05:13:27.054 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.055 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.055 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:27.055 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.343557+0000", 2023-10-09T05:13:27.055 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.16686296 2023-10-09T05:13:27.055 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.055 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.055 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:13:27.055 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.343630+0000", 2023-10-09T05:13:27.055 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.2867000000000006e-05 2023-10-09T05:13:27.055 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.055 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.055 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:13:27.055 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.343689+0000", 2023-10-09T05:13:27.056 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.8386e-05 2023-10-09T05:13:27.056 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.056 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:13:27.056 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.056 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.056 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.056 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_repop(client.34845.0:154181 9.f e165/152 9:f9ff56ed:::benchmark_data_smithi133_153303_object64212:head v 165'9843, mlcod=0'0)", 2023-10-09T05:13:27.056 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:12:27.177594+0000", 2023-10-09T05:13:27.056 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 59.815884953999998, 2023-10-09T05:13:27.056 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.16695955700000001, 2023-10-09T05:13:27.056 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:13:27.056 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:27.056 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:13:27.057 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.057 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:13:27.057 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.177594+0000", 2023-10-09T05:13:27.057 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.057 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.057 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.057 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:13:27.057 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.177594+0000", 2023-10-09T05:13:27.057 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.057 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.057 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.057 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:13:27.058 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.177594+0000", 2023-10-09T05:13:27.058 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999995 2023-10-09T05:13:27.058 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.058 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.058 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:13:27.058 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.177607+0000", 2023-10-09T05:13:27.058 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.3818999999999999e-05 2023-10-09T05:13:27.058 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.058 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.058 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:13:27.058 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.177619+0000", 2023-10-09T05:13:27.058 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.1949e-05 2023-10-09T05:13:27.058 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.059 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.059 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:13:27.059 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.177627+0000", 2023-10-09T05:13:27.059 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.2640000000000002e-06 2023-10-09T05:13:27.059 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.059 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.059 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:13:27.059 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.177676+0000", 2023-10-09T05:13:27.059 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.9413000000000003e-05 2023-10-09T05:13:27.059 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.059 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.059 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:13:27.060 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.177700+0000", 2023-10-09T05:13:27.060 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.4196e-05 2023-10-09T05:13:27.060 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.060 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.060 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:13:27.060 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.344499+0000", 2023-10-09T05:13:27.060 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.166798475 2023-10-09T05:13:27.060 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.060 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.060 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:13:27.060 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.344554+0000", 2023-10-09T05:13:27.060 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.5081999999999999e-05 2023-10-09T05:13:27.061 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.061 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:13:27.061 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.061 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.061 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.061 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_repop(client.34845.0:154182 9.3 e165/161 9:c00da17b:::benchmark_data_smithi133_153303_object64213:head v 165'8277, mlcod=157'5458)", 2023-10-09T05:13:27.061 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:12:27.177954+0000", 2023-10-09T05:13:27.061 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 59.815524777999997, 2023-10-09T05:13:27.061 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.17409061100000001, 2023-10-09T05:13:27.061 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:13:27.061 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:27.061 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:13:27.062 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.062 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:13:27.062 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.177954+0000", 2023-10-09T05:13:27.062 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.062 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.062 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.062 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:13:27.062 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.177954+0000", 2023-10-09T05:13:27.062 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.062 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.062 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.062 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:13:27.063 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.177954+0000", 2023-10-09T05:13:27.063 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999995 2023-10-09T05:13:27.063 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.063 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.063 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:13:27.063 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.177962+0000", 2023-10-09T05:13:27.063 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.2970000000000004e-06 2023-10-09T05:13:27.063 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.063 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.063 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:13:27.063 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.177969+0000", 2023-10-09T05:13:27.063 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.7469999999999999e-06 2023-10-09T05:13:27.064 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.064 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.064 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:13:27.064 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.177973+0000", 2023-10-09T05:13:27.064 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.54e-06 2023-10-09T05:13:27.064 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.064 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.064 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:13:27.064 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.178014+0000", 2023-10-09T05:13:27.064 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.1023999999999999e-05 2023-10-09T05:13:27.064 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.064 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.064 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:13:27.065 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.178034+0000", 2023-10-09T05:13:27.065 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.0295000000000001e-05 2023-10-09T05:13:27.065 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.065 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.065 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:13:27.065 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.351919+0000", 2023-10-09T05:13:27.065 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.173885505 2023-10-09T05:13:27.065 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.065 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.065 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:13:27.065 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.352045+0000", 2023-10-09T05:13:27.065 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00012564400000000001 2023-10-09T05:13:27.066 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.066 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:13:27.066 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.066 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.066 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.066 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_repop_reply(client.34845.0:154179 9.c e165/139 ondisk, result = 0)", 2023-10-09T05:13:27.066 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:12:27.178395+0000", 2023-10-09T05:13:27.066 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 59.815084202999998, 2023-10-09T05:13:27.066 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.165301427, 2023-10-09T05:13:27.066 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:13:27.066 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "started", 2023-10-09T05:13:27.066 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:13:27.067 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.067 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:13:27.067 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.178395+0000", 2023-10-09T05:13:27.067 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.067 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.067 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.067 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:13:27.067 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.178395+0000", 2023-10-09T05:13:27.067 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.067 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.067 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.067 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:13:27.068 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.178394+0000", 2023-10-09T05:13:27.068 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999995 2023-10-09T05:13:27.068 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.068 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.068 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:13:27.068 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.178400+0000", 2023-10-09T05:13:27.068 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.835e-06 2023-10-09T05:13:27.068 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.068 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.068 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:13:27.068 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.343369+0000", 2023-10-09T05:13:27.068 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.16496882800000001 2023-10-09T05:13:27.068 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.069 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.069 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:13:27.069 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.343382+0000", 2023-10-09T05:13:27.069 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.3473e-05 2023-10-09T05:13:27.069 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.069 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.069 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:13:27.069 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.343537+0000", 2023-10-09T05:13:27.069 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00015416599999999999 2023-10-09T05:13:27.069 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.069 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.069 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:13:27.070 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.343549+0000", 2023-10-09T05:13:27.070 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.2533e-05 2023-10-09T05:13:27.070 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.070 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.070 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:13:27.070 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.343696+0000", 2023-10-09T05:13:27.070 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00014729000000000001 2023-10-09T05:13:27.070 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.070 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:13:27.070 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.070 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.070 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.071 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_repop(client.34845.0:154188 9.4 e165/160 9:22b5369a:::benchmark_data_smithi133_153303_object64219:head v 165'8594, mlcod=165'8592)", 2023-10-09T05:13:27.071 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:12:27.184670+0000", 2023-10-09T05:13:27.071 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 59.808808907, 2023-10-09T05:13:27.071 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.17017385900000001, 2023-10-09T05:13:27.071 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:13:27.071 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:27.071 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:13:27.071 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.071 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:13:27.071 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.184670+0000", 2023-10-09T05:13:27.071 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.071 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.072 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.072 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:13:27.072 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.184670+0000", 2023-10-09T05:13:27.072 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.072 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.072 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.072 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:13:27.072 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.184669+0000", 2023-10-09T05:13:27.072 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:13:27.072 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.072 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.072 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:13:27.072 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.184677+0000", 2023-10-09T05:13:27.073 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.1089999999999998e-06 2023-10-09T05:13:27.073 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.073 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.073 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:13:27.073 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.344469+0000", 2023-10-09T05:13:27.073 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.15979195500000001 2023-10-09T05:13:27.073 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.073 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.073 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:13:27.073 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.344477+0000", 2023-10-09T05:13:27.073 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.7370000000000006e-06 2023-10-09T05:13:27.073 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.074 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.074 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:13:27.074 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.346306+0000", 2023-10-09T05:13:27.074 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.001829097 2023-10-09T05:13:27.074 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.074 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.074 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:13:27.074 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.346329+0000", 2023-10-09T05:13:27.074 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.3017999999999999e-05 2023-10-09T05:13:27.074 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.074 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.075 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:13:27.075 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.354651+0000", 2023-10-09T05:13:27.075 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0083220340000000007 2023-10-09T05:13:27.075 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.075 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.075 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:13:27.075 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.354844+0000", 2023-10-09T05:13:27.075 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00019304700000000001 2023-10-09T05:13:27.075 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.075 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:13:27.075 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.075 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.076 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.076 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.34845.0:154191 9.18 9:1f504b53:::benchmark_data_smithi133_153303_object64222:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-10-09T05:13:27.076 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:12:27.186451+0000", 2023-10-09T05:13:27.076 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 59.807028064999997, 2023-10-09T05:13:27.076 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.17772421099999999, 2023-10-09T05:13:27.076 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:13:27.076 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:27.076 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:13:27.076 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.34845", 2023-10-09T05:13:27.076 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:13:27.076 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 154191 2023-10-09T05:13:27.076 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.077 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:13:27.077 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.077 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:13:27.077 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.186451+0000", 2023-10-09T05:13:27.077 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.077 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.077 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.077 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:13:27.077 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.186451+0000", 2023-10-09T05:13:27.077 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.077 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.077 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.077 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:13:27.078 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.186449+0000", 2023-10-09T05:13:27.078 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999976 2023-10-09T05:13:27.078 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.078 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.078 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:13:27.078 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.186458+0000", 2023-10-09T05:13:27.078 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.49e-06 2023-10-09T05:13:27.078 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.078 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.078 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:13:27.078 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.343304+0000", 2023-10-09T05:13:27.078 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.15684627200000001 2023-10-09T05:13:27.079 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.079 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.079 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:13:27.079 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.343362+0000", 2023-10-09T05:13:27.079 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.8644000000000003e-05 2023-10-09T05:13:27.079 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.079 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.079 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:13:27.079 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.343595+0000", 2023-10-09T05:13:27.079 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00023268200000000001 2023-10-09T05:13:27.079 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.079 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.080 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:13:27.080 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.343748+0000", 2023-10-09T05:13:27.080 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00015239600000000001 2023-10-09T05:13:27.080 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.080 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.080 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:13:27.080 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.344171+0000", 2023-10-09T05:13:27.080 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00042381100000000001 2023-10-09T05:13:27.080 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.080 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.080 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:27.080 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.351580+0000", 2023-10-09T05:13:27.080 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0074082039999999998 2023-10-09T05:13:27.081 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.081 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.081 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:13:27.081 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.351592+0000", 2023-10-09T05:13:27.081 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.2184999999999999e-05 2023-10-09T05:13:27.081 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.081 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.081 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:27.081 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.364089+0000", 2023-10-09T05:13:27.081 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.012497337000000001 2023-10-09T05:13:27.081 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.081 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.081 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:13:27.082 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.364136+0000", 2023-10-09T05:13:27.082 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.6900000000000002e-05 2023-10-09T05:13:27.082 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.082 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.082 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:13:27.082 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:27.364175+0000", 2023-10-09T05:13:27.082 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.9441999999999999e-05 2023-10-09T05:13:27.082 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.082 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:13:27.082 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.082 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.082 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.083 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.34845.0:158247 9.12 9:489fcfb2:::benchmark_data_smithi133_153303_object68278:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e166)", 2023-10-09T05:13:27.083 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:12:32.029931+0000", 2023-10-09T05:13:27.083 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 54.963547793000004, 2023-10-09T05:13:27.083 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.14494370100000001, 2023-10-09T05:13:27.083 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:13:27.083 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:27.083 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:13:27.083 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.34845", 2023-10-09T05:13:27.083 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:13:27.083 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 158247 2023-10-09T05:13:27.083 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.083 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:13:27.083 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.084 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:13:27.084 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.029931+0000", 2023-10-09T05:13:27.084 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.084 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.084 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.084 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:13:27.084 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.029931+0000", 2023-10-09T05:13:27.084 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.084 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.084 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.084 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:13:27.084 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.029930+0000", 2023-10-09T05:13:27.084 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999981 2023-10-09T05:13:27.085 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.085 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.085 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:13:27.085 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.029935+0000", 2023-10-09T05:13:27.085 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.7570000000000001e-06 2023-10-09T05:13:27.085 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.085 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.085 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:13:27.085 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.029944+0000", 2023-10-09T05:13:27.085 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.0469999999999994e-06 2023-10-09T05:13:27.085 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.085 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.086 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:13:27.086 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.029951+0000", 2023-10-09T05:13:27.086 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.0400000000000004e-06 2023-10-09T05:13:27.086 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.086 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.086 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:13:27.086 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.033374+0000", 2023-10-09T05:13:27.086 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0034236420000000002 2023-10-09T05:13:27.086 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.086 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.086 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:13:27.086 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.033510+0000", 2023-10-09T05:13:27.087 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00013609700000000001 2023-10-09T05:13:27.087 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.087 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.087 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:13:27.087 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.033901+0000", 2023-10-09T05:13:27.087 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000390992 2023-10-09T05:13:27.087 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.087 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.087 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:27.087 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.039405+0000", 2023-10-09T05:13:27.087 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0055034250000000002 2023-10-09T05:13:27.087 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.088 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.088 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:13:27.088 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.039413+0000", 2023-10-09T05:13:27.088 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.473e-06 2023-10-09T05:13:27.088 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.088 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.088 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:27.088 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.174155+0000", 2023-10-09T05:13:27.088 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.134741954 2023-10-09T05:13:27.088 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.088 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.088 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:13:27.089 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.174348+0000", 2023-10-09T05:13:27.089 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000192592 2023-10-09T05:13:27.089 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.089 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.089 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:13:27.089 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.174875+0000", 2023-10-09T05:13:27.089 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00052742200000000005 2023-10-09T05:13:27.089 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.089 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:13:27.089 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.089 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.089 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.090 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_repop(client.34845.0:158252 9.13 e166/160 9:ce0d6480:::benchmark_data_smithi133_153303_object68283:head v 166'8442, mlcod=157'5457)", 2023-10-09T05:13:27.090 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:12:32.039496+0000", 2023-10-09T05:13:27.090 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 54.953983186999999, 2023-10-09T05:13:27.090 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.197040511, 2023-10-09T05:13:27.090 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:13:27.090 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:27.090 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:13:27.090 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.090 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:13:27.090 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.039496+0000", 2023-10-09T05:13:27.090 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.090 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.091 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.091 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:13:27.091 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.039496+0000", 2023-10-09T05:13:27.091 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.091 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.091 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.091 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:13:27.091 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.039495+0000", 2023-10-09T05:13:27.091 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:13:27.091 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.091 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.091 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:13:27.091 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.039504+0000", 2023-10-09T05:13:27.092 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.9760000000000001e-06 2023-10-09T05:13:27.092 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.092 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.092 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:13:27.092 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.039516+0000", 2023-10-09T05:13:27.092 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.1571e-05 2023-10-09T05:13:27.092 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.092 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.092 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:13:27.092 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.039523+0000", 2023-10-09T05:13:27.092 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.5179999999999998e-06 2023-10-09T05:13:27.092 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.093 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.093 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:13:27.093 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.039660+0000", 2023-10-09T05:13:27.093 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000136785 2023-10-09T05:13:27.093 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.093 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.093 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:13:27.093 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.039709+0000", 2023-10-09T05:13:27.093 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.8575000000000002e-05 2023-10-09T05:13:27.093 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.093 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.093 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:13:27.094 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.236489+0000", 2023-10-09T05:13:27.094 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.196780433 2023-10-09T05:13:27.094 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.094 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.094 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:13:27.094 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.236537+0000", 2023-10-09T05:13:27.094 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.7453999999999997e-05 2023-10-09T05:13:27.094 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.094 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:13:27.094 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.094 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.094 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.095 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_repop_reply(client.34845.0:158247 9.12 e166/160 ondisk, result = 0)", 2023-10-09T05:13:27.095 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:12:32.042280+0000", 2023-10-09T05:13:27.095 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 54.951199285999998, 2023-10-09T05:13:27.095 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.13262268499999999, 2023-10-09T05:13:27.095 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:13:27.095 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "started", 2023-10-09T05:13:27.095 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:13:27.095 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.095 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:13:27.095 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.042280+0000", 2023-10-09T05:13:27.095 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.095 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.096 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.096 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:13:27.096 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.042280+0000", 2023-10-09T05:13:27.096 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.096 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.096 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.096 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:13:27.096 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.042280+0000", 2023-10-09T05:13:27.096 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999995 2023-10-09T05:13:27.096 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.096 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.096 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:13:27.097 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.042283+0000", 2023-10-09T05:13:27.097 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.4460000000000001e-06 2023-10-09T05:13:27.097 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.097 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.097 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:13:27.097 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.042290+0000", 2023-10-09T05:13:27.097 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.0729999999999999e-06 2023-10-09T05:13:27.097 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.097 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.097 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:13:27.097 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.042295+0000", 2023-10-09T05:13:27.097 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.6920000000000001e-06 2023-10-09T05:13:27.098 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.098 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.098 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:13:27.098 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.174101+0000", 2023-10-09T05:13:27.098 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.13180636600000001 2023-10-09T05:13:27.098 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.098 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.098 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:13:27.098 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.174132+0000", 2023-10-09T05:13:27.098 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.0546999999999997e-05 2023-10-09T05:13:27.098 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.098 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.098 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:13:27.099 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.174903+0000", 2023-10-09T05:13:27.099 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00077099099999999995 2023-10-09T05:13:27.099 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.099 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:13:27.099 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.099 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.099 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.099 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.34845.0:158258 9.2 9.eac212e2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e166)", 2023-10-09T05:13:27.099 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:12:32.077627+0000", 2023-10-09T05:13:27.099 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 54.915852155000003, 2023-10-09T05:13:27.099 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.18680211699999999, 2023-10-09T05:13:27.099 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:13:27.100 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "reached pg", 2023-10-09T05:13:27.100 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:13:27.100 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.34845", 2023-10-09T05:13:27.100 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:13:27.100 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 158258 2023-10-09T05:13:27.100 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.100 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:13:27.100 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.100 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:13:27.100 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.077627+0000", 2023-10-09T05:13:27.100 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.100 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.101 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.101 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:13:27.101 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.077627+0000", 2023-10-09T05:13:27.101 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.101 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.101 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.101 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:13:27.101 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.077625+0000", 2023-10-09T05:13:27.101 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999976 2023-10-09T05:13:27.101 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.101 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.101 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:13:27.101 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.077638+0000", 2023-10-09T05:13:27.102 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.2576e-05 2023-10-09T05:13:27.102 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.102 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.102 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:13:27.102 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.077655+0000", 2023-10-09T05:13:27.102 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.7203000000000001e-05 2023-10-09T05:13:27.102 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.102 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.102 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:13:27.102 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.077666+0000", 2023-10-09T05:13:27.102 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.1106e-05 2023-10-09T05:13:27.103 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.103 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.103 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:13:27.103 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.264418+0000", 2023-10-09T05:13:27.103 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.18675255499999999 2023-10-09T05:13:27.103 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.103 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.103 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:13:27.103 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.264429+0000", 2023-10-09T05:13:27.103 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.0845e-05 2023-10-09T05:13:27.103 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.103 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:13:27.104 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.104 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.104 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.104 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.34845.0:158258 9.22 9:47484357:::benchmark_data_smithi133_153303_object68289:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T05:13:27.104 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:12:32.241754+0000", 2023-10-09T05:13:27.104 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 54.751724906, 2023-10-09T05:13:27.104 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.87858276800000001, 2023-10-09T05:13:27.104 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:13:27.104 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:27.104 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:13:27.104 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.34845", 2023-10-09T05:13:27.104 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:13:27.105 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 158258 2023-10-09T05:13:27.105 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.105 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:13:27.105 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.105 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:13:27.105 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.241754+0000", 2023-10-09T05:13:27.105 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.105 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.105 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.105 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:13:27.105 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.241754+0000", 2023-10-09T05:13:27.105 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.106 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.106 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.106 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:13:27.106 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.241752+0000", 2023-10-09T05:13:27.106 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999976 2023-10-09T05:13:27.106 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.106 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.106 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:13:27.106 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.241766+0000", 2023-10-09T05:13:27.106 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.4345000000000001e-05 2023-10-09T05:13:27.106 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.106 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.107 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:13:27.107 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.241794+0000", 2023-10-09T05:13:27.107 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.7174000000000001e-05 2023-10-09T05:13:27.107 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.107 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.107 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:13:27.107 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.241808+0000", 2023-10-09T05:13:27.107 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.415e-05 2023-10-09T05:13:27.107 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.107 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.107 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:13:27.107 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.264452+0000", 2023-10-09T05:13:27.108 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.022644087 2023-10-09T05:13:27.108 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.108 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.108 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:13:27.108 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.264460+0000", 2023-10-09T05:13:27.108 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.2090000000000005e-06 2023-10-09T05:13:27.108 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.108 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.108 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:13:27.108 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:33.113883+0000", 2023-10-09T05:13:27.108 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.84942296100000003 2023-10-09T05:13:27.108 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.108 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.109 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:13:27.109 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:33.114041+0000", 2023-10-09T05:13:27.109 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000158028 2023-10-09T05:13:27.109 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.109 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.109 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:13:27.109 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:33.114262+0000", 2023-10-09T05:13:27.109 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00022079700000000001 2023-10-09T05:13:27.109 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.109 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.109 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:13:27.109 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:33.120078+0000", 2023-10-09T05:13:27.110 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0058162049999999996 2023-10-09T05:13:27.110 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.110 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.110 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:27.110 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:33.120122+0000", 2023-10-09T05:13:27.110 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.4428000000000002e-05 2023-10-09T05:13:27.110 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.110 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.110 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:27.110 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:33.120160+0000", 2023-10-09T05:13:27.110 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.7277000000000002e-05 2023-10-09T05:13:27.110 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.111 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.111 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:13:27.111 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:33.120222+0000", 2023-10-09T05:13:27.111 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.2213999999999998e-05 2023-10-09T05:13:27.111 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.111 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.111 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:13:27.111 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:33.120337+0000", 2023-10-09T05:13:27.111 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000115238 2023-10-09T05:13:27.111 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.111 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:13:27.111 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.112 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.112 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.112 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.34845.0:158266 9.1c 9:39f2987b:::benchmark_data_smithi133_153303_object68297:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T05:13:27.112 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:12:32.379780+0000", 2023-10-09T05:13:27.112 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 54.613699472, 2023-10-09T05:13:27.112 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.75083452299999998, 2023-10-09T05:13:27.112 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:13:27.112 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:27.112 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:13:27.112 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.34845", 2023-10-09T05:13:27.112 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:13:27.112 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 158266 2023-10-09T05:13:27.113 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.113 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:13:27.113 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.113 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:13:27.113 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.379780+0000", 2023-10-09T05:13:27.113 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.113 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.113 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.113 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:13:27.113 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.379780+0000", 2023-10-09T05:13:27.113 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.113 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.113 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.114 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:13:27.114 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.379778+0000", 2023-10-09T05:13:27.114 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:13:27.114 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.114 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.114 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:13:27.114 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.379786+0000", 2023-10-09T05:13:27.114 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.0469999999999994e-06 2023-10-09T05:13:27.114 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.114 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.114 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:13:27.114 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.379800+0000", 2023-10-09T05:13:27.115 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.3287e-05 2023-10-09T05:13:27.115 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.115 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.115 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:13:27.115 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.379808+0000", 2023-10-09T05:13:27.115 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.0849999999999997e-06 2023-10-09T05:13:27.115 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.115 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.115 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:13:27.115 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.379880+0000", 2023-10-09T05:13:27.115 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.2519000000000001e-05 2023-10-09T05:13:27.115 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.116 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.116 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:13:27.116 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:32.379895+0000", 2023-10-09T05:13:27.116 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.429e-05 2023-10-09T05:13:27.116 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.116 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.116 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:13:27.116 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:33.125466+0000", 2023-10-09T05:13:27.116 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.74557135699999999 2023-10-09T05:13:27.116 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.116 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.116 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:13:27.116 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:33.125637+0000", 2023-10-09T05:13:27.117 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00017062100000000001 2023-10-09T05:13:27.117 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.117 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.117 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 4,7", 2023-10-09T05:13:27.117 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:33.125895+0000", 2023-10-09T05:13:27.117 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00025817100000000002 2023-10-09T05:13:27.117 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.117 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.117 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:13:27.117 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:33.128735+0000", 2023-10-09T05:13:27.117 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0028402169999999999 2023-10-09T05:13:27.117 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.118 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.118 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:27.118 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:33.129394+0000", 2023-10-09T05:13:27.118 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00065857199999999996 2023-10-09T05:13:27.118 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.118 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.118 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:27.118 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:33.130509+0000", 2023-10-09T05:13:27.118 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.001115801 2023-10-09T05:13:27.118 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.118 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.118 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:13:27.119 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:33.130568+0000", 2023-10-09T05:13:27.119 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.8795000000000002e-05 2023-10-09T05:13:27.119 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.119 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.119 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:13:27.119 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:12:33.130614+0000", 2023-10-09T05:13:27.119 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.6161000000000001e-05 2023-10-09T05:13:27.119 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.119 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:13:27.119 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.119 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.119 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.120 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "MOSDPGRecoveryDeleteReply(9.8 e176,173 9:119c0a21:::benchmark_data_smithi133_153303_object64034:head,165'8423)", 2023-10-09T05:13:27.120 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:13:16.476398+0000", 2023-10-09T05:13:27.120 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 10.517080757, 2023-10-09T05:13:27.120 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.100885823, 2023-10-09T05:13:27.120 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:13:27.120 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "reached pg", 2023-10-09T05:13:27.120 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:13:27.120 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.120 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:13:27.120 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:13:16.476398+0000", 2023-10-09T05:13:27.120 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.120 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.121 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.121 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:13:27.121 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:13:16.476398+0000", 2023-10-09T05:13:27.121 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.121 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.121 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.121 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:13:27.121 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:13:16.476397+0000", 2023-10-09T05:13:27.121 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:13:27.121 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.121 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.121 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:13:27.122 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:13:16.476404+0000", 2023-10-09T05:13:27.122 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.285e-06 2023-10-09T05:13:27.122 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.122 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.122 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:13:27.122 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:13:16.476415+0000", 2023-10-09T05:13:27.122 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.1833e-05 2023-10-09T05:13:27.122 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.122 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.122 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:13:27.122 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:13:16.476423+0000", 2023-10-09T05:13:27.122 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.7319999999999998e-06 2023-10-09T05:13:27.123 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.123 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.123 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:13:27.123 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:13:16.476461+0000", 2023-10-09T05:13:27.123 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.7558000000000002e-05 2023-10-09T05:13:27.123 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.123 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.123 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:13:27.123 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:13:16.577284+0000", 2023-10-09T05:13:27.123 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.10082358399999999 2023-10-09T05:13:27.123 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.123 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:13:27.124 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.124 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.124 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.124 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "MOSDPGRecoveryDeleteReply(9.8 e176,173 9:15217e34:::benchmark_data_smithi133_153303_object73361:head,170'8699)", 2023-10-09T05:13:27.124 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:13:17.615546+0000", 2023-10-09T05:13:27.124 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 9.3779335709999998, 2023-10-09T05:13:27.124 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.105601822, 2023-10-09T05:13:27.124 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:13:27.124 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "reached pg", 2023-10-09T05:13:27.124 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:13:27.124 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.124 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:13:27.125 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:13:17.615546+0000", 2023-10-09T05:13:27.125 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.125 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.125 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.125 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:13:27.125 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:13:17.615546+0000", 2023-10-09T05:13:27.125 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.125 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.125 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.125 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:13:27.125 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:13:17.615545+0000", 2023-10-09T05:13:27.125 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:13:27.125 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.126 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.126 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:13:27.126 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:13:17.615551+0000", 2023-10-09T05:13:27.126 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.7400000000000001e-06 2023-10-09T05:13:27.126 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.126 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.126 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:13:27.126 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:13:17.615561+0000", 2023-10-09T05:13:27.126 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.0397000000000001e-05 2023-10-09T05:13:27.126 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.126 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.126 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:13:27.127 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:13:17.615568+0000", 2023-10-09T05:13:27.127 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.7519999999999999e-06 2023-10-09T05:13:27.127 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.127 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.127 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:13:27.127 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:13:17.615610+0000", 2023-10-09T05:13:27.127 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.1952000000000001e-05 2023-10-09T05:13:27.127 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.127 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.127 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:13:27.127 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:13:17.721148+0000", 2023-10-09T05:13:27.127 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.1055378 2023-10-09T05:13:27.128 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.128 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:13:27.128 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.128 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.128 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.128 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "MOSDPGRecoveryDeleteReply(9.8 e176,173 9:1571ca06:::benchmark_data_smithi133_153303_object87448:head,171'9113)", 2023-10-09T05:13:27.128 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:13:19.069242+0000", 2023-10-09T05:13:27.128 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 7.9242368780000003, 2023-10-09T05:13:27.128 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.120607537, 2023-10-09T05:13:27.128 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:13:27.128 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "reached pg", 2023-10-09T05:13:27.128 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:13:27.128 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.129 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:13:27.129 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:13:19.069242+0000", 2023-10-09T05:13:27.129 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.129 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.129 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.129 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:13:27.129 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:13:19.069242+0000", 2023-10-09T05:13:27.129 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:13:27.129 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.129 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.129 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:13:27.129 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:13:19.069242+0000", 2023-10-09T05:13:27.130 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:13:27.130 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.130 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.130 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:13:27.130 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:13:19.069250+0000", 2023-10-09T05:13:27.130 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.7609999999999996e-06 2023-10-09T05:13:27.130 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.130 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.130 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:13:27.130 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:13:19.069262+0000", 2023-10-09T05:13:27.130 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.1473e-05 2023-10-09T05:13:27.130 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.131 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.131 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:13:27.131 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:13:19.069269+0000", 2023-10-09T05:13:27.131 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.4880000000000001e-06 2023-10-09T05:13:27.131 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.131 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.131 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:13:27.131 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:13:19.069298+0000", 2023-10-09T05:13:27.131 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.8306e-05 2023-10-09T05:13:27.131 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:13:27.131 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:13:27.131 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:13:27.132 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:13:19.189850+0000", 2023-10-09T05:13:27.132 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.120552303 2023-10-09T05:13:27.132 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.132 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:13:27.132 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.132 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:13:27.132 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:13:27.132 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:13:27.133 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:26 smithi133 bash[102858]: cluster 2023-10-09T05:13:25.858078+0000 mgr.y (mgr.25277) 747 : cluster 0 pgmap v734: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:27.133 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:26 smithi133 bash[105589]: cluster 2023-10-09T05:13:25.858078+0000 mgr.y (mgr.25277) 747 : cluster 0 pgmap v734: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:27.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:27.134 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:27.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:27.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:26 smithi194 bash[92666]: cluster 2023-10-09T05:13:25.858078+0000 mgr.y (mgr.25277) 747 : cluster 0 pgmap v734: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:27.235 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:27.259 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:27.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:27.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:27.362 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:27.393 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:27.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:27.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:27.497 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:27.532 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:27.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:27.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:27.635 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:27.661 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:27.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:27.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:27.765 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:27.795 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:27.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:27.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:27.899 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:27.929 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:27.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:27.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:28.032 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:28.062 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:28.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:28.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:28.166 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:28.193 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:28.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:28.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:28.297 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:28.330 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:28.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:28.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:28.434 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:28.467 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:28.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:28.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:28.570 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:28.605 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:28.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:28.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:28.709 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:28.741 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:28.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:28.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:28.844 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:28.877 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:28.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:28.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:28.982 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:29.013 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:29.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:29.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:29.117 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:29.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:28 smithi194 bash[92666]: cluster 2023-10-09T05:13:27.858727+0000 mgr.y (mgr.25277) 748 : cluster 0 pgmap v735: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:29.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:28 smithi194 bash[92666]: audit 2023-10-09T05:13:28.449599+0000 mon.a (mon.0) 956 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:13:29.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:28 smithi194 bash[92666]: audit 2023-10-09T05:13:28.450644+0000 mon.a (mon.0) 957 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:13:29.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:28 smithi194 bash[92666]: audit 2023-10-09T05:13:28.833180+0000 mon.a (mon.0) 958 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:13:29.138 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:29.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:29.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:29.241 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:29.268 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:29.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:29.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:29.291 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:28 smithi133 bash[102858]: cluster 2023-10-09T05:13:27.858727+0000 mgr.y (mgr.25277) 748 : cluster 0 pgmap v735: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:29.291 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:28 smithi133 bash[102858]: audit 2023-10-09T05:13:28.449599+0000 mon.a (mon.0) 956 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:13:29.291 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:28 smithi133 bash[102858]: audit 2023-10-09T05:13:28.450644+0000 mon.a (mon.0) 957 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:13:29.291 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:28 smithi133 bash[102858]: audit 2023-10-09T05:13:28.833180+0000 mon.a (mon.0) 958 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:13:29.292 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:28 smithi133 bash[105589]: cluster 2023-10-09T05:13:27.858727+0000 mgr.y (mgr.25277) 748 : cluster 0 pgmap v735: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:29.292 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:28 smithi133 bash[105589]: audit 2023-10-09T05:13:28.449599+0000 mon.a (mon.0) 956 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:13:29.292 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:28 smithi133 bash[105589]: audit 2023-10-09T05:13:28.450644+0000 mon.a (mon.0) 957 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:13:29.292 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:28 smithi133 bash[105589]: audit 2023-10-09T05:13:28.833180+0000 mon.a (mon.0) 958 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:13:29.371 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:29.396 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:29.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:29.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:29.501 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:29.532 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:29.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:29.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:29.636 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:29.669 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:29.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:29.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:29.772 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:29.807 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:29.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:29.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:29.910 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:29.935 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T05:13:29.940 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:29.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:29.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:30.044 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:30.072 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:30.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:30.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:30.176 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:30.207 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:30.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:30.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:30.311 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:30.336 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:30.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:30.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:30.439 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:30.471 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:30.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:30.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:30.574 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:30.607 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:30.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:30.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:30.711 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:30.738 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:30.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:30.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:30.842 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:30.870 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:30.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:30.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:30.972 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:31.004 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:31.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:31.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:31.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:30 smithi133 bash[102858]: cluster 2023-10-09T05:13:29.859658+0000 mgr.y (mgr.25277) 749 : cluster 0 pgmap v736: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:31.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:30 smithi133 bash[105589]: cluster 2023-10-09T05:13:29.859658+0000 mgr.y (mgr.25277) 749 : cluster 0 pgmap v736: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:31.107 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:31.133 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:31.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:31.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:31.237 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:31.257 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:31.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:31.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:31.273 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:13:31.273 INFO:teuthology.orchestra.run.smithi133.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T05:13:31.273 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:13:31.360 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:31.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:30 smithi194 bash[92666]: cluster 2023-10-09T05:13:29.859658+0000 mgr.y (mgr.25277) 749 : cluster 0 pgmap v736: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:31.391 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:31.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:31.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:31.483 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:13:31.496 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:31.521 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:31.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:31.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:31.625 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:31.656 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:31.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:31.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:31.760 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:31.793 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:31.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:31.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:31.897 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:31.932 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:31.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:31.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:32.036 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:32.057 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:32.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:32.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:32.160 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:32.190 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:32.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:32.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:32.294 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:32.316 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:32.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:32.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:32.420 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:32.452 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:32.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:32.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:32.556 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:32.588 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:32.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:32.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:32.692 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:32.717 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:32.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:32.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:32.821 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:32.852 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:32.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:32.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:32.955 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:32.980 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:32.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:32.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:33.058 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:32 smithi133 bash[102858]: cluster 2023-10-09T05:13:31.860234+0000 mgr.y (mgr.25277) 750 : cluster 0 pgmap v737: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:33.058 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:32 smithi133 bash[105589]: cluster 2023-10-09T05:13:31.860234+0000 mgr.y (mgr.25277) 750 : cluster 0 pgmap v737: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:33.083 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:33.097 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:32 smithi194 bash[92666]: cluster 2023-10-09T05:13:31.860234+0000 mgr.y (mgr.25277) 750 : cluster 0 pgmap v737: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:33.118 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:33.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:33.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:33.222 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:33.253 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:33.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:33.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:33.357 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:33.391 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:33.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:33.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:33.495 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:33.526 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:33.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:33.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:33.630 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:33.655 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:33.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:33.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:33.758 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:33.793 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:33.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:33.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:33.897 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:33.921 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:33.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:33.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:34.024 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:34.056 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:34.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:34.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:34.160 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:34.196 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:34.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:34.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:34.299 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:34.330 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:34.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:34.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:34.433 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:34.455 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:34.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:34.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:34.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:34 smithi133 bash[105589]: cluster 2023-10-09T05:13:33.860684+0000 mgr.y (mgr.25277) 751 : cluster 0 pgmap v738: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:34.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:34 smithi133 bash[102858]: cluster 2023-10-09T05:13:33.860684+0000 mgr.y (mgr.25277) 751 : cluster 0 pgmap v738: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:34.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:13:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:13:34] "GET /metrics HTTP/1.1" 200 34053 "" "Prometheus/2.43.0" 2023-10-09T05:13:34.558 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:34.586 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:34.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:34.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:34.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:34 smithi194 bash[92666]: cluster 2023-10-09T05:13:33.860684+0000 mgr.y (mgr.25277) 751 : cluster 0 pgmap v738: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:34.689 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:34.713 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:34.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:34.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:34.724 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:13:34.816 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:34.838 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:34.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:34.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:34.942 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:34.962 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:13:34.968 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:34.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:34.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:34.984 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:13:35.004 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:13:35.031 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:13:35.054 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:13:35.072 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:35.083 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:13:35.099 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:35.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:35.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:35.108 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:13:35.135 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:13:35.137 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:13:35.137 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:13:35.137 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:13:35.137 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:13:35.137 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:13:35.137 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:13:35.137 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:13:35.137 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:13:35.203 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:35.232 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:35.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:35.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:35.336 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:35.359 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:35 smithi133 bash[102858]: audit 2023-10-09T05:13:34.936874+0000 mon.c (mon.2) 71 : audit 0 from='client.? 172.21.15.133:0/3266144255' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:13:35.360 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:35 smithi133 bash[105589]: audit 2023-10-09T05:13:34.936874+0000 mon.c (mon.2) 71 : audit 0 from='client.? 172.21.15.133:0/3266144255' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:13:35.361 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:35.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:35.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:35.465 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:35.497 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:35.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:35.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:35.601 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:35.614 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:35 smithi194 bash[92666]: audit 2023-10-09T05:13:34.936874+0000 mon.c (mon.2) 71 : audit 0 from='client.? 172.21.15.133:0/3266144255' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:13:35.632 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:35.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:35.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:35.735 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:35.760 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:35.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:35.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:35.863 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:35.887 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:35.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:35.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:35.991 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:36.022 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:36.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:36.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:36.125 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:36.153 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:36.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:36.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:36.257 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:36.284 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:36.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:36.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:36.327 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is unset 2023-10-09T05:13:36.388 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:36.413 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:36.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:36.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:36.516 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:36.525 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:36 smithi133 bash[105589]: audit 2023-10-09T05:13:35.733338+0000 mon.a (mon.0) 959 : audit 1 from='client.? 172.21.15.133:0/2999416691' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:13:36.526 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:36 smithi133 bash[105589]: cluster 2023-10-09T05:13:35.861469+0000 mgr.y (mgr.25277) 752 : cluster 0 pgmap v739: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:36.526 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:36 smithi133 bash[102858]: audit 2023-10-09T05:13:35.733338+0000 mon.a (mon.0) 959 : audit 1 from='client.? 172.21.15.133:0/2999416691' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:13:36.526 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:36 smithi133 bash[102858]: cluster 2023-10-09T05:13:35.861469+0000 mgr.y (mgr.25277) 752 : cluster 0 pgmap v739: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:36.543 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:36.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:36.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:36.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:36 smithi194 bash[92666]: audit 2023-10-09T05:13:35.733338+0000 mon.a (mon.0) 959 : audit 1 from='client.? 172.21.15.133:0/2999416691' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:13:36.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:36 smithi194 bash[92666]: cluster 2023-10-09T05:13:35.861469+0000 mgr.y (mgr.25277) 752 : cluster 0 pgmap v739: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:36.650 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:36.676 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:36.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:36.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:36.779 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:36.804 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:36.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:36.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:36.908 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:36.941 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:36.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:36.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:37.045 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:37.072 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:37.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:37.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:37.175 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:37.202 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:37.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:37.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:37.306 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:37.331 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:37.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:37.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:37.435 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:37.468 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:37.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:37.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:37.573 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:37.586 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:37 smithi133 bash[105589]: cluster 2023-10-09T05:13:36.306479+0000 mon.a (mon.0) 960 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:13:37.587 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:37 smithi133 bash[105589]: audit 2023-10-09T05:13:36.322200+0000 mon.a (mon.0) 961 : audit 1 from='client.? 172.21.15.133:0/2999416691' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:13:37.587 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:37 smithi133 bash[105589]: cluster 2023-10-09T05:13:36.322290+0000 mon.a (mon.0) 962 : cluster 0 osdmap e178: 8 total, 8 up, 7 in 2023-10-09T05:13:37.587 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:37 smithi133 bash[105589]: cluster 2023-10-09T05:13:36.350256+0000 osd.7 (osd.7) 108 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T05:13:37.587 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:37 smithi133 bash[105589]: cluster 2023-10-09T05:13:36.351347+0000 osd.7 (osd.7) 109 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T05:13:37.587 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:37 smithi133 bash[105589]: cluster 2023-10-09T05:13:36.352406+0000 osd.7 (osd.7) 110 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T05:13:37.588 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:37 smithi133 bash[102858]: cluster 2023-10-09T05:13:36.306479+0000 mon.a (mon.0) 960 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:13:37.588 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:37 smithi133 bash[102858]: audit 2023-10-09T05:13:36.322200+0000 mon.a (mon.0) 961 : audit 1 from='client.? 172.21.15.133:0/2999416691' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:13:37.588 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:37 smithi133 bash[102858]: cluster 2023-10-09T05:13:36.322290+0000 mon.a (mon.0) 962 : cluster 0 osdmap e178: 8 total, 8 up, 7 in 2023-10-09T05:13:37.588 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:37 smithi133 bash[102858]: cluster 2023-10-09T05:13:36.350256+0000 osd.7 (osd.7) 108 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T05:13:37.589 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:37 smithi133 bash[102858]: cluster 2023-10-09T05:13:36.351347+0000 osd.7 (osd.7) 109 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T05:13:37.589 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:37 smithi133 bash[102858]: cluster 2023-10-09T05:13:36.352406+0000 osd.7 (osd.7) 110 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T05:13:37.605 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:37.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:37.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:37.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:37 smithi194 bash[92666]: cluster 2023-10-09T05:13:36.306479+0000 mon.a (mon.0) 960 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:13:37.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:37 smithi194 bash[92666]: audit 2023-10-09T05:13:36.322200+0000 mon.a (mon.0) 961 : audit 1 from='client.? 172.21.15.133:0/2999416691' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:13:37.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:37 smithi194 bash[92666]: cluster 2023-10-09T05:13:36.322290+0000 mon.a (mon.0) 962 : cluster 0 osdmap e178: 8 total, 8 up, 7 in 2023-10-09T05:13:37.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:37 smithi194 bash[92666]: cluster 2023-10-09T05:13:36.350256+0000 osd.7 (osd.7) 108 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T05:13:37.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:37 smithi194 bash[92666]: cluster 2023-10-09T05:13:36.351347+0000 osd.7 (osd.7) 109 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T05:13:37.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:37 smithi194 bash[92666]: cluster 2023-10-09T05:13:36.352406+0000 osd.7 (osd.7) 110 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T05:13:37.708 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:37.736 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:37.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:37.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:37.840 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:37.871 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:37.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:37.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:37.974 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:37.999 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:38.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:38.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:38.103 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:38.130 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:38.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:38.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:38.233 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:38.265 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:38.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:38.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:38.369 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:38.400 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:38.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:38.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:38.505 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:38.538 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:38.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:38.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:38.599 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:13:38.611 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:38 smithi133 bash[102858]: cluster 2023-10-09T05:13:37.392629+0000 osd.6 (osd.6) 104 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T05:13:38.611 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:38 smithi133 bash[102858]: cluster 2023-10-09T05:13:37.392728+0000 osd.5 (osd.5) 169 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T05:13:38.611 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:38 smithi133 bash[102858]: cluster 2023-10-09T05:13:37.392980+0000 osd.4 (osd.4) 115 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T05:13:38.611 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:38 smithi133 bash[102858]: cluster 2023-10-09T05:13:37.861885+0000 mgr.y (mgr.25277) 753 : cluster 0 pgmap v741: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:38.612 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:38 smithi133 bash[105589]: cluster 2023-10-09T05:13:37.392629+0000 osd.6 (osd.6) 104 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T05:13:38.612 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:38 smithi133 bash[105589]: cluster 2023-10-09T05:13:37.392728+0000 osd.5 (osd.5) 169 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T05:13:38.612 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:38 smithi133 bash[105589]: cluster 2023-10-09T05:13:37.392980+0000 osd.4 (osd.4) 115 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T05:13:38.612 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:38 smithi133 bash[105589]: cluster 2023-10-09T05:13:37.861885+0000 mgr.y (mgr.25277) 753 : cluster 0 pgmap v741: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:38.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:38 smithi194 bash[92666]: cluster 2023-10-09T05:13:37.392629+0000 osd.6 (osd.6) 104 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T05:13:38.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:38 smithi194 bash[92666]: cluster 2023-10-09T05:13:37.392728+0000 osd.5 (osd.5) 169 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T05:13:38.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:38 smithi194 bash[92666]: cluster 2023-10-09T05:13:37.392980+0000 osd.4 (osd.4) 115 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T05:13:38.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:38 smithi194 bash[92666]: cluster 2023-10-09T05:13:37.861885+0000 mgr.y (mgr.25277) 753 : cluster 0 pgmap v741: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:38.641 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:38.666 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:38.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:38.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:38.678 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:13:38.678 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:13:38.771 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:38.797 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:38.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:38.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:38.900 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:38.933 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:38.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:38.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:39.036 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:39.067 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:39.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:39.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:39.170 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:39.202 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:39.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:39.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:39.307 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:39.342 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:39.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:39.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:39.446 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:39.476 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:39.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:39.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:39.544 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T05:13:39.580 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:39.621 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:39.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:39.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:39.726 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:39.756 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:39.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:39.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:39.859 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:39.879 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 6, 7, 1] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:13:39.879 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:13:39.879 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [0, 2, 3, 4, 6, 7, 1] 2023-10-09T05:13:39.879 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd out 7 2023-10-09T05:13:39.886 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:39.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:39.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:39.968 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:13:39.989 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:40.040 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:40.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:40.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:40.144 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:40.170 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:40.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:40.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:40.265 INFO:teuthology.orchestra.run.smithi133.stderr:pool 'unique_pool_5' created 2023-10-09T05:13:40.273 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:40.300 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:40.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:40.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:40.404 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:40.431 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:40.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:40.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:40.535 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:40.546 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:40 smithi194 bash[92666]: audit 2023-10-09T05:13:39.743400+0000 mon.a (mon.0) 963 : audit 1 from='client.? 172.21.15.133:0/3662666153' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T05:13:40.546 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:40 smithi194 bash[92666]: cluster 2023-10-09T05:13:39.862720+0000 mgr.y (mgr.25277) 754 : cluster 0 pgmap v742: 73 pgs: 73 active+clean; 582 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:40.569 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:40.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:40.571 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:40 smithi133 bash[102858]: audit 2023-10-09T05:13:39.743400+0000 mon.a (mon.0) 963 : audit 1 from='client.? 172.21.15.133:0/3662666153' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T05:13:40.571 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:40 smithi133 bash[102858]: cluster 2023-10-09T05:13:39.862720+0000 mgr.y (mgr.25277) 754 : cluster 0 pgmap v742: 73 pgs: 73 active+clean; 582 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:40.571 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:40 smithi133 bash[105589]: audit 2023-10-09T05:13:39.743400+0000 mon.a (mon.0) 963 : audit 1 from='client.? 172.21.15.133:0/3662666153' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T05:13:40.572 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:40 smithi133 bash[105589]: cluster 2023-10-09T05:13:39.862720+0000 mgr.y (mgr.25277) 754 : cluster 0 pgmap v742: 73 pgs: 73 active+clean; 582 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:40.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:40.672 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:40.703 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:40.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:40.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:40.807 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:40.840 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:40.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:40.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:40.944 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:40.972 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:40.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:40.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:41.076 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:41.110 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:41.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:41.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:41.214 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:41.244 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:41.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:41.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:41.263 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:41 smithi133 bash[102858]: cluster 2023-10-09T05:13:40.249142+0000 osd.4 (osd.4) 116 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T05:13:41.263 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:41 smithi133 bash[102858]: cluster 2023-10-09T05:13:40.249582+0000 osd.5 (osd.5) 170 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T05:13:41.263 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:41 smithi133 bash[102858]: cluster 2023-10-09T05:13:40.253768+0000 osd.4 (osd.4) 117 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T05:13:41.264 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:41 smithi133 bash[102858]: audit 2023-10-09T05:13:40.254554+0000 mon.a (mon.0) 964 : audit 1 from='client.? 172.21.15.133:0/3662666153' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T05:13:41.264 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:41 smithi133 bash[102858]: cluster 2023-10-09T05:13:40.254708+0000 mon.a (mon.0) 965 : cluster 0 osdmap e179: 8 total, 8 up, 7 in 2023-10-09T05:13:41.264 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:41 smithi133 bash[102858]: cluster 2023-10-09T05:13:40.257825+0000 osd.7 (osd.7) 111 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T05:13:41.264 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:41 smithi133 bash[102858]: cluster 2023-10-09T05:13:41.256933+0000 mon.a (mon.0) 966 : cluster 0 osdmap e180: 8 total, 8 up, 7 in 2023-10-09T05:13:41.277 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2023-10-09T05:13:41.347 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:41.360 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:41 smithi194 bash[92666]: cluster 2023-10-09T05:13:40.249142+0000 osd.4 (osd.4) 116 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T05:13:41.360 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:41 smithi194 bash[92666]: cluster 2023-10-09T05:13:40.249582+0000 osd.5 (osd.5) 170 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T05:13:41.360 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:41 smithi194 bash[92666]: cluster 2023-10-09T05:13:40.253768+0000 osd.4 (osd.4) 117 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T05:13:41.360 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:41 smithi194 bash[92666]: audit 2023-10-09T05:13:40.254554+0000 mon.a (mon.0) 964 : audit 1 from='client.? 172.21.15.133:0/3662666153' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T05:13:41.360 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:41 smithi194 bash[92666]: cluster 2023-10-09T05:13:40.254708+0000 mon.a (mon.0) 965 : cluster 0 osdmap e179: 8 total, 8 up, 7 in 2023-10-09T05:13:41.361 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:41 smithi194 bash[92666]: cluster 2023-10-09T05:13:40.257825+0000 osd.7 (osd.7) 111 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T05:13:41.361 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:41 smithi194 bash[92666]: cluster 2023-10-09T05:13:41.256933+0000 mon.a (mon.0) 966 : cluster 0 osdmap e180: 8 total, 8 up, 7 in 2023-10-09T05:13:41.379 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:41.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:41.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:41.484 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:41.520 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:41.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:41.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:41.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:41 smithi133 bash[105589]: cluster 2023-10-09T05:13:40.249142+0000 osd.4 (osd.4) 116 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T05:13:41.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:41 smithi133 bash[105589]: cluster 2023-10-09T05:13:40.249582+0000 osd.5 (osd.5) 170 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T05:13:41.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:41 smithi133 bash[105589]: cluster 2023-10-09T05:13:40.253768+0000 osd.4 (osd.4) 117 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T05:13:41.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:41 smithi133 bash[105589]: audit 2023-10-09T05:13:40.254554+0000 mon.a (mon.0) 964 : audit 1 from='client.? 172.21.15.133:0/3662666153' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T05:13:41.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:41 smithi133 bash[105589]: cluster 2023-10-09T05:13:40.254708+0000 mon.a (mon.0) 965 : cluster 0 osdmap e179: 8 total, 8 up, 7 in 2023-10-09T05:13:41.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:41 smithi133 bash[105589]: cluster 2023-10-09T05:13:40.257825+0000 osd.7 (osd.7) 111 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T05:13:41.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:41 smithi133 bash[105589]: cluster 2023-10-09T05:13:41.256933+0000 mon.a (mon.0) 966 : cluster 0 osdmap e180: 8 total, 8 up, 7 in 2023-10-09T05:13:41.624 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:41.661 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:41.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:41.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:41.764 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:41.796 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:41.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:41.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:41.899 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:41.933 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:41.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:41.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:42.038 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:42.073 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:42.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:42.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:42.177 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:42.208 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:42.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:42.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:42.312 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:42.321 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:42 smithi194 bash[92666]: cluster 2023-10-09T05:13:41.252161+0000 osd.5 (osd.5) 171 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T05:13:42.322 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:42 smithi194 bash[92666]: cluster 2023-10-09T05:13:41.257416+0000 osd.4 (osd.4) 118 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T05:13:42.322 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:42 smithi194 bash[92666]: cluster 2023-10-09T05:13:41.257712+0000 osd.6 (osd.6) 105 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T05:13:42.322 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:42 smithi194 bash[92666]: cluster 2023-10-09T05:13:41.257731+0000 osd.7 (osd.7) 112 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T05:13:42.322 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:42 smithi194 bash[92666]: cluster 2023-10-09T05:13:41.863396+0000 mgr.y (mgr.25277) 755 : cluster 0 pgmap v745: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:42.339 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:42.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:42.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:42.444 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:42.476 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:42.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:42.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:42.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:42 smithi133 bash[102858]: cluster 2023-10-09T05:13:41.252161+0000 osd.5 (osd.5) 171 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T05:13:42.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:42 smithi133 bash[102858]: cluster 2023-10-09T05:13:41.257416+0000 osd.4 (osd.4) 118 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T05:13:42.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:42 smithi133 bash[102858]: cluster 2023-10-09T05:13:41.257712+0000 osd.6 (osd.6) 105 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T05:13:42.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:42 smithi133 bash[102858]: cluster 2023-10-09T05:13:41.257731+0000 osd.7 (osd.7) 112 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T05:13:42.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:42 smithi133 bash[102858]: cluster 2023-10-09T05:13:41.863396+0000 mgr.y (mgr.25277) 755 : cluster 0 pgmap v745: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:42.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:42 smithi133 bash[105589]: cluster 2023-10-09T05:13:41.252161+0000 osd.5 (osd.5) 171 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T05:13:42.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:42 smithi133 bash[105589]: cluster 2023-10-09T05:13:41.257416+0000 osd.4 (osd.4) 118 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T05:13:42.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:42 smithi133 bash[105589]: cluster 2023-10-09T05:13:41.257712+0000 osd.6 (osd.6) 105 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T05:13:42.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:42 smithi133 bash[105589]: cluster 2023-10-09T05:13:41.257731+0000 osd.7 (osd.7) 112 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T05:13:42.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:42 smithi133 bash[105589]: cluster 2023-10-09T05:13:41.863396+0000 mgr.y (mgr.25277) 755 : cluster 0 pgmap v745: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:42.580 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:42.612 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:42.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:42.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:42.715 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:42.744 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:42.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:42.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:42.847 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:42.878 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:42.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:42.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:42.982 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:43.010 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:43.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:43.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:43.113 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:43.142 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:43.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:43.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:43.245 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:43.275 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:43.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:43.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:43.379 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:43.392 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:43 smithi133 bash[105589]: cluster 2023-10-09T05:13:42.277142+0000 mon.a (mon.0) 967 : cluster 0 osdmap e181: 8 total, 8 up, 7 in 2023-10-09T05:13:43.392 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:43 smithi133 bash[102858]: cluster 2023-10-09T05:13:42.277142+0000 mon.a (mon.0) 967 : cluster 0 osdmap e181: 8 total, 8 up, 7 in 2023-10-09T05:13:43.410 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:43.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:43.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:43.514 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:43.525 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:43 smithi194 bash[92666]: cluster 2023-10-09T05:13:42.277142+0000 mon.a (mon.0) 967 : cluster 0 osdmap e181: 8 total, 8 up, 7 in 2023-10-09T05:13:43.540 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:43.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:43.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:43.644 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:43.676 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:43.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:43.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:43.772 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:13:43.780 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:43.818 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:43.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:43.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:43.921 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:43.950 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:43.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:43.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:44.054 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:44.085 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:44.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:44.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:44.189 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:44.219 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:44.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:44.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:44.322 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:13:44.322 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:44.333 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:44 smithi194 bash[92666]: cluster 2023-10-09T05:13:42.991919+0000 osd.7 (osd.7) 113 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T05:13:44.334 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:44 smithi194 bash[92666]: cluster 2023-10-09T05:13:43.049829+0000 osd.4 (osd.4) 119 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T05:13:44.334 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:44 smithi194 bash[92666]: audit 2023-10-09T05:13:43.446329+0000 mon.a (mon.0) 968 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:13:44.334 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:44 smithi194 bash[92666]: cluster 2023-10-09T05:13:43.517443+0000 osd.6 (osd.6) 106 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T05:13:44.334 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:44 smithi194 bash[92666]: cluster 2023-10-09T05:13:43.864167+0000 mgr.y (mgr.25277) 756 : cluster 0 pgmap v747: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:44.350 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:44.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:44.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:44.455 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:44.463 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:13:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:13:44] "GET /metrics HTTP/1.1" 200 36397 "" "Prometheus/2.43.0" 2023-10-09T05:13:44.463 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:44 smithi133 bash[105589]: cluster 2023-10-09T05:13:42.991919+0000 osd.7 (osd.7) 113 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T05:13:44.464 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:44 smithi133 bash[105589]: cluster 2023-10-09T05:13:43.049829+0000 osd.4 (osd.4) 119 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T05:13:44.464 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:44 smithi133 bash[105589]: audit 2023-10-09T05:13:43.446329+0000 mon.a (mon.0) 968 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:13:44.464 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:44 smithi133 bash[105589]: cluster 2023-10-09T05:13:43.517443+0000 osd.6 (osd.6) 106 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T05:13:44.464 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:44 smithi133 bash[105589]: cluster 2023-10-09T05:13:43.864167+0000 mgr.y (mgr.25277) 756 : cluster 0 pgmap v747: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:44.464 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:44 smithi133 bash[102858]: cluster 2023-10-09T05:13:42.991919+0000 osd.7 (osd.7) 113 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T05:13:44.464 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:44 smithi133 bash[102858]: cluster 2023-10-09T05:13:43.049829+0000 osd.4 (osd.4) 119 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T05:13:44.464 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:44 smithi133 bash[102858]: audit 2023-10-09T05:13:43.446329+0000 mon.a (mon.0) 968 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:13:44.464 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:44 smithi133 bash[102858]: cluster 2023-10-09T05:13:43.517443+0000 osd.6 (osd.6) 106 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T05:13:44.464 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:44 smithi133 bash[102858]: cluster 2023-10-09T05:13:43.864167+0000 mgr.y (mgr.25277) 756 : cluster 0 pgmap v747: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:44.482 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:44.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:44.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:44.585 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:44.616 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:44.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:44.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:44.720 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:44.751 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:44.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:44.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:44.789 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:13:44.790 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:13:44.856 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:44.882 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:44.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:44.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:44.985 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:45.006 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:45.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:45.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:45.109 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:45.139 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:45.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:45.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:45.243 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:45.274 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:45.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:45.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:45.377 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:45.403 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:45.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:45.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:45.506 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:45.533 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:45.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:45.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:45.636 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:45.665 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:45.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:45.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:45.769 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:45.791 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:45.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:45.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:45.804 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:13:45.894 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:45.917 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:45.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:45.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:46.021 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:46.043 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:46.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:46.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:46.146 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:46.179 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:46.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:46.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:46.283 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:46.313 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:46.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:46.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:46.416 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:46.445 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:46.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:46.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:46.547 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:46.575 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:46.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:46.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:46.678 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:46.708 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:46.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:46.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:46.811 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:46.834 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:46.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:46.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:46.939 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:46.964 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:46.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:46.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:47.068 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:47.081 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:46 smithi194 bash[92666]: cluster 2023-10-09T05:13:45.865191+0000 mgr.y (mgr.25277) 757 : cluster 0 pgmap v748: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 3.7 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:47.100 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:47.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:47.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:47.121 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:46 smithi133 bash[105589]: cluster 2023-10-09T05:13:45.865191+0000 mgr.y (mgr.25277) 757 : cluster 0 pgmap v748: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 3.7 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:47.121 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:46 smithi133 bash[102858]: cluster 2023-10-09T05:13:45.865191+0000 mgr.y (mgr.25277) 757 : cluster 0 pgmap v748: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 3.7 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:47.205 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:47.234 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:47.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:47.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:47.337 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:47.364 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:47.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:47.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:47.467 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:47.495 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:47.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:47.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:47.599 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:47.621 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:47.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:47.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:47.709 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:13:47.709 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:13:47.724 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:47.753 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:47.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:47.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:47.858 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:47.890 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:47.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:47.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:47.994 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:48.016 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:48.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:48.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:48.120 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:48.150 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:48.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:48.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:48.254 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:48.276 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:48.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:48.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:48.383 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:48.407 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:48.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:48.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:48.511 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:48.517 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T05:13:48.534 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:48.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:48.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:48.638 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:48.664 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:48.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:48.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:48.768 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:48.791 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:48.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:48.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:48.894 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:48.914 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:48.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:48.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:49.017 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:49.046 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:49.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:49.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:49.149 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:49.175 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:49.176 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:48 smithi133 bash[102858]: cluster 2023-10-09T05:13:47.865971+0000 mgr.y (mgr.25277) 758 : cluster 0 pgmap v749: 89 pgs: 89 active+clean; 582 KiB data, 3.7 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:49.177 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:48 smithi133 bash[102858]: audit 2023-10-09T05:13:48.753588+0000 mon.a (mon.0) 969 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:13:49.177 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:48 smithi133 bash[105589]: cluster 2023-10-09T05:13:47.865971+0000 mgr.y (mgr.25277) 758 : cluster 0 pgmap v749: 89 pgs: 89 active+clean; 582 KiB data, 3.7 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:49.177 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:48 smithi133 bash[105589]: audit 2023-10-09T05:13:48.753588+0000 mon.a (mon.0) 969 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:13:49.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:49.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:49.278 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:49.307 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:49.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:49.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:49.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:48 smithi194 bash[92666]: cluster 2023-10-09T05:13:47.865971+0000 mgr.y (mgr.25277) 758 : cluster 0 pgmap v749: 89 pgs: 89 active+clean; 582 KiB data, 3.7 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:49.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:48 smithi194 bash[92666]: audit 2023-10-09T05:13:48.753588+0000 mon.a (mon.0) 969 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:13:49.411 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:49.437 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:49.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:49.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:49.540 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:49.561 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:49.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:49.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:49.664 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:49.689 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:49.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:49.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:49.794 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:49.825 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:49.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:49.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:49.928 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:49.958 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:49.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:49.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:50.062 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:50.095 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:50.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:50.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:50.198 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:50.225 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:50.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:50.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:50.328 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:50.355 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:50.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:50.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:50.459 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:50.492 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:50.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:50.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:50.596 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:50.618 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:50.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:50.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:50.721 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:50.755 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:50.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:50.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:50.859 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:50.881 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:50.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:50.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:50.984 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:51.007 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:51.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:51.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:51.110 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:51.124 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:50 smithi194 bash[92666]: cluster 2023-10-09T05:13:49.866725+0000 mgr.y (mgr.25277) 759 : cluster 0 pgmap v750: 89 pgs: 89 active+clean; 582 KiB data, 3.7 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:51.142 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:51.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:51.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:51.246 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:51.255 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:50 smithi133 bash[102858]: cluster 2023-10-09T05:13:49.866725+0000 mgr.y (mgr.25277) 759 : cluster 0 pgmap v750: 89 pgs: 89 active+clean; 582 KiB data, 3.7 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:51.255 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:50 smithi133 bash[105589]: cluster 2023-10-09T05:13:49.866725+0000 mgr.y (mgr.25277) 759 : cluster 0 pgmap v750: 89 pgs: 89 active+clean; 582 KiB data, 3.7 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:51.268 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:51.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:51.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:51.372 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:51.396 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:51.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:51.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:51.499 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:51.521 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:51.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:51.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:51.625 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:51.651 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:51.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:51.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:51.754 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:51.784 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:51.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:51.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:51.888 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:51.912 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:51.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:51.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:52.015 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:52.040 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:52.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:52.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:52.143 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:52.169 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:52.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:52.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:52.272 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:52.306 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:52.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:52.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:52.410 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:52.439 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:52.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:52.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:52.543 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:52.574 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:52.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:52.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:52.678 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:52.699 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:52.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:52.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:52.803 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:52.833 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:52.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:52.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:52.937 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:52.945 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:52 smithi133 bash[102858]: cluster 2023-10-09T05:13:51.867273+0000 mgr.y (mgr.25277) 760 : cluster 0 pgmap v751: 89 pgs: 89 active+clean; 582 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:52.945 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:52 smithi133 bash[105589]: cluster 2023-10-09T05:13:51.867273+0000 mgr.y (mgr.25277) 760 : cluster 0 pgmap v751: 89 pgs: 89 active+clean; 582 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:52.963 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:52.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:52.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:53.068 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:53.082 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:52 smithi194 bash[92666]: cluster 2023-10-09T05:13:51.867273+0000 mgr.y (mgr.25277) 760 : cluster 0 pgmap v751: 89 pgs: 89 active+clean; 582 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:53.102 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:53.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:53.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:53.206 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:53.241 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:53.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:53.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:53.298 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:13:53.345 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:53.370 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:53.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:53.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:53.473 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:53.496 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:53.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:53.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:53.599 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:53.620 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:53.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:53.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:53.723 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:53.752 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:53.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:53.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:53.855 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:53.879 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:53.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:53.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:53.982 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:54.010 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:54.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:54.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:54.113 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:54.140 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:54.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:54.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:54.243 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:54.270 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:54.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:54.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:54.373 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:54.385 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:13:54.394 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:54.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:54.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:54.409 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:13:54.435 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:13:54.457 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:13:54.483 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:13:54.497 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:54.509 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:13:54.531 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:54.535 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:13:54.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:54.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:54.559 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:13:54.561 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:13:54.561 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:13:54.561 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:13:54.561 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:13:54.562 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:13:54.562 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:13:54.562 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:13:54.562 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:13:54.637 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:54.660 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:54.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:54.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:54.763 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:54.776 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:13:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:13:54] "GET /metrics HTTP/1.1" 200 36397 "" "Prometheus/2.43.0" 2023-10-09T05:13:54.794 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:54.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:54.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:54.896 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:54.920 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:54.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:54.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:55.023 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:55.035 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:54 smithi133 bash[105589]: audit 2023-10-09T05:13:53.853944+0000 mon.a (mon.0) 970 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:13:55.036 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:54 smithi133 bash[105589]: cluster 2023-10-09T05:13:53.867770+0000 mgr.y (mgr.25277) 761 : cluster 0 pgmap v752: 89 pgs: 89 active+clean; 582 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:55.036 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:54 smithi133 bash[105589]: audit 2023-10-09T05:13:54.360752+0000 mon.a (mon.0) 971 : audit 0 from='client.? 172.21.15.133:0/3129308717' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:13:55.036 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:54 smithi133 bash[105589]: audit 2023-10-09T05:13:54.840232+0000 mon.a (mon.0) 972 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:13:55.036 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:54 smithi133 bash[105589]: audit 2023-10-09T05:13:54.848401+0000 mon.a (mon.0) 973 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:13:55.036 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:54 smithi133 bash[102858]: audit 2023-10-09T05:13:53.853944+0000 mon.a (mon.0) 970 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:13:55.036 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:54 smithi133 bash[102858]: cluster 2023-10-09T05:13:53.867770+0000 mgr.y (mgr.25277) 761 : cluster 0 pgmap v752: 89 pgs: 89 active+clean; 582 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:55.036 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:54 smithi133 bash[102858]: audit 2023-10-09T05:13:54.360752+0000 mon.a (mon.0) 971 : audit 0 from='client.? 172.21.15.133:0/3129308717' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:13:55.036 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:54 smithi133 bash[102858]: audit 2023-10-09T05:13:54.840232+0000 mon.a (mon.0) 972 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:13:55.037 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:54 smithi133 bash[102858]: audit 2023-10-09T05:13:54.848401+0000 mon.a (mon.0) 973 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:13:55.051 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:55.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:55.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:55.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:54 smithi194 bash[92666]: audit 2023-10-09T05:13:53.853944+0000 mon.a (mon.0) 970 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:13:55.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:54 smithi194 bash[92666]: cluster 2023-10-09T05:13:53.867770+0000 mgr.y (mgr.25277) 761 : cluster 0 pgmap v752: 89 pgs: 89 active+clean; 582 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:55.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:54 smithi194 bash[92666]: audit 2023-10-09T05:13:54.360752+0000 mon.a (mon.0) 971 : audit 0 from='client.? 172.21.15.133:0/3129308717' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:13:55.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:54 smithi194 bash[92666]: audit 2023-10-09T05:13:54.840232+0000 mon.a (mon.0) 972 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:13:55.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:54 smithi194 bash[92666]: audit 2023-10-09T05:13:54.848401+0000 mon.a (mon.0) 973 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:13:55.155 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:55.182 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:55.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:55.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:55.285 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:55.316 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:55.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:55.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:55.419 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:55.450 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:55.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:55.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:55.554 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:55.575 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:55.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:55.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:55.678 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:55.707 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:13:55.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:55.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:13:55.811 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:55.845 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:55.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:55.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:55.950 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:55.977 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:55.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:55.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:56.081 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:56.108 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:56.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:56.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:56.212 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:56.241 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:56.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:56.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:56.345 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:56.367 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:56.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:56.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:56.471 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:56.494 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:56.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:56.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:56.550 INFO:teuthology.orchestra.run.smithi194.stdout:{ 2023-10-09T05:13:56.551 INFO:teuthology.orchestra.run.smithi194.stdout: "size": 20, 2023-10-09T05:13:56.551 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 600, 2023-10-09T05:13:56.551 INFO:teuthology.orchestra.run.smithi194.stdout: "ops": [ 2023-10-09T05:13:56.551 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.551 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:2196 9.4 9:2eedd0a6:::benchmark_data_smithi133_153303_object2195:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e143)", 2023-10-09T05:13:56.551 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:09:30.066277+0000", 2023-10-09T05:13:56.551 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 266.479294912, 2023-10-09T05:13:56.551 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.0433124999999999, 2023-10-09T05:13:56.551 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:13:56.551 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:56.552 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:13:56.552 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:13:56.552 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:13:56.552 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 2196 2023-10-09T05:13:56.552 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.552 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:13:56.552 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.552 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:13:56.552 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.066277+0000", 2023-10-09T05:13:56.552 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.552 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.552 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.553 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:13:56.553 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.066277+0000", 2023-10-09T05:13:56.553 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.553 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.553 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.553 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:13:56.553 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.066275+0000", 2023-10-09T05:13:56.553 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999981 2023-10-09T05:13:56.553 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.553 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.553 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:13:56.553 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.066565+0000", 2023-10-09T05:13:56.554 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00028957100000000002 2023-10-09T05:13:56.554 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.554 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.554 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:13:56.554 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.066578+0000", 2023-10-09T05:13:56.554 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.2995e-05 2023-10-09T05:13:56.554 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.554 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.554 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:13:56.554 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.066587+0000", 2023-10-09T05:13:56.554 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.7919999999999998e-06 2023-10-09T05:13:56.554 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.555 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.555 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:13:56.555 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.066703+0000", 2023-10-09T05:13:56.555 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000116618 2023-10-09T05:13:56.555 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.555 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.555 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:13:56.555 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.066717+0000", 2023-10-09T05:13:56.555 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.3794999999999999e-05 2023-10-09T05:13:56.555 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.555 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.555 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:13:56.556 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:31.067485+0000", 2023-10-09T05:13:56.556 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.000768348 2023-10-09T05:13:56.556 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.556 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.556 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:13:56.556 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:31.067501+0000", 2023-10-09T05:13:56.556 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.5797000000000001e-05 2023-10-09T05:13:56.556 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.556 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.556 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:13:56.556 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:32.105177+0000", 2023-10-09T05:13:56.556 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.037675584 2023-10-09T05:13:56.557 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.557 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.557 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:13:56.557 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:32.105412+0000", 2023-10-09T05:13:56.557 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000235512 2023-10-09T05:13:56.557 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.557 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.557 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:13:56.557 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:32.105611+0000", 2023-10-09T05:13:56.557 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000198457 2023-10-09T05:13:56.557 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.557 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.558 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:56.558 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:32.108613+0000", 2023-10-09T05:13:56.558 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0030019389999999999 2023-10-09T05:13:56.558 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.558 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.558 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:56.558 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:32.109246+0000", 2023-10-09T05:13:56.558 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000633159 2023-10-09T05:13:56.558 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.558 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.558 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:13:56.558 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:32.109261+0000", 2023-10-09T05:13:56.559 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.5603999999999998e-05 2023-10-09T05:13:56.559 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.559 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.559 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:13:56.559 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:32.109335+0000", 2023-10-09T05:13:56.559 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.3367000000000004e-05 2023-10-09T05:13:56.559 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.559 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.559 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:13:56.559 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:32.109590+0000", 2023-10-09T05:13:56.559 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00025471899999999998 2023-10-09T05:13:56.559 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.560 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:13:56.560 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.560 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.560 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.560 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:4897 9.4 9:2d187545:::benchmark_data_smithi133_153303_object4896:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-10-09T05:13:56.560 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:09:35.118732+0000", 2023-10-09T05:13:56.560 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 261.42683969199999, 2023-10-09T05:13:56.560 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0038634909999999, 2023-10-09T05:13:56.560 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:13:56.560 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:56.560 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:13:56.560 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:13:56.561 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:13:56.561 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 4897 2023-10-09T05:13:56.561 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.561 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:13:56.561 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.561 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:13:56.561 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.118732+0000", 2023-10-09T05:13:56.561 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.561 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.561 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.561 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:13:56.561 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.118732+0000", 2023-10-09T05:13:56.561 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.562 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.562 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.562 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:13:56.562 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.118730+0000", 2023-10-09T05:13:56.562 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999971 2023-10-09T05:13:56.562 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.562 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.562 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:13:56.562 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.118892+0000", 2023-10-09T05:13:56.562 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00016267300000000001 2023-10-09T05:13:56.562 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.563 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.563 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:13:56.563 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.118920+0000", 2023-10-09T05:13:56.563 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.8157000000000002e-05 2023-10-09T05:13:56.563 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.563 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.563 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:13:56.563 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.118931+0000", 2023-10-09T05:13:56.563 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0868e-05 2023-10-09T05:13:56.563 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.563 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.563 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:13:56.564 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.119003+0000", 2023-10-09T05:13:56.564 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.2207999999999997e-05 2023-10-09T05:13:56.564 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.564 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.564 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:13:56.564 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.119014+0000", 2023-10-09T05:13:56.564 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0178e-05 2023-10-09T05:13:56.564 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.564 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.564 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:13:56.564 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.118631+0000", 2023-10-09T05:13:56.564 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.99961730100000001 2023-10-09T05:13:56.564 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.565 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.565 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:13:56.565 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.118809+0000", 2023-10-09T05:13:56.565 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000178355 2023-10-09T05:13:56.565 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.565 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.565 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:13:56.565 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.119018+0000", 2023-10-09T05:13:56.565 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00020862699999999999 2023-10-09T05:13:56.565 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.565 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.566 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:56.566 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.121767+0000", 2023-10-09T05:13:56.566 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0027487990000000001 2023-10-09T05:13:56.566 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.566 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.566 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:56.566 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.122455+0000", 2023-10-09T05:13:56.566 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000688639 2023-10-09T05:13:56.566 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.566 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.566 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:13:56.566 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.122467+0000", 2023-10-09T05:13:56.567 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.1192000000000001e-05 2023-10-09T05:13:56.567 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.567 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.567 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:13:56.567 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.122550+0000", 2023-10-09T05:13:56.567 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.3129999999999999e-05 2023-10-09T05:13:56.567 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.567 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.567 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:13:56.567 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.122596+0000", 2023-10-09T05:13:56.567 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.6044999999999997e-05 2023-10-09T05:13:56.568 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.568 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:13:56.568 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.568 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.568 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.568 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:4902 9.4 9:210be32c:::benchmark_data_smithi133_153303_object4901:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-10-09T05:13:56.568 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:09:35.121136+0000", 2023-10-09T05:13:56.568 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 261.42443636899998, 2023-10-09T05:13:56.568 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0021351489999999, 2023-10-09T05:13:56.568 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:13:56.568 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:56.568 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:13:56.569 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:13:56.569 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:13:56.569 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 4902 2023-10-09T05:13:56.569 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.569 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:13:56.569 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.569 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:13:56.569 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.121136+0000", 2023-10-09T05:13:56.569 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.569 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.569 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.569 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:13:56.570 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.121136+0000", 2023-10-09T05:13:56.570 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.570 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.570 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.570 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:13:56.570 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.121134+0000", 2023-10-09T05:13:56.570 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999981 2023-10-09T05:13:56.570 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.570 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.570 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:13:56.570 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.121197+0000", 2023-10-09T05:13:56.571 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.3800999999999994e-05 2023-10-09T05:13:56.571 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.571 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.571 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:13:56.571 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.121217+0000", 2023-10-09T05:13:56.571 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.9539e-05 2023-10-09T05:13:56.571 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.571 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.571 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:13:56.571 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.121226+0000", 2023-10-09T05:13:56.571 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.1859999999999998e-06 2023-10-09T05:13:56.572 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.572 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.572 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:13:56.572 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.121305+0000", 2023-10-09T05:13:56.572 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.9023999999999994e-05 2023-10-09T05:13:56.572 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.572 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.572 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:13:56.572 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.121319+0000", 2023-10-09T05:13:56.572 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.3909e-05 2023-10-09T05:13:56.572 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.573 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.573 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:13:56.573 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.120282+0000", 2023-10-09T05:13:56.573 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.99896322400000004 2023-10-09T05:13:56.573 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.573 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.573 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:13:56.573 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.120421+0000", 2023-10-09T05:13:56.573 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00013880699999999999 2023-10-09T05:13:56.573 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.573 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.574 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:13:56.574 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.120595+0000", 2023-10-09T05:13:56.574 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000174019 2023-10-09T05:13:56.574 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.574 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.574 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:13:56.574 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.122679+0000", 2023-10-09T05:13:56.574 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0020843120000000001 2023-10-09T05:13:56.574 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.574 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.574 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:56.574 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.122724+0000", 2023-10-09T05:13:56.575 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.4113000000000003e-05 2023-10-09T05:13:56.575 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.575 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.575 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:56.575 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.123184+0000", 2023-10-09T05:13:56.575 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00046050300000000002 2023-10-09T05:13:56.575 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.575 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.575 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:13:56.575 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.123232+0000", 2023-10-09T05:13:56.575 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.7618999999999999e-05 2023-10-09T05:13:56.575 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.576 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.576 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:13:56.576 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.123271+0000", 2023-10-09T05:13:56.576 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.9011999999999999e-05 2023-10-09T05:13:56.576 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.576 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:13:56.576 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.576 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.576 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.576 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:4913 9.4 9:252ad98e:::benchmark_data_smithi133_153303_object4912:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-10-09T05:13:56.576 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:09:35.126406+0000", 2023-10-09T05:13:56.577 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 261.41916624100003, 2023-10-09T05:13:56.577 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.99875375, 2023-10-09T05:13:56.577 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:13:56.577 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:56.577 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:13:56.577 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:13:56.577 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:13:56.577 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 4913 2023-10-09T05:13:56.577 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.577 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:13:56.577 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.578 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:13:56.578 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.126406+0000", 2023-10-09T05:13:56.578 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.578 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.578 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.578 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:13:56.578 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.126406+0000", 2023-10-09T05:13:56.578 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.578 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.578 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.578 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:13:56.579 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.126404+0000", 2023-10-09T05:13:56.579 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:13:56.579 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.579 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.579 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:13:56.579 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.126470+0000", 2023-10-09T05:13:56.579 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.5983999999999997e-05 2023-10-09T05:13:56.579 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.579 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.579 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:13:56.579 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.126497+0000", 2023-10-09T05:13:56.579 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.6721999999999999e-05 2023-10-09T05:13:56.580 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.580 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.580 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:13:56.580 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.126506+0000", 2023-10-09T05:13:56.580 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.0080000000000008e-06 2023-10-09T05:13:56.580 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.580 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.580 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:13:56.580 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.126567+0000", 2023-10-09T05:13:56.580 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.0534e-05 2023-10-09T05:13:56.580 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.580 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.581 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:13:56.581 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.126576+0000", 2023-10-09T05:13:56.581 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.3340000000000008e-06 2023-10-09T05:13:56.581 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.581 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.581 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:13:56.581 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.121062+0000", 2023-10-09T05:13:56.581 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.99448622799999997 2023-10-09T05:13:56.581 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.581 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.581 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:13:56.581 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.121188+0000", 2023-10-09T05:13:56.582 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00012550500000000001 2023-10-09T05:13:56.582 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.582 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.582 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:13:56.582 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.121336+0000", 2023-10-09T05:13:56.582 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00014797599999999999 2023-10-09T05:13:56.582 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.582 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.582 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:13:56.582 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.122685+0000", 2023-10-09T05:13:56.582 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.001349486 2023-10-09T05:13:56.582 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.583 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.583 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:56.583 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.123156+0000", 2023-10-09T05:13:56.583 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000470483 2023-10-09T05:13:56.583 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.583 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.583 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:56.583 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.125041+0000", 2023-10-09T05:13:56.583 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0018853640000000001 2023-10-09T05:13:56.583 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.583 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.584 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:13:56.584 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.125110+0000", 2023-10-09T05:13:56.584 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.9565000000000002e-05 2023-10-09T05:13:56.584 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.584 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.584 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:13:56.584 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.125159+0000", 2023-10-09T05:13:56.584 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.8992000000000001e-05 2023-10-09T05:13:56.584 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.584 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:13:56.584 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.585 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.585 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.585 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:4914 9.4 9:28522720:::benchmark_data_smithi133_153303_object4913:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-10-09T05:13:56.585 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:09:35.126907+0000", 2023-10-09T05:13:56.585 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 261.41866468799998, 2023-10-09T05:13:56.585 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.99838625000000003, 2023-10-09T05:13:56.585 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:13:56.585 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:56.585 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:13:56.585 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:13:56.585 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:13:56.585 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 4914 2023-10-09T05:13:56.586 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.586 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:13:56.586 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.586 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:13:56.586 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.126907+0000", 2023-10-09T05:13:56.586 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.586 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.586 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.586 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:13:56.586 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.126907+0000", 2023-10-09T05:13:56.586 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.587 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.587 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.587 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:13:56.587 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.126907+0000", 2023-10-09T05:13:56.587 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:13:56.587 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.587 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.587 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:13:56.587 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.126976+0000", 2023-10-09T05:13:56.587 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.9214999999999994e-05 2023-10-09T05:13:56.587 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.587 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.588 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:13:56.588 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.126989+0000", 2023-10-09T05:13:56.588 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.3461e-05 2023-10-09T05:13:56.588 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.588 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.588 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:13:56.588 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.126996+0000", 2023-10-09T05:13:56.588 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.5509999999999996e-06 2023-10-09T05:13:56.588 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.588 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.588 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:13:56.588 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.127038+0000", 2023-10-09T05:13:56.589 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.2357000000000001e-05 2023-10-09T05:13:56.589 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.589 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.589 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:13:56.589 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.127045+0000", 2023-10-09T05:13:56.589 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.1219999999999997e-06 2023-10-09T05:13:56.589 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.589 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.589 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:13:56.590 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.121801+0000", 2023-10-09T05:13:56.590 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.99475603499999998 2023-10-09T05:13:56.590 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.590 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.591 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:13:56.591 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.121896+0000", 2023-10-09T05:13:56.591 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.4314000000000005e-05 2023-10-09T05:13:56.591 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.591 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.591 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:13:56.591 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.122062+0000", 2023-10-09T05:13:56.591 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00016600400000000001 2023-10-09T05:13:56.591 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.592 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.592 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:13:56.592 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.122837+0000", 2023-10-09T05:13:56.592 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00077533099999999998 2023-10-09T05:13:56.592 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.592 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.592 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:56.592 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.124776+0000", 2023-10-09T05:13:56.592 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.001939315 2023-10-09T05:13:56.592 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.592 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.593 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:56.593 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.125209+0000", 2023-10-09T05:13:56.593 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00043252699999999999 2023-10-09T05:13:56.593 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.593 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.593 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:13:56.593 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.125253+0000", 2023-10-09T05:13:56.593 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.4391999999999998e-05 2023-10-09T05:13:56.593 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.593 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.593 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:13:56.594 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.125294+0000", 2023-10-09T05:13:56.594 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.0361999999999997e-05 2023-10-09T05:13:56.594 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.594 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:13:56.594 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.594 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.594 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.594 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:26395 9.4 9:28614660:::benchmark_data_smithi133_153303_object26394:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T05:13:56.594 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:09:46.570080+0000", 2023-10-09T05:13:56.594 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 249.97549154500001, 2023-10-09T05:13:56.594 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.12138963, 2023-10-09T05:13:56.595 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:13:56.595 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:56.595 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:13:56.595 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:13:56.595 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:13:56.595 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 26395 2023-10-09T05:13:56.595 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.595 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:13:56.595 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.595 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:13:56.595 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:46.570080+0000", 2023-10-09T05:13:56.595 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.595 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.596 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.596 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:13:56.596 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:46.570080+0000", 2023-10-09T05:13:56.596 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.596 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.596 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.596 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:13:56.596 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:46.570061+0000", 2023-10-09T05:13:56.596 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.99998 2023-10-09T05:13:56.596 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.596 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.596 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:13:56.597 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:46.570128+0000", 2023-10-09T05:13:56.597 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.7518e-05 2023-10-09T05:13:56.597 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.597 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.597 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:13:56.597 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:46.570140+0000", 2023-10-09T05:13:56.597 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.1688000000000001e-05 2023-10-09T05:13:56.597 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.597 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.597 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:13:56.597 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:46.570146+0000", 2023-10-09T05:13:56.598 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.2659999999999998e-06 2023-10-09T05:13:56.598 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.598 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.598 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:13:56.598 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:46.570186+0000", 2023-10-09T05:13:56.598 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.0017e-05 2023-10-09T05:13:56.598 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.598 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.598 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:13:56.598 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:46.570313+0000", 2023-10-09T05:13:56.598 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000126644 2023-10-09T05:13:56.598 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.599 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.599 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:13:56.599 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:46.570673+0000", 2023-10-09T05:13:56.599 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000360095 2023-10-09T05:13:56.599 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.599 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.599 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:13:56.599 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:46.573304+0000", 2023-10-09T05:13:56.599 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0026311780000000001 2023-10-09T05:13:56.599 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.599 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.599 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:56.600 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:46.574282+0000", 2023-10-09T05:13:56.600 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00097822300000000007 2023-10-09T05:13:56.600 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.600 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.600 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:56.600 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:46.691260+0000", 2023-10-09T05:13:56.600 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.11697772100000001 2023-10-09T05:13:56.600 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.600 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.600 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:13:56.600 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:46.691358+0000", 2023-10-09T05:13:56.601 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.7798000000000001e-05 2023-10-09T05:13:56.601 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.601 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.601 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:13:56.601 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:46.691470+0000", 2023-10-09T05:13:56.601 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000112281 2023-10-09T05:13:56.601 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.601 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:13:56.601 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.601 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.601 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.601 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:26408 9.4 9:2f9e70f5:::benchmark_data_smithi133_153303_object26407:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T05:13:56.602 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:09:46.579175+0000", 2023-10-09T05:13:56.602 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 249.966396465, 2023-10-09T05:13:56.602 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.118815432, 2023-10-09T05:13:56.602 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:13:56.602 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:56.602 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:13:56.602 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:13:56.602 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:13:56.602 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 26408 2023-10-09T05:13:56.602 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.602 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:13:56.603 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.603 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:13:56.603 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:46.579175+0000", 2023-10-09T05:13:56.603 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.603 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.603 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.603 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:13:56.603 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:46.579175+0000", 2023-10-09T05:13:56.603 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.603 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.603 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.604 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:13:56.604 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:46.579174+0000", 2023-10-09T05:13:56.604 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:13:56.604 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.604 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.604 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:13:56.604 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:46.579281+0000", 2023-10-09T05:13:56.604 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000107116 2023-10-09T05:13:56.604 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.604 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.604 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:13:56.605 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:46.579294+0000", 2023-10-09T05:13:56.605 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.2347999999999999e-05 2023-10-09T05:13:56.605 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.605 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.605 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:13:56.605 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:46.579299+0000", 2023-10-09T05:13:56.605 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.1919999999999996e-06 2023-10-09T05:13:56.605 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.605 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.605 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:13:56.605 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:46.579336+0000", 2023-10-09T05:13:56.606 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.7389999999999999e-05 2023-10-09T05:13:56.606 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.606 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.606 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:13:56.606 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:46.579443+0000", 2023-10-09T05:13:56.606 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00010659500000000001 2023-10-09T05:13:56.606 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.606 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.606 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:13:56.606 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:46.579539+0000", 2023-10-09T05:13:56.606 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.6268999999999996e-05 2023-10-09T05:13:56.607 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.607 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.607 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:13:56.607 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:46.580550+0000", 2023-10-09T05:13:56.607 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.001011407 2023-10-09T05:13:56.607 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.607 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.607 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:56.607 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:46.696879+0000", 2023-10-09T05:13:56.607 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.116328817 2023-10-09T05:13:56.607 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.607 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.607 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:56.608 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:46.697800+0000", 2023-10-09T05:13:56.608 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00092076600000000005 2023-10-09T05:13:56.608 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.608 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.608 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:13:56.608 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:46.697911+0000", 2023-10-09T05:13:56.608 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000110616 2023-10-09T05:13:56.608 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.608 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.608 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:13:56.608 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:46.697991+0000", 2023-10-09T05:13:56.608 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.0222999999999994e-05 2023-10-09T05:13:56.609 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.609 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:13:56.609 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.609 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.609 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.609 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:27317 9.4 9:270a37db:::benchmark_data_smithi133_153303_object27316:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T05:13:56.609 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:09:47.183245+0000", 2023-10-09T05:13:56.609 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 249.36232685900001, 2023-10-09T05:13:56.609 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.100051318, 2023-10-09T05:13:56.609 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:13:56.609 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:56.609 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:13:56.610 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:13:56.610 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:13:56.610 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 27317 2023-10-09T05:13:56.610 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.610 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:13:56.610 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.610 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:13:56.610 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:47.183245+0000", 2023-10-09T05:13:56.610 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.610 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.611 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.611 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:13:56.611 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:47.183245+0000", 2023-10-09T05:13:56.611 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.611 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.611 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.611 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:13:56.611 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:47.183244+0000", 2023-10-09T05:13:56.611 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:13:56.611 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.611 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.612 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:13:56.612 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:47.183350+0000", 2023-10-09T05:13:56.612 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000106325 2023-10-09T05:13:56.612 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.612 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.612 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:13:56.612 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:47.183365+0000", 2023-10-09T05:13:56.612 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.4802e-05 2023-10-09T05:13:56.612 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.612 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.612 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:13:56.612 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:47.183371+0000", 2023-10-09T05:13:56.613 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.4400000000000002e-06 2023-10-09T05:13:56.613 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.613 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.613 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:13:56.613 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:47.183408+0000", 2023-10-09T05:13:56.613 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.7083999999999999e-05 2023-10-09T05:13:56.613 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.613 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.613 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:13:56.613 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:47.183565+0000", 2023-10-09T05:13:56.613 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000156804 2023-10-09T05:13:56.614 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.614 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.614 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:13:56.614 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:47.183738+0000", 2023-10-09T05:13:56.614 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00017301400000000001 2023-10-09T05:13:56.614 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.614 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.614 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:13:56.614 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:47.186815+0000", 2023-10-09T05:13:56.614 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0030772040000000001 2023-10-09T05:13:56.614 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.614 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.614 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:56.615 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:47.188333+0000", 2023-10-09T05:13:56.615 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.001517914 2023-10-09T05:13:56.615 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.615 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.615 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:56.615 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:47.283117+0000", 2023-10-09T05:13:56.615 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.094783706999999995 2023-10-09T05:13:56.615 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.615 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.615 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:13:56.615 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:47.283231+0000", 2023-10-09T05:13:56.615 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00011380800000000001 2023-10-09T05:13:56.616 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.616 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.616 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:13:56.616 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:47.283296+0000", 2023-10-09T05:13:56.616 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.5698999999999996e-05 2023-10-09T05:13:56.616 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.616 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:13:56.616 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.616 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.616 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.616 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:27326 9.4 9:2b74f445:::benchmark_data_smithi133_153303_object27325:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T05:13:56.616 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:09:47.189712+0000", 2023-10-09T05:13:56.617 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 249.355859796, 2023-10-09T05:13:56.617 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.096900072000000004, 2023-10-09T05:13:56.617 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:13:56.617 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:56.617 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:13:56.617 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:13:56.617 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:13:56.617 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 27326 2023-10-09T05:13:56.617 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.617 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:13:56.617 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.617 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:13:56.617 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:47.189712+0000", 2023-10-09T05:13:56.618 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.618 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.618 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.618 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:13:56.618 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:47.189712+0000", 2023-10-09T05:13:56.618 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.618 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.618 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.618 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:13:56.618 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:47.189710+0000", 2023-10-09T05:13:56.619 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999981 2023-10-09T05:13:56.619 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.619 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.619 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:13:56.619 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:47.189778+0000", 2023-10-09T05:13:56.619 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.7760000000000002e-05 2023-10-09T05:13:56.619 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.619 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.619 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:13:56.619 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:47.189797+0000", 2023-10-09T05:13:56.619 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.9120999999999999e-05 2023-10-09T05:13:56.620 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.620 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.620 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:13:56.620 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:47.189805+0000", 2023-10-09T05:13:56.620 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.3510000000000002e-06 2023-10-09T05:13:56.620 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.620 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.620 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:13:56.620 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:47.189857+0000", 2023-10-09T05:13:56.620 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.2042000000000001e-05 2023-10-09T05:13:56.620 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.621 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.621 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:13:56.621 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:47.190010+0000", 2023-10-09T05:13:56.621 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00015275400000000001 2023-10-09T05:13:56.621 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.621 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.621 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:13:56.621 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:47.190185+0000", 2023-10-09T05:13:56.621 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00017491699999999999 2023-10-09T05:13:56.621 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.621 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.622 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:13:56.622 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:47.191429+0000", 2023-10-09T05:13:56.622 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0012435899999999999 2023-10-09T05:13:56.622 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.622 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.622 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:56.622 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:47.192768+0000", 2023-10-09T05:13:56.622 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0013394349999999999 2023-10-09T05:13:56.622 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.622 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.622 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:13:56.623 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:47.286423+0000", 2023-10-09T05:13:56.623 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.093655156000000003 2023-10-09T05:13:56.623 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.623 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.623 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:13:56.623 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:47.286522+0000", 2023-10-09T05:13:56.623 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.8852999999999997e-05 2023-10-09T05:13:56.623 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.623 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.623 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:13:56.623 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:47.286612+0000", 2023-10-09T05:13:56.623 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.0163999999999999e-05 2023-10-09T05:13:56.624 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.624 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:13:56.624 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.624 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.624 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.624 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_repop(client.34845.0:117232 9.4 e158/156 9:2f946786:::benchmark_data_smithi133_153303_object27263:head v 158'7361, mlcod=158'7359)", 2023-10-09T05:13:56.624 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:11:38.331398+0000", 2023-10-09T05:13:56.624 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 138.21417345699999, 2023-10-09T05:13:56.624 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.025065258999999999, 2023-10-09T05:13:56.624 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:13:56.624 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:56.624 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:13:56.625 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.625 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:13:56.625 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:11:38.331398+0000", 2023-10-09T05:13:56.625 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.625 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.625 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.625 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:13:56.625 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:11:38.331398+0000", 2023-10-09T05:13:56.625 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.625 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.625 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.625 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:13:56.626 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:11:38.331398+0000", 2023-10-09T05:13:56.626 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999995 2023-10-09T05:13:56.626 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.626 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.626 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:13:56.626 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:11:38.331404+0000", 2023-10-09T05:13:56.626 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.5960000000000003e-06 2023-10-09T05:13:56.626 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.626 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.626 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:13:56.626 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:11:38.331420+0000", 2023-10-09T05:13:56.627 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.6011e-05 2023-10-09T05:13:56.627 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.627 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.627 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:13:56.627 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:11:38.331425+0000", 2023-10-09T05:13:56.627 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.1930000000000001e-06 2023-10-09T05:13:56.627 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.627 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.627 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:13:56.627 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:11:38.334539+0000", 2023-10-09T05:13:56.627 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0031138149999999998 2023-10-09T05:13:56.627 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.628 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.628 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:13:56.628 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:11:38.334566+0000", 2023-10-09T05:13:56.628 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.7382e-05 2023-10-09T05:13:56.628 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.628 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.628 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:13:56.628 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:11:38.356420+0000", 2023-10-09T05:13:56.628 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.021853912 2023-10-09T05:13:56.628 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.628 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.628 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:13:56.629 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:11:38.356464+0000", 2023-10-09T05:13:56.629 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.3860999999999999e-05 2023-10-09T05:13:56.629 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.629 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:13:56.629 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.629 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.629 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.629 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_repop(client.34845.0:141489 9.c e165/139 9:36c2c6f7:::benchmark_data_smithi133_153303_object51520:head v 165'9044, mlcod=165'9042)", 2023-10-09T05:13:56.629 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:12:12.877153+0000", 2023-10-09T05:13:56.629 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 103.66841875900001, 2023-10-09T05:13:56.629 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.031841207000000003, 2023-10-09T05:13:56.629 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:13:56.630 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:56.630 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:13:56.630 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.630 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:13:56.630 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:12.877153+0000", 2023-10-09T05:13:56.630 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.630 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.630 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.630 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:13:56.630 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:12.877153+0000", 2023-10-09T05:13:56.630 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.630 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.631 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.631 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:13:56.631 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:12.877152+0000", 2023-10-09T05:13:56.631 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:13:56.631 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.631 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.631 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:13:56.631 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:12.877163+0000", 2023-10-09T05:13:56.631 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.1073000000000001e-05 2023-10-09T05:13:56.631 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.631 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.632 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:13:56.632 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:12.877180+0000", 2023-10-09T05:13:56.632 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.6174000000000001e-05 2023-10-09T05:13:56.632 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.632 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.632 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:13:56.632 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:12.877187+0000", 2023-10-09T05:13:56.632 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.0709999999999997e-06 2023-10-09T05:13:56.632 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.632 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.632 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:13:56.632 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:12.886544+0000", 2023-10-09T05:13:56.632 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0093577439999999994 2023-10-09T05:13:56.633 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.633 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.633 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:13:56.633 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:12.886573+0000", 2023-10-09T05:13:56.633 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.8690000000000001e-05 2023-10-09T05:13:56.633 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.633 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.633 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:13:56.633 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:12.908869+0000", 2023-10-09T05:13:56.633 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.022295691999999999 2023-10-09T05:13:56.633 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.634 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.634 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:13:56.634 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:12.908994+0000", 2023-10-09T05:13:56.634 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000125616 2023-10-09T05:13:56.634 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.634 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:13:56.634 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.634 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.634 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.634 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_repop(client.34845.0:141490 9.c e165/139 9:3c5522ff:::benchmark_data_smithi133_153303_object51521:head v 165'9045, mlcod=165'9042)", 2023-10-09T05:13:56.634 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:12:12.880771+0000", 2023-10-09T05:13:56.635 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 103.66480101400001, 2023-10-09T05:13:56.635 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.028339164, 2023-10-09T05:13:56.635 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:13:56.635 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:56.635 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:13:56.635 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.635 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:13:56.635 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:12.880771+0000", 2023-10-09T05:13:56.635 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.635 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.635 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.636 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:13:56.636 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:12.880771+0000", 2023-10-09T05:13:56.636 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.636 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.636 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.636 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:13:56.636 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:12.880770+0000", 2023-10-09T05:13:56.636 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:13:56.636 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.636 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.636 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:13:56.636 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:12.880776+0000", 2023-10-09T05:13:56.636 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.1029999999999998e-06 2023-10-09T05:13:56.636 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.637 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.637 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:13:56.637 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:12.880805+0000", 2023-10-09T05:13:56.637 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.8649999999999998e-05 2023-10-09T05:13:56.637 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.637 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.637 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:13:56.637 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:12.880809+0000", 2023-10-09T05:13:56.637 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.5329999999999996e-06 2023-10-09T05:13:56.637 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.637 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.637 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:13:56.637 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:12.891194+0000", 2023-10-09T05:13:56.638 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.010384632 2023-10-09T05:13:56.638 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.638 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.638 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:13:56.638 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:12.891220+0000", 2023-10-09T05:13:56.638 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.5783999999999999e-05 2023-10-09T05:13:56.638 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.638 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.638 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:13:56.638 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:12.908998+0000", 2023-10-09T05:13:56.638 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.017777674 2023-10-09T05:13:56.638 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.639 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.639 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:13:56.639 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:12.909110+0000", 2023-10-09T05:13:56.639 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00011254700000000001 2023-10-09T05:13:56.639 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.639 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:13:56.639 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.639 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.639 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.639 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_repop(client.34845.0:141494 9.c e165/139 9:34b2f4ad:::benchmark_data_smithi133_153303_object51525:head v 165'9046, mlcod=165'9042)", 2023-10-09T05:13:56.639 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:12:12.884531+0000", 2023-10-09T05:13:56.639 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 103.661040535, 2023-10-09T05:13:56.639 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.024713329999999999, 2023-10-09T05:13:56.640 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:13:56.640 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:56.640 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:13:56.640 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.640 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:13:56.640 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:12.884531+0000", 2023-10-09T05:13:56.640 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.640 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.640 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.640 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:13:56.640 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:12.884531+0000", 2023-10-09T05:13:56.641 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.641 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.641 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.641 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:13:56.641 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:12.884531+0000", 2023-10-09T05:13:56.641 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:13:56.641 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.641 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.641 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:13:56.641 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:12.884543+0000", 2023-10-09T05:13:56.641 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.2094e-05 2023-10-09T05:13:56.641 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.641 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.641 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:13:56.642 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:12.884561+0000", 2023-10-09T05:13:56.642 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.8229000000000001e-05 2023-10-09T05:13:56.642 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.642 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.642 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:13:56.642 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:12.884569+0000", 2023-10-09T05:13:56.642 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.2579999999999995e-06 2023-10-09T05:13:56.642 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.642 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.642 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:13:56.642 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:12.894925+0000", 2023-10-09T05:13:56.643 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.010356295 2023-10-09T05:13:56.643 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.643 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.643 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:13:56.643 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:12.894987+0000", 2023-10-09T05:13:56.643 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.1999000000000001e-05 2023-10-09T05:13:56.643 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.643 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.643 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:13:56.643 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:12.909115+0000", 2023-10-09T05:13:56.643 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.014127129 2023-10-09T05:13:56.643 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.644 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.644 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:13:56.644 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:12.909245+0000", 2023-10-09T05:13:56.644 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00013015100000000001 2023-10-09T05:13:56.644 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.644 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:13:56.644 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.644 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.644 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.644 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_repop(client.34845.0:148323 9.c e165/139 9:30866c6c:::benchmark_data_smithi133_153303_object58354:head v 165'9463, mlcod=165'9461)", 2023-10-09T05:13:56.644 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:12:20.261761+0000", 2023-10-09T05:13:56.644 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 96.283810931000005, 2023-10-09T05:13:56.645 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.026243882, 2023-10-09T05:13:56.645 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:13:56.645 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:56.645 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:13:56.645 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.645 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:13:56.645 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.261761+0000", 2023-10-09T05:13:56.645 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.645 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.645 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.645 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:13:56.645 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.261761+0000", 2023-10-09T05:13:56.645 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.646 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.646 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.646 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:13:56.646 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.261760+0000", 2023-10-09T05:13:56.646 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:13:56.646 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.646 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.646 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:13:56.646 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.261770+0000", 2023-10-09T05:13:56.646 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0074e-05 2023-10-09T05:13:56.646 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.646 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.647 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:13:56.647 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.261784+0000", 2023-10-09T05:13:56.647 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.3678e-05 2023-10-09T05:13:56.647 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.647 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.647 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:13:56.647 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.261789+0000", 2023-10-09T05:13:56.647 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.3419999999999998e-06 2023-10-09T05:13:56.647 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.647 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.647 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:13:56.647 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.261843+0000", 2023-10-09T05:13:56.647 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.4061999999999998e-05 2023-10-09T05:13:56.648 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.648 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.648 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:13:56.648 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.261864+0000", 2023-10-09T05:13:56.648 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.0213000000000001e-05 2023-10-09T05:13:56.648 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.648 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.648 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:13:56.648 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.287955+0000", 2023-10-09T05:13:56.648 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.026091692999999999 2023-10-09T05:13:56.648 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.648 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.649 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:13:56.649 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.288005+0000", 2023-10-09T05:13:56.649 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.9632000000000002e-05 2023-10-09T05:13:56.649 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.649 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:13:56.649 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.649 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.649 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.649 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_repop(client.34845.0:148327 9.c e165/139 9:30870a9e:::benchmark_data_smithi133_153303_object58358:head v 165'9464, mlcod=165'9461)", 2023-10-09T05:13:56.649 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:12:20.262332+0000", 2023-10-09T05:13:56.649 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 96.283240077000002, 2023-10-09T05:13:56.650 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.029037604000000002, 2023-10-09T05:13:56.650 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:13:56.650 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:56.650 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:13:56.650 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.650 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:13:56.650 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.262332+0000", 2023-10-09T05:13:56.650 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.650 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.650 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.651 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:13:56.651 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.262332+0000", 2023-10-09T05:13:56.651 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.651 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.651 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.651 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:13:56.651 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.262331+0000", 2023-10-09T05:13:56.651 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999995 2023-10-09T05:13:56.651 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.651 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.651 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:13:56.651 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.262339+0000", 2023-10-09T05:13:56.652 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.4490000000000002e-06 2023-10-09T05:13:56.652 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.652 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.652 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:13:56.652 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.262346+0000", 2023-10-09T05:13:56.652 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.8970000000000002e-06 2023-10-09T05:13:56.652 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.652 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.652 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:13:56.652 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.262349+0000", 2023-10-09T05:13:56.653 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.631e-06 2023-10-09T05:13:56.653 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.653 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.653 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:13:56.653 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.268137+0000", 2023-10-09T05:13:56.653 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0057878340000000004 2023-10-09T05:13:56.653 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.653 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.653 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:13:56.653 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.268166+0000", 2023-10-09T05:13:56.653 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.8962000000000002e-05 2023-10-09T05:13:56.654 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.654 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.654 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:13:56.654 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.291334+0000", 2023-10-09T05:13:56.654 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.023167918999999999 2023-10-09T05:13:56.654 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.654 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.654 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:13:56.654 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.291369+0000", 2023-10-09T05:13:56.654 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.5496e-05 2023-10-09T05:13:56.654 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.655 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:13:56.655 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.655 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.655 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.655 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_repop(client.34845.0:148328 9.c e165/139 9:37696802:::benchmark_data_smithi133_153303_object58359:head v 165'9465, mlcod=165'9461)", 2023-10-09T05:13:56.655 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:12:20.267438+0000", 2023-10-09T05:13:56.655 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 96.278134159000004, 2023-10-09T05:13:56.655 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.024232432000000002, 2023-10-09T05:13:56.655 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:13:56.655 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:56.655 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:13:56.656 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.656 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:13:56.656 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.267438+0000", 2023-10-09T05:13:56.656 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.656 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.656 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.656 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:13:56.656 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.267438+0000", 2023-10-09T05:13:56.656 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.656 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.656 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.657 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:13:56.657 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.267437+0000", 2023-10-09T05:13:56.657 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:13:56.657 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.657 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.657 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:13:56.657 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.267450+0000", 2023-10-09T05:13:56.657 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.3376e-05 2023-10-09T05:13:56.657 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.657 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.657 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:13:56.657 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.267470+0000", 2023-10-09T05:13:56.658 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.9647999999999999e-05 2023-10-09T05:13:56.658 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.658 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.658 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:13:56.658 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.267477+0000", 2023-10-09T05:13:56.658 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.7680000000000008e-06 2023-10-09T05:13:56.658 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.658 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.658 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:13:56.658 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.272264+0000", 2023-10-09T05:13:56.659 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0047865499999999997 2023-10-09T05:13:56.659 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.659 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.659 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:13:56.659 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.272288+0000", 2023-10-09T05:13:56.659 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.3626999999999999e-05 2023-10-09T05:13:56.659 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.659 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.659 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:13:56.659 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.291373+0000", 2023-10-09T05:13:56.659 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.019085622999999999 2023-10-09T05:13:56.659 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.660 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.660 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:13:56.660 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.291670+0000", 2023-10-09T05:13:56.660 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000297013 2023-10-09T05:13:56.660 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.660 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:13:56.660 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.660 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.660 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.660 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_repop(client.34845.0:148330 9.c e165/139 9:3ee70f95:::benchmark_data_smithi133_153303_object58361:head v 165'9466, mlcod=165'9461)", 2023-10-09T05:13:56.660 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:12:20.269340+0000", 2023-10-09T05:13:56.661 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 96.276231784000004, 2023-10-09T05:13:56.661 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.022494292999999999, 2023-10-09T05:13:56.661 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:13:56.661 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:56.661 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:13:56.661 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.661 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:13:56.661 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.269340+0000", 2023-10-09T05:13:56.661 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.661 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.661 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.662 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:13:56.662 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.269340+0000", 2023-10-09T05:13:56.662 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.662 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.662 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.662 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:13:56.662 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.269339+0000", 2023-10-09T05:13:56.662 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:13:56.662 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.662 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.662 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:13:56.663 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.269347+0000", 2023-10-09T05:13:56.663 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.5499999999999997e-06 2023-10-09T05:13:56.663 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.663 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.663 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:13:56.663 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.269360+0000", 2023-10-09T05:13:56.663 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.3275000000000001e-05 2023-10-09T05:13:56.663 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.663 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.663 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:13:56.663 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.269366+0000", 2023-10-09T05:13:56.663 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.896e-06 2023-10-09T05:13:56.663 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.664 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.664 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:13:56.664 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.276002+0000", 2023-10-09T05:13:56.664 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0066362260000000003 2023-10-09T05:13:56.664 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.664 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.664 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:13:56.664 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.276058+0000", 2023-10-09T05:13:56.665 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.6022e-05 2023-10-09T05:13:56.665 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.665 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.665 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:13:56.665 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.291677+0000", 2023-10-09T05:13:56.665 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.015618622 2023-10-09T05:13:56.665 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.665 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.665 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:13:56.665 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.291834+0000", 2023-10-09T05:13:56.666 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00015758600000000001 2023-10-09T05:13:56.666 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.666 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:13:56.666 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.666 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.666 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.666 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_repop(client.34845.0:148337 9.14 e165/160 9:2beca73e:::benchmark_data_smithi133_153303_object58368:head v 165'8385, mlcod=165'8382)", 2023-10-09T05:13:56.666 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:12:20.270518+0000", 2023-10-09T05:13:56.666 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 96.275053678000006, 2023-10-09T05:13:56.666 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.021477942, 2023-10-09T05:13:56.666 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:13:56.667 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:56.667 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:13:56.667 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.667 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:13:56.667 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.270518+0000", 2023-10-09T05:13:56.667 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.667 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.667 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.667 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:13:56.667 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.270518+0000", 2023-10-09T05:13:56.667 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.667 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.668 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.668 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:13:56.668 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.270517+0000", 2023-10-09T05:13:56.668 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:13:56.668 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.668 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.668 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:13:56.668 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.270525+0000", 2023-10-09T05:13:56.668 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.9789999999999993e-06 2023-10-09T05:13:56.668 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.668 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.669 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:13:56.669 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.270542+0000", 2023-10-09T05:13:56.669 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.6237000000000001e-05 2023-10-09T05:13:56.669 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.669 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.669 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:13:56.669 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.270548+0000", 2023-10-09T05:13:56.669 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.981e-06 2023-10-09T05:13:56.669 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.669 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.669 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:13:56.670 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.279776+0000", 2023-10-09T05:13:56.670 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0092286680000000006 2023-10-09T05:13:56.670 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.670 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.670 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:13:56.670 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.279799+0000", 2023-10-09T05:13:56.670 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.2903000000000001e-05 2023-10-09T05:13:56.670 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.670 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.670 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:13:56.670 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.291837+0000", 2023-10-09T05:13:56.670 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.012038085 2023-10-09T05:13:56.671 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.671 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.671 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:13:56.671 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.291996+0000", 2023-10-09T05:13:56.671 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00015889699999999999 2023-10-09T05:13:56.671 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.671 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:13:56.671 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.671 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.671 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.671 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_repop(client.34845.0:148335 9.c e165/139 9:355342dd:::benchmark_data_smithi133_153303_object58366:head v 165'9468, mlcod=165'9461)", 2023-10-09T05:13:56.672 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:12:20.276498+0000", 2023-10-09T05:13:56.672 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 96.269074258000003, 2023-10-09T05:13:56.672 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.023967820000000001, 2023-10-09T05:13:56.672 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:13:56.672 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:56.672 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:13:56.672 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.672 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:13:56.672 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.276498+0000", 2023-10-09T05:13:56.672 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.672 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.673 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.673 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:13:56.673 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.276498+0000", 2023-10-09T05:13:56.673 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.673 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.673 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.673 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:13:56.673 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.276497+0000", 2023-10-09T05:13:56.673 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:13:56.673 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.674 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.674 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:13:56.674 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.276505+0000", 2023-10-09T05:13:56.674 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.0360000000000006e-06 2023-10-09T05:13:56.674 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.674 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.674 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:13:56.674 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.276521+0000", 2023-10-09T05:13:56.674 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.6473999999999998e-05 2023-10-09T05:13:56.674 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.674 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.674 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:13:56.675 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.276528+0000", 2023-10-09T05:13:56.675 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.9940000000000003e-06 2023-10-09T05:13:56.675 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.675 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.675 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:13:56.675 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.288048+0000", 2023-10-09T05:13:56.675 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.01151977 2023-10-09T05:13:56.675 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.675 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.675 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:13:56.675 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.288070+0000", 2023-10-09T05:13:56.675 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.1710000000000001e-05 2023-10-09T05:13:56.676 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.676 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.676 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:13:56.676 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.300339+0000", 2023-10-09T05:13:56.676 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.012269795 2023-10-09T05:13:56.676 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.676 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.676 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:13:56.676 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:20.300466+0000", 2023-10-09T05:13:56.676 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00012608000000000001 2023-10-09T05:13:56.676 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.677 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:13:56.677 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.677 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.677 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.677 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_repop(client.34845.0:157660 9.c e166/139 9:380b5e91:::benchmark_data_smithi133_153303_object67691:head v 166'10069, mlcod=166'10064)", 2023-10-09T05:13:56.677 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:12:31.352145+0000", 2023-10-09T05:13:56.677 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 85.193426531, 2023-10-09T05:13:56.677 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.024586543999999998, 2023-10-09T05:13:56.677 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:13:56.677 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:13:56.677 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:13:56.677 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.677 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:13:56.678 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:31.352145+0000", 2023-10-09T05:13:56.678 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.678 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.678 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.678 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:13:56.678 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:31.352145+0000", 2023-10-09T05:13:56.678 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:13:56.678 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.678 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.678 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:13:56.678 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:31.352144+0000", 2023-10-09T05:13:56.679 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:13:56.679 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.679 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.679 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:13:56.679 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:31.352157+0000", 2023-10-09T05:13:56.679 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.2850999999999999e-05 2023-10-09T05:13:56.679 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.679 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.679 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:13:56.679 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:31.352172+0000", 2023-10-09T05:13:56.679 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.5031999999999999e-05 2023-10-09T05:13:56.680 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.680 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.680 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:13:56.680 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:31.352179+0000", 2023-10-09T05:13:56.680 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.905e-06 2023-10-09T05:13:56.680 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.680 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.680 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:13:56.680 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:31.352233+0000", 2023-10-09T05:13:56.680 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.4088999999999997e-05 2023-10-09T05:13:56.680 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.681 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.681 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:13:56.681 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:31.352263+0000", 2023-10-09T05:13:56.681 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.9918000000000001e-05 2023-10-09T05:13:56.681 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.681 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.681 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:13:56.681 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:31.376551+0000", 2023-10-09T05:13:56.681 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.024287507 2023-10-09T05:13:56.681 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:13:56.681 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:13:56.681 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:13:56.682 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:31.376732+0000", 2023-10-09T05:13:56.682 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00018116000000000001 2023-10-09T05:13:56.682 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.682 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:13:56.682 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.682 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:13:56.682 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:13:56.682 INFO:teuthology.orchestra.run.smithi194.stdout:} 2023-10-09T05:13:56.682 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:56.710 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:56.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:56.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:56.813 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:56.839 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:56.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:56.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:56.942 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:56.974 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:56.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:56.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:57.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:56 smithi133 bash[105589]: cluster 2023-10-09T05:13:55.868848+0000 mgr.y (mgr.25277) 762 : cluster 0 pgmap v753: 89 pgs: 89 active+clean; 582 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:57.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:56 smithi133 bash[102858]: cluster 2023-10-09T05:13:55.868848+0000 mgr.y (mgr.25277) 762 : cluster 0 pgmap v753: 89 pgs: 89 active+clean; 582 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:57.078 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:57.104 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:57.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:57.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:57.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:56 smithi194 bash[92666]: cluster 2023-10-09T05:13:55.868848+0000 mgr.y (mgr.25277) 762 : cluster 0 pgmap v753: 89 pgs: 89 active+clean; 582 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:57.208 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:57.230 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:13:57.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:57.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:13:57.333 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:57.361 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:57.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:57.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:57.379 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T05:13:57.465 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:57.494 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:57.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:57.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:57.597 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:57.626 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:57.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:57.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:57.730 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:57.761 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:57.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:57.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:57.865 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:57.892 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:57.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:57.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:57.996 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:58.025 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:58.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:58.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:58.129 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:58.158 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:58.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:58.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:58.262 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:58.297 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:58.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:58.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:58.401 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:58.432 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:58.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:58.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:58.537 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:58.559 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:58.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:58.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:58.662 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:58.687 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:13:58.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:58.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:13:58.790 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:58.816 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:58.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:58.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:58.919 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:58.955 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:58.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:58.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:59.058 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:59.070 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:58 smithi133 bash[102858]: cluster 2023-10-09T05:13:57.869593+0000 mgr.y (mgr.25277) 763 : cluster 0 pgmap v754: 89 pgs: 89 active+clean; 582 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:59.071 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:58 smithi133 bash[102858]: audit 2023-10-09T05:13:58.445544+0000 mon.a (mon.0) 974 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:13:59.071 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:58 smithi133 bash[102858]: audit 2023-10-09T05:13:58.497292+0000 mon.c (mon.2) 72 : audit 1 from='client.? 172.21.15.133:0/1609908824' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:13:59.071 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:58 smithi133 bash[102858]: audit 2023-10-09T05:13:58.497990+0000 mon.a (mon.0) 975 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:13:59.071 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:58 smithi133 bash[105589]: cluster 2023-10-09T05:13:57.869593+0000 mgr.y (mgr.25277) 763 : cluster 0 pgmap v754: 89 pgs: 89 active+clean; 582 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:59.071 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:58 smithi133 bash[105589]: audit 2023-10-09T05:13:58.445544+0000 mon.a (mon.0) 974 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:13:59.071 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:58 smithi133 bash[105589]: audit 2023-10-09T05:13:58.497292+0000 mon.c (mon.2) 72 : audit 1 from='client.? 172.21.15.133:0/1609908824' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:13:59.071 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:58 smithi133 bash[105589]: audit 2023-10-09T05:13:58.497990+0000 mon.a (mon.0) 975 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:13:59.091 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:59.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:59.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:59.195 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:59.207 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:58 smithi194 bash[92666]: cluster 2023-10-09T05:13:57.869593+0000 mgr.y (mgr.25277) 763 : cluster 0 pgmap v754: 89 pgs: 89 active+clean; 582 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:13:59.208 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:58 smithi194 bash[92666]: audit 2023-10-09T05:13:58.445544+0000 mon.a (mon.0) 974 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:13:59.208 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:58 smithi194 bash[92666]: audit 2023-10-09T05:13:58.497292+0000 mon.c (mon.2) 72 : audit 1 from='client.? 172.21.15.133:0/1609908824' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:13:59.208 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:58 smithi194 bash[92666]: audit 2023-10-09T05:13:58.497990+0000 mon.a (mon.0) 975 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:13:59.228 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:13:59.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:59.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:13:59.314 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is unset 2023-10-09T05:13:59.331 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:59.360 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:13:59.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:59.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:13:59.463 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:59.494 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:59.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:59.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:13:59.598 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:59.633 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:13:59.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:59.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:13:59.738 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:59.766 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:13:59.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:59.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:13:59.870 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:59.905 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:13:59.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:13:59.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:00.009 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:00.040 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:00.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:00.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:00.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:59 smithi194 bash[92666]: cluster 2023-10-09T05:13:59.302543+0000 mon.a (mon.0) 976 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:14:00.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:59 smithi194 bash[92666]: cluster 2023-10-09T05:13:59.302622+0000 mon.a (mon.0) 977 : cluster 1 Cluster is now healthy 2023-10-09T05:14:00.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:59 smithi194 bash[92666]: cluster 2023-10-09T05:13:59.306132+0000 osd.6 (osd.6) 107 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T05:14:00.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:59 smithi194 bash[92666]: audit 2023-10-09T05:13:59.309098+0000 mon.a (mon.0) 978 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:14:00.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:13:59 smithi194 bash[92666]: cluster 2023-10-09T05:13:59.309195+0000 mon.a (mon.0) 979 : cluster 0 osdmap e182: 8 total, 8 up, 7 in 2023-10-09T05:14:00.145 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:00.170 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:00.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:00.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:00.188 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:59 smithi133 bash[102858]: cluster 2023-10-09T05:13:59.302543+0000 mon.a (mon.0) 976 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:14:00.188 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:59 smithi133 bash[102858]: cluster 2023-10-09T05:13:59.302622+0000 mon.a (mon.0) 977 : cluster 1 Cluster is now healthy 2023-10-09T05:14:00.188 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:59 smithi133 bash[102858]: cluster 2023-10-09T05:13:59.306132+0000 osd.6 (osd.6) 107 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T05:14:00.188 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:59 smithi133 bash[102858]: audit 2023-10-09T05:13:59.309098+0000 mon.a (mon.0) 978 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:14:00.188 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:13:59 smithi133 bash[102858]: cluster 2023-10-09T05:13:59.309195+0000 mon.a (mon.0) 979 : cluster 0 osdmap e182: 8 total, 8 up, 7 in 2023-10-09T05:14:00.189 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:59 smithi133 bash[105589]: cluster 2023-10-09T05:13:59.302543+0000 mon.a (mon.0) 976 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:14:00.189 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:59 smithi133 bash[105589]: cluster 2023-10-09T05:13:59.302622+0000 mon.a (mon.0) 977 : cluster 1 Cluster is now healthy 2023-10-09T05:14:00.189 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:59 smithi133 bash[105589]: cluster 2023-10-09T05:13:59.306132+0000 osd.6 (osd.6) 107 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T05:14:00.189 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:59 smithi133 bash[105589]: audit 2023-10-09T05:13:59.309098+0000 mon.a (mon.0) 978 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:14:00.189 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:13:59 smithi133 bash[105589]: cluster 2023-10-09T05:13:59.309195+0000 mon.a (mon.0) 979 : cluster 0 osdmap e182: 8 total, 8 up, 7 in 2023-10-09T05:14:00.275 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:00.298 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:00.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:00.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:00.401 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:00.431 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:00.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:00.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:00.535 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:00.563 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:00.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:00.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:00.665 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:00.692 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:00.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:00.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:00.749 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:14:00.796 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:00.826 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:00.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:00.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:00.931 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:00.950 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:00.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:00.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:01.054 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:01.082 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:01.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:01.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:01.186 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:01.199 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:00 smithi194 bash[92666]: cluster 2023-10-09T05:13:59.870430+0000 mgr.y (mgr.25277) 764 : cluster 0 pgmap v756: 89 pgs: 89 active+clean; 582 KiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:14:01.219 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:01.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:01.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:01.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:00 smithi133 bash[105589]: cluster 2023-10-09T05:13:59.870430+0000 mgr.y (mgr.25277) 764 : cluster 0 pgmap v756: 89 pgs: 89 active+clean; 582 KiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:14:01.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:00 smithi133 bash[102858]: cluster 2023-10-09T05:13:59.870430+0000 mgr.y (mgr.25277) 764 : cluster 0 pgmap v756: 89 pgs: 89 active+clean; 582 KiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:14:01.323 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:01.347 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:01.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:01.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:01.451 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:01.482 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:01.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:01.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:01.586 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:01.598 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:01 smithi133 bash[102858]: debug 2023-10-09T05:14:01.403+0000 7f5eb4d1d700 -1 mon.a@0(leader).osd e182 definitely_dead 0 2023-10-09T05:14:01.621 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:01.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:01.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:01.725 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:01.756 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:01.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:01.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:01.860 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:01.891 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:01.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:01.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:01.961 INFO:teuthology.orchestra.run.smithi133.stderr:enabled application 'rados' on pool 'unique_pool_5' 2023-10-09T05:14:01.964 INFO:teuthology.orchestra.run.smithi133.stderr:marked out osd.7. 2023-10-09T05:14:01.995 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:02.023 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:02.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:02.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:02.126 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:02.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:01 smithi194 bash[92666]: audit 2023-10-09T05:14:01.405768+0000 mon.c (mon.2) 73 : audit 1 from='client.? 172.21.15.133:0/3779854269' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T05:14:02.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:01 smithi194 bash[92666]: audit 2023-10-09T05:14:01.406243+0000 mon.a (mon.0) 980 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T05:14:02.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:01 smithi194 bash[92666]: cluster 2023-10-09T05:14:01.406320+0000 mon.a (mon.0) 981 : cluster 1 Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T05:14:02.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:01 smithi194 bash[92666]: audit 2023-10-09T05:14:01.412662+0000 mon.a (mon.0) 982 : audit 1 from='client.? 172.21.15.133:0/650656022' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:14:02.153 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:02.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:02.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:02.201 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:14:02.256 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:02.267 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:01 smithi133 bash[105589]: audit 2023-10-09T05:14:01.405768+0000 mon.c (mon.2) 73 : audit 1 from='client.? 172.21.15.133:0/3779854269' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T05:14:02.267 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:01 smithi133 bash[105589]: audit 2023-10-09T05:14:01.406243+0000 mon.a (mon.0) 980 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T05:14:02.268 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:01 smithi133 bash[105589]: cluster 2023-10-09T05:14:01.406320+0000 mon.a (mon.0) 981 : cluster 1 Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T05:14:02.268 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:01 smithi133 bash[105589]: audit 2023-10-09T05:14:01.412662+0000 mon.a (mon.0) 982 : audit 1 from='client.? 172.21.15.133:0/650656022' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:14:02.268 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:01 smithi133 bash[102858]: audit 2023-10-09T05:14:01.405768+0000 mon.c (mon.2) 73 : audit 1 from='client.? 172.21.15.133:0/3779854269' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T05:14:02.268 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:01 smithi133 bash[102858]: audit 2023-10-09T05:14:01.406243+0000 mon.a (mon.0) 980 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T05:14:02.268 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:01 smithi133 bash[102858]: cluster 2023-10-09T05:14:01.406320+0000 mon.a (mon.0) 981 : cluster 1 Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T05:14:02.268 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:01 smithi133 bash[102858]: audit 2023-10-09T05:14:01.412662+0000 mon.a (mon.0) 982 : audit 1 from='client.? 172.21.15.133:0/650656022' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:14:02.286 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:02.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:02.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:02.390 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:02.414 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:02.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:02.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:02.518 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:02.539 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:02.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:02.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:02.642 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:02.669 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:02.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:02.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:02.729 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:14:02.772 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:02.797 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:02.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:02.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:02.900 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:02.931 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:02.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:02.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:03.035 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:03.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:02 smithi133 bash[102858]: cluster 2023-10-09T05:14:01.871335+0000 mgr.y (mgr.25277) 765 : cluster 0 pgmap v757: 89 pgs: 89 active+clean; 582 KiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:14:03.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:02 smithi133 bash[102858]: cluster 2023-10-09T05:14:01.943323+0000 osd.5 (osd.5) 172 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T05:14:03.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:02 smithi133 bash[102858]: audit 2023-10-09T05:14:01.945666+0000 mon.a (mon.0) 983 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T05:14:03.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:02 smithi133 bash[102858]: cluster 2023-10-09T05:14:01.945671+0000 osd.6 (osd.6) 108 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T05:14:03.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:02 smithi133 bash[102858]: audit 2023-10-09T05:14:01.945768+0000 mon.a (mon.0) 984 : audit 1 from='client.? 172.21.15.133:0/650656022' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:14:03.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:02 smithi133 bash[102858]: cluster 2023-10-09T05:14:01.945832+0000 mon.a (mon.0) 985 : cluster 0 osdmap e183: 8 total, 8 up, 6 in 2023-10-09T05:14:03.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:02 smithi133 bash[102858]: cluster 2023-10-09T05:14:01.945866+0000 osd.7 (osd.7) 114 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T05:14:03.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:02 smithi133 bash[102858]: cluster 2023-10-09T05:14:01.946681+0000 osd.4 (osd.4) 120 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T05:14:03.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:02 smithi133 bash[102858]: cluster 2023-10-09T05:14:01.951850+0000 osd.7 (osd.7) 115 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T05:14:03.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:02 smithi133 bash[105589]: cluster 2023-10-09T05:14:01.871335+0000 mgr.y (mgr.25277) 765 : cluster 0 pgmap v757: 89 pgs: 89 active+clean; 582 KiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:14:03.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:02 smithi133 bash[105589]: cluster 2023-10-09T05:14:01.943323+0000 osd.5 (osd.5) 172 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T05:14:03.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:02 smithi133 bash[105589]: audit 2023-10-09T05:14:01.945666+0000 mon.a (mon.0) 983 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T05:14:03.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:02 smithi133 bash[105589]: cluster 2023-10-09T05:14:01.945671+0000 osd.6 (osd.6) 108 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T05:14:03.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:02 smithi133 bash[105589]: audit 2023-10-09T05:14:01.945768+0000 mon.a (mon.0) 984 : audit 1 from='client.? 172.21.15.133:0/650656022' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:14:03.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:02 smithi133 bash[105589]: cluster 2023-10-09T05:14:01.945832+0000 mon.a (mon.0) 985 : cluster 0 osdmap e183: 8 total, 8 up, 6 in 2023-10-09T05:14:03.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:02 smithi133 bash[105589]: cluster 2023-10-09T05:14:01.945866+0000 osd.7 (osd.7) 114 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T05:14:03.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:02 smithi133 bash[105589]: cluster 2023-10-09T05:14:01.946681+0000 osd.4 (osd.4) 120 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T05:14:03.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:02 smithi133 bash[105589]: cluster 2023-10-09T05:14:01.951850+0000 osd.7 (osd.7) 115 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T05:14:03.066 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:03.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:03.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:03.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:02 smithi194 bash[92666]: cluster 2023-10-09T05:14:01.871335+0000 mgr.y (mgr.25277) 765 : cluster 0 pgmap v757: 89 pgs: 89 active+clean; 582 KiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:14:03.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:02 smithi194 bash[92666]: cluster 2023-10-09T05:14:01.943323+0000 osd.5 (osd.5) 172 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T05:14:03.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:02 smithi194 bash[92666]: audit 2023-10-09T05:14:01.945666+0000 mon.a (mon.0) 983 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T05:14:03.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:02 smithi194 bash[92666]: cluster 2023-10-09T05:14:01.945671+0000 osd.6 (osd.6) 108 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T05:14:03.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:02 smithi194 bash[92666]: audit 2023-10-09T05:14:01.945768+0000 mon.a (mon.0) 984 : audit 1 from='client.? 172.21.15.133:0/650656022' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:14:03.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:02 smithi194 bash[92666]: cluster 2023-10-09T05:14:01.945832+0000 mon.a (mon.0) 985 : cluster 0 osdmap e183: 8 total, 8 up, 6 in 2023-10-09T05:14:03.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:02 smithi194 bash[92666]: cluster 2023-10-09T05:14:01.945866+0000 osd.7 (osd.7) 114 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T05:14:03.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:02 smithi194 bash[92666]: cluster 2023-10-09T05:14:01.946681+0000 osd.4 (osd.4) 120 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T05:14:03.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:02 smithi194 bash[92666]: cluster 2023-10-09T05:14:01.951850+0000 osd.7 (osd.7) 115 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T05:14:03.169 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:03.200 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:03.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:03.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:03.304 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:03.333 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:03.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:03.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:03.437 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:03.471 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:03.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:03.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:03.575 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:03.604 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:03.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:03.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:03.708 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:03.740 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:03.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:03.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:03.844 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:03.875 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:03.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:03.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:03.978 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:04.004 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:04.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:04.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:04.107 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:04.115 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:03 smithi194 bash[92666]: cluster 2023-10-09T05:14:02.956432+0000 osd.5 (osd.5) 173 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T05:14:04.115 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:03 smithi194 bash[92666]: cluster 2023-10-09T05:14:02.956834+0000 osd.6 (osd.6) 109 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T05:14:04.116 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:03 smithi194 bash[92666]: cluster 2023-10-09T05:14:02.958469+0000 osd.4 (osd.4) 121 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T05:14:04.116 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:03 smithi194 bash[92666]: cluster 2023-10-09T05:14:02.958488+0000 mon.a (mon.0) 986 : cluster 0 osdmap e184: 8 total, 8 up, 6 in 2023-10-09T05:14:04.116 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:03 smithi194 bash[92666]: cluster 2023-10-09T05:14:02.960365+0000 osd.6 (osd.6) 110 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T05:14:04.116 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:03 smithi194 bash[92666]: cluster 2023-10-09T05:14:02.961437+0000 osd.4 (osd.4) 122 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T05:14:04.116 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:03 smithi194 bash[92666]: cluster 2023-10-09T05:14:02.961764+0000 osd.7 (osd.7) 116 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T05:14:04.116 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:03 smithi194 bash[92666]: cluster 2023-10-09T05:14:02.963752+0000 osd.4 (osd.4) 123 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T05:14:04.132 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:04.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:04.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:04.235 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:04.260 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:04.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:04.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:04.282 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:03 smithi133 bash[105589]: cluster 2023-10-09T05:14:02.956432+0000 osd.5 (osd.5) 173 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T05:14:04.282 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:03 smithi133 bash[105589]: cluster 2023-10-09T05:14:02.956834+0000 osd.6 (osd.6) 109 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T05:14:04.282 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:03 smithi133 bash[105589]: cluster 2023-10-09T05:14:02.958469+0000 osd.4 (osd.4) 121 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T05:14:04.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:03 smithi133 bash[105589]: cluster 2023-10-09T05:14:02.958488+0000 mon.a (mon.0) 986 : cluster 0 osdmap e184: 8 total, 8 up, 6 in 2023-10-09T05:14:04.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:03 smithi133 bash[105589]: cluster 2023-10-09T05:14:02.960365+0000 osd.6 (osd.6) 110 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T05:14:04.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:03 smithi133 bash[105589]: cluster 2023-10-09T05:14:02.961437+0000 osd.4 (osd.4) 122 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T05:14:04.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:03 smithi133 bash[105589]: cluster 2023-10-09T05:14:02.961764+0000 osd.7 (osd.7) 116 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T05:14:04.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:03 smithi133 bash[105589]: cluster 2023-10-09T05:14:02.963752+0000 osd.4 (osd.4) 123 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T05:14:04.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:03 smithi133 bash[102858]: cluster 2023-10-09T05:14:02.956432+0000 osd.5 (osd.5) 173 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T05:14:04.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:03 smithi133 bash[102858]: cluster 2023-10-09T05:14:02.956834+0000 osd.6 (osd.6) 109 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T05:14:04.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:03 smithi133 bash[102858]: cluster 2023-10-09T05:14:02.958469+0000 osd.4 (osd.4) 121 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T05:14:04.284 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:03 smithi133 bash[102858]: cluster 2023-10-09T05:14:02.958488+0000 mon.a (mon.0) 986 : cluster 0 osdmap e184: 8 total, 8 up, 6 in 2023-10-09T05:14:04.284 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:03 smithi133 bash[102858]: cluster 2023-10-09T05:14:02.960365+0000 osd.6 (osd.6) 110 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T05:14:04.284 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:03 smithi133 bash[102858]: cluster 2023-10-09T05:14:02.961437+0000 osd.4 (osd.4) 122 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T05:14:04.284 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:03 smithi133 bash[102858]: cluster 2023-10-09T05:14:02.961764+0000 osd.7 (osd.7) 116 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T05:14:04.284 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:03 smithi133 bash[102858]: cluster 2023-10-09T05:14:02.963752+0000 osd.4 (osd.4) 123 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T05:14:04.364 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:04.389 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:04.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:04.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:04.493 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:04.520 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:04.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:04.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:04.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:14:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:14:04] "GET /metrics HTTP/1.1" 200 36399 "" "Prometheus/2.43.0" 2023-10-09T05:14:04.551 DEBUG:teuthology.orchestra.run.smithi133:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2023-10-09T05:14:04.553 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T05:14:04.553 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T05:14:04.624 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:04.638 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:hints = 1 2023-10-09T05:14:04.638 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T05:14:04.638 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Object prefix: benchmark_data_smithi133_173023 2023-10-09T05:14:04.639 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:14:04.639 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 0 0 0 0 0 0 - 0 2023-10-09T05:14:04.654 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:04.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:04.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:04.759 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:04.786 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:04.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:04.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:04.889 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:04.913 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:04.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:04.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:05.017 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:05.080 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:05.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:05.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:05.183 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:05.209 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:05.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:05.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:05.257 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:04 smithi194 bash[92666]: cluster 2023-10-09T05:14:03.872194+0000 mgr.y (mgr.25277) 766 : cluster 0 pgmap v760: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 534 GiB / 536 GiB avail; 14 B/s, 0 objects/s recovering 2023-10-09T05:14:05.257 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:04 smithi194 bash[92666]: cluster 2023-10-09T05:14:03.962773+0000 osd.5 (osd.5) 174 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T05:14:05.257 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:04 smithi194 bash[92666]: cluster 2023-10-09T05:14:03.965960+0000 mon.a (mon.0) 987 : cluster 0 osdmap e185: 8 total, 8 up, 6 in 2023-10-09T05:14:05.257 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:04 smithi194 bash[92666]: cluster 2023-10-09T05:14:03.971020+0000 osd.6 (osd.6) 111 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T05:14:05.257 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:04 smithi194 bash[92666]: cluster 2023-10-09T05:14:03.971132+0000 osd.7 (osd.7) 117 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T05:14:05.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:04 smithi133 bash[105589]: cluster 2023-10-09T05:14:03.872194+0000 mgr.y (mgr.25277) 766 : cluster 0 pgmap v760: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 534 GiB / 536 GiB avail; 14 B/s, 0 objects/s recovering 2023-10-09T05:14:05.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:04 smithi133 bash[105589]: cluster 2023-10-09T05:14:03.962773+0000 osd.5 (osd.5) 174 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T05:14:05.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:04 smithi133 bash[105589]: cluster 2023-10-09T05:14:03.965960+0000 mon.a (mon.0) 987 : cluster 0 osdmap e185: 8 total, 8 up, 6 in 2023-10-09T05:14:05.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:04 smithi133 bash[105589]: cluster 2023-10-09T05:14:03.971020+0000 osd.6 (osd.6) 111 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T05:14:05.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:04 smithi133 bash[105589]: cluster 2023-10-09T05:14:03.971132+0000 osd.7 (osd.7) 117 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T05:14:05.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:04 smithi133 bash[102858]: cluster 2023-10-09T05:14:03.872194+0000 mgr.y (mgr.25277) 766 : cluster 0 pgmap v760: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 534 GiB / 536 GiB avail; 14 B/s, 0 objects/s recovering 2023-10-09T05:14:05.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:04 smithi133 bash[102858]: cluster 2023-10-09T05:14:03.962773+0000 osd.5 (osd.5) 174 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T05:14:05.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:04 smithi133 bash[102858]: cluster 2023-10-09T05:14:03.965960+0000 mon.a (mon.0) 987 : cluster 0 osdmap e185: 8 total, 8 up, 6 in 2023-10-09T05:14:05.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:04 smithi133 bash[102858]: cluster 2023-10-09T05:14:03.971020+0000 osd.6 (osd.6) 111 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T05:14:05.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:04 smithi133 bash[102858]: cluster 2023-10-09T05:14:03.971132+0000 osd.7 (osd.7) 117 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T05:14:05.312 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:05.338 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:05.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:05.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:05.440 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:05.475 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:05.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:05.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:05.579 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:05.607 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:05.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:05.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:05.639 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 1 16 2436 2420 151.242 151.25 0.00611317 0.00655945 2023-10-09T05:14:05.711 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:05.748 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:05.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:05.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:05.852 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:05.891 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:05.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:05.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:05.996 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:06.026 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:06.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:06.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:06.083 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:14:06.131 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:06.174 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:06.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:06.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:06.237 INFO:teuthology.orchestra.run.smithi194.stdout:{ 2023-10-09T05:14:06.237 INFO:teuthology.orchestra.run.smithi194.stdout: "ops": [], 2023-10-09T05:14:06.237 INFO:teuthology.orchestra.run.smithi194.stdout: "num_ops": 0 2023-10-09T05:14:06.237 INFO:teuthology.orchestra.run.smithi194.stdout:} 2023-10-09T05:14:06.279 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:06.289 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:05 smithi133 bash[102858]: cluster 2023-10-09T05:14:04.978162+0000 mon.a (mon.0) 988 : cluster 0 osdmap e186: 8 total, 8 up, 6 in 2023-10-09T05:14:06.289 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:05 smithi133 bash[102858]: cluster 2023-10-09T05:14:04.978699+0000 osd.6 (osd.6) 112 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T05:14:06.289 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:05 smithi133 bash[102858]: cluster 2023-10-09T05:14:04.989201+0000 osd.7 (osd.7) 118 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T05:14:06.289 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:05 smithi133 bash[102858]: audit 2023-10-09T05:14:05.375980+0000 mon.a (mon.0) 989 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:14:06.290 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:05 smithi133 bash[102858]: audit 2023-10-09T05:14:05.384634+0000 mon.a (mon.0) 990 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:14:06.290 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:05 smithi133 bash[105589]: cluster 2023-10-09T05:14:04.978162+0000 mon.a (mon.0) 988 : cluster 0 osdmap e186: 8 total, 8 up, 6 in 2023-10-09T05:14:06.290 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:05 smithi133 bash[105589]: cluster 2023-10-09T05:14:04.978699+0000 osd.6 (osd.6) 112 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T05:14:06.290 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:05 smithi133 bash[105589]: cluster 2023-10-09T05:14:04.989201+0000 osd.7 (osd.7) 118 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T05:14:06.290 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:05 smithi133 bash[105589]: audit 2023-10-09T05:14:05.375980+0000 mon.a (mon.0) 989 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:14:06.290 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:05 smithi133 bash[105589]: audit 2023-10-09T05:14:05.384634+0000 mon.a (mon.0) 990 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:14:06.290 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:05 smithi194 bash[92666]: cluster 2023-10-09T05:14:04.978162+0000 mon.a (mon.0) 988 : cluster 0 osdmap e186: 8 total, 8 up, 6 in 2023-10-09T05:14:06.291 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:05 smithi194 bash[92666]: cluster 2023-10-09T05:14:04.978699+0000 osd.6 (osd.6) 112 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T05:14:06.291 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:05 smithi194 bash[92666]: cluster 2023-10-09T05:14:04.989201+0000 osd.7 (osd.7) 118 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T05:14:06.291 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:05 smithi194 bash[92666]: audit 2023-10-09T05:14:05.375980+0000 mon.a (mon.0) 989 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:14:06.291 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:05 smithi194 bash[92666]: audit 2023-10-09T05:14:05.384634+0000 mon.a (mon.0) 990 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:14:06.306 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:06.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:06.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:06.410 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:06.437 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:06.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:06.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:06.542 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:06.568 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:06.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:06.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:06.639 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 2 16 4895 4879 152.459 153.688 0.00517765 0.00654563 2023-10-09T05:14:06.672 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:06.700 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:06.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:06.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:06.805 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:06.832 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:06.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:06.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:06.936 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:06.965 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:06.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:06.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:07.070 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:07.098 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:07.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:07.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:07.202 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:07.211 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:06 smithi133 bash[105589]: cluster 2023-10-09T05:14:04.981694+0000 osd.4 (osd.4) 124 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T05:14:07.211 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:06 smithi133 bash[105589]: cluster 2023-10-09T05:14:04.983670+0000 osd.4 (osd.4) 125 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T05:14:07.211 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:06 smithi133 bash[105589]: cluster 2023-10-09T05:14:05.399497+0000 osd.5 (osd.5) 175 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T05:14:07.212 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:06 smithi133 bash[105589]: cluster 2023-10-09T05:14:05.872768+0000 mgr.y (mgr.25277) 767 : cluster 0 pgmap v763: 89 pgs: 1 active+undersized+remapped, 88 active+clean; 29 MiB data, 2.6 GiB used, 534 GiB / 536 GiB avail; 7.1 MiB/s wr, 113 op/s; 145 KiB/s, 2 keys/s, 2 objects/s recovering 2023-10-09T05:14:07.212 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:06 smithi133 bash[105589]: cluster 2023-10-09T05:14:05.991188+0000 mon.a (mon.0) 991 : cluster 0 osdmap e187: 8 total, 8 up, 6 in 2023-10-09T05:14:07.212 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:06 smithi133 bash[105589]: cluster 2023-10-09T05:14:05.996999+0000 osd.6 (osd.6) 113 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T05:14:07.212 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:06 smithi133 bash[105589]: cluster 2023-10-09T05:14:05.999217+0000 osd.7 (osd.7) 119 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T05:14:07.212 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:06 smithi133 bash[105589]: cluster 2023-10-09T05:14:06.002762+0000 osd.6 (osd.6) 114 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T05:14:07.212 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:06 smithi133 bash[105589]: audit 2023-10-09T05:14:06.059028+0000 mon.a (mon.0) 992 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:14:07.212 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:06 smithi133 bash[105589]: audit 2023-10-09T05:14:06.060412+0000 mon.a (mon.0) 993 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:14:07.212 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:06 smithi133 bash[105589]: audit 2023-10-09T05:14:06.069234+0000 mon.a (mon.0) 994 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:14:07.213 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:06 smithi133 bash[102858]: cluster 2023-10-09T05:14:04.981694+0000 osd.4 (osd.4) 124 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T05:14:07.213 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:06 smithi133 bash[102858]: cluster 2023-10-09T05:14:04.983670+0000 osd.4 (osd.4) 125 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T05:14:07.213 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:06 smithi133 bash[102858]: cluster 2023-10-09T05:14:05.399497+0000 osd.5 (osd.5) 175 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T05:14:07.213 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:06 smithi133 bash[102858]: cluster 2023-10-09T05:14:05.872768+0000 mgr.y (mgr.25277) 767 : cluster 0 pgmap v763: 89 pgs: 1 active+undersized+remapped, 88 active+clean; 29 MiB data, 2.6 GiB used, 534 GiB / 536 GiB avail; 7.1 MiB/s wr, 113 op/s; 145 KiB/s, 2 keys/s, 2 objects/s recovering 2023-10-09T05:14:07.213 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:06 smithi133 bash[102858]: cluster 2023-10-09T05:14:05.991188+0000 mon.a (mon.0) 991 : cluster 0 osdmap e187: 8 total, 8 up, 6 in 2023-10-09T05:14:07.213 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:06 smithi133 bash[102858]: cluster 2023-10-09T05:14:05.996999+0000 osd.6 (osd.6) 113 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T05:14:07.213 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:06 smithi133 bash[102858]: cluster 2023-10-09T05:14:05.999217+0000 osd.7 (osd.7) 119 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T05:14:07.213 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:06 smithi133 bash[102858]: cluster 2023-10-09T05:14:06.002762+0000 osd.6 (osd.6) 114 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T05:14:07.213 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:06 smithi133 bash[102858]: audit 2023-10-09T05:14:06.059028+0000 mon.a (mon.0) 992 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:14:07.213 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:06 smithi133 bash[102858]: audit 2023-10-09T05:14:06.060412+0000 mon.a (mon.0) 993 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:14:07.214 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:06 smithi133 bash[102858]: audit 2023-10-09T05:14:06.069234+0000 mon.a (mon.0) 994 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:14:07.227 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:07.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:07.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:07.331 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:07.341 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:06 smithi194 bash[92666]: cluster 2023-10-09T05:14:04.981694+0000 osd.4 (osd.4) 124 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T05:14:07.341 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:06 smithi194 bash[92666]: cluster 2023-10-09T05:14:04.983670+0000 osd.4 (osd.4) 125 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T05:14:07.341 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:06 smithi194 bash[92666]: cluster 2023-10-09T05:14:05.399497+0000 osd.5 (osd.5) 175 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T05:14:07.341 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:06 smithi194 bash[92666]: cluster 2023-10-09T05:14:05.872768+0000 mgr.y (mgr.25277) 767 : cluster 0 pgmap v763: 89 pgs: 1 active+undersized+remapped, 88 active+clean; 29 MiB data, 2.6 GiB used, 534 GiB / 536 GiB avail; 7.1 MiB/s wr, 113 op/s; 145 KiB/s, 2 keys/s, 2 objects/s recovering 2023-10-09T05:14:07.341 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:06 smithi194 bash[92666]: cluster 2023-10-09T05:14:05.991188+0000 mon.a (mon.0) 991 : cluster 0 osdmap e187: 8 total, 8 up, 6 in 2023-10-09T05:14:07.341 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:06 smithi194 bash[92666]: cluster 2023-10-09T05:14:05.996999+0000 osd.6 (osd.6) 113 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T05:14:07.341 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:06 smithi194 bash[92666]: cluster 2023-10-09T05:14:05.999217+0000 osd.7 (osd.7) 119 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T05:14:07.341 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:06 smithi194 bash[92666]: cluster 2023-10-09T05:14:06.002762+0000 osd.6 (osd.6) 114 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T05:14:07.342 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:06 smithi194 bash[92666]: audit 2023-10-09T05:14:06.059028+0000 mon.a (mon.0) 992 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:14:07.342 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:06 smithi194 bash[92666]: audit 2023-10-09T05:14:06.060412+0000 mon.a (mon.0) 993 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:14:07.342 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:06 smithi194 bash[92666]: audit 2023-10-09T05:14:06.069234+0000 mon.a (mon.0) 994 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:14:07.361 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:07.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:07.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:07.464 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:07.492 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:07.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:07.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:07.595 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:07.619 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:07.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:07.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:07.639 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 3 16 7423 7407 154.298 158 0.00322825 0.00646912 2023-10-09T05:14:07.707 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T05:14:07.723 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:07.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:07.752 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:07.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:07.853 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:07.888 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:07.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:07.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:07.989 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:07 smithi133 bash[102858]: cluster 2023-10-09T05:14:05.990789+0000 osd.5 (osd.5) 176 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T05:14:07.993 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:08.021 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:08.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:08.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:08.117 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:14:08.126 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:08.158 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:08.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:08.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:08.262 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:08.272 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:07 smithi133 bash[105589]: cluster 2023-10-09T05:14:05.990789+0000 osd.5 (osd.5) 176 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T05:14:08.272 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:07 smithi133 bash[105589]: cluster 2023-10-09T05:14:06.002316+0000 osd.4 (osd.4) 126 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T05:14:08.272 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:07 smithi133 bash[105589]: cluster 2023-10-09T05:14:06.004282+0000 osd.4 (osd.4) 127 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T05:14:08.273 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:07 smithi133 bash[102858]: cluster 2023-10-09T05:14:06.002316+0000 osd.4 (osd.4) 126 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T05:14:08.273 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:07 smithi133 bash[102858]: cluster 2023-10-09T05:14:06.004282+0000 osd.4 (osd.4) 127 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T05:14:08.294 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:08.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:08.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:08.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:07 smithi194 bash[92666]: cluster 2023-10-09T05:14:05.990789+0000 osd.5 (osd.5) 176 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T05:14:08.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:07 smithi194 bash[92666]: cluster 2023-10-09T05:14:06.002316+0000 osd.4 (osd.4) 126 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T05:14:08.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:07 smithi194 bash[92666]: cluster 2023-10-09T05:14:06.004282+0000 osd.4 (osd.4) 127 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T05:14:08.397 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:08.427 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:08.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:08.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:08.533 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:08.559 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:08.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:08.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:08.562 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 6, 1] out_osds: [5, 7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:14:08.563 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:14:08.563 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-10-09T05:14:08.563 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T05:14:08.563 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T05:14:08.563 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-10-09T05:14:08.640 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 4 16 10020 10004 156.289 162.312 0.00458515 0.00638975 2023-10-09T05:14:08.663 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:08.692 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:08.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:08.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:08.796 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:08.828 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:08.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:08.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:08.931 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:08.960 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:08.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:08.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:09.065 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:09.094 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:09.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:09.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:09.197 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:09.207 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:08 smithi194 bash[92666]: cluster 2023-10-09T05:14:07.873250+0000 mgr.y (mgr.25277) 768 : cluster 0 pgmap v765: 89 pgs: 1 active+undersized+remapped, 88 active+clean; 65 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 13 MiB/s wr, 209 op/s; 118 KiB/s, 2 keys/s, 2 objects/s recovering 2023-10-09T05:14:09.225 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:09.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:09.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:09.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:08 smithi133 bash[105589]: cluster 2023-10-09T05:14:07.873250+0000 mgr.y (mgr.25277) 768 : cluster 0 pgmap v765: 89 pgs: 1 active+undersized+remapped, 88 active+clean; 65 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 13 MiB/s wr, 209 op/s; 118 KiB/s, 2 keys/s, 2 objects/s recovering 2023-10-09T05:14:09.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:08 smithi133 bash[102858]: cluster 2023-10-09T05:14:07.873250+0000 mgr.y (mgr.25277) 768 : cluster 0 pgmap v765: 89 pgs: 1 active+undersized+remapped, 88 active+clean; 65 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 13 MiB/s wr, 209 op/s; 118 KiB/s, 2 keys/s, 2 objects/s recovering 2023-10-09T05:14:09.328 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:09.358 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:09.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:09.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:09.461 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:09.493 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:09.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:09.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:09.598 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:09.621 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:09.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:09.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:09.639 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 5 16 12188 12172 152.128 135.5 0.0056328 0.00656361 2023-10-09T05:14:09.724 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:09.741 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:09.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:09.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:09.844 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:09.871 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:09.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:09.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:09.974 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:10.004 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:10.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:10.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:10.108 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:10.134 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:10.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:10.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:10.238 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:10.265 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:10.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:10.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:10.372 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:10.396 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:10.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:10.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:10.501 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:10.528 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:10.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:10.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:10.632 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:10.640 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 6 16 14235 14219 148.095 127.938 0.0108936 0.00674438 2023-10-09T05:14:10.663 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:10.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:10.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:10.767 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:10.793 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:10.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:10.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:10.897 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:10.915 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:10.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:10.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:11.021 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:11.047 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:11.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:11.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:11.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:10 smithi194 bash[92666]: cluster 2023-10-09T05:14:09.873755+0000 mgr.y (mgr.25277) 769 : cluster 0 pgmap v766: 89 pgs: 89 active+clean; 337 MiB data, 3.7 GiB used, 533 GiB / 536 GiB avail; 56 MiB/s wr, 897 op/s; 97 KiB/s, 1 keys/s, 1 objects/s recovering 2023-10-09T05:14:11.151 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:11.170 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:11.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:11.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:11.273 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:11.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:10 smithi133 bash[102858]: cluster 2023-10-09T05:14:09.873755+0000 mgr.y (mgr.25277) 769 : cluster 0 pgmap v766: 89 pgs: 89 active+clean; 337 MiB data, 3.7 GiB used, 533 GiB / 536 GiB avail; 56 MiB/s wr, 897 op/s; 97 KiB/s, 1 keys/s, 1 objects/s recovering 2023-10-09T05:14:11.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:10 smithi133 bash[105589]: cluster 2023-10-09T05:14:09.873755+0000 mgr.y (mgr.25277) 769 : cluster 0 pgmap v766: 89 pgs: 89 active+clean; 337 MiB data, 3.7 GiB used, 533 GiB / 536 GiB avail; 56 MiB/s wr, 897 op/s; 97 KiB/s, 1 keys/s, 1 objects/s recovering 2023-10-09T05:14:11.299 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:11.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:11.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:11.404 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:11.427 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:11.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:11.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:11.531 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:11.550 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:11.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:11.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:11.640 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 7 16 15985 15969 142.562 109.375 0.0015944 0.00689725 2023-10-09T05:14:11.652 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:11.674 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:11.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:11.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:11.780 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:11.805 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:11.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:11.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:11.908 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:11.937 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:11.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:11.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:12.041 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:12.070 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:12.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:12.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:12.175 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:12.201 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:12.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:12.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:12.305 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:12.332 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:12.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:12.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:12.436 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:12.466 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:12.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:12.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:12.570 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:12.572 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:14:12.594 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:12.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:12.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:12.640 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 8 16 17991 17975 140.412 125.375 0.0055565 0.00711448 2023-10-09T05:14:12.699 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:12.729 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:12.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:12.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:12.832 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:12.861 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:12.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:12.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:12.964 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:12.994 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:12.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:12.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:13.098 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:13.106 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:12 smithi194 bash[92666]: cluster 2023-10-09T05:14:11.874418+0000 mgr.y (mgr.25277) 770 : cluster 0 pgmap v767: 89 pgs: 89 active+clean; 801 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 101 MiB/s wr, 1.62k op/s; 73 KiB/s, 1 keys/s, 1 objects/s recovering 2023-10-09T05:14:13.123 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:13.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:13.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:13.227 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:13.254 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:13.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:13.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:13.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:12 smithi133 bash[105589]: cluster 2023-10-09T05:14:11.874418+0000 mgr.y (mgr.25277) 770 : cluster 0 pgmap v767: 89 pgs: 89 active+clean; 801 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 101 MiB/s wr, 1.62k op/s; 73 KiB/s, 1 keys/s, 1 objects/s recovering 2023-10-09T05:14:13.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:12 smithi133 bash[102858]: cluster 2023-10-09T05:14:11.874418+0000 mgr.y (mgr.25277) 770 : cluster 0 pgmap v767: 89 pgs: 89 active+clean; 801 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 101 MiB/s wr, 1.62k op/s; 73 KiB/s, 1 keys/s, 1 objects/s recovering 2023-10-09T05:14:13.392 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:13.420 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:13.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:13.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:13.523 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:13.550 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:13.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:13.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:13.640 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 9 16 19850 19834 137.719 116.188 0.00510259 0.00725404 2023-10-09T05:14:13.656 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:13.689 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:13.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:13.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:13.792 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:13.829 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:13.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:13.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:13.932 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:13.960 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:13.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:13.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:14.064 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:14.092 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:14.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:14.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:14.195 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:14.220 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:14.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:14.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:14.324 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:14.341 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:14.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:14.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:14.443 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:14.473 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:14.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:14.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:14.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:14 smithi133 bash[102858]: audit 2023-10-09T05:14:13.420555+0000 mon.a (mon.0) 995 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 2]}]: dispatch 2023-10-09T05:14:14.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:14 smithi133 bash[102858]: audit 2023-10-09T05:14:13.421733+0000 mon.a (mon.0) 996 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2023-10-09T05:14:14.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:14 smithi133 bash[102858]: audit 2023-10-09T05:14:13.422043+0000 mon.a (mon.0) 997 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-10-09T05:14:14.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:14 smithi133 bash[102858]: audit 2023-10-09T05:14:13.452353+0000 mon.a (mon.0) 998 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:14:14.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:14 smithi133 bash[102858]: audit 2023-10-09T05:14:13.454403+0000 mon.a (mon.0) 999 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:14:14.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:14 smithi133 bash[102858]: cluster 2023-10-09T05:14:13.875066+0000 mgr.y (mgr.25277) 771 : cluster 0 pgmap v768: 89 pgs: 89 active+clean; 801 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T05:14:14.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:14:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:14:14] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.43.0" 2023-10-09T05:14:14.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:14 smithi133 bash[105589]: audit 2023-10-09T05:14:13.420555+0000 mon.a (mon.0) 995 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 2]}]: dispatch 2023-10-09T05:14:14.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:14 smithi133 bash[105589]: audit 2023-10-09T05:14:13.421733+0000 mon.a (mon.0) 996 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2023-10-09T05:14:14.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:14 smithi133 bash[105589]: audit 2023-10-09T05:14:13.422043+0000 mon.a (mon.0) 997 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-10-09T05:14:14.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:14 smithi133 bash[105589]: audit 2023-10-09T05:14:13.452353+0000 mon.a (mon.0) 998 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:14:14.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:14 smithi133 bash[105589]: audit 2023-10-09T05:14:13.454403+0000 mon.a (mon.0) 999 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:14:14.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:14 smithi133 bash[105589]: cluster 2023-10-09T05:14:13.875066+0000 mgr.y (mgr.25277) 771 : cluster 0 pgmap v768: 89 pgs: 89 active+clean; 801 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T05:14:14.576 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:14.586 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:14 smithi194 bash[92666]: audit 2023-10-09T05:14:13.420555+0000 mon.a (mon.0) 995 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 2]}]: dispatch 2023-10-09T05:14:14.586 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:14 smithi194 bash[92666]: audit 2023-10-09T05:14:13.421733+0000 mon.a (mon.0) 996 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2023-10-09T05:14:14.586 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:14 smithi194 bash[92666]: audit 2023-10-09T05:14:13.422043+0000 mon.a (mon.0) 997 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-10-09T05:14:14.586 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:14 smithi194 bash[92666]: audit 2023-10-09T05:14:13.452353+0000 mon.a (mon.0) 998 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:14:14.587 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:14 smithi194 bash[92666]: audit 2023-10-09T05:14:13.454403+0000 mon.a (mon.0) 999 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:14:14.587 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:14 smithi194 bash[92666]: cluster 2023-10-09T05:14:13.875066+0000 mgr.y (mgr.25277) 771 : cluster 0 pgmap v768: 89 pgs: 89 active+clean; 801 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T05:14:14.604 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:14.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:14.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:14.640 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 10 16 21393 21377 133.59 96.4375 0.00841759 0.00747839 2023-10-09T05:14:14.707 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:14.736 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:14.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:14.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:14.840 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:14.864 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:14.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:14.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:14.969 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:14.996 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:14.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:14.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:15.099 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:15.128 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:15.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:15.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:15.233 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:15.260 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:15.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:15.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:15.362 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:15.391 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:15.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:15.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:15.494 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:15.521 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:15.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:15.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:15.624 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:15.633 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:15 smithi194 bash[92666]: cluster 2023-10-09T05:14:14.309694+0000 osd.5 (osd.5) 177 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T05:14:15.633 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:15 smithi194 bash[92666]: cluster 2023-10-09T05:14:14.311468+0000 osd.4 (osd.4) 128 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T05:14:15.633 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:15 smithi194 bash[92666]: audit 2023-10-09T05:14:14.311959+0000 mon.a (mon.0) 1000 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 2]}]': finished 2023-10-09T05:14:15.633 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:15 smithi194 bash[92666]: audit 2023-10-09T05:14:14.312044+0000 mon.a (mon.0) 1001 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2023-10-09T05:14:15.633 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:15 smithi194 bash[92666]: audit 2023-10-09T05:14:14.312083+0000 mon.a (mon.0) 1002 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-10-09T05:14:15.633 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:15 smithi194 bash[92666]: cluster 2023-10-09T05:14:14.312202+0000 mon.a (mon.0) 1003 : cluster 0 osdmap e188: 8 total, 8 up, 6 in 2023-10-09T05:14:15.633 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:15 smithi194 bash[92666]: cluster 2023-10-09T05:14:14.315719+0000 osd.6 (osd.6) 115 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T05:14:15.633 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:15 smithi194 bash[92666]: cluster 2023-10-09T05:14:14.455382+0000 osd.7 (osd.7) 120 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T05:14:15.640 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 11 16 23250 23234 131.996 116.062 0.00521235 0.00756966 2023-10-09T05:14:15.650 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:15.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:15.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:15.753 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:15.763 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:15 smithi133 bash[102858]: cluster 2023-10-09T05:14:14.309694+0000 osd.5 (osd.5) 177 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T05:14:15.763 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:15 smithi133 bash[102858]: cluster 2023-10-09T05:14:14.311468+0000 osd.4 (osd.4) 128 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T05:14:15.763 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:15 smithi133 bash[102858]: audit 2023-10-09T05:14:14.311959+0000 mon.a (mon.0) 1000 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 2]}]': finished 2023-10-09T05:14:15.763 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:15 smithi133 bash[102858]: audit 2023-10-09T05:14:14.312044+0000 mon.a (mon.0) 1001 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2023-10-09T05:14:15.763 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:15 smithi133 bash[102858]: audit 2023-10-09T05:14:14.312083+0000 mon.a (mon.0) 1002 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-10-09T05:14:15.763 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:15 smithi133 bash[102858]: cluster 2023-10-09T05:14:14.312202+0000 mon.a (mon.0) 1003 : cluster 0 osdmap e188: 8 total, 8 up, 6 in 2023-10-09T05:14:15.763 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:15 smithi133 bash[102858]: cluster 2023-10-09T05:14:14.315719+0000 osd.6 (osd.6) 115 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T05:14:15.763 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:15 smithi133 bash[102858]: cluster 2023-10-09T05:14:14.455382+0000 osd.7 (osd.7) 120 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T05:14:15.764 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:15 smithi133 bash[105589]: cluster 2023-10-09T05:14:14.309694+0000 osd.5 (osd.5) 177 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T05:14:15.764 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:15 smithi133 bash[105589]: cluster 2023-10-09T05:14:14.311468+0000 osd.4 (osd.4) 128 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T05:14:15.764 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:15 smithi133 bash[105589]: audit 2023-10-09T05:14:14.311959+0000 mon.a (mon.0) 1000 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 2]}]': finished 2023-10-09T05:14:15.764 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:15 smithi133 bash[105589]: audit 2023-10-09T05:14:14.312044+0000 mon.a (mon.0) 1001 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2023-10-09T05:14:15.764 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:15 smithi133 bash[105589]: audit 2023-10-09T05:14:14.312083+0000 mon.a (mon.0) 1002 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-10-09T05:14:15.764 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:15 smithi133 bash[105589]: cluster 2023-10-09T05:14:14.312202+0000 mon.a (mon.0) 1003 : cluster 0 osdmap e188: 8 total, 8 up, 6 in 2023-10-09T05:14:15.764 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:15 smithi133 bash[105589]: cluster 2023-10-09T05:14:14.315719+0000 osd.6 (osd.6) 115 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T05:14:15.764 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:15 smithi133 bash[105589]: cluster 2023-10-09T05:14:14.455382+0000 osd.7 (osd.7) 120 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T05:14:15.780 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:15.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:15.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:15.885 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:15.906 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:15.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:15.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:16.008 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:16.030 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:16.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:16.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:16.133 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:16.164 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:16.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:16.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:16.268 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:16.290 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:16.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:16.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:16.393 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:16.420 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:16.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:16.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:16.524 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:16.538 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:16 smithi194 bash[92666]: cluster 2023-10-09T05:14:15.331471+0000 osd.5 (osd.5) 178 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T05:14:16.538 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:16 smithi194 bash[92666]: cluster 2023-10-09T05:14:15.335595+0000 osd.4 (osd.4) 129 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T05:14:16.538 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:16 smithi194 bash[92666]: cluster 2023-10-09T05:14:15.336626+0000 osd.6 (osd.6) 116 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T05:14:16.538 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:16 smithi194 bash[92666]: cluster 2023-10-09T05:14:15.336891+0000 osd.4 (osd.4) 130 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T05:14:16.538 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:16 smithi194 bash[92666]: cluster 2023-10-09T05:14:15.340308+0000 mon.a (mon.0) 1004 : cluster 0 osdmap e189: 8 total, 8 up, 6 in 2023-10-09T05:14:16.538 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:16 smithi194 bash[92666]: cluster 2023-10-09T05:14:15.875542+0000 mgr.y (mgr.25277) 772 : cluster 0 pgmap v771: 89 pgs: 89 active+clean; 1.2 GiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 151 MiB/s wr, 2.41k op/s 2023-10-09T05:14:16.553 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:16.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:16.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:16.588 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:16 smithi133 bash[105589]: cluster 2023-10-09T05:14:15.331471+0000 osd.5 (osd.5) 178 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T05:14:16.588 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:16 smithi133 bash[105589]: cluster 2023-10-09T05:14:15.335595+0000 osd.4 (osd.4) 129 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T05:14:16.588 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:16 smithi133 bash[105589]: cluster 2023-10-09T05:14:15.336626+0000 osd.6 (osd.6) 116 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T05:14:16.588 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:16 smithi133 bash[105589]: cluster 2023-10-09T05:14:15.336891+0000 osd.4 (osd.4) 130 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T05:14:16.588 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:16 smithi133 bash[105589]: cluster 2023-10-09T05:14:15.340308+0000 mon.a (mon.0) 1004 : cluster 0 osdmap e189: 8 total, 8 up, 6 in 2023-10-09T05:14:16.588 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:16 smithi133 bash[105589]: cluster 2023-10-09T05:14:15.875542+0000 mgr.y (mgr.25277) 772 : cluster 0 pgmap v771: 89 pgs: 89 active+clean; 1.2 GiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 151 MiB/s wr, 2.41k op/s 2023-10-09T05:14:16.589 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:16 smithi133 bash[102858]: cluster 2023-10-09T05:14:15.331471+0000 osd.5 (osd.5) 178 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T05:14:16.589 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:16 smithi133 bash[102858]: cluster 2023-10-09T05:14:15.335595+0000 osd.4 (osd.4) 129 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T05:14:16.589 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:16 smithi133 bash[102858]: cluster 2023-10-09T05:14:15.336626+0000 osd.6 (osd.6) 116 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T05:14:16.589 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:16 smithi133 bash[102858]: cluster 2023-10-09T05:14:15.336891+0000 osd.4 (osd.4) 130 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T05:14:16.589 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:16 smithi133 bash[102858]: cluster 2023-10-09T05:14:15.340308+0000 mon.a (mon.0) 1004 : cluster 0 osdmap e189: 8 total, 8 up, 6 in 2023-10-09T05:14:16.589 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:16 smithi133 bash[102858]: cluster 2023-10-09T05:14:15.875542+0000 mgr.y (mgr.25277) 772 : cluster 0 pgmap v771: 89 pgs: 89 active+clean; 1.2 GiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 151 MiB/s wr, 2.41k op/s 2023-10-09T05:14:16.640 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 12 16 24904 24888 129.61 103.375 0.00730005 0.007709 2023-10-09T05:14:16.657 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:16.689 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:16.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:16.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:16.794 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:16.823 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:16.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:16.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:16.929 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:16.955 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:16.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:16.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:17.059 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:17.106 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:17.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:17.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:17.211 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:17.237 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:17.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:17.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:17.343 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:17.345 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:17 smithi133 bash[105589]: cluster 2023-10-09T05:14:15.957014+0000 osd.7 (osd.7) 121 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T05:14:17.345 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:17 smithi133 bash[105589]: cluster 2023-10-09T05:14:15.959014+0000 osd.7 (osd.7) 122 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T05:14:17.345 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:17 smithi133 bash[102858]: cluster 2023-10-09T05:14:15.957014+0000 osd.7 (osd.7) 121 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T05:14:17.345 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:17 smithi133 bash[102858]: cluster 2023-10-09T05:14:15.959014+0000 osd.7 (osd.7) 122 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T05:14:17.368 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:17.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:17.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:17.474 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:17.503 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:17.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:17.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:17.514 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:14:17.606 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:17.617 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:17 smithi194 bash[92666]: cluster 2023-10-09T05:14:15.957014+0000 osd.7 (osd.7) 121 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T05:14:17.618 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:17 smithi194 bash[92666]: cluster 2023-10-09T05:14:15.959014+0000 osd.7 (osd.7) 122 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T05:14:17.634 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:17.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:17.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:17.640 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 13 16 26644 26628 128.005 108.75 0.0108194 0.00780541 2023-10-09T05:14:17.740 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:17.768 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:17.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:17.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:17.873 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:17.912 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:17.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:17.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:18.017 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:18.044 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:18.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:18.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:18.151 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:18.180 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:18.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:18.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:18.284 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:18.313 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:18.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:18.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:18.419 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:18.454 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:18.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:18.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:18.474 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:18 smithi133 bash[102858]: cluster 2023-10-09T05:14:17.876127+0000 mgr.y (mgr.25277) 773 : cluster 0 pgmap v772: 89 pgs: 89 active+clean; 1.3 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-10-09T05:14:18.475 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:18 smithi133 bash[105589]: cluster 2023-10-09T05:14:17.876127+0000 mgr.y (mgr.25277) 773 : cluster 0 pgmap v772: 89 pgs: 89 active+clean; 1.3 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-10-09T05:14:18.559 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:18.586 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:18.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:18.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:18.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:18 smithi194 bash[92666]: cluster 2023-10-09T05:14:17.876127+0000 mgr.y (mgr.25277) 773 : cluster 0 pgmap v772: 89 pgs: 89 active+clean; 1.3 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-10-09T05:14:18.640 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 14 16 28340 28324 126.432 106 0.00379032 0.00790378 2023-10-09T05:14:18.690 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:18.716 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:18.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:18.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:18.821 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:18.857 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:18.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:18.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:18.963 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:18.991 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:18.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:18.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:19.098 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:19.120 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:19.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:19.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:19.224 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:19.246 INFO:teuthology.orchestra.run.smithi194.stdout:{ 2023-10-09T05:14:19.246 INFO:teuthology.orchestra.run.smithi194.stdout: "ops": [], 2023-10-09T05:14:19.247 INFO:teuthology.orchestra.run.smithi194.stdout: "complaint_time": 30, 2023-10-09T05:14:19.247 INFO:teuthology.orchestra.run.smithi194.stdout: "num_blocked_ops": 0 2023-10-09T05:14:19.247 INFO:teuthology.orchestra.run.smithi194.stdout:} 2023-10-09T05:14:19.251 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:19.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:19.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:19.356 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:19.380 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:19.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:19.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:19.485 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:19.515 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:19.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:19.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:19.621 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:19.641 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 15 16 30036 30020 125.069 106 0.00875653 0.00798875 2023-10-09T05:14:19.648 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:19.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:19.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:19.750 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:19.774 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:19.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:19.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:19.878 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:19.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:19 smithi194 bash[92666]: audit 2023-10-09T05:14:18.871211+0000 mon.a (mon.0) 1005 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:14:19.909 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:19.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:19.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:20.013 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:20.042 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:20.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:20.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:20.148 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:20.157 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:19 smithi133 bash[105589]: audit 2023-10-09T05:14:18.871211+0000 mon.a (mon.0) 1005 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:14:20.158 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:19 smithi133 bash[102858]: audit 2023-10-09T05:14:18.871211+0000 mon.a (mon.0) 1005 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:14:20.181 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:20.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:20.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:20.288 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:20.314 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:20.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:20.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:20.417 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:20.441 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:20.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:20.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:20.545 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:20.569 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:20.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:20.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:20.641 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 16 16 31619 31603 123.436 98.9375 0.0121648 0.00809358 2023-10-09T05:14:20.673 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:20.699 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:20.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:20.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:20.804 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:20.828 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:20.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:20.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:20.866 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T05:14:20.931 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:20.965 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:20.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:20.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:20.975 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:14:21.019 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:14:21.066 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:14:21.071 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:21.082 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:20 smithi194 bash[92666]: cluster 2023-10-09T05:14:19.876906+0000 mgr.y (mgr.25277) 774 : cluster 0 pgmap v773: 89 pgs: 89 active+clean; 1.5 GiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-09T05:14:21.096 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:21.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:21.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:21.104 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:14:21.138 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:20 smithi133 bash[102858]: cluster 2023-10-09T05:14:19.876906+0000 mgr.y (mgr.25277) 774 : cluster 0 pgmap v773: 89 pgs: 89 active+clean; 1.5 GiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-09T05:14:21.138 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:20 smithi133 bash[105589]: cluster 2023-10-09T05:14:19.876906+0000 mgr.y (mgr.25277) 774 : cluster 0 pgmap v773: 89 pgs: 89 active+clean; 1.5 GiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-09T05:14:21.141 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:14:21.182 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:14:21.199 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:21.228 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:14:21.236 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:21.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:21.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:21.272 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:14:21.277 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:14:21.277 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:14:21.278 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:14:21.278 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:14:21.278 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:14:21.278 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:14:21.278 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:14:21.278 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:14:21.340 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:21.367 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:21.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:21.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:21.470 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:21.497 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:21.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:21.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:21.600 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:21.635 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:21.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:21.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:21.641 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 17 16 32975 32959 121.159 84.75 0.015398 0.00824632 2023-10-09T05:14:21.741 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:21.769 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:21.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:21.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:21.873 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:21.890 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:21 smithi133 bash[102858]: audit 2023-10-09T05:14:20.912960+0000 mon.b (mon.1) 17 : audit 0 from='client.? 172.21.15.133:0/4230426381' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:14:21.890 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:21 smithi133 bash[102858]: audit 2023-10-09T05:14:21.622073+0000 mon.c (mon.2) 74 : audit 1 from='client.? 172.21.15.133:0/26948204' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:14:21.890 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:21 smithi133 bash[102858]: audit 2023-10-09T05:14:21.623281+0000 mon.a (mon.0) 1006 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:14:21.890 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:21 smithi133 bash[105589]: audit 2023-10-09T05:14:20.912960+0000 mon.b (mon.1) 17 : audit 0 from='client.? 172.21.15.133:0/4230426381' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:14:21.891 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:21 smithi133 bash[105589]: audit 2023-10-09T05:14:21.622073+0000 mon.c (mon.2) 74 : audit 1 from='client.? 172.21.15.133:0/26948204' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:14:21.891 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:21 smithi133 bash[105589]: audit 2023-10-09T05:14:21.623281+0000 mon.a (mon.0) 1006 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:14:21.897 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:21.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:21.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:21.909 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is set 2023-10-09T05:14:22.001 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:22.039 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:22.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:22.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:22.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:21 smithi194 bash[92666]: audit 2023-10-09T05:14:20.912960+0000 mon.b (mon.1) 17 : audit 0 from='client.? 172.21.15.133:0/4230426381' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:14:22.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:21 smithi194 bash[92666]: audit 2023-10-09T05:14:21.622073+0000 mon.c (mon.2) 74 : audit 1 from='client.? 172.21.15.133:0/26948204' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:14:22.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:21 smithi194 bash[92666]: audit 2023-10-09T05:14:21.623281+0000 mon.a (mon.0) 1006 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:14:22.145 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:22.171 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:22.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:22.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:22.276 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:22.307 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:22.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:22.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:22.412 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:22.451 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:22.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:22.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:22.558 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:22.585 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:22.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:22.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:22.641 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 18 16 34587 34571 120.025 100.75 0.0091027 0.00832575 2023-10-09T05:14:22.690 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:22.724 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:22.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:22.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:22.828 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:22.862 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:22.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:22.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:22.968 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:22.976 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:22 smithi194 bash[92666]: cluster 2023-10-09T05:14:21.877546+0000 mgr.y (mgr.25277) 775 : cluster 0 pgmap v774: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-10-09T05:14:22.976 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:22 smithi194 bash[92666]: cluster 2023-10-09T05:14:21.885421+0000 mon.a (mon.0) 1007 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:14:22.976 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:22 smithi194 bash[92666]: cluster 2023-10-09T05:14:21.895579+0000 osd.5 (osd.5) 179 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T05:14:22.976 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:22 smithi194 bash[92666]: audit 2023-10-09T05:14:21.898975+0000 mon.a (mon.0) 1008 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:14:22.977 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:22 smithi194 bash[92666]: cluster 2023-10-09T05:14:21.899270+0000 mon.a (mon.0) 1009 : cluster 0 osdmap e190: 8 total, 8 up, 6 in 2023-10-09T05:14:22.977 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:22 smithi194 bash[92666]: cluster 2023-10-09T05:14:21.900849+0000 osd.6 (osd.6) 117 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T05:14:22.977 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:22 smithi194 bash[92666]: cluster 2023-10-09T05:14:21.902117+0000 osd.6 (osd.6) 118 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T05:14:22.977 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:22 smithi194 bash[92666]: cluster 2023-10-09T05:14:21.904665+0000 osd.4 (osd.4) 131 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T05:14:22.977 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:22 smithi194 bash[92666]: cluster 2023-10-09T05:14:21.965511+0000 osd.7 (osd.7) 123 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T05:14:22.994 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:22.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:22.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:23.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:22 smithi133 bash[102858]: cluster 2023-10-09T05:14:21.877546+0000 mgr.y (mgr.25277) 775 : cluster 0 pgmap v774: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-10-09T05:14:23.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:22 smithi133 bash[102858]: cluster 2023-10-09T05:14:21.885421+0000 mon.a (mon.0) 1007 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:14:23.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:22 smithi133 bash[102858]: cluster 2023-10-09T05:14:21.895579+0000 osd.5 (osd.5) 179 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T05:14:23.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:22 smithi133 bash[102858]: audit 2023-10-09T05:14:21.898975+0000 mon.a (mon.0) 1008 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:14:23.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:22 smithi133 bash[102858]: cluster 2023-10-09T05:14:21.899270+0000 mon.a (mon.0) 1009 : cluster 0 osdmap e190: 8 total, 8 up, 6 in 2023-10-09T05:14:23.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:22 smithi133 bash[102858]: cluster 2023-10-09T05:14:21.900849+0000 osd.6 (osd.6) 117 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T05:14:23.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:22 smithi133 bash[102858]: cluster 2023-10-09T05:14:21.902117+0000 osd.6 (osd.6) 118 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T05:14:23.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:22 smithi133 bash[102858]: cluster 2023-10-09T05:14:21.904665+0000 osd.4 (osd.4) 131 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T05:14:23.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:22 smithi133 bash[102858]: cluster 2023-10-09T05:14:21.965511+0000 osd.7 (osd.7) 123 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T05:14:23.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:22 smithi133 bash[105589]: cluster 2023-10-09T05:14:21.877546+0000 mgr.y (mgr.25277) 775 : cluster 0 pgmap v774: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-10-09T05:14:23.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:22 smithi133 bash[105589]: cluster 2023-10-09T05:14:21.885421+0000 mon.a (mon.0) 1007 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:14:23.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:22 smithi133 bash[105589]: cluster 2023-10-09T05:14:21.895579+0000 osd.5 (osd.5) 179 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T05:14:23.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:22 smithi133 bash[105589]: audit 2023-10-09T05:14:21.898975+0000 mon.a (mon.0) 1008 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:14:23.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:22 smithi133 bash[105589]: cluster 2023-10-09T05:14:21.899270+0000 mon.a (mon.0) 1009 : cluster 0 osdmap e190: 8 total, 8 up, 6 in 2023-10-09T05:14:23.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:22 smithi133 bash[105589]: cluster 2023-10-09T05:14:21.900849+0000 osd.6 (osd.6) 117 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T05:14:23.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:22 smithi133 bash[105589]: cluster 2023-10-09T05:14:21.902117+0000 osd.6 (osd.6) 118 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T05:14:23.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:22 smithi133 bash[105589]: cluster 2023-10-09T05:14:21.904665+0000 osd.4 (osd.4) 131 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T05:14:23.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:22 smithi133 bash[105589]: cluster 2023-10-09T05:14:21.965511+0000 osd.7 (osd.7) 123 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T05:14:23.097 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:23.126 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:23.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:23.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:23.230 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:23.258 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:23.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:23.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:23.362 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:23.397 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:23.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:23.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:23.501 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:23.530 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:23.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:23.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:23.634 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:23.641 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 19 16 36071 36055 118.589 92.75 0.00509805 0.00842561 2023-10-09T05:14:23.661 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:23.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:23.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:23.766 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:23.794 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:23.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:23.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:23.898 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:23.931 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:23.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:23.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:24.036 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:24.061 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:24.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:24.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:24.165 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:24.199 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:24.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:24.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:24.302 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:24.332 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:24.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:24.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:24.436 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:24.445 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:14:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:14:24] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.43.0" 2023-10-09T05:14:24.446 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:24 smithi133 bash[102858]: cluster 2023-10-09T05:14:23.878070+0000 mgr.y (mgr.25277) 776 : cluster 0 pgmap v776: 89 pgs: 89 active+clean; 1.9 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T05:14:24.446 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:24 smithi133 bash[105589]: cluster 2023-10-09T05:14:23.878070+0000 mgr.y (mgr.25277) 776 : cluster 0 pgmap v776: 89 pgs: 89 active+clean; 1.9 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T05:14:24.463 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:24.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:24.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:24.567 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:24.581 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:24 smithi194 bash[92666]: cluster 2023-10-09T05:14:23.878070+0000 mgr.y (mgr.25277) 776 : cluster 0 pgmap v776: 89 pgs: 89 active+clean; 1.9 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T05:14:24.600 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:24.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:24.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:24.641 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T05:14:24.641431+0000 min lat: 0.00128168 max lat: 0.157772 avg lat: 0.00857819 2023-10-09T05:14:24.641 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:14:24.641 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 20 16 37299 37283 116.496 76.75 0.00669294 0.00857819 2023-10-09T05:14:24.705 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:24.730 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:24.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:24.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:24.834 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:24.870 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:24.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:24.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:24.975 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:25.000 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:25.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:25.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:25.103 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:25.135 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:25.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:25.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:25.239 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:25.275 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:25.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:25.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:25.380 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:25.416 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:25.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:25.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:25.520 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:25.549 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:25.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:25.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:25.641 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 21 16 38847 38831 115.556 96.75 0.0075562 0.00864812 2023-10-09T05:14:25.653 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:25.682 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:25.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:25.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:25.723 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:14:25.787 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:25.816 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:25.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:25.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:25.920 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:25.946 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:25.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:25.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:26.049 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:26.080 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:26.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:26.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:26.184 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:26.210 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:26.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:26.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:26.316 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:26.346 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:26.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:26.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:26.451 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:26.476 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:26.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:26.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:26.581 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:26.610 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:26.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:26.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:26.641 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 22 16 40368 40352 114.624 95.0625 0.0142564 0.00871829 2023-10-09T05:14:26.713 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:26.741 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:26.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:26.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:26.845 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:26.870 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:26.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:26.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:26.973 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:27.003 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:27.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:27.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:27.108 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:27.115 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:26 smithi194 bash[92666]: cluster 2023-10-09T05:14:25.878802+0000 mgr.y (mgr.25277) 777 : cluster 0 pgmap v777: 89 pgs: 89 active+clean; 2.2 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-10-09T05:14:27.134 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:27.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:27.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:27.205 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:14:27.214 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:26 smithi133 bash[102858]: cluster 2023-10-09T05:14:25.878802+0000 mgr.y (mgr.25277) 777 : cluster 0 pgmap v777: 89 pgs: 89 active+clean; 2.2 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-10-09T05:14:27.215 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:26 smithi133 bash[105589]: cluster 2023-10-09T05:14:25.878802+0000 mgr.y (mgr.25277) 777 : cluster 0 pgmap v777: 89 pgs: 89 active+clean; 2.2 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-10-09T05:14:27.238 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:27.261 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:27.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:27.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:27.364 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:27.405 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:27.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:27.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:27.417 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:14:27.448 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:14:27.448 INFO:teuthology.orchestra.run.smithi133.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T05:14:27.448 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:14:27.509 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:27.538 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:27.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:27.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:27.641 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:27.642 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 23 16 41633 41617 113.077 79.0625 0.0116769 0.00883613 2023-10-09T05:14:27.668 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:27.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:27.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:27.772 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:27.799 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:27.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:27.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:27.904 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:27.926 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:27.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:27.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:28.030 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:28.055 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:28.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:28.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:28.159 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:28.188 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:28.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:28.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:28.290 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:28.319 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:28.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:28.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:28.424 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:28.449 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:28.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:28.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:28.555 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:28.586 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:28.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:28.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:28.642 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 24 16 43016 43000 111.967 86.4375 0.0274358 0.00892468 2023-10-09T05:14:28.693 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:28.721 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:28.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:28.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:28.824 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:28.848 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:28.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:28.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:28.952 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:28.977 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:28.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:28.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:29.081 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:29.106 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:29.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:29.111 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:28 smithi133 bash[102858]: cluster 2023-10-09T05:14:27.879370+0000 mgr.y (mgr.25277) 778 : cluster 0 pgmap v778: 89 pgs: 89 active+clean; 2.3 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T05:14:29.111 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:28 smithi133 bash[102858]: audit 2023-10-09T05:14:28.454539+0000 mon.a (mon.0) 1010 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:14:29.112 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:28 smithi133 bash[102858]: audit 2023-10-09T05:14:28.455986+0000 mon.a (mon.0) 1011 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:14:29.112 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:28 smithi133 bash[105589]: cluster 2023-10-09T05:14:27.879370+0000 mgr.y (mgr.25277) 778 : cluster 0 pgmap v778: 89 pgs: 89 active+clean; 2.3 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T05:14:29.112 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:28 smithi133 bash[105589]: audit 2023-10-09T05:14:28.454539+0000 mon.a (mon.0) 1010 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:14:29.112 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:28 smithi133 bash[105589]: audit 2023-10-09T05:14:28.455986+0000 mon.a (mon.0) 1011 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:14:29.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:29.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:28 smithi194 bash[92666]: cluster 2023-10-09T05:14:27.879370+0000 mgr.y (mgr.25277) 778 : cluster 0 pgmap v778: 89 pgs: 89 active+clean; 2.3 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T05:14:29.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:28 smithi194 bash[92666]: audit 2023-10-09T05:14:28.454539+0000 mon.a (mon.0) 1010 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:14:29.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:28 smithi194 bash[92666]: audit 2023-10-09T05:14:28.455986+0000 mon.a (mon.0) 1011 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:14:29.213 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:29.241 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:29.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:29.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:29.347 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:29.380 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:29.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:29.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:29.484 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:29.515 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:29.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:29.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:29.620 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:29.642 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 25 16 44374 44358 110.883 84.875 0.00966148 0.00901236 2023-10-09T05:14:29.649 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:29.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:29.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:29.752 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:29.778 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:29.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:29.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:29.883 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:29.914 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:29.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:29.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:30.020 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:30.042 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:30.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:30.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:30.148 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:30.172 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:30.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:30.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:30.277 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:30.300 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:30.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:30.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:30.403 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:30.428 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:30.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:30.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:30.532 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:30.558 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:30.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:30.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:30.642 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 26 16 45728 45712 109.873 84.625 0.00663215 0.00909515 2023-10-09T05:14:30.661 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:30.686 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:30.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:30.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:30.792 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:30.818 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:30.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:30.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:30.927 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:30.951 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:30.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:30.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:31.055 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:31.085 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:31.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:31.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:31.188 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:31.197 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:30 smithi133 bash[105589]: cluster 2023-10-09T05:14:29.880024+0000 mgr.y (mgr.25277) 779 : cluster 0 pgmap v779: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T05:14:31.198 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:30 smithi133 bash[102858]: cluster 2023-10-09T05:14:29.880024+0000 mgr.y (mgr.25277) 779 : cluster 0 pgmap v779: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T05:14:31.226 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:31.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:31.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:31.283 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:30 smithi194 bash[92666]: cluster 2023-10-09T05:14:29.880024+0000 mgr.y (mgr.25277) 779 : cluster 0 pgmap v779: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T05:14:31.333 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:31.358 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:31.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:31.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:31.461 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:31.494 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:31.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:31.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:31.568 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:14:31.569 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:14:31.598 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:31.627 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:31.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:31.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:31.642 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 27 15 47048 47033 108.861 82.5625 0.00992928 0.00917943 2023-10-09T05:14:31.732 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:31.755 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:31.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:31.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:31.859 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:31.885 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:31.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:31.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:31.990 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:32.021 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:32.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:32.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:32.070 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:14:32.071 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:14:32.125 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:32.151 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:32.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:32.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:32.254 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:32.283 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:32.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:32.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:32.387 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:32.425 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:32.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:32.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:32.528 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:32.577 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:32.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:32.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:32.623 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T05:14:32.642 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 28 16 48298 48282 107.761 78.0625 0.010907 0.0092739 2023-10-09T05:14:32.681 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:32.706 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:32.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:32.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:32.809 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:32.835 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:32.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:32.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:32.939 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:32.974 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:32.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:32.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:33.079 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:33.105 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:33.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:33.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:33.210 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:33.219 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:32 smithi133 bash[105589]: cluster 2023-10-09T05:14:31.880897+0000 mgr.y (mgr.25277) 780 : cluster 0 pgmap v780: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T05:14:33.220 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:32 smithi133 bash[102858]: cluster 2023-10-09T05:14:31.880897+0000 mgr.y (mgr.25277) 780 : cluster 0 pgmap v780: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T05:14:33.240 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:33.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:33.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:33.344 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:33.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:33.375 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:33.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:33.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:32 smithi194 bash[92666]: cluster 2023-10-09T05:14:31.880897+0000 mgr.y (mgr.25277) 780 : cluster 0 pgmap v780: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T05:14:33.476 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:33.501 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:33.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:33.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:33.605 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:33.633 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:33.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:33.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:33.642 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 29 16 49595 49579 106.84 81.0625 0.00641891 0.00935454 2023-10-09T05:14:33.738 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:33.766 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:33.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:33.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:33.858 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 6, 1] out_osds: [5, 7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:14:33.858 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:14:33.858 INFO:tasks.thrashosds.thrasher:Reweighting osd 2 to 0.5045905919794463 2023-10-09T05:14:33.859 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 0.5045905919794463 2023-10-09T05:14:33.871 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:33.901 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:33.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:33.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:34.008 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:34.039 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:34.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:34.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:34.144 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:34.168 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:34.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:34.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:34.271 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:34.293 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:34.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:34.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:34.396 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:34.404 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:34 smithi194 bash[92666]: cluster 2023-10-09T05:14:33.881424+0000 mgr.y (mgr.25277) 781 : cluster 0 pgmap v781: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-09T05:14:34.422 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:34.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:34.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:34.525 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:34.543 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:34 smithi133 bash[102858]: cluster 2023-10-09T05:14:33.881424+0000 mgr.y (mgr.25277) 781 : cluster 0 pgmap v781: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-09T05:14:34.543 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:14:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:14:34] "GET /metrics HTTP/1.1" 200 36463 "" "Prometheus/2.43.0" 2023-10-09T05:14:34.543 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:34 smithi133 bash[105589]: cluster 2023-10-09T05:14:33.881424+0000 mgr.y (mgr.25277) 781 : cluster 0 pgmap v781: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-09T05:14:34.548 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:34.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:34.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:34.642 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 30 16 50858 50842 105.91 78.9375 0.00368972 0.00943555 2023-10-09T05:14:34.651 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:34.675 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:34.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:34.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:34.780 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:34.807 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:34.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:34.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:34.911 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:34.946 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:34.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:34.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:35.050 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:35.079 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:35.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:35.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:35.183 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:35.219 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:35.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:35.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:35.324 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:35.353 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:35.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:35.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:35.457 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:35.482 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:35.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:35.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:35.589 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:35.618 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:35.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:35.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:35.642 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 31 16 52121 52105 105.039 78.9375 0.00639641 0.0095149 2023-10-09T05:14:35.722 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:35.750 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:35.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:35.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:35.855 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:35.882 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:35.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:35.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:35.986 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:36.012 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:36.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:36.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:36.116 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:36.148 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:36.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:36.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:36.252 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:36.277 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:36.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:36.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:36.381 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:36.409 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:36.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:36.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:36.513 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:36.548 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:36.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:36.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:36.642 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 32 16 53345 53329 104.147 76.5 0.00851341 0.00959511 2023-10-09T05:14:36.651 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:36.681 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:36.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:36.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:36.783 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:36.814 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:36.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:36.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:36.919 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:36.944 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:36.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:36.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:37.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:36 smithi133 bash[102858]: cluster 2023-10-09T05:14:35.882005+0000 mgr.y (mgr.25277) 782 : cluster 0 pgmap v782: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T05:14:37.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:36 smithi133 bash[105589]: cluster 2023-10-09T05:14:35.882005+0000 mgr.y (mgr.25277) 782 : cluster 0 pgmap v782: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T05:14:37.049 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:37.076 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:37.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:37.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:37.180 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:37.204 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:37.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:37.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:37.308 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:37.332 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:37.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:37.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:37.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:36 smithi194 bash[92666]: cluster 2023-10-09T05:14:35.882005+0000 mgr.y (mgr.25277) 782 : cluster 0 pgmap v782: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T05:14:37.435 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:37.440 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:14:37.463 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:37.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:37.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:37.568 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:37.594 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:37.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:37.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:37.642 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 33 16 54573 54557 103.317 76.75 0.0037298 0.00967317 2023-10-09T05:14:37.698 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:37.722 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:37.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:37.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:37.825 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:37.861 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:37.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:37.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:37.965 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:37.992 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:37.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:37.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:38.096 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:38.129 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:38.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:38.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:38.232 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:38.257 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:38.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:38.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:38.365 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:38.393 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:38.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:38.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:38.498 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:38.536 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:38.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:38.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:38.641 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:38.642 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 34 16 55739 55723 102.421 72.875 0.00864678 0.00975749 2023-10-09T05:14:38.664 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:38.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:38.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:38.771 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:38.808 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:38.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:38.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:38.912 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:38.939 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:38.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:38.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:39.045 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:39.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:38 smithi133 bash[102858]: cluster 2023-10-09T05:14:37.882473+0000 mgr.y (mgr.25277) 783 : cluster 0 pgmap v783: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T05:14:39.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:38 smithi133 bash[105589]: cluster 2023-10-09T05:14:37.882473+0000 mgr.y (mgr.25277) 783 : cluster 0 pgmap v783: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T05:14:39.069 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:39.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:39.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:39.173 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:39.199 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:39.201 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:38 smithi194 bash[92666]: cluster 2023-10-09T05:14:37.882473+0000 mgr.y (mgr.25277) 783 : cluster 0 pgmap v783: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T05:14:39.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:39.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:39.303 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:39.326 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:39.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:39.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:39.431 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:39.462 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:39.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:39.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:39.567 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:39.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:39.604 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:39.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:39.642 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 35 16 56919 56903 101.602 73.75 0.0101983 0.00983638 2023-10-09T05:14:39.705 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:39.731 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:39.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:39.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:39.813 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:14:39.834 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:39.877 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:39.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:39.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:39.986 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:40.021 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:40.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:40.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:40.126 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:40.149 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:40.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:40.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:40.257 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:40.282 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:40.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:40.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:40.389 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:40.417 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:40.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:40.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:40.524 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:40.558 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:40.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:40.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:40.643 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 36 16 58075 58059 100.786 72.25 0.00761213 0.00991519 2023-10-09T05:14:40.661 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:40.684 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:40.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:40.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:40.787 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:40.813 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:40.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:40.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:40.918 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:40.950 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:40.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:40.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:41.055 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:41.093 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:41.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:41.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:41.198 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:41.207 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:40 smithi194 bash[92666]: cluster 2023-10-09T05:14:39.884277+0000 mgr.y (mgr.25277) 784 : cluster 0 pgmap v784: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 82 MiB/s wr, 1.30k op/s 2023-10-09T05:14:41.225 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:41.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:41.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:41.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:40 smithi133 bash[102858]: cluster 2023-10-09T05:14:39.884277+0000 mgr.y (mgr.25277) 784 : cluster 0 pgmap v784: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 82 MiB/s wr, 1.30k op/s 2023-10-09T05:14:41.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:40 smithi133 bash[105589]: cluster 2023-10-09T05:14:39.884277+0000 mgr.y (mgr.25277) 784 : cluster 0 pgmap v784: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 82 MiB/s wr, 1.30k op/s 2023-10-09T05:14:41.329 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:41.357 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:41.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:41.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:41.463 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:41.492 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:41.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:41.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:41.597 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:41.624 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:41.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:41.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:41.643 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 37 16 59289 59273 100.113 75.875 0.0100292 0.00998214 2023-10-09T05:14:41.731 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:41.756 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:41.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:41.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:41.861 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:41.886 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:41.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:41.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:41.990 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:42.019 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:42.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:42.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:42.124 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:42.150 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:42.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:42.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:42.262 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:42.285 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:42.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:42.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:42.388 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:42.425 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:42.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:42.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:42.528 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:42.562 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:42.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:42.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:42.643 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 38 16 60460 60444 99.4041 73.1875 0.0155449 0.0100547 2023-10-09T05:14:42.667 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:42.697 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:42.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:42.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:42.802 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:42.828 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:42.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:42.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:42.892 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:14:42.892 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:14:42.932 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:42.958 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:42.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:42.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:43.061 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:43.095 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:43.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:43.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:43.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:42 smithi194 bash[92666]: cluster 2023-10-09T05:14:41.884834+0000 mgr.y (mgr.25277) 785 : cluster 0 pgmap v785: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-10-09T05:14:43.199 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:43.225 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:43.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:43.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:43.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:42 smithi133 bash[102858]: cluster 2023-10-09T05:14:41.884834+0000 mgr.y (mgr.25277) 785 : cluster 0 pgmap v785: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-10-09T05:14:43.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:42 smithi133 bash[105589]: cluster 2023-10-09T05:14:41.884834+0000 mgr.y (mgr.25277) 785 : cluster 0 pgmap v785: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-10-09T05:14:43.330 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:43.358 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:43.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:43.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:43.464 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:43.495 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:43.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:43.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:43.599 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:43.643 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 39 16 61413 61397 98.3824 59.5625 0.0138698 0.0101566 2023-10-09T05:14:43.650 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:43.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:43.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:43.681 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:14:43.721 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:14:43.755 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:43.763 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:14:43.779 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:43.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:43.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:43.820 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:14:43.860 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:14:43.882 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:43.905 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:14:43.918 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:43.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:43.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:43.944 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:14:43.990 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:14:43.992 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:14:43.992 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:14:43.992 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:14:43.992 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:14:43.992 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:14:43.992 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:14:43.992 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:14:43.992 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:14:44.006 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T05:14:44.025 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:44.076 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:44.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:44.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:44.180 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:44.210 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:44.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:44.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:44.316 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:44.346 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is set 2023-10-09T05:14:44.352 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:44.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:44.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:44.456 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:44.485 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:44.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:44.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:44.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:14:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:14:44] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.43.0" 2023-10-09T05:14:44.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:44 smithi133 bash[105589]: audit 2023-10-09T05:14:43.447739+0000 mon.a (mon.0) 1012 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:14:44.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:44 smithi133 bash[105589]: audit 2023-10-09T05:14:43.643571+0000 mon.c (mon.2) 75 : audit 0 from='client.? 172.21.15.133:0/2095386813' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:14:44.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:44 smithi133 bash[105589]: cluster 2023-10-09T05:14:43.885501+0000 mgr.y (mgr.25277) 786 : cluster 0 pgmap v786: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T05:14:44.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:44 smithi133 bash[105589]: audit 2023-10-09T05:14:43.933858+0000 mon.c (mon.2) 76 : audit 1 from='client.? 172.21.15.133:0/2045299549' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:14:44.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:44 smithi133 bash[105589]: audit 2023-10-09T05:14:43.934465+0000 mon.a (mon.0) 1013 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:14:44.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:44 smithi133 bash[102858]: audit 2023-10-09T05:14:43.447739+0000 mon.a (mon.0) 1012 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:14:44.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:44 smithi133 bash[102858]: audit 2023-10-09T05:14:43.643571+0000 mon.c (mon.2) 75 : audit 0 from='client.? 172.21.15.133:0/2095386813' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:14:44.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:44 smithi133 bash[102858]: cluster 2023-10-09T05:14:43.885501+0000 mgr.y (mgr.25277) 786 : cluster 0 pgmap v786: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T05:14:44.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:44 smithi133 bash[102858]: audit 2023-10-09T05:14:43.933858+0000 mon.c (mon.2) 76 : audit 1 from='client.? 172.21.15.133:0/2045299549' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:14:44.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:44 smithi133 bash[102858]: audit 2023-10-09T05:14:43.934465+0000 mon.a (mon.0) 1013 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:14:44.590 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:44.599 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:44 smithi194 bash[92666]: audit 2023-10-09T05:14:43.447739+0000 mon.a (mon.0) 1012 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:14:44.599 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:44 smithi194 bash[92666]: audit 2023-10-09T05:14:43.643571+0000 mon.c (mon.2) 75 : audit 0 from='client.? 172.21.15.133:0/2095386813' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:14:44.599 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:44 smithi194 bash[92666]: cluster 2023-10-09T05:14:43.885501+0000 mgr.y (mgr.25277) 786 : cluster 0 pgmap v786: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T05:14:44.599 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:44 smithi194 bash[92666]: audit 2023-10-09T05:14:43.933858+0000 mon.c (mon.2) 76 : audit 1 from='client.? 172.21.15.133:0/2045299549' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:14:44.599 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:44 smithi194 bash[92666]: audit 2023-10-09T05:14:43.934465+0000 mon.a (mon.0) 1013 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:14:44.624 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:44.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:44.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:44.643 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T05:14:44.643386+0000 min lat: 0.00128168 max lat: 0.157772 avg lat: 0.0102434 2023-10-09T05:14:44.643 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:14:44.643 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 40 16 62441 62425 97.5289 64.25 0.00305207 0.0102434 2023-10-09T05:14:44.730 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:44.755 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:44.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:44.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:44.860 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:44.892 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:44.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:44.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:44.996 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:45.023 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:45.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:45.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:45.127 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:45.156 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:45.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:45.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:45.261 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:45.288 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:45.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:45.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:45.393 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:45.403 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:45 smithi133 bash[102858]: cluster 2023-10-09T05:14:44.316247+0000 mon.a (mon.0) 1014 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:14:45.403 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:45 smithi133 bash[102858]: audit 2023-10-09T05:14:44.336138+0000 mon.a (mon.0) 1015 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:14:45.403 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:45 smithi133 bash[102858]: cluster 2023-10-09T05:14:44.336242+0000 mon.a (mon.0) 1016 : cluster 0 osdmap e191: 8 total, 8 up, 6 in 2023-10-09T05:14:45.403 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:45 smithi133 bash[102858]: cluster 2023-10-09T05:14:44.337544+0000 osd.6 (osd.6) 119 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T05:14:45.403 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:45 smithi133 bash[105589]: cluster 2023-10-09T05:14:44.316247+0000 mon.a (mon.0) 1014 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:14:45.403 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:45 smithi133 bash[105589]: audit 2023-10-09T05:14:44.336138+0000 mon.a (mon.0) 1015 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:14:45.403 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:45 smithi133 bash[105589]: cluster 2023-10-09T05:14:44.336242+0000 mon.a (mon.0) 1016 : cluster 0 osdmap e191: 8 total, 8 up, 6 in 2023-10-09T05:14:45.404 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:45 smithi133 bash[105589]: cluster 2023-10-09T05:14:44.337544+0000 osd.6 (osd.6) 119 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T05:14:45.421 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:45.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:45.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:45.524 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:45.542 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:45.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:45.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:45.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:45 smithi194 bash[92666]: cluster 2023-10-09T05:14:44.316247+0000 mon.a (mon.0) 1014 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:14:45.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:45 smithi194 bash[92666]: audit 2023-10-09T05:14:44.336138+0000 mon.a (mon.0) 1015 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:14:45.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:45 smithi194 bash[92666]: cluster 2023-10-09T05:14:44.336242+0000 mon.a (mon.0) 1016 : cluster 0 osdmap e191: 8 total, 8 up, 6 in 2023-10-09T05:14:45.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:45 smithi194 bash[92666]: cluster 2023-10-09T05:14:44.337544+0000 osd.6 (osd.6) 119 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T05:14:45.643 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 41 16 63346 63330 96.5294 56.5625 0.00427167 0.0103181 2023-10-09T05:14:45.646 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:45.681 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:45.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:45.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:45.791 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:45.816 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:45.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:45.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:45.920 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:45.946 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:45.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:45.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:46.051 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:46.083 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:46.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:46.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:46.188 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:46.215 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:46.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:46.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:46.318 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:46.347 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:46 smithi194 bash[92666]: cluster 2023-10-09T05:14:45.886001+0000 mgr.y (mgr.25277) 787 : cluster 0 pgmap v788: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T05:14:46.355 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:46.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:46.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:46.458 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:46.483 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:46.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:46.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:46.589 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:46.621 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:46.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:46.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:46.643 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 42 16 64376 64360 95.7637 64.375 0.0156711 0.0104363 2023-10-09T05:14:46.724 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:46.754 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:46.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:46.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:46.852 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:46 smithi133 bash[102858]: cluster 2023-10-09T05:14:45.886001+0000 mgr.y (mgr.25277) 787 : cluster 0 pgmap v788: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T05:14:46.852 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:46 smithi133 bash[105589]: cluster 2023-10-09T05:14:45.886001+0000 mgr.y (mgr.25277) 787 : cluster 0 pgmap v788: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T05:14:46.859 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:46.883 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:46.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:46.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:46.987 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:47.013 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:47.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:47.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:47.117 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:47.145 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:47.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:47.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:47.248 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:47.277 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:47.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:47.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:47.381 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:47.406 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:47.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:47.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:47.511 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:47.540 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:47.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:47.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:47.643 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:47.644 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 43 16 65366 65350 94.9754 61.875 0.0049932 0.0105232 2023-10-09T05:14:47.678 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:47.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:47.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:47.784 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:47.813 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:47.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:47.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:47.916 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:47.951 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:47.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:47.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:48.056 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:48.083 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:48.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:48.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:48.187 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:48.212 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:48.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:48.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:48.317 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:48.340 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:48.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:48.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:48.444 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:48.475 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:48.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:48.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:48.585 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:48.613 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:48.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:48.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:48.643 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 44 16 66453 66437 94.3608 67.9375 0.00615071 0.0105906 2023-10-09T05:14:48.716 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:48.742 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:48.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:48.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:48.832 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:14:48.845 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:48.874 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:48.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:48.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:48.978 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:49.007 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:49.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:49.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:49.012 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:48 smithi133 bash[105589]: cluster 2023-10-09T05:14:47.886442+0000 mgr.y (mgr.25277) 788 : cluster 0 pgmap v789: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T05:14:49.012 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:48 smithi133 bash[102858]: cluster 2023-10-09T05:14:47.886442+0000 mgr.y (mgr.25277) 788 : cluster 0 pgmap v789: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T05:14:49.113 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:49.142 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:49.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:49.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:49.245 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:49.255 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:48 smithi194 bash[92666]: cluster 2023-10-09T05:14:47.886442+0000 mgr.y (mgr.25277) 788 : cluster 0 pgmap v789: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T05:14:49.271 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:49.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:49.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:49.375 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:49.402 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:49.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:49.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:49.505 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:49.529 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:49.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:49.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:49.632 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:49.643 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 45 16 67491 67475 93.7054 64.875 0.00699209 0.0106653 2023-10-09T05:14:49.666 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:49.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:49.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:49.770 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:49.795 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:49.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:49.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:49.899 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:49.922 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:49.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:49.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:49.961 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:49 smithi133 bash[105589]: audit 2023-10-09T05:14:49.828120+0000 mon.a (mon.0) 1017 : audit 1 from='client.? 172.21.15.133:0/824891517' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.5045905919794463}]: dispatch 2023-10-09T05:14:49.961 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:49 smithi133 bash[102858]: audit 2023-10-09T05:14:49.828120+0000 mon.a (mon.0) 1017 : audit 1 from='client.? 172.21.15.133:0/824891517' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.5045905919794463}]: dispatch 2023-10-09T05:14:49.972 INFO:teuthology.orchestra.run.smithi133.stderr:reweighted osd.2 to 0.504591 (812c) 2023-10-09T05:14:50.028 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:50.036 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:49 smithi194 bash[92666]: audit 2023-10-09T05:14:49.828120+0000 mon.a (mon.0) 1017 : audit 1 from='client.? 172.21.15.133:0/824891517' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.5045905919794463}]: dispatch 2023-10-09T05:14:50.049 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:50.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:50.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:50.151 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:50.186 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:50.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:50.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:50.290 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:50.316 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:50.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:50.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:50.420 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:50.452 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:50.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:50.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:50.556 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:50.580 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:14:50.587 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:50.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:50.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:50.644 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 46 16 67851 67835 92.1574 22.5 0.00391693 0.0106965 2023-10-09T05:14:50.691 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:50.721 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:50.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:50.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:50.746 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:14:50.824 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:50.856 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:50.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:50.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:50.959 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:50.969 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:50 smithi133 bash[102858]: cluster 2023-10-09T05:14:49.887161+0000 mgr.y (mgr.25277) 789 : cluster 0 pgmap v790: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T05:14:50.972 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:50 smithi194 bash[92666]: cluster 2023-10-09T05:14:49.887161+0000 mgr.y (mgr.25277) 789 : cluster 0 pgmap v790: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T05:14:50.972 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:50 smithi194 bash[92666]: cluster 2023-10-09T05:14:49.950648+0000 osd.6 (osd.6) 120 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T05:14:50.972 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:50 smithi194 bash[92666]: cluster 2023-10-09T05:14:49.950779+0000 osd.5 (osd.5) 180 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T05:14:50.972 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:50 smithi194 bash[92666]: audit 2023-10-09T05:14:49.953010+0000 mon.a (mon.0) 1018 : audit 1 from='client.? 172.21.15.133:0/824891517' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.5045905919794463}]': finished 2023-10-09T05:14:50.972 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:50 smithi194 bash[92666]: cluster 2023-10-09T05:14:49.953150+0000 mon.a (mon.0) 1019 : cluster 0 osdmap e192: 8 total, 8 up, 6 in 2023-10-09T05:14:50.972 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:50 smithi194 bash[92666]: cluster 2023-10-09T05:14:50.578683+0000 osd.7 (osd.7) 124 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T05:14:50.989 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:50.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:50.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:51.092 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:51.118 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:51.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:51.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:51.222 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:51.252 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:51.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:51.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:51.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:50 smithi133 bash[102858]: cluster 2023-10-09T05:14:49.950648+0000 osd.6 (osd.6) 120 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T05:14:51.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:50 smithi133 bash[102858]: cluster 2023-10-09T05:14:49.950779+0000 osd.5 (osd.5) 180 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T05:14:51.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:50 smithi133 bash[102858]: audit 2023-10-09T05:14:49.953010+0000 mon.a (mon.0) 1018 : audit 1 from='client.? 172.21.15.133:0/824891517' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.5045905919794463}]': finished 2023-10-09T05:14:51.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:50 smithi133 bash[102858]: cluster 2023-10-09T05:14:49.953150+0000 mon.a (mon.0) 1019 : cluster 0 osdmap e192: 8 total, 8 up, 6 in 2023-10-09T05:14:51.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:50 smithi133 bash[102858]: cluster 2023-10-09T05:14:50.578683+0000 osd.7 (osd.7) 124 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T05:14:51.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:50 smithi133 bash[105589]: cluster 2023-10-09T05:14:49.887161+0000 mgr.y (mgr.25277) 789 : cluster 0 pgmap v790: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T05:14:51.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:50 smithi133 bash[105589]: cluster 2023-10-09T05:14:49.950648+0000 osd.6 (osd.6) 120 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T05:14:51.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:50 smithi133 bash[105589]: cluster 2023-10-09T05:14:49.950779+0000 osd.5 (osd.5) 180 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T05:14:51.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:50 smithi133 bash[105589]: audit 2023-10-09T05:14:49.953010+0000 mon.a (mon.0) 1018 : audit 1 from='client.? 172.21.15.133:0/824891517' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.5045905919794463}]': finished 2023-10-09T05:14:51.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:50 smithi133 bash[105589]: cluster 2023-10-09T05:14:49.953150+0000 mon.a (mon.0) 1019 : cluster 0 osdmap e192: 8 total, 8 up, 6 in 2023-10-09T05:14:51.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:50 smithi133 bash[105589]: cluster 2023-10-09T05:14:50.578683+0000 osd.7 (osd.7) 124 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T05:14:51.355 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:51.385 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:51.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:51.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:51.489 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:51.520 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:51.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:51.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:51.623 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:51.644 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 47 16 67851 67835 90.1966 0 - 0.0106965 2023-10-09T05:14:51.653 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:51.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:51.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:51.757 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:51.793 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:51.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:51.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:51.897 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:51.923 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:51.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:51.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:51.974 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:51 smithi133 bash[102858]: cluster 2023-10-09T05:14:50.960001+0000 osd.4 (osd.4) 132 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T05:14:51.974 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:51 smithi133 bash[102858]: cluster 2023-10-09T05:14:50.964169+0000 osd.4 (osd.4) 133 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T05:14:51.974 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:51 smithi133 bash[102858]: cluster 2023-10-09T05:14:50.965334+0000 osd.6 (osd.6) 121 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T05:14:51.975 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:51 smithi133 bash[102858]: cluster 2023-10-09T05:14:50.966597+0000 mon.a (mon.0) 1020 : cluster 0 osdmap e193: 8 total, 8 up, 6 in 2023-10-09T05:14:51.975 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:51 smithi133 bash[102858]: cluster 2023-10-09T05:14:50.967338+0000 osd.6 (osd.6) 122 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T05:14:51.975 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:51 smithi133 bash[102858]: cluster 2023-10-09T05:14:50.967725+0000 osd.4 (osd.4) 134 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T05:14:51.975 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:51 smithi133 bash[102858]: cluster 2023-10-09T05:14:50.969535+0000 osd.4 (osd.4) 135 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T05:14:51.975 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:51 smithi133 bash[102858]: cluster 2023-10-09T05:14:50.972785+0000 osd.4 (osd.4) 136 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T05:14:51.975 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:51 smithi133 bash[102858]: cluster 2023-10-09T05:14:51.566871+0000 osd.7 (osd.7) 125 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T05:14:51.975 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:51 smithi133 bash[102858]: cluster 2023-10-09T05:14:51.968231+0000 mon.a (mon.0) 1021 : cluster 0 osdmap e194: 8 total, 8 up, 6 in 2023-10-09T05:14:52.027 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:52.050 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:52.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:52.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:52.157 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:52.195 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:52.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:52.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:52.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:51 smithi133 bash[105589]: cluster 2023-10-09T05:14:50.960001+0000 osd.4 (osd.4) 132 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T05:14:52.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:51 smithi133 bash[105589]: cluster 2023-10-09T05:14:50.964169+0000 osd.4 (osd.4) 133 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T05:14:52.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:51 smithi133 bash[105589]: cluster 2023-10-09T05:14:50.965334+0000 osd.6 (osd.6) 121 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T05:14:52.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:51 smithi133 bash[105589]: cluster 2023-10-09T05:14:50.966597+0000 mon.a (mon.0) 1020 : cluster 0 osdmap e193: 8 total, 8 up, 6 in 2023-10-09T05:14:52.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:51 smithi133 bash[105589]: cluster 2023-10-09T05:14:50.967338+0000 osd.6 (osd.6) 122 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T05:14:52.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:51 smithi133 bash[105589]: cluster 2023-10-09T05:14:50.967725+0000 osd.4 (osd.4) 134 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T05:14:52.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:51 smithi133 bash[105589]: cluster 2023-10-09T05:14:50.969535+0000 osd.4 (osd.4) 135 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T05:14:52.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:51 smithi133 bash[105589]: cluster 2023-10-09T05:14:50.972785+0000 osd.4 (osd.4) 136 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T05:14:52.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:51 smithi133 bash[105589]: cluster 2023-10-09T05:14:51.566871+0000 osd.7 (osd.7) 125 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T05:14:52.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:51 smithi133 bash[105589]: cluster 2023-10-09T05:14:51.968231+0000 mon.a (mon.0) 1021 : cluster 0 osdmap e194: 8 total, 8 up, 6 in 2023-10-09T05:14:52.300 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:52.309 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:51 smithi194 bash[92666]: cluster 2023-10-09T05:14:50.960001+0000 osd.4 (osd.4) 132 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T05:14:52.310 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:51 smithi194 bash[92666]: cluster 2023-10-09T05:14:50.964169+0000 osd.4 (osd.4) 133 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T05:14:52.310 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:51 smithi194 bash[92666]: cluster 2023-10-09T05:14:50.965334+0000 osd.6 (osd.6) 121 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T05:14:52.310 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:51 smithi194 bash[92666]: cluster 2023-10-09T05:14:50.966597+0000 mon.a (mon.0) 1020 : cluster 0 osdmap e193: 8 total, 8 up, 6 in 2023-10-09T05:14:52.310 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:51 smithi194 bash[92666]: cluster 2023-10-09T05:14:50.967338+0000 osd.6 (osd.6) 122 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T05:14:52.310 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:51 smithi194 bash[92666]: cluster 2023-10-09T05:14:50.967725+0000 osd.4 (osd.4) 134 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T05:14:52.310 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:51 smithi194 bash[92666]: cluster 2023-10-09T05:14:50.969535+0000 osd.4 (osd.4) 135 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T05:14:52.310 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:51 smithi194 bash[92666]: cluster 2023-10-09T05:14:50.972785+0000 osd.4 (osd.4) 136 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T05:14:52.311 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:51 smithi194 bash[92666]: cluster 2023-10-09T05:14:51.566871+0000 osd.7 (osd.7) 125 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T05:14:52.311 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:51 smithi194 bash[92666]: cluster 2023-10-09T05:14:51.968231+0000 mon.a (mon.0) 1021 : cluster 0 osdmap e194: 8 total, 8 up, 6 in 2023-10-09T05:14:52.327 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:52.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:52.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:52.430 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:52.459 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:52.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:52.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:52.567 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:52.595 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:52.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:52.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:52.644 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 48 16 68268 68252 88.8605 13.0312 0.00415164 0.0112465 2023-10-09T05:14:52.700 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:52.734 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:52.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:52.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:52.840 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:52.872 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:52.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:52.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:52.977 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:53.003 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:53.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:53.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:53.111 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:53.133 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:53.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:53.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:53.237 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:53.247 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:52 smithi194 bash[92666]: cluster 2023-10-09T05:14:51.887940+0000 mgr.y (mgr.25277) 790 : cluster 0 pgmap v793: 89 pgs: 4 peering, 85 active+clean; 4.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-09T05:14:53.247 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:52 smithi194 bash[92666]: cluster 2023-10-09T05:14:51.964975+0000 osd.4 (osd.4) 137 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T05:14:53.247 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:52 smithi194 bash[92666]: cluster 2023-10-09T05:14:51.965145+0000 osd.5 (osd.5) 181 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T05:14:53.247 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:52 smithi194 bash[92666]: cluster 2023-10-09T05:14:51.969005+0000 osd.4 (osd.4) 138 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T05:14:53.248 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:52 smithi194 bash[92666]: cluster 2023-10-09T05:14:51.969012+0000 osd.6 (osd.6) 123 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T05:14:53.248 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:52 smithi194 bash[92666]: cluster 2023-10-09T05:14:51.969601+0000 mon.a (mon.0) 1022 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:14:53.248 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:52 smithi194 bash[92666]: cluster 2023-10-09T05:14:52.067606+0000 osd.7 (osd.7) 126 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T05:14:53.263 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:53.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:53.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:53.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:52 smithi133 bash[105589]: cluster 2023-10-09T05:14:51.887940+0000 mgr.y (mgr.25277) 790 : cluster 0 pgmap v793: 89 pgs: 4 peering, 85 active+clean; 4.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-09T05:14:53.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:52 smithi133 bash[105589]: cluster 2023-10-09T05:14:51.964975+0000 osd.4 (osd.4) 137 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T05:14:53.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:52 smithi133 bash[105589]: cluster 2023-10-09T05:14:51.965145+0000 osd.5 (osd.5) 181 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T05:14:53.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:52 smithi133 bash[105589]: cluster 2023-10-09T05:14:51.969005+0000 osd.4 (osd.4) 138 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T05:14:53.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:52 smithi133 bash[105589]: cluster 2023-10-09T05:14:51.969012+0000 osd.6 (osd.6) 123 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T05:14:53.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:52 smithi133 bash[105589]: cluster 2023-10-09T05:14:51.969601+0000 mon.a (mon.0) 1022 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:14:53.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:52 smithi133 bash[105589]: cluster 2023-10-09T05:14:52.067606+0000 osd.7 (osd.7) 126 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T05:14:53.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:52 smithi133 bash[102858]: cluster 2023-10-09T05:14:51.887940+0000 mgr.y (mgr.25277) 790 : cluster 0 pgmap v793: 89 pgs: 4 peering, 85 active+clean; 4.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-09T05:14:53.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:52 smithi133 bash[102858]: cluster 2023-10-09T05:14:51.964975+0000 osd.4 (osd.4) 137 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T05:14:53.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:52 smithi133 bash[102858]: cluster 2023-10-09T05:14:51.965145+0000 osd.5 (osd.5) 181 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T05:14:53.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:52 smithi133 bash[102858]: cluster 2023-10-09T05:14:51.969005+0000 osd.4 (osd.4) 138 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T05:14:53.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:52 smithi133 bash[102858]: cluster 2023-10-09T05:14:51.969012+0000 osd.6 (osd.6) 123 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T05:14:53.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:52 smithi133 bash[102858]: cluster 2023-10-09T05:14:51.969601+0000 mon.a (mon.0) 1022 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:14:53.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:52 smithi133 bash[102858]: cluster 2023-10-09T05:14:52.067606+0000 osd.7 (osd.7) 126 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T05:14:53.368 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:53.395 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:53.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:53.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:53.500 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:53.532 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:53.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:53.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:53.633 INFO:teuthology.orchestra.run.smithi194.stdout:{ 2023-10-09T05:14:53.634 INFO:teuthology.orchestra.run.smithi194.stdout: "ops": [], 2023-10-09T05:14:53.634 INFO:teuthology.orchestra.run.smithi194.stdout: "complaint_time": 30, 2023-10-09T05:14:53.634 INFO:teuthology.orchestra.run.smithi194.stdout: "num_blocked_ops": 0 2023-10-09T05:14:53.634 INFO:teuthology.orchestra.run.smithi194.stdout:} 2023-10-09T05:14:53.635 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:53.644 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 49 16 69153 69137 88.1757 55.3125 0.0354038 0.0113346 2023-10-09T05:14:53.668 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:53.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:53.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:53.772 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:53.804 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:53.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:53.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:53.912 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:53.941 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:53.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:53.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:54.045 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:54.073 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:54.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:54.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:54.177 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:54.202 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:54.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:54.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:54.307 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:54.335 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:54.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:54.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:54.440 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:54.448 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:54 smithi133 bash[102858]: cluster 2023-10-09T05:14:53.888615+0000 mgr.y (mgr.25277) 791 : cluster 0 pgmap v795: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 peering, 84 active+clean; 4.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 51 MiB/s wr, 822 op/s; 4147/202998 objects degraded (2.043%); 11 KiB/s, 0 objects/s recovering 2023-10-09T05:14:54.449 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:14:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:14:54] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.43.0" 2023-10-09T05:14:54.449 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:54 smithi133 bash[105589]: cluster 2023-10-09T05:14:53.888615+0000 mgr.y (mgr.25277) 791 : cluster 0 pgmap v795: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 peering, 84 active+clean; 4.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 51 MiB/s wr, 822 op/s; 4147/202998 objects degraded (2.043%); 11 KiB/s, 0 objects/s recovering 2023-10-09T05:14:54.470 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:54.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:54.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:54.573 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:54.601 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:54.603 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:54 smithi194 bash[92666]: cluster 2023-10-09T05:14:53.888615+0000 mgr.y (mgr.25277) 791 : cluster 0 pgmap v795: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 peering, 84 active+clean; 4.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 51 MiB/s wr, 822 op/s; 4147/202998 objects degraded (2.043%); 11 KiB/s, 0 objects/s recovering 2023-10-09T05:14:54.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:54.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:54.644 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 50 16 70047 70031 87.5296 55.875 0.00972441 0.0114185 2023-10-09T05:14:54.705 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:54.746 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:54.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:54.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:54.855 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:54.886 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:54.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:54.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:54.993 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:55.008 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T05:14:55.018 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:55.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:55.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:55.122 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:55.161 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:55.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:55.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:55.266 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:55.293 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:55.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:55.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:55.336 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:55 smithi194 bash[92666]: cluster 2023-10-09T05:14:54.318503+0000 mon.a (mon.0) 1023 : cluster 3 Health check failed: Degraded data redundancy: 4147/202998 objects degraded (2.043%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:14:55.365 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:14:55.365 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:14:55.401 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:55.426 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:55.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:55.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:55.531 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:55.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:55 smithi133 bash[102858]: cluster 2023-10-09T05:14:54.318503+0000 mon.a (mon.0) 1023 : cluster 3 Health check failed: Degraded data redundancy: 4147/202998 objects degraded (2.043%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:14:55.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:55 smithi133 bash[105589]: cluster 2023-10-09T05:14:54.318503+0000 mon.a (mon.0) 1023 : cluster 3 Health check failed: Degraded data redundancy: 4147/202998 objects degraded (2.043%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:14:55.561 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:14:55.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:55.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:14:55.644 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 51 16 70910 70894 86.8708 53.9375 0.0199863 0.0115053 2023-10-09T05:14:55.665 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:55.689 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:55.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:55.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:55.796 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:55.820 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:55.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:55.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:55.928 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:55.960 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:55.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:55.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:56.063 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:56.092 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:56.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:56.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:56.195 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:56.221 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:56.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:56.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:56.324 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:56.348 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 6, 1] out_osds: [5, 7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:14:56.348 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:14:56.349 INFO:tasks.thrashosds.thrasher:Adding osd 7 2023-10-09T05:14:56.349 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd in 7 2023-10-09T05:14:56.357 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:56.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:56.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:56.461 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:56.470 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:56 smithi133 bash[102858]: cluster 2023-10-09T05:14:55.889531+0000 mgr.y (mgr.25277) 792 : cluster 0 pgmap v796: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 53 MiB/s wr, 852 op/s; 8310/209292 objects degraded (3.971%); 4228/209292 objects misplaced (2.020%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T05:14:56.471 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:56 smithi133 bash[105589]: cluster 2023-10-09T05:14:55.889531+0000 mgr.y (mgr.25277) 792 : cluster 0 pgmap v796: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 53 MiB/s wr, 852 op/s; 8310/209292 objects degraded (3.971%); 4228/209292 objects misplaced (2.020%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T05:14:56.499 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:56.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:56.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:56.605 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:56.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:56 smithi194 bash[92666]: cluster 2023-10-09T05:14:55.889531+0000 mgr.y (mgr.25277) 792 : cluster 0 pgmap v796: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 53 MiB/s wr, 852 op/s; 8310/209292 objects degraded (3.971%); 4228/209292 objects misplaced (2.020%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T05:14:56.637 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:56.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:56.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:56.644 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 52 16 71466 71450 85.8684 34.75 0.0109051 0.0116392 2023-10-09T05:14:56.741 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:56.765 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:56.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:56.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:56.869 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:56.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:56.905 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:56.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:57.006 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:57.028 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:57.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:57.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:57.133 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:57.162 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:57.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:57.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:57.266 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:57.294 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:57.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:57.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:57.397 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:57.431 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:57.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:57.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:57.537 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:57.559 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:57.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:57.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:57.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:57 smithi194 bash[92666]: cluster 2023-10-09T05:14:56.335381+0000 mon.a (mon.0) 1024 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:14:57.644 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 53 16 72411 72395 85.3625 59.0625 0.0104672 0.0117088 2023-10-09T05:14:57.665 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:57.703 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:57.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:57.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:57.808 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:57.818 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:57 smithi133 bash[102858]: cluster 2023-10-09T05:14:56.335381+0000 mon.a (mon.0) 1024 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:14:57.818 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:57 smithi133 bash[105589]: cluster 2023-10-09T05:14:56.335381+0000 mon.a (mon.0) 1024 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:14:57.841 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:57.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:57.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:57.946 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:57.971 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:14:57.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:57.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:14:58.077 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:58.107 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:58.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:58.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:58.214 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:58.240 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:58.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:58.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:58.344 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:58.376 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:14:58.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:58.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:14:58.482 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:58.492 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:58 smithi133 bash[102858]: cluster 2023-10-09T05:14:57.890008+0000 mgr.y (mgr.25277) 793 : cluster 0 pgmap v797: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 28 MiB/s wr, 455 op/s; 8310/210477 objects degraded (3.948%); 4228/210477 objects misplaced (2.009%); 879 KiB/s, 13 objects/s recovering 2023-10-09T05:14:58.492 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:58 smithi133 bash[105589]: cluster 2023-10-09T05:14:57.890008+0000 mgr.y (mgr.25277) 793 : cluster 0 pgmap v797: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 28 MiB/s wr, 455 op/s; 8310/210477 objects degraded (3.948%); 4228/210477 objects misplaced (2.009%); 879 KiB/s, 13 objects/s recovering 2023-10-09T05:14:58.509 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:14:58.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:58.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:14:58.616 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:58.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:58 smithi194 bash[92666]: cluster 2023-10-09T05:14:57.890008+0000 mgr.y (mgr.25277) 793 : cluster 0 pgmap v797: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 28 MiB/s wr, 455 op/s; 8310/210477 objects degraded (3.948%); 4228/210477 objects misplaced (2.009%); 879 KiB/s, 13 objects/s recovering 2023-10-09T05:14:58.644 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:58.646 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 54 16 73307 73291 84.8166 56 0.00924222 0.0117826 2023-10-09T05:14:58.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:58.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:58.747 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:58.785 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:58.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:58.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:58.893 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:58.925 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:58.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:58.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:59.032 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:59.062 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:59.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:59.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:59.165 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:59.189 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:14:59.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:59.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:14:59.293 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:59.317 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:59.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:59.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:59.420 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:59.446 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:14:59.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:59.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:14:59.452 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:59 smithi133 bash[102858]: audit 2023-10-09T05:14:58.452519+0000 mon.a (mon.0) 1025 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:14:59.453 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:14:59 smithi133 bash[102858]: audit 2023-10-09T05:14:58.453566+0000 mon.a (mon.0) 1026 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:14:59.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:59 smithi133 bash[105589]: audit 2023-10-09T05:14:58.452519+0000 mon.a (mon.0) 1025 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:14:59.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:14:59 smithi133 bash[105589]: audit 2023-10-09T05:14:58.453566+0000 mon.a (mon.0) 1026 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:14:59.551 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:59.589 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:59.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:59.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:59.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:59 smithi194 bash[92666]: audit 2023-10-09T05:14:58.452519+0000 mon.a (mon.0) 1025 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:14:59.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:14:59 smithi194 bash[92666]: audit 2023-10-09T05:14:58.453566+0000 mon.a (mon.0) 1026 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:14:59.646 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 55 16 74113 74097 84.1903 50.375 0.014808 0.0118712 2023-10-09T05:14:59.695 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:59.729 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:14:59.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:59.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:14:59.836 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:59.838 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:14:59.871 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:14:59.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:14:59.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:14:59.976 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:00.017 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:00.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:00.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:00.122 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:00.152 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:00.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:00.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:00.255 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:00.285 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:00.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:00.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:00.391 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:00.424 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:00.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:00.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:00.532 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:00.565 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:00.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:00.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:00.646 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 56 16 74963 74947 83.6355 53.125 0.0138055 0.0119511 2023-10-09T05:15:00.672 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:00.696 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:00.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:00.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:00.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:00 smithi133 bash[102858]: cluster 2023-10-09T05:14:59.890825+0000 mgr.y (mgr.25277) 794 : cluster 0 pgmap v798: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 36 MiB/s wr, 574 op/s; 8310/215847 objects degraded (3.850%); 4228/215847 objects misplaced (1.959%); 781 KiB/s, 12 objects/s recovering 2023-10-09T05:15:00.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:00 smithi133 bash[105589]: cluster 2023-10-09T05:14:59.890825+0000 mgr.y (mgr.25277) 794 : cluster 0 pgmap v798: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 36 MiB/s wr, 574 op/s; 8310/215847 objects degraded (3.850%); 4228/215847 objects misplaced (1.959%); 781 KiB/s, 12 objects/s recovering 2023-10-09T05:15:00.801 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:00.839 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:00.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:00.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:00.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:00 smithi194 bash[92666]: cluster 2023-10-09T05:14:59.890825+0000 mgr.y (mgr.25277) 794 : cluster 0 pgmap v798: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 36 MiB/s wr, 574 op/s; 8310/215847 objects degraded (3.850%); 4228/215847 objects misplaced (1.959%); 781 KiB/s, 12 objects/s recovering 2023-10-09T05:15:00.944 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:00.983 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:00.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:00.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:01.087 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:01.111 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:01.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:01.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:01.218 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:01.249 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:01.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:01.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:01.352 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:01.376 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:01.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:01.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:01.477 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:01 smithi133 bash[105589]: cluster 2023-10-09T05:15:00.450287+0000 mon.a (mon.0) 1027 : cluster 3 Health check update: Degraded data redundancy: 8310/215847 objects degraded (3.850%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:01.480 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:01.506 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:15:01.510 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:01.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:01.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:01.614 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:01.643 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:01.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:01.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:01.646 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 57 16 75828 75812 83.1166 54.0625 0.0470032 0.0120239 2023-10-09T05:15:01.742 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:01 smithi133 bash[102858]: cluster 2023-10-09T05:15:00.450287+0000 mon.a (mon.0) 1027 : cluster 3 Health check update: Degraded data redundancy: 8310/215847 objects degraded (3.850%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:01.745 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:01.755 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:01 smithi194 bash[92666]: cluster 2023-10-09T05:15:00.450287+0000 mon.a (mon.0) 1027 : cluster 3 Health check update: Degraded data redundancy: 8310/215847 objects degraded (3.850%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:01.772 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:01.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:01.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:01.877 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:01.902 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:01.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:01.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:02.005 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:02.031 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:02.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:02.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:02.136 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:02.161 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:02.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:02.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:02.265 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:02.291 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:02.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:02.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:02.394 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:02.419 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:02.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:02.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:02.525 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:02.554 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:02.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:02.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:02.620 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:02 smithi194 bash[92666]: cluster 2023-10-09T05:15:01.891362+0000 mgr.y (mgr.25277) 795 : cluster 0 pgmap v799: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 44 MiB/s wr, 696 op/s; 8310/222882 objects degraded (3.728%); 4025/222882 objects misplaced (1.806%); 1.9 MiB/s, 31 objects/s recovering 2023-10-09T05:15:02.646 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 58 16 76663 76647 82.5833 52.1875 0.00436561 0.0121014 2023-10-09T05:15:02.659 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:02.685 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:02.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:02.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:02.789 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:02.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:02 smithi133 bash[102858]: cluster 2023-10-09T05:15:01.891362+0000 mgr.y (mgr.25277) 795 : cluster 0 pgmap v799: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 44 MiB/s wr, 696 op/s; 8310/222882 objects degraded (3.728%); 4025/222882 objects misplaced (1.806%); 1.9 MiB/s, 31 objects/s recovering 2023-10-09T05:15:02.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:02 smithi133 bash[105589]: cluster 2023-10-09T05:15:01.891362+0000 mgr.y (mgr.25277) 795 : cluster 0 pgmap v799: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 44 MiB/s wr, 696 op/s; 8310/222882 objects degraded (3.728%); 4025/222882 objects misplaced (1.806%); 1.9 MiB/s, 31 objects/s recovering 2023-10-09T05:15:02.824 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:02.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:02.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:02.929 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:02.959 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:02.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:02.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:03.065 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:03.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:03.102 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:03.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:03.203 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:03.232 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:03.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:03.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:03.337 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:03.364 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:03.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:03.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:03.469 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:03.499 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:03.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:03.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:03.607 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:03.634 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:03.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:03.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:03.646 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 59 16 77562 77546 82.1358 56.1875 0.014493 0.0121688 2023-10-09T05:15:03.738 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:03.770 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:03.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:03.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:03.877 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:04.001 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:04.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:04.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:04.105 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:04.140 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:04.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:04.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:04.244 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:04.275 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:04.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:04.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:04.378 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:04.402 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:04.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:04.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:04.490 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:15:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:15:04] "GET /metrics HTTP/1.1" 200 36502 "" "Prometheus/2.43.0" 2023-10-09T05:15:04.508 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:04.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:04.540 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:04.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:04.641 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:04.646 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T05:15:04.646769+0000 min lat: 0.00128168 max lat: 2.27779 avg lat: 0.0122362 2023-10-09T05:15:04.646 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:15:04.647 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 60 16 78437 78421 81.6783 54.6875 0.0191978 0.0122362 2023-10-09T05:15:04.667 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:04.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:04.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:04.774 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:04.803 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:04.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:04.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:04.907 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:04.930 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:04.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:04.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:04.941 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:04 smithi194 bash[92666]: cluster 2023-10-09T05:15:03.891867+0000 mgr.y (mgr.25277) 796 : cluster 0 pgmap v800: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 38 MiB/s wr, 608 op/s; 8310/223755 objects degraded (3.714%); 4025/223755 objects misplaced (1.799%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T05:15:04.941 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:04 smithi194 bash[92666]: audit 2023-10-09T05:15:03.892646+0000 mon.a (mon.0) 1028 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:15:05.036 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:05.082 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:05.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:05.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:05.187 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:05.212 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:05.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:05.215 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:04 smithi133 bash[105589]: cluster 2023-10-09T05:15:03.891867+0000 mgr.y (mgr.25277) 796 : cluster 0 pgmap v800: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 38 MiB/s wr, 608 op/s; 8310/223755 objects degraded (3.714%); 4025/223755 objects misplaced (1.799%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T05:15:05.215 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:04 smithi133 bash[105589]: audit 2023-10-09T05:15:03.892646+0000 mon.a (mon.0) 1028 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:15:05.216 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:04 smithi133 bash[102858]: cluster 2023-10-09T05:15:03.891867+0000 mgr.y (mgr.25277) 796 : cluster 0 pgmap v800: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 38 MiB/s wr, 608 op/s; 8310/223755 objects degraded (3.714%); 4025/223755 objects misplaced (1.799%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T05:15:05.216 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:04 smithi133 bash[102858]: audit 2023-10-09T05:15:03.892646+0000 mon.a (mon.0) 1028 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:15:05.216 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:15:05.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:05.261 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:15:05.315 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:15:05.318 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:05.343 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:05.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:05.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:05.359 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:15:05.405 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:15:05.441 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:15:05.448 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:05.481 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:15:05.482 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:05.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:05.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:05.521 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:15:05.529 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:15:05.529 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:15:05.529 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:15:05.529 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:15:05.529 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:15:05.529 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:15:05.529 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:15:05.529 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:15:05.587 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:05.610 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:05.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:05.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:05.646 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 61 16 79195 79179 81.1158 47.375 0.00982983 0.0123216 2023-10-09T05:15:05.716 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:05.746 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:05.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:05.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:05.852 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:05.889 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:05.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:05.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:05.932 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is unset 2023-10-09T05:15:05.996 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:06.006 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:05 smithi194 bash[92666]: audit 2023-10-09T05:15:05.172612+0000 mon.a (mon.0) 1029 : audit 0 from='client.? 172.21.15.133:0/324084726' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:15:06.006 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:05 smithi194 bash[92666]: audit 2023-10-09T05:15:05.178046+0000 mon.c (mon.2) 77 : audit 1 from='client.? 172.21.15.133:0/3060826754' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:15:06.007 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:05 smithi194 bash[92666]: audit 2023-10-09T05:15:05.178657+0000 mon.a (mon.0) 1030 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:15:06.023 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:06.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:06.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:06.129 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:06.140 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:05 smithi133 bash[102858]: audit 2023-10-09T05:15:05.172612+0000 mon.a (mon.0) 1029 : audit 0 from='client.? 172.21.15.133:0/324084726' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:15:06.140 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:05 smithi133 bash[102858]: audit 2023-10-09T05:15:05.178046+0000 mon.c (mon.2) 77 : audit 1 from='client.? 172.21.15.133:0/3060826754' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:15:06.140 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:05 smithi133 bash[102858]: audit 2023-10-09T05:15:05.178657+0000 mon.a (mon.0) 1030 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:15:06.140 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:05 smithi133 bash[105589]: audit 2023-10-09T05:15:05.172612+0000 mon.a (mon.0) 1029 : audit 0 from='client.? 172.21.15.133:0/324084726' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:15:06.141 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:05 smithi133 bash[105589]: audit 2023-10-09T05:15:05.178046+0000 mon.c (mon.2) 77 : audit 1 from='client.? 172.21.15.133:0/3060826754' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:15:06.141 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:05 smithi133 bash[105589]: audit 2023-10-09T05:15:05.178657+0000 mon.a (mon.0) 1030 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:15:06.160 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:06.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:06.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:06.266 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:06.293 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:06.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:06.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:06.403 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:06.444 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:06.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:06.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:06.549 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:06.576 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:06.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:06.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:06.647 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 62 16 79975 79959 80.5937 48.75 0.00530197 0.0124017 2023-10-09T05:15:06.678 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:06.701 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:06.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:06.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:06.807 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:06.833 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:06.835 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:15:06.835 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:15:06.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:06.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:06.938 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:06.972 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:06.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:06.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:07.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:06 smithi133 bash[102858]: cluster 2023-10-09T05:15:05.892585+0000 mgr.y (mgr.25277) 797 : cluster 0 pgmap v801: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 54 MiB/s wr, 867 op/s; 8310/234222 objects degraded (3.548%); 3833/234222 objects misplaced (1.636%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T05:15:07.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:06 smithi133 bash[102858]: cluster 2023-10-09T05:15:05.902107+0000 mon.a (mon.0) 1031 : cluster 3 Health check update: Degraded data redundancy: 8310/234222 objects degraded (3.548%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:07.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:06 smithi133 bash[102858]: cluster 2023-10-09T05:15:05.902834+0000 mon.a (mon.0) 1032 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:15:07.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:06 smithi133 bash[102858]: cluster 2023-10-09T05:15:05.911014+0000 osd.4 (osd.4) 139 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T05:15:07.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:06 smithi133 bash[102858]: audit 2023-10-09T05:15:05.918920+0000 mon.a (mon.0) 1033 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:15:07.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:06 smithi133 bash[102858]: cluster 2023-10-09T05:15:05.919058+0000 mon.a (mon.0) 1034 : cluster 0 osdmap e195: 8 total, 8 up, 6 in 2023-10-09T05:15:07.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:06 smithi133 bash[102858]: cluster 2023-10-09T05:15:05.920345+0000 osd.6 (osd.6) 124 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T05:15:07.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:06 smithi133 bash[102858]: audit 2023-10-09T05:15:06.156624+0000 mon.a (mon.0) 1035 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:15:07.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:06 smithi133 bash[102858]: cluster 2023-10-09T05:15:06.658087+0000 osd.7 (osd.7) 127 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T05:15:07.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:06 smithi133 bash[102858]: audit 2023-10-09T05:15:06.684948+0000 mon.a (mon.0) 1036 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:15:07.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:06 smithi133 bash[102858]: audit 2023-10-09T05:15:06.692656+0000 mon.a (mon.0) 1037 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:15:07.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:06 smithi133 bash[105589]: cluster 2023-10-09T05:15:05.892585+0000 mgr.y (mgr.25277) 797 : cluster 0 pgmap v801: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 54 MiB/s wr, 867 op/s; 8310/234222 objects degraded (3.548%); 3833/234222 objects misplaced (1.636%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T05:15:07.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:06 smithi133 bash[105589]: cluster 2023-10-09T05:15:05.902107+0000 mon.a (mon.0) 1031 : cluster 3 Health check update: Degraded data redundancy: 8310/234222 objects degraded (3.548%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:07.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:06 smithi133 bash[105589]: cluster 2023-10-09T05:15:05.902834+0000 mon.a (mon.0) 1032 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:15:07.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:06 smithi133 bash[105589]: cluster 2023-10-09T05:15:05.911014+0000 osd.4 (osd.4) 139 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T05:15:07.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:06 smithi133 bash[105589]: audit 2023-10-09T05:15:05.918920+0000 mon.a (mon.0) 1033 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:15:07.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:06 smithi133 bash[105589]: cluster 2023-10-09T05:15:05.919058+0000 mon.a (mon.0) 1034 : cluster 0 osdmap e195: 8 total, 8 up, 6 in 2023-10-09T05:15:07.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:06 smithi133 bash[105589]: cluster 2023-10-09T05:15:05.920345+0000 osd.6 (osd.6) 124 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T05:15:07.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:06 smithi133 bash[105589]: audit 2023-10-09T05:15:06.156624+0000 mon.a (mon.0) 1035 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:15:07.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:06 smithi133 bash[105589]: cluster 2023-10-09T05:15:06.658087+0000 osd.7 (osd.7) 127 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T05:15:07.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:06 smithi133 bash[105589]: audit 2023-10-09T05:15:06.684948+0000 mon.a (mon.0) 1036 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:15:07.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:06 smithi133 bash[105589]: audit 2023-10-09T05:15:06.692656+0000 mon.a (mon.0) 1037 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:15:07.079 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:07.110 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:07.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:07.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:07.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:06 smithi194 bash[92666]: cluster 2023-10-09T05:15:05.892585+0000 mgr.y (mgr.25277) 797 : cluster 0 pgmap v801: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 54 MiB/s wr, 867 op/s; 8310/234222 objects degraded (3.548%); 3833/234222 objects misplaced (1.636%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T05:15:07.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:06 smithi194 bash[92666]: cluster 2023-10-09T05:15:05.902107+0000 mon.a (mon.0) 1031 : cluster 3 Health check update: Degraded data redundancy: 8310/234222 objects degraded (3.548%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:07.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:06 smithi194 bash[92666]: cluster 2023-10-09T05:15:05.902834+0000 mon.a (mon.0) 1032 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:15:07.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:06 smithi194 bash[92666]: cluster 2023-10-09T05:15:05.911014+0000 osd.4 (osd.4) 139 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T05:15:07.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:06 smithi194 bash[92666]: audit 2023-10-09T05:15:05.918920+0000 mon.a (mon.0) 1033 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:15:07.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:06 smithi194 bash[92666]: cluster 2023-10-09T05:15:05.919058+0000 mon.a (mon.0) 1034 : cluster 0 osdmap e195: 8 total, 8 up, 6 in 2023-10-09T05:15:07.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:06 smithi194 bash[92666]: cluster 2023-10-09T05:15:05.920345+0000 osd.6 (osd.6) 124 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T05:15:07.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:06 smithi194 bash[92666]: audit 2023-10-09T05:15:06.156624+0000 mon.a (mon.0) 1035 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:15:07.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:06 smithi194 bash[92666]: cluster 2023-10-09T05:15:06.658087+0000 osd.7 (osd.7) 127 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T05:15:07.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:06 smithi194 bash[92666]: audit 2023-10-09T05:15:06.684948+0000 mon.a (mon.0) 1036 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:15:07.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:06 smithi194 bash[92666]: audit 2023-10-09T05:15:06.692656+0000 mon.a (mon.0) 1037 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:15:07.216 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:07.252 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:07.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:07.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:07.356 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:07.384 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:07.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:07.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:07.491 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:07.515 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:07.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:07.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:07.619 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:07.647 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 63 16 80686 80670 80.0198 44.4375 0.034453 0.0124906 2023-10-09T05:15:07.650 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:07.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:07.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:07.753 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:07.779 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:07.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:07.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:07.885 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:07.912 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:07.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:07.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:08.016 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:08.046 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:08.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:08.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:08.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:07 smithi194 bash[92666]: cluster 2023-10-09T05:15:06.658035+0000 osd.5 (osd.5) 182 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T05:15:08.153 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:08.178 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:08.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:08.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:08.286 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:08.293 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:07 smithi133 bash[102858]: cluster 2023-10-09T05:15:06.658035+0000 osd.5 (osd.5) 182 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T05:15:08.294 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:07 smithi133 bash[105589]: cluster 2023-10-09T05:15:06.658035+0000 osd.5 (osd.5) 182 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T05:15:08.327 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:08.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:08.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:08.403 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T05:15:08.434 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:08.468 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:08.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:08.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:08.571 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:08.599 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:08.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:08.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:08.647 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 64 16 81547 81531 79.6102 53.8125 0.0129729 0.0125555 2023-10-09T05:15:08.704 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:08.737 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:08.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:08.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:08.840 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:08.866 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:08.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:08.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:08.924 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:08 smithi133 bash[102858]: cluster 2023-10-09T05:15:07.893243+0000 mgr.y (mgr.25277) 798 : cluster 0 pgmap v803: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 53 MiB/s wr, 841 op/s; 8310/235728 objects degraded (3.525%); 3833/235728 objects misplaced (1.626%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T05:15:08.973 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:09.010 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:09.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:09.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:09.115 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:09.125 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:08 smithi194 bash[92666]: cluster 2023-10-09T05:15:07.893243+0000 mgr.y (mgr.25277) 798 : cluster 0 pgmap v803: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 53 MiB/s wr, 841 op/s; 8310/235728 objects degraded (3.525%); 3833/235728 objects misplaced (1.626%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T05:15:09.143 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:09.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:09.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:09.249 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:09.276 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:09.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:09.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:09.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:08 smithi133 bash[105589]: cluster 2023-10-09T05:15:07.893243+0000 mgr.y (mgr.25277) 798 : cluster 0 pgmap v803: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 53 MiB/s wr, 841 op/s; 8310/235728 objects degraded (3.525%); 3833/235728 objects misplaced (1.626%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T05:15:09.379 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:09.407 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:09.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:09.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:09.512 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:09.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:09.541 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:09.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:09.642 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:09.647 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 65 16 82379 82363 79.1854 52 0.017271 0.0126218 2023-10-09T05:15:09.675 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:09.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:09.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:09.781 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:09.806 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:09.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:09.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:09.911 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:09.947 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:09.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:09.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:10.050 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:10.077 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:10.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:10.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:10.180 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:10.207 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:10.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:10.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:10.310 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:10.335 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:10.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:10.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:10.437 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:10.464 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:10.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:10.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:10.573 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:10.603 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:10.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:10.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:10.647 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 66 16 83208 83192 78.7706 51.8125 0.037582 0.0126891 2023-10-09T05:15:10.709 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:10.738 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:10.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:10.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:10.845 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:10.878 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:10.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:10.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:10.984 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:10.993 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:10 smithi133 bash[105589]: cluster 2023-10-09T05:15:09.893963+0000 mgr.y (mgr.25277) 799 : cluster 0 pgmap v804: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 52 MiB/s wr, 839 op/s; 8310/241041 objects degraded (3.448%); 3833/241041 objects misplaced (1.590%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T05:15:10.994 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:10 smithi133 bash[102858]: cluster 2023-10-09T05:15:09.893963+0000 mgr.y (mgr.25277) 799 : cluster 0 pgmap v804: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 52 MiB/s wr, 839 op/s; 8310/241041 objects degraded (3.448%); 3833/241041 objects misplaced (1.590%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T05:15:11.017 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:11.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:11.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:11.123 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:11.149 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:11.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:11.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:11.252 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:11.292 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:11.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:11.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:11.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:10 smithi194 bash[92666]: cluster 2023-10-09T05:15:09.893963+0000 mgr.y (mgr.25277) 799 : cluster 0 pgmap v804: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 52 MiB/s wr, 839 op/s; 8310/241041 objects degraded (3.448%); 3833/241041 objects misplaced (1.590%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T05:15:11.401 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:11.429 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:11.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:11.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:11.535 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:11.544 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:11 smithi133 bash[102858]: debug 2023-10-09T05:15:11.414+0000 7f5eb4d1d700 -1 mon.a@0(leader).osd e195 definitely_dead 0 2023-10-09T05:15:11.566 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:11.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:11.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:11.647 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 67 16 83978 83962 78.3131 48.125 0.0311376 0.0127613 2023-10-09T05:15:11.671 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:11.715 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:11.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:11.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:11.819 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:11.841 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:11.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:11.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:11.858 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:15:11.944 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:11.969 INFO:teuthology.orchestra.run.smithi133.stderr:marked in osd.7. 2023-10-09T05:15:11.979 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:11.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:11.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:12.085 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:12.088 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:15:12.097 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:11 smithi133 bash[102858]: audit 2023-10-09T05:15:11.417494+0000 mon.a (mon.0) 1038 : audit 0 from='client.? 172.21.15.133:0/2343565495' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T05:15:12.097 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:11 smithi133 bash[105589]: audit 2023-10-09T05:15:11.417494+0000 mon.a (mon.0) 1038 : audit 0 from='client.? 172.21.15.133:0/2343565495' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T05:15:12.114 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:12.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:12.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:12.239 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:12.249 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:11 smithi194 bash[92666]: audit 2023-10-09T05:15:11.417494+0000 mon.a (mon.0) 1038 : audit 0 from='client.? 172.21.15.133:0/2343565495' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T05:15:12.265 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:12.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:12.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:12.367 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:12.399 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:12.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:12.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:12.502 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:12.543 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:12.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:12.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:12.646 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:12.647 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 68 16 84286 84270 77.4446 19.25 0.00595466 0.0127941 2023-10-09T05:15:12.658 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:15:12.673 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:12.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:12.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:12.776 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:12.805 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:12.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:12.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:12.908 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:12.929 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:12.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:12.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:13.032 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:13.057 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:13.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:13.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:13.160 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:13.167 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:12 smithi194 bash[92666]: cluster 2023-10-09T05:15:11.894717+0000 mgr.y (mgr.25277) 800 : cluster 0 pgmap v805: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 52 MiB/s wr, 827 op/s; 8310/247722 objects degraded (3.355%); 3622/247722 objects misplaced (1.462%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T05:15:13.167 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:12 smithi194 bash[92666]: cluster 2023-10-09T05:15:11.937934+0000 mon.a (mon.0) 1039 : cluster 3 Health check update: Degraded data redundancy: 8310/247722 objects degraded (3.355%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:13.167 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:12 smithi194 bash[92666]: cluster 2023-10-09T05:15:11.954667+0000 osd.6 (osd.6) 125 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T05:15:13.167 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:12 smithi194 bash[92666]: cluster 2023-10-09T05:15:11.958136+0000 osd.4 (osd.4) 140 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T05:15:13.167 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:12 smithi194 bash[92666]: cluster 2023-10-09T05:15:11.961283+0000 osd.6 (osd.6) 126 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T05:15:13.167 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:12 smithi194 bash[92666]: cluster 2023-10-09T05:15:11.961342+0000 osd.7 (osd.7) 128 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T05:15:13.167 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:12 smithi194 bash[92666]: cluster 2023-10-09T05:15:11.961435+0000 osd.4 (osd.4) 141 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T05:15:13.167 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:12 smithi194 bash[92666]: audit 2023-10-09T05:15:11.961508+0000 mon.a (mon.0) 1040 : audit 1 from='client.? 172.21.15.133:0/2343565495' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T05:15:13.167 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:12 smithi194 bash[92666]: cluster 2023-10-09T05:15:11.961769+0000 mon.a (mon.0) 1041 : cluster 0 osdmap e196: 8 total, 8 up, 7 in 2023-10-09T05:15:13.181 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:13.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:13.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:13.286 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:13.295 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:12 smithi133 bash[102858]: cluster 2023-10-09T05:15:11.894717+0000 mgr.y (mgr.25277) 800 : cluster 0 pgmap v805: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 52 MiB/s wr, 827 op/s; 8310/247722 objects degraded (3.355%); 3622/247722 objects misplaced (1.462%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T05:15:13.295 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:12 smithi133 bash[102858]: cluster 2023-10-09T05:15:11.937934+0000 mon.a (mon.0) 1039 : cluster 3 Health check update: Degraded data redundancy: 8310/247722 objects degraded (3.355%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:13.295 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:12 smithi133 bash[102858]: cluster 2023-10-09T05:15:11.954667+0000 osd.6 (osd.6) 125 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T05:15:13.295 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:12 smithi133 bash[102858]: cluster 2023-10-09T05:15:11.958136+0000 osd.4 (osd.4) 140 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T05:15:13.295 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:12 smithi133 bash[102858]: cluster 2023-10-09T05:15:11.961283+0000 osd.6 (osd.6) 126 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T05:15:13.295 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:12 smithi133 bash[102858]: cluster 2023-10-09T05:15:11.961342+0000 osd.7 (osd.7) 128 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T05:15:13.295 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:12 smithi133 bash[102858]: cluster 2023-10-09T05:15:11.961435+0000 osd.4 (osd.4) 141 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T05:15:13.296 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:12 smithi133 bash[102858]: audit 2023-10-09T05:15:11.961508+0000 mon.a (mon.0) 1040 : audit 1 from='client.? 172.21.15.133:0/2343565495' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T05:15:13.296 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:12 smithi133 bash[102858]: cluster 2023-10-09T05:15:11.961769+0000 mon.a (mon.0) 1041 : cluster 0 osdmap e196: 8 total, 8 up, 7 in 2023-10-09T05:15:13.296 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:12 smithi133 bash[105589]: cluster 2023-10-09T05:15:11.894717+0000 mgr.y (mgr.25277) 800 : cluster 0 pgmap v805: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 52 MiB/s wr, 827 op/s; 8310/247722 objects degraded (3.355%); 3622/247722 objects misplaced (1.462%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T05:15:13.296 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:12 smithi133 bash[105589]: cluster 2023-10-09T05:15:11.937934+0000 mon.a (mon.0) 1039 : cluster 3 Health check update: Degraded data redundancy: 8310/247722 objects degraded (3.355%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:13.296 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:12 smithi133 bash[105589]: cluster 2023-10-09T05:15:11.954667+0000 osd.6 (osd.6) 125 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T05:15:13.296 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:12 smithi133 bash[105589]: cluster 2023-10-09T05:15:11.958136+0000 osd.4 (osd.4) 140 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T05:15:13.296 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:12 smithi133 bash[105589]: cluster 2023-10-09T05:15:11.961283+0000 osd.6 (osd.6) 126 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T05:15:13.296 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:12 smithi133 bash[105589]: cluster 2023-10-09T05:15:11.961342+0000 osd.7 (osd.7) 128 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T05:15:13.296 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:12 smithi133 bash[105589]: cluster 2023-10-09T05:15:11.961435+0000 osd.4 (osd.4) 141 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T05:15:13.297 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:12 smithi133 bash[105589]: audit 2023-10-09T05:15:11.961508+0000 mon.a (mon.0) 1040 : audit 1 from='client.? 172.21.15.133:0/2343565495' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T05:15:13.297 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:12 smithi133 bash[105589]: cluster 2023-10-09T05:15:11.961769+0000 mon.a (mon.0) 1041 : cluster 0 osdmap e196: 8 total, 8 up, 7 in 2023-10-09T05:15:13.310 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:13.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:13.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:13.412 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:13.439 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:13.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:13.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:13.512 INFO:tasks.thrashosds.thrasher:Added osd 7 2023-10-09T05:15:13.542 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:13.570 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:13.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:13.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:13.647 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 69 16 84286 84270 76.3222 0 - 0.0127941 2023-10-09T05:15:13.675 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:13.706 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:13.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:13.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:13.808 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:13.843 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:13.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:13.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:13.947 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:13.968 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:13 smithi133 bash[102858]: cluster 2023-10-09T05:15:12.953879+0000 mon.a (mon.0) 1042 : cluster 0 osdmap e197: 8 total, 8 up, 7 in 2023-10-09T05:15:13.968 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:13 smithi133 bash[102858]: cluster 2023-10-09T05:15:12.954608+0000 osd.4 (osd.4) 142 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T05:15:13.968 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:13 smithi133 bash[102858]: cluster 2023-10-09T05:15:12.954975+0000 osd.7 (osd.7) 129 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T05:15:13.968 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:13 smithi133 bash[102858]: cluster 2023-10-09T05:15:12.956129+0000 osd.6 (osd.6) 127 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T05:15:13.968 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:13 smithi133 bash[102858]: cluster 2023-10-09T05:15:12.957558+0000 osd.4 (osd.4) 143 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T05:15:13.968 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:13 smithi133 bash[102858]: cluster 2023-10-09T05:15:12.958308+0000 osd.6 (osd.6) 128 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T05:15:13.968 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:13 smithi133 bash[102858]: cluster 2023-10-09T05:15:12.958548+0000 osd.7 (osd.7) 130 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T05:15:13.968 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:13 smithi133 bash[102858]: cluster 2023-10-09T05:15:12.959654+0000 osd.4 (osd.4) 144 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T05:15:13.969 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:13 smithi133 bash[102858]: audit 2023-10-09T05:15:13.252877+0000 mon.a (mon.0) 1043 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:15:13.969 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:13 smithi133 bash[102858]: audit 2023-10-09T05:15:13.259553+0000 mon.a (mon.0) 1044 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:15:13.969 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:13 smithi133 bash[102858]: audit 2023-10-09T05:15:13.448559+0000 mon.a (mon.0) 1045 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:15:13.969 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:13 smithi133 bash[102858]: cluster 2023-10-09T05:15:13.958896+0000 mon.a (mon.0) 1046 : cluster 0 osdmap e198: 8 total, 8 up, 7 in 2023-10-09T05:15:13.979 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:13.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:13.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:14.082 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:14.095 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:13 smithi194 bash[92666]: cluster 2023-10-09T05:15:12.953879+0000 mon.a (mon.0) 1042 : cluster 0 osdmap e197: 8 total, 8 up, 7 in 2023-10-09T05:15:14.095 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:13 smithi194 bash[92666]: cluster 2023-10-09T05:15:12.954608+0000 osd.4 (osd.4) 142 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T05:15:14.095 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:13 smithi194 bash[92666]: cluster 2023-10-09T05:15:12.954975+0000 osd.7 (osd.7) 129 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T05:15:14.095 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:13 smithi194 bash[92666]: cluster 2023-10-09T05:15:12.956129+0000 osd.6 (osd.6) 127 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T05:15:14.095 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:13 smithi194 bash[92666]: cluster 2023-10-09T05:15:12.957558+0000 osd.4 (osd.4) 143 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T05:15:14.095 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:13 smithi194 bash[92666]: cluster 2023-10-09T05:15:12.958308+0000 osd.6 (osd.6) 128 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T05:15:14.095 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:13 smithi194 bash[92666]: cluster 2023-10-09T05:15:12.958548+0000 osd.7 (osd.7) 130 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T05:15:14.096 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:13 smithi194 bash[92666]: cluster 2023-10-09T05:15:12.959654+0000 osd.4 (osd.4) 144 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T05:15:14.096 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:13 smithi194 bash[92666]: audit 2023-10-09T05:15:13.252877+0000 mon.a (mon.0) 1043 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:15:14.096 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:13 smithi194 bash[92666]: audit 2023-10-09T05:15:13.259553+0000 mon.a (mon.0) 1044 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:15:14.096 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:13 smithi194 bash[92666]: audit 2023-10-09T05:15:13.448559+0000 mon.a (mon.0) 1045 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:15:14.096 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:13 smithi194 bash[92666]: cluster 2023-10-09T05:15:13.958896+0000 mon.a (mon.0) 1046 : cluster 0 osdmap e198: 8 total, 8 up, 7 in 2023-10-09T05:15:14.111 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:14.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:14.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:14.215 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:14.243 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:14.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:14.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:14.282 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:13 smithi133 bash[105589]: cluster 2023-10-09T05:15:12.953879+0000 mon.a (mon.0) 1042 : cluster 0 osdmap e197: 8 total, 8 up, 7 in 2023-10-09T05:15:14.282 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:13 smithi133 bash[105589]: cluster 2023-10-09T05:15:12.954608+0000 osd.4 (osd.4) 142 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T05:15:14.282 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:13 smithi133 bash[105589]: cluster 2023-10-09T05:15:12.954975+0000 osd.7 (osd.7) 129 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T05:15:14.282 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:13 smithi133 bash[105589]: cluster 2023-10-09T05:15:12.956129+0000 osd.6 (osd.6) 127 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T05:15:14.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:13 smithi133 bash[105589]: cluster 2023-10-09T05:15:12.957558+0000 osd.4 (osd.4) 143 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T05:15:14.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:13 smithi133 bash[105589]: cluster 2023-10-09T05:15:12.958308+0000 osd.6 (osd.6) 128 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T05:15:14.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:13 smithi133 bash[105589]: cluster 2023-10-09T05:15:12.958548+0000 osd.7 (osd.7) 130 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T05:15:14.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:13 smithi133 bash[105589]: cluster 2023-10-09T05:15:12.959654+0000 osd.4 (osd.4) 144 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T05:15:14.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:13 smithi133 bash[105589]: audit 2023-10-09T05:15:13.252877+0000 mon.a (mon.0) 1043 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:15:14.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:13 smithi133 bash[105589]: audit 2023-10-09T05:15:13.259553+0000 mon.a (mon.0) 1044 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:15:14.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:13 smithi133 bash[105589]: audit 2023-10-09T05:15:13.448559+0000 mon.a (mon.0) 1045 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:15:14.283 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:13 smithi133 bash[105589]: cluster 2023-10-09T05:15:13.958896+0000 mon.a (mon.0) 1046 : cluster 0 osdmap e198: 8 total, 8 up, 7 in 2023-10-09T05:15:14.346 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:14.376 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:14.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:14.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:14.479 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:14.508 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:14.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:14.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:14.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:15:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:15:14] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.43.0" 2023-10-09T05:15:14.612 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:14.635 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:14.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:14.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:14.647 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 70 16 84286 84270 75.2319 0 - 0.0127941 2023-10-09T05:15:14.739 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:14.792 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:14.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:14.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:14.896 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:14.921 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:14.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:14.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:15.015 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:14 smithi133 bash[102858]: cluster 2023-10-09T05:15:13.072331+0000 osd.5 (osd.5) 183 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T05:15:15.015 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:14 smithi133 bash[102858]: cluster 2023-10-09T05:15:13.895569+0000 mgr.y (mgr.25277) 801 : cluster 0 pgmap v808: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 37 MiB/s wr, 586 op/s; 4163/248259 objects degraded (1.677%); 7722/248259 objects misplaced (3.110%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T05:15:15.015 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:14 smithi133 bash[102858]: cluster 2023-10-09T05:15:13.954584+0000 osd.6 (osd.6) 129 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T05:15:15.015 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:14 smithi133 bash[102858]: cluster 2023-10-09T05:15:13.957946+0000 osd.4 (osd.4) 145 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T05:15:15.015 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:14 smithi133 bash[102858]: cluster 2023-10-09T05:15:13.959694+0000 osd.4 (osd.4) 146 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T05:15:15.015 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:14 smithi133 bash[102858]: cluster 2023-10-09T05:15:14.483597+0000 osd.7 (osd.7) 131 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T05:15:15.016 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:14 smithi133 bash[105589]: cluster 2023-10-09T05:15:13.072331+0000 osd.5 (osd.5) 183 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T05:15:15.016 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:14 smithi133 bash[105589]: cluster 2023-10-09T05:15:13.895569+0000 mgr.y (mgr.25277) 801 : cluster 0 pgmap v808: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 37 MiB/s wr, 586 op/s; 4163/248259 objects degraded (1.677%); 7722/248259 objects misplaced (3.110%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T05:15:15.016 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:14 smithi133 bash[105589]: cluster 2023-10-09T05:15:13.954584+0000 osd.6 (osd.6) 129 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T05:15:15.016 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:14 smithi133 bash[105589]: cluster 2023-10-09T05:15:13.957946+0000 osd.4 (osd.4) 145 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T05:15:15.016 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:14 smithi133 bash[105589]: cluster 2023-10-09T05:15:13.959694+0000 osd.4 (osd.4) 146 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T05:15:15.016 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:14 smithi133 bash[105589]: cluster 2023-10-09T05:15:14.483597+0000 osd.7 (osd.7) 131 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T05:15:15.024 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:15.051 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:15.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:15.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:15.156 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:15.191 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:15.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:15.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:15.286 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:14 smithi194 bash[92666]: cluster 2023-10-09T05:15:13.072331+0000 osd.5 (osd.5) 183 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T05:15:15.286 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:14 smithi194 bash[92666]: cluster 2023-10-09T05:15:13.895569+0000 mgr.y (mgr.25277) 801 : cluster 0 pgmap v808: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 37 MiB/s wr, 586 op/s; 4163/248259 objects degraded (1.677%); 7722/248259 objects misplaced (3.110%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T05:15:15.286 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:14 smithi194 bash[92666]: cluster 2023-10-09T05:15:13.954584+0000 osd.6 (osd.6) 129 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T05:15:15.286 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:14 smithi194 bash[92666]: cluster 2023-10-09T05:15:13.957946+0000 osd.4 (osd.4) 145 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T05:15:15.286 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:14 smithi194 bash[92666]: cluster 2023-10-09T05:15:13.959694+0000 osd.4 (osd.4) 146 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T05:15:15.286 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:14 smithi194 bash[92666]: cluster 2023-10-09T05:15:14.483597+0000 osd.7 (osd.7) 131 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T05:15:15.298 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:15.333 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:15.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:15.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:15.438 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:15.477 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:15.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:15.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:15.583 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:15.629 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:15.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:15.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:15.647 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 71 16 84810 84794 74.6335 10.9167 0.0332346 0.0133924 2023-10-09T05:15:15.734 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:15.779 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:15.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:15.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:15.883 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:15.927 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:15.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:15.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:16.032 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:16.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:15 smithi133 bash[102858]: cluster 2023-10-09T05:15:14.634689+0000 osd.5 (osd.5) 184 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T05:15:16.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:15 smithi133 bash[102858]: audit 2023-10-09T05:15:15.336755+0000 mon.a (mon.0) 1047 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:15:16.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:15 smithi133 bash[102858]: cluster 2023-10-09T05:15:15.347653+0000 osd.7 (osd.7) 132 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T05:15:16.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:15 smithi133 bash[102858]: cluster 2023-10-09T05:15:15.354033+0000 mon.a (mon.0) 1048 : cluster 0 osdmap e199: 8 total, 8 up, 7 in 2023-10-09T05:15:16.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:15 smithi133 bash[102858]: cluster 2023-10-09T05:15:15.355205+0000 osd.6 (osd.6) 130 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T05:15:16.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:15 smithi133 bash[102858]: cluster 2023-10-09T05:15:15.357956+0000 osd.6 (osd.6) 131 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T05:15:16.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:15 smithi133 bash[102858]: cluster 2023-10-09T05:15:15.360823+0000 osd.6 (osd.6) 132 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T05:15:16.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:15 smithi133 bash[102858]: audit 2023-10-09T05:15:15.361336+0000 mon.a (mon.0) 1049 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:15:16.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:15 smithi133 bash[105589]: cluster 2023-10-09T05:15:14.634689+0000 osd.5 (osd.5) 184 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T05:15:16.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:15 smithi133 bash[105589]: audit 2023-10-09T05:15:15.336755+0000 mon.a (mon.0) 1047 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:15:16.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:15 smithi133 bash[105589]: cluster 2023-10-09T05:15:15.347653+0000 osd.7 (osd.7) 132 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T05:15:16.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:15 smithi133 bash[105589]: cluster 2023-10-09T05:15:15.354033+0000 mon.a (mon.0) 1048 : cluster 0 osdmap e199: 8 total, 8 up, 7 in 2023-10-09T05:15:16.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:15 smithi133 bash[105589]: cluster 2023-10-09T05:15:15.355205+0000 osd.6 (osd.6) 130 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T05:15:16.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:15 smithi133 bash[105589]: cluster 2023-10-09T05:15:15.357956+0000 osd.6 (osd.6) 131 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T05:15:16.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:15 smithi133 bash[105589]: cluster 2023-10-09T05:15:15.360823+0000 osd.6 (osd.6) 132 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T05:15:16.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:15 smithi133 bash[105589]: audit 2023-10-09T05:15:15.361336+0000 mon.a (mon.0) 1049 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:15:16.069 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:16.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:16.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:16.176 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:16.205 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:16.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:16.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:16.311 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:16.337 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:16.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:16.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:16.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:15 smithi194 bash[92666]: cluster 2023-10-09T05:15:14.634689+0000 osd.5 (osd.5) 184 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T05:15:16.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:15 smithi194 bash[92666]: audit 2023-10-09T05:15:15.336755+0000 mon.a (mon.0) 1047 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:15:16.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:15 smithi194 bash[92666]: cluster 2023-10-09T05:15:15.347653+0000 osd.7 (osd.7) 132 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T05:15:16.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:15 smithi194 bash[92666]: cluster 2023-10-09T05:15:15.354033+0000 mon.a (mon.0) 1048 : cluster 0 osdmap e199: 8 total, 8 up, 7 in 2023-10-09T05:15:16.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:15 smithi194 bash[92666]: cluster 2023-10-09T05:15:15.355205+0000 osd.6 (osd.6) 130 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T05:15:16.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:15 smithi194 bash[92666]: cluster 2023-10-09T05:15:15.357956+0000 osd.6 (osd.6) 131 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T05:15:16.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:15 smithi194 bash[92666]: cluster 2023-10-09T05:15:15.360823+0000 osd.6 (osd.6) 132 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T05:15:16.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:15 smithi194 bash[92666]: audit 2023-10-09T05:15:15.361336+0000 mon.a (mon.0) 1049 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:15:16.441 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:16.489 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:16.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:16.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:16.594 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:16.621 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:16.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:16.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:16.647 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 72 16 85442 85426 74.1455 39.5 0.0294762 0.0134797 2023-10-09T05:15:16.728 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:16.754 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:16.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:16.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:16.858 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:16.884 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:16.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:16.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:16.991 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:17.036 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:17.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:17.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:17.049 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:15:17.050 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:15:17.140 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:17.172 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:17.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:17.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:17.277 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:17.287 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:16 smithi133 bash[102858]: cluster 2023-10-09T05:15:15.354795+0000 osd.4 (osd.4) 147 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T05:15:17.287 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:16 smithi133 bash[102858]: cluster 2023-10-09T05:15:15.364211+0000 osd.4 (osd.4) 148 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T05:15:17.287 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:16 smithi133 bash[102858]: cluster 2023-10-09T05:15:15.897860+0000 mgr.y (mgr.25277) 802 : cluster 0 pgmap v811: 89 pgs: 4 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 255 B/s rd, 27 MiB/s wr, 437 op/s; 23400/252969 objects degraded (9.250%); 4100/252969 objects misplaced (1.621%); 3.2 MiB/s, 52 objects/s recovering 2023-10-09T05:15:17.287 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:16 smithi133 bash[102858]: audit 2023-10-09T05:15:15.987091+0000 mon.a (mon.0) 1050 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:15:17.288 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:16 smithi133 bash[102858]: audit 2023-10-09T05:15:15.991565+0000 mon.a (mon.0) 1051 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:15:17.288 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:16 smithi133 bash[102858]: audit 2023-10-09T05:15:15.999270+0000 mon.a (mon.0) 1052 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:15:17.288 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:16 smithi133 bash[102858]: cluster 2023-10-09T05:15:16.382206+0000 mon.a (mon.0) 1053 : cluster 0 osdmap e200: 8 total, 8 up, 7 in 2023-10-09T05:15:17.288 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:16 smithi133 bash[102858]: cluster 2023-10-09T05:15:16.382768+0000 osd.7 (osd.7) 133 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T05:15:17.288 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:16 smithi133 bash[102858]: cluster 2023-10-09T05:15:16.384043+0000 osd.7 (osd.7) 134 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T05:15:17.288 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:16 smithi133 bash[102858]: cluster 2023-10-09T05:15:16.384867+0000 osd.6 (osd.6) 133 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T05:15:17.288 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:16 smithi133 bash[102858]: cluster 2023-10-09T05:15:16.385695+0000 osd.7 (osd.7) 135 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T05:15:17.288 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:16 smithi133 bash[105589]: cluster 2023-10-09T05:15:15.354795+0000 osd.4 (osd.4) 147 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T05:15:17.288 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:16 smithi133 bash[105589]: cluster 2023-10-09T05:15:15.364211+0000 osd.4 (osd.4) 148 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T05:15:17.288 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:16 smithi133 bash[105589]: cluster 2023-10-09T05:15:15.897860+0000 mgr.y (mgr.25277) 802 : cluster 0 pgmap v811: 89 pgs: 4 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 255 B/s rd, 27 MiB/s wr, 437 op/s; 23400/252969 objects degraded (9.250%); 4100/252969 objects misplaced (1.621%); 3.2 MiB/s, 52 objects/s recovering 2023-10-09T05:15:17.289 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:16 smithi133 bash[105589]: audit 2023-10-09T05:15:15.987091+0000 mon.a (mon.0) 1050 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:15:17.289 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:16 smithi133 bash[105589]: audit 2023-10-09T05:15:15.991565+0000 mon.a (mon.0) 1051 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:15:17.289 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:16 smithi133 bash[105589]: audit 2023-10-09T05:15:15.999270+0000 mon.a (mon.0) 1052 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:15:17.289 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:16 smithi133 bash[105589]: cluster 2023-10-09T05:15:16.382206+0000 mon.a (mon.0) 1053 : cluster 0 osdmap e200: 8 total, 8 up, 7 in 2023-10-09T05:15:17.289 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:16 smithi133 bash[105589]: cluster 2023-10-09T05:15:16.382768+0000 osd.7 (osd.7) 133 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T05:15:17.289 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:16 smithi133 bash[105589]: cluster 2023-10-09T05:15:16.384043+0000 osd.7 (osd.7) 134 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T05:15:17.289 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:16 smithi133 bash[105589]: cluster 2023-10-09T05:15:16.384867+0000 osd.6 (osd.6) 133 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T05:15:17.289 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:16 smithi133 bash[105589]: cluster 2023-10-09T05:15:16.385695+0000 osd.7 (osd.7) 135 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T05:15:17.304 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:17.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:17.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:17.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:16 smithi194 bash[92666]: cluster 2023-10-09T05:15:15.354795+0000 osd.4 (osd.4) 147 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T05:15:17.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:16 smithi194 bash[92666]: cluster 2023-10-09T05:15:15.364211+0000 osd.4 (osd.4) 148 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T05:15:17.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:16 smithi194 bash[92666]: cluster 2023-10-09T05:15:15.897860+0000 mgr.y (mgr.25277) 802 : cluster 0 pgmap v811: 89 pgs: 4 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 255 B/s rd, 27 MiB/s wr, 437 op/s; 23400/252969 objects degraded (9.250%); 4100/252969 objects misplaced (1.621%); 3.2 MiB/s, 52 objects/s recovering 2023-10-09T05:15:17.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:16 smithi194 bash[92666]: audit 2023-10-09T05:15:15.987091+0000 mon.a (mon.0) 1050 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:15:17.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:16 smithi194 bash[92666]: audit 2023-10-09T05:15:15.991565+0000 mon.a (mon.0) 1051 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:15:17.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:16 smithi194 bash[92666]: audit 2023-10-09T05:15:15.999270+0000 mon.a (mon.0) 1052 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:15:17.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:16 smithi194 bash[92666]: cluster 2023-10-09T05:15:16.382206+0000 mon.a (mon.0) 1053 : cluster 0 osdmap e200: 8 total, 8 up, 7 in 2023-10-09T05:15:17.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:16 smithi194 bash[92666]: cluster 2023-10-09T05:15:16.382768+0000 osd.7 (osd.7) 133 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T05:15:17.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:16 smithi194 bash[92666]: cluster 2023-10-09T05:15:16.384043+0000 osd.7 (osd.7) 134 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T05:15:17.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:16 smithi194 bash[92666]: cluster 2023-10-09T05:15:16.384867+0000 osd.6 (osd.6) 133 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T05:15:17.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:16 smithi194 bash[92666]: cluster 2023-10-09T05:15:16.385695+0000 osd.7 (osd.7) 135 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T05:15:17.409 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:17.439 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:17.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:17.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:17.546 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:17.570 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:17.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:17.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:17.648 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 73 16 86080 86064 73.676 39.875 0.0118439 0.013566 2023-10-09T05:15:17.673 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:17.704 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:17.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:17.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:17.808 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:17.832 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:17.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:17.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:17.937 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:17.974 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:17.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:17.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:17.994 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:15:17.994 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:15:17.994 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:17 smithi194 bash[92666]: cluster 2023-10-09T05:15:16.084600+0000 osd.5 (osd.5) 185 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T05:15:17.994 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:17 smithi194 bash[92666]: cluster 2023-10-09T05:15:16.377935+0000 osd.5 (osd.5) 186 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T05:15:17.994 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:17 smithi194 bash[92666]: cluster 2023-10-09T05:15:16.384844+0000 osd.4 (osd.4) 149 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T05:15:18.082 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:18.112 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:18.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:18.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:18.216 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:18.246 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:18.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:18.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:18.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:17 smithi133 bash[105589]: cluster 2023-10-09T05:15:16.084600+0000 osd.5 (osd.5) 185 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T05:15:18.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:17 smithi133 bash[105589]: cluster 2023-10-09T05:15:16.377935+0000 osd.5 (osd.5) 186 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T05:15:18.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:17 smithi133 bash[105589]: cluster 2023-10-09T05:15:16.384844+0000 osd.4 (osd.4) 149 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T05:15:18.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:17 smithi133 bash[102858]: cluster 2023-10-09T05:15:16.084600+0000 osd.5 (osd.5) 185 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T05:15:18.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:17 smithi133 bash[102858]: cluster 2023-10-09T05:15:16.377935+0000 osd.5 (osd.5) 186 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T05:15:18.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:17 smithi133 bash[102858]: cluster 2023-10-09T05:15:16.384844+0000 osd.4 (osd.4) 149 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T05:15:18.350 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:18.383 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:18.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:18.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:18.486 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:18.514 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 6, 1, 7] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:15:18.514 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:15:18.514 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [0, 2, 3, 4, 6, 1, 7] 2023-10-09T05:15:18.514 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd out 7 2023-10-09T05:15:18.530 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:18.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:18.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:18.634 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:18.648 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 74 16 86771 86755 73.264 43.1875 0.018705 0.0136412 2023-10-09T05:15:18.660 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:18.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:18.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:18.765 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:18.787 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:18.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:18.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:18.892 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:18.935 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:18.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:18.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:18.980 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T05:15:19.040 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:19.051 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:18 smithi194 bash[92666]: cluster 2023-10-09T05:15:17.898420+0000 mgr.y (mgr.25277) 803 : cluster 0 pgmap v813: 89 pgs: 4 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 207 B/s rd, 24 MiB/s wr, 390 op/s; 28586/253509 objects degraded (11.276%); 4100/253509 objects misplaced (1.617%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T05:15:19.051 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:18 smithi194 bash[92666]: cluster 2023-10-09T05:15:17.984091+0000 mon.a (mon.0) 1054 : cluster 3 Health check update: Degraded data redundancy: 28586/253509 objects degraded (11.276%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:19.079 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:19.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:19.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:19.185 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:19.193 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:18 smithi133 bash[102858]: cluster 2023-10-09T05:15:17.898420+0000 mgr.y (mgr.25277) 803 : cluster 0 pgmap v813: 89 pgs: 4 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 207 B/s rd, 24 MiB/s wr, 390 op/s; 28586/253509 objects degraded (11.276%); 4100/253509 objects misplaced (1.617%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T05:15:19.193 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:18 smithi133 bash[102858]: cluster 2023-10-09T05:15:17.984091+0000 mon.a (mon.0) 1054 : cluster 3 Health check update: Degraded data redundancy: 28586/253509 objects degraded (11.276%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:19.193 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:18 smithi133 bash[105589]: cluster 2023-10-09T05:15:17.898420+0000 mgr.y (mgr.25277) 803 : cluster 0 pgmap v813: 89 pgs: 4 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 207 B/s rd, 24 MiB/s wr, 390 op/s; 28586/253509 objects degraded (11.276%); 4100/253509 objects misplaced (1.617%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T05:15:19.194 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:18 smithi133 bash[105589]: cluster 2023-10-09T05:15:17.984091+0000 mon.a (mon.0) 1054 : cluster 3 Health check update: Degraded data redundancy: 28586/253509 objects degraded (11.276%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:19.225 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:19.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:19.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:19.329 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:19.356 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:19.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:19.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:19.468 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:19.491 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:19.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:19.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:19.596 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:19.633 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:19.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:19.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:19.648 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 75 16 87462 87446 72.8629 43.1875 0.00984562 0.0137184 2023-10-09T05:15:19.736 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:19.769 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:19.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:19.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:19.875 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:19.906 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:19.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:19.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:20.012 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:20.058 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:20.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:20.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:20.165 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:20.199 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:20.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:20.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:20.304 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:20.337 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:20.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:20.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:20.441 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:20.474 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:20.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:20.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:20.579 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:20.603 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:20.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:20.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:20.648 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 76 16 88174 88158 72.4897 44.5 0.0663277 0.0137883 2023-10-09T05:15:20.707 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:20.725 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:20.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:20.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:20.828 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:20.859 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:20.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:20.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:20.965 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:20.997 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:21.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:21.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:21.056 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:21 smithi133 bash[105589]: cluster 2023-10-09T05:15:19.899197+0000 mgr.y (mgr.25277) 804 : cluster 0 pgmap v814: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 170 B/s rd, 32 MiB/s wr, 505 op/s; 23274/257364 objects degraded (9.043%); 9161/257364 objects misplaced (3.560%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T05:15:21.057 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:21 smithi133 bash[102858]: cluster 2023-10-09T05:15:19.899197+0000 mgr.y (mgr.25277) 804 : cluster 0 pgmap v814: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 170 B/s rd, 32 MiB/s wr, 505 op/s; 23274/257364 objects degraded (9.043%); 9161/257364 objects misplaced (3.560%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T05:15:21.101 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:21.130 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:21.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:21.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:21.241 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:21.284 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:21.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:21.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:21.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:21 smithi194 bash[92666]: cluster 2023-10-09T05:15:19.899197+0000 mgr.y (mgr.25277) 804 : cluster 0 pgmap v814: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 170 B/s rd, 32 MiB/s wr, 505 op/s; 23274/257364 objects degraded (9.043%); 9161/257364 objects misplaced (3.560%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T05:15:21.388 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:21.417 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:21.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:21.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:21.522 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:21.550 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:21.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:21.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:21.648 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 77 16 88863 88847 72.1075 43.0625 0.0156926 0.0138616 2023-10-09T05:15:21.658 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:21.685 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:21.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:21.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:21.788 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:21.811 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:21.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:21.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:21.915 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:21.942 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:21.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:21.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:22.045 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:22.073 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:22.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:22.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:22.176 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:22.205 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:22.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:22.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:22.310 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:22.339 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:22.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:22.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:22.444 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:22.474 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:22.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:22.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:22.583 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:22.607 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:22.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:22.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:22.648 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 78 16 89523 89507 71.7118 41.25 0.05132 0.0139381 2023-10-09T05:15:22.711 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:22.734 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:22.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:22.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:22.843 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:22.870 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:22.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:22.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:22.975 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:23.012 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:23 smithi133 bash[102858]: cluster 2023-10-09T05:15:21.901880+0000 mgr.y (mgr.25277) 805 : cluster 0 pgmap v815: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 128 B/s rd, 39 MiB/s wr, 620 op/s; 23274/263040 objects degraded (8.848%); 9161/263040 objects misplaced (3.483%); 3.3 MiB/s, 51 objects/s recovering 2023-10-09T05:15:23.023 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:23.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:23.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:23.128 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:23.166 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:23.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:23.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:23.272 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:23.280 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:23 smithi133 bash[105589]: cluster 2023-10-09T05:15:21.901880+0000 mgr.y (mgr.25277) 805 : cluster 0 pgmap v815: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 128 B/s rd, 39 MiB/s wr, 620 op/s; 23274/263040 objects degraded (8.848%); 9161/263040 objects misplaced (3.483%); 3.3 MiB/s, 51 objects/s recovering 2023-10-09T05:15:23.301 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:23.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:23.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:23.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:23 smithi194 bash[92666]: cluster 2023-10-09T05:15:21.901880+0000 mgr.y (mgr.25277) 805 : cluster 0 pgmap v815: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 128 B/s rd, 39 MiB/s wr, 620 op/s; 23274/263040 objects degraded (8.848%); 9161/263040 objects misplaced (3.483%); 3.3 MiB/s, 51 objects/s recovering 2023-10-09T05:15:23.407 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:23.438 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:23.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:23.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:23.541 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:23.574 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:23.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:23.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:23.648 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 79 16 90218 90202 71.3539 43.4375 0.010882 0.0140076 2023-10-09T05:15:23.681 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:23.711 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:23.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:23.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:23.806 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:15:23.815 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:23.839 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:23.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:23.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:23.942 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:23.975 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:23.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:23.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:24.013 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:24 smithi133 bash[102858]: cluster 2023-10-09T05:15:23.325514+0000 mon.a (mon.0) 1055 : cluster 3 Health check update: Degraded data redundancy: 23274/263040 objects degraded (8.848%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:24.080 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:24.090 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:24 smithi194 bash[92666]: cluster 2023-10-09T05:15:23.325514+0000 mon.a (mon.0) 1055 : cluster 3 Health check update: Degraded data redundancy: 23274/263040 objects degraded (8.848%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:24.111 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:24.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:24.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:24.217 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:24.244 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:24.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:24.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:24.285 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:15:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:15:24] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.43.0" 2023-10-09T05:15:24.286 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:24 smithi133 bash[105589]: cluster 2023-10-09T05:15:23.325514+0000 mon.a (mon.0) 1055 : cluster 3 Health check update: Degraded data redundancy: 23274/263040 objects degraded (8.848%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:24.348 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:24.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:24.379 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:24.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:24.480 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:24.504 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:24.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:24.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:24.609 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:24.648 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T05:15:24.648696+0000 min lat: 0.00128168 max lat: 2.78751 avg lat: 0.0140793 2023-10-09T05:15:24.648 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:15:24.649 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 80 16 90899 90883 70.9939 42.5625 0.0216216 0.0140793 2023-10-09T05:15:24.650 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:24.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:24.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:24.753 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:24.791 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:24.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:24.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:24.896 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:24.926 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:24.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:24.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:25.032 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:25.064 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:25.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:25.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:25.168 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:25.195 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:25.197 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:25 smithi133 bash[105589]: cluster 2023-10-09T05:15:23.902478+0000 mgr.y (mgr.25277) 806 : cluster 0 pgmap v816: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 27 MiB/s wr, 430 op/s; 23274/263673 objects degraded (8.827%); 9045/263673 objects misplaced (3.430%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T05:15:25.197 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:25 smithi133 bash[102858]: cluster 2023-10-09T05:15:23.902478+0000 mgr.y (mgr.25277) 806 : cluster 0 pgmap v816: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 27 MiB/s wr, 430 op/s; 23274/263673 objects degraded (8.827%); 9045/263673 objects misplaced (3.430%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T05:15:25.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:25.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:25.299 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:25.324 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:25.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:25.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:25.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:25 smithi194 bash[92666]: cluster 2023-10-09T05:15:23.902478+0000 mgr.y (mgr.25277) 806 : cluster 0 pgmap v816: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 27 MiB/s wr, 430 op/s; 23274/263673 objects degraded (8.827%); 9045/263673 objects misplaced (3.430%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T05:15:25.434 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:25.465 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:25.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:25.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:25.572 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:25.610 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:25.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:25.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:25.648 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 81 16 91587 91571 70.6483 43 0.059118 0.0141471 2023-10-09T05:15:25.716 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:25.747 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:25.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:25.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:25.854 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:25.890 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:25.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:25.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:25.907 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:15:25.994 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:26.023 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:26.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:26.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:26.125 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:26.152 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:26.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:26.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:26.257 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:26.293 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:26.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:26.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:26.397 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:26.428 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:26.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:26.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:26.531 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:26.568 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:26.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:26.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:26.649 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 82 16 92273 92257 70.3095 42.875 0.0242907 0.0142158 2023-10-09T05:15:26.672 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:26.707 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:26.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:26.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:26.813 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:26.841 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:26.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:26.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:26.943 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:26.979 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:26.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:26.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:27.087 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:27.096 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:27 smithi133 bash[105589]: cluster 2023-10-09T05:15:25.903459+0000 mgr.y (mgr.25277) 807 : cluster 0 pgmap v817: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 40 MiB/s wr, 632 op/s; 23274/271962 objects degraded (8.558%); 8932/271962 objects misplaced (3.284%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T05:15:27.096 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:27 smithi133 bash[102858]: cluster 2023-10-09T05:15:25.903459+0000 mgr.y (mgr.25277) 807 : cluster 0 pgmap v817: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 40 MiB/s wr, 632 op/s; 23274/271962 objects degraded (8.558%); 8932/271962 objects misplaced (3.284%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T05:15:27.113 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:27.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:27.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:27.217 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:27.259 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:27.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:27.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:27.363 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:27.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:27 smithi194 bash[92666]: cluster 2023-10-09T05:15:25.903459+0000 mgr.y (mgr.25277) 807 : cluster 0 pgmap v817: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 40 MiB/s wr, 632 op/s; 23274/271962 objects degraded (8.558%); 8932/271962 objects misplaced (3.284%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T05:15:27.398 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:27.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:27.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:27.504 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:27.533 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:27.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:27.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:27.638 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:27.649 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 83 16 92966 92950 69.9842 43.3125 0.0139014 0.0142813 2023-10-09T05:15:27.668 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:27.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:27.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:27.772 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:27.806 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:27.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:27.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:27.912 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:27.941 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:27.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:27.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:28.044 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:28.076 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:28.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:28.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:28.184 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:28.211 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:15:28.211 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:15:28.220 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:28.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:28.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:28.325 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:28.362 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:28.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:28.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:28.464 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:28.498 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:28.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:28.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:28.603 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:28.626 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:28.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:28.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:28.649 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 84 16 93610 93594 69.6302 40.25 0.0141267 0.0143541 2023-10-09T05:15:28.730 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:28.756 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:28.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:28.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:28.860 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:28.897 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:28.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:28.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:29.001 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:29.025 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:29 smithi133 bash[105589]: cluster 2023-10-09T05:15:27.904022+0000 mgr.y (mgr.25277) 808 : cluster 0 pgmap v818: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 36 MiB/s wr, 569 op/s; 23274/273192 objects degraded (8.519%); 8932/273192 objects misplaced (3.269%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:15:29.026 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:29 smithi133 bash[105589]: cluster 2023-10-09T05:15:28.326638+0000 mon.a (mon.0) 1056 : cluster 3 Health check update: Degraded data redundancy: 23274/273192 objects degraded (8.519%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:29.026 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:29 smithi133 bash[105589]: audit 2023-10-09T05:15:28.448135+0000 mon.a (mon.0) 1057 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:15:29.026 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:29 smithi133 bash[102858]: cluster 2023-10-09T05:15:27.904022+0000 mgr.y (mgr.25277) 808 : cluster 0 pgmap v818: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 36 MiB/s wr, 569 op/s; 23274/273192 objects degraded (8.519%); 8932/273192 objects misplaced (3.269%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:15:29.026 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:29 smithi133 bash[102858]: cluster 2023-10-09T05:15:28.326638+0000 mon.a (mon.0) 1056 : cluster 3 Health check update: Degraded data redundancy: 23274/273192 objects degraded (8.519%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:29.026 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:29 smithi133 bash[102858]: audit 2023-10-09T05:15:28.448135+0000 mon.a (mon.0) 1057 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:15:29.041 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:29.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:29.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:29.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:29 smithi194 bash[92666]: cluster 2023-10-09T05:15:27.904022+0000 mgr.y (mgr.25277) 808 : cluster 0 pgmap v818: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 36 MiB/s wr, 569 op/s; 23274/273192 objects degraded (8.519%); 8932/273192 objects misplaced (3.269%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:15:29.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:29 smithi194 bash[92666]: cluster 2023-10-09T05:15:28.326638+0000 mon.a (mon.0) 1056 : cluster 3 Health check update: Degraded data redundancy: 23274/273192 objects degraded (8.519%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:29.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:29 smithi194 bash[92666]: audit 2023-10-09T05:15:28.448135+0000 mon.a (mon.0) 1057 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:15:29.147 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:29.173 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:29.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:29.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:29.266 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T05:15:29.279 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:29.339 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:29.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:29.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:29.448 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:29.471 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:29.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:29.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:29.574 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:29.602 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:29.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:29.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:29.649 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 85 16 94264 94248 69.2919 40.875 0.0119006 0.0144248 2023-10-09T05:15:29.705 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:29.739 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:29.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:29.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:29.845 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:29.867 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:29.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:29.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:29.971 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:30.003 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:30.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:30.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:30.107 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:30.132 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:30.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:30.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:30.238 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:30.283 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:30.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:30.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:30.387 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:30.418 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:30.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:30.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:30.522 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:30.550 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:30.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:30.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:30.649 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 86 16 94903 94887 68.9505 39.9375 0.00951913 0.0144961 2023-10-09T05:15:30.656 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:30.691 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:30.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:30.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:30.798 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:30.829 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:30.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:30.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:30.933 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:30.972 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:30.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:30.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:31.075 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:31.111 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:31.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:31.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:31.217 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:31.249 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:31.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:31.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:31.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:31 smithi133 bash[102858]: cluster 2023-10-09T05:15:29.904768+0000 mgr.y (mgr.25277) 809 : cluster 0 pgmap v819: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 42 MiB/s wr, 671 op/s; 23274/277686 objects degraded (8.381%); 8692/277686 objects misplaced (3.130%); 3.8 MiB/s, 59 objects/s recovering 2023-10-09T05:15:31.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:31 smithi133 bash[105589]: cluster 2023-10-09T05:15:29.904768+0000 mgr.y (mgr.25277) 809 : cluster 0 pgmap v819: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 42 MiB/s wr, 671 op/s; 23274/277686 objects degraded (8.381%); 8692/277686 objects misplaced (3.130%); 3.8 MiB/s, 59 objects/s recovering 2023-10-09T05:15:31.353 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:31.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:31 smithi194 bash[92666]: cluster 2023-10-09T05:15:29.904768+0000 mgr.y (mgr.25277) 809 : cluster 0 pgmap v819: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 42 MiB/s wr, 671 op/s; 23274/277686 objects degraded (8.381%); 8692/277686 objects misplaced (3.130%); 3.8 MiB/s, 59 objects/s recovering 2023-10-09T05:15:31.391 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:31.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:31.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:31.495 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:31.522 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:31.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:31.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:31.632 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:31.651 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 87 16 95555 95539 68.6244 40.75 0.0428236 0.0145631 2023-10-09T05:15:31.669 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:31.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:31.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:31.774 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:31.793 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:15:31.802 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:31.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:31.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:31.840 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:15:31.877 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:15:31.905 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:31.932 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:15:31.935 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:31.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:31.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:31.981 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:15:32.023 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:15:32.033 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:32 smithi133 bash[102858]: audit 2023-10-09T05:15:31.760249+0000 mon.c (mon.2) 78 : audit 0 from='client.? 172.21.15.133:0/3145786641' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:15:32.034 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:32 smithi133 bash[105589]: audit 2023-10-09T05:15:31.760249+0000 mon.c (mon.2) 78 : audit 0 from='client.? 172.21.15.133:0/3145786641' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:15:32.041 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:32.064 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:15:32.068 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:32.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:32.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:32.097 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:15:32.103 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:15:32.103 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:15:32.104 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:15:32.104 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:15:32.104 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:15:32.104 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:15:32.104 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:15:32.104 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:15:32.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:32 smithi194 bash[92666]: audit 2023-10-09T05:15:31.760249+0000 mon.c (mon.2) 78 : audit 0 from='client.? 172.21.15.133:0/3145786641' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:15:32.172 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:32.208 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:32.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:32.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:32.315 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:32.346 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:32.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:32.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:32.450 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:32.485 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:32.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:32.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:32.589 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:32.613 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:32.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:32.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:32.651 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 88 16 96184 96168 68.2912 39.3125 0.0143321 0.0146366 2023-10-09T05:15:32.717 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:32.757 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:32.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:32.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:32.859 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:32.894 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:32.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:32.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:32.997 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:33.021 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:33.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:33.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:33.081 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is unset 2023-10-09T05:15:33.128 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:33.159 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:33.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:33.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:33.262 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:33.271 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:33 smithi194 bash[92666]: cluster 2023-10-09T05:15:31.905294+0000 mgr.y (mgr.25277) 810 : cluster 0 pgmap v820: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 45 MiB/s wr, 718 op/s; 23274/283257 objects degraded (8.217%); 8692/283257 objects misplaced (3.069%); 2.4 MiB/s, 39 objects/s recovering 2023-10-09T05:15:33.272 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:33 smithi194 bash[92666]: audit 2023-10-09T05:15:32.085639+0000 mon.a (mon.0) 1058 : audit 1 from='client.? 172.21.15.133:0/563527921' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:15:33.292 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:33.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:33.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:33.396 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:33.434 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:33.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:33.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:33.541 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:33.559 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:33.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:33.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:33.570 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:33 smithi133 bash[102858]: cluster 2023-10-09T05:15:31.905294+0000 mgr.y (mgr.25277) 810 : cluster 0 pgmap v820: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 45 MiB/s wr, 718 op/s; 23274/283257 objects degraded (8.217%); 8692/283257 objects misplaced (3.069%); 2.4 MiB/s, 39 objects/s recovering 2023-10-09T05:15:33.570 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:33 smithi133 bash[102858]: audit 2023-10-09T05:15:32.085639+0000 mon.a (mon.0) 1058 : audit 1 from='client.? 172.21.15.133:0/563527921' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:15:33.570 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:33 smithi133 bash[105589]: cluster 2023-10-09T05:15:31.905294+0000 mgr.y (mgr.25277) 810 : cluster 0 pgmap v820: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 45 MiB/s wr, 718 op/s; 23274/283257 objects degraded (8.217%); 8692/283257 objects misplaced (3.069%); 2.4 MiB/s, 39 objects/s recovering 2023-10-09T05:15:33.570 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:33 smithi133 bash[105589]: audit 2023-10-09T05:15:32.085639+0000 mon.a (mon.0) 1058 : audit 1 from='client.? 172.21.15.133:0/563527921' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:15:33.652 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 89 16 96832 96816 67.979 40.5 0.00508125 0.0147028 2023-10-09T05:15:33.662 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:33.696 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:33.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:33.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:33.800 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:33.840 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:33.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:33.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:33.944 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:33.980 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:33.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:33.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:34.087 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:34.096 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:34 smithi194 bash[92666]: cluster 2023-10-09T05:15:33.050344+0000 mon.a (mon.0) 1059 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:15:34.096 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:34 smithi194 bash[92666]: audit 2023-10-09T05:15:33.075604+0000 mon.a (mon.0) 1060 : audit 1 from='client.? 172.21.15.133:0/563527921' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:15:34.097 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:34 smithi194 bash[92666]: cluster 2023-10-09T05:15:33.076327+0000 mon.a (mon.0) 1061 : cluster 0 osdmap e201: 8 total, 8 up, 7 in 2023-10-09T05:15:34.097 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:34 smithi194 bash[92666]: cluster 2023-10-09T05:15:33.077311+0000 osd.5 (osd.5) 187 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T05:15:34.097 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:34 smithi194 bash[92666]: cluster 2023-10-09T05:15:33.089123+0000 osd.6 (osd.6) 134 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T05:15:34.097 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:34 smithi194 bash[92666]: cluster 2023-10-09T05:15:33.089123+0000 osd.4 (osd.4) 150 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T05:15:34.097 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:34 smithi194 bash[92666]: cluster 2023-10-09T05:15:33.093562+0000 osd.7 (osd.7) 136 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T05:15:34.097 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:34 smithi194 bash[92666]: cluster 2023-10-09T05:15:33.327690+0000 mon.a (mon.0) 1062 : cluster 3 Health check update: Degraded data redundancy: 23274/283257 objects degraded (8.217%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:34.119 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:15:34.121 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:34.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:34.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:34.228 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:34.238 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:34 smithi133 bash[102858]: cluster 2023-10-09T05:15:33.050344+0000 mon.a (mon.0) 1059 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:15:34.238 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:34 smithi133 bash[102858]: audit 2023-10-09T05:15:33.075604+0000 mon.a (mon.0) 1060 : audit 1 from='client.? 172.21.15.133:0/563527921' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:15:34.238 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:34 smithi133 bash[102858]: cluster 2023-10-09T05:15:33.076327+0000 mon.a (mon.0) 1061 : cluster 0 osdmap e201: 8 total, 8 up, 7 in 2023-10-09T05:15:34.238 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:34 smithi133 bash[102858]: cluster 2023-10-09T05:15:33.077311+0000 osd.5 (osd.5) 187 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T05:15:34.238 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:34 smithi133 bash[102858]: cluster 2023-10-09T05:15:33.089123+0000 osd.6 (osd.6) 134 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T05:15:34.238 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:34 smithi133 bash[102858]: cluster 2023-10-09T05:15:33.089123+0000 osd.4 (osd.4) 150 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T05:15:34.238 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:34 smithi133 bash[102858]: cluster 2023-10-09T05:15:33.093562+0000 osd.7 (osd.7) 136 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T05:15:34.238 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:34 smithi133 bash[102858]: cluster 2023-10-09T05:15:33.327690+0000 mon.a (mon.0) 1062 : cluster 3 Health check update: Degraded data redundancy: 23274/283257 objects degraded (8.217%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:34.239 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:34 smithi133 bash[105589]: cluster 2023-10-09T05:15:33.050344+0000 mon.a (mon.0) 1059 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:15:34.239 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:34 smithi133 bash[105589]: audit 2023-10-09T05:15:33.075604+0000 mon.a (mon.0) 1060 : audit 1 from='client.? 172.21.15.133:0/563527921' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:15:34.239 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:34 smithi133 bash[105589]: cluster 2023-10-09T05:15:33.076327+0000 mon.a (mon.0) 1061 : cluster 0 osdmap e201: 8 total, 8 up, 7 in 2023-10-09T05:15:34.239 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:34 smithi133 bash[105589]: cluster 2023-10-09T05:15:33.077311+0000 osd.5 (osd.5) 187 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T05:15:34.239 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:34 smithi133 bash[105589]: cluster 2023-10-09T05:15:33.089123+0000 osd.6 (osd.6) 134 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T05:15:34.239 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:34 smithi133 bash[105589]: cluster 2023-10-09T05:15:33.089123+0000 osd.4 (osd.4) 150 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T05:15:34.239 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:34 smithi133 bash[105589]: cluster 2023-10-09T05:15:33.093562+0000 osd.7 (osd.7) 136 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T05:15:34.239 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:34 smithi133 bash[105589]: cluster 2023-10-09T05:15:33.327690+0000 mon.a (mon.0) 1062 : cluster 3 Health check update: Degraded data redundancy: 23274/283257 objects degraded (8.217%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:34.263 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:34.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:34.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:34.367 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:34.398 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:34.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:34.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:34.506 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:34.514 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:15:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:15:34] "GET /metrics HTTP/1.1" 200 36510 "" "Prometheus/2.43.0" 2023-10-09T05:15:34.544 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:34.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:34.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:34.650 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:34.652 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 90 11 97433 97422 67.6444 37.875 0.0134383 0.0147762 2023-10-09T05:15:34.672 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:34.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:34.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:34.775 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:34.804 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:34.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:34.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:34.910 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:34.935 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:34.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:34.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:35.039 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:35.067 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:35.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:35.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:35.169 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:35.201 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:35.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:35.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:35.305 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:35.316 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:35 smithi133 bash[102858]: cluster 2023-10-09T05:15:33.905725+0000 mgr.y (mgr.25277) 811 : cluster 0 pgmap v822: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 42 MiB/s wr, 673 op/s; 23274/283881 objects degraded (8.199%); 8582/283881 objects misplaced (3.023%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:15:35.316 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:35 smithi133 bash[105589]: cluster 2023-10-09T05:15:33.905725+0000 mgr.y (mgr.25277) 811 : cluster 0 pgmap v822: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 42 MiB/s wr, 673 op/s; 23274/283881 objects degraded (8.199%); 8582/283881 objects misplaced (3.023%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:15:35.329 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:35.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:35.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:35.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:35 smithi194 bash[92666]: cluster 2023-10-09T05:15:33.905725+0000 mgr.y (mgr.25277) 811 : cluster 0 pgmap v822: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 42 MiB/s wr, 673 op/s; 23274/283881 objects degraded (8.199%); 8582/283881 objects misplaced (3.023%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:15:35.432 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:35.456 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:35.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:35.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:35.559 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:35.585 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:35.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:35.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:35.652 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Total time run: 90.0262 2023-10-09T05:15:35.652 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Total writes made: 97433 2023-10-09T05:15:35.652 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Write size: 65536 2023-10-09T05:15:35.652 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Object size: 65536 2023-10-09T05:15:35.653 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Bandwidth (MB/sec): 67.6421 2023-10-09T05:15:35.653 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Stddev Bandwidth: 34.5212 2023-10-09T05:15:35.653 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Max bandwidth (MB/sec): 162.312 2023-10-09T05:15:35.653 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Min bandwidth (MB/sec): 0 2023-10-09T05:15:35.653 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Average IOPS: 1082 2023-10-09T05:15:35.653 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Stddev IOPS: 552.36 2023-10-09T05:15:35.653 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Max IOPS: 2597 2023-10-09T05:15:35.653 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Min IOPS: 0 2023-10-09T05:15:35.653 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Average Latency(s): 0.0147774 2023-10-09T05:15:35.653 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Stddev Latency(s): 0.0462459 2023-10-09T05:15:35.654 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Max latency(s): 2.78751 2023-10-09T05:15:35.654 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Min latency(s): 0.00128168 2023-10-09T05:15:35.660 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T05:15:35.688 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:35.725 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:35.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:35.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:35.832 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:35.855 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:35.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:35.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:35.961 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:35.989 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:35.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:35.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:36.093 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:36.134 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:36.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:36.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:36.241 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:36.266 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:36.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:36.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:36.372 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:36.402 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:36.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:36.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:36.506 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:36.532 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:36.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:36.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:36.636 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:36.660 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:36.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:36.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:36.763 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:36.789 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:36.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:36.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:36.892 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:36.932 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:36.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:36.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:37.036 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:37.062 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:37.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:37.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:37.165 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:37.200 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:37.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:37.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:37.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:37 smithi133 bash[105589]: cluster 2023-10-09T05:15:35.906535+0000 mgr.y (mgr.25277) 812 : cluster 0 pgmap v823: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 40 MiB/s wr, 643 op/s; 23274/291267 objects degraded (7.991%); 8462/291267 objects misplaced (2.905%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:15:37.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:37 smithi133 bash[102858]: cluster 2023-10-09T05:15:35.906535+0000 mgr.y (mgr.25277) 812 : cluster 0 pgmap v823: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 40 MiB/s wr, 643 op/s; 23274/291267 objects degraded (7.991%); 8462/291267 objects misplaced (2.905%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:15:37.306 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:37.357 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:37.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:37.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:37.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:37 smithi194 bash[92666]: cluster 2023-10-09T05:15:35.906535+0000 mgr.y (mgr.25277) 812 : cluster 0 pgmap v823: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 40 MiB/s wr, 643 op/s; 23274/291267 objects degraded (7.991%); 8462/291267 objects misplaced (2.905%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:15:37.464 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:37.496 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:37.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:37.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:37.600 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:37.625 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:37.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:37.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:37.729 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:37.774 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:37.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:37.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:37.882 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:37.907 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:37.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:37.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:38.014 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:38.047 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:38.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:38.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:38.153 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:38.186 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:38.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:38.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:38.292 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:38.331 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:38.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:38.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:38.437 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:38.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:38.468 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:38.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:38.570 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:38.627 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:38.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:38.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:38.732 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:38.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:38.765 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:38.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:38.843 INFO:teuthology.orchestra.run.smithi194.stdout:{ 2023-10-09T05:15:38.843 INFO:teuthology.orchestra.run.smithi194.stdout: "size": 20, 2023-10-09T05:15:38.844 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 600, 2023-10-09T05:15:38.844 INFO:teuthology.orchestra.run.smithi194.stdout: "ops": [ 2023-10-09T05:15:38.844 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.844 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:2186 9.e 9:79cde488:::benchmark_data_smithi133_153303_object2185:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-10-09T05:15:38.844 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:09:30.046855+0000", 2023-10-09T05:15:38.844 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 368.78782590899999, 2023-10-09T05:15:38.844 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.0578218499999998, 2023-10-09T05:15:38.844 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:15:38.844 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "reached pg", 2023-10-09T05:15:38.844 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:15:38.844 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:15:38.845 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:15:38.845 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 2186 2023-10-09T05:15:38.845 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.845 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:15:38.845 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.845 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:15:38.845 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.046855+0000", 2023-10-09T05:15:38.845 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.845 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.845 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.845 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:15:38.845 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.046855+0000", 2023-10-09T05:15:38.846 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.846 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.846 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.846 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:15:38.846 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.046854+0000", 2023-10-09T05:15:38.846 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:15:38.846 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.846 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.846 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:15:38.846 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.046911+0000", 2023-10-09T05:15:38.846 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.6975000000000003e-05 2023-10-09T05:15:38.846 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.847 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.847 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:15:38.847 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.046924+0000", 2023-10-09T05:15:38.847 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.2704000000000001e-05 2023-10-09T05:15:38.847 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.847 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.847 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:15:38.847 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.046929+0000", 2023-10-09T05:15:38.847 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.7060000000000001e-06 2023-10-09T05:15:38.848 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.848 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.848 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.848 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.046974+0000", 2023-10-09T05:15:38.848 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.5253999999999997e-05 2023-10-09T05:15:38.848 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.848 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.848 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:15:38.848 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.047072+0000", 2023-10-09T05:15:38.848 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.7924999999999995e-05 2023-10-09T05:15:38.848 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.849 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.849 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:15:38.849 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.047168+0000", 2023-10-09T05:15:38.849 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.5655999999999994e-05 2023-10-09T05:15:38.849 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.849 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.849 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:15:38.849 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.047623+0000", 2023-10-09T05:15:38.849 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00045538000000000002 2023-10-09T05:15:38.849 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.849 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.849 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:15:38.850 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.048874+0000", 2023-10-09T05:15:38.850 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0012501210000000001 2023-10-09T05:15:38.850 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.850 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.850 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.850 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.067946+0000", 2023-10-09T05:15:38.850 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0190723 2023-10-09T05:15:38.850 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.850 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.850 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:15:38.850 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.067956+0000", 2023-10-09T05:15:38.850 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0259e-05 2023-10-09T05:15:38.851 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.851 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.851 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.851 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:31.063908+0000", 2023-10-09T05:15:38.851 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.99595230899999998 2023-10-09T05:15:38.851 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.851 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.851 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:15:38.851 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:31.063921+0000", 2023-10-09T05:15:38.851 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.2387e-05 2023-10-09T05:15:38.851 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.851 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.852 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.852 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:32.104582+0000", 2023-10-09T05:15:38.852 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0406616099999999 2023-10-09T05:15:38.852 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.852 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.852 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:15:38.852 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:32.104677+0000", 2023-10-09T05:15:38.852 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.4148999999999996e-05 2023-10-09T05:15:38.852 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.852 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:15:38.852 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.852 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.853 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.853 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:2190 9.d 9:b085df71:::benchmark_data_smithi133_153303_object2189:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-10-09T05:15:38.853 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:09:30.047729+0000", 2023-10-09T05:15:38.853 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 368.78695202900002, 2023-10-09T05:15:38.853 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.0463494350000002, 2023-10-09T05:15:38.853 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:15:38.853 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "reached pg", 2023-10-09T05:15:38.853 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:15:38.853 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:15:38.853 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:15:38.853 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 2190 2023-10-09T05:15:38.853 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.854 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:15:38.854 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.854 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:15:38.854 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.047729+0000", 2023-10-09T05:15:38.854 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.854 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.854 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.854 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:15:38.854 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.047729+0000", 2023-10-09T05:15:38.854 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.854 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.854 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.855 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:15:38.855 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.047728+0000", 2023-10-09T05:15:38.855 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:15:38.855 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.855 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.855 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:15:38.855 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.047762+0000", 2023-10-09T05:15:38.855 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.4754e-05 2023-10-09T05:15:38.855 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.855 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.855 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:15:38.856 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.047780+0000", 2023-10-09T05:15:38.856 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.7694999999999999e-05 2023-10-09T05:15:38.856 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.856 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.856 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:15:38.856 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.047787+0000", 2023-10-09T05:15:38.856 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.1130000000000003e-06 2023-10-09T05:15:38.856 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.856 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.856 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.856 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.047823+0000", 2023-10-09T05:15:38.856 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.5877999999999998e-05 2023-10-09T05:15:38.856 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.857 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.857 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:15:38.857 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.047927+0000", 2023-10-09T05:15:38.857 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00010358599999999999 2023-10-09T05:15:38.857 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.857 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.857 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,5", 2023-10-09T05:15:38.857 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.048139+0000", 2023-10-09T05:15:38.857 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00021249499999999999 2023-10-09T05:15:38.858 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.858 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.858 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:15:38.858 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.048755+0000", 2023-10-09T05:15:38.858 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00061594700000000004 2023-10-09T05:15:38.858 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.858 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.858 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:15:38.858 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.049782+0000", 2023-10-09T05:15:38.858 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0010268639999999999 2023-10-09T05:15:38.858 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.858 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.859 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.859 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.071847+0000", 2023-10-09T05:15:38.859 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.022065596 2023-10-09T05:15:38.859 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.859 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.859 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:15:38.859 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.071856+0000", 2023-10-09T05:15:38.859 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.8890000000000008e-06 2023-10-09T05:15:38.859 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.859 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.859 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.859 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:31.059790+0000", 2023-10-09T05:15:38.860 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.98793316600000003 2023-10-09T05:15:38.860 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.860 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.860 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:15:38.860 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:31.059806+0000", 2023-10-09T05:15:38.860 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.6498999999999999e-05 2023-10-09T05:15:38.860 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.860 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.860 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.860 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:32.093974+0000", 2023-10-09T05:15:38.860 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0341678830000001 2023-10-09T05:15:38.860 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.861 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.861 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:15:38.861 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:32.094078+0000", 2023-10-09T05:15:38.861 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00010414199999999999 2023-10-09T05:15:38.861 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.861 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:15:38.861 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.861 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.861 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.861 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:2193 9.d 9:b772f3d5:::benchmark_data_smithi133_153303_object2192:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-10-09T05:15:38.861 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:09:30.050303+0000", 2023-10-09T05:15:38.861 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 368.78437788899998, 2023-10-09T05:15:38.862 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.0442809209999999, 2023-10-09T05:15:38.862 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:15:38.862 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "reached pg", 2023-10-09T05:15:38.862 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:15:38.862 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:15:38.862 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:15:38.862 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 2193 2023-10-09T05:15:38.862 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.862 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:15:38.862 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.862 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:15:38.862 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.050303+0000", 2023-10-09T05:15:38.863 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.863 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.863 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.863 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:15:38.863 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.050303+0000", 2023-10-09T05:15:38.863 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.863 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.863 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.863 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:15:38.863 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.050302+0000", 2023-10-09T05:15:38.863 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:15:38.863 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.863 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.864 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:15:38.864 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.050331+0000", 2023-10-09T05:15:38.864 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.9417999999999999e-05 2023-10-09T05:15:38.864 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.864 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.864 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:15:38.864 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.050342+0000", 2023-10-09T05:15:38.864 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.1189000000000001e-05 2023-10-09T05:15:38.864 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.864 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.864 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:15:38.864 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.050350+0000", 2023-10-09T05:15:38.865 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.1049999999999997e-06 2023-10-09T05:15:38.865 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.865 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.865 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.865 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.050385+0000", 2023-10-09T05:15:38.865 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.5692999999999999e-05 2023-10-09T05:15:38.865 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.865 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.865 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:15:38.865 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.050495+0000", 2023-10-09T05:15:38.865 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00010950000000000001 2023-10-09T05:15:38.865 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.866 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.866 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,5", 2023-10-09T05:15:38.866 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.050659+0000", 2023-10-09T05:15:38.866 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00016445699999999999 2023-10-09T05:15:38.866 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.866 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.866 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:15:38.866 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.051207+0000", 2023-10-09T05:15:38.866 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00054784300000000001 2023-10-09T05:15:38.866 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.866 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.866 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:15:38.867 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.053535+0000", 2023-10-09T05:15:38.867 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.002327778 2023-10-09T05:15:38.867 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.867 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.867 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.867 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.072040+0000", 2023-10-09T05:15:38.867 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.018505612000000001 2023-10-09T05:15:38.867 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.867 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.867 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:15:38.867 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.072049+0000", 2023-10-09T05:15:38.867 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.4540000000000007e-06 2023-10-09T05:15:38.868 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.868 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.868 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.868 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:31.059843+0000", 2023-10-09T05:15:38.868 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.98779448599999997 2023-10-09T05:15:38.868 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.868 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.868 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:15:38.868 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:31.059852+0000", 2023-10-09T05:15:38.868 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.0210000000000005e-06 2023-10-09T05:15:38.868 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.868 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.868 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.869 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:32.094525+0000", 2023-10-09T05:15:38.869 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0346724199999999 2023-10-09T05:15:38.869 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.869 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.869 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:15:38.869 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:32.094584+0000", 2023-10-09T05:15:38.869 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.8805999999999999e-05 2023-10-09T05:15:38.869 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.869 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:15:38.869 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.869 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.869 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.870 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:2205 9.e 9:75b1b20e:::benchmark_data_smithi133_153303_object2204:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-10-09T05:15:38.870 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:09:30.069979+0000", 2023-10-09T05:15:38.870 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 368.764701699, 2023-10-09T05:15:38.870 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.042229029, 2023-10-09T05:15:38.870 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:15:38.870 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:15:38.870 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:15:38.870 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:15:38.870 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:15:38.870 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 2205 2023-10-09T05:15:38.870 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.871 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:15:38.871 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.871 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:15:38.871 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.069979+0000", 2023-10-09T05:15:38.871 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.871 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.871 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.871 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:15:38.871 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.069979+0000", 2023-10-09T05:15:38.871 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.871 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.871 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.872 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:15:38.872 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.069977+0000", 2023-10-09T05:15:38.872 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999981 2023-10-09T05:15:38.872 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.872 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.872 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:15:38.872 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.070021+0000", 2023-10-09T05:15:38.872 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.4165999999999997e-05 2023-10-09T05:15:38.872 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.872 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.872 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:15:38.872 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.070035+0000", 2023-10-09T05:15:38.873 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.4046000000000001e-05 2023-10-09T05:15:38.873 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.873 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.873 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:15:38.873 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.070043+0000", 2023-10-09T05:15:38.873 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.238e-06 2023-10-09T05:15:38.873 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.873 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.873 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.873 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.070093+0000", 2023-10-09T05:15:38.873 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.0333000000000001e-05 2023-10-09T05:15:38.873 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.874 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.874 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:15:38.874 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.070102+0000", 2023-10-09T05:15:38.874 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.4429999999999992e-06 2023-10-09T05:15:38.874 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.874 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.874 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.874 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:31.063945+0000", 2023-10-09T05:15:38.874 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.99384280800000002 2023-10-09T05:15:38.874 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.874 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.874 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:15:38.875 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:31.063953+0000", 2023-10-09T05:15:38.875 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.6510000000000001e-06 2023-10-09T05:15:38.875 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.875 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.875 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.875 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:32.105133+0000", 2023-10-09T05:15:38.875 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.041179917 2023-10-09T05:15:38.875 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.875 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.875 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:15:38.875 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:32.105462+0000", 2023-10-09T05:15:38.876 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00032897100000000001 2023-10-09T05:15:38.876 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.876 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.876 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:15:38.876 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:32.105890+0000", 2023-10-09T05:15:38.876 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000428038 2023-10-09T05:15:38.876 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.876 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.876 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:15:38.876 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:32.109596+0000", 2023-10-09T05:15:38.876 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0037058550000000001 2023-10-09T05:15:38.876 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.877 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.877 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:15:38.877 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:32.110786+0000", 2023-10-09T05:15:38.877 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.001190092 2023-10-09T05:15:38.877 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.877 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.877 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:15:38.877 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:32.111866+0000", 2023-10-09T05:15:38.877 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.001080358 2023-10-09T05:15:38.877 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.877 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.877 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:15:38.878 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:32.111921+0000", 2023-10-09T05:15:38.878 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.5278000000000002e-05 2023-10-09T05:15:38.878 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.878 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.878 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:15:38.878 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:32.112208+0000", 2023-10-09T05:15:38.878 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00028653700000000001 2023-10-09T05:15:38.878 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.878 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:15:38.878 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.878 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.879 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.879 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:2207 9.e 9:70d1e0fd:::benchmark_data_smithi133_153303_object2206:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-10-09T05:15:38.879 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:09:30.077324+0000", 2023-10-09T05:15:38.879 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 368.75735699699999, 2023-10-09T05:15:38.879 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.0351857720000002, 2023-10-09T05:15:38.879 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:15:38.879 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:15:38.879 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:15:38.879 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:15:38.879 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:15:38.879 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 2207 2023-10-09T05:15:38.879 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.880 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:15:38.880 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.880 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:15:38.880 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.077324+0000", 2023-10-09T05:15:38.880 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.880 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.880 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.880 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:15:38.880 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.077324+0000", 2023-10-09T05:15:38.880 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.880 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.880 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.881 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:15:38.881 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.077322+0000", 2023-10-09T05:15:38.881 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:15:38.881 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.881 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.881 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:15:38.881 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.077371+0000", 2023-10-09T05:15:38.881 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.8535999999999997e-05 2023-10-09T05:15:38.881 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.881 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.881 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:15:38.881 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.077384+0000", 2023-10-09T05:15:38.882 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.3225e-05 2023-10-09T05:15:38.882 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.882 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.882 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:15:38.882 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.077392+0000", 2023-10-09T05:15:38.882 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.6620000000000005e-06 2023-10-09T05:15:38.882 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.882 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.882 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.882 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.077442+0000", 2023-10-09T05:15:38.882 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.0192000000000003e-05 2023-10-09T05:15:38.883 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.883 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.883 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:15:38.883 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:30.077453+0000", 2023-10-09T05:15:38.883 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.1151999999999999e-05 2023-10-09T05:15:38.883 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.883 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.883 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.883 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:31.063976+0000", 2023-10-09T05:15:38.883 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.98652265299999997 2023-10-09T05:15:38.883 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.883 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.884 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:15:38.884 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:31.063983+0000", 2023-10-09T05:15:38.884 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.4800000000000004e-06 2023-10-09T05:15:38.884 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.884 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.884 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.884 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:32.106471+0000", 2023-10-09T05:15:38.884 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.042488332 2023-10-09T05:15:38.884 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.884 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.884 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:15:38.884 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:32.107130+0000", 2023-10-09T05:15:38.885 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00065828399999999997 2023-10-09T05:15:38.885 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.885 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.885 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:15:38.885 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:32.107318+0000", 2023-10-09T05:15:38.885 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00018831100000000001 2023-10-09T05:15:38.885 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.885 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.885 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:15:38.885 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:32.110115+0000", 2023-10-09T05:15:38.885 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0027972269999999998 2023-10-09T05:15:38.886 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.886 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.886 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:15:38.886 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:32.110795+0000", 2023-10-09T05:15:38.886 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00068029699999999995 2023-10-09T05:15:38.886 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.886 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.886 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:15:38.886 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:32.112278+0000", 2023-10-09T05:15:38.886 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.001482913 2023-10-09T05:15:38.886 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.886 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.887 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:15:38.887 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:32.112338+0000", 2023-10-09T05:15:38.887 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.9206000000000002e-05 2023-10-09T05:15:38.887 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.887 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.887 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:15:38.887 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:32.112509+0000", 2023-10-09T05:15:38.887 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00017180099999999999 2023-10-09T05:15:38.887 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.887 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:15:38.887 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.888 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.888 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.888 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:4681 9.d 9:bc6f9ba3:::benchmark_data_smithi133_153303_object4680:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-10-09T05:15:38.888 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:09:33.086024+0000", 2023-10-09T05:15:38.888 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 365.74865614100003, 2023-10-09T05:15:38.888 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.017114407, 2023-10-09T05:15:38.888 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:15:38.888 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "reached pg", 2023-10-09T05:15:38.888 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:15:38.888 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:15:38.888 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:15:38.888 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 4681 2023-10-09T05:15:38.889 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.889 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:15:38.889 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.889 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:15:38.889 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:33.086024+0000", 2023-10-09T05:15:38.889 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.889 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.889 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.889 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:15:38.889 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:33.086024+0000", 2023-10-09T05:15:38.889 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.889 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.890 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.890 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:15:38.890 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:33.086023+0000", 2023-10-09T05:15:38.890 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999981 2023-10-09T05:15:38.890 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.890 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.890 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:15:38.890 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:33.086067+0000", 2023-10-09T05:15:38.890 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.4540999999999999e-05 2023-10-09T05:15:38.890 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.890 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.890 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:15:38.891 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:33.086081+0000", 2023-10-09T05:15:38.891 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.3777999999999999e-05 2023-10-09T05:15:38.891 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.891 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.891 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:15:38.891 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:33.086088+0000", 2023-10-09T05:15:38.891 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.3640000000000001e-06 2023-10-09T05:15:38.891 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.891 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.891 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.891 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:33.086156+0000", 2023-10-09T05:15:38.891 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.7665999999999999e-05 2023-10-09T05:15:38.892 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.892 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.892 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:15:38.892 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:33.086302+0000", 2023-10-09T05:15:38.892 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00014646400000000001 2023-10-09T05:15:38.892 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.892 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.892 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 4,5", 2023-10-09T05:15:38.892 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:33.086470+0000", 2023-10-09T05:15:38.893 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00016749199999999999 2023-10-09T05:15:38.893 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.893 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.893 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:15:38.893 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:33.087601+0000", 2023-10-09T05:15:38.893 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.001130757 2023-10-09T05:15:38.893 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.893 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.893 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.894 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:33.090997+0000", 2023-10-09T05:15:38.894 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0033959630000000001 2023-10-09T05:15:38.894 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.894 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.894 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:15:38.894 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:33.091005+0000", 2023-10-09T05:15:38.894 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.4400000000000005e-06 2023-10-09T05:15:38.894 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.894 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.894 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.894 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:34.099563+0000", 2023-10-09T05:15:38.895 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0085580670000001 2023-10-09T05:15:38.895 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.895 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.895 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for degraded object", 2023-10-09T05:15:38.895 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:34.100181+0000", 2023-10-09T05:15:38.895 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00061774300000000002 2023-10-09T05:15:38.895 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.895 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.895 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.895 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:34.103024+0000", 2023-10-09T05:15:38.895 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0028430690000000002 2023-10-09T05:15:38.895 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.896 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.896 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:15:38.896 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:34.103139+0000", 2023-10-09T05:15:38.896 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00011493999999999999 2023-10-09T05:15:38.896 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.896 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:15:38.896 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.896 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.896 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.896 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:4693 9.d 9:b501a234:::benchmark_data_smithi133_153303_object4692:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-10-09T05:15:38.896 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:09:33.094736+0000", 2023-10-09T05:15:38.897 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 365.73994424300002, 2023-10-09T05:15:38.897 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.011266102, 2023-10-09T05:15:38.897 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:15:38.897 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:15:38.897 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:15:38.897 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:15:38.897 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:15:38.897 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 4693 2023-10-09T05:15:38.897 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.897 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:15:38.897 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.897 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:15:38.898 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:33.094736+0000", 2023-10-09T05:15:38.898 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.898 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.898 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.898 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:15:38.898 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:33.094736+0000", 2023-10-09T05:15:38.898 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.898 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.898 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.898 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:15:38.898 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:33.094735+0000", 2023-10-09T05:15:38.899 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:15:38.899 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.899 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.899 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:15:38.899 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:33.094771+0000", 2023-10-09T05:15:38.899 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.5703e-05 2023-10-09T05:15:38.899 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.899 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.899 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:15:38.899 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:33.094783+0000", 2023-10-09T05:15:38.899 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.2119e-05 2023-10-09T05:15:38.899 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.899 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.900 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:15:38.900 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:33.094789+0000", 2023-10-09T05:15:38.900 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.1149999999999999e-06 2023-10-09T05:15:38.900 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.900 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.900 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.900 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:33.095097+0000", 2023-10-09T05:15:38.900 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00030841899999999999 2023-10-09T05:15:38.900 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.900 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.900 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:15:38.900 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:33.095108+0000", 2023-10-09T05:15:38.901 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0351e-05 2023-10-09T05:15:38.901 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.901 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.901 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.901 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:34.100231+0000", 2023-10-09T05:15:38.901 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.005123145 2023-10-09T05:15:38.901 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.901 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.901 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:15:38.901 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:34.100393+0000", 2023-10-09T05:15:38.901 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00016193699999999999 2023-10-09T05:15:38.901 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.902 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.902 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 4,5", 2023-10-09T05:15:38.902 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:34.100559+0000", 2023-10-09T05:15:38.902 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000165942 2023-10-09T05:15:38.902 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.902 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.902 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:15:38.902 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:34.103216+0000", 2023-10-09T05:15:38.902 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0026568849999999999 2023-10-09T05:15:38.902 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.902 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.902 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:15:38.903 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:34.103253+0000", 2023-10-09T05:15:38.903 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.7236000000000001e-05 2023-10-09T05:15:38.903 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.903 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.903 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:15:38.903 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:34.105837+0000", 2023-10-09T05:15:38.903 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0025839679999999999 2023-10-09T05:15:38.903 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.903 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.903 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:15:38.903 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:34.105894+0000", 2023-10-09T05:15:38.903 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.7219000000000002e-05 2023-10-09T05:15:38.904 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.904 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.904 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:15:38.904 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:34.106002+0000", 2023-10-09T05:15:38.904 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000108447 2023-10-09T05:15:38.904 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.904 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:15:38.904 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.904 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.904 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.904 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:4701 9.d 9:b05a0620:::benchmark_data_smithi133_153303_object4700:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-10-09T05:15:38.904 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:09:33.098158+0000", 2023-10-09T05:15:38.905 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 365.73652234100001, 2023-10-09T05:15:38.905 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0107389680000001, 2023-10-09T05:15:38.905 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:15:38.905 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:15:38.905 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:15:38.905 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:15:38.905 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:15:38.905 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 4701 2023-10-09T05:15:38.905 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.905 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:15:38.905 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.905 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:15:38.906 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:33.098158+0000", 2023-10-09T05:15:38.906 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.906 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.906 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.906 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:15:38.906 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:33.098158+0000", 2023-10-09T05:15:38.906 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.906 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.906 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.906 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:15:38.906 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:33.098157+0000", 2023-10-09T05:15:38.907 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:15:38.907 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.907 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.907 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:15:38.907 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:33.098188+0000", 2023-10-09T05:15:38.907 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.0930999999999999e-05 2023-10-09T05:15:38.907 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.907 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.907 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:15:38.907 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:33.098199+0000", 2023-10-09T05:15:38.907 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.1505000000000001e-05 2023-10-09T05:15:38.907 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.908 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.908 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:15:38.908 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:33.098205+0000", 2023-10-09T05:15:38.908 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.7780000000000004e-06 2023-10-09T05:15:38.908 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.908 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.908 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.908 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:33.098265+0000", 2023-10-09T05:15:38.908 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.0436999999999998e-05 2023-10-09T05:15:38.908 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.908 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.909 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:15:38.909 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:33.098275+0000", 2023-10-09T05:15:38.909 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.944e-06 2023-10-09T05:15:38.909 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.909 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.909 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.909 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:34.101114+0000", 2023-10-09T05:15:38.909 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0028390949999999 2023-10-09T05:15:38.909 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.909 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.909 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:15:38.909 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:34.101265+0000", 2023-10-09T05:15:38.910 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000150187 2023-10-09T05:15:38.910 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.910 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.910 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 4,5", 2023-10-09T05:15:38.910 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:34.101388+0000", 2023-10-09T05:15:38.910 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00012338 2023-10-09T05:15:38.910 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.910 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.910 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:15:38.910 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:34.103298+0000", 2023-10-09T05:15:38.910 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0019101680000000001 2023-10-09T05:15:38.910 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.911 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.911 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:15:38.911 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:34.106015+0000", 2023-10-09T05:15:38.911 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0027167929999999999 2023-10-09T05:15:38.911 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.911 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.911 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:15:38.911 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:34.108717+0000", 2023-10-09T05:15:38.911 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0027021660000000002 2023-10-09T05:15:38.911 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.911 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.911 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:15:38.912 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:34.108781+0000", 2023-10-09T05:15:38.912 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.3665999999999997e-05 2023-10-09T05:15:38.912 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.912 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.912 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:15:38.912 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:34.108897+0000", 2023-10-09T05:15:38.912 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000116443 2023-10-09T05:15:38.912 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.912 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:15:38.912 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.912 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.912 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.913 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:4873 9.e 9:701ee628:::benchmark_data_smithi133_153303_object4872:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-10-09T05:15:38.913 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:09:35.110260+0000", 2023-10-09T05:15:38.913 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 363.724420619, 2023-10-09T05:15:38.913 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0126051629999999, 2023-10-09T05:15:38.913 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:15:38.913 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:15:38.913 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:15:38.913 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:15:38.913 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:15:38.913 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 4873 2023-10-09T05:15:38.913 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.913 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:15:38.914 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.914 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:15:38.914 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.110260+0000", 2023-10-09T05:15:38.914 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.914 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.914 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.914 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:15:38.914 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.110260+0000", 2023-10-09T05:15:38.914 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.914 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.914 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.915 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:15:38.915 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.110258+0000", 2023-10-09T05:15:38.915 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999976 2023-10-09T05:15:38.915 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.915 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.915 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:15:38.915 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.110427+0000", 2023-10-09T05:15:38.915 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00016919099999999999 2023-10-09T05:15:38.915 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.915 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.915 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:15:38.915 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.110439+0000", 2023-10-09T05:15:38.916 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.2069000000000001e-05 2023-10-09T05:15:38.916 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.916 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.916 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:15:38.916 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.110446+0000", 2023-10-09T05:15:38.916 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.8680000000000001e-06 2023-10-09T05:15:38.916 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.916 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.916 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.916 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.110547+0000", 2023-10-09T05:15:38.916 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000101432 2023-10-09T05:15:38.916 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.916 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.917 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:15:38.917 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.110561+0000", 2023-10-09T05:15:38.917 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.3349999999999999e-05 2023-10-09T05:15:38.917 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.917 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.917 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.917 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.119661+0000", 2023-10-09T05:15:38.917 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0091000020000001 2023-10-09T05:15:38.917 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.917 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.917 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:15:38.917 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.119804+0000", 2023-10-09T05:15:38.918 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000143101 2023-10-09T05:15:38.918 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.918 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.918 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:15:38.918 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.119997+0000", 2023-10-09T05:15:38.918 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00019292499999999999 2023-10-09T05:15:38.918 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.918 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.918 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:15:38.918 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.121912+0000", 2023-10-09T05:15:38.918 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0019150090000000001 2023-10-09T05:15:38.919 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.919 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.919 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:15:38.919 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.122346+0000", 2023-10-09T05:15:38.919 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00043386199999999997 2023-10-09T05:15:38.919 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.919 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.919 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:15:38.919 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.122777+0000", 2023-10-09T05:15:38.919 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00043157999999999998 2023-10-09T05:15:38.919 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.919 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.919 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:15:38.920 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.122831+0000", 2023-10-09T05:15:38.920 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.4104999999999998e-05 2023-10-09T05:15:38.920 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.920 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.920 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:15:38.920 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.122865+0000", 2023-10-09T05:15:38.920 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.383e-05 2023-10-09T05:15:38.920 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.920 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:15:38.920 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.920 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.920 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.921 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:4879 9.e 9:7bf1e0b1:::benchmark_data_smithi133_153303_object4878:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-10-09T05:15:38.921 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:09:35.113115+0000", 2023-10-09T05:15:38.921 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 363.72156568299999, 2023-10-09T05:15:38.921 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.010506417, 2023-10-09T05:15:38.921 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:15:38.921 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:15:38.921 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:15:38.921 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:15:38.921 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:15:38.922 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 4879 2023-10-09T05:15:38.922 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.922 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:15:38.922 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.922 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:15:38.922 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.113115+0000", 2023-10-09T05:15:38.922 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.922 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.922 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.922 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:15:38.922 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.113115+0000", 2023-10-09T05:15:38.923 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.923 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.923 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.923 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:15:38.923 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.113113+0000", 2023-10-09T05:15:38.923 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:15:38.923 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.923 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.923 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:15:38.923 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.113156+0000", 2023-10-09T05:15:38.923 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.2293000000000003e-05 2023-10-09T05:15:38.923 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.923 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.924 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:15:38.924 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.113171+0000", 2023-10-09T05:15:38.924 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.5634999999999999e-05 2023-10-09T05:15:38.924 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.924 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.924 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:15:38.924 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.113179+0000", 2023-10-09T05:15:38.924 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.7870000000000002e-06 2023-10-09T05:15:38.924 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.924 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.924 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.925 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.113256+0000", 2023-10-09T05:15:38.925 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.6631000000000005e-05 2023-10-09T05:15:38.925 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.925 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.925 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:15:38.925 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:35.113271+0000", 2023-10-09T05:15:38.925 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.5167999999999999e-05 2023-10-09T05:15:38.925 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.925 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.925 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.925 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.121109+0000", 2023-10-09T05:15:38.925 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0078378589999999 2023-10-09T05:15:38.926 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.926 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.926 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:15:38.926 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.121254+0000", 2023-10-09T05:15:38.926 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000145372 2023-10-09T05:15:38.926 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.926 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.926 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:15:38.926 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.121448+0000", 2023-10-09T05:15:38.926 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00019430000000000001 2023-10-09T05:15:38.926 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.926 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.927 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:15:38.927 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.122139+0000", 2023-10-09T05:15:38.927 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00069094899999999995 2023-10-09T05:15:38.927 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.927 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.927 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:15:38.927 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.123351+0000", 2023-10-09T05:15:38.927 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.001212039 2023-10-09T05:15:38.927 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.927 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.927 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:15:38.927 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.123552+0000", 2023-10-09T05:15:38.928 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000200607 2023-10-09T05:15:38.928 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.928 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.928 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:15:38.928 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.123597+0000", 2023-10-09T05:15:38.928 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.5153e-05 2023-10-09T05:15:38.928 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.928 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.928 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:15:38.928 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:09:36.123621+0000", 2023-10-09T05:15:38.928 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.4239999999999998e-05 2023-10-09T05:15:38.928 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.929 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:15:38.929 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.929 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.929 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.929 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:158242 9.e 9:7050cadf:::benchmark_data_smithi133_153303_object68273:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e166)", 2023-10-09T05:15:38.929 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:12:32.025649+0000", 2023-10-09T05:15:38.929 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 186.80903130499999, 2023-10-09T05:15:38.929 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.269990177, 2023-10-09T05:15:38.929 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:15:38.929 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "reached pg", 2023-10-09T05:15:38.929 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:15:38.929 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:15:38.929 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:15:38.930 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 158242 2023-10-09T05:15:38.930 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.930 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:15:38.930 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.930 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:15:38.930 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.025649+0000", 2023-10-09T05:15:38.930 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.930 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.930 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.930 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:15:38.930 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.025649+0000", 2023-10-09T05:15:38.931 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.931 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.931 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.931 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:15:38.931 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.025648+0000", 2023-10-09T05:15:38.931 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999981 2023-10-09T05:15:38.931 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.931 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.931 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:15:38.931 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.025656+0000", 2023-10-09T05:15:38.931 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.6759999999999996e-06 2023-10-09T05:15:38.931 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.931 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.932 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:15:38.932 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.025669+0000", 2023-10-09T05:15:38.932 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.2544999999999999e-05 2023-10-09T05:15:38.932 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.932 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.932 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:15:38.932 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.025677+0000", 2023-10-09T05:15:38.932 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.7889999999999995e-06 2023-10-09T05:15:38.932 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.932 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.932 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.932 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.025776+0000", 2023-10-09T05:15:38.933 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.9469000000000006e-05 2023-10-09T05:15:38.933 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.933 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.933 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:15:38.933 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.025924+0000", 2023-10-09T05:15:38.933 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000148307 2023-10-09T05:15:38.933 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.933 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.933 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:15:38.933 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.026059+0000", 2023-10-09T05:15:38.933 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00013493199999999999 2023-10-09T05:15:38.933 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.933 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.934 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:15:38.934 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.031080+0000", 2023-10-09T05:15:38.934 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.005020989 2023-10-09T05:15:38.934 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.934 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.934 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:15:38.934 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.032527+0000", 2023-10-09T05:15:38.934 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0014469159999999999 2023-10-09T05:15:38.934 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.934 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.934 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.934 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.295629+0000", 2023-10-09T05:15:38.935 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.263101735 2023-10-09T05:15:38.935 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.935 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.935 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:15:38.935 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.295640+0000", 2023-10-09T05:15:38.935 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0531e-05 2023-10-09T05:15:38.935 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.935 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:15:38.935 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.935 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.935 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.935 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:158249 9.d 9:bb24e501:::benchmark_data_smithi133_153303_object68280:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e166)", 2023-10-09T05:15:38.936 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:12:32.030395+0000", 2023-10-09T05:15:38.936 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 186.804285988, 2023-10-09T05:15:38.936 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.247645481, 2023-10-09T05:15:38.936 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:15:38.936 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "reached pg", 2023-10-09T05:15:38.936 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:15:38.936 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:15:38.936 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:15:38.936 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 158249 2023-10-09T05:15:38.936 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.936 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:15:38.936 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.937 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:15:38.937 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.030395+0000", 2023-10-09T05:15:38.937 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.937 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.937 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.937 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:15:38.937 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.030395+0000", 2023-10-09T05:15:38.937 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.937 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.937 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.937 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:15:38.937 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.030393+0000", 2023-10-09T05:15:38.937 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999981 2023-10-09T05:15:38.938 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.938 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.938 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:15:38.938 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.030400+0000", 2023-10-09T05:15:38.938 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.2239999999999998e-06 2023-10-09T05:15:38.938 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.938 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.938 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:15:38.938 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.030418+0000", 2023-10-09T05:15:38.938 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.7770000000000001e-05 2023-10-09T05:15:38.938 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.938 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.939 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:15:38.939 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.030425+0000", 2023-10-09T05:15:38.939 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.5819999999999998e-06 2023-10-09T05:15:38.939 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.939 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.939 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.939 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.035754+0000", 2023-10-09T05:15:38.939 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0053298429999999999 2023-10-09T05:15:38.939 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.939 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.939 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:15:38.939 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.035884+0000", 2023-10-09T05:15:38.940 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00012952999999999999 2023-10-09T05:15:38.940 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.940 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.940 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,5", 2023-10-09T05:15:38.940 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.036015+0000", 2023-10-09T05:15:38.940 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00013157300000000001 2023-10-09T05:15:38.940 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.940 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.940 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:15:38.940 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.040668+0000", 2023-10-09T05:15:38.940 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0046526839999999998 2023-10-09T05:15:38.940 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.941 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.941 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:15:38.941 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.040764+0000", 2023-10-09T05:15:38.941 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.5486e-05 2023-10-09T05:15:38.941 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.941 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.941 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.941 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.278024+0000", 2023-10-09T05:15:38.941 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.23726077400000001 2023-10-09T05:15:38.941 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.941 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.941 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:15:38.942 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.278040+0000", 2023-10-09T05:15:38.942 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.5736999999999999e-05 2023-10-09T05:15:38.942 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.942 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:15:38.942 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.942 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.942 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.942 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:158242 9.e 9:7050cadf:::benchmark_data_smithi133_153303_object68273:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T05:15:38.942 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:12:32.242322+0000", 2023-10-09T05:15:38.942 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 186.592358631, 2023-10-09T05:15:38.942 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.076416432, 2023-10-09T05:15:38.942 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:15:38.943 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "reached pg", 2023-10-09T05:15:38.943 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:15:38.943 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:15:38.943 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:15:38.943 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 158242 2023-10-09T05:15:38.943 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.943 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:15:38.943 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.943 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:15:38.943 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.242322+0000", 2023-10-09T05:15:38.943 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.943 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.944 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.944 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:15:38.944 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.242322+0000", 2023-10-09T05:15:38.944 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.944 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.944 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.944 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:15:38.944 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.242320+0000", 2023-10-09T05:15:38.944 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999981 2023-10-09T05:15:38.944 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.944 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.944 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:15:38.945 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.242327+0000", 2023-10-09T05:15:38.945 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.4189999999999999e-06 2023-10-09T05:15:38.945 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.945 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.945 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:15:38.945 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.242341+0000", 2023-10-09T05:15:38.945 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.4698000000000001e-05 2023-10-09T05:15:38.945 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.945 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.945 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:15:38.945 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.242351+0000", 2023-10-09T05:15:38.945 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.7559999999999994e-06 2023-10-09T05:15:38.946 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.946 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.946 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.946 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.295695+0000", 2023-10-09T05:15:38.946 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.053344025000000003 2023-10-09T05:15:38.946 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.946 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.946 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:15:38.946 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.295704+0000", 2023-10-09T05:15:38.946 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.4710000000000007e-06 2023-10-09T05:15:38.946 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.947 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.947 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.947 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.055183+0000", 2023-10-09T05:15:38.947 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.75947893099999997 2023-10-09T05:15:38.947 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.947 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.947 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:15:38.947 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.055200+0000", 2023-10-09T05:15:38.947 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.7136e-05 2023-10-09T05:15:38.947 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.947 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.947 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.948 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.318661+0000", 2023-10-09T05:15:38.948 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.26346133500000002 2023-10-09T05:15:38.948 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.948 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.948 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:15:38.948 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.318738+0000", 2023-10-09T05:15:38.948 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.7367000000000007e-05 2023-10-09T05:15:38.948 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.948 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:15:38.948 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.948 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.948 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.949 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:158249 9.1d 9:bb24e501:::benchmark_data_smithi133_153303_object68280:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T05:15:38.949 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:12:32.242358+0000", 2023-10-09T05:15:38.949 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 186.59232273399999, 2023-10-09T05:15:38.949 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.90621389699999999, 2023-10-09T05:15:38.949 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:15:38.949 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "reached pg", 2023-10-09T05:15:38.949 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:15:38.949 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:15:38.949 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:15:38.949 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 158249 2023-10-09T05:15:38.949 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.949 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:15:38.950 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.950 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:15:38.950 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.242358+0000", 2023-10-09T05:15:38.950 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.950 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.950 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.950 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:15:38.950 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.242358+0000", 2023-10-09T05:15:38.950 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.950 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.950 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.950 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:15:38.951 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.242358+0000", 2023-10-09T05:15:38.951 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999995 2023-10-09T05:15:38.951 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.951 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.951 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:15:38.951 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.242360+0000", 2023-10-09T05:15:38.951 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.745e-06 2023-10-09T05:15:38.951 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.951 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.951 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:15:38.951 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.242366+0000", 2023-10-09T05:15:38.951 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.6720000000000001e-06 2023-10-09T05:15:38.952 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.952 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.952 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:15:38.952 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.242370+0000", 2023-10-09T05:15:38.952 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.6909999999999999e-06 2023-10-09T05:15:38.952 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.952 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.952 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.952 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.278815+0000", 2023-10-09T05:15:38.952 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.036444923999999997 2023-10-09T05:15:38.952 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.952 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.953 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:15:38.953 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.278823+0000", 2023-10-09T05:15:38.953 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.9530000000000005e-06 2023-10-09T05:15:38.953 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.953 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.953 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.953 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.145888+0000", 2023-10-09T05:15:38.953 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.86706470999999996 2023-10-09T05:15:38.953 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.953 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.953 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for missing object", 2023-10-09T05:15:38.953 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.146052+0000", 2023-10-09T05:15:38.954 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00016355300000000001 2023-10-09T05:15:38.954 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.954 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.954 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.954 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.148491+0000", 2023-10-09T05:15:38.954 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.002439119 2023-10-09T05:15:38.954 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.954 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.954 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:15:38.954 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.148572+0000", 2023-10-09T05:15:38.954 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.0921000000000007e-05 2023-10-09T05:15:38.955 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.955 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:15:38.955 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.955 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.955 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.955 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:158259 9.e 9:72f8432f:::benchmark_data_smithi133_153303_object68290:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T05:15:38.955 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:12:32.242374+0000", 2023-10-09T05:15:38.955 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 186.59230670700001, 2023-10-09T05:15:38.955 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0951018130000001, 2023-10-09T05:15:38.955 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:15:38.955 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:15:38.955 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:15:38.956 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:15:38.956 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:15:38.956 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 158259 2023-10-09T05:15:38.956 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.956 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:15:38.956 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.956 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:15:38.956 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.242374+0000", 2023-10-09T05:15:38.956 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.956 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.956 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.956 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:15:38.957 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.242374+0000", 2023-10-09T05:15:38.957 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.957 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.957 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.957 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:15:38.957 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.242374+0000", 2023-10-09T05:15:38.957 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999995 2023-10-09T05:15:38.957 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.957 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.957 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:15:38.957 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.242376+0000", 2023-10-09T05:15:38.958 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.2800000000000002e-06 2023-10-09T05:15:38.958 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.958 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.958 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:15:38.958 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.242381+0000", 2023-10-09T05:15:38.958 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.9749999999999998e-06 2023-10-09T05:15:38.958 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.958 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.958 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:15:38.958 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.242384+0000", 2023-10-09T05:15:38.958 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.3189999999999999e-06 2023-10-09T05:15:38.958 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.959 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.959 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.959 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.297246+0000", 2023-10-09T05:15:38.959 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.054862326000000003 2023-10-09T05:15:38.959 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.959 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.959 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:15:38.959 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.297259+0000", 2023-10-09T05:15:38.959 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.2952999999999999e-05 2023-10-09T05:15:38.959 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.959 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.959 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.960 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.056659+0000", 2023-10-09T05:15:38.960 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.75939975599999998 2023-10-09T05:15:38.960 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.960 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.960 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:15:38.960 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.056671+0000", 2023-10-09T05:15:38.960 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.173e-05 2023-10-09T05:15:38.960 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.960 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.960 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.960 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.327890+0000", 2023-10-09T05:15:38.961 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.27121921300000001 2023-10-09T05:15:38.961 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.961 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.961 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:15:38.961 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.328052+0000", 2023-10-09T05:15:38.961 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000161471 2023-10-09T05:15:38.961 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.961 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.961 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:15:38.961 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.328519+0000", 2023-10-09T05:15:38.961 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00046742399999999998 2023-10-09T05:15:38.961 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.961 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.962 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:15:38.962 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.333939+0000", 2023-10-09T05:15:38.962 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0054198079999999999 2023-10-09T05:15:38.962 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.962 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.962 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:15:38.962 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.337127+0000", 2023-10-09T05:15:38.962 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0031883269999999999 2023-10-09T05:15:38.962 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.962 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.962 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:15:38.962 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.337235+0000", 2023-10-09T05:15:38.963 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00010746700000000001 2023-10-09T05:15:38.963 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.963 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.963 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:15:38.963 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.337276+0000", 2023-10-09T05:15:38.963 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.1783000000000003e-05 2023-10-09T05:15:38.963 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.963 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.963 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:15:38.963 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.337476+0000", 2023-10-09T05:15:38.963 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000199326 2023-10-09T05:15:38.963 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.964 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:15:38.964 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.964 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.964 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.964 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:158262 9.1d 9:b94f2bcf:::benchmark_data_smithi133_153303_object68293:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T05:15:38.964 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:12:32.353297+0000", 2023-10-09T05:15:38.964 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 186.481383459, 2023-10-09T05:15:38.964 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.79723211800000005, 2023-10-09T05:15:38.964 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:15:38.964 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:15:38.964 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:15:38.964 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:15:38.965 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:15:38.965 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 158262 2023-10-09T05:15:38.965 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.965 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:15:38.965 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.965 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:15:38.965 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.353297+0000", 2023-10-09T05:15:38.965 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.965 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.965 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.965 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:15:38.965 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.353297+0000", 2023-10-09T05:15:38.966 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.966 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.966 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.966 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:15:38.966 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.353294+0000", 2023-10-09T05:15:38.966 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999962 2023-10-09T05:15:38.966 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.966 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.966 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:15:38.966 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.353311+0000", 2023-10-09T05:15:38.966 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.7759999999999999e-05 2023-10-09T05:15:38.966 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.967 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.967 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:15:38.967 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.353340+0000", 2023-10-09T05:15:38.967 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.8931000000000001e-05 2023-10-09T05:15:38.967 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.967 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.967 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:15:38.967 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.353352+0000", 2023-10-09T05:15:38.967 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.2106000000000001e-05 2023-10-09T05:15:38.967 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.967 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.967 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.968 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.353419+0000", 2023-10-09T05:15:38.968 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.6766000000000004e-05 2023-10-09T05:15:38.968 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.968 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.968 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:15:38.968 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.353430+0000", 2023-10-09T05:15:38.968 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0903999999999999e-05 2023-10-09T05:15:38.968 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.968 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.968 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.968 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.146302+0000", 2023-10-09T05:15:38.968 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.79287152400000005 2023-10-09T05:15:38.969 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.969 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.969 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:15:38.969 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.146437+0000", 2023-10-09T05:15:38.969 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000135325 2023-10-09T05:15:38.969 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.969 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.969 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,5", 2023-10-09T05:15:38.969 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.146762+0000", 2023-10-09T05:15:38.969 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00032545700000000002 2023-10-09T05:15:38.969 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.969 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.969 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:15:38.970 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.148647+0000", 2023-10-09T05:15:38.970 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0018850760000000001 2023-10-09T05:15:38.970 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.970 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.970 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:15:38.970 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.149759+0000", 2023-10-09T05:15:38.970 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0011119960000000001 2023-10-09T05:15:38.970 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.970 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.970 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:15:38.970 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.150348+0000", 2023-10-09T05:15:38.971 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00058828899999999998 2023-10-09T05:15:38.971 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.971 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.971 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:15:38.971 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.150405+0000", 2023-10-09T05:15:38.971 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.7055000000000002e-05 2023-10-09T05:15:38.971 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.971 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.971 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:15:38.971 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.150529+0000", 2023-10-09T05:15:38.971 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000124518 2023-10-09T05:15:38.971 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.972 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:15:38.972 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.972 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.972 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.972 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:158268 9.1e 9:7bd3fa9b:::benchmark_data_smithi133_153303_object68299:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T05:15:38.972 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:12:32.406406+0000", 2023-10-09T05:15:38.972 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 186.42827482000001, 2023-10-09T05:15:38.972 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.74728764800000003, 2023-10-09T05:15:38.972 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:15:38.972 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:15:38.972 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:15:38.973 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:15:38.973 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:15:38.973 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 158268 2023-10-09T05:15:38.973 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.973 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:15:38.973 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.973 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:15:38.973 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.406406+0000", 2023-10-09T05:15:38.973 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.973 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.973 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.973 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:15:38.974 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.406406+0000", 2023-10-09T05:15:38.974 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.974 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.974 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.974 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:15:38.974 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.406404+0000", 2023-10-09T05:15:38.974 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999981 2023-10-09T05:15:38.974 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.974 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.974 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:15:38.974 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.406415+0000", 2023-10-09T05:15:38.974 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.1481000000000001e-05 2023-10-09T05:15:38.975 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.975 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.975 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:15:38.975 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.406435+0000", 2023-10-09T05:15:38.975 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.9989000000000001e-05 2023-10-09T05:15:38.975 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.975 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.975 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:15:38.975 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.406444+0000", 2023-10-09T05:15:38.975 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.1620000000000008e-06 2023-10-09T05:15:38.975 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.976 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.976 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.976 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.406501+0000", 2023-10-09T05:15:38.976 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.7500000000000002e-05 2023-10-09T05:15:38.976 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.976 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.976 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:15:38.976 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.406509+0000", 2023-10-09T05:15:38.976 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.9130000000000003e-06 2023-10-09T05:15:38.976 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.976 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.976 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.977 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.146488+0000", 2023-10-09T05:15:38.977 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.73997920299999997 2023-10-09T05:15:38.977 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.977 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.977 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:15:38.977 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.146600+0000", 2023-10-09T05:15:38.977 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00011184 2023-10-09T05:15:38.977 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.977 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.977 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:15:38.977 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.146918+0000", 2023-10-09T05:15:38.977 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00031838800000000002 2023-10-09T05:15:38.978 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.978 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.978 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:15:38.978 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.148326+0000", 2023-10-09T05:15:38.978 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.001407199 2023-10-09T05:15:38.978 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.978 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.978 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:15:38.978 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.150015+0000", 2023-10-09T05:15:38.978 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0016890200000000001 2023-10-09T05:15:38.978 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.978 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.979 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:15:38.979 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.151118+0000", 2023-10-09T05:15:38.979 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.001103092 2023-10-09T05:15:38.979 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.979 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.979 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:15:38.979 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.151166+0000", 2023-10-09T05:15:38.979 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.8019999999999998e-05 2023-10-09T05:15:38.979 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.979 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.979 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:15:38.979 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.153693+0000", 2023-10-09T05:15:38.980 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0025277099999999999 2023-10-09T05:15:38.980 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.980 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:15:38.980 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.980 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.980 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.980 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:158275 9.e 9:720c7c44:::benchmark_data_smithi133_153303_object68306:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-10-09T05:15:38.980 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:12:33.103445+0000", 2023-10-09T05:15:38.980 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 185.73123540200001, 2023-10-09T05:15:38.980 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.234297385, 2023-10-09T05:15:38.980 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:15:38.980 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:15:38.981 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:15:38.981 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:15:38.981 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:15:38.981 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 158275 2023-10-09T05:15:38.981 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.981 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:15:38.981 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.981 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:15:38.981 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.103445+0000", 2023-10-09T05:15:38.981 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.981 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.981 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.982 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:15:38.982 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.103445+0000", 2023-10-09T05:15:38.982 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.982 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.982 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.982 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:15:38.982 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.103443+0000", 2023-10-09T05:15:38.982 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999981 2023-10-09T05:15:38.982 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.982 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.982 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:15:38.982 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.103451+0000", 2023-10-09T05:15:38.983 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.3540000000000002e-06 2023-10-09T05:15:38.983 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.983 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.983 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:15:38.983 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.103466+0000", 2023-10-09T05:15:38.983 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.5594999999999999e-05 2023-10-09T05:15:38.983 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.983 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.983 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:15:38.983 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.103476+0000", 2023-10-09T05:15:38.983 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0183000000000001e-05 2023-10-09T05:15:38.983 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.984 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.984 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.984 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.112718+0000", 2023-10-09T05:15:38.984 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0092413589999999993 2023-10-09T05:15:38.984 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.984 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.984 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:15:38.984 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.112729+0000", 2023-10-09T05:15:38.984 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.1647e-05 2023-10-09T05:15:38.984 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.984 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.985 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.985 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.330704+0000", 2023-10-09T05:15:38.985 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.21797418900000001 2023-10-09T05:15:38.985 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.985 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.985 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:15:38.985 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.330857+0000", 2023-10-09T05:15:38.985 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00015367000000000001 2023-10-09T05:15:38.985 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.985 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.985 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:15:38.985 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.331092+0000", 2023-10-09T05:15:38.985 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00023427100000000001 2023-10-09T05:15:38.986 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.986 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.986 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:15:38.986 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.337053+0000", 2023-10-09T05:15:38.986 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0059619030000000002 2023-10-09T05:15:38.986 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.986 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.986 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:15:38.986 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.337189+0000", 2023-10-09T05:15:38.986 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000135296 2023-10-09T05:15:38.986 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.986 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.987 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:15:38.987 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.337527+0000", 2023-10-09T05:15:38.987 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000338532 2023-10-09T05:15:38.987 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.987 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.987 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:15:38.987 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.337584+0000", 2023-10-09T05:15:38.987 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.7182999999999998e-05 2023-10-09T05:15:38.987 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.987 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.987 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:15:38.987 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.337743+0000", 2023-10-09T05:15:38.988 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00015815999999999999 2023-10-09T05:15:38.988 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.988 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:15:38.988 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.988 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.988 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.988 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:158276 9.e 9:73d2d973:::benchmark_data_smithi133_153303_object68307:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-10-09T05:15:38.988 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:12:33.103485+0000", 2023-10-09T05:15:38.988 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 185.73119610800001, 2023-10-09T05:15:38.988 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.24156750099999999, 2023-10-09T05:15:38.988 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:15:38.988 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:15:38.989 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:15:38.989 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:15:38.989 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:15:38.989 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 158276 2023-10-09T05:15:38.989 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.989 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:15:38.989 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.989 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:15:38.989 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.103485+0000", 2023-10-09T05:15:38.989 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.989 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.989 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.990 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:15:38.990 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.103485+0000", 2023-10-09T05:15:38.990 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.990 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.990 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.990 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:15:38.990 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.103484+0000", 2023-10-09T05:15:38.990 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999995 2023-10-09T05:15:38.990 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.990 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.990 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:15:38.990 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.103487+0000", 2023-10-09T05:15:38.991 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.92e-06 2023-10-09T05:15:38.991 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.991 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.991 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:15:38.991 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.103494+0000", 2023-10-09T05:15:38.991 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.6429999999999998e-06 2023-10-09T05:15:38.991 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.991 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.991 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:15:38.991 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.103498+0000", 2023-10-09T05:15:38.991 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.5460000000000002e-06 2023-10-09T05:15:38.991 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.992 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.992 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.992 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.112754+0000", 2023-10-09T05:15:38.992 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0092554349999999994 2023-10-09T05:15:38.992 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.992 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.992 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:15:38.992 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.112761+0000", 2023-10-09T05:15:38.992 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.0759999999999997e-06 2023-10-09T05:15:38.992 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.992 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.992 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:38.993 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.332533+0000", 2023-10-09T05:15:38.993 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.21977247499999999 2023-10-09T05:15:38.993 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.993 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.993 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:15:38.993 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.332659+0000", 2023-10-09T05:15:38.993 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00012614700000000001 2023-10-09T05:15:38.993 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.993 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.993 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:15:38.993 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.332929+0000", 2023-10-09T05:15:38.993 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000269201 2023-10-09T05:15:38.994 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.994 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.994 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:15:38.994 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.337198+0000", 2023-10-09T05:15:38.994 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0042695820000000001 2023-10-09T05:15:38.994 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.994 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.994 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:15:38.994 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.343149+0000", 2023-10-09T05:15:38.994 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.005950922 2023-10-09T05:15:38.994 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.994 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.995 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:15:38.995 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.344762+0000", 2023-10-09T05:15:38.995 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.001613223 2023-10-09T05:15:38.995 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.995 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.995 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:15:38.995 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.344831+0000", 2023-10-09T05:15:38.995 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.8285000000000001e-05 2023-10-09T05:15:38.995 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.995 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.995 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:15:38.995 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.345052+0000", 2023-10-09T05:15:38.996 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000221487 2023-10-09T05:15:38.996 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.996 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:15:38.996 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:38.996 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.996 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.996 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:158281 9.e 9:7711964e:::benchmark_data_smithi133_153303_object68312:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-10-09T05:15:38.996 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:12:33.133760+0000", 2023-10-09T05:15:38.996 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 185.70092063999999, 2023-10-09T05:15:38.996 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.21617122699999999, 2023-10-09T05:15:38.996 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:15:38.996 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:15:38.997 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:15:38.997 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:15:38.997 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:15:38.997 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 158281 2023-10-09T05:15:38.997 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.997 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:15:38.997 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.997 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:15:38.997 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.133760+0000", 2023-10-09T05:15:38.997 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.997 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.997 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.998 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:15:38.998 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.133760+0000", 2023-10-09T05:15:38.998 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:15:38.998 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.998 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.998 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:15:38.998 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.133759+0000", 2023-10-09T05:15:38.998 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:15:38.998 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.998 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.998 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:15:38.999 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.133767+0000", 2023-10-09T05:15:38.999 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.7229999999999993e-06 2023-10-09T05:15:38.999 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.999 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.999 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:15:38.999 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.133779+0000", 2023-10-09T05:15:38.999 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.2031999999999999e-05 2023-10-09T05:15:38.999 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:38.999 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:38.999 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:15:38.999 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.133785+0000", 2023-10-09T05:15:38.999 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.908e-06 2023-10-09T05:15:39.000 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:39.000 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:39.000 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:39.000 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.148302+0000", 2023-10-09T05:15:39.000 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.014516782000000001 2023-10-09T05:15:39.000 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:39.000 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:39.000 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:15:39.000 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.148311+0000", 2023-10-09T05:15:39.000 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.6980000000000004e-06 2023-10-09T05:15:39.000 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:39.000 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:39.001 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:15:39.001 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.333998+0000", 2023-10-09T05:15:39.001 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.18568744700000001 2023-10-09T05:15:39.001 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:39.001 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:39.001 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:15:39.001 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.334164+0000", 2023-10-09T05:15:39.001 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00016624200000000001 2023-10-09T05:15:39.001 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:39.001 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:39.001 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:15:39.001 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.334493+0000", 2023-10-09T05:15:39.002 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00032905599999999999 2023-10-09T05:15:39.002 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:39.002 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:39.002 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:15:39.002 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.337206+0000", 2023-10-09T05:15:39.002 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0027129739999999999 2023-10-09T05:15:39.002 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:39.002 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:39.002 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:15:39.002 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.343194+0000", 2023-10-09T05:15:39.002 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0059877120000000001 2023-10-09T05:15:39.002 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:39.003 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:39.003 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:15:39.003 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.349714+0000", 2023-10-09T05:15:39.003 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0065199500000000001 2023-10-09T05:15:39.003 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:39.003 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:39.003 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:15:39.003 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.349762+0000", 2023-10-09T05:15:39.003 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.8167000000000001e-05 2023-10-09T05:15:39.003 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:15:39.003 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:15:39.003 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:15:39.004 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.349931+0000", 2023-10-09T05:15:39.004 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00016903099999999999 2023-10-09T05:15:39.004 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:39.004 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:15:39.004 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:39.004 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:15:39.004 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:15:39.004 INFO:teuthology.orchestra.run.smithi194.stdout:} 2023-10-09T05:15:39.004 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:39.044 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:39.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:39.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:39.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:39 smithi194 bash[92666]: cluster 2023-10-09T05:15:37.907084+0000 mgr.y (mgr.25277) 813 : cluster 0 pgmap v824: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 39 MiB/s wr, 635 op/s; 23274/291849 objects degraded (7.975%); 8462/291849 objects misplaced (2.899%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:15:39.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:39 smithi194 bash[92666]: cluster 2023-10-09T05:15:38.328863+0000 mon.a (mon.0) 1063 : cluster 3 Health check update: Degraded data redundancy: 23274/291849 objects degraded (7.975%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:39.152 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:39.189 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:39.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:39.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:39.295 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:39.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:39 smithi133 bash[105589]: cluster 2023-10-09T05:15:37.907084+0000 mgr.y (mgr.25277) 813 : cluster 0 pgmap v824: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 39 MiB/s wr, 635 op/s; 23274/291849 objects degraded (7.975%); 8462/291849 objects misplaced (2.899%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:15:39.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:39 smithi133 bash[105589]: cluster 2023-10-09T05:15:38.328863+0000 mon.a (mon.0) 1063 : cluster 3 Health check update: Degraded data redundancy: 23274/291849 objects degraded (7.975%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:39.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:39 smithi133 bash[102858]: cluster 2023-10-09T05:15:37.907084+0000 mgr.y (mgr.25277) 813 : cluster 0 pgmap v824: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 39 MiB/s wr, 635 op/s; 23274/291849 objects degraded (7.975%); 8462/291849 objects misplaced (2.899%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:15:39.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:39 smithi133 bash[102858]: cluster 2023-10-09T05:15:38.328863+0000 mon.a (mon.0) 1063 : cluster 3 Health check update: Degraded data redundancy: 23274/291849 objects degraded (7.975%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:39.326 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:39.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:39.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:39.430 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:39.462 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:39.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:39.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:39.569 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:39.595 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:39.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:39.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:39.698 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:39.708 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:39 smithi133 bash[102858]: debug 2023-10-09T05:15:39.410+0000 7f5eb4d1d700 -1 mon.a@0(leader).osd e201 definitely_dead 0 2023-10-09T05:15:39.734 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:39.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:39.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:39.839 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:39.868 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:39.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:39.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:39.963 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T05:15:39.973 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:40.001 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:40.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:40.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:40.081 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:15:40.100 INFO:teuthology.orchestra.run.smithi133.stderr:marked out osd.7. 2023-10-09T05:15:40.107 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:40.118 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:40 smithi194 bash[92666]: audit 2023-10-09T05:15:39.415319+0000 mon.c (mon.2) 79 : audit 1 from='client.? 172.21.15.133:0/825969365' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T05:15:40.118 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:40 smithi194 bash[92666]: audit 2023-10-09T05:15:39.415878+0000 mon.a (mon.0) 1064 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T05:15:40.118 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:40 smithi194 bash[92666]: cluster 2023-10-09T05:15:39.415948+0000 mon.a (mon.0) 1065 : cluster 1 Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T05:15:40.131 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:40.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:40.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:40.234 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:40.245 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:40 smithi133 bash[105589]: audit 2023-10-09T05:15:39.415319+0000 mon.c (mon.2) 79 : audit 1 from='client.? 172.21.15.133:0/825969365' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T05:15:40.245 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:40 smithi133 bash[105589]: audit 2023-10-09T05:15:39.415878+0000 mon.a (mon.0) 1064 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T05:15:40.245 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:40 smithi133 bash[105589]: cluster 2023-10-09T05:15:39.415948+0000 mon.a (mon.0) 1065 : cluster 1 Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T05:15:40.245 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:40 smithi133 bash[102858]: audit 2023-10-09T05:15:39.415319+0000 mon.c (mon.2) 79 : audit 1 from='client.? 172.21.15.133:0/825969365' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T05:15:40.245 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:40 smithi133 bash[102858]: audit 2023-10-09T05:15:39.415878+0000 mon.a (mon.0) 1064 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T05:15:40.245 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:40 smithi133 bash[102858]: cluster 2023-10-09T05:15:39.415948+0000 mon.a (mon.0) 1065 : cluster 1 Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T05:15:40.264 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:40.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:40.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:40.369 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:40.403 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:40.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:40.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:40.410 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:15:40.506 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:40.531 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:40.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:40.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:40.634 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:40.678 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:40.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:40.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:40.783 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:40.818 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:40.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:40.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:40.921 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:40.957 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:40.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:40.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:41.060 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:41.086 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:41.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:41.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:41.190 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:41.220 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:41.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:41.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:41.324 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:41.332 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:41 smithi194 bash[92666]: cluster 2023-10-09T05:15:39.907875+0000 mgr.y (mgr.25277) 814 : cluster 0 pgmap v825: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 31 MiB/s wr, 606 op/s; 23274/288846 objects degraded (8.058%); 8133/288846 objects misplaced (2.816%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T05:15:41.332 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:41 smithi194 bash[92666]: cluster 2023-10-09T05:15:40.077072+0000 osd.4 (osd.4) 151 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T05:15:41.333 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:41 smithi194 bash[92666]: cluster 2023-10-09T05:15:40.079987+0000 osd.5 (osd.5) 188 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T05:15:41.333 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:41 smithi194 bash[92666]: audit 2023-10-09T05:15:40.080148+0000 mon.a (mon.0) 1066 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T05:15:41.333 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:41 smithi194 bash[92666]: cluster 2023-10-09T05:15:40.080353+0000 mon.a (mon.0) 1067 : cluster 0 osdmap e202: 8 total, 8 up, 6 in 2023-10-09T05:15:41.348 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:41.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:41.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:41.453 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:41.464 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:41 smithi133 bash[102858]: cluster 2023-10-09T05:15:39.907875+0000 mgr.y (mgr.25277) 814 : cluster 0 pgmap v825: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 31 MiB/s wr, 606 op/s; 23274/288846 objects degraded (8.058%); 8133/288846 objects misplaced (2.816%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T05:15:41.464 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:41 smithi133 bash[102858]: cluster 2023-10-09T05:15:40.077072+0000 osd.4 (osd.4) 151 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T05:15:41.464 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:41 smithi133 bash[102858]: cluster 2023-10-09T05:15:40.079987+0000 osd.5 (osd.5) 188 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T05:15:41.464 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:41 smithi133 bash[102858]: audit 2023-10-09T05:15:40.080148+0000 mon.a (mon.0) 1066 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T05:15:41.464 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:41 smithi133 bash[102858]: cluster 2023-10-09T05:15:40.080353+0000 mon.a (mon.0) 1067 : cluster 0 osdmap e202: 8 total, 8 up, 6 in 2023-10-09T05:15:41.465 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:41 smithi133 bash[105589]: cluster 2023-10-09T05:15:39.907875+0000 mgr.y (mgr.25277) 814 : cluster 0 pgmap v825: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 31 MiB/s wr, 606 op/s; 23274/288846 objects degraded (8.058%); 8133/288846 objects misplaced (2.816%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T05:15:41.465 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:41 smithi133 bash[105589]: cluster 2023-10-09T05:15:40.077072+0000 osd.4 (osd.4) 151 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T05:15:41.465 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:41 smithi133 bash[105589]: cluster 2023-10-09T05:15:40.079987+0000 osd.5 (osd.5) 188 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T05:15:41.465 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:41 smithi133 bash[105589]: audit 2023-10-09T05:15:40.080148+0000 mon.a (mon.0) 1066 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T05:15:41.465 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:41 smithi133 bash[105589]: cluster 2023-10-09T05:15:40.080353+0000 mon.a (mon.0) 1067 : cluster 0 osdmap e202: 8 total, 8 up, 6 in 2023-10-09T05:15:41.481 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:41.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:41.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:41.584 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:41.616 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:41.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:41.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:41.720 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:41.758 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:41.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:41.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:41.863 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:41.893 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:41.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:41.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:41.996 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:42.026 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:42.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:42.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:42.130 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:42.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:42 smithi194 bash[92666]: cluster 2023-10-09T05:15:41.089290+0000 mon.a (mon.0) 1068 : cluster 0 osdmap e203: 8 total, 8 up, 6 in 2023-10-09T05:15:42.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:42 smithi194 bash[92666]: cluster 2023-10-09T05:15:41.089435+0000 osd.5 (osd.5) 189 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T05:15:42.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:42 smithi194 bash[92666]: cluster 2023-10-09T05:15:41.091132+0000 osd.4 (osd.4) 152 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T05:15:42.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:42 smithi194 bash[92666]: cluster 2023-10-09T05:15:41.091233+0000 osd.6 (osd.6) 135 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T05:15:42.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:42 smithi194 bash[92666]: cluster 2023-10-09T05:15:41.094020+0000 osd.4 (osd.4) 153 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T05:15:42.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:42 smithi194 bash[92666]: cluster 2023-10-09T05:15:41.095250+0000 osd.6 (osd.6) 136 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T05:15:42.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:42 smithi194 bash[92666]: cluster 2023-10-09T05:15:41.096835+0000 osd.4 (osd.4) 154 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T05:15:42.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:42 smithi194 bash[92666]: cluster 2023-10-09T05:15:41.097486+0000 osd.6 (osd.6) 137 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T05:15:42.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:42 smithi194 bash[92666]: cluster 2023-10-09T05:15:41.098983+0000 osd.7 (osd.7) 137 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T05:15:42.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:42 smithi194 bash[92666]: cluster 2023-10-09T05:15:41.099135+0000 osd.6 (osd.6) 138 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T05:15:42.161 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:42.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:42.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:42.264 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:42.292 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:42.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:42.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:42.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:42 smithi133 bash[105589]: cluster 2023-10-09T05:15:41.089290+0000 mon.a (mon.0) 1068 : cluster 0 osdmap e203: 8 total, 8 up, 6 in 2023-10-09T05:15:42.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:42 smithi133 bash[105589]: cluster 2023-10-09T05:15:41.089435+0000 osd.5 (osd.5) 189 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T05:15:42.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:42 smithi133 bash[105589]: cluster 2023-10-09T05:15:41.091132+0000 osd.4 (osd.4) 152 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T05:15:42.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:42 smithi133 bash[105589]: cluster 2023-10-09T05:15:41.091233+0000 osd.6 (osd.6) 135 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T05:15:42.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:42 smithi133 bash[105589]: cluster 2023-10-09T05:15:41.094020+0000 osd.4 (osd.4) 153 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T05:15:42.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:42 smithi133 bash[105589]: cluster 2023-10-09T05:15:41.095250+0000 osd.6 (osd.6) 136 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T05:15:42.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:42 smithi133 bash[105589]: cluster 2023-10-09T05:15:41.096835+0000 osd.4 (osd.4) 154 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T05:15:42.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:42 smithi133 bash[105589]: cluster 2023-10-09T05:15:41.097486+0000 osd.6 (osd.6) 137 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T05:15:42.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:42 smithi133 bash[105589]: cluster 2023-10-09T05:15:41.098983+0000 osd.7 (osd.7) 137 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T05:15:42.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:42 smithi133 bash[105589]: cluster 2023-10-09T05:15:41.099135+0000 osd.6 (osd.6) 138 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T05:15:42.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:42 smithi133 bash[102858]: cluster 2023-10-09T05:15:41.089290+0000 mon.a (mon.0) 1068 : cluster 0 osdmap e203: 8 total, 8 up, 6 in 2023-10-09T05:15:42.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:42 smithi133 bash[102858]: cluster 2023-10-09T05:15:41.089435+0000 osd.5 (osd.5) 189 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T05:15:42.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:42 smithi133 bash[102858]: cluster 2023-10-09T05:15:41.091132+0000 osd.4 (osd.4) 152 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T05:15:42.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:42 smithi133 bash[102858]: cluster 2023-10-09T05:15:41.091233+0000 osd.6 (osd.6) 135 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T05:15:42.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:42 smithi133 bash[102858]: cluster 2023-10-09T05:15:41.094020+0000 osd.4 (osd.4) 153 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T05:15:42.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:42 smithi133 bash[102858]: cluster 2023-10-09T05:15:41.095250+0000 osd.6 (osd.6) 136 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T05:15:42.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:42 smithi133 bash[102858]: cluster 2023-10-09T05:15:41.096835+0000 osd.4 (osd.4) 154 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T05:15:42.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:42 smithi133 bash[102858]: cluster 2023-10-09T05:15:41.097486+0000 osd.6 (osd.6) 137 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T05:15:42.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:42 smithi133 bash[102858]: cluster 2023-10-09T05:15:41.098983+0000 osd.7 (osd.7) 137 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T05:15:42.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:42 smithi133 bash[102858]: cluster 2023-10-09T05:15:41.099135+0000 osd.6 (osd.6) 138 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T05:15:42.395 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:42.432 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:42.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:42.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:42.537 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:42.569 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:42.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:42.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:42.677 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:42.707 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:42.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:42.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:42.813 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:42.834 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:42.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:42.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:42.937 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:42.964 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:42.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:42.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:43.071 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:43.098 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:43 smithi194 bash[92666]: cluster 2023-10-09T05:15:41.908718+0000 mgr.y (mgr.25277) 815 : cluster 0 pgmap v828: 89 pgs: 1 peering, 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.8 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 127 B/s rd, 22 MiB/s wr, 704 op/s; 19119/283950 objects degraded (6.733%); 8133/283950 objects misplaced (2.864%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T05:15:43.098 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:43 smithi194 bash[92666]: cluster 2023-10-09T05:15:42.097997+0000 osd.5 (osd.5) 190 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T05:15:43.098 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:43 smithi194 bash[92666]: cluster 2023-10-09T05:15:42.102345+0000 osd.4 (osd.4) 155 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T05:15:43.098 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:43 smithi194 bash[92666]: cluster 2023-10-09T05:15:42.103377+0000 mon.a (mon.0) 1069 : cluster 0 osdmap e204: 8 total, 8 up, 6 in 2023-10-09T05:15:43.098 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:43 smithi194 bash[92666]: cluster 2023-10-09T05:15:42.104258+0000 osd.4 (osd.4) 156 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T05:15:43.099 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:43 smithi194 bash[92666]: cluster 2023-10-09T05:15:42.104774+0000 osd.6 (osd.6) 139 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T05:15:43.099 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:43 smithi194 bash[92666]: cluster 2023-10-09T05:15:42.104777+0000 osd.7 (osd.7) 138 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T05:15:43.099 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:43.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:43.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:43.202 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:43.240 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:43.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:43.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:43.344 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:43.377 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:43.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:43.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:43.484 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:43.512 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:43.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:43.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:43.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:43 smithi133 bash[105589]: cluster 2023-10-09T05:15:41.908718+0000 mgr.y (mgr.25277) 815 : cluster 0 pgmap v828: 89 pgs: 1 peering, 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.8 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 127 B/s rd, 22 MiB/s wr, 704 op/s; 19119/283950 objects degraded (6.733%); 8133/283950 objects misplaced (2.864%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T05:15:43.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:43 smithi133 bash[105589]: cluster 2023-10-09T05:15:42.097997+0000 osd.5 (osd.5) 190 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T05:15:43.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:43 smithi133 bash[105589]: cluster 2023-10-09T05:15:42.102345+0000 osd.4 (osd.4) 155 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T05:15:43.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:43 smithi133 bash[105589]: cluster 2023-10-09T05:15:42.103377+0000 mon.a (mon.0) 1069 : cluster 0 osdmap e204: 8 total, 8 up, 6 in 2023-10-09T05:15:43.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:43 smithi133 bash[105589]: cluster 2023-10-09T05:15:42.104258+0000 osd.4 (osd.4) 156 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T05:15:43.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:43 smithi133 bash[105589]: cluster 2023-10-09T05:15:42.104774+0000 osd.6 (osd.6) 139 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T05:15:43.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:43 smithi133 bash[105589]: cluster 2023-10-09T05:15:42.104777+0000 osd.7 (osd.7) 138 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T05:15:43.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:43 smithi133 bash[102858]: cluster 2023-10-09T05:15:41.908718+0000 mgr.y (mgr.25277) 815 : cluster 0 pgmap v828: 89 pgs: 1 peering, 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.8 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 127 B/s rd, 22 MiB/s wr, 704 op/s; 19119/283950 objects degraded (6.733%); 8133/283950 objects misplaced (2.864%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T05:15:43.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:43 smithi133 bash[102858]: cluster 2023-10-09T05:15:42.097997+0000 osd.5 (osd.5) 190 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T05:15:43.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:43 smithi133 bash[102858]: cluster 2023-10-09T05:15:42.102345+0000 osd.4 (osd.4) 155 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T05:15:43.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:43 smithi133 bash[102858]: cluster 2023-10-09T05:15:42.103377+0000 mon.a (mon.0) 1069 : cluster 0 osdmap e204: 8 total, 8 up, 6 in 2023-10-09T05:15:43.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:43 smithi133 bash[102858]: cluster 2023-10-09T05:15:42.104258+0000 osd.4 (osd.4) 156 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T05:15:43.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:43 smithi133 bash[102858]: cluster 2023-10-09T05:15:42.104774+0000 osd.6 (osd.6) 139 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T05:15:43.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:43 smithi133 bash[102858]: cluster 2023-10-09T05:15:42.104777+0000 osd.7 (osd.7) 138 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T05:15:43.617 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:43.647 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:43.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:43.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:43.751 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:43.777 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:43.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:43.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:43.883 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:43.908 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:43.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:43.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:44.011 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:44.042 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:44.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:44.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:44.145 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:44.153 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:44 smithi194 bash[92666]: cluster 2023-10-09T05:15:43.329940+0000 mon.a (mon.0) 1070 : cluster 3 Health check update: Degraded data redundancy: 19119/283950 objects degraded (6.733%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:44.154 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:44 smithi194 bash[92666]: audit 2023-10-09T05:15:43.456163+0000 mon.a (mon.0) 1071 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:15:44.154 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:44 smithi194 bash[92666]: audit 2023-10-09T05:15:43.457358+0000 mon.a (mon.0) 1072 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:15:44.171 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:44.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:44.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:44.275 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:44.300 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:44.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:44.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:44.405 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:44.414 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:44 smithi133 bash[105589]: cluster 2023-10-09T05:15:43.329940+0000 mon.a (mon.0) 1070 : cluster 3 Health check update: Degraded data redundancy: 19119/283950 objects degraded (6.733%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:44.414 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:44 smithi133 bash[105589]: audit 2023-10-09T05:15:43.456163+0000 mon.a (mon.0) 1071 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:15:44.414 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:44 smithi133 bash[105589]: audit 2023-10-09T05:15:43.457358+0000 mon.a (mon.0) 1072 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:15:44.414 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:15:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:15:44] "GET /metrics HTTP/1.1" 200 36505 "" "Prometheus/2.43.0" 2023-10-09T05:15:44.415 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:44 smithi133 bash[102858]: cluster 2023-10-09T05:15:43.329940+0000 mon.a (mon.0) 1070 : cluster 3 Health check update: Degraded data redundancy: 19119/283950 objects degraded (6.733%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:44.415 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:44 smithi133 bash[102858]: audit 2023-10-09T05:15:43.456163+0000 mon.a (mon.0) 1071 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:15:44.415 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:44 smithi133 bash[102858]: audit 2023-10-09T05:15:43.457358+0000 mon.a (mon.0) 1072 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:15:44.429 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:44.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:44.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:44.535 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:44.569 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:44.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:44.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:44.672 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:44.702 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:44.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:44.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:44.804 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:44.822 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:15:44.823 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:15:44.823 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:15:44.840 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:44.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:44.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:44.945 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:44.973 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:44.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:44.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:45.080 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:45.110 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:45.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:45.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:45.217 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:45.254 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:45.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:45.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:45.360 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:45.370 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:45 smithi133 bash[102858]: cluster 2023-10-09T05:15:43.909368+0000 mgr.y (mgr.25277) 816 : cluster 0 pgmap v830: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.8 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 170 B/s rd, 1.1 MiB/s wr, 493 op/s; 22435/283584 objects degraded (7.911%); 4713/283584 objects misplaced (1.662%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T05:15:45.370 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:45 smithi133 bash[105589]: cluster 2023-10-09T05:15:43.909368+0000 mgr.y (mgr.25277) 816 : cluster 0 pgmap v830: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.8 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 170 B/s rd, 1.1 MiB/s wr, 493 op/s; 22435/283584 objects degraded (7.911%); 4713/283584 objects misplaced (1.662%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T05:15:45.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:45 smithi194 bash[92666]: cluster 2023-10-09T05:15:43.909368+0000 mgr.y (mgr.25277) 816 : cluster 0 pgmap v830: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.8 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 170 B/s rd, 1.1 MiB/s wr, 493 op/s; 22435/283584 objects degraded (7.911%); 4713/283584 objects misplaced (1.662%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T05:15:45.388 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:45.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:45.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:45.493 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:45.517 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:45.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:45.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:45.621 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:45.646 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:45.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:45.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:45.753 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:45.782 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:45.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:45.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:45.885 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:45.909 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:45.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:45.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:46.015 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:46.043 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:46.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:46.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:46.149 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:46.182 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:46.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:46.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:46.286 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:46.316 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:46.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:46.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:46.421 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:46.455 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:46.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:46.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:46.474 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 6, 1] out_osds: [5, 7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:15:46.474 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:15:46.475 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_5 2023-10-09T05:15:46.475 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:15:46.560 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:46.591 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:46.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:46.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:46.696 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:46.727 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:46.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:46.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:46.833 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:46.868 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:46.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:46.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:46.972 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:47.014 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:47.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:47.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:47.120 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:47.156 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:47.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:47.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:47.262 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:47.287 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:47.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:47.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:47.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:47 smithi194 bash[92666]: cluster 2023-10-09T05:15:45.910186+0000 mgr.y (mgr.25277) 817 : cluster 0 pgmap v831: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 170 B/s wr, 591 op/s; 11543/278193 objects degraded (4.149%); 3275/278193 objects misplaced (1.177%); 3.2 MiB/s, 1 keys/s, 53 objects/s recovering 2023-10-09T05:15:47.395 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:47.405 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:47 smithi133 bash[102858]: cluster 2023-10-09T05:15:45.910186+0000 mgr.y (mgr.25277) 817 : cluster 0 pgmap v831: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 170 B/s wr, 591 op/s; 11543/278193 objects degraded (4.149%); 3275/278193 objects misplaced (1.177%); 3.2 MiB/s, 1 keys/s, 53 objects/s recovering 2023-10-09T05:15:47.406 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:47 smithi133 bash[105589]: cluster 2023-10-09T05:15:45.910186+0000 mgr.y (mgr.25277) 817 : cluster 0 pgmap v831: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 170 B/s wr, 591 op/s; 11543/278193 objects degraded (4.149%); 3275/278193 objects misplaced (1.177%); 3.2 MiB/s, 1 keys/s, 53 objects/s recovering 2023-10-09T05:15:47.421 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:47.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:47.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:47.527 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:47.551 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:47.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:47.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:47.656 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:47.689 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:47.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:47.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:47.793 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:47.817 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:47.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:47.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:47.921 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:47.946 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:47.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:47.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:48.051 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:48.083 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:48.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:48.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:48.188 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:48.209 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:48.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:48.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:48.313 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:48.336 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:48.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:48.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:48.441 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:48.485 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:48.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:48.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:48.589 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:48.619 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:48.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:48.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:48.726 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:48.751 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:48.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:48.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:48.856 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:48.890 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:48.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:48.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:48.999 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:49.035 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:49.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:49.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:49.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:49 smithi194 bash[92666]: cluster 2023-10-09T05:15:47.910611+0000 mgr.y (mgr.25277) 818 : cluster 0 pgmap v832: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 357 op/s; 11543/277098 objects degraded (4.166%); 3275/277098 objects misplaced (1.182%); 2.5 MiB/s, 1 keys/s, 41 objects/s recovering 2023-10-09T05:15:49.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:49 smithi194 bash[92666]: cluster 2023-10-09T05:15:48.331117+0000 mon.a (mon.0) 1073 : cluster 3 Health check update: Degraded data redundancy: 11543/277098 objects degraded (4.166%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:49.140 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:49.174 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:49.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:49.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:49.263 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:49 smithi133 bash[102858]: cluster 2023-10-09T05:15:47.910611+0000 mgr.y (mgr.25277) 818 : cluster 0 pgmap v832: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 357 op/s; 11543/277098 objects degraded (4.166%); 3275/277098 objects misplaced (1.182%); 2.5 MiB/s, 1 keys/s, 41 objects/s recovering 2023-10-09T05:15:49.263 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:49 smithi133 bash[102858]: cluster 2023-10-09T05:15:48.331117+0000 mon.a (mon.0) 1073 : cluster 3 Health check update: Degraded data redundancy: 11543/277098 objects degraded (4.166%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:49.263 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:49 smithi133 bash[105589]: cluster 2023-10-09T05:15:47.910611+0000 mgr.y (mgr.25277) 818 : cluster 0 pgmap v832: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 357 op/s; 11543/277098 objects degraded (4.166%); 3275/277098 objects misplaced (1.182%); 2.5 MiB/s, 1 keys/s, 41 objects/s recovering 2023-10-09T05:15:49.263 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:49 smithi133 bash[105589]: cluster 2023-10-09T05:15:48.331117+0000 mon.a (mon.0) 1073 : cluster 3 Health check update: Degraded data redundancy: 11543/277098 objects degraded (4.166%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:49.280 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:49.307 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:49.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:49.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:49.412 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:49.438 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:49.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:49.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:49.546 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:49.579 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:49.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:49.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:49.685 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:49.711 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:49.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:49.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:49.817 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:49.866 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:49.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:49.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:49.972 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:49.993 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:49.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:49.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:50.096 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:50.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:50.127 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:50.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:50.228 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:50.254 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:50.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:50.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:50.360 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:50.388 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:50.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:50.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:50.493 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:50.530 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:50.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:50.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:50.635 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:50.661 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:50.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:50.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:50.768 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:50.786 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:50.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:50.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:50.889 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:50.925 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:50.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:50.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:51.030 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:51.056 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:51.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:51.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:51.133 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:51 smithi133 bash[102858]: cluster 2023-10-09T05:15:49.911330+0000 mgr.y (mgr.25277) 819 : cluster 0 pgmap v833: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 484 op/s; 11543/272031 objects degraded (4.243%); 3275/272031 objects misplaced (1.204%); 2.2 MiB/s, 1 keys/s, 36 objects/s recovering 2023-10-09T05:15:51.134 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:51 smithi133 bash[105589]: cluster 2023-10-09T05:15:49.911330+0000 mgr.y (mgr.25277) 819 : cluster 0 pgmap v833: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 484 op/s; 11543/272031 objects degraded (4.243%); 3275/272031 objects misplaced (1.204%); 2.2 MiB/s, 1 keys/s, 36 objects/s recovering 2023-10-09T05:15:51.162 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:51.202 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:51.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:51.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:51.308 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:51.339 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:51.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:51.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:51.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:51 smithi194 bash[92666]: cluster 2023-10-09T05:15:49.911330+0000 mgr.y (mgr.25277) 819 : cluster 0 pgmap v833: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 484 op/s; 11543/272031 objects degraded (4.243%); 3275/272031 objects misplaced (1.204%); 2.2 MiB/s, 1 keys/s, 36 objects/s recovering 2023-10-09T05:15:51.446 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:51.475 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:51.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:51.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:51.578 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:51.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:51.610 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:51.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:51.711 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:51.747 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:51.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:51.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:51.853 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:51.879 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:51.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:51.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:51.982 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:52.004 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:52.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:52.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:52.108 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:52.133 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:52.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:52.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:52.236 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:52.265 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:52.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:52.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:52.371 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:52.401 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:52.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:52.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:52.504 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:52.533 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:52.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:52.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:52.640 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:52.667 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:52.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:52.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:52.772 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:52.811 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:52.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:52.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:52.918 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:52.945 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:52.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:52.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:53.048 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:53.073 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:53.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:53.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:53.177 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:53.186 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:53 smithi194 bash[92666]: cluster 2023-10-09T05:15:51.912008+0000 mgr.y (mgr.25277) 820 : cluster 0 pgmap v834: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 615 op/s; 11543/265467 objects degraded (4.348%); 3069/265467 objects misplaced (1.156%); 3.2 MiB/s, 0 keys/s, 52 objects/s recovering 2023-10-09T05:15:53.205 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:53.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:53.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:53.311 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:53.335 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:53.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:53.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:53.440 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:53.450 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:53 smithi133 bash[105589]: cluster 2023-10-09T05:15:51.912008+0000 mgr.y (mgr.25277) 820 : cluster 0 pgmap v834: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 615 op/s; 11543/265467 objects degraded (4.348%); 3069/265467 objects misplaced (1.156%); 3.2 MiB/s, 0 keys/s, 52 objects/s recovering 2023-10-09T05:15:53.450 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:53 smithi133 bash[102858]: cluster 2023-10-09T05:15:51.912008+0000 mgr.y (mgr.25277) 820 : cluster 0 pgmap v834: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 615 op/s; 11543/265467 objects degraded (4.348%); 3069/265467 objects misplaced (1.156%); 3.2 MiB/s, 0 keys/s, 52 objects/s recovering 2023-10-09T05:15:53.470 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:53.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:53.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:53.574 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:53.602 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:53.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:53.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:53.708 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:53.739 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:53.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:53.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:53.845 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:53.906 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:53.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:53.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:54.013 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:54.038 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:54.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:54.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:54.144 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:54.172 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:54.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:54.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:54.278 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:54.301 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:54.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:54.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:54.316 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:15:54.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:54 smithi194 bash[92666]: cluster 2023-10-09T05:15:53.332283+0000 mon.a (mon.0) 1074 : cluster 3 Health check update: Degraded data redundancy: 11543/265467 objects degraded (4.348%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:54.405 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:54.431 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:54.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:54.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:54.534 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:54.543 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:54 smithi133 bash[102858]: cluster 2023-10-09T05:15:53.332283+0000 mon.a (mon.0) 1074 : cluster 3 Health check update: Degraded data redundancy: 11543/265467 objects degraded (4.348%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:54.543 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:15:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:15:54] "GET /metrics HTTP/1.1" 200 36505 "" "Prometheus/2.43.0" 2023-10-09T05:15:54.543 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:54 smithi133 bash[105589]: cluster 2023-10-09T05:15:53.332283+0000 mon.a (mon.0) 1074 : cluster 3 Health check update: Degraded data redundancy: 11543/265467 objects degraded (4.348%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:15:54.568 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:54.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:54.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:54.676 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:54.699 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:54.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:54.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:54.805 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:54.829 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:54.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:54.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:54.935 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:54.959 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:54.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:54.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:55.062 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:55.091 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:55.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:55.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:55.195 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:55.224 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:55.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:55.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:55.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:55 smithi133 bash[105589]: cluster 2023-10-09T05:15:53.913678+0000 mgr.y (mgr.25277) 821 : cluster 0 pgmap v835: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 539 op/s; 11543/264810 objects degraded (4.359%); 3069/264810 objects misplaced (1.159%); 2.7 MiB/s, 0 keys/s, 44 objects/s recovering 2023-10-09T05:15:55.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:55 smithi133 bash[102858]: cluster 2023-10-09T05:15:53.913678+0000 mgr.y (mgr.25277) 821 : cluster 0 pgmap v835: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 539 op/s; 11543/264810 objects degraded (4.359%); 3069/264810 objects misplaced (1.159%); 2.7 MiB/s, 0 keys/s, 44 objects/s recovering 2023-10-09T05:15:55.331 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:55.360 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:55.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:55.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:55.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:55 smithi194 bash[92666]: cluster 2023-10-09T05:15:53.913678+0000 mgr.y (mgr.25277) 821 : cluster 0 pgmap v835: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 539 op/s; 11543/264810 objects degraded (4.359%); 3069/264810 objects misplaced (1.159%); 2.7 MiB/s, 0 keys/s, 44 objects/s recovering 2023-10-09T05:15:55.466 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:55.500 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:55.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:55.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:55.605 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:55.629 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:55.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:55.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:55.734 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:55.762 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:55.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:55.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:55.867 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:55.893 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:55.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:55.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:55.998 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:56.053 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:56.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:56.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:56.161 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:56.194 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:56.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:56.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:56.297 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:56.325 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:56.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:56.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:56.429 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:56.466 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:56.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:56.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:56.570 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:56.606 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:56.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:56.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:56.710 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:56.741 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:56.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:56.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:56.847 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:56.871 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:56.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:56.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:56.975 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:57.006 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:15:57.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:57.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:15:57.112 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:57.142 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:57.144 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is set 2023-10-09T05:15:57.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:57.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:57.250 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:57.275 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:57.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:57.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:57.380 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:57.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:57 smithi194 bash[92666]: cluster 2023-10-09T05:15:55.914495+0000 mgr.y (mgr.25277) 822 : cluster 0 pgmap v836: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 777 op/s; 11543/255588 objects degraded (4.516%); 2831/255588 objects misplaced (1.108%); 3.4 MiB/s, 0 keys/s, 54 objects/s recovering 2023-10-09T05:15:57.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:57 smithi194 bash[92666]: audit 2023-10-09T05:15:56.852384+0000 mon.a (mon.0) 1075 : audit 1 from='client.? 172.21.15.133:0/2936289007' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:15:57.406 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:57.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:57.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:57.509 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:57.525 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:57 smithi133 bash[102858]: cluster 2023-10-09T05:15:55.914495+0000 mgr.y (mgr.25277) 822 : cluster 0 pgmap v836: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 777 op/s; 11543/255588 objects degraded (4.516%); 2831/255588 objects misplaced (1.108%); 3.4 MiB/s, 0 keys/s, 54 objects/s recovering 2023-10-09T05:15:57.525 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:57 smithi133 bash[102858]: audit 2023-10-09T05:15:56.852384+0000 mon.a (mon.0) 1075 : audit 1 from='client.? 172.21.15.133:0/2936289007' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:15:57.525 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:57 smithi133 bash[105589]: cluster 2023-10-09T05:15:55.914495+0000 mgr.y (mgr.25277) 822 : cluster 0 pgmap v836: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 777 op/s; 11543/255588 objects degraded (4.516%); 2831/255588 objects misplaced (1.108%); 3.4 MiB/s, 0 keys/s, 54 objects/s recovering 2023-10-09T05:15:57.525 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:57 smithi133 bash[105589]: audit 2023-10-09T05:15:56.852384+0000 mon.a (mon.0) 1075 : audit 1 from='client.? 172.21.15.133:0/2936289007' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:15:57.539 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:57.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:57.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:57.644 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:57.670 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:57.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:57.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:57.775 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:57.808 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:57.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:57.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:57.913 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:57.937 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:57.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:57.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:58.041 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:58.068 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:58.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:58.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:58.174 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:58.204 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:58.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:58.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:58.311 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:58.345 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:58.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:58.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:58.448 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:58.456 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:58 smithi194 bash[92666]: cluster 2023-10-09T05:15:57.129626+0000 mon.a (mon.0) 1076 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:15:58.456 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:58 smithi194 bash[92666]: cluster 2023-10-09T05:15:57.135504+0000 osd.5 (osd.5) 191 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T05:15:58.456 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:58 smithi194 bash[92666]: audit 2023-10-09T05:15:57.136632+0000 mon.a (mon.0) 1077 : audit 1 from='client.? 172.21.15.133:0/2936289007' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:15:58.456 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:58 smithi194 bash[92666]: cluster 2023-10-09T05:15:57.136778+0000 mon.a (mon.0) 1078 : cluster 0 osdmap e205: 8 total, 8 up, 6 in 2023-10-09T05:15:58.457 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:58 smithi194 bash[92666]: cluster 2023-10-09T05:15:57.154632+0000 osd.6 (osd.6) 140 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T05:15:58.457 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:58 smithi194 bash[92666]: cluster 2023-10-09T05:15:57.156087+0000 osd.6 (osd.6) 141 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T05:15:58.457 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:58 smithi194 bash[92666]: cluster 2023-10-09T05:15:57.188006+0000 osd.7 (osd.7) 139 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T05:15:58.475 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:58.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:58.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:58.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:58 smithi133 bash[105589]: cluster 2023-10-09T05:15:57.129626+0000 mon.a (mon.0) 1076 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:15:58.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:58 smithi133 bash[105589]: cluster 2023-10-09T05:15:57.135504+0000 osd.5 (osd.5) 191 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T05:15:58.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:58 smithi133 bash[105589]: audit 2023-10-09T05:15:57.136632+0000 mon.a (mon.0) 1077 : audit 1 from='client.? 172.21.15.133:0/2936289007' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:15:58.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:58 smithi133 bash[105589]: cluster 2023-10-09T05:15:57.136778+0000 mon.a (mon.0) 1078 : cluster 0 osdmap e205: 8 total, 8 up, 6 in 2023-10-09T05:15:58.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:58 smithi133 bash[105589]: cluster 2023-10-09T05:15:57.154632+0000 osd.6 (osd.6) 140 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T05:15:58.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:58 smithi133 bash[105589]: cluster 2023-10-09T05:15:57.156087+0000 osd.6 (osd.6) 141 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T05:15:58.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:58 smithi133 bash[105589]: cluster 2023-10-09T05:15:57.188006+0000 osd.7 (osd.7) 139 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T05:15:58.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:58 smithi133 bash[102858]: cluster 2023-10-09T05:15:57.129626+0000 mon.a (mon.0) 1076 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:15:58.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:58 smithi133 bash[102858]: cluster 2023-10-09T05:15:57.135504+0000 osd.5 (osd.5) 191 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T05:15:58.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:58 smithi133 bash[102858]: audit 2023-10-09T05:15:57.136632+0000 mon.a (mon.0) 1077 : audit 1 from='client.? 172.21.15.133:0/2936289007' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:15:58.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:58 smithi133 bash[102858]: cluster 2023-10-09T05:15:57.136778+0000 mon.a (mon.0) 1078 : cluster 0 osdmap e205: 8 total, 8 up, 6 in 2023-10-09T05:15:58.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:58 smithi133 bash[102858]: cluster 2023-10-09T05:15:57.154632+0000 osd.6 (osd.6) 140 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T05:15:58.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:58 smithi133 bash[102858]: cluster 2023-10-09T05:15:57.156087+0000 osd.6 (osd.6) 141 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T05:15:58.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:58 smithi133 bash[102858]: cluster 2023-10-09T05:15:57.188006+0000 osd.7 (osd.7) 139 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T05:15:58.580 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:58.605 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:58.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:58.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:58.708 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:58.735 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:15:58.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:58.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:15:58.840 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:58.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:58.870 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:58.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:58.971 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:59.008 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:15:59.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:59.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:15:59.111 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:59.141 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:15:59.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:59.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:15:59.154 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:59 smithi133 bash[102858]: cluster 2023-10-09T05:15:57.156652+0000 osd.4 (osd.4) 157 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T05:15:59.245 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:59.255 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:59 smithi133 bash[102858]: cluster 2023-10-09T05:15:57.159328+0000 osd.4 (osd.4) 158 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T05:15:59.256 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:59 smithi133 bash[102858]: cluster 2023-10-09T05:15:57.915055+0000 mgr.y (mgr.25277) 823 : cluster 0 pgmap v838: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 767 op/s; 11543/254067 objects degraded (4.543%); 2831/254067 objects misplaced (1.114%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T05:15:59.256 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:59 smithi133 bash[102858]: cluster 2023-10-09T05:15:58.333855+0000 mon.a (mon.0) 1079 : cluster 3 Health check update: Degraded data redundancy: 11543/254067 objects degraded (4.543%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:15:59.256 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:59 smithi133 bash[102858]: audit 2023-10-09T05:15:58.453886+0000 mon.a (mon.0) 1080 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:15:59.256 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:15:59 smithi133 bash[102858]: audit 2023-10-09T05:15:58.455047+0000 mon.a (mon.0) 1081 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:15:59.256 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:59 smithi133 bash[105589]: cluster 2023-10-09T05:15:57.156652+0000 osd.4 (osd.4) 157 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T05:15:59.256 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:59 smithi133 bash[105589]: cluster 2023-10-09T05:15:57.159328+0000 osd.4 (osd.4) 158 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T05:15:59.256 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:59 smithi133 bash[105589]: cluster 2023-10-09T05:15:57.915055+0000 mgr.y (mgr.25277) 823 : cluster 0 pgmap v838: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 767 op/s; 11543/254067 objects degraded (4.543%); 2831/254067 objects misplaced (1.114%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T05:15:59.256 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:59 smithi133 bash[105589]: cluster 2023-10-09T05:15:58.333855+0000 mon.a (mon.0) 1079 : cluster 3 Health check update: Degraded data redundancy: 11543/254067 objects degraded (4.543%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:15:59.256 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:59 smithi133 bash[105589]: audit 2023-10-09T05:15:58.453886+0000 mon.a (mon.0) 1080 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:15:59.257 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:15:59 smithi133 bash[105589]: audit 2023-10-09T05:15:58.455047+0000 mon.a (mon.0) 1081 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:15:59.268 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:59.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:59.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:15:59.373 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:59.380 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:59 smithi194 bash[92666]: cluster 2023-10-09T05:15:57.156652+0000 osd.4 (osd.4) 157 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T05:15:59.380 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:59 smithi194 bash[92666]: cluster 2023-10-09T05:15:57.159328+0000 osd.4 (osd.4) 158 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T05:15:59.381 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:59 smithi194 bash[92666]: cluster 2023-10-09T05:15:57.915055+0000 mgr.y (mgr.25277) 823 : cluster 0 pgmap v838: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 767 op/s; 11543/254067 objects degraded (4.543%); 2831/254067 objects misplaced (1.114%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T05:15:59.381 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:59 smithi194 bash[92666]: cluster 2023-10-09T05:15:58.333855+0000 mon.a (mon.0) 1079 : cluster 3 Health check update: Degraded data redundancy: 11543/254067 objects degraded (4.543%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:15:59.381 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:59 smithi194 bash[92666]: audit 2023-10-09T05:15:58.453886+0000 mon.a (mon.0) 1080 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:15:59.381 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:15:59 smithi194 bash[92666]: audit 2023-10-09T05:15:58.455047+0000 mon.a (mon.0) 1081 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:15:59.400 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:59.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:59.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:59.505 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:59.538 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:15:59.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:59.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:15:59.642 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:59.671 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:15:59.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:59.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:15:59.776 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:59.803 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:15:59.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:59.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:15:59.908 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:59.936 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:15:59.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:15:59.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:00.038 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:00.066 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:00.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:00.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:00.170 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:00.196 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:00.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:00.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:00.299 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:00.332 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:00.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:00.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:00.436 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:00.465 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:00.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:00.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:00.570 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:00.597 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:00.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:00.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:00.701 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:00.734 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:00.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:00.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:00.839 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:00.869 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:00.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:00.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:00.901 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:16:00.901 INFO:teuthology.orchestra.run.smithi133.stdout: "ops": [ 2023-10-09T05:16:00.901 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.901 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_repop(client.35022.0:114584 10.1 e205/202 10:833a5d9f:::benchmark_data_smithi133_173023_object17148:head v 205'7175, mlcod=195'5261)", 2023-10-09T05:16:00.902 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:16:00.893408+0000", 2023-10-09T05:16:00.902 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.0061868009999999996, 2023-10-09T05:16:00.902 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0062089750000000003, 2023-10-09T05:16:00.902 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:16:00.902 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "started", 2023-10-09T05:16:00.902 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:16:00.902 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.902 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:16:00.902 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.893408+0000", 2023-10-09T05:16:00.902 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:00.902 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.903 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.903 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:16:00.903 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.893408+0000", 2023-10-09T05:16:00.903 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:00.903 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.903 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.903 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:16:00.903 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.893407+0000", 2023-10-09T05:16:00.903 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:16:00.903 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.903 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.903 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:16:00.904 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.893418+0000", 2023-10-09T05:16:00.904 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.0645999999999999e-05 2023-10-09T05:16:00.904 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.904 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.904 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:16:00.904 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.893429+0000", 2023-10-09T05:16:00.904 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.1161999999999999e-05 2023-10-09T05:16:00.904 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.904 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.904 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:16:00.904 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.893436+0000", 2023-10-09T05:16:00.905 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.0879999999999997e-06 2023-10-09T05:16:00.905 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.905 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.905 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:00.905 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.895784+0000", 2023-10-09T05:16:00.905 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.002347338 2023-10-09T05:16:00.905 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.905 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.905 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:16:00.905 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.895810+0000", 2023-10-09T05:16:00.905 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.6815e-05 2023-10-09T05:16:00.905 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:00.905 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:16:00.906 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:00.906 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.906 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.906 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_repop(client.35022.0:114594 10.2 e205/179 10:47a8410d:::benchmark_data_smithi133_173023_object17158:head v 205'7291, mlcod=205'7289)", 2023-10-09T05:16:00.906 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:16:00.897819+0000", 2023-10-09T05:16:00.906 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.0017755239999999999, 2023-10-09T05:16:00.906 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.001839264, 2023-10-09T05:16:00.906 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:16:00.906 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "started", 2023-10-09T05:16:00.906 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:16:00.906 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.906 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:16:00.907 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.897819+0000", 2023-10-09T05:16:00.907 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:00.907 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.907 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.907 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:16:00.907 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.897819+0000", 2023-10-09T05:16:00.907 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:00.907 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.907 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.907 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:16:00.907 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.897818+0000", 2023-10-09T05:16:00.907 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:16:00.908 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.908 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.908 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:16:00.908 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.897826+0000", 2023-10-09T05:16:00.908 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.2479999999999996e-06 2023-10-09T05:16:00.908 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.908 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.908 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:16:00.908 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.897843+0000", 2023-10-09T05:16:00.908 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.6779999999999999e-05 2023-10-09T05:16:00.908 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.908 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.908 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:16:00.909 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.897851+0000", 2023-10-09T05:16:00.909 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.7500000000000003e-06 2023-10-09T05:16:00.909 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.909 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.909 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:00.909 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.898172+0000", 2023-10-09T05:16:00.909 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00032128799999999998 2023-10-09T05:16:00.909 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.909 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.909 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:16:00.909 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.898194+0000", 2023-10-09T05:16:00.909 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.1988999999999999e-05 2023-10-09T05:16:00.910 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:00.910 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:16:00.910 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:00.910 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.910 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.910 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35022.0:114585 10.4 10:2e1aed41:::benchmark_data_smithi133_173023_object17149:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e205)", 2023-10-09T05:16:00.910 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:16:00.887977+0000", 2023-10-09T05:16:00.910 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.011618253, 2023-10-09T05:16:00.910 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.011720516, 2023-10-09T05:16:00.910 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:16:00.910 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:16:00.910 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:16:00.911 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35022", 2023-10-09T05:16:00.911 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2582323369", 2023-10-09T05:16:00.911 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 114585 2023-10-09T05:16:00.911 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.911 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:16:00.911 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.911 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:16:00.911 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.887977+0000", 2023-10-09T05:16:00.911 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:00.911 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.911 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.911 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:16:00.912 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.887977+0000", 2023-10-09T05:16:00.912 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:00.912 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.912 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.912 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:16:00.912 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.887976+0000", 2023-10-09T05:16:00.912 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:16:00.912 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.912 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.912 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:16:00.912 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.887979+0000", 2023-10-09T05:16:00.912 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.6270000000000002e-06 2023-10-09T05:16:00.912 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.913 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.913 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:16:00.913 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.887988+0000", 2023-10-09T05:16:00.913 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.0499999999999997e-06 2023-10-09T05:16:00.913 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.913 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.913 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:16:00.913 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.887994+0000", 2023-10-09T05:16:00.913 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.5600000000000001e-06 2023-10-09T05:16:00.913 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.913 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.913 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:00.914 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.888038+0000", 2023-10-09T05:16:00.914 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.4230999999999997e-05 2023-10-09T05:16:00.914 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.914 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.914 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:16:00.914 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.888147+0000", 2023-10-09T05:16:00.914 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000108658 2023-10-09T05:16:00.914 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.914 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.914 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:16:00.914 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.888222+0000", 2023-10-09T05:16:00.914 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.5133e-05 2023-10-09T05:16:00.915 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.915 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.915 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:16:00.915 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.899289+0000", 2023-10-09T05:16:00.915 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.011067507000000001 2023-10-09T05:16:00.915 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.915 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.915 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:16:00.915 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.899350+0000", 2023-10-09T05:16:00.915 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.0355000000000001e-05 2023-10-09T05:16:00.915 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:00.915 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:16:00.916 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:00.916 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.916 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.916 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_repop(client.35022.0:114591 10.a e205/179 10:5bce4949:::benchmark_data_smithi133_173023_object17155:head v 205'7276, mlcod=205'7274)", 2023-10-09T05:16:00.916 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:16:00.889152+0000", 2023-10-09T05:16:00.916 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.010442912, 2023-10-09T05:16:00.916 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.010595284, 2023-10-09T05:16:00.916 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:16:00.916 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "started", 2023-10-09T05:16:00.916 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:16:00.916 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.916 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:16:00.916 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.889152+0000", 2023-10-09T05:16:00.917 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:00.917 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.917 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.917 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:16:00.917 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.889152+0000", 2023-10-09T05:16:00.917 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:00.917 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.917 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.917 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:16:00.918 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.889151+0000", 2023-10-09T05:16:00.918 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999995 2023-10-09T05:16:00.918 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.918 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.918 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:16:00.918 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.889159+0000", 2023-10-09T05:16:00.918 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.3799999999999996e-06 2023-10-09T05:16:00.918 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.918 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.918 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:16:00.918 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.889166+0000", 2023-10-09T05:16:00.919 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.5719999999999997e-06 2023-10-09T05:16:00.919 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.919 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.919 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:16:00.919 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.889172+0000", 2023-10-09T05:16:00.919 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.1789999999999999e-06 2023-10-09T05:16:00.919 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.919 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.919 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:00.919 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.889226+0000", 2023-10-09T05:16:00.919 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.4836999999999997e-05 2023-10-09T05:16:00.919 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:00.919 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:00.920 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:16:00.920 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:00.889244+0000", 2023-10-09T05:16:00.920 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.7943000000000001e-05 2023-10-09T05:16:00.920 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:00.920 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:16:00.920 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:00.920 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:00.920 INFO:teuthology.orchestra.run.smithi133.stdout: ], 2023-10-09T05:16:00.920 INFO:teuthology.orchestra.run.smithi133.stdout: "num_ops": 4 2023-10-09T05:16:00.920 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:16:00.977 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:01.003 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:01.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:01.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:01.108 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:01.133 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:01.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:01.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:01.237 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:01.245 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:01 smithi133 bash[102858]: cluster 2023-10-09T05:15:59.915818+0000 mgr.y (mgr.25277) 824 : cluster 0 pgmap v839: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 761 op/s; 11543/249168 objects degraded (4.633%); 2831/249168 objects misplaced (1.136%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T05:16:01.245 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:01 smithi133 bash[105589]: cluster 2023-10-09T05:15:59.915818+0000 mgr.y (mgr.25277) 824 : cluster 0 pgmap v839: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 761 op/s; 11543/249168 objects degraded (4.633%); 2831/249168 objects misplaced (1.136%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T05:16:01.270 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:01.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:01.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:01.304 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:16:01.349 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:16:01.405 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:01.412 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:16:01.440 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:01.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:01.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:01.467 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:16:01.503 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:16:01.536 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:16:01.547 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:01.556 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:01 smithi194 bash[92666]: cluster 2023-10-09T05:15:59.915818+0000 mgr.y (mgr.25277) 824 : cluster 0 pgmap v839: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 761 op/s; 11543/249168 objects degraded (4.633%); 2831/249168 objects misplaced (1.136%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T05:16:01.569 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:16:01.574 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:01.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:01.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:01.603 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:16:01.605 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:16:01.605 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:16:01.605 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:16:01.605 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:16:01.605 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:16:01.605 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:16:01.605 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:16:01.605 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:16:01.680 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:01.717 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:01.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:01.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:01.822 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:01.851 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:01.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:01.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:01.955 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:01.981 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:01.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:01.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:02.084 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:02.112 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:02.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:02.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:02.159 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:02 smithi133 bash[102858]: audit 2023-10-09T05:16:01.234551+0000 mon.a (mon.0) 1082 : audit 0 from='client.? 172.21.15.133:0/2131792615' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:16:02.217 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:02.246 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:02.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:02.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:02.352 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:02.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:02 smithi194 bash[92666]: audit 2023-10-09T05:16:01.234551+0000 mon.a (mon.0) 1082 : audit 0 from='client.? 172.21.15.133:0/2131792615' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:16:02.388 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:02.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:02.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:02.491 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:02.520 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:02.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:02.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:02.605 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:02 smithi133 bash[105589]: audit 2023-10-09T05:16:01.234551+0000 mon.a (mon.0) 1082 : audit 0 from='client.? 172.21.15.133:0/2131792615' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:16:02.625 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:02.653 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:02.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:02.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:02.681 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:16:02.759 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:02.776 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:02.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:02.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:02.878 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:02.921 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:02.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:02.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:03.028 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:03.047 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:03.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:03.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:03.154 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:03.170 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T05:16:03.175 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:03.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:03.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:03.278 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:03.304 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:03.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:03.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:03.411 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:03.421 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:03 smithi133 bash[102858]: cluster 2023-10-09T05:16:01.916647+0000 mgr.y (mgr.25277) 825 : cluster 0 pgmap v840: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 757 op/s; 11543/242721 objects degraded (4.756%); 2585/242721 objects misplaced (1.065%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T05:16:03.421 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:03 smithi133 bash[105589]: cluster 2023-10-09T05:16:01.916647+0000 mgr.y (mgr.25277) 825 : cluster 0 pgmap v840: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 757 op/s; 11543/242721 objects degraded (4.756%); 2585/242721 objects misplaced (1.065%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T05:16:03.445 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:03.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:03.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:03.548 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:03.577 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:03.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:03.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:03.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:03 smithi194 bash[92666]: cluster 2023-10-09T05:16:01.916647+0000 mgr.y (mgr.25277) 825 : cluster 0 pgmap v840: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 757 op/s; 11543/242721 objects degraded (4.756%); 2585/242721 objects misplaced (1.065%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T05:16:03.681 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:03.713 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:03.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:03.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:03.816 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:03.843 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:03.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:03.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:03.946 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:03.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:03.979 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:03.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:04.080 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:04.106 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:04.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:04.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:04.209 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:04.241 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:04.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:04.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:04.250 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:04 smithi133 bash[102858]: cluster 2023-10-09T05:16:03.335099+0000 mon.a (mon.0) 1083 : cluster 3 Health check update: Degraded data redundancy: 11543/242721 objects degraded (4.756%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:04.250 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:04 smithi133 bash[105589]: cluster 2023-10-09T05:16:03.335099+0000 mon.a (mon.0) 1083 : cluster 3 Health check update: Degraded data redundancy: 11543/242721 objects degraded (4.756%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:04.345 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:04.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:04.378 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:04.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:04.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:04 smithi194 bash[92666]: cluster 2023-10-09T05:16:03.335099+0000 mon.a (mon.0) 1083 : cluster 3 Health check update: Degraded data redundancy: 11543/242721 objects degraded (4.756%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:04.479 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:04.524 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:04.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:04.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:04.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:16:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:16:04] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.43.0" 2023-10-09T05:16:04.629 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:04.674 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:04.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:04.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:04.779 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:04.813 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:04.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:04.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:04.916 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:04.941 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:04.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:04.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:05.044 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:05.071 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:05.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:05.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:05.174 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:05.202 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:05.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:05.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:05.305 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:05.335 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:05.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:05.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:05.372 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:05 smithi133 bash[105589]: cluster 2023-10-09T05:16:03.917229+0000 mgr.y (mgr.25277) 826 : cluster 0 pgmap v841: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 758 op/s; 11543/242043 objects degraded (4.769%); 2585/242043 objects misplaced (1.068%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T05:16:05.372 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:05 smithi133 bash[102858]: cluster 2023-10-09T05:16:03.917229+0000 mgr.y (mgr.25277) 826 : cluster 0 pgmap v841: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 758 op/s; 11543/242043 objects degraded (4.769%); 2585/242043 objects misplaced (1.068%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T05:16:05.439 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:05.477 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:05.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:05.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:05.584 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:05.623 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:05.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:05.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:05.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:05 smithi194 bash[92666]: cluster 2023-10-09T05:16:03.917229+0000 mgr.y (mgr.25277) 826 : cluster 0 pgmap v841: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 758 op/s; 11543/242043 objects degraded (4.769%); 2585/242043 objects misplaced (1.068%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T05:16:05.727 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:05.751 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:05.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:05.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:05.856 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:05.881 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:05.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:05.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:05.988 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:06.019 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:06.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:06.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:06.120 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:06.155 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:06.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:06.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:06.264 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:06.288 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:06.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:06.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:06.393 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:06.428 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:06.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:06.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:06.532 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:06.560 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:06.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:06.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:06.664 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:06.700 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:06.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:06.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:06.804 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:06.832 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:06.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:06.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:06.936 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:06.952 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:16:06.981 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:06.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:06.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:07.088 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:07.121 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:07.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:07.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:07.226 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:07.273 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:07.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:07.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:07.309 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:16:07.318 INFO:teuthology.orchestra.run.smithi133.stderr:dumped all 2023-10-09T05:16:07.357 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:07 smithi133 bash[102858]: cluster 2023-10-09T05:16:05.918054+0000 mgr.y (mgr.25277) 827 : cluster 0 pgmap v842: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 733 op/s; 11543/233586 objects degraded (4.942%); 2289/233586 objects misplaced (0.980%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T05:16:07.357 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:07 smithi133 bash[105589]: cluster 2023-10-09T05:16:05.918054+0000 mgr.y (mgr.25277) 827 : cluster 0 pgmap v842: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 733 op/s; 11543/233586 objects degraded (4.942%); 2289/233586 objects misplaced (0.980%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T05:16:07.383 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:07.413 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:07.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:07.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:07.516 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:07.527 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:07 smithi194 bash[92666]: cluster 2023-10-09T05:16:05.918054+0000 mgr.y (mgr.25277) 827 : cluster 0 pgmap v842: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 733 op/s; 11543/233586 objects degraded (4.942%); 2289/233586 objects misplaced (0.980%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T05:16:07.543 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:07.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:07.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:07.649 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:07.678 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:07.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:07.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:07.782 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:07.812 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:07.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:07.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:07.917 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:07.943 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:07.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:07.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:08.046 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:08.076 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:08.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:08.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:08.180 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:08.210 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:08.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:08.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:08.313 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:08.345 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:08.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:08.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:08.360 INFO:teuthology.orchestra.run.smithi133.stdout:{"pg_ready":true,"pg_map":{"version":842,"stamp":"2023-10-09T05:16:05.917417+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5101721528,"num_objects":77862,"num_object_clones":0,"num_object_copies":233586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11543,"num_objects_misplaced":2289,"num_objects_unfound":0,"num_objects_dirty":77862,"num_whiteouts":0,"num_read":506,"num_read_kb":931,"num_write":117330,"num_write_kb":6238158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3541,"num_bytes_recovered":231622686,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":103300,"ondisk_log_size":103300,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":270,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":18781944,"kb_used_data":15146504,"kb_used_omap":20,"kb_used_meta":3635051,"kb_avail":543688968,"statfs":{"total":575970213888,"available":556737503232,"internally_reserved":0,"allocated":15510020096,"data_stored":15504378624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":3722292878},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-480641024,"num_objects":-7334,"num_object_clones":0,"num_object_copies":-22002,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-542,"num_objects_unfound":0,"num_objects_dirty":-7334,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7334,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":542,"num_bytes_recovered":35520512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003496"},"pg_stats":[{"pgid":"3.1a","version":"69'2","reported_seq":205,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.109901+0000","last_change":"2023-10-09T05:09:57.873320+0000","last_active":"2023-10-09T05:15:42.109901+0000","last_peered":"2023-10-09T05:15:42.109901+0000","last_clean":"2023-10-09T05:15:42.109901+0000","last_became_active":"2023-10-09T05:09:57.868376+0000","last_became_peered":"2023-10-09T05:09:57.868376+0000","last_unstale":"2023-10-09T05:15:42.109901+0000","last_undegraded":"2023-10-09T05:15:42.109901+0000","last_fullsized":"2023-10-09T05:15:42.109901+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:27.186610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":183,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.337691+0000","last_change":"2023-10-09T05:14:15.345991+0000","last_active":"2023-10-09T05:15:42.337691+0000","last_peered":"2023-10-09T05:15:42.337691+0000","last_clean":"2023-10-09T05:15:42.337691+0000","last_became_active":"2023-10-09T05:14:15.345511+0000","last_became_peered":"2023-10-09T05:14:15.345511+0000","last_unstale":"2023-10-09T05:15:42.337691+0000","last_undegraded":"2023-10-09T05:15:42.337691+0000","last_fullsized":"2023-10-09T05:15:42.337691+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:49.234025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":66,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.100121+0000","last_change":"2023-10-09T05:09:57.857511+0000","last_active":"2023-10-09T05:15:42.100121+0000","last_peered":"2023-10-09T05:15:42.100121+0000","last_clean":"2023-10-09T05:15:42.100121+0000","last_became_active":"2023-10-09T05:09:57.857193+0000","last_became_peered":"2023-10-09T05:09:57.857193+0000","last_unstale":"2023-10-09T05:15:42.100121+0000","last_undegraded":"2023-10-09T05:15:42.100121+0000","last_fullsized":"2023-10-09T05:15:42.100121+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:32.842980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":190,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.101997+0000","last_change":"2023-10-09T05:15:41.118305+0000","last_active":"2023-10-09T05:15:42.101997+0000","last_peered":"2023-10-09T05:15:42.101997+0000","last_clean":"2023-10-09T05:15:42.101997+0000","last_became_active":"2023-10-09T05:15:41.117975+0000","last_became_peered":"2023-10-09T05:15:41.117975+0000","last_unstale":"2023-10-09T05:15:42.101997+0000","last_undegraded":"2023-10-09T05:15:42.101997+0000","last_fullsized":"2023-10-09T05:15:42.101997+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:03.469849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":192,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.337017+0000","last_change":"2023-10-09T05:09:57.849878+0000","last_active":"2023-10-09T05:15:42.337017+0000","last_peered":"2023-10-09T05:15:42.337017+0000","last_clean":"2023-10-09T05:15:42.337017+0000","last_became_active":"2023-10-09T05:09:57.849541+0000","last_became_peered":"2023-10-09T05:09:57.849541+0000","last_unstale":"2023-10-09T05:15:42.337017+0000","last_undegraded":"2023-10-09T05:15:42.337017+0000","last_fullsized":"2023-10-09T05:15:42.337017+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:49:03.330682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":195,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.114893+0000","last_change":"2023-10-09T05:13:03.039637+0000","last_active":"2023-10-09T05:15:42.114893+0000","last_peered":"2023-10-09T05:15:42.114893+0000","last_clean":"2023-10-09T05:15:42.114893+0000","last_became_active":"2023-10-09T05:13:03.039268+0000","last_became_peered":"2023-10-09T05:13:03.039268+0000","last_unstale":"2023-10-09T05:15:42.114893+0000","last_undegraded":"2023-10-09T05:15:42.114893+0000","last_fullsized":"2023-10-09T05:15:42.114893+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:19.875767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":169,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.102403+0000","last_change":"2023-10-09T05:13:03.035264+0000","last_active":"2023-10-09T05:15:42.102403+0000","last_peered":"2023-10-09T05:15:42.102403+0000","last_clean":"2023-10-09T05:15:42.102403+0000","last_became_active":"2023-10-09T05:13:03.034947+0000","last_became_peered":"2023-10-09T05:13:03.034947+0000","last_unstale":"2023-10-09T05:15:42.102403+0000","last_undegraded":"2023-10-09T05:15:42.102403+0000","last_fullsized":"2023-10-09T05:15:42.102403+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:44.842601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":174,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.337172+0000","last_change":"2023-10-09T05:09:57.849695+0000","last_active":"2023-10-09T05:15:42.337172+0000","last_peered":"2023-10-09T05:15:42.337172+0000","last_clean":"2023-10-09T05:15:42.337172+0000","last_became_active":"2023-10-09T05:09:57.848478+0000","last_became_peered":"2023-10-09T05:09:57.848478+0000","last_unstale":"2023-10-09T05:15:42.337172+0000","last_undegraded":"2023-10-09T05:15:42.337172+0000","last_fullsized":"2023-10-09T05:15:42.337172+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:26.115312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":172,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.099900+0000","last_change":"2023-10-09T05:15:41.105234+0000","last_active":"2023-10-09T05:15:42.099900+0000","last_peered":"2023-10-09T05:15:42.099900+0000","last_clean":"2023-10-09T05:15:42.099900+0000","last_became_active":"2023-10-09T05:15:41.104693+0000","last_became_peered":"2023-10-09T05:15:41.104693+0000","last_unstale":"2023-10-09T05:15:42.099900+0000","last_undegraded":"2023-10-09T05:15:42.099900+0000","last_fullsized":"2023-10-09T05:15:42.099900+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:56.682903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":178,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.337155+0000","last_change":"2023-10-09T05:15:41.126268+0000","last_active":"2023-10-09T05:15:42.337155+0000","last_peered":"2023-10-09T05:15:42.337155+0000","last_clean":"2023-10-09T05:15:42.337155+0000","last_became_active":"2023-10-09T05:15:41.125924+0000","last_became_peered":"2023-10-09T05:15:41.125924+0000","last_unstale":"2023-10-09T05:15:42.337155+0000","last_undegraded":"2023-10-09T05:15:42.337155+0000","last_fullsized":"2023-10-09T05:15:42.337155+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:30:20.022340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":58,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.337784+0000","last_change":"2023-10-09T05:15:41.154411+0000","last_active":"2023-10-09T05:15:42.337784+0000","last_peered":"2023-10-09T05:15:42.337784+0000","last_clean":"2023-10-09T05:15:42.337784+0000","last_became_active":"2023-10-09T05:15:41.126702+0000","last_became_peered":"2023-10-09T05:15:41.126702+0000","last_unstale":"2023-10-09T05:15:42.337784+0000","last_undegraded":"2023-10-09T05:15:42.337784+0000","last_fullsized":"2023-10-09T05:15:42.337784+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:51.580016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":65,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.099136+0000","last_change":"2023-10-09T05:09:57.848297+0000","last_active":"2023-10-09T05:15:42.099136+0000","last_peered":"2023-10-09T05:15:42.099136+0000","last_clean":"2023-10-09T05:15:42.099136+0000","last_became_active":"2023-10-09T05:09:57.847987+0000","last_became_peered":"2023-10-09T05:09:57.847987+0000","last_unstale":"2023-10-09T05:15:42.099136+0000","last_undegraded":"2023-10-09T05:15:42.099136+0000","last_fullsized":"2023-10-09T05:15:42.099136+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:16.385601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'23","reported_seq":237,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.120359+0000","last_change":"2023-10-09T05:15:41.217507+0000","last_active":"2023-10-09T05:15:42.120359+0000","last_peered":"2023-10-09T05:15:42.120359+0000","last_clean":"2023-10-09T05:15:42.120359+0000","last_became_active":"2023-10-09T05:15:41.137408+0000","last_became_peered":"2023-10-09T05:15:41.137408+0000","last_unstale":"2023-10-09T05:15:42.120359+0000","last_undegraded":"2023-10-09T05:15:42.120359+0000","last_fullsized":"2023-10-09T05:15:42.120359+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:14.672364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":185,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.110316+0000","last_change":"2023-10-09T05:09:57.868761+0000","last_active":"2023-10-09T05:15:42.110316+0000","last_peered":"2023-10-09T05:15:42.110316+0000","last_clean":"2023-10-09T05:15:42.110316+0000","last_became_active":"2023-10-09T05:09:57.868121+0000","last_became_peered":"2023-10-09T05:09:57.868121+0000","last_unstale":"2023-10-09T05:15:42.110316+0000","last_undegraded":"2023-10-09T05:15:42.110316+0000","last_fullsized":"2023-10-09T05:15:42.110316+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:43.412235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":61,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.111411+0000","last_change":"2023-10-09T05:15:41.207763+0000","last_active":"2023-10-09T05:15:42.111411+0000","last_peered":"2023-10-09T05:15:42.111411+0000","last_clean":"2023-10-09T05:15:42.111411+0000","last_became_active":"2023-10-09T05:15:41.118356+0000","last_became_peered":"2023-10-09T05:15:41.118356+0000","last_unstale":"2023-10-09T05:15:42.111411+0000","last_undegraded":"2023-10-09T05:15:42.111411+0000","last_fullsized":"2023-10-09T05:15:42.111411+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:24.587392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":201,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.374462+0000","last_change":"2023-10-09T05:15:41.108580+0000","last_active":"2023-10-09T05:15:42.374462+0000","last_peered":"2023-10-09T05:15:42.374462+0000","last_clean":"2023-10-09T05:15:42.374462+0000","last_became_active":"2023-10-09T05:15:41.108248+0000","last_became_peered":"2023-10-09T05:15:41.108248+0000","last_unstale":"2023-10-09T05:15:42.374462+0000","last_undegraded":"2023-10-09T05:15:42.374462+0000","last_fullsized":"2023-10-09T05:15:42.374462+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:28:41.740128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":188,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.102064+0000","last_change":"2023-10-09T05:15:41.127748+0000","last_active":"2023-10-09T05:15:42.102064+0000","last_peered":"2023-10-09T05:15:42.102064+0000","last_clean":"2023-10-09T05:15:42.102064+0000","last_became_active":"2023-10-09T05:15:41.127425+0000","last_became_peered":"2023-10-09T05:15:41.127425+0000","last_unstale":"2023-10-09T05:15:42.102064+0000","last_undegraded":"2023-10-09T05:15:42.102064+0000","last_fullsized":"2023-10-09T05:15:42.102064+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:47.895225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":114,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.337885+0000","last_change":"2023-10-09T05:15:41.127088+0000","last_active":"2023-10-09T05:15:42.337885+0000","last_peered":"2023-10-09T05:15:42.337885+0000","last_clean":"2023-10-09T05:15:42.337885+0000","last_became_active":"2023-10-09T05:15:41.126752+0000","last_became_peered":"2023-10-09T05:15:41.126752+0000","last_unstale":"2023-10-09T05:15:42.337885+0000","last_undegraded":"2023-10-09T05:15:42.337885+0000","last_fullsized":"2023-10-09T05:15:42.337885+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:15.831662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":38,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.110761+0000","last_change":"2023-10-09T05:15:41.151137+0000","last_active":"2023-10-09T05:15:42.110761+0000","last_peered":"2023-10-09T05:15:42.110761+0000","last_clean":"2023-10-09T05:15:42.110761+0000","last_became_active":"2023-10-09T05:15:41.150762+0000","last_became_peered":"2023-10-09T05:15:41.150762+0000","last_unstale":"2023-10-09T05:15:42.110761+0000","last_undegraded":"2023-10-09T05:15:42.110761+0000","last_fullsized":"2023-10-09T05:15:42.110761+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:04.112124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":188,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.102119+0000","last_change":"2023-10-09T05:15:41.121586+0000","last_active":"2023-10-09T05:15:42.102119+0000","last_peered":"2023-10-09T05:15:42.102119+0000","last_clean":"2023-10-09T05:15:42.102119+0000","last_became_active":"2023-10-09T05:15:41.118694+0000","last_became_peered":"2023-10-09T05:15:41.118694+0000","last_unstale":"2023-10-09T05:15:42.102119+0000","last_undegraded":"2023-10-09T05:15:42.102119+0000","last_fullsized":"2023-10-09T05:15:42.102119+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:48.566568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":198,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.107946+0000","last_change":"2023-10-09T05:13:03.047339+0000","last_active":"2023-10-09T05:15:42.107946+0000","last_peered":"2023-10-09T05:15:42.107946+0000","last_clean":"2023-10-09T05:15:42.107946+0000","last_became_active":"2023-10-09T05:13:03.040038+0000","last_became_peered":"2023-10-09T05:13:03.040038+0000","last_unstale":"2023-10-09T05:15:42.107946+0000","last_undegraded":"2023-10-09T05:15:42.107946+0000","last_fullsized":"2023-10-09T05:15:42.107946+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:56.726249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":201,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.338170+0000","last_change":"2023-10-09T05:15:41.111547+0000","last_active":"2023-10-09T05:15:42.338170+0000","last_peered":"2023-10-09T05:15:42.338170+0000","last_clean":"2023-10-09T05:15:42.338170+0000","last_became_active":"2023-10-09T05:15:41.108466+0000","last_became_peered":"2023-10-09T05:15:41.108466+0000","last_unstale":"2023-10-09T05:15:42.338170+0000","last_undegraded":"2023-10-09T05:15:42.338170+0000","last_fullsized":"2023-10-09T05:15:42.338170+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:01:12.583497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"205'7388","reported_seq":7423,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:16:04.597039+0000","last_change":"2023-10-09T05:13:41.269806+0000","last_active":"2023-10-09T05:16:04.597039+0000","last_peered":"2023-10-09T05:16:04.597039+0000","last_clean":"2023-10-09T05:16:04.597039+0000","last_became_active":"2023-10-09T05:13:41.269499+0000","last_became_peered":"2023-10-09T05:13:41.269499+0000","last_unstale":"2023-10-09T05:16:04.597039+0000","last_undegraded":"2023-10-09T05:16:04.597039+0000","last_fullsized":"2023-10-09T05:16:04.597039+0000","mapping_epoch":179,"log_start":"189'1457","ondisk_log_start":"189'1457","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":5931,"log_dups_size":0,"ondisk_log_size":5931,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:53:31.740811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319946752,"num_objects":4882,"num_object_clones":0,"num_object_copies":14646,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4882,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7388,"num_write_kb":392640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":57,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.113068+0000","last_change":"2023-10-09T05:15:41.190598+0000","last_active":"2023-10-09T05:15:42.113068+0000","last_peered":"2023-10-09T05:15:42.113068+0000","last_clean":"2023-10-09T05:15:42.113068+0000","last_became_active":"2023-10-09T05:15:41.115228+0000","last_became_peered":"2023-10-09T05:15:41.115228+0000","last_unstale":"2023-10-09T05:15:42.113068+0000","last_undegraded":"2023-10-09T05:15:42.113068+0000","last_fullsized":"2023-10-09T05:15:42.113068+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:39.062081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":43,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.108059+0000","last_change":"2023-10-09T05:15:41.123633+0000","last_active":"2023-10-09T05:15:42.108059+0000","last_peered":"2023-10-09T05:15:42.108059+0000","last_clean":"2023-10-09T05:15:42.108059+0000","last_became_active":"2023-10-09T05:15:41.123197+0000","last_became_peered":"2023-10-09T05:15:41.123197+0000","last_unstale":"2023-10-09T05:15:42.108059+0000","last_undegraded":"2023-10-09T05:15:42.108059+0000","last_fullsized":"2023-10-09T05:15:42.108059+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:53.952474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.7","version":"205'7430","reported_seq":11580,"reported_epoch":205,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T05:16:05.008861+0000","last_change":"2023-10-09T05:15:41.221962+0000","last_active":"2023-10-09T05:16:05.008861+0000","last_peered":"2023-10-09T05:16:05.008861+0000","last_clean":"2023-10-09T05:14:49.957493+0000","last_became_active":"2023-10-09T05:14:52.119953+0000","last_became_peered":"2023-10-09T05:14:52.119953+0000","last_unstale":"2023-10-09T05:16:05.008861+0000","last_undegraded":"2023-10-09T05:16:05.008861+0000","last_fullsized":"2023-10-09T05:14:51.974092+0000","mapping_epoch":193,"log_start":"187'600","ondisk_log_start":"187'600","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":6830,"log_dups_size":0,"ondisk_log_size":6830,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:02.866929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317456384,"num_objects":4844,"num_object_clones":0,"num_object_copies":14532,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2289,"num_objects_unfound":0,"num_objects_dirty":4844,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7430,"num_write_kb":392768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2047,"num_bytes_recovered":134152192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3],"avail_no_missing":["6","2","3"],"object_location_counts":[{"shards":"2,3,6","objects":2289},{"shards":"3,6","objects":2555}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":57,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.112956+0000","last_change":"2023-10-09T05:15:41.210326+0000","last_active":"2023-10-09T05:15:42.112956+0000","last_peered":"2023-10-09T05:15:42.112956+0000","last_clean":"2023-10-09T05:15:42.112956+0000","last_became_active":"2023-10-09T05:15:41.135518+0000","last_became_peered":"2023-10-09T05:15:41.135518+0000","last_unstale":"2023-10-09T05:15:42.112956+0000","last_undegraded":"2023-10-09T05:15:42.112956+0000","last_fullsized":"2023-10-09T05:15:42.112956+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:35.282510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":202,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.110588+0000","last_change":"2023-10-09T05:09:57.868031+0000","last_active":"2023-10-09T05:15:42.110588+0000","last_peered":"2023-10-09T05:15:42.110588+0000","last_clean":"2023-10-09T05:15:42.110588+0000","last_became_active":"2023-10-09T05:09:57.866477+0000","last_became_peered":"2023-10-09T05:09:57.866477+0000","last_unstale":"2023-10-09T05:15:42.110588+0000","last_undegraded":"2023-10-09T05:15:42.110588+0000","last_fullsized":"2023-10-09T05:15:42.110588+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:27.102583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"205'7290","reported_seq":7326,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:16:04.594512+0000","last_change":"2023-10-09T05:13:41.270633+0000","last_active":"2023-10-09T05:16:04.594512+0000","last_peered":"2023-10-09T05:16:04.594512+0000","last_clean":"2023-10-09T05:16:04.594512+0000","last_became_active":"2023-10-09T05:13:41.270167+0000","last_became_peered":"2023-10-09T05:13:41.270167+0000","last_unstale":"2023-10-09T05:16:04.594512+0000","last_undegraded":"2023-10-09T05:16:04.594512+0000","last_fullsized":"2023-10-09T05:16:04.594512+0000","mapping_epoch":179,"log_start":"188'1377","ondisk_log_start":"188'1377","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":5913,"log_dups_size":0,"ondisk_log_size":5913,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:29.153072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315621376,"num_objects":4816,"num_object_clones":0,"num_object_copies":14448,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4816,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7290,"num_write_kb":387392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":57,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.108833+0000","last_change":"2023-10-09T05:15:41.143716+0000","last_active":"2023-10-09T05:15:42.108833+0000","last_peered":"2023-10-09T05:15:42.108833+0000","last_clean":"2023-10-09T05:15:42.108833+0000","last_became_active":"2023-10-09T05:15:41.127905+0000","last_became_peered":"2023-10-09T05:15:41.127905+0000","last_unstale":"2023-10-09T05:15:42.108833+0000","last_undegraded":"2023-10-09T05:15:42.108833+0000","last_fullsized":"2023-10-09T05:15:42.108833+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:19:16.919154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":225,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.108661+0000","last_change":"2023-10-09T05:15:41.117071+0000","last_active":"2023-10-09T05:15:42.108661+0000","last_peered":"2023-10-09T05:15:42.108661+0000","last_clean":"2023-10-09T05:15:42.108661+0000","last_became_active":"2023-10-09T05:15:41.116599+0000","last_became_peered":"2023-10-09T05:15:41.116599+0000","last_unstale":"2023-10-09T05:15:42.108661+0000","last_undegraded":"2023-10-09T05:15:42.108661+0000","last_fullsized":"2023-10-09T05:15:42.108661+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:02.978515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"205'7166","reported_seq":7200,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:16:01.327505+0000","last_change":"2023-10-09T05:13:41.263623+0000","last_active":"2023-10-09T05:16:01.327505+0000","last_peered":"2023-10-09T05:16:01.327505+0000","last_clean":"2023-10-09T05:16:01.327505+0000","last_became_active":"2023-10-09T05:13:41.263309+0000","last_became_peered":"2023-10-09T05:13:41.263309+0000","last_unstale":"2023-10-09T05:16:01.327505+0000","last_undegraded":"2023-10-09T05:16:01.327505+0000","last_fullsized":"2023-10-09T05:16:01.327505+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":7166,"log_dups_size":0,"ondisk_log_size":7166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:08.600564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328859648,"num_objects":5018,"num_object_clones":0,"num_object_copies":15054,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5018,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7166,"num_write_kb":389888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":121,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.374639+0000","last_change":"2023-10-09T05:13:03.037926+0000","last_active":"2023-10-09T05:15:42.374639+0000","last_peered":"2023-10-09T05:15:42.374639+0000","last_clean":"2023-10-09T05:15:42.374639+0000","last_became_active":"2023-10-09T05:13:03.037639+0000","last_became_peered":"2023-10-09T05:13:03.037639+0000","last_unstale":"2023-10-09T05:15:42.374639+0000","last_undegraded":"2023-10-09T05:15:42.374639+0000","last_fullsized":"2023-10-09T05:15:42.374639+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:52.977143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":180,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.102828+0000","last_change":"2023-10-09T05:09:57.852914+0000","last_active":"2023-10-09T05:15:42.102828+0000","last_peered":"2023-10-09T05:15:42.102828+0000","last_clean":"2023-10-09T05:15:42.102828+0000","last_became_active":"2023-10-09T05:09:57.852522+0000","last_became_peered":"2023-10-09T05:09:57.852522+0000","last_unstale":"2023-10-09T05:15:42.102828+0000","last_undegraded":"2023-10-09T05:15:42.102828+0000","last_fullsized":"2023-10-09T05:15:42.102828+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:24.848494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"205'7513","reported_seq":7549,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:16:05.692215+0000","last_change":"2023-10-09T05:13:41.269005+0000","last_active":"2023-10-09T05:16:05.692215+0000","last_peered":"2023-10-09T05:16:05.692215+0000","last_clean":"2023-10-09T05:16:05.692215+0000","last_became_active":"2023-10-09T05:13:41.268729+0000","last_became_peered":"2023-10-09T05:13:41.268729+0000","last_unstale":"2023-10-09T05:16:05.692215+0000","last_undegraded":"2023-10-09T05:16:05.692215+0000","last_fullsized":"2023-10-09T05:16:05.692215+0000","mapping_epoch":179,"log_start":"189'1570","ondisk_log_start":"189'1570","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":5943,"log_dups_size":0,"ondisk_log_size":5943,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:56:22.245861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324206592,"num_objects":4947,"num_object_clones":0,"num_object_copies":14841,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4947,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7513,"num_write_kb":398720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"69'3","reported_seq":154,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.337917+0000","last_change":"2023-10-09T05:02:09.650579+0000","last_active":"2023-10-09T05:15:42.337917+0000","last_peered":"2023-10-09T05:15:42.337917+0000","last_clean":"2023-10-09T05:15:42.337917+0000","last_became_active":"2023-10-09T05:02:09.650160+0000","last_became_peered":"2023-10-09T05:02:09.650160+0000","last_unstale":"2023-10-09T05:15:42.337917+0000","last_undegraded":"2023-10-09T05:15:42.337917+0000","last_fullsized":"2023-10-09T05:15:42.337917+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:18.292670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":165,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.111307+0000","last_change":"2023-10-09T05:02:09.652449+0000","last_active":"2023-10-09T05:15:42.111307+0000","last_peered":"2023-10-09T05:15:42.111307+0000","last_clean":"2023-10-09T05:15:42.111307+0000","last_became_active":"2023-10-09T05:02:09.651994+0000","last_became_peered":"2023-10-09T05:02:09.651994+0000","last_unstale":"2023-10-09T05:15:42.111307+0000","last_undegraded":"2023-10-09T05:15:42.111307+0000","last_fullsized":"2023-10-09T05:15:42.111307+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:14.480513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"205'7220","reported_seq":8937,"reported_epoch":205,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:16:03.812651+0000","last_change":"2023-10-09T05:15:41.213441+0000","last_active":"2023-10-09T05:16:03.812651+0000","last_peered":"2023-10-09T05:16:03.812651+0000","last_clean":"2023-10-09T05:14:49.931887+0000","last_became_active":"2023-10-09T05:14:52.187809+0000","last_became_peered":"2023-10-09T05:14:52.187809+0000","last_unstale":"2023-10-09T05:16:03.812651+0000","last_undegraded":"2023-10-09T05:15:41.209633+0000","last_fullsized":"2023-10-09T05:14:51.971097+0000","mapping_epoch":193,"log_start":"187'800","ondisk_log_start":"187'800","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":6420,"log_dups_size":0,"ondisk_log_size":6420,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:44.859098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323485696,"num_objects":4936,"num_object_clones":0,"num_object_copies":14808,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3316,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4936,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7220,"num_write_kb":388992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":832,"num_bytes_recovered":54525952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":3005},{"shards":"3,4","objects":1620}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":186,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.108555+0000","last_change":"2023-10-09T05:09:57.849784+0000","last_active":"2023-10-09T05:15:42.108555+0000","last_peered":"2023-10-09T05:15:42.108555+0000","last_clean":"2023-10-09T05:15:42.108555+0000","last_became_active":"2023-10-09T05:09:57.849216+0000","last_became_peered":"2023-10-09T05:09:57.849216+0000","last_unstale":"2023-10-09T05:15:42.108555+0000","last_undegraded":"2023-10-09T05:15:42.108555+0000","last_fullsized":"2023-10-09T05:15:42.108555+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:29.764343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":212,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.111345+0000","last_change":"2023-10-09T05:14:50.981149+0000","last_active":"2023-10-09T05:15:42.111345+0000","last_peered":"2023-10-09T05:15:42.111345+0000","last_clean":"2023-10-09T05:15:42.111345+0000","last_became_active":"2023-10-09T05:14:50.980163+0000","last_became_peered":"2023-10-09T05:14:50.980163+0000","last_unstale":"2023-10-09T05:15:42.111345+0000","last_undegraded":"2023-10-09T05:15:42.111345+0000","last_fullsized":"2023-10-09T05:15:42.111345+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:40.014674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"205'7369","reported_seq":7403,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:16:04.919302+0000","last_change":"2023-10-09T05:13:41.273384+0000","last_active":"2023-10-09T05:16:04.919302+0000","last_peered":"2023-10-09T05:16:04.919302+0000","last_clean":"2023-10-09T05:16:04.919302+0000","last_became_active":"2023-10-09T05:13:41.271638+0000","last_became_peered":"2023-10-09T05:13:41.271638+0000","last_unstale":"2023-10-09T05:16:04.919302+0000","last_undegraded":"2023-10-09T05:16:04.919302+0000","last_fullsized":"2023-10-09T05:16:04.919302+0000","mapping_epoch":179,"log_start":"187'500","ondisk_log_start":"187'500","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":6869,"log_dups_size":0,"ondisk_log_size":6869,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:36:47.447983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317784064,"num_objects":4849,"num_object_clones":0,"num_object_copies":14547,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4849,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7369,"num_write_kb":390976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":196,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.110418+0000","last_change":"2023-10-09T05:15:41.129613+0000","last_active":"2023-10-09T05:15:42.110418+0000","last_peered":"2023-10-09T05:15:42.110418+0000","last_clean":"2023-10-09T05:15:42.110418+0000","last_became_active":"2023-10-09T05:15:41.129321+0000","last_became_peered":"2023-10-09T05:15:41.129321+0000","last_unstale":"2023-10-09T05:15:42.110418+0000","last_undegraded":"2023-10-09T05:15:42.110418+0000","last_fullsized":"2023-10-09T05:15:42.110418+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:03:07.043445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":185,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.111979+0000","last_change":"2023-10-09T05:09:57.868378+0000","last_active":"2023-10-09T05:15:42.111979+0000","last_peered":"2023-10-09T05:15:42.111979+0000","last_clean":"2023-10-09T05:15:42.111979+0000","last_became_active":"2023-10-09T05:09:57.867507+0000","last_became_peered":"2023-10-09T05:09:57.867507+0000","last_unstale":"2023-10-09T05:15:42.111979+0000","last_undegraded":"2023-10-09T05:15:42.111979+0000","last_fullsized":"2023-10-09T05:15:42.111979+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:14:35.199631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"205'7323","reported_seq":7384,"reported_epoch":205,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:16:04.514669+0000","last_change":"2023-10-09T05:15:41.236917+0000","last_active":"2023-10-09T05:16:04.514669+0000","last_peered":"2023-10-09T05:16:04.514669+0000","last_clean":"2023-10-09T05:15:11.955351+0000","last_became_active":"2023-10-09T05:15:41.222692+0000","last_became_peered":"2023-10-09T05:15:41.222692+0000","last_unstale":"2023-10-09T05:16:04.514669+0000","last_undegraded":"2023-10-09T05:15:41.127560+0000","last_fullsized":"2023-10-09T05:15:41.100880+0000","mapping_epoch":202,"log_start":"189'1474","ondisk_log_start":"189'1474","created":179,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":5849,"log_dups_size":0,"ondisk_log_size":5849,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:36:21.188540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316211200,"num_objects":4825,"num_object_clones":0,"num_object_copies":14475,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":982,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4825,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7323,"num_write_kb":388736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":4012},{"shards":"0,4,7","objects":811}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":203,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.336790+0000","last_change":"2023-10-09T05:15:41.121292+0000","last_active":"2023-10-09T05:15:42.336790+0000","last_peered":"2023-10-09T05:15:42.336790+0000","last_clean":"2023-10-09T05:15:42.336790+0000","last_became_active":"2023-10-09T05:15:41.120913+0000","last_became_peered":"2023-10-09T05:15:41.120913+0000","last_unstale":"2023-10-09T05:15:42.336790+0000","last_undegraded":"2023-10-09T05:15:42.336790+0000","last_fullsized":"2023-10-09T05:15:42.336790+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:20.270641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":66,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.099226+0000","last_change":"2023-10-09T05:09:57.856764+0000","last_active":"2023-10-09T05:15:42.099226+0000","last_peered":"2023-10-09T05:15:42.099226+0000","last_clean":"2023-10-09T05:15:42.099226+0000","last_became_active":"2023-10-09T05:09:57.856461+0000","last_became_peered":"2023-10-09T05:09:57.856461+0000","last_unstale":"2023-10-09T05:15:42.099226+0000","last_undegraded":"2023-10-09T05:15:42.099226+0000","last_fullsized":"2023-10-09T05:15:42.099226+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:06.848082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"205'7252","reported_seq":7298,"reported_epoch":205,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:16:04.508757+0000","last_change":"2023-10-09T05:15:42.397778+0000","last_active":"2023-10-09T05:16:04.508757+0000","last_peered":"2023-10-09T05:16:04.508757+0000","last_clean":"2023-10-09T05:15:11.925322+0000","last_became_active":"2023-10-09T05:15:42.378312+0000","last_became_peered":"2023-10-09T05:15:42.378312+0000","last_unstale":"2023-10-09T05:16:04.508757+0000","last_undegraded":"2023-10-09T05:15:42.119765+0000","last_fullsized":"2023-10-09T05:15:42.108930+0000","mapping_epoch":203,"log_start":"189'1464","ondisk_log_start":"189'1464","created":179,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":5788,"log_dups_size":0,"ondisk_log_size":5788,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:44:28.758473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316801024,"num_objects":4834,"num_object_clones":0,"num_object_copies":14502,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1036,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4834,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7252,"num_write_kb":386752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":3988},{"shards":"0,4,7","objects":846}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":93,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.098624+0000","last_change":"2023-10-09T05:15:41.098919+0000","last_active":"2023-10-09T05:15:42.098624+0000","last_peered":"2023-10-09T05:15:42.098624+0000","last_clean":"2023-10-09T05:15:42.098624+0000","last_became_active":"2023-10-09T05:15:41.098592+0000","last_became_peered":"2023-10-09T05:15:41.098592+0000","last_unstale":"2023-10-09T05:15:42.098624+0000","last_undegraded":"2023-10-09T05:15:42.098624+0000","last_fullsized":"2023-10-09T05:15:42.098624+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:25:09.306798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":138,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.337412+0000","last_change":"2023-10-09T05:13:03.031243+0000","last_active":"2023-10-09T05:15:42.337412+0000","last_peered":"2023-10-09T05:15:42.337412+0000","last_clean":"2023-10-09T05:15:42.337412+0000","last_became_active":"2023-10-09T05:13:03.028769+0000","last_became_peered":"2023-10-09T05:13:03.028769+0000","last_unstale":"2023-10-09T05:15:42.337412+0000","last_undegraded":"2023-10-09T05:15:42.337412+0000","last_fullsized":"2023-10-09T05:15:42.337412+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:25:25.239799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":74,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.099264+0000","last_change":"2023-10-09T05:13:03.033358+0000","last_active":"2023-10-09T05:15:42.099264+0000","last_peered":"2023-10-09T05:15:42.099264+0000","last_clean":"2023-10-09T05:15:42.099264+0000","last_became_active":"2023-10-09T05:13:03.032967+0000","last_became_peered":"2023-10-09T05:13:03.032967+0000","last_unstale":"2023-10-09T05:15:42.099264+0000","last_undegraded":"2023-10-09T05:15:42.099264+0000","last_fullsized":"2023-10-09T05:15:42.099264+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:33.726009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"205'7463","reported_seq":7497,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:16:04.999816+0000","last_change":"2023-10-09T05:13:41.268554+0000","last_active":"2023-10-09T05:16:04.999816+0000","last_peered":"2023-10-09T05:16:04.999816+0000","last_clean":"2023-10-09T05:16:04.999816+0000","last_became_active":"2023-10-09T05:13:41.268278+0000","last_became_peered":"2023-10-09T05:13:41.268278+0000","last_unstale":"2023-10-09T05:16:04.999816+0000","last_undegraded":"2023-10-09T05:16:04.999816+0000","last_fullsized":"2023-10-09T05:16:04.999816+0000","mapping_epoch":179,"log_start":"187'600","ondisk_log_start":"187'600","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":6863,"log_dups_size":0,"ondisk_log_size":6863,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:53.822644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317390848,"num_objects":4843,"num_object_clones":0,"num_object_copies":14529,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4843,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7463,"num_write_kb":393792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":172,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.098496+0000","last_change":"2023-10-09T05:13:03.031386+0000","last_active":"2023-10-09T05:15:42.098496+0000","last_peered":"2023-10-09T05:15:42.098496+0000","last_clean":"2023-10-09T05:15:42.098496+0000","last_became_active":"2023-10-09T05:13:03.030816+0000","last_became_peered":"2023-10-09T05:13:03.030816+0000","last_unstale":"2023-10-09T05:15:42.098496+0000","last_undegraded":"2023-10-09T05:15:42.098496+0000","last_fullsized":"2023-10-09T05:15:42.098496+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:28.088661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":166,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.110200+0000","last_change":"2023-10-09T04:56:56.087320+0000","last_active":"2023-10-09T05:15:42.110200+0000","last_peered":"2023-10-09T05:15:42.110200+0000","last_clean":"2023-10-09T05:15:42.110200+0000","last_became_active":"2023-10-09T04:56:56.086552+0000","last_became_peered":"2023-10-09T04:56:56.086552+0000","last_unstale":"2023-10-09T05:15:42.110200+0000","last_undegraded":"2023-10-09T05:15:42.110200+0000","last_fullsized":"2023-10-09T05:15:42.110200+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:14.205288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":75,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.098675+0000","last_change":"2023-10-09T05:14:50.972904+0000","last_active":"2023-10-09T05:15:42.098675+0000","last_peered":"2023-10-09T05:15:42.098675+0000","last_clean":"2023-10-09T05:15:42.098675+0000","last_became_active":"2023-10-09T05:14:50.972487+0000","last_became_peered":"2023-10-09T05:14:50.972487+0000","last_unstale":"2023-10-09T05:15:42.098675+0000","last_undegraded":"2023-10-09T05:15:42.098675+0000","last_fullsized":"2023-10-09T05:15:42.098675+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:35:23.517359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"205'7114","reported_seq":7175,"reported_epoch":205,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:16:01.324515+0000","last_change":"2023-10-09T05:15:41.190211+0000","last_active":"2023-10-09T05:16:01.324515+0000","last_peered":"2023-10-09T05:16:01.324515+0000","last_clean":"2023-10-09T05:15:11.887753+0000","last_became_active":"2023-10-09T05:15:41.181391+0000","last_became_peered":"2023-10-09T05:15:41.181391+0000","last_unstale":"2023-10-09T05:16:01.324515+0000","last_undegraded":"2023-10-09T05:15:41.107106+0000","last_fullsized":"2023-10-09T05:15:41.094646+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":179,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":7114,"log_dups_size":0,"ondisk_log_size":7114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:20.531558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324927488,"num_objects":4958,"num_object_clones":0,"num_object_copies":14874,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1031,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4958,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7114,"num_write_kb":386304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":4108},{"shards":"2,4,7","objects":850}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":114,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.337525+0000","last_change":"2023-10-09T05:14:50.977695+0000","last_active":"2023-10-09T05:15:42.337525+0000","last_peered":"2023-10-09T05:15:42.337525+0000","last_clean":"2023-10-09T05:15:42.337525+0000","last_became_active":"2023-10-09T05:14:50.977126+0000","last_became_peered":"2023-10-09T05:14:50.977126+0000","last_unstale":"2023-10-09T05:15:42.337525+0000","last_undegraded":"2023-10-09T05:15:42.337525+0000","last_fullsized":"2023-10-09T05:15:42.337525+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:50:57.273402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":67,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:15:57.152862+0000","last_change":"2023-10-09T05:09:57.854064+0000","last_active":"2023-10-09T05:15:57.152862+0000","last_peered":"2023-10-09T05:15:57.152862+0000","last_clean":"2023-10-09T05:15:57.152862+0000","last_became_active":"2023-10-09T05:09:57.847953+0000","last_became_peered":"2023-10-09T05:09:57.847953+0000","last_unstale":"2023-10-09T05:15:57.152862+0000","last_undegraded":"2023-10-09T05:15:57.152862+0000","last_fullsized":"2023-10-09T05:15:57.152862+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:45.914976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":45,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.413363+0000","last_change":"2023-10-09T05:15:41.120111+0000","last_active":"2023-10-09T05:15:42.413363+0000","last_peered":"2023-10-09T05:15:42.413363+0000","last_clean":"2023-10-09T05:15:42.413363+0000","last_became_active":"2023-10-09T05:15:41.119795+0000","last_became_peered":"2023-10-09T05:15:41.119795+0000","last_unstale":"2023-10-09T05:15:42.413363+0000","last_undegraded":"2023-10-09T05:15:42.413363+0000","last_fullsized":"2023-10-09T05:15:42.413363+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:42.300486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.b","version":"205'7195","reported_seq":7229,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:16:04.983215+0000","last_change":"2023-10-09T05:13:41.272258+0000","last_active":"2023-10-09T05:16:04.983215+0000","last_peered":"2023-10-09T05:16:04.983215+0000","last_clean":"2023-10-09T05:16:04.983215+0000","last_became_active":"2023-10-09T05:13:41.271935+0000","last_became_peered":"2023-10-09T05:13:41.271935+0000","last_unstale":"2023-10-09T05:16:04.983215+0000","last_undegraded":"2023-10-09T05:16:04.983215+0000","last_fullsized":"2023-10-09T05:16:04.983215+0000","mapping_epoch":179,"log_start":"187'300","ondisk_log_start":"187'300","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":6895,"log_dups_size":0,"ondisk_log_size":6895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:23:02.858628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309526528,"num_objects":4723,"num_object_clones":0,"num_object_copies":14169,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4723,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7195,"num_write_kb":381376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":135,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.412811+0000","last_change":"2023-10-09T05:15:41.119772+0000","last_active":"2023-10-09T05:15:42.412811+0000","last_peered":"2023-10-09T05:15:42.412811+0000","last_clean":"2023-10-09T05:15:42.412811+0000","last_became_active":"2023-10-09T05:15:41.119403+0000","last_became_peered":"2023-10-09T05:15:41.119403+0000","last_unstale":"2023-10-09T05:15:42.412811+0000","last_undegraded":"2023-10-09T05:15:42.412811+0000","last_fullsized":"2023-10-09T05:15:42.412811+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:12:16.579928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":464,"reported_epoch":205,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:15:57.159160+0000","last_change":"2023-10-09T05:15:42.111959+0000","last_active":"2023-10-09T05:15:57.159160+0000","last_peered":"2023-10-09T05:15:57.159160+0000","last_clean":"2023-10-09T05:15:05.934157+0000","last_became_active":"2023-10-09T05:15:42.111532+0000","last_became_peered":"2023-10-09T05:15:42.111532+0000","last_unstale":"2023-10-09T05:15:57.159160+0000","last_undegraded":"2023-10-09T05:15:42.103440+0000","last_fullsized":"2023-10-09T05:15:42.100931+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_clean_scrub_stamp":"2023-10-09T04:36:44.261013+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:16.948809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":161,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.337081+0000","last_change":"2023-10-09T05:13:03.027260+0000","last_active":"2023-10-09T05:15:42.337081+0000","last_peered":"2023-10-09T05:15:42.337081+0000","last_clean":"2023-10-09T05:15:42.337081+0000","last_became_active":"2023-10-09T05:13:03.026416+0000","last_became_peered":"2023-10-09T05:13:03.026416+0000","last_unstale":"2023-10-09T05:15:42.337081+0000","last_undegraded":"2023-10-09T05:15:42.337081+0000","last_fullsized":"2023-10-09T05:15:42.337081+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:05.057338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":178,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.108290+0000","last_change":"2023-10-09T05:15:41.119704+0000","last_active":"2023-10-09T05:15:42.108290+0000","last_peered":"2023-10-09T05:15:42.108290+0000","last_clean":"2023-10-09T05:15:42.108290+0000","last_became_active":"2023-10-09T05:15:41.118928+0000","last_became_peered":"2023-10-09T05:15:41.118928+0000","last_unstale":"2023-10-09T05:15:42.108290+0000","last_undegraded":"2023-10-09T05:15:42.108290+0000","last_fullsized":"2023-10-09T05:15:42.108290+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:47:24.556059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"205'7383","reported_seq":8660,"reported_epoch":205,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:16:04.609321+0000","last_change":"2023-10-09T05:15:41.235565+0000","last_active":"2023-10-09T05:16:04.609321+0000","last_peered":"2023-10-09T05:16:04.609321+0000","last_clean":"2023-10-09T05:15:11.934574+0000","last_became_active":"2023-10-09T05:15:41.227723+0000","last_became_peered":"2023-10-09T05:15:41.227723+0000","last_unstale":"2023-10-09T05:16:04.609321+0000","last_undegraded":"2023-10-09T05:15:41.147342+0000","last_fullsized":"2023-10-09T05:15:41.096227+0000","mapping_epoch":202,"log_start":"188'1444","ondisk_log_start":"188'1444","created":179,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":5939,"log_dups_size":0,"ondisk_log_size":5939,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:04:18.080835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317784064,"num_objects":4849,"num_object_clones":0,"num_object_copies":14547,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1017,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4849,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7383,"num_write_kb":391424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":607,"num_bytes_recovered":39714816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":4041},{"shards":"0,2,7","objects":808}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":181,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:15:57.152381+0000","last_change":"2023-10-09T05:15:41.115902+0000","last_active":"2023-10-09T05:15:57.152381+0000","last_peered":"2023-10-09T05:15:57.152381+0000","last_clean":"2023-10-09T05:15:57.152381+0000","last_became_active":"2023-10-09T05:15:41.115576+0000","last_became_peered":"2023-10-09T05:15:41.115576+0000","last_unstale":"2023-10-09T05:15:57.152381+0000","last_undegraded":"2023-10-09T05:15:57.152381+0000","last_fullsized":"2023-10-09T05:15:57.152381+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:42:21.916563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":162,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.101827+0000","last_change":"2023-10-09T05:02:09.648781+0000","last_active":"2023-10-09T05:15:42.101827+0000","last_peered":"2023-10-09T05:15:42.101827+0000","last_clean":"2023-10-09T05:15:42.101827+0000","last_became_active":"2023-10-09T05:02:09.648078+0000","last_became_peered":"2023-10-09T05:02:09.648078+0000","last_unstale":"2023-10-09T05:15:42.101827+0000","last_undegraded":"2023-10-09T05:15:42.101827+0000","last_fullsized":"2023-10-09T05:15:42.101827+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:35.287091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":222,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.101743+0000","last_change":"2023-10-09T05:15:41.107646+0000","last_active":"2023-10-09T05:15:42.101743+0000","last_peered":"2023-10-09T05:15:42.101743+0000","last_clean":"2023-10-09T05:15:42.101743+0000","last_became_active":"2023-10-09T05:15:41.105974+0000","last_became_peered":"2023-10-09T05:15:41.105974+0000","last_unstale":"2023-10-09T05:15:42.101743+0000","last_undegraded":"2023-10-09T05:15:42.101743+0000","last_fullsized":"2023-10-09T05:15:42.101743+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:01.702393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"205'7305","reported_seq":7340,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:16:04.992354+0000","last_change":"2023-10-09T05:13:41.273522+0000","last_active":"2023-10-09T05:16:04.992354+0000","last_peered":"2023-10-09T05:16:04.992354+0000","last_clean":"2023-10-09T05:16:04.992354+0000","last_became_active":"2023-10-09T05:13:41.273159+0000","last_became_peered":"2023-10-09T05:13:41.273159+0000","last_unstale":"2023-10-09T05:16:04.992354+0000","last_undegraded":"2023-10-09T05:16:04.992354+0000","last_fullsized":"2023-10-09T05:16:04.992354+0000","mapping_epoch":179,"log_start":"187'400","ondisk_log_start":"187'400","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":6905,"log_dups_size":0,"ondisk_log_size":6905,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:53.695799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312999960,"num_objects":4777,"num_object_clones":0,"num_object_copies":14331,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4777,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7305,"num_write_kb":386561,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":80,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.102949+0000","last_change":"2023-10-09T05:15:41.113499+0000","last_active":"2023-10-09T05:15:42.102949+0000","last_peered":"2023-10-09T05:15:42.102949+0000","last_clean":"2023-10-09T05:15:42.102949+0000","last_became_active":"2023-10-09T05:15:41.107504+0000","last_became_peered":"2023-10-09T05:15:41.107504+0000","last_unstale":"2023-10-09T05:15:42.102949+0000","last_undegraded":"2023-10-09T05:15:42.102949+0000","last_fullsized":"2023-10-09T05:15:42.102949+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:48.704963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":84,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.098902+0000","last_change":"2023-10-09T05:14:50.977609+0000","last_active":"2023-10-09T05:15:42.098902+0000","last_peered":"2023-10-09T05:15:42.098902+0000","last_clean":"2023-10-09T05:15:42.098902+0000","last_became_active":"2023-10-09T05:14:50.977225+0000","last_became_peered":"2023-10-09T05:14:50.977225+0000","last_unstale":"2023-10-09T05:15:42.098902+0000","last_undegraded":"2023-10-09T05:15:42.098902+0000","last_fullsized":"2023-10-09T05:15:42.098902+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:06.711748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":193,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.103119+0000","last_change":"2023-10-09T05:15:41.111318+0000","last_active":"2023-10-09T05:15:42.103119+0000","last_peered":"2023-10-09T05:15:42.103119+0000","last_clean":"2023-10-09T05:15:42.103119+0000","last_became_active":"2023-10-09T05:15:41.110878+0000","last_became_peered":"2023-10-09T05:15:41.110878+0000","last_unstale":"2023-10-09T05:15:42.103119+0000","last_undegraded":"2023-10-09T05:15:42.103119+0000","last_fullsized":"2023-10-09T05:15:42.103119+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:22.225260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"205'7271","reported_seq":7329,"reported_epoch":205,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:16:04.513199+0000","last_change":"2023-10-09T05:15:41.203168+0000","last_active":"2023-10-09T05:16:04.513199+0000","last_peered":"2023-10-09T05:16:04.513199+0000","last_clean":"2023-10-09T05:14:49.945496+0000","last_became_active":"2023-10-09T05:14:52.149107+0000","last_became_peered":"2023-10-09T05:14:52.149107+0000","last_unstale":"2023-10-09T05:16:04.513199+0000","last_undegraded":"2023-10-09T05:15:13.027503+0000","last_fullsized":"2023-10-09T05:14:51.974129+0000","mapping_epoch":193,"log_start":"189'1466","ondisk_log_start":"189'1466","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":5805,"log_dups_size":0,"ondisk_log_size":5805,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:21.041496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317652992,"num_objects":4847,"num_object_clones":0,"num_object_copies":14541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4159,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7271,"num_write_kb":387776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":2951},{"shards":"4,6","objects":688}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":192,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.338272+0000","last_change":"2023-10-09T05:15:41.193256+0000","last_active":"2023-10-09T05:15:42.338272+0000","last_peered":"2023-10-09T05:15:42.338272+0000","last_clean":"2023-10-09T05:15:42.338272+0000","last_became_active":"2023-10-09T05:15:41.137091+0000","last_became_peered":"2023-10-09T05:15:41.137091+0000","last_unstale":"2023-10-09T05:15:42.338272+0000","last_undegraded":"2023-10-09T05:15:42.338272+0000","last_fullsized":"2023-10-09T05:15:42.338272+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:42.820583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":66,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.099857+0000","last_change":"2023-10-09T05:09:57.858617+0000","last_active":"2023-10-09T05:15:42.099857+0000","last_peered":"2023-10-09T05:15:42.099857+0000","last_clean":"2023-10-09T05:15:42.099857+0000","last_became_active":"2023-10-09T05:09:57.858128+0000","last_became_peered":"2023-10-09T05:09:57.858128+0000","last_unstale":"2023-10-09T05:15:42.099857+0000","last_undegraded":"2023-10-09T05:15:42.099857+0000","last_fullsized":"2023-10-09T05:15:42.099857+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:02.502985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":176,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.336931+0000","last_change":"2023-10-09T05:15:41.154789+0000","last_active":"2023-10-09T05:15:42.336931+0000","last_peered":"2023-10-09T05:15:42.336931+0000","last_clean":"2023-10-09T05:15:42.336931+0000","last_became_active":"2023-10-09T05:15:41.154291+0000","last_became_peered":"2023-10-09T05:15:41.154291+0000","last_unstale":"2023-10-09T05:15:42.336931+0000","last_undegraded":"2023-10-09T05:15:42.336931+0000","last_fullsized":"2023-10-09T05:15:42.336931+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:25.269156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"205'7348","reported_seq":7382,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:16:04.996653+0000","last_change":"2023-10-09T05:13:41.271193+0000","last_active":"2023-10-09T05:16:04.996653+0000","last_peered":"2023-10-09T05:16:04.996653+0000","last_clean":"2023-10-09T05:16:04.996653+0000","last_became_active":"2023-10-09T05:13:41.269862+0000","last_became_peered":"2023-10-09T05:13:41.269862+0000","last_unstale":"2023-10-09T05:16:04.996653+0000","last_undegraded":"2023-10-09T05:16:04.996653+0000","last_fullsized":"2023-10-09T05:16:04.996653+0000","mapping_epoch":179,"log_start":"187'500","ondisk_log_start":"187'500","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":6848,"log_dups_size":0,"ondisk_log_size":6848,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:36.314853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320471040,"num_objects":4890,"num_object_clones":0,"num_object_copies":14670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4890,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7348,"num_write_kb":391616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":200,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.109683+0000","last_change":"2023-10-09T05:15:41.123233+0000","last_active":"2023-10-09T05:15:42.109683+0000","last_peered":"2023-10-09T05:15:42.109683+0000","last_clean":"2023-10-09T05:15:42.109683+0000","last_became_active":"2023-10-09T05:15:41.122768+0000","last_became_peered":"2023-10-09T05:15:41.122768+0000","last_unstale":"2023-10-09T05:15:42.109683+0000","last_undegraded":"2023-10-09T05:15:42.109683+0000","last_fullsized":"2023-10-09T05:15:42.109683+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:39.717521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":192,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.101877+0000","last_change":"2023-10-09T05:09:57.878002+0000","last_active":"2023-10-09T05:15:42.101877+0000","last_peered":"2023-10-09T05:15:42.101877+0000","last_clean":"2023-10-09T05:15:42.101877+0000","last_became_active":"2023-10-09T05:09:57.852792+0000","last_became_peered":"2023-10-09T05:09:57.852792+0000","last_unstale":"2023-10-09T05:15:42.101877+0000","last_undegraded":"2023-10-09T05:15:42.101877+0000","last_fullsized":"2023-10-09T05:15:42.101877+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:18.763275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":158,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.110212+0000","last_change":"2023-10-09T05:13:03.026541+0000","last_active":"2023-10-09T05:15:42.110212+0000","last_peered":"2023-10-09T05:15:42.110212+0000","last_clean":"2023-10-09T05:15:42.110212+0000","last_became_active":"2023-10-09T05:13:03.025729+0000","last_became_peered":"2023-10-09T05:13:03.025729+0000","last_unstale":"2023-10-09T05:15:42.110212+0000","last_undegraded":"2023-10-09T05:15:42.110212+0000","last_fullsized":"2023-10-09T05:15:42.110212+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:22.938235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'2","reported_seq":187,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.103073+0000","last_change":"2023-10-09T05:15:41.118210+0000","last_active":"2023-10-09T05:15:42.103073+0000","last_peered":"2023-10-09T05:15:42.103073+0000","last_clean":"2023-10-09T05:15:42.103073+0000","last_became_active":"2023-10-09T05:15:41.116521+0000","last_became_peered":"2023-10-09T05:15:41.116521+0000","last_unstale":"2023-10-09T05:15:42.103073+0000","last_undegraded":"2023-10-09T05:15:42.103073+0000","last_fullsized":"2023-10-09T05:15:42.103073+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:01.287668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":166,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.098797+0000","last_change":"2023-10-09T05:14:50.976630+0000","last_active":"2023-10-09T05:15:42.098797+0000","last_peered":"2023-10-09T05:15:42.098797+0000","last_clean":"2023-10-09T05:15:42.098797+0000","last_became_active":"2023-10-09T05:14:50.976283+0000","last_became_peered":"2023-10-09T05:14:50.976283+0000","last_unstale":"2023-10-09T05:15:42.098797+0000","last_undegraded":"2023-10-09T05:15:42.098797+0000","last_fullsized":"2023-10-09T05:15:42.098797+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:04.614827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":60,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.120437+0000","last_change":"2023-10-09T05:13:03.055887+0000","last_active":"2023-10-09T05:15:42.120437+0000","last_peered":"2023-10-09T05:15:42.120437+0000","last_clean":"2023-10-09T05:15:42.120437+0000","last_became_active":"2023-10-09T05:13:03.028782+0000","last_became_peered":"2023-10-09T05:13:03.028782+0000","last_unstale":"2023-10-09T05:15:42.120437+0000","last_undegraded":"2023-10-09T05:15:42.120437+0000","last_fullsized":"2023-10-09T05:15:42.120437+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:38.828205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":202,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.110961+0000","last_change":"2023-10-09T05:09:57.867865+0000","last_active":"2023-10-09T05:15:42.110961+0000","last_peered":"2023-10-09T05:15:42.110961+0000","last_clean":"2023-10-09T05:15:42.110961+0000","last_became_active":"2023-10-09T05:09:57.866723+0000","last_became_peered":"2023-10-09T05:09:57.866723+0000","last_unstale":"2023-10-09T05:15:42.110961+0000","last_undegraded":"2023-10-09T05:15:42.110961+0000","last_fullsized":"2023-10-09T05:15:42.110961+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:58.946358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":60,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.111453+0000","last_change":"2023-10-09T05:13:03.049546+0000","last_active":"2023-10-09T05:15:42.111453+0000","last_peered":"2023-10-09T05:15:42.111453+0000","last_clean":"2023-10-09T05:15:42.111453+0000","last_became_active":"2023-10-09T05:13:03.028676+0000","last_became_peered":"2023-10-09T05:13:03.028676+0000","last_unstale":"2023-10-09T05:15:42.111453+0000","last_undegraded":"2023-10-09T05:15:42.111453+0000","last_fullsized":"2023-10-09T05:15:42.111453+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:55.089350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":182,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.110021+0000","last_change":"2023-10-09T05:02:09.649748+0000","last_active":"2023-10-09T05:15:42.110021+0000","last_peered":"2023-10-09T05:15:42.110021+0000","last_clean":"2023-10-09T05:15:42.110021+0000","last_became_active":"2023-10-09T05:02:09.649444+0000","last_became_peered":"2023-10-09T05:02:09.649444+0000","last_unstale":"2023-10-09T05:15:42.110021+0000","last_undegraded":"2023-10-09T05:15:42.110021+0000","last_fullsized":"2023-10-09T05:15:42.110021+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:07.436820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'2","reported_seq":179,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.337746+0000","last_change":"2023-10-09T05:14:50.973251+0000","last_active":"2023-10-09T05:15:42.337746+0000","last_peered":"2023-10-09T05:15:42.337746+0000","last_clean":"2023-10-09T05:15:42.337746+0000","last_became_active":"2023-10-09T05:14:50.972870+0000","last_became_peered":"2023-10-09T05:14:50.972870+0000","last_unstale":"2023-10-09T05:15:42.337746+0000","last_undegraded":"2023-10-09T05:15:42.337746+0000","last_fullsized":"2023-10-09T05:15:42.337746+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:03.879853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":183,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.336729+0000","last_change":"2023-10-09T05:02:09.649674+0000","last_active":"2023-10-09T05:15:42.336729+0000","last_peered":"2023-10-09T05:15:42.336729+0000","last_clean":"2023-10-09T05:15:42.336729+0000","last_became_active":"2023-10-09T05:02:09.649041+0000","last_became_peered":"2023-10-09T05:02:09.649041+0000","last_unstale":"2023-10-09T05:15:42.336729+0000","last_undegraded":"2023-10-09T05:15:42.336729+0000","last_fullsized":"2023-10-09T05:15:42.336729+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:18.524071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'5","reported_seq":202,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.102558+0000","last_change":"2023-10-09T05:14:50.985019+0000","last_active":"2023-10-09T05:15:42.102558+0000","last_peered":"2023-10-09T05:15:42.102558+0000","last_clean":"2023-10-09T05:15:42.102558+0000","last_became_active":"2023-10-09T05:14:50.979874+0000","last_became_peered":"2023-10-09T05:14:50.979874+0000","last_unstale":"2023-10-09T05:15:42.102558+0000","last_undegraded":"2023-10-09T05:15:42.102558+0000","last_fullsized":"2023-10-09T05:15:42.102558+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:47.166329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":185,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:15:42.109601+0000","last_change":"2023-10-09T05:09:57.866732+0000","last_active":"2023-10-09T05:15:42.109601+0000","last_peered":"2023-10-09T05:15:42.109601+0000","last_clean":"2023-10-09T05:15:42.109601+0000","last_became_active":"2023-10-09T05:09:57.866117+0000","last_became_peered":"2023-10-09T05:09:57.866117+0000","last_unstale":"2023-10-09T05:15:42.109601+0000","last_undegraded":"2023-10-09T05:15:42.109601+0000","last_fullsized":"2023-10-09T05:15:42.109601+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:49:14.815342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5101125656,"num_objects":77838,"num_object_clones":0,"num_object_copies":233514,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11541,"num_objects_misplaced":2289,"num_objects_unfound":0,"num_objects_dirty":77838,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":117030,"num_write_kb":6235713,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3490,"num_bytes_recovered":228655104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15676682240,"data_stored":15676670024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":103078,"ondisk_log_size":103078,"up":48,"acting":41,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":19,"num_read_kb":18,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41,"num_bytes_recovered":15742,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":2,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234406,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":46,"seq":197568496048,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2761272,"kb_used_data":2171504,"kb_used_omap":0,"kb_used_meta":589760,"kb_avail":90983880,"statfs":{"total":95995035648,"available":93167493120,"internally_reserved":0,"allocated":2223620096,"data_stored":2222898430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603914240},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:15:30 2023","interfaces":[{"interface":"back","average":{"1min":2.327,"5min":1.6679999999999999,"15min":1.8109999999999999},"min":{"1min":0.35499999999999998,"5min":0.26500000000000001,"15min":0.214},"max":{"1min":13.359,"5min":23.908999999999999,"15min":98.275000000000006},"last":50.978999999999999},{"interface":"front","average":{"1min":2.4300000000000002,"5min":1.778,"15min":1.9330000000000001},"min":{"1min":0.377,"5min":0.33200000000000002,"15min":0.27200000000000002},"max":{"1min":13.403,"5min":23.838000000000001,"15min":98.334999999999994},"last":51.051000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:15:30 2023","interfaces":[{"interface":"back","average":{"1min":1.1559999999999999,"5min":2.2290000000000001,"15min":1.679},"min":{"1min":0.313,"5min":0.26700000000000002,"15min":0.25800000000000001},"max":{"1min":6.3849999999999998,"5min":60.149999999999999,"15min":60.149999999999999},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.4650000000000001,"5min":2.2370000000000001,"15min":1.8140000000000001},"min":{"1min":0.36799999999999999,"5min":0.28899999999999998,"15min":0.27900000000000003},"max":{"1min":7.6239999999999997,"5min":60.268000000000001,"15min":60.268000000000001},"last":0.82899999999999996}]},{"osd":2,"last update":"Mon Oct 9 05:15:30 2023","interfaces":[{"interface":"back","average":{"1min":1.861,"5min":0.99099999999999999,"15min":1.3240000000000001},"min":{"1min":0.29399999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":13.568,"5min":13.568,"15min":13.568},"last":0.74399999999999999},{"interface":"front","average":{"1min":2.1840000000000002,"5min":1.093,"15min":1.21},"min":{"1min":0.33800000000000002,"5min":0.32400000000000001,"15min":0.30099999999999999},"max":{"1min":7.7409999999999997,"5min":7.7409999999999997,"15min":8.3810000000000002},"last":0.65300000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:15:30 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":4.1900000000000004,"15min":2.5030000000000001},"min":{"1min":0.38700000000000001,"5min":0.32400000000000001,"15min":0.315},"max":{"1min":3.226,"5min":79.328000000000003,"15min":79.328000000000003},"last":0.5},{"interface":"front","average":{"1min":1.8240000000000001,"5min":4.327,"15min":2.472},"min":{"1min":0.32800000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":7.7069999999999999,"5min":79.209999999999994,"15min":79.209999999999994},"last":1.137}]},{"osd":4,"last update":"Mon Oct 9 05:15:30 2023","interfaces":[{"interface":"back","average":{"1min":1.6279999999999999,"5min":1.0660000000000001,"15min":1.1899999999999999},"min":{"1min":0.42799999999999999,"5min":0.30299999999999999,"15min":0.28199999999999997},"max":{"1min":8.3670000000000009,"5min":8.3670000000000009,"15min":16.119},"last":0.57299999999999995},{"interface":"front","average":{"1min":1.839,"5min":1.1539999999999999,"15min":1.2749999999999999},"min":{"1min":0.35399999999999998,"5min":0.34100000000000003,"15min":0.247},"max":{"1min":13.473000000000001,"5min":13.473000000000001,"15min":13.473000000000001},"last":1.228}]},{"osd":5,"last update":"Mon Oct 9 05:15:30 2023","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":1.1379999999999999,"15min":1.3200000000000001},"min":{"1min":0.42399999999999999,"5min":0.36399999999999999,"15min":0.29299999999999998},"max":{"1min":3.1440000000000001,"5min":6.9210000000000003,"15min":7.7149999999999999},"last":0.62},{"interface":"front","average":{"1min":1.524,"5min":1.222,"15min":1.4950000000000001},"min":{"1min":0.378,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":13.292999999999999,"5min":13.292999999999999,"15min":13.292999999999999},"last":1.4410000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:16:02 2023","interfaces":[{"interface":"back","average":{"1min":1.627,"5min":1.387,"15min":1.357},"min":{"1min":0.40799999999999997,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":7.8259999999999996,"5min":7.8259999999999996,"15min":9.3780000000000001},"last":1.1830000000000001},{"interface":"front","average":{"1min":1.9490000000000001,"5min":1.5569999999999999,"15min":1.4790000000000001},"min":{"1min":0.47699999999999998,"5min":0.441,"15min":0.434},"max":{"1min":7.8760000000000003,"5min":7.8760000000000003,"15min":9.4060000000000006},"last":1.2490000000000001}]}]},{"osd":1,"up_from":96,"seq":412316860639,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2155488,"kb_used_data":1503832,"kb_used_omap":4,"kb_used_meta":651643,"kb_avail":91589664,"statfs":{"total":95995035648,"available":93787815936,"internally_reserved":0,"allocated":1539923968,"data_stored":1538818568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":667283342},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:15:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0289999999999999,"5min":1.3759999999999999,"15min":1.5429999999999999},"min":{"1min":0.20100000000000001,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":5.6280000000000001,"5min":7.1900000000000004,"15min":22.382999999999999},"last":3.9300000000000002},{"interface":"front","average":{"1min":2.3260000000000001,"5min":1.653,"15min":1.7549999999999999},"min":{"1min":0.31,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":6.1059999999999999,"5min":7.0259999999999998,"15min":10.268000000000001},"last":4.4029999999999996}]},{"osd":2,"last update":"Mon Oct 9 05:15:32 2023","interfaces":[{"interface":"back","average":{"1min":1.3919999999999999,"5min":1.171,"15min":1.3300000000000001},"min":{"1min":0.23699999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":4.3479999999999999,"5min":5.0510000000000002,"15min":10.115},"last":0.377},{"interface":"front","average":{"1min":1.387,"5min":1.329,"15min":1.417},"min":{"1min":0.29899999999999999,"5min":0.27700000000000002,"15min":0.252},"max":{"1min":4.9349999999999996,"5min":9.9429999999999996,"15min":11.917},"last":0.437}]},{"osd":3,"last update":"Mon Oct 9 05:15:32 2023","interfaces":[{"interface":"back","average":{"1min":2.2360000000000002,"5min":2.5950000000000002,"15min":2.181},"min":{"1min":0.33600000000000002,"5min":0.24099999999999999,"15min":0.23100000000000001},"max":{"1min":11.029999999999999,"5min":61.280000000000001,"15min":61.280000000000001},"last":1.6399999999999999},{"interface":"front","average":{"1min":1.163,"5min":2.5,"15min":1.9610000000000001},"min":{"1min":0.34499999999999997,"5min":0.26800000000000002,"15min":0.24399999999999999},"max":{"1min":3.911,"5min":65.046000000000006,"15min":65.046000000000006},"last":0.89300000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.6579999999999999,"5min":1.4430000000000001,"15min":1.4019999999999999},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.34300000000000003},"max":{"1min":9.4830000000000005,"5min":9.4830000000000005,"15min":9.9749999999999996},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.9710000000000001,"5min":1.5489999999999999,"15min":1.5169999999999999},"min":{"1min":0.33200000000000002,"5min":0.32100000000000001,"15min":0.30099999999999999},"max":{"1min":9.6519999999999992,"5min":10.944000000000001,"15min":22.350000000000001},"last":3.9049999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.6439999999999999,"5min":1.9159999999999999,"15min":1.925},"min":{"1min":0.46400000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.2439999999999998,"5min":9.0530000000000008,"15min":12.454000000000001},"last":1.6160000000000001},{"interface":"front","average":{"1min":1.129,"5min":1.54,"15min":1.6439999999999999},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.316},"max":{"1min":3.2120000000000002,"5min":6.6660000000000004,"15min":10.035},"last":0.49299999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.4239999999999999,"5min":1.5600000000000001,"15min":1.8149999999999999},"min":{"1min":0.40699999999999997,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.1509999999999998,"5min":8.5630000000000006,"15min":10.409000000000001},"last":1.581},{"interface":"front","average":{"1min":1.409,"5min":1.508,"15min":1.573},"min":{"1min":0.373,"5min":0.373,"15min":0.35099999999999998},"max":{"1min":3.6419999999999999,"5min":8.9540000000000006,"15min":10.007},"last":0.71599999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":1.802,"15min":1.6890000000000001},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.40699999999999997},"max":{"1min":3.2349999999999999,"5min":11.282,"15min":11.282},"last":0.51600000000000001},{"interface":"front","average":{"1min":1.393,"5min":1.724,"15min":1.71},"min":{"1min":0.41499999999999998,"5min":0.39000000000000001,"15min":0.36399999999999999},"max":{"1min":3.944,"5min":11.167,"15min":22.175999999999998},"last":3.8650000000000002}]}]},{"osd":0,"up_from":88,"seq":377957122280,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3838084,"kb_used_data":3244732,"kb_used_omap":5,"kb_used_meta":593338,"kb_avail":89907068,"statfs":{"total":95995035648,"available":92064837632,"internally_reserved":0,"allocated":3322605568,"data_stored":3321525504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":607578746},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:15:14 2023","interfaces":[{"interface":"back","average":{"1min":1.3999999999999999,"5min":3.274,"15min":2.1619999999999999},"min":{"1min":0.29499999999999998,"5min":0.214,"15min":0.214},"max":{"1min":4.968,"5min":109.88,"15min":109.88},"last":0.33200000000000002},{"interface":"front","average":{"1min":1.2270000000000001,"5min":3.4940000000000002,"15min":2.242},"min":{"1min":0.222,"5min":0.222,"15min":0.218},"max":{"1min":3.7000000000000002,"5min":109.435,"15min":109.435},"last":1.702}]},{"osd":2,"last update":"Mon Oct 9 05:15:27 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":0.98399999999999999,"15min":1.329},"min":{"1min":0.313,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":3.7599999999999998,"5min":10.609999999999999,"15min":10.609999999999999},"last":1.647},{"interface":"front","average":{"1min":2.4350000000000001,"5min":1.28,"15min":1.26},"min":{"1min":0.248,"5min":0.248,"15min":0.23300000000000001},"max":{"1min":11.622,"5min":11.622,"15min":11.622},"last":0.51800000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:15:32 2023","interfaces":[{"interface":"back","average":{"1min":1.8080000000000001,"5min":2.2120000000000002,"15min":1.7549999999999999},"min":{"1min":0.30199999999999999,"5min":0.223,"15min":0.223},"max":{"1min":11.137,"5min":40.649999999999999,"15min":40.649999999999999},"last":3.5960000000000001},{"interface":"front","average":{"1min":2.524,"5min":2.3050000000000002,"15min":1.7709999999999999},"min":{"1min":0.316,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":12.52,"5min":42.758000000000003,"15min":42.758000000000003},"last":2.5339999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:15:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":1.1890000000000001,"15min":1.2849999999999999},"min":{"1min":0.47099999999999997,"5min":0.36499999999999999,"15min":0.32600000000000001},"max":{"1min":3.5329999999999999,"5min":10.632999999999999,"15min":10.632999999999999},"last":1.679},{"interface":"front","average":{"1min":2.1819999999999999,"5min":1.4119999999999999,"15min":1.385},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":5.8150000000000004,"5min":6.4649999999999999,"15min":7.202},"last":1.1459999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:15:32 2023","interfaces":[{"interface":"back","average":{"1min":1.371,"5min":1.478,"15min":1.476},"min":{"1min":0.42499999999999999,"5min":0.34699999999999998,"15min":0.29899999999999999},"max":{"1min":8.7149999999999999,"5min":8.7149999999999999,"15min":11.577999999999999},"last":0.57399999999999995},{"interface":"front","average":{"1min":1.5149999999999999,"5min":1.5600000000000001,"15min":1.623},"min":{"1min":0.375,"5min":0.36699999999999999,"15min":0.316},"max":{"1min":5.6280000000000001,"5min":10.676,"15min":10.676},"last":1.7250000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:15:32 2023","interfaces":[{"interface":"back","average":{"1min":2.5960000000000001,"5min":1.514,"15min":1.53},"min":{"1min":0.432,"5min":0.42599999999999999,"15min":0.371},"max":{"1min":6.218,"5min":6.79,"15min":8.8710000000000004},"last":0.75700000000000001},{"interface":"front","average":{"1min":2.3220000000000001,"5min":1.589,"15min":1.4450000000000001},"min":{"1min":0.45500000000000002,"5min":0.40400000000000003,"15min":0.36499999999999999},"max":{"1min":8.4580000000000002,"5min":8.4580000000000002,"15min":10.429},"last":0.77500000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:15:32 2023","interfaces":[{"interface":"back","average":{"1min":1.6339999999999999,"5min":1.8089999999999999,"15min":1.5309999999999999},"min":{"1min":0.48599999999999999,"5min":0.39800000000000002,"15min":0.36599999999999999},"max":{"1min":8.75,"5min":8.75,"15min":10.314},"last":0.74099999999999999},{"interface":"front","average":{"1min":1.327,"5min":1.5429999999999999,"15min":1.6060000000000001},"min":{"1min":0.46600000000000003,"5min":0.42399999999999999,"15min":0.41699999999999998},"max":{"1min":3.5899999999999999,"5min":10.694000000000001,"15min":10.694000000000001},"last":6.9950000000000001}]}]},{"osd":2,"up_from":106,"seq":455266533558,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2796048,"kb_used_data":2415688,"kb_used_omap":7,"kb_used_meta":380344,"kb_avail":90949104,"statfs":{"total":95995035648,"available":93131882496,"internally_reserved":0,"allocated":2473664512,"data_stored":2472633332,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":389472337},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.7989999999999999,"5min":1.5589999999999999,"15min":1.335},"min":{"1min":0.378,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":11.949,"5min":53.088000000000001,"15min":53.088000000000001},"last":15.099},{"interface":"front","average":{"1min":2.032,"5min":1.6080000000000001,"15min":1.4690000000000001},"min":{"1min":0.29499999999999998,"5min":0.215,"15min":0.215},"max":{"1min":9.1519999999999992,"5min":46.762,"15min":46.762},"last":0.83399999999999996}]},{"osd":1,"last update":"Mon Oct 9 05:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":2.488,"15min":1.657},"min":{"1min":0.372,"5min":0.251,"15min":0.251},"max":{"1min":4.3280000000000003,"5min":68.933999999999997,"15min":68.933999999999997},"last":7.7009999999999996},{"interface":"front","average":{"1min":1.4990000000000001,"5min":2.6699999999999999,"15min":1.883},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.23100000000000001},"max":{"1min":4.327,"5min":68.846000000000004,"15min":68.846000000000004},"last":0.58199999999999996}]},{"osd":3,"last update":"Mon Oct 9 05:15:37 2023","interfaces":[{"interface":"back","average":{"1min":1.7989999999999999,"5min":1.948,"15min":1.7430000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.27300000000000002},"max":{"1min":4.3840000000000003,"5min":68.968000000000004,"15min":68.968000000000004},"last":1.2250000000000001},{"interface":"front","average":{"1min":1.218,"5min":1.7490000000000001,"15min":1.595},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.2290000000000001,"5min":68.891999999999996,"15min":68.891999999999996},"last":0.65500000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.796,"5min":1.03,"15min":1.296},"min":{"1min":0.46800000000000003,"5min":0.34000000000000002,"15min":0.33300000000000002},"max":{"1min":6.1909999999999998,"5min":6.1909999999999998,"15min":20.712},"last":0.79300000000000004},{"interface":"front","average":{"1min":1.3899999999999999,"5min":1.024,"15min":1.3500000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":4.2679999999999998,"5min":4.2679999999999998,"15min":23.547000000000001},"last":0.61399999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.3340000000000001,"5min":1.2290000000000001,"15min":1.4850000000000001},"min":{"1min":0.379,"5min":0.35799999999999998,"15min":0.35399999999999998},"max":{"1min":4.9249999999999998,"5min":5.532,"15min":20.782},"last":2.2370000000000001},{"interface":"front","average":{"1min":1.484,"5min":1.4630000000000001,"15min":1.6859999999999999},"min":{"1min":0.434,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":4.5039999999999996,"5min":7.391,"15min":20.792000000000002},"last":0.76100000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.7829999999999999,"5min":1.2150000000000001,"15min":1.4319999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.34999999999999998},"max":{"1min":6.5800000000000001,"5min":6.5800000000000001,"15min":23.510999999999999},"last":0.63800000000000001},{"interface":"front","average":{"1min":1.8919999999999999,"5min":1.2490000000000001,"15min":1.4059999999999999},"min":{"1min":0.50900000000000001,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":5.0179999999999998,"5min":6.3390000000000004,"15min":20.684999999999999},"last":2.3519999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.536,"5min":1.278,"15min":1.4019999999999999},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44},"max":{"1min":4.9720000000000004,"5min":4.9720000000000004,"15min":20.887},"last":2.2789999999999999},{"interface":"front","average":{"1min":1.625,"5min":1.345,"15min":1.472},"min":{"1min":0.51400000000000001,"5min":0.50900000000000001,"15min":0.45400000000000001},"max":{"1min":4.9000000000000004,"5min":4.9000000000000004,"15min":20.863},"last":0.69799999999999995}]}]},{"osd":3,"up_from":123,"seq":528280977577,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2772236,"kb_used_data":2244804,"kb_used_omap":2,"kb_used_meta":527421,"kb_avail":90972916,"statfs":{"total":95995035648,"available":93156265984,"internally_reserved":0,"allocated":2298679296,"data_stored":2297706159,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":540079157},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:15:34 2023","interfaces":[{"interface":"back","average":{"1min":5.75,"5min":2.5640000000000001,"15min":1.927},"min":{"1min":0.20499999999999999,"5min":0.20000000000000001,"15min":0.187},"max":{"1min":29.814,"5min":29.814,"15min":29.814},"last":3.8599999999999999},{"interface":"front","average":{"1min":6.9470000000000001,"5min":2.4399999999999999,"15min":2.0659999999999998},"min":{"1min":0.22600000000000001,"5min":0.188,"15min":0.184},"max":{"1min":36.302,"5min":36.302,"15min":36.302},"last":4.093}]},{"osd":1,"last update":"Mon Oct 9 05:15:34 2023","interfaces":[{"interface":"back","average":{"1min":1.5880000000000001,"5min":2.27,"15min":1.946},"min":{"1min":0.24299999999999999,"5min":0.222,"15min":0.222},"max":{"1min":7.2130000000000001,"5min":44.206000000000003,"15min":44.206000000000003},"last":3.8759999999999999},{"interface":"front","average":{"1min":1.4590000000000001,"5min":2.4540000000000002,"15min":1.944},"min":{"1min":0.218,"5min":0.218,"15min":0.20999999999999999},"max":{"1min":7.6360000000000001,"5min":44.496000000000002,"15min":44.496000000000002},"last":3.5470000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:15:34 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":1.429,"15min":1.5820000000000001},"min":{"1min":0.254,"5min":0.246,"15min":0.23699999999999999},"max":{"1min":7.4050000000000002,"5min":9.5730000000000004,"15min":9.6129999999999995},"last":5.2290000000000001},{"interface":"front","average":{"1min":2.298,"5min":1.4570000000000001,"15min":1.5960000000000001},"min":{"1min":0.27300000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":8.7309999999999999,"5min":9.0310000000000006,"15min":14.754},"last":6.0519999999999996}]},{"osd":4,"last update":"Mon Oct 9 05:15:34 2023","interfaces":[{"interface":"back","average":{"1min":1.609,"5min":1.268,"15min":1.47},"min":{"1min":0.33400000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.3040000000000003,"5min":7.3040000000000003,"15min":14.833},"last":3.9729999999999999},{"interface":"front","average":{"1min":1.246,"5min":1.417,"15min":1.3260000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":6.056,"5min":9.6709999999999994,"15min":9.6709999999999994},"last":3.7999999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:15:34 2023","interfaces":[{"interface":"back","average":{"1min":1.508,"5min":1.5449999999999999,"15min":1.4490000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":6.6289999999999996,"5min":9.6189999999999998,"15min":9.6189999999999998},"last":3.7669999999999999},{"interface":"front","average":{"1min":1.4890000000000001,"5min":1.772,"15min":1.8140000000000001},"min":{"1min":0.39300000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":7.54,"5min":10.456,"15min":10.456},"last":4.0199999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:15:34 2023","interfaces":[{"interface":"back","average":{"1min":1.9059999999999999,"5min":1.669,"15min":1.675},"min":{"1min":0.38500000000000001,"5min":0.36099999999999999,"15min":0.33200000000000002},"max":{"1min":7.4989999999999997,"5min":8.0899999999999999,"15min":9.6590000000000007},"last":3.9910000000000001},{"interface":"front","average":{"1min":2.4550000000000001,"5min":1.643,"15min":1.762},"min":{"1min":0.48699999999999999,"5min":0.36899999999999999,"15min":0.34699999999999998},"max":{"1min":7.2530000000000001,"5min":7.2530000000000001,"15min":14.881},"last":5.9939999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:15:34 2023","interfaces":[{"interface":"back","average":{"1min":1.5740000000000001,"5min":1.4550000000000001,"15min":1.792},"min":{"1min":0.41799999999999998,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":7.282,"5min":7.282,"15min":14.856},"last":6.0289999999999999},{"interface":"front","average":{"1min":1.597,"5min":1.788,"15min":1.6439999999999999},"min":{"1min":0.41099999999999998,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":6.0030000000000001,"5min":9.6969999999999992,"15min":9.6969999999999992},"last":5.2830000000000004}]}]},{"osd":4,"up_from":30,"seq":128849019334,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4458816,"kb_used_data":3565944,"kb_used_omap":0,"kb_used_meta":892544,"kb_avail":89286336,"statfs":{"total":95995035648,"available":91429208064,"internally_reserved":0,"allocated":3651526656,"data_stored":3650796631,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":913965056},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:15:28 2023","interfaces":[{"interface":"back","average":{"1min":1.5600000000000001,"5min":2.1640000000000001,"15min":2.0409999999999999},"min":{"1min":0.316,"5min":0.28399999999999997,"15min":0.254},"max":{"1min":8.548,"5min":67.224999999999994,"15min":102.669},"last":4.1779999999999999},{"interface":"front","average":{"1min":1.1819999999999999,"5min":2.073,"15min":1.972},"min":{"1min":0.308,"5min":0.308,"15min":0.26900000000000002},"max":{"1min":6.3730000000000002,"5min":67.191000000000003,"15min":102.72499999999999},"last":4.1609999999999996}]},{"osd":1,"last update":"Mon Oct 9 05:15:15 2023","interfaces":[{"interface":"back","average":{"1min":1.6160000000000001,"5min":1.413,"15min":1.7110000000000001},"min":{"1min":0.315,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.4269999999999996,"5min":10.387,"15min":59.408000000000001},"last":3.883},{"interface":"front","average":{"1min":1.121,"5min":1.2849999999999999,"15min":1.625},"min":{"1min":0.42899999999999999,"5min":0.36299999999999999,"15min":0.28199999999999997},"max":{"1min":2.8359999999999999,"5min":10.305999999999999,"15min":60.207000000000001},"last":1.2629999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:15:28 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":1.651,"15min":1.4319999999999999},"min":{"1min":0.41599999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.8780000000000001,"5min":66.201999999999998,"15min":66.201999999999998},"last":1.3049999999999999},{"interface":"front","average":{"1min":1.2569999999999999,"5min":1.6850000000000001,"15min":1.341},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":4.3419999999999996,"5min":66.361000000000004,"15min":66.361000000000004},"last":4.0819999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:15:36 2023","interfaces":[{"interface":"back","average":{"1min":1.5920000000000001,"5min":3.5019999999999998,"15min":2.2360000000000002},"min":{"1min":0.495,"5min":0.36399999999999999,"15min":0.309},"max":{"1min":8.5869999999999997,"5min":68,"15min":68},"last":1.335},{"interface":"front","average":{"1min":1.0309999999999999,"5min":3.113,"15min":2.0630000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35399999999999998},"max":{"1min":4.0949999999999998,"5min":67.775000000000006,"15min":67.775000000000006},"last":4.5609999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:15:36 2023","interfaces":[{"interface":"back","average":{"1min":0.78900000000000003,"5min":0.92400000000000004,"15min":1.177},"min":{"1min":0.42299999999999999,"5min":0.33400000000000002,"15min":0.314},"max":{"1min":2.9260000000000002,"5min":4.21,"15min":10.214},"last":1.3620000000000001},{"interface":"front","average":{"1min":0.72099999999999997,"5min":1.1890000000000001,"15min":1.29},"min":{"1min":0.435,"5min":0.34799999999999998,"15min":0.31900000000000001},"max":{"1min":1.77,"5min":6.0460000000000003,"15min":6.8310000000000004},"last":0.75600000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:15:15 2023","interfaces":[{"interface":"back","average":{"1min":1.718,"5min":1.0860000000000001,"15min":1.3300000000000001},"min":{"1min":0.40600000000000003,"5min":0.34200000000000003,"15min":0.32800000000000001},"max":{"1min":5.258,"5min":5.258,"15min":11.557},"last":0.97899999999999998},{"interface":"front","average":{"1min":1.264,"5min":1.022,"15min":1.256},"min":{"1min":0.47499999999999998,"5min":0.39800000000000002,"15min":0.34699999999999998},"max":{"1min":8.4930000000000003,"5min":8.4930000000000003,"15min":10.689},"last":1.552}]},{"osd":7,"last update":"Mon Oct 9 05:15:15 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":1.0369999999999999,"15min":1.1299999999999999},"min":{"1min":0.44500000000000001,"5min":0.41499999999999998,"15min":0.36299999999999999},"max":{"1min":1.264,"5min":4.3799999999999999,"15min":6.4820000000000002},"last":0.95599999999999996},{"interface":"front","average":{"1min":1.3620000000000001,"5min":1.492,"15min":1.407},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":5.3019999999999996,"5min":8.9689999999999994,"15min":8.9689999999999994},"last":1.0189999999999999}]}]},{"osd":5,"up_from":0,"seq":163208757690,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3302559744,"data_stored":3302555672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1532559360,"data_stored":1532559360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2469986304,"data_stored":2469986304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2283798528,"data_stored":2283798528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3632467968,"data_stored":3632463896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2212040704,"data_stored":2212036632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":243269632,"data_stored":243269632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:16:08.367 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_5 pgp_num 16 2023-10-09T05:16:08.451 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:08.486 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:08.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:08.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:08.592 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:08.618 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:08.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:08.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:08.726 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:08.758 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:08.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:08.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:08.776 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:16:08.776 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:16:08.862 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:08.889 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:08.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:08.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:08.996 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:09.027 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:09.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:09.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:09.130 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:09.157 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:09.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:09.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:09.260 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:09.291 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:09.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:09.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:09.396 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:09.425 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:09.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:09.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:09.529 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:09.537 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:09 smithi194 bash[92666]: audit 2023-10-09T05:16:07.290565+0000 mgr.y (mgr.25277) 828 : audit 0 from='client.45032 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:16:09.537 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:09 smithi194 bash[92666]: cluster 2023-10-09T05:16:07.918538+0000 mgr.y (mgr.25277) 829 : cluster 0 pgmap v843: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 678 op/s; 11543/232140 objects degraded (4.972%); 2289/232140 objects misplaced (0.986%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T05:16:09.537 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:09 smithi194 bash[92666]: cluster 2023-10-09T05:16:08.337415+0000 mon.a (mon.0) 1084 : cluster 3 Health check update: Degraded data redundancy: 11543/232140 objects degraded (4.972%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:09.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:09 smithi133 bash[102858]: audit 2023-10-09T05:16:07.290565+0000 mgr.y (mgr.25277) 828 : audit 0 from='client.45032 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:16:09.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:09 smithi133 bash[102858]: cluster 2023-10-09T05:16:07.918538+0000 mgr.y (mgr.25277) 829 : cluster 0 pgmap v843: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 678 op/s; 11543/232140 objects degraded (4.972%); 2289/232140 objects misplaced (0.986%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T05:16:09.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:09 smithi133 bash[102858]: cluster 2023-10-09T05:16:08.337415+0000 mon.a (mon.0) 1084 : cluster 3 Health check update: Degraded data redundancy: 11543/232140 objects degraded (4.972%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:09.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:09 smithi133 bash[105589]: audit 2023-10-09T05:16:07.290565+0000 mgr.y (mgr.25277) 828 : audit 0 from='client.45032 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:16:09.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:09 smithi133 bash[105589]: cluster 2023-10-09T05:16:07.918538+0000 mgr.y (mgr.25277) 829 : cluster 0 pgmap v843: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 678 op/s; 11543/232140 objects degraded (4.972%); 2289/232140 objects misplaced (0.986%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T05:16:09.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:09 smithi133 bash[105589]: cluster 2023-10-09T05:16:08.337415+0000 mon.a (mon.0) 1084 : cluster 3 Health check update: Degraded data redundancy: 11543/232140 objects degraded (4.972%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:09.552 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:09.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:09.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:09.656 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:09.694 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:09.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:09.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:09.797 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:09.837 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:09.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:09.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:09.941 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:09.975 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:09.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:09.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:10.082 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:10.108 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:10.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:10.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:10.220 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:10.260 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:10.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:10.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:10.363 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:10.386 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:10.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:10.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:10.490 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:10.516 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:10.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:10.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:10.619 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:10.644 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:10.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:10.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:10.749 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:10.775 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:10.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:10.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:10.881 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:10.906 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:10.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:10.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:11.013 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:11.041 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:11.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:11.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:11.146 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:11.180 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:11.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:11.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:11.284 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:11.321 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:11.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:11.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:11.425 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:11.441 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:11 smithi133 bash[105589]: cluster 2023-10-09T05:16:09.919207+0000 mgr.y (mgr.25277) 830 : cluster 0 pgmap v844: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 733 op/s; 11543/227658 objects degraded (5.070%); 2289/227658 objects misplaced (1.005%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T05:16:11.442 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:11 smithi133 bash[102858]: cluster 2023-10-09T05:16:09.919207+0000 mgr.y (mgr.25277) 830 : cluster 0 pgmap v844: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 733 op/s; 11543/227658 objects degraded (5.070%); 2289/227658 objects misplaced (1.005%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T05:16:11.449 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:11.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:11.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:11.554 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:11.588 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:11.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:11.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:11.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:11 smithi194 bash[92666]: cluster 2023-10-09T05:16:09.919207+0000 mgr.y (mgr.25277) 830 : cluster 0 pgmap v844: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 733 op/s; 11543/227658 objects degraded (5.070%); 2289/227658 objects misplaced (1.005%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T05:16:11.694 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:11.731 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:11.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:11.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:11.834 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:11.865 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:11.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:11.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:11.908 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:16:11.968 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:12.001 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:12.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:12.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:12.105 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:12.133 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:12.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:12.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:12.237 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:12.270 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:12.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:12.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:12.374 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:12.406 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:12.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:12.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:12.513 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:12.537 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:12.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:12.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:12.641 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:12.676 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:12.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:12.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:12.780 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:12.811 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:12.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:12.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:12.914 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:12.952 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:12.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:12.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:13.058 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:13.094 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:13.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:13.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:13.193 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:13 smithi133 bash[102858]: cluster 2023-10-09T05:16:11.919784+0000 mgr.y (mgr.25277) 831 : cluster 0 pgmap v845: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 773 op/s; 11543/221301 objects degraded (5.216%); 1998/221301 objects misplaced (0.903%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T05:16:13.200 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:13.224 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:13.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:13.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:13.327 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:13.335 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:13 smithi194 bash[92666]: cluster 2023-10-09T05:16:11.919784+0000 mgr.y (mgr.25277) 831 : cluster 0 pgmap v845: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 773 op/s; 11543/221301 objects degraded (5.216%); 1998/221301 objects misplaced (0.903%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T05:16:13.355 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:13.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:13.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:13.458 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:13.467 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:13 smithi133 bash[105589]: cluster 2023-10-09T05:16:11.919784+0000 mgr.y (mgr.25277) 831 : cluster 0 pgmap v845: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 773 op/s; 11543/221301 objects degraded (5.216%); 1998/221301 objects misplaced (0.903%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T05:16:13.488 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:13.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:13.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:13.590 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:13.615 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:13.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:13.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:13.718 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:13.741 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:13.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:13.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:13.844 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:13.874 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:13.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:13.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:13.978 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:14.018 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:14.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:14.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:14.124 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:14.161 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:14.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:14.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:14.264 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:14.273 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:14 smithi133 bash[102858]: cluster 2023-10-09T05:16:13.340178+0000 mon.a (mon.0) 1085 : cluster 3 Health check update: Degraded data redundancy: 11543/221301 objects degraded (5.216%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:14.274 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:14 smithi133 bash[102858]: audit 2023-10-09T05:16:13.449350+0000 mon.a (mon.0) 1086 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:16:14.274 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:14 smithi133 bash[105589]: cluster 2023-10-09T05:16:13.340178+0000 mon.a (mon.0) 1085 : cluster 3 Health check update: Degraded data redundancy: 11543/221301 objects degraded (5.216%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:14.274 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:14 smithi133 bash[105589]: audit 2023-10-09T05:16:13.449350+0000 mon.a (mon.0) 1086 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:16:14.291 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:14.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:14.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:14.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:14 smithi194 bash[92666]: cluster 2023-10-09T05:16:13.340178+0000 mon.a (mon.0) 1085 : cluster 3 Health check update: Degraded data redundancy: 11543/221301 objects degraded (5.216%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:14.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:14 smithi194 bash[92666]: audit 2023-10-09T05:16:13.449350+0000 mon.a (mon.0) 1086 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:16:14.396 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:14.426 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:14.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:14.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:14.531 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:14.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:16:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:16:14] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.43.0" 2023-10-09T05:16:14.556 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:14.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:14.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:14.660 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:14.698 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:14.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:14.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:14.805 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:14.825 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:16:14.837 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:14.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:14.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:14.941 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:14.967 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:14.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:14.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:15.070 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:15.096 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:15.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:15.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:15.204 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:15.213 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:15 smithi194 bash[92666]: cluster 2023-10-09T05:16:13.920376+0000 mgr.y (mgr.25277) 832 : cluster 0 pgmap v846: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 614 op/s; 11543/220590 objects degraded (5.233%); 1998/220590 objects misplaced (0.906%); 3.1 MiB/s, 48 objects/s recovering 2023-10-09T05:16:15.230 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:15.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:15.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:15.337 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:15.346 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:15 smithi133 bash[105589]: cluster 2023-10-09T05:16:13.920376+0000 mgr.y (mgr.25277) 832 : cluster 0 pgmap v846: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 614 op/s; 11543/220590 objects degraded (5.233%); 1998/220590 objects misplaced (0.906%); 3.1 MiB/s, 48 objects/s recovering 2023-10-09T05:16:15.346 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:15 smithi133 bash[102858]: cluster 2023-10-09T05:16:13.920376+0000 mgr.y (mgr.25277) 832 : cluster 0 pgmap v846: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 614 op/s; 11543/220590 objects degraded (5.233%); 1998/220590 objects misplaced (0.906%); 3.1 MiB/s, 48 objects/s recovering 2023-10-09T05:16:15.362 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:15.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:15.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:15.467 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:15.497 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:15.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:15.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:15.601 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:15.632 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:15.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:15.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:15.740 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:15.765 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:15.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:15.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:15.869 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:15.889 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:15.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:15.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:15.993 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:16.021 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:16.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:16.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:16.126 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:16.150 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:16.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:16.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:16.257 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:16.266 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:16 smithi194 bash[92666]: audit 2023-10-09T05:16:16.087036+0000 mon.a (mon.0) 1087 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:16:16.293 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:16.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:16.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:16.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:16 smithi133 bash[105589]: audit 2023-10-09T05:16:16.087036+0000 mon.a (mon.0) 1087 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:16:16.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:16 smithi133 bash[102858]: audit 2023-10-09T05:16:16.087036+0000 mon.a (mon.0) 1087 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:16:16.396 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:16.414 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:16.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:16.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:16.517 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:16.554 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:16.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:16.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:16.659 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:16.694 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:16.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:16.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:16.799 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:16.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:16.847 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:16.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:16.947 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:16.986 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:16.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:16.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:17.092 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:17.126 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:17.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:17.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:17.212 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:17 smithi133 bash[105589]: cluster 2023-10-09T05:16:15.921612+0000 mgr.y (mgr.25277) 833 : cluster 0 pgmap v847: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 845 op/s; 11543/211611 objects degraded (5.455%); 1673/211611 objects misplaced (0.791%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T05:16:17.231 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:17.266 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:17.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:17.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:17.373 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:17.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:17 smithi194 bash[92666]: cluster 2023-10-09T05:16:15.921612+0000 mgr.y (mgr.25277) 833 : cluster 0 pgmap v847: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 845 op/s; 11543/211611 objects degraded (5.455%); 1673/211611 objects misplaced (0.791%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T05:16:17.403 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:17.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:17.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:17.481 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:17 smithi133 bash[102858]: cluster 2023-10-09T05:16:15.921612+0000 mgr.y (mgr.25277) 833 : cluster 0 pgmap v847: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 845 op/s; 11543/211611 objects degraded (5.455%); 1673/211611 objects misplaced (0.791%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T05:16:17.508 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:17.533 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:17.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:17.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:17.639 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:17.680 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:17.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:17.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:17.788 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:17.818 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:17.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:17.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:17.925 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:17.951 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:17.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:17.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:18.054 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:18.093 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:18.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:18.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:18.200 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:18.229 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:18.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:18.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:18.333 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:18.357 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:18.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:18.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:18.467 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:18.499 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:18.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:18.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:18.606 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:18.635 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:18.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:18.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:18.744 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:18.780 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:18.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:18.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:18.884 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:18.911 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:18.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:18.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:19.018 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:19.052 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:19.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:19.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:19.156 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:19.182 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:19.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:19.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:19.285 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:19.313 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:19.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:19.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:19.336 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:19 smithi133 bash[102858]: cluster 2023-10-09T05:16:17.922230+0000 mgr.y (mgr.25277) 834 : cluster 0 pgmap v848: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 647 op/s; 11543/210258 objects degraded (5.490%); 1673/210258 objects misplaced (0.796%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T05:16:19.336 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:19 smithi133 bash[102858]: cluster 2023-10-09T05:16:18.341490+0000 mon.a (mon.0) 1088 : cluster 3 Health check update: Degraded data redundancy: 11543/210258 objects degraded (5.490%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:19.336 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:19 smithi133 bash[105589]: cluster 2023-10-09T05:16:17.922230+0000 mgr.y (mgr.25277) 834 : cluster 0 pgmap v848: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 647 op/s; 11543/210258 objects degraded (5.490%); 1673/210258 objects misplaced (0.796%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T05:16:19.336 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:19 smithi133 bash[105589]: cluster 2023-10-09T05:16:18.341490+0000 mon.a (mon.0) 1088 : cluster 3 Health check update: Degraded data redundancy: 11543/210258 objects degraded (5.490%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:19.418 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:19.455 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:19.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:19.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:19.560 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:19.592 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:19.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:19.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:19.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:19 smithi194 bash[92666]: cluster 2023-10-09T05:16:17.922230+0000 mgr.y (mgr.25277) 834 : cluster 0 pgmap v848: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 647 op/s; 11543/210258 objects degraded (5.490%); 1673/210258 objects misplaced (0.796%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T05:16:19.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:19 smithi194 bash[92666]: cluster 2023-10-09T05:16:18.341490+0000 mon.a (mon.0) 1088 : cluster 3 Health check update: Degraded data redundancy: 11543/210258 objects degraded (5.490%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:19.698 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:19.738 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:19.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:19.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:19.842 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:19.877 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:19.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:19.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:19.981 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:20.016 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:20.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:20.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:20.123 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:20.152 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:20.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:20.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:20.256 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:20.291 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:20.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:20.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:20.394 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:20.418 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:20.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:20.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:20.522 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:20.549 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:20.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:20.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:20.654 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:20.682 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:20.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:20.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:20.785 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:20.812 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:20.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:20.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:20.916 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:20.945 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:20.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:20.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:21.048 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:21.083 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:21.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:21.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:21.189 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:21.216 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:21.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:21.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:21.319 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:21.327 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:21 smithi194 bash[92666]: cluster 2023-10-09T05:16:19.922859+0000 mgr.y (mgr.25277) 835 : cluster 0 pgmap v849: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 737 op/s; 11543/205581 objects degraded (5.615%); 1673/205581 objects misplaced (0.814%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T05:16:21.344 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:21.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:21.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:21.452 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:21.461 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:21 smithi133 bash[105589]: cluster 2023-10-09T05:16:19.922859+0000 mgr.y (mgr.25277) 835 : cluster 0 pgmap v849: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 737 op/s; 11543/205581 objects degraded (5.615%); 1673/205581 objects misplaced (0.814%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T05:16:21.461 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:21 smithi133 bash[102858]: cluster 2023-10-09T05:16:19.922859+0000 mgr.y (mgr.25277) 835 : cluster 0 pgmap v849: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 737 op/s; 11543/205581 objects degraded (5.615%); 1673/205581 objects misplaced (0.814%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T05:16:21.487 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:21.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:21.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:21.592 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:21.621 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:21.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:21.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:21.728 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:21.774 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:21.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:21.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:21.880 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:21.919 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:21.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:21.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:22.025 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:22.046 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:22.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:22.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:22.150 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:22.179 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:22.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:22.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:22.283 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:22.305 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:22.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:22.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:22.407 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:22.437 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:22.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:22.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:22.540 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:22.570 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:22.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:22.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:22.674 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:22.698 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:16:22.698 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:16:22.716 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:22.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:22.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:22.820 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:22.861 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:22.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:22.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:22.967 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:22.991 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:22.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:22.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:23.096 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:23.124 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:23.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:23.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:23.230 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:23.239 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:23 smithi194 bash[92666]: cluster 2023-10-09T05:16:21.923614+0000 mgr.y (mgr.25277) 836 : cluster 0 pgmap v850: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 784 op/s; 11543/199410 objects degraded (5.789%); 1334/199410 objects misplaced (0.669%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T05:16:23.239 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:23 smithi194 bash[92666]: audit 2023-10-09T05:16:22.791928+0000 mon.a (mon.0) 1089 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:16:23.239 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:23 smithi194 bash[92666]: audit 2023-10-09T05:16:22.799120+0000 mon.a (mon.0) 1090 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:16:23.260 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:23.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:23.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:23.363 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:23.394 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:23.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:23.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:23.500 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:23.526 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:23.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:23.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:23.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:23 smithi133 bash[105589]: cluster 2023-10-09T05:16:21.923614+0000 mgr.y (mgr.25277) 836 : cluster 0 pgmap v850: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 784 op/s; 11543/199410 objects degraded (5.789%); 1334/199410 objects misplaced (0.669%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T05:16:23.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:23 smithi133 bash[105589]: audit 2023-10-09T05:16:22.791928+0000 mon.a (mon.0) 1089 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:16:23.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:23 smithi133 bash[105589]: audit 2023-10-09T05:16:22.799120+0000 mon.a (mon.0) 1090 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:16:23.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:23 smithi133 bash[102858]: cluster 2023-10-09T05:16:21.923614+0000 mgr.y (mgr.25277) 836 : cluster 0 pgmap v850: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 784 op/s; 11543/199410 objects degraded (5.789%); 1334/199410 objects misplaced (0.669%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T05:16:23.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:23 smithi133 bash[102858]: audit 2023-10-09T05:16:22.791928+0000 mon.a (mon.0) 1089 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:16:23.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:23 smithi133 bash[102858]: audit 2023-10-09T05:16:22.799120+0000 mon.a (mon.0) 1090 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:16:23.629 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:23.655 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:23.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:23.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:23.758 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:23.791 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:23.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:23.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:23.897 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:23.930 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:23.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:23.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:24.035 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:24.058 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:24.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:24.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:24.163 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:24.200 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:24.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:24.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:24.239 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is set 2023-10-09T05:16:24.305 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:24.355 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:24.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:24.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:24.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:24 smithi194 bash[92666]: audit 2023-10-09T05:16:23.228052+0000 mon.c (mon.2) 80 : audit 1 from='client.? 172.21.15.133:0/721959286' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:16:24.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:24 smithi194 bash[92666]: audit 2023-10-09T05:16:23.228500+0000 mon.a (mon.0) 1091 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:16:24.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:24 smithi194 bash[92666]: cluster 2023-10-09T05:16:23.345023+0000 mon.a (mon.0) 1092 : cluster 3 Health check update: Degraded data redundancy: 11543/199410 objects degraded (5.789%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:24.458 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:24.482 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:24.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:24.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:24.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:16:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:16:24] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.43.0" 2023-10-09T05:16:24.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:24 smithi133 bash[105589]: audit 2023-10-09T05:16:23.228052+0000 mon.c (mon.2) 80 : audit 1 from='client.? 172.21.15.133:0/721959286' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:16:24.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:24 smithi133 bash[105589]: audit 2023-10-09T05:16:23.228500+0000 mon.a (mon.0) 1091 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:16:24.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:24 smithi133 bash[105589]: cluster 2023-10-09T05:16:23.345023+0000 mon.a (mon.0) 1092 : cluster 3 Health check update: Degraded data redundancy: 11543/199410 objects degraded (5.789%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:24.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:24 smithi133 bash[102858]: audit 2023-10-09T05:16:23.228052+0000 mon.c (mon.2) 80 : audit 1 from='client.? 172.21.15.133:0/721959286' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:16:24.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:24 smithi133 bash[102858]: audit 2023-10-09T05:16:23.228500+0000 mon.a (mon.0) 1091 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:16:24.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:24 smithi133 bash[102858]: cluster 2023-10-09T05:16:23.345023+0000 mon.a (mon.0) 1092 : cluster 3 Health check update: Degraded data redundancy: 11543/199410 objects degraded (5.789%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:24.588 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:24.615 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:24.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:24.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:24.721 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:24.757 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:24.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:24.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:24.860 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:24.895 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:24.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:24.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:24.998 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:25.041 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:25.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:25.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:25.147 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:25.178 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:25.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:25.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:25.283 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:25.300 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:25.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:25.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:25.403 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:25.415 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:25 smithi194 bash[92666]: cluster 2023-10-09T05:16:23.924248+0000 mgr.y (mgr.25277) 837 : cluster 0 pgmap v851: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 626 op/s; 11543/198747 objects degraded (5.808%); 1334/198747 objects misplaced (0.671%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T05:16:25.415 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:25 smithi194 bash[92666]: cluster 2023-10-09T05:16:24.223013+0000 mon.a (mon.0) 1093 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:16:25.415 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:25 smithi194 bash[92666]: cluster 2023-10-09T05:16:24.226148+0000 osd.7 (osd.7) 140 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T05:16:25.415 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:25 smithi194 bash[92666]: cluster 2023-10-09T05:16:24.230226+0000 osd.5 (osd.5) 192 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T05:16:25.415 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:25 smithi194 bash[92666]: cluster 2023-10-09T05:16:24.233779+0000 osd.6 (osd.6) 142 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T05:16:25.416 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:25 smithi194 bash[92666]: audit 2023-10-09T05:16:24.234664+0000 mon.a (mon.0) 1094 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:16:25.416 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:25 smithi194 bash[92666]: cluster 2023-10-09T05:16:24.234813+0000 mon.a (mon.0) 1095 : cluster 0 osdmap e206: 8 total, 8 up, 6 in 2023-10-09T05:16:25.416 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:25 smithi194 bash[92666]: cluster 2023-10-09T05:16:24.234825+0000 osd.4 (osd.4) 159 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T05:16:25.433 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:25.434 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:25 smithi133 bash[102858]: cluster 2023-10-09T05:16:23.924248+0000 mgr.y (mgr.25277) 837 : cluster 0 pgmap v851: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 626 op/s; 11543/198747 objects degraded (5.808%); 1334/198747 objects misplaced (0.671%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T05:16:25.434 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:25 smithi133 bash[102858]: cluster 2023-10-09T05:16:24.223013+0000 mon.a (mon.0) 1093 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:16:25.434 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:25 smithi133 bash[102858]: cluster 2023-10-09T05:16:24.226148+0000 osd.7 (osd.7) 140 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T05:16:25.435 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:25 smithi133 bash[102858]: cluster 2023-10-09T05:16:24.230226+0000 osd.5 (osd.5) 192 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T05:16:25.435 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:25 smithi133 bash[102858]: cluster 2023-10-09T05:16:24.233779+0000 osd.6 (osd.6) 142 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T05:16:25.435 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:25 smithi133 bash[102858]: audit 2023-10-09T05:16:24.234664+0000 mon.a (mon.0) 1094 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:16:25.435 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:25 smithi133 bash[102858]: cluster 2023-10-09T05:16:24.234813+0000 mon.a (mon.0) 1095 : cluster 0 osdmap e206: 8 total, 8 up, 6 in 2023-10-09T05:16:25.435 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:25 smithi133 bash[102858]: cluster 2023-10-09T05:16:24.234825+0000 osd.4 (osd.4) 159 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T05:16:25.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:25.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:25.537 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:25.575 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:25.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:25.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:25.680 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:25.714 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:25.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:25.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:25.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:25 smithi133 bash[105589]: cluster 2023-10-09T05:16:23.924248+0000 mgr.y (mgr.25277) 837 : cluster 0 pgmap v851: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 626 op/s; 11543/198747 objects degraded (5.808%); 1334/198747 objects misplaced (0.671%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T05:16:25.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:25 smithi133 bash[105589]: cluster 2023-10-09T05:16:24.223013+0000 mon.a (mon.0) 1093 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:16:25.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:25 smithi133 bash[105589]: cluster 2023-10-09T05:16:24.226148+0000 osd.7 (osd.7) 140 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T05:16:25.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:25 smithi133 bash[105589]: cluster 2023-10-09T05:16:24.230226+0000 osd.5 (osd.5) 192 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T05:16:25.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:25 smithi133 bash[105589]: cluster 2023-10-09T05:16:24.233779+0000 osd.6 (osd.6) 142 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T05:16:25.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:25 smithi133 bash[105589]: audit 2023-10-09T05:16:24.234664+0000 mon.a (mon.0) 1094 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:16:25.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:25 smithi133 bash[105589]: cluster 2023-10-09T05:16:24.234813+0000 mon.a (mon.0) 1095 : cluster 0 osdmap e206: 8 total, 8 up, 6 in 2023-10-09T05:16:25.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:25 smithi133 bash[105589]: cluster 2023-10-09T05:16:24.234825+0000 osd.4 (osd.4) 159 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T05:16:25.819 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:25.849 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:25.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:25.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:25.952 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:25.999 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:26.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:26.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:26.103 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:26.119 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T05:16:26.126 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:26.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:26.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:26.237 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:26.261 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:26.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:26.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:26.365 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:26.389 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:26.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:26.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:26.495 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:26.523 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:26.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:26.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:26.628 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:26.653 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:26.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:26.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:26.758 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:26.795 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:26.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:26.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:26.902 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:26.929 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:26.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:26.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:27.035 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:27.068 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:27.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:27.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:27.174 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:27.205 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:27.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:27.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:27.308 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:27.337 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:27 smithi133 bash[102858]: cluster 2023-10-09T05:16:25.925019+0000 mgr.y (mgr.25277) 838 : cluster 0 pgmap v853: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 715 op/s; 11543/190143 objects degraded (6.071%); 972/190143 objects misplaced (0.511%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T05:16:27.337 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:27 smithi133 bash[105589]: cluster 2023-10-09T05:16:25.925019+0000 mgr.y (mgr.25277) 838 : cluster 0 pgmap v853: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 715 op/s; 11543/190143 objects degraded (6.071%); 972/190143 objects misplaced (0.511%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T05:16:27.339 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:27.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:27.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:27.446 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:27.454 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:27 smithi194 bash[92666]: cluster 2023-10-09T05:16:25.925019+0000 mgr.y (mgr.25277) 838 : cluster 0 pgmap v853: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 715 op/s; 11543/190143 objects degraded (6.071%); 972/190143 objects misplaced (0.511%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T05:16:27.475 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:27.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:27.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:27.580 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:27.618 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:27.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:27.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:27.724 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:27.759 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:27.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:27.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:27.863 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:27.893 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:27.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:27.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:27.986 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:16:28.002 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:28.022 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:16:28.031 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:28.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:28.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:28.061 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:16:28.108 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:16:28.136 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:28.153 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:16:28.173 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:28.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:28.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:28.192 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:16:28.230 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:16:28.273 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:16:28.276 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:28.284 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:16:28.284 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:16:28.284 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:16:28.284 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:16:28.284 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:16:28.284 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:16:28.284 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:16:28.285 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:16:28.304 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:28.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:28.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:28.409 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:28.418 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:28 smithi133 bash[105589]: audit 2023-10-09T05:16:27.932514+0000 mon.c (mon.2) 81 : audit 0 from='client.? 172.21.15.133:0/767748453' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:16:28.418 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:28 smithi133 bash[102858]: audit 2023-10-09T05:16:27.932514+0000 mon.c (mon.2) 81 : audit 0 from='client.? 172.21.15.133:0/767748453' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:16:28.438 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:28.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:28.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:28.544 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:28.552 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:28 smithi194 bash[92666]: audit 2023-10-09T05:16:27.932514+0000 mon.c (mon.2) 81 : audit 0 from='client.? 172.21.15.133:0/767748453' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:16:28.568 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:28.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:28.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:28.672 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:28.701 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:28.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:28.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:28.805 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:28.835 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:28.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:28.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:28.944 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:28.972 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:28.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:28.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:29.077 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:29.104 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:29.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:29.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:29.208 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:29.237 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:29.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:29.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:29.340 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:29.363 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:29 smithi133 bash[102858]: cluster 2023-10-09T05:16:27.925589+0000 mgr.y (mgr.25277) 839 : cluster 0 pgmap v854: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 713 op/s; 11543/188859 objects degraded (6.112%); 972/188859 objects misplaced (0.515%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T05:16:29.363 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:29 smithi133 bash[102858]: cluster 2023-10-09T05:16:28.346345+0000 mon.a (mon.0) 1096 : cluster 3 Health check update: Degraded data redundancy: 11543/188859 objects degraded (6.112%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:29.363 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:29 smithi133 bash[102858]: audit 2023-10-09T05:16:28.449853+0000 mon.a (mon.0) 1097 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:16:29.363 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:29 smithi133 bash[105589]: cluster 2023-10-09T05:16:27.925589+0000 mgr.y (mgr.25277) 839 : cluster 0 pgmap v854: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 713 op/s; 11543/188859 objects degraded (6.112%); 972/188859 objects misplaced (0.515%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T05:16:29.363 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:29 smithi133 bash[105589]: cluster 2023-10-09T05:16:28.346345+0000 mon.a (mon.0) 1096 : cluster 3 Health check update: Degraded data redundancy: 11543/188859 objects degraded (6.112%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:29.364 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:29 smithi133 bash[105589]: audit 2023-10-09T05:16:28.449853+0000 mon.a (mon.0) 1097 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:16:29.366 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:29.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:29.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:29.470 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:29.518 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:29.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:29.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:29.524 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:29 smithi194 bash[92666]: cluster 2023-10-09T05:16:27.925589+0000 mgr.y (mgr.25277) 839 : cluster 0 pgmap v854: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 713 op/s; 11543/188859 objects degraded (6.112%); 972/188859 objects misplaced (0.515%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T05:16:29.524 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:29 smithi194 bash[92666]: cluster 2023-10-09T05:16:28.346345+0000 mon.a (mon.0) 1096 : cluster 3 Health check update: Degraded data redundancy: 11543/188859 objects degraded (6.112%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:29.524 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:29 smithi194 bash[92666]: audit 2023-10-09T05:16:28.449853+0000 mon.a (mon.0) 1097 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:16:29.624 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:29.666 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:29.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:29.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:29.769 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:29.791 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:29.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:29.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:29.894 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:29.926 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:29.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:29.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:30.030 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:30.074 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:30.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:30.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:30.179 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:30.201 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:30.202 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:16:30.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:30.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:30.306 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:30.354 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:30.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:30.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:30.460 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:30.485 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:30.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:30.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:30.592 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:30.626 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:30.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:30.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:30.730 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:30.752 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:30.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:30.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:30.855 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:30.912 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:30.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:30.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:31.040 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:31.084 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:31.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:31.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:31.189 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:31.220 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:31.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:31.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:31.278 INFO:teuthology.orchestra.run.smithi133.stderr:set pool 10 pgp_num to 16 2023-10-09T05:16:31.323 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:31.332 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:31 smithi194 bash[92666]: cluster 2023-10-09T05:16:29.926098+0000 mgr.y (mgr.25277) 840 : cluster 0 pgmap v855: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 704 op/s; 11543/184428 objects degraded (6.259%); 972/184428 objects misplaced (0.527%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T05:16:31.332 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:31 smithi194 bash[92666]: audit 2023-10-09T05:16:31.143482+0000 mon.c (mon.2) 82 : audit 1 from='client.? 172.21.15.133:0/3153720259' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:16:31.332 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:31 smithi194 bash[92666]: audit 2023-10-09T05:16:31.144460+0000 mon.a (mon.0) 1098 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:16:31.349 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:31.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:31.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:31.453 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:31.463 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:31 smithi133 bash[102858]: cluster 2023-10-09T05:16:29.926098+0000 mgr.y (mgr.25277) 840 : cluster 0 pgmap v855: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 704 op/s; 11543/184428 objects degraded (6.259%); 972/184428 objects misplaced (0.527%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T05:16:31.464 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:31 smithi133 bash[102858]: audit 2023-10-09T05:16:31.143482+0000 mon.c (mon.2) 82 : audit 1 from='client.? 172.21.15.133:0/3153720259' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:16:31.464 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:31 smithi133 bash[102858]: audit 2023-10-09T05:16:31.144460+0000 mon.a (mon.0) 1098 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:16:31.464 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:31 smithi133 bash[105589]: cluster 2023-10-09T05:16:29.926098+0000 mgr.y (mgr.25277) 840 : cluster 0 pgmap v855: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 704 op/s; 11543/184428 objects degraded (6.259%); 972/184428 objects misplaced (0.527%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T05:16:31.464 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:31 smithi133 bash[105589]: audit 2023-10-09T05:16:31.143482+0000 mon.c (mon.2) 82 : audit 1 from='client.? 172.21.15.133:0/3153720259' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:16:31.464 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:31 smithi133 bash[105589]: audit 2023-10-09T05:16:31.144460+0000 mon.a (mon.0) 1098 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:16:31.493 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:31.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:31.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:31.617 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:31.654 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:31.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:31.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:31.758 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:31.780 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:31.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:31.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:31.886 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:31.916 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:31.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:31.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:32.021 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:32.048 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:32.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:32.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:32.151 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:32.188 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:32.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:32.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:32.286 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:32 smithi194 bash[92666]: audit 2023-10-09T05:16:31.267560+0000 mon.a (mon.0) 1099 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T05:16:32.286 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:32 smithi194 bash[92666]: cluster 2023-10-09T05:16:31.267632+0000 mon.a (mon.0) 1100 : cluster 0 osdmap e207: 8 total, 8 up, 6 in 2023-10-09T05:16:32.286 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:32 smithi194 bash[92666]: cluster 2023-10-09T05:16:31.268471+0000 osd.6 (osd.6) 143 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T05:16:32.286 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:32 smithi194 bash[92666]: cluster 2023-10-09T05:16:31.302334+0000 osd.5 (osd.5) 193 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T05:16:32.286 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:32 smithi194 bash[92666]: cluster 2023-10-09T05:16:31.303863+0000 osd.5 (osd.5) 194 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T05:16:32.286 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:32 smithi194 bash[92666]: cluster 2023-10-09T05:16:31.404880+0000 osd.7 (osd.7) 141 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T05:16:32.287 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:32 smithi194 bash[92666]: cluster 2023-10-09T05:16:31.926664+0000 mgr.y (mgr.25277) 841 : cluster 0 pgmap v857: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 840 op/s; 11543/178575 objects degraded (6.464%); 501/178575 objects misplaced (0.281%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T05:16:32.292 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:32.317 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:32.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:32.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:32.369 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:16:32.420 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:32 smithi133 bash[102858]: audit 2023-10-09T05:16:31.267560+0000 mon.a (mon.0) 1099 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T05:16:32.420 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:32 smithi133 bash[102858]: cluster 2023-10-09T05:16:31.267632+0000 mon.a (mon.0) 1100 : cluster 0 osdmap e207: 8 total, 8 up, 6 in 2023-10-09T05:16:32.421 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:32 smithi133 bash[102858]: cluster 2023-10-09T05:16:31.268471+0000 osd.6 (osd.6) 143 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T05:16:32.421 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:32 smithi133 bash[102858]: cluster 2023-10-09T05:16:31.302334+0000 osd.5 (osd.5) 193 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T05:16:32.421 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:32 smithi133 bash[102858]: cluster 2023-10-09T05:16:31.303863+0000 osd.5 (osd.5) 194 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T05:16:32.421 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:32 smithi133 bash[102858]: cluster 2023-10-09T05:16:31.404880+0000 osd.7 (osd.7) 141 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T05:16:32.421 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:32 smithi133 bash[102858]: cluster 2023-10-09T05:16:31.926664+0000 mgr.y (mgr.25277) 841 : cluster 0 pgmap v857: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 840 op/s; 11543/178575 objects degraded (6.464%); 501/178575 objects misplaced (0.281%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T05:16:32.421 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:32 smithi133 bash[105589]: audit 2023-10-09T05:16:31.267560+0000 mon.a (mon.0) 1099 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T05:16:32.421 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:32 smithi133 bash[105589]: cluster 2023-10-09T05:16:31.267632+0000 mon.a (mon.0) 1100 : cluster 0 osdmap e207: 8 total, 8 up, 6 in 2023-10-09T05:16:32.421 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:32 smithi133 bash[105589]: cluster 2023-10-09T05:16:31.268471+0000 osd.6 (osd.6) 143 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T05:16:32.421 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:32 smithi133 bash[105589]: cluster 2023-10-09T05:16:31.302334+0000 osd.5 (osd.5) 193 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T05:16:32.422 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:32 smithi133 bash[105589]: cluster 2023-10-09T05:16:31.303863+0000 osd.5 (osd.5) 194 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T05:16:32.422 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:32 smithi133 bash[105589]: cluster 2023-10-09T05:16:31.404880+0000 osd.7 (osd.7) 141 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T05:16:32.422 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:32 smithi133 bash[105589]: cluster 2023-10-09T05:16:31.926664+0000 mgr.y (mgr.25277) 841 : cluster 0 pgmap v857: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 840 op/s; 11543/178575 objects degraded (6.464%); 501/178575 objects misplaced (0.281%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T05:16:32.462 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:32.506 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:32.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:32.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:32.609 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:32.637 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:32.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:32.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:32.741 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:32.790 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:32.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:32.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:32.897 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:32.926 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:32.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:32.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:33.032 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:33.072 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:33.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:33.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:33.176 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:33.209 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:33.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:33.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:33.316 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:33.341 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:33.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:33.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:33.445 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:33.469 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:33.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:33.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:33.576 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:33.607 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:33.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:33.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:33.713 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:33.747 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:33.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:33.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:33.852 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:33.879 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:33.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:33.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:33.985 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:34.019 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:34.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:34.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:34.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:33 smithi133 bash[102858]: audit 2023-10-09T05:16:32.845620+0000 mon.a (mon.0) 1101 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:16:34.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:33 smithi133 bash[102858]: audit 2023-10-09T05:16:32.853123+0000 mon.a (mon.0) 1102 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:16:34.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:33 smithi133 bash[102858]: cluster 2023-10-09T05:16:33.347556+0000 mon.a (mon.0) 1103 : cluster 3 Health check update: Degraded data redundancy: 11543/178575 objects degraded (6.464%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:34.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:33 smithi133 bash[102858]: audit 2023-10-09T05:16:33.478080+0000 mon.a (mon.0) 1104 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:34.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:33 smithi133 bash[102858]: audit 2023-10-09T05:16:33.479244+0000 mon.a (mon.0) 1105 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:16:34.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:33 smithi133 bash[102858]: audit 2023-10-09T05:16:33.486458+0000 mon.a (mon.0) 1106 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:16:34.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:33 smithi133 bash[105589]: audit 2023-10-09T05:16:32.845620+0000 mon.a (mon.0) 1101 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:16:34.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:33 smithi133 bash[105589]: audit 2023-10-09T05:16:32.853123+0000 mon.a (mon.0) 1102 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:16:34.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:33 smithi133 bash[105589]: cluster 2023-10-09T05:16:33.347556+0000 mon.a (mon.0) 1103 : cluster 3 Health check update: Degraded data redundancy: 11543/178575 objects degraded (6.464%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:34.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:33 smithi133 bash[105589]: audit 2023-10-09T05:16:33.478080+0000 mon.a (mon.0) 1104 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:34.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:33 smithi133 bash[105589]: audit 2023-10-09T05:16:33.479244+0000 mon.a (mon.0) 1105 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:16:34.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:33 smithi133 bash[105589]: audit 2023-10-09T05:16:33.486458+0000 mon.a (mon.0) 1106 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:16:34.126 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:34.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:33 smithi194 bash[92666]: audit 2023-10-09T05:16:32.845620+0000 mon.a (mon.0) 1101 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:16:34.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:33 smithi194 bash[92666]: audit 2023-10-09T05:16:32.853123+0000 mon.a (mon.0) 1102 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:16:34.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:33 smithi194 bash[92666]: cluster 2023-10-09T05:16:33.347556+0000 mon.a (mon.0) 1103 : cluster 3 Health check update: Degraded data redundancy: 11543/178575 objects degraded (6.464%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:34.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:33 smithi194 bash[92666]: audit 2023-10-09T05:16:33.478080+0000 mon.a (mon.0) 1104 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:34.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:33 smithi194 bash[92666]: audit 2023-10-09T05:16:33.479244+0000 mon.a (mon.0) 1105 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:16:34.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:33 smithi194 bash[92666]: audit 2023-10-09T05:16:33.486458+0000 mon.a (mon.0) 1106 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:16:34.150 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:34.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:34.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:34.254 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:34.282 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:34.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:34.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:34.388 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:34.396 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:16:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:16:34] "GET /metrics HTTP/1.1" 200 36503 "" "Prometheus/2.43.0" 2023-10-09T05:16:34.419 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:34.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:34.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:34.527 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:34.551 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:34.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:34.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:34.657 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:34.679 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:34.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:34.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:34.785 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:34.811 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:34.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:34.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:34.915 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:34.941 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:34.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:34.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:35.045 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:35.058 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:34 smithi194 bash[92666]: cluster 2023-10-09T05:16:33.927207+0000 mgr.y (mgr.25277) 842 : cluster 0 pgmap v858: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 693 op/s; 11543/178575 objects degraded (6.464%); 501/178575 objects misplaced (0.281%); 5.4 MiB/s, 85 objects/s recovering 2023-10-09T05:16:35.079 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:35.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:35.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:35.185 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:35.212 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:35.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:35.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:35.285 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:16:35.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:34 smithi133 bash[102858]: cluster 2023-10-09T05:16:33.927207+0000 mgr.y (mgr.25277) 842 : cluster 0 pgmap v858: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 693 op/s; 11543/178575 objects degraded (6.464%); 501/178575 objects misplaced (0.281%); 5.4 MiB/s, 85 objects/s recovering 2023-10-09T05:16:35.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:34 smithi133 bash[105589]: cluster 2023-10-09T05:16:33.927207+0000 mgr.y (mgr.25277) 842 : cluster 0 pgmap v858: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 693 op/s; 11543/178575 objects degraded (6.464%); 501/178575 objects misplaced (0.281%); 5.4 MiB/s, 85 objects/s recovering 2023-10-09T05:16:35.316 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:35.347 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:35.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:35.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:35.450 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:35.477 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:35.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:35.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:35.580 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:35.604 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:35.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:35.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:35.707 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:35.735 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:35.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:35.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:35.840 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:35.866 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:35.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:35.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:35.970 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:35.997 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:35.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:35.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:36.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:35 smithi133 bash[102858]: cluster 2023-10-09T05:16:34.854825+0000 osd.5 (osd.5) 195 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T05:16:36.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:35 smithi133 bash[102858]: cluster 2023-10-09T05:16:34.858503+0000 mon.a (mon.0) 1107 : cluster 0 osdmap e208: 8 total, 8 up, 6 in 2023-10-09T05:16:36.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:35 smithi133 bash[102858]: cluster 2023-10-09T05:16:34.859170+0000 osd.6 (osd.6) 144 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T05:16:36.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:35 smithi133 bash[102858]: cluster 2023-10-09T05:16:34.865819+0000 osd.4 (osd.4) 160 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T05:16:36.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:35 smithi133 bash[105589]: cluster 2023-10-09T05:16:34.854825+0000 osd.5 (osd.5) 195 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T05:16:36.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:35 smithi133 bash[105589]: cluster 2023-10-09T05:16:34.858503+0000 mon.a (mon.0) 1107 : cluster 0 osdmap e208: 8 total, 8 up, 6 in 2023-10-09T05:16:36.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:35 smithi133 bash[105589]: cluster 2023-10-09T05:16:34.859170+0000 osd.6 (osd.6) 144 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T05:16:36.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:35 smithi133 bash[105589]: cluster 2023-10-09T05:16:34.865819+0000 osd.4 (osd.4) 160 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T05:16:36.099 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:36.124 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:36.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:36.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:36.130 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:35 smithi194 bash[92666]: cluster 2023-10-09T05:16:34.854825+0000 osd.5 (osd.5) 195 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T05:16:36.130 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:35 smithi194 bash[92666]: cluster 2023-10-09T05:16:34.858503+0000 mon.a (mon.0) 1107 : cluster 0 osdmap e208: 8 total, 8 up, 6 in 2023-10-09T05:16:36.130 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:35 smithi194 bash[92666]: cluster 2023-10-09T05:16:34.859170+0000 osd.6 (osd.6) 144 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T05:16:36.130 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:35 smithi194 bash[92666]: cluster 2023-10-09T05:16:34.865819+0000 osd.4 (osd.4) 160 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T05:16:36.230 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:36.264 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:36.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:36.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:36.302 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:16:36.369 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:36.394 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:36.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:36.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:36.497 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:36.528 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:36.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:36.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:36.630 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:36.669 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:36.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:36.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:36.775 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:36.801 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:36.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:36.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:36.906 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:36.938 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:36.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:36.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:37.043 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:37.052 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:36 smithi194 bash[92666]: cluster 2023-10-09T05:16:35.853385+0000 osd.5 (osd.5) 196 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T05:16:37.052 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:36 smithi194 bash[92666]: cluster 2023-10-09T05:16:35.854714+0000 mon.a (mon.0) 1108 : cluster 0 osdmap e209: 8 total, 8 up, 6 in 2023-10-09T05:16:37.052 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:36 smithi194 bash[92666]: cluster 2023-10-09T05:16:35.855611+0000 osd.6 (osd.6) 145 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T05:16:37.052 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:36 smithi194 bash[92666]: cluster 2023-10-09T05:16:35.927758+0000 mgr.y (mgr.25277) 843 : cluster 0 pgmap v861: 89 pgs: 1 peering, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 808 op/s; 11543/169875 objects degraded (6.795%); 10 MiB/s, 161 objects/s recovering 2023-10-09T05:16:37.077 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:37.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:37.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:37.181 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:37.205 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:37.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:37.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:37.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:36 smithi133 bash[105589]: cluster 2023-10-09T05:16:35.853385+0000 osd.5 (osd.5) 196 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T05:16:37.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:36 smithi133 bash[105589]: cluster 2023-10-09T05:16:35.854714+0000 mon.a (mon.0) 1108 : cluster 0 osdmap e209: 8 total, 8 up, 6 in 2023-10-09T05:16:37.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:36 smithi133 bash[105589]: cluster 2023-10-09T05:16:35.855611+0000 osd.6 (osd.6) 145 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T05:16:37.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:36 smithi133 bash[105589]: cluster 2023-10-09T05:16:35.927758+0000 mgr.y (mgr.25277) 843 : cluster 0 pgmap v861: 89 pgs: 1 peering, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 808 op/s; 11543/169875 objects degraded (6.795%); 10 MiB/s, 161 objects/s recovering 2023-10-09T05:16:37.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:36 smithi133 bash[102858]: cluster 2023-10-09T05:16:35.853385+0000 osd.5 (osd.5) 196 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T05:16:37.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:36 smithi133 bash[102858]: cluster 2023-10-09T05:16:35.854714+0000 mon.a (mon.0) 1108 : cluster 0 osdmap e209: 8 total, 8 up, 6 in 2023-10-09T05:16:37.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:36 smithi133 bash[102858]: cluster 2023-10-09T05:16:35.855611+0000 osd.6 (osd.6) 145 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T05:16:37.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:36 smithi133 bash[102858]: cluster 2023-10-09T05:16:35.927758+0000 mgr.y (mgr.25277) 843 : cluster 0 pgmap v861: 89 pgs: 1 peering, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 808 op/s; 11543/169875 objects degraded (6.795%); 10 MiB/s, 161 objects/s recovering 2023-10-09T05:16:37.309 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:37.315 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:16:37.336 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:37.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:37.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:37.439 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:37.467 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:37.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:37.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:37.573 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:37.598 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:37.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:37.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:37.702 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:37.746 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:37.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:37.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:37.780 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 6, 1] out_osds: [5, 7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:16:37.780 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:16:37.780 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-10-09T05:16:37.780 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:16:37.855 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:37.887 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:37.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:37.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:37.984 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:37 smithi133 bash[102858]: cluster 2023-10-09T05:16:36.706397+0000 osd.7 (osd.7) 142 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T05:16:37.985 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:37 smithi133 bash[105589]: cluster 2023-10-09T05:16:36.706397+0000 osd.7 (osd.7) 142 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T05:16:37.990 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:38.043 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:38.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:38.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:38.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:37 smithi194 bash[92666]: cluster 2023-10-09T05:16:36.706397+0000 osd.7 (osd.7) 142 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T05:16:38.159 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:38.188 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:38.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:38.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:38.293 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:38.323 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:38.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:38.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:38.430 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:38.475 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:38.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:38.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:38.579 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:38.610 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:38.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:38.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:38.713 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:38.748 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:38.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:38.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:38.852 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:38.885 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:38.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:38.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:38.988 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:39.032 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:39.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:39.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:39.083 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:39 smithi133 bash[105589]: cluster 2023-10-09T05:16:37.928297+0000 mgr.y (mgr.25277) 844 : cluster 0 pgmap v862: 89 pgs: 1 peering, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 565 op/s; 11543/168642 objects degraded (6.845%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T05:16:39.083 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:39 smithi133 bash[105589]: cluster 2023-10-09T05:16:38.348641+0000 mon.a (mon.0) 1109 : cluster 3 Health check update: Degraded data redundancy: 11543/169875 objects degraded (6.795%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:39.084 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:39 smithi133 bash[102858]: cluster 2023-10-09T05:16:37.928297+0000 mgr.y (mgr.25277) 844 : cluster 0 pgmap v862: 89 pgs: 1 peering, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 565 op/s; 11543/168642 objects degraded (6.845%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T05:16:39.084 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:39 smithi133 bash[102858]: cluster 2023-10-09T05:16:38.348641+0000 mon.a (mon.0) 1109 : cluster 3 Health check update: Degraded data redundancy: 11543/169875 objects degraded (6.795%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:39.139 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:39.164 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:39.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:39.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:39.267 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:39.285 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:39.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:39.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:39.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:38 smithi194 bash[92666]: cluster 2023-10-09T05:16:37.928297+0000 mgr.y (mgr.25277) 844 : cluster 0 pgmap v862: 89 pgs: 1 peering, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 565 op/s; 11543/168642 objects degraded (6.845%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T05:16:39.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:38 smithi194 bash[92666]: cluster 2023-10-09T05:16:38.348641+0000 mon.a (mon.0) 1109 : cluster 3 Health check update: Degraded data redundancy: 11543/169875 objects degraded (6.795%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:39.387 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:39.417 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:39.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:39.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:39.522 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:39.553 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:39.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:39.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:39.656 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:39.694 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:39.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:39.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:39.800 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:39.838 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:39.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:39.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:39.942 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:39.966 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:39.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:39.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:40.070 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:40.100 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:40.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:40.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:40.203 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:40.229 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:40.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:40.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:40.332 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:40.368 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:40.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:40.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:40.472 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:40.503 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:40.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:40.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:40.606 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:40.634 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:40.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:40.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:40.737 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:40.761 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:40.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:40.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:40.863 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:40.893 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:40.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:40.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:40.997 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:41.022 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:41.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:41.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:41.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:40 smithi194 bash[92666]: cluster 2023-10-09T05:16:39.929104+0000 mgr.y (mgr.25277) 845 : cluster 0 pgmap v863: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 577 op/s; 9953/164706 objects degraded (6.043%); 1469/164706 objects misplaced (0.892%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T05:16:41.138 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:41.165 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:41.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:41.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:41.270 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:41.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:40 smithi133 bash[102858]: cluster 2023-10-09T05:16:39.929104+0000 mgr.y (mgr.25277) 845 : cluster 0 pgmap v863: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 577 op/s; 9953/164706 objects degraded (6.043%); 1469/164706 objects misplaced (0.892%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T05:16:41.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:40 smithi133 bash[105589]: cluster 2023-10-09T05:16:39.929104+0000 mgr.y (mgr.25277) 845 : cluster 0 pgmap v863: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 577 op/s; 9953/164706 objects degraded (6.043%); 1469/164706 objects misplaced (0.892%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T05:16:41.299 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:41.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:41.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:41.405 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:41.440 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:41.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:41.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:41.544 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:41.579 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:41.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:41.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:41.681 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:41.713 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:41.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:41.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:41.817 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:41.852 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:41.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:41.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:41.955 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:41.980 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:41.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:41.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:42.084 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:42.109 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:42.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:42.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:42.211 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:42.244 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:42.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:42.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:42.351 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:42.386 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:42.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:42.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:42.493 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:42.517 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:42.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:42.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:42.620 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:42.649 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:42.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:42.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:42.753 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:42.773 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:42.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:42.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:42.877 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:42.897 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:42.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:42.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:43.000 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:43.028 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:43.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:43.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:43.134 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:43.146 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:42 smithi133 bash[105589]: cluster 2023-10-09T05:16:41.929816+0000 mgr.y (mgr.25277) 846 : cluster 0 pgmap v864: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 789 op/s; 9953/159633 objects degraded (6.235%); 1469/159633 objects misplaced (0.920%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T05:16:43.147 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:42 smithi133 bash[102858]: cluster 2023-10-09T05:16:41.929816+0000 mgr.y (mgr.25277) 846 : cluster 0 pgmap v864: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 789 op/s; 9953/159633 objects degraded (6.235%); 1469/159633 objects misplaced (0.920%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T05:16:43.157 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:43.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:43.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:43.260 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:43.310 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:43.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:43.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:43.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:42 smithi194 bash[92666]: cluster 2023-10-09T05:16:41.929816+0000 mgr.y (mgr.25277) 846 : cluster 0 pgmap v864: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 789 op/s; 9953/159633 objects degraded (6.235%); 1469/159633 objects misplaced (0.920%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T05:16:43.416 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:43.442 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:43.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:43.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:43.546 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:43.583 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:43.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:43.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:43.689 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:43.720 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:43.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:43.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:43.823 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:43.849 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:43.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:43.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:43.953 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:43.985 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:43.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:43.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:44.088 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:44.113 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:44.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:44.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:44.217 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:44.254 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:44.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:44.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:44.258 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:43 smithi133 bash[105589]: cluster 2023-10-09T05:16:43.349804+0000 mon.a (mon.0) 1110 : cluster 3 Health check update: Degraded data redundancy: 9953/159633 objects degraded (6.235%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:44.258 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:43 smithi133 bash[105589]: audit 2023-10-09T05:16:43.450020+0000 mon.a (mon.0) 1111 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:16:44.258 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:43 smithi133 bash[102858]: cluster 2023-10-09T05:16:43.349804+0000 mon.a (mon.0) 1110 : cluster 3 Health check update: Degraded data redundancy: 9953/159633 objects degraded (6.235%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:44.258 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:43 smithi133 bash[102858]: audit 2023-10-09T05:16:43.450020+0000 mon.a (mon.0) 1111 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:16:44.358 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:44.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:43 smithi194 bash[92666]: cluster 2023-10-09T05:16:43.349804+0000 mon.a (mon.0) 1110 : cluster 3 Health check update: Degraded data redundancy: 9953/159633 objects degraded (6.235%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:44.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:43 smithi194 bash[92666]: audit 2023-10-09T05:16:43.450020+0000 mon.a (mon.0) 1111 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:16:44.386 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:44.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:44.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:44.490 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:44.518 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:44.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:44.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:44.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:16:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:16:44] "GET /metrics HTTP/1.1" 200 36508 "" "Prometheus/2.43.0" 2023-10-09T05:16:44.627 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:44.651 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:44.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:44.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:44.756 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:44.795 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:44.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:44.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:44.899 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:44.935 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:44.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:44.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:45.036 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:45.080 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:45.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:45.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:45.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:44 smithi194 bash[92666]: cluster 2023-10-09T05:16:43.930474+0000 mgr.y (mgr.25277) 847 : cluster 0 pgmap v865: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 534 op/s; 9953/159633 objects degraded (6.235%); 1469/159633 objects misplaced (0.920%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T05:16:45.183 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:45.212 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:45.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:45.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:45.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:44 smithi133 bash[102858]: cluster 2023-10-09T05:16:43.930474+0000 mgr.y (mgr.25277) 847 : cluster 0 pgmap v865: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 534 op/s; 9953/159633 objects degraded (6.235%); 1469/159633 objects misplaced (0.920%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T05:16:45.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:44 smithi133 bash[105589]: cluster 2023-10-09T05:16:43.930474+0000 mgr.y (mgr.25277) 847 : cluster 0 pgmap v865: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 534 op/s; 9953/159633 objects degraded (6.235%); 1469/159633 objects misplaced (0.920%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T05:16:45.316 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:45.350 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:45.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:45.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:45.455 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:45.491 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:45.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:45.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:45.595 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:45.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:45.630 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:45.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:45.731 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:45.755 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:45.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:45.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:45.860 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:45.892 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:45.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:45.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:45.995 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:46.025 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:46.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:46.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:46.128 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:46.162 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:46.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:46.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:46.270 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:46.297 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:46.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:46.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:46.403 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:46.461 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:46.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:46.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:46.570 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:46.597 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:46.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:46.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:46.700 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:46.725 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:46.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:46.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:46.829 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:46.850 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:46.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:46.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:46.916 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:16:46.954 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:46.986 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:46.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:46.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:47.093 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:47.117 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:47.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:47.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:47.221 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:47.230 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:47 smithi133 bash[105589]: cluster 2023-10-09T05:16:45.931337+0000 mgr.y (mgr.25277) 848 : cluster 0 pgmap v866: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 619 op/s; 10064/151134 objects degraded (6.659%); 1266/151134 objects misplaced (0.838%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T05:16:47.230 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:47 smithi133 bash[102858]: cluster 2023-10-09T05:16:45.931337+0000 mgr.y (mgr.25277) 848 : cluster 0 pgmap v866: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 619 op/s; 10064/151134 objects degraded (6.659%); 1266/151134 objects misplaced (0.838%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T05:16:47.254 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:47.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:47.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:47.360 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:47.367 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:47 smithi194 bash[92666]: cluster 2023-10-09T05:16:45.931337+0000 mgr.y (mgr.25277) 848 : cluster 0 pgmap v866: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 619 op/s; 10064/151134 objects degraded (6.659%); 1266/151134 objects misplaced (0.838%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T05:16:47.385 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:47.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:47.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:47.489 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:47.517 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:47.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:47.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:47.620 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:47.644 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:47.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:47.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:47.751 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:47.779 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:47.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:47.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:47.886 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:47.912 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:47.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:47.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:48.016 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:48.034 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:48.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:48.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:48.136 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:48.170 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:48.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:48.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:48.275 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:48.307 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:48.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:48.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:48.414 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:48.444 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:48.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:48.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:48.548 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:48.572 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:48.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:48.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:48.675 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:48.699 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:48.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:48.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:48.802 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:48.841 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:48.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:48.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:48.944 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:48.968 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:48.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:48.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:49.072 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:49.100 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:49.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:49.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:49.204 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:49.213 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:49 smithi194 bash[92666]: cluster 2023-10-09T05:16:47.931892+0000 mgr.y (mgr.25277) 849 : cluster 0 pgmap v867: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 555 op/s; 10064/149874 objects degraded (6.715%); 1266/149874 objects misplaced (0.845%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T05:16:49.213 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:49 smithi194 bash[92666]: cluster 2023-10-09T05:16:48.351006+0000 mon.a (mon.0) 1112 : cluster 3 Health check update: Degraded data redundancy: 10064/149874 objects degraded (6.715%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:49.230 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:49.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:49.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:49.268 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:16:49.268 INFO:teuthology.orchestra.run.smithi133.stdout: "size": 20, 2023-10-09T05:16:49.268 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 600, 2023-10-09T05:16:49.268 INFO:teuthology.orchestra.run.smithi133.stdout: "ops": [ 2023-10-09T05:16:49.268 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.269 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.34845.0:102326 9.0 9:04cde4f1:::benchmark_data_smithi133_153303_object12357:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-10-09T05:16:49.269 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:11:16.765441+0000", 2023-10-09T05:16:49.269 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 332.50037285600001, 2023-10-09T05:16:49.269 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.186535588, 2023-10-09T05:16:49.269 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:16:49.269 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:16:49.269 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:16:49.269 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.34845", 2023-10-09T05:16:49.269 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:16:49.269 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 102326 2023-10-09T05:16:49.269 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.269 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:16:49.270 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.270 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:16:49.270 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:11:16.765441+0000", 2023-10-09T05:16:49.270 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.270 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.270 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.270 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:16:49.270 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:11:16.765441+0000", 2023-10-09T05:16:49.270 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.270 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.270 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.270 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:16:49.271 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:11:16.765440+0000", 2023-10-09T05:16:49.271 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:16:49.271 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.271 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.271 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:16:49.271 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:11:16.765448+0000", 2023-10-09T05:16:49.271 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.9640000000000003e-06 2023-10-09T05:16:49.271 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.271 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.271 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:16:49.271 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:11:16.921999+0000", 2023-10-09T05:16:49.272 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.15655185699999999 2023-10-09T05:16:49.272 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.272 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.272 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:16:49.272 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:11:16.922028+0000", 2023-10-09T05:16:49.272 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.8459e-05 2023-10-09T05:16:49.272 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.272 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.272 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.272 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:11:16.929385+0000", 2023-10-09T05:16:49.272 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0073566600000000001 2023-10-09T05:16:49.272 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.273 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.273 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:16:49.273 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:11:16.929508+0000", 2023-10-09T05:16:49.273 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00012315000000000001 2023-10-09T05:16:49.273 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.273 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.273 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T05:16:49.273 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:11:16.929631+0000", 2023-10-09T05:16:49.273 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000123084 2023-10-09T05:16:49.273 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.273 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.273 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:16:49.273 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:11:16.938254+0000", 2023-10-09T05:16:49.274 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0086235649999999997 2023-10-09T05:16:49.274 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.274 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.274 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:16:49.274 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:11:16.945497+0000", 2023-10-09T05:16:49.274 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0072425240000000002 2023-10-09T05:16:49.274 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.274 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.274 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:16:49.274 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:11:16.946228+0000", 2023-10-09T05:16:49.274 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000731531 2023-10-09T05:16:49.274 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.275 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.275 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:16:49.275 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:11:16.946288+0000", 2023-10-09T05:16:49.275 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.9577000000000002e-05 2023-10-09T05:16:49.275 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.275 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.275 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:16:49.275 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:11:16.946341+0000", 2023-10-09T05:16:49.275 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.2834000000000003e-05 2023-10-09T05:16:49.275 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.275 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.275 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:16:49.276 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:11:16.951977+0000", 2023-10-09T05:16:49.276 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0056359920000000003 2023-10-09T05:16:49.276 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.276 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:16:49.276 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.276 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.276 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.276 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35022.0:71163 10.8 10:1f2e0a33:::benchmark_data_smithi133_173023_object71162:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e194)", 2023-10-09T05:16:49.276 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:14:55.922194+0000", 2023-10-09T05:16:49.276 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 113.34361977099999, 2023-10-09T05:16:49.276 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.40668251500000002, 2023-10-09T05:16:49.276 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:16:49.277 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:16:49.277 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:16:49.277 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35022", 2023-10-09T05:16:49.277 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2582323369", 2023-10-09T05:16:49.277 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 71163 2023-10-09T05:16:49.277 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.277 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:16:49.277 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.277 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:16:49.277 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.922194+0000", 2023-10-09T05:16:49.277 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.277 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.278 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.278 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:16:49.278 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.922194+0000", 2023-10-09T05:16:49.278 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.278 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.278 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.278 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:16:49.278 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.922194+0000", 2023-10-09T05:16:49.278 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999995 2023-10-09T05:16:49.278 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.278 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.278 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:16:49.279 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.922216+0000", 2023-10-09T05:16:49.279 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.1894000000000001e-05 2023-10-09T05:16:49.279 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.279 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.279 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:16:49.279 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.922223+0000", 2023-10-09T05:16:49.279 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.7970000000000001e-06 2023-10-09T05:16:49.279 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.279 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.279 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:16:49.279 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.922230+0000", 2023-10-09T05:16:49.279 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.6200000000000001e-06 2023-10-09T05:16:49.279 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.280 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.280 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.280 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.936125+0000", 2023-10-09T05:16:49.280 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.013894778 2023-10-09T05:16:49.280 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.280 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.280 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:16:49.280 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.936314+0000", 2023-10-09T05:16:49.280 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000188875 2023-10-09T05:16:49.280 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.280 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.280 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,4", 2023-10-09T05:16:49.281 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.936497+0000", 2023-10-09T05:16:49.281 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000183172 2023-10-09T05:16:49.281 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.281 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.281 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:16:49.281 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.942528+0000", 2023-10-09T05:16:49.281 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0060308050000000002 2023-10-09T05:16:49.281 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.281 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.281 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:16:49.281 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.943373+0000", 2023-10-09T05:16:49.281 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000845465 2023-10-09T05:16:49.282 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.282 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.282 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:16:49.282 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:56.328736+0000", 2023-10-09T05:16:49.282 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.38536289699999998 2023-10-09T05:16:49.282 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.282 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.282 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:16:49.282 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:56.328832+0000", 2023-10-09T05:16:49.282 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.5797000000000005e-05 2023-10-09T05:16:49.282 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.282 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.283 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:16:49.283 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:56.328877+0000", 2023-10-09T05:16:49.283 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.5048e-05 2023-10-09T05:16:49.283 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.283 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:16:49.283 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.283 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.283 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.283 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35022.0:71177 10.8 10:106180a5:::benchmark_data_smithi133_173023_object71176:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e194)", 2023-10-09T05:16:49.283 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:14:55.946091+0000", 2023-10-09T05:16:49.283 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 113.319722785, 2023-10-09T05:16:49.283 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.38317306099999998, 2023-10-09T05:16:49.284 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:16:49.285 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:16:49.285 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:16:49.285 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35022", 2023-10-09T05:16:49.285 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2582323369", 2023-10-09T05:16:49.285 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 71177 2023-10-09T05:16:49.285 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.285 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:16:49.286 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.286 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:16:49.286 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.946091+0000", 2023-10-09T05:16:49.286 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.286 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.286 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.286 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:16:49.286 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.946091+0000", 2023-10-09T05:16:49.286 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.286 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.286 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.286 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:16:49.287 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.946090+0000", 2023-10-09T05:16:49.287 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:16:49.287 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.287 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.287 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:16:49.287 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.946112+0000", 2023-10-09T05:16:49.287 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.2493e-05 2023-10-09T05:16:49.287 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.287 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.287 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:16:49.287 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.946123+0000", 2023-10-09T05:16:49.287 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.1063000000000001e-05 2023-10-09T05:16:49.288 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.288 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.288 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:16:49.288 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.946130+0000", 2023-10-09T05:16:49.288 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.7050000000000002e-06 2023-10-09T05:16:49.288 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.288 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.288 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.288 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.946173+0000", 2023-10-09T05:16:49.288 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.2617000000000002e-05 2023-10-09T05:16:49.288 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.288 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.289 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:16:49.289 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.946290+0000", 2023-10-09T05:16:49.289 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000117012 2023-10-09T05:16:49.289 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.289 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.289 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,4", 2023-10-09T05:16:49.289 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.946394+0000", 2023-10-09T05:16:49.289 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000103795 2023-10-09T05:16:49.289 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.289 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.289 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:16:49.289 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.948629+0000", 2023-10-09T05:16:49.290 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0022356670000000002 2023-10-09T05:16:49.290 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.290 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.290 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:16:49.290 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.949049+0000", 2023-10-09T05:16:49.290 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00041946799999999999 2023-10-09T05:16:49.290 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.290 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.290 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:16:49.290 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:56.328917+0000", 2023-10-09T05:16:49.290 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.37986786099999997 2023-10-09T05:16:49.290 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.291 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.291 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:16:49.291 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:56.329226+0000", 2023-10-09T05:16:49.291 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00030909699999999999 2023-10-09T05:16:49.291 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.291 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.291 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:16:49.291 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:56.329264+0000", 2023-10-09T05:16:49.291 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.8742000000000003e-05 2023-10-09T05:16:49.291 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.291 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:16:49.291 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.292 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.292 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.292 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35022.0:71178 10.8 10:142db4e1:::benchmark_data_smithi133_173023_object71177:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e194)", 2023-10-09T05:16:49.292 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:14:55.949584+0000", 2023-10-09T05:16:49.292 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 113.31622967, 2023-10-09T05:16:49.292 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.37998182800000002, 2023-10-09T05:16:49.292 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:16:49.292 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:16:49.292 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:16:49.292 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35022", 2023-10-09T05:16:49.292 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2582323369", 2023-10-09T05:16:49.292 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 71178 2023-10-09T05:16:49.293 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.293 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:16:49.293 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.293 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:16:49.293 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.949584+0000", 2023-10-09T05:16:49.293 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.293 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.293 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.293 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:16:49.293 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.949584+0000", 2023-10-09T05:16:49.293 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.293 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.293 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.294 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:16:49.294 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.949582+0000", 2023-10-09T05:16:49.294 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999976 2023-10-09T05:16:49.294 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.294 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.294 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:16:49.294 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.949615+0000", 2023-10-09T05:16:49.294 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.2706000000000002e-05 2023-10-09T05:16:49.294 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.294 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.294 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:16:49.294 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.949631+0000", 2023-10-09T05:16:49.295 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.6654e-05 2023-10-09T05:16:49.295 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.295 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.295 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:16:49.295 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.949640+0000", 2023-10-09T05:16:49.295 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.6910000000000003e-06 2023-10-09T05:16:49.295 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.295 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.295 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.295 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.949693+0000", 2023-10-09T05:16:49.295 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.3310999999999998e-05 2023-10-09T05:16:49.295 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.296 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.296 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:16:49.296 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.949896+0000", 2023-10-09T05:16:49.296 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00020301499999999999 2023-10-09T05:16:49.296 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.296 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.296 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,4", 2023-10-09T05:16:49.296 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.950116+0000", 2023-10-09T05:16:49.296 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000219244 2023-10-09T05:16:49.296 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.296 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.296 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:16:49.297 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.953368+0000", 2023-10-09T05:16:49.297 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0032522380000000002 2023-10-09T05:16:49.297 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.297 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.297 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:16:49.297 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.956777+0000", 2023-10-09T05:16:49.297 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0034094379999999999 2023-10-09T05:16:49.297 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.297 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.297 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:16:49.297 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:56.329302+0000", 2023-10-09T05:16:49.297 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.37252489799999999 2023-10-09T05:16:49.298 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.298 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.298 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:16:49.298 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:56.329521+0000", 2023-10-09T05:16:49.298 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00021856100000000001 2023-10-09T05:16:49.298 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.298 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.298 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:16:49.298 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:56.329566+0000", 2023-10-09T05:16:49.298 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.5473999999999999e-05 2023-10-09T05:16:49.298 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.298 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:16:49.299 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.299 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.299 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.299 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35022.0:71181 10.8 10:190499d6:::benchmark_data_smithi133_173023_object71180:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e194)", 2023-10-09T05:16:49.299 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:14:55.950368+0000", 2023-10-09T05:16:49.299 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 113.315445865, 2023-10-09T05:16:49.299 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.37934546099999999, 2023-10-09T05:16:49.299 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:16:49.299 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:16:49.299 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:16:49.299 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35022", 2023-10-09T05:16:49.299 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2582323369", 2023-10-09T05:16:49.300 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 71181 2023-10-09T05:16:49.300 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.300 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:16:49.300 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.300 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:16:49.300 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.950368+0000", 2023-10-09T05:16:49.300 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.300 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.300 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.300 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:16:49.300 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.950368+0000", 2023-10-09T05:16:49.300 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.301 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.301 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.301 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:16:49.301 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.950367+0000", 2023-10-09T05:16:49.301 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:16:49.301 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.301 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.301 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:16:49.301 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.950399+0000", 2023-10-09T05:16:49.301 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.2419000000000003e-05 2023-10-09T05:16:49.301 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.301 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.302 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:16:49.302 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.950414+0000", 2023-10-09T05:16:49.302 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.4876e-05 2023-10-09T05:16:49.302 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.302 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.302 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:16:49.302 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.950423+0000", 2023-10-09T05:16:49.302 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.2799999999999992e-06 2023-10-09T05:16:49.302 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.302 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.302 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.303 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.953395+0000", 2023-10-09T05:16:49.303 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0029717889999999999 2023-10-09T05:16:49.303 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.303 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.303 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:16:49.303 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.953570+0000", 2023-10-09T05:16:49.303 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000174857 2023-10-09T05:16:49.303 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.303 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.303 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,4", 2023-10-09T05:16:49.303 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.953807+0000", 2023-10-09T05:16:49.303 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00023655099999999999 2023-10-09T05:16:49.303 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.304 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.304 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:16:49.304 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.959212+0000", 2023-10-09T05:16:49.304 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0054052639999999999 2023-10-09T05:16:49.304 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.304 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.304 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:16:49.304 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.960382+0000", 2023-10-09T05:16:49.304 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.001170101 2023-10-09T05:16:49.304 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.304 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.305 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:16:49.305 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:56.329633+0000", 2023-10-09T05:16:49.305 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.36925110799999999 2023-10-09T05:16:49.305 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.305 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.305 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:16:49.305 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:56.329680+0000", 2023-10-09T05:16:49.305 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.6647999999999998e-05 2023-10-09T05:16:49.305 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.305 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.306 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:16:49.306 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:56.329714+0000", 2023-10-09T05:16:49.306 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.3915999999999999e-05 2023-10-09T05:16:49.306 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.306 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:16:49.306 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.306 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.306 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.306 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35022.0:71193 10.8 10:1da497f7:::benchmark_data_smithi133_173023_object71192:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e194)", 2023-10-09T05:16:49.307 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:14:55.971344+0000", 2023-10-09T05:16:49.307 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 113.294470023, 2023-10-09T05:16:49.307 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.358567839, 2023-10-09T05:16:49.307 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:16:49.307 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:16:49.307 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:16:49.307 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35022", 2023-10-09T05:16:49.307 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2582323369", 2023-10-09T05:16:49.307 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 71193 2023-10-09T05:16:49.307 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.307 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:16:49.308 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.308 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:16:49.308 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.971344+0000", 2023-10-09T05:16:49.308 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.308 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.308 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.308 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:16:49.308 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.971344+0000", 2023-10-09T05:16:49.308 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.308 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.309 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.309 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:16:49.309 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.971343+0000", 2023-10-09T05:16:49.309 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:16:49.309 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.309 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.309 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:16:49.309 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.971368+0000", 2023-10-09T05:16:49.309 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.5307999999999999e-05 2023-10-09T05:16:49.309 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.310 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.310 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:16:49.310 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.971383+0000", 2023-10-09T05:16:49.310 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.5007e-05 2023-10-09T05:16:49.310 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.310 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.310 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:16:49.310 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.971392+0000", 2023-10-09T05:16:49.310 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.9630000000000004e-06 2023-10-09T05:16:49.310 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.311 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.311 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.311 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.971447+0000", 2023-10-09T05:16:49.311 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.4936999999999999e-05 2023-10-09T05:16:49.311 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.311 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.311 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:16:49.311 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.971599+0000", 2023-10-09T05:16:49.311 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00015254799999999999 2023-10-09T05:16:49.311 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.311 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.312 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,4", 2023-10-09T05:16:49.312 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.971746+0000", 2023-10-09T05:16:49.312 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00014700600000000001 2023-10-09T05:16:49.312 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.312 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.312 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:16:49.312 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.973997+0000", 2023-10-09T05:16:49.312 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.002250181 2023-10-09T05:16:49.312 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.312 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.313 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:16:49.313 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.977419+0000", 2023-10-09T05:16:49.313 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0034226930000000001 2023-10-09T05:16:49.313 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.313 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.314 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:16:49.314 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:56.329825+0000", 2023-10-09T05:16:49.314 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.35240569199999999 2023-10-09T05:16:49.314 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.314 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.314 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:16:49.314 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:56.329874+0000", 2023-10-09T05:16:49.314 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.8785000000000001e-05 2023-10-09T05:16:49.314 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.314 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.314 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:16:49.315 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:56.329912+0000", 2023-10-09T05:16:49.315 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.8238999999999998e-05 2023-10-09T05:16:49.315 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.315 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:16:49.315 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.315 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.315 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.315 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35022.0:71204 10.8 10:1d6cd1ac:::benchmark_data_smithi133_173023_object71203:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e194)", 2023-10-09T05:16:49.315 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:14:55.999793+0000", 2023-10-09T05:16:49.315 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 113.26602061200001, 2023-10-09T05:16:49.315 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.33046932200000001, 2023-10-09T05:16:49.315 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:16:49.315 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:16:49.316 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:16:49.316 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35022", 2023-10-09T05:16:49.317 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2582323369", 2023-10-09T05:16:49.318 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 71204 2023-10-09T05:16:49.318 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.318 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:16:49.318 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.318 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:16:49.318 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.999793+0000", 2023-10-09T05:16:49.318 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.318 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.318 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.318 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:16:49.318 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.999793+0000", 2023-10-09T05:16:49.319 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.319 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.319 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.319 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:16:49.319 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.999791+0000", 2023-10-09T05:16:49.319 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999971 2023-10-09T05:16:49.319 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.319 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.319 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:16:49.319 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.999841+0000", 2023-10-09T05:16:49.319 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.0624999999999997e-05 2023-10-09T05:16:49.319 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.320 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.320 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:16:49.320 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.999879+0000", 2023-10-09T05:16:49.320 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.7978999999999997e-05 2023-10-09T05:16:49.320 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.320 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.320 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:16:49.321 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.999896+0000", 2023-10-09T05:16:49.321 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.6157000000000001e-05 2023-10-09T05:16:49.321 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.321 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.321 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.321 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:55.999984+0000", 2023-10-09T05:16:49.321 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.8815999999999998e-05 2023-10-09T05:16:49.321 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.321 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.321 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:16:49.321 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:56.000193+0000", 2023-10-09T05:16:49.322 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00020811399999999999 2023-10-09T05:16:49.322 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.322 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.322 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,4", 2023-10-09T05:16:49.322 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:56.000403+0000", 2023-10-09T05:16:49.322 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00021057899999999999 2023-10-09T05:16:49.322 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.322 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.322 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:16:49.322 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:56.003398+0000", 2023-10-09T05:16:49.322 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0029944260000000001 2023-10-09T05:16:49.323 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.323 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.323 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:16:49.323 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:56.004392+0000", 2023-10-09T05:16:49.323 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00099468400000000002 2023-10-09T05:16:49.323 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.323 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.323 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:16:49.323 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:56.329973+0000", 2023-10-09T05:16:49.323 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.32558029199999999 2023-10-09T05:16:49.323 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.323 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.324 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:16:49.324 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:56.330209+0000", 2023-10-09T05:16:49.324 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00023662800000000001 2023-10-09T05:16:49.324 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.324 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.324 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:16:49.324 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:14:56.330263+0000", 2023-10-09T05:16:49.324 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.3653000000000003e-05 2023-10-09T05:16:49.324 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.324 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:16:49.324 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.324 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.325 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.325 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35022.0:100457 10.8 10:1b58c5db:::benchmark_data_smithi133_173023_object3021:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2023-10-09T05:16:49.325 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:15:40.082812+0000", 2023-10-09T05:16:49.325 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 69.183002231000003, 2023-10-09T05:16:49.325 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.1714878259999999, 2023-10-09T05:16:49.325 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:16:49.325 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "reached pg", 2023-10-09T05:16:49.325 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:16:49.325 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35022", 2023-10-09T05:16:49.325 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2582323369", 2023-10-09T05:16:49.325 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 100457 2023-10-09T05:16:49.326 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.326 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:16:49.326 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.326 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:16:49.326 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.082812+0000", 2023-10-09T05:16:49.326 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.326 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.326 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.326 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:16:49.326 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.082812+0000", 2023-10-09T05:16:49.326 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.326 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.327 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.327 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:16:49.327 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.082810+0000", 2023-10-09T05:16:49.327 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999981 2023-10-09T05:16:49.327 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.327 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.327 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:16:49.327 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.082819+0000", 2023-10-09T05:16:49.327 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.6580000000000008e-06 2023-10-09T05:16:49.327 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.327 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.327 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:16:49.327 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.089833+0000", 2023-10-09T05:16:49.328 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.007013717 2023-10-09T05:16:49.328 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.328 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.328 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:16:49.328 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.089844+0000", 2023-10-09T05:16:49.328 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.1147e-05 2023-10-09T05:16:49.328 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.328 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.328 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.328 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.102878+0000", 2023-10-09T05:16:49.328 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.013034502999999999 2023-10-09T05:16:49.328 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.329 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.329 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:16:49.329 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.103065+0000", 2023-10-09T05:16:49.329 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00018720999999999999 2023-10-09T05:16:49.329 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.329 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.329 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,7", 2023-10-09T05:16:49.329 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.103335+0000", 2023-10-09T05:16:49.329 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00026951600000000003 2023-10-09T05:16:49.329 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.329 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.329 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.330 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.157458+0000", 2023-10-09T05:16:49.330 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.054123157999999998 2023-10-09T05:16:49.330 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.330 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.330 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:16:49.330 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.157469+0000", 2023-10-09T05:16:49.330 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.0696999999999999e-05 2023-10-09T05:16:49.330 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.330 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.330 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.330 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.235063+0000", 2023-10-09T05:16:49.330 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.0775942940000001 2023-10-09T05:16:49.331 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.331 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.331 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for missing object", 2023-10-09T05:16:49.331 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.235352+0000", 2023-10-09T05:16:49.331 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00028922900000000001 2023-10-09T05:16:49.331 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.331 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.331 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.331 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.254210+0000", 2023-10-09T05:16:49.331 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.018857714000000001 2023-10-09T05:16:49.331 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.331 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.332 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:16:49.332 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.254300+0000", 2023-10-09T05:16:49.332 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.9720000000000002e-05 2023-10-09T05:16:49.332 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.332 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:16:49.332 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.332 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.332 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.332 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35022.0:100462 10.c 10.1d8b671c (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e202)", 2023-10-09T05:16:49.332 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:15:40.106005+0000", 2023-10-09T05:16:49.332 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 69.159808752999993, 2023-10-09T05:16:49.332 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.99112038199999997, 2023-10-09T05:16:49.333 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:16:49.333 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "reached pg", 2023-10-09T05:16:49.333 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:16:49.333 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35022", 2023-10-09T05:16:49.333 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2582323369", 2023-10-09T05:16:49.333 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 100462 2023-10-09T05:16:49.333 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.333 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:16:49.333 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.333 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:16:49.333 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.106005+0000", 2023-10-09T05:16:49.333 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.334 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.334 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.334 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:16:49.334 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.106005+0000", 2023-10-09T05:16:49.334 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.334 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.334 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.334 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:16:49.334 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.106004+0000", 2023-10-09T05:16:49.334 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:16:49.334 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.334 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.335 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:16:49.335 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.106010+0000", 2023-10-09T05:16:49.335 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.6270000000000002e-06 2023-10-09T05:16:49.335 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.335 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.335 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:16:49.335 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.106026+0000", 2023-10-09T05:16:49.335 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.5423000000000001e-05 2023-10-09T05:16:49.335 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.335 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.335 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:16:49.335 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.106050+0000", 2023-10-09T05:16:49.336 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.3972999999999999e-05 2023-10-09T05:16:49.336 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.336 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.336 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.336 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.112314+0000", 2023-10-09T05:16:49.336 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0062640760000000004 2023-10-09T05:16:49.336 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.336 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.336 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op must wait for map", 2023-10-09T05:16:49.336 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.112323+0000", 2023-10-09T05:16:49.336 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.0729999999999994e-06 2023-10-09T05:16:49.336 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.337 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.337 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.337 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.156677+0000", 2023-10-09T05:16:49.337 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.044353837 2023-10-09T05:16:49.337 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.337 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.337 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:16:49.337 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.156689+0000", 2023-10-09T05:16:49.337 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.2146e-05 2023-10-09T05:16:49.337 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.337 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.337 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.338 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.097109+0000", 2023-10-09T05:16:49.338 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.94042031699999995 2023-10-09T05:16:49.338 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.338 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.338 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:16:49.338 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.097126+0000", 2023-10-09T05:16:49.338 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.6560999999999999e-05 2023-10-09T05:16:49.338 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.338 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:16:49.338 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.338 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.338 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.339 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35022.0:100465 10.8 10:1d66f240:::benchmark_data_smithi133_173023_object3029:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e202)", 2023-10-09T05:16:49.339 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:15:40.124254+0000", 2023-10-09T05:16:49.339 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 69.141560308999999, 2023-10-09T05:16:49.339 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.1308084819999999, 2023-10-09T05:16:49.339 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:16:49.339 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:16:49.339 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:16:49.339 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35022", 2023-10-09T05:16:49.339 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2582323369", 2023-10-09T05:16:49.339 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 100465 2023-10-09T05:16:49.339 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.339 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:16:49.340 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.340 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:16:49.340 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.124254+0000", 2023-10-09T05:16:49.340 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.340 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.340 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.340 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:16:49.340 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.124254+0000", 2023-10-09T05:16:49.340 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.340 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.340 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.340 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:16:49.341 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.124252+0000", 2023-10-09T05:16:49.341 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999981 2023-10-09T05:16:49.341 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.341 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.341 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:16:49.341 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.124258+0000", 2023-10-09T05:16:49.341 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.1240000000000002e-06 2023-10-09T05:16:49.341 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.341 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.341 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:16:49.341 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.124269+0000", 2023-10-09T05:16:49.341 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.0886e-05 2023-10-09T05:16:49.341 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.342 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.342 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:16:49.342 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.124277+0000", 2023-10-09T05:16:49.342 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.7049999999999999e-06 2023-10-09T05:16:49.342 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.342 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.342 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.342 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.157533+0000", 2023-10-09T05:16:49.342 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0332567 2023-10-09T05:16:49.342 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.342 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.342 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:16:49.343 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.157541+0000", 2023-10-09T05:16:49.343 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.8129999999999996e-06 2023-10-09T05:16:49.343 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.343 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.343 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.343 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.236230+0000", 2023-10-09T05:16:49.343 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.078688656 2023-10-09T05:16:49.343 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.343 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.343 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:16:49.343 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.236386+0000", 2023-10-09T05:16:49.343 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00015585100000000001 2023-10-09T05:16:49.344 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.344 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.344 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,4", 2023-10-09T05:16:49.344 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.236961+0000", 2023-10-09T05:16:49.344 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00057499199999999997 2023-10-09T05:16:49.344 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.344 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.344 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:16:49.344 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.252011+0000", 2023-10-09T05:16:49.344 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.015050071 2023-10-09T05:16:49.344 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.344 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.345 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:16:49.345 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.253064+0000", 2023-10-09T05:16:49.345 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.001053062 2023-10-09T05:16:49.345 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.345 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.345 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:16:49.345 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.254543+0000", 2023-10-09T05:16:49.345 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0014793829999999999 2023-10-09T05:16:49.345 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.345 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.345 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:16:49.345 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.254585+0000", 2023-10-09T05:16:49.345 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.1514000000000003e-05 2023-10-09T05:16:49.346 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.346 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.346 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:16:49.346 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.255062+0000", 2023-10-09T05:16:49.346 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00047764699999999999 2023-10-09T05:16:49.346 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.346 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:16:49.346 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.346 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.346 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.346 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35022.0:100467 10.c 10.12aad26c (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e202)", 2023-10-09T05:16:49.346 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:15:40.124286+0000", 2023-10-09T05:16:49.347 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 69.141527996999997, 2023-10-09T05:16:49.347 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.97287657699999996, 2023-10-09T05:16:49.347 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:16:49.347 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "reached pg", 2023-10-09T05:16:49.347 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:16:49.347 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35022", 2023-10-09T05:16:49.347 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2582323369", 2023-10-09T05:16:49.347 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 100467 2023-10-09T05:16:49.347 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.347 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:16:49.347 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.347 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:16:49.348 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.124286+0000", 2023-10-09T05:16:49.348 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.348 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.348 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.348 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:16:49.348 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.124286+0000", 2023-10-09T05:16:49.348 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.348 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.348 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.348 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:16:49.348 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.124286+0000", 2023-10-09T05:16:49.348 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999995 2023-10-09T05:16:49.349 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.349 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.349 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:16:49.349 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.124289+0000", 2023-10-09T05:16:49.349 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.005e-06 2023-10-09T05:16:49.349 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.349 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.349 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:16:49.349 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.124296+0000", 2023-10-09T05:16:49.349 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.8589999999999999e-06 2023-10-09T05:16:49.349 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.349 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.350 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:16:49.350 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.124301+0000", 2023-10-09T05:16:49.350 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.5639999999999996e-06 2023-10-09T05:16:49.350 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.350 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.350 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.350 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.156721+0000", 2023-10-09T05:16:49.350 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.032419612 2023-10-09T05:16:49.350 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.350 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.350 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:16:49.350 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.156730+0000", 2023-10-09T05:16:49.351 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.3999999999999998e-06 2023-10-09T05:16:49.351 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.351 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.351 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.351 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.097152+0000", 2023-10-09T05:16:49.351 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.94042143700000003 2023-10-09T05:16:49.351 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.351 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.351 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:16:49.351 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.097163+0000", 2023-10-09T05:16:49.351 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.1107999999999999e-05 2023-10-09T05:16:49.351 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.352 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:16:49.352 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.352 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.352 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.352 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35022.0:100471 10.c 10.a43a5a2c (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e202)", 2023-10-09T05:16:49.352 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:15:40.136667+0000", 2023-10-09T05:16:49.352 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 69.129147583000005, 2023-10-09T05:16:49.352 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.96052951600000003, 2023-10-09T05:16:49.352 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:16:49.352 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "reached pg", 2023-10-09T05:16:49.352 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:16:49.352 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35022", 2023-10-09T05:16:49.353 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2582323369", 2023-10-09T05:16:49.353 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 100471 2023-10-09T05:16:49.353 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.353 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:16:49.353 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.353 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:16:49.353 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.136667+0000", 2023-10-09T05:16:49.353 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.353 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.353 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.353 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:16:49.353 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.136667+0000", 2023-10-09T05:16:49.353 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.354 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.354 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.354 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:16:49.354 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.136664+0000", 2023-10-09T05:16:49.354 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999976 2023-10-09T05:16:49.354 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.354 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.354 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:16:49.354 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.136672+0000", 2023-10-09T05:16:49.354 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.9629999999999998e-06 2023-10-09T05:16:49.354 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.355 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.355 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:16:49.355 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.137259+0000", 2023-10-09T05:16:49.355 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00058638899999999999 2023-10-09T05:16:49.355 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.355 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.355 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:16:49.355 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.137272+0000", 2023-10-09T05:16:49.355 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.3198e-05 2023-10-09T05:16:49.355 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.355 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.355 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.355 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.156757+0000", 2023-10-09T05:16:49.356 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.019484853999999999 2023-10-09T05:16:49.356 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.356 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.356 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:16:49.356 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.156767+0000", 2023-10-09T05:16:49.356 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.9219999999999992e-06 2023-10-09T05:16:49.356 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.356 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.356 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.356 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.097186+0000", 2023-10-09T05:16:49.356 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.94041930100000004 2023-10-09T05:16:49.356 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.357 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.357 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:16:49.357 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.097196+0000", 2023-10-09T05:16:49.357 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.0162e-05 2023-10-09T05:16:49.357 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.357 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:16:49.357 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.357 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.357 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.357 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35022.0:100473 10.8 10:12a330a9:::benchmark_data_smithi133_173023_object3037:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e202)", 2023-10-09T05:16:49.357 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:15:40.145874+0000", 2023-10-09T05:16:49.357 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 69.119939896000005, 2023-10-09T05:16:49.358 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.1203388780000001, 2023-10-09T05:16:49.358 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:16:49.358 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:16:49.358 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:16:49.358 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35022", 2023-10-09T05:16:49.358 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2582323369", 2023-10-09T05:16:49.358 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 100473 2023-10-09T05:16:49.358 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.358 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:16:49.358 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.358 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:16:49.358 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.145874+0000", 2023-10-09T05:16:49.359 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.359 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.359 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.359 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:16:49.359 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.145874+0000", 2023-10-09T05:16:49.359 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.359 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.359 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.359 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:16:49.359 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.145872+0000", 2023-10-09T05:16:49.359 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999981 2023-10-09T05:16:49.359 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.360 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.360 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:16:49.360 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.145883+0000", 2023-10-09T05:16:49.360 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.0915e-05 2023-10-09T05:16:49.360 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.360 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.360 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:16:49.360 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.145903+0000", 2023-10-09T05:16:49.360 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.0102000000000002e-05 2023-10-09T05:16:49.360 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.360 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.360 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:16:49.361 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.145916+0000", 2023-10-09T05:16:49.361 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.2914999999999999e-05 2023-10-09T05:16:49.361 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.361 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.361 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.361 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.157562+0000", 2023-10-09T05:16:49.361 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.011645984999999999 2023-10-09T05:16:49.361 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.361 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.361 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:16:49.361 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.157569+0000", 2023-10-09T05:16:49.361 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.2729999999999997e-06 2023-10-09T05:16:49.361 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.362 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.362 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.362 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.242268+0000", 2023-10-09T05:16:49.362 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.0846983910000001 2023-10-09T05:16:49.362 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.362 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.362 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:16:49.362 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.242430+0000", 2023-10-09T05:16:49.362 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00016167500000000001 2023-10-09T05:16:49.362 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.362 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.362 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,4", 2023-10-09T05:16:49.363 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.242521+0000", 2023-10-09T05:16:49.363 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.1903000000000004e-05 2023-10-09T05:16:49.363 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.363 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.363 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:16:49.363 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.252028+0000", 2023-10-09T05:16:49.363 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0095068159999999995 2023-10-09T05:16:49.363 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.363 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.363 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:16:49.363 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.254355+0000", 2023-10-09T05:16:49.363 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0023270169999999998 2023-10-09T05:16:49.363 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.364 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.364 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:16:49.364 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.265228+0000", 2023-10-09T05:16:49.364 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.010872565000000001 2023-10-09T05:16:49.364 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.364 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.364 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:16:49.364 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.265288+0000", 2023-10-09T05:16:49.364 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.0526999999999998e-05 2023-10-09T05:16:49.364 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.364 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.364 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:16:49.365 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.266213+0000", 2023-10-09T05:16:49.365 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00092473800000000004 2023-10-09T05:16:49.365 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.365 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:16:49.365 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.365 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.365 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.365 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35022.0:100474 10.8 10:141ac169:::benchmark_data_smithi133_173023_object3038:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e202)", 2023-10-09T05:16:49.365 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:15:40.146304+0000", 2023-10-09T05:16:49.365 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 69.119510251999998, 2023-10-09T05:16:49.365 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.120507089, 2023-10-09T05:16:49.365 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:16:49.366 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:16:49.366 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:16:49.366 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35022", 2023-10-09T05:16:49.366 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2582323369", 2023-10-09T05:16:49.366 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 100474 2023-10-09T05:16:49.366 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.366 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:16:49.366 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.366 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:16:49.366 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.146304+0000", 2023-10-09T05:16:49.366 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.366 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.367 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.367 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:16:49.367 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.146304+0000", 2023-10-09T05:16:49.367 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.367 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.367 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.367 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:16:49.367 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.146303+0000", 2023-10-09T05:16:49.367 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:16:49.367 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.367 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.367 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:16:49.367 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.146308+0000", 2023-10-09T05:16:49.368 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.3029999999999999e-06 2023-10-09T05:16:49.368 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.368 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.368 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:16:49.368 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.146325+0000", 2023-10-09T05:16:49.368 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.6390000000000001e-05 2023-10-09T05:16:49.368 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.368 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.368 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:16:49.368 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.146334+0000", 2023-10-09T05:16:49.368 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.6970000000000005e-06 2023-10-09T05:16:49.368 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.369 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.369 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.369 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.157596+0000", 2023-10-09T05:16:49.369 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.011262059 2023-10-09T05:16:49.369 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.369 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.369 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:16:49.369 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.157604+0000", 2023-10-09T05:16:49.369 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.4569999999999999e-06 2023-10-09T05:16:49.369 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.369 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.370 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.370 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.244632+0000", 2023-10-09T05:16:49.370 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.087028678 2023-10-09T05:16:49.370 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.370 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.370 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:16:49.370 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.244848+0000", 2023-10-09T05:16:49.370 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000215808 2023-10-09T05:16:49.370 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.370 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.370 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,4", 2023-10-09T05:16:49.370 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.245100+0000", 2023-10-09T05:16:49.370 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00025201599999999998 2023-10-09T05:16:49.371 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.371 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.371 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:16:49.371 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.253008+0000", 2023-10-09T05:16:49.371 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.007907931 2023-10-09T05:16:49.371 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.371 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.371 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:16:49.371 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.254429+0000", 2023-10-09T05:16:49.371 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.001420925 2023-10-09T05:16:49.371 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.371 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.372 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:16:49.372 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.266267+0000", 2023-10-09T05:16:49.372 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.011837792999999999 2023-10-09T05:16:49.372 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.372 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.372 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:16:49.372 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.266320+0000", 2023-10-09T05:16:49.372 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.3452000000000003e-05 2023-10-09T05:16:49.372 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.372 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.372 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:16:49.372 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.266811+0000", 2023-10-09T05:16:49.373 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00049061899999999997 2023-10-09T05:16:49.373 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.373 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:16:49.373 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.373 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.373 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.373 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35022.0:100491 10.c 10.57e661bc (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e202)", 2023-10-09T05:16:49.373 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:15:40.205661+0000", 2023-10-09T05:16:49.373 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 69.060153200000002, 2023-10-09T05:16:49.373 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.89156803699999998, 2023-10-09T05:16:49.373 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:16:49.373 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "reached pg", 2023-10-09T05:16:49.374 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:16:49.374 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35022", 2023-10-09T05:16:49.374 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2582323369", 2023-10-09T05:16:49.374 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 100491 2023-10-09T05:16:49.374 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.374 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:16:49.374 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.374 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:16:49.374 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.205661+0000", 2023-10-09T05:16:49.374 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.374 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.374 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.375 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:16:49.375 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.205661+0000", 2023-10-09T05:16:49.375 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.375 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.375 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.375 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:16:49.375 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.205659+0000", 2023-10-09T05:16:49.375 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:16:49.375 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.375 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.375 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:16:49.375 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.205896+0000", 2023-10-09T05:16:49.376 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000236102 2023-10-09T05:16:49.376 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.376 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.376 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:16:49.376 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.205910+0000", 2023-10-09T05:16:49.376 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.4212999999999999e-05 2023-10-09T05:16:49.376 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.376 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.376 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:16:49.376 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.205919+0000", 2023-10-09T05:16:49.376 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.2129999999999997e-06 2023-10-09T05:16:49.376 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.376 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.377 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.377 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.205988+0000", 2023-10-09T05:16:49.377 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.8546000000000004e-05 2023-10-09T05:16:49.377 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.377 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.377 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:16:49.377 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.205999+0000", 2023-10-09T05:16:49.377 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.1382e-05 2023-10-09T05:16:49.377 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.377 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.377 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.377 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.097219+0000", 2023-10-09T05:16:49.378 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.891219698 2023-10-09T05:16:49.378 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.378 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.378 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:16:49.378 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.097229+0000", 2023-10-09T05:16:49.378 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.031e-05 2023-10-09T05:16:49.378 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.378 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:16:49.378 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.378 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.378 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.378 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35022.0:100492 10.8 10:10127fd7:::benchmark_data_smithi133_173023_object3056:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e202)", 2023-10-09T05:16:49.379 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:15:40.209973+0000", 2023-10-09T05:16:49.379 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 69.055841325000003, 2023-10-09T05:16:49.379 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.057361341, 2023-10-09T05:16:49.379 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:16:49.379 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:16:49.379 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:16:49.379 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35022", 2023-10-09T05:16:49.379 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2582323369", 2023-10-09T05:16:49.379 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 100492 2023-10-09T05:16:49.379 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.379 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:16:49.379 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.380 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:16:49.380 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.209973+0000", 2023-10-09T05:16:49.380 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.380 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.380 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.380 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:16:49.380 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.209973+0000", 2023-10-09T05:16:49.380 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.380 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.380 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.380 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:16:49.380 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.209971+0000", 2023-10-09T05:16:49.380 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999981 2023-10-09T05:16:49.381 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.381 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.381 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:16:49.381 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.209981+0000", 2023-10-09T05:16:49.381 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.9920000000000002e-06 2023-10-09T05:16:49.381 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.381 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.381 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:16:49.381 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.209996+0000", 2023-10-09T05:16:49.381 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.5636000000000001e-05 2023-10-09T05:16:49.381 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.381 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.382 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:16:49.382 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.210007+0000", 2023-10-09T05:16:49.382 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.0387000000000001e-05 2023-10-09T05:16:49.382 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.382 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.382 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.382 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.210083+0000", 2023-10-09T05:16:49.382 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.5811999999999999e-05 2023-10-09T05:16:49.382 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.382 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.382 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:16:49.382 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.210095+0000", 2023-10-09T05:16:49.383 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.2119999999999999e-05 2023-10-09T05:16:49.383 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.383 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.383 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.383 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.247904+0000", 2023-10-09T05:16:49.383 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.037809376 2023-10-09T05:16:49.383 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.383 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.383 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:16:49.383 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.248135+0000", 2023-10-09T05:16:49.383 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00023052700000000001 2023-10-09T05:16:49.383 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.384 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.384 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,4", 2023-10-09T05:16:49.384 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.248392+0000", 2023-10-09T05:16:49.384 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00025730000000000002 2023-10-09T05:16:49.384 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.384 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.384 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:16:49.384 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.253075+0000", 2023-10-09T05:16:49.384 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0046834889999999999 2023-10-09T05:16:49.384 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.384 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.384 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:16:49.385 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.254509+0000", 2023-10-09T05:16:49.385 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0014330429999999999 2023-10-09T05:16:49.385 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.385 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.385 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:16:49.385 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.266870+0000", 2023-10-09T05:16:49.385 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.012361023 2023-10-09T05:16:49.385 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.385 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.385 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:16:49.385 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.266926+0000", 2023-10-09T05:16:49.385 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.6127e-05 2023-10-09T05:16:49.386 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.386 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.386 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:16:49.386 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.267334+0000", 2023-10-09T05:16:49.386 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00040845999999999998 2023-10-09T05:16:49.386 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.386 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:16:49.386 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.386 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.386 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.386 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35022.0:100507 10.8 10:128dc5ff:::benchmark_data_smithi133_173023_object3071:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e202)", 2023-10-09T05:16:49.387 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:15:40.245871+0000", 2023-10-09T05:16:49.387 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 69.019943523999999, 2023-10-09T05:16:49.387 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.0238061519999999, 2023-10-09T05:16:49.387 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:16:49.387 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:16:49.387 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:16:49.387 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35022", 2023-10-09T05:16:49.387 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2582323369", 2023-10-09T05:16:49.387 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 100507 2023-10-09T05:16:49.387 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.387 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:16:49.387 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.388 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:16:49.388 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.245871+0000", 2023-10-09T05:16:49.388 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.388 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.388 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.388 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:16:49.388 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.245871+0000", 2023-10-09T05:16:49.388 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.388 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.388 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.388 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:16:49.388 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.245868+0000", 2023-10-09T05:16:49.389 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999976 2023-10-09T05:16:49.389 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.389 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.389 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:16:49.389 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.245878+0000", 2023-10-09T05:16:49.389 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.0319000000000001e-05 2023-10-09T05:16:49.389 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.389 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.389 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:16:49.389 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.245902+0000", 2023-10-09T05:16:49.389 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.3254000000000001e-05 2023-10-09T05:16:49.389 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.390 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.390 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:16:49.390 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.245912+0000", 2023-10-09T05:16:49.390 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.0772e-05 2023-10-09T05:16:49.390 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.390 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.390 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.390 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.245979+0000", 2023-10-09T05:16:49.390 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.6631999999999995e-05 2023-10-09T05:16:49.390 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.390 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.390 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:16:49.391 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.245999+0000", 2023-10-09T05:16:49.391 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.0126e-05 2023-10-09T05:16:49.391 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.391 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.391 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.391 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.255233+0000", 2023-10-09T05:16:49.391 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.00923411 2023-10-09T05:16:49.391 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.391 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.391 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:16:49.391 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.255416+0000", 2023-10-09T05:16:49.391 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00018273399999999999 2023-10-09T05:16:49.392 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.392 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.392 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,4", 2023-10-09T05:16:49.392 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.255597+0000", 2023-10-09T05:16:49.392 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00018068399999999999 2023-10-09T05:16:49.392 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.392 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.392 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:16:49.392 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.267393+0000", 2023-10-09T05:16:49.392 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.011796200999999999 2023-10-09T05:16:49.392 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.392 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.392 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:16:49.393 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.267427+0000", 2023-10-09T05:16:49.393 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.4659999999999997e-05 2023-10-09T05:16:49.393 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.393 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.393 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:16:49.393 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.268917+0000", 2023-10-09T05:16:49.393 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0014892009999999999 2023-10-09T05:16:49.393 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.393 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.393 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:16:49.393 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.268970+0000", 2023-10-09T05:16:49.393 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.3674000000000001e-05 2023-10-09T05:16:49.394 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.394 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.394 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:16:49.394 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.269677+0000", 2023-10-09T05:16:49.394 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00070638299999999999 2023-10-09T05:16:49.394 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.394 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:16:49.394 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.394 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.394 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.394 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35022.0:100509 10.c 10.8f29a56c (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e202)", 2023-10-09T05:16:49.394 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:15:40.247958+0000", 2023-10-09T05:16:49.395 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 69.017855892, 2023-10-09T05:16:49.395 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.84930609800000001, 2023-10-09T05:16:49.395 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:16:49.395 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "reached pg", 2023-10-09T05:16:49.395 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:16:49.395 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35022", 2023-10-09T05:16:49.395 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2582323369", 2023-10-09T05:16:49.395 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 100509 2023-10-09T05:16:49.395 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.395 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:16:49.395 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.395 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:16:49.396 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.247958+0000", 2023-10-09T05:16:49.396 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.396 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.396 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.396 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:16:49.396 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.247958+0000", 2023-10-09T05:16:49.396 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.396 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.396 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.396 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:16:49.396 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.247957+0000", 2023-10-09T05:16:49.396 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:16:49.397 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.397 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.397 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:16:49.397 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.247964+0000", 2023-10-09T05:16:49.397 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.1219999999999997e-06 2023-10-09T05:16:49.397 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.397 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.397 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:16:49.397 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.247977+0000", 2023-10-09T05:16:49.397 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.3271000000000001e-05 2023-10-09T05:16:49.397 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.397 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.397 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:16:49.398 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.247985+0000", 2023-10-09T05:16:49.398 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.1860000000000003e-06 2023-10-09T05:16:49.398 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.398 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.398 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.398 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.248044+0000", 2023-10-09T05:16:49.398 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.9488999999999999e-05 2023-10-09T05:16:49.398 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.398 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.398 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:16:49.398 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.248058+0000", 2023-10-09T05:16:49.398 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.4371e-05 2023-10-09T05:16:49.399 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.399 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.399 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.399 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.097254+0000", 2023-10-09T05:16:49.399 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.84919559499999997 2023-10-09T05:16:49.399 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.399 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.399 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:16:49.399 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.097264+0000", 2023-10-09T05:16:49.399 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.0329000000000001e-05 2023-10-09T05:16:49.399 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.399 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:16:49.400 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.400 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.400 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.400 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35022.0:100513 10.8 10:146cfde8:::benchmark_data_smithi133_173023_object3077:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e202)", 2023-10-09T05:16:49.400 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:15:40.256507+0000", 2023-10-09T05:16:49.400 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 69.009307397000001, 2023-10-09T05:16:49.400 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.0251934119999999, 2023-10-09T05:16:49.400 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:16:49.400 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:16:49.400 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:16:49.400 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35022", 2023-10-09T05:16:49.400 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2582323369", 2023-10-09T05:16:49.401 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 100513 2023-10-09T05:16:49.401 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.401 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:16:49.401 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.401 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:16:49.401 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.256507+0000", 2023-10-09T05:16:49.401 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.401 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.401 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.401 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:16:49.401 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.256507+0000", 2023-10-09T05:16:49.401 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.402 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.402 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.402 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:16:49.402 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.256505+0000", 2023-10-09T05:16:49.402 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999981 2023-10-09T05:16:49.402 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.402 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.402 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:16:49.402 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.256514+0000", 2023-10-09T05:16:49.402 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.0410000000000003e-06 2023-10-09T05:16:49.402 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.402 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.403 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:16:49.403 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.256533+0000", 2023-10-09T05:16:49.403 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.9459000000000001e-05 2023-10-09T05:16:49.403 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.403 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.403 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:16:49.403 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.256543+0000", 2023-10-09T05:16:49.403 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.3279999999999995e-06 2023-10-09T05:16:49.403 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.403 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.403 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.403 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.256615+0000", 2023-10-09T05:16:49.403 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.1758e-05 2023-10-09T05:16:49.404 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.404 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.404 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:16:49.404 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:40.256631+0000", 2023-10-09T05:16:49.404 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.6138999999999999e-05 2023-10-09T05:16:49.404 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.404 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.404 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.404 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.261361+0000", 2023-10-09T05:16:49.404 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.0047305179999999 2023-10-09T05:16:49.404 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.404 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.405 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:16:49.405 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.261505+0000", 2023-10-09T05:16:49.405 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00014394700000000001 2023-10-09T05:16:49.405 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.405 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.405 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,4", 2023-10-09T05:16:49.405 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.261653+0000", 2023-10-09T05:16:49.405 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00014737100000000001 2023-10-09T05:16:49.405 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.405 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.405 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:16:49.405 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.267878+0000", 2023-10-09T05:16:49.406 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.006225235 2023-10-09T05:16:49.406 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.406 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.406 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:16:49.406 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.272491+0000", 2023-10-09T05:16:49.406 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0046127900000000003 2023-10-09T05:16:49.406 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.406 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.406 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:16:49.406 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.280722+0000", 2023-10-09T05:16:49.406 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0082316049999999995 2023-10-09T05:16:49.406 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.407 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.407 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:16:49.407 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.280778+0000", 2023-10-09T05:16:49.407 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.6292000000000002e-05 2023-10-09T05:16:49.407 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.407 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.407 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:16:49.407 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:15:41.281700+0000", 2023-10-09T05:16:49.407 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000921651 2023-10-09T05:16:49.407 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.407 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:16:49.407 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.407 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.408 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.408 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35022.0:147125 10.4 10:2ce7f0b7:::benchmark_data_smithi133_173023_object49689:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e209)", 2023-10-09T05:16:49.408 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:16:48.352907+0000", 2023-10-09T05:16:49.408 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.91290755199999996, 2023-10-09T05:16:49.408 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.191321782, 2023-10-09T05:16:49.408 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:16:49.408 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:16:49.408 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:16:49.408 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35022", 2023-10-09T05:16:49.408 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2582323369", 2023-10-09T05:16:49.408 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 147125 2023-10-09T05:16:49.408 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.409 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:16:49.409 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.409 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:16:49.409 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:48.352907+0000", 2023-10-09T05:16:49.409 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.409 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.409 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.409 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:16:49.409 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:48.352907+0000", 2023-10-09T05:16:49.409 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:16:49.409 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.409 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.409 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:16:49.410 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:48.352905+0000", 2023-10-09T05:16:49.410 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:16:49.410 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.410 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.410 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:16:49.410 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:48.352911+0000", 2023-10-09T05:16:49.410 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.4790000000000001e-06 2023-10-09T05:16:49.410 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.410 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.410 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:16:49.410 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:48.352923+0000", 2023-10-09T05:16:49.410 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.1545e-05 2023-10-09T05:16:49.411 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.411 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.411 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:16:49.411 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:48.352930+0000", 2023-10-09T05:16:49.411 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.2359999999999998e-06 2023-10-09T05:16:49.411 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.411 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.411 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:16:49.411 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:48.372854+0000", 2023-10-09T05:16:49.411 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.019923685 2023-10-09T05:16:49.411 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.411 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.412 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:16:49.412 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:48.372965+0000", 2023-10-09T05:16:49.412 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000110963 2023-10-09T05:16:49.412 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.412 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.412 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:16:49.412 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:48.373073+0000", 2023-10-09T05:16:49.412 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000108473 2023-10-09T05:16:49.412 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.412 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.412 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:16:49.412 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:48.533633+0000", 2023-10-09T05:16:49.413 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.160559705 2023-10-09T05:16:49.413 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.413 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.413 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:16:49.413 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:48.537428+0000", 2023-10-09T05:16:49.413 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.003794511 2023-10-09T05:16:49.413 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.413 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.413 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:16:49.413 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:48.543885+0000", 2023-10-09T05:16:49.413 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.006457005 2023-10-09T05:16:49.413 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.414 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.414 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:16:49.414 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:48.544108+0000", 2023-10-09T05:16:49.414 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00022297500000000001 2023-10-09T05:16:49.414 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:16:49.414 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:16:49.414 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:16:49.414 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:16:48.544228+0000", 2023-10-09T05:16:49.414 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000120822 2023-10-09T05:16:49.414 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.414 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:16:49.415 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.415 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:16:49.415 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:16:49.415 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:16:49.416 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:49.416 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:49 smithi133 bash[105589]: cluster 2023-10-09T05:16:47.931892+0000 mgr.y (mgr.25277) 849 : cluster 0 pgmap v867: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 555 op/s; 10064/149874 objects degraded (6.715%); 1266/149874 objects misplaced (0.845%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T05:16:49.417 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:49 smithi133 bash[105589]: cluster 2023-10-09T05:16:48.351006+0000 mon.a (mon.0) 1112 : cluster 3 Health check update: Degraded data redundancy: 10064/149874 objects degraded (6.715%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:49.417 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:49 smithi133 bash[102858]: cluster 2023-10-09T05:16:47.931892+0000 mgr.y (mgr.25277) 849 : cluster 0 pgmap v867: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 555 op/s; 10064/149874 objects degraded (6.715%); 1266/149874 objects misplaced (0.845%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T05:16:49.417 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:49 smithi133 bash[102858]: cluster 2023-10-09T05:16:48.351006+0000 mon.a (mon.0) 1112 : cluster 3 Health check update: Degraded data redundancy: 10064/149874 objects degraded (6.715%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:49.441 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:49.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:49.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:49.548 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:49.580 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:49.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:49.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:49.684 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:49.716 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:49.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:49.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:49.822 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:49.861 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:49.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:49.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:49.968 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:50.001 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:50.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:50.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:50.107 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:50.132 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:50.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:50.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:50.238 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:50.264 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:50.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:50.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:50.368 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:50.398 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:50.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:50.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:50.502 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:50.530 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:50.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:50.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:50.634 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:50.651 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:50.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:50.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:50.753 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:50.791 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:50.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:50.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:50.894 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:50.935 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:50.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:50.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:51.010 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:51 smithi133 bash[105589]: cluster 2023-10-09T05:16:49.932921+0000 mgr.y (mgr.25277) 850 : cluster 0 pgmap v868: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 588 op/s; 10182/147447 objects degraded (6.906%); 1050/147447 objects misplaced (0.712%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T05:16:51.011 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:51 smithi133 bash[102858]: cluster 2023-10-09T05:16:49.932921+0000 mgr.y (mgr.25277) 850 : cluster 0 pgmap v868: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 588 op/s; 10182/147447 objects degraded (6.906%); 1050/147447 objects misplaced (0.712%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T05:16:51.042 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:51.065 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:51.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:51.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:51.169 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:51.197 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:51.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:51.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:51.300 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:51.324 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:51.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:51.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:51.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:51 smithi194 bash[92666]: cluster 2023-10-09T05:16:49.932921+0000 mgr.y (mgr.25277) 850 : cluster 0 pgmap v868: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 588 op/s; 10182/147447 objects degraded (6.906%); 1050/147447 objects misplaced (0.712%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T05:16:51.427 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:51.456 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:51.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:51.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:51.559 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:51.598 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:51.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:51.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:51.706 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:51.732 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:51.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:51.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:51.837 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:51.867 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:51.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:51.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:51.972 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:52.003 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:52.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:52.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:52.113 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:52.149 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:52.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:52.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:52.253 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:52.292 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:52.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:52.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:52.398 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:52.424 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:52.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:52.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:52.529 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:52.559 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:52.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:52.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:52.665 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:52.698 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:52.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:52.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:52.804 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:52.830 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:52.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:52.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:52.844 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T05:16:52.937 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:52.974 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:52.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:52.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:53.015 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:53 smithi194 bash[92666]: cluster 2023-10-09T05:16:51.933704+0000 mgr.y (mgr.25277) 851 : cluster 0 pgmap v869: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 684 op/s; 10182/140040 objects degraded (7.271%); 1050/140040 objects misplaced (0.750%); 1013 KiB/s, 15 objects/s recovering 2023-10-09T05:16:53.083 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:53.095 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:53 smithi133 bash[102858]: cluster 2023-10-09T05:16:51.933704+0000 mgr.y (mgr.25277) 851 : cluster 0 pgmap v869: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 684 op/s; 10182/140040 objects degraded (7.271%); 1050/140040 objects misplaced (0.750%); 1013 KiB/s, 15 objects/s recovering 2023-10-09T05:16:53.095 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:53 smithi133 bash[105589]: cluster 2023-10-09T05:16:51.933704+0000 mgr.y (mgr.25277) 851 : cluster 0 pgmap v869: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 684 op/s; 10182/140040 objects degraded (7.271%); 1050/140040 objects misplaced (0.750%); 1013 KiB/s, 15 objects/s recovering 2023-10-09T05:16:53.110 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:53.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:53.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:53.218 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:53.254 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:53.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:53.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:53.360 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:53.400 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:53.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:53.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:53.506 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:53.531 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:53.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:53.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:53.637 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:53.664 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:53.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:53.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:53.769 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:53.800 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:53.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:53.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:53.870 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:16:53.903 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:53.915 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:16:53.928 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:53.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:53.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:53.954 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:16:54.003 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:16:54.036 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:54.042 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:16:54.076 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:54.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:54.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:54.081 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:16:54.110 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:16:54.156 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:16:54.159 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:16:54.159 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:16:54.159 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:16:54.159 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:16:54.159 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:16:54.160 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:16:54.160 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:16:54.160 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:16:54.178 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:54.197 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:54 smithi133 bash[102858]: cluster 2023-10-09T05:16:53.352393+0000 mon.a (mon.0) 1113 : cluster 3 Health check update: Degraded data redundancy: 10182/140040 objects degraded (7.271%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:54.198 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:54 smithi133 bash[102858]: audit 2023-10-09T05:16:53.809114+0000 mon.c (mon.2) 83 : audit 0 from='client.? 172.21.15.133:0/1460141716' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:16:54.198 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:54 smithi133 bash[102858]: audit 2023-10-09T05:16:53.829198+0000 mon.a (mon.0) 1114 : audit 1 from='client.? 172.21.15.133:0/1915423154' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:16:54.198 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:54 smithi133 bash[105589]: cluster 2023-10-09T05:16:53.352393+0000 mon.a (mon.0) 1113 : cluster 3 Health check update: Degraded data redundancy: 10182/140040 objects degraded (7.271%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:54.198 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:54 smithi133 bash[105589]: audit 2023-10-09T05:16:53.809114+0000 mon.c (mon.2) 83 : audit 0 from='client.? 172.21.15.133:0/1460141716' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:16:54.198 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:54 smithi133 bash[105589]: audit 2023-10-09T05:16:53.829198+0000 mon.a (mon.0) 1114 : audit 1 from='client.? 172.21.15.133:0/1915423154' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:16:54.206 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:54.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:54.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:54.308 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:54.320 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:54 smithi194 bash[92666]: cluster 2023-10-09T05:16:53.352393+0000 mon.a (mon.0) 1113 : cluster 3 Health check update: Degraded data redundancy: 10182/140040 objects degraded (7.271%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:54.320 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:54 smithi194 bash[92666]: audit 2023-10-09T05:16:53.809114+0000 mon.c (mon.2) 83 : audit 0 from='client.? 172.21.15.133:0/1460141716' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:16:54.320 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:54 smithi194 bash[92666]: audit 2023-10-09T05:16:53.829198+0000 mon.a (mon.0) 1114 : audit 1 from='client.? 172.21.15.133:0/1915423154' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:16:54.335 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:54.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:54.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:54.426 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is unset 2023-10-09T05:16:54.441 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:54.449 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:16:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:16:54] "GET /metrics HTTP/1.1" 200 36508 "" "Prometheus/2.43.0" 2023-10-09T05:16:54.466 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:54.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:54.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:54.572 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:54.612 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:54.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:54.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:54.719 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:54.751 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:54.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:54.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:54.856 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:54.882 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:54.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:54.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:54.986 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:55.011 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:55.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:55.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:55.017 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:55 smithi133 bash[102858]: cluster 2023-10-09T05:16:53.934250+0000 mgr.y (mgr.25277) 852 : cluster 0 pgmap v870: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 544 op/s; 10182/140040 objects degraded (7.271%); 1050/140040 objects misplaced (0.750%); 1013 KiB/s, 15 objects/s recovering 2023-10-09T05:16:55.017 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:55 smithi133 bash[102858]: cluster 2023-10-09T05:16:54.401331+0000 mon.a (mon.0) 1115 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:16:55.017 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:55 smithi133 bash[102858]: audit 2023-10-09T05:16:54.418754+0000 mon.a (mon.0) 1116 : audit 1 from='client.? 172.21.15.133:0/1915423154' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:16:55.017 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:55 smithi133 bash[102858]: cluster 2023-10-09T05:16:54.418892+0000 mon.a (mon.0) 1117 : cluster 0 osdmap e210: 8 total, 8 up, 6 in 2023-10-09T05:16:55.017 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:55 smithi133 bash[102858]: cluster 2023-10-09T05:16:54.419038+0000 osd.5 (osd.5) 197 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T05:16:55.017 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:55 smithi133 bash[102858]: cluster 2023-10-09T05:16:54.429790+0000 osd.6 (osd.6) 146 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T05:16:55.018 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:55 smithi133 bash[102858]: cluster 2023-10-09T05:16:54.432004+0000 osd.6 (osd.6) 147 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T05:16:55.116 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:55.123 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:55 smithi194 bash[92666]: cluster 2023-10-09T05:16:53.934250+0000 mgr.y (mgr.25277) 852 : cluster 0 pgmap v870: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 544 op/s; 10182/140040 objects degraded (7.271%); 1050/140040 objects misplaced (0.750%); 1013 KiB/s, 15 objects/s recovering 2023-10-09T05:16:55.124 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:55 smithi194 bash[92666]: cluster 2023-10-09T05:16:54.401331+0000 mon.a (mon.0) 1115 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:16:55.124 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:55 smithi194 bash[92666]: audit 2023-10-09T05:16:54.418754+0000 mon.a (mon.0) 1116 : audit 1 from='client.? 172.21.15.133:0/1915423154' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:16:55.124 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:55 smithi194 bash[92666]: cluster 2023-10-09T05:16:54.418892+0000 mon.a (mon.0) 1117 : cluster 0 osdmap e210: 8 total, 8 up, 6 in 2023-10-09T05:16:55.124 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:55 smithi194 bash[92666]: cluster 2023-10-09T05:16:54.419038+0000 osd.5 (osd.5) 197 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T05:16:55.124 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:55 smithi194 bash[92666]: cluster 2023-10-09T05:16:54.429790+0000 osd.6 (osd.6) 146 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T05:16:55.124 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:55 smithi194 bash[92666]: cluster 2023-10-09T05:16:54.432004+0000 osd.6 (osd.6) 147 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T05:16:55.144 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:55.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:55.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:55.246 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:55.272 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:55.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:55.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:55.321 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:55 smithi133 bash[105589]: cluster 2023-10-09T05:16:53.934250+0000 mgr.y (mgr.25277) 852 : cluster 0 pgmap v870: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 544 op/s; 10182/140040 objects degraded (7.271%); 1050/140040 objects misplaced (0.750%); 1013 KiB/s, 15 objects/s recovering 2023-10-09T05:16:55.321 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:55 smithi133 bash[105589]: cluster 2023-10-09T05:16:54.401331+0000 mon.a (mon.0) 1115 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:16:55.321 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:55 smithi133 bash[105589]: audit 2023-10-09T05:16:54.418754+0000 mon.a (mon.0) 1116 : audit 1 from='client.? 172.21.15.133:0/1915423154' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:16:55.322 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:55 smithi133 bash[105589]: cluster 2023-10-09T05:16:54.418892+0000 mon.a (mon.0) 1117 : cluster 0 osdmap e210: 8 total, 8 up, 6 in 2023-10-09T05:16:55.322 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:55 smithi133 bash[105589]: cluster 2023-10-09T05:16:54.419038+0000 osd.5 (osd.5) 197 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T05:16:55.322 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:55 smithi133 bash[105589]: cluster 2023-10-09T05:16:54.429790+0000 osd.6 (osd.6) 146 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T05:16:55.322 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:55 smithi133 bash[105589]: cluster 2023-10-09T05:16:54.432004+0000 osd.6 (osd.6) 147 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T05:16:55.377 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:55.402 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:55.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:55.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:55.505 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:55.528 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:55.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:55.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:55.633 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:55.663 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:55.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:55.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:55.765 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:55.800 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:55.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:55.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:55.905 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:55.931 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:55.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:55.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:56.033 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:56.061 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:56.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:56.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:56.168 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:56.193 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:56.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:56.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:56.297 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:56.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:56 smithi133 bash[102858]: cluster 2023-10-09T05:16:54.432448+0000 osd.4 (osd.4) 161 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T05:16:56.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:56 smithi133 bash[105589]: cluster 2023-10-09T05:16:54.432448+0000 osd.4 (osd.4) 161 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T05:16:56.323 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:56.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:56.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:56.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:56 smithi194 bash[92666]: cluster 2023-10-09T05:16:54.432448+0000 osd.4 (osd.4) 161 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T05:16:56.427 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:56.459 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:56.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:56.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:56.563 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:56.596 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:56.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:56.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:56.699 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:56.722 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:56.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:56.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:56.825 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:56.858 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:56.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:56.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:56.965 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:56.992 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:56.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:56.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:57.090 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:57 smithi133 bash[102858]: cluster 2023-10-09T05:16:55.934879+0000 mgr.y (mgr.25277) 853 : cluster 0 pgmap v872: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 666 op/s; 10255/131121 objects degraded (7.821%); 865/131121 objects misplaced (0.660%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:16:57.090 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:57 smithi133 bash[105589]: cluster 2023-10-09T05:16:55.934879+0000 mgr.y (mgr.25277) 853 : cluster 0 pgmap v872: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 666 op/s; 10255/131121 objects degraded (7.821%); 865/131121 objects misplaced (0.660%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:16:57.096 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:57.124 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:57.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:57.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:57.227 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:57.235 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:57 smithi194 bash[92666]: cluster 2023-10-09T05:16:55.934879+0000 mgr.y (mgr.25277) 853 : cluster 0 pgmap v872: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 666 op/s; 10255/131121 objects degraded (7.821%); 865/131121 objects misplaced (0.660%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:16:57.252 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:57.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:57.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:57.356 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:57.396 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:57.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:57.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:57.503 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:57.533 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:57.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:57.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:57.636 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:57.668 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:57.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:57.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:57.771 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:57.796 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:57.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:57.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:57.900 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:57.934 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:57.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:57.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:58.036 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:58.068 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:16:58.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:58.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:16:58.173 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:58.213 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:16:58.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:58.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:16:58.316 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:58.347 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:58.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:58.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:58.453 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:58.479 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:58.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:58.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:16:58.586 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:58.615 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:58.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:58.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:58.720 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:58.754 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:58.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:58.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:58.858 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:58.880 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:16:58.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:58.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:16:58.985 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:59.032 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:59.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:59.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:59.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:59 smithi194 bash[92666]: cluster 2023-10-09T05:16:57.935406+0000 mgr.y (mgr.25277) 854 : cluster 0 pgmap v873: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 665 op/s; 10255/129909 objects degraded (7.894%); 865/129909 objects misplaced (0.666%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:16:59.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:59 smithi194 bash[92666]: cluster 2023-10-09T05:16:58.357897+0000 mon.a (mon.0) 1118 : cluster 3 Health check update: Degraded data redundancy: 10255/129909 objects degraded (7.894%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:59.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:16:59 smithi194 bash[92666]: audit 2023-10-09T05:16:58.450287+0000 mon.a (mon.0) 1119 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:16:59.138 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:59.164 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:59.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:59.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:59.269 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:59.279 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:59 smithi133 bash[102858]: cluster 2023-10-09T05:16:57.935406+0000 mgr.y (mgr.25277) 854 : cluster 0 pgmap v873: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 665 op/s; 10255/129909 objects degraded (7.894%); 865/129909 objects misplaced (0.666%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:16:59.279 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:59 smithi133 bash[102858]: cluster 2023-10-09T05:16:58.357897+0000 mon.a (mon.0) 1118 : cluster 3 Health check update: Degraded data redundancy: 10255/129909 objects degraded (7.894%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:59.279 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:16:59 smithi133 bash[102858]: audit 2023-10-09T05:16:58.450287+0000 mon.a (mon.0) 1119 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:16:59.279 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:59 smithi133 bash[105589]: cluster 2023-10-09T05:16:57.935406+0000 mgr.y (mgr.25277) 854 : cluster 0 pgmap v873: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 665 op/s; 10255/129909 objects degraded (7.894%); 865/129909 objects misplaced (0.666%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:16:59.279 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:59 smithi133 bash[105589]: cluster 2023-10-09T05:16:58.357897+0000 mon.a (mon.0) 1118 : cluster 3 Health check update: Degraded data redundancy: 10255/129909 objects degraded (7.894%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:16:59.279 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:16:59 smithi133 bash[105589]: audit 2023-10-09T05:16:58.450287+0000 mon.a (mon.0) 1119 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:16:59.297 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:16:59.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:59.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:16:59.400 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:59.431 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:16:59.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:59.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:16:59.536 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:59.566 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:59.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:59.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:59.670 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:59.694 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:16:59.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:59.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:16:59.800 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:59.827 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:16:59.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:59.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:16:59.933 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:59.964 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:16:59.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:59.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:00.069 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:00.109 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:00.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:00.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:00.214 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:00.252 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:00.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:00.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:00.258 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:17:00.265 INFO:teuthology.orchestra.run.smithi133.stderr:dumped all 2023-10-09T05:17:00.357 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:00.406 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:00.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:00.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:00.512 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:00.539 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:00.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:00.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:00.644 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:00.667 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:00.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:00.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:00.771 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:00.806 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:00.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:00.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:00.912 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:00.937 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:00.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:00.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:01.037 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:01 smithi133 bash[105589]: cluster 2023-10-09T05:16:59.938112+0000 mgr.y (mgr.25277) 855 : cluster 0 pgmap v874: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 668 op/s; 10331/127380 objects degraded (8.110%); 667/127380 objects misplaced (0.524%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T05:17:01.038 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:01 smithi133 bash[105589]: audit 2023-10-09T05:17:00.243691+0000 mgr.y (mgr.25277) 856 : audit 0 from='client.54835 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:17:01.038 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:01 smithi133 bash[102858]: cluster 2023-10-09T05:16:59.938112+0000 mgr.y (mgr.25277) 855 : cluster 0 pgmap v874: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 668 op/s; 10331/127380 objects degraded (8.110%); 667/127380 objects misplaced (0.524%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T05:17:01.038 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:01 smithi133 bash[102858]: audit 2023-10-09T05:17:00.243691+0000 mgr.y (mgr.25277) 856 : audit 0 from='client.54835 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:17:01.040 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:01.068 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:01.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:01.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:01.132 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:17:01.156 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:17:01.172 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:01.220 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:01.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:01.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:01.325 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:01.368 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:01.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:01.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:01.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:01 smithi194 bash[92666]: cluster 2023-10-09T05:16:59.938112+0000 mgr.y (mgr.25277) 855 : cluster 0 pgmap v874: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 668 op/s; 10331/127380 objects degraded (8.110%); 667/127380 objects misplaced (0.524%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T05:17:01.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:01 smithi194 bash[92666]: audit 2023-10-09T05:17:00.243691+0000 mgr.y (mgr.25277) 856 : audit 0 from='client.54835 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:17:01.475 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:01.511 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:01.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:01.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:01.614 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:01.645 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:01.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:01.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:01.749 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:01.782 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:01.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:01.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:01.890 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:01.921 INFO:teuthology.orchestra.run.smithi133.stdout:{"pg_ready":true,"pg_map":{"version":874,"stamp":"2023-10-09T05:16:59.937428+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2781616056,"num_objects":42460,"num_object_clones":0,"num_object_copies":127380,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10331,"num_objects_misplaced":667,"num_objects_unfound":0,"num_objects_dirty":42460,"num_whiteouts":0,"num_read":506,"num_read_kb":931,"num_write":152732,"num_write_kb":6238158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6375,"num_bytes_recovered":417351710,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108202,"ondisk_log_size":108202,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":270,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":12461328,"kb_used_data":8689248,"kb_used_omap":20,"kb_used_meta":3772011,"kb_avail":550009584,"statfs":{"total":575970213888,"available":563209814016,"internally_reserved":0,"allocated":8897789952,"data_stored":8891979551,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":3862539918},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,1,4,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-438370304,"num_objects":-6689,"num_object_clones":0,"num_object_copies":-20067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":149,"num_objects_misplaced":-383,"num_objects_unfound":0,"num_objects_dirty":-6689,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6689,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":234,"num_bytes_recovered":15335424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005339"},"pg_stats":[{"pgid":"3.1a","version":"69'2","reported_seq":210,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.863953+0000","last_change":"2023-10-09T05:09:57.873320+0000","last_active":"2023-10-09T05:16:35.863953+0000","last_peered":"2023-10-09T05:16:35.863953+0000","last_clean":"2023-10-09T05:16:35.863953+0000","last_became_active":"2023-10-09T05:09:57.868376+0000","last_became_peered":"2023-10-09T05:09:57.868376+0000","last_unstale":"2023-10-09T05:16:35.863953+0000","last_undegraded":"2023-10-09T05:16:35.863953+0000","last_fullsized":"2023-10-09T05:16:35.863953+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:27.186610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":188,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.864547+0000","last_change":"2023-10-09T05:14:15.345991+0000","last_active":"2023-10-09T05:16:35.864547+0000","last_peered":"2023-10-09T05:16:35.864547+0000","last_clean":"2023-10-09T05:16:35.864547+0000","last_became_active":"2023-10-09T05:14:15.345511+0000","last_became_peered":"2023-10-09T05:14:15.345511+0000","last_unstale":"2023-10-09T05:16:35.864547+0000","last_undegraded":"2023-10-09T05:16:35.864547+0000","last_fullsized":"2023-10-09T05:16:35.864547+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:49.234025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":71,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.855574+0000","last_change":"2023-10-09T05:09:57.857511+0000","last_active":"2023-10-09T05:16:35.855574+0000","last_peered":"2023-10-09T05:16:35.855574+0000","last_clean":"2023-10-09T05:16:35.855574+0000","last_became_active":"2023-10-09T05:09:57.857193+0000","last_became_peered":"2023-10-09T05:09:57.857193+0000","last_unstale":"2023-10-09T05:16:35.855574+0000","last_undegraded":"2023-10-09T05:16:35.855574+0000","last_fullsized":"2023-10-09T05:16:35.855574+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:32.842980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":195,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.871203+0000","last_change":"2023-10-09T05:15:41.118305+0000","last_active":"2023-10-09T05:16:35.871203+0000","last_peered":"2023-10-09T05:16:35.871203+0000","last_clean":"2023-10-09T05:16:35.871203+0000","last_became_active":"2023-10-09T05:15:41.117975+0000","last_became_peered":"2023-10-09T05:15:41.117975+0000","last_unstale":"2023-10-09T05:16:35.871203+0000","last_undegraded":"2023-10-09T05:16:35.871203+0000","last_fullsized":"2023-10-09T05:16:35.871203+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:03.469849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":198,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:38.973673+0000","last_change":"2023-10-09T05:09:57.849878+0000","last_active":"2023-10-09T05:16:38.973673+0000","last_peered":"2023-10-09T05:16:38.973673+0000","last_clean":"2023-10-09T05:16:38.973673+0000","last_became_active":"2023-10-09T05:09:57.849541+0000","last_became_peered":"2023-10-09T05:09:57.849541+0000","last_unstale":"2023-10-09T05:16:38.973673+0000","last_undegraded":"2023-10-09T05:16:38.973673+0000","last_fullsized":"2023-10-09T05:16:38.973673+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:49:03.330682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":200,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.861482+0000","last_change":"2023-10-09T05:13:03.039637+0000","last_active":"2023-10-09T05:16:35.861482+0000","last_peered":"2023-10-09T05:16:35.861482+0000","last_clean":"2023-10-09T05:16:35.861482+0000","last_became_active":"2023-10-09T05:13:03.039268+0000","last_became_peered":"2023-10-09T05:13:03.039268+0000","last_unstale":"2023-10-09T05:16:35.861482+0000","last_undegraded":"2023-10-09T05:16:35.861482+0000","last_fullsized":"2023-10-09T05:16:35.861482+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:19.875767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":175,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.870742+0000","last_change":"2023-10-09T05:13:03.035264+0000","last_active":"2023-10-09T05:16:35.870742+0000","last_peered":"2023-10-09T05:16:35.870742+0000","last_clean":"2023-10-09T05:16:35.870742+0000","last_became_active":"2023-10-09T05:13:03.034947+0000","last_became_peered":"2023-10-09T05:13:03.034947+0000","last_unstale":"2023-10-09T05:16:35.870742+0000","last_undegraded":"2023-10-09T05:16:35.870742+0000","last_fullsized":"2023-10-09T05:16:35.870742+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:44.842601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":180,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.853964+0000","last_change":"2023-10-09T05:09:57.849695+0000","last_active":"2023-10-09T05:16:35.853964+0000","last_peered":"2023-10-09T05:16:35.853964+0000","last_clean":"2023-10-09T05:16:35.853964+0000","last_became_active":"2023-10-09T05:09:57.848478+0000","last_became_peered":"2023-10-09T05:09:57.848478+0000","last_unstale":"2023-10-09T05:16:35.853964+0000","last_undegraded":"2023-10-09T05:16:35.853964+0000","last_fullsized":"2023-10-09T05:16:35.853964+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:26.115312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":178,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.863727+0000","last_change":"2023-10-09T05:15:41.105234+0000","last_active":"2023-10-09T05:16:35.863727+0000","last_peered":"2023-10-09T05:16:35.863727+0000","last_clean":"2023-10-09T05:16:35.863727+0000","last_became_active":"2023-10-09T05:15:41.104693+0000","last_became_peered":"2023-10-09T05:15:41.104693+0000","last_unstale":"2023-10-09T05:16:35.863727+0000","last_undegraded":"2023-10-09T05:16:35.863727+0000","last_fullsized":"2023-10-09T05:16:35.863727+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:56.682903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":184,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.864709+0000","last_change":"2023-10-09T05:15:41.126268+0000","last_active":"2023-10-09T05:16:35.864709+0000","last_peered":"2023-10-09T05:16:35.864709+0000","last_clean":"2023-10-09T05:16:35.864709+0000","last_became_active":"2023-10-09T05:15:41.125924+0000","last_became_peered":"2023-10-09T05:15:41.125924+0000","last_unstale":"2023-10-09T05:16:35.864709+0000","last_undegraded":"2023-10-09T05:16:35.864709+0000","last_fullsized":"2023-10-09T05:16:35.864709+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:30:20.022340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":64,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.864473+0000","last_change":"2023-10-09T05:15:41.154411+0000","last_active":"2023-10-09T05:16:35.864473+0000","last_peered":"2023-10-09T05:16:35.864473+0000","last_clean":"2023-10-09T05:16:35.864473+0000","last_became_active":"2023-10-09T05:15:41.126702+0000","last_became_peered":"2023-10-09T05:15:41.126702+0000","last_unstale":"2023-10-09T05:16:35.864473+0000","last_undegraded":"2023-10-09T05:16:35.864473+0000","last_fullsized":"2023-10-09T05:16:35.864473+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:51.580016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":70,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.855883+0000","last_change":"2023-10-09T05:09:57.848297+0000","last_active":"2023-10-09T05:16:35.855883+0000","last_peered":"2023-10-09T05:16:35.855883+0000","last_clean":"2023-10-09T05:16:35.855883+0000","last_became_active":"2023-10-09T05:09:57.847987+0000","last_became_peered":"2023-10-09T05:09:57.847987+0000","last_unstale":"2023-10-09T05:16:35.855883+0000","last_undegraded":"2023-10-09T05:16:35.855883+0000","last_fullsized":"2023-10-09T05:16:35.855883+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:16.385601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'23","reported_seq":242,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.866651+0000","last_change":"2023-10-09T05:15:41.217507+0000","last_active":"2023-10-09T05:16:35.866651+0000","last_peered":"2023-10-09T05:16:35.866651+0000","last_clean":"2023-10-09T05:16:35.866651+0000","last_became_active":"2023-10-09T05:15:41.137408+0000","last_became_peered":"2023-10-09T05:15:41.137408+0000","last_unstale":"2023-10-09T05:16:35.866651+0000","last_undegraded":"2023-10-09T05:16:35.866651+0000","last_fullsized":"2023-10-09T05:16:35.866651+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:14.672364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":190,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.864558+0000","last_change":"2023-10-09T05:09:57.868761+0000","last_active":"2023-10-09T05:16:35.864558+0000","last_peered":"2023-10-09T05:16:35.864558+0000","last_clean":"2023-10-09T05:16:35.864558+0000","last_became_active":"2023-10-09T05:09:57.868121+0000","last_became_peered":"2023-10-09T05:09:57.868121+0000","last_unstale":"2023-10-09T05:16:35.864558+0000","last_undegraded":"2023-10-09T05:16:35.864558+0000","last_fullsized":"2023-10-09T05:16:35.864558+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:43.412235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":66,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.864686+0000","last_change":"2023-10-09T05:15:41.207763+0000","last_active":"2023-10-09T05:16:35.864686+0000","last_peered":"2023-10-09T05:16:35.864686+0000","last_clean":"2023-10-09T05:16:35.864686+0000","last_became_active":"2023-10-09T05:15:41.118356+0000","last_became_peered":"2023-10-09T05:15:41.118356+0000","last_unstale":"2023-10-09T05:16:35.864686+0000","last_undegraded":"2023-10-09T05:16:35.864686+0000","last_fullsized":"2023-10-09T05:16:35.864686+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:24.587392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":207,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.866799+0000","last_change":"2023-10-09T05:15:41.108580+0000","last_active":"2023-10-09T05:16:35.866799+0000","last_peered":"2023-10-09T05:16:35.866799+0000","last_clean":"2023-10-09T05:16:35.866799+0000","last_became_active":"2023-10-09T05:15:41.108248+0000","last_became_peered":"2023-10-09T05:15:41.108248+0000","last_unstale":"2023-10-09T05:16:35.866799+0000","last_undegraded":"2023-10-09T05:16:35.866799+0000","last_fullsized":"2023-10-09T05:16:35.866799+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:28:41.740128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":193,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.870502+0000","last_change":"2023-10-09T05:15:41.127748+0000","last_active":"2023-10-09T05:16:35.870502+0000","last_peered":"2023-10-09T05:16:35.870502+0000","last_clean":"2023-10-09T05:16:35.870502+0000","last_became_active":"2023-10-09T05:15:41.127425+0000","last_became_peered":"2023-10-09T05:15:41.127425+0000","last_unstale":"2023-10-09T05:16:35.870502+0000","last_undegraded":"2023-10-09T05:16:35.870502+0000","last_fullsized":"2023-10-09T05:16:35.870502+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:47.895225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":120,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.864624+0000","last_change":"2023-10-09T05:15:41.127088+0000","last_active":"2023-10-09T05:16:35.864624+0000","last_peered":"2023-10-09T05:16:35.864624+0000","last_clean":"2023-10-09T05:16:35.864624+0000","last_became_active":"2023-10-09T05:15:41.126752+0000","last_became_peered":"2023-10-09T05:15:41.126752+0000","last_unstale":"2023-10-09T05:16:35.864624+0000","last_undegraded":"2023-10-09T05:16:35.864624+0000","last_fullsized":"2023-10-09T05:16:35.864624+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:15.831662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":43,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.864107+0000","last_change":"2023-10-09T05:15:41.151137+0000","last_active":"2023-10-09T05:16:35.864107+0000","last_peered":"2023-10-09T05:16:35.864107+0000","last_clean":"2023-10-09T05:16:35.864107+0000","last_became_active":"2023-10-09T05:15:41.150762+0000","last_became_peered":"2023-10-09T05:15:41.150762+0000","last_unstale":"2023-10-09T05:16:35.864107+0000","last_undegraded":"2023-10-09T05:16:35.864107+0000","last_fullsized":"2023-10-09T05:16:35.864107+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:04.112124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":194,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.871213+0000","last_change":"2023-10-09T05:15:41.121586+0000","last_active":"2023-10-09T05:16:35.871213+0000","last_peered":"2023-10-09T05:16:35.871213+0000","last_clean":"2023-10-09T05:16:35.871213+0000","last_became_active":"2023-10-09T05:15:41.118694+0000","last_became_peered":"2023-10-09T05:15:41.118694+0000","last_unstale":"2023-10-09T05:16:35.871213+0000","last_undegraded":"2023-10-09T05:16:35.871213+0000","last_fullsized":"2023-10-09T05:16:35.871213+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:48.566568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":203,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.859892+0000","last_change":"2023-10-09T05:13:03.047339+0000","last_active":"2023-10-09T05:16:35.859892+0000","last_peered":"2023-10-09T05:16:35.859892+0000","last_clean":"2023-10-09T05:16:35.859892+0000","last_became_active":"2023-10-09T05:13:03.040038+0000","last_became_peered":"2023-10-09T05:13:03.040038+0000","last_unstale":"2023-10-09T05:16:35.859892+0000","last_undegraded":"2023-10-09T05:16:35.859892+0000","last_fullsized":"2023-10-09T05:16:35.859892+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:56.726249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":207,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:38.974442+0000","last_change":"2023-10-09T05:15:41.111547+0000","last_active":"2023-10-09T05:16:38.974442+0000","last_peered":"2023-10-09T05:16:38.974442+0000","last_clean":"2023-10-09T05:16:38.974442+0000","last_became_active":"2023-10-09T05:15:41.108466+0000","last_became_peered":"2023-10-09T05:15:41.108466+0000","last_unstale":"2023-10-09T05:16:38.974442+0000","last_undegraded":"2023-10-09T05:16:38.974442+0000","last_fullsized":"2023-10-09T05:16:38.974442+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:01:12.583497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"210'9527","reported_seq":9567,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T05:16:54.933238+0000","last_change":"2023-10-09T05:13:41.269806+0000","last_active":"2023-10-09T05:16:54.933238+0000","last_peered":"2023-10-09T05:16:54.933238+0000","last_clean":"2023-10-09T05:16:54.933238+0000","last_became_active":"2023-10-09T05:13:41.269499+0000","last_became_peered":"2023-10-09T05:13:41.269499+0000","last_unstale":"2023-10-09T05:16:54.933238+0000","last_undegraded":"2023-10-09T05:16:54.933238+0000","last_fullsized":"2023-10-09T05:16:54.933238+0000","mapping_epoch":179,"log_start":"190'3557","ondisk_log_start":"190'3557","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":5970,"log_dups_size":0,"ondisk_log_size":5970,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:53:31.740811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179765248,"num_objects":2743,"num_object_clones":0,"num_object_copies":8229,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2743,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9527,"num_write_kb":392640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":62,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.864421+0000","last_change":"2023-10-09T05:15:41.190598+0000","last_active":"2023-10-09T05:16:35.864421+0000","last_peered":"2023-10-09T05:16:35.864421+0000","last_clean":"2023-10-09T05:16:35.864421+0000","last_became_active":"2023-10-09T05:15:41.115228+0000","last_became_peered":"2023-10-09T05:15:41.115228+0000","last_unstale":"2023-10-09T05:16:35.864421+0000","last_undegraded":"2023-10-09T05:16:35.864421+0000","last_fullsized":"2023-10-09T05:16:35.864421+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:39.062081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":48,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.859961+0000","last_change":"2023-10-09T05:15:41.123633+0000","last_active":"2023-10-09T05:16:35.859961+0000","last_peered":"2023-10-09T05:16:35.859961+0000","last_clean":"2023-10-09T05:16:35.859961+0000","last_became_active":"2023-10-09T05:15:41.123197+0000","last_became_peered":"2023-10-09T05:15:41.123197+0000","last_unstale":"2023-10-09T05:16:35.859961+0000","last_undegraded":"2023-10-09T05:16:35.859961+0000","last_fullsized":"2023-10-09T05:16:35.859961+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:53.952474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.7","version":"210'9596","reported_seq":18339,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T05:16:55.010041+0000","last_change":"2023-10-09T05:16:35.933149+0000","last_active":"2023-10-09T05:16:55.010041+0000","last_peered":"2023-10-09T05:16:55.010041+0000","last_clean":"2023-10-09T05:16:55.010041+0000","last_became_active":"2023-10-09T05:16:35.867205+0000","last_became_peered":"2023-10-09T05:16:35.867205+0000","last_unstale":"2023-10-09T05:16:55.010041+0000","last_undegraded":"2023-10-09T05:16:55.010041+0000","last_fullsized":"2023-10-09T05:16:55.010041+0000","mapping_epoch":208,"log_start":"190'2700","ondisk_log_start":"190'2700","created":179,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":6896,"log_dups_size":0,"ondisk_log_size":6896,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:53.851870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175505408,"num_objects":2678,"num_object_clones":0,"num_object_copies":8034,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2678,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9596,"num_write_kb":392768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4336,"num_bytes_recovered":284164096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":62,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.864839+0000","last_change":"2023-10-09T05:15:41.210326+0000","last_active":"2023-10-09T05:16:35.864839+0000","last_peered":"2023-10-09T05:16:35.864839+0000","last_clean":"2023-10-09T05:16:35.864839+0000","last_became_active":"2023-10-09T05:15:41.135518+0000","last_became_peered":"2023-10-09T05:15:41.135518+0000","last_unstale":"2023-10-09T05:16:35.864839+0000","last_undegraded":"2023-10-09T05:16:35.864839+0000","last_fullsized":"2023-10-09T05:16:35.864839+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:35.282510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":207,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.863926+0000","last_change":"2023-10-09T05:09:57.868031+0000","last_active":"2023-10-09T05:16:35.863926+0000","last_peered":"2023-10-09T05:16:35.863926+0000","last_clean":"2023-10-09T05:16:35.863926+0000","last_became_active":"2023-10-09T05:09:57.866477+0000","last_became_peered":"2023-10-09T05:09:57.866477+0000","last_unstale":"2023-10-09T05:16:35.863926+0000","last_undegraded":"2023-10-09T05:16:35.863926+0000","last_fullsized":"2023-10-09T05:16:35.863926+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:27.102583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"210'9407","reported_seq":9448,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T05:16:54.940119+0000","last_change":"2023-10-09T05:13:41.270633+0000","last_active":"2023-10-09T05:16:54.940119+0000","last_peered":"2023-10-09T05:16:54.940119+0000","last_clean":"2023-10-09T05:16:54.940119+0000","last_became_active":"2023-10-09T05:13:41.270167+0000","last_became_peered":"2023-10-09T05:13:41.270167+0000","last_unstale":"2023-10-09T05:16:54.940119+0000","last_undegraded":"2023-10-09T05:16:54.940119+0000","last_fullsized":"2023-10-09T05:16:54.940119+0000","mapping_epoch":179,"log_start":"190'3477","ondisk_log_start":"190'3477","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":5930,"log_dups_size":0,"ondisk_log_size":5930,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:29.153072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176881664,"num_objects":2699,"num_object_clones":0,"num_object_copies":8097,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2699,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9407,"num_write_kb":387392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":62,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.860998+0000","last_change":"2023-10-09T05:15:41.143716+0000","last_active":"2023-10-09T05:16:35.860998+0000","last_peered":"2023-10-09T05:16:35.860998+0000","last_clean":"2023-10-09T05:16:35.860998+0000","last_became_active":"2023-10-09T05:15:41.127905+0000","last_became_peered":"2023-10-09T05:15:41.127905+0000","last_unstale":"2023-10-09T05:16:35.860998+0000","last_undegraded":"2023-10-09T05:16:35.860998+0000","last_fullsized":"2023-10-09T05:16:35.860998+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:19:16.919154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":230,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.860345+0000","last_change":"2023-10-09T05:15:41.117071+0000","last_active":"2023-10-09T05:16:35.860345+0000","last_peered":"2023-10-09T05:16:35.860345+0000","last_clean":"2023-10-09T05:16:35.860345+0000","last_became_active":"2023-10-09T05:15:41.116599+0000","last_became_peered":"2023-10-09T05:15:41.116599+0000","last_unstale":"2023-10-09T05:16:35.860345+0000","last_undegraded":"2023-10-09T05:16:35.860345+0000","last_fullsized":"2023-10-09T05:16:35.860345+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:02.978515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"210'9577","reported_seq":9618,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T05:16:56.512343+0000","last_change":"2023-10-09T05:13:41.263623+0000","last_active":"2023-10-09T05:16:56.512343+0000","last_peered":"2023-10-09T05:16:56.512343+0000","last_clean":"2023-10-09T05:16:56.512343+0000","last_became_active":"2023-10-09T05:13:41.263309+0000","last_became_peered":"2023-10-09T05:13:41.263309+0000","last_unstale":"2023-10-09T05:16:56.512343+0000","last_undegraded":"2023-10-09T05:16:56.512343+0000","last_fullsized":"2023-10-09T05:16:56.512343+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":9577,"log_dups_size":0,"ondisk_log_size":9577,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:08.600564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170852352,"num_objects":2607,"num_object_clones":0,"num_object_copies":7821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9577,"num_write_kb":389888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":126,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.864418+0000","last_change":"2023-10-09T05:13:03.037926+0000","last_active":"2023-10-09T05:16:35.864418+0000","last_peered":"2023-10-09T05:16:35.864418+0000","last_clean":"2023-10-09T05:16:35.864418+0000","last_became_active":"2023-10-09T05:13:03.037639+0000","last_became_peered":"2023-10-09T05:13:03.037639+0000","last_unstale":"2023-10-09T05:16:35.864418+0000","last_undegraded":"2023-10-09T05:16:35.864418+0000","last_fullsized":"2023-10-09T05:16:35.864418+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:52.977143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":185,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.871121+0000","last_change":"2023-10-09T05:09:57.852914+0000","last_active":"2023-10-09T05:16:35.871121+0000","last_peered":"2023-10-09T05:16:35.871121+0000","last_clean":"2023-10-09T05:16:35.871121+0000","last_became_active":"2023-10-09T05:09:57.852522+0000","last_became_peered":"2023-10-09T05:09:57.852522+0000","last_unstale":"2023-10-09T05:16:35.871121+0000","last_undegraded":"2023-10-09T05:16:35.871121+0000","last_fullsized":"2023-10-09T05:16:35.871121+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:24.848494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"210'9757","reported_seq":9798,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T05:16:55.658540+0000","last_change":"2023-10-09T05:13:41.269005+0000","last_active":"2023-10-09T05:16:55.658540+0000","last_peered":"2023-10-09T05:16:55.658540+0000","last_clean":"2023-10-09T05:16:55.658540+0000","last_became_active":"2023-10-09T05:13:41.268729+0000","last_became_peered":"2023-10-09T05:13:41.268729+0000","last_unstale":"2023-10-09T05:16:55.658540+0000","last_undegraded":"2023-10-09T05:16:55.658540+0000","last_fullsized":"2023-10-09T05:16:55.658540+0000","mapping_epoch":179,"log_start":"190'3870","ondisk_log_start":"190'3870","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":5887,"log_dups_size":0,"ondisk_log_size":5887,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:56:22.245861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177143808,"num_objects":2703,"num_object_clones":0,"num_object_copies":8109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2703,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9757,"num_write_kb":398720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"69'3","reported_seq":159,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.868814+0000","last_change":"2023-10-09T05:02:09.650579+0000","last_active":"2023-10-09T05:16:35.868814+0000","last_peered":"2023-10-09T05:16:35.868814+0000","last_clean":"2023-10-09T05:16:35.868814+0000","last_became_active":"2023-10-09T05:02:09.650160+0000","last_became_peered":"2023-10-09T05:02:09.650160+0000","last_unstale":"2023-10-09T05:16:35.868814+0000","last_undegraded":"2023-10-09T05:16:35.868814+0000","last_fullsized":"2023-10-09T05:16:35.868814+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:18.292670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":170,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.865739+0000","last_change":"2023-10-09T05:02:09.652449+0000","last_active":"2023-10-09T05:16:35.865739+0000","last_peered":"2023-10-09T05:16:35.865739+0000","last_clean":"2023-10-09T05:16:35.865739+0000","last_became_active":"2023-10-09T05:02:09.651994+0000","last_became_peered":"2023-10-09T05:02:09.651994+0000","last_unstale":"2023-10-09T05:16:35.865739+0000","last_undegraded":"2023-10-09T05:16:35.865739+0000","last_fullsized":"2023-10-09T05:16:35.865739+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:14.480513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"210'9555","reported_seq":12369,"reported_epoch":210,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:16:59.011052+0000","last_change":"2023-10-09T05:16:34.194233+0000","last_active":"2023-10-09T05:16:59.011052+0000","last_peered":"2023-10-09T05:16:59.011052+0000","last_clean":"2023-10-09T05:14:49.931887+0000","last_became_active":"2023-10-09T05:14:52.187809+0000","last_became_peered":"2023-10-09T05:14:52.187809+0000","last_unstale":"2023-10-09T05:16:59.011052+0000","last_undegraded":"2023-10-09T05:15:41.209633+0000","last_fullsized":"2023-10-09T05:14:51.971097+0000","mapping_epoch":193,"log_start":"190'3100","ondisk_log_start":"190'3100","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":6455,"log_dups_size":0,"ondisk_log_size":6455,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:44.859098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170459136,"num_objects":2601,"num_object_clones":0,"num_object_copies":7803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2104,"num_objects_misplaced":667,"num_objects_unfound":0,"num_objects_dirty":2601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9555,"num_write_kb":388992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1377,"num_bytes_recovered":90243072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":667},{"shards":"3,4","objects":-170}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":191,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.860632+0000","last_change":"2023-10-09T05:09:57.849784+0000","last_active":"2023-10-09T05:16:35.860632+0000","last_peered":"2023-10-09T05:16:35.860632+0000","last_clean":"2023-10-09T05:16:35.860632+0000","last_became_active":"2023-10-09T05:09:57.849216+0000","last_became_peered":"2023-10-09T05:09:57.849216+0000","last_unstale":"2023-10-09T05:16:35.860632+0000","last_undegraded":"2023-10-09T05:16:35.860632+0000","last_fullsized":"2023-10-09T05:16:35.860632+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:29.764343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":217,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.864330+0000","last_change":"2023-10-09T05:14:50.981149+0000","last_active":"2023-10-09T05:16:35.864330+0000","last_peered":"2023-10-09T05:16:35.864330+0000","last_clean":"2023-10-09T05:16:35.864330+0000","last_became_active":"2023-10-09T05:14:50.980163+0000","last_became_peered":"2023-10-09T05:14:50.980163+0000","last_unstale":"2023-10-09T05:16:35.864330+0000","last_undegraded":"2023-10-09T05:16:35.864330+0000","last_fullsized":"2023-10-09T05:16:35.864330+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:40.014674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"210'9450","reported_seq":9485,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T05:16:55.022019+0000","last_change":"2023-10-09T05:13:41.273384+0000","last_active":"2023-10-09T05:16:55.022019+0000","last_peered":"2023-10-09T05:16:55.022019+0000","last_clean":"2023-10-09T05:16:55.022019+0000","last_became_active":"2023-10-09T05:13:41.271638+0000","last_became_peered":"2023-10-09T05:13:41.271638+0000","last_unstale":"2023-10-09T05:16:55.022019+0000","last_undegraded":"2023-10-09T05:16:55.022019+0000","last_fullsized":"2023-10-09T05:16:55.022019+0000","mapping_epoch":179,"log_start":"190'2600","ondisk_log_start":"190'2600","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":6850,"log_dups_size":0,"ondisk_log_size":6850,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:36:47.447983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181403648,"num_objects":2768,"num_object_clones":0,"num_object_copies":8304,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2768,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9450,"num_write_kb":390976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":201,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.863837+0000","last_change":"2023-10-09T05:15:41.129613+0000","last_active":"2023-10-09T05:16:35.863837+0000","last_peered":"2023-10-09T05:16:35.863837+0000","last_clean":"2023-10-09T05:16:35.863837+0000","last_became_active":"2023-10-09T05:15:41.129321+0000","last_became_peered":"2023-10-09T05:15:41.129321+0000","last_unstale":"2023-10-09T05:16:35.863837+0000","last_undegraded":"2023-10-09T05:16:35.863837+0000","last_fullsized":"2023-10-09T05:16:35.863837+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:03:07.043445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":190,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.864432+0000","last_change":"2023-10-09T05:09:57.868378+0000","last_active":"2023-10-09T05:16:35.864432+0000","last_peered":"2023-10-09T05:16:35.864432+0000","last_clean":"2023-10-09T05:16:35.864432+0000","last_became_active":"2023-10-09T05:09:57.867507+0000","last_became_peered":"2023-10-09T05:09:57.867507+0000","last_unstale":"2023-10-09T05:16:35.864432+0000","last_undegraded":"2023-10-09T05:16:35.864432+0000","last_fullsized":"2023-10-09T05:16:35.864432+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:14:35.199631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"210'9701","reported_seq":9767,"reported_epoch":210,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:16:59.515580+0000","last_change":"2023-10-09T05:15:41.236917+0000","last_active":"2023-10-09T05:16:59.515580+0000","last_peered":"2023-10-09T05:16:59.515580+0000","last_clean":"2023-10-09T05:15:11.955351+0000","last_became_active":"2023-10-09T05:15:41.222692+0000","last_became_peered":"2023-10-09T05:15:41.222692+0000","last_unstale":"2023-10-09T05:16:59.515580+0000","last_undegraded":"2023-10-09T05:15:41.127560+0000","last_fullsized":"2023-10-09T05:15:41.100880+0000","mapping_epoch":202,"log_start":"190'3874","ondisk_log_start":"190'3874","created":179,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":5827,"log_dups_size":0,"ondisk_log_size":5827,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:36:21.188540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160366592,"num_objects":2447,"num_object_clones":0,"num_object_copies":7341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":982,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2447,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9701,"num_write_kb":388736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":1634},{"shards":"0,4,7","objects":811}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":208,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.853820+0000","last_change":"2023-10-09T05:15:41.121292+0000","last_active":"2023-10-09T05:16:35.853820+0000","last_peered":"2023-10-09T05:16:35.853820+0000","last_clean":"2023-10-09T05:16:35.853820+0000","last_became_active":"2023-10-09T05:15:41.120913+0000","last_became_peered":"2023-10-09T05:15:41.120913+0000","last_unstale":"2023-10-09T05:16:35.853820+0000","last_undegraded":"2023-10-09T05:16:35.853820+0000","last_fullsized":"2023-10-09T05:16:35.853820+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:20.270641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":72,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:50.748110+0000","last_change":"2023-10-09T05:09:57.856764+0000","last_active":"2023-10-09T05:16:50.748110+0000","last_peered":"2023-10-09T05:16:50.748110+0000","last_clean":"2023-10-09T05:16:50.748110+0000","last_became_active":"2023-10-09T05:09:57.856461+0000","last_became_peered":"2023-10-09T05:09:57.856461+0000","last_unstale":"2023-10-09T05:16:50.748110+0000","last_undegraded":"2023-10-09T05:16:50.748110+0000","last_fullsized":"2023-10-09T05:16:50.748110+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:06.848082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"210'9592","reported_seq":9641,"reported_epoch":210,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:16:59.586843+0000","last_change":"2023-10-09T05:15:42.397778+0000","last_active":"2023-10-09T05:16:59.586843+0000","last_peered":"2023-10-09T05:16:59.586843+0000","last_clean":"2023-10-09T05:15:11.925322+0000","last_became_active":"2023-10-09T05:15:42.378312+0000","last_became_peered":"2023-10-09T05:15:42.378312+0000","last_unstale":"2023-10-09T05:16:59.586843+0000","last_undegraded":"2023-10-09T05:15:42.119765+0000","last_fullsized":"2023-10-09T05:15:42.108930+0000","mapping_epoch":203,"log_start":"190'3764","ondisk_log_start":"190'3764","created":179,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":5828,"log_dups_size":0,"ondisk_log_size":5828,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:44:28.758473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163446784,"num_objects":2494,"num_object_clones":0,"num_object_copies":7482,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1036,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2494,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9592,"num_write_kb":386752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":1648},{"shards":"0,4,7","objects":846}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":99,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.856038+0000","last_change":"2023-10-09T05:15:41.098919+0000","last_active":"2023-10-09T05:16:35.856038+0000","last_peered":"2023-10-09T05:16:35.856038+0000","last_clean":"2023-10-09T05:16:35.856038+0000","last_became_active":"2023-10-09T05:15:41.098592+0000","last_became_peered":"2023-10-09T05:15:41.098592+0000","last_unstale":"2023-10-09T05:16:35.856038+0000","last_undegraded":"2023-10-09T05:16:35.856038+0000","last_fullsized":"2023-10-09T05:16:35.856038+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:25:09.306798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":143,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.864296+0000","last_change":"2023-10-09T05:13:03.031243+0000","last_active":"2023-10-09T05:16:35.864296+0000","last_peered":"2023-10-09T05:16:35.864296+0000","last_clean":"2023-10-09T05:16:35.864296+0000","last_became_active":"2023-10-09T05:13:03.028769+0000","last_became_peered":"2023-10-09T05:13:03.028769+0000","last_unstale":"2023-10-09T05:16:35.864296+0000","last_undegraded":"2023-10-09T05:16:35.864296+0000","last_fullsized":"2023-10-09T05:16:35.864296+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:25:25.239799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":80,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.855994+0000","last_change":"2023-10-09T05:13:03.033358+0000","last_active":"2023-10-09T05:16:35.855994+0000","last_peered":"2023-10-09T05:16:35.855994+0000","last_clean":"2023-10-09T05:16:35.855994+0000","last_became_active":"2023-10-09T05:13:03.032967+0000","last_became_peered":"2023-10-09T05:13:03.032967+0000","last_unstale":"2023-10-09T05:16:35.855994+0000","last_undegraded":"2023-10-09T05:16:35.855994+0000","last_fullsized":"2023-10-09T05:16:35.855994+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:33.726009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"210'9604","reported_seq":9643,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T05:16:54.982696+0000","last_change":"2023-10-09T05:13:41.268554+0000","last_active":"2023-10-09T05:16:54.982696+0000","last_peered":"2023-10-09T05:16:54.982696+0000","last_clean":"2023-10-09T05:16:54.982696+0000","last_became_active":"2023-10-09T05:13:41.268278+0000","last_became_peered":"2023-10-09T05:13:41.268278+0000","last_unstale":"2023-10-09T05:16:54.982696+0000","last_undegraded":"2023-10-09T05:16:54.982696+0000","last_fullsized":"2023-10-09T05:16:54.982696+0000","mapping_epoch":179,"log_start":"190'2700","ondisk_log_start":"190'2700","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":6904,"log_dups_size":0,"ondisk_log_size":6904,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:53.822644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177078272,"num_objects":2702,"num_object_clones":0,"num_object_copies":8106,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2702,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9604,"num_write_kb":393792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":177,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.855572+0000","last_change":"2023-10-09T05:13:03.031386+0000","last_active":"2023-10-09T05:16:35.855572+0000","last_peered":"2023-10-09T05:16:35.855572+0000","last_clean":"2023-10-09T05:16:35.855572+0000","last_became_active":"2023-10-09T05:13:03.030816+0000","last_became_peered":"2023-10-09T05:13:03.030816+0000","last_unstale":"2023-10-09T05:16:35.855572+0000","last_undegraded":"2023-10-09T05:16:35.855572+0000","last_fullsized":"2023-10-09T05:16:35.855572+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:28.088661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":171,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.864427+0000","last_change":"2023-10-09T04:56:56.087320+0000","last_active":"2023-10-09T05:16:35.864427+0000","last_peered":"2023-10-09T05:16:35.864427+0000","last_clean":"2023-10-09T05:16:35.864427+0000","last_became_active":"2023-10-09T04:56:56.086552+0000","last_became_peered":"2023-10-09T04:56:56.086552+0000","last_unstale":"2023-10-09T05:16:35.864427+0000","last_undegraded":"2023-10-09T05:16:35.864427+0000","last_fullsized":"2023-10-09T05:16:35.864427+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:14.205288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":81,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.864363+0000","last_change":"2023-10-09T05:14:50.972904+0000","last_active":"2023-10-09T05:16:35.864363+0000","last_peered":"2023-10-09T05:16:35.864363+0000","last_clean":"2023-10-09T05:16:35.864363+0000","last_became_active":"2023-10-09T05:14:50.972487+0000","last_became_peered":"2023-10-09T05:14:50.972487+0000","last_unstale":"2023-10-09T05:16:35.864363+0000","last_undegraded":"2023-10-09T05:16:35.864363+0000","last_fullsized":"2023-10-09T05:16:35.864363+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:35:23.517359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"210'9487","reported_seq":9552,"reported_epoch":210,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:16:56.477729+0000","last_change":"2023-10-09T05:15:41.190211+0000","last_active":"2023-10-09T05:16:56.477729+0000","last_peered":"2023-10-09T05:16:56.477729+0000","last_clean":"2023-10-09T05:15:11.887753+0000","last_became_active":"2023-10-09T05:15:41.181391+0000","last_became_peered":"2023-10-09T05:15:41.181391+0000","last_unstale":"2023-10-09T05:16:56.477729+0000","last_undegraded":"2023-10-09T05:15:41.107106+0000","last_fullsized":"2023-10-09T05:15:41.094646+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":179,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":9487,"log_dups_size":0,"ondisk_log_size":9487,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:20.531558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169410560,"num_objects":2585,"num_object_clones":0,"num_object_copies":7755,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1031,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2585,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9487,"num_write_kb":386304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":1735},{"shards":"2,4,7","objects":850}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":120,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T05:16:54.424284+0000","last_change":"2023-10-09T05:14:50.977695+0000","last_active":"2023-10-09T05:16:54.424284+0000","last_peered":"2023-10-09T05:16:54.424284+0000","last_clean":"2023-10-09T05:16:54.424284+0000","last_became_active":"2023-10-09T05:14:50.977126+0000","last_became_peered":"2023-10-09T05:14:50.977126+0000","last_unstale":"2023-10-09T05:16:54.424284+0000","last_undegraded":"2023-10-09T05:16:54.424284+0000","last_fullsized":"2023-10-09T05:16:54.424284+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:50:57.273402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":72,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.855574+0000","last_change":"2023-10-09T05:09:57.854064+0000","last_active":"2023-10-09T05:16:35.855574+0000","last_peered":"2023-10-09T05:16:35.855574+0000","last_clean":"2023-10-09T05:16:35.855574+0000","last_became_active":"2023-10-09T05:09:57.847953+0000","last_became_peered":"2023-10-09T05:09:57.847953+0000","last_unstale":"2023-10-09T05:16:35.855574+0000","last_undegraded":"2023-10-09T05:16:35.855574+0000","last_fullsized":"2023-10-09T05:16:35.855574+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:45.914976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":51,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.936034+0000","last_change":"2023-10-09T05:15:41.120111+0000","last_active":"2023-10-09T05:16:35.936034+0000","last_peered":"2023-10-09T05:16:35.936034+0000","last_clean":"2023-10-09T05:16:35.936034+0000","last_became_active":"2023-10-09T05:15:41.119795+0000","last_became_peered":"2023-10-09T05:15:41.119795+0000","last_unstale":"2023-10-09T05:16:35.936034+0000","last_undegraded":"2023-10-09T05:16:35.936034+0000","last_fullsized":"2023-10-09T05:16:35.936034+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:42.300486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.b","version":"210'9251","reported_seq":9290,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T05:16:54.990150+0000","last_change":"2023-10-09T05:13:41.272258+0000","last_active":"2023-10-09T05:16:54.990150+0000","last_peered":"2023-10-09T05:16:54.990150+0000","last_clean":"2023-10-09T05:16:54.990150+0000","last_became_active":"2023-10-09T05:13:41.271935+0000","last_became_peered":"2023-10-09T05:13:41.271935+0000","last_unstale":"2023-10-09T05:16:54.990150+0000","last_undegraded":"2023-10-09T05:16:54.990150+0000","last_fullsized":"2023-10-09T05:16:54.990150+0000","mapping_epoch":179,"log_start":"190'2400","ondisk_log_start":"190'2400","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":6851,"log_dups_size":0,"ondisk_log_size":6851,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:23:02.858628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174784512,"num_objects":2667,"num_object_clones":0,"num_object_copies":8001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9251,"num_write_kb":381376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":140,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.936030+0000","last_change":"2023-10-09T05:15:41.119772+0000","last_active":"2023-10-09T05:16:35.936030+0000","last_peered":"2023-10-09T05:16:35.936030+0000","last_clean":"2023-10-09T05:16:35.936030+0000","last_became_active":"2023-10-09T05:15:41.119403+0000","last_became_peered":"2023-10-09T05:15:41.119403+0000","last_unstale":"2023-10-09T05:16:35.936030+0000","last_undegraded":"2023-10-09T05:16:35.936030+0000","last_fullsized":"2023-10-09T05:16:35.936030+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:12:16.579928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":469,"reported_epoch":209,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:16:35.870170+0000","last_change":"2023-10-09T05:15:42.111959+0000","last_active":"2023-10-09T05:16:35.870170+0000","last_peered":"2023-10-09T05:16:35.870170+0000","last_clean":"2023-10-09T05:15:05.934157+0000","last_became_active":"2023-10-09T05:15:42.111532+0000","last_became_peered":"2023-10-09T05:15:42.111532+0000","last_unstale":"2023-10-09T05:16:35.870170+0000","last_undegraded":"2023-10-09T05:15:42.103440+0000","last_fullsized":"2023-10-09T05:15:42.100931+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_clean_scrub_stamp":"2023-10-09T04:36:44.261013+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:16.948809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":167,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:43.975014+0000","last_change":"2023-10-09T05:13:03.027260+0000","last_active":"2023-10-09T05:16:43.975014+0000","last_peered":"2023-10-09T05:16:43.975014+0000","last_clean":"2023-10-09T05:16:43.975014+0000","last_became_active":"2023-10-09T05:13:03.026416+0000","last_became_peered":"2023-10-09T05:13:03.026416+0000","last_unstale":"2023-10-09T05:16:43.975014+0000","last_undegraded":"2023-10-09T05:16:43.975014+0000","last_fullsized":"2023-10-09T05:16:43.975014+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:05.057338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":183,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.860791+0000","last_change":"2023-10-09T05:15:41.119704+0000","last_active":"2023-10-09T05:16:35.860791+0000","last_peered":"2023-10-09T05:16:35.860791+0000","last_clean":"2023-10-09T05:16:35.860791+0000","last_became_active":"2023-10-09T05:15:41.118928+0000","last_became_peered":"2023-10-09T05:15:41.118928+0000","last_unstale":"2023-10-09T05:16:35.860791+0000","last_undegraded":"2023-10-09T05:16:35.860791+0000","last_fullsized":"2023-10-09T05:16:35.860791+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:47:24.556059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"210'9503","reported_seq":10785,"reported_epoch":210,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:16:54.885547+0000","last_change":"2023-10-09T05:15:41.235565+0000","last_active":"2023-10-09T05:16:54.885547+0000","last_peered":"2023-10-09T05:16:54.885547+0000","last_clean":"2023-10-09T05:15:11.934574+0000","last_became_active":"2023-10-09T05:15:41.227723+0000","last_became_peered":"2023-10-09T05:15:41.227723+0000","last_unstale":"2023-10-09T05:16:54.885547+0000","last_undegraded":"2023-10-09T05:15:41.147342+0000","last_fullsized":"2023-10-09T05:15:41.096227+0000","mapping_epoch":202,"log_start":"190'3544","ondisk_log_start":"190'3544","created":179,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":5959,"log_dups_size":0,"ondisk_log_size":5959,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:04:18.080835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178847744,"num_objects":2729,"num_object_clones":0,"num_object_copies":8187,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1017,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2729,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9503,"num_write_kb":391424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":607,"num_bytes_recovered":39714816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":1921},{"shards":"0,2,7","objects":808}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":186,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.855355+0000","last_change":"2023-10-09T05:15:41.115902+0000","last_active":"2023-10-09T05:16:35.855355+0000","last_peered":"2023-10-09T05:16:35.855355+0000","last_clean":"2023-10-09T05:16:35.855355+0000","last_became_active":"2023-10-09T05:15:41.115576+0000","last_became_peered":"2023-10-09T05:15:41.115576+0000","last_unstale":"2023-10-09T05:16:35.855355+0000","last_undegraded":"2023-10-09T05:16:35.855355+0000","last_fullsized":"2023-10-09T05:16:35.855355+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:42:21.916563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":167,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.870505+0000","last_change":"2023-10-09T05:02:09.648781+0000","last_active":"2023-10-09T05:16:35.870505+0000","last_peered":"2023-10-09T05:16:35.870505+0000","last_clean":"2023-10-09T05:16:35.870505+0000","last_became_active":"2023-10-09T05:02:09.648078+0000","last_became_peered":"2023-10-09T05:02:09.648078+0000","last_unstale":"2023-10-09T05:16:35.870505+0000","last_undegraded":"2023-10-09T05:16:35.870505+0000","last_fullsized":"2023-10-09T05:16:35.870505+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:35.287091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":228,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.871097+0000","last_change":"2023-10-09T05:15:41.107646+0000","last_active":"2023-10-09T05:16:35.871097+0000","last_peered":"2023-10-09T05:16:35.871097+0000","last_clean":"2023-10-09T05:16:35.871097+0000","last_became_active":"2023-10-09T05:15:41.105974+0000","last_became_peered":"2023-10-09T05:15:41.105974+0000","last_unstale":"2023-10-09T05:16:35.871097+0000","last_undegraded":"2023-10-09T05:16:35.871097+0000","last_fullsized":"2023-10-09T05:16:35.871097+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:01.702393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"210'9360","reported_seq":9400,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T05:16:55.041341+0000","last_change":"2023-10-09T05:13:41.273522+0000","last_active":"2023-10-09T05:16:55.041341+0000","last_peered":"2023-10-09T05:16:55.041341+0000","last_clean":"2023-10-09T05:16:55.041341+0000","last_became_active":"2023-10-09T05:13:41.273159+0000","last_became_peered":"2023-10-09T05:13:41.273159+0000","last_unstale":"2023-10-09T05:16:55.041341+0000","last_undegraded":"2023-10-09T05:16:55.041341+0000","last_fullsized":"2023-10-09T05:16:55.041341+0000","mapping_epoch":179,"log_start":"190'2500","ondisk_log_start":"190'2500","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":6860,"log_dups_size":0,"ondisk_log_size":6860,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:53.695799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178323480,"num_objects":2722,"num_object_clones":0,"num_object_copies":8166,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2722,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9360,"num_write_kb":386561,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":86,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.870874+0000","last_change":"2023-10-09T05:15:41.113499+0000","last_active":"2023-10-09T05:16:35.870874+0000","last_peered":"2023-10-09T05:16:35.870874+0000","last_clean":"2023-10-09T05:16:35.870874+0000","last_became_active":"2023-10-09T05:15:41.107504+0000","last_became_peered":"2023-10-09T05:15:41.107504+0000","last_unstale":"2023-10-09T05:16:35.870874+0000","last_undegraded":"2023-10-09T05:16:35.870874+0000","last_fullsized":"2023-10-09T05:16:35.870874+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:48.704963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":90,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.863929+0000","last_change":"2023-10-09T05:14:50.977609+0000","last_active":"2023-10-09T05:16:35.863929+0000","last_peered":"2023-10-09T05:16:35.863929+0000","last_clean":"2023-10-09T05:16:35.863929+0000","last_became_active":"2023-10-09T05:14:50.977225+0000","last_became_peered":"2023-10-09T05:14:50.977225+0000","last_unstale":"2023-10-09T05:16:35.863929+0000","last_undegraded":"2023-10-09T05:16:35.863929+0000","last_fullsized":"2023-10-09T05:16:35.863929+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:06.711748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":198,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.871326+0000","last_change":"2023-10-09T05:15:41.111318+0000","last_active":"2023-10-09T05:16:35.871326+0000","last_peered":"2023-10-09T05:16:35.871326+0000","last_clean":"2023-10-09T05:16:35.871326+0000","last_became_active":"2023-10-09T05:15:41.110878+0000","last_became_peered":"2023-10-09T05:15:41.110878+0000","last_unstale":"2023-10-09T05:16:35.871326+0000","last_undegraded":"2023-10-09T05:16:35.871326+0000","last_fullsized":"2023-10-09T05:16:35.871326+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:22.225260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"210'9570","reported_seq":9632,"reported_epoch":210,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:16:59.556046+0000","last_change":"2023-10-09T05:15:41.203168+0000","last_active":"2023-10-09T05:16:59.556046+0000","last_peered":"2023-10-09T05:16:59.556046+0000","last_clean":"2023-10-09T05:14:49.945496+0000","last_became_active":"2023-10-09T05:14:52.149107+0000","last_became_peered":"2023-10-09T05:14:52.149107+0000","last_unstale":"2023-10-09T05:16:59.556046+0000","last_undegraded":"2023-10-09T05:15:13.027503+0000","last_fullsized":"2023-10-09T05:14:51.974129+0000","mapping_epoch":193,"log_start":"190'3766","ondisk_log_start":"190'3766","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":5804,"log_dups_size":0,"ondisk_log_size":5804,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:21.041496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166985728,"num_objects":2548,"num_object_clones":0,"num_object_copies":7644,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4159,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2548,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9570,"num_write_kb":387776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":652},{"shards":"4,6","objects":-1611}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":197,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.865470+0000","last_change":"2023-10-09T05:15:41.193256+0000","last_active":"2023-10-09T05:16:35.865470+0000","last_peered":"2023-10-09T05:16:35.865470+0000","last_clean":"2023-10-09T05:16:35.865470+0000","last_became_active":"2023-10-09T05:15:41.137091+0000","last_became_peered":"2023-10-09T05:15:41.137091+0000","last_unstale":"2023-10-09T05:16:35.865470+0000","last_undegraded":"2023-10-09T05:16:35.865470+0000","last_fullsized":"2023-10-09T05:16:35.865470+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:42.820583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":72,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.863989+0000","last_change":"2023-10-09T05:09:57.858617+0000","last_active":"2023-10-09T05:16:35.863989+0000","last_peered":"2023-10-09T05:16:35.863989+0000","last_clean":"2023-10-09T05:16:35.863989+0000","last_became_active":"2023-10-09T05:09:57.858128+0000","last_became_peered":"2023-10-09T05:09:57.858128+0000","last_unstale":"2023-10-09T05:16:35.863989+0000","last_undegraded":"2023-10-09T05:16:35.863989+0000","last_fullsized":"2023-10-09T05:16:35.863989+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:02.502985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":182,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.853877+0000","last_change":"2023-10-09T05:15:41.154789+0000","last_active":"2023-10-09T05:16:35.853877+0000","last_peered":"2023-10-09T05:16:35.853877+0000","last_clean":"2023-10-09T05:16:35.853877+0000","last_became_active":"2023-10-09T05:15:41.154291+0000","last_became_peered":"2023-10-09T05:15:41.154291+0000","last_unstale":"2023-10-09T05:16:35.853877+0000","last_undegraded":"2023-10-09T05:16:35.853877+0000","last_fullsized":"2023-10-09T05:16:35.853877+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:25.269156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"210'9495","reported_seq":9534,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T05:16:55.036090+0000","last_change":"2023-10-09T05:13:41.271193+0000","last_active":"2023-10-09T05:16:55.036090+0000","last_peered":"2023-10-09T05:16:55.036090+0000","last_clean":"2023-10-09T05:16:55.036090+0000","last_became_active":"2023-10-09T05:13:41.269862+0000","last_became_peered":"2023-10-09T05:13:41.269862+0000","last_unstale":"2023-10-09T05:16:55.036090+0000","last_undegraded":"2023-10-09T05:16:55.036090+0000","last_fullsized":"2023-10-09T05:16:55.036090+0000","mapping_epoch":179,"log_start":"190'2600","ondisk_log_start":"190'2600","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:13:40.243953+0000","last_clean_scrub_stamp":"2023-10-09T05:13:40.243953+0000","objects_scrubbed":0,"log_size":6895,"log_dups_size":0,"ondisk_log_size":6895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:36.314853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179765248,"num_objects":2743,"num_object_clones":0,"num_object_copies":8229,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2743,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9495,"num_write_kb":391616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":205,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.861233+0000","last_change":"2023-10-09T05:15:41.123233+0000","last_active":"2023-10-09T05:16:35.861233+0000","last_peered":"2023-10-09T05:16:35.861233+0000","last_clean":"2023-10-09T05:16:35.861233+0000","last_became_active":"2023-10-09T05:15:41.122768+0000","last_became_peered":"2023-10-09T05:15:41.122768+0000","last_unstale":"2023-10-09T05:16:35.861233+0000","last_undegraded":"2023-10-09T05:16:35.861233+0000","last_fullsized":"2023-10-09T05:16:35.861233+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:39.717521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":198,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.871101+0000","last_change":"2023-10-09T05:09:57.878002+0000","last_active":"2023-10-09T05:16:35.871101+0000","last_peered":"2023-10-09T05:16:35.871101+0000","last_clean":"2023-10-09T05:16:35.871101+0000","last_became_active":"2023-10-09T05:09:57.852792+0000","last_became_peered":"2023-10-09T05:09:57.852792+0000","last_unstale":"2023-10-09T05:16:35.871101+0000","last_undegraded":"2023-10-09T05:16:35.871101+0000","last_fullsized":"2023-10-09T05:16:35.871101+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:18.763275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":163,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.863592+0000","last_change":"2023-10-09T05:13:03.026541+0000","last_active":"2023-10-09T05:16:35.863592+0000","last_peered":"2023-10-09T05:16:35.863592+0000","last_clean":"2023-10-09T05:16:35.863592+0000","last_became_active":"2023-10-09T05:13:03.025729+0000","last_became_peered":"2023-10-09T05:13:03.025729+0000","last_unstale":"2023-10-09T05:16:35.863592+0000","last_undegraded":"2023-10-09T05:16:35.863592+0000","last_fullsized":"2023-10-09T05:16:35.863592+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:22.938235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'2","reported_seq":193,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T05:16:54.432474+0000","last_change":"2023-10-09T05:15:41.118210+0000","last_active":"2023-10-09T05:16:54.432474+0000","last_peered":"2023-10-09T05:16:54.432474+0000","last_clean":"2023-10-09T05:16:54.432474+0000","last_became_active":"2023-10-09T05:15:41.116521+0000","last_became_peered":"2023-10-09T05:15:41.116521+0000","last_unstale":"2023-10-09T05:16:54.432474+0000","last_undegraded":"2023-10-09T05:16:54.432474+0000","last_fullsized":"2023-10-09T05:16:54.432474+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:01.287668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":171,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.863812+0000","last_change":"2023-10-09T05:14:50.976630+0000","last_active":"2023-10-09T05:16:35.863812+0000","last_peered":"2023-10-09T05:16:35.863812+0000","last_clean":"2023-10-09T05:16:35.863812+0000","last_became_active":"2023-10-09T05:14:50.976283+0000","last_became_peered":"2023-10-09T05:14:50.976283+0000","last_unstale":"2023-10-09T05:16:35.863812+0000","last_undegraded":"2023-10-09T05:16:35.863812+0000","last_fullsized":"2023-10-09T05:16:35.863812+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:04.614827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":65,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.866873+0000","last_change":"2023-10-09T05:13:03.055887+0000","last_active":"2023-10-09T05:16:35.866873+0000","last_peered":"2023-10-09T05:16:35.866873+0000","last_clean":"2023-10-09T05:16:35.866873+0000","last_became_active":"2023-10-09T05:13:03.028782+0000","last_became_peered":"2023-10-09T05:13:03.028782+0000","last_unstale":"2023-10-09T05:16:35.866873+0000","last_undegraded":"2023-10-09T05:16:35.866873+0000","last_fullsized":"2023-10-09T05:16:35.866873+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:38.828205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":207,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.864828+0000","last_change":"2023-10-09T05:09:57.867865+0000","last_active":"2023-10-09T05:16:35.864828+0000","last_peered":"2023-10-09T05:16:35.864828+0000","last_clean":"2023-10-09T05:16:35.864828+0000","last_became_active":"2023-10-09T05:09:57.866723+0000","last_became_peered":"2023-10-09T05:09:57.866723+0000","last_unstale":"2023-10-09T05:16:35.864828+0000","last_undegraded":"2023-10-09T05:16:35.864828+0000","last_fullsized":"2023-10-09T05:16:35.864828+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:58.946358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":65,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.864451+0000","last_change":"2023-10-09T05:13:03.049546+0000","last_active":"2023-10-09T05:16:35.864451+0000","last_peered":"2023-10-09T05:16:35.864451+0000","last_clean":"2023-10-09T05:16:35.864451+0000","last_became_active":"2023-10-09T05:13:03.028676+0000","last_became_peered":"2023-10-09T05:13:03.028676+0000","last_unstale":"2023-10-09T05:16:35.864451+0000","last_undegraded":"2023-10-09T05:16:35.864451+0000","last_fullsized":"2023-10-09T05:16:35.864451+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:55.089350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":187,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.864260+0000","last_change":"2023-10-09T05:02:09.649748+0000","last_active":"2023-10-09T05:16:35.864260+0000","last_peered":"2023-10-09T05:16:35.864260+0000","last_clean":"2023-10-09T05:16:35.864260+0000","last_became_active":"2023-10-09T05:02:09.649444+0000","last_became_peered":"2023-10-09T05:02:09.649444+0000","last_unstale":"2023-10-09T05:16:35.864260+0000","last_undegraded":"2023-10-09T05:16:35.864260+0000","last_fullsized":"2023-10-09T05:16:35.864260+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:07.436820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'2","reported_seq":185,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.864585+0000","last_change":"2023-10-09T05:14:50.973251+0000","last_active":"2023-10-09T05:16:35.864585+0000","last_peered":"2023-10-09T05:16:35.864585+0000","last_clean":"2023-10-09T05:16:35.864585+0000","last_became_active":"2023-10-09T05:14:50.972870+0000","last_became_peered":"2023-10-09T05:14:50.972870+0000","last_unstale":"2023-10-09T05:16:35.864585+0000","last_undegraded":"2023-10-09T05:16:35.864585+0000","last_fullsized":"2023-10-09T05:16:35.864585+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:03.879853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":189,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:38.973700+0000","last_change":"2023-10-09T05:02:09.649674+0000","last_active":"2023-10-09T05:16:38.973700+0000","last_peered":"2023-10-09T05:16:38.973700+0000","last_clean":"2023-10-09T05:16:38.973700+0000","last_became_active":"2023-10-09T05:02:09.649041+0000","last_became_peered":"2023-10-09T05:02:09.649041+0000","last_unstale":"2023-10-09T05:16:38.973700+0000","last_undegraded":"2023-10-09T05:16:38.973700+0000","last_fullsized":"2023-10-09T05:16:38.973700+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:18.524071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'5","reported_seq":207,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.870839+0000","last_change":"2023-10-09T05:14:50.985019+0000","last_active":"2023-10-09T05:16:35.870839+0000","last_peered":"2023-10-09T05:16:35.870839+0000","last_clean":"2023-10-09T05:16:35.870839+0000","last_became_active":"2023-10-09T05:14:50.979874+0000","last_became_peered":"2023-10-09T05:14:50.979874+0000","last_unstale":"2023-10-09T05:16:35.870839+0000","last_undegraded":"2023-10-09T05:16:35.870839+0000","last_fullsized":"2023-10-09T05:16:35.870839+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:47.166329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":190,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T05:16:35.863844+0000","last_change":"2023-10-09T05:09:57.866732+0000","last_active":"2023-10-09T05:16:35.863844+0000","last_peered":"2023-10-09T05:16:35.863844+0000","last_clean":"2023-10-09T05:16:35.863844+0000","last_became_active":"2023-10-09T05:09:57.866117+0000","last_became_peered":"2023-10-09T05:09:57.866117+0000","last_unstale":"2023-10-09T05:16:35.863844+0000","last_undegraded":"2023-10-09T05:16:35.863844+0000","last_fullsized":"2023-10-09T05:16:35.863844+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:49:14.815342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":2781020184,"num_objects":42436,"num_object_clones":0,"num_object_copies":127308,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10329,"num_objects_misplaced":667,"num_objects_unfound":0,"num_objects_dirty":42436,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":152432,"num_write_kb":6235713,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6324,"num_bytes_recovered":414384128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9043456000,"data_stored":9043443784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107980,"ondisk_log_size":107980,"up":48,"acting":42,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":19,"num_read_kb":18,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41,"num_bytes_recovered":15742,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":2,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234417,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":46,"seq":197568496058,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1825140,"kb_used_data":1227180,"kb_used_omap":0,"kb_used_meta":597952,"kb_avail":91920012,"statfs":{"total":95995035648,"available":94126092288,"internally_reserved":0,"allocated":1256632320,"data_stored":1255891067,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612302848},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:16:32 2023","interfaces":[{"interface":"back","average":{"1min":7.4180000000000001,"5min":2.8119999999999998,"15min":2.1960000000000002},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":80.391999999999996,"5min":80.391999999999996,"15min":98.275000000000006},"last":2.8700000000000001},{"interface":"front","average":{"1min":7.4900000000000002,"5min":2.919,"15min":2.3180000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.30399999999999999},"max":{"1min":80.453999999999994,"5min":80.453999999999994,"15min":98.334999999999994},"last":2.8300000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:16:32 2023","interfaces":[{"interface":"back","average":{"1min":1.6299999999999999,"5min":2.2850000000000001,"15min":1.6990000000000001},"min":{"1min":0.30199999999999999,"5min":0.26700000000000002,"15min":0.25800000000000001},"max":{"1min":6.625,"5min":60.149999999999999,"15min":60.149999999999999},"last":3.629},{"interface":"front","average":{"1min":1.4159999999999999,"5min":2.266,"15min":1.8009999999999999},"min":{"1min":0.32200000000000001,"5min":0.28899999999999998,"15min":0.27900000000000003},"max":{"1min":6.7220000000000004,"5min":60.268000000000001,"15min":60.268000000000001},"last":0.32000000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:16:32 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":1.111,"15min":1.3160000000000001},"min":{"1min":0.31900000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.0140000000000002,"5min":13.568,"15min":13.568},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.6779999999999999,"5min":1.2809999999999999,"15min":1.236},"min":{"1min":0.34100000000000003,"5min":0.32400000000000001,"15min":0.30099999999999999},"max":{"1min":8.4339999999999993,"5min":8.4339999999999993,"15min":8.4339999999999993},"last":0.51800000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:16:32 2023","interfaces":[{"interface":"back","average":{"1min":5.1959999999999997,"5min":4.9000000000000004,"15min":2.754},"min":{"1min":0.32700000000000001,"5min":0.32400000000000001,"15min":0.315},"max":{"1min":80.489999999999995,"5min":80.489999999999995,"15min":80.489999999999995},"last":0.71299999999999997},{"interface":"front","average":{"1min":5.4050000000000002,"5min":5.0519999999999996,"15min":2.746},"min":{"1min":0.34899999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":80.085999999999999,"5min":80.085999999999999,"15min":80.085999999999999},"last":0.40500000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:16:32 2023","interfaces":[{"interface":"back","average":{"1min":1.5720000000000001,"5min":1.2,"15min":1.2390000000000001},"min":{"1min":0.44900000000000001,"5min":0.30299999999999999,"15min":0.28199999999999997},"max":{"1min":4.9630000000000001,"5min":8.3670000000000009,"15min":16.119},"last":0.78200000000000003},{"interface":"front","average":{"1min":1.3759999999999999,"5min":1.224,"15min":1.2989999999999999},"min":{"1min":0.377,"5min":0.35399999999999998,"15min":0.32300000000000001},"max":{"1min":4.0099999999999998,"5min":13.473000000000001,"15min":13.473000000000001},"last":0.73399999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:16:32 2023","interfaces":[{"interface":"back","average":{"1min":1.1890000000000001,"5min":1.1060000000000001,"15min":1.331},"min":{"1min":0.39500000000000002,"5min":0.36399999999999999,"15min":0.312},"max":{"1min":4.2649999999999997,"5min":6.9210000000000003,"15min":7.7149999999999999},"last":0.754},{"interface":"front","average":{"1min":1.3109999999999999,"5min":1.274,"15min":1.5009999999999999},"min":{"1min":0.39500000000000002,"5min":0.378,"15min":0.35899999999999999},"max":{"1min":6.6459999999999999,"5min":13.292999999999999,"15min":13.292999999999999},"last":0.47499999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:16:02 2023","interfaces":[{"interface":"back","average":{"1min":1.627,"5min":1.387,"15min":1.357},"min":{"1min":0.40799999999999997,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":7.8259999999999996,"5min":7.8259999999999996,"15min":9.3780000000000001},"last":0.56499999999999995},{"interface":"front","average":{"1min":1.9490000000000001,"5min":1.5569999999999999,"15min":1.4790000000000001},"min":{"1min":0.47699999999999998,"5min":0.441,"15min":0.434},"max":{"1min":7.8760000000000003,"5min":7.8760000000000003,"15min":9.4060000000000006},"last":0.59599999999999997}]}]},{"osd":1,"up_from":96,"seq":412316860649,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1493168,"kb_used_data":831656,"kb_used_omap":4,"kb_used_meta":661499,"kb_avail":92251984,"statfs":{"total":95995035648,"available":94466031616,"internally_reserved":0,"allocated":851615744,"data_stored":850472309,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":677375886},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:16:21 2023","interfaces":[{"interface":"back","average":{"1min":2.2040000000000002,"5min":1.4890000000000001,"15min":1.514},"min":{"1min":0.223,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":10.866,"5min":10.866,"15min":22.382999999999999},"last":7.5839999999999996},{"interface":"front","average":{"1min":1.1819999999999999,"5min":1.502,"15min":1.5940000000000001},"min":{"1min":0.21199999999999999,"5min":0.20999999999999999,"15min":0.20799999999999999},"max":{"1min":5.1950000000000003,"5min":6.6879999999999997,"15min":10.159000000000001},"last":4.585}]},{"osd":2,"last update":"Mon Oct 9 05:16:33 2023","interfaces":[{"interface":"back","average":{"1min":1.627,"5min":1.1799999999999999,"15min":1.339},"min":{"1min":0.33500000000000002,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":5.6200000000000001,"5min":5.6200000000000001,"15min":10.115},"last":3.161},{"interface":"front","average":{"1min":1.76,"5min":1.401,"15min":1.4410000000000001},"min":{"1min":0.35999999999999999,"5min":0.27700000000000002,"15min":0.252},"max":{"1min":6.3040000000000003,"5min":9.9429999999999996,"15min":11.917},"last":0.29799999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:16:33 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":1.96,"15min":2.1339999999999999},"min":{"1min":0.28000000000000003,"5min":0.252,"15min":0.23100000000000001},"max":{"1min":6.2569999999999997,"5min":46.750999999999998,"15min":61.280000000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.379,"5min":1.837,"15min":1.962},"min":{"1min":0.33000000000000002,"5min":0.26800000000000002,"15min":0.24399999999999999},"max":{"1min":5.2510000000000003,"5min":46.671999999999997,"15min":65.046000000000006},"last":0.38}]},{"osd":4,"last update":"Mon Oct 9 05:16:21 2023","interfaces":[{"interface":"back","average":{"1min":1.573,"5min":1.3600000000000001,"15min":1.395},"min":{"1min":0.41299999999999998,"5min":0.39700000000000002,"15min":0.34300000000000003},"max":{"1min":5.2839999999999998,"5min":9.4830000000000005,"15min":9.9749999999999996},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.74,"5min":1.5569999999999999,"15min":1.514},"min":{"1min":0.317,"5min":0.317,"15min":0.30099999999999999},"max":{"1min":4.1150000000000002,"5min":10.944000000000001,"15min":22.350000000000001},"last":1.716}]},{"osd":5,"last update":"Mon Oct 9 05:16:21 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":1.5800000000000001,"15min":1.7849999999999999},"min":{"1min":0.40000000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.5920000000000001,"5min":8.0090000000000003,"15min":12.454000000000001},"last":2.738},{"interface":"front","average":{"1min":1.4239999999999999,"5min":1.333,"15min":1.6180000000000001},"min":{"1min":0.44400000000000001,"5min":0.38200000000000001,"15min":0.316},"max":{"1min":5.1749999999999998,"5min":5.1749999999999998,"15min":10.035},"last":0.46300000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:16:21 2023","interfaces":[{"interface":"back","average":{"1min":1.552,"5min":1.593,"15min":1.71},"min":{"1min":0.38600000000000001,"5min":0.36399999999999999,"15min":0.34699999999999998},"max":{"1min":4.3630000000000004,"5min":8.5630000000000006,"15min":10.409000000000001},"last":2.9340000000000002},{"interface":"front","average":{"1min":1.7729999999999999,"5min":1.4350000000000001,"15min":1.5649999999999999},"min":{"1min":0.45300000000000001,"5min":0.373,"15min":0.35099999999999998},"max":{"1min":6.327,"5min":8.9540000000000006,"15min":10.007},"last":0.43099999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:16:21 2023","interfaces":[{"interface":"back","average":{"1min":1.615,"5min":1.6599999999999999,"15min":1.627},"min":{"1min":0.51200000000000001,"5min":0.46500000000000002,"15min":0.41899999999999998},"max":{"1min":5.7359999999999998,"5min":11.282,"15min":11.282},"last":3.1000000000000001},{"interface":"front","average":{"1min":1.7809999999999999,"5min":1.708,"15min":1.681},"min":{"1min":0.45300000000000001,"5min":0.39000000000000001,"15min":0.36399999999999999},"max":{"1min":4.1589999999999998,"5min":11.167,"15min":22.175999999999998},"last":1.754}]}]},{"osd":0,"up_from":88,"seq":377957122290,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2498436,"kb_used_data":1874748,"kb_used_omap":5,"kb_used_meta":623674,"kb_avail":91246716,"statfs":{"total":95995035648,"available":93436637184,"internally_reserved":0,"allocated":1919741952,"data_stored":1918631547,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":638642810},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,4,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:16:17 2023","interfaces":[{"interface":"back","average":{"1min":1.8700000000000001,"5min":2.0870000000000002,"15min":2.1139999999999999},"min":{"1min":0.184,"5min":0.184,"15min":0.184},"max":{"1min":10.669,"5min":73.811000000000007,"15min":109.88},"last":0.32500000000000001},{"interface":"front","average":{"1min":1.5609999999999999,"5min":2.278,"15min":2.1539999999999999},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":10.164999999999999,"5min":74.650999999999996,"15min":109.435},"last":0.38700000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:16:32 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":1.1970000000000001,"15min":1.3200000000000001},"min":{"1min":0.28599999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":10.298999999999999,"5min":10.609999999999999,"15min":10.609999999999999},"last":4.9459999999999997},{"interface":"front","average":{"1min":2.3679999999999999,"5min":1.4770000000000001,"15min":1.2909999999999999},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":9.3309999999999995,"5min":11.622,"15min":11.622},"last":5.3769999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:16:32 2023","interfaces":[{"interface":"back","average":{"1min":2.4729999999999999,"5min":2.3730000000000002,"15min":1.8240000000000001},"min":{"1min":0.35399999999999998,"5min":0.223,"15min":0.223},"max":{"1min":6.6020000000000003,"5min":40.649999999999999,"15min":40.649999999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":2.7290000000000001,"5min":2.5920000000000001,"15min":1.875},"min":{"1min":0.29899999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":15.69,"5min":42.758000000000003,"15min":42.758000000000003},"last":4.9080000000000004}]},{"osd":4,"last update":"Mon Oct 9 05:16:32 2023","interfaces":[{"interface":"back","average":{"1min":1.756,"5min":1.3680000000000001,"15min":1.282},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.33500000000000002},"max":{"1min":9.9930000000000003,"5min":10.632999999999999,"15min":10.632999999999999},"last":0.57199999999999995},{"interface":"front","average":{"1min":2.1829999999999998,"5min":1.4950000000000001,"15min":1.379},"min":{"1min":0.45000000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":6.6420000000000003,"5min":6.6420000000000003,"15min":7.202},"last":0.36299999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:16:32 2023","interfaces":[{"interface":"back","average":{"1min":1.9610000000000001,"5min":1.5369999999999999,"15min":1.423},"min":{"1min":0.41499999999999998,"5min":0.34999999999999998,"15min":0.29899999999999999},"max":{"1min":5.3579999999999997,"5min":8.7149999999999999,"15min":8.7149999999999999},"last":0.36899999999999999},{"interface":"front","average":{"1min":1.8200000000000001,"5min":1.6200000000000001,"15min":1.5960000000000001},"min":{"1min":0.375,"5min":0.36699999999999999,"15min":0.316},"max":{"1min":10.465,"5min":10.676,"15min":10.676},"last":0.49099999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:16:32 2023","interfaces":[{"interface":"back","average":{"1min":2.1890000000000001,"5min":1.6000000000000001,"15min":1.5369999999999999},"min":{"1min":0.44900000000000001,"5min":0.42599999999999999,"15min":0.371},"max":{"1min":6.5650000000000004,"5min":6.5650000000000004,"15min":8.8710000000000004},"last":5.3390000000000004},{"interface":"front","average":{"1min":2.2370000000000001,"5min":1.716,"15min":1.4750000000000001},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.36499999999999999},"max":{"1min":5.7789999999999999,"5min":8.4580000000000002,"15min":8.4580000000000002},"last":0.42599999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:16:32 2023","interfaces":[{"interface":"back","average":{"1min":2.1200000000000001,"5min":1.7170000000000001,"15min":1.554},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.375},"max":{"1min":5.2679999999999998,"5min":8.75,"15min":8.75},"last":0.39800000000000002},{"interface":"front","average":{"1min":2.153,"5min":1.5209999999999999,"15min":1.5780000000000001},"min":{"1min":0.503,"5min":0.42399999999999999,"15min":0.41699999999999998},"max":{"1min":10.429,"5min":10.694000000000001,"15min":10.694000000000001},"last":0.51300000000000001}]}]},{"osd":2,"up_from":106,"seq":455266533569,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2096216,"kb_used_data":1667472,"kb_used_omap":7,"kb_used_meta":428728,"kb_avail":91648936,"statfs":{"total":95995035648,"available":93848510464,"internally_reserved":0,"allocated":1707491328,"data_stored":1706429614,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":439017553},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:16:21 2023","interfaces":[{"interface":"back","average":{"1min":3.27,"5min":1.4390000000000001,"15min":1.478},"min":{"1min":0.28999999999999998,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":15.099,"5min":15.099,"15min":53.088000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":2.79,"5min":1.494,"15min":1.589},"min":{"1min":0.66200000000000003,"5min":0.215,"15min":0.215},"max":{"1min":8.3249999999999993,"5min":9.1519999999999992,"15min":46.762},"last":2.9119999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:16:21 2023","interfaces":[{"interface":"back","average":{"1min":2.2149999999999999,"5min":2.6240000000000001,"15min":1.732},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":7.7009999999999996,"5min":68.933999999999997,"15min":68.933999999999997},"last":3.843},{"interface":"front","average":{"1min":1.804,"5min":2.7040000000000002,"15min":1.921},"min":{"1min":0.29499999999999998,"5min":0.26700000000000002,"15min":0.23100000000000001},"max":{"1min":12.691000000000001,"5min":68.846000000000004,"15min":68.846000000000004},"last":3.6880000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:16:41 2023","interfaces":[{"interface":"back","average":{"1min":3.6280000000000001,"5min":2.4500000000000002,"15min":1.8320000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.27300000000000002},"max":{"1min":11.779,"5min":68.968000000000004,"15min":68.968000000000004},"last":2.9489999999999998},{"interface":"front","average":{"1min":1.8859999999999999,"5min":1.8899999999999999,"15min":1.6040000000000001},"min":{"1min":0.34799999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":11.474,"5min":68.891999999999996,"15min":68.891999999999996},"last":3.8069999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:16:21 2023","interfaces":[{"interface":"back","average":{"1min":2.2890000000000001,"5min":1.3069999999999999,"15min":1.361},"min":{"1min":0.51700000000000002,"5min":0.34599999999999997,"15min":0.33300000000000002},"max":{"1min":6.9249999999999998,"5min":6.9249999999999998,"15min":20.712},"last":2.996},{"interface":"front","average":{"1min":1.8069999999999999,"5min":1.246,"15min":1.3859999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":11.420999999999999,"5min":11.420999999999999,"15min":23.547000000000001},"last":3.637}]},{"osd":5,"last update":"Mon Oct 9 05:16:21 2023","interfaces":[{"interface":"back","average":{"1min":1.802,"5min":1.319,"15min":1.5249999999999999},"min":{"1min":0.38400000000000001,"5min":0.36799999999999999,"15min":0.35399999999999998},"max":{"1min":6.6269999999999998,"5min":6.6269999999999998,"15min":20.782},"last":3.3140000000000001},{"interface":"front","average":{"1min":2.0630000000000002,"5min":1.645,"15min":1.7010000000000001},"min":{"1min":0.438,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":6.9500000000000002,"5min":7.391,"15min":20.792000000000002},"last":2.9700000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:16:21 2023","interfaces":[{"interface":"back","average":{"1min":1.984,"5min":1.391,"15min":1.47},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":11.353,"5min":11.353,"15min":23.510999999999999},"last":3.665},{"interface":"front","average":{"1min":2.3740000000000001,"5min":1.4850000000000001,"15min":1.4710000000000001},"min":{"1min":0.42599999999999999,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":6.577,"5min":6.577,"15min":20.684999999999999},"last":0.45500000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:16:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0009999999999999,"5min":1.4330000000000001,"15min":1.4399999999999999},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":6.6699999999999999,"5min":6.6699999999999999,"15min":20.887},"last":0.47999999999999998},{"interface":"front","average":{"1min":2.169,"5min":1.508,"15min":1.5329999999999999},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.45400000000000001},"max":{"1min":6.968,"5min":6.968,"15min":20.863},"last":3.0190000000000001}]}]},{"osd":3,"up_from":123,"seq":528280977588,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1745364,"kb_used_data":1177740,"kb_used_omap":2,"kb_used_meta":567613,"kb_avail":91999788,"statfs":{"total":95995035648,"available":94207782912,"internally_reserved":0,"allocated":1206005760,"data_stored":1205001778,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":581235765},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:16:34 2023","interfaces":[{"interface":"back","average":{"1min":2.4969999999999999,"5min":2.4020000000000001,"15min":1.9419999999999999},"min":{"1min":0.30199999999999999,"5min":0.20000000000000001,"15min":0.187},"max":{"1min":10.903,"5min":29.814,"15min":29.814},"last":5.5949999999999998},{"interface":"front","average":{"1min":2.7250000000000001,"5min":2.6120000000000001,"15min":2.0529999999999999},"min":{"1min":0.33600000000000002,"5min":0.188,"15min":0.184},"max":{"1min":11.26,"5min":36.302,"15min":36.302},"last":65.882999999999996}]},{"osd":1,"last update":"Mon Oct 9 05:16:34 2023","interfaces":[{"interface":"back","average":{"1min":3.335,"5min":2.5419999999999998,"15min":1.986},"min":{"1min":0.245,"5min":0.24299999999999999,"15min":0.222},"max":{"1min":12.377000000000001,"5min":44.206000000000003,"15min":44.206000000000003},"last":1.9039999999999999},{"interface":"front","average":{"1min":2.5299999999999998,"5min":2.3980000000000001,"15min":1.9139999999999999},"min":{"1min":0.23599999999999999,"5min":0.218,"15min":0.20999999999999999},"max":{"1min":10.839,"5min":44.496000000000002,"15min":44.496000000000002},"last":1.8}]},{"osd":2,"last update":"Mon Oct 9 05:16:34 2023","interfaces":[{"interface":"back","average":{"1min":2.9449999999999998,"5min":1.6339999999999999,"15min":1.6319999999999999},"min":{"1min":0.253,"5min":0.246,"15min":0.23699999999999999},"max":{"1min":11.199999999999999,"5min":11.199999999999999,"15min":11.199999999999999},"last":1.9159999999999999},{"interface":"front","average":{"1min":3.9129999999999998,"5min":1.9610000000000001,"15min":1.7330000000000001},"min":{"1min":0.28999999999999998,"5min":0.27300000000000002,"15min":0.24099999999999999},"max":{"1min":12.324999999999999,"5min":12.324999999999999,"15min":14.754},"last":2.702}]},{"osd":4,"last update":"Mon Oct 9 05:16:34 2023","interfaces":[{"interface":"back","average":{"1min":3.2029999999999998,"5min":1.6299999999999999,"15min":1.5629999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":12.438000000000001,"5min":12.438000000000001,"15min":14.833},"last":0.34399999999999997},{"interface":"front","average":{"1min":2.145,"5min":1.452,"15min":1.385},"min":{"1min":0.34399999999999997,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":10.811999999999999,"5min":10.811999999999999,"15min":10.811999999999999},"last":0.41699999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:16:34 2023","interfaces":[{"interface":"back","average":{"1min":2.169,"5min":1.5629999999999999,"15min":1.514},"min":{"1min":0.39100000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":10.720000000000001,"5min":10.720000000000001,"15min":10.720000000000001},"last":0.379},{"interface":"front","average":{"1min":2.1899999999999999,"5min":1.6020000000000001,"15min":1.821},"min":{"1min":0.42899999999999999,"5min":0.33900000000000002,"15min":0.33400000000000002},"max":{"1min":11.135,"5min":11.135,"15min":11.135},"last":1.9370000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:16:34 2023","interfaces":[{"interface":"back","average":{"1min":2.1949999999999998,"5min":1.6779999999999999,"15min":1.6930000000000001},"min":{"1min":0.40200000000000002,"5min":0.36099999999999999,"15min":0.33200000000000002},"max":{"1min":11.173999999999999,"5min":11.173999999999999,"15min":11.173999999999999},"last":1.9590000000000001},{"interface":"front","average":{"1min":3.544,"5min":2.0179999999999998,"15min":1.8640000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.34699999999999998},"max":{"1min":12.407,"5min":12.407,"15min":14.881},"last":1.847}]},{"osd":7,"last update":"Mon Oct 9 05:16:34 2023","interfaces":[{"interface":"back","average":{"1min":3.3809999999999998,"5min":1.833,"15min":1.867},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":12.468,"5min":12.468,"15min":14.856},"last":1.8799999999999999},{"interface":"front","average":{"1min":2.1760000000000002,"5min":1.748,"15min":1.6879999999999999},"min":{"1min":0.47999999999999998,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":11.233000000000001,"5min":11.233000000000001,"15min":11.233000000000001},"last":1.754}]}]},{"osd":4,"up_from":30,"seq":128849019345,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2803004,"kb_used_data":1910452,"kb_used_omap":0,"kb_used_meta":892544,"kb_avail":90942148,"statfs":{"total":95995035648,"available":93124759552,"internally_reserved":0,"allocated":1956302848,"data_stored":1955553236,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":913965056},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:16:29 2023","interfaces":[{"interface":"back","average":{"1min":2.1840000000000002,"5min":2.3969999999999998,"15min":2.109},"min":{"1min":0.32800000000000001,"5min":0.29099999999999998,"15min":0.26600000000000001},"max":{"1min":7.7000000000000002,"5min":67.224999999999994,"15min":102.669},"last":2.6560000000000001},{"interface":"front","average":{"1min":2.0960000000000001,"5min":2.2549999999999999,"15min":2.0270000000000001},"min":{"1min":0.38800000000000001,"5min":0.308,"15min":0.26900000000000002},"max":{"1min":6.9409999999999998,"5min":67.191000000000003,"15min":102.72499999999999},"last":2.6400000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:16:19 2023","interfaces":[{"interface":"back","average":{"1min":1.518,"5min":1.446,"15min":1.736},"min":{"1min":0.30299999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.423,"5min":10.387,"15min":59.408000000000001},"last":0.41699999999999998},{"interface":"front","average":{"1min":1.3320000000000001,"5min":1.3080000000000001,"15min":1.5800000000000001},"min":{"1min":0.39400000000000002,"5min":0.36299999999999999,"15min":0.28199999999999997},"max":{"1min":7.8289999999999997,"5min":10.305999999999999,"15min":60.207000000000001},"last":0.45700000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:16:29 2023","interfaces":[{"interface":"back","average":{"1min":1.4850000000000001,"5min":1.8200000000000001,"15min":1.47},"min":{"1min":0.375,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.8799999999999999,"5min":66.201999999999998,"15min":66.201999999999998},"last":1.569},{"interface":"front","average":{"1min":2.0859999999999999,"5min":1.9350000000000001,"15min":1.4159999999999999},"min":{"1min":0.36299999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":6.6189999999999998,"5min":66.361000000000004,"15min":66.361000000000004},"last":1.292}]},{"osd":3,"last update":"Mon Oct 9 05:16:38 2023","interfaces":[{"interface":"back","average":{"1min":1.581,"5min":3.4460000000000002,"15min":2.23},"min":{"1min":0.39100000000000001,"5min":0.36399999999999999,"15min":0.309},"max":{"1min":7.7939999999999996,"5min":68,"15min":68},"last":0.498},{"interface":"front","average":{"1min":1.4610000000000001,"5min":3.1190000000000002,"15min":2.056},"min":{"1min":0.432,"5min":0.35699999999999998,"15min":0.35399999999999998},"max":{"1min":5.0460000000000003,"5min":67.775000000000006,"15min":67.775000000000006},"last":0.376}]},{"osd":5,"last update":"Mon Oct 9 05:16:38 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":0.99199999999999999,"15min":1.1839999999999999},"min":{"1min":0.42599999999999999,"5min":0.375,"15min":0.32500000000000001},"max":{"1min":7.7549999999999999,"5min":7.7549999999999999,"15min":10.214},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.98099999999999998,"5min":1.1479999999999999,"15min":1.2969999999999999},"min":{"1min":0.435,"5min":0.34799999999999998,"15min":0.34200000000000003},"max":{"1min":4.9770000000000003,"5min":6.0460000000000003,"15min":6.8310000000000004},"last":0.51800000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:16:19 2023","interfaces":[{"interface":"back","average":{"1min":1.5369999999999999,"5min":1.133,"15min":1.371},"min":{"1min":0.441,"5min":0.34200000000000003,"15min":0.32800000000000001},"max":{"1min":6.3760000000000003,"5min":6.3760000000000003,"15min":11.557},"last":0.47699999999999998},{"interface":"front","average":{"1min":1.4970000000000001,"5min":1.1679999999999999,"15min":1.2889999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.34699999999999998},"max":{"1min":7.923,"5min":8.4930000000000003,"15min":10.689},"last":0.69799999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:16:19 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":1.0880000000000001,"15min":1.1399999999999999},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.36299999999999999},"max":{"1min":5.0060000000000002,"5min":5.0060000000000002,"15min":6.4820000000000002},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.415,"5min":1.3919999999999999,"15min":1.4390000000000001},"min":{"1min":0.45500000000000002,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":6.452,"5min":8.9689999999999994,"15min":8.9689999999999994},"last":0.68000000000000005}]}]},{"osd":5,"up_from":0,"seq":163208757701,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1896222720,"data_stored":1896218648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":836763648,"data_stored":836763648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1692139520,"data_stored":1692139520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1193607168,"data_stored":1193607168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1934954496,"data_stored":1934950424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1246498816,"data_stored":1246494744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":243269632,"data_stored":243269632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:17:01.930 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 1 2023-10-09T05:17:01.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:01.931 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:01.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:02.032 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:02.058 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:02.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:02.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:02.161 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:02.200 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:02.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:02.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:02.309 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:02.336 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:02.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:02.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:02.396 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:17:02.440 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:02.465 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:02.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:02.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:02.569 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:02.597 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:02.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:02.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:02.699 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:02.726 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:02.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:02.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:02.830 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:02.858 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:02.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:02.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:02.961 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:02.996 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:02.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:02.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:03.100 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:03.133 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:03.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:03.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:03.237 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:03 smithi133 bash[102858]: cluster 2023-10-09T05:17:01.938944+0000 mgr.y (mgr.25277) 857 : cluster 0 pgmap v875: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 692 op/s; 10331/119268 objects degraded (8.662%); 667/119268 objects misplaced (0.559%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T05:17:03.237 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:03 smithi133 bash[105589]: cluster 2023-10-09T05:17:01.938944+0000 mgr.y (mgr.25277) 857 : cluster 0 pgmap v875: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 692 op/s; 10331/119268 objects degraded (8.662%); 667/119268 objects misplaced (0.559%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T05:17:03.237 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:03.269 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:03.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:03.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:03.374 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:03.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:03 smithi194 bash[92666]: cluster 2023-10-09T05:17:01.938944+0000 mgr.y (mgr.25277) 857 : cluster 0 pgmap v875: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 692 op/s; 10331/119268 objects degraded (8.662%); 667/119268 objects misplaced (0.559%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T05:17:03.399 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:03.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:03.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:03.502 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:03.532 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:03.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:03.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:03.638 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:03.663 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:03.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:03.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:03.767 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:03.794 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:03.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:03.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:03.901 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:03.928 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:03.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:03.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:04.032 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:04.058 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:04.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:04.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:04.163 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:04.177 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:04 smithi133 bash[102858]: cluster 2023-10-09T05:17:03.359182+0000 mon.a (mon.0) 1120 : cluster 3 Health check update: Degraded data redundancy: 10331/119268 objects degraded (8.662%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:04.177 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:04 smithi133 bash[105589]: cluster 2023-10-09T05:17:03.359182+0000 mon.a (mon.0) 1120 : cluster 3 Health check update: Degraded data redundancy: 10331/119268 objects degraded (8.662%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:04.197 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:04.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:04.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:04.301 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:04.348 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:04.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:04.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:04.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:04 smithi194 bash[92666]: cluster 2023-10-09T05:17:03.359182+0000 mon.a (mon.0) 1120 : cluster 3 Health check update: Degraded data redundancy: 10331/119268 objects degraded (8.662%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:04.454 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:04.470 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:17:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:17:04] "GET /metrics HTTP/1.1" 200 36514 "" "Prometheus/2.43.0" 2023-10-09T05:17:04.488 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:04.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:04.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:04.601 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:04.639 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:04.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:04.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:04.743 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:04.773 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:04.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:04.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:04.880 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:04.912 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:04.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:04.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:05.017 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:05.053 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:05.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:05.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:05.159 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:05.169 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:05 smithi194 bash[92666]: cluster 2023-10-09T05:17:03.939489+0000 mgr.y (mgr.25277) 858 : cluster 0 pgmap v876: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 692 op/s; 10331/119268 objects degraded (8.662%); 667/119268 objects misplaced (0.559%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T05:17:05.191 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:05.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:05.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:05.293 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:05.318 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:05.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:05.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:05.416 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:05 smithi133 bash[102858]: cluster 2023-10-09T05:17:03.939489+0000 mgr.y (mgr.25277) 858 : cluster 0 pgmap v876: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 692 op/s; 10331/119268 objects degraded (8.662%); 667/119268 objects misplaced (0.559%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T05:17:05.416 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:05 smithi133 bash[105589]: cluster 2023-10-09T05:17:03.939489+0000 mgr.y (mgr.25277) 858 : cluster 0 pgmap v876: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 692 op/s; 10331/119268 objects degraded (8.662%); 667/119268 objects misplaced (0.559%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T05:17:05.424 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:05.457 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:05.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:05.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:05.563 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:05.587 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:05.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:05.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:05.689 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:05.715 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:05.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:05.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:05.820 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:05.861 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:05.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:05.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:05.971 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:05.996 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:06.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:06.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:06.103 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:06.132 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:06.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:06.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:06.237 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:06.270 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:06.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:06.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:06.374 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:06.399 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:06.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:06.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:06.468 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:17:06.471 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:17:06.471 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:17:06.504 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:06.525 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:06.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:06.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:06.633 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:06.665 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:06.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:06.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:06.771 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:06.806 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:06.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:06.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:06.912 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:06.942 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:06.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:06.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:07.045 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:07.078 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:07.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:07.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:07.184 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:07.215 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:07.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:07.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:07.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:07 smithi133 bash[102858]: cluster 2023-10-09T05:17:05.940356+0000 mgr.y (mgr.25277) 859 : cluster 0 pgmap v877: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 846 op/s; 10426/110214 objects degraded (9.460%); 459/110214 objects misplaced (0.416%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:17:07.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:07 smithi133 bash[105589]: cluster 2023-10-09T05:17:05.940356+0000 mgr.y (mgr.25277) 859 : cluster 0 pgmap v877: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 846 op/s; 10426/110214 objects degraded (9.460%); 459/110214 objects misplaced (0.416%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:17:07.318 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:07.349 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:07.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:07.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:07.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:07 smithi194 bash[92666]: cluster 2023-10-09T05:17:05.940356+0000 mgr.y (mgr.25277) 859 : cluster 0 pgmap v877: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 846 op/s; 10426/110214 objects degraded (9.460%); 459/110214 objects misplaced (0.416%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:17:07.454 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:07.483 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:07.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:07.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:07.586 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:07.612 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:07.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:07.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:07.718 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:07.740 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:07.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:07.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:07.847 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:07.871 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:07.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:07.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:07.974 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:07.999 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:08.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:08.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:08.102 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:08.127 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:08.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:08.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:08.231 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:08.254 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:08.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:08.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:08.361 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:08.385 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:08.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:08.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:08.488 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:08.515 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:08.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:08.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:08.621 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:08.647 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:08.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:08.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:08.751 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:08.770 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:08.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:08.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:08.876 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:08.926 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:08.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:08.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:09.030 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:09.057 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:09.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:09.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:09.166 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:09.199 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:09.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:09.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:09.308 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:09.338 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:09.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:09.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:09.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:09 smithi194 bash[92666]: cluster 2023-10-09T05:17:07.941035+0000 mgr.y (mgr.25277) 860 : cluster 0 pgmap v878: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 630 op/s; 10500/108402 objects degraded (9.686%); 301/108402 objects misplaced (0.278%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T05:17:09.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:09 smithi194 bash[92666]: cluster 2023-10-09T05:17:08.360288+0000 mon.a (mon.0) 1121 : cluster 3 Health check update: Degraded data redundancy: 10500/108402 objects degraded (9.686%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:09.444 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:09.472 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:09.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:09.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:09.479 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:09 smithi133 bash[102858]: cluster 2023-10-09T05:17:07.941035+0000 mgr.y (mgr.25277) 860 : cluster 0 pgmap v878: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 630 op/s; 10500/108402 objects degraded (9.686%); 301/108402 objects misplaced (0.278%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T05:17:09.479 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:09 smithi133 bash[102858]: cluster 2023-10-09T05:17:08.360288+0000 mon.a (mon.0) 1121 : cluster 3 Health check update: Degraded data redundancy: 10500/108402 objects degraded (9.686%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:09.479 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:09 smithi133 bash[105589]: cluster 2023-10-09T05:17:07.941035+0000 mgr.y (mgr.25277) 860 : cluster 0 pgmap v878: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 630 op/s; 10500/108402 objects degraded (9.686%); 301/108402 objects misplaced (0.278%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T05:17:09.479 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:09 smithi133 bash[105589]: cluster 2023-10-09T05:17:08.360288+0000 mon.a (mon.0) 1121 : cluster 3 Health check update: Degraded data redundancy: 10500/108402 objects degraded (9.686%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:09.576 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:09.613 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:09.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:09.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:09.720 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:09.738 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:09.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:09.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:09.842 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:09.859 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:09.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:09.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:09.961 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:10.004 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:10.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:10.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:10.107 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:10.136 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:10.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:10.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:10.241 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:10.271 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:10.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:10.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:10.374 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:10.397 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:10.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:10.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:10.500 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:10.543 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:10.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:10.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:10.648 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:10.682 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:10.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:10.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:10.787 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:10.818 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:10.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:10.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:10.924 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:10.960 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:10.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:10.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:11.066 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:11.084 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:17:11.084 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:17:11.089 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:11 smithi133 bash[102858]: cluster 2023-10-09T05:17:09.941748+0000 mgr.y (mgr.25277) 861 : cluster 0 pgmap v879: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 650 op/s; 10500/106461 objects degraded (9.863%); 301/106461 objects misplaced (0.283%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T05:17:11.089 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:11 smithi133 bash[105589]: cluster 2023-10-09T05:17:09.941748+0000 mgr.y (mgr.25277) 861 : cluster 0 pgmap v879: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 650 op/s; 10500/106461 objects degraded (9.863%); 301/106461 objects misplaced (0.283%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T05:17:11.090 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:11.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:11.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:11.193 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:11.219 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:11.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:11.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:11.326 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:11.365 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:11.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:11.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:11.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:11 smithi194 bash[92666]: cluster 2023-10-09T05:17:09.941748+0000 mgr.y (mgr.25277) 861 : cluster 0 pgmap v879: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 650 op/s; 10500/106461 objects degraded (9.863%); 301/106461 objects misplaced (0.283%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T05:17:11.472 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:11.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:11.502 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:11.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:11.603 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:11.638 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:11.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:11.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:11.742 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:11.771 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:11.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:11.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:11.876 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:11.918 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:11.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:11.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:12.024 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:12.061 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:12.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:12.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:12.164 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:12.198 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:12.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:12.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:12.303 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:12.327 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:12.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:12.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:12.430 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:12.447 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:12.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:12.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:12.549 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:12.574 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:12.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:12.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:12.677 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:12.716 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:12.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:12.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:12.819 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:12.850 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:12.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:12.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:12.956 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:12.980 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:12.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:12.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:13.083 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:13.105 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:13.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:13.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:13.209 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:13.216 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:13 smithi194 bash[92666]: cluster 2023-10-09T05:17:11.942485+0000 mgr.y (mgr.25277) 862 : cluster 0 pgmap v880: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 798 op/s; 10500/98628 objects degraded (10.646%); 301/98628 objects misplaced (0.305%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T05:17:13.245 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:13.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:13.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:13.353 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:13.386 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:13.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:13.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:13.389 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:13 smithi133 bash[102858]: cluster 2023-10-09T05:17:11.942485+0000 mgr.y (mgr.25277) 862 : cluster 0 pgmap v880: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 798 op/s; 10500/98628 objects degraded (10.646%); 301/98628 objects misplaced (0.305%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T05:17:13.389 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:13 smithi133 bash[105589]: cluster 2023-10-09T05:17:11.942485+0000 mgr.y (mgr.25277) 862 : cluster 0 pgmap v880: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 798 op/s; 10500/98628 objects degraded (10.646%); 301/98628 objects misplaced (0.305%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T05:17:13.490 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:13.533 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:13.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:13.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:13.642 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:13.666 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:13.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:13.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:13.769 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:13.793 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:13.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:13.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:13.899 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:13.922 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:13.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:13.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:14.028 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:14.057 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:14.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:14.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:14.162 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:14.171 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:14 smithi133 bash[105589]: cluster 2023-10-09T05:17:13.361322+0000 mon.a (mon.0) 1122 : cluster 3 Health check update: Degraded data redundancy: 10500/98628 objects degraded (10.646%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:14.171 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:14 smithi133 bash[105589]: audit 2023-10-09T05:17:13.450524+0000 mon.a (mon.0) 1123 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:17:14.172 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:14 smithi133 bash[102858]: cluster 2023-10-09T05:17:13.361322+0000 mon.a (mon.0) 1122 : cluster 3 Health check update: Degraded data redundancy: 10500/98628 objects degraded (10.646%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:14.172 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:14 smithi133 bash[102858]: audit 2023-10-09T05:17:13.450524+0000 mon.a (mon.0) 1123 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:17:14.199 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:14.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:14.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:14.302 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:14.340 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:14.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:14.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:14.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:14 smithi194 bash[92666]: cluster 2023-10-09T05:17:13.361322+0000 mon.a (mon.0) 1122 : cluster 3 Health check update: Degraded data redundancy: 10500/98628 objects degraded (10.646%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:14.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:14 smithi194 bash[92666]: audit 2023-10-09T05:17:13.450524+0000 mon.a (mon.0) 1123 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:17:14.445 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:14.454 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:17:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:17:14] "GET /metrics HTTP/1.1" 200 36514 "" "Prometheus/2.43.0" 2023-10-09T05:17:14.479 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:14.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:14.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:14.584 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:14.617 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:14.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:14.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:14.721 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:14.756 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:14.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:14.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:14.861 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:14.891 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:14.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:14.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:14.994 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:15.019 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:15.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:15.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:15.128 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:15.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:15 smithi194 bash[92666]: cluster 2023-10-09T05:17:13.943110+0000 mgr.y (mgr.25277) 863 : cluster 0 pgmap v881: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 589 op/s; 10585/98031 objects degraded (10.798%); 82/98031 objects misplaced (0.084%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T05:17:15.153 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:15.155 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T05:17:15.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:15.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:15.257 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:15.287 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:15.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:15.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:15.342 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:15 smithi133 bash[105589]: cluster 2023-10-09T05:17:13.943110+0000 mgr.y (mgr.25277) 863 : cluster 0 pgmap v881: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 589 op/s; 10585/98031 objects degraded (10.798%); 82/98031 objects misplaced (0.084%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T05:17:15.342 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:15 smithi133 bash[102858]: cluster 2023-10-09T05:17:13.943110+0000 mgr.y (mgr.25277) 863 : cluster 0 pgmap v881: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 589 op/s; 10585/98031 objects degraded (10.798%); 82/98031 objects misplaced (0.084%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T05:17:15.390 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:15.424 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:15.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:15.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:15.528 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:15.554 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:15.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:15.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:15.657 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:15.684 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:15.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:15.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:15.795 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:15.824 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:15.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:15.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:15.927 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:15.954 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:15.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:15.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:16.063 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:16.088 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:16.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:16.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:16.192 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:16.222 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:16.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:16.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:16.326 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:16.352 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:16.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:16.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:16.457 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:16.482 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:16.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:16.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:16.586 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:16.622 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:16.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:16.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:16.729 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:16.768 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:16.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:16.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:16.872 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:16.905 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:16.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:16.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:17.012 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:17.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:17.060 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:17.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:17.161 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:17.191 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:17.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:17.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:17.295 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:17.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:17 smithi133 bash[102858]: cluster 2023-10-09T05:17:15.943962+0000 mgr.y (mgr.25277) 864 : cluster 0 pgmap v882: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 815 op/s; 10585/89910 objects degraded (11.773%); 82/89910 objects misplaced (0.091%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T05:17:17.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:17 smithi133 bash[105589]: cluster 2023-10-09T05:17:15.943962+0000 mgr.y (mgr.25277) 864 : cluster 0 pgmap v882: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 815 op/s; 10585/89910 objects degraded (11.773%); 82/89910 objects misplaced (0.091%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T05:17:17.331 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:17.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:17.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:17.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:17 smithi194 bash[92666]: cluster 2023-10-09T05:17:15.943962+0000 mgr.y (mgr.25277) 864 : cluster 0 pgmap v882: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 815 op/s; 10585/89910 objects degraded (11.773%); 82/89910 objects misplaced (0.091%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T05:17:17.440 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:17.466 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:17.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:17.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:17.569 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:17.596 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:17.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:17.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:17.700 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:17.730 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:17.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:17.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:17.837 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:17.864 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:17.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:17.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:17.967 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:18.002 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:18.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:18.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:18.107 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:18.130 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:18.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:18.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:18.237 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:18.277 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:18.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:18.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:18.381 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:18.407 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:18.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:18.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:18.515 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:18.550 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:18.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:18.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:18.655 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:18.683 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:18.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:18.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:18.787 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:18.818 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:18.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:18.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:18.923 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:18.950 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:18.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:18.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:19.056 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:19.078 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:19.082 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:19 smithi133 bash[102858]: cluster 2023-10-09T05:17:17.944726+0000 mgr.y (mgr.25277) 865 : cluster 0 pgmap v883: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 617 op/s; 10548/87963 objects degraded (11.991%); 1.4 MiB/s, 28 objects/s recovering 2023-10-09T05:17:19.082 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:19 smithi133 bash[102858]: cluster 2023-10-09T05:17:18.362526+0000 mon.a (mon.0) 1124 : cluster 3 Health check update: Degraded data redundancy: 10548/87963 objects degraded (11.991%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:19.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:19.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:19.187 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:19.194 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:19 smithi194 bash[92666]: cluster 2023-10-09T05:17:17.944726+0000 mgr.y (mgr.25277) 865 : cluster 0 pgmap v883: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 617 op/s; 10548/87963 objects degraded (11.991%); 1.4 MiB/s, 28 objects/s recovering 2023-10-09T05:17:19.194 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:19 smithi194 bash[92666]: cluster 2023-10-09T05:17:18.362526+0000 mon.a (mon.0) 1124 : cluster 3 Health check update: Degraded data redundancy: 10548/87963 objects degraded (11.991%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:19.210 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:19.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:19.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:19.314 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:19.342 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:19.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:19.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:19.444 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:19.471 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:19.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:19.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:19.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:19 smithi133 bash[105589]: cluster 2023-10-09T05:17:17.944726+0000 mgr.y (mgr.25277) 865 : cluster 0 pgmap v883: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 617 op/s; 10548/87963 objects degraded (11.991%); 1.4 MiB/s, 28 objects/s recovering 2023-10-09T05:17:19.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:19 smithi133 bash[105589]: cluster 2023-10-09T05:17:18.362526+0000 mon.a (mon.0) 1124 : cluster 3 Health check update: Degraded data redundancy: 10548/87963 objects degraded (11.991%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:19.574 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:19.599 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:19.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:19.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:19.702 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:19.729 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:19.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:19.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:19.837 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:19.876 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:19.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:19.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:19.959 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:17:19.983 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:20.011 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:20.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:20.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:20.115 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:20.139 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:20.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:20.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:20.242 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:20.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:20.286 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:20.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:20.387 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:20.418 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:20.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:20.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:20.525 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:20.560 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:20.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:20.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:20.665 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:20.693 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:20.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:20.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:20.796 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:20.826 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:20.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:20.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:20.931 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:20.954 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:20.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:20.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:21.057 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:21.085 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:21.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:21.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:21.191 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:21.199 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:21 smithi194 bash[92666]: cluster 2023-10-09T05:17:19.945447+0000 mgr.y (mgr.25277) 866 : cluster 0 pgmap v884: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.7 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 624 op/s; 10548/85926 objects degraded (12.276%); 1002 KiB/s, 21 objects/s recovering 2023-10-09T05:17:21.218 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:21.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:21.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:21.324 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:21.345 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:21.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:21.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:21.354 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:21 smithi133 bash[105589]: cluster 2023-10-09T05:17:19.945447+0000 mgr.y (mgr.25277) 866 : cluster 0 pgmap v884: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.7 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 624 op/s; 10548/85926 objects degraded (12.276%); 1002 KiB/s, 21 objects/s recovering 2023-10-09T05:17:21.354 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:21 smithi133 bash[102858]: cluster 2023-10-09T05:17:19.945447+0000 mgr.y (mgr.25277) 866 : cluster 0 pgmap v884: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.7 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 624 op/s; 10548/85926 objects degraded (12.276%); 1002 KiB/s, 21 objects/s recovering 2023-10-09T05:17:21.448 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:21.472 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:21.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:21.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:21.575 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:21.604 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:21.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:21.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:21.715 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:21.737 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:21.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:21.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:21.841 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:21.879 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:21.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:21.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:21.986 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:22.021 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:22.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:22.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:22.125 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:22.152 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:22.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:22.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:22.256 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:22.281 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:22.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:22.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:22.385 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:22.425 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:22.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:22.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:22.531 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:22.558 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:22.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:22.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:22.616 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:17:22.661 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:22.693 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:17:22.694 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:22.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:22.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:22.733 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:17:22.773 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:17:22.799 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:22.813 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:17:22.822 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:22.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:22.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:22.854 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:17:22.894 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:17:22.924 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:22.933 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:17:22.939 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:17:22.939 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:17:22.939 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:17:22.940 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:17:22.940 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:17:22.940 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:17:22.940 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:17:22.940 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:17:22.962 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:22.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:22.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:23.068 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:23.100 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:23.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:23.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:23.203 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:23.228 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:23.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:23.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:23.234 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:23 smithi133 bash[102858]: cluster 2023-10-09T05:17:21.946271+0000 mgr.y (mgr.25277) 867 : cluster 0 pgmap v885: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 784 op/s; 10548/78201 objects degraded (13.488%); 1002 KiB/s, 21 objects/s recovering 2023-10-09T05:17:23.234 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:23 smithi133 bash[102858]: audit 2023-10-09T05:17:22.592524+0000 mon.a (mon.0) 1125 : audit 0 from='client.? 172.21.15.133:0/1655394595' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:17:23.234 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:23 smithi133 bash[105589]: cluster 2023-10-09T05:17:21.946271+0000 mgr.y (mgr.25277) 867 : cluster 0 pgmap v885: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 784 op/s; 10548/78201 objects degraded (13.488%); 1002 KiB/s, 21 objects/s recovering 2023-10-09T05:17:23.235 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:23 smithi133 bash[105589]: audit 2023-10-09T05:17:22.592524+0000 mon.a (mon.0) 1125 : audit 0 from='client.? 172.21.15.133:0/1655394595' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:17:23.332 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:23.342 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:23 smithi194 bash[92666]: cluster 2023-10-09T05:17:21.946271+0000 mgr.y (mgr.25277) 867 : cluster 0 pgmap v885: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 784 op/s; 10548/78201 objects degraded (13.488%); 1002 KiB/s, 21 objects/s recovering 2023-10-09T05:17:23.342 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:23 smithi194 bash[92666]: audit 2023-10-09T05:17:22.592524+0000 mon.a (mon.0) 1125 : audit 0 from='client.? 172.21.15.133:0/1655394595' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:17:23.356 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:23.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:23.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:23.460 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:23.494 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:23.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:23.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:23.600 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:23.625 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:23.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:23.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:23.729 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:23.756 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:23.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:23.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:23.861 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:23.888 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:23.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:23.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:23.992 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:24.016 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:24.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:24.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:24.120 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:24.132 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:24 smithi133 bash[102858]: cluster 2023-10-09T05:17:23.363696+0000 mon.a (mon.0) 1126 : cluster 3 Health check update: Degraded data redundancy: 10548/78201 objects degraded (13.488%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:24.133 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:24 smithi133 bash[102858]: audit 2023-10-09T05:17:23.661670+0000 mon.c (mon.2) 84 : audit 1 from='client.? 172.21.15.133:0/3712934027' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:17:24.133 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:24 smithi133 bash[102858]: audit 2023-10-09T05:17:23.662044+0000 mon.a (mon.0) 1127 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:17:24.133 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:24 smithi133 bash[102858]: audit 2023-10-09T05:17:23.822166+0000 mon.c (mon.2) 85 : audit 1 from='client.? 172.21.15.133:0/3586361763' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T05:17:24.133 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:24 smithi133 bash[102858]: audit 2023-10-09T05:17:23.822623+0000 mon.a (mon.0) 1128 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T05:17:24.133 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:24 smithi133 bash[105589]: cluster 2023-10-09T05:17:23.363696+0000 mon.a (mon.0) 1126 : cluster 3 Health check update: Degraded data redundancy: 10548/78201 objects degraded (13.488%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:24.134 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:24 smithi133 bash[105589]: audit 2023-10-09T05:17:23.661670+0000 mon.c (mon.2) 84 : audit 1 from='client.? 172.21.15.133:0/3712934027' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:17:24.134 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:24 smithi133 bash[105589]: audit 2023-10-09T05:17:23.662044+0000 mon.a (mon.0) 1127 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:17:24.134 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:24 smithi133 bash[105589]: audit 2023-10-09T05:17:23.822166+0000 mon.c (mon.2) 85 : audit 1 from='client.? 172.21.15.133:0/3586361763' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T05:17:24.134 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:24 smithi133 bash[105589]: audit 2023-10-09T05:17:23.822623+0000 mon.a (mon.0) 1128 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T05:17:24.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:24 smithi194 bash[92666]: cluster 2023-10-09T05:17:23.363696+0000 mon.a (mon.0) 1126 : cluster 3 Health check update: Degraded data redundancy: 10548/78201 objects degraded (13.488%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:24.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:24 smithi194 bash[92666]: audit 2023-10-09T05:17:23.661670+0000 mon.c (mon.2) 84 : audit 1 from='client.? 172.21.15.133:0/3712934027' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:17:24.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:24 smithi194 bash[92666]: audit 2023-10-09T05:17:23.662044+0000 mon.a (mon.0) 1127 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:17:24.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:24 smithi194 bash[92666]: audit 2023-10-09T05:17:23.822166+0000 mon.c (mon.2) 85 : audit 1 from='client.? 172.21.15.133:0/3586361763' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T05:17:24.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:24 smithi194 bash[92666]: audit 2023-10-09T05:17:23.822623+0000 mon.a (mon.0) 1128 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T05:17:24.163 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:24.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:24.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:24.267 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:24.295 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:24.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:24.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:24.398 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:24.401 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is unset 2023-10-09T05:17:24.405 INFO:teuthology.orchestra.run.smithi133.stderr:set pool 1 pgp_num to 1 2023-10-09T05:17:24.425 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:24.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:24.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:24.446 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:17:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:17:24] "GET /metrics HTTP/1.1" 200 36514 "" "Prometheus/2.43.0" 2023-10-09T05:17:24.530 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:24.567 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:24.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:24.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:24.672 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:24.713 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:24.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:24.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:24.818 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:24.849 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:24.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:24.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:24.955 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:24.982 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:24.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:24.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:25.088 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:25.119 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:25.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:25.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:25.223 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:25.248 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:25.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:25.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:25.312 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:25 smithi133 bash[102858]: cluster 2023-10-09T05:17:23.946927+0000 mgr.y (mgr.25277) 868 : cluster 0 pgmap v886: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 586 op/s; 10413/77508 objects degraded (13.435%); 1002 KiB/s, 32 objects/s recovering 2023-10-09T05:17:25.312 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:25 smithi133 bash[102858]: cluster 2023-10-09T05:17:24.367850+0000 mon.a (mon.0) 1129 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:17:25.312 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:25 smithi133 bash[102858]: cluster 2023-10-09T05:17:24.383268+0000 osd.5 (osd.5) 198 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T05:17:25.312 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:25 smithi133 bash[102858]: cluster 2023-10-09T05:17:24.391277+0000 osd.6 (osd.6) 148 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T05:17:25.313 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:25 smithi133 bash[102858]: audit 2023-10-09T05:17:24.392052+0000 mon.a (mon.0) 1130 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:17:25.313 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:25 smithi133 bash[102858]: audit 2023-10-09T05:17:24.392293+0000 mon.a (mon.0) 1131 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-10-09T05:17:25.313 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:25 smithi133 bash[102858]: cluster 2023-10-09T05:17:24.392519+0000 mon.a (mon.0) 1132 : cluster 0 osdmap e211: 8 total, 8 up, 6 in 2023-10-09T05:17:25.313 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:25 smithi133 bash[102858]: cluster 2023-10-09T05:17:24.485647+0000 osd.7 (osd.7) 143 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T05:17:25.313 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:25 smithi133 bash[102858]: cluster 2023-10-09T05:17:24.487910+0000 osd.7 (osd.7) 144 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T05:17:25.313 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:25 smithi133 bash[105589]: cluster 2023-10-09T05:17:23.946927+0000 mgr.y (mgr.25277) 868 : cluster 0 pgmap v886: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 586 op/s; 10413/77508 objects degraded (13.435%); 1002 KiB/s, 32 objects/s recovering 2023-10-09T05:17:25.313 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:25 smithi133 bash[105589]: cluster 2023-10-09T05:17:24.367850+0000 mon.a (mon.0) 1129 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:17:25.313 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:25 smithi133 bash[105589]: cluster 2023-10-09T05:17:24.383268+0000 osd.5 (osd.5) 198 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T05:17:25.313 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:25 smithi133 bash[105589]: cluster 2023-10-09T05:17:24.391277+0000 osd.6 (osd.6) 148 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T05:17:25.314 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:25 smithi133 bash[105589]: audit 2023-10-09T05:17:24.392052+0000 mon.a (mon.0) 1130 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:17:25.314 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:25 smithi133 bash[105589]: audit 2023-10-09T05:17:24.392293+0000 mon.a (mon.0) 1131 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-10-09T05:17:25.314 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:25 smithi133 bash[105589]: cluster 2023-10-09T05:17:24.392519+0000 mon.a (mon.0) 1132 : cluster 0 osdmap e211: 8 total, 8 up, 6 in 2023-10-09T05:17:25.314 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:25 smithi133 bash[105589]: cluster 2023-10-09T05:17:24.485647+0000 osd.7 (osd.7) 143 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T05:17:25.314 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:25 smithi133 bash[105589]: cluster 2023-10-09T05:17:24.487910+0000 osd.7 (osd.7) 144 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T05:17:25.352 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:25.384 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:25.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:25 smithi194 bash[92666]: cluster 2023-10-09T05:17:23.946927+0000 mgr.y (mgr.25277) 868 : cluster 0 pgmap v886: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 586 op/s; 10413/77508 objects degraded (13.435%); 1002 KiB/s, 32 objects/s recovering 2023-10-09T05:17:25.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:25 smithi194 bash[92666]: cluster 2023-10-09T05:17:24.367850+0000 mon.a (mon.0) 1129 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:17:25.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:25 smithi194 bash[92666]: cluster 2023-10-09T05:17:24.383268+0000 osd.5 (osd.5) 198 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T05:17:25.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:25 smithi194 bash[92666]: cluster 2023-10-09T05:17:24.391277+0000 osd.6 (osd.6) 148 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T05:17:25.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:25 smithi194 bash[92666]: audit 2023-10-09T05:17:24.392052+0000 mon.a (mon.0) 1130 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:17:25.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:25 smithi194 bash[92666]: audit 2023-10-09T05:17:24.392293+0000 mon.a (mon.0) 1131 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-10-09T05:17:25.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:25 smithi194 bash[92666]: cluster 2023-10-09T05:17:24.392519+0000 mon.a (mon.0) 1132 : cluster 0 osdmap e211: 8 total, 8 up, 6 in 2023-10-09T05:17:25.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:25 smithi194 bash[92666]: cluster 2023-10-09T05:17:24.485647+0000 osd.7 (osd.7) 143 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T05:17:25.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:25 smithi194 bash[92666]: cluster 2023-10-09T05:17:24.487910+0000 osd.7 (osd.7) 144 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T05:17:25.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:25.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:25.488 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:25.526 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:25.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:25.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:25.631 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:25.655 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:25.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:25.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:25.760 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:25.839 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:25.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:25.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:25.943 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:25.969 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:25.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:25.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:26.075 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:26.099 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:26.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:26.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:26.207 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:26.233 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:26.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:26.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:26.338 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:26.369 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:26.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:26.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:26.375 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:26 smithi133 bash[105589]: cluster 2023-10-09T05:17:24.399424+0000 osd.4 (osd.4) 162 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T05:17:26.375 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:26 smithi133 bash[105589]: cluster 2023-10-09T05:17:24.401258+0000 osd.4 (osd.4) 163 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T05:17:26.375 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:26 smithi133 bash[105589]: cluster 2023-10-09T05:17:24.403780+0000 osd.4 (osd.4) 164 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T05:17:26.375 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:26 smithi133 bash[102858]: cluster 2023-10-09T05:17:24.399424+0000 osd.4 (osd.4) 162 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T05:17:26.376 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:26 smithi133 bash[102858]: cluster 2023-10-09T05:17:24.401258+0000 osd.4 (osd.4) 163 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T05:17:26.376 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:26 smithi133 bash[102858]: cluster 2023-10-09T05:17:24.403780+0000 osd.4 (osd.4) 164 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T05:17:26.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:26 smithi194 bash[92666]: cluster 2023-10-09T05:17:24.399424+0000 osd.4 (osd.4) 162 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T05:17:26.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:26 smithi194 bash[92666]: cluster 2023-10-09T05:17:24.401258+0000 osd.4 (osd.4) 163 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T05:17:26.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:26 smithi194 bash[92666]: cluster 2023-10-09T05:17:24.403780+0000 osd.4 (osd.4) 164 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T05:17:26.475 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:26.499 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:26.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:26.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:26.605 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:26.630 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:26.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:26.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:26.733 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:26.760 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:26.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:26.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:26.865 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:26.902 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:26.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:26.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:27.007 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:27.048 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:27.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:27.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:27.155 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:27.197 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:27.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:27.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:27.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:27 smithi133 bash[105589]: cluster 2023-10-09T05:17:25.947713+0000 mgr.y (mgr.25277) 869 : cluster 0 pgmap v888: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 690 op/s; 10413/69201 objects degraded (15.047%); 345 KiB/s, 25 objects/s recovering 2023-10-09T05:17:27.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:27 smithi133 bash[102858]: cluster 2023-10-09T05:17:25.947713+0000 mgr.y (mgr.25277) 869 : cluster 0 pgmap v888: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 690 op/s; 10413/69201 objects degraded (15.047%); 345 KiB/s, 25 objects/s recovering 2023-10-09T05:17:27.304 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:27.327 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:27.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:27.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:27.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:27 smithi194 bash[92666]: cluster 2023-10-09T05:17:25.947713+0000 mgr.y (mgr.25277) 869 : cluster 0 pgmap v888: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 690 op/s; 10413/69201 objects degraded (15.047%); 345 KiB/s, 25 objects/s recovering 2023-10-09T05:17:27.429 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:27.453 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:27.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:27.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:27.557 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:27.624 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:27.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:27.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:27.729 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:27.752 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:27.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:27.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:27.762 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:17:27.857 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:27.876 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:27.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:27.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:27.979 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:28.004 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:28.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:28.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:28.107 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:28.134 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:28.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:28.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:28.238 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:28.263 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:28.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:28.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:28.369 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:28.393 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:28.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:28.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:28.497 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:28.529 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:28.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:28.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:28.632 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:28.686 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:28.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:28.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:28.797 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:28.835 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:28.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:28.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:28.941 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:28.971 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:28.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:28.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:29.077 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:29.110 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:29.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:29.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:29.213 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:29.221 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:29 smithi133 bash[105589]: cluster 2023-10-09T05:17:27.948361+0000 mgr.y (mgr.25277) 870 : cluster 0 pgmap v889: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 691 op/s; 10298/67206 objects degraded (15.323%); 0 B/s, 24 objects/s recovering 2023-10-09T05:17:29.221 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:29 smithi133 bash[105589]: cluster 2023-10-09T05:17:28.365035+0000 mon.a (mon.0) 1133 : cluster 3 Health check update: Degraded data redundancy: 10298/67206 objects degraded (15.323%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:29.221 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:29 smithi133 bash[105589]: audit 2023-10-09T05:17:28.460051+0000 mon.a (mon.0) 1134 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:17:29.222 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:29 smithi133 bash[105589]: audit 2023-10-09T05:17:28.464709+0000 mon.a (mon.0) 1135 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:17:29.222 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:29 smithi133 bash[102858]: cluster 2023-10-09T05:17:27.948361+0000 mgr.y (mgr.25277) 870 : cluster 0 pgmap v889: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 691 op/s; 10298/67206 objects degraded (15.323%); 0 B/s, 24 objects/s recovering 2023-10-09T05:17:29.222 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:29 smithi133 bash[102858]: cluster 2023-10-09T05:17:28.365035+0000 mon.a (mon.0) 1133 : cluster 3 Health check update: Degraded data redundancy: 10298/67206 objects degraded (15.323%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:29.222 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:29 smithi133 bash[102858]: audit 2023-10-09T05:17:28.460051+0000 mon.a (mon.0) 1134 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:17:29.222 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:29 smithi133 bash[102858]: audit 2023-10-09T05:17:28.464709+0000 mon.a (mon.0) 1135 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:17:29.245 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:29.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:29.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:29.351 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:29.375 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:29.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:29.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:29.381 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:29 smithi194 bash[92666]: cluster 2023-10-09T05:17:27.948361+0000 mgr.y (mgr.25277) 870 : cluster 0 pgmap v889: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 691 op/s; 10298/67206 objects degraded (15.323%); 0 B/s, 24 objects/s recovering 2023-10-09T05:17:29.381 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:29 smithi194 bash[92666]: cluster 2023-10-09T05:17:28.365035+0000 mon.a (mon.0) 1133 : cluster 3 Health check update: Degraded data redundancy: 10298/67206 objects degraded (15.323%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:29.381 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:29 smithi194 bash[92666]: audit 2023-10-09T05:17:28.460051+0000 mon.a (mon.0) 1134 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:17:29.381 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:29 smithi194 bash[92666]: audit 2023-10-09T05:17:28.464709+0000 mon.a (mon.0) 1135 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:17:29.481 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:29.508 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:29.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:29.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:29.612 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:29.648 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:29.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:29.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:29.752 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:29.782 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:29.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:29.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:29.886 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:29.913 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:29.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:29.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:30.019 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:30.043 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:30.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:30.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:30.146 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:30.173 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:30.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:30.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:30.277 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:30.304 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:30.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:30.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:30.409 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:30.450 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:30.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:30.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:30.556 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:30.592 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:17:30.595 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:30.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:30.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:30.703 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:30.729 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:30.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:30.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:30.833 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:30.856 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:17:30.856 INFO:teuthology.orchestra.run.smithi133.stdout: "ops": [], 2023-10-09T05:17:30.856 INFO:teuthology.orchestra.run.smithi133.stdout: "complaint_time": 30, 2023-10-09T05:17:30.856 INFO:teuthology.orchestra.run.smithi133.stdout: "num_blocked_ops": 0 2023-10-09T05:17:30.857 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:17:30.883 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:30.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:30.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:30.987 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:31.017 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:31.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:31.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:31.123 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:31.130 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:31 smithi133 bash[105589]: cluster 2023-10-09T05:17:29.949002+0000 mgr.y (mgr.25277) 871 : cluster 0 pgmap v890: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.3 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 689 op/s; 10298/65223 objects degraded (15.789%); 0 B/s, 24 objects/s recovering 2023-10-09T05:17:31.130 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:31 smithi133 bash[102858]: cluster 2023-10-09T05:17:29.949002+0000 mgr.y (mgr.25277) 871 : cluster 0 pgmap v890: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.3 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 689 op/s; 10298/65223 objects degraded (15.789%); 0 B/s, 24 objects/s recovering 2023-10-09T05:17:31.148 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:31.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:31.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:31.253 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:31.262 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:31 smithi194 bash[92666]: cluster 2023-10-09T05:17:29.949002+0000 mgr.y (mgr.25277) 871 : cluster 0 pgmap v890: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.3 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 689 op/s; 10298/65223 objects degraded (15.789%); 0 B/s, 24 objects/s recovering 2023-10-09T05:17:31.276 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:31.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:31.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:31.387 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:31.412 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:31.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:31.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:31.516 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:31.555 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:31.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:31.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:31.660 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:31.705 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:31.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:31.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:31.809 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:31.833 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:31.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:31.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:31.937 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:31.962 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:31.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:31.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:32.066 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:32.091 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:32.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:32.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:32.197 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:32.222 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:32.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:32.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:32.328 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:32.365 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:32.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:32.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:32.454 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T05:17:32.470 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:32.493 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:32.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:32.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:32.600 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:32.652 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:32.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:32.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:32.758 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:32.802 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:32.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:32.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:32.908 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:32.935 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:32.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:32.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:33.039 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:33.084 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:33.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:33.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:33.192 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:33.202 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:33 smithi194 bash[92666]: cluster 2023-10-09T05:17:31.949803+0000 mgr.y (mgr.25277) 872 : cluster 0 pgmap v891: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.2 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 702 op/s; 10298/57129 objects degraded (18.026%); 0 B/s, 24 objects/s recovering 2023-10-09T05:17:33.220 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:33.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:33.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:33.323 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:33.353 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:33.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:33.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:33.456 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:33.483 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:33.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:33.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:33.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:33 smithi133 bash[102858]: cluster 2023-10-09T05:17:31.949803+0000 mgr.y (mgr.25277) 872 : cluster 0 pgmap v891: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.2 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 702 op/s; 10298/57129 objects degraded (18.026%); 0 B/s, 24 objects/s recovering 2023-10-09T05:17:33.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:33 smithi133 bash[105589]: cluster 2023-10-09T05:17:31.949803+0000 mgr.y (mgr.25277) 872 : cluster 0 pgmap v891: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.2 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 702 op/s; 10298/57129 objects degraded (18.026%); 0 B/s, 24 objects/s recovering 2023-10-09T05:17:33.589 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:33.633 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:33.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:33.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:33.704 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 6, 1] out_osds: [5, 7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:17:33.704 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:17:33.704 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-10-09T05:17:33.704 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T05:17:33.704 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T05:17:33.705 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-10-09T05:17:33.741 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:33.809 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:33.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:33.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:33.915 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:33.952 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:33.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:33.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:33.960 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:17:34.056 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:34.108 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:34.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:34.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:34.215 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:34.249 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:34.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:34.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:34.281 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:17:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:17:34] "GET /metrics HTTP/1.1" 200 36495 "" "Prometheus/2.43.0" 2023-10-09T05:17:34.281 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:34 smithi133 bash[105589]: cluster 2023-10-09T05:17:33.366331+0000 mon.a (mon.0) 1136 : cluster 3 Health check update: Degraded data redundancy: 10298/57129 objects degraded (18.026%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:34.282 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:34 smithi133 bash[105589]: audit 2023-10-09T05:17:33.567112+0000 mon.a (mon.0) 1137 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:17:34.282 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:34 smithi133 bash[102858]: cluster 2023-10-09T05:17:33.366331+0000 mon.a (mon.0) 1136 : cluster 3 Health check update: Degraded data redundancy: 10298/57129 objects degraded (18.026%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:34.282 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:34 smithi133 bash[102858]: audit 2023-10-09T05:17:33.567112+0000 mon.a (mon.0) 1137 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:17:34.355 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:34.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:34 smithi194 bash[92666]: cluster 2023-10-09T05:17:33.366331+0000 mon.a (mon.0) 1136 : cluster 3 Health check update: Degraded data redundancy: 10298/57129 objects degraded (18.026%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:34.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:34 smithi194 bash[92666]: audit 2023-10-09T05:17:33.567112+0000 mon.a (mon.0) 1137 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:17:34.392 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:34.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:34.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:34.497 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:34.532 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:34.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:34.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:34.637 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:34.667 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:34.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:34.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:34.772 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:34.797 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:34.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:34.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:34.900 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:34.937 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:34.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:34.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:35.040 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:35.071 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:35.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:35.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:35.176 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:35.200 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:35.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:35.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:35.306 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:35.314 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:35 smithi194 bash[92666]: cluster 2023-10-09T05:17:33.951059+0000 mgr.y (mgr.25277) 873 : cluster 0 pgmap v892: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.1 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 698 op/s; 10154/56535 objects degraded (17.961%); 0 B/s, 25 objects/s recovering 2023-10-09T05:17:35.330 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:35.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:35.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:35.436 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:35.445 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:35 smithi133 bash[102858]: cluster 2023-10-09T05:17:33.951059+0000 mgr.y (mgr.25277) 873 : cluster 0 pgmap v892: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.1 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 698 op/s; 10154/56535 objects degraded (17.961%); 0 B/s, 25 objects/s recovering 2023-10-09T05:17:35.445 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:35 smithi133 bash[105589]: cluster 2023-10-09T05:17:33.951059+0000 mgr.y (mgr.25277) 873 : cluster 0 pgmap v892: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.1 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 698 op/s; 10154/56535 objects degraded (17.961%); 0 B/s, 25 objects/s recovering 2023-10-09T05:17:35.459 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:35.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:35.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:35.567 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:35.599 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:35.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:35.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:35.704 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:35.739 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:35.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:35.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:35.844 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:35.867 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:35.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:35.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:35.970 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:35.987 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:17:35.997 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:35.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:35.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:36.099 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:36.135 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:36.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:36.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:36.240 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:36.272 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:36.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:36.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:36.376 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:36.410 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:36.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:36.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:36.516 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:36.547 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:36.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:36.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:36.649 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:36.677 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:36.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:36.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:36.782 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:36.804 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:36.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:36.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:36.907 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:36.957 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:36.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:36.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:37.063 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:37.090 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:37.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:37.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:37.197 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:37.228 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:37.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:37.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:37.335 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:37.356 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:37.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:37.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:37.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:37 smithi194 bash[92666]: cluster 2023-10-09T05:17:35.951837+0000 mgr.y (mgr.25277) 874 : cluster 0 pgmap v893: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 997 MiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 853 op/s; 10154/47880 objects degraded (21.207%); 0 B/s, 22 objects/s recovering 2023-10-09T05:17:37.460 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:37.489 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:37.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:37.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:37.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:37 smithi133 bash[102858]: cluster 2023-10-09T05:17:35.951837+0000 mgr.y (mgr.25277) 874 : cluster 0 pgmap v893: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 997 MiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 853 op/s; 10154/47880 objects degraded (21.207%); 0 B/s, 22 objects/s recovering 2023-10-09T05:17:37.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:37 smithi133 bash[105589]: cluster 2023-10-09T05:17:35.951837+0000 mgr.y (mgr.25277) 874 : cluster 0 pgmap v893: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 997 MiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 853 op/s; 10154/47880 objects degraded (21.207%); 0 B/s, 22 objects/s recovering 2023-10-09T05:17:37.593 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:37.619 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:37.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:37.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:37.722 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:37.750 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:37.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:37.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:37.854 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:37.887 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:37.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:37.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:37.992 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:38.022 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:38.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:38.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:38.127 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:38.157 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:38.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:38.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:38.261 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:38.284 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:38.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:38.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:38.391 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:38.412 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:38.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:38.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:38.516 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:38.542 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:38.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:38.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:38.647 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:38.676 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:38.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:38.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:38.781 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:38.807 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:38.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:38.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:38.912 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:38.933 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:38.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:38.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:39.040 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:39.078 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:39.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:39.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:39.183 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:39.192 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:39 smithi133 bash[105589]: cluster 2023-10-09T05:17:37.952490+0000 mgr.y (mgr.25277) 875 : cluster 0 pgmap v894: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 956 MiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 645 op/s; 10005/45954 objects degraded (21.772%); 0 B/s, 33 objects/s recovering 2023-10-09T05:17:39.192 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:39 smithi133 bash[105589]: cluster 2023-10-09T05:17:38.369395+0000 mon.a (mon.0) 1138 : cluster 3 Health check update: Degraded data redundancy: 10005/45954 objects degraded (21.772%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:39.193 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:39 smithi133 bash[102858]: cluster 2023-10-09T05:17:37.952490+0000 mgr.y (mgr.25277) 875 : cluster 0 pgmap v894: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 956 MiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 645 op/s; 10005/45954 objects degraded (21.772%); 0 B/s, 33 objects/s recovering 2023-10-09T05:17:39.193 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:39 smithi133 bash[102858]: cluster 2023-10-09T05:17:38.369395+0000 mon.a (mon.0) 1138 : cluster 3 Health check update: Degraded data redundancy: 10005/45954 objects degraded (21.772%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:39.211 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:39.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:39.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:39.281 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:17:39.281 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:17:39.317 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:39.352 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:39.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:39.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:39.460 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:39.468 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:39 smithi194 bash[92666]: cluster 2023-10-09T05:17:37.952490+0000 mgr.y (mgr.25277) 875 : cluster 0 pgmap v894: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 956 MiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 645 op/s; 10005/45954 objects degraded (21.772%); 0 B/s, 33 objects/s recovering 2023-10-09T05:17:39.468 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:39 smithi194 bash[92666]: cluster 2023-10-09T05:17:38.369395+0000 mon.a (mon.0) 1138 : cluster 3 Health check update: Degraded data redundancy: 10005/45954 objects degraded (21.772%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:39.490 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:39.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:39.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:39.599 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:39.632 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:39.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:39.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:39.736 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:39.777 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:39.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:39.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:39.881 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:39.931 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:39.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:39.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:40.038 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:40.075 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:40.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:40.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:40.181 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:40.238 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:40.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:40.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:40.341 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:40.372 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:40.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:40.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:40.475 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:40.510 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:40.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:40.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:40.612 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:40.641 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:40.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:40.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:40.749 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:40.785 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:40.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:40.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:40.889 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:40.915 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:40.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:40.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:41.019 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:41.049 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:41.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:41.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:41.089 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:41 smithi133 bash[102858]: cluster 2023-10-09T05:17:39.953847+0000 mgr.y (mgr.25277) 876 : cluster 0 pgmap v895: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 916 MiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 643 op/s; 10005/44034 objects degraded (22.721%); 0 B/s, 24 objects/s recovering 2023-10-09T05:17:41.089 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:41 smithi133 bash[102858]: audit 2023-10-09T05:17:40.089044+0000 mon.a (mon.0) 1139 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:17:41.089 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:41 smithi133 bash[102858]: audit 2023-10-09T05:17:40.098627+0000 mon.a (mon.0) 1140 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:17:41.089 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:41 smithi133 bash[102858]: audit 2023-10-09T05:17:40.713508+0000 mon.a (mon.0) 1141 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:17:41.089 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:41 smithi133 bash[102858]: audit 2023-10-09T05:17:40.725878+0000 mon.a (mon.0) 1142 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:17:41.089 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:41 smithi133 bash[105589]: cluster 2023-10-09T05:17:39.953847+0000 mgr.y (mgr.25277) 876 : cluster 0 pgmap v895: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 916 MiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 643 op/s; 10005/44034 objects degraded (22.721%); 0 B/s, 24 objects/s recovering 2023-10-09T05:17:41.090 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:41 smithi133 bash[105589]: audit 2023-10-09T05:17:40.089044+0000 mon.a (mon.0) 1139 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:17:41.090 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:41 smithi133 bash[105589]: audit 2023-10-09T05:17:40.098627+0000 mon.a (mon.0) 1140 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:17:41.090 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:41 smithi133 bash[105589]: audit 2023-10-09T05:17:40.713508+0000 mon.a (mon.0) 1141 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:17:41.090 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:41 smithi133 bash[105589]: audit 2023-10-09T05:17:40.725878+0000 mon.a (mon.0) 1142 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:17:41.153 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:41.189 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:41.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:41.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:41.297 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:41.338 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:41.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:41.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:41.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:41 smithi194 bash[92666]: cluster 2023-10-09T05:17:39.953847+0000 mgr.y (mgr.25277) 876 : cluster 0 pgmap v895: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 916 MiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 643 op/s; 10005/44034 objects degraded (22.721%); 0 B/s, 24 objects/s recovering 2023-10-09T05:17:41.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:41 smithi194 bash[92666]: audit 2023-10-09T05:17:40.089044+0000 mon.a (mon.0) 1139 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:17:41.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:41 smithi194 bash[92666]: audit 2023-10-09T05:17:40.098627+0000 mon.a (mon.0) 1140 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:17:41.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:41 smithi194 bash[92666]: audit 2023-10-09T05:17:40.713508+0000 mon.a (mon.0) 1141 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:17:41.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:41 smithi194 bash[92666]: audit 2023-10-09T05:17:40.725878+0000 mon.a (mon.0) 1142 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:17:41.441 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:41.480 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:41.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:41.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:41.588 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:41.613 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:41.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:41.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:41.716 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:41.742 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:41.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:41.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:41.852 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:41.884 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:41.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:41.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:41.989 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:42.010 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:42.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:42.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:42.113 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:42.140 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:42.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:42.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:42.243 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:42.271 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:42.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:42.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:42.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:42 smithi133 bash[102858]: audit 2023-10-09T05:17:41.313740+0000 mon.a (mon.0) 1143 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:42.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:42 smithi133 bash[102858]: audit 2023-10-09T05:17:41.314818+0000 mon.a (mon.0) 1144 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:17:42.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:42 smithi133 bash[102858]: audit 2023-10-09T05:17:41.322168+0000 mon.a (mon.0) 1145 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:17:42.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:42 smithi133 bash[105589]: audit 2023-10-09T05:17:41.313740+0000 mon.a (mon.0) 1143 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:42.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:42 smithi133 bash[105589]: audit 2023-10-09T05:17:41.314818+0000 mon.a (mon.0) 1144 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:17:42.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:42 smithi133 bash[105589]: audit 2023-10-09T05:17:41.322168+0000 mon.a (mon.0) 1145 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:17:42.375 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:42.384 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:42 smithi194 bash[92666]: audit 2023-10-09T05:17:41.313740+0000 mon.a (mon.0) 1143 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:42.384 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:42 smithi194 bash[92666]: audit 2023-10-09T05:17:41.314818+0000 mon.a (mon.0) 1144 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:17:42.384 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:42 smithi194 bash[92666]: audit 2023-10-09T05:17:41.322168+0000 mon.a (mon.0) 1145 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:17:42.399 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:42.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:42.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:42.502 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:42.533 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:42.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:42.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:42.635 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:42.670 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:42.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:42.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:42.775 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:42.805 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:42.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:42.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:42.910 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:42.935 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:42.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:42.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:43.038 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:43.063 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:43.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:43.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:43.169 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:43.200 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:43.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:43.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:43.307 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:43.317 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:43 smithi194 bash[92666]: cluster 2023-10-09T05:17:41.954581+0000 mgr.y (mgr.25277) 877 : cluster 0 pgmap v896: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 765 MiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 789 op/s; 10005/36783 objects degraded (27.200%); 0 B/s, 24 objects/s recovering 2023-10-09T05:17:43.336 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:43.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:43.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:43.439 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:43.464 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:43.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:43.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:43.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:43 smithi133 bash[102858]: cluster 2023-10-09T05:17:41.954581+0000 mgr.y (mgr.25277) 877 : cluster 0 pgmap v896: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 765 MiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 789 op/s; 10005/36783 objects degraded (27.200%); 0 B/s, 24 objects/s recovering 2023-10-09T05:17:43.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:43 smithi133 bash[105589]: cluster 2023-10-09T05:17:41.954581+0000 mgr.y (mgr.25277) 877 : cluster 0 pgmap v896: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 765 MiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 789 op/s; 10005/36783 objects degraded (27.200%); 0 B/s, 24 objects/s recovering 2023-10-09T05:17:43.568 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:43.591 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:43.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:43.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:43.694 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:43.721 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:43.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:43.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:43.824 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:43.852 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:43.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:43.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:43.956 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:43.981 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:43.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:43.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:44.087 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:44.095 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:44 smithi133 bash[102858]: cluster 2023-10-09T05:17:43.370419+0000 mon.a (mon.0) 1146 : cluster 3 Health check update: Degraded data redundancy: 10005/36783 objects degraded (27.200%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:44.095 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:44 smithi133 bash[102858]: audit 2023-10-09T05:17:43.451610+0000 mon.a (mon.0) 1147 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:17:44.096 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:44 smithi133 bash[105589]: cluster 2023-10-09T05:17:43.370419+0000 mon.a (mon.0) 1146 : cluster 3 Health check update: Degraded data redundancy: 10005/36783 objects degraded (27.200%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:44.096 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:44 smithi133 bash[105589]: audit 2023-10-09T05:17:43.451610+0000 mon.a (mon.0) 1147 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:17:44.115 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:44.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:44.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:44.218 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:44.243 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:44.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:44.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:44.349 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:44.357 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:44 smithi194 bash[92666]: cluster 2023-10-09T05:17:43.370419+0000 mon.a (mon.0) 1146 : cluster 3 Health check update: Degraded data redundancy: 10005/36783 objects degraded (27.200%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:44.357 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:44 smithi194 bash[92666]: audit 2023-10-09T05:17:43.451610+0000 mon.a (mon.0) 1147 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:17:44.373 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:44.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:44.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:44.477 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:44.499 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:44.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:44.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:44.512 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:17:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:17:44] "GET /metrics HTTP/1.1" 200 36497 "" "Prometheus/2.43.0" 2023-10-09T05:17:44.605 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:44.635 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:44.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:44.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:44.738 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:44.789 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:44.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:44.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:44.894 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:44.918 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:44.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:44.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:45.021 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:45.051 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:45.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:45.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:45.154 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:45.182 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:45.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:45.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:45.288 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:45.319 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:45.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:45.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:45.325 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:45 smithi133 bash[105589]: cluster 2023-10-09T05:17:43.955178+0000 mgr.y (mgr.25277) 878 : cluster 0 pgmap v897: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 753 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 581 op/s; 9853/36171 objects degraded (27.240%); 0 B/s, 37 objects/s recovering 2023-10-09T05:17:45.325 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:45 smithi133 bash[102858]: cluster 2023-10-09T05:17:43.955178+0000 mgr.y (mgr.25277) 878 : cluster 0 pgmap v897: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 753 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 581 op/s; 9853/36171 objects degraded (27.240%); 0 B/s, 37 objects/s recovering 2023-10-09T05:17:45.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:45 smithi194 bash[92666]: cluster 2023-10-09T05:17:43.955178+0000 mgr.y (mgr.25277) 878 : cluster 0 pgmap v897: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 753 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 581 op/s; 9853/36171 objects degraded (27.240%); 0 B/s, 37 objects/s recovering 2023-10-09T05:17:45.425 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:45.463 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:45.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:45.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:45.567 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:45.605 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:45.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:45.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:45.709 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:45.745 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:45.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:45.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:45.848 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:45.873 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:45.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:45.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:45.979 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:46.012 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:46.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:46.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:46.116 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:46.137 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:46.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:46.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:46.245 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:46.269 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:46.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:46.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:46.372 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:46.390 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:46.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:46.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:46.493 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:46.518 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:46.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:46.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:46.626 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:46.653 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:46.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:46.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:46.759 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:46.784 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:46.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:46.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:46.862 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:17:46.888 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:46.913 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:46.913 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:17:46.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:46.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:46.958 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:17:46.992 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:17:47.015 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:47.036 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:17:47.040 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:47.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:47.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:47.064 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:17:47.100 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:17:47.133 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:17:47.141 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:17:47.142 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:17:47.142 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:17:47.142 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:17:47.142 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:17:47.142 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:17:47.142 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:17:47.142 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:17:47.144 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:47.177 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:47.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:47.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:47.283 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:47.314 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:47.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:47.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:47.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:47 smithi194 bash[92666]: cluster 2023-10-09T05:17:45.955942+0000 mgr.y (mgr.25277) 879 : cluster 0 pgmap v898: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 590 MiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 782 op/s; 9853/28362 objects degraded (34.740%); 0 B/s, 25 objects/s recovering 2023-10-09T05:17:47.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:47 smithi194 bash[92666]: audit 2023-10-09T05:17:46.809262+0000 mon.c (mon.2) 86 : audit 0 from='client.? 172.21.15.133:0/3559587756' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:17:47.420 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:47.446 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:47.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:47.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:47.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:47 smithi133 bash[102858]: cluster 2023-10-09T05:17:45.955942+0000 mgr.y (mgr.25277) 879 : cluster 0 pgmap v898: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 590 MiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 782 op/s; 9853/28362 objects degraded (34.740%); 0 B/s, 25 objects/s recovering 2023-10-09T05:17:47.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:47 smithi133 bash[102858]: audit 2023-10-09T05:17:46.809262+0000 mon.c (mon.2) 86 : audit 0 from='client.? 172.21.15.133:0/3559587756' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:17:47.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:47 smithi133 bash[105589]: cluster 2023-10-09T05:17:45.955942+0000 mgr.y (mgr.25277) 879 : cluster 0 pgmap v898: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 590 MiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 782 op/s; 9853/28362 objects degraded (34.740%); 0 B/s, 25 objects/s recovering 2023-10-09T05:17:47.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:47 smithi133 bash[105589]: audit 2023-10-09T05:17:46.809262+0000 mon.c (mon.2) 86 : audit 0 from='client.? 172.21.15.133:0/3559587756' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:17:47.552 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:47.580 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:47.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:47.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:47.684 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:47.714 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:47.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:47.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:47.820 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:47.848 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:47.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:47.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:47.951 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:47.988 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:47.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:47.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:48.094 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:48.125 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:48.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:48.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:48.229 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:48.257 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:48.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:48.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:48.360 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:48.393 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:48.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:48.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:48.496 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:48.523 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:48.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:48.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:48.629 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:48.668 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:48.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:48.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:48.772 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:48.801 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:48.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:48.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:48.906 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:48.934 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:48.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:48.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:49.037 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:49.072 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:49.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:49.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:49.176 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:49.202 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:49.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:49.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:49.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:49 smithi133 bash[102858]: cluster 2023-10-09T05:17:47.956625+0000 mgr.y (mgr.25277) 880 : cluster 0 pgmap v899: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 550 MiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 595 op/s; 9707/26445 objects degraded (36.706%); 0 B/s, 37 objects/s recovering 2023-10-09T05:17:49.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:49 smithi133 bash[102858]: cluster 2023-10-09T05:17:48.371426+0000 mon.a (mon.0) 1148 : cluster 3 Health check update: Degraded data redundancy: 9707/26445 objects degraded (36.706%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:49.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:49 smithi133 bash[105589]: cluster 2023-10-09T05:17:47.956625+0000 mgr.y (mgr.25277) 880 : cluster 0 pgmap v899: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 550 MiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 595 op/s; 9707/26445 objects degraded (36.706%); 0 B/s, 37 objects/s recovering 2023-10-09T05:17:49.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:49 smithi133 bash[105589]: cluster 2023-10-09T05:17:48.371426+0000 mon.a (mon.0) 1148 : cluster 3 Health check update: Degraded data redundancy: 9707/26445 objects degraded (36.706%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:49.306 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:49.315 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:49 smithi194 bash[92666]: cluster 2023-10-09T05:17:47.956625+0000 mgr.y (mgr.25277) 880 : cluster 0 pgmap v899: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 550 MiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 595 op/s; 9707/26445 objects degraded (36.706%); 0 B/s, 37 objects/s recovering 2023-10-09T05:17:49.315 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:49 smithi194 bash[92666]: cluster 2023-10-09T05:17:48.371426+0000 mon.a (mon.0) 1148 : cluster 3 Health check update: Degraded data redundancy: 9707/26445 objects degraded (36.706%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:49.330 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:49.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:49.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:49.437 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:49.469 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:49.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:49.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:49.576 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:49.614 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:49.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:49.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:49.720 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:49.747 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:49.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:49.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:49.852 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:49.876 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:49.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:49.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:49.985 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:50.010 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:50.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:50.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:50.111 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:50 smithi133 bash[102858]: audit 2023-10-09T05:17:49.819896+0000 mon.b (mon.1) 18 : audit 1 from='client.? 172.21.15.133:0/3753945536' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:17:50.111 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:50 smithi133 bash[102858]: audit 2023-10-09T05:17:49.820121+0000 mon.a (mon.0) 1149 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:17:50.117 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:50.124 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is set 2023-10-09T05:17:50.150 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:50.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:50.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:50.252 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:50.263 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:50 smithi194 bash[92666]: audit 2023-10-09T05:17:49.819896+0000 mon.b (mon.1) 18 : audit 1 from='client.? 172.21.15.133:0/3753945536' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:17:50.263 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:50 smithi194 bash[92666]: audit 2023-10-09T05:17:49.820121+0000 mon.a (mon.0) 1149 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:17:50.282 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:50.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:50.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:50.389 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:50.412 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:50.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:50.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:50.519 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:50.527 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:50 smithi133 bash[105589]: audit 2023-10-09T05:17:49.819896+0000 mon.b (mon.1) 18 : audit 1 from='client.? 172.21.15.133:0/3753945536' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:17:50.527 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:50 smithi133 bash[105589]: audit 2023-10-09T05:17:49.820121+0000 mon.a (mon.0) 1149 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:17:50.553 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:50.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:50.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:50.659 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:50.685 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:50.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:50.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:50.788 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:50.814 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:50.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:50.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:50.918 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:50.949 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:50.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:50.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:51.053 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:51.080 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:51.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:51.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:51.186 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:51.195 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:51 smithi194 bash[92666]: cluster 2023-10-09T05:17:49.957236+0000 mgr.y (mgr.25277) 881 : cluster 0 pgmap v900: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 511 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 593 op/s; 9707/24573 objects degraded (39.503%); 0 B/s, 24 objects/s recovering 2023-10-09T05:17:51.195 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:51 smithi194 bash[92666]: cluster 2023-10-09T05:17:50.102961+0000 mon.a (mon.0) 1150 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:17:51.195 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:51 smithi194 bash[92666]: cluster 2023-10-09T05:17:50.111077+0000 osd.6 (osd.6) 149 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T05:17:51.195 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:51 smithi194 bash[92666]: audit 2023-10-09T05:17:50.114437+0000 mon.a (mon.0) 1151 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:17:51.195 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:51 smithi194 bash[92666]: cluster 2023-10-09T05:17:50.114571+0000 mon.a (mon.0) 1152 : cluster 0 osdmap e212: 8 total, 8 up, 6 in 2023-10-09T05:17:51.195 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:51 smithi194 bash[92666]: cluster 2023-10-09T05:17:50.115591+0000 osd.5 (osd.5) 199 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T05:17:51.209 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:51.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:51.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:51.313 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:51.337 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:51.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:51.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:51.440 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:51.448 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:51 smithi133 bash[102858]: cluster 2023-10-09T05:17:49.957236+0000 mgr.y (mgr.25277) 881 : cluster 0 pgmap v900: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 511 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 593 op/s; 9707/24573 objects degraded (39.503%); 0 B/s, 24 objects/s recovering 2023-10-09T05:17:51.448 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:51 smithi133 bash[102858]: cluster 2023-10-09T05:17:50.102961+0000 mon.a (mon.0) 1150 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:17:51.448 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:51 smithi133 bash[102858]: cluster 2023-10-09T05:17:50.111077+0000 osd.6 (osd.6) 149 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T05:17:51.448 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:51 smithi133 bash[102858]: audit 2023-10-09T05:17:50.114437+0000 mon.a (mon.0) 1151 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:17:51.448 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:51 smithi133 bash[102858]: cluster 2023-10-09T05:17:50.114571+0000 mon.a (mon.0) 1152 : cluster 0 osdmap e212: 8 total, 8 up, 6 in 2023-10-09T05:17:51.448 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:51 smithi133 bash[102858]: cluster 2023-10-09T05:17:50.115591+0000 osd.5 (osd.5) 199 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T05:17:51.449 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:51 smithi133 bash[105589]: cluster 2023-10-09T05:17:49.957236+0000 mgr.y (mgr.25277) 881 : cluster 0 pgmap v900: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 511 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 593 op/s; 9707/24573 objects degraded (39.503%); 0 B/s, 24 objects/s recovering 2023-10-09T05:17:51.449 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:51 smithi133 bash[105589]: cluster 2023-10-09T05:17:50.102961+0000 mon.a (mon.0) 1150 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:17:51.449 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:51 smithi133 bash[105589]: cluster 2023-10-09T05:17:50.111077+0000 osd.6 (osd.6) 149 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T05:17:51.449 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:51 smithi133 bash[105589]: audit 2023-10-09T05:17:50.114437+0000 mon.a (mon.0) 1151 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:17:51.449 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:51 smithi133 bash[105589]: cluster 2023-10-09T05:17:50.114571+0000 mon.a (mon.0) 1152 : cluster 0 osdmap e212: 8 total, 8 up, 6 in 2023-10-09T05:17:51.449 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:51 smithi133 bash[105589]: cluster 2023-10-09T05:17:50.115591+0000 osd.5 (osd.5) 199 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T05:17:51.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:51.466 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:51.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:51.567 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:51.594 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:51.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:51.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:51.697 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:51.723 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:51.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:51.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:51.828 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:51.866 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:51.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:51.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:51.970 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:52.000 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:52.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:52.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:52.106 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:52.133 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:52.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:52.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:52.237 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:52.264 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:52.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:52.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:52.369 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:52.397 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:52.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:52.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:52.500 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:52.534 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:52.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:52.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:52.641 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:52.669 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:52.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:52.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:52.776 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:52.815 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:52.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:52.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:52.923 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:52.952 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:52.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:52.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:53.054 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:53.080 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:53.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:53.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:53.187 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:53.195 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:53 smithi194 bash[92666]: cluster 2023-10-09T05:17:51.957894+0000 mgr.y (mgr.25277) 882 : cluster 0 pgmap v902: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 352 MiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 661 op/s; 9707/16932 objects degraded (57.329%); 0 B/s, 29 objects/s recovering 2023-10-09T05:17:53.211 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:53.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:53.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:53.314 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:53.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:53.354 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:53.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:53.375 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:53 smithi133 bash[102858]: cluster 2023-10-09T05:17:51.957894+0000 mgr.y (mgr.25277) 882 : cluster 0 pgmap v902: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 352 MiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 661 op/s; 9707/16932 objects degraded (57.329%); 0 B/s, 29 objects/s recovering 2023-10-09T05:17:53.375 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:53 smithi133 bash[105589]: cluster 2023-10-09T05:17:51.957894+0000 mgr.y (mgr.25277) 882 : cluster 0 pgmap v902: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 352 MiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 661 op/s; 9707/16932 objects degraded (57.329%); 0 B/s, 29 objects/s recovering 2023-10-09T05:17:53.455 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:53.484 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:53.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:53.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:53.589 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:53.616 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:53.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:53.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:53.721 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:53.752 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:17:53.770 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:53.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:53.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:53.873 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:53.899 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:53.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:53.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:54.002 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:54.039 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:54.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:54.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:54.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:54 smithi194 bash[92666]: cluster 2023-10-09T05:17:53.372329+0000 mon.a (mon.0) 1153 : cluster 3 Health check update: Degraded data redundancy: 9707/16932 objects degraded (57.329%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:54.142 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:54.188 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:54.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:54.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:54.282 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:54 smithi133 bash[102858]: cluster 2023-10-09T05:17:53.372329+0000 mon.a (mon.0) 1153 : cluster 3 Health check update: Degraded data redundancy: 9707/16932 objects degraded (57.329%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:54.282 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:54 smithi133 bash[105589]: cluster 2023-10-09T05:17:53.372329+0000 mon.a (mon.0) 1153 : cluster 3 Health check update: Degraded data redundancy: 9707/16932 objects degraded (57.329%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:54.293 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:54.331 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:54.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:54.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:54.440 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:54.471 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:54.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:54.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:54.575 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:54.586 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:17:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:17:54] "GET /metrics HTTP/1.1" 200 36497 "" "Prometheus/2.43.0" 2023-10-09T05:17:54.607 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:54.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:54.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:54.711 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:54.743 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:54.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:54.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:54.847 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:54.874 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:54.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:54.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:54.977 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:55.007 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:55.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:55.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:55.112 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:55.145 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:55.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:55.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:55.248 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:55.270 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:55 smithi133 bash[105589]: cluster 2023-10-09T05:17:53.958529+0000 mgr.y (mgr.25277) 883 : cluster 0 pgmap v903: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 337 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 664 op/s; 9545/16215 objects degraded (58.865%); 0 B/s, 30 objects/s recovering 2023-10-09T05:17:55.270 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:55 smithi133 bash[102858]: cluster 2023-10-09T05:17:53.958529+0000 mgr.y (mgr.25277) 883 : cluster 0 pgmap v903: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 337 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 664 op/s; 9545/16215 objects degraded (58.865%); 0 B/s, 30 objects/s recovering 2023-10-09T05:17:55.277 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:55.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:55.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:55.380 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:55.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:55 smithi194 bash[92666]: cluster 2023-10-09T05:17:53.958529+0000 mgr.y (mgr.25277) 883 : cluster 0 pgmap v903: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 337 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 664 op/s; 9545/16215 objects degraded (58.865%); 0 B/s, 30 objects/s recovering 2023-10-09T05:17:55.411 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:55.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:55.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:55.496 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:17:55.496 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:17:55.514 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:55.539 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:55.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:55.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:55.644 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:55.668 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:55.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:55.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:55.771 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:55.803 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:55.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:55.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:55.910 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:55.936 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:55.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:55.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:56.040 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:56.069 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:56.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:56.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:56.098 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:17:56.176 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:56.215 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:56.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:56.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:56.318 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:56.349 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:56.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:56.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:56.459 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:56.494 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:56.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:56.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:56.601 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:56.636 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:56.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:56.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:56.743 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:56.776 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:56.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:56.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:56.826 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:17:56.827 INFO:teuthology.orchestra.run.smithi133.stdout: "success": "" 2023-10-09T05:17:56.827 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:17:56.882 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:56.911 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:56.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:56.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:57.015 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:57.045 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:57.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:57.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:57.147 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:57.170 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:57.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:57.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:57.273 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:57.287 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:57 smithi133 bash[105589]: cluster 2023-10-09T05:17:55.959135+0000 mgr.y (mgr.25277) 884 : cluster 0 pgmap v904: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 153 MiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 698 op/s; 9545/7395 objects degraded (129.074%); 0 B/s, 30 objects/s recovering 2023-10-09T05:17:57.287 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:57 smithi133 bash[102858]: cluster 2023-10-09T05:17:55.959135+0000 mgr.y (mgr.25277) 884 : cluster 0 pgmap v904: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 153 MiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 698 op/s; 9545/7395 objects degraded (129.074%); 0 B/s, 30 objects/s recovering 2023-10-09T05:17:57.296 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:57.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:57.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:57.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:57 smithi194 bash[92666]: cluster 2023-10-09T05:17:55.959135+0000 mgr.y (mgr.25277) 884 : cluster 0 pgmap v904: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 153 MiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 698 op/s; 9545/7395 objects degraded (129.074%); 0 B/s, 30 objects/s recovering 2023-10-09T05:17:57.408 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:57.434 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:17:57.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:57.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:17:57.537 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:57.566 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:57.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:57.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:57.671 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:57.697 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:57.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:57.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:57.800 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:57.948 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T05:17:57.981 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:57.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:57.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:58.086 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:58.113 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:58.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:58.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:58.217 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:58.240 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Removed 97433 objects 2023-10-09T05:17:58.241 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Clean up completed and total clean up time :142.58 2023-10-09T05:17:58.243 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:58.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:58.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:58.347 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:58.372 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:17:58.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:58.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:17:58.476 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:58.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:58.498 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:58.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:58.599 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:58.627 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:58.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:58.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:58.730 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:58.765 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:58.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:58.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:58.869 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:58.903 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:17:58.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:58.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:17:59.006 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:59.038 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:59.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:59.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:17:59.142 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:59.171 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:59.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:59.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:59.274 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:59.303 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:17:59.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:59.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:17:59.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:59 smithi194 bash[92666]: cluster 2023-10-09T05:17:57.959722+0000 mgr.y (mgr.25277) 885 : cluster 0 pgmap v905: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 111 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 702 op/s; 9406/5376 objects degraded (174.963%); 0 B/s, 30 objects/s recovering 2023-10-09T05:17:59.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:59 smithi194 bash[92666]: cluster 2023-10-09T05:17:58.375381+0000 mon.a (mon.0) 1154 : cluster 3 Health check update: Degraded data redundancy: 9406/5376 objects degraded (174.963%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:59.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:59 smithi194 bash[92666]: audit 2023-10-09T05:17:58.455494+0000 mon.a (mon.0) 1155 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:17:59.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:17:59 smithi194 bash[92666]: audit 2023-10-09T05:17:58.456659+0000 mon.a (mon.0) 1156 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:17:59.406 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:59.419 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:59 smithi133 bash[105589]: cluster 2023-10-09T05:17:57.959722+0000 mgr.y (mgr.25277) 885 : cluster 0 pgmap v905: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 111 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 702 op/s; 9406/5376 objects degraded (174.963%); 0 B/s, 30 objects/s recovering 2023-10-09T05:17:59.419 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:59 smithi133 bash[105589]: cluster 2023-10-09T05:17:58.375381+0000 mon.a (mon.0) 1154 : cluster 3 Health check update: Degraded data redundancy: 9406/5376 objects degraded (174.963%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:59.419 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:59 smithi133 bash[105589]: audit 2023-10-09T05:17:58.455494+0000 mon.a (mon.0) 1155 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:17:59.419 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:17:59 smithi133 bash[105589]: audit 2023-10-09T05:17:58.456659+0000 mon.a (mon.0) 1156 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:17:59.419 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:59 smithi133 bash[102858]: cluster 2023-10-09T05:17:57.959722+0000 mgr.y (mgr.25277) 885 : cluster 0 pgmap v905: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 111 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 702 op/s; 9406/5376 objects degraded (174.963%); 0 B/s, 30 objects/s recovering 2023-10-09T05:17:59.419 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:59 smithi133 bash[102858]: cluster 2023-10-09T05:17:58.375381+0000 mon.a (mon.0) 1154 : cluster 3 Health check update: Degraded data redundancy: 9406/5376 objects degraded (174.963%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:17:59.419 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:59 smithi133 bash[102858]: audit 2023-10-09T05:17:58.455494+0000 mon.a (mon.0) 1155 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:17:59.419 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:17:59 smithi133 bash[102858]: audit 2023-10-09T05:17:58.456659+0000 mon.a (mon.0) 1156 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:17:59.435 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:17:59.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:59.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:17:59.539 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:59.571 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:59.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:59.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:59.675 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:59.711 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:17:59.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:59.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:17:59.814 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:59.843 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:17:59.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:59.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:17:59.946 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:59.980 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:17:59.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:59.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:00.084 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:00.110 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:00.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:00.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:00.213 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:00.241 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:00.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:00.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:00.345 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:00.372 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:00.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:00.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:00.476 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:00.507 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:00.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:00.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:00.611 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:00.639 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:00.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:00.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:00.744 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:00.774 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:00.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:00.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:00.877 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:00.910 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:00.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:00.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:01.014 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:01.045 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:01.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:01.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:01.150 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:01.180 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:01.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:01.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:01.284 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:01.296 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:01 smithi194 bash[92666]: cluster 2023-10-09T05:17:59.960842+0000 mgr.y (mgr.25277) 886 : cluster 0 pgmap v906: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 84 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 682 op/s; 9406/4086 objects degraded (230.201%); 0 B/s, 30 objects/s recovering 2023-10-09T05:18:01.315 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:01.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:01.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:01.419 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:01.451 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:01.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:01.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:01.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:01 smithi133 bash[102858]: cluster 2023-10-09T05:17:59.960842+0000 mgr.y (mgr.25277) 886 : cluster 0 pgmap v906: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 84 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 682 op/s; 9406/4086 objects degraded (230.201%); 0 B/s, 30 objects/s recovering 2023-10-09T05:18:01.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:01 smithi133 bash[105589]: cluster 2023-10-09T05:17:59.960842+0000 mgr.y (mgr.25277) 886 : cluster 0 pgmap v906: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 84 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 682 op/s; 9406/4086 objects degraded (230.201%); 0 B/s, 30 objects/s recovering 2023-10-09T05:18:01.555 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:01.586 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:01.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:01.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:01.630 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:18:01.630 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:18:01.690 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:01.718 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:01.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:01.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:01.822 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:01.857 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:01.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:01.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:01.960 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:01.986 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:01.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:01.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:02.090 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:02.123 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:02.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:02.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:02.227 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:02.254 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:02.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:02.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:02.358 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:02.384 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:02.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:02.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:02.487 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:02.522 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:02.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:02.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:02.626 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:02.632 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2023-10-09T05:18:02.632 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2023-10-09T05:18:02.644 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:18:02.660 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:02.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:02.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:02.763 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:02.785 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:02.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:02.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:02.889 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:02.909 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:02.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:02.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:03.012 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:03.039 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:03.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:03.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:03.142 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:03.152 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:03 smithi133 bash[105589]: cluster 2023-10-09T05:18:01.961616+0000 mgr.y (mgr.25277) 887 : cluster 0 pgmap v907: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.7 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 581 op/s; 9406/126 objects degraded (7465.079%); 0 B/s, 25 objects/s recovering 2023-10-09T05:18:03.152 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:03 smithi133 bash[102858]: cluster 2023-10-09T05:18:01.961616+0000 mgr.y (mgr.25277) 887 : cluster 0 pgmap v907: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.7 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 581 op/s; 9406/126 objects degraded (7465.079%); 0 B/s, 25 objects/s recovering 2023-10-09T05:18:03.168 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:03.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:03.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:03.273 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:03.303 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:03.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:03.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:03.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:03 smithi194 bash[92666]: cluster 2023-10-09T05:18:01.961616+0000 mgr.y (mgr.25277) 887 : cluster 0 pgmap v907: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.7 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 581 op/s; 9406/126 objects degraded (7465.079%); 0 B/s, 25 objects/s recovering 2023-10-09T05:18:03.407 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:03.440 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:03.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:03.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:03.544 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:03.579 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:03.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:03.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:03.655 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 6, 1] out_osds: [5, 7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:18:03.656 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:18:03.656 INFO:tasks.thrashosds.thrasher:Adding osd 7 2023-10-09T05:18:03.656 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd in 7 2023-10-09T05:18:03.683 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:03.725 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:03.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:03.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:03.828 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:03.860 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:03.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:03.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:03.964 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:03.998 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:04.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:04.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:04.102 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:04.135 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:04.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:04.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:04.240 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:04.267 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:04.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:04.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:04.370 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:04.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:04 smithi194 bash[92666]: cluster 2023-10-09T05:18:03.376467+0000 mon.a (mon.0) 1157 : cluster 3 Health check update: Degraded data redundancy: 9406/126 objects degraded (7465.079%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:18:04.399 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:04.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:04.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:04.461 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:18:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:18:04] "GET /metrics HTTP/1.1" 200 36507 "" "Prometheus/2.43.0" 2023-10-09T05:18:04.463 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:04 smithi133 bash[105589]: cluster 2023-10-09T05:18:03.376467+0000 mon.a (mon.0) 1157 : cluster 3 Health check update: Degraded data redundancy: 9406/126 objects degraded (7465.079%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:18:04.463 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:04 smithi133 bash[102858]: cluster 2023-10-09T05:18:03.376467+0000 mon.a (mon.0) 1157 : cluster 3 Health check update: Degraded data redundancy: 9406/126 objects degraded (7465.079%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:18:04.503 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:04.529 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:04.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:04.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:04.634 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:04.659 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:04.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:04.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:04.762 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:04.792 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:04.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:04.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:04.896 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:04.928 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:04.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:04.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:05.031 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:05.056 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:05.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:05.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:05.160 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:05.188 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:05.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:05.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:05.293 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:05.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:05 smithi133 bash[102858]: cluster 2023-10-09T05:18:03.962568+0000 mgr.y (mgr.25277) 888 : cluster 0 pgmap v908: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 468 op/s; 8446/72 objects degraded (11730.556%); 0 B/s, 105 objects/s recovering 2023-10-09T05:18:05.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:05 smithi133 bash[102858]: cluster 2023-10-09T05:18:04.384560+0000 osd.7 (osd.7) 145 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T05:18:05.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:05 smithi133 bash[102858]: cluster 2023-10-09T05:18:04.387441+0000 mon.a (mon.0) 1158 : cluster 0 osdmap e213: 8 total, 8 up, 6 in 2023-10-09T05:18:05.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:05 smithi133 bash[102858]: cluster 2023-10-09T05:18:04.748059+0000 osd.5 (osd.5) 200 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T05:18:05.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:05 smithi133 bash[102858]: cluster 2023-10-09T05:18:04.748059+0000 osd.6 (osd.6) 150 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T05:18:05.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:05 smithi133 bash[105589]: cluster 2023-10-09T05:18:03.962568+0000 mgr.y (mgr.25277) 888 : cluster 0 pgmap v908: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 468 op/s; 8446/72 objects degraded (11730.556%); 0 B/s, 105 objects/s recovering 2023-10-09T05:18:05.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:05 smithi133 bash[105589]: cluster 2023-10-09T05:18:04.384560+0000 osd.7 (osd.7) 145 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T05:18:05.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:05 smithi133 bash[105589]: cluster 2023-10-09T05:18:04.387441+0000 mon.a (mon.0) 1158 : cluster 0 osdmap e213: 8 total, 8 up, 6 in 2023-10-09T05:18:05.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:05 smithi133 bash[105589]: cluster 2023-10-09T05:18:04.748059+0000 osd.5 (osd.5) 200 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T05:18:05.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:05 smithi133 bash[105589]: cluster 2023-10-09T05:18:04.748059+0000 osd.6 (osd.6) 150 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T05:18:05.319 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:05.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:05.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:05.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:05 smithi194 bash[92666]: cluster 2023-10-09T05:18:03.962568+0000 mgr.y (mgr.25277) 888 : cluster 0 pgmap v908: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 468 op/s; 8446/72 objects degraded (11730.556%); 0 B/s, 105 objects/s recovering 2023-10-09T05:18:05.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:05 smithi194 bash[92666]: cluster 2023-10-09T05:18:04.384560+0000 osd.7 (osd.7) 145 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T05:18:05.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:05 smithi194 bash[92666]: cluster 2023-10-09T05:18:04.387441+0000 mon.a (mon.0) 1158 : cluster 0 osdmap e213: 8 total, 8 up, 6 in 2023-10-09T05:18:05.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:05 smithi194 bash[92666]: cluster 2023-10-09T05:18:04.748059+0000 osd.5 (osd.5) 200 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T05:18:05.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:05 smithi194 bash[92666]: cluster 2023-10-09T05:18:04.748059+0000 osd.6 (osd.6) 150 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T05:18:05.421 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:05.450 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:05.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:05.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:05.554 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:05.585 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:05.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:05.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:05.690 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:05.715 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:05.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:05.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:05.819 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:05.843 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:05.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:05.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:05.946 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:05.974 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:05.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:05.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:06.077 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:06.110 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:06.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:06.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:06.214 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:06.222 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:06 smithi133 bash[105589]: cluster 2023-10-09T05:18:04.393763+0000 osd.4 (osd.4) 165 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T05:18:06.223 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:06 smithi133 bash[105589]: cluster 2023-10-09T05:18:05.386911+0000 osd.7 (osd.7) 146 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T05:18:06.223 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:06 smithi133 bash[105589]: cluster 2023-10-09T05:18:05.387801+0000 osd.5 (osd.5) 201 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T05:18:06.223 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:06 smithi133 bash[105589]: cluster 2023-10-09T05:18:05.390385+0000 mon.a (mon.0) 1159 : cluster 0 osdmap e214: 8 total, 8 up, 6 in 2023-10-09T05:18:06.223 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:06 smithi133 bash[102858]: cluster 2023-10-09T05:18:04.393763+0000 osd.4 (osd.4) 165 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T05:18:06.223 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:06 smithi133 bash[102858]: cluster 2023-10-09T05:18:05.386911+0000 osd.7 (osd.7) 146 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T05:18:06.223 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:06 smithi133 bash[102858]: cluster 2023-10-09T05:18:05.387801+0000 osd.5 (osd.5) 201 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T05:18:06.223 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:06 smithi133 bash[102858]: cluster 2023-10-09T05:18:05.390385+0000 mon.a (mon.0) 1159 : cluster 0 osdmap e214: 8 total, 8 up, 6 in 2023-10-09T05:18:06.240 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:06.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:06.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:06.344 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:06.369 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:06.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:06.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:06.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:06 smithi194 bash[92666]: cluster 2023-10-09T05:18:04.393763+0000 osd.4 (osd.4) 165 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T05:18:06.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:06 smithi194 bash[92666]: cluster 2023-10-09T05:18:05.386911+0000 osd.7 (osd.7) 146 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T05:18:06.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:06 smithi194 bash[92666]: cluster 2023-10-09T05:18:05.387801+0000 osd.5 (osd.5) 201 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T05:18:06.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:06 smithi194 bash[92666]: cluster 2023-10-09T05:18:05.390385+0000 mon.a (mon.0) 1159 : cluster 0 osdmap e214: 8 total, 8 up, 6 in 2023-10-09T05:18:06.473 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:06.505 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:06.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:06.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:06.608 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:06.641 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:06.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:06.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:06.699 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:18:06.745 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:06.773 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:06.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:06.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:06.877 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:06.906 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:06.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:06.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:07.010 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:07.034 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:07.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:07.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:07.137 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:07.166 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:07.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:07.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:07.269 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:07.278 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:07 smithi133 bash[102858]: cluster 2023-10-09T05:18:05.394881+0000 osd.4 (osd.4) 166 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T05:18:07.278 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:07 smithi133 bash[102858]: cluster 2023-10-09T05:18:05.963326+0000 mgr.y (mgr.25277) 889 : cluster 0 pgmap v911: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 220 op/s; 7897/72 objects degraded (10968.056%); 0 B/s, 188 objects/s recovering 2023-10-09T05:18:07.278 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:07 smithi133 bash[102858]: cluster 2023-10-09T05:18:05.970668+0000 osd.6 (osd.6) 151 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T05:18:07.278 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:07 smithi133 bash[102858]: cluster 2023-10-09T05:18:05.972311+0000 osd.6 (osd.6) 152 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T05:18:07.278 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:07 smithi133 bash[102858]: cluster 2023-10-09T05:18:05.973736+0000 osd.6 (osd.6) 153 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T05:18:07.278 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:07 smithi133 bash[105589]: cluster 2023-10-09T05:18:05.394881+0000 osd.4 (osd.4) 166 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T05:18:07.278 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:07 smithi133 bash[105589]: cluster 2023-10-09T05:18:05.963326+0000 mgr.y (mgr.25277) 889 : cluster 0 pgmap v911: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 220 op/s; 7897/72 objects degraded (10968.056%); 0 B/s, 188 objects/s recovering 2023-10-09T05:18:07.278 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:07 smithi133 bash[105589]: cluster 2023-10-09T05:18:05.970668+0000 osd.6 (osd.6) 151 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T05:18:07.279 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:07 smithi133 bash[105589]: cluster 2023-10-09T05:18:05.972311+0000 osd.6 (osd.6) 152 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T05:18:07.279 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:07 smithi133 bash[105589]: cluster 2023-10-09T05:18:05.973736+0000 osd.6 (osd.6) 153 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T05:18:07.293 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:07.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:07.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:07.396 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:07.407 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:07 smithi194 bash[92666]: cluster 2023-10-09T05:18:05.394881+0000 osd.4 (osd.4) 166 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T05:18:07.408 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:07 smithi194 bash[92666]: cluster 2023-10-09T05:18:05.963326+0000 mgr.y (mgr.25277) 889 : cluster 0 pgmap v911: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 220 op/s; 7897/72 objects degraded (10968.056%); 0 B/s, 188 objects/s recovering 2023-10-09T05:18:07.408 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:07 smithi194 bash[92666]: cluster 2023-10-09T05:18:05.970668+0000 osd.6 (osd.6) 151 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T05:18:07.408 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:07 smithi194 bash[92666]: cluster 2023-10-09T05:18:05.972311+0000 osd.6 (osd.6) 152 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T05:18:07.408 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:07 smithi194 bash[92666]: cluster 2023-10-09T05:18:05.973736+0000 osd.6 (osd.6) 153 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T05:18:07.423 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:07.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:07.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:07.527 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:07.548 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:07.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:07.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:07.651 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:07.676 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:07.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:07.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:07.713 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:18:07.780 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:07.806 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:07.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:07.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:07.910 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:07.934 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:07.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:07.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:08.040 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:08.069 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:08.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:08.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:08.172 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:08.195 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:08.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:08.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:08.299 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:08.332 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:08.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:08.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:08.435 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:08.464 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:08.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:08.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:08.568 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:08.587 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:08.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:08.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:08.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:08 smithi194 bash[92666]: cluster 2023-10-09T05:18:07.394766+0000 osd.5 (osd.5) 202 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T05:18:08.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:08 smithi194 bash[92666]: cluster 2023-10-09T05:18:07.399046+0000 osd.7 (osd.7) 147 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T05:18:08.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:08 smithi194 bash[92666]: cluster 2023-10-09T05:18:07.399214+0000 osd.6 (osd.6) 154 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T05:18:08.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:08 smithi194 bash[92666]: cluster 2023-10-09T05:18:07.399254+0000 mon.a (mon.0) 1160 : cluster 0 osdmap e215: 8 total, 8 up, 6 in 2023-10-09T05:18:08.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:08 smithi194 bash[92666]: cluster 2023-10-09T05:18:07.522580+0000 osd.4 (osd.4) 167 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T05:18:08.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:08 smithi194 bash[92666]: cluster 2023-10-09T05:18:07.963998+0000 mgr.y (mgr.25277) 890 : cluster 0 pgmap v913: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 2 op/s; 7678/72 objects degraded (10663.889%); 0 B/s, 287 objects/s recovering 2023-10-09T05:18:08.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:08 smithi194 bash[92666]: cluster 2023-10-09T05:18:08.377930+0000 mon.a (mon.0) 1161 : cluster 3 Health check update: Degraded data redundancy: 7897/72 objects degraded (10968.056%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:18:08.690 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:08.723 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:08.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:08.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:08.747 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:08 smithi133 bash[105589]: cluster 2023-10-09T05:18:07.394766+0000 osd.5 (osd.5) 202 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T05:18:08.747 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:08 smithi133 bash[105589]: cluster 2023-10-09T05:18:07.399046+0000 osd.7 (osd.7) 147 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T05:18:08.747 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:08 smithi133 bash[105589]: cluster 2023-10-09T05:18:07.399214+0000 osd.6 (osd.6) 154 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T05:18:08.747 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:08 smithi133 bash[105589]: cluster 2023-10-09T05:18:07.399254+0000 mon.a (mon.0) 1160 : cluster 0 osdmap e215: 8 total, 8 up, 6 in 2023-10-09T05:18:08.747 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:08 smithi133 bash[105589]: cluster 2023-10-09T05:18:07.522580+0000 osd.4 (osd.4) 167 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T05:18:08.747 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:08 smithi133 bash[105589]: cluster 2023-10-09T05:18:07.963998+0000 mgr.y (mgr.25277) 890 : cluster 0 pgmap v913: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 2 op/s; 7678/72 objects degraded (10663.889%); 0 B/s, 287 objects/s recovering 2023-10-09T05:18:08.747 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:08 smithi133 bash[105589]: cluster 2023-10-09T05:18:08.377930+0000 mon.a (mon.0) 1161 : cluster 3 Health check update: Degraded data redundancy: 7897/72 objects degraded (10968.056%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:18:08.747 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:08 smithi133 bash[102858]: cluster 2023-10-09T05:18:07.394766+0000 osd.5 (osd.5) 202 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T05:18:08.748 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:08 smithi133 bash[102858]: cluster 2023-10-09T05:18:07.399046+0000 osd.7 (osd.7) 147 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T05:18:08.748 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:08 smithi133 bash[102858]: cluster 2023-10-09T05:18:07.399214+0000 osd.6 (osd.6) 154 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T05:18:08.748 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:08 smithi133 bash[102858]: cluster 2023-10-09T05:18:07.399254+0000 mon.a (mon.0) 1160 : cluster 0 osdmap e215: 8 total, 8 up, 6 in 2023-10-09T05:18:08.748 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:08 smithi133 bash[102858]: cluster 2023-10-09T05:18:07.522580+0000 osd.4 (osd.4) 167 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T05:18:08.748 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:08 smithi133 bash[102858]: cluster 2023-10-09T05:18:07.963998+0000 mgr.y (mgr.25277) 890 : cluster 0 pgmap v913: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 2 op/s; 7678/72 objects degraded (10663.889%); 0 B/s, 287 objects/s recovering 2023-10-09T05:18:08.748 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:08 smithi133 bash[102858]: cluster 2023-10-09T05:18:08.377930+0000 mon.a (mon.0) 1161 : cluster 3 Health check update: Degraded data redundancy: 7897/72 objects degraded (10968.056%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:18:08.826 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:08.846 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:08.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:08.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:08.949 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:08.982 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:08.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:08.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:09.086 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:09.105 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:09.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:09.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:09.209 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:09.228 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:09.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:09.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:09.332 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:09.357 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:09.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:09.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:09.460 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:09.484 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:09.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:09.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:09.586 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:09.594 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:09 smithi133 bash[105589]: cluster 2023-10-09T05:18:08.395012+0000 mon.a (mon.0) 1162 : cluster 0 osdmap e216: 8 total, 8 up, 6 in 2023-10-09T05:18:09.595 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:09 smithi133 bash[105589]: cluster 2023-10-09T05:18:08.396658+0000 osd.5 (osd.5) 203 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T05:18:09.595 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:09 smithi133 bash[105589]: cluster 2023-10-09T05:18:08.400505+0000 osd.6 (osd.6) 155 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T05:18:09.595 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:09 smithi133 bash[105589]: cluster 2023-10-09T05:18:08.400758+0000 osd.4 (osd.4) 168 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T05:18:09.595 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:09 smithi133 bash[105589]: cluster 2023-10-09T05:18:08.431279+0000 osd.7 (osd.7) 148 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T05:18:09.595 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:09 smithi133 bash[102858]: cluster 2023-10-09T05:18:08.395012+0000 mon.a (mon.0) 1162 : cluster 0 osdmap e216: 8 total, 8 up, 6 in 2023-10-09T05:18:09.595 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:09 smithi133 bash[102858]: cluster 2023-10-09T05:18:08.396658+0000 osd.5 (osd.5) 203 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T05:18:09.595 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:09 smithi133 bash[102858]: cluster 2023-10-09T05:18:08.400505+0000 osd.6 (osd.6) 155 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T05:18:09.595 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:09 smithi133 bash[102858]: cluster 2023-10-09T05:18:08.400758+0000 osd.4 (osd.4) 168 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T05:18:09.595 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:09 smithi133 bash[102858]: cluster 2023-10-09T05:18:08.431279+0000 osd.7 (osd.7) 148 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T05:18:09.608 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:09.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:09.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:09.712 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:09.723 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:09 smithi194 bash[92666]: cluster 2023-10-09T05:18:08.395012+0000 mon.a (mon.0) 1162 : cluster 0 osdmap e216: 8 total, 8 up, 6 in 2023-10-09T05:18:09.723 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:09 smithi194 bash[92666]: cluster 2023-10-09T05:18:08.396658+0000 osd.5 (osd.5) 203 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T05:18:09.724 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:09 smithi194 bash[92666]: cluster 2023-10-09T05:18:08.400505+0000 osd.6 (osd.6) 155 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T05:18:09.724 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:09 smithi194 bash[92666]: cluster 2023-10-09T05:18:08.400758+0000 osd.4 (osd.4) 168 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T05:18:09.724 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:09 smithi194 bash[92666]: cluster 2023-10-09T05:18:08.431279+0000 osd.7 (osd.7) 148 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T05:18:09.742 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:09.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:09.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:09.845 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:09.876 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:09.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:09.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:09.981 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:10.012 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:10.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:10.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:10.115 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:10.145 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:10.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:10.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:10.249 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:10.274 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:10.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:10.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:10.377 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:10.414 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:10.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:10.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:10.517 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:10.524 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:10 smithi133 bash[102858]: cluster 2023-10-09T05:18:09.964485+0000 mgr.y (mgr.25277) 891 : cluster 0 pgmap v915: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 7678/72 objects degraded (10663.889%); 0 B/s, 137 objects/s recovering 2023-10-09T05:18:10.525 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:10 smithi133 bash[105589]: cluster 2023-10-09T05:18:09.964485+0000 mgr.y (mgr.25277) 891 : cluster 0 pgmap v915: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 7678/72 objects degraded (10663.889%); 0 B/s, 137 objects/s recovering 2023-10-09T05:18:10.537 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:10.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:10.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:10.640 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:10.655 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:10 smithi194 bash[92666]: cluster 2023-10-09T05:18:09.964485+0000 mgr.y (mgr.25277) 891 : cluster 0 pgmap v915: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 7678/72 objects degraded (10663.889%); 0 B/s, 137 objects/s recovering 2023-10-09T05:18:10.674 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:10.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:10.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:10.779 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:10.809 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:10.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:10.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:10.914 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:10.946 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:10.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:10.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:11.050 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:11.080 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:11.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:11.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:11.184 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:11.213 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:11.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:11.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:11.317 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:11.343 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:11.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:11.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:11.446 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:11.477 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:11.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:11.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:11.581 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:11.611 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:11.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:11.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:11.715 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:11.744 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:11.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:11.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:11.847 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:11.871 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:11.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:11.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:11.977 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:12.007 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:12.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:12.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:12.111 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:12.136 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:12.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:12.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:12.240 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:12.268 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:12.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:12.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:12.371 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:12.400 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:12.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:12.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:12.503 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:12.514 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:12 smithi194 bash[92666]: cluster 2023-10-09T05:18:11.415165+0000 osd.5 (osd.5) 204 : cluster 3 failed to encode map e217 with expected crc 2023-10-09T05:18:12.514 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:12 smithi194 bash[92666]: cluster 2023-10-09T05:18:11.418967+0000 mon.a (mon.0) 1163 : cluster 0 osdmap e217: 8 total, 8 up, 6 in 2023-10-09T05:18:12.514 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:12 smithi194 bash[92666]: cluster 2023-10-09T05:18:11.419871+0000 osd.4 (osd.4) 169 : cluster 3 failed to encode map e217 with expected crc 2023-10-09T05:18:12.514 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:12 smithi194 bash[92666]: cluster 2023-10-09T05:18:11.428337+0000 osd.7 (osd.7) 149 : cluster 3 failed to encode map e217 with expected crc 2023-10-09T05:18:12.514 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:12 smithi194 bash[92666]: cluster 2023-10-09T05:18:11.965248+0000 mgr.y (mgr.25277) 892 : cluster 0 pgmap v917: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 6177/72 objects degraded (8579.167%); 96 KiB/s, 286 objects/s recovering 2023-10-09T05:18:12.550 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:12.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:12.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:12.654 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:12.665 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:12 smithi133 bash[105589]: cluster 2023-10-09T05:18:11.415165+0000 osd.5 (osd.5) 204 : cluster 3 failed to encode map e217 with expected crc 2023-10-09T05:18:12.665 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:12 smithi133 bash[105589]: cluster 2023-10-09T05:18:11.418967+0000 mon.a (mon.0) 1163 : cluster 0 osdmap e217: 8 total, 8 up, 6 in 2023-10-09T05:18:12.665 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:12 smithi133 bash[105589]: cluster 2023-10-09T05:18:11.419871+0000 osd.4 (osd.4) 169 : cluster 3 failed to encode map e217 with expected crc 2023-10-09T05:18:12.665 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:12 smithi133 bash[105589]: cluster 2023-10-09T05:18:11.428337+0000 osd.7 (osd.7) 149 : cluster 3 failed to encode map e217 with expected crc 2023-10-09T05:18:12.665 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:12 smithi133 bash[105589]: cluster 2023-10-09T05:18:11.965248+0000 mgr.y (mgr.25277) 892 : cluster 0 pgmap v917: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 6177/72 objects degraded (8579.167%); 96 KiB/s, 286 objects/s recovering 2023-10-09T05:18:12.665 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:12 smithi133 bash[102858]: cluster 2023-10-09T05:18:11.415165+0000 osd.5 (osd.5) 204 : cluster 3 failed to encode map e217 with expected crc 2023-10-09T05:18:12.665 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:12 smithi133 bash[102858]: cluster 2023-10-09T05:18:11.418967+0000 mon.a (mon.0) 1163 : cluster 0 osdmap e217: 8 total, 8 up, 6 in 2023-10-09T05:18:12.665 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:12 smithi133 bash[102858]: cluster 2023-10-09T05:18:11.419871+0000 osd.4 (osd.4) 169 : cluster 3 failed to encode map e217 with expected crc 2023-10-09T05:18:12.666 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:12 smithi133 bash[102858]: cluster 2023-10-09T05:18:11.428337+0000 osd.7 (osd.7) 149 : cluster 3 failed to encode map e217 with expected crc 2023-10-09T05:18:12.666 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:12 smithi133 bash[102858]: cluster 2023-10-09T05:18:11.965248+0000 mgr.y (mgr.25277) 892 : cluster 0 pgmap v917: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 6177/72 objects degraded (8579.167%); 96 KiB/s, 286 objects/s recovering 2023-10-09T05:18:12.678 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:12.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:12.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:12.781 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:12.804 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:12.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:12.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:12.907 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:12.937 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:12.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:12.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:13.040 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:13.066 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:13.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:13.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:13.169 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:13.199 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:13.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:13.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:13.302 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:13.331 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:13.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:13.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:13.434 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:13.446 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:13 smithi194 bash[92666]: cluster 2023-10-09T05:18:12.425607+0000 osd.4 (osd.4) 170 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T05:18:13.447 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:13 smithi194 bash[92666]: cluster 2023-10-09T05:18:12.426832+0000 mon.a (mon.0) 1164 : cluster 0 osdmap e218: 8 total, 8 up, 6 in 2023-10-09T05:18:13.447 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:13 smithi194 bash[92666]: cluster 2023-10-09T05:18:12.756861+0000 osd.7 (osd.7) 150 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T05:18:13.447 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:13 smithi194 bash[92666]: cluster 2023-10-09T05:18:13.379041+0000 mon.a (mon.0) 1165 : cluster 3 Health check update: Degraded data redundancy: 6177/72 objects degraded (8579.167%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:18:13.465 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:13.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:13.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:13.568 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:13.576 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:13 smithi133 bash[102858]: cluster 2023-10-09T05:18:12.425607+0000 osd.4 (osd.4) 170 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T05:18:13.576 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:13 smithi133 bash[102858]: cluster 2023-10-09T05:18:12.426832+0000 mon.a (mon.0) 1164 : cluster 0 osdmap e218: 8 total, 8 up, 6 in 2023-10-09T05:18:13.576 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:13 smithi133 bash[102858]: cluster 2023-10-09T05:18:12.756861+0000 osd.7 (osd.7) 150 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T05:18:13.576 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:13 smithi133 bash[102858]: cluster 2023-10-09T05:18:13.379041+0000 mon.a (mon.0) 1165 : cluster 3 Health check update: Degraded data redundancy: 6177/72 objects degraded (8579.167%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:18:13.576 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:13 smithi133 bash[105589]: cluster 2023-10-09T05:18:12.425607+0000 osd.4 (osd.4) 170 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T05:18:13.576 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:13 smithi133 bash[105589]: cluster 2023-10-09T05:18:12.426832+0000 mon.a (mon.0) 1164 : cluster 0 osdmap e218: 8 total, 8 up, 6 in 2023-10-09T05:18:13.576 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:13 smithi133 bash[105589]: cluster 2023-10-09T05:18:12.756861+0000 osd.7 (osd.7) 150 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T05:18:13.577 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:13 smithi133 bash[105589]: cluster 2023-10-09T05:18:13.379041+0000 mon.a (mon.0) 1165 : cluster 3 Health check update: Degraded data redundancy: 6177/72 objects degraded (8579.167%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:18:13.590 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:13.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:13.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:13.694 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:13.722 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:13.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:13.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:13.825 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:13.851 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:13.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:13.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:13.955 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:13.984 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:13.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:13.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:14.088 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:14.113 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:14.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:14.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:14.216 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:14.241 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:14.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:14.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:14.344 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:14.370 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:14.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:14.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:14.474 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:14.483 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:18:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:18:14] "GET /metrics HTTP/1.1" 200 36500 "" "Prometheus/2.43.0" 2023-10-09T05:18:14.483 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:14 smithi133 bash[105589]: audit 2023-10-09T05:18:13.452107+0000 mon.a (mon.0) 1166 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:18:14.483 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:14 smithi133 bash[105589]: cluster 2023-10-09T05:18:13.548697+0000 osd.6 (osd.6) 156 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T05:18:14.484 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:14 smithi133 bash[105589]: cluster 2023-10-09T05:18:13.548745+0000 osd.5 (osd.5) 205 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T05:18:14.484 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:14 smithi133 bash[105589]: cluster 2023-10-09T05:18:13.965869+0000 mgr.y (mgr.25277) 893 : cluster 0 pgmap v919: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 6177/72 objects degraded (8579.167%); 96 KiB/s, 250 objects/s recovering 2023-10-09T05:18:14.484 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:14 smithi133 bash[102858]: audit 2023-10-09T05:18:13.452107+0000 mon.a (mon.0) 1166 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:18:14.484 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:14 smithi133 bash[102858]: cluster 2023-10-09T05:18:13.548697+0000 osd.6 (osd.6) 156 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T05:18:14.484 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:14 smithi133 bash[102858]: cluster 2023-10-09T05:18:13.548745+0000 osd.5 (osd.5) 205 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T05:18:14.484 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:14 smithi133 bash[102858]: cluster 2023-10-09T05:18:13.965869+0000 mgr.y (mgr.25277) 893 : cluster 0 pgmap v919: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 6177/72 objects degraded (8579.167%); 96 KiB/s, 250 objects/s recovering 2023-10-09T05:18:14.502 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:14.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:14.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:14.606 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:14.618 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:14 smithi194 bash[92666]: audit 2023-10-09T05:18:13.452107+0000 mon.a (mon.0) 1166 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:18:14.619 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:14 smithi194 bash[92666]: cluster 2023-10-09T05:18:13.548697+0000 osd.6 (osd.6) 156 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T05:18:14.619 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:14 smithi194 bash[92666]: cluster 2023-10-09T05:18:13.548745+0000 osd.5 (osd.5) 205 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T05:18:14.619 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:14 smithi194 bash[92666]: cluster 2023-10-09T05:18:13.965869+0000 mgr.y (mgr.25277) 893 : cluster 0 pgmap v919: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 6177/72 objects degraded (8579.167%); 96 KiB/s, 250 objects/s recovering 2023-10-09T05:18:14.639 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:14.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:14.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:14.744 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:14.772 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:14.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:14.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:14.877 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:14.898 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:14.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:14.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:15.001 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:15.032 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:15.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:15.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:15.136 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:15.169 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:15.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:15.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:15.272 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:15.297 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:15.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:15.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:15.400 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:15.421 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:15.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:15.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:15.524 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:15.544 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:15.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:15.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:15.647 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:15.678 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:15.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:15.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:15.782 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:15.801 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:15.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:15.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:15.904 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:15.933 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:15.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:15.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:16.036 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:16.058 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:16.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:16.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:16.161 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:16.185 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:16.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:16.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:16.288 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:16.312 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:16.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:16.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:16.416 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:16.439 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:16.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:16.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:16.543 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:16.569 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:16.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:16.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:16.672 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:16.697 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:16.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:16.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:16.800 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:16.836 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:16.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:16.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:16.940 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:16.967 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:16.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:16.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:17.070 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:17.101 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:17.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:17.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:17.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:17 smithi194 bash[92666]: cluster 2023-10-09T05:18:15.966623+0000 mgr.y (mgr.25277) 894 : cluster 0 pgmap v920: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 5819/72 objects degraded (8081.944%); 76 KiB/s, 245 objects/s recovering 2023-10-09T05:18:17.204 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:17.216 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:17 smithi133 bash[105589]: cluster 2023-10-09T05:18:15.966623+0000 mgr.y (mgr.25277) 894 : cluster 0 pgmap v920: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 5819/72 objects degraded (8081.944%); 76 KiB/s, 245 objects/s recovering 2023-10-09T05:18:17.216 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:17 smithi133 bash[102858]: cluster 2023-10-09T05:18:15.966623+0000 mgr.y (mgr.25277) 894 : cluster 0 pgmap v920: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 5819/72 objects degraded (8081.944%); 76 KiB/s, 245 objects/s recovering 2023-10-09T05:18:17.232 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:17.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:17.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:17.335 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:17.363 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:17.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:17.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:17.467 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:17.489 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:17.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:17.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:17.593 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:17.619 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:17.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:17.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:17.723 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:17.754 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:17.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:17.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:17.858 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:17.885 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:17.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:17.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:17.988 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:18.011 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:18.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:18.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:18.113 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:18.142 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:18.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:18.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:18.245 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:18.278 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:18.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:18.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:18.381 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:18.399 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:18:18.399 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:18:18.407 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:18.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:18.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:18.512 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:18.535 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:18.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:18.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:18.639 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:18.667 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:18.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:18.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:18.770 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:18.797 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:18.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:18.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:18.901 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:18.928 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:18.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:18.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:19.031 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:19.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:19 smithi133 bash[102858]: cluster 2023-10-09T05:18:17.967437+0000 mgr.y (mgr.25277) 895 : cluster 0 pgmap v921: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 5819/72 objects degraded (8081.944%); 72 KiB/s, 232 objects/s recovering 2023-10-09T05:18:19.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:19 smithi133 bash[102858]: cluster 2023-10-09T05:18:18.380073+0000 mon.a (mon.0) 1167 : cluster 3 Health check update: Degraded data redundancy: 5819/72 objects degraded (8081.944%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:18:19.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:19 smithi133 bash[105589]: cluster 2023-10-09T05:18:17.967437+0000 mgr.y (mgr.25277) 895 : cluster 0 pgmap v921: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 5819/72 objects degraded (8081.944%); 72 KiB/s, 232 objects/s recovering 2023-10-09T05:18:19.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:19 smithi133 bash[105589]: cluster 2023-10-09T05:18:18.380073+0000 mon.a (mon.0) 1167 : cluster 3 Health check update: Degraded data redundancy: 5819/72 objects degraded (8081.944%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:18:19.062 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:19.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:19.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:19.166 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:19.193 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:19.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:19.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:19.297 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:19.320 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:19.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:19.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:19.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:19 smithi194 bash[92666]: cluster 2023-10-09T05:18:17.967437+0000 mgr.y (mgr.25277) 895 : cluster 0 pgmap v921: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 5819/72 objects degraded (8081.944%); 72 KiB/s, 232 objects/s recovering 2023-10-09T05:18:19.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:19 smithi194 bash[92666]: cluster 2023-10-09T05:18:18.380073+0000 mon.a (mon.0) 1167 : cluster 3 Health check update: Degraded data redundancy: 5819/72 objects degraded (8081.944%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:18:19.424 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:19.451 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:19.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:19.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:19.555 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:19.576 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:19.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:19.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:19.679 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:19.711 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:19.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:19.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:19.814 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:19.837 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:19.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:19.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:19.940 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:19.969 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:19.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:19.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:20.072 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:20.094 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:20.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:20.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:20.198 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:20.217 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:20.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:20.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:20.321 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:20.344 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:20.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:20.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:20.447 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:20.468 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:20.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:20.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:20.571 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:20.611 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:20.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:20.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:20.636 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:18:20.657 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:18:20.681 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:18:20.702 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:18:20.717 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:20.730 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:18:20.736 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:20.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:20.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:20.752 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:18:20.775 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:18:20.801 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:18:20.802 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:18:20.802 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:18:20.802 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:18:20.802 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:18:20.802 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:18:20.803 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:18:20.803 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:18:20.803 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:18:20.839 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:20.861 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:20.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:20.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:20.964 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:20.990 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:20.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:20.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:21.043 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is set 2023-10-09T05:18:21.093 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:21.113 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:21.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:21.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:21.216 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:21.237 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:21.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:21.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:21.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:21 smithi133 bash[105589]: cluster 2023-10-09T05:18:19.967999+0000 mgr.y (mgr.25277) 896 : cluster 0 pgmap v922: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 5315/72 objects degraded (7381.944%); 0 B/s, 221 objects/s recovering 2023-10-09T05:18:21.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:21 smithi133 bash[105589]: audit 2023-10-09T05:18:20.482102+0000 mon.c (mon.2) 87 : audit 1 from='client.? 172.21.15.133:0/3226741925' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:18:21.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:21 smithi133 bash[105589]: audit 2023-10-09T05:18:20.482515+0000 mon.a (mon.0) 1168 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:18:21.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:21 smithi133 bash[105589]: audit 2023-10-09T05:18:20.609772+0000 mon.a (mon.0) 1169 : audit 0 from='client.? 172.21.15.133:0/866610943' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:18:21.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:21 smithi133 bash[102858]: cluster 2023-10-09T05:18:19.967999+0000 mgr.y (mgr.25277) 896 : cluster 0 pgmap v922: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 5315/72 objects degraded (7381.944%); 0 B/s, 221 objects/s recovering 2023-10-09T05:18:21.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:21 smithi133 bash[102858]: audit 2023-10-09T05:18:20.482102+0000 mon.c (mon.2) 87 : audit 1 from='client.? 172.21.15.133:0/3226741925' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:18:21.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:21 smithi133 bash[102858]: audit 2023-10-09T05:18:20.482515+0000 mon.a (mon.0) 1168 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:18:21.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:21 smithi133 bash[102858]: audit 2023-10-09T05:18:20.609772+0000 mon.a (mon.0) 1169 : audit 0 from='client.? 172.21.15.133:0/866610943' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:18:21.340 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:21.367 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:21.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:21.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:21.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:21 smithi194 bash[92666]: cluster 2023-10-09T05:18:19.967999+0000 mgr.y (mgr.25277) 896 : cluster 0 pgmap v922: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 5315/72 objects degraded (7381.944%); 0 B/s, 221 objects/s recovering 2023-10-09T05:18:21.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:21 smithi194 bash[92666]: audit 2023-10-09T05:18:20.482102+0000 mon.c (mon.2) 87 : audit 1 from='client.? 172.21.15.133:0/3226741925' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:18:21.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:21 smithi194 bash[92666]: audit 2023-10-09T05:18:20.482515+0000 mon.a (mon.0) 1168 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:18:21.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:21 smithi194 bash[92666]: audit 2023-10-09T05:18:20.609772+0000 mon.a (mon.0) 1169 : audit 0 from='client.? 172.21.15.133:0/866610943' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:18:21.470 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:21.501 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:21.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:21.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:21.604 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:21.627 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:21.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:21.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:21.732 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:21.759 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:21.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:21.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:21.863 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:21.888 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:21.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:21.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:21.990 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:22.014 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:22.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:22.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:22.117 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:22.143 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:22.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:22.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:22.247 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:22.254 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:22 smithi194 bash[92666]: cluster 2023-10-09T05:18:21.028402+0000 mon.a (mon.0) 1170 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:18:22.255 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:22 smithi194 bash[92666]: cluster 2023-10-09T05:18:21.032819+0000 osd.6 (osd.6) 157 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T05:18:22.255 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:22 smithi194 bash[92666]: cluster 2023-10-09T05:18:21.035662+0000 osd.5 (osd.5) 206 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T05:18:22.255 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:22 smithi194 bash[92666]: audit 2023-10-09T05:18:21.035983+0000 mon.a (mon.0) 1171 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:18:22.255 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:22 smithi194 bash[92666]: cluster 2023-10-09T05:18:21.036145+0000 mon.a (mon.0) 1172 : cluster 0 osdmap e219: 8 total, 8 up, 6 in 2023-10-09T05:18:22.268 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:22.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:22.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:22.371 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:22.396 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:22.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:22.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:22.418 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:22 smithi133 bash[102858]: cluster 2023-10-09T05:18:21.028402+0000 mon.a (mon.0) 1170 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:18:22.418 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:22 smithi133 bash[102858]: cluster 2023-10-09T05:18:21.032819+0000 osd.6 (osd.6) 157 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T05:18:22.418 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:22 smithi133 bash[102858]: cluster 2023-10-09T05:18:21.035662+0000 osd.5 (osd.5) 206 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T05:18:22.419 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:22 smithi133 bash[102858]: audit 2023-10-09T05:18:21.035983+0000 mon.a (mon.0) 1171 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:18:22.419 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:22 smithi133 bash[102858]: cluster 2023-10-09T05:18:21.036145+0000 mon.a (mon.0) 1172 : cluster 0 osdmap e219: 8 total, 8 up, 6 in 2023-10-09T05:18:22.419 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:22 smithi133 bash[105589]: cluster 2023-10-09T05:18:21.028402+0000 mon.a (mon.0) 1170 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:18:22.419 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:22 smithi133 bash[105589]: cluster 2023-10-09T05:18:21.032819+0000 osd.6 (osd.6) 157 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T05:18:22.419 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:22 smithi133 bash[105589]: cluster 2023-10-09T05:18:21.035662+0000 osd.5 (osd.5) 206 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T05:18:22.420 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:22 smithi133 bash[105589]: audit 2023-10-09T05:18:21.035983+0000 mon.a (mon.0) 1171 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:18:22.420 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:22 smithi133 bash[105589]: cluster 2023-10-09T05:18:21.036145+0000 mon.a (mon.0) 1172 : cluster 0 osdmap e219: 8 total, 8 up, 6 in 2023-10-09T05:18:22.421 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T05:18:22.499 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:22.524 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:22.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:22.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:22.626 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:22.653 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:22.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:22.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:22.759 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:22.790 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:22.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:22.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:22.894 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:22.926 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:22.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:22.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:23.030 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:23.060 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:23.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:23.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:23.164 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:23.189 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:23.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:23.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:23.292 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:23.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:23 smithi133 bash[102858]: cluster 2023-10-09T05:18:21.968532+0000 mgr.y (mgr.25277) 897 : cluster 0 pgmap v924: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 5315/72 objects degraded (7381.944%); 0 B/s, 90 objects/s recovering 2023-10-09T05:18:23.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:23 smithi133 bash[105589]: cluster 2023-10-09T05:18:21.968532+0000 mgr.y (mgr.25277) 897 : cluster 0 pgmap v924: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 5315/72 objects degraded (7381.944%); 0 B/s, 90 objects/s recovering 2023-10-09T05:18:23.317 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:23.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:23.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:23.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:23 smithi194 bash[92666]: cluster 2023-10-09T05:18:21.968532+0000 mgr.y (mgr.25277) 897 : cluster 0 pgmap v924: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 5315/72 objects degraded (7381.944%); 0 B/s, 90 objects/s recovering 2023-10-09T05:18:23.420 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:23.455 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:23.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:23.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:23.559 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:23.587 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:23.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:23.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:23.691 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:23.722 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:23.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:23.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:23.826 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:23.855 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:23.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:23.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:23.959 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:23.995 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:23.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:23.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:24.099 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:24.124 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:24.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:24.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:24.228 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:24.252 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:24.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:24.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:24.347 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:24 smithi133 bash[102858]: cluster 2023-10-09T05:18:23.381022+0000 mon.a (mon.0) 1173 : cluster 3 Health check update: Degraded data redundancy: 5315/72 objects degraded (7381.944%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:18:24.347 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:24 smithi133 bash[105589]: cluster 2023-10-09T05:18:23.381022+0000 mon.a (mon.0) 1173 : cluster 3 Health check update: Degraded data redundancy: 5315/72 objects degraded (7381.944%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:18:24.348 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:18:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:18:24] "GET /metrics HTTP/1.1" 200 36500 "" "Prometheus/2.43.0" 2023-10-09T05:18:24.355 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:24.384 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:24.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:24 smithi194 bash[92666]: cluster 2023-10-09T05:18:23.381022+0000 mon.a (mon.0) 1173 : cluster 3 Health check update: Degraded data redundancy: 5315/72 objects degraded (7381.944%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:18:24.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:24.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:24.488 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:24.522 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:24.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:24.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:24.625 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:24.651 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:24.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:24.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:24.756 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:24.791 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:24.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:24.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:24.895 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:24.923 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:24.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:24.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:25.026 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:25.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:24 smithi133 bash[102858]: debug 2023-10-09T05:18:24.827+0000 7f5eb4d1d700 -1 mon.a@0(leader).osd e219 definitely_dead 0 2023-10-09T05:18:25.058 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:25.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:25.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:25.161 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:25.191 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:25.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:25.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:25.294 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:25.307 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:25 smithi194 bash[92666]: cluster 2023-10-09T05:18:23.969510+0000 mgr.y (mgr.25277) 898 : cluster 0 pgmap v925: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 5315/72 objects degraded (7381.944%); 0 B/s, 86 objects/s recovering 2023-10-09T05:18:25.307 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:25 smithi194 bash[92666]: audit 2023-10-09T05:18:24.832893+0000 mon.c (mon.2) 88 : audit 0 from='client.? 172.21.15.133:0/1974527867' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T05:18:25.307 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:25 smithi194 bash[92666]: audit 2023-10-09T05:18:24.833265+0000 mon.a (mon.0) 1174 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T05:18:25.322 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:25.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:25.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:25.411 INFO:teuthology.orchestra.run.smithi133.stderr:marked in osd.7. 2023-10-09T05:18:25.426 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:25.427 INFO:teuthology.orchestra.run.smithi133.stderr:pool 'unique_pool_5' removed 2023-10-09T05:18:25.467 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:25.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:25.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:25.484 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:25 smithi133 bash[102858]: cluster 2023-10-09T05:18:23.969510+0000 mgr.y (mgr.25277) 898 : cluster 0 pgmap v925: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 5315/72 objects degraded (7381.944%); 0 B/s, 86 objects/s recovering 2023-10-09T05:18:25.484 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:25 smithi133 bash[102858]: audit 2023-10-09T05:18:24.832893+0000 mon.c (mon.2) 88 : audit 0 from='client.? 172.21.15.133:0/1974527867' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T05:18:25.484 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:25 smithi133 bash[102858]: audit 2023-10-09T05:18:24.833265+0000 mon.a (mon.0) 1174 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T05:18:25.484 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:25 smithi133 bash[105589]: cluster 2023-10-09T05:18:23.969510+0000 mgr.y (mgr.25277) 898 : cluster 0 pgmap v925: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 5315/72 objects degraded (7381.944%); 0 B/s, 86 objects/s recovering 2023-10-09T05:18:25.485 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:25 smithi133 bash[105589]: audit 2023-10-09T05:18:24.832893+0000 mon.c (mon.2) 88 : audit 0 from='client.? 172.21.15.133:0/1974527867' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T05:18:25.485 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:25 smithi133 bash[105589]: audit 2023-10-09T05:18:24.833265+0000 mon.a (mon.0) 1174 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T05:18:25.572 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:25.594 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:25.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:25.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:25.697 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:25.722 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:25.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:25.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:25.825 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:25.855 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:25.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:25.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:25.959 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:25.992 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:25.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:25.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:26.096 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:26.108 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:26 smithi194 bash[92666]: audit 2023-10-09T05:18:25.165255+0000 mon.a (mon.0) 1175 : audit 1 from='client.? 172.21.15.133:0/2914653170' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:18:26.108 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:26 smithi194 bash[92666]: cluster 2023-10-09T05:18:25.391548+0000 osd.5 (osd.5) 207 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T05:18:26.108 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:26 smithi194 bash[92666]: cluster 2023-10-09T05:18:25.393539+0000 osd.7 (osd.7) 151 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T05:18:26.108 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:26 smithi194 bash[92666]: audit 2023-10-09T05:18:25.394979+0000 mon.a (mon.0) 1176 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T05:18:26.108 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:26 smithi194 bash[92666]: cluster 2023-10-09T05:18:25.396748+0000 osd.7 (osd.7) 152 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T05:18:26.108 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:26 smithi194 bash[92666]: audit 2023-10-09T05:18:25.397729+0000 mon.a (mon.0) 1177 : audit 1 from='client.? 172.21.15.133:0/2914653170' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:18:26.108 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:26 smithi194 bash[92666]: cluster 2023-10-09T05:18:25.397975+0000 mon.a (mon.0) 1178 : cluster 0 osdmap e220: 8 total, 8 up, 7 in 2023-10-09T05:18:26.108 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:26 smithi194 bash[92666]: cluster 2023-10-09T05:18:25.404104+0000 osd.6 (osd.6) 158 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T05:18:26.127 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:26.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:26.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:26.232 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:26.267 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:26.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:26.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:26.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:26 smithi133 bash[102858]: audit 2023-10-09T05:18:25.165255+0000 mon.a (mon.0) 1175 : audit 1 from='client.? 172.21.15.133:0/2914653170' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:18:26.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:26 smithi133 bash[102858]: cluster 2023-10-09T05:18:25.391548+0000 osd.5 (osd.5) 207 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T05:18:26.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:26 smithi133 bash[102858]: cluster 2023-10-09T05:18:25.393539+0000 osd.7 (osd.7) 151 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T05:18:26.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:26 smithi133 bash[102858]: audit 2023-10-09T05:18:25.394979+0000 mon.a (mon.0) 1176 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T05:18:26.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:26 smithi133 bash[102858]: cluster 2023-10-09T05:18:25.396748+0000 osd.7 (osd.7) 152 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T05:18:26.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:26 smithi133 bash[102858]: audit 2023-10-09T05:18:25.397729+0000 mon.a (mon.0) 1177 : audit 1 from='client.? 172.21.15.133:0/2914653170' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:18:26.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:26 smithi133 bash[102858]: cluster 2023-10-09T05:18:25.397975+0000 mon.a (mon.0) 1178 : cluster 0 osdmap e220: 8 total, 8 up, 7 in 2023-10-09T05:18:26.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:26 smithi133 bash[102858]: cluster 2023-10-09T05:18:25.404104+0000 osd.6 (osd.6) 158 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T05:18:26.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:26 smithi133 bash[105589]: audit 2023-10-09T05:18:25.165255+0000 mon.a (mon.0) 1175 : audit 1 from='client.? 172.21.15.133:0/2914653170' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:18:26.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:26 smithi133 bash[105589]: cluster 2023-10-09T05:18:25.391548+0000 osd.5 (osd.5) 207 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T05:18:26.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:26 smithi133 bash[105589]: cluster 2023-10-09T05:18:25.393539+0000 osd.7 (osd.7) 151 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T05:18:26.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:26 smithi133 bash[105589]: audit 2023-10-09T05:18:25.394979+0000 mon.a (mon.0) 1176 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T05:18:26.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:26 smithi133 bash[105589]: cluster 2023-10-09T05:18:25.396748+0000 osd.7 (osd.7) 152 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T05:18:26.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:26 smithi133 bash[105589]: audit 2023-10-09T05:18:25.397729+0000 mon.a (mon.0) 1177 : audit 1 from='client.? 172.21.15.133:0/2914653170' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:18:26.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:26 smithi133 bash[105589]: cluster 2023-10-09T05:18:25.397975+0000 mon.a (mon.0) 1178 : cluster 0 osdmap e220: 8 total, 8 up, 7 in 2023-10-09T05:18:26.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:26 smithi133 bash[105589]: cluster 2023-10-09T05:18:25.404104+0000 osd.6 (osd.6) 158 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T05:18:26.371 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:26.403 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:26.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:26.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:26.436 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:18:26.449 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:18:26.507 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:26.549 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:26.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:26.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:26.655 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:26.675 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:26.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:26.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:26.778 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:26.814 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:26.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:26.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:26.918 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:26.949 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:26.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:26.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:27.052 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:27.082 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:27.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:27.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:27.186 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:27.221 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:27.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:27.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:27.325 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:27.337 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:27 smithi194 bash[92666]: cluster 2023-10-09T05:18:25.395037+0000 osd.4 (osd.4) 171 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T05:18:27.337 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:27 smithi194 bash[92666]: cluster 2023-10-09T05:18:25.970214+0000 mgr.y (mgr.25277) 899 : cluster 0 pgmap v927: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail 2023-10-09T05:18:27.337 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:27 smithi194 bash[92666]: cluster 2023-10-09T05:18:26.388182+0000 mon.a (mon.0) 1179 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5315/72 objects degraded (7381.944%), 3 pgs degraded, 3 pgs undersized) 2023-10-09T05:18:27.337 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:27 smithi194 bash[92666]: cluster 2023-10-09T05:18:26.402480+0000 mon.a (mon.0) 1180 : cluster 0 osdmap e221: 8 total, 8 up, 7 in 2023-10-09T05:18:27.337 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:27 smithi194 bash[92666]: cluster 2023-10-09T05:18:26.403371+0000 osd.7 (osd.7) 153 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T05:18:27.338 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:27 smithi194 bash[92666]: cluster 2023-10-09T05:18:26.403405+0000 osd.6 (osd.6) 159 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T05:18:27.338 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:27 smithi194 bash[92666]: cluster 2023-10-09T05:18:26.403617+0000 osd.5 (osd.5) 208 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T05:18:27.338 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:27 smithi194 bash[92666]: cluster 2023-10-09T05:18:26.404919+0000 osd.7 (osd.7) 154 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T05:18:27.338 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:27 smithi194 bash[92666]: cluster 2023-10-09T05:18:26.408145+0000 osd.6 (osd.6) 160 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T05:18:27.338 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:27 smithi194 bash[92666]: cluster 2023-10-09T05:18:26.411006+0000 osd.6 (osd.6) 161 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T05:18:27.355 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:27.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:27.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:27.459 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:27.467 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:27 smithi133 bash[105589]: cluster 2023-10-09T05:18:25.395037+0000 osd.4 (osd.4) 171 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T05:18:27.468 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:27 smithi133 bash[105589]: cluster 2023-10-09T05:18:25.970214+0000 mgr.y (mgr.25277) 899 : cluster 0 pgmap v927: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail 2023-10-09T05:18:27.468 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:27 smithi133 bash[105589]: cluster 2023-10-09T05:18:26.388182+0000 mon.a (mon.0) 1179 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5315/72 objects degraded (7381.944%), 3 pgs degraded, 3 pgs undersized) 2023-10-09T05:18:27.468 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:27 smithi133 bash[105589]: cluster 2023-10-09T05:18:26.402480+0000 mon.a (mon.0) 1180 : cluster 0 osdmap e221: 8 total, 8 up, 7 in 2023-10-09T05:18:27.468 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:27 smithi133 bash[105589]: cluster 2023-10-09T05:18:26.403371+0000 osd.7 (osd.7) 153 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T05:18:27.468 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:27 smithi133 bash[105589]: cluster 2023-10-09T05:18:26.403405+0000 osd.6 (osd.6) 159 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T05:18:27.468 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:27 smithi133 bash[105589]: cluster 2023-10-09T05:18:26.403617+0000 osd.5 (osd.5) 208 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T05:18:27.468 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:27 smithi133 bash[105589]: cluster 2023-10-09T05:18:26.404919+0000 osd.7 (osd.7) 154 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T05:18:27.469 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:27 smithi133 bash[105589]: cluster 2023-10-09T05:18:26.408145+0000 osd.6 (osd.6) 160 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T05:18:27.469 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:27 smithi133 bash[105589]: cluster 2023-10-09T05:18:26.411006+0000 osd.6 (osd.6) 161 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T05:18:27.469 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:27 smithi133 bash[102858]: cluster 2023-10-09T05:18:25.395037+0000 osd.4 (osd.4) 171 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T05:18:27.469 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:27 smithi133 bash[102858]: cluster 2023-10-09T05:18:25.970214+0000 mgr.y (mgr.25277) 899 : cluster 0 pgmap v927: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail 2023-10-09T05:18:27.469 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:27 smithi133 bash[102858]: cluster 2023-10-09T05:18:26.388182+0000 mon.a (mon.0) 1179 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5315/72 objects degraded (7381.944%), 3 pgs degraded, 3 pgs undersized) 2023-10-09T05:18:27.470 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:27 smithi133 bash[102858]: cluster 2023-10-09T05:18:26.402480+0000 mon.a (mon.0) 1180 : cluster 0 osdmap e221: 8 total, 8 up, 7 in 2023-10-09T05:18:27.470 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:27 smithi133 bash[102858]: cluster 2023-10-09T05:18:26.403371+0000 osd.7 (osd.7) 153 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T05:18:27.470 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:27 smithi133 bash[102858]: cluster 2023-10-09T05:18:26.403405+0000 osd.6 (osd.6) 159 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T05:18:27.470 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:27 smithi133 bash[102858]: cluster 2023-10-09T05:18:26.403617+0000 osd.5 (osd.5) 208 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T05:18:27.470 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:27 smithi133 bash[102858]: cluster 2023-10-09T05:18:26.404919+0000 osd.7 (osd.7) 154 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T05:18:27.470 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:27 smithi133 bash[102858]: cluster 2023-10-09T05:18:26.408145+0000 osd.6 (osd.6) 160 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T05:18:27.471 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:27 smithi133 bash[102858]: cluster 2023-10-09T05:18:26.411006+0000 osd.6 (osd.6) 161 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T05:18:27.472 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T05:18:27.473 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T05:18:27.473 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_6 2023-10-09T05:18:27.473 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_6 16 2023-10-09T05:18:27.494 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:27.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:27.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:27.512 INFO:tasks.thrashosds.thrasher:Added osd 7 2023-10-09T05:18:27.598 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:27.631 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:27.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:27.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:27.735 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:27.764 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:27.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:27.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:27.868 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:27.899 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:27.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:27.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:28.003 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:28.037 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:28.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:28.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:28.142 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:28.174 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:28.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:28.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:28.278 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:28.291 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:28 smithi133 bash[105589]: cluster 2023-10-09T05:18:26.404856+0000 osd.4 (osd.4) 172 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T05:18:28.291 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:28 smithi133 bash[105589]: cluster 2023-10-09T05:18:26.406849+0000 osd.4 (osd.4) 173 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T05:18:28.291 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:28 smithi133 bash[105589]: cluster 2023-10-09T05:18:26.411053+0000 osd.4 (osd.4) 174 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T05:18:28.291 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:28 smithi133 bash[105589]: cluster 2023-10-09T05:18:27.404878+0000 osd.5 (osd.5) 209 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T05:18:28.291 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:28 smithi133 bash[105589]: cluster 2023-10-09T05:18:27.405236+0000 mon.a (mon.0) 1181 : cluster 0 osdmap e222: 8 total, 8 up, 7 in 2023-10-09T05:18:28.291 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:28 smithi133 bash[105589]: cluster 2023-10-09T05:18:27.409453+0000 osd.6 (osd.6) 162 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T05:18:28.292 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:28 smithi133 bash[105589]: cluster 2023-10-09T05:18:27.775154+0000 osd.7 (osd.7) 155 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T05:18:28.292 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:28 smithi133 bash[102858]: cluster 2023-10-09T05:18:26.404856+0000 osd.4 (osd.4) 172 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T05:18:28.292 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:28 smithi133 bash[102858]: cluster 2023-10-09T05:18:26.406849+0000 osd.4 (osd.4) 173 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T05:18:28.292 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:28 smithi133 bash[102858]: cluster 2023-10-09T05:18:26.411053+0000 osd.4 (osd.4) 174 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T05:18:28.292 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:28 smithi133 bash[102858]: cluster 2023-10-09T05:18:27.404878+0000 osd.5 (osd.5) 209 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T05:18:28.292 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:28 smithi133 bash[102858]: cluster 2023-10-09T05:18:27.405236+0000 mon.a (mon.0) 1181 : cluster 0 osdmap e222: 8 total, 8 up, 7 in 2023-10-09T05:18:28.292 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:28 smithi133 bash[102858]: cluster 2023-10-09T05:18:27.409453+0000 osd.6 (osd.6) 162 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T05:18:28.292 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:28 smithi133 bash[102858]: cluster 2023-10-09T05:18:27.775154+0000 osd.7 (osd.7) 155 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T05:18:28.313 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:28.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:28.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:28.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:28 smithi194 bash[92666]: cluster 2023-10-09T05:18:26.404856+0000 osd.4 (osd.4) 172 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T05:18:28.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:28 smithi194 bash[92666]: cluster 2023-10-09T05:18:26.406849+0000 osd.4 (osd.4) 173 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T05:18:28.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:28 smithi194 bash[92666]: cluster 2023-10-09T05:18:26.411053+0000 osd.4 (osd.4) 174 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T05:18:28.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:28 smithi194 bash[92666]: cluster 2023-10-09T05:18:27.404878+0000 osd.5 (osd.5) 209 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T05:18:28.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:28 smithi194 bash[92666]: cluster 2023-10-09T05:18:27.405236+0000 mon.a (mon.0) 1181 : cluster 0 osdmap e222: 8 total, 8 up, 7 in 2023-10-09T05:18:28.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:28 smithi194 bash[92666]: cluster 2023-10-09T05:18:27.409453+0000 osd.6 (osd.6) 162 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T05:18:28.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:28 smithi194 bash[92666]: cluster 2023-10-09T05:18:27.775154+0000 osd.7 (osd.7) 155 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T05:18:28.417 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:28.445 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:28.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:28.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:28.549 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:28.581 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:28.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:28.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:28.685 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:28.716 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:28.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:28.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:28.820 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:28.857 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:28.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:28.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:28.960 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:28.995 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:28.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:28.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:29.098 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:29.109 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:29 smithi194 bash[92666]: cluster 2023-10-09T05:18:27.557454+0000 osd.4 (osd.4) 175 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T05:18:29.110 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:29 smithi194 bash[92666]: cluster 2023-10-09T05:18:27.560509+0000 osd.4 (osd.4) 176 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T05:18:29.110 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:29 smithi194 bash[92666]: cluster 2023-10-09T05:18:27.562838+0000 osd.4 (osd.4) 177 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T05:18:29.110 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:29 smithi194 bash[92666]: cluster 2023-10-09T05:18:27.970911+0000 mgr.y (mgr.25277) 900 : cluster 0 pgmap v930: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail 2023-10-09T05:18:29.110 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:29 smithi194 bash[92666]: cluster 2023-10-09T05:18:28.392596+0000 mon.a (mon.0) 1182 : cluster 0 osdmap e223: 8 total, 8 up, 7 in 2023-10-09T05:18:29.110 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:29 smithi194 bash[92666]: cluster 2023-10-09T05:18:28.395679+0000 osd.7 (osd.7) 156 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T05:18:29.110 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:29 smithi194 bash[92666]: cluster 2023-10-09T05:18:28.398428+0000 osd.7 (osd.7) 157 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T05:18:29.110 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:29 smithi194 bash[92666]: audit 2023-10-09T05:18:28.460706+0000 mon.a (mon.0) 1183 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:18:29.110 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:29 smithi194 bash[92666]: audit 2023-10-09T05:18:28.462276+0000 mon.a (mon.0) 1184 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:18:29.127 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:29.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:29.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:29.232 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:29.260 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:29.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:29.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:29.363 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:29.375 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:29 smithi133 bash[102858]: cluster 2023-10-09T05:18:27.557454+0000 osd.4 (osd.4) 175 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T05:18:29.376 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:29 smithi133 bash[102858]: cluster 2023-10-09T05:18:27.560509+0000 osd.4 (osd.4) 176 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T05:18:29.376 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:29 smithi133 bash[102858]: cluster 2023-10-09T05:18:27.562838+0000 osd.4 (osd.4) 177 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T05:18:29.376 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:29 smithi133 bash[102858]: cluster 2023-10-09T05:18:27.970911+0000 mgr.y (mgr.25277) 900 : cluster 0 pgmap v930: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail 2023-10-09T05:18:29.376 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:29 smithi133 bash[102858]: cluster 2023-10-09T05:18:28.392596+0000 mon.a (mon.0) 1182 : cluster 0 osdmap e223: 8 total, 8 up, 7 in 2023-10-09T05:18:29.376 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:29 smithi133 bash[102858]: cluster 2023-10-09T05:18:28.395679+0000 osd.7 (osd.7) 156 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T05:18:29.376 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:29 smithi133 bash[102858]: cluster 2023-10-09T05:18:28.398428+0000 osd.7 (osd.7) 157 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T05:18:29.376 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:29 smithi133 bash[102858]: audit 2023-10-09T05:18:28.460706+0000 mon.a (mon.0) 1183 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:18:29.376 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:29 smithi133 bash[102858]: audit 2023-10-09T05:18:28.462276+0000 mon.a (mon.0) 1184 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:18:29.376 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:29 smithi133 bash[105589]: cluster 2023-10-09T05:18:27.557454+0000 osd.4 (osd.4) 175 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T05:18:29.377 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:29 smithi133 bash[105589]: cluster 2023-10-09T05:18:27.560509+0000 osd.4 (osd.4) 176 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T05:18:29.377 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:29 smithi133 bash[105589]: cluster 2023-10-09T05:18:27.562838+0000 osd.4 (osd.4) 177 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T05:18:29.377 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:29 smithi133 bash[105589]: cluster 2023-10-09T05:18:27.970911+0000 mgr.y (mgr.25277) 900 : cluster 0 pgmap v930: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail 2023-10-09T05:18:29.377 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:29 smithi133 bash[105589]: cluster 2023-10-09T05:18:28.392596+0000 mon.a (mon.0) 1182 : cluster 0 osdmap e223: 8 total, 8 up, 7 in 2023-10-09T05:18:29.377 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:29 smithi133 bash[105589]: cluster 2023-10-09T05:18:28.395679+0000 osd.7 (osd.7) 156 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T05:18:29.377 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:29 smithi133 bash[105589]: cluster 2023-10-09T05:18:28.398428+0000 osd.7 (osd.7) 157 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T05:18:29.377 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:29 smithi133 bash[105589]: audit 2023-10-09T05:18:28.460706+0000 mon.a (mon.0) 1183 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:18:29.377 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:29 smithi133 bash[105589]: audit 2023-10-09T05:18:28.462276+0000 mon.a (mon.0) 1184 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:18:29.392 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:29.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:29.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:29.496 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:29.529 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:29.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:29.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:29.633 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:29.666 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:29.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:29.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:29.770 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:29.801 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:29.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:29.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:29.904 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:29.935 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:29.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:29.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:30.020 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:18:30.040 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:30.070 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:30.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:30.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:30.175 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:30.210 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:30.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:30.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:30.313 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:30.338 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:30.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:30.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:30.441 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:30.468 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:30.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:30.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:30.571 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:30.603 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:30.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:30.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:30.707 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:30.721 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:30 smithi133 bash[102858]: cluster 2023-10-09T05:18:29.335475+0000 osd.5 (osd.5) 210 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T05:18:30.721 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:30 smithi133 bash[102858]: cluster 2023-10-09T05:18:29.335742+0000 osd.4 (osd.4) 178 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T05:18:30.721 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:30 smithi133 bash[102858]: cluster 2023-10-09T05:18:29.404640+0000 osd.5 (osd.5) 211 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T05:18:30.722 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:30 smithi133 bash[102858]: cluster 2023-10-09T05:18:29.407083+0000 osd.7 (osd.7) 158 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T05:18:30.722 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:30 smithi133 bash[102858]: cluster 2023-10-09T05:18:29.408197+0000 mon.a (mon.0) 1185 : cluster 0 osdmap e224: 8 total, 8 up, 7 in 2023-10-09T05:18:30.722 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:30 smithi133 bash[102858]: cluster 2023-10-09T05:18:29.409885+0000 osd.7 (osd.7) 159 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T05:18:30.722 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:30 smithi133 bash[102858]: cluster 2023-10-09T05:18:29.971859+0000 mgr.y (mgr.25277) 901 : cluster 0 pgmap v933: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 255 B/s rd, 0 op/s; 145 KiB/s, 0 objects/s recovering 2023-10-09T05:18:30.722 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:30 smithi133 bash[102858]: cluster 2023-10-09T05:18:30.076432+0000 osd.4 (osd.4) 179 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T05:18:30.722 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:30 smithi133 bash[105589]: cluster 2023-10-09T05:18:29.335475+0000 osd.5 (osd.5) 210 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T05:18:30.722 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:30 smithi133 bash[105589]: cluster 2023-10-09T05:18:29.335742+0000 osd.4 (osd.4) 178 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T05:18:30.722 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:30 smithi133 bash[105589]: cluster 2023-10-09T05:18:29.404640+0000 osd.5 (osd.5) 211 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T05:18:30.722 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:30 smithi133 bash[105589]: cluster 2023-10-09T05:18:29.407083+0000 osd.7 (osd.7) 158 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T05:18:30.723 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:30 smithi133 bash[105589]: cluster 2023-10-09T05:18:29.408197+0000 mon.a (mon.0) 1185 : cluster 0 osdmap e224: 8 total, 8 up, 7 in 2023-10-09T05:18:30.723 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:30 smithi133 bash[105589]: cluster 2023-10-09T05:18:29.409885+0000 osd.7 (osd.7) 159 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T05:18:30.723 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:30 smithi133 bash[105589]: cluster 2023-10-09T05:18:29.971859+0000 mgr.y (mgr.25277) 901 : cluster 0 pgmap v933: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 255 B/s rd, 0 op/s; 145 KiB/s, 0 objects/s recovering 2023-10-09T05:18:30.723 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:30 smithi133 bash[105589]: cluster 2023-10-09T05:18:30.076432+0000 osd.4 (osd.4) 179 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T05:18:30.739 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:30.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:30.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:30.842 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:30.855 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:30 smithi194 bash[92666]: cluster 2023-10-09T05:18:29.335475+0000 osd.5 (osd.5) 210 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T05:18:30.855 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:30 smithi194 bash[92666]: cluster 2023-10-09T05:18:29.335742+0000 osd.4 (osd.4) 178 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T05:18:30.856 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:30 smithi194 bash[92666]: cluster 2023-10-09T05:18:29.404640+0000 osd.5 (osd.5) 211 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T05:18:30.856 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:30 smithi194 bash[92666]: cluster 2023-10-09T05:18:29.407083+0000 osd.7 (osd.7) 158 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T05:18:30.856 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:30 smithi194 bash[92666]: cluster 2023-10-09T05:18:29.408197+0000 mon.a (mon.0) 1185 : cluster 0 osdmap e224: 8 total, 8 up, 7 in 2023-10-09T05:18:30.856 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:30 smithi194 bash[92666]: cluster 2023-10-09T05:18:29.409885+0000 osd.7 (osd.7) 159 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T05:18:30.856 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:30 smithi194 bash[92666]: cluster 2023-10-09T05:18:29.971859+0000 mgr.y (mgr.25277) 901 : cluster 0 pgmap v933: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 255 B/s rd, 0 op/s; 145 KiB/s, 0 objects/s recovering 2023-10-09T05:18:30.856 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:30 smithi194 bash[92666]: cluster 2023-10-09T05:18:30.076432+0000 osd.4 (osd.4) 179 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T05:18:30.878 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:30.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:30.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:30.982 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:31.016 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:31.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:31.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:31.120 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:31.144 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:31.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:31.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:31.247 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:31.281 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:31.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:31.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:31.385 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:31.419 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:31.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:31.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:31.523 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:31.553 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:31.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:31.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:31.656 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:31.686 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:31.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:31.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:31.790 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:31.817 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:31.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:31.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:31.920 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:31.954 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:31.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:31.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:32.052 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:18:32.053 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:18:32.053 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:18:32.058 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:32.083 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:32.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:32.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:32.185 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:32.217 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:32.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:32.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:32.320 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:32.344 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:32.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:32.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:32.447 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:32.479 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:32.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:32.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:32.513 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 6, 1, 7] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:18:32.513 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:18:32.514 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-09T05:18:32.514 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T05:18:32.514 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T05:18:32.514 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-09T05:18:32.583 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:32.610 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:32.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:32.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:32.714 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:32.737 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:32.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:32.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:32.839 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:32.870 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:32.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:32.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:32.974 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:33.005 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:33.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:33.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:33.109 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:33.142 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:33.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:33.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:33.245 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:33.258 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:33 smithi194 bash[92666]: cluster 2023-10-09T05:18:31.972812+0000 mgr.y (mgr.25277) 902 : cluster 0 pgmap v934: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 183 B/s rd, 0 op/s; 104 KiB/s, 0 objects/s recovering 2023-10-09T05:18:33.280 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:33.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:33.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:33.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:33 smithi133 bash[102858]: cluster 2023-10-09T05:18:31.972812+0000 mgr.y (mgr.25277) 902 : cluster 0 pgmap v934: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 183 B/s rd, 0 op/s; 104 KiB/s, 0 objects/s recovering 2023-10-09T05:18:33.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:33 smithi133 bash[105589]: cluster 2023-10-09T05:18:31.972812+0000 mgr.y (mgr.25277) 902 : cluster 0 pgmap v934: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 183 B/s rd, 0 op/s; 104 KiB/s, 0 objects/s recovering 2023-10-09T05:18:33.384 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:33.419 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:33.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:33.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:33.522 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:33.553 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:33.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:33.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:33.656 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:33.689 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:33.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:33.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:33.792 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:33.823 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:33.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:33.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:33.926 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:33.959 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:33.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:33.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:34.063 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:34.089 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:34.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:34.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:34.193 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:34.226 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:34.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:34.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:34.330 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:34.364 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:34.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:34.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:34.397 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:18:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:18:34] "GET /metrics HTTP/1.1" 200 34074 "" "Prometheus/2.43.0" 2023-10-09T05:18:34.468 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:34.480 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:34 smithi194 bash[92666]: cluster 2023-10-09T05:18:33.973624+0000 mgr.y (mgr.25277) 903 : cluster 0 pgmap v935: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 155 B/s rd, 0 op/s; 88 KiB/s, 0 objects/s recovering 2023-10-09T05:18:34.500 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:34.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:34.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:34.605 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:34.638 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:34.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:34.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:34.741 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:34.749 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:34 smithi133 bash[105589]: cluster 2023-10-09T05:18:33.973624+0000 mgr.y (mgr.25277) 903 : cluster 0 pgmap v935: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 155 B/s rd, 0 op/s; 88 KiB/s, 0 objects/s recovering 2023-10-09T05:18:34.749 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:34 smithi133 bash[102858]: cluster 2023-10-09T05:18:33.973624+0000 mgr.y (mgr.25277) 903 : cluster 0 pgmap v935: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 155 B/s rd, 0 op/s; 88 KiB/s, 0 objects/s recovering 2023-10-09T05:18:34.766 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:34.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:34.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:34.870 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:34.901 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:34.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:34.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:35.006 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:35.029 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:35.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:35.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:35.134 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:35.164 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:35.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:35.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:35.266 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:35.293 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:35.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:35.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:35.397 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:35.428 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:35.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:35.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:35.531 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:35.567 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:35.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:35.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:35.671 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:35.905 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:35.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:35.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:36.009 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:36.037 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:36.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:36.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:36.140 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:36.160 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:36.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:36.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:36.170 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:18:36.174 INFO:teuthology.orchestra.run.smithi133.stdout: "ops": [], 2023-10-09T05:18:36.174 INFO:teuthology.orchestra.run.smithi133.stdout: "complaint_time": 30, 2023-10-09T05:18:36.174 INFO:teuthology.orchestra.run.smithi133.stdout: "num_blocked_ops": 0 2023-10-09T05:18:36.175 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:18:36.264 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:36.301 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:36.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:36.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:36.405 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:36.428 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:36.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:36.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:36.532 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:36.564 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:36.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:36.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:36.668 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:36.689 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:36.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:36.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:36.793 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:36.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:36.833 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:36.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:36.934 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:36.966 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:36.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:36.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:37.070 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:37.099 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:37.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:37.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:37.203 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:37.236 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:37.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:37.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:37.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:37 smithi133 bash[105589]: cluster 2023-10-09T05:18:35.974477+0000 mgr.y (mgr.25277) 904 : cluster 0 pgmap v936: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 127 B/s rd, 0 op/s; 73 KiB/s, 0 objects/s recovering 2023-10-09T05:18:37.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:37 smithi133 bash[102858]: cluster 2023-10-09T05:18:35.974477+0000 mgr.y (mgr.25277) 904 : cluster 0 pgmap v936: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 127 B/s rd, 0 op/s; 73 KiB/s, 0 objects/s recovering 2023-10-09T05:18:37.301 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:18:37.330 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:37 smithi194 bash[92666]: cluster 2023-10-09T05:18:35.974477+0000 mgr.y (mgr.25277) 904 : cluster 0 pgmap v936: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 127 B/s rd, 0 op/s; 73 KiB/s, 0 objects/s recovering 2023-10-09T05:18:37.341 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:37.372 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:37.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:37.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:37.476 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:37.506 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:37.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:37.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:37.610 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:37.636 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:37.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:37.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:37.740 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:37.773 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:37.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:37.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:37.877 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:37.911 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:37.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:37.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:38.016 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:38.045 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:38.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:38.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:38.149 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:38.179 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:38.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:38.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:38.283 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:38.309 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:38.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:38.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:38.413 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:38.441 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:38.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:38.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:38.544 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:38.574 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:38.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:38.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:38.678 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:38.705 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:38.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:38.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:38.809 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:38.836 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:38.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:38.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:38.939 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:38.966 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:38.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:38.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:39.070 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:39.082 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:39 smithi133 bash[105589]: cluster 2023-10-09T05:18:37.974962+0000 mgr.y (mgr.25277) 905 : cluster 0 pgmap v937: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 106 B/s rd, 0 op/s; 61 KiB/s, 0 objects/s recovering 2023-10-09T05:18:39.082 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:39 smithi133 bash[105589]: audit 2023-10-09T05:18:38.953075+0000 mon.a (mon.0) 1186 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:18:39.083 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:39 smithi133 bash[102858]: cluster 2023-10-09T05:18:37.974962+0000 mgr.y (mgr.25277) 905 : cluster 0 pgmap v937: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 106 B/s rd, 0 op/s; 61 KiB/s, 0 objects/s recovering 2023-10-09T05:18:39.083 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:39 smithi133 bash[102858]: audit 2023-10-09T05:18:38.953075+0000 mon.a (mon.0) 1186 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:18:39.099 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:39.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:39.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:39.202 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:39.222 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:39.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:39.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:39.325 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:39.338 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:39 smithi194 bash[92666]: cluster 2023-10-09T05:18:37.974962+0000 mgr.y (mgr.25277) 905 : cluster 0 pgmap v937: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 106 B/s rd, 0 op/s; 61 KiB/s, 0 objects/s recovering 2023-10-09T05:18:39.339 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:39 smithi194 bash[92666]: audit 2023-10-09T05:18:38.953075+0000 mon.a (mon.0) 1186 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:18:39.359 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:39.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:39.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:39.462 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:39.483 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:39.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:39.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:39.587 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:39.612 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:39.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:39.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:39.715 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:39.744 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:39.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:39.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:39.848 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:39.878 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:39.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:39.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:39.982 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:40.008 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:40.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:40.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:40.111 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:40.136 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:40.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:40.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:40.239 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:40.268 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:40.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:40.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:40.372 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:40.399 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:40.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:40.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:40.502 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:40.527 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:40.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:40.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:40.631 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:40.654 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:40.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:40.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:40.757 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:40.777 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T05:18:40.789 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:40.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:40.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:40.892 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:40.918 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:40.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:40.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:41.022 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:41.053 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:41.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:41.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:41.157 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:41.182 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:41.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:41.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:41.286 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:41.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:41 smithi133 bash[102858]: cluster 2023-10-09T05:18:39.975688+0000 mgr.y (mgr.25277) 906 : cluster 0 pgmap v938: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 96 B/s rd, 0 op/s; 55 KiB/s, 0 objects/s recovering 2023-10-09T05:18:41.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:41 smithi133 bash[105589]: cluster 2023-10-09T05:18:39.975688+0000 mgr.y (mgr.25277) 906 : cluster 0 pgmap v938: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 96 B/s rd, 0 op/s; 55 KiB/s, 0 objects/s recovering 2023-10-09T05:18:41.299 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:41 smithi194 bash[92666]: cluster 2023-10-09T05:18:39.975688+0000 mgr.y (mgr.25277) 906 : cluster 0 pgmap v938: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 96 B/s rd, 0 op/s; 55 KiB/s, 0 objects/s recovering 2023-10-09T05:18:41.313 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:41.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:41.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:41.417 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:41.442 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:41.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:41.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:41.545 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:41.570 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:41.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:41.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:41.673 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:41.697 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:41.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:41.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:41.800 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:41.814 INFO:teuthology.orchestra.run.smithi194.stdout:{ 2023-10-09T05:18:41.815 INFO:teuthology.orchestra.run.smithi194.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T05:18:41.815 INFO:teuthology.orchestra.run.smithi194.stdout:} 2023-10-09T05:18:41.829 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:41.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:41.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:41.932 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:41.961 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:41.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:41.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:42.065 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:42.078 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:42 smithi133 bash[105589]: audit 2023-10-09T05:18:41.365399+0000 mon.a (mon.0) 1187 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:18:42.078 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:42 smithi133 bash[102858]: audit 2023-10-09T05:18:41.365399+0000 mon.a (mon.0) 1187 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:18:42.093 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:42.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:42.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:42.197 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:42.221 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:42.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:42.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:42.325 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:42.347 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:42.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:42.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:42.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:42 smithi194 bash[92666]: audit 2023-10-09T05:18:41.365399+0000 mon.a (mon.0) 1187 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:18:42.451 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:42.482 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:42.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:42.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:42.586 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:42.618 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:42.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:42.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:42.721 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:42.755 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:42.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:42.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:42.789 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:18:42.814 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:18:42.837 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:18:42.860 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:42.876 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:18:42.889 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:42.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:42.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:42.906 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:18:42.932 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:18:42.961 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:18:42.988 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:18:42.990 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:18:42.990 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:18:42.990 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:18:42.990 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:18:42.990 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:18:42.990 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:18:42.990 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:18:42.991 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:18:42.993 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:43.019 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:43.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:43.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:43.122 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:43.152 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:43.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:43.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:43.257 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:43.292 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:43.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:43.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:43.396 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:43.423 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:43.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:43.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:43.509 INFO:teuthology.orchestra.run.smithi133.stderr:pool 'unique_pool_6' created 2023-10-09T05:18:43.510 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is unset 2023-10-09T05:18:43.527 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:43.549 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:43.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:43.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:43.652 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:43.687 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:43.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:43.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:43.790 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:43.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:43 smithi133 bash[102858]: cluster 2023-10-09T05:18:41.976713+0000 mgr.y (mgr.25277) 907 : cluster 0 pgmap v939: 73 pgs: 73 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:18:43.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:43 smithi133 bash[102858]: audit 2023-10-09T05:18:42.759567+0000 mon.a (mon.0) 1188 : audit 0 from='client.? 172.21.15.133:0/2110966142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:18:43.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:43 smithi133 bash[102858]: audit 2023-10-09T05:18:42.839225+0000 mon.c (mon.2) 89 : audit 1 from='client.? 172.21.15.133:0/2944835738' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:18:43.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:43 smithi133 bash[102858]: audit 2023-10-09T05:18:42.839825+0000 mon.a (mon.0) 1189 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:18:43.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:43 smithi133 bash[105589]: cluster 2023-10-09T05:18:41.976713+0000 mgr.y (mgr.25277) 907 : cluster 0 pgmap v939: 73 pgs: 73 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:18:43.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:43 smithi133 bash[105589]: audit 2023-10-09T05:18:42.759567+0000 mon.a (mon.0) 1188 : audit 0 from='client.? 172.21.15.133:0/2110966142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:18:43.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:43 smithi133 bash[105589]: audit 2023-10-09T05:18:42.839225+0000 mon.c (mon.2) 89 : audit 1 from='client.? 172.21.15.133:0/2944835738' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:18:43.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:43 smithi133 bash[105589]: audit 2023-10-09T05:18:42.839825+0000 mon.a (mon.0) 1189 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:18:43.803 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:43 smithi194 bash[92666]: cluster 2023-10-09T05:18:41.976713+0000 mgr.y (mgr.25277) 907 : cluster 0 pgmap v939: 73 pgs: 73 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:18:43.803 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:43 smithi194 bash[92666]: audit 2023-10-09T05:18:42.759567+0000 mon.a (mon.0) 1188 : audit 0 from='client.? 172.21.15.133:0/2110966142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:18:43.803 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:43 smithi194 bash[92666]: audit 2023-10-09T05:18:42.839225+0000 mon.c (mon.2) 89 : audit 1 from='client.? 172.21.15.133:0/2944835738' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:18:43.803 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:43 smithi194 bash[92666]: audit 2023-10-09T05:18:42.839825+0000 mon.a (mon.0) 1189 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:18:43.823 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:43.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:43.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:43.926 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:43.951 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:43.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:43.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:44.054 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:44.086 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:44.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:44.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:44.191 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:44.220 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:44.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:44.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:44.324 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:44.354 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:44.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:44.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:44.459 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:44.472 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:18:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:18:44] "GET /metrics HTTP/1.1" 200 35400 "" "Prometheus/2.43.0" 2023-10-09T05:18:44.495 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:44.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:44.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:44.600 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:44.614 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:44 smithi194 bash[92666]: audit 2023-10-09T05:18:43.037535+0000 mon.a (mon.0) 1190 : audit 1 from='client.? 172.21.15.133:0/511829260' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-10-09T05:18:44.614 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:44 smithi194 bash[92666]: cluster 2023-10-09T05:18:43.483080+0000 mon.a (mon.0) 1191 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:18:44.614 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:44 smithi194 bash[92666]: audit 2023-10-09T05:18:43.484054+0000 mon.a (mon.0) 1192 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:18:44.615 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:44 smithi194 bash[92666]: cluster 2023-10-09T05:18:43.494047+0000 osd.7 (osd.7) 160 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T05:18:44.615 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:44 smithi194 bash[92666]: cluster 2023-10-09T05:18:43.494289+0000 osd.5 (osd.5) 212 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T05:18:44.615 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:44 smithi194 bash[92666]: audit 2023-10-09T05:18:43.496598+0000 mon.a (mon.0) 1193 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:18:44.615 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:44 smithi194 bash[92666]: audit 2023-10-09T05:18:43.496822+0000 mon.a (mon.0) 1194 : audit 1 from='client.? 172.21.15.133:0/511829260' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2023-10-09T05:18:44.615 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:44 smithi194 bash[92666]: cluster 2023-10-09T05:18:43.496922+0000 mon.a (mon.0) 1195 : cluster 0 osdmap e225: 8 total, 8 up, 7 in 2023-10-09T05:18:44.615 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:44 smithi194 bash[92666]: cluster 2023-10-09T05:18:43.504198+0000 osd.4 (osd.4) 180 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T05:18:44.615 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:44 smithi194 bash[92666]: cluster 2023-10-09T05:18:43.977368+0000 mgr.y (mgr.25277) 908 : cluster 0 pgmap v941: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:18:44.631 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:44.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:44.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:44.734 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:44.747 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:44 smithi133 bash[105589]: audit 2023-10-09T05:18:43.037535+0000 mon.a (mon.0) 1190 : audit 1 from='client.? 172.21.15.133:0/511829260' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-10-09T05:18:44.747 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:44 smithi133 bash[105589]: cluster 2023-10-09T05:18:43.483080+0000 mon.a (mon.0) 1191 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:18:44.748 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:44 smithi133 bash[105589]: audit 2023-10-09T05:18:43.484054+0000 mon.a (mon.0) 1192 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:18:44.748 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:44 smithi133 bash[105589]: cluster 2023-10-09T05:18:43.494047+0000 osd.7 (osd.7) 160 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T05:18:44.748 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:44 smithi133 bash[105589]: cluster 2023-10-09T05:18:43.494289+0000 osd.5 (osd.5) 212 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T05:18:44.748 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:44 smithi133 bash[105589]: audit 2023-10-09T05:18:43.496598+0000 mon.a (mon.0) 1193 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:18:44.748 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:44 smithi133 bash[105589]: audit 2023-10-09T05:18:43.496822+0000 mon.a (mon.0) 1194 : audit 1 from='client.? 172.21.15.133:0/511829260' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2023-10-09T05:18:44.748 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:44 smithi133 bash[105589]: cluster 2023-10-09T05:18:43.496922+0000 mon.a (mon.0) 1195 : cluster 0 osdmap e225: 8 total, 8 up, 7 in 2023-10-09T05:18:44.749 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:44 smithi133 bash[105589]: cluster 2023-10-09T05:18:43.504198+0000 osd.4 (osd.4) 180 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T05:18:44.749 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:44 smithi133 bash[105589]: cluster 2023-10-09T05:18:43.977368+0000 mgr.y (mgr.25277) 908 : cluster 0 pgmap v941: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:18:44.749 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:44 smithi133 bash[102858]: audit 2023-10-09T05:18:43.037535+0000 mon.a (mon.0) 1190 : audit 1 from='client.? 172.21.15.133:0/511829260' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-10-09T05:18:44.749 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:44 smithi133 bash[102858]: cluster 2023-10-09T05:18:43.483080+0000 mon.a (mon.0) 1191 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:18:44.749 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:44 smithi133 bash[102858]: audit 2023-10-09T05:18:43.484054+0000 mon.a (mon.0) 1192 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:18:44.750 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:44 smithi133 bash[102858]: cluster 2023-10-09T05:18:43.494047+0000 osd.7 (osd.7) 160 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T05:18:44.750 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:44 smithi133 bash[102858]: cluster 2023-10-09T05:18:43.494289+0000 osd.5 (osd.5) 212 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T05:18:44.750 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:44 smithi133 bash[102858]: audit 2023-10-09T05:18:43.496598+0000 mon.a (mon.0) 1193 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:18:44.750 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:44 smithi133 bash[102858]: audit 2023-10-09T05:18:43.496822+0000 mon.a (mon.0) 1194 : audit 1 from='client.? 172.21.15.133:0/511829260' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2023-10-09T05:18:44.750 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:44 smithi133 bash[102858]: cluster 2023-10-09T05:18:43.496922+0000 mon.a (mon.0) 1195 : cluster 0 osdmap e225: 8 total, 8 up, 7 in 2023-10-09T05:18:44.750 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:44 smithi133 bash[102858]: cluster 2023-10-09T05:18:43.504198+0000 osd.4 (osd.4) 180 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T05:18:44.750 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:44 smithi133 bash[102858]: cluster 2023-10-09T05:18:43.977368+0000 mgr.y (mgr.25277) 908 : cluster 0 pgmap v941: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:18:44.767 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:44.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:44.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:44.871 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:44.896 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:44.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:44.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:45.000 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:45.031 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:45.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:45.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:45.082 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_6 rados --yes-i-really-mean-it || true 2023-10-09T05:18:45.135 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:45.167 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:45.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:45.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:45.270 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:45.302 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:45.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:45.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:45.406 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:45.441 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:45.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:45.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:45.546 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:45.577 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:45.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:45.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:45.684 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:45.697 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:45 smithi194 bash[92666]: cluster 2023-10-09T05:18:44.506708+0000 osd.4 (osd.4) 181 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T05:18:45.698 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:45 smithi194 bash[92666]: cluster 2023-10-09T05:18:44.509918+0000 mon.a (mon.0) 1196 : cluster 0 osdmap e226: 8 total, 8 up, 7 in 2023-10-09T05:18:45.698 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:45 smithi194 bash[92666]: cluster 2023-10-09T05:18:44.510904+0000 osd.4 (osd.4) 182 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T05:18:45.698 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:45 smithi194 bash[92666]: cluster 2023-10-09T05:18:44.510905+0000 osd.7 (osd.7) 161 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T05:18:45.698 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:45 smithi194 bash[92666]: cluster 2023-10-09T05:18:44.510934+0000 osd.6 (osd.6) 163 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T05:18:45.698 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:45 smithi194 bash[92666]: cluster 2023-10-09T05:18:44.748131+0000 osd.5 (osd.5) 213 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T05:18:45.716 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:45.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:45.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:45.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:45 smithi133 bash[105589]: cluster 2023-10-09T05:18:44.506708+0000 osd.4 (osd.4) 181 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T05:18:45.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:45 smithi133 bash[105589]: cluster 2023-10-09T05:18:44.509918+0000 mon.a (mon.0) 1196 : cluster 0 osdmap e226: 8 total, 8 up, 7 in 2023-10-09T05:18:45.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:45 smithi133 bash[105589]: cluster 2023-10-09T05:18:44.510904+0000 osd.4 (osd.4) 182 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T05:18:45.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:45 smithi133 bash[105589]: cluster 2023-10-09T05:18:44.510905+0000 osd.7 (osd.7) 161 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T05:18:45.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:45 smithi133 bash[105589]: cluster 2023-10-09T05:18:44.510934+0000 osd.6 (osd.6) 163 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T05:18:45.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:45 smithi133 bash[105589]: cluster 2023-10-09T05:18:44.748131+0000 osd.5 (osd.5) 213 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T05:18:45.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:45 smithi133 bash[102858]: cluster 2023-10-09T05:18:44.506708+0000 osd.4 (osd.4) 181 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T05:18:45.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:45 smithi133 bash[102858]: cluster 2023-10-09T05:18:44.509918+0000 mon.a (mon.0) 1196 : cluster 0 osdmap e226: 8 total, 8 up, 7 in 2023-10-09T05:18:45.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:45 smithi133 bash[102858]: cluster 2023-10-09T05:18:44.510904+0000 osd.4 (osd.4) 182 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T05:18:45.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:45 smithi133 bash[102858]: cluster 2023-10-09T05:18:44.510905+0000 osd.7 (osd.7) 161 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T05:18:45.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:45 smithi133 bash[102858]: cluster 2023-10-09T05:18:44.510934+0000 osd.6 (osd.6) 163 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T05:18:45.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:45 smithi133 bash[102858]: cluster 2023-10-09T05:18:44.748131+0000 osd.5 (osd.5) 213 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T05:18:45.820 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:45.852 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:45.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:45.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:45.956 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:45.990 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:45.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:45.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:46.092 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:46.125 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:46.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:46.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:46.153 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:18:46.229 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:46.260 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:46.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:46.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:46.364 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:46.410 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:46.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:46.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:46.513 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:46.537 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:46.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:46.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:46.641 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:46.670 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:46.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:46.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:46.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:46 smithi133 bash[102858]: cluster 2023-10-09T05:18:45.506765+0000 osd.6 (osd.6) 164 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T05:18:46.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:46 smithi133 bash[102858]: cluster 2023-10-09T05:18:45.507115+0000 osd.5 (osd.5) 214 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T05:18:46.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:46 smithi133 bash[102858]: cluster 2023-10-09T05:18:45.508967+0000 mon.a (mon.0) 1197 : cluster 0 osdmap e227: 8 total, 8 up, 7 in 2023-10-09T05:18:46.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:46 smithi133 bash[102858]: cluster 2023-10-09T05:18:45.961311+0000 osd.4 (osd.4) 183 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T05:18:46.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:46 smithi133 bash[102858]: cluster 2023-10-09T05:18:45.977873+0000 mgr.y (mgr.25277) 909 : cluster 0 pgmap v944: 89 pgs: 5 unknown, 84 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:18:46.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:46 smithi133 bash[105589]: cluster 2023-10-09T05:18:45.506765+0000 osd.6 (osd.6) 164 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T05:18:46.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:46 smithi133 bash[105589]: cluster 2023-10-09T05:18:45.507115+0000 osd.5 (osd.5) 214 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T05:18:46.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:46 smithi133 bash[105589]: cluster 2023-10-09T05:18:45.508967+0000 mon.a (mon.0) 1197 : cluster 0 osdmap e227: 8 total, 8 up, 7 in 2023-10-09T05:18:46.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:46 smithi133 bash[105589]: cluster 2023-10-09T05:18:45.961311+0000 osd.4 (osd.4) 183 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T05:18:46.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:46 smithi133 bash[105589]: cluster 2023-10-09T05:18:45.977873+0000 mgr.y (mgr.25277) 909 : cluster 0 pgmap v944: 89 pgs: 5 unknown, 84 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:18:46.807 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:46.840 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:46.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:46.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:46.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:46 smithi194 bash[92666]: cluster 2023-10-09T05:18:45.506765+0000 osd.6 (osd.6) 164 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T05:18:46.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:46 smithi194 bash[92666]: cluster 2023-10-09T05:18:45.507115+0000 osd.5 (osd.5) 214 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T05:18:46.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:46 smithi194 bash[92666]: cluster 2023-10-09T05:18:45.508967+0000 mon.a (mon.0) 1197 : cluster 0 osdmap e227: 8 total, 8 up, 7 in 2023-10-09T05:18:46.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:46 smithi194 bash[92666]: cluster 2023-10-09T05:18:45.961311+0000 osd.4 (osd.4) 183 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T05:18:46.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:46 smithi194 bash[92666]: cluster 2023-10-09T05:18:45.977873+0000 mgr.y (mgr.25277) 909 : cluster 0 pgmap v944: 89 pgs: 5 unknown, 84 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:18:46.943 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:46.973 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:46.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:46.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:47.077 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:47.104 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:18:47.134 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:47.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:47.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:47.238 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:47.269 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:47.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:47.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:47.374 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:47.404 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:47.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:47.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:47.507 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:47.531 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:47.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:47.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:47.609 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:18:47.634 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:47.657 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:47.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:47.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:47.710 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 6, 1, 7] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:18:47.710 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:18:47.710 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-09T05:18:47.710 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 1.0 2023-10-09T05:18:47.760 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:47.804 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:47.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:47.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:47.849 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:47 smithi133 bash[102858]: cluster 2023-10-09T05:18:46.238116+0000 osd.7 (osd.7) 162 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T05:18:47.849 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:47 smithi133 bash[105589]: cluster 2023-10-09T05:18:46.238116+0000 osd.7 (osd.7) 162 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T05:18:47.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:47 smithi194 bash[92666]: cluster 2023-10-09T05:18:46.238116+0000 osd.7 (osd.7) 162 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T05:18:47.908 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:47.934 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:47.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:47.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:48.037 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:48.068 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:48.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:48.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:48.171 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:48.238 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:48.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:48.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:48.341 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:48.384 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:48.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:48.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:48.487 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:48.520 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:48.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:48.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:48.624 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:48.646 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:48.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:48.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:48.749 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:48.782 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:48.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:48.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:48.885 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:48.919 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:48.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:48.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:49.023 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:49.034 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:48 smithi133 bash[102858]: audit 2023-10-09T05:18:47.807509+0000 mon.a (mon.0) 1198 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:18:49.035 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:48 smithi133 bash[102858]: audit 2023-10-09T05:18:47.814279+0000 mon.a (mon.0) 1199 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:18:49.035 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:48 smithi133 bash[102858]: cluster 2023-10-09T05:18:47.978590+0000 mgr.y (mgr.25277) 910 : cluster 0 pgmap v945: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:18:49.035 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:48 smithi133 bash[102858]: audit 2023-10-09T05:18:48.768910+0000 mon.a (mon.0) 1200 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:18:49.035 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:48 smithi133 bash[102858]: audit 2023-10-09T05:18:48.775150+0000 mon.a (mon.0) 1201 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:18:49.035 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:48 smithi133 bash[105589]: audit 2023-10-09T05:18:47.807509+0000 mon.a (mon.0) 1198 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:18:49.035 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:48 smithi133 bash[105589]: audit 2023-10-09T05:18:47.814279+0000 mon.a (mon.0) 1199 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:18:49.035 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:48 smithi133 bash[105589]: cluster 2023-10-09T05:18:47.978590+0000 mgr.y (mgr.25277) 910 : cluster 0 pgmap v945: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:18:49.035 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:48 smithi133 bash[105589]: audit 2023-10-09T05:18:48.768910+0000 mon.a (mon.0) 1200 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:18:49.035 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:48 smithi133 bash[105589]: audit 2023-10-09T05:18:48.775150+0000 mon.a (mon.0) 1201 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:18:49.052 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:49.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:49.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:49.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:48 smithi194 bash[92666]: audit 2023-10-09T05:18:47.807509+0000 mon.a (mon.0) 1198 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:18:49.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:48 smithi194 bash[92666]: audit 2023-10-09T05:18:47.814279+0000 mon.a (mon.0) 1199 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:18:49.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:48 smithi194 bash[92666]: cluster 2023-10-09T05:18:47.978590+0000 mgr.y (mgr.25277) 910 : cluster 0 pgmap v945: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:18:49.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:48 smithi194 bash[92666]: audit 2023-10-09T05:18:48.768910+0000 mon.a (mon.0) 1200 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:18:49.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:48 smithi194 bash[92666]: audit 2023-10-09T05:18:48.775150+0000 mon.a (mon.0) 1201 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:18:49.156 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:49.187 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:49.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:49.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:49.292 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:49.325 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:49.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:49.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:49.429 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:49.460 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:49.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:49.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:49.564 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:49.592 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:49.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:49.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:49.635 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:18:49.695 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:49.729 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:49.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:49.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:49.833 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:49.864 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:49.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:49.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:49.968 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:49.998 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:50.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:50.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:50.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:49 smithi133 bash[102858]: audit 2023-10-09T05:18:49.247303+0000 mon.a (mon.0) 1202 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:50.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:49 smithi133 bash[102858]: audit 2023-10-09T05:18:49.248895+0000 mon.a (mon.0) 1203 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:18:50.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:49 smithi133 bash[102858]: audit 2023-10-09T05:18:49.256757+0000 mon.a (mon.0) 1204 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:18:50.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:49 smithi133 bash[105589]: audit 2023-10-09T05:18:49.247303+0000 mon.a (mon.0) 1202 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:50.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:49 smithi133 bash[105589]: audit 2023-10-09T05:18:49.248895+0000 mon.a (mon.0) 1203 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:18:50.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:49 smithi133 bash[105589]: audit 2023-10-09T05:18:49.256757+0000 mon.a (mon.0) 1204 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:18:50.102 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:50.127 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:50.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:50.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:50.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:49 smithi194 bash[92666]: audit 2023-10-09T05:18:49.247303+0000 mon.a (mon.0) 1202 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:50.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:49 smithi194 bash[92666]: audit 2023-10-09T05:18:49.248895+0000 mon.a (mon.0) 1203 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:18:50.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:49 smithi194 bash[92666]: audit 2023-10-09T05:18:49.256757+0000 mon.a (mon.0) 1204 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:18:50.231 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:50.263 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:50.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:50.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:50.366 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:50.396 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:50.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:50.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:50.499 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:50.523 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:50.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:50.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:50.626 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:50.661 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:50.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:50.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:50.765 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:50.801 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:50.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:50.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:50.905 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:50.933 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:50.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:50.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:51.036 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:51.061 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:51.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:51.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:51.093 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:50 smithi133 bash[102858]: cluster 2023-10-09T05:18:49.979512+0000 mgr.y (mgr.25277) 911 : cluster 0 pgmap v946: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:18:51.094 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:50 smithi133 bash[105589]: cluster 2023-10-09T05:18:49.979512+0000 mgr.y (mgr.25277) 911 : cluster 0 pgmap v946: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:18:51.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:50 smithi194 bash[92666]: cluster 2023-10-09T05:18:49.979512+0000 mgr.y (mgr.25277) 911 : cluster 0 pgmap v946: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:18:51.165 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:51.187 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:51.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:51.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:51.291 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:51.317 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:51.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:51.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:51.419 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:51.445 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:51.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:51.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:51.504 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:18:51.548 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:51.572 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:51.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:51.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:51.667 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:18:51.668 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:18:51.675 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:51.708 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:51.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:51.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:51.812 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:51.843 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:51.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:51.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:51.946 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:51.976 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:51.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:51.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:52.080 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:52.110 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:52.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:52.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:52.214 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:52.243 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:52.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:52.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:52.347 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:52.380 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:52.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:52.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:52.484 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:52.507 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:52.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:52.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:52.611 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:52.644 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:52.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:52.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:52.748 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:52.778 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:52.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:52.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:52.881 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:52.902 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:52.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:52.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:53.005 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:53.039 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:53.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:53.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:53.142 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:53.173 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:53.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:53.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:53.277 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:53.289 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:53 smithi133 bash[105589]: cluster 2023-10-09T05:18:51.980330+0000 mgr.y (mgr.25277) 912 : cluster 0 pgmap v947: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:18:53.289 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:53 smithi133 bash[102858]: cluster 2023-10-09T05:18:51.980330+0000 mgr.y (mgr.25277) 912 : cluster 0 pgmap v947: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:18:53.305 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:53.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:53.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:53.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:53 smithi194 bash[92666]: cluster 2023-10-09T05:18:51.980330+0000 mgr.y (mgr.25277) 912 : cluster 0 pgmap v947: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:18:53.409 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:53.434 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:53.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:53.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:53.538 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:53.568 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:53.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:53.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:53.673 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:53.708 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:53.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:53.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:53.813 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:53.849 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:53.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:53.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:53.952 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:53.979 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:53.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:53.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:54.083 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:54.114 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:54.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:54.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:54.218 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:54.249 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:54.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:54.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:54.353 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:54.386 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:54.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:54.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:54.489 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:54.516 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:54.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:54.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:54.556 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:18:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:18:54] "GET /metrics HTTP/1.1" 200 35400 "" "Prometheus/2.43.0" 2023-10-09T05:18:54.620 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:54.651 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:54.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:54.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:54.755 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:54.781 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:54.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:54.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:54.884 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:54.917 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:54.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:54.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:55.020 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:55.046 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:18:55.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:55.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:18:55.150 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:55.161 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:54 smithi194 bash[92666]: audit 2023-10-09T05:18:53.965799+0000 mon.a (mon.0) 1205 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:18:55.162 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:54 smithi194 bash[92666]: cluster 2023-10-09T05:18:53.980843+0000 mgr.y (mgr.25277) 913 : cluster 0 pgmap v948: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:18:55.180 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:55.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:55.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:55.284 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:55.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:54 smithi133 bash[102858]: audit 2023-10-09T05:18:53.965799+0000 mon.a (mon.0) 1205 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:18:55.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:54 smithi133 bash[102858]: cluster 2023-10-09T05:18:53.980843+0000 mgr.y (mgr.25277) 913 : cluster 0 pgmap v948: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:18:55.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:54 smithi133 bash[105589]: audit 2023-10-09T05:18:53.965799+0000 mon.a (mon.0) 1205 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:18:55.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:54 smithi133 bash[105589]: cluster 2023-10-09T05:18:53.980843+0000 mgr.y (mgr.25277) 913 : cluster 0 pgmap v948: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:18:55.320 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:55.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:55.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:55.425 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:55.454 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:55.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:55.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:55.559 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:55.582 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:55.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:55.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:55.685 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:55.711 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:55.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:55.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:55.815 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:55.846 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:55.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:55.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:55.950 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:55.975 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:55.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:55.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:56.078 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:56.110 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:56.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:56.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:56.214 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:56.236 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:56.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:56.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:56.339 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:56.374 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:56.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:56.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:56.478 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:56.508 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:56.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:56.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:56.612 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:56.645 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:56.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:56.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:56.749 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:56.780 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:56.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:56.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:56.883 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:56.912 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:56.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:56.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:57.016 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:57.053 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:57.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:57.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:57.157 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:57.168 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:57 smithi133 bash[102858]: cluster 2023-10-09T05:18:55.981553+0000 mgr.y (mgr.25277) 914 : cluster 0 pgmap v949: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:18:57.169 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:57 smithi133 bash[105589]: cluster 2023-10-09T05:18:55.981553+0000 mgr.y (mgr.25277) 914 : cluster 0 pgmap v949: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:18:57.186 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:57.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:57.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:57.289 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:57.317 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:57.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:57.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:57.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:57 smithi194 bash[92666]: cluster 2023-10-09T05:18:55.981553+0000 mgr.y (mgr.25277) 914 : cluster 0 pgmap v949: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:18:57.420 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:57.444 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:18:57.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:57.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:18:57.494 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:18:57.494 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:18:57.548 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:57.580 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:57.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:57.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:57.684 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:57.712 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:57.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:57.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:57.816 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:57.850 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:57.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:57.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:57.954 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:57.983 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:18:57.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:57.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:18:58.089 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:58.120 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:58.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:58.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:58.225 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:58.251 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:58.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:58.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:58.354 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:58.390 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:58.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:58.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:58.494 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:58.512 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:58.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:58.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:58.615 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:58.648 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:58.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:58.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:58.752 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:58.787 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:18:58.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:58.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:18:58.890 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:58.919 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:58.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:58.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:59.022 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:59.053 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:59.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:59.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:59.158 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:59.193 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:59.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:59.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:59.297 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:59.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:59 smithi133 bash[102858]: cluster 2023-10-09T05:18:57.982309+0000 mgr.y (mgr.25277) 915 : cluster 0 pgmap v950: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:18:59.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:59 smithi133 bash[102858]: audit 2023-10-09T05:18:58.453245+0000 mon.a (mon.0) 1206 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:18:59.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:59 smithi133 bash[102858]: audit 2023-10-09T05:18:58.649765+0000 mon.c (mon.2) 90 : audit 1 from='client.? 172.21.15.133:0/3626502417' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:18:59.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:18:59 smithi133 bash[102858]: audit 2023-10-09T05:18:58.650150+0000 mon.a (mon.0) 1207 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:18:59.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:59 smithi133 bash[105589]: cluster 2023-10-09T05:18:57.982309+0000 mgr.y (mgr.25277) 915 : cluster 0 pgmap v950: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:18:59.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:59 smithi133 bash[105589]: audit 2023-10-09T05:18:58.453245+0000 mon.a (mon.0) 1206 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:18:59.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:59 smithi133 bash[105589]: audit 2023-10-09T05:18:58.649765+0000 mon.c (mon.2) 90 : audit 1 from='client.? 172.21.15.133:0/3626502417' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:18:59.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:18:59 smithi133 bash[105589]: audit 2023-10-09T05:18:58.650150+0000 mon.a (mon.0) 1207 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:18:59.311 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:59 smithi194 bash[92666]: cluster 2023-10-09T05:18:57.982309+0000 mgr.y (mgr.25277) 915 : cluster 0 pgmap v950: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:18:59.312 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:59 smithi194 bash[92666]: audit 2023-10-09T05:18:58.453245+0000 mon.a (mon.0) 1206 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:18:59.312 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:59 smithi194 bash[92666]: audit 2023-10-09T05:18:58.649765+0000 mon.c (mon.2) 90 : audit 1 from='client.? 172.21.15.133:0/3626502417' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:18:59.312 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:18:59 smithi194 bash[92666]: audit 2023-10-09T05:18:58.650150+0000 mon.a (mon.0) 1207 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:18:59.334 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:18:59.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:59.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:18:59.438 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:59.469 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:18:59.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:59.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:18:59.504 INFO:teuthology.orchestra.run.smithi133.stderr:enabled application 'rados' on pool 'unique_pool_6' 2023-10-09T05:18:59.573 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:59.603 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:59.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:59.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:59.707 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:59.732 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:18:59.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:59.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:18:59.836 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:59.868 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:18:59.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:18:59.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:18:59.972 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:00.006 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:00.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:00.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:00.110 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:00.141 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:00.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:00.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:00.247 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:00.278 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:00.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:00.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:00.382 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:00.405 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:00.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:00.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:00.508 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:00.532 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:00.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:00.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:00.593 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T05:19:00.636 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:00.649 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:00 smithi133 bash[102858]: cluster 2023-10-09T05:18:59.497210+0000 osd.5 (osd.5) 215 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T05:19:00.649 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:00 smithi133 bash[102858]: cluster 2023-10-09T05:18:59.497448+0000 osd.7 (osd.7) 163 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T05:19:00.649 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:00 smithi133 bash[102858]: audit 2023-10-09T05:18:59.499423+0000 mon.a (mon.0) 1208 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:19:00.649 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:00 smithi133 bash[102858]: cluster 2023-10-09T05:18:59.499497+0000 mon.a (mon.0) 1209 : cluster 0 osdmap e228: 8 total, 8 up, 7 in 2023-10-09T05:19:00.649 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:00 smithi133 bash[102858]: cluster 2023-10-09T05:18:59.663997+0000 osd.4 (osd.4) 184 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T05:19:00.649 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:00 smithi133 bash[102858]: cluster 2023-10-09T05:18:59.983226+0000 mgr.y (mgr.25277) 916 : cluster 0 pgmap v952: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:19:00.649 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:00 smithi133 bash[105589]: cluster 2023-10-09T05:18:59.497210+0000 osd.5 (osd.5) 215 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T05:19:00.650 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:00 smithi133 bash[105589]: cluster 2023-10-09T05:18:59.497448+0000 osd.7 (osd.7) 163 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T05:19:00.650 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:00 smithi133 bash[105589]: audit 2023-10-09T05:18:59.499423+0000 mon.a (mon.0) 1208 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:19:00.650 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:00 smithi133 bash[105589]: cluster 2023-10-09T05:18:59.499497+0000 mon.a (mon.0) 1209 : cluster 0 osdmap e228: 8 total, 8 up, 7 in 2023-10-09T05:19:00.650 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:00 smithi133 bash[105589]: cluster 2023-10-09T05:18:59.663997+0000 osd.4 (osd.4) 184 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T05:19:00.650 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:00 smithi133 bash[105589]: cluster 2023-10-09T05:18:59.983226+0000 mgr.y (mgr.25277) 916 : cluster 0 pgmap v952: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:19:00.668 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:00.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:00.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:00.771 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:00.784 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:00 smithi194 bash[92666]: cluster 2023-10-09T05:18:59.497210+0000 osd.5 (osd.5) 215 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T05:19:00.784 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:00 smithi194 bash[92666]: cluster 2023-10-09T05:18:59.497448+0000 osd.7 (osd.7) 163 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T05:19:00.784 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:00 smithi194 bash[92666]: audit 2023-10-09T05:18:59.499423+0000 mon.a (mon.0) 1208 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:19:00.784 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:00 smithi194 bash[92666]: cluster 2023-10-09T05:18:59.499497+0000 mon.a (mon.0) 1209 : cluster 0 osdmap e228: 8 total, 8 up, 7 in 2023-10-09T05:19:00.784 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:00 smithi194 bash[92666]: cluster 2023-10-09T05:18:59.663997+0000 osd.4 (osd.4) 184 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T05:19:00.784 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:00 smithi194 bash[92666]: cluster 2023-10-09T05:18:59.983226+0000 mgr.y (mgr.25277) 916 : cluster 0 pgmap v952: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:19:00.800 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:00.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:00.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:00.904 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:00.930 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:00.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:00.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:01.034 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:01.068 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:01.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:01.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:01.172 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:01.202 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:01.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:01.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:01.306 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:01.336 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:01.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:01.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:01.440 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:01.478 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:01.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:01.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:01.582 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:01.609 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:01.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:01.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:01.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:01 smithi194 bash[92666]: cluster 2023-10-09T05:19:00.141372+0000 osd.6 (osd.6) 165 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T05:19:01.712 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:01.746 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:01.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:01.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:01.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:01 smithi133 bash[102858]: cluster 2023-10-09T05:19:00.141372+0000 osd.6 (osd.6) 165 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T05:19:01.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:01 smithi133 bash[105589]: cluster 2023-10-09T05:19:00.141372+0000 osd.6 (osd.6) 165 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T05:19:01.850 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:01.874 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:01.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:01.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:01.977 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:02.012 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:02.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:02.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:02.116 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:02.145 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:02.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:02.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:02.249 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:02.280 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:02.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:02.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:02.384 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:02.413 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:02.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:02.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:02.516 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:02.546 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:02.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:02.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:02.650 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:02.676 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:02.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:02.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:02.780 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:02.794 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:02 smithi194 bash[92666]: cluster 2023-10-09T05:19:01.984207+0000 mgr.y (mgr.25277) 917 : cluster 0 pgmap v953: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:19:02.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:02 smithi133 bash[105589]: cluster 2023-10-09T05:19:01.984207+0000 mgr.y (mgr.25277) 917 : cluster 0 pgmap v953: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:19:02.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:02 smithi133 bash[102858]: cluster 2023-10-09T05:19:01.984207+0000 mgr.y (mgr.25277) 917 : cluster 0 pgmap v953: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:19:02.812 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:02.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:02.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:02.870 DEBUG:teuthology.orchestra.run.smithi133:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_6 bench 90 write' 2023-10-09T05:19:02.872 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T05:19:02.872 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T05:19:02.914 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:hints = 1 2023-10-09T05:19:02.915 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T05:19:02.915 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Object prefix: benchmark_data_smithi133_194104 2023-10-09T05:19:02.915 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:19:02.916 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 0 0 0 0 0 0 - 0 2023-10-09T05:19:02.917 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:02.964 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:02.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:02.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:03.073 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:03.102 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:03.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:03.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:03.208 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:03.248 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:03.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:03.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:03.352 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:03.380 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:03.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:03.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:03.484 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:03.513 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:03.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:03.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:03.617 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:03.652 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:03.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:03.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:03.755 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:03.783 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:03.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:03.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:03.884 INFO:teuthology.orchestra.run.smithi133.stdout:moved 0 / 267 (0%) 2023-10-09T05:19:03.885 INFO:teuthology.orchestra.run.smithi133.stdout:avg 38.1429 2023-10-09T05:19:03.885 INFO:teuthology.orchestra.run.smithi133.stdout:stddev 10.3844 -> 10.3844 (expected baseline 5.71786) 2023-10-09T05:19:03.885 INFO:teuthology.orchestra.run.smithi133.stdout:min osd.2 with 14 -> 14 pgs (0.367041 -> 0.367041 * mean) 2023-10-09T05:19:03.885 INFO:teuthology.orchestra.run.smithi133.stdout:max osd.1 with 49 -> 49 pgs (1.28464 -> 1.28464 * mean) 2023-10-09T05:19:03.885 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:19:03.885 INFO:teuthology.orchestra.run.smithi133.stdout:oload 1000 2023-10-09T05:19:03.885 INFO:teuthology.orchestra.run.smithi133.stdout:max_change 1 2023-10-09T05:19:03.885 INFO:teuthology.orchestra.run.smithi133.stdout:max_change_osds 4 2023-10-09T05:19:03.885 INFO:teuthology.orchestra.run.smithi133.stdout:average_utilization 0.0086 2023-10-09T05:19:03.886 INFO:teuthology.orchestra.run.smithi133.stdout:overload_utilization 0.0858 2023-10-09T05:19:03.887 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:03.890 INFO:teuthology.orchestra.run.smithi133.stderr:no change 2023-10-09T05:19:03.909 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:03.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:03.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:03.915 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 1 16 2405 2389 149.304 149.312 0.00527805 0.00653065 2023-10-09T05:19:04.013 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:04.044 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:04.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:04.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:04.149 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:04.179 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:04.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:04.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:04.283 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:04.309 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:04.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:04.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:04.412 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:04.419 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:19:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:19:04] "GET /metrics HTTP/1.1" 200 36420 "" "Prometheus/2.43.0" 2023-10-09T05:19:04.436 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:04.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:04.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:04.540 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:04.563 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:04.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:04.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:04.666 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:04.695 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:04.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:04.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:04.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:04 smithi133 bash[105589]: audit 2023-10-09T05:19:03.883674+0000 mgr.y (mgr.25277) 918 : audit 0 from='client.35190 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:19:04.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:04 smithi133 bash[105589]: cluster 2023-10-09T05:19:03.984792+0000 mgr.y (mgr.25277) 919 : cluster 0 pgmap v954: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:19:04.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:04 smithi133 bash[102858]: audit 2023-10-09T05:19:03.883674+0000 mgr.y (mgr.25277) 918 : audit 0 from='client.35190 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:19:04.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:04 smithi133 bash[102858]: cluster 2023-10-09T05:19:03.984792+0000 mgr.y (mgr.25277) 919 : cluster 0 pgmap v954: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:19:04.799 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:04.824 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:04.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:04.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:04.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:04 smithi194 bash[92666]: audit 2023-10-09T05:19:03.883674+0000 mgr.y (mgr.25277) 918 : audit 0 from='client.35190 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:19:04.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:04 smithi194 bash[92666]: cluster 2023-10-09T05:19:03.984792+0000 mgr.y (mgr.25277) 919 : cluster 0 pgmap v954: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:19:04.916 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 2 16 5053 5037 157.395 165.5 0.00237697 0.00616095 2023-10-09T05:19:04.927 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:04.965 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:04.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:04.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:05.069 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:05.096 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:05.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:05.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:05.200 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:05.237 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:05.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:05.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:05.341 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:05.369 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:05.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:05.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:05.473 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:05.501 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:05.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:05.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:05.605 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:05.630 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:05.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:05.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:05.734 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:05.758 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:05.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:05.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:05.864 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:05.893 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:05.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:05.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:05.916 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 3 15 7784 7769 161.842 170.75 0.00611138 0.00616601 2023-10-09T05:19:05.996 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:06.027 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:06.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:06.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:06.131 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:06.159 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:06.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:06.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:06.264 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:06.300 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:06.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:06.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:06.404 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:06.433 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:06.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:06.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:06.537 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:06.569 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:06.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:06.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:06.672 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:06.702 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:06.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:06.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:06.806 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:06.836 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:06.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:06.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:06.916 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 4 16 10209 10193 159.253 151.5 0.00215684 0.00626987 2023-10-09T05:19:06.963 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:07.004 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:07.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:07.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:07.108 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:07.118 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:07 smithi133 bash[105589]: cluster 2023-10-09T05:19:05.985565+0000 mgr.y (mgr.25277) 920 : cluster 0 pgmap v955: 89 pgs: 89 active+clean; 248 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 25 MiB/s wr, 395 op/s 2023-10-09T05:19:07.118 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:07 smithi133 bash[102858]: cluster 2023-10-09T05:19:05.985565+0000 mgr.y (mgr.25277) 920 : cluster 0 pgmap v955: 89 pgs: 89 active+clean; 248 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 25 MiB/s wr, 395 op/s 2023-10-09T05:19:07.136 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:07.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:07.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:07.240 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:07.266 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:07.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:07.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:07.369 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:07.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:07 smithi194 bash[92666]: cluster 2023-10-09T05:19:05.985565+0000 mgr.y (mgr.25277) 920 : cluster 0 pgmap v955: 89 pgs: 89 active+clean; 248 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 25 MiB/s wr, 395 op/s 2023-10-09T05:19:07.395 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:07.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:07.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:07.501 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:07.530 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:07.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:07.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:07.633 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:07.656 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:07.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:07.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:07.760 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:07.790 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:07.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:07.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:07.894 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:07.916 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 5 16 12668 12652 158.137 153.688 0.00684479 0.0063163 2023-10-09T05:19:07.920 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:07.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:07.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:08.023 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:08.054 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:08.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:08.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:08.159 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:08.187 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:08.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:08.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:08.290 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:08.317 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:08.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:08.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:08.419 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:08.448 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:08.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:08.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:08.552 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:08.580 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:08.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:08.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:08.683 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:08.695 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-10-09T05:19:08.695 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 110 0.05 2023-10-09T05:19:08.708 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:08.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:08.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:08.811 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:08.844 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:08.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:08.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:08.916 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 6 16 14769 14753 153.664 131.312 0.0152363 0.00649416 2023-10-09T05:19:08.953 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:08.980 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:08.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:08.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:09.085 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:09.113 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:09.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:09.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:09.161 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:19:09.195 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:09 smithi133 bash[102858]: cluster 2023-10-09T05:19:07.986198+0000 mgr.y (mgr.25277) 921 : cluster 0 pgmap v956: 89 pgs: 89 active+clean; 480 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 48 MiB/s wr, 766 op/s 2023-10-09T05:19:09.195 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:09 smithi133 bash[105589]: cluster 2023-10-09T05:19:07.986198+0000 mgr.y (mgr.25277) 921 : cluster 0 pgmap v956: 89 pgs: 89 active+clean; 480 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 48 MiB/s wr, 766 op/s 2023-10-09T05:19:09.199 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:19:09.216 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:09.242 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:19:09.255 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:09.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:09.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:09.274 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:19:09.333 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:19:09.359 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:09.381 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:19:09.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:09 smithi194 bash[92666]: cluster 2023-10-09T05:19:07.986198+0000 mgr.y (mgr.25277) 921 : cluster 0 pgmap v956: 89 pgs: 89 active+clean; 480 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 48 MiB/s wr, 766 op/s 2023-10-09T05:19:09.387 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:09.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:09.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:09.423 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:19:09.477 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:19:09.494 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:09.516 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:19:09.520 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:19:09.520 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:19:09.520 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:19:09.520 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:19:09.520 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:19:09.520 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:19:09.521 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:19:09.521 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:19:09.522 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:09.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:09.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:09.625 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:09.653 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:09.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:09.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:09.757 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:09.786 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:09.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:09.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:09.889 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:09.915 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:09.916 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 7 16 16440 16424 146.63 104.438 0.00663215 0.00681214 2023-10-09T05:19:09.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:09.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:10.019 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:10.047 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:10.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:10.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:10.074 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is unset 2023-10-09T05:19:10.149 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:10.159 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:10 smithi194 bash[92666]: audit 2023-10-09T05:19:09.157749+0000 mon.c (mon.2) 91 : audit 0 from='client.? 172.21.15.133:0/1118662234' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:19:10.159 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:10 smithi194 bash[92666]: audit 2023-10-09T05:19:09.609008+0000 mon.a (mon.0) 1210 : audit 1 from='client.? 172.21.15.133:0/644565521' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:19:10.179 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:10.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:10.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:10.283 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:10.311 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:10.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:10.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:10.417 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:10.445 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:10.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:10.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:10.473 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:10 smithi133 bash[102858]: audit 2023-10-09T05:19:09.157749+0000 mon.c (mon.2) 91 : audit 0 from='client.? 172.21.15.133:0/1118662234' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:19:10.474 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:10 smithi133 bash[102858]: audit 2023-10-09T05:19:09.609008+0000 mon.a (mon.0) 1210 : audit 1 from='client.? 172.21.15.133:0/644565521' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:19:10.474 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:10 smithi133 bash[105589]: audit 2023-10-09T05:19:09.157749+0000 mon.c (mon.2) 91 : audit 0 from='client.? 172.21.15.133:0/1118662234' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:19:10.474 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:10 smithi133 bash[105589]: audit 2023-10-09T05:19:09.609008+0000 mon.a (mon.0) 1210 : audit 1 from='client.? 172.21.15.133:0/644565521' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:19:10.550 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:10.582 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:10.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:10.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:10.688 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:10.715 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:10.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:10.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:10.819 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:10.845 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:10.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:10.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:10.916 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 8 16 18573 18557 144.964 133.312 0.0031446 0.00689084 2023-10-09T05:19:10.949 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:10.976 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:10.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:10.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:11.079 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:11.104 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:11.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:11.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:11.209 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:11.218 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:11 smithi133 bash[105589]: cluster 2023-10-09T05:19:09.986908+0000 mgr.y (mgr.25277) 922 : cluster 0 pgmap v957: 89 pgs: 89 active+clean; 708 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T05:19:11.218 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:11 smithi133 bash[105589]: cluster 2023-10-09T05:19:10.047084+0000 mon.a (mon.0) 1211 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:19:11.218 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:11 smithi133 bash[105589]: cluster 2023-10-09T05:19:10.047122+0000 mon.a (mon.0) 1212 : cluster 1 Cluster is now healthy 2023-10-09T05:19:11.218 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:11 smithi133 bash[105589]: cluster 2023-10-09T05:19:10.062203+0000 osd.5 (osd.5) 216 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T05:19:11.218 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:11 smithi133 bash[105589]: audit 2023-10-09T05:19:10.066132+0000 mon.a (mon.0) 1213 : audit 1 from='client.? 172.21.15.133:0/644565521' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:19:11.218 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:11 smithi133 bash[105589]: cluster 2023-10-09T05:19:10.066173+0000 osd.7 (osd.7) 164 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T05:19:11.219 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:11 smithi133 bash[105589]: cluster 2023-10-09T05:19:10.066368+0000 mon.a (mon.0) 1214 : cluster 0 osdmap e229: 8 total, 8 up, 7 in 2023-10-09T05:19:11.219 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:11 smithi133 bash[105589]: cluster 2023-10-09T05:19:10.066406+0000 osd.4 (osd.4) 185 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T05:19:11.219 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:11 smithi133 bash[105589]: cluster 2023-10-09T05:19:10.071348+0000 osd.6 (osd.6) 166 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T05:19:11.219 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:11 smithi133 bash[105589]: cluster 2023-10-09T05:19:10.073438+0000 osd.6 (osd.6) 167 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T05:19:11.219 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:11 smithi133 bash[102858]: cluster 2023-10-09T05:19:09.986908+0000 mgr.y (mgr.25277) 922 : cluster 0 pgmap v957: 89 pgs: 89 active+clean; 708 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T05:19:11.219 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:11 smithi133 bash[102858]: cluster 2023-10-09T05:19:10.047084+0000 mon.a (mon.0) 1211 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:19:11.219 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:11 smithi133 bash[102858]: cluster 2023-10-09T05:19:10.047122+0000 mon.a (mon.0) 1212 : cluster 1 Cluster is now healthy 2023-10-09T05:19:11.220 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:11 smithi133 bash[102858]: cluster 2023-10-09T05:19:10.062203+0000 osd.5 (osd.5) 216 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T05:19:11.220 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:11 smithi133 bash[102858]: audit 2023-10-09T05:19:10.066132+0000 mon.a (mon.0) 1213 : audit 1 from='client.? 172.21.15.133:0/644565521' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:19:11.220 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:11 smithi133 bash[102858]: cluster 2023-10-09T05:19:10.066173+0000 osd.7 (osd.7) 164 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T05:19:11.220 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:11 smithi133 bash[102858]: cluster 2023-10-09T05:19:10.066368+0000 mon.a (mon.0) 1214 : cluster 0 osdmap e229: 8 total, 8 up, 7 in 2023-10-09T05:19:11.220 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:11 smithi133 bash[102858]: cluster 2023-10-09T05:19:10.066406+0000 osd.4 (osd.4) 185 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T05:19:11.220 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:11 smithi133 bash[102858]: cluster 2023-10-09T05:19:10.071348+0000 osd.6 (osd.6) 166 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T05:19:11.220 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:11 smithi133 bash[102858]: cluster 2023-10-09T05:19:10.073438+0000 osd.6 (osd.6) 167 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T05:19:11.234 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:11.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:11.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:11.339 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:11.349 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:11 smithi194 bash[92666]: cluster 2023-10-09T05:19:09.986908+0000 mgr.y (mgr.25277) 922 : cluster 0 pgmap v957: 89 pgs: 89 active+clean; 708 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T05:19:11.350 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:11 smithi194 bash[92666]: cluster 2023-10-09T05:19:10.047084+0000 mon.a (mon.0) 1211 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:19:11.350 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:11 smithi194 bash[92666]: cluster 2023-10-09T05:19:10.047122+0000 mon.a (mon.0) 1212 : cluster 1 Cluster is now healthy 2023-10-09T05:19:11.350 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:11 smithi194 bash[92666]: cluster 2023-10-09T05:19:10.062203+0000 osd.5 (osd.5) 216 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T05:19:11.350 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:11 smithi194 bash[92666]: audit 2023-10-09T05:19:10.066132+0000 mon.a (mon.0) 1213 : audit 1 from='client.? 172.21.15.133:0/644565521' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:19:11.350 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:11 smithi194 bash[92666]: cluster 2023-10-09T05:19:10.066173+0000 osd.7 (osd.7) 164 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T05:19:11.350 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:11 smithi194 bash[92666]: cluster 2023-10-09T05:19:10.066368+0000 mon.a (mon.0) 1214 : cluster 0 osdmap e229: 8 total, 8 up, 7 in 2023-10-09T05:19:11.350 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:11 smithi194 bash[92666]: cluster 2023-10-09T05:19:10.066406+0000 osd.4 (osd.4) 185 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T05:19:11.350 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:11 smithi194 bash[92666]: cluster 2023-10-09T05:19:10.071348+0000 osd.6 (osd.6) 166 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T05:19:11.350 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:11 smithi194 bash[92666]: cluster 2023-10-09T05:19:10.073438+0000 osd.6 (osd.6) 167 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T05:19:11.371 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:11.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:11.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:11.477 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:11.505 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:11.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:11.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:11.611 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:11.640 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:11.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:11.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:11.745 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:11.771 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:11.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:11.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:11.875 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:11.902 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:11.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:11.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:11.916 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 9 16 20550 20534 142.584 123.562 0.00543653 0.00700657 2023-10-09T05:19:12.006 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:12.036 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:12.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:12.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:12.140 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:12.163 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:12.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:12.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:12.266 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:12.292 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:12.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:12.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:12.396 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:12.422 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:12.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:12.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:12.526 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:12.553 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:12.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:12.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:12.658 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:12.679 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:12.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:12.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:12.783 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:12.815 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:12.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:12.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:12.917 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 10 16 22342 22326 139.525 112 0.00258046 0.00715923 2023-10-09T05:19:12.920 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:12.946 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:12.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:12.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:13.049 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:13.077 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:13.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:13.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:13.181 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:13.206 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:13.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:13.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:13.311 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:13.319 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:13 smithi194 bash[92666]: cluster 2023-10-09T05:19:11.987676+0000 mgr.y (mgr.25277) 923 : cluster 0 pgmap v959: 89 pgs: 89 active+clean; 998 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-10-09T05:19:13.337 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:13.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:13.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:13.441 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:13.474 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:13.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:13.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:13.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:13 smithi133 bash[105589]: cluster 2023-10-09T05:19:11.987676+0000 mgr.y (mgr.25277) 923 : cluster 0 pgmap v959: 89 pgs: 89 active+clean; 998 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-10-09T05:19:13.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:13 smithi133 bash[102858]: cluster 2023-10-09T05:19:11.987676+0000 mgr.y (mgr.25277) 923 : cluster 0 pgmap v959: 89 pgs: 89 active+clean; 998 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-10-09T05:19:13.578 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:13.602 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:13.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:13.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:13.705 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:13.729 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:13.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:13.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:13.832 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:13.858 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:13.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:13.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:13.917 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 11 16 24124 24108 136.965 111.375 0.0112241 0.00729237 2023-10-09T05:19:13.961 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:13.989 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:13.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:13.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:14.092 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:14.120 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:14.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:14.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:14.223 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:14.249 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:14.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:14.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:14.353 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:14.362 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:19:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:19:14] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.43.0" 2023-10-09T05:19:14.391 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:14.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:14.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:14.495 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:14.524 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:14.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:14.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:14.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:14 smithi133 bash[102858]: audit 2023-10-09T05:19:13.461975+0000 mon.a (mon.0) 1215 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:19:14.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:14 smithi133 bash[102858]: audit 2023-10-09T05:19:13.463102+0000 mon.a (mon.0) 1216 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:19:14.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:14 smithi133 bash[102858]: cluster 2023-10-09T05:19:13.988419+0000 mgr.y (mgr.25277) 924 : cluster 0 pgmap v960: 89 pgs: 89 active+clean; 1.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-10-09T05:19:14.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:14 smithi133 bash[105589]: audit 2023-10-09T05:19:13.461975+0000 mon.a (mon.0) 1215 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:19:14.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:14 smithi133 bash[105589]: audit 2023-10-09T05:19:13.463102+0000 mon.a (mon.0) 1216 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:19:14.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:14 smithi133 bash[105589]: cluster 2023-10-09T05:19:13.988419+0000 mgr.y (mgr.25277) 924 : cluster 0 pgmap v960: 89 pgs: 89 active+clean; 1.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-10-09T05:19:14.628 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:14.655 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:14.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:14.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:14.759 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:14.769 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:14 smithi194 bash[92666]: audit 2023-10-09T05:19:13.461975+0000 mon.a (mon.0) 1215 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:19:14.769 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:14 smithi194 bash[92666]: audit 2023-10-09T05:19:13.463102+0000 mon.a (mon.0) 1216 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:19:14.769 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:14 smithi194 bash[92666]: cluster 2023-10-09T05:19:13.988419+0000 mgr.y (mgr.25277) 924 : cluster 0 pgmap v960: 89 pgs: 89 active+clean; 1.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-10-09T05:19:14.787 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:14.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:14.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:14.890 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:14.918 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 12 16 25821 25805 134.378 106.062 0.016018 0.00743458 2023-10-09T05:19:14.921 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:14.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:14.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:15.026 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:15.055 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:15.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:15.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:15.159 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:15.189 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:15.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:15.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:15.293 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:15.319 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:15.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:15.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:15.424 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:15.450 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:15.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:15.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:15.553 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:15.578 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:15.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:15.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:15.620 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:19:15.644 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:19:15.683 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:15.736 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:15.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:15.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:15.842 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:15.870 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:15.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:15.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:15.918 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 13 16 27461 27445 131.925 102.5 0.0116517 0.00757248 2023-10-09T05:19:15.973 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:16.020 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:16.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:16.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:16.124 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:16.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:16.154 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:16.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:16.255 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:16.293 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:16.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:16.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:16.400 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:16.428 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:16.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:16.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:16.533 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:16.561 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:16.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:16.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:16.665 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:16.697 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:16.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:16.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:16.800 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:16.827 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:16.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:16.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:16.918 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 14 16 29070 29054 129.685 100.562 0.00799336 0.00770544 2023-10-09T05:19:16.930 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:16.955 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:16.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:16.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:17.060 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:17.091 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:17.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:17.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:17.196 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:17.205 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:17 smithi194 bash[92666]: cluster 2023-10-09T05:19:15.992928+0000 mgr.y (mgr.25277) 925 : cluster 0 pgmap v961: 89 pgs: 89 active+clean; 1.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-10-09T05:19:17.226 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:17.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:17.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:17.330 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:17.340 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:17 smithi133 bash[105589]: cluster 2023-10-09T05:19:15.992928+0000 mgr.y (mgr.25277) 925 : cluster 0 pgmap v961: 89 pgs: 89 active+clean; 1.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-10-09T05:19:17.340 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:17 smithi133 bash[102858]: cluster 2023-10-09T05:19:15.992928+0000 mgr.y (mgr.25277) 925 : cluster 0 pgmap v961: 89 pgs: 89 active+clean; 1.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-10-09T05:19:17.359 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:17.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:17.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:17.465 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:17.491 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:17.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:17.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:17.594 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:17.623 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:17.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:17.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:17.726 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:17.763 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:17.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:17.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:17.867 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:17.897 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:17.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:17.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:17.918 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 15 16 30797 30781 128.234 107.938 0.0150343 0.00779204 2023-10-09T05:19:18.000 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:18.041 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:18.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:18.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:18.142 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:19:18.143 INFO:teuthology.orchestra.run.smithi133.stdout: "ops": [ 2023-10-09T05:19:18.143 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:19:18.143 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35187.0:31101 11.b 11:d2b8616c:::benchmark_data_smithi133_194104_object31100:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-10-09T05:19:18.143 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:19:18.135596+0000", 2023-10-09T05:19:18.143 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.0055799489999999998, 2023-10-09T05:19:18.143 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0056197039999999997, 2023-10-09T05:19:18.143 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:19:18.143 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:19:18.143 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:19:18.143 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35187", 2023-10-09T05:19:18.144 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2118344251", 2023-10-09T05:19:18.144 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 31101 2023-10-09T05:19:18.144 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:19:18.144 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:19:18.144 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:19:18.144 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:19:18.144 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:19:18.135596+0000", 2023-10-09T05:19:18.144 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:19:18.144 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:19:18.144 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:19:18.144 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:19:18.145 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:19:18.135596+0000", 2023-10-09T05:19:18.145 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:19:18.145 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:19:18.145 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:19:18.145 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:19:18.145 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:19:18.135594+0000", 2023-10-09T05:19:18.145 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:19:18.145 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:19:18.145 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:19:18.145 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:19:18.145 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:19:18.135626+0000", 2023-10-09T05:19:18.145 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.1269000000000002e-05 2023-10-09T05:19:18.146 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:19:18.146 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:19:18.146 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:19:18.146 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:19:18.135639+0000", 2023-10-09T05:19:18.146 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.3923999999999999e-05 2023-10-09T05:19:18.146 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:19:18.146 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:19:18.146 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:19:18.146 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:19:18.135649+0000", 2023-10-09T05:19:18.146 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.1619999999999997e-06 2023-10-09T05:19:18.146 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:19:18.146 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:19:18.147 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:19:18.147 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:19:18.135702+0000", 2023-10-09T05:19:18.147 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.3798999999999998e-05 2023-10-09T05:19:18.147 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:19:18.147 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:19:18.147 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:19:18.147 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:19:18.135859+0000", 2023-10-09T05:19:18.147 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00015639899999999999 2023-10-09T05:19:18.147 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:19:18.147 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:19:18.147 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:19:18.147 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:19:18.136032+0000", 2023-10-09T05:19:18.148 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00017359100000000001 2023-10-09T05:19:18.148 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:19:18.148 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:19:18.148 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:19:18.148 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:19:18.138300+0000", 2023-10-09T05:19:18.148 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0022675350000000002 2023-10-09T05:19:18.148 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:19:18.148 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:19:18.148 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:19:18.148 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:19:18.148 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:19:18.148 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35187.0:31103 11.a 11:5c37d9a2:::benchmark_data_smithi133_194104_object31102:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-10-09T05:19:18.148 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:19:18.137053+0000", 2023-10-09T05:19:18.149 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.0041227579999999998, 2023-10-09T05:19:18.149 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.004259975, 2023-10-09T05:19:18.149 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:19:18.149 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:19:18.149 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:19:18.149 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35187", 2023-10-09T05:19:18.149 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2118344251", 2023-10-09T05:19:18.149 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 31103 2023-10-09T05:19:18.149 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:19:18.149 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:19:18.149 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:19:18.149 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:19:18.150 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:19:18.137053+0000", 2023-10-09T05:19:18.150 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:19:18.150 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:19:18.150 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:19:18.150 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:19:18.150 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:19:18.137053+0000", 2023-10-09T05:19:18.150 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:19:18.150 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:19:18.150 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:19:18.151 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:19:18.151 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:19:18.137052+0000", 2023-10-09T05:19:18.151 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:19:18.151 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:19:18.151 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:19:18.151 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:19:18.151 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:19:18.137109+0000", 2023-10-09T05:19:18.152 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.6773e-05 2023-10-09T05:19:18.152 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:19:18.152 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:19:18.152 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:19:18.152 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:19:18.137127+0000", 2023-10-09T05:19:18.152 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.7835000000000001e-05 2023-10-09T05:19:18.152 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:19:18.152 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:19:18.152 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:19:18.153 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:19:18.137134+0000", 2023-10-09T05:19:18.153 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.9970000000000001e-06 2023-10-09T05:19:18.153 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:19:18.153 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:19:18.153 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:19:18.153 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:19:18.137168+0000", 2023-10-09T05:19:18.153 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.4156000000000003e-05 2023-10-09T05:19:18.154 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:19:18.154 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:19:18.154 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:19:18.154 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:19:18.137295+0000", 2023-10-09T05:19:18.154 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00012760199999999999 2023-10-09T05:19:18.154 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:19:18.155 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:19:18.155 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 1,6", 2023-10-09T05:19:18.155 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:19:18.137512+0000", 2023-10-09T05:19:18.155 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00021694800000000001 2023-10-09T05:19:18.155 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:19:18.155 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:19:18.156 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:19:18.156 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:19:18.139839+0000", 2023-10-09T05:19:18.156 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0023264800000000001 2023-10-09T05:19:18.156 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:19:18.156 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:19:18.156 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:19:18.156 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:19:18.156 INFO:teuthology.orchestra.run.smithi133.stdout: ], 2023-10-09T05:19:18.156 INFO:teuthology.orchestra.run.smithi133.stdout: "num_ops": 2 2023-10-09T05:19:18.157 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:19:18.157 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:18.188 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:18.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:18.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:18.292 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:18.319 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:18.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:18.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:18.426 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:18.452 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:18.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:18.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:18.485 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:19:18.557 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:18.605 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:18.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:18.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:18.713 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:18.743 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:18.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:18.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:18.847 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:18.876 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:18.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:18.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:18.918 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 16 16 32239 32223 125.852 90.125 0.00533855 0.00793852 2023-10-09T05:19:18.980 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:19.001 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:19.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:19.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:19.104 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:19.134 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:19.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:19.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:19.237 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:19.280 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:19.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:19.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:19.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:19 smithi133 bash[105589]: cluster 2023-10-09T05:19:17.993679+0000 mgr.y (mgr.25277) 926 : cluster 0 pgmap v962: 89 pgs: 89 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-10-09T05:19:19.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:19 smithi133 bash[102858]: cluster 2023-10-09T05:19:17.993679+0000 mgr.y (mgr.25277) 926 : cluster 0 pgmap v962: 89 pgs: 89 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-10-09T05:19:19.385 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:19.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:19 smithi194 bash[92666]: cluster 2023-10-09T05:19:17.993679+0000 mgr.y (mgr.25277) 926 : cluster 0 pgmap v962: 89 pgs: 89 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-10-09T05:19:19.409 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:19.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:19.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:19.512 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:19.540 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:19.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:19.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:19.645 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:19.671 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:19.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:19.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:19.776 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:19.804 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:19.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:19.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:19.909 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:19.918 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 17 16 33719 33703 123.89 92.5 0.00823713 0.00806652 2023-10-09T05:19:19.932 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:19.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:19.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:20.036 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:20.064 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:20.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:20.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:20.168 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:20.195 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:20.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:20.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:20.299 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:20.327 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:20.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:20.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:20.431 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:20.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:20.465 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:20.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:20.566 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:20.599 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:20.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:20.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:20.705 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:20.733 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:20.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:20.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:20.836 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:20.864 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:20.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:20.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:20.918 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 18 15 35153 35138 121.989 89.6875 0.00245964 0.00818969 2023-10-09T05:19:20.968 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:20.993 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:20.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:20.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:21.096 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:21.106 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:21 smithi194 bash[92666]: cluster 2023-10-09T05:19:19.995087+0000 mgr.y (mgr.25277) 927 : cluster 0 pgmap v963: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-10-09T05:19:21.128 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:21.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:21.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:21.233 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:21.263 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:21.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:21.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:21.368 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:21.377 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:21 smithi133 bash[102858]: cluster 2023-10-09T05:19:19.995087+0000 mgr.y (mgr.25277) 927 : cluster 0 pgmap v963: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-10-09T05:19:21.378 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:21 smithi133 bash[105589]: cluster 2023-10-09T05:19:19.995087+0000 mgr.y (mgr.25277) 927 : cluster 0 pgmap v963: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-10-09T05:19:21.395 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T05:19:21.405 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:21.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:21.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:21.509 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:21.552 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:21.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:21.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:21.659 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:21.691 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:21.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:21.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:21.796 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:21.822 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:21.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:21.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:21.860 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:19:21.860 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:19:21.918 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 19 16 36456 36440 119.85 81.375 0.0130023 0.00833797 2023-10-09T05:19:21.927 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:21.954 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:21.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:21.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:22.060 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:22.092 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:22.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:22.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:22.196 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:22.221 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:22.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:22.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:22.326 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:22.366 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:22.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:22.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:22.470 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:22.498 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:22.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:22.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:22.601 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:22.634 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:22.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:22.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:22.737 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:22.761 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:22.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:22.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:22.864 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:22.890 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:22.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:22.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:22.919 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T05:19:22.918867+0000 min lat: 0.0015063 max lat: 0.11307 avg lat: 0.00843373 2023-10-09T05:19:22.919 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:19:22.919 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 20 16 37938 37922 118.489 92.625 0.00911414 0.00843373 2023-10-09T05:19:22.996 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:23.026 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:23.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:23.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:23.132 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:23.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:23 smithi194 bash[92666]: cluster 2023-10-09T05:19:21.995594+0000 mgr.y (mgr.25277) 928 : cluster 0 pgmap v964: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-10-09T05:19:23.163 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:23.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:23.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:23.267 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:23.297 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:23.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:23.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:23.402 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:23.411 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:23 smithi133 bash[105589]: cluster 2023-10-09T05:19:21.995594+0000 mgr.y (mgr.25277) 928 : cluster 0 pgmap v964: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-10-09T05:19:23.411 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:23 smithi133 bash[102858]: cluster 2023-10-09T05:19:21.995594+0000 mgr.y (mgr.25277) 928 : cluster 0 pgmap v964: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-10-09T05:19:23.429 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:23.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:23.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:23.534 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:23.567 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:23.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:23.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:23.670 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:23.706 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:23.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:23.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:23.808 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:23.919 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 21 16 39236 39220 116.709 81.125 0.0113777 0.00856299 2023-10-09T05:19:24.097 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:24.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:24.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:24.200 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:24.229 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:24.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:24.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:24.333 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:24.360 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:24.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:24.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:24.465 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:24.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:24.500 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:24.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:24.601 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:24.637 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:24.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:24.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:24.741 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:24.770 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:24.773 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:24 smithi194 bash[92666]: cluster 2023-10-09T05:19:23.996109+0000 mgr.y (mgr.25277) 929 : cluster 0 pgmap v965: 89 pgs: 89 active+clean; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-10-09T05:19:24.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:24.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:24.798 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:19:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:19:24] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.43.0" 2023-10-09T05:19:24.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:24 smithi133 bash[105589]: cluster 2023-10-09T05:19:23.996109+0000 mgr.y (mgr.25277) 929 : cluster 0 pgmap v965: 89 pgs: 89 active+clean; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-10-09T05:19:24.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:24 smithi133 bash[102858]: cluster 2023-10-09T05:19:23.996109+0000 mgr.y (mgr.25277) 929 : cluster 0 pgmap v965: 89 pgs: 89 active+clean; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-10-09T05:19:24.875 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:24.901 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:24.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:24.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:24.919 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 22 16 40597 40581 115.27 85.0625 0.00719339 0.0086698 2023-10-09T05:19:25.005 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:25.030 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:25.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:25.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:25.133 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:25.160 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:25.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:25.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:25.265 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:25.284 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:25.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:25.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:25.388 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:25.414 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:25.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:25.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:25.521 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:25.549 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:25.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:25.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:25.653 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:25.678 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:25.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:25.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:25.782 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:25.813 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:25.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:25.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:25.916 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:25.919 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 23 16 42000 41984 114.071 87.6875 0.00848171 0.00875893 2023-10-09T05:19:25.945 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:25.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:25.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:26.048 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:26.075 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:26.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:26.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:26.178 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:26.208 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:26.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:26.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:26.315 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:26.341 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:26.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:26.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:26.445 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:26.472 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:26.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:26.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:26.576 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:26.602 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:26.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:26.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:26.705 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:26.832 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:26.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:26.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:26.919 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 24 16 43368 43352 112.88 85.5 0.00873486 0.00885343 2023-10-09T05:19:26.936 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:26.964 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:26.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:26.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:27.067 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:27.099 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:27.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:27.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:27.202 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:27.212 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:27 smithi194 bash[92666]: cluster 2023-10-09T05:19:25.996654+0000 mgr.y (mgr.25277) 930 : cluster 0 pgmap v966: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T05:19:27.227 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:27.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:27.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:27.332 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:27.364 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:27.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:27.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:27.468 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:27.497 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:27.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:27.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:27.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:27 smithi133 bash[102858]: cluster 2023-10-09T05:19:25.996654+0000 mgr.y (mgr.25277) 930 : cluster 0 pgmap v966: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T05:19:27.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:27 smithi133 bash[105589]: cluster 2023-10-09T05:19:25.996654+0000 mgr.y (mgr.25277) 930 : cluster 0 pgmap v966: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T05:19:27.602 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:27.628 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:27.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:27.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:27.732 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:27.763 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:27.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:27.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:27.867 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:27.892 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:27.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:27.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:27.919 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 25 16 44461 44445 111.097 68.3125 0.00772136 0.00897267 2023-10-09T05:19:27.995 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:28.031 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:28.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:28.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:28.136 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:28.166 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:28.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:28.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:28.270 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:28.294 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:28.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:28.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:28.397 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:28.424 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:28.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:28.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:28.530 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:28.571 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:28.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:28.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:28.677 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:28.709 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:28.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:28.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:28.816 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:28.847 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:28.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:28.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:28.919 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 26 16 45624 45608 109.62 72.6875 0.00600059 0.00911658 2023-10-09T05:19:28.953 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:28.976 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:28.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:28.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:29.059 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:29 smithi133 bash[102858]: cluster 2023-10-09T05:19:27.997190+0000 mgr.y (mgr.25277) 931 : cluster 0 pgmap v967: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T05:19:29.059 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:29 smithi133 bash[102858]: audit 2023-10-09T05:19:28.454351+0000 mon.a (mon.0) 1217 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:19:29.079 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:29.089 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:29 smithi194 bash[92666]: cluster 2023-10-09T05:19:27.997190+0000 mgr.y (mgr.25277) 931 : cluster 0 pgmap v967: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T05:19:29.089 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:29 smithi194 bash[92666]: audit 2023-10-09T05:19:28.454351+0000 mon.a (mon.0) 1217 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:19:29.108 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:29.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:29.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:29.210 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:29.238 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:29.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:29.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:29.310 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:29 smithi133 bash[105589]: cluster 2023-10-09T05:19:27.997190+0000 mgr.y (mgr.25277) 931 : cluster 0 pgmap v967: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T05:19:29.310 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:29 smithi133 bash[105589]: audit 2023-10-09T05:19:28.454351+0000 mon.a (mon.0) 1217 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:19:29.342 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:29.370 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:29.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:29.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:29.476 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:29.501 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:29.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:29.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:29.604 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:29.640 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:29.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:29.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:29.663 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:19:29.744 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:29.771 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:29.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:29.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:29.874 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:29.906 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:29.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:29.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:29.919 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 27 16 46823 46807 108.335 74.9375 0.0115246 0.00922478 2023-10-09T05:19:30.011 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:30.038 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:30.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:30.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:30.141 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:30.166 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:30.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:30.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:30.269 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:30.316 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:30.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:30.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:30.420 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:30.451 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:30.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:30.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:30.554 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:30.582 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:30.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:30.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:30.688 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:30.707 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:30.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:30.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:30.809 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:30.830 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:30.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:30.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:30.919 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 28 16 47906 47890 106.883 67.6875 0.0127124 0.00934317 2023-10-09T05:19:30.933 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:30.961 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:30.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:30.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:31.065 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:31.087 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:31.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:31.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:31.191 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:31.201 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:31 smithi133 bash[105589]: cluster 2023-10-09T05:19:29.997792+0000 mgr.y (mgr.25277) 932 : cluster 0 pgmap v968: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T05:19:31.201 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:31 smithi133 bash[102858]: cluster 2023-10-09T05:19:29.997792+0000 mgr.y (mgr.25277) 932 : cluster 0 pgmap v968: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T05:19:31.214 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:31.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:31.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:31.318 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:31.347 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:31.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:31.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:31.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:31 smithi194 bash[92666]: cluster 2023-10-09T05:19:29.997792+0000 mgr.y (mgr.25277) 932 : cluster 0 pgmap v968: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T05:19:31.453 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:31.479 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:31.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:31.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:31.585 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:31.612 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:31.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:31.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:31.718 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:31.745 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:31.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:31.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:31.851 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:31.882 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:31.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:31.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:31.920 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 29 16 49169 49153 105.919 78.9375 0.00804308 0.00942882 2023-10-09T05:19:31.987 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:32.015 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:32.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:32.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:32.121 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:32.150 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:32.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:32.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:32.255 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:32.287 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:32.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:32.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:32.392 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:32.413 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:32.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:32.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:32.516 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:32.547 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:32.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:32.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:32.653 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:32.670 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:32.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:32.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:32.772 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:32.807 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:32.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:32.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:32.911 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:32.919 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 30 16 50322 50306 104.79 72.0625 0.00832737 0.0095365 2023-10-09T05:19:32.937 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:32.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:32.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:33.041 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:33.072 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:33.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:33.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:33.087 INFO:teuthology.orchestra.run.smithi133.stdout:moved 5 / 267 (1.87266%) 2023-10-09T05:19:33.087 INFO:teuthology.orchestra.run.smithi133.stdout:avg 38.1429 2023-10-09T05:19:33.087 INFO:teuthology.orchestra.run.smithi133.stdout:stddev 10.3844 -> 9.78962 (expected baseline 5.71786) 2023-10-09T05:19:33.087 INFO:teuthology.orchestra.run.smithi133.stdout:min osd.2 with 14 -> 15 pgs (0.367041 -> 0.393258 * mean) 2023-10-09T05:19:33.087 INFO:teuthology.orchestra.run.smithi133.stdout:max osd.1 with 49 -> 46 pgs (1.28464 -> 1.20599 * mean) 2023-10-09T05:19:33.087 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:19:33.087 INFO:teuthology.orchestra.run.smithi133.stdout:oload 110 2023-10-09T05:19:33.087 INFO:teuthology.orchestra.run.smithi133.stdout:max_change 0.05 2023-10-09T05:19:33.087 INFO:teuthology.orchestra.run.smithi133.stdout:max_change_osds 4 2023-10-09T05:19:33.087 INFO:teuthology.orchestra.run.smithi133.stdout:average_utilization 0.0221 2023-10-09T05:19:33.087 INFO:teuthology.orchestra.run.smithi133.stdout:overload_utilization 0.0243 2023-10-09T05:19:33.088 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2 weight 0.5046 -> 0.5546 2023-10-09T05:19:33.088 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1 weight 1.0000 -> 0.9500 2023-10-09T05:19:33.088 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6 weight 1.0000 -> 0.9500 2023-10-09T05:19:33.176 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:33.186 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:33 smithi133 bash[105589]: cluster 2023-10-09T05:19:31.998297+0000 mgr.y (mgr.25277) 933 : cluster 0 pgmap v969: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-10-09T05:19:33.186 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:33 smithi133 bash[105589]: audit 2023-10-09T05:19:32.120529+0000 mgr.y (mgr.25277) 934 : audit 0 from='client.54910 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:19:33.186 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:33 smithi133 bash[105589]: audit 2023-10-09T05:19:32.121288+0000 mon.a (mon.0) 1218 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'62260','2':'36344','6':'62260'}"}]: dispatch 2023-10-09T05:19:33.186 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:33 smithi133 bash[102858]: cluster 2023-10-09T05:19:31.998297+0000 mgr.y (mgr.25277) 933 : cluster 0 pgmap v969: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-10-09T05:19:33.187 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:33 smithi133 bash[102858]: audit 2023-10-09T05:19:32.120529+0000 mgr.y (mgr.25277) 934 : audit 0 from='client.54910 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:19:33.187 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:33 smithi133 bash[102858]: audit 2023-10-09T05:19:32.121288+0000 mon.a (mon.0) 1218 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'62260','2':'36344','6':'62260'}"}]: dispatch 2023-10-09T05:19:33.203 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:33.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:33.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:33.306 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:33.317 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:33 smithi194 bash[92666]: cluster 2023-10-09T05:19:31.998297+0000 mgr.y (mgr.25277) 933 : cluster 0 pgmap v969: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-10-09T05:19:33.317 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:33 smithi194 bash[92666]: audit 2023-10-09T05:19:32.120529+0000 mgr.y (mgr.25277) 934 : audit 0 from='client.54910 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:19:33.317 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:33 smithi194 bash[92666]: audit 2023-10-09T05:19:32.121288+0000 mon.a (mon.0) 1218 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'62260','2':'36344','6':'62260'}"}]: dispatch 2023-10-09T05:19:33.335 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:33.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:33.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:33.438 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:33.455 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:33.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:33.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:33.558 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:33.584 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:33.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:33.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:33.687 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:33.714 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:33.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:33.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:33.817 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:33.848 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:33.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:33.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:33.920 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 31 16 50752 50736 102.277 26.875 0.00290966 0.00955512 2023-10-09T05:19:33.951 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:33.976 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:33.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:33.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:34.080 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:34.104 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:34.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:34.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:34.207 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:34.216 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:34 smithi133 bash[105589]: cluster 2023-10-09T05:19:33.083110+0000 osd.6 (osd.6) 168 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T05:19:34.216 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:34 smithi133 bash[105589]: cluster 2023-10-09T05:19:33.084554+0000 osd.6 (osd.6) 169 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T05:19:34.216 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:34 smithi133 bash[105589]: audit 2023-10-09T05:19:33.086061+0000 mon.a (mon.0) 1219 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'62260','2':'36344','6':'62260'}"}]': finished 2023-10-09T05:19:34.216 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:34 smithi133 bash[105589]: cluster 2023-10-09T05:19:33.088834+0000 osd.7 (osd.7) 165 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T05:19:34.216 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:34 smithi133 bash[105589]: cluster 2023-10-09T05:19:33.089761+0000 mon.a (mon.0) 1220 : cluster 0 osdmap e230: 8 total, 8 up, 7 in 2023-10-09T05:19:34.216 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:34 smithi133 bash[105589]: cluster 2023-10-09T05:19:33.091342+0000 osd.7 (osd.7) 166 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T05:19:34.216 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:34 smithi133 bash[105589]: cluster 2023-10-09T05:19:33.091597+0000 osd.4 (osd.4) 186 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T05:19:34.216 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:34 smithi133 bash[105589]: cluster 2023-10-09T05:19:33.093490+0000 osd.4 (osd.4) 187 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T05:19:34.217 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:34 smithi133 bash[105589]: cluster 2023-10-09T05:19:33.096604+0000 osd.4 (osd.4) 188 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T05:19:34.217 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:34 smithi133 bash[105589]: cluster 2023-10-09T05:19:33.153189+0000 osd.5 (osd.5) 217 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T05:19:34.217 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:34 smithi133 bash[105589]: cluster 2023-10-09T05:19:33.502112+0000 osd.6 (osd.6) 170 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T05:19:34.217 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:34 smithi133 bash[105589]: cluster 2023-10-09T05:19:33.503445+0000 osd.7 (osd.7) 167 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T05:19:34.217 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:34 smithi133 bash[105589]: cluster 2023-10-09T05:19:33.504004+0000 mon.a (mon.0) 1221 : cluster 0 osdmap e231: 8 total, 8 up, 7 in 2023-10-09T05:19:34.217 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:34 smithi133 bash[105589]: cluster 2023-10-09T05:19:33.506631+0000 osd.6 (osd.6) 171 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T05:19:34.217 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:34 smithi133 bash[105589]: cluster 2023-10-09T05:19:33.506795+0000 osd.7 (osd.7) 168 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T05:19:34.217 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:34 smithi133 bash[102858]: cluster 2023-10-09T05:19:33.083110+0000 osd.6 (osd.6) 168 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T05:19:34.217 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:34 smithi133 bash[102858]: cluster 2023-10-09T05:19:33.084554+0000 osd.6 (osd.6) 169 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T05:19:34.217 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:34 smithi133 bash[102858]: audit 2023-10-09T05:19:33.086061+0000 mon.a (mon.0) 1219 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'62260','2':'36344','6':'62260'}"}]': finished 2023-10-09T05:19:34.218 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:34 smithi133 bash[102858]: cluster 2023-10-09T05:19:33.088834+0000 osd.7 (osd.7) 165 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T05:19:34.218 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:34 smithi133 bash[102858]: cluster 2023-10-09T05:19:33.089761+0000 mon.a (mon.0) 1220 : cluster 0 osdmap e230: 8 total, 8 up, 7 in 2023-10-09T05:19:34.218 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:34 smithi133 bash[102858]: cluster 2023-10-09T05:19:33.091342+0000 osd.7 (osd.7) 166 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T05:19:34.218 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:34 smithi133 bash[102858]: cluster 2023-10-09T05:19:33.091597+0000 osd.4 (osd.4) 186 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T05:19:34.218 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:34 smithi133 bash[102858]: cluster 2023-10-09T05:19:33.093490+0000 osd.4 (osd.4) 187 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T05:19:34.218 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:34 smithi133 bash[102858]: cluster 2023-10-09T05:19:33.096604+0000 osd.4 (osd.4) 188 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T05:19:34.218 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:34 smithi133 bash[102858]: cluster 2023-10-09T05:19:33.153189+0000 osd.5 (osd.5) 217 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T05:19:34.218 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:34 smithi133 bash[102858]: cluster 2023-10-09T05:19:33.502112+0000 osd.6 (osd.6) 170 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T05:19:34.218 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:34 smithi133 bash[102858]: cluster 2023-10-09T05:19:33.503445+0000 osd.7 (osd.7) 167 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T05:19:34.218 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:34 smithi133 bash[102858]: cluster 2023-10-09T05:19:33.504004+0000 mon.a (mon.0) 1221 : cluster 0 osdmap e231: 8 total, 8 up, 7 in 2023-10-09T05:19:34.219 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:34 smithi133 bash[102858]: cluster 2023-10-09T05:19:33.506631+0000 osd.6 (osd.6) 171 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T05:19:34.219 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:34 smithi133 bash[102858]: cluster 2023-10-09T05:19:33.506795+0000 osd.7 (osd.7) 168 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T05:19:34.228 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:34.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:34.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:34.332 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:34.344 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:34 smithi194 bash[92666]: cluster 2023-10-09T05:19:33.083110+0000 osd.6 (osd.6) 168 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T05:19:34.345 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:34 smithi194 bash[92666]: cluster 2023-10-09T05:19:33.084554+0000 osd.6 (osd.6) 169 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T05:19:34.345 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:34 smithi194 bash[92666]: audit 2023-10-09T05:19:33.086061+0000 mon.a (mon.0) 1219 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'62260','2':'36344','6':'62260'}"}]': finished 2023-10-09T05:19:34.345 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:34 smithi194 bash[92666]: cluster 2023-10-09T05:19:33.088834+0000 osd.7 (osd.7) 165 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T05:19:34.345 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:34 smithi194 bash[92666]: cluster 2023-10-09T05:19:33.089761+0000 mon.a (mon.0) 1220 : cluster 0 osdmap e230: 8 total, 8 up, 7 in 2023-10-09T05:19:34.345 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:34 smithi194 bash[92666]: cluster 2023-10-09T05:19:33.091342+0000 osd.7 (osd.7) 166 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T05:19:34.345 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:34 smithi194 bash[92666]: cluster 2023-10-09T05:19:33.091597+0000 osd.4 (osd.4) 186 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T05:19:34.345 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:34 smithi194 bash[92666]: cluster 2023-10-09T05:19:33.093490+0000 osd.4 (osd.4) 187 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T05:19:34.345 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:34 smithi194 bash[92666]: cluster 2023-10-09T05:19:33.096604+0000 osd.4 (osd.4) 188 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T05:19:34.345 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:34 smithi194 bash[92666]: cluster 2023-10-09T05:19:33.153189+0000 osd.5 (osd.5) 217 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T05:19:34.345 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:34 smithi194 bash[92666]: cluster 2023-10-09T05:19:33.502112+0000 osd.6 (osd.6) 170 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T05:19:34.346 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:34 smithi194 bash[92666]: cluster 2023-10-09T05:19:33.503445+0000 osd.7 (osd.7) 167 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T05:19:34.346 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:34 smithi194 bash[92666]: cluster 2023-10-09T05:19:33.504004+0000 mon.a (mon.0) 1221 : cluster 0 osdmap e231: 8 total, 8 up, 7 in 2023-10-09T05:19:34.346 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:34 smithi194 bash[92666]: cluster 2023-10-09T05:19:33.506631+0000 osd.6 (osd.6) 171 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T05:19:34.346 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:34 smithi194 bash[92666]: cluster 2023-10-09T05:19:33.506795+0000 osd.7 (osd.7) 168 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T05:19:34.363 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:34.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:34.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:34.466 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:34.477 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:19:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:19:34] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.43.0" 2023-10-09T05:19:34.496 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:34.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:34.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:34.599 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:34.633 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:34.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:34.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:34.736 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:34.766 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:34.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:34.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:34.871 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:34.895 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:34.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:34.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:34.920 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 32 16 51025 51009 99.614 17.0625 0.013508 0.0100328 2023-10-09T05:19:35.000 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:35.026 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:35.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:35.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:35.132 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:35.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:35 smithi194 bash[92666]: cluster 2023-10-09T05:19:33.503289+0000 osd.4 (osd.4) 189 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T05:19:35.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:35 smithi194 bash[92666]: cluster 2023-10-09T05:19:33.506467+0000 osd.4 (osd.4) 190 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T05:19:35.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:35 smithi194 bash[92666]: cluster 2023-10-09T05:19:33.998895+0000 mgr.y (mgr.25277) 935 : cluster 0 pgmap v972: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T05:19:35.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:35 smithi194 bash[92666]: cluster 2023-10-09T05:19:34.519457+0000 osd.7 (osd.7) 169 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T05:19:35.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:35 smithi194 bash[92666]: cluster 2023-10-09T05:19:34.519582+0000 osd.5 (osd.5) 218 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T05:19:35.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:35 smithi194 bash[92666]: cluster 2023-10-09T05:19:34.521046+0000 mon.a (mon.0) 1222 : cluster 0 osdmap e232: 8 total, 8 up, 7 in 2023-10-09T05:19:35.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:35 smithi194 bash[92666]: cluster 2023-10-09T05:19:34.523838+0000 osd.7 (osd.7) 170 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T05:19:35.164 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:35.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:35.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:35.269 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:35.277 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:35 smithi133 bash[102858]: cluster 2023-10-09T05:19:33.503289+0000 osd.4 (osd.4) 189 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T05:19:35.277 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:35 smithi133 bash[102858]: cluster 2023-10-09T05:19:33.506467+0000 osd.4 (osd.4) 190 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T05:19:35.277 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:35 smithi133 bash[102858]: cluster 2023-10-09T05:19:33.998895+0000 mgr.y (mgr.25277) 935 : cluster 0 pgmap v972: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T05:19:35.277 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:35 smithi133 bash[102858]: cluster 2023-10-09T05:19:34.519457+0000 osd.7 (osd.7) 169 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T05:19:35.277 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:35 smithi133 bash[102858]: cluster 2023-10-09T05:19:34.519582+0000 osd.5 (osd.5) 218 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T05:19:35.277 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:35 smithi133 bash[102858]: cluster 2023-10-09T05:19:34.521046+0000 mon.a (mon.0) 1222 : cluster 0 osdmap e232: 8 total, 8 up, 7 in 2023-10-09T05:19:35.278 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:35 smithi133 bash[102858]: cluster 2023-10-09T05:19:34.523838+0000 osd.7 (osd.7) 170 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T05:19:35.278 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:35 smithi133 bash[105589]: cluster 2023-10-09T05:19:33.503289+0000 osd.4 (osd.4) 189 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T05:19:35.278 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:35 smithi133 bash[105589]: cluster 2023-10-09T05:19:33.506467+0000 osd.4 (osd.4) 190 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T05:19:35.278 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:35 smithi133 bash[105589]: cluster 2023-10-09T05:19:33.998895+0000 mgr.y (mgr.25277) 935 : cluster 0 pgmap v972: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T05:19:35.278 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:35 smithi133 bash[105589]: cluster 2023-10-09T05:19:34.519457+0000 osd.7 (osd.7) 169 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T05:19:35.278 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:35 smithi133 bash[105589]: cluster 2023-10-09T05:19:34.519582+0000 osd.5 (osd.5) 218 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T05:19:35.279 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:35 smithi133 bash[105589]: cluster 2023-10-09T05:19:34.521046+0000 mon.a (mon.0) 1222 : cluster 0 osdmap e232: 8 total, 8 up, 7 in 2023-10-09T05:19:35.279 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:35 smithi133 bash[105589]: cluster 2023-10-09T05:19:34.523838+0000 osd.7 (osd.7) 170 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T05:19:35.298 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:35.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:35.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:35.403 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:35.430 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:35.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:35.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:35.533 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:35.557 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:35.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:35.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:35.660 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:35.689 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:35.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:35.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:35.795 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:35.827 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:35.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:35.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:35.920 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 33 16 52069 52053 98.5723 65.25 0.0117505 0.010121 2023-10-09T05:19:35.932 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:35.955 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:35.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:35.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:36.058 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:36.083 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:36.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:36.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:36.187 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:36.222 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:36.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:36.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:36.328 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:36.336 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:36 smithi133 bash[102858]: cluster 2023-10-09T05:19:34.519135+0000 osd.4 (osd.4) 191 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T05:19:36.336 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:36 smithi133 bash[105589]: cluster 2023-10-09T05:19:34.519135+0000 osd.4 (osd.4) 191 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T05:19:36.361 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:36.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:36.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:36.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:36 smithi194 bash[92666]: cluster 2023-10-09T05:19:34.519135+0000 osd.4 (osd.4) 191 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T05:19:36.465 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:36.489 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:36.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:36.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:36.598 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:36.625 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:36.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:36.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:36.728 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:36.756 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:36.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:36.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:36.859 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:36.885 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:36.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:36.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:36.920 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 34 16 53096 53080 97.5608 64.1875 0.0101446 0.0102433 2023-10-09T05:19:36.989 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:37.020 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:37.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:37.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:37.127 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:37.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:37 smithi194 bash[92666]: cluster 2023-10-09T05:19:35.999597+0000 mgr.y (mgr.25277) 936 : cluster 0 pgmap v974: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 3131/152790 objects misplaced (2.049%); 11 KiB/s, 0 objects/s recovering 2023-10-09T05:19:37.158 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:37.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:37.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:37.266 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:37.292 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:37.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:37.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:37.397 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:37.429 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:37.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:37.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:37.534 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:37.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:37 smithi133 bash[102858]: cluster 2023-10-09T05:19:35.999597+0000 mgr.y (mgr.25277) 936 : cluster 0 pgmap v974: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 3131/152790 objects misplaced (2.049%); 11 KiB/s, 0 objects/s recovering 2023-10-09T05:19:37.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:37 smithi133 bash[105589]: cluster 2023-10-09T05:19:35.999597+0000 mgr.y (mgr.25277) 936 : cluster 0 pgmap v974: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 3131/152790 objects misplaced (2.049%); 11 KiB/s, 0 objects/s recovering 2023-10-09T05:19:37.561 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:37.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:37.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:37.666 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:37.692 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:37.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:37.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:37.797 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:37.827 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:37.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:37.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:37.920 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 35 16 54173 54157 96.6964 67.3125 0.0142517 0.0103361 2023-10-09T05:19:37.930 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:37.966 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:37.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:37.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:38.071 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:38.106 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:38.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:38.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:38.210 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:38.233 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:38.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:38.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:38.339 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:38.361 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:38.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:38.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:38.420 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-10-09T05:19:38.421 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 3.0 2023-10-09T05:19:38.465 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:38.489 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:38.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:38.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:38.593 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:38.615 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:38.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:38.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:38.721 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:38.745 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:38.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:38.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:38.849 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:38.876 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:38.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:38.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:38.920 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 36 16 55248 55232 95.8765 67.1875 0.00901336 0.0104236 2023-10-09T05:19:38.981 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:39.012 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:39.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:39.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:39.119 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:39.151 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:39.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:39.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:39.256 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:39.282 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:39.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:39.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:39.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:39 smithi194 bash[92666]: cluster 2023-10-09T05:19:38.000107+0000 mgr.y (mgr.25277) 937 : cluster 0 pgmap v975: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 58 MiB/s wr, 924 op/s; 3131/156378 objects misplaced (2.002%); 11 KiB/s, 0 objects/s recovering 2023-10-09T05:19:39.387 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:39.411 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:39.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:39.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:39.515 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:39.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:39 smithi133 bash[105589]: cluster 2023-10-09T05:19:38.000107+0000 mgr.y (mgr.25277) 937 : cluster 0 pgmap v975: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 58 MiB/s wr, 924 op/s; 3131/156378 objects misplaced (2.002%); 11 KiB/s, 0 objects/s recovering 2023-10-09T05:19:39.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:39 smithi133 bash[102858]: cluster 2023-10-09T05:19:38.000107+0000 mgr.y (mgr.25277) 937 : cluster 0 pgmap v975: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 58 MiB/s wr, 924 op/s; 3131/156378 objects misplaced (2.002%); 11 KiB/s, 0 objects/s recovering 2023-10-09T05:19:39.550 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:39.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:39.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:39.655 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:39.678 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:39.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:39.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:39.783 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:39.813 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:39.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:39.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:39.918 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:39.921 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 37 16 56412 56396 95.2513 72.75 0.0112917 0.0104931 2023-10-09T05:19:39.941 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:39.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:39.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:40.049 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:40.079 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:40.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:40.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:40.182 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:40.214 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:40.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:40.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:40.317 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:40.355 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:40.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:40.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:40.459 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:40.487 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:40.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:40.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:40.591 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:40.617 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:40.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:40.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:40.721 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:40.741 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:40.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:40.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:40.844 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:40.866 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:40.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:40.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:40.920 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 38 16 57429 57413 94.4173 63.5625 0.0123892 0.0105852 2023-10-09T05:19:40.976 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:41.002 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:41.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:41.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:41.106 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:41.161 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:41.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:41.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:41.266 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:41.291 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:41.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:41.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:41.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:41 smithi194 bash[92666]: cluster 2023-10-09T05:19:40.000779+0000 mgr.y (mgr.25277) 938 : cluster 0 pgmap v976: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 52 MiB/s wr, 835 op/s; 2919/161499 objects misplaced (1.807%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:19:41.396 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:41.408 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:41 smithi133 bash[105589]: cluster 2023-10-09T05:19:40.000779+0000 mgr.y (mgr.25277) 938 : cluster 0 pgmap v976: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 52 MiB/s wr, 835 op/s; 2919/161499 objects misplaced (1.807%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:19:41.408 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:41 smithi133 bash[102858]: cluster 2023-10-09T05:19:40.000779+0000 mgr.y (mgr.25277) 938 : cluster 0 pgmap v976: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 52 MiB/s wr, 835 op/s; 2919/161499 objects misplaced (1.807%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:19:41.439 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:41.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:41.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:41.544 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:41.573 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:41.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:41.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:41.678 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:41.718 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:41.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:41.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:41.823 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:41.858 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:41.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:41.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:41.920 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 39 16 58389 58373 93.5347 60 0.00740013 0.0106845 2023-10-09T05:19:41.962 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:42.001 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:42.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:42.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:42.111 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:42.134 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:42.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:42.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:42.238 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:42.267 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:42.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:42.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:42.364 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:19:42.374 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:42.401 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:42.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:42.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:42.418 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:19:42.455 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:19:42.506 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:19:42.508 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:42.534 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:42.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:42.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:42.548 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:19:42.588 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:19:42.624 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:19:42.641 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:42.656 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:19:42.660 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:19:42.660 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:19:42.660 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:19:42.660 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:19:42.660 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:19:42.660 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:19:42.660 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:19:42.660 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:19:42.664 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:42.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:42.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:42.766 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:42.792 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:42.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:42.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:42.898 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:42.921 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T05:19:42.921375+0000 min lat: 0.0015063 max lat: 1.53414 avg lat: 0.0107718 2023-10-09T05:19:42.921 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:19:42.921 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 40 16 59403 59387 92.7795 63.375 0.0275833 0.0107718 2023-10-09T05:19:42.927 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:42.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:42.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:43.032 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:43.054 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:43.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:43.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:43.109 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:43 smithi194 bash[92666]: cluster 2023-10-09T05:19:42.001481+0000 mgr.y (mgr.25277) 939 : cluster 0 pgmap v977: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 60 MiB/s wr, 955 op/s; 2919/168534 objects misplaced (1.732%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T05:19:43.110 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:43 smithi194 bash[92666]: audit 2023-10-09T05:19:42.205919+0000 mon.a (mon.0) 1223 : audit 1 from='client.? 172.21.15.133:0/2435078754' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:19:43.110 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:43 smithi194 bash[92666]: audit 2023-10-09T05:19:42.300273+0000 mon.a (mon.0) 1224 : audit 0 from='client.? 172.21.15.133:0/2419428848' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:19:43.128 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is set 2023-10-09T05:19:43.157 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:43.184 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:43.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:43.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:43.288 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:43.316 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:43.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:43.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:43.424 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:43.451 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:43.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:43.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:43.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:43 smithi133 bash[102858]: cluster 2023-10-09T05:19:42.001481+0000 mgr.y (mgr.25277) 939 : cluster 0 pgmap v977: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 60 MiB/s wr, 955 op/s; 2919/168534 objects misplaced (1.732%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T05:19:43.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:43 smithi133 bash[102858]: audit 2023-10-09T05:19:42.205919+0000 mon.a (mon.0) 1223 : audit 1 from='client.? 172.21.15.133:0/2435078754' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:19:43.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:43 smithi133 bash[102858]: audit 2023-10-09T05:19:42.300273+0000 mon.a (mon.0) 1224 : audit 0 from='client.? 172.21.15.133:0/2419428848' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:19:43.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:43 smithi133 bash[105589]: cluster 2023-10-09T05:19:42.001481+0000 mgr.y (mgr.25277) 939 : cluster 0 pgmap v977: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 60 MiB/s wr, 955 op/s; 2919/168534 objects misplaced (1.732%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T05:19:43.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:43 smithi133 bash[105589]: audit 2023-10-09T05:19:42.205919+0000 mon.a (mon.0) 1223 : audit 1 from='client.? 172.21.15.133:0/2435078754' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:19:43.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:43 smithi133 bash[105589]: audit 2023-10-09T05:19:42.300273+0000 mon.a (mon.0) 1224 : audit 0 from='client.? 172.21.15.133:0/2419428848' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:19:43.556 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:43.584 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:43.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:43.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:43.687 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:43.710 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:43.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:43.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:43.816 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:43.842 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:43.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:43.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:43.921 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 41 16 60394 60378 92.0271 61.9375 0.0139542 0.0108411 2023-10-09T05:19:43.947 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:43.977 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:43.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:43.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:44.080 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:44.107 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:44.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:44.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:44.113 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:44 smithi194 bash[92666]: cluster 2023-10-09T05:19:43.107045+0000 mon.a (mon.0) 1225 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:19:44.113 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:44 smithi194 bash[92666]: cluster 2023-10-09T05:19:43.116550+0000 osd.5 (osd.5) 219 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T05:19:44.113 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:44 smithi194 bash[92666]: audit 2023-10-09T05:19:43.118375+0000 mon.a (mon.0) 1226 : audit 1 from='client.? 172.21.15.133:0/2435078754' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:19:44.210 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:44.248 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:44.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:44.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:44.352 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:44.363 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:44 smithi194 bash[92666]: cluster 2023-10-09T05:19:43.118429+0000 osd.6 (osd.6) 172 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T05:19:44.364 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:44 smithi194 bash[92666]: cluster 2023-10-09T05:19:43.118449+0000 mon.a (mon.0) 1227 : cluster 0 osdmap e233: 8 total, 8 up, 7 in 2023-10-09T05:19:44.364 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:44 smithi194 bash[92666]: cluster 2023-10-09T05:19:43.125132+0000 osd.7 (osd.7) 171 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T05:19:44.364 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:44 smithi194 bash[92666]: cluster 2023-10-09T05:19:43.125445+0000 osd.4 (osd.4) 192 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T05:19:44.364 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:44 smithi194 bash[92666]: cluster 2023-10-09T05:19:43.126420+0000 osd.7 (osd.7) 172 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T05:19:44.364 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:44 smithi194 bash[92666]: cluster 2023-10-09T05:19:43.126651+0000 osd.4 (osd.4) 193 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T05:19:44.364 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:44 smithi194 bash[92666]: audit 2023-10-09T05:19:43.459798+0000 mon.a (mon.0) 1228 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:19:44.364 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:44 smithi194 bash[92666]: audit 2023-10-09T05:19:43.461253+0000 mon.a (mon.0) 1229 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:19:44.371 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:44 smithi133 bash[102858]: cluster 2023-10-09T05:19:43.107045+0000 mon.a (mon.0) 1225 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:19:44.371 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:44 smithi133 bash[102858]: cluster 2023-10-09T05:19:43.116550+0000 osd.5 (osd.5) 219 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T05:19:44.372 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:44 smithi133 bash[102858]: audit 2023-10-09T05:19:43.118375+0000 mon.a (mon.0) 1226 : audit 1 from='client.? 172.21.15.133:0/2435078754' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:19:44.372 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:44 smithi133 bash[102858]: cluster 2023-10-09T05:19:43.118429+0000 osd.6 (osd.6) 172 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T05:19:44.372 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:44 smithi133 bash[102858]: cluster 2023-10-09T05:19:43.118449+0000 mon.a (mon.0) 1227 : cluster 0 osdmap e233: 8 total, 8 up, 7 in 2023-10-09T05:19:44.372 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:44 smithi133 bash[102858]: cluster 2023-10-09T05:19:43.125132+0000 osd.7 (osd.7) 171 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T05:19:44.372 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:44 smithi133 bash[102858]: cluster 2023-10-09T05:19:43.125445+0000 osd.4 (osd.4) 192 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T05:19:44.372 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:44 smithi133 bash[102858]: cluster 2023-10-09T05:19:43.126420+0000 osd.7 (osd.7) 172 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T05:19:44.372 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:44 smithi133 bash[102858]: cluster 2023-10-09T05:19:43.126651+0000 osd.4 (osd.4) 193 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T05:19:44.372 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:44 smithi133 bash[102858]: audit 2023-10-09T05:19:43.459798+0000 mon.a (mon.0) 1228 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:19:44.372 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:44 smithi133 bash[102858]: audit 2023-10-09T05:19:43.461253+0000 mon.a (mon.0) 1229 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:19:44.372 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:44 smithi133 bash[105589]: cluster 2023-10-09T05:19:43.107045+0000 mon.a (mon.0) 1225 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:19:44.373 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:44 smithi133 bash[105589]: cluster 2023-10-09T05:19:43.116550+0000 osd.5 (osd.5) 219 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T05:19:44.373 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:44 smithi133 bash[105589]: audit 2023-10-09T05:19:43.118375+0000 mon.a (mon.0) 1226 : audit 1 from='client.? 172.21.15.133:0/2435078754' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:19:44.373 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:44 smithi133 bash[105589]: cluster 2023-10-09T05:19:43.118429+0000 osd.6 (osd.6) 172 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T05:19:44.373 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:44 smithi133 bash[105589]: cluster 2023-10-09T05:19:43.118449+0000 mon.a (mon.0) 1227 : cluster 0 osdmap e233: 8 total, 8 up, 7 in 2023-10-09T05:19:44.373 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:44 smithi133 bash[105589]: cluster 2023-10-09T05:19:43.125132+0000 osd.7 (osd.7) 171 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T05:19:44.373 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:44 smithi133 bash[105589]: cluster 2023-10-09T05:19:43.125445+0000 osd.4 (osd.4) 192 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T05:19:44.373 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:44 smithi133 bash[105589]: cluster 2023-10-09T05:19:43.126420+0000 osd.7 (osd.7) 172 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T05:19:44.373 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:44 smithi133 bash[105589]: cluster 2023-10-09T05:19:43.126651+0000 osd.4 (osd.4) 193 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T05:19:44.373 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:44 smithi133 bash[105589]: audit 2023-10-09T05:19:43.459798+0000 mon.a (mon.0) 1228 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:19:44.373 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:44 smithi133 bash[105589]: audit 2023-10-09T05:19:43.461253+0000 mon.a (mon.0) 1229 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:19:44.378 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:44.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:44.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:44.435 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:19:44.435 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:19:44.482 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:44.513 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:44.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:44.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:44.619 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:44.628 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:19:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:19:44] "GET /metrics HTTP/1.1" 200 36529 "" "Prometheus/2.43.0" 2023-10-09T05:19:44.644 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:44.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:44.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:44.748 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:44.781 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:44.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:44.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:44.885 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:44.919 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:44.921 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 42 16 61303 61287 91.1886 56.8125 0.0154704 0.0109588 2023-10-09T05:19:44.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:44.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:45.024 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:45.050 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:45.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:45.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:45.155 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:45.182 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:45.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:45.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:45.285 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:45.292 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:45 smithi133 bash[102858]: cluster 2023-10-09T05:19:44.001915+0000 mgr.y (mgr.25277) 940 : cluster 0 pgmap v979: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 62 MiB/s wr, 994 op/s; 2919/172461 objects misplaced (1.693%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T05:19:45.292 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:45 smithi133 bash[105589]: cluster 2023-10-09T05:19:44.001915+0000 mgr.y (mgr.25277) 940 : cluster 0 pgmap v979: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 62 MiB/s wr, 994 op/s; 2919/172461 objects misplaced (1.693%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T05:19:45.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:45.318 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:45.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:45.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:45 smithi194 bash[92666]: cluster 2023-10-09T05:19:44.001915+0000 mgr.y (mgr.25277) 940 : cluster 0 pgmap v979: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 62 MiB/s wr, 994 op/s; 2919/172461 objects misplaced (1.693%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T05:19:45.419 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:45.449 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:45.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:45.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:45.552 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:45.581 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:45.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:45.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:45.685 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:45.722 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:45.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:45.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:45.830 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:45.859 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:45.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:45.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:45.921 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 43 15 62178 62163 90.3411 54.75 0.0198067 0.0110637 2023-10-09T05:19:45.966 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:45.994 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:45.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:45.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:46.100 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:46.137 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:46.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:46.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:46.241 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:46.274 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:46.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:46.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:46.380 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:46.404 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:46.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:46.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:46.513 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:46.542 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:46.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:46.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:46.649 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:46.678 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:46.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:46.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:46.781 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:46.812 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:46.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:46.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:46.915 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:46.921 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 44 16 63165 63149 89.6884 61.625 0.0230046 0.0111442 2023-10-09T05:19:46.943 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:46.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:46.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:47.045 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:47.076 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:47.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:47.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:47.179 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:47.205 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:47.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:47.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:47.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:47 smithi133 bash[102858]: cluster 2023-10-09T05:19:46.002973+0000 mgr.y (mgr.25277) 941 : cluster 0 pgmap v980: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 62 MiB/s wr, 988 op/s; 2687/182466 objects misplaced (1.473%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T05:19:47.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:47 smithi133 bash[105589]: cluster 2023-10-09T05:19:46.002973+0000 mgr.y (mgr.25277) 941 : cluster 0 pgmap v980: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 62 MiB/s wr, 988 op/s; 2687/182466 objects misplaced (1.473%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T05:19:47.309 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:47.340 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T05:19:47.342 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:47.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:47.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:47.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:47 smithi194 bash[92666]: cluster 2023-10-09T05:19:46.002973+0000 mgr.y (mgr.25277) 941 : cluster 0 pgmap v980: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 62 MiB/s wr, 988 op/s; 2687/182466 objects misplaced (1.473%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T05:19:47.445 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:47.474 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:47.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:47.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:47.576 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:47.600 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:47.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:47.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:47.706 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:47.743 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:47.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:47.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:47.803 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:19:47.849 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:47.889 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:47.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:47.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:47.905 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:19:47.922 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 45 16 64159 64143 89.0754 62.125 0.00940979 0.0112201 2023-10-09T05:19:47.995 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:48.038 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:48.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:48.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:48.146 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:48.174 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:48.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:48.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:48.281 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:48.311 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:48.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:48.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:48.418 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:48.449 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:48.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:48.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:48.553 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:48.589 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:48.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:48.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:48.693 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:48.722 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:48.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:48.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:48.827 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:48.860 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:48.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:48.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:48.922 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 46 16 65109 65093 88.4297 59.375 0.00693763 0.0113032 2023-10-09T05:19:48.964 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:49.111 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:49.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:49.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:49.131 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:19:49.216 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:49.239 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:49.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:49.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:49.345 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:49.375 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:49.376 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:49 smithi194 bash[92666]: cluster 2023-10-09T05:19:48.003625+0000 mgr.y (mgr.25277) 942 : cluster 0 pgmap v981: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 56 MiB/s wr, 898 op/s; 2687/183333 objects misplaced (1.466%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T05:19:49.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:49.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:49.478 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:49.518 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:49.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:49.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:49.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:49 smithi133 bash[105589]: cluster 2023-10-09T05:19:48.003625+0000 mgr.y (mgr.25277) 942 : cluster 0 pgmap v981: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 56 MiB/s wr, 898 op/s; 2687/183333 objects misplaced (1.466%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T05:19:49.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:49 smithi133 bash[102858]: cluster 2023-10-09T05:19:48.003625+0000 mgr.y (mgr.25277) 942 : cluster 0 pgmap v981: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 56 MiB/s wr, 898 op/s; 2687/183333 objects misplaced (1.466%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T05:19:49.624 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:49.650 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:49.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:49.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:49.756 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:49.782 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:49.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:49.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:49.887 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:49.912 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:49.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:49.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:49.922 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 47 16 66003 65987 87.7369 55.875 0.0138089 0.0113917 2023-10-09T05:19:50.016 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:50.044 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:50.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:50.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:50.147 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:50.155 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:50 smithi194 bash[92666]: audit 2023-10-09T05:19:49.336446+0000 mon.a (mon.0) 1230 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:19:50.183 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:50.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:50.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:50.287 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:50.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:50 smithi133 bash[102858]: audit 2023-10-09T05:19:49.336446+0000 mon.a (mon.0) 1230 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:19:50.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:50 smithi133 bash[105589]: audit 2023-10-09T05:19:49.336446+0000 mon.a (mon.0) 1230 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:19:50.316 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:50.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:50.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:50.419 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:50.447 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:50.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:50.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:50.553 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:50.583 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:50.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:50.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:50.689 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:50.718 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:50.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:50.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:50.822 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:50.848 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:50.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:50.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:50.922 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 48 16 67018 67002 87.2306 63.4375 0.00393223 0.0114568 2023-10-09T05:19:50.951 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:50.977 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:50.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:50.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:51.084 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:51.111 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:51.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:51.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:51.216 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:51.243 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:51.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:51.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:51.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:51 smithi133 bash[102858]: cluster 2023-10-09T05:19:50.004303+0000 mgr.y (mgr.25277) 943 : cluster 0 pgmap v982: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 62 MiB/s wr, 995 op/s; 2455/191361 objects misplaced (1.283%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:19:51.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:51 smithi133 bash[105589]: cluster 2023-10-09T05:19:50.004303+0000 mgr.y (mgr.25277) 943 : cluster 0 pgmap v982: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 62 MiB/s wr, 995 op/s; 2455/191361 objects misplaced (1.283%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:19:51.349 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:51.382 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:51.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:51 smithi194 bash[92666]: cluster 2023-10-09T05:19:50.004303+0000 mgr.y (mgr.25277) 943 : cluster 0 pgmap v982: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 62 MiB/s wr, 995 op/s; 2455/191361 objects misplaced (1.283%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:19:51.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:51.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:51.488 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:51.515 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:51.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:51.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:51.619 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:51.645 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:51.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:51.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:51.688 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:19:51.749 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:51.786 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:51.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:51.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:51.892 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:51.917 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:51.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:51.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:51.922 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 49 16 67967 67951 86.6607 59.3125 0.0162355 0.0115329 2023-10-09T05:19:52.023 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:52.057 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:52.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:52.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:52.164 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:52.200 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:52.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:52.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:52.305 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:52.335 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:52.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:52.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:52.440 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:52.465 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:52.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:52.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:52.570 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:52.597 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:52.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:52.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:52.700 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:52.725 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:52.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:52.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:52.831 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:52.860 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:52.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:52.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:52.922 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 50 16 68973 68957 86.1849 62.875 0.00727303 0.0115963 2023-10-09T05:19:52.965 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:52.991 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:52.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:53.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:53.101 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:53.129 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:53.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:53.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:53.237 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:53.247 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:53 smithi194 bash[92666]: cluster 2023-10-09T05:19:52.004949+0000 mgr.y (mgr.25277) 944 : cluster 0 pgmap v983: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 60 MiB/s wr, 965 op/s; 2455/197514 objects misplaced (1.243%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:19:53.271 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:53.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:53.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:53.377 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:53.412 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:53.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:53.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:53.516 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:53.526 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:53 smithi133 bash[102858]: cluster 2023-10-09T05:19:52.004949+0000 mgr.y (mgr.25277) 944 : cluster 0 pgmap v983: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 60 MiB/s wr, 965 op/s; 2455/197514 objects misplaced (1.243%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:19:53.526 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:53 smithi133 bash[105589]: cluster 2023-10-09T05:19:52.004949+0000 mgr.y (mgr.25277) 944 : cluster 0 pgmap v983: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 60 MiB/s wr, 965 op/s; 2455/197514 objects misplaced (1.243%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:19:53.552 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:53.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:53.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:53.655 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:53.682 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:53.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:53.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:53.723 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:19:53.723 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:19:53.789 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:53.815 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:53.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:53.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:53.922 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:53.923 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 51 16 69926 69910 85.6623 59.5625 0.00564691 0.0116671 2023-10-09T05:19:53.951 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:53.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:53.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:54.056 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:54.089 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:54.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:54.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:54.194 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:54.220 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:54.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:54.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:54.324 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:54.335 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:19:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:19:54] "GET /metrics HTTP/1.1" 200 36529 "" "Prometheus/2.43.0" 2023-10-09T05:19:54.335 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:19:54 smithi133 bash[91128]: debug 2023-10-09T05:19:54.329+0000 7ff45d5ed700 -1 mgr.server reply reply (3) No such process no change 2023-10-09T05:19:54.335 INFO:teuthology.orchestra.run.smithi133.stdout:moved 22 / 267 (8.2397%) 2023-10-09T05:19:54.336 INFO:teuthology.orchestra.run.smithi133.stdout:avg 38.1429 2023-10-09T05:19:54.336 INFO:teuthology.orchestra.run.smithi133.stdout:stddev 9.78962 -> 5.22162 (expected baseline 5.71786) 2023-10-09T05:19:54.336 INFO:teuthology.orchestra.run.smithi133.stdout:min osd.2 with 15 -> 32 pgs (0.393258 -> 0.838951 * mean) 2023-10-09T05:19:54.336 INFO:teuthology.orchestra.run.smithi133.stdout:max osd.1 with 46 -> 31 pgs (1.20599 -> 0.812734 * mean) 2023-10-09T05:19:54.336 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:19:54.336 INFO:teuthology.orchestra.run.smithi133.stdout:oload 110 2023-10-09T05:19:54.336 INFO:teuthology.orchestra.run.smithi133.stdout:max_change 3 2023-10-09T05:19:54.336 INFO:teuthology.orchestra.run.smithi133.stdout:max_change_osds 4 2023-10-09T05:19:54.336 INFO:teuthology.orchestra.run.smithi133.stdout:average_utilization 0.0279 2023-10-09T05:19:54.336 INFO:teuthology.orchestra.run.smithi133.stdout:overload_utilization 0.0307 2023-10-09T05:19:54.336 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2 weight 0.5546 -> 1.0000 2023-10-09T05:19:54.337 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1 weight 0.9500 -> 0.6837 2023-10-09T05:19:54.337 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6 weight 0.9500 -> 0.8271 2023-10-09T05:19:54.344 INFO:teuthology.orchestra.run.smithi133.stderr:no change 2023-10-09T05:19:54.355 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:54.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:54.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:54.461 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:54.491 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:54.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:54.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:54.596 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:54.628 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:54.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:54.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:54.733 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:54.752 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:54 smithi133 bash[102858]: cluster 2023-10-09T05:19:54.005501+0000 mgr.y (mgr.25277) 945 : cluster 0 pgmap v984: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 55 MiB/s wr, 883 op/s; 2455/201324 objects misplaced (1.219%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T05:19:54.752 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:54 smithi133 bash[105589]: cluster 2023-10-09T05:19:54.005501+0000 mgr.y (mgr.25277) 945 : cluster 0 pgmap v984: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 55 MiB/s wr, 883 op/s; 2455/201324 objects misplaced (1.219%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T05:19:54.770 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:54.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:54.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:54.873 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:54.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:54 smithi194 bash[92666]: cluster 2023-10-09T05:19:54.005501+0000 mgr.y (mgr.25277) 945 : cluster 0 pgmap v984: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 55 MiB/s wr, 883 op/s; 2455/201324 objects misplaced (1.219%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T05:19:54.899 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:54.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:54.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:54.925 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 52 16 70645 70629 84.8752 44.9375 0.0157247 0.011776 2023-10-09T05:19:55.008 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:55.038 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:55.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:55.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:55.143 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:55.174 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:55.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:55.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:55.277 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:55.309 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:55.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:55.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:55.420 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:55.452 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:55.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:55.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:55.516 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:55 smithi133 bash[102858]: audit 2023-10-09T05:19:54.333974+0000 mgr.y (mgr.25277) 946 : audit 0 from='client.45188 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:19:55.517 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:55 smithi133 bash[102858]: audit 2023-10-09T05:19:55.360855+0000 mon.a (mon.0) 1231 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:19:55.517 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:55 smithi133 bash[102858]: audit 2023-10-09T05:19:55.372205+0000 mon.a (mon.0) 1232 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:19:55.555 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:55.592 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:55.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:55.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:55.695 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:55.705 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:55 smithi133 bash[105589]: audit 2023-10-09T05:19:54.333974+0000 mgr.y (mgr.25277) 946 : audit 0 from='client.45188 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:19:55.705 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:55 smithi133 bash[105589]: audit 2023-10-09T05:19:55.360855+0000 mon.a (mon.0) 1231 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:19:55.705 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:55 smithi133 bash[105589]: audit 2023-10-09T05:19:55.372205+0000 mon.a (mon.0) 1232 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:19:55.719 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:55.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:55.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:55.824 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:55.865 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:55.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:55.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:55.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:55 smithi194 bash[92666]: audit 2023-10-09T05:19:54.333974+0000 mgr.y (mgr.25277) 946 : audit 0 from='client.45188 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:19:55.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:55 smithi194 bash[92666]: audit 2023-10-09T05:19:55.360855+0000 mon.a (mon.0) 1231 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:19:55.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:55 smithi194 bash[92666]: audit 2023-10-09T05:19:55.372205+0000 mon.a (mon.0) 1232 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:19:55.925 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 53 16 71496 71480 84.2773 53.1875 0.00861473 0.0118592 2023-10-09T05:19:55.970 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:55.987 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:55.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:55.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:56.090 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:56.122 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:56.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:56.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:56.229 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:56.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:56.267 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:56.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:56.368 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:56.393 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:56.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:56.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:56.496 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:56.518 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:56.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:56.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:56.623 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:56.653 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:56.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:56.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:56.763 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:56.786 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:56.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:56.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:56.893 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:56.920 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:56.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:56.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:56.925 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 54 16 72394 72378 83.7559 56.125 0.0216656 0.0119335 2023-10-09T05:19:57.024 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:57.047 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:57.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:56 smithi133 bash[102858]: audit 2023-10-09T05:19:55.956927+0000 mon.a (mon.0) 1233 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:19:57.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:56 smithi133 bash[102858]: audit 2023-10-09T05:19:55.961944+0000 mon.a (mon.0) 1234 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:19:57.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:56 smithi133 bash[102858]: cluster 2023-10-09T05:19:56.006286+0000 mgr.y (mgr.25277) 947 : cluster 0 pgmap v985: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 67 MiB/s wr, 1.06k op/s; 2221/210810 objects misplaced (1.054%); 3.6 MiB/s, 58 objects/s recovering 2023-10-09T05:19:57.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:56 smithi133 bash[102858]: audit 2023-10-09T05:19:56.539532+0000 mon.a (mon.0) 1235 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:57.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:56 smithi133 bash[102858]: audit 2023-10-09T05:19:56.540895+0000 mon.a (mon.0) 1236 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:19:57.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:56 smithi133 bash[102858]: audit 2023-10-09T05:19:56.553342+0000 mon.a (mon.0) 1237 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:19:57.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:56 smithi133 bash[105589]: audit 2023-10-09T05:19:55.956927+0000 mon.a (mon.0) 1233 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:19:57.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:56 smithi133 bash[105589]: audit 2023-10-09T05:19:55.961944+0000 mon.a (mon.0) 1234 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:19:57.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:56 smithi133 bash[105589]: cluster 2023-10-09T05:19:56.006286+0000 mgr.y (mgr.25277) 947 : cluster 0 pgmap v985: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 67 MiB/s wr, 1.06k op/s; 2221/210810 objects misplaced (1.054%); 3.6 MiB/s, 58 objects/s recovering 2023-10-09T05:19:57.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:56 smithi133 bash[105589]: audit 2023-10-09T05:19:56.539532+0000 mon.a (mon.0) 1235 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:57.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:56 smithi133 bash[105589]: audit 2023-10-09T05:19:56.540895+0000 mon.a (mon.0) 1236 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:19:57.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:56 smithi133 bash[105589]: audit 2023-10-09T05:19:56.553342+0000 mon.a (mon.0) 1237 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:19:57.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:57.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:57.151 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:57.181 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:57.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:57.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:57.285 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:57.313 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:57.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:57.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:57.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:56 smithi194 bash[92666]: audit 2023-10-09T05:19:55.956927+0000 mon.a (mon.0) 1233 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:19:57.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:56 smithi194 bash[92666]: audit 2023-10-09T05:19:55.961944+0000 mon.a (mon.0) 1234 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:19:57.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:56 smithi194 bash[92666]: cluster 2023-10-09T05:19:56.006286+0000 mgr.y (mgr.25277) 947 : cluster 0 pgmap v985: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 67 MiB/s wr, 1.06k op/s; 2221/210810 objects misplaced (1.054%); 3.6 MiB/s, 58 objects/s recovering 2023-10-09T05:19:57.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:56 smithi194 bash[92666]: audit 2023-10-09T05:19:56.539532+0000 mon.a (mon.0) 1235 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:57.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:56 smithi194 bash[92666]: audit 2023-10-09T05:19:56.540895+0000 mon.a (mon.0) 1236 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:19:57.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:56 smithi194 bash[92666]: audit 2023-10-09T05:19:56.553342+0000 mon.a (mon.0) 1237 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:19:57.419 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:57.449 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:57.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:57.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:57.555 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:57.582 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:57.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:57.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:57.689 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:57.716 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:57.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:57.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:57.819 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:57.845 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:19:57.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:57.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:19:57.925 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 55 16 73346 73330 83.3148 59.5 0.0160333 0.0119974 2023-10-09T05:19:57.950 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:57.988 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:57.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:57.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:58.093 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:58.118 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:58.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:58.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:19:58.221 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:58.249 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:58.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:58.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:58.353 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:58.395 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:58.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:58.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:58.500 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:58.538 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:19:58.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:58.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:19:58.645 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:58.672 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:58.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:58.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:58.748 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-10-09T05:19:58.748 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-10-09T05:19:58.779 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:58.805 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:58.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:58.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:58.908 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:58.927 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 56 16 74188 74172 82.7643 52.625 0.0135016 0.0120752 2023-10-09T05:19:58.932 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:58.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:58.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:59.037 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:59.065 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:19:59.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:59.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:19:59.168 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:59.192 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:59.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:59.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:59.297 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:59.304 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:59 smithi194 bash[92666]: cluster 2023-10-09T05:19:58.006823+0000 mgr.y (mgr.25277) 948 : cluster 0 pgmap v986: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 51 MiB/s wr, 809 op/s; 2221/211635 objects misplaced (1.049%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:19:59.304 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:19:59 smithi194 bash[92666]: audit 2023-10-09T05:19:58.454418+0000 mon.a (mon.0) 1238 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:19:59.320 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:59.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:59.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:59.423 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:59.461 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:19:59.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:59.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:19:59.563 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:59.585 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:19:59.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:59.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:19:59.590 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:59 smithi133 bash[102858]: cluster 2023-10-09T05:19:58.006823+0000 mgr.y (mgr.25277) 948 : cluster 0 pgmap v986: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 51 MiB/s wr, 809 op/s; 2221/211635 objects misplaced (1.049%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:19:59.590 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:19:59 smithi133 bash[102858]: audit 2023-10-09T05:19:58.454418+0000 mon.a (mon.0) 1238 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:19:59.590 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:59 smithi133 bash[105589]: cluster 2023-10-09T05:19:58.006823+0000 mgr.y (mgr.25277) 948 : cluster 0 pgmap v986: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 51 MiB/s wr, 809 op/s; 2221/211635 objects misplaced (1.049%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:19:59.591 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:19:59 smithi133 bash[105589]: audit 2023-10-09T05:19:58.454418+0000 mon.a (mon.0) 1238 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:19:59.691 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:59.722 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:19:59.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:59.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:19:59.828 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:59.849 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:19:59.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:59.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:19:59.927 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 57 16 74934 74918 82.1303 46.625 0.00435388 0.0121516 2023-10-09T05:19:59.952 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:59.985 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:19:59.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:19:59.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:00.089 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:00.113 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:00.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:00.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:00.153 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:00 smithi133 bash[105589]: cluster 2023-10-09T05:20:00.000109+0000 mon.a (mon.0) 1239 : cluster 3 Health detail: HEALTH_WARN noscrub flag(s) set 2023-10-09T05:20:00.153 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:00 smithi133 bash[105589]: cluster 2023-10-09T05:20:00.000136+0000 mon.a (mon.0) 1240 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-09T05:20:00.154 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:00 smithi133 bash[102858]: cluster 2023-10-09T05:20:00.000109+0000 mon.a (mon.0) 1239 : cluster 3 Health detail: HEALTH_WARN noscrub flag(s) set 2023-10-09T05:20:00.154 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:00 smithi133 bash[102858]: cluster 2023-10-09T05:20:00.000136+0000 mon.a (mon.0) 1240 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-09T05:20:00.216 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:00.223 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:00 smithi194 bash[92666]: cluster 2023-10-09T05:20:00.000109+0000 mon.a (mon.0) 1239 : cluster 3 Health detail: HEALTH_WARN noscrub flag(s) set 2023-10-09T05:20:00.223 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:00 smithi194 bash[92666]: cluster 2023-10-09T05:20:00.000136+0000 mon.a (mon.0) 1240 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-09T05:20:00.242 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:00.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:00.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:00.349 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:00.377 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:00.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:00.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:00.483 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:00.509 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:00.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:00.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:00.616 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:00.643 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:00.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:00.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:00.746 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:00.780 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:00.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:00.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:00.883 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:00.914 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:00.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:00.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:00.927 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 58 16 75690 75674 81.5289 47.25 0.011126 0.0122598 2023-10-09T05:20:01.020 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:01.052 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:01.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:01.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:01.156 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:01.165 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:01 smithi194 bash[92666]: cluster 2023-10-09T05:20:00.007439+0000 mgr.y (mgr.25277) 949 : cluster 0 pgmap v987: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 62 MiB/s wr, 989 op/s; 1961/218976 objects misplaced (0.896%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T05:20:01.183 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:01.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:01.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:01.287 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:01.312 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:01.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:01.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:01.341 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:01 smithi133 bash[102858]: cluster 2023-10-09T05:20:00.007439+0000 mgr.y (mgr.25277) 949 : cluster 0 pgmap v987: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 62 MiB/s wr, 989 op/s; 1961/218976 objects misplaced (0.896%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T05:20:01.341 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:01 smithi133 bash[105589]: cluster 2023-10-09T05:20:00.007439+0000 mgr.y (mgr.25277) 949 : cluster 0 pgmap v987: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 62 MiB/s wr, 989 op/s; 1961/218976 objects misplaced (0.896%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T05:20:01.416 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:01.439 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:01.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:01.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:01.545 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:01.571 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:01.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:01.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:01.676 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:01.711 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:01.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:01.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:01.817 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:01.927 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 59 16 76652 76636 81.166 60.125 0.020046 0.0123154 2023-10-09T05:20:02.054 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:02.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:02.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:02.157 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:02.190 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:02.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:02.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:02.300 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:02.338 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:02.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:02.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:02.444 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:02.472 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:02.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:02.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:02.576 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:02.595 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:02.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:02.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:02.697 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:02.728 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:02.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:02.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:02.836 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:02.870 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:02.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:02.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:02.927 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T05:20:02.927746+0000 min lat: 0.0015063 max lat: 1.53414 avg lat: 0.0123888 2023-10-09T05:20:02.928 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:20:02.928 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 60 16 77488 77472 80.6841 52.25 0.0322341 0.0123888 2023-10-09T05:20:02.972 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:03.000 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:03.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:03.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:03.098 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:03 smithi133 bash[102858]: cluster 2023-10-09T05:20:02.008065+0000 mgr.y (mgr.25277) 950 : cluster 0 pgmap v988: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 57 MiB/s wr, 919 op/s; 1961/224472 objects misplaced (0.874%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:20:03.103 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:03.135 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:03.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:03.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:03.240 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:03.274 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:03.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:03.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:03.380 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:03.389 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:03 smithi194 bash[92666]: cluster 2023-10-09T05:20:02.008065+0000 mgr.y (mgr.25277) 950 : cluster 0 pgmap v988: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 57 MiB/s wr, 919 op/s; 1961/224472 objects misplaced (0.874%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:20:03.407 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:03.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:03.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:03.511 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:03.536 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:03.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:03.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:03.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:03 smithi133 bash[105589]: cluster 2023-10-09T05:20:02.008065+0000 mgr.y (mgr.25277) 950 : cluster 0 pgmap v988: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 57 MiB/s wr, 919 op/s; 1961/224472 objects misplaced (0.874%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:20:03.640 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:03.668 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:03.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:03.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:03.770 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:03.806 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:03.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:03.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:03.911 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:03.929 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 61 16 78433 78417 80.3274 59.0625 0.0244548 0.0124432 2023-10-09T05:20:03.966 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:03.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:03.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:04.072 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:04.102 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:04.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:04.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:04.208 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:04.243 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:04.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:04.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:04.347 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:04.371 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:04.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:04.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:04.473 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:20:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:20:04] "GET /metrics HTTP/1.1" 200 36535 "" "Prometheus/2.43.0" 2023-10-09T05:20:04.477 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:04.501 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:04.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:04.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:04.577 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:20:04.577 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:20:04.609 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:04.617 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:04 smithi194 bash[92666]: cluster 2023-10-09T05:20:04.008625+0000 mgr.y (mgr.25277) 951 : cluster 0 pgmap v989: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 52 MiB/s wr, 834 op/s; 1961/227571 objects misplaced (0.862%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:20:04.633 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:04.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:04.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:04.737 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:04.769 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:04.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:04.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:04.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:04 smithi133 bash[105589]: cluster 2023-10-09T05:20:04.008625+0000 mgr.y (mgr.25277) 951 : cluster 0 pgmap v989: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 52 MiB/s wr, 834 op/s; 1961/227571 objects misplaced (0.862%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:20:04.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:04 smithi133 bash[102858]: cluster 2023-10-09T05:20:04.008625+0000 mgr.y (mgr.25277) 951 : cluster 0 pgmap v989: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 52 MiB/s wr, 834 op/s; 1961/227571 objects misplaced (0.862%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:20:04.872 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:04.899 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:04.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:04.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:04.929 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 62 16 79340 79324 79.946 56.6875 0.0280123 0.0125029 2023-10-09T05:20:05.006 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:05.030 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:05.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:05.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:05.135 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:05.160 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:05.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:05.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:05.264 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:05.289 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:05.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:05.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:05.394 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:05.425 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:05.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:05.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:05.528 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:05.552 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:05.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:05.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:05.656 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:05.687 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:05.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:05.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:05.793 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:05.823 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:05.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:05.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:05.929 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 63 16 80205 80189 79.5352 54.0625 0.00396234 0.0125607 2023-10-09T05:20:05.934 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:05.969 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:05.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:05.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:06.073 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:06.096 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:06.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:06.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:06.199 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:06.231 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:06.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:06.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:06.337 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:06.360 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:06.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:06.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:06.463 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:06.494 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:06.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:06.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:06.597 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:06.624 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:06.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:06.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:06.727 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:06.756 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:06.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:06.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:06.862 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:06.897 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:06.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:06.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:06.929 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 64 16 81030 81014 79.0981 51.5625 0.00842491 0.0126361 2023-10-09T05:20:07.004 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:07.028 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:07.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:07.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:07.132 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:07.141 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:07 smithi133 bash[102858]: cluster 2023-10-09T05:20:06.009185+0000 mgr.y (mgr.25277) 952 : cluster 0 pgmap v990: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 61 MiB/s wr, 983 op/s; 1655/236727 objects misplaced (0.699%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T05:20:07.141 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:07 smithi133 bash[105589]: cluster 2023-10-09T05:20:06.009185+0000 mgr.y (mgr.25277) 952 : cluster 0 pgmap v990: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 61 MiB/s wr, 983 op/s; 1655/236727 objects misplaced (0.699%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T05:20:07.168 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:07.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:07.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:07.274 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:07.313 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:07.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:07.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:07.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:07 smithi194 bash[92666]: cluster 2023-10-09T05:20:06.009185+0000 mgr.y (mgr.25277) 952 : cluster 0 pgmap v990: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 61 MiB/s wr, 983 op/s; 1655/236727 objects misplaced (0.699%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T05:20:07.421 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:07.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:07.458 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:07.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:07.490 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T05:20:07.559 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:07.595 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:07.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:07.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:07.703 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:07.736 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:07.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:07.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:07.840 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:07.890 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:07.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:07.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:07.929 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 65 16 81928 81912 78.7446 56.125 0.0140032 0.0126931 2023-10-09T05:20:07.945 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:20:07.993 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:08.022 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:08.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:08.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:08.126 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:08.161 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:08.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:08.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:08.200 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:20:08.253 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:20:08.272 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:08.293 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:20:08.300 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:08.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:08.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:08.346 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:20:08.392 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:20:08.405 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:08.435 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:20:08.439 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:08.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:08.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:08.492 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:20:08.536 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:20:08.539 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:20:08.539 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:20:08.539 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:20:08.539 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:20:08.539 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:20:08.539 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:20:08.539 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:20:08.539 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:20:08.542 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:08.568 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:08.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:08.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:08.672 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:08.697 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:08.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:08.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:08.800 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:08.830 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:08.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:08.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:08.930 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 66 16 82720 82704 78.3015 49.5 0.0259502 0.0127651 2023-10-09T05:20:08.995 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:09.022 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:09.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:09.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:09.127 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:09.154 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:09.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:09.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:09.258 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:09.265 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:09 smithi133 bash[105589]: cluster 2023-10-09T05:20:08.013754+0000 mgr.y (mgr.25277) 953 : cluster 0 pgmap v991: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 46 MiB/s wr, 741 op/s; 1655/237531 objects misplaced (0.697%); 2.9 MiB/s, 47 objects/s recovering 2023-10-09T05:20:09.265 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:09 smithi133 bash[105589]: audit 2023-10-09T05:20:08.143960+0000 mon.b (mon.1) 19 : audit 0 from='client.? 172.21.15.133:0/36491209' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:20:09.266 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:09 smithi133 bash[105589]: audit 2023-10-09T05:20:08.538066+0000 mon.a (mon.0) 1241 : audit 1 from='client.? 172.21.15.133:0/2690965721' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:20:09.266 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:09 smithi133 bash[102858]: cluster 2023-10-09T05:20:08.013754+0000 mgr.y (mgr.25277) 953 : cluster 0 pgmap v991: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 46 MiB/s wr, 741 op/s; 1655/237531 objects misplaced (0.697%); 2.9 MiB/s, 47 objects/s recovering 2023-10-09T05:20:09.266 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:09 smithi133 bash[102858]: audit 2023-10-09T05:20:08.143960+0000 mon.b (mon.1) 19 : audit 0 from='client.? 172.21.15.133:0/36491209' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:20:09.266 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:09 smithi133 bash[102858]: audit 2023-10-09T05:20:08.538066+0000 mon.a (mon.0) 1241 : audit 1 from='client.? 172.21.15.133:0/2690965721' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:20:09.280 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:09.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:09.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:09.383 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:09.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:09 smithi194 bash[92666]: cluster 2023-10-09T05:20:08.013754+0000 mgr.y (mgr.25277) 953 : cluster 0 pgmap v991: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 46 MiB/s wr, 741 op/s; 1655/237531 objects misplaced (0.697%); 2.9 MiB/s, 47 objects/s recovering 2023-10-09T05:20:09.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:09 smithi194 bash[92666]: audit 2023-10-09T05:20:08.143960+0000 mon.b (mon.1) 19 : audit 0 from='client.? 172.21.15.133:0/36491209' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:20:09.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:09 smithi194 bash[92666]: audit 2023-10-09T05:20:08.538066+0000 mon.a (mon.0) 1241 : audit 1 from='client.? 172.21.15.133:0/2690965721' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:20:09.415 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:09.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:09.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:09.518 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is set 2023-10-09T05:20:09.520 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:09.545 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:09.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:09.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:09.651 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:09.685 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:09.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:09.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:09.791 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:09.822 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:09.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:09.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:09.926 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:09.929 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 67 16 83542 83526 77.8996 51.375 0.0104834 0.0128307 2023-10-09T05:20:09.950 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:09.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:09.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:10.055 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:10.077 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:10 smithi133 bash[102858]: cluster 2023-10-09T05:20:09.506177+0000 mon.a (mon.0) 1242 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:20:10.077 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:10 smithi133 bash[102858]: cluster 2023-10-09T05:20:09.510248+0000 osd.5 (osd.5) 220 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T05:20:10.077 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:10 smithi133 bash[102858]: audit 2023-10-09T05:20:09.512130+0000 mon.a (mon.0) 1243 : audit 1 from='client.? 172.21.15.133:0/2690965721' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:20:10.077 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:10 smithi133 bash[102858]: cluster 2023-10-09T05:20:09.512192+0000 mon.a (mon.0) 1244 : cluster 0 osdmap e234: 8 total, 8 up, 7 in 2023-10-09T05:20:10.077 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:10 smithi133 bash[102858]: cluster 2023-10-09T05:20:09.522499+0000 osd.7 (osd.7) 173 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T05:20:10.087 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:10.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:10.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:10.191 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:10.229 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:10.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:10.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:10.336 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:10.345 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:10 smithi133 bash[105589]: cluster 2023-10-09T05:20:09.506177+0000 mon.a (mon.0) 1242 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:20:10.345 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:10 smithi133 bash[105589]: cluster 2023-10-09T05:20:09.510248+0000 osd.5 (osd.5) 220 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T05:20:10.345 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:10 smithi133 bash[105589]: audit 2023-10-09T05:20:09.512130+0000 mon.a (mon.0) 1243 : audit 1 from='client.? 172.21.15.133:0/2690965721' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:20:10.345 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:10 smithi133 bash[105589]: cluster 2023-10-09T05:20:09.512192+0000 mon.a (mon.0) 1244 : cluster 0 osdmap e234: 8 total, 8 up, 7 in 2023-10-09T05:20:10.345 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:10 smithi133 bash[105589]: cluster 2023-10-09T05:20:09.522499+0000 osd.7 (osd.7) 173 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T05:20:10.366 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:10.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:10.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:10.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:10 smithi194 bash[92666]: cluster 2023-10-09T05:20:09.506177+0000 mon.a (mon.0) 1242 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:20:10.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:10 smithi194 bash[92666]: cluster 2023-10-09T05:20:09.510248+0000 osd.5 (osd.5) 220 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T05:20:10.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:10 smithi194 bash[92666]: audit 2023-10-09T05:20:09.512130+0000 mon.a (mon.0) 1243 : audit 1 from='client.? 172.21.15.133:0/2690965721' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:20:10.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:10 smithi194 bash[92666]: cluster 2023-10-09T05:20:09.512192+0000 mon.a (mon.0) 1244 : cluster 0 osdmap e234: 8 total, 8 up, 7 in 2023-10-09T05:20:10.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:10 smithi194 bash[92666]: cluster 2023-10-09T05:20:09.522499+0000 osd.7 (osd.7) 173 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T05:20:10.471 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:10.496 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:10.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:10.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:10.604 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:10.641 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:10.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:10.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:10.744 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:10.778 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:10.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:10.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:10.882 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:10.909 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:10.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:10.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:10.930 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 68 16 84374 84358 77.5187 52 0.015237 0.0128944 2023-10-09T05:20:11.012 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:11.035 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:11.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:11.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:11.139 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:11.147 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:11 smithi194 bash[92666]: cluster 2023-10-09T05:20:09.510407+0000 osd.4 (osd.4) 194 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T05:20:11.147 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:11 smithi194 bash[92666]: cluster 2023-10-09T05:20:10.014239+0000 mgr.y (mgr.25277) 954 : cluster 0 pgmap v993: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 54 MiB/s wr, 858 op/s; 1347/244758 objects misplaced (0.550%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T05:20:11.164 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:11.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:11.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:11.267 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:11.304 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:11.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:11.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:11.408 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:11.416 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:11 smithi133 bash[105589]: cluster 2023-10-09T05:20:09.510407+0000 osd.4 (osd.4) 194 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T05:20:11.416 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:11 smithi133 bash[105589]: cluster 2023-10-09T05:20:10.014239+0000 mgr.y (mgr.25277) 954 : cluster 0 pgmap v993: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 54 MiB/s wr, 858 op/s; 1347/244758 objects misplaced (0.550%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T05:20:11.417 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:11 smithi133 bash[102858]: cluster 2023-10-09T05:20:09.510407+0000 osd.4 (osd.4) 194 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T05:20:11.417 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:11 smithi133 bash[102858]: cluster 2023-10-09T05:20:10.014239+0000 mgr.y (mgr.25277) 954 : cluster 0 pgmap v993: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 54 MiB/s wr, 858 op/s; 1347/244758 objects misplaced (0.550%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T05:20:11.433 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:11.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:11.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:11.540 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:11.569 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:11.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:11.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:11.672 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:11.699 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:11.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:11.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:11.804 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:11.844 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:11.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:11.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:11.930 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 69 16 85272 85256 77.2086 56.125 0.00776459 0.0129459 2023-10-09T05:20:11.953 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:11.978 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:11.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:11.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:12.085 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:12.108 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:12.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:12.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:12.212 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:12.245 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:12.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:12.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:12.301 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:20:12.351 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:12.379 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:12.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:12.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:12.485 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:12.519 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:12.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:12.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:12.624 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:12.651 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:12.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:12.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:12.755 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:12.780 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:12.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:12.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:12.884 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:12.902 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:12.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:12.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:12.930 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 70 16 86067 86051 76.8154 49.6875 0.00429351 0.0130032 2023-10-09T05:20:13.005 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:13.031 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:13.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:13.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:13.139 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:13.165 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:13.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:13.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:13.268 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:13.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:13 smithi133 bash[102858]: cluster 2023-10-09T05:20:12.014955+0000 mgr.y (mgr.25277) 955 : cluster 0 pgmap v994: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 863 op/s; 1347/250398 objects misplaced (0.538%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T05:20:13.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:13 smithi133 bash[105589]: cluster 2023-10-09T05:20:12.014955+0000 mgr.y (mgr.25277) 955 : cluster 0 pgmap v994: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 863 op/s; 1347/250398 objects misplaced (0.538%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T05:20:13.300 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:13.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:13.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:13.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:13 smithi194 bash[92666]: cluster 2023-10-09T05:20:12.014955+0000 mgr.y (mgr.25277) 955 : cluster 0 pgmap v994: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 863 op/s; 1347/250398 objects misplaced (0.538%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T05:20:13.404 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:13.428 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:13.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:13.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:13.532 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:13.571 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:13.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:13.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:13.677 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:13.709 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:13.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:13.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:13.813 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:13.853 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:13.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:13.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:13.933 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 71 16 86876 86860 76.4425 50.5625 0.0243537 0.0130766 2023-10-09T05:20:13.959 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:13.988 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:13.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:13.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:14.083 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:14 smithi133 bash[102858]: audit 2023-10-09T05:20:13.455143+0000 mon.a (mon.0) 1245 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:20:14.083 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:14 smithi133 bash[105589]: audit 2023-10-09T05:20:13.455143+0000 mon.a (mon.0) 1245 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:20:14.091 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:14.120 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:14.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:14.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:14.138 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:20:14.172 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:20:14.224 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:14.289 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:14.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:14.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:14.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:14 smithi194 bash[92666]: audit 2023-10-09T05:20:13.455143+0000 mon.a (mon.0) 1245 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:20:14.399 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:14.410 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:20:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:20:14] "GET /metrics HTTP/1.1" 200 36530 "" "Prometheus/2.43.0" 2023-10-09T05:20:14.476 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:14.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:14.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:14.579 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:14.605 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:14.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:14.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:14.710 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:14.737 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:14.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:14.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:14.841 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:14.874 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:14.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:14.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:14.934 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 72 16 87688 87672 76.0856 50.75 0.026 0.0131369 2023-10-09T05:20:14.981 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:15.009 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:15.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:15.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:15.114 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:15.147 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:15.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:15.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:15.256 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:15.261 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:15 smithi194 bash[92666]: cluster 2023-10-09T05:20:14.015565+0000 mgr.y (mgr.25277) 956 : cluster 0 pgmap v995: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 5.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 856 op/s; 1347/253293 objects misplaced (0.532%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T05:20:15.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:15.292 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:15.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:15.393 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:15.403 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:15 smithi133 bash[102858]: cluster 2023-10-09T05:20:14.015565+0000 mgr.y (mgr.25277) 956 : cluster 0 pgmap v995: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 5.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 856 op/s; 1347/253293 objects misplaced (0.532%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T05:20:15.403 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:15 smithi133 bash[105589]: cluster 2023-10-09T05:20:14.015565+0000 mgr.y (mgr.25277) 956 : cluster 0 pgmap v995: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 5.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 856 op/s; 1347/253293 objects misplaced (0.532%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T05:20:15.427 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:15.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:15.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:15.532 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:15.554 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:15.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:15.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:15.656 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:15.686 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:15.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:15.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:15.792 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:15.823 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:15.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:15.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:15.926 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:15.933 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 73 16 88437 88421 75.6847 46.8125 0.018613 0.0132068 2023-10-09T05:20:15.966 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:15.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:15.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:16.073 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:16.106 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:16.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:16.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:16.212 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:16.239 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:16.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:16.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:16.342 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:16.372 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:16.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:16.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:16.477 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:16.507 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:16.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:16.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:16.614 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:16.648 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:16.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:16.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:16.753 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:16.789 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:16.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:16.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:16.892 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:16.920 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:16.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:16.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:16.933 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 74 16 89245 89229 75.3443 50.5 0.0246869 0.0132666 2023-10-09T05:20:17.029 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:17.059 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:17.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:17.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:17.106 INFO:teuthology.orchestra.run.smithi133.stdout:moved 27 / 267 (10.1124%) 2023-10-09T05:20:17.106 INFO:teuthology.orchestra.run.smithi133.stdout:avg 38.1429 2023-10-09T05:20:17.106 INFO:teuthology.orchestra.run.smithi133.stdout:stddev 9.78962 -> 5.11101 (expected baseline 5.71786) 2023-10-09T05:20:17.106 INFO:teuthology.orchestra.run.smithi133.stdout:min osd.2 with 15 -> 33 pgs (0.393258 -> 0.865169 * mean) 2023-10-09T05:20:17.106 INFO:teuthology.orchestra.run.smithi133.stdout:max osd.1 with 46 -> 34 pgs (1.20599 -> 0.891386 * mean) 2023-10-09T05:20:17.106 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:20:17.106 INFO:teuthology.orchestra.run.smithi133.stdout:oload 110 2023-10-09T05:20:17.106 INFO:teuthology.orchestra.run.smithi133.stdout:max_change 3 2023-10-09T05:20:17.106 INFO:teuthology.orchestra.run.smithi133.stdout:max_change_osds 4 2023-10-09T05:20:17.106 INFO:teuthology.orchestra.run.smithi133.stdout:average_utilization 0.0340 2023-10-09T05:20:17.107 INFO:teuthology.orchestra.run.smithi133.stdout:overload_utilization 0.0375 2023-10-09T05:20:17.107 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2 weight 0.5546 -> 1.0000 2023-10-09T05:20:17.107 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1 weight 0.9500 -> 0.6711 2023-10-09T05:20:17.107 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6 weight 0.9500 -> 0.8195 2023-10-09T05:20:17.107 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0 weight 1.0000 -> 0.9067 2023-10-09T05:20:17.172 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:17.193 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:17.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:17.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:17.297 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:17.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:17 smithi133 bash[105589]: cluster 2023-10-09T05:20:16.016307+0000 mgr.y (mgr.25277) 957 : cluster 0 pgmap v996: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 52 MiB/s wr, 837 op/s; 1018/261858 objects misplaced (0.389%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T05:20:17.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:17 smithi133 bash[105589]: audit 2023-10-09T05:20:16.151262+0000 mgr.y (mgr.25277) 958 : audit 0 from='client.45206 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:20:17.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:17 smithi133 bash[105589]: audit 2023-10-09T05:20:16.152301+0000 mon.a (mon.0) 1246 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'59423','1':'43980','2':'65536','6':'53704'}"}]: dispatch 2023-10-09T05:20:17.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:17 smithi133 bash[102858]: cluster 2023-10-09T05:20:16.016307+0000 mgr.y (mgr.25277) 957 : cluster 0 pgmap v996: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 52 MiB/s wr, 837 op/s; 1018/261858 objects misplaced (0.389%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T05:20:17.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:17 smithi133 bash[102858]: audit 2023-10-09T05:20:16.151262+0000 mgr.y (mgr.25277) 958 : audit 0 from='client.45206 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:20:17.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:17 smithi133 bash[102858]: audit 2023-10-09T05:20:16.152301+0000 mon.a (mon.0) 1246 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'59423','1':'43980','2':'65536','6':'53704'}"}]: dispatch 2023-10-09T05:20:17.309 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:17 smithi194 bash[92666]: cluster 2023-10-09T05:20:16.016307+0000 mgr.y (mgr.25277) 957 : cluster 0 pgmap v996: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 52 MiB/s wr, 837 op/s; 1018/261858 objects misplaced (0.389%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T05:20:17.309 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:17 smithi194 bash[92666]: audit 2023-10-09T05:20:16.151262+0000 mgr.y (mgr.25277) 958 : audit 0 from='client.45206 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:20:17.309 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:17 smithi194 bash[92666]: audit 2023-10-09T05:20:16.152301+0000 mon.a (mon.0) 1246 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'59423','1':'43980','2':'65536','6':'53704'}"}]: dispatch 2023-10-09T05:20:17.322 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:17.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:17.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:17.425 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:17.451 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:17.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:17.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:17.554 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:17.582 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:17.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:17.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:17.686 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:17.715 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:17.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:17.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:17.818 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:17.844 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:17.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:17.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:17.933 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 75 16 89392 89376 74.4623 9.1875 0.00586814 0.0132783 2023-10-09T05:20:17.948 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:17.980 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:17.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:17.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:18.085 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:18.112 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:18.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:18.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:18.216 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:18.225 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:18 smithi194 bash[92666]: cluster 2023-10-09T05:20:17.099206+0000 osd.6 (osd.6) 173 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T05:20:18.225 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:18 smithi194 bash[92666]: cluster 2023-10-09T05:20:17.101960+0000 osd.5 (osd.5) 221 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T05:20:18.225 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:18 smithi194 bash[92666]: audit 2023-10-09T05:20:17.105059+0000 mon.a (mon.0) 1247 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'59423','1':'43980','2':'65536','6':'53704'}"}]': finished 2023-10-09T05:20:18.225 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:18 smithi194 bash[92666]: cluster 2023-10-09T05:20:17.105206+0000 mon.a (mon.0) 1248 : cluster 0 osdmap e235: 8 total, 8 up, 7 in 2023-10-09T05:20:18.225 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:18 smithi194 bash[92666]: cluster 2023-10-09T05:20:17.116749+0000 osd.7 (osd.7) 174 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T05:20:18.225 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:18 smithi194 bash[92666]: cluster 2023-10-09T05:20:17.118690+0000 osd.7 (osd.7) 175 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T05:20:18.225 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:18 smithi194 bash[92666]: cluster 2023-10-09T05:20:17.120627+0000 osd.7 (osd.7) 176 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T05:20:18.240 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:18.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:18.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:18.313 INFO:teuthology.orchestra.run.smithi194.stdout:{ 2023-10-09T05:20:18.313 INFO:teuthology.orchestra.run.smithi194.stdout: "ops": [], 2023-10-09T05:20:18.313 INFO:teuthology.orchestra.run.smithi194.stdout: "num_ops": 0 2023-10-09T05:20:18.313 INFO:teuthology.orchestra.run.smithi194.stdout:} 2023-10-09T05:20:18.344 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:18.352 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:18 smithi133 bash[102858]: cluster 2023-10-09T05:20:17.099206+0000 osd.6 (osd.6) 173 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T05:20:18.352 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:18 smithi133 bash[102858]: cluster 2023-10-09T05:20:17.101960+0000 osd.5 (osd.5) 221 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T05:20:18.352 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:18 smithi133 bash[102858]: audit 2023-10-09T05:20:17.105059+0000 mon.a (mon.0) 1247 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'59423','1':'43980','2':'65536','6':'53704'}"}]': finished 2023-10-09T05:20:18.353 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:18 smithi133 bash[102858]: cluster 2023-10-09T05:20:17.105206+0000 mon.a (mon.0) 1248 : cluster 0 osdmap e235: 8 total, 8 up, 7 in 2023-10-09T05:20:18.353 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:18 smithi133 bash[102858]: cluster 2023-10-09T05:20:17.116749+0000 osd.7 (osd.7) 174 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T05:20:18.353 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:18 smithi133 bash[102858]: cluster 2023-10-09T05:20:17.118690+0000 osd.7 (osd.7) 175 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T05:20:18.353 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:18 smithi133 bash[102858]: cluster 2023-10-09T05:20:17.120627+0000 osd.7 (osd.7) 176 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T05:20:18.353 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:18 smithi133 bash[105589]: cluster 2023-10-09T05:20:17.099206+0000 osd.6 (osd.6) 173 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T05:20:18.353 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:18 smithi133 bash[105589]: cluster 2023-10-09T05:20:17.101960+0000 osd.5 (osd.5) 221 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T05:20:18.353 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:18 smithi133 bash[105589]: audit 2023-10-09T05:20:17.105059+0000 mon.a (mon.0) 1247 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'59423','1':'43980','2':'65536','6':'53704'}"}]': finished 2023-10-09T05:20:18.353 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:18 smithi133 bash[105589]: cluster 2023-10-09T05:20:17.105206+0000 mon.a (mon.0) 1248 : cluster 0 osdmap e235: 8 total, 8 up, 7 in 2023-10-09T05:20:18.353 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:18 smithi133 bash[105589]: cluster 2023-10-09T05:20:17.116749+0000 osd.7 (osd.7) 174 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T05:20:18.354 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:18 smithi133 bash[105589]: cluster 2023-10-09T05:20:17.118690+0000 osd.7 (osd.7) 175 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T05:20:18.354 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:18 smithi133 bash[105589]: cluster 2023-10-09T05:20:17.120627+0000 osd.7 (osd.7) 176 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T05:20:18.369 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:18.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:18.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:18.473 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:18.501 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:18.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:18.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:18.604 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:18.632 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:18.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:18.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:18.736 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:18.769 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:18.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:18.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:18.797 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-09T05:20:18.798 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 1.0 2023-10-09T05:20:18.873 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:18.903 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:18.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:18.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:18.933 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 76 16 89392 89376 73.4827 0 - 0.0132783 2023-10-09T05:20:19.007 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:19.033 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:19.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:19.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:19.136 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:19.146 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:19 smithi194 bash[92666]: cluster 2023-10-09T05:20:18.016721+0000 mgr.y (mgr.25277) 959 : cluster 0 pgmap v998: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 5.3 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 54 MiB/s wr, 860 op/s; 1018/262650 objects misplaced (0.388%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T05:20:19.147 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:19 smithi194 bash[92666]: cluster 2023-10-09T05:20:18.103783+0000 osd.4 (osd.4) 195 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.147 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:19 smithi194 bash[92666]: cluster 2023-10-09T05:20:18.106871+0000 osd.6 (osd.6) 174 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.147 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:19 smithi194 bash[92666]: cluster 2023-10-09T05:20:18.107193+0000 osd.4 (osd.4) 196 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.147 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:19 smithi194 bash[92666]: cluster 2023-10-09T05:20:18.107625+0000 mon.a (mon.0) 1249 : cluster 0 osdmap e236: 8 total, 8 up, 7 in 2023-10-09T05:20:19.147 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:19 smithi194 bash[92666]: cluster 2023-10-09T05:20:18.107920+0000 osd.7 (osd.7) 177 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.147 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:19 smithi194 bash[92666]: cluster 2023-10-09T05:20:18.110108+0000 osd.7 (osd.7) 178 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.147 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:19 smithi194 bash[92666]: cluster 2023-10-09T05:20:18.110223+0000 osd.4 (osd.4) 197 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.147 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:19 smithi194 bash[92666]: cluster 2023-10-09T05:20:18.111601+0000 osd.7 (osd.7) 179 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.147 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:19 smithi194 bash[92666]: cluster 2023-10-09T05:20:18.112570+0000 osd.4 (osd.4) 198 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.147 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:19 smithi194 bash[92666]: cluster 2023-10-09T05:20:18.112746+0000 osd.6 (osd.6) 175 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.147 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:19 smithi194 bash[92666]: cluster 2023-10-09T05:20:18.113162+0000 osd.7 (osd.7) 180 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.148 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:19 smithi194 bash[92666]: cluster 2023-10-09T05:20:18.667786+0000 osd.5 (osd.5) 222 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.164 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:19.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:19.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:19.266 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:19.294 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:19.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:19.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:19.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:19 smithi133 bash[102858]: cluster 2023-10-09T05:20:18.016721+0000 mgr.y (mgr.25277) 959 : cluster 0 pgmap v998: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 5.3 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 54 MiB/s wr, 860 op/s; 1018/262650 objects misplaced (0.388%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T05:20:19.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:19 smithi133 bash[102858]: cluster 2023-10-09T05:20:18.103783+0000 osd.4 (osd.4) 195 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:19 smithi133 bash[102858]: cluster 2023-10-09T05:20:18.106871+0000 osd.6 (osd.6) 174 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:19 smithi133 bash[102858]: cluster 2023-10-09T05:20:18.107193+0000 osd.4 (osd.4) 196 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:19 smithi133 bash[102858]: cluster 2023-10-09T05:20:18.107625+0000 mon.a (mon.0) 1249 : cluster 0 osdmap e236: 8 total, 8 up, 7 in 2023-10-09T05:20:19.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:19 smithi133 bash[102858]: cluster 2023-10-09T05:20:18.107920+0000 osd.7 (osd.7) 177 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:19 smithi133 bash[102858]: cluster 2023-10-09T05:20:18.110108+0000 osd.7 (osd.7) 178 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:19 smithi133 bash[102858]: cluster 2023-10-09T05:20:18.110223+0000 osd.4 (osd.4) 197 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:19 smithi133 bash[102858]: cluster 2023-10-09T05:20:18.111601+0000 osd.7 (osd.7) 179 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:19 smithi133 bash[102858]: cluster 2023-10-09T05:20:18.112570+0000 osd.4 (osd.4) 198 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:19 smithi133 bash[102858]: cluster 2023-10-09T05:20:18.112746+0000 osd.6 (osd.6) 175 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:19 smithi133 bash[102858]: cluster 2023-10-09T05:20:18.113162+0000 osd.7 (osd.7) 180 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:19 smithi133 bash[102858]: cluster 2023-10-09T05:20:18.667786+0000 osd.5 (osd.5) 222 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:19 smithi133 bash[105589]: cluster 2023-10-09T05:20:18.016721+0000 mgr.y (mgr.25277) 959 : cluster 0 pgmap v998: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 5.3 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 54 MiB/s wr, 860 op/s; 1018/262650 objects misplaced (0.388%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T05:20:19.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:19 smithi133 bash[105589]: cluster 2023-10-09T05:20:18.103783+0000 osd.4 (osd.4) 195 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:19 smithi133 bash[105589]: cluster 2023-10-09T05:20:18.106871+0000 osd.6 (osd.6) 174 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:19 smithi133 bash[105589]: cluster 2023-10-09T05:20:18.107193+0000 osd.4 (osd.4) 196 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:19 smithi133 bash[105589]: cluster 2023-10-09T05:20:18.107625+0000 mon.a (mon.0) 1249 : cluster 0 osdmap e236: 8 total, 8 up, 7 in 2023-10-09T05:20:19.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:19 smithi133 bash[105589]: cluster 2023-10-09T05:20:18.107920+0000 osd.7 (osd.7) 177 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:19 smithi133 bash[105589]: cluster 2023-10-09T05:20:18.110108+0000 osd.7 (osd.7) 178 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:19 smithi133 bash[105589]: cluster 2023-10-09T05:20:18.110223+0000 osd.4 (osd.4) 197 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:19 smithi133 bash[105589]: cluster 2023-10-09T05:20:18.111601+0000 osd.7 (osd.7) 179 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:19 smithi133 bash[105589]: cluster 2023-10-09T05:20:18.112570+0000 osd.4 (osd.4) 198 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:19 smithi133 bash[105589]: cluster 2023-10-09T05:20:18.112746+0000 osd.6 (osd.6) 175 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:19 smithi133 bash[105589]: cluster 2023-10-09T05:20:18.113162+0000 osd.7 (osd.7) 180 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.302 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:19 smithi133 bash[105589]: cluster 2023-10-09T05:20:18.667786+0000 osd.5 (osd.5) 222 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T05:20:19.354 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T05:20:19.396 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:19.435 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:19.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:19.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:19.547 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:19.665 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:19.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:19.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:19.768 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:19.800 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:19.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:19.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:19.905 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:19.934 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 77 16 89545 89529 72.6526 4.78125 0.00906363 0.0137554 2023-10-09T05:20:19.940 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:19.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:19.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:20.043 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:20.067 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:20.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:20.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:20.173 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:20.212 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:20.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:20.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:20.322 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:20.332 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:20 smithi194 bash[92666]: cluster 2023-10-09T05:20:19.109147+0000 mon.a (mon.0) 1250 : cluster 0 osdmap e237: 8 total, 8 up, 7 in 2023-10-09T05:20:20.332 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:20 smithi194 bash[92666]: cluster 2023-10-09T05:20:19.109549+0000 osd.5 (osd.5) 223 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T05:20:20.333 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:20 smithi194 bash[92666]: cluster 2023-10-09T05:20:19.110006+0000 osd.7 (osd.7) 181 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T05:20:20.333 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:20 smithi194 bash[92666]: cluster 2023-10-09T05:20:19.110070+0000 osd.6 (osd.6) 176 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T05:20:20.333 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:20 smithi194 bash[92666]: cluster 2023-10-09T05:20:19.111755+0000 osd.7 (osd.7) 182 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T05:20:20.333 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:20 smithi194 bash[92666]: cluster 2023-10-09T05:20:19.112741+0000 osd.6 (osd.6) 177 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T05:20:20.333 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:20 smithi194 bash[92666]: cluster 2023-10-09T05:20:19.152169+0000 osd.4 (osd.4) 199 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T05:20:20.348 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:20.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:20.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:20.453 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:20.480 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:20.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:20.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:20.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:20 smithi133 bash[102858]: cluster 2023-10-09T05:20:19.109147+0000 mon.a (mon.0) 1250 : cluster 0 osdmap e237: 8 total, 8 up, 7 in 2023-10-09T05:20:20.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:20 smithi133 bash[102858]: cluster 2023-10-09T05:20:19.109549+0000 osd.5 (osd.5) 223 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T05:20:20.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:20 smithi133 bash[102858]: cluster 2023-10-09T05:20:19.110006+0000 osd.7 (osd.7) 181 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T05:20:20.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:20 smithi133 bash[102858]: cluster 2023-10-09T05:20:19.110070+0000 osd.6 (osd.6) 176 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T05:20:20.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:20 smithi133 bash[102858]: cluster 2023-10-09T05:20:19.111755+0000 osd.7 (osd.7) 182 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T05:20:20.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:20 smithi133 bash[102858]: cluster 2023-10-09T05:20:19.112741+0000 osd.6 (osd.6) 177 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T05:20:20.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:20 smithi133 bash[102858]: cluster 2023-10-09T05:20:19.152169+0000 osd.4 (osd.4) 199 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T05:20:20.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:20 smithi133 bash[105589]: cluster 2023-10-09T05:20:19.109147+0000 mon.a (mon.0) 1250 : cluster 0 osdmap e237: 8 total, 8 up, 7 in 2023-10-09T05:20:20.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:20 smithi133 bash[105589]: cluster 2023-10-09T05:20:19.109549+0000 osd.5 (osd.5) 223 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T05:20:20.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:20 smithi133 bash[105589]: cluster 2023-10-09T05:20:19.110006+0000 osd.7 (osd.7) 181 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T05:20:20.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:20 smithi133 bash[105589]: cluster 2023-10-09T05:20:19.110070+0000 osd.6 (osd.6) 176 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T05:20:20.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:20 smithi133 bash[105589]: cluster 2023-10-09T05:20:19.111755+0000 osd.7 (osd.7) 182 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T05:20:20.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:20 smithi133 bash[105589]: cluster 2023-10-09T05:20:19.112741+0000 osd.6 (osd.6) 177 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T05:20:20.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:20 smithi133 bash[105589]: cluster 2023-10-09T05:20:19.152169+0000 osd.4 (osd.4) 199 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T05:20:20.583 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:20.621 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:20.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:20.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:20.725 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:20.752 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:20.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:20.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:20.855 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:20.890 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:20.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:20.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:20.934 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 78 16 90051 90035 72.1267 31.625 0.00613395 0.0138563 2023-10-09T05:20:20.996 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:21.023 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:21.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:21.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:21.124 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:21 smithi133 bash[102858]: cluster 2023-10-09T05:20:20.017440+0000 mgr.y (mgr.25277) 960 : cluster 0 pgmap v1001: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 peering, 80 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 46 MiB/s wr, 729 op/s; 5589/266433 objects degraded (2.098%); 484/266433 objects misplaced (0.182%); 9.0 MiB/s, 143 objects/s recovering 2023-10-09T05:20:21.125 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:21 smithi133 bash[102858]: cluster 2023-10-09T05:20:20.113258+0000 mon.a (mon.0) 1251 : cluster 3 Health check failed: Degraded data redundancy: 5589/266433 objects degraded (2.098%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:20:21.125 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:21 smithi133 bash[105589]: cluster 2023-10-09T05:20:20.017440+0000 mgr.y (mgr.25277) 960 : cluster 0 pgmap v1001: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 peering, 80 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 46 MiB/s wr, 729 op/s; 5589/266433 objects degraded (2.098%); 484/266433 objects misplaced (0.182%); 9.0 MiB/s, 143 objects/s recovering 2023-10-09T05:20:21.125 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:21 smithi133 bash[105589]: cluster 2023-10-09T05:20:20.113258+0000 mon.a (mon.0) 1251 : cluster 3 Health check failed: Degraded data redundancy: 5589/266433 objects degraded (2.098%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:20:21.126 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:21.165 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:21.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:21.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:21.270 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:21.278 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:21 smithi194 bash[92666]: cluster 2023-10-09T05:20:20.017440+0000 mgr.y (mgr.25277) 960 : cluster 0 pgmap v1001: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 peering, 80 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 46 MiB/s wr, 729 op/s; 5589/266433 objects degraded (2.098%); 484/266433 objects misplaced (0.182%); 9.0 MiB/s, 143 objects/s recovering 2023-10-09T05:20:21.278 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:21 smithi194 bash[92666]: cluster 2023-10-09T05:20:20.113258+0000 mon.a (mon.0) 1251 : cluster 3 Health check failed: Degraded data redundancy: 5589/266433 objects degraded (2.098%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:20:21.292 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:21.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:21.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:21.330 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:20:21.336 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:20:21.395 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:21.423 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:21.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:21.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:21.529 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:21.556 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:21.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:21.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:21.661 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:21.693 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:21.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:21.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:21.797 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:21.831 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:21.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:21.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:21.934 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 79 16 90567 90551 71.6219 32.25 0.00790081 0.0139543 2023-10-09T05:20:21.936 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:21.961 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:21.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:21.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:22.066 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:22.090 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:22.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:22.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:22.196 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:22.220 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:22.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:22.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:22.326 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:22.354 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:22.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:22.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:22.461 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:22.499 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:22.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:22.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:22.604 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:22.645 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:22.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:22.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:22.751 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:22.778 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:22.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:22.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:22.880 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:22.925 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:22.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:22.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:22.934 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T05:20:22.934223+0000 min lat: 0.0015063 max lat: 2.60968 avg lat: 0.0140432 2023-10-09T05:20:22.934 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:20:22.934 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 80 16 91144 91128 71.1775 36.0625 0.0143237 0.0140432 2023-10-09T05:20:23.032 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:23.059 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:23.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:23.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:23.163 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:23.199 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:23.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:23.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:23.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:23 smithi133 bash[102858]: cluster 2023-10-09T05:20:22.018122+0000 mgr.y (mgr.25277) 961 : cluster 0 pgmap v1002: 89 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 peering, 77 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 25 MiB/s wr, 405 op/s; 16766/269160 objects degraded (6.229%); 6162/269160 objects misplaced (2.289%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T05:20:23.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:23 smithi133 bash[105589]: cluster 2023-10-09T05:20:22.018122+0000 mgr.y (mgr.25277) 961 : cluster 0 pgmap v1002: 89 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 peering, 77 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 25 MiB/s wr, 405 op/s; 16766/269160 objects degraded (6.229%); 6162/269160 objects misplaced (2.289%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T05:20:23.305 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:23.341 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:23.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:23.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:23.362 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:20:23.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:23 smithi194 bash[92666]: cluster 2023-10-09T05:20:22.018122+0000 mgr.y (mgr.25277) 961 : cluster 0 pgmap v1002: 89 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 peering, 77 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 25 MiB/s wr, 405 op/s; 16766/269160 objects degraded (6.229%); 6162/269160 objects misplaced (2.289%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T05:20:23.449 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:23.482 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:23.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:23.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:23.585 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:23.621 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:23.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:23.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:23.724 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:23.750 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:23.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:23.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:23.855 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:23.885 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:23.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:23.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:23.934 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 81 16 91694 91678 70.7231 34.375 0.0108856 0.0141309 2023-10-09T05:20:23.990 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:24.033 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:24.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:24.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:24.129 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:20:24.136 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:24.173 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:24.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:24.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:24.277 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:24.281 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:20:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:20:24] "GET /metrics HTTP/1.1" 200 36530 "" "Prometheus/2.43.0" 2023-10-09T05:20:24.299 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:24.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:24.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:24.402 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:24.432 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:24.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:24.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:24.536 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:24.545 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:24 smithi133 bash[105589]: cluster 2023-10-09T05:20:24.018739+0000 mgr.y (mgr.25277) 962 : cluster 0 pgmap v1003: 89 pgs: 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 81 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 24 MiB/s wr, 379 op/s; 28026/270519 objects degraded (10.360%); 11664/270519 objects misplaced (4.312%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T05:20:24.545 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:24 smithi133 bash[102858]: cluster 2023-10-09T05:20:24.018739+0000 mgr.y (mgr.25277) 962 : cluster 0 pgmap v1003: 89 pgs: 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 81 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 24 MiB/s wr, 379 op/s; 28026/270519 objects degraded (10.360%); 11664/270519 objects misplaced (4.312%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T05:20:24.574 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:24.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:24.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:24.680 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:24.713 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:24.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:24.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:24.819 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:24.827 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:24 smithi194 bash[92666]: cluster 2023-10-09T05:20:24.018739+0000 mgr.y (mgr.25277) 962 : cluster 0 pgmap v1003: 89 pgs: 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 81 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 24 MiB/s wr, 379 op/s; 28026/270519 objects degraded (10.360%); 11664/270519 objects misplaced (4.312%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T05:20:24.849 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:24.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:24.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:24.934 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 82 16 92229 92213 70.2684 33.4375 0.052096 0.0142251 2023-10-09T05:20:24.952 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:24.988 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:24.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:24.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:25.093 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:25.130 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:25.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:25.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:25.235 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:25.265 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:25.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:25.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:25.372 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:25.406 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:25.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:25.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:25.517 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:25.540 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:25.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:25.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:25.646 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:25.690 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:25.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:25.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:25.794 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:25.827 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:25.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:25.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:25.931 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:25.934 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 83 16 92767 92751 69.827 33.625 0.0388789 0.014311 2023-10-09T05:20:25.955 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:25.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:25.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:26.059 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:26.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:26.094 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:26.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:26.195 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:26.203 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:26 smithi194 bash[92666]: cluster 2023-10-09T05:20:26.070762+0000 mon.a (mon.0) 1252 : cluster 3 Health check update: Degraded data redundancy: 28026/276120 objects degraded (10.150%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:20:26.224 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:26.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:26.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:26.328 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:26.350 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:26.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:26.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:26.453 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:26.462 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:26 smithi133 bash[105589]: cluster 2023-10-09T05:20:26.070762+0000 mon.a (mon.0) 1252 : cluster 3 Health check update: Degraded data redundancy: 28026/276120 objects degraded (10.150%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:20:26.462 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:26 smithi133 bash[102858]: cluster 2023-10-09T05:20:26.070762+0000 mon.a (mon.0) 1252 : cluster 3 Health check update: Degraded data redundancy: 28026/276120 objects degraded (10.150%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:20:26.482 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:26.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:26.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:26.588 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:26.629 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:26.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:26.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:26.734 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:26.760 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:26.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:26.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:26.863 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:26.893 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:26.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:26.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:26.934 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 84 16 93202 93186 69.3194 27.1875 0.0108094 0.0144178 2023-10-09T05:20:26.996 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:27.024 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:27.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:27.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:27.131 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:27.156 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:27.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:27.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:27.259 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:27.291 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:27.295 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:27 smithi133 bash[102858]: cluster 2023-10-09T05:20:26.019531+0000 mgr.y (mgr.25277) 963 : cluster 0 pgmap v1004: 89 pgs: 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 81 active+clean; 5.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 35 MiB/s wr, 561 op/s; 28026/276120 objects degraded (10.150%); 11361/276120 objects misplaced (4.115%); 7.1 MiB/s, 114 objects/s recovering 2023-10-09T05:20:27.295 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:27 smithi133 bash[105589]: cluster 2023-10-09T05:20:26.019531+0000 mgr.y (mgr.25277) 963 : cluster 0 pgmap v1004: 89 pgs: 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 81 active+clean; 5.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 35 MiB/s wr, 561 op/s; 28026/276120 objects degraded (10.150%); 11361/276120 objects misplaced (4.115%); 7.1 MiB/s, 114 objects/s recovering 2023-10-09T05:20:27.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:27.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:27.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:27 smithi194 bash[92666]: cluster 2023-10-09T05:20:26.019531+0000 mgr.y (mgr.25277) 963 : cluster 0 pgmap v1004: 89 pgs: 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 81 active+clean; 5.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 35 MiB/s wr, 561 op/s; 28026/276120 objects degraded (10.150%); 11361/276120 objects misplaced (4.115%); 7.1 MiB/s, 114 objects/s recovering 2023-10-09T05:20:27.397 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:27.427 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:27.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:27.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:27.530 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:27.556 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:27.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:27.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:27.660 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:27.683 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:27.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:27.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:27.787 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:27.808 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:27.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:27.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:27.911 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:27.933 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:27.934 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 85 16 93726 93710 68.8889 32.75 0.0260244 0.0145082 2023-10-09T05:20:27.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:27.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:28.041 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:28.075 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:28.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:28.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:28.180 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:28.204 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:28.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:28.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:28.307 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:28.330 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:28.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:28.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:28.433 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:28.446 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 05:20:28 smithi194 bash[80638]: logger=cleanup t=2023-10-09T05:20:28.348524606Z level=info msg="Completed cleanup jobs" duration=442.000153ms 2023-10-09T05:20:28.461 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:28.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:28.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:28.564 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:28.587 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:28.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:28.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:28.690 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:28.719 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:28.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:28.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:28.823 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:28.849 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:28.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:28.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:28.935 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 86 16 94029 94013 68.3081 18.9375 0.00446795 0.0145336 2023-10-09T05:20:28.947 INFO:teuthology.orchestra.run.smithi194.stdout:{ 2023-10-09T05:20:28.947 INFO:teuthology.orchestra.run.smithi194.stdout: "ops": [], 2023-10-09T05:20:28.948 INFO:teuthology.orchestra.run.smithi194.stdout: "complaint_time": 30, 2023-10-09T05:20:28.948 INFO:teuthology.orchestra.run.smithi194.stdout: "num_blocked_ops": 0 2023-10-09T05:20:28.948 INFO:teuthology.orchestra.run.smithi194.stdout:} 2023-10-09T05:20:28.952 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:28.975 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:28.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:28.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:29.078 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:29.097 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:29.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:29.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:29.201 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:29.229 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:29.231 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:29 smithi194 bash[92666]: cluster 2023-10-09T05:20:28.020256+0000 mgr.y (mgr.25277) 964 : cluster 0 pgmap v1005: 89 pgs: 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 81 active+clean; 5.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 24 MiB/s wr, 386 op/s; 28026/276636 objects degraded (10.131%); 11361/276636 objects misplaced (4.107%); 5.7 MiB/s, 92 objects/s recovering 2023-10-09T05:20:29.231 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:29 smithi194 bash[92666]: cluster 2023-10-09T05:20:28.153590+0000 osd.7 (osd.7) 183 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T05:20:29.231 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:29 smithi194 bash[92666]: cluster 2023-10-09T05:20:28.154202+0000 mon.a (mon.0) 1253 : cluster 0 osdmap e238: 8 total, 8 up, 7 in 2023-10-09T05:20:29.231 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:29 smithi194 bash[92666]: cluster 2023-10-09T05:20:28.155622+0000 osd.7 (osd.7) 184 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T05:20:29.231 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:29 smithi194 bash[92666]: cluster 2023-10-09T05:20:28.155661+0000 osd.4 (osd.4) 200 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T05:20:29.231 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:29 smithi194 bash[92666]: audit 2023-10-09T05:20:28.459514+0000 mon.a (mon.0) 1254 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:20:29.231 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:29 smithi194 bash[92666]: audit 2023-10-09T05:20:28.461600+0000 mon.a (mon.0) 1255 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:20:29.231 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:29 smithi194 bash[92666]: cluster 2023-10-09T05:20:28.648690+0000 osd.5 (osd.5) 224 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T05:20:29.232 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:29 smithi194 bash[92666]: cluster 2023-10-09T05:20:28.650749+0000 osd.5 (osd.5) 225 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T05:20:29.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:29.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:29.333 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:29.353 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:29.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:29.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:29.407 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:29 smithi133 bash[102858]: cluster 2023-10-09T05:20:28.020256+0000 mgr.y (mgr.25277) 964 : cluster 0 pgmap v1005: 89 pgs: 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 81 active+clean; 5.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 24 MiB/s wr, 386 op/s; 28026/276636 objects degraded (10.131%); 11361/276636 objects misplaced (4.107%); 5.7 MiB/s, 92 objects/s recovering 2023-10-09T05:20:29.407 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:29 smithi133 bash[102858]: cluster 2023-10-09T05:20:28.153590+0000 osd.7 (osd.7) 183 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T05:20:29.407 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:29 smithi133 bash[102858]: cluster 2023-10-09T05:20:28.154202+0000 mon.a (mon.0) 1253 : cluster 0 osdmap e238: 8 total, 8 up, 7 in 2023-10-09T05:20:29.408 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:29 smithi133 bash[102858]: cluster 2023-10-09T05:20:28.155622+0000 osd.7 (osd.7) 184 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T05:20:29.408 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:29 smithi133 bash[102858]: cluster 2023-10-09T05:20:28.155661+0000 osd.4 (osd.4) 200 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T05:20:29.408 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:29 smithi133 bash[102858]: audit 2023-10-09T05:20:28.459514+0000 mon.a (mon.0) 1254 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:20:29.408 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:29 smithi133 bash[102858]: audit 2023-10-09T05:20:28.461600+0000 mon.a (mon.0) 1255 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:20:29.408 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:29 smithi133 bash[102858]: cluster 2023-10-09T05:20:28.648690+0000 osd.5 (osd.5) 224 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T05:20:29.408 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:29 smithi133 bash[102858]: cluster 2023-10-09T05:20:28.650749+0000 osd.5 (osd.5) 225 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T05:20:29.408 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:29 smithi133 bash[105589]: cluster 2023-10-09T05:20:28.020256+0000 mgr.y (mgr.25277) 964 : cluster 0 pgmap v1005: 89 pgs: 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 81 active+clean; 5.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 24 MiB/s wr, 386 op/s; 28026/276636 objects degraded (10.131%); 11361/276636 objects misplaced (4.107%); 5.7 MiB/s, 92 objects/s recovering 2023-10-09T05:20:29.408 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:29 smithi133 bash[105589]: cluster 2023-10-09T05:20:28.153590+0000 osd.7 (osd.7) 183 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T05:20:29.408 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:29 smithi133 bash[105589]: cluster 2023-10-09T05:20:28.154202+0000 mon.a (mon.0) 1253 : cluster 0 osdmap e238: 8 total, 8 up, 7 in 2023-10-09T05:20:29.409 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:29 smithi133 bash[105589]: cluster 2023-10-09T05:20:28.155622+0000 osd.7 (osd.7) 184 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T05:20:29.409 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:29 smithi133 bash[105589]: cluster 2023-10-09T05:20:28.155661+0000 osd.4 (osd.4) 200 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T05:20:29.409 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:29 smithi133 bash[105589]: audit 2023-10-09T05:20:28.459514+0000 mon.a (mon.0) 1254 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:20:29.409 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:29 smithi133 bash[105589]: audit 2023-10-09T05:20:28.461600+0000 mon.a (mon.0) 1255 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:20:29.409 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:29 smithi133 bash[105589]: cluster 2023-10-09T05:20:28.648690+0000 osd.5 (osd.5) 224 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T05:20:29.409 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:29 smithi133 bash[105589]: cluster 2023-10-09T05:20:28.650749+0000 osd.5 (osd.5) 225 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T05:20:29.457 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:29.483 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:29.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:29.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:29.588 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:29.618 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:29.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:29.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:29.724 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:29.757 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:29.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:29.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:29.864 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:29.911 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:29.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:29.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:29.927 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T05:20:29.935 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 87 16 94406 94390 67.7938 23.5625 0.0389669 0.0147439 2023-10-09T05:20:30.014 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:30.049 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:30.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:30.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:30.156 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:30.190 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:30.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:30.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:30.297 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:30.321 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:30.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:30.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:30.424 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:30.433 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:30 smithi133 bash[102858]: cluster 2023-10-09T05:20:29.151205+0000 osd.6 (osd.6) 178 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T05:20:30.433 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:30 smithi133 bash[102858]: cluster 2023-10-09T05:20:29.152334+0000 mon.a (mon.0) 1256 : cluster 0 osdmap e239: 8 total, 8 up, 7 in 2023-10-09T05:20:30.433 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:30 smithi133 bash[102858]: cluster 2023-10-09T05:20:29.153139+0000 osd.7 (osd.7) 185 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T05:20:30.434 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:30 smithi133 bash[102858]: cluster 2023-10-09T05:20:29.154536+0000 osd.7 (osd.7) 186 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T05:20:30.434 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:30 smithi133 bash[102858]: cluster 2023-10-09T05:20:29.156297+0000 osd.7 (osd.7) 187 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T05:20:30.434 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:30 smithi133 bash[105589]: cluster 2023-10-09T05:20:29.151205+0000 osd.6 (osd.6) 178 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T05:20:30.434 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:30 smithi133 bash[105589]: cluster 2023-10-09T05:20:29.152334+0000 mon.a (mon.0) 1256 : cluster 0 osdmap e239: 8 total, 8 up, 7 in 2023-10-09T05:20:30.434 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:30 smithi133 bash[105589]: cluster 2023-10-09T05:20:29.153139+0000 osd.7 (osd.7) 185 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T05:20:30.434 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:30 smithi133 bash[105589]: cluster 2023-10-09T05:20:29.154536+0000 osd.7 (osd.7) 186 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T05:20:30.434 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:30 smithi133 bash[105589]: cluster 2023-10-09T05:20:29.156297+0000 osd.7 (osd.7) 187 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T05:20:30.466 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:30.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:30.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:30.570 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:30.581 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:30 smithi194 bash[92666]: cluster 2023-10-09T05:20:29.151205+0000 osd.6 (osd.6) 178 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T05:20:30.581 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:30 smithi194 bash[92666]: cluster 2023-10-09T05:20:29.152334+0000 mon.a (mon.0) 1256 : cluster 0 osdmap e239: 8 total, 8 up, 7 in 2023-10-09T05:20:30.581 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:30 smithi194 bash[92666]: cluster 2023-10-09T05:20:29.153139+0000 osd.7 (osd.7) 185 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T05:20:30.582 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:30 smithi194 bash[92666]: cluster 2023-10-09T05:20:29.154536+0000 osd.7 (osd.7) 186 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T05:20:30.582 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:30 smithi194 bash[92666]: cluster 2023-10-09T05:20:29.156297+0000 osd.7 (osd.7) 187 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T05:20:30.598 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:30.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:30.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:30.703 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:30.726 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:30.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:30.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:30.832 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:30.865 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:30.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:30.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:30.935 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 88 15 94946 94931 67.4073 33.8125 0.0218934 0.0148281 2023-10-09T05:20:30.975 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:31.016 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:31.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:31.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:31.120 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:31.146 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:31.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:31.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:31.253 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:31.288 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:31.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:31.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:31.358 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:31 smithi133 bash[105589]: cluster 2023-10-09T05:20:29.208739+0000 osd.4 (osd.4) 201 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T05:20:31.358 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:31 smithi133 bash[105589]: cluster 2023-10-09T05:20:29.211382+0000 osd.4 (osd.4) 202 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T05:20:31.359 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:31 smithi133 bash[105589]: cluster 2023-10-09T05:20:30.020766+0000 mgr.y (mgr.25277) 965 : cluster 0 pgmap v1008: 89 pgs: 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 82 active+clean; 5.7 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 29 MiB/s wr, 457 op/s; 28026/280140 objects degraded (10.004%); 11076/280140 objects misplaced (3.954%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T05:20:31.359 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:31 smithi133 bash[102858]: cluster 2023-10-09T05:20:29.208739+0000 osd.4 (osd.4) 201 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T05:20:31.359 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:31 smithi133 bash[102858]: cluster 2023-10-09T05:20:29.211382+0000 osd.4 (osd.4) 202 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T05:20:31.359 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:31 smithi133 bash[102858]: cluster 2023-10-09T05:20:30.020766+0000 mgr.y (mgr.25277) 965 : cluster 0 pgmap v1008: 89 pgs: 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 82 active+clean; 5.7 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 29 MiB/s wr, 457 op/s; 28026/280140 objects degraded (10.004%); 11076/280140 objects misplaced (3.954%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T05:20:31.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:31 smithi194 bash[92666]: cluster 2023-10-09T05:20:29.208739+0000 osd.4 (osd.4) 201 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T05:20:31.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:31 smithi194 bash[92666]: cluster 2023-10-09T05:20:29.211382+0000 osd.4 (osd.4) 202 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T05:20:31.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:31 smithi194 bash[92666]: cluster 2023-10-09T05:20:30.020766+0000 mgr.y (mgr.25277) 965 : cluster 0 pgmap v1008: 89 pgs: 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 82 active+clean; 5.7 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 29 MiB/s wr, 457 op/s; 28026/280140 objects degraded (10.004%); 11076/280140 objects misplaced (3.954%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T05:20:31.395 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:31.422 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:31.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:31.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:31.531 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:31.556 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:31.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:31.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:31.659 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:31.690 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:31.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:31.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:31.798 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:31.823 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:31.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:31.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:31.926 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:31.936 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 89 16 95441 95425 66.9968 30.875 0.00864788 0.0148959 2023-10-09T05:20:31.951 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:31.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:31.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:32.055 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:32.089 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:32.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:32.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:32.197 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:32.225 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:32 smithi133 bash[102858]: cluster 2023-10-09T05:20:32.171463+0000 mon.a (mon.0) 1257 : cluster 3 Health check update: Degraded data redundancy: 28026/283491 objects degraded (9.886%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:20:32.228 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:32.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:32.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:32.332 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:32.366 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:32.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:32.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:32.468 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:32.477 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:32 smithi133 bash[105589]: cluster 2023-10-09T05:20:32.171463+0000 mon.a (mon.0) 1257 : cluster 3 Health check update: Degraded data redundancy: 28026/283491 objects degraded (9.886%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:20:32.504 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:32.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:32.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:32.610 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:32.620 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:32 smithi194 bash[92666]: cluster 2023-10-09T05:20:32.171463+0000 mon.a (mon.0) 1257 : cluster 3 Health check update: Degraded data redundancy: 28026/283491 objects degraded (9.886%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:20:32.634 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:32.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:32.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:32.738 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:32.770 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:32.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:32.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:32.875 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:32.900 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:32.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:32.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:32.936 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 90 3 95917 95914 66.5921 30.5625 0.0416841 0.0150089 2023-10-09T05:20:33.003 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:33.022 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:33.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:33.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:33.125 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:33.150 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:33.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:33.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:33.253 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:33.263 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:33 smithi133 bash[105589]: cluster 2023-10-09T05:20:32.021791+0000 mgr.y (mgr.25277) 966 : cluster 0 pgmap v1009: 89 pgs: 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 82 active+clean; 5.8 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 34 MiB/s wr, 540 op/s; 28026/283491 objects degraded (9.886%); 11076/283491 objects misplaced (3.907%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T05:20:33.264 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:33 smithi133 bash[102858]: cluster 2023-10-09T05:20:32.021791+0000 mgr.y (mgr.25277) 966 : cluster 0 pgmap v1009: 89 pgs: 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 82 active+clean; 5.8 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 34 MiB/s wr, 540 op/s; 28026/283491 objects degraded (9.886%); 11076/283491 objects misplaced (3.907%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T05:20:33.275 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:33.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:33.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:33.378 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:33.403 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:33.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:33.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:33.506 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:33.531 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:33.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:33.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:33.635 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:33.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:33 smithi194 bash[92666]: cluster 2023-10-09T05:20:32.021791+0000 mgr.y (mgr.25277) 966 : cluster 0 pgmap v1009: 89 pgs: 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 82 active+clean; 5.8 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 34 MiB/s wr, 540 op/s; 28026/283491 objects degraded (9.886%); 11076/283491 objects misplaced (3.907%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T05:20:33.657 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:33.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:33.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:33.760 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:33.792 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:33.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:33.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:33.896 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:33.932 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:33.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:33.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:33.936 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Total time run: 90.0272 2023-10-09T05:20:33.936 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Total writes made: 95917 2023-10-09T05:20:33.936 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Write size: 65536 2023-10-09T05:20:33.936 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Object size: 65536 2023-10-09T05:20:33.936 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Bandwidth (MB/sec): 66.5889 2023-10-09T05:20:33.937 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Stddev Bandwidth: 34.8258 2023-10-09T05:20:33.937 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Max bandwidth (MB/sec): 170.75 2023-10-09T05:20:33.937 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Min bandwidth (MB/sec): 0 2023-10-09T05:20:33.937 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Average IOPS: 1065 2023-10-09T05:20:33.937 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Stddev IOPS: 557.222 2023-10-09T05:20:33.937 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Max IOPS: 2732 2023-10-09T05:20:33.937 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Min IOPS: 0 2023-10-09T05:20:33.937 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Average Latency(s): 0.0150106 2023-10-09T05:20:33.938 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Stddev Latency(s): 0.0416624 2023-10-09T05:20:33.938 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Max latency(s): 2.60968 2023-10-09T05:20:33.938 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Min latency(s): 0.0015063 2023-10-09T05:20:33.950 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T05:20:34.036 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:34.066 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:34.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:34.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:34.169 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:34.222 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:34.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:34.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:34.331 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:34.358 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:34.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:34.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:34.466 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:34.476 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:20:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:20:34] "GET /metrics HTTP/1.1" 200 36543 "" "Prometheus/2.43.0" 2023-10-09T05:20:34.505 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:34.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:34.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:34.607 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:34.640 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:34.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:34.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:34.746 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:34.769 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:34.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:34.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:34.777 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:20:34.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:34 smithi133 bash[105589]: cluster 2023-10-09T05:20:34.022420+0000 mgr.y (mgr.25277) 967 : cluster 0 pgmap v1010: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 23 MiB/s wr, 370 op/s; 28026/285015 objects degraded (9.833%); 10943/285015 objects misplaced (3.839%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T05:20:34.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:34 smithi133 bash[102858]: cluster 2023-10-09T05:20:34.022420+0000 mgr.y (mgr.25277) 967 : cluster 0 pgmap v1010: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 23 MiB/s wr, 370 op/s; 28026/285015 objects degraded (9.833%); 10943/285015 objects misplaced (3.839%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T05:20:34.809 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:34 smithi194 bash[92666]: cluster 2023-10-09T05:20:34.022420+0000 mgr.y (mgr.25277) 967 : cluster 0 pgmap v1010: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 23 MiB/s wr, 370 op/s; 28026/285015 objects degraded (9.833%); 10943/285015 objects misplaced (3.839%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T05:20:34.872 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:34.912 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:34.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:34.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:35.015 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:35.037 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:35.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:35.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:35.142 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:35.169 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:35.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:35.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:35.272 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:35.315 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:35.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:35.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:35.420 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:35.457 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:35.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:35.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:35.561 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:35.589 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:35.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:35.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:35.692 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:35.727 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:35.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:35.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:35.834 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:35.858 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:35.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:35.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:35.962 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:35.994 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:35.999 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:20:36.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:36.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:36.051 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:20:36.066 INFO:teuthology.orchestra.run.smithi133.stdout:moved 0 / 267 (0%) 2023-10-09T05:20:36.067 INFO:teuthology.orchestra.run.smithi133.stdout:avg 38.1429 2023-10-09T05:20:36.067 INFO:teuthology.orchestra.run.smithi133.stdout:stddev 5.11101 -> 5.11101 (expected baseline 5.71786) 2023-10-09T05:20:36.067 INFO:teuthology.orchestra.run.smithi133.stdout:min osd.2 with 33 -> 33 pgs (0.865169 -> 0.865169 * mean) 2023-10-09T05:20:36.067 INFO:teuthology.orchestra.run.smithi133.stdout:max osd.3 with 49 -> 49 pgs (1.28464 -> 1.28464 * mean) 2023-10-09T05:20:36.067 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:20:36.067 INFO:teuthology.orchestra.run.smithi133.stdout:oload 1000 2023-10-09T05:20:36.067 INFO:teuthology.orchestra.run.smithi133.stdout:max_change 1 2023-10-09T05:20:36.067 INFO:teuthology.orchestra.run.smithi133.stdout:max_change_osds 4 2023-10-09T05:20:36.067 INFO:teuthology.orchestra.run.smithi133.stdout:average_utilization 0.0366 2023-10-09T05:20:36.067 INFO:teuthology.orchestra.run.smithi133.stdout:overload_utilization 0.3663 2023-10-09T05:20:36.072 INFO:teuthology.orchestra.run.smithi133.stderr:no change 2023-10-09T05:20:36.095 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:20:36.106 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:36.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:36 smithi194 bash[92666]: audit 2023-10-09T05:20:35.952522+0000 mon.a (mon.0) 1258 : audit 0 from='client.? 172.21.15.133:0/4233738853' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:20:36.151 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:36.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:36.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:36.169 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:20:36.221 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:20:36.254 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:36.261 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:20:36.276 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:36.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:36.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:36.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:36 smithi133 bash[102858]: audit 2023-10-09T05:20:35.952522+0000 mon.a (mon.0) 1258 : audit 0 from='client.? 172.21.15.133:0/4233738853' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:20:36.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:36 smithi133 bash[105589]: audit 2023-10-09T05:20:35.952522+0000 mon.a (mon.0) 1258 : audit 0 from='client.? 172.21.15.133:0/4233738853' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:20:36.308 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:20:36.349 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:20:36.352 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:20:36.352 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:20:36.352 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:20:36.353 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:20:36.353 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:20:36.353 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:20:36.353 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:20:36.353 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:20:36.380 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:36.404 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:36.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:36.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:36.508 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:36.539 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:36.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:36.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:36.647 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:36.683 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:36.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:36.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:36.789 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:36.819 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:36.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:36.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:36.925 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:36.956 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:36.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:36.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:37.016 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:37 smithi133 bash[102858]: cluster 2023-10-09T05:20:36.023211+0000 mgr.y (mgr.25277) 968 : cluster 0 pgmap v1011: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 127 B/s rd, 29 MiB/s wr, 528 op/s; 28026/286086 objects degraded (9.796%); 10943/286086 objects misplaced (3.825%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T05:20:37.017 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:37 smithi133 bash[102858]: audit 2023-10-09T05:20:36.060976+0000 mgr.y (mgr.25277) 969 : audit 0 from='client.35226 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:20:37.038 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is unset 2023-10-09T05:20:37.059 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:37.084 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:37.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:37.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:37.188 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:37.219 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:37.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:37.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:37.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:37 smithi133 bash[102858]: audit 2023-10-09T05:20:36.370495+0000 mon.a (mon.0) 1259 : audit 1 from='client.? 172.21.15.133:0/3739540309' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:20:37.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:37 smithi133 bash[105589]: cluster 2023-10-09T05:20:36.023211+0000 mgr.y (mgr.25277) 968 : cluster 0 pgmap v1011: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 127 B/s rd, 29 MiB/s wr, 528 op/s; 28026/286086 objects degraded (9.796%); 10943/286086 objects misplaced (3.825%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T05:20:37.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:37 smithi133 bash[105589]: audit 2023-10-09T05:20:36.060976+0000 mgr.y (mgr.25277) 969 : audit 0 from='client.35226 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:20:37.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:37 smithi133 bash[105589]: audit 2023-10-09T05:20:36.370495+0000 mon.a (mon.0) 1259 : audit 1 from='client.? 172.21.15.133:0/3739540309' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:20:37.323 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:37.331 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:37 smithi194 bash[92666]: cluster 2023-10-09T05:20:36.023211+0000 mgr.y (mgr.25277) 968 : cluster 0 pgmap v1011: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 127 B/s rd, 29 MiB/s wr, 528 op/s; 28026/286086 objects degraded (9.796%); 10943/286086 objects misplaced (3.825%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T05:20:37.331 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:37 smithi194 bash[92666]: audit 2023-10-09T05:20:36.060976+0000 mgr.y (mgr.25277) 969 : audit 0 from='client.35226 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:20:37.331 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:37 smithi194 bash[92666]: audit 2023-10-09T05:20:36.370495+0000 mon.a (mon.0) 1259 : audit 1 from='client.? 172.21.15.133:0/3739540309' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:20:37.350 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:37.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:37.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:37.455 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:37.479 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:37.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:37.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:37.582 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:37.623 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:37.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:37.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:37.727 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:37.749 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:37.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:37.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:37.853 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:37.886 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:37.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:37.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:37.990 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:38.025 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:38.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:38.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:38.130 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:38.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:38 smithi194 bash[92666]: cluster 2023-10-09T05:20:37.005279+0000 mon.a (mon.0) 1260 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:20:38.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:38 smithi194 bash[92666]: cluster 2023-10-09T05:20:37.016243+0000 osd.6 (osd.6) 179 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T05:20:38.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:38 smithi194 bash[92666]: audit 2023-10-09T05:20:37.021590+0000 mon.a (mon.0) 1261 : audit 1 from='client.? 172.21.15.133:0/3739540309' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:20:38.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:38 smithi194 bash[92666]: cluster 2023-10-09T05:20:37.021775+0000 mon.a (mon.0) 1262 : cluster 0 osdmap e240: 8 total, 8 up, 7 in 2023-10-09T05:20:38.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:38 smithi194 bash[92666]: cluster 2023-10-09T05:20:37.023291+0000 osd.4 (osd.4) 203 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T05:20:38.151 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:38.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:38.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:38.255 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:38.263 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:38 smithi133 bash[105589]: cluster 2023-10-09T05:20:37.005279+0000 mon.a (mon.0) 1260 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:20:38.263 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:38 smithi133 bash[105589]: cluster 2023-10-09T05:20:37.016243+0000 osd.6 (osd.6) 179 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T05:20:38.263 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:38 smithi133 bash[105589]: audit 2023-10-09T05:20:37.021590+0000 mon.a (mon.0) 1261 : audit 1 from='client.? 172.21.15.133:0/3739540309' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:20:38.264 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:38 smithi133 bash[105589]: cluster 2023-10-09T05:20:37.021775+0000 mon.a (mon.0) 1262 : cluster 0 osdmap e240: 8 total, 8 up, 7 in 2023-10-09T05:20:38.264 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:38 smithi133 bash[105589]: cluster 2023-10-09T05:20:37.023291+0000 osd.4 (osd.4) 203 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T05:20:38.264 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:38 smithi133 bash[102858]: cluster 2023-10-09T05:20:37.005279+0000 mon.a (mon.0) 1260 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:20:38.264 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:38 smithi133 bash[102858]: cluster 2023-10-09T05:20:37.016243+0000 osd.6 (osd.6) 179 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T05:20:38.264 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:38 smithi133 bash[102858]: audit 2023-10-09T05:20:37.021590+0000 mon.a (mon.0) 1261 : audit 1 from='client.? 172.21.15.133:0/3739540309' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:20:38.264 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:38 smithi133 bash[102858]: cluster 2023-10-09T05:20:37.021775+0000 mon.a (mon.0) 1262 : cluster 0 osdmap e240: 8 total, 8 up, 7 in 2023-10-09T05:20:38.264 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:38 smithi133 bash[102858]: cluster 2023-10-09T05:20:37.023291+0000 osd.4 (osd.4) 203 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T05:20:38.280 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:38.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:38.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:38.385 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:38.418 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:38.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:38.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:38.522 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:38.551 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:38.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:38.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:38.656 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:38.686 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:38.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:38.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:38.793 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:38.816 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:38.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:38.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:38.921 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:38.943 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:38.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:38.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:39.051 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:39.060 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:39 smithi133 bash[102858]: cluster 2023-10-09T05:20:37.298301+0000 osd.5 (osd.5) 226 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T05:20:39.060 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:39 smithi133 bash[102858]: cluster 2023-10-09T05:20:38.023805+0000 mgr.y (mgr.25277) 970 : cluster 0 pgmap v1013: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 115 B/s rd, 22 MiB/s wr, 423 op/s; 28026/285924 objects degraded (9.802%); 10943/285924 objects misplaced (3.827%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T05:20:39.061 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:39 smithi133 bash[102858]: cluster 2023-10-09T05:20:38.511051+0000 mon.a (mon.0) 1263 : cluster 3 Health check update: Degraded data redundancy: 28026/286086 objects degraded (9.796%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:20:39.061 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:39 smithi133 bash[105589]: cluster 2023-10-09T05:20:37.298301+0000 osd.5 (osd.5) 226 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T05:20:39.061 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:39 smithi133 bash[105589]: cluster 2023-10-09T05:20:38.023805+0000 mgr.y (mgr.25277) 970 : cluster 0 pgmap v1013: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 115 B/s rd, 22 MiB/s wr, 423 op/s; 28026/285924 objects degraded (9.802%); 10943/285924 objects misplaced (3.827%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T05:20:39.061 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:39 smithi133 bash[105589]: cluster 2023-10-09T05:20:38.511051+0000 mon.a (mon.0) 1263 : cluster 3 Health check update: Degraded data redundancy: 28026/286086 objects degraded (9.796%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:20:39.090 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:39.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:39.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:39.193 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:39.226 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:39.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:39.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:39.332 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:39.362 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:39.363 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:20:39.364 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:20:39.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:39.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:39.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:39 smithi194 bash[92666]: cluster 2023-10-09T05:20:37.298301+0000 osd.5 (osd.5) 226 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T05:20:39.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:39 smithi194 bash[92666]: cluster 2023-10-09T05:20:38.023805+0000 mgr.y (mgr.25277) 970 : cluster 0 pgmap v1013: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 115 B/s rd, 22 MiB/s wr, 423 op/s; 28026/285924 objects degraded (9.802%); 10943/285924 objects misplaced (3.827%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T05:20:39.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:39 smithi194 bash[92666]: cluster 2023-10-09T05:20:38.511051+0000 mon.a (mon.0) 1263 : cluster 3 Health check update: Degraded data redundancy: 28026/286086 objects degraded (9.796%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:20:39.468 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:39.498 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:39.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:39.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:39.601 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:39.643 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:39.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:39.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:39.747 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:39.776 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:39.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:39.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:39.880 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:39.946 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:39.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:39.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:40.049 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:40.084 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:40.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:40.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:40.188 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:40.216 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:40.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:40.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:40.319 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:40.357 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T05:20:40.361 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:40.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:40.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:40.464 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:40.495 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:40.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:40.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:40.602 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:40.636 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:40.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:40.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:40.742 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:40.765 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:40.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:40.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:40.868 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:40.895 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:40.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:40.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:40.999 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:41.037 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:41.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:41.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:41.144 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:41.174 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:41.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:41.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:41.278 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:41.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:41 smithi133 bash[102858]: cluster 2023-10-09T05:20:40.024502+0000 mgr.y (mgr.25277) 971 : cluster 0 pgmap v1014: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 102 B/s rd, 16 MiB/s wr, 440 op/s; 28026/282282 objects degraded (9.928%); 10799/282282 objects misplaced (3.826%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T05:20:41.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:41 smithi133 bash[105589]: cluster 2023-10-09T05:20:40.024502+0000 mgr.y (mgr.25277) 971 : cluster 0 pgmap v1014: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 102 B/s rd, 16 MiB/s wr, 440 op/s; 28026/282282 objects degraded (9.928%); 10799/282282 objects misplaced (3.826%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T05:20:41.304 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:41.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:41.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:41.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:41 smithi194 bash[92666]: cluster 2023-10-09T05:20:40.024502+0000 mgr.y (mgr.25277) 971 : cluster 0 pgmap v1014: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 102 B/s rd, 16 MiB/s wr, 440 op/s; 28026/282282 objects degraded (9.928%); 10799/282282 objects misplaced (3.826%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T05:20:41.408 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:41.435 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:41.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:41.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:41.543 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:41.567 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:41.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:41.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:41.671 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:41.695 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:41.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:41.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:41.797 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:41.824 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:41.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:41.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:41.929 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:41.965 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:41.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:41.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:42.069 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:42.108 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:42.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:42.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:42.214 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:42.246 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:42.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:42.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:42.351 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:42.390 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:42.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:42.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:42.499 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:42.533 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:42.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:42.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:42.639 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:42.668 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:42.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:42.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:42.674 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:20:42.678 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:20:42.771 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:42.803 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:42.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:42.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:42.906 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:42.932 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:42.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:42.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:43.035 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:43.071 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:43.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:43.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:43.178 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:43.210 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:43.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:43.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:43.316 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:43.326 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:43 smithi194 bash[92666]: cluster 2023-10-09T05:20:42.025193+0000 mgr.y (mgr.25277) 972 : cluster 0 pgmap v1015: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 102 B/s rd, 9.0 MiB/s wr, 462 op/s; 28026/278292 objects degraded (10.071%); 10799/278292 objects misplaced (3.880%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T05:20:43.347 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:43.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:43.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:43.450 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:43.475 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:43.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:43.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:43.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:43 smithi133 bash[105589]: cluster 2023-10-09T05:20:42.025193+0000 mgr.y (mgr.25277) 972 : cluster 0 pgmap v1015: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 102 B/s rd, 9.0 MiB/s wr, 462 op/s; 28026/278292 objects degraded (10.071%); 10799/278292 objects misplaced (3.880%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T05:20:43.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:43 smithi133 bash[102858]: cluster 2023-10-09T05:20:42.025193+0000 mgr.y (mgr.25277) 972 : cluster 0 pgmap v1015: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 102 B/s rd, 9.0 MiB/s wr, 462 op/s; 28026/278292 objects degraded (10.071%); 10799/278292 objects misplaced (3.880%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T05:20:43.583 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:43.609 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:43.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:43.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:43.713 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:43.737 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:43.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:43.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:43.843 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:43.875 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:43.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:43.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:43.981 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:44.018 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:44.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:44.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:44.123 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:44.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:44 smithi194 bash[92666]: audit 2023-10-09T05:20:43.455251+0000 mon.a (mon.0) 1264 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:20:44.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:44 smithi194 bash[92666]: cluster 2023-10-09T05:20:43.512294+0000 mon.a (mon.0) 1265 : cluster 3 Health check update: Degraded data redundancy: 28026/278292 objects degraded (10.071%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:20:44.153 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:44.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:44.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:44.257 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:44.267 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:44 smithi133 bash[105589]: audit 2023-10-09T05:20:43.455251+0000 mon.a (mon.0) 1264 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:20:44.267 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:44 smithi133 bash[105589]: cluster 2023-10-09T05:20:43.512294+0000 mon.a (mon.0) 1265 : cluster 3 Health check update: Degraded data redundancy: 28026/278292 objects degraded (10.071%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:20:44.268 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:44 smithi133 bash[102858]: audit 2023-10-09T05:20:43.455251+0000 mon.a (mon.0) 1264 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:20:44.268 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:44 smithi133 bash[102858]: cluster 2023-10-09T05:20:43.512294+0000 mon.a (mon.0) 1265 : cluster 3 Health check update: Degraded data redundancy: 28026/278292 objects degraded (10.071%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:20:44.296 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:44.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:44.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:44.403 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:44.433 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:44.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:44.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:44.451 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 6, 1, 7] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:20:44.452 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T05:20:44.536 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:44.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:20:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:20:44] "GET /metrics HTTP/1.1" 200 36541 "" "Prometheus/2.43.0" 2023-10-09T05:20:44.559 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:44.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:44.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:44.662 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:44.690 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:44.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:44.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:44.796 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:44.842 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:44.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:44.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:44.950 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:44.972 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:44.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:44.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:45.075 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:45.108 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:45.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:45.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:45.148 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:20:45.214 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:45.234 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:45.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:45.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:45.339 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:45.348 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:45 smithi133 bash[102858]: cluster 2023-10-09T05:20:44.025819+0000 mgr.y (mgr.25277) 973 : cluster 0 pgmap v1016: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 102 B/s rd, 5.8 MiB/s wr, 484 op/s; 28026/276093 objects degraded (10.151%); 10700/276093 objects misplaced (3.876%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T05:20:45.348 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:45 smithi133 bash[105589]: cluster 2023-10-09T05:20:44.025819+0000 mgr.y (mgr.25277) 973 : cluster 0 pgmap v1016: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 102 B/s rd, 5.8 MiB/s wr, 484 op/s; 28026/276093 objects degraded (10.151%); 10700/276093 objects misplaced (3.876%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T05:20:45.369 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:45.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:45.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:45.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:45 smithi194 bash[92666]: cluster 2023-10-09T05:20:44.025819+0000 mgr.y (mgr.25277) 973 : cluster 0 pgmap v1016: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 102 B/s rd, 5.8 MiB/s wr, 484 op/s; 28026/276093 objects degraded (10.151%); 10700/276093 objects misplaced (3.876%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T05:20:45.472 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:45.508 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:45.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:45.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:45.611 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:45.635 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:45.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:45.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:45.738 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:45.774 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:45.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:45.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:45.878 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:45.903 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:45.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:45.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:46.008 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:46.032 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:46.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:46.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:46.134 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:46.170 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:46.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:46.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:46.275 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:46.310 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:46.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:46.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:46.416 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:46.446 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:46.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:46.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:46.552 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:46.595 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:46.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:46.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:46.701 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:46.732 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:46.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:46.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:46.837 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:46.866 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:46.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:46.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:46.975 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:46.999 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:47.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:47.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:47.103 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:47.125 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:47.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:47.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:47.228 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:47.236 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:47 smithi133 bash[105589]: cluster 2023-10-09T05:20:46.026529+0000 mgr.y (mgr.25277) 974 : cluster 0 pgmap v1017: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 269 KiB/s wr, 465 op/s; 28026/272364 objects degraded (10.290%); 10700/272364 objects misplaced (3.929%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T05:20:47.237 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:47 smithi133 bash[102858]: cluster 2023-10-09T05:20:46.026529+0000 mgr.y (mgr.25277) 974 : cluster 0 pgmap v1017: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 269 KiB/s wr, 465 op/s; 28026/272364 objects degraded (10.290%); 10700/272364 objects misplaced (3.929%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T05:20:47.264 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:47.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:47.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:47.367 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:47.377 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:47 smithi194 bash[92666]: cluster 2023-10-09T05:20:46.026529+0000 mgr.y (mgr.25277) 974 : cluster 0 pgmap v1017: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 269 KiB/s wr, 465 op/s; 28026/272364 objects degraded (10.290%); 10700/272364 objects misplaced (3.929%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T05:20:47.393 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:47.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:47.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:47.495 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:47.525 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:47.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:47.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:47.566 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:20:47.571 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:20:47.631 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:47.656 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:47.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:47.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:47.759 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:47.804 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:47.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:47.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:47.907 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:47.941 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:47.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:47.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:48.046 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:48.076 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:48.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:48.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:48.179 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:48.228 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:48.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:48.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:48.332 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:48.357 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:48.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:48.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:48.465 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:48.499 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:48.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:48.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:48.585 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:20:48.606 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:48.630 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:48.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:48.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:48.733 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:48.764 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:48.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:48.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:48.870 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:48.894 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:48.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:48.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:48.997 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:49.025 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:49.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:49.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:49.129 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:49.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:49 smithi194 bash[92666]: cluster 2023-10-09T05:20:48.027158+0000 mgr.y (mgr.25277) 975 : cluster 0 pgmap v1018: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 28026/269784 objects degraded (10.388%); 10700/269784 objects misplaced (3.966%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T05:20:49.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:49 smithi194 bash[92666]: cluster 2023-10-09T05:20:48.513392+0000 mon.a (mon.0) 1266 : cluster 3 Health check update: Degraded data redundancy: 28026/269784 objects degraded (10.388%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:20:49.160 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:49.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:49.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:49.268 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:49.290 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:49.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:49 smithi133 bash[102858]: cluster 2023-10-09T05:20:48.027158+0000 mgr.y (mgr.25277) 975 : cluster 0 pgmap v1018: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 28026/269784 objects degraded (10.388%); 10700/269784 objects misplaced (3.966%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T05:20:49.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:49 smithi133 bash[102858]: cluster 2023-10-09T05:20:48.513392+0000 mon.a (mon.0) 1266 : cluster 3 Health check update: Degraded data redundancy: 28026/269784 objects degraded (10.388%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:20:49.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:49 smithi133 bash[105589]: cluster 2023-10-09T05:20:48.027158+0000 mgr.y (mgr.25277) 975 : cluster 0 pgmap v1018: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 28026/269784 objects degraded (10.388%); 10700/269784 objects misplaced (3.966%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T05:20:49.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:49 smithi133 bash[105589]: cluster 2023-10-09T05:20:48.513392+0000 mon.a (mon.0) 1266 : cluster 3 Health check update: Degraded data redundancy: 28026/269784 objects degraded (10.388%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:20:49.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:49.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:49.400 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:49.437 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:49.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:49.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:49.546 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:49.584 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:49.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:49.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:49.607 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:20:49.607 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:20:49.688 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:49.717 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:49.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:49.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:49.819 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:49.854 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:49.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:49.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:49.960 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:49.989 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:49.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:49.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:50.097 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:50.122 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:50.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:50.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:50.228 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:50.255 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:50.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:50.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:50.358 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:50.382 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:50.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:50.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:50.485 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:50.508 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:50.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:50.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:50.547 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T05:20:50.613 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:50.651 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:50.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:50.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:50.756 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:50.784 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:50.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:50.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:50.888 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:50.914 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:50.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:50.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:51.018 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:51.041 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:51.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:51.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:51.146 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:51.173 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:51.175 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:51 smithi133 bash[102858]: cluster 2023-10-09T05:20:50.027859+0000 mgr.y (mgr.25277) 976 : cluster 0 pgmap v1019: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 564 op/s; 28026/265581 objects degraded (10.553%); 10588/265581 objects misplaced (3.987%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T05:20:51.175 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:51 smithi133 bash[105589]: cluster 2023-10-09T05:20:50.027859+0000 mgr.y (mgr.25277) 976 : cluster 0 pgmap v1019: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 564 op/s; 28026/265581 objects degraded (10.553%); 10588/265581 objects misplaced (3.987%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T05:20:51.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:51.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:51.277 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:51.303 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:51.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:51.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:51.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:51 smithi194 bash[92666]: cluster 2023-10-09T05:20:50.027859+0000 mgr.y (mgr.25277) 976 : cluster 0 pgmap v1019: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 564 op/s; 28026/265581 objects degraded (10.553%); 10588/265581 objects misplaced (3.987%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T05:20:51.408 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:51.435 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:51.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:51.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:51.539 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:51.570 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:51.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:51.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:51.678 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:51.704 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:51.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:51.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:51.808 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:51.833 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:51.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:51.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:51.935 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:51.969 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:51.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:51.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:52.083 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:52.114 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:52.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:52.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:52.217 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:52.247 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:52.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:52.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:52.353 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:52.397 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:52.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:52.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:52.504 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:52.534 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:52.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:52.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:52.638 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:52.662 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:52.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:52.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:52.768 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:52.813 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:52.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:52.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:52.918 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:52.945 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:52.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:52.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:53.049 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:53.089 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:53.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:53.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:53.195 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:53.203 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:53 smithi194 bash[92666]: cluster 2023-10-09T05:20:52.028554+0000 mgr.y (mgr.25277) 977 : cluster 0 pgmap v1020: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 556 op/s; 28026/262248 objects degraded (10.687%); 10588/262248 objects misplaced (4.037%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T05:20:53.218 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:53.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:53.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:53.321 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:53.333 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:53 smithi133 bash[102858]: cluster 2023-10-09T05:20:52.028554+0000 mgr.y (mgr.25277) 977 : cluster 0 pgmap v1020: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 556 op/s; 28026/262248 objects degraded (10.687%); 10588/262248 objects misplaced (4.037%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T05:20:53.333 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:53 smithi133 bash[105589]: cluster 2023-10-09T05:20:52.028554+0000 mgr.y (mgr.25277) 977 : cluster 0 pgmap v1020: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 556 op/s; 28026/262248 objects degraded (10.687%); 10588/262248 objects misplaced (4.037%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T05:20:53.350 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:53.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:53.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:53.456 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:53.486 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:53.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:53.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:53.593 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:53.617 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:53.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:53.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:53.722 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:53.758 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:53.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:53.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:53.865 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:53.902 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:53.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:53.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:54.008 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:54.036 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:54.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:54.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:54.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:54 smithi194 bash[92666]: cluster 2023-10-09T05:20:53.514526+0000 mon.a (mon.0) 1267 : cluster 3 Health check update: Degraded data redundancy: 28026/262248 objects degraded (10.687%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:20:54.140 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:54.166 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:54.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:54.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:54.264 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:54 smithi133 bash[102858]: cluster 2023-10-09T05:20:53.514526+0000 mon.a (mon.0) 1267 : cluster 3 Health check update: Degraded data redundancy: 28026/262248 objects degraded (10.687%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:20:54.264 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:54 smithi133 bash[105589]: cluster 2023-10-09T05:20:53.514526+0000 mon.a (mon.0) 1267 : cluster 3 Health check update: Degraded data redundancy: 28026/262248 objects degraded (10.687%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:20:54.273 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:54.312 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:54.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:54.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:54.417 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:54.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:54.459 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:54.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:54.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:20:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:20:54] "GET /metrics HTTP/1.1" 200 36541 "" "Prometheus/2.43.0" 2023-10-09T05:20:54.560 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:54.589 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:54.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:54.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:54.694 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:54.716 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:54.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:54.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:54.820 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:54.848 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:54.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:54.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:54.951 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:54.988 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:54.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:54.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:55.091 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:55.117 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:55.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:55.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:55.224 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:55.249 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:55.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:55.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:55.276 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:55 smithi133 bash[102858]: cluster 2023-10-09T05:20:54.029157+0000 mgr.y (mgr.25277) 978 : cluster 0 pgmap v1021: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 28026/259722 objects degraded (10.791%); 10452/259722 objects misplaced (4.024%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T05:20:55.276 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:55 smithi133 bash[105589]: cluster 2023-10-09T05:20:54.029157+0000 mgr.y (mgr.25277) 978 : cluster 0 pgmap v1021: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 28026/259722 objects degraded (10.791%); 10452/259722 objects misplaced (4.024%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T05:20:55.353 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:55.362 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:55 smithi194 bash[92666]: cluster 2023-10-09T05:20:54.029157+0000 mgr.y (mgr.25277) 978 : cluster 0 pgmap v1021: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 28026/259722 objects degraded (10.791%); 10452/259722 objects misplaced (4.024%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T05:20:55.377 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:55.379 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:20:55.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:55.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:55.481 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:55.503 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:55.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:55.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:55.607 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:55.628 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:55.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:55.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:55.731 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:55.763 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:55.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:55.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:55.867 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:55.889 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:55.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:55.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:55.993 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:56.014 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:56.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:56.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:56.119 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:56.149 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:56.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:56.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:56.254 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:56.275 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:56.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:56.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:56.377 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:56.403 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:56.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:56.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:56.511 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:56.559 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:56.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:56.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:20:56.665 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:56.692 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:56.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:56.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:56.798 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:56.818 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:56.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:56.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:56.921 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:56.954 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:56.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:56.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:57.067 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:57.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:57.114 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:57.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:57.215 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:57.224 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:57 smithi133 bash[102858]: cluster 2023-10-09T05:20:56.029729+0000 mgr.y (mgr.25277) 979 : cluster 0 pgmap v1022: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 545 op/s; 28026/256437 objects degraded (10.929%); 10452/256437 objects misplaced (4.076%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:20:57.224 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:57 smithi133 bash[102858]: audit 2023-10-09T05:20:56.781147+0000 mon.a (mon.0) 1268 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:20:57.224 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:57 smithi133 bash[105589]: cluster 2023-10-09T05:20:56.029729+0000 mgr.y (mgr.25277) 979 : cluster 0 pgmap v1022: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 545 op/s; 28026/256437 objects degraded (10.929%); 10452/256437 objects misplaced (4.076%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:20:57.224 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:57 smithi133 bash[105589]: audit 2023-10-09T05:20:56.781147+0000 mon.a (mon.0) 1268 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:20:57.265 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:20:57.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:57.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:20:57.374 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:57.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:57 smithi194 bash[92666]: cluster 2023-10-09T05:20:56.029729+0000 mgr.y (mgr.25277) 979 : cluster 0 pgmap v1022: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 545 op/s; 28026/256437 objects degraded (10.929%); 10452/256437 objects misplaced (4.076%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:20:57.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:57 smithi194 bash[92666]: audit 2023-10-09T05:20:56.781147+0000 mon.a (mon.0) 1268 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:20:57.419 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:57.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:57.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:57.524 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:57.562 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:57.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:57.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:57.670 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:57.699 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:57.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:57.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:57.805 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:57.834 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:57.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:57.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:57.938 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:57.962 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:57.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:57.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:58.066 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:58.093 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:58.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:58.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:58.197 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:58.223 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:20:58.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:58.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:20:58.330 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:58.354 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:58.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:58.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:58.458 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:58.485 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:58.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:58.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:58.588 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:58.620 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:58.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:58.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:58.724 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:58.748 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:58.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:58.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:58.854 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:58.875 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:20:58.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:58.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:20:58.981 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:59.012 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:59.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:59.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:59.119 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:59.151 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:20:59.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:59.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:20:59.258 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:59.287 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:59.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:59.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:59.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:59 smithi133 bash[102858]: cluster 2023-10-09T05:20:58.030292+0000 mgr.y (mgr.25277) 980 : cluster 0 pgmap v1023: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 28026/253908 objects degraded (11.038%); 10452/253908 objects misplaced (4.116%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:20:59.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:59 smithi133 bash[102858]: audit 2023-10-09T05:20:58.455562+0000 mon.a (mon.0) 1269 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:20:59.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:20:59 smithi133 bash[102858]: cluster 2023-10-09T05:20:58.515909+0000 mon.a (mon.0) 1270 : cluster 3 Health check update: Degraded data redundancy: 28026/253908 objects degraded (11.038%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:20:59.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:59 smithi133 bash[105589]: cluster 2023-10-09T05:20:58.030292+0000 mgr.y (mgr.25277) 980 : cluster 0 pgmap v1023: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 28026/253908 objects degraded (11.038%); 10452/253908 objects misplaced (4.116%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:20:59.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:59 smithi133 bash[105589]: audit 2023-10-09T05:20:58.455562+0000 mon.a (mon.0) 1269 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:20:59.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:20:59 smithi133 bash[105589]: cluster 2023-10-09T05:20:58.515909+0000 mon.a (mon.0) 1270 : cluster 3 Health check update: Degraded data redundancy: 28026/253908 objects degraded (11.038%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:20:59.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:59 smithi194 bash[92666]: cluster 2023-10-09T05:20:58.030292+0000 mgr.y (mgr.25277) 980 : cluster 0 pgmap v1023: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 28026/253908 objects degraded (11.038%); 10452/253908 objects misplaced (4.116%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:20:59.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:59 smithi194 bash[92666]: audit 2023-10-09T05:20:58.455562+0000 mon.a (mon.0) 1269 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:20:59.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:20:59 smithi194 bash[92666]: cluster 2023-10-09T05:20:58.515909+0000 mon.a (mon.0) 1270 : cluster 3 Health check update: Degraded data redundancy: 28026/253908 objects degraded (11.038%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:20:59.390 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:59.417 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:59.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:59.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:59.521 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:59.545 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:20:59.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:59.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:20:59.648 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:59.677 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:20:59.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:59.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:20:59.745 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:20:59.745 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:20:59.781 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:59.818 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:20:59.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:59.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:20:59.922 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:59.951 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:20:59.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:20:59.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:00.059 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:00.083 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:00.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:00.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:00.188 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:00.213 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:00.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:00.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:00.321 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:00.361 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:00.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:00.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:00.465 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:00.504 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:00.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:00.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:00.612 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:00.646 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:00.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:00.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:00.751 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:00.768 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:00.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:00.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:00.872 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:00.873 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T05:21:00.901 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:00.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:00.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:01.004 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:01.033 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:01.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:01.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:01.122 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:01 smithi133 bash[102858]: cluster 2023-10-09T05:21:00.030994+0000 mgr.y (mgr.25277) 981 : cluster 0 pgmap v1024: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 28026/249831 objects degraded (11.218%); 10326/249831 objects misplaced (4.133%); 1.9 MiB/s, 31 objects/s recovering 2023-10-09T05:21:01.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:01 smithi194 bash[92666]: cluster 2023-10-09T05:21:00.030994+0000 mgr.y (mgr.25277) 981 : cluster 0 pgmap v1024: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 28026/249831 objects degraded (11.218%); 10326/249831 objects misplaced (4.133%); 1.9 MiB/s, 31 objects/s recovering 2023-10-09T05:21:01.135 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:01.162 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:01.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:01.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:01.266 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:01.295 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:01.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:01.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:01.401 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:01.427 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:01.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:01.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:01.533 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:01.542 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:01 smithi133 bash[105589]: cluster 2023-10-09T05:21:00.030994+0000 mgr.y (mgr.25277) 981 : cluster 0 pgmap v1024: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 28026/249831 objects degraded (11.218%); 10326/249831 objects misplaced (4.133%); 1.9 MiB/s, 31 objects/s recovering 2023-10-09T05:21:01.561 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:01.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:01.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:01.664 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:01.705 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:01.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:01.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:01.811 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:01.836 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:01.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:01.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:01.939 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:01.967 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:01.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:01.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:02.073 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:02.101 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:02.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:02.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:02.204 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:02.227 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:02.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:02.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:02.330 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:02.370 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:02.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:02.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:02.480 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:02.531 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:02.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:02.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:02.605 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:21:02.636 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:02.662 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:21:02.689 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:02.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:02.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:02.702 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:21:02.743 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:21:02.786 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:21:02.800 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:02.836 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:21:02.851 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:02.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:02.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:02.881 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:21:02.937 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:21:02.945 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:21:02.945 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:21:02.945 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:21:02.945 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:21:02.945 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:21:02.945 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:21:02.946 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:21:02.946 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:21:02.957 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:02.987 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:02.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:02.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:03.095 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:03.120 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:03.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:03.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:03.225 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:03.237 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:03 smithi133 bash[102858]: cluster 2023-10-09T05:21:02.031656+0000 mgr.y (mgr.25277) 982 : cluster 0 pgmap v1025: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 28026/246351 objects degraded (11.376%); 10326/246351 objects misplaced (4.192%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T05:21:03.237 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:03 smithi133 bash[102858]: audit 2023-10-09T05:21:02.548424+0000 mon.c (mon.2) 92 : audit 0 from='client.? 172.21.15.133:0/2195106343' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:21:03.237 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:03 smithi133 bash[105589]: cluster 2023-10-09T05:21:02.031656+0000 mgr.y (mgr.25277) 982 : cluster 0 pgmap v1025: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 28026/246351 objects degraded (11.376%); 10326/246351 objects misplaced (4.192%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T05:21:03.238 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:03 smithi133 bash[105589]: audit 2023-10-09T05:21:02.548424+0000 mon.c (mon.2) 92 : audit 0 from='client.? 172.21.15.133:0/2195106343' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:21:03.262 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:03.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:03.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:03.375 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:03.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:03 smithi194 bash[92666]: cluster 2023-10-09T05:21:02.031656+0000 mgr.y (mgr.25277) 982 : cluster 0 pgmap v1025: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 28026/246351 objects degraded (11.376%); 10326/246351 objects misplaced (4.192%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T05:21:03.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:03 smithi194 bash[92666]: audit 2023-10-09T05:21:02.548424+0000 mon.c (mon.2) 92 : audit 0 from='client.? 172.21.15.133:0/2195106343' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:21:03.406 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:03.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:03.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:03.509 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:03.537 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:03.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:03.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:03.641 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:03.667 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:03.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:03.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:03.773 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:03.799 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:03.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:03.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:03.903 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:03.936 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:03.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:03.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:04.039 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:04.062 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:04.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:04.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:04.165 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:04.206 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:04.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:04.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:04.311 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:04.320 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:04 smithi194 bash[92666]: cluster 2023-10-09T05:21:03.521271+0000 mon.a (mon.0) 1271 : cluster 3 Health check update: Degraded data redundancy: 28026/246351 objects degraded (11.376%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:21:04.321 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:04 smithi194 bash[92666]: audit 2023-10-09T05:21:03.560985+0000 mon.a (mon.0) 1272 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:21:04.321 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:04 smithi194 bash[92666]: audit 2023-10-09T05:21:03.569268+0000 mon.a (mon.0) 1273 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:21:04.321 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:04 smithi194 bash[92666]: audit 2023-10-09T05:21:04.015513+0000 mon.a (mon.0) 1274 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:21:04.321 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:04 smithi194 bash[92666]: audit 2023-10-09T05:21:04.032265+0000 mon.a (mon.0) 1275 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:21:04.345 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:04.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:04.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:04.449 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:04.457 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:21:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:21:04] "GET /metrics HTTP/1.1" 200 36546 "" "Prometheus/2.43.0" 2023-10-09T05:21:04.458 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:04 smithi133 bash[105589]: cluster 2023-10-09T05:21:03.521271+0000 mon.a (mon.0) 1271 : cluster 3 Health check update: Degraded data redundancy: 28026/246351 objects degraded (11.376%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:21:04.458 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:04 smithi133 bash[105589]: audit 2023-10-09T05:21:03.560985+0000 mon.a (mon.0) 1272 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:21:04.458 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:04 smithi133 bash[105589]: audit 2023-10-09T05:21:03.569268+0000 mon.a (mon.0) 1273 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:21:04.458 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:04 smithi133 bash[105589]: audit 2023-10-09T05:21:04.015513+0000 mon.a (mon.0) 1274 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:21:04.458 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:04 smithi133 bash[105589]: audit 2023-10-09T05:21:04.032265+0000 mon.a (mon.0) 1275 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:21:04.458 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:04 smithi133 bash[102858]: cluster 2023-10-09T05:21:03.521271+0000 mon.a (mon.0) 1271 : cluster 3 Health check update: Degraded data redundancy: 28026/246351 objects degraded (11.376%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:21:04.458 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:04 smithi133 bash[102858]: audit 2023-10-09T05:21:03.560985+0000 mon.a (mon.0) 1272 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:21:04.458 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:04 smithi133 bash[102858]: audit 2023-10-09T05:21:03.569268+0000 mon.a (mon.0) 1273 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:21:04.459 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:04 smithi133 bash[102858]: audit 2023-10-09T05:21:04.015513+0000 mon.a (mon.0) 1274 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:21:04.459 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:04 smithi133 bash[102858]: audit 2023-10-09T05:21:04.032265+0000 mon.a (mon.0) 1275 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:21:04.485 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:04.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:04.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:04.589 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:04.612 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:04.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:04.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:04.721 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:04.746 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:04.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:04.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:04.850 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:04.876 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:04.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:04.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:04.980 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:05.038 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:05.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:05.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:05.143 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:05.184 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:05.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:05.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:05.288 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:05.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:05 smithi133 bash[102858]: cluster 2023-10-09T05:21:04.035261+0000 mgr.y (mgr.25277) 983 : cluster 0 pgmap v1026: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 507 op/s; 28026/243972 objects degraded (11.487%); 10220/243972 objects misplaced (4.189%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:21:05.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:05 smithi133 bash[102858]: audit 2023-10-09T05:21:04.675651+0000 mon.a (mon.0) 1276 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:21:05.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:05 smithi133 bash[102858]: audit 2023-10-09T05:21:04.676985+0000 mon.a (mon.0) 1277 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:21:05.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:05 smithi133 bash[102858]: audit 2023-10-09T05:21:04.686516+0000 mon.a (mon.0) 1278 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:21:05.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:05 smithi133 bash[105589]: cluster 2023-10-09T05:21:04.035261+0000 mgr.y (mgr.25277) 983 : cluster 0 pgmap v1026: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 507 op/s; 28026/243972 objects degraded (11.487%); 10220/243972 objects misplaced (4.189%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:21:05.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:05 smithi133 bash[105589]: audit 2023-10-09T05:21:04.675651+0000 mon.a (mon.0) 1276 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:21:05.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:05 smithi133 bash[105589]: audit 2023-10-09T05:21:04.676985+0000 mon.a (mon.0) 1277 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:21:05.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:05 smithi133 bash[105589]: audit 2023-10-09T05:21:04.686516+0000 mon.a (mon.0) 1278 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:21:05.315 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:05.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:05.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:05.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:05 smithi194 bash[92666]: cluster 2023-10-09T05:21:04.035261+0000 mgr.y (mgr.25277) 983 : cluster 0 pgmap v1026: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 507 op/s; 28026/243972 objects degraded (11.487%); 10220/243972 objects misplaced (4.189%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:21:05.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:05 smithi194 bash[92666]: audit 2023-10-09T05:21:04.675651+0000 mon.a (mon.0) 1276 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:21:05.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:05 smithi194 bash[92666]: audit 2023-10-09T05:21:04.676985+0000 mon.a (mon.0) 1277 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:21:05.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:05 smithi194 bash[92666]: audit 2023-10-09T05:21:04.686516+0000 mon.a (mon.0) 1278 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:21:05.401 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:21:05.422 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:05.460 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:05.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:05.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:05.565 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:05.603 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:05.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:05.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:05.709 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:05.736 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:05.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:05.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:05.839 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:05.860 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:05.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:05.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:05.967 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:05.992 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:05.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:05.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:06.100 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:06.128 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:06.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:06.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:06.150 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is unset 2023-10-09T05:21:06.233 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:06.269 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:06.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:06.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:06.376 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:06.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:06 smithi194 bash[92666]: audit 2023-10-09T05:21:05.651517+0000 mon.c (mon.2) 93 : audit 1 from='client.? 172.21.15.133:0/165203166' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:21:06.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:06 smithi194 bash[92666]: audit 2023-10-09T05:21:05.652202+0000 mon.a (mon.0) 1279 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:21:06.401 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:06.403 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:06 smithi133 bash[102858]: audit 2023-10-09T05:21:05.651517+0000 mon.c (mon.2) 93 : audit 1 from='client.? 172.21.15.133:0/165203166' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:21:06.403 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:06 smithi133 bash[102858]: audit 2023-10-09T05:21:05.652202+0000 mon.a (mon.0) 1279 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:21:06.404 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:06 smithi133 bash[105589]: audit 2023-10-09T05:21:05.651517+0000 mon.c (mon.2) 93 : audit 1 from='client.? 172.21.15.133:0/165203166' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:21:06.404 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:06 smithi133 bash[105589]: audit 2023-10-09T05:21:05.652202+0000 mon.a (mon.0) 1279 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:21:06.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:06.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:06.507 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:06.544 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:06.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:06.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:06.647 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:06.675 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:06.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:06.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:06.784 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:06.811 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:06.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:06.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:06.917 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:06.956 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:06.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:06.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:07.062 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:07.086 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:07.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:07.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:07.157 INFO:teuthology.orchestra.run.smithi133.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T05:21:07.186 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:07 smithi133 bash[105589]: cluster 2023-10-09T05:21:06.035979+0000 mgr.y (mgr.25277) 984 : cluster 0 pgmap v1027: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 524 op/s; 28026/240813 objects degraded (11.638%); 10220/240813 objects misplaced (4.244%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T05:21:07.186 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:07 smithi133 bash[105589]: cluster 2023-10-09T05:21:06.129931+0000 mon.a (mon.0) 1280 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:21:07.187 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:07 smithi133 bash[105589]: cluster 2023-10-09T05:21:06.133333+0000 osd.5 (osd.5) 227 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T05:21:07.187 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:07 smithi133 bash[105589]: audit 2023-10-09T05:21:06.136479+0000 mon.a (mon.0) 1281 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:21:07.187 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:07 smithi133 bash[105589]: cluster 2023-10-09T05:21:06.136567+0000 mon.a (mon.0) 1282 : cluster 0 osdmap e241: 8 total, 8 up, 7 in 2023-10-09T05:21:07.187 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:07 smithi133 bash[105589]: cluster 2023-10-09T05:21:06.137747+0000 osd.7 (osd.7) 188 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T05:21:07.187 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:07 smithi133 bash[105589]: cluster 2023-10-09T05:21:06.150686+0000 osd.4 (osd.4) 204 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T05:21:07.187 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:07 smithi133 bash[105589]: cluster 2023-10-09T05:21:06.153375+0000 osd.4 (osd.4) 205 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T05:21:07.187 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:07 smithi133 bash[105589]: cluster 2023-10-09T05:21:06.173778+0000 osd.6 (osd.6) 180 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T05:21:07.187 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:07 smithi133 bash[105589]: cluster 2023-10-09T05:21:06.176838+0000 osd.6 (osd.6) 181 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T05:21:07.187 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:07 smithi133 bash[105589]: audit 2023-10-09T05:21:06.245562+0000 mon.b (mon.1) 20 : audit 1 from='client.? 172.21.15.133:0/2347068960' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:21:07.188 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:07 smithi133 bash[105589]: audit 2023-10-09T05:21:06.246574+0000 mon.a (mon.0) 1283 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:21:07.188 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:07 smithi133 bash[102858]: cluster 2023-10-09T05:21:06.035979+0000 mgr.y (mgr.25277) 984 : cluster 0 pgmap v1027: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 524 op/s; 28026/240813 objects degraded (11.638%); 10220/240813 objects misplaced (4.244%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T05:21:07.188 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:07 smithi133 bash[102858]: cluster 2023-10-09T05:21:06.129931+0000 mon.a (mon.0) 1280 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:21:07.188 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:07 smithi133 bash[102858]: cluster 2023-10-09T05:21:06.133333+0000 osd.5 (osd.5) 227 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T05:21:07.188 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:07 smithi133 bash[102858]: audit 2023-10-09T05:21:06.136479+0000 mon.a (mon.0) 1281 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:21:07.189 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:07 smithi133 bash[102858]: cluster 2023-10-09T05:21:06.136567+0000 mon.a (mon.0) 1282 : cluster 0 osdmap e241: 8 total, 8 up, 7 in 2023-10-09T05:21:07.189 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:07 smithi133 bash[102858]: cluster 2023-10-09T05:21:06.137747+0000 osd.7 (osd.7) 188 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T05:21:07.189 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:07 smithi133 bash[102858]: cluster 2023-10-09T05:21:06.150686+0000 osd.4 (osd.4) 204 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T05:21:07.189 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:07 smithi133 bash[102858]: cluster 2023-10-09T05:21:06.153375+0000 osd.4 (osd.4) 205 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T05:21:07.189 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:07 smithi133 bash[102858]: cluster 2023-10-09T05:21:06.173778+0000 osd.6 (osd.6) 180 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T05:21:07.189 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:07 smithi133 bash[102858]: cluster 2023-10-09T05:21:06.176838+0000 osd.6 (osd.6) 181 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T05:21:07.189 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:07 smithi133 bash[102858]: audit 2023-10-09T05:21:06.245562+0000 mon.b (mon.1) 20 : audit 1 from='client.? 172.21.15.133:0/2347068960' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:21:07.189 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:07 smithi133 bash[102858]: audit 2023-10-09T05:21:06.246574+0000 mon.a (mon.0) 1283 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:21:07.190 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:07.222 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:07.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:07.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:07.328 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:07.357 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:07.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:07.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:07.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:07 smithi194 bash[92666]: cluster 2023-10-09T05:21:06.035979+0000 mgr.y (mgr.25277) 984 : cluster 0 pgmap v1027: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 524 op/s; 28026/240813 objects degraded (11.638%); 10220/240813 objects misplaced (4.244%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T05:21:07.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:07 smithi194 bash[92666]: cluster 2023-10-09T05:21:06.129931+0000 mon.a (mon.0) 1280 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:21:07.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:07 smithi194 bash[92666]: cluster 2023-10-09T05:21:06.133333+0000 osd.5 (osd.5) 227 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T05:21:07.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:07 smithi194 bash[92666]: audit 2023-10-09T05:21:06.136479+0000 mon.a (mon.0) 1281 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:21:07.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:07 smithi194 bash[92666]: cluster 2023-10-09T05:21:06.136567+0000 mon.a (mon.0) 1282 : cluster 0 osdmap e241: 8 total, 8 up, 7 in 2023-10-09T05:21:07.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:07 smithi194 bash[92666]: cluster 2023-10-09T05:21:06.137747+0000 osd.7 (osd.7) 188 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T05:21:07.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:07 smithi194 bash[92666]: cluster 2023-10-09T05:21:06.150686+0000 osd.4 (osd.4) 204 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T05:21:07.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:07 smithi194 bash[92666]: cluster 2023-10-09T05:21:06.153375+0000 osd.4 (osd.4) 205 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T05:21:07.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:07 smithi194 bash[92666]: cluster 2023-10-09T05:21:06.173778+0000 osd.6 (osd.6) 180 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T05:21:07.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:07 smithi194 bash[92666]: cluster 2023-10-09T05:21:06.176838+0000 osd.6 (osd.6) 181 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T05:21:07.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:07 smithi194 bash[92666]: audit 2023-10-09T05:21:06.245562+0000 mon.b (mon.1) 20 : audit 1 from='client.? 172.21.15.133:0/2347068960' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:21:07.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:07 smithi194 bash[92666]: audit 2023-10-09T05:21:06.246574+0000 mon.a (mon.0) 1283 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:21:07.464 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:07.492 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:07.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:07.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:07.595 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:07.616 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:07.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:07.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:07.719 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:07.742 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:07.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:07.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:07.845 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:07.866 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:07.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:07.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:07.890 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:21:07.970 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:07.997 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:07.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:08.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:08.100 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:08.131 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:08.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:08.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:08.158 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:08 smithi133 bash[102858]: audit 2023-10-09T05:21:07.146311+0000 mon.a (mon.0) 1284 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:21:08.158 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:08 smithi133 bash[102858]: cluster 2023-10-09T05:21:07.146443+0000 mon.a (mon.0) 1285 : cluster 0 osdmap e242: 8 total, 8 up, 7 in 2023-10-09T05:21:08.159 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:08 smithi133 bash[102858]: cluster 2023-10-09T05:21:07.154276+0000 osd.7 (osd.7) 189 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T05:21:08.159 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:08 smithi133 bash[102858]: cluster 2023-10-09T05:21:07.155187+0000 osd.4 (osd.4) 206 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T05:21:08.159 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:08 smithi133 bash[102858]: cluster 2023-10-09T05:21:07.156662+0000 osd.6 (osd.6) 182 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T05:21:08.159 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:08 smithi133 bash[102858]: cluster 2023-10-09T05:21:07.157203+0000 osd.7 (osd.7) 190 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T05:21:08.234 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:08.257 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:08.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:08.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:08.364 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:08.398 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:08.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:08.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:08.507 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:08.516 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:08 smithi133 bash[105589]: audit 2023-10-09T05:21:07.146311+0000 mon.a (mon.0) 1284 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:21:08.516 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:08 smithi133 bash[105589]: cluster 2023-10-09T05:21:07.146443+0000 mon.a (mon.0) 1285 : cluster 0 osdmap e242: 8 total, 8 up, 7 in 2023-10-09T05:21:08.516 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:08 smithi133 bash[105589]: cluster 2023-10-09T05:21:07.154276+0000 osd.7 (osd.7) 189 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T05:21:08.516 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:08 smithi133 bash[105589]: cluster 2023-10-09T05:21:07.155187+0000 osd.4 (osd.4) 206 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T05:21:08.517 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:08 smithi133 bash[105589]: cluster 2023-10-09T05:21:07.156662+0000 osd.6 (osd.6) 182 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T05:21:08.517 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:08 smithi133 bash[105589]: cluster 2023-10-09T05:21:07.157203+0000 osd.7 (osd.7) 190 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T05:21:08.532 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:08.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:08.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:08.635 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:08.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:08 smithi194 bash[92666]: audit 2023-10-09T05:21:07.146311+0000 mon.a (mon.0) 1284 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:21:08.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:08 smithi194 bash[92666]: cluster 2023-10-09T05:21:07.146443+0000 mon.a (mon.0) 1285 : cluster 0 osdmap e242: 8 total, 8 up, 7 in 2023-10-09T05:21:08.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:08 smithi194 bash[92666]: cluster 2023-10-09T05:21:07.154276+0000 osd.7 (osd.7) 189 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T05:21:08.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:08 smithi194 bash[92666]: cluster 2023-10-09T05:21:07.155187+0000 osd.4 (osd.4) 206 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T05:21:08.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:08 smithi194 bash[92666]: cluster 2023-10-09T05:21:07.156662+0000 osd.6 (osd.6) 182 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T05:21:08.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:08 smithi194 bash[92666]: cluster 2023-10-09T05:21:07.157203+0000 osd.7 (osd.7) 190 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T05:21:08.658 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:08.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:08.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:08.767 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:08.815 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:08.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:08.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:08.924 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:08.961 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:08.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:08.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:09.067 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:09.105 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:09.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:09.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:09.141 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T05:21:09.212 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:09.240 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:09.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:09.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:09.344 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:09.369 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:09.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:09.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:09.473 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:09.481 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:09 smithi194 bash[92666]: cluster 2023-10-09T05:21:08.036741+0000 mgr.y (mgr.25277) 985 : cluster 0 pgmap v1030: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 471 op/s; 28026/238503 objects degraded (11.751%); 10220/238503 objects misplaced (4.285%); 847 KiB/s, 13 objects/s recovering 2023-10-09T05:21:09.481 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:09 smithi194 bash[92666]: cluster 2023-10-09T05:21:08.159871+0000 osd.5 (osd.5) 228 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T05:21:09.481 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:09 smithi194 bash[92666]: cluster 2023-10-09T05:21:08.161079+0000 mon.a (mon.0) 1286 : cluster 0 osdmap e243: 8 total, 8 up, 7 in 2023-10-09T05:21:09.481 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:09 smithi194 bash[92666]: cluster 2023-10-09T05:21:08.166064+0000 osd.7 (osd.7) 191 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T05:21:09.481 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:09 smithi194 bash[92666]: cluster 2023-10-09T05:21:08.166606+0000 osd.4 (osd.4) 207 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T05:21:09.481 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:09 smithi194 bash[92666]: cluster 2023-10-09T05:21:08.170540+0000 osd.6 (osd.6) 183 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T05:21:09.481 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:09 smithi194 bash[92666]: cluster 2023-10-09T05:21:08.522507+0000 mon.a (mon.0) 1287 : cluster 3 Health check update: Degraded data redundancy: 28026/238503 objects degraded (11.751%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:21:09.495 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:09.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:09.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:09.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:09 smithi133 bash[105589]: cluster 2023-10-09T05:21:08.036741+0000 mgr.y (mgr.25277) 985 : cluster 0 pgmap v1030: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 471 op/s; 28026/238503 objects degraded (11.751%); 10220/238503 objects misplaced (4.285%); 847 KiB/s, 13 objects/s recovering 2023-10-09T05:21:09.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:09 smithi133 bash[105589]: cluster 2023-10-09T05:21:08.159871+0000 osd.5 (osd.5) 228 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T05:21:09.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:09 smithi133 bash[105589]: cluster 2023-10-09T05:21:08.161079+0000 mon.a (mon.0) 1286 : cluster 0 osdmap e243: 8 total, 8 up, 7 in 2023-10-09T05:21:09.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:09 smithi133 bash[105589]: cluster 2023-10-09T05:21:08.166064+0000 osd.7 (osd.7) 191 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T05:21:09.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:09 smithi133 bash[105589]: cluster 2023-10-09T05:21:08.166606+0000 osd.4 (osd.4) 207 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T05:21:09.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:09 smithi133 bash[105589]: cluster 2023-10-09T05:21:08.170540+0000 osd.6 (osd.6) 183 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T05:21:09.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:09 smithi133 bash[105589]: cluster 2023-10-09T05:21:08.522507+0000 mon.a (mon.0) 1287 : cluster 3 Health check update: Degraded data redundancy: 28026/238503 objects degraded (11.751%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:21:09.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:09 smithi133 bash[102858]: cluster 2023-10-09T05:21:08.036741+0000 mgr.y (mgr.25277) 985 : cluster 0 pgmap v1030: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 471 op/s; 28026/238503 objects degraded (11.751%); 10220/238503 objects misplaced (4.285%); 847 KiB/s, 13 objects/s recovering 2023-10-09T05:21:09.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:09 smithi133 bash[102858]: cluster 2023-10-09T05:21:08.159871+0000 osd.5 (osd.5) 228 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T05:21:09.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:09 smithi133 bash[102858]: cluster 2023-10-09T05:21:08.161079+0000 mon.a (mon.0) 1286 : cluster 0 osdmap e243: 8 total, 8 up, 7 in 2023-10-09T05:21:09.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:09 smithi133 bash[102858]: cluster 2023-10-09T05:21:08.166064+0000 osd.7 (osd.7) 191 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T05:21:09.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:09 smithi133 bash[102858]: cluster 2023-10-09T05:21:08.166606+0000 osd.4 (osd.4) 207 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T05:21:09.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:09 smithi133 bash[102858]: cluster 2023-10-09T05:21:08.170540+0000 osd.6 (osd.6) 183 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T05:21:09.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:09 smithi133 bash[102858]: cluster 2023-10-09T05:21:08.522507+0000 mon.a (mon.0) 1287 : cluster 3 Health check update: Degraded data redundancy: 28026/238503 objects degraded (11.751%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:21:09.599 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:09.636 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:09.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:09.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:09.739 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:09.772 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:09.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:09.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:09.877 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:09.895 INFO:teuthology.orchestra.run.smithi194.stdout:{ 2023-10-09T05:21:09.896 INFO:teuthology.orchestra.run.smithi194.stdout: "size": 20, 2023-10-09T05:21:09.896 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 600, 2023-10-09T05:21:09.896 INFO:teuthology.orchestra.run.smithi194.stdout: "ops": [ 2023-10-09T05:21:09.896 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.896 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:158229 9.f 9:f3688f26:::benchmark_data_smithi133_153303_object68260:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T05:21:09.896 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:12:32.243288+0000", 2023-10-09T05:21:09.896 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 517.65087737900001, 2023-10-09T05:21:09.896 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0733340659999999, 2023-10-09T05:21:09.896 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:21:09.896 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "reached pg", 2023-10-09T05:21:09.896 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:21:09.897 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:21:09.897 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:21:09.897 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 158229 2023-10-09T05:21:09.897 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.897 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:21:09.897 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.897 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:21:09.897 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.243288+0000", 2023-10-09T05:21:09.897 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:09.897 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.897 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.897 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:21:09.898 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.243288+0000", 2023-10-09T05:21:09.898 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:09.898 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.898 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.898 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:21:09.898 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.243288+0000", 2023-10-09T05:21:09.898 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999995 2023-10-09T05:21:09.898 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.898 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.899 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:21:09.899 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.243291+0000", 2023-10-09T05:21:09.899 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.472e-06 2023-10-09T05:21:09.899 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.899 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.899 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:21:09.899 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.243298+0000", 2023-10-09T05:21:09.899 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.4119999999999998e-06 2023-10-09T05:21:09.899 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.899 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.899 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:21:09.900 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.243301+0000", 2023-10-09T05:21:09.900 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.7079999999999999e-06 2023-10-09T05:21:09.900 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.900 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.900 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:09.900 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.377390+0000", 2023-10-09T05:21:09.901 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.13408895000000001 2023-10-09T05:21:09.901 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.901 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.901 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:21:09.901 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.377403+0000", 2023-10-09T05:21:09.902 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.2683999999999999e-05 2023-10-09T05:21:09.902 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.902 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.902 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:09.902 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.060570+0000", 2023-10-09T05:21:09.903 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.68316669500000005 2023-10-09T05:21:09.903 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.903 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.904 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:21:09.904 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.060584+0000", 2023-10-09T05:21:09.904 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.3901e-05 2023-10-09T05:21:09.904 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.904 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.905 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:09.905 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.316545+0000", 2023-10-09T05:21:09.905 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.255961404 2023-10-09T05:21:09.905 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.905 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.905 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:21:09.905 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.316622+0000", 2023-10-09T05:21:09.905 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.7342000000000006e-05 2023-10-09T05:21:09.905 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:09.906 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:21:09.906 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:09.906 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.906 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.906 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:158265 9.1a 9:5952e389:::benchmark_data_smithi133_153303_object68296:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T05:21:09.906 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:12:32.379388+0000", 2023-10-09T05:21:09.906 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 517.51477807100002, 2023-10-09T05:21:09.906 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.9586892090000001, 2023-10-09T05:21:09.906 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:21:09.906 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:21:09.906 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:21:09.906 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:21:09.907 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:21:09.907 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 158265 2023-10-09T05:21:09.907 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.907 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:21:09.907 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.907 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:21:09.907 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.379388+0000", 2023-10-09T05:21:09.907 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:09.907 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.907 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.907 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:21:09.907 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.379388+0000", 2023-10-09T05:21:09.907 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:09.908 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.908 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.908 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:21:09.908 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.379385+0000", 2023-10-09T05:21:09.908 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999976 2023-10-09T05:21:09.908 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.908 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.908 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:21:09.908 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.379397+0000", 2023-10-09T05:21:09.909 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.1294000000000001e-05 2023-10-09T05:21:09.909 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.909 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.909 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:21:09.909 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.379418+0000", 2023-10-09T05:21:09.909 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.1435000000000001e-05 2023-10-09T05:21:09.909 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.909 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.909 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:21:09.909 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.379432+0000", 2023-10-09T05:21:09.909 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.3412e-05 2023-10-09T05:21:09.909 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.910 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.910 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:09.910 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.381288+0000", 2023-10-09T05:21:09.910 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0018564300000000001 2023-10-09T05:21:09.910 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.910 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.910 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:21:09.910 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:32.381297+0000", 2023-10-09T05:21:09.910 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.6309999999999992e-06 2023-10-09T05:21:09.910 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.910 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.910 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:09.911 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.298632+0000", 2023-10-09T05:21:09.911 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.91733491 2023-10-09T05:21:09.911 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.911 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.911 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:21:09.911 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.298649+0000", 2023-10-09T05:21:09.911 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.7226000000000001e-05 2023-10-09T05:21:09.911 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.911 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.911 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:09.911 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.327319+0000", 2023-10-09T05:21:09.911 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0286702700000001 2023-10-09T05:21:09.912 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.912 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.912 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:21:09.912 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.327438+0000", 2023-10-09T05:21:09.912 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00011914200000000001 2023-10-09T05:21:09.912 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.912 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.912 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:21:09.912 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.327706+0000", 2023-10-09T05:21:09.912 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00026727000000000001 2023-10-09T05:21:09.912 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.912 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.912 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:21:09.913 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.333710+0000", 2023-10-09T05:21:09.913 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0060048369999999998 2023-10-09T05:21:09.913 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.913 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.913 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:21:09.913 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.333745+0000", 2023-10-09T05:21:09.913 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.4903000000000001e-05 2023-10-09T05:21:09.913 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.913 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.913 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:21:09.913 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.337919+0000", 2023-10-09T05:21:09.913 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0041737730000000004 2023-10-09T05:21:09.914 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.914 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.914 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:21:09.914 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.337965+0000", 2023-10-09T05:21:09.914 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.6375000000000002e-05 2023-10-09T05:21:09.914 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.914 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.914 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:21:09.914 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.338077+0000", 2023-10-09T05:21:09.914 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000111506 2023-10-09T05:21:09.914 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:09.914 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:21:09.915 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:09.915 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.915 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.915 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:158324 9.1a 9:5fd6673b:::benchmark_data_smithi133_153303_object68355:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-10-09T05:21:09.915 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:12:33.188885+0000", 2023-10-09T05:21:09.915 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 516.70528097800002, 2023-10-09T05:21:09.915 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.1557682920000001, 2023-10-09T05:21:09.915 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:21:09.915 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:21:09.915 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:21:09.915 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:21:09.915 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:21:09.916 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 158324 2023-10-09T05:21:09.916 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.916 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:21:09.916 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.916 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:21:09.916 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.188885+0000", 2023-10-09T05:21:09.916 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:09.916 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.916 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.916 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:21:09.916 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.188885+0000", 2023-10-09T05:21:09.916 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:09.917 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.917 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.917 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:21:09.917 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.188883+0000", 2023-10-09T05:21:09.917 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999981 2023-10-09T05:21:09.917 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.917 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.917 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:21:09.917 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.188890+0000", 2023-10-09T05:21:09.917 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.5780000000000004e-06 2023-10-09T05:21:09.917 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.917 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.917 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:21:09.918 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.188906+0000", 2023-10-09T05:21:09.918 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.6262000000000001e-05 2023-10-09T05:21:09.918 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.918 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.918 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:21:09.918 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.188913+0000", 2023-10-09T05:21:09.918 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.6229999999999997e-06 2023-10-09T05:21:09.918 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.918 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.918 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:09.918 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.188965+0000", 2023-10-09T05:21:09.919 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.1483999999999997e-05 2023-10-09T05:21:09.919 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.919 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.919 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:21:09.919 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.188971+0000", 2023-10-09T05:21:09.919 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.4110000000000001e-06 2023-10-09T05:21:09.919 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.919 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.919 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:09.919 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.299802+0000", 2023-10-09T05:21:09.919 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.110830597 2023-10-09T05:21:09.919 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.919 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.920 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:21:09.920 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.299808+0000", 2023-10-09T05:21:09.920 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.4250000000000003e-06 2023-10-09T05:21:09.920 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.920 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.920 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:09.920 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.333771+0000", 2023-10-09T05:21:09.920 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0339623469999999 2023-10-09T05:21:09.920 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.920 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.920 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:21:09.920 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.333902+0000", 2023-10-09T05:21:09.921 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00013147500000000001 2023-10-09T05:21:09.921 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.921 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.921 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:21:09.921 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.334120+0000", 2023-10-09T05:21:09.921 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00021756100000000001 2023-10-09T05:21:09.921 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.921 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.921 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:21:09.921 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.342368+0000", 2023-10-09T05:21:09.921 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0082480599999999998 2023-10-09T05:21:09.921 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.921 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.922 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:21:09.922 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.344309+0000", 2023-10-09T05:21:09.922 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.001941396 2023-10-09T05:21:09.922 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.922 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.922 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:21:09.922 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.344351+0000", 2023-10-09T05:21:09.922 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.1949000000000001e-05 2023-10-09T05:21:09.922 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.922 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.922 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:21:09.922 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.344407+0000", 2023-10-09T05:21:09.923 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.5717000000000002e-05 2023-10-09T05:21:09.923 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.923 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.923 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:21:09.923 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.344653+0000", 2023-10-09T05:21:09.923 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000246185 2023-10-09T05:21:09.923 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:09.923 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:21:09.923 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:09.923 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.923 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.923 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:158326 9.1a 9:5f6f61db:::benchmark_data_smithi133_153303_object68357:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-10-09T05:21:09.924 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:12:33.191808+0000", 2023-10-09T05:21:09.924 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 516.70235728900002, 2023-10-09T05:21:09.924 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.153272385, 2023-10-09T05:21:09.924 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:21:09.924 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:21:09.924 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:21:09.924 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:21:09.924 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:21:09.924 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 158326 2023-10-09T05:21:09.924 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.924 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:21:09.924 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.925 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:21:09.925 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.191808+0000", 2023-10-09T05:21:09.925 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:09.925 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.925 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.925 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:21:09.925 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.191808+0000", 2023-10-09T05:21:09.925 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:09.925 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.925 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.925 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:21:09.925 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.191807+0000", 2023-10-09T05:21:09.925 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:21:09.926 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.926 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.926 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:21:09.926 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.191815+0000", 2023-10-09T05:21:09.926 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.8180000000000004e-06 2023-10-09T05:21:09.926 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.926 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.926 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:21:09.926 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.191829+0000", 2023-10-09T05:21:09.926 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.3974999999999999e-05 2023-10-09T05:21:09.926 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.926 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.927 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:21:09.927 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.191837+0000", 2023-10-09T05:21:09.927 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.5269999999999998e-06 2023-10-09T05:21:09.927 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.927 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.927 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:09.927 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.191894+0000", 2023-10-09T05:21:09.927 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.6650999999999997e-05 2023-10-09T05:21:09.927 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.927 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.927 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:21:09.927 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.191904+0000", 2023-10-09T05:21:09.928 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.9620000000000005e-06 2023-10-09T05:21:09.928 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.928 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.928 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:09.928 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.299833+0000", 2023-10-09T05:21:09.928 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.10792955 2023-10-09T05:21:09.928 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.928 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.928 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:21:09.928 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.299837+0000", 2023-10-09T05:21:09.928 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.0350000000000003e-06 2023-10-09T05:21:09.928 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.928 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.929 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:09.929 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.335246+0000", 2023-10-09T05:21:09.929 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0354082979999999 2023-10-09T05:21:09.929 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.929 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.929 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:21:09.929 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.335372+0000", 2023-10-09T05:21:09.929 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00012598199999999999 2023-10-09T05:21:09.929 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.929 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.929 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:21:09.929 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.335579+0000", 2023-10-09T05:21:09.930 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00020737800000000001 2023-10-09T05:21:09.930 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.930 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.930 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:21:09.930 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.342382+0000", 2023-10-09T05:21:09.930 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0068025630000000002 2023-10-09T05:21:09.930 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.930 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.930 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:21:09.930 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.344734+0000", 2023-10-09T05:21:09.930 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0023527140000000001 2023-10-09T05:21:09.930 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.931 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.931 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:21:09.931 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.344832+0000", 2023-10-09T05:21:09.931 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.7571000000000005e-05 2023-10-09T05:21:09.931 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.931 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.931 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:21:09.931 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.344905+0000", 2023-10-09T05:21:09.931 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.3326999999999998e-05 2023-10-09T05:21:09.931 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.931 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.931 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:21:09.932 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.345081+0000", 2023-10-09T05:21:09.932 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00017564800000000001 2023-10-09T05:21:09.932 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:09.932 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:21:09.932 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:09.932 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.932 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.932 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:158347 9.1a 9:5cc01c15:::benchmark_data_smithi133_153303_object68378:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-10-09T05:21:09.932 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:12:33.215313+0000", 2023-10-09T05:21:09.932 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 516.67885242600005, 2023-10-09T05:21:09.932 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.1302003119999999, 2023-10-09T05:21:09.932 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:21:09.933 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:21:09.933 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:21:09.933 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:21:09.933 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:21:09.933 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 158347 2023-10-09T05:21:09.933 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.933 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:21:09.933 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.933 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:21:09.933 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.215313+0000", 2023-10-09T05:21:09.933 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:09.933 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.933 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.934 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:21:09.934 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.215313+0000", 2023-10-09T05:21:09.934 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:09.934 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.934 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.934 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:21:09.934 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.215312+0000", 2023-10-09T05:21:09.934 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:21:09.934 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.934 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.934 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:21:09.934 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.215320+0000", 2023-10-09T05:21:09.935 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.6899999999999992e-06 2023-10-09T05:21:09.935 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.935 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.935 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:21:09.935 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.215335+0000", 2023-10-09T05:21:09.935 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.5729000000000001e-05 2023-10-09T05:21:09.935 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.935 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.935 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:21:09.935 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.215345+0000", 2023-10-09T05:21:09.935 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.7899999999999994e-06 2023-10-09T05:21:09.935 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.935 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.936 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:09.936 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.215398+0000", 2023-10-09T05:21:09.936 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.2584999999999999e-05 2023-10-09T05:21:09.936 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.936 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.936 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:21:09.936 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.215409+0000", 2023-10-09T05:21:09.936 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0709e-05 2023-10-09T05:21:09.936 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.936 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.936 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:09.936 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.299860+0000", 2023-10-09T05:21:09.937 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.084451708 2023-10-09T05:21:09.937 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.937 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.937 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:21:09.937 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.299864+0000", 2023-10-09T05:21:09.937 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.517e-06 2023-10-09T05:21:09.937 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.937 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.937 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:09.937 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.336661+0000", 2023-10-09T05:21:09.937 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.036797172 2023-10-09T05:21:09.937 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.938 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.938 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:21:09.938 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.336766+0000", 2023-10-09T05:21:09.938 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000105472 2023-10-09T05:21:09.938 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.938 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.938 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:21:09.938 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.336957+0000", 2023-10-09T05:21:09.938 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000190588 2023-10-09T05:21:09.938 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.938 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.938 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:21:09.938 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.344674+0000", 2023-10-09T05:21:09.939 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0077167909999999998 2023-10-09T05:21:09.939 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.939 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.939 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:21:09.939 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.345183+0000", 2023-10-09T05:21:09.939 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00050915400000000005 2023-10-09T05:21:09.939 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.939 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.939 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:21:09.939 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.345261+0000", 2023-10-09T05:21:09.939 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.7614999999999995e-05 2023-10-09T05:21:09.939 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.940 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.940 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:21:09.940 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.345335+0000", 2023-10-09T05:21:09.940 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.4179999999999998e-05 2023-10-09T05:21:09.940 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.940 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.940 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:21:09.940 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.345514+0000", 2023-10-09T05:21:09.940 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000178942 2023-10-09T05:21:09.940 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:09.940 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:21:09.940 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:09.941 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.941 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.941 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:158359 9.1a 9:5c4d2121:::benchmark_data_smithi133_153303_object68390:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-10-09T05:21:09.941 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:12:33.267400+0000", 2023-10-09T05:21:09.941 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 516.626765439, 2023-10-09T05:21:09.941 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0785495410000001, 2023-10-09T05:21:09.941 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:21:09.941 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:21:09.941 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:21:09.941 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:21:09.941 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:21:09.941 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 158359 2023-10-09T05:21:09.941 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.942 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:21:09.942 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.942 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:21:09.942 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.267400+0000", 2023-10-09T05:21:09.942 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:09.942 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.942 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.942 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:21:09.942 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.267400+0000", 2023-10-09T05:21:09.942 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:09.942 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.942 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.943 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:21:09.943 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.267399+0000", 2023-10-09T05:21:09.943 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:21:09.943 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.943 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.943 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:21:09.943 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.267409+0000", 2023-10-09T05:21:09.943 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.8819999999999996e-06 2023-10-09T05:21:09.943 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.943 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.943 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:21:09.943 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.267446+0000", 2023-10-09T05:21:09.944 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.7366000000000001e-05 2023-10-09T05:21:09.944 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.944 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.944 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:21:09.944 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.267462+0000", 2023-10-09T05:21:09.944 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.6045e-05 2023-10-09T05:21:09.944 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.944 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.944 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:09.944 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.267549+0000", 2023-10-09T05:21:09.944 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.7532000000000002e-05 2023-10-09T05:21:09.944 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.944 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.945 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:21:09.945 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.267568+0000", 2023-10-09T05:21:09.945 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.8556999999999999e-05 2023-10-09T05:21:09.945 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.945 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.945 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:09.945 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.299883+0000", 2023-10-09T05:21:09.945 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.032314717999999999 2023-10-09T05:21:09.945 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.945 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.945 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:21:09.946 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.299889+0000", 2023-10-09T05:21:09.946 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.3289999999999999e-06 2023-10-09T05:21:09.946 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.946 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.946 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:09.946 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.338119+0000", 2023-10-09T05:21:09.946 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0382302800000001 2023-10-09T05:21:09.946 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.946 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.946 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:21:09.946 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.338214+0000", 2023-10-09T05:21:09.947 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.4914000000000006e-05 2023-10-09T05:21:09.947 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.947 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.947 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:21:09.947 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.338432+0000", 2023-10-09T05:21:09.947 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00021755200000000001 2023-10-09T05:21:09.947 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.947 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.947 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:21:09.947 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.344683+0000", 2023-10-09T05:21:09.947 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0062509460000000003 2023-10-09T05:21:09.947 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.948 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.948 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:21:09.948 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.345600+0000", 2023-10-09T05:21:09.948 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00091759000000000003 2023-10-09T05:21:09.948 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.948 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.948 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:21:09.948 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.345688+0000", 2023-10-09T05:21:09.948 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.7929000000000005e-05 2023-10-09T05:21:09.948 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.948 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.948 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:21:09.949 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.345758+0000", 2023-10-09T05:21:09.949 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.9540000000000002e-05 2023-10-09T05:21:09.949 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.949 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.949 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:21:09.949 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.345950+0000", 2023-10-09T05:21:09.949 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00019201799999999999 2023-10-09T05:21:09.949 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:09.949 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:21:09.949 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:09.949 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.950 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.950 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:158365 9.1a 9:5d9864a9:::benchmark_data_smithi133_153303_object68396:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-10-09T05:21:09.950 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:12:33.331895+0000", 2023-10-09T05:21:09.950 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 516.56227101100001, 2023-10-09T05:21:09.950 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.014532134, 2023-10-09T05:21:09.950 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:21:09.950 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:21:09.950 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:21:09.950 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:21:09.950 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:21:09.950 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 158365 2023-10-09T05:21:09.951 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.951 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:21:09.951 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.951 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:21:09.951 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.331895+0000", 2023-10-09T05:21:09.951 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:09.951 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.951 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.951 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:21:09.951 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.331895+0000", 2023-10-09T05:21:09.951 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:09.951 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.952 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.952 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:21:09.952 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.331893+0000", 2023-10-09T05:21:09.952 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999981 2023-10-09T05:21:09.952 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.952 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.952 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:21:09.952 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.331901+0000", 2023-10-09T05:21:09.952 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.5199999999999997e-06 2023-10-09T05:21:09.952 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.952 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.952 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:21:09.953 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.331915+0000", 2023-10-09T05:21:09.953 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.3618000000000001e-05 2023-10-09T05:21:09.953 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.953 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.953 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:21:09.953 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.331923+0000", 2023-10-09T05:21:09.953 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.4929999999999998e-06 2023-10-09T05:21:09.953 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.953 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.953 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:09.953 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.331989+0000", 2023-10-09T05:21:09.953 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.5223999999999998e-05 2023-10-09T05:21:09.954 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.954 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.954 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:21:09.954 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.331999+0000", 2023-10-09T05:21:09.954 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0396e-05 2023-10-09T05:21:09.954 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.954 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.954 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:09.954 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.339552+0000", 2023-10-09T05:21:09.954 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.007553473 2023-10-09T05:21:09.954 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.954 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.955 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:21:09.955 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.339740+0000", 2023-10-09T05:21:09.955 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00018765899999999999 2023-10-09T05:21:09.955 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.955 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.955 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:21:09.955 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.340133+0000", 2023-10-09T05:21:09.955 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00039271899999999998 2023-10-09T05:21:09.955 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.955 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.955 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:21:09.955 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.344750+0000", 2023-10-09T05:21:09.956 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0046170209999999998 2023-10-09T05:21:09.956 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.956 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.956 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:21:09.956 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.346028+0000", 2023-10-09T05:21:09.956 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0012782990000000001 2023-10-09T05:21:09.956 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.956 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.956 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:21:09.956 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.346101+0000", 2023-10-09T05:21:09.956 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.3044000000000001e-05 2023-10-09T05:21:09.956 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.957 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.957 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:21:09.957 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.346196+0000", 2023-10-09T05:21:09.957 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.4819000000000001e-05 2023-10-09T05:21:09.957 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.957 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.957 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:21:09.957 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.346427+0000", 2023-10-09T05:21:09.957 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00023087100000000001 2023-10-09T05:21:09.957 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:09.957 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:21:09.958 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:09.958 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.958 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.958 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:158400 9.1a 9:5f602589:::benchmark_data_smithi133_153303_object68431:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-10-09T05:21:09.958 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:12:33.362389+0000", 2023-10-09T05:21:09.958 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 516.53177676600001, 2023-10-09T05:21:09.958 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.98450426300000005, 2023-10-09T05:21:09.958 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:21:09.958 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:21:09.958 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:21:09.958 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:21:09.959 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:21:09.959 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 158400 2023-10-09T05:21:09.959 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.959 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:21:09.959 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.959 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:21:09.959 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.362389+0000", 2023-10-09T05:21:09.959 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:09.959 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.959 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.959 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:21:09.959 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.362389+0000", 2023-10-09T05:21:09.960 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:09.960 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.960 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.960 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:21:09.960 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.362388+0000", 2023-10-09T05:21:09.960 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:21:09.960 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.960 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.960 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:21:09.960 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.362394+0000", 2023-10-09T05:21:09.960 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.0800000000000002e-06 2023-10-09T05:21:09.961 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.961 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.961 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:21:09.961 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.362411+0000", 2023-10-09T05:21:09.961 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.7078000000000001e-05 2023-10-09T05:21:09.961 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.961 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.961 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:21:09.961 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.362418+0000", 2023-10-09T05:21:09.961 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.8800000000000002e-06 2023-10-09T05:21:09.961 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.962 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.962 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:09.962 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.362470+0000", 2023-10-09T05:21:09.962 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.2905999999999998e-05 2023-10-09T05:21:09.962 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.962 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.962 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:21:09.962 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:33.362493+0000", 2023-10-09T05:21:09.962 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.2744000000000001e-05 2023-10-09T05:21:09.962 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.962 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.963 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:09.963 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.342452+0000", 2023-10-09T05:21:09.963 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.97995883500000003 2023-10-09T05:21:09.963 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.963 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.963 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:21:09.963 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.342689+0000", 2023-10-09T05:21:09.963 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000237202 2023-10-09T05:21:09.963 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.963 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.963 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:21:09.964 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.343033+0000", 2023-10-09T05:21:09.964 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00034360700000000001 2023-10-09T05:21:09.964 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.964 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.964 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:21:09.964 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.344759+0000", 2023-10-09T05:21:09.964 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00172602 2023-10-09T05:21:09.964 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.964 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.964 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:21:09.964 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.346534+0000", 2023-10-09T05:21:09.964 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.001775455 2023-10-09T05:21:09.965 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.965 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.965 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:21:09.965 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.346601+0000", 2023-10-09T05:21:09.965 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.6715000000000001e-05 2023-10-09T05:21:09.965 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.965 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.965 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:21:09.965 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.346688+0000", 2023-10-09T05:21:09.965 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.6829999999999994e-05 2023-10-09T05:21:09.965 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.966 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.966 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:21:09.966 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:34.346893+0000", 2023-10-09T05:21:09.966 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00020536299999999999 2023-10-09T05:21:09.966 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:09.966 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:21:09.966 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:09.966 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.966 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.966 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:162973 9.6 9:6612fd45:::benchmark_data_smithi133_153303_object73004:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e170)", 2023-10-09T05:21:09.966 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:12:37.885195+0000", 2023-10-09T05:21:09.967 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 512.00897076599995, 2023-10-09T05:21:09.967 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.063710338000000005, 2023-10-09T05:21:09.967 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:21:09.967 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:21:09.967 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:21:09.967 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:21:09.967 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:21:09.967 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 162973 2023-10-09T05:21:09.967 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.967 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:21:09.967 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.967 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:21:09.968 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:37.885195+0000", 2023-10-09T05:21:09.968 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:09.968 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.968 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.968 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:21:09.968 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:37.885195+0000", 2023-10-09T05:21:09.968 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:09.968 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.968 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.968 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:21:09.968 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:37.885194+0000", 2023-10-09T05:21:09.969 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:21:09.969 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.969 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.969 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:21:09.969 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:37.885199+0000", 2023-10-09T05:21:09.969 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.468e-06 2023-10-09T05:21:09.969 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.969 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.969 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:21:09.969 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:37.885215+0000", 2023-10-09T05:21:09.969 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.5342e-05 2023-10-09T05:21:09.969 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.970 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.970 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:21:09.970 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:37.885220+0000", 2023-10-09T05:21:09.970 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.8239999999999996e-06 2023-10-09T05:21:09.970 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.970 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.970 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:09.970 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:37.885268+0000", 2023-10-09T05:21:09.970 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.7994000000000002e-05 2023-10-09T05:21:09.970 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.970 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.971 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:21:09.971 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:37.885423+0000", 2023-10-09T05:21:09.971 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000154661 2023-10-09T05:21:09.971 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.971 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.971 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:21:09.971 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:37.885985+0000", 2023-10-09T05:21:09.971 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00056212500000000002 2023-10-09T05:21:09.971 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.971 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.971 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:21:09.971 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:37.888797+0000", 2023-10-09T05:21:09.972 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0028116109999999999 2023-10-09T05:21:09.972 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.972 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.972 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:21:09.972 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:37.896729+0000", 2023-10-09T05:21:09.972 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0079318389999999996 2023-10-09T05:21:09.972 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.972 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.972 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:21:09.972 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:37.948615+0000", 2023-10-09T05:21:09.972 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.051886775000000003 2023-10-09T05:21:09.972 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.973 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.973 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:21:09.973 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:37.948687+0000", 2023-10-09T05:21:09.973 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.1514999999999995e-05 2023-10-09T05:21:09.973 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.973 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.973 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:21:09.973 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:37.948905+0000", 2023-10-09T05:21:09.973 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000218414 2023-10-09T05:21:09.973 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:09.973 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:21:09.974 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:09.974 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.974 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.974 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.34845.0:162977 9.16 9:6848efeb:::benchmark_data_smithi133_153303_object73008:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e170)", 2023-10-09T05:21:09.974 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:12:37.890030+0000", 2023-10-09T05:21:09.974 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 512.00413624099997, 2023-10-09T05:21:09.974 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.059576979000000002, 2023-10-09T05:21:09.974 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:21:09.974 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:21:09.974 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:21:09.974 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.34845", 2023-10-09T05:21:09.975 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/3615533870", 2023-10-09T05:21:09.975 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 162977 2023-10-09T05:21:09.975 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.975 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:21:09.975 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.975 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:21:09.975 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:37.890030+0000", 2023-10-09T05:21:09.975 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:09.975 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.975 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.975 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:21:09.975 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:37.890030+0000", 2023-10-09T05:21:09.976 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:09.976 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.976 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.976 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:21:09.976 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:37.890028+0000", 2023-10-09T05:21:09.976 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:21:09.976 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.976 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.976 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:21:09.976 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:37.890034+0000", 2023-10-09T05:21:09.976 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.0299999999999999e-06 2023-10-09T05:21:09.977 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.977 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.977 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:21:09.977 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:37.890045+0000", 2023-10-09T05:21:09.977 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0985000000000001e-05 2023-10-09T05:21:09.977 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.977 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.977 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:21:09.977 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:37.890052+0000", 2023-10-09T05:21:09.977 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.1019999999999999e-06 2023-10-09T05:21:09.977 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.977 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.978 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:09.978 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:37.890105+0000", 2023-10-09T05:21:09.978 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.2151999999999998e-05 2023-10-09T05:21:09.978 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.978 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.978 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:21:09.978 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:37.890233+0000", 2023-10-09T05:21:09.978 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00012843899999999999 2023-10-09T05:21:09.978 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.978 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.978 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:21:09.979 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:37.890632+0000", 2023-10-09T05:21:09.979 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00039882200000000001 2023-10-09T05:21:09.979 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.979 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.979 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:21:09.979 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:37.893153+0000", 2023-10-09T05:21:09.979 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0025208190000000001 2023-10-09T05:21:09.979 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.979 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.979 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:21:09.979 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:37.897115+0000", 2023-10-09T05:21:09.979 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0039626849999999996 2023-10-09T05:21:09.980 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.980 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.980 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:21:09.980 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:37.949280+0000", 2023-10-09T05:21:09.980 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.052165135000000001 2023-10-09T05:21:09.980 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.980 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.980 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:21:09.980 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:37.949353+0000", 2023-10-09T05:21:09.980 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.3052000000000005e-05 2023-10-09T05:21:09.980 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:09.980 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:09.980 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:21:09.981 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:12:37.949606+0000", 2023-10-09T05:21:09.981 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00025300499999999999 2023-10-09T05:21:09.981 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:09.981 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:21:09.981 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:09.981 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:09.981 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:21:09.981 INFO:teuthology.orchestra.run.smithi194.stdout:} 2023-10-09T05:21:09.981 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:21:09.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:09.983 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:09.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:10.084 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:10.141 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:10.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:10.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:10.245 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:10.286 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:10.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:10.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:10.392 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:10.434 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:10.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:10.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:10.539 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:10.563 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:10.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:10.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:10.668 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:10.704 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:10.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:10.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:10.808 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:10.969 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:10.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:10.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:11.075 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:11.100 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T05:21:11.114 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:11.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:11.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:11.221 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:11.255 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:11.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:11.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:11.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:11 smithi133 bash[105589]: cluster 2023-10-09T05:21:10.037566+0000 mgr.y (mgr.25277) 986 : cluster 0 pgmap v1032: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 23957/235011 objects degraded (10.194%); 10081/235011 objects misplaced (4.290%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T05:21:11.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:11 smithi133 bash[102858]: cluster 2023-10-09T05:21:10.037566+0000 mgr.y (mgr.25277) 986 : cluster 0 pgmap v1032: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 23957/235011 objects degraded (10.194%); 10081/235011 objects misplaced (4.290%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T05:21:11.360 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:11.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:11 smithi194 bash[92666]: cluster 2023-10-09T05:21:10.037566+0000 mgr.y (mgr.25277) 986 : cluster 0 pgmap v1032: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 23957/235011 objects degraded (10.194%); 10081/235011 objects misplaced (4.290%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T05:21:11.396 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:11.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:11.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:11.500 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:11.531 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:11.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:11.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:11.637 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:11.686 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:11.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:11.696 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:11.797 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:11.826 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:11.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:11.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:11.928 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:11.964 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:11.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:11.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:12.068 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:12.095 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:12.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:12.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:12.201 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:12.240 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:12.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:12.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:12.342 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:12.378 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:12.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:12.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:12.485 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:12.511 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:12.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:12.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:12.616 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:12.646 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:12.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:12.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:12.672 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:21:12.753 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:12.791 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:12.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:12.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:12.897 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:12.928 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:12.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:12.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:13.033 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:13.061 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:13.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:13.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:13.164 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:13.170 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:13 smithi194 bash[92666]: cluster 2023-10-09T05:21:12.038221+0000 mgr.y (mgr.25277) 987 : cluster 0 pgmap v1033: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 480 op/s; 19532/232161 objects degraded (8.413%); 14464/232161 objects misplaced (6.230%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:21:13.195 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:13.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:13.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:13.300 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:13.332 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:13.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:13.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:13.435 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:13.460 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:13.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:13.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:13.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:13 smithi133 bash[105589]: cluster 2023-10-09T05:21:12.038221+0000 mgr.y (mgr.25277) 987 : cluster 0 pgmap v1033: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 480 op/s; 19532/232161 objects degraded (8.413%); 14464/232161 objects misplaced (6.230%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:21:13.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:13 smithi133 bash[102858]: cluster 2023-10-09T05:21:12.038221+0000 mgr.y (mgr.25277) 987 : cluster 0 pgmap v1033: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 480 op/s; 19532/232161 objects degraded (8.413%); 14464/232161 objects misplaced (6.230%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:21:13.564 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:13.599 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:13.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:13.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:13.690 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:21:13.703 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:13.775 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:13.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:13.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:13.883 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:13.925 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:13.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:13.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:14.027 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:14.059 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:14.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:14.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:14.166 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:14.207 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:14.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:14.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:14.298 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:21:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:21:14] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.43.0" 2023-10-09T05:21:14.312 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:14.337 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:14.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:14.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:14.441 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:14.476 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:14.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:14.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:14.581 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:14.621 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:14.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:14.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:14.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:14 smithi194 bash[92666]: audit 2023-10-09T05:21:13.462262+0000 mon.a (mon.0) 1288 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:21:14.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:14 smithi194 bash[92666]: audit 2023-10-09T05:21:13.463438+0000 mon.a (mon.0) 1289 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:21:14.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:14 smithi194 bash[92666]: cluster 2023-10-09T05:21:13.523579+0000 mon.a (mon.0) 1290 : cluster 3 Health check update: Degraded data redundancy: 19532/232161 objects degraded (8.413%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:21:14.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:14 smithi194 bash[92666]: cluster 2023-10-09T05:21:14.038871+0000 mgr.y (mgr.25277) 988 : cluster 0 pgmap v1034: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 19532/229722 objects degraded (8.502%); 14336/229722 objects misplaced (6.241%); 2.4 MiB/s, 39 objects/s recovering 2023-10-09T05:21:14.712 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:21:14.724 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:14.757 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:14.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:14.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:14.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:14 smithi133 bash[102858]: audit 2023-10-09T05:21:13.462262+0000 mon.a (mon.0) 1288 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:21:14.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:14 smithi133 bash[102858]: audit 2023-10-09T05:21:13.463438+0000 mon.a (mon.0) 1289 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:21:14.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:14 smithi133 bash[102858]: cluster 2023-10-09T05:21:13.523579+0000 mon.a (mon.0) 1290 : cluster 3 Health check update: Degraded data redundancy: 19532/232161 objects degraded (8.413%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:21:14.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:14 smithi133 bash[102858]: cluster 2023-10-09T05:21:14.038871+0000 mgr.y (mgr.25277) 988 : cluster 0 pgmap v1034: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 19532/229722 objects degraded (8.502%); 14336/229722 objects misplaced (6.241%); 2.4 MiB/s, 39 objects/s recovering 2023-10-09T05:21:14.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:14 smithi133 bash[105589]: audit 2023-10-09T05:21:13.462262+0000 mon.a (mon.0) 1288 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:21:14.865 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:14.893 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:14.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:14.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:14.999 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:15.026 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:15.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:15.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:15.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:14 smithi133 bash[105589]: audit 2023-10-09T05:21:13.463438+0000 mon.a (mon.0) 1289 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:21:15.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:14 smithi133 bash[105589]: cluster 2023-10-09T05:21:13.523579+0000 mon.a (mon.0) 1290 : cluster 3 Health check update: Degraded data redundancy: 19532/232161 objects degraded (8.413%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:21:15.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:14 smithi133 bash[105589]: cluster 2023-10-09T05:21:14.038871+0000 mgr.y (mgr.25277) 988 : cluster 0 pgmap v1034: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 19532/229722 objects degraded (8.502%); 14336/229722 objects misplaced (6.241%); 2.4 MiB/s, 39 objects/s recovering 2023-10-09T05:21:15.129 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:15.156 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:15.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:15.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:15.260 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:15.317 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:15.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:15.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:15.423 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:15.461 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:15.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:15.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:15.566 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:15.592 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:15.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:15.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:15.699 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:15.749 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:15.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:15.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:15.852 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:15.889 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:15.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:15.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:15.957 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:21:15.998 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:16.023 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:16.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:16.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:16.126 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:16.154 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:16.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:16.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:16.259 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:16.291 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:16.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:16.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:16.395 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:16.422 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:16.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:16.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:16.525 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:16.546 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:16.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:16.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:16.649 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:16.676 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:16.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:16.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:16.781 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:16.820 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:16.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:16.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:16.926 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:16.949 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:16.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:16.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:17.052 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:17.082 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:17.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:17.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:17.185 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:17.209 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:17.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:17.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:17.312 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:17.334 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:17.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:17.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:17.444 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:17.453 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:17 smithi133 bash[102858]: cluster 2023-10-09T05:21:16.039574+0000 mgr.y (mgr.25277) 989 : cluster 0 pgmap v1035: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 480 op/s; 19615/226131 objects degraded (8.674%); 14145/226131 objects misplaced (6.255%); 2.9 MiB/s, 47 objects/s recovering 2023-10-09T05:21:17.453 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:17 smithi133 bash[105589]: cluster 2023-10-09T05:21:16.039574+0000 mgr.y (mgr.25277) 989 : cluster 0 pgmap v1035: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 480 op/s; 19615/226131 objects degraded (8.674%); 14145/226131 objects misplaced (6.255%); 2.9 MiB/s, 47 objects/s recovering 2023-10-09T05:21:17.471 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:17.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:17.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:17.577 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:17.587 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:17 smithi194 bash[92666]: cluster 2023-10-09T05:21:16.039574+0000 mgr.y (mgr.25277) 989 : cluster 0 pgmap v1035: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 480 op/s; 19615/226131 objects degraded (8.674%); 14145/226131 objects misplaced (6.255%); 2.9 MiB/s, 47 objects/s recovering 2023-10-09T05:21:17.607 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:17.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:17.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:17.717 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:17.755 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:17.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:17.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:17.859 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:17.884 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:17.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:17.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:17.987 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:18.007 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:18.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:18.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:18.110 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:18.148 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:18.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:18.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:18.252 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:18.275 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:18.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:18.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:18.380 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:18.412 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:18.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:18.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:18.518 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:18.559 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:18.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:18.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:18.664 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:18.688 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:18.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:18.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:18.794 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:18.827 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:18.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:18.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:18.934 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:18.959 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:18.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:18.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:19.066 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:19.106 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:19.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:19.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:19.212 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:19.219 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:19 smithi194 bash[92666]: cluster 2023-10-09T05:21:18.041749+0000 mgr.y (mgr.25277) 990 : cluster 0 pgmap v1036: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 499 op/s; 19615/223509 objects degraded (8.776%); 14145/223509 objects misplaced (6.329%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:21:19.219 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:19 smithi194 bash[92666]: cluster 2023-10-09T05:21:18.524776+0000 mon.a (mon.0) 1291 : cluster 3 Health check update: Degraded data redundancy: 19615/223509 objects degraded (8.776%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:21:19.238 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:19.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:19.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:19.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:19 smithi133 bash[102858]: cluster 2023-10-09T05:21:18.041749+0000 mgr.y (mgr.25277) 990 : cluster 0 pgmap v1036: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 499 op/s; 19615/223509 objects degraded (8.776%); 14145/223509 objects misplaced (6.329%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:21:19.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:19 smithi133 bash[102858]: cluster 2023-10-09T05:21:18.524776+0000 mon.a (mon.0) 1291 : cluster 3 Health check update: Degraded data redundancy: 19615/223509 objects degraded (8.776%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:21:19.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:19 smithi133 bash[105589]: cluster 2023-10-09T05:21:18.041749+0000 mgr.y (mgr.25277) 990 : cluster 0 pgmap v1036: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 499 op/s; 19615/223509 objects degraded (8.776%); 14145/223509 objects misplaced (6.329%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:21:19.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:19 smithi133 bash[105589]: cluster 2023-10-09T05:21:18.524776+0000 mon.a (mon.0) 1291 : cluster 3 Health check update: Degraded data redundancy: 19615/223509 objects degraded (8.776%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:21:19.345 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:19.381 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:19.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:19.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:19.487 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:19.529 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:19.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:19.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:19.634 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:19.654 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:19.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:19.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:19.758 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:19.784 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:19.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:19.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:19.889 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:19.913 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:19.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:19.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:20.016 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:20.041 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:20.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:20.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:20.144 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:20.173 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:20.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:20.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:20.279 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:20.305 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:20.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:20.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:20.412 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:20.435 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:20.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:20.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:20.541 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:20.562 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:20.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:20.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:20.668 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:20.693 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:20.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:20.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:20.796 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:20.836 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:20.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:20.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:20.939 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:20.970 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:20.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:20.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:21.074 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:21.104 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:21.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:21.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:21.207 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:21.232 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:21.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:21.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:21.336 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:21.366 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:21.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:21.368 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:21 smithi194 bash[92666]: cluster 2023-10-09T05:21:20.042415+0000 mgr.y (mgr.25277) 991 : cluster 0 pgmap v1037: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 19615/219462 objects degraded (8.938%); 14024/219462 objects misplaced (6.390%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T05:21:21.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:21.470 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:21.497 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:21.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:21.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:21.599 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:21.631 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:21.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:21.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:21.635 INFO:teuthology.orchestra.run.smithi194.stdout:{ 2023-10-09T05:21:21.635 INFO:teuthology.orchestra.run.smithi194.stdout: "ops": [ 2023-10-09T05:21:21.635 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.635 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35187.0:120911 11.3 11:c096b94c:::benchmark_data_smithi133_194104_object24991:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-10-09T05:21:21.635 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:21:21.610675+0000", 2023-10-09T05:21:21.635 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 0.022904642999999999, 2023-10-09T05:21:21.635 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.022937394, 2023-10-09T05:21:21.636 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:21:21.636 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:21:21.636 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:21:21.636 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35187", 2023-10-09T05:21:21.636 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2118344251", 2023-10-09T05:21:21.636 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 120911 2023-10-09T05:21:21.636 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.636 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:21:21.636 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.637 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:21:21.637 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.610675+0000", 2023-10-09T05:21:21.637 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:21.637 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.637 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.637 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:21:21.637 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.610675+0000", 2023-10-09T05:21:21.637 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:21.637 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.637 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.637 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:21:21.638 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.610673+0000", 2023-10-09T05:21:21.638 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999981 2023-10-09T05:21:21.638 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.638 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.638 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:21:21.638 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.610685+0000", 2023-10-09T05:21:21.638 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.1881e-05 2023-10-09T05:21:21.638 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.638 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.638 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:21:21.638 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.610698+0000", 2023-10-09T05:21:21.638 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.2432999999999999e-05 2023-10-09T05:21:21.639 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.639 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.639 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:21:21.639 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.610703+0000", 2023-10-09T05:21:21.639 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.7690000000000002e-06 2023-10-09T05:21:21.639 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.639 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.639 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:21.639 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.619178+0000", 2023-10-09T05:21:21.639 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0084744060000000003 2023-10-09T05:21:21.639 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.640 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.640 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:21:21.640 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.619317+0000", 2023-10-09T05:21:21.640 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00013910300000000001 2023-10-09T05:21:21.640 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.640 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.640 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 2,7", 2023-10-09T05:21:21.640 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.619516+0000", 2023-10-09T05:21:21.640 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00019890199999999999 2023-10-09T05:21:21.640 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.640 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.640 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:21:21.641 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.628701+0000", 2023-10-09T05:21:21.641 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0091849419999999998 2023-10-09T05:21:21.641 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.641 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.641 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:21:21.641 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.628709+0000", 2023-10-09T05:21:21.641 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.7409999999999998e-06 2023-10-09T05:21:21.641 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:21.641 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:21:21.641 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:21.641 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.641 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.641 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35187.0:120914 11.3 11:c9a739a2:::benchmark_data_smithi133_194104_object24994:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-10-09T05:21:21.642 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:21:21.613523+0000", 2023-10-09T05:21:21.642 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 0.020057116, 2023-10-09T05:21:21.642 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.020187723000000001, 2023-10-09T05:21:21.642 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:21:21.642 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "waiting for rw locks", 2023-10-09T05:21:21.642 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:21:21.642 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35187", 2023-10-09T05:21:21.642 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2118344251", 2023-10-09T05:21:21.642 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 120914 2023-10-09T05:21:21.642 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.642 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:21:21.643 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.643 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:21:21.643 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.613523+0000", 2023-10-09T05:21:21.643 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:21.643 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.643 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.643 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:21:21.643 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.613523+0000", 2023-10-09T05:21:21.643 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:21.643 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.643 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.643 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:21:21.644 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.613521+0000", 2023-10-09T05:21:21.644 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999981 2023-10-09T05:21:21.644 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.644 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.644 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:21:21.644 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.613528+0000", 2023-10-09T05:21:21.644 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.9380000000000003e-06 2023-10-09T05:21:21.644 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.644 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.644 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:21:21.644 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.613542+0000", 2023-10-09T05:21:21.645 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.4197e-05 2023-10-09T05:21:21.645 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.645 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.645 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:21:21.645 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.613550+0000", 2023-10-09T05:21:21.645 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.5349999999999999e-06 2023-10-09T05:21:21.645 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.645 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.645 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:21.645 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.628107+0000", 2023-10-09T05:21:21.645 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.014556949 2023-10-09T05:21:21.645 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.645 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.646 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for rw locks", 2023-10-09T05:21:21.646 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.628248+0000", 2023-10-09T05:21:21.646 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000141874 2023-10-09T05:21:21.646 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:21.646 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:21:21.646 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:21.646 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.646 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.646 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_repop(client.35187.0:120918 11.5 e243/225 11:abe1ea3c:::benchmark_data_smithi133_194104_object24998:head v 243'7640, mlcod=243'7638)", 2023-10-09T05:21:21.646 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:21:21.624404+0000", 2023-10-09T05:21:21.646 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 0.009175875, 2023-10-09T05:21:21.647 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0093767940000000008, 2023-10-09T05:21:21.647 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:21:21.647 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "started", 2023-10-09T05:21:21.647 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:21:21.647 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.647 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:21:21.647 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.624404+0000", 2023-10-09T05:21:21.647 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:21.647 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.647 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.647 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:21:21.647 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.624404+0000", 2023-10-09T05:21:21.648 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:21.648 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.648 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.648 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:21:21.648 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.624403+0000", 2023-10-09T05:21:21.648 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:21:21.648 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.648 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.648 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:21:21.648 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.624410+0000", 2023-10-09T05:21:21.648 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.4569999999999999e-06 2023-10-09T05:21:21.649 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.649 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.649 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:21:21.649 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.624420+0000", 2023-10-09T05:21:21.649 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0062e-05 2023-10-09T05:21:21.649 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.649 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.649 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:21:21.649 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.624425+0000", 2023-10-09T05:21:21.649 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.0390000000000004e-06 2023-10-09T05:21:21.649 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.649 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.650 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:21.650 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.624482+0000", 2023-10-09T05:21:21.650 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.6994999999999999e-05 2023-10-09T05:21:21.650 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.650 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.650 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:21:21.650 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.624497+0000", 2023-10-09T05:21:21.650 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.4589999999999999e-05 2023-10-09T05:21:21.650 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:21.650 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:21:21.650 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:21.651 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.651 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.651 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_repop(client.35187.0:120917 11.d e243/236 11:baa21f53:::benchmark_data_smithi133_194104_object24997:head v 243'7677, mlcod=0'0)", 2023-10-09T05:21:21.651 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:21:21.624957+0000", 2023-10-09T05:21:21.651 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 0.0086225209999999993, 2023-10-09T05:21:21.651 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0088860950000000001, 2023-10-09T05:21:21.651 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:21:21.651 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "started", 2023-10-09T05:21:21.651 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:21:21.651 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.651 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:21:21.651 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.624957+0000", 2023-10-09T05:21:21.652 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:21.652 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.652 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.652 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:21:21.652 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.624957+0000", 2023-10-09T05:21:21.652 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:21.652 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.652 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.652 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:21:21.652 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.624957+0000", 2023-10-09T05:21:21.652 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999995 2023-10-09T05:21:21.653 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.653 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.653 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:21:21.653 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.624966+0000", 2023-10-09T05:21:21.653 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.2650000000000002e-06 2023-10-09T05:21:21.653 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.653 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.653 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:21:21.653 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.624976+0000", 2023-10-09T05:21:21.653 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.6339999999999996e-06 2023-10-09T05:21:21.653 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.653 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.654 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:21:21.654 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.624981+0000", 2023-10-09T05:21:21.654 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.8490000000000002e-06 2023-10-09T05:21:21.654 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.654 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.654 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:21.654 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.628175+0000", 2023-10-09T05:21:21.654 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0031938930000000002 2023-10-09T05:21:21.654 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.654 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.654 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:21:21.654 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.628210+0000", 2023-10-09T05:21:21.655 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.4901999999999999e-05 2023-10-09T05:21:21.655 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:21.655 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:21:21.655 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:21.655 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.655 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.655 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35187.0:120926 11.3 11:cb0e6e19:::benchmark_data_smithi133_194104_object25006:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-10-09T05:21:21.655 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:21:21.632193+0000", 2023-10-09T05:21:21.655 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 0.001387044, 2023-10-09T05:21:21.655 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.001714077, 2023-10-09T05:21:21.655 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:21:21.656 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:21:21.656 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:21:21.656 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35187", 2023-10-09T05:21:21.656 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2118344251", 2023-10-09T05:21:21.656 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 120926 2023-10-09T05:21:21.656 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.656 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:21:21.656 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.656 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:21:21.656 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.632193+0000", 2023-10-09T05:21:21.656 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:21.656 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.656 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.657 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:21:21.657 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.632193+0000", 2023-10-09T05:21:21.657 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:21.657 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.657 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.657 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:21:21.657 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.632191+0000", 2023-10-09T05:21:21.657 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:21:21.657 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.657 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.657 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:21:21.657 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.632199+0000", 2023-10-09T05:21:21.658 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.841e-06 2023-10-09T05:21:21.658 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.658 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.658 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:21:21.658 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.632216+0000", 2023-10-09T05:21:21.658 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.628e-05 2023-10-09T05:21:21.658 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.658 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.658 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:21:21.658 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.632225+0000", 2023-10-09T05:21:21.658 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.2750000000000001e-06 2023-10-09T05:21:21.658 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.659 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.659 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:21.659 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.632288+0000", 2023-10-09T05:21:21.659 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.3602999999999994e-05 2023-10-09T05:21:21.659 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.659 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.659 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:21:21.659 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.632433+0000", 2023-10-09T05:21:21.659 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00014504599999999999 2023-10-09T05:21:21.659 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:21.659 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:21.659 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 2,7", 2023-10-09T05:21:21.660 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:21.632564+0000", 2023-10-09T05:21:21.660 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000130548 2023-10-09T05:21:21.660 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:21.660 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:21:21.660 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:21.660 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:21.660 INFO:teuthology.orchestra.run.smithi194.stdout: ], 2023-10-09T05:21:21.660 INFO:teuthology.orchestra.run.smithi194.stdout: "num_ops": 5 2023-10-09T05:21:21.660 INFO:teuthology.orchestra.run.smithi194.stdout:} 2023-10-09T05:21:21.736 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:21.745 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:21 smithi133 bash[102858]: cluster 2023-10-09T05:21:20.042415+0000 mgr.y (mgr.25277) 991 : cluster 0 pgmap v1037: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 19615/219462 objects degraded (8.938%); 14024/219462 objects misplaced (6.390%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T05:21:21.745 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:21 smithi133 bash[105589]: cluster 2023-10-09T05:21:20.042415+0000 mgr.y (mgr.25277) 991 : cluster 0 pgmap v1037: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 19615/219462 objects degraded (8.938%); 14024/219462 objects misplaced (6.390%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T05:21:21.777 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:21.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:21.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:21.883 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:21.906 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:21.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:21.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:22.013 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:22.040 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:22.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:22.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:22.143 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:22.168 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:22.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:22.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:22.272 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:22.301 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:22.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:22.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:22.409 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:22.439 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:22.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:22.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:22.542 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:22.582 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:22.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:22.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:22.685 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:22.716 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:22.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:22.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:22.824 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:22.860 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:22.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:22.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:22.942 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T05:21:22.970 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:23.025 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:23.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:23.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:23.128 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:23.154 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:23.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:23.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:23.258 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:23.293 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:23.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:23.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:23.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:23 smithi194 bash[92666]: cluster 2023-10-09T05:21:22.042959+0000 mgr.y (mgr.25277) 992 : cluster 0 pgmap v1038: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 19678/215934 objects degraded (9.113%); 13843/215934 objects misplaced (6.411%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:21:23.402 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:23.439 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:23.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:23.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:23.474 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:23 smithi133 bash[102858]: cluster 2023-10-09T05:21:22.042959+0000 mgr.y (mgr.25277) 992 : cluster 0 pgmap v1038: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 19678/215934 objects degraded (9.113%); 13843/215934 objects misplaced (6.411%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:21:23.474 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:23 smithi133 bash[105589]: cluster 2023-10-09T05:21:22.042959+0000 mgr.y (mgr.25277) 992 : cluster 0 pgmap v1038: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 19678/215934 objects degraded (9.113%); 13843/215934 objects misplaced (6.411%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:21:23.542 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:23.568 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:23.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:23.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:23.674 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:23.700 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:23.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:23.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:23.805 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:23.845 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:23.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:23.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:23.956 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:23.981 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:23.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:23.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:24.084 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:24.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:24.118 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:24.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:24.219 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:24.251 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:24.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:24.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:24.345 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:21:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:21:24] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.43.0" 2023-10-09T05:21:24.345 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:24 smithi133 bash[105589]: cluster 2023-10-09T05:21:23.525757+0000 mon.a (mon.0) 1292 : cluster 3 Health check update: Degraded data redundancy: 19678/215934 objects degraded (9.113%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:21:24.346 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:24 smithi133 bash[102858]: cluster 2023-10-09T05:21:23.525757+0000 mon.a (mon.0) 1292 : cluster 3 Health check update: Degraded data redundancy: 19678/215934 objects degraded (9.113%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:21:24.361 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:24.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:24 smithi194 bash[92666]: cluster 2023-10-09T05:21:23.525757+0000 mon.a (mon.0) 1292 : cluster 3 Health check update: Degraded data redundancy: 19678/215934 objects degraded (9.113%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:21:24.391 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:24.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:24.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:24.495 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:24.533 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:24.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:24.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:24.636 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:24.662 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:24.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:24.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:24.769 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:24.799 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:24.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:24.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:24.905 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:24.931 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:24.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:24.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:25.037 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:25.065 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:25.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:25.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:25.172 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:25.198 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:25.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:25.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:25.302 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:25.327 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:25.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:25.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:25.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:25 smithi194 bash[92666]: cluster 2023-10-09T05:21:24.043569+0000 mgr.y (mgr.25277) 993 : cluster 0 pgmap v1039: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 19693/213330 objects degraded (9.231%); 13714/213330 objects misplaced (6.429%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T05:21:25.431 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:25.475 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:25.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:25.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:25.580 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:25.590 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:25 smithi133 bash[102858]: cluster 2023-10-09T05:21:24.043569+0000 mgr.y (mgr.25277) 993 : cluster 0 pgmap v1039: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 19693/213330 objects degraded (9.231%); 13714/213330 objects misplaced (6.429%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T05:21:25.591 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:25 smithi133 bash[105589]: cluster 2023-10-09T05:21:24.043569+0000 mgr.y (mgr.25277) 993 : cluster 0 pgmap v1039: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 19693/213330 objects degraded (9.231%); 13714/213330 objects misplaced (6.429%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T05:21:25.607 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:25.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:25.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:25.710 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:25.737 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:25.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:25.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:25.843 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:25.870 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:25.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:25.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:25.973 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:26.002 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:26.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:26.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:26.108 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:26.147 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:26.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:26.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:26.249 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:26.277 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:26.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:26.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:26.380 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:26.410 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:26.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:26.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:26.515 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:26.542 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:26.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:26.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:26.645 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:26.674 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:26.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:26.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:26.782 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:26.814 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:26.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:26.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:26.922 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:26.950 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:26.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:26.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:27.055 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:27.083 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:27.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:27.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:27.181 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:27 smithi133 bash[102858]: cluster 2023-10-09T05:21:26.044248+0000 mgr.y (mgr.25277) 994 : cluster 0 pgmap v1040: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 553 op/s; 19738/209778 objects degraded (9.409%); 13550/209778 objects misplaced (6.459%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T05:21:27.182 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:27 smithi133 bash[105589]: cluster 2023-10-09T05:21:26.044248+0000 mgr.y (mgr.25277) 994 : cluster 0 pgmap v1040: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 553 op/s; 19738/209778 objects degraded (9.409%); 13550/209778 objects misplaced (6.459%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T05:21:27.187 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:27.216 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:27.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:27.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:27.321 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:27.346 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:27.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:27.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:27.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:27 smithi194 bash[92666]: cluster 2023-10-09T05:21:26.044248+0000 mgr.y (mgr.25277) 994 : cluster 0 pgmap v1040: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 553 op/s; 19738/209778 objects degraded (9.409%); 13550/209778 objects misplaced (6.459%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T05:21:27.449 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:27.484 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:27.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:27.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:27.548 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:21:27.588 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:27.614 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:21:27.622 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:27.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:27.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:27.667 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:21:27.716 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:21:27.728 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:27.748 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:21:27.756 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:21:27.772 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:27.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:27.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:27.801 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:21:27.849 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:21:27.880 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:27.885 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:21:27.900 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:21:27.901 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:21:27.901 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:21:27.901 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:21:27.901 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:21:27.901 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:21:27.901 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:21:27.901 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:21:27.904 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:27.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:27.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:28.008 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:28.035 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:28.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:28.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:28.138 INFO:teuthology.orchestra.run.smithi133.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T05:21:28.140 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is set 2023-10-09T05:21:28.142 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:28.170 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:28.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:28.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:28.279 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:28.288 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:28 smithi133 bash[105589]: audit 2023-10-09T05:21:27.491880+0000 mon.a (mon.0) 1293 : audit 0 from='client.? 172.21.15.133:0/2906292207' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:21:28.288 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:28 smithi133 bash[105589]: audit 2023-10-09T05:21:27.920036+0000 mon.c (mon.2) 94 : audit 1 from='client.? 172.21.15.133:0/2893309733' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:21:28.288 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:28 smithi133 bash[105589]: audit 2023-10-09T05:21:27.920617+0000 mon.a (mon.0) 1294 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:21:28.288 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:28 smithi133 bash[105589]: audit 2023-10-09T05:21:27.925424+0000 mon.b (mon.1) 21 : audit 1 from='client.? 172.21.15.133:0/1671103967' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:21:28.289 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:28 smithi133 bash[105589]: audit 2023-10-09T05:21:27.925546+0000 mon.a (mon.0) 1295 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:21:28.289 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:28 smithi133 bash[102858]: audit 2023-10-09T05:21:27.491880+0000 mon.a (mon.0) 1293 : audit 0 from='client.? 172.21.15.133:0/2906292207' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:21:28.289 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:28 smithi133 bash[102858]: audit 2023-10-09T05:21:27.920036+0000 mon.c (mon.2) 94 : audit 1 from='client.? 172.21.15.133:0/2893309733' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:21:28.289 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:28 smithi133 bash[102858]: audit 2023-10-09T05:21:27.920617+0000 mon.a (mon.0) 1294 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:21:28.289 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:28 smithi133 bash[102858]: audit 2023-10-09T05:21:27.925424+0000 mon.b (mon.1) 21 : audit 1 from='client.? 172.21.15.133:0/1671103967' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:21:28.289 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:28 smithi133 bash[102858]: audit 2023-10-09T05:21:27.925546+0000 mon.a (mon.0) 1295 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:21:28.308 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:28.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:28.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:28.415 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:28.451 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:28.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:28.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:28.555 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:28.565 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:28 smithi194 bash[92666]: audit 2023-10-09T05:21:27.491880+0000 mon.a (mon.0) 1293 : audit 0 from='client.? 172.21.15.133:0/2906292207' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:21:28.565 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:28 smithi194 bash[92666]: audit 2023-10-09T05:21:27.920036+0000 mon.c (mon.2) 94 : audit 1 from='client.? 172.21.15.133:0/2893309733' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:21:28.565 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:28 smithi194 bash[92666]: audit 2023-10-09T05:21:27.920617+0000 mon.a (mon.0) 1294 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:21:28.565 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:28 smithi194 bash[92666]: audit 2023-10-09T05:21:27.925424+0000 mon.b (mon.1) 21 : audit 1 from='client.? 172.21.15.133:0/1671103967' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:21:28.565 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:28 smithi194 bash[92666]: audit 2023-10-09T05:21:27.925546+0000 mon.a (mon.0) 1295 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:21:28.580 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:28.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:28.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:28.683 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:28.716 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:28.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:28.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:28.825 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:28.859 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:28.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:28.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:28.962 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:28.996 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:29.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:29.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:29.103 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:29.131 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:29.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:29.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:29.238 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:29.264 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:29.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:29.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:29.320 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:29 smithi133 bash[105589]: cluster 2023-10-09T05:21:28.044780+0000 mgr.y (mgr.25277) 995 : cluster 0 pgmap v1041: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 19738/207297 objects degraded (9.522%); 13550/207297 objects misplaced (6.537%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T05:21:29.320 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:29 smithi133 bash[105589]: cluster 2023-10-09T05:21:28.113933+0000 mon.a (mon.0) 1296 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:21:29.320 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:29 smithi133 bash[105589]: audit 2023-10-09T05:21:28.124940+0000 mon.a (mon.0) 1297 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:21:29.320 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:29 smithi133 bash[105589]: cluster 2023-10-09T05:21:28.127881+0000 osd.5 (osd.5) 229 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T05:21:29.320 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:29 smithi133 bash[105589]: audit 2023-10-09T05:21:28.132386+0000 mon.a (mon.0) 1298 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:21:29.320 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:29 smithi133 bash[105589]: cluster 2023-10-09T05:21:28.132608+0000 mon.a (mon.0) 1299 : cluster 0 osdmap e244: 8 total, 8 up, 7 in 2023-10-09T05:21:29.320 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:29 smithi133 bash[105589]: cluster 2023-10-09T05:21:28.138761+0000 osd.6 (osd.6) 184 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T05:21:29.320 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:29 smithi133 bash[105589]: cluster 2023-10-09T05:21:28.145728+0000 osd.6 (osd.6) 185 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T05:21:29.320 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:29 smithi133 bash[105589]: cluster 2023-10-09T05:21:28.149756+0000 osd.4 (osd.4) 208 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T05:21:29.321 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:29 smithi133 bash[105589]: audit 2023-10-09T05:21:28.466269+0000 mon.a (mon.0) 1300 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:21:29.321 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:29 smithi133 bash[105589]: audit 2023-10-09T05:21:28.469880+0000 mon.a (mon.0) 1301 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:21:29.321 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:29 smithi133 bash[105589]: cluster 2023-10-09T05:21:28.526785+0000 mon.a (mon.0) 1302 : cluster 3 Health check update: Degraded data redundancy: 19738/207297 objects degraded (9.522%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:21:29.321 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:29 smithi133 bash[102858]: cluster 2023-10-09T05:21:28.044780+0000 mgr.y (mgr.25277) 995 : cluster 0 pgmap v1041: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 19738/207297 objects degraded (9.522%); 13550/207297 objects misplaced (6.537%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T05:21:29.321 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:29 smithi133 bash[102858]: cluster 2023-10-09T05:21:28.113933+0000 mon.a (mon.0) 1296 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:21:29.321 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:29 smithi133 bash[102858]: audit 2023-10-09T05:21:28.124940+0000 mon.a (mon.0) 1297 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:21:29.321 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:29 smithi133 bash[102858]: cluster 2023-10-09T05:21:28.127881+0000 osd.5 (osd.5) 229 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T05:21:29.321 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:29 smithi133 bash[102858]: audit 2023-10-09T05:21:28.132386+0000 mon.a (mon.0) 1298 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:21:29.321 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:29 smithi133 bash[102858]: cluster 2023-10-09T05:21:28.132608+0000 mon.a (mon.0) 1299 : cluster 0 osdmap e244: 8 total, 8 up, 7 in 2023-10-09T05:21:29.322 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:29 smithi133 bash[102858]: cluster 2023-10-09T05:21:28.138761+0000 osd.6 (osd.6) 184 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T05:21:29.322 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:29 smithi133 bash[102858]: cluster 2023-10-09T05:21:28.145728+0000 osd.6 (osd.6) 185 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T05:21:29.322 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:29 smithi133 bash[102858]: cluster 2023-10-09T05:21:28.149756+0000 osd.4 (osd.4) 208 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T05:21:29.322 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:29 smithi133 bash[102858]: audit 2023-10-09T05:21:28.466269+0000 mon.a (mon.0) 1300 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:21:29.322 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:29 smithi133 bash[102858]: audit 2023-10-09T05:21:28.469880+0000 mon.a (mon.0) 1301 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:21:29.322 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:29 smithi133 bash[102858]: cluster 2023-10-09T05:21:28.526785+0000 mon.a (mon.0) 1302 : cluster 3 Health check update: Degraded data redundancy: 19738/207297 objects degraded (9.522%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:21:29.369 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:29.390 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:29.391 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:29 smithi194 bash[92666]: cluster 2023-10-09T05:21:28.044780+0000 mgr.y (mgr.25277) 995 : cluster 0 pgmap v1041: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 19738/207297 objects degraded (9.522%); 13550/207297 objects misplaced (6.537%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T05:21:29.391 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:29 smithi194 bash[92666]: cluster 2023-10-09T05:21:28.113933+0000 mon.a (mon.0) 1296 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:21:29.391 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:29 smithi194 bash[92666]: audit 2023-10-09T05:21:28.124940+0000 mon.a (mon.0) 1297 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:21:29.392 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:29 smithi194 bash[92666]: cluster 2023-10-09T05:21:28.127881+0000 osd.5 (osd.5) 229 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T05:21:29.392 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:29 smithi194 bash[92666]: audit 2023-10-09T05:21:28.132386+0000 mon.a (mon.0) 1298 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:21:29.392 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:29 smithi194 bash[92666]: cluster 2023-10-09T05:21:28.132608+0000 mon.a (mon.0) 1299 : cluster 0 osdmap e244: 8 total, 8 up, 7 in 2023-10-09T05:21:29.392 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:29 smithi194 bash[92666]: cluster 2023-10-09T05:21:28.138761+0000 osd.6 (osd.6) 184 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T05:21:29.392 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:29 smithi194 bash[92666]: cluster 2023-10-09T05:21:28.145728+0000 osd.6 (osd.6) 185 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T05:21:29.392 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:29 smithi194 bash[92666]: cluster 2023-10-09T05:21:28.149756+0000 osd.4 (osd.4) 208 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T05:21:29.392 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:29 smithi194 bash[92666]: audit 2023-10-09T05:21:28.466269+0000 mon.a (mon.0) 1300 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:21:29.392 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:29 smithi194 bash[92666]: audit 2023-10-09T05:21:28.469880+0000 mon.a (mon.0) 1301 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:21:29.392 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:29 smithi194 bash[92666]: cluster 2023-10-09T05:21:28.526785+0000 mon.a (mon.0) 1302 : cluster 3 Health check update: Degraded data redundancy: 19738/207297 objects degraded (9.522%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:21:29.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:29.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:29.494 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:29.520 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:29.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:29.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:29.625 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:29.662 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:29.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:29.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:29.774 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:29.799 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:29.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:29.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:29.902 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:29.933 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:29.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:29.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:30.035 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:30.057 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:30.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:30.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:30.160 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:30.183 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:30.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:30.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:30.286 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:30.325 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:30.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:30.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:30.428 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:30.455 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:30.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:30.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:30.558 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:30.595 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:30.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:30.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:30.703 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:30.733 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:30.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:30.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:30.839 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:30.866 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:30.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:30.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:30.972 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:31.006 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:31.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:31.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:31.109 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:31.134 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:31.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:31.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:31.237 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:31.256 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:31.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:31.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:31.359 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:31.383 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:31.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:31.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:31.486 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:31.514 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:31.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:31.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:31.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:31 smithi133 bash[102858]: cluster 2023-10-09T05:21:30.045337+0000 mgr.y (mgr.25277) 996 : cluster 0 pgmap v1043: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 19794/203190 objects degraded (9.742%); 13376/203190 objects misplaced (6.583%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:21:31.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:31 smithi133 bash[105589]: cluster 2023-10-09T05:21:30.045337+0000 mgr.y (mgr.25277) 996 : cluster 0 pgmap v1043: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 19794/203190 objects degraded (9.742%); 13376/203190 objects misplaced (6.583%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:21:31.618 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:31.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:31 smithi194 bash[92666]: cluster 2023-10-09T05:21:30.045337+0000 mgr.y (mgr.25277) 996 : cluster 0 pgmap v1043: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 19794/203190 objects degraded (9.742%); 13376/203190 objects misplaced (6.583%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:21:31.648 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:31.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:31.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:31.751 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:31.778 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:31.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:31.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:31.888 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:31.918 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:31.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:31.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:32.024 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:32.068 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:32.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:32.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:32.173 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:32.205 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:32.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:32.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:32.265 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T05:21:32.312 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:32.334 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:32.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:32.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:32.436 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:32.476 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:32.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:32.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:32.580 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:32.607 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:32.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:32.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:32.710 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:32.740 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:32.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:32.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:32.844 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:32.865 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:32.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:32.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:32.968 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:33.017 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:33.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:33.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:33.121 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:33.156 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:21:33.162 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:33.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:33.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:33.271 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:33.296 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:33.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:33 smithi133 bash[102858]: cluster 2023-10-09T05:21:32.046045+0000 mgr.y (mgr.25277) 997 : cluster 0 pgmap v1044: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 19826/199809 objects degraded (9.922%); 13227/199809 objects misplaced (6.620%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:21:33.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:33 smithi133 bash[105589]: cluster 2023-10-09T05:21:32.046045+0000 mgr.y (mgr.25277) 997 : cluster 0 pgmap v1044: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 19826/199809 objects degraded (9.922%); 13227/199809 objects misplaced (6.620%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:21:33.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:33.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:33.400 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:33.407 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:33 smithi194 bash[92666]: cluster 2023-10-09T05:21:32.046045+0000 mgr.y (mgr.25277) 997 : cluster 0 pgmap v1044: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 19826/199809 objects degraded (9.922%); 13227/199809 objects misplaced (6.620%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:21:33.432 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:33.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:33.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:33.541 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:33.563 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:33.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:33.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:33.619 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:21:33.619 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:21:33.666 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:33.702 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:33.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:33.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:33.808 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:33.835 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:33.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:33.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:33.938 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:33.969 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:33.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:33.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:34.076 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:34.103 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:34.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:34.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:34.210 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:34.219 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:34 smithi133 bash[102858]: cluster 2023-10-09T05:21:33.527981+0000 mon.a (mon.0) 1303 : cluster 3 Health check update: Degraded data redundancy: 19826/199809 objects degraded (9.922%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:21:34.220 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:34 smithi133 bash[105589]: cluster 2023-10-09T05:21:33.527981+0000 mon.a (mon.0) 1303 : cluster 3 Health check update: Degraded data redundancy: 19826/199809 objects degraded (9.922%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:21:34.247 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:34.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:34.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:34.284 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:21:34.353 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:34.381 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:34.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:34.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:34.484 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:34.493 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:21:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:21:34] "GET /metrics HTTP/1.1" 200 36527 "" "Prometheus/2.43.0" 2023-10-09T05:21:34.524 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:34.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:34.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:34.633 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:34.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:34 smithi194 bash[92666]: cluster 2023-10-09T05:21:33.527981+0000 mon.a (mon.0) 1303 : cluster 3 Health check update: Degraded data redundancy: 19826/199809 objects degraded (9.922%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:21:34.681 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:34.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:34.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:34.787 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:34.821 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T05:21:34.825 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:34.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:34.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:34.932 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:34.974 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:34.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:34.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:35.077 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:35.107 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:35.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:35.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:35.210 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:35.235 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:35.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:35.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:35.340 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:35.348 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:35 smithi194 bash[92666]: cluster 2023-10-09T05:21:34.046699+0000 mgr.y (mgr.25277) 998 : cluster 0 pgmap v1045: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 531 op/s; 19904/197373 objects degraded (10.084%); 13021/197373 objects misplaced (6.597%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T05:21:35.370 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:35.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:35.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:35.473 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:35.482 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:35 smithi133 bash[105589]: cluster 2023-10-09T05:21:34.046699+0000 mgr.y (mgr.25277) 998 : cluster 0 pgmap v1045: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 531 op/s; 19904/197373 objects degraded (10.084%); 13021/197373 objects misplaced (6.597%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T05:21:35.482 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:35 smithi133 bash[102858]: cluster 2023-10-09T05:21:34.046699+0000 mgr.y (mgr.25277) 998 : cluster 0 pgmap v1045: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 531 op/s; 19904/197373 objects degraded (10.084%); 13021/197373 objects misplaced (6.597%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T05:21:35.517 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:35.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:35.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:35.625 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:35.665 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:35.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:35.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:35.773 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:35.806 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:35.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:35.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:35.911 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:35.934 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:35.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:35.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:36.040 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:36.067 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:36.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:36.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:36.175 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:36.207 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:36.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:36.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:36.314 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:36.341 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:36.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:36.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:36.446 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:36.469 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:36.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:36.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:36.577 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:36.598 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:36.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:36.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:36.704 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:36.740 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:36.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:36.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:36.846 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:36.868 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:36.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:36.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:36.971 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:37.007 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:37.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:37.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:37.112 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:37.118 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:21:37.118 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:21:37.162 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:37.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:37.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:37.269 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:37.279 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:37 smithi133 bash[102858]: cluster 2023-10-09T05:21:36.047339+0000 mgr.y (mgr.25277) 999 : cluster 0 pgmap v1046: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 19942/194100 objects degraded (10.274%); 12871/194100 objects misplaced (6.631%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T05:21:37.279 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:37 smithi133 bash[105589]: cluster 2023-10-09T05:21:36.047339+0000 mgr.y (mgr.25277) 999 : cluster 0 pgmap v1046: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 19942/194100 objects degraded (10.274%); 12871/194100 objects misplaced (6.631%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T05:21:37.302 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:37.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:37.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:37.405 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:37.414 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:37 smithi194 bash[92666]: cluster 2023-10-09T05:21:36.047339+0000 mgr.y (mgr.25277) 999 : cluster 0 pgmap v1046: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 19942/194100 objects degraded (10.274%); 12871/194100 objects misplaced (6.631%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T05:21:37.430 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:37.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:37.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:37.535 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:37.559 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:37.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:37.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:37.664 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:37.695 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:37.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:37.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:37.801 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:37.829 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:37.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:37.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:37.937 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:37.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:37.974 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:37.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:38.075 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:38.103 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:38.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:38.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:38.206 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:38.232 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:38.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:38.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:38.339 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:38.391 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:38.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:38.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:38.493 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:38.518 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:38.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:38.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:38.624 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:38.650 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:38.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:38.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:38.754 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:38.779 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:38.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:38.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:38.887 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:38.918 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:38.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:38.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:39.024 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:39.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:39.070 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:39.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:39.154 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:21:39.171 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:39.199 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:39.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:39.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:39.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:39 smithi133 bash[102858]: cluster 2023-10-09T05:21:38.047997+0000 mgr.y (mgr.25277) 1000 : cluster 0 pgmap v1047: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 19942/191628 objects degraded (10.407%); 12871/191628 objects misplaced (6.717%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T05:21:39.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:39 smithi133 bash[102858]: cluster 2023-10-09T05:21:38.529054+0000 mon.a (mon.0) 1304 : cluster 3 Health check update: Degraded data redundancy: 19942/191628 objects degraded (10.407%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:21:39.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:39 smithi133 bash[105589]: cluster 2023-10-09T05:21:38.047997+0000 mgr.y (mgr.25277) 1000 : cluster 0 pgmap v1047: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 19942/191628 objects degraded (10.407%); 12871/191628 objects misplaced (6.717%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T05:21:39.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:39 smithi133 bash[105589]: cluster 2023-10-09T05:21:38.529054+0000 mon.a (mon.0) 1304 : cluster 3 Health check update: Degraded data redundancy: 19942/191628 objects degraded (10.407%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:21:39.304 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:39.330 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:39.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:39.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:39.437 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:39.446 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:39 smithi194 bash[92666]: cluster 2023-10-09T05:21:38.047997+0000 mgr.y (mgr.25277) 1000 : cluster 0 pgmap v1047: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 19942/191628 objects degraded (10.407%); 12871/191628 objects misplaced (6.717%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T05:21:39.446 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:39 smithi194 bash[92666]: cluster 2023-10-09T05:21:38.529054+0000 mon.a (mon.0) 1304 : cluster 3 Health check update: Degraded data redundancy: 19942/191628 objects degraded (10.407%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:21:39.467 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:39.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:39.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:39.573 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:39.604 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:39.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:39.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:39.621 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:21:39.707 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:39.744 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:39.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:39.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:39.848 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:39.879 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:39.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:39.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:39.984 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:40.012 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:40.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:40.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:40.120 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:40.144 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:40.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:40.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:40.251 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:40.272 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:40.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:40.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:40.375 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:40.403 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:40.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:40.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:40.508 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:40.540 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:40.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:40.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:40.643 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:40.665 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:40.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:40.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:40.767 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:40.820 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:40.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:40.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:40.924 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:40.947 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:40.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:40.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:41.053 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:41.087 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:41.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:41.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:41.191 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:41.230 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:41.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:41.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:41.342 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:41.350 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:41 smithi194 bash[92666]: cluster 2023-10-09T05:21:40.048792+0000 mgr.y (mgr.25277) 1001 : cluster 0 pgmap v1048: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 556 op/s; 19989/187377 objects degraded (10.668%); 12698/187377 objects misplaced (6.777%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T05:21:41.363 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:41.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:41.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:41.467 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:41.497 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:41.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:41.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:41.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:41 smithi133 bash[102858]: cluster 2023-10-09T05:21:40.048792+0000 mgr.y (mgr.25277) 1001 : cluster 0 pgmap v1048: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 556 op/s; 19989/187377 objects degraded (10.668%); 12698/187377 objects misplaced (6.777%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T05:21:41.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:41 smithi133 bash[105589]: cluster 2023-10-09T05:21:40.048792+0000 mgr.y (mgr.25277) 1001 : cluster 0 pgmap v1048: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 556 op/s; 19989/187377 objects degraded (10.668%); 12698/187377 objects misplaced (6.777%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T05:21:41.600 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:41.631 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:41.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:41.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:41.733 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:41.757 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:41.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:41.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:41.924 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:41.961 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:41.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:41.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:42.069 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:42.091 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:42.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:42.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:42.194 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:42.233 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:42.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:42.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:42.340 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:42.372 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:42.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:42.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:42.475 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:42.502 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:42.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:42.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:42.608 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:42.635 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:42.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:42.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:42.743 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:42.794 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:42.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:42.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:42.900 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:42.925 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:42.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:42.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:43.028 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:43.056 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:43.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:43.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:43.164 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:43.207 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:43.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:43.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:43.312 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:43.357 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:43.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:43.366 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:43 smithi133 bash[102858]: cluster 2023-10-09T05:21:42.049547+0000 mgr.y (mgr.25277) 1002 : cluster 0 pgmap v1049: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 539 op/s; 20042/183777 objects degraded (10.906%); 12525/183777 objects misplaced (6.815%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T05:21:43.367 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:43 smithi133 bash[105589]: cluster 2023-10-09T05:21:42.049547+0000 mgr.y (mgr.25277) 1002 : cluster 0 pgmap v1049: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 539 op/s; 20042/183777 objects degraded (10.906%); 12525/183777 objects misplaced (6.815%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T05:21:43.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:43.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:43 smithi194 bash[92666]: cluster 2023-10-09T05:21:42.049547+0000 mgr.y (mgr.25277) 1002 : cluster 0 pgmap v1049: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 539 op/s; 20042/183777 objects degraded (10.906%); 12525/183777 objects misplaced (6.815%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T05:21:43.468 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:43.497 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:43.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:43.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:43.603 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:43.632 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:43.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:43.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:43.740 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:43.769 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:43.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:43.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:43.875 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:43.905 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:43.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:43.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:44.009 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:44.045 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:44.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:44.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:44.150 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:44.176 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:44.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:44.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:44.280 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:44.307 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:44.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:44.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:44.410 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:44.418 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:21:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:21:44] "GET /metrics HTTP/1.1" 200 36527 "" "Prometheus/2.43.0" 2023-10-09T05:21:44.419 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:44 smithi133 bash[102858]: audit 2023-10-09T05:21:43.456731+0000 mon.a (mon.0) 1305 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:21:44.419 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:44 smithi133 bash[102858]: cluster 2023-10-09T05:21:43.530147+0000 mon.a (mon.0) 1306 : cluster 3 Health check update: Degraded data redundancy: 20042/183777 objects degraded (10.906%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:21:44.420 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:44 smithi133 bash[105589]: audit 2023-10-09T05:21:43.456731+0000 mon.a (mon.0) 1305 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:21:44.420 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:44 smithi133 bash[105589]: cluster 2023-10-09T05:21:43.530147+0000 mon.a (mon.0) 1306 : cluster 3 Health check update: Degraded data redundancy: 20042/183777 objects degraded (10.906%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:21:44.434 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:44.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:44.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:44.541 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:44.574 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:44.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:44.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:44.642 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:44 smithi194 bash[92666]: audit 2023-10-09T05:21:43.456731+0000 mon.a (mon.0) 1305 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:21:44.643 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:44 smithi194 bash[92666]: cluster 2023-10-09T05:21:43.530147+0000 mon.a (mon.0) 1306 : cluster 3 Health check update: Degraded data redundancy: 20042/183777 objects degraded (10.906%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:21:44.683 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:44.712 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:44.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:44.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:44.821 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:44.851 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:44.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:44.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:44.956 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:44.980 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:44.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:44.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:45.089 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:45.127 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:45.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:45.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:45.230 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:45.237 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:45 smithi194 bash[92666]: cluster 2023-10-09T05:21:44.050202+0000 mgr.y (mgr.25277) 1003 : cluster 0 pgmap v1050: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 20086/181341 objects degraded (11.076%); 12356/181341 objects misplaced (6.814%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T05:21:45.254 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:45.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:45.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:45.359 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:45.388 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:45.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:45.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:45.496 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:45.505 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:45 smithi133 bash[105589]: cluster 2023-10-09T05:21:44.050202+0000 mgr.y (mgr.25277) 1003 : cluster 0 pgmap v1050: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 20086/181341 objects degraded (11.076%); 12356/181341 objects misplaced (6.814%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T05:21:45.507 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:45 smithi133 bash[102858]: cluster 2023-10-09T05:21:44.050202+0000 mgr.y (mgr.25277) 1003 : cluster 0 pgmap v1050: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 20086/181341 objects degraded (11.076%); 12356/181341 objects misplaced (6.814%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T05:21:45.532 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:45.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:45.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:45.583 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:21:45.583 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:21:45.637 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:45.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:45.682 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:45.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:45.783 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:45.807 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:45.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:45.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:45.911 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:45.946 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:45.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:45.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:46.056 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:46.085 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:46.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:46.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:46.192 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:46.229 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:46.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:46.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:46.336 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:46.358 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:46.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:46.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:46.461 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:46.511 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:46.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:46.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:46.616 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:46.649 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:46.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:46.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:46.755 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:46.782 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:46.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:46.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:46.791 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T05:21:46.888 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:46.916 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:46.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:46.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:47.020 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:47.054 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:47.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:47.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:47.161 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:47.184 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:47.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:47.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:47.288 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:47.296 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:47 smithi194 bash[92666]: cluster 2023-10-09T05:21:46.051010+0000 mgr.y (mgr.25277) 1004 : cluster 0 pgmap v1051: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 539 op/s; 20106/177957 objects degraded (11.298%); 12209/177957 objects misplaced (6.861%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T05:21:47.318 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:47.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:47.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:47.425 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:47.459 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:47.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:47.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:47.465 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:47 smithi133 bash[102858]: cluster 2023-10-09T05:21:46.051010+0000 mgr.y (mgr.25277) 1004 : cluster 0 pgmap v1051: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 539 op/s; 20106/177957 objects degraded (11.298%); 12209/177957 objects misplaced (6.861%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T05:21:47.465 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:47 smithi133 bash[105589]: cluster 2023-10-09T05:21:46.051010+0000 mgr.y (mgr.25277) 1004 : cluster 0 pgmap v1051: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 539 op/s; 20106/177957 objects degraded (11.298%); 12209/177957 objects misplaced (6.861%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T05:21:47.566 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:47.590 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:47.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:47.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:47.696 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:47.723 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:47.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:47.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:47.826 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:47.857 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:47.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:47.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:47.961 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:47.993 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:48.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:48.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:48.102 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:48.131 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:48.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:48.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:48.235 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:48.258 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:48.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:48.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:48.364 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:48.388 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:48.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:48.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:48.492 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:48.514 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:48.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:48.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:48.617 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:48.647 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:48.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:48.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:48.757 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:48.786 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:48.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:48.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:48.892 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:48.916 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:48.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:48.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:49.020 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:49.044 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:49.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:49.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:49.148 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:49.184 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:49.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:49.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:49.287 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:49.325 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:49.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:49.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:49.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:49 smithi194 bash[92666]: cluster 2023-10-09T05:21:48.051628+0000 mgr.y (mgr.25277) 1005 : cluster 0 pgmap v1052: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 517 op/s; 20106/175467 objects degraded (11.459%); 12209/175467 objects misplaced (6.958%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:21:49.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:49 smithi194 bash[92666]: cluster 2023-10-09T05:21:48.531280+0000 mon.a (mon.0) 1307 : cluster 3 Health check update: Degraded data redundancy: 20106/175467 objects degraded (11.459%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:21:49.430 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:49.461 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:49.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:49.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:49.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:49 smithi133 bash[102858]: cluster 2023-10-09T05:21:48.051628+0000 mgr.y (mgr.25277) 1005 : cluster 0 pgmap v1052: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 517 op/s; 20106/175467 objects degraded (11.459%); 12209/175467 objects misplaced (6.958%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:21:49.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:49 smithi133 bash[102858]: cluster 2023-10-09T05:21:48.531280+0000 mon.a (mon.0) 1307 : cluster 3 Health check update: Degraded data redundancy: 20106/175467 objects degraded (11.459%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:21:49.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:49 smithi133 bash[105589]: cluster 2023-10-09T05:21:48.051628+0000 mgr.y (mgr.25277) 1005 : cluster 0 pgmap v1052: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 517 op/s; 20106/175467 objects degraded (11.459%); 12209/175467 objects misplaced (6.958%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:21:49.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:49 smithi133 bash[105589]: cluster 2023-10-09T05:21:48.531280+0000 mon.a (mon.0) 1307 : cluster 3 Health check update: Degraded data redundancy: 20106/175467 objects degraded (11.459%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:21:49.565 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:49.609 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:49.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:49.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:49.715 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:49.741 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:49.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:49.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:49.848 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:49.876 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:49.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:49.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:49.979 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:50.008 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:50.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:50.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:50.117 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:50.156 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:50.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:50.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:50.260 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:50.294 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:50.295 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:21:50.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:50.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:50.343 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:21:50.391 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:21:50.399 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:50.424 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:50.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:50.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:50.438 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:21:50.478 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:21:50.511 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:21:50.528 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:50.557 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:21:50.565 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:50.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:50.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:50.596 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:21:50.598 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:21:50.599 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:21:50.599 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:21:50.599 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:21:50.599 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:21:50.599 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:21:50.599 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:21:50.599 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:21:50.673 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:50.700 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:50.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:50.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:50.805 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:50.835 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:50.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:50.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:50.942 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:50.983 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:50.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:50.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:51.087 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:51.128 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:51.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:51.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:51.233 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:51.267 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:51.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:51.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:51.369 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:51.378 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:51 smithi133 bash[105589]: cluster 2023-10-09T05:21:50.052414+0000 mgr.y (mgr.25277) 1006 : cluster 0 pgmap v1053: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 20131/171450 objects degraded (11.742%); 12045/171450 objects misplaced (7.025%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T05:21:51.378 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:51 smithi133 bash[105589]: audit 2023-10-09T05:21:50.244756+0000 mon.a (mon.0) 1308 : audit 0 from='client.? 172.21.15.133:0/3510411716' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:21:51.379 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:51 smithi133 bash[102858]: cluster 2023-10-09T05:21:50.052414+0000 mgr.y (mgr.25277) 1006 : cluster 0 pgmap v1053: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 20131/171450 objects degraded (11.742%); 12045/171450 objects misplaced (7.025%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T05:21:51.379 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:51 smithi133 bash[102858]: audit 2023-10-09T05:21:50.244756+0000 mon.a (mon.0) 1308 : audit 0 from='client.? 172.21.15.133:0/3510411716' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:21:51.394 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:51.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:51.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:51.502 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:51.528 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:51.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:51.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:51.633 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:51.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:51 smithi194 bash[92666]: cluster 2023-10-09T05:21:50.052414+0000 mgr.y (mgr.25277) 1006 : cluster 0 pgmap v1053: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 20131/171450 objects degraded (11.742%); 12045/171450 objects misplaced (7.025%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T05:21:51.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:51 smithi194 bash[92666]: audit 2023-10-09T05:21:50.244756+0000 mon.a (mon.0) 1308 : audit 0 from='client.? 172.21.15.133:0/3510411716' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:21:51.647 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:21:51.662 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:51.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:51.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:51.777 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:51.812 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:51.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:51.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:51.916 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:51.962 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:51.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:51.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:52.068 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:52.091 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:52.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:52.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:52.195 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:52.218 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:52.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:52.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:52.325 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:52.354 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:52.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:52.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:52.462 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:52.487 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:52.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:52.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:52.593 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:52.616 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:52.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:52.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:52.721 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:52.753 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:52.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:52.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:52.861 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:52.884 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:52.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:52.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:52.991 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:53.018 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:53.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:53.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:53.124 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:53.146 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:53.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:53.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:53.249 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:53.278 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:53.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:53.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:53.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:53 smithi133 bash[102858]: cluster 2023-10-09T05:21:52.053247+0000 mgr.y (mgr.25277) 1007 : cluster 0 pgmap v1054: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 20172/167925 objects degraded (12.013%); 11876/167925 objects misplaced (7.072%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T05:21:53.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:53 smithi133 bash[105589]: cluster 2023-10-09T05:21:52.053247+0000 mgr.y (mgr.25277) 1007 : cluster 0 pgmap v1054: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 20172/167925 objects degraded (12.013%); 11876/167925 objects misplaced (7.072%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T05:21:53.384 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:53.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:53 smithi194 bash[92666]: cluster 2023-10-09T05:21:52.053247+0000 mgr.y (mgr.25277) 1007 : cluster 0 pgmap v1054: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 20172/167925 objects degraded (12.013%); 11876/167925 objects misplaced (7.072%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T05:21:53.422 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:53.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:53.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:53.531 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:53.571 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:53.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:53.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:53.675 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:53.699 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:53.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:53.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:53.805 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:53.829 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:53.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:53.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:53.932 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:53.965 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:53.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:53.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:54.072 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:54.097 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:54.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:54.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:54.201 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:54.210 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:54 smithi194 bash[92666]: cluster 2023-10-09T05:21:53.532179+0000 mon.a (mon.0) 1309 : cluster 3 Health check update: Degraded data redundancy: 20172/167925 objects degraded (12.013%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:21:54.210 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:54 smithi194 bash[92666]: audit 2023-10-09T05:21:53.738792+0000 mon.c (mon.2) 95 : audit 1 from='client.? 172.21.15.133:0/3358600109' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:21:54.210 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:54 smithi194 bash[92666]: audit 2023-10-09T05:21:53.740494+0000 mon.a (mon.0) 1310 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:21:54.225 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:54.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:54.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:54.328 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:54.359 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:54.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:54.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:54.461 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:54.472 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:54 smithi194 bash[92666]: audit 2023-10-09T05:21:53.798821+0000 mon.a (mon.0) 1311 : audit 1 from='client.? 172.21.15.133:0/4182173157' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:21:54.499 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:54.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:54.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:54.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:54 smithi133 bash[102858]: cluster 2023-10-09T05:21:53.532179+0000 mon.a (mon.0) 1309 : cluster 3 Health check update: Degraded data redundancy: 20172/167925 objects degraded (12.013%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:21:54.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:54 smithi133 bash[102858]: audit 2023-10-09T05:21:53.738792+0000 mon.c (mon.2) 95 : audit 1 from='client.? 172.21.15.133:0/3358600109' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:21:54.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:54 smithi133 bash[102858]: audit 2023-10-09T05:21:53.740494+0000 mon.a (mon.0) 1310 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:21:54.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:54 smithi133 bash[102858]: audit 2023-10-09T05:21:53.798821+0000 mon.a (mon.0) 1311 : audit 1 from='client.? 172.21.15.133:0/4182173157' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:21:54.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:21:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:21:54] "GET /metrics HTTP/1.1" 200 36527 "" "Prometheus/2.43.0" 2023-10-09T05:21:54.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:54 smithi133 bash[105589]: cluster 2023-10-09T05:21:53.532179+0000 mon.a (mon.0) 1309 : cluster 3 Health check update: Degraded data redundancy: 20172/167925 objects degraded (12.013%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:21:54.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:54 smithi133 bash[105589]: audit 2023-10-09T05:21:53.738792+0000 mon.c (mon.2) 95 : audit 1 from='client.? 172.21.15.133:0/3358600109' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:21:54.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:54 smithi133 bash[105589]: audit 2023-10-09T05:21:53.740494+0000 mon.a (mon.0) 1310 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:21:54.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:54 smithi133 bash[105589]: audit 2023-10-09T05:21:53.798821+0000 mon.a (mon.0) 1311 : audit 1 from='client.? 172.21.15.133:0/4182173157' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:21:54.575 INFO:teuthology.orchestra.run.smithi133.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T05:21:54.575 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is set 2023-10-09T05:21:54.603 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:54.638 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:54.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:54.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:54.740 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:54.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:54.790 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:54.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:54.890 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:54.918 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:54.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:54.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:55.023 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:55.051 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:21:55.052 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:55.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:55.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:55.156 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:55.182 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:55.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:55.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:55.219 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:55 smithi133 bash[102858]: cluster 2023-10-09T05:21:54.053908+0000 mgr.y (mgr.25277) 1008 : cluster 0 pgmap v1055: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 507 op/s; 20172/165516 objects degraded (12.187%); 11730/165516 objects misplaced (7.087%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T05:21:55.219 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:55 smithi133 bash[102858]: cluster 2023-10-09T05:21:54.541825+0000 mon.a (mon.0) 1312 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:21:55.219 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:55 smithi133 bash[102858]: cluster 2023-10-09T05:21:54.557299+0000 osd.5 (osd.5) 230 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T05:21:55.219 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:55 smithi133 bash[102858]: cluster 2023-10-09T05:21:54.559011+0000 osd.6 (osd.6) 186 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T05:21:55.219 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:55 smithi133 bash[102858]: cluster 2023-10-09T05:21:54.562835+0000 osd.4 (osd.4) 209 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T05:21:55.219 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:55 smithi133 bash[102858]: audit 2023-10-09T05:21:54.564185+0000 mon.a (mon.0) 1313 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:21:55.219 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:55 smithi133 bash[102858]: audit 2023-10-09T05:21:54.564482+0000 mon.a (mon.0) 1314 : audit 1 from='client.? 172.21.15.133:0/4182173157' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:21:55.219 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:55 smithi133 bash[102858]: cluster 2023-10-09T05:21:54.564975+0000 mon.a (mon.0) 1315 : cluster 0 osdmap e245: 8 total, 8 up, 7 in 2023-10-09T05:21:55.220 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:55 smithi133 bash[102858]: cluster 2023-10-09T05:21:54.569692+0000 osd.7 (osd.7) 192 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T05:21:55.289 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:55.334 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:55.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:55.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:55.440 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:55.485 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:55.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:55.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:55.492 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:55 smithi133 bash[105589]: cluster 2023-10-09T05:21:54.053908+0000 mgr.y (mgr.25277) 1008 : cluster 0 pgmap v1055: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 507 op/s; 20172/165516 objects degraded (12.187%); 11730/165516 objects misplaced (7.087%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T05:21:55.492 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:55 smithi133 bash[105589]: cluster 2023-10-09T05:21:54.541825+0000 mon.a (mon.0) 1312 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:21:55.492 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:55 smithi133 bash[105589]: cluster 2023-10-09T05:21:54.557299+0000 osd.5 (osd.5) 230 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T05:21:55.492 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:55 smithi133 bash[105589]: cluster 2023-10-09T05:21:54.559011+0000 osd.6 (osd.6) 186 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T05:21:55.492 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:55 smithi133 bash[105589]: cluster 2023-10-09T05:21:54.562835+0000 osd.4 (osd.4) 209 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T05:21:55.492 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:55 smithi133 bash[105589]: audit 2023-10-09T05:21:54.564185+0000 mon.a (mon.0) 1313 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:21:55.492 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:55 smithi133 bash[105589]: audit 2023-10-09T05:21:54.564482+0000 mon.a (mon.0) 1314 : audit 1 from='client.? 172.21.15.133:0/4182173157' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:21:55.492 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:55 smithi133 bash[105589]: cluster 2023-10-09T05:21:54.564975+0000 mon.a (mon.0) 1315 : cluster 0 osdmap e245: 8 total, 8 up, 7 in 2023-10-09T05:21:55.492 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:55 smithi133 bash[105589]: cluster 2023-10-09T05:21:54.569692+0000 osd.7 (osd.7) 192 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T05:21:55.592 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:55.595 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:55 smithi194 bash[92666]: cluster 2023-10-09T05:21:54.053908+0000 mgr.y (mgr.25277) 1008 : cluster 0 pgmap v1055: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 507 op/s; 20172/165516 objects degraded (12.187%); 11730/165516 objects misplaced (7.087%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T05:21:55.595 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:55 smithi194 bash[92666]: cluster 2023-10-09T05:21:54.541825+0000 mon.a (mon.0) 1312 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:21:55.595 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:55 smithi194 bash[92666]: cluster 2023-10-09T05:21:54.557299+0000 osd.5 (osd.5) 230 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T05:21:55.595 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:55 smithi194 bash[92666]: cluster 2023-10-09T05:21:54.559011+0000 osd.6 (osd.6) 186 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T05:21:55.595 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:55 smithi194 bash[92666]: cluster 2023-10-09T05:21:54.562835+0000 osd.4 (osd.4) 209 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T05:21:55.595 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:55 smithi194 bash[92666]: audit 2023-10-09T05:21:54.564185+0000 mon.a (mon.0) 1313 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:21:55.595 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:55 smithi194 bash[92666]: audit 2023-10-09T05:21:54.564482+0000 mon.a (mon.0) 1314 : audit 1 from='client.? 172.21.15.133:0/4182173157' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:21:55.596 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:55 smithi194 bash[92666]: cluster 2023-10-09T05:21:54.564975+0000 mon.a (mon.0) 1315 : cluster 0 osdmap e245: 8 total, 8 up, 7 in 2023-10-09T05:21:55.596 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:55 smithi194 bash[92666]: cluster 2023-10-09T05:21:54.569692+0000 osd.7 (osd.7) 192 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T05:21:55.615 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:55.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:55.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:55.719 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:55.759 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:55.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:55.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:55.866 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:55.892 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:55.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:55.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:55.998 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:56.033 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:56.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:56.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:56.138 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:56.170 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:56.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:56.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:56.273 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:56.295 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:56.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:56.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:56.397 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:56.431 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:56.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:56.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:56.539 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:56.562 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:21:56.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:56.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:21:56.665 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:56.699 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:56.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:56.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:56.803 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:56.835 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:56.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:56.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:56.941 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:56.968 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:56.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:56.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:57.049 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T05:21:57.076 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:57.138 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:57.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:57.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:57.226 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:57 smithi133 bash[102858]: cluster 2023-10-09T05:21:56.054661+0000 mgr.y (mgr.25277) 1009 : cluster 0 pgmap v1057: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 518 op/s; 20194/162402 objects degraded (12.435%); 11590/162402 objects misplaced (7.137%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T05:21:57.247 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:57.257 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:57 smithi133 bash[105589]: cluster 2023-10-09T05:21:56.054661+0000 mgr.y (mgr.25277) 1009 : cluster 0 pgmap v1057: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 518 op/s; 20194/162402 objects degraded (12.435%); 11590/162402 objects misplaced (7.137%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T05:21:57.285 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:57.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:57.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:57.388 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:57.422 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:57.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:57.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:57.476 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:57 smithi194 bash[92666]: cluster 2023-10-09T05:21:56.054661+0000 mgr.y (mgr.25277) 1009 : cluster 0 pgmap v1057: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 518 op/s; 20194/162402 objects degraded (12.435%); 11590/162402 objects misplaced (7.137%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T05:21:57.527 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:57.551 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:57.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:57.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:57.654 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:57.679 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:57.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:57.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:57.773 INFO:teuthology.orchestra.run.smithi194.stdout:{ 2023-10-09T05:21:57.773 INFO:teuthology.orchestra.run.smithi194.stdout: "ops": [ 2023-10-09T05:21:57.774 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.774 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35187.0:139858 11.f 11:f0848141:::benchmark_data_smithi133_194104_object43938:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e245)", 2023-10-09T05:21:57.774 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:21:57.761019+0000", 2023-10-09T05:21:57.774 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 0.011399085, 2023-10-09T05:21:57.774 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.011442117999999999, 2023-10-09T05:21:57.774 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:21:57.774 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:21:57.774 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:21:57.775 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35187", 2023-10-09T05:21:57.775 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2118344251", 2023-10-09T05:21:57.775 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 139858 2023-10-09T05:21:57.775 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.775 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:21:57.775 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.775 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:21:57.775 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.761019+0000", 2023-10-09T05:21:57.776 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:57.776 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.776 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.776 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:21:57.776 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.761019+0000", 2023-10-09T05:21:57.776 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:57.777 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.777 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.777 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:21:57.777 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.761018+0000", 2023-10-09T05:21:57.777 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:21:57.777 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.778 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.778 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:21:57.778 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.761024+0000", 2023-10-09T05:21:57.778 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.677e-06 2023-10-09T05:21:57.778 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.778 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.779 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:21:57.779 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.761035+0000", 2023-10-09T05:21:57.779 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.1242e-05 2023-10-09T05:21:57.779 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.779 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.779 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:21:57.779 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.761041+0000", 2023-10-09T05:21:57.780 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.4650000000000002e-06 2023-10-09T05:21:57.780 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.780 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.780 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:57.780 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.761094+0000", 2023-10-09T05:21:57.780 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.3276000000000003e-05 2023-10-09T05:21:57.780 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.780 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.780 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:21:57.781 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.761288+0000", 2023-10-09T05:21:57.781 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000194408 2023-10-09T05:21:57.781 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.781 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.781 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:21:57.781 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.761490+0000", 2023-10-09T05:21:57.781 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00020120000000000001 2023-10-09T05:21:57.781 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.782 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.782 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:21:57.782 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.764873+0000", 2023-10-09T05:21:57.782 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0033829860000000002 2023-10-09T05:21:57.782 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:57.782 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:21:57.782 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:57.783 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.783 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.783 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35187.0:139860 11.4 11:2707f6a7:::benchmark_data_smithi133_194104_object43940:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e245)", 2023-10-09T05:21:57.783 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:21:57.765847+0000", 2023-10-09T05:21:57.783 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 0.0065711670000000002, 2023-10-09T05:21:57.783 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0067389620000000003, 2023-10-09T05:21:57.783 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:21:57.783 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:21:57.783 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:21:57.784 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35187", 2023-10-09T05:21:57.784 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2118344251", 2023-10-09T05:21:57.784 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 139860 2023-10-09T05:21:57.784 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.784 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:21:57.784 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.784 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:21:57.784 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.765847+0000", 2023-10-09T05:21:57.784 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:57.784 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.785 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.785 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:21:57.785 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.765847+0000", 2023-10-09T05:21:57.785 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:57.785 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.785 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.785 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:21:57.785 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.765846+0000", 2023-10-09T05:21:57.785 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:21:57.785 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.786 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.786 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:21:57.786 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.765851+0000", 2023-10-09T05:21:57.786 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.8389999999999997e-06 2023-10-09T05:21:57.786 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.786 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.787 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:21:57.787 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.765863+0000", 2023-10-09T05:21:57.787 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.2082000000000001e-05 2023-10-09T05:21:57.787 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.787 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.787 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:21:57.788 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.765871+0000", 2023-10-09T05:21:57.788 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.9240000000000007e-06 2023-10-09T05:21:57.788 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.788 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.788 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:57.788 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.765904+0000", 2023-10-09T05:21:57.788 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.3045e-05 2023-10-09T05:21:57.788 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.789 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.789 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:21:57.789 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.766005+0000", 2023-10-09T05:21:57.789 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000100828 2023-10-09T05:21:57.789 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.789 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.789 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:21:57.789 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.766092+0000", 2023-10-09T05:21:57.790 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.6551000000000005e-05 2023-10-09T05:21:57.790 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.790 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.790 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:21:57.790 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.769596+0000", 2023-10-09T05:21:57.790 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0035040879999999998 2023-10-09T05:21:57.790 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.790 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.791 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:21:57.791 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.770845+0000", 2023-10-09T05:21:57.791 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0012492390000000001 2023-10-09T05:21:57.791 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:57.791 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:21:57.791 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:57.791 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.791 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.791 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_repop(client.35187.0:139862 11.0 e245/225 11:0e36c9f5:::benchmark_data_smithi133_194104_object43942:head v 245'8817, mlcod=245'8814)", 2023-10-09T05:21:57.791 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:21:57.769853+0000", 2023-10-09T05:21:57.791 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 0.0025656059999999998, 2023-10-09T05:21:57.791 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0028527890000000001, 2023-10-09T05:21:57.792 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:21:57.792 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "started", 2023-10-09T05:21:57.792 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:21:57.792 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.792 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:21:57.792 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.769853+0000", 2023-10-09T05:21:57.792 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:57.792 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.792 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.792 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:21:57.792 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.769853+0000", 2023-10-09T05:21:57.793 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:57.793 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.793 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.793 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:21:57.793 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.769852+0000", 2023-10-09T05:21:57.793 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:21:57.793 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.793 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.793 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:21:57.793 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.769863+0000", 2023-10-09T05:21:57.793 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.1117999999999999e-05 2023-10-09T05:21:57.793 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.794 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.794 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:21:57.794 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.769874+0000", 2023-10-09T05:21:57.794 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.1209000000000001e-05 2023-10-09T05:21:57.794 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.794 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.794 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:21:57.794 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.769880+0000", 2023-10-09T05:21:57.794 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.8359999999999997e-06 2023-10-09T05:21:57.794 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.794 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.794 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:57.795 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.769926+0000", 2023-10-09T05:21:57.795 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.5416e-05 2023-10-09T05:21:57.795 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.795 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.795 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:21:57.795 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.769944+0000", 2023-10-09T05:21:57.795 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.8518000000000001e-05 2023-10-09T05:21:57.795 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:57.795 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:21:57.796 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:57.796 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.796 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.796 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35187.0:139841 11.f 11:fe82a903:::benchmark_data_smithi133_194104_object43921:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e245)", 2023-10-09T05:21:57.796 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:21:57.730723+0000", 2023-10-09T05:21:57.796 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 0.041695528000000003, 2023-10-09T05:21:57.796 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.042070385000000002, 2023-10-09T05:21:57.796 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:21:57.796 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:21:57.796 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:21:57.796 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35187", 2023-10-09T05:21:57.796 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2118344251", 2023-10-09T05:21:57.797 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 139841 2023-10-09T05:21:57.797 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.797 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:21:57.797 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.797 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:21:57.797 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.730723+0000", 2023-10-09T05:21:57.797 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:57.797 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.797 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.797 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:21:57.797 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.730723+0000", 2023-10-09T05:21:57.797 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:57.798 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.798 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.798 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:21:57.798 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.730722+0000", 2023-10-09T05:21:57.798 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:21:57.798 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.798 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.798 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:21:57.798 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.730728+0000", 2023-10-09T05:21:57.798 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.0519999999999998e-06 2023-10-09T05:21:57.798 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.798 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.799 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:21:57.799 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.730739+0000", 2023-10-09T05:21:57.799 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0668e-05 2023-10-09T05:21:57.799 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.799 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.799 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:21:57.799 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.730746+0000", 2023-10-09T05:21:57.799 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.1829999999999997e-06 2023-10-09T05:21:57.799 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.799 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.799 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:57.799 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.746817+0000", 2023-10-09T05:21:57.800 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.016070605000000002 2023-10-09T05:21:57.800 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.800 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.800 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:21:57.800 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.746971+0000", 2023-10-09T05:21:57.800 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00015431 2023-10-09T05:21:57.800 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.800 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.800 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:21:57.800 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.747149+0000", 2023-10-09T05:21:57.800 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00017771700000000001 2023-10-09T05:21:57.800 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.801 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.801 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:21:57.801 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.757609+0000", 2023-10-09T05:21:57.801 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.010460348 2023-10-09T05:21:57.801 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.801 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.801 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:21:57.801 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.757627+0000", 2023-10-09T05:21:57.801 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.8204000000000001e-05 2023-10-09T05:21:57.801 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:57.801 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:21:57.801 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:57.801 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.802 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.802 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35187.0:139849 11.f 11:f4cbb3aa:::benchmark_data_smithi133_194104_object43929:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e245)", 2023-10-09T05:21:57.802 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:21:57.739028+0000", 2023-10-09T05:21:57.802 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 0.033390575999999998, 2023-10-09T05:21:57.802 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.03388617, 2023-10-09T05:21:57.802 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:21:57.802 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:21:57.802 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:21:57.802 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35187", 2023-10-09T05:21:57.802 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2118344251", 2023-10-09T05:21:57.802 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 139849 2023-10-09T05:21:57.803 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.803 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:21:57.803 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.803 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:21:57.803 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.739028+0000", 2023-10-09T05:21:57.803 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:57.803 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.803 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.803 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:21:57.803 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.739028+0000", 2023-10-09T05:21:57.803 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:21:57.804 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.804 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.804 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:21:57.804 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.739027+0000", 2023-10-09T05:21:57.804 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:21:57.804 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.804 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.804 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:21:57.804 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.739032+0000", 2023-10-09T05:21:57.804 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.1569999999999999e-06 2023-10-09T05:21:57.804 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.804 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.804 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:21:57.805 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.739044+0000", 2023-10-09T05:21:57.805 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.1979e-05 2023-10-09T05:21:57.805 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.805 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.805 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:21:57.805 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.739052+0000", 2023-10-09T05:21:57.805 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.0020000000000006e-06 2023-10-09T05:21:57.805 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.805 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.805 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:21:57.806 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.752406+0000", 2023-10-09T05:21:57.806 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.013354138999999999 2023-10-09T05:21:57.806 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.806 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.806 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:21:57.806 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.752620+0000", 2023-10-09T05:21:57.806 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000214117 2023-10-09T05:21:57.806 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.806 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.806 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:21:57.807 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.752844+0000", 2023-10-09T05:21:57.807 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00022314700000000001 2023-10-09T05:21:57.807 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.807 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.807 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:21:57.807 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.757770+0000", 2023-10-09T05:21:57.807 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0049268309999999996 2023-10-09T05:21:57.807 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:21:57.807 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:21:57.808 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:21:57.808 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:21:57.764770+0000", 2023-10-09T05:21:57.808 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0070000230000000002 2023-10-09T05:21:57.808 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:57.808 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:21:57.808 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:57.808 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:21:57.808 INFO:teuthology.orchestra.run.smithi194.stdout: ], 2023-10-09T05:21:57.809 INFO:teuthology.orchestra.run.smithi194.stdout: "num_ops": 5 2023-10-09T05:21:57.809 INFO:teuthology.orchestra.run.smithi194.stdout:} 2023-10-09T05:21:57.809 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:57.862 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:57.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:57.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:57.976 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:58.007 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:58.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:58.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:58.111 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:58.134 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:58.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:58.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:58.241 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:58.279 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:58.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:58.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:21:58.381 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:58.411 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:58.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:58.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:58.516 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:58.536 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:58.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:58.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:58.639 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:58.674 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:58.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:58.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:58.784 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:58.805 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:21:58.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:58.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:21:58.909 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:58.933 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T05:21:58.938 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:58.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:58.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:59.043 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:59.063 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:21:59.086 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:59.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:59.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:59.192 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:59.235 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:21:59.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:59.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:21:59.342 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:59.353 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:59 smithi133 bash[102858]: cluster 2023-10-09T05:21:58.055273+0000 mgr.y (mgr.25277) 1010 : cluster 0 pgmap v1058: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 20194/160014 objects degraded (12.620%); 11590/160014 objects misplaced (7.243%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T05:21:59.353 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:59 smithi133 bash[102858]: audit 2023-10-09T05:21:58.457107+0000 mon.a (mon.0) 1316 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:21:59.353 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:21:59 smithi133 bash[102858]: cluster 2023-10-09T05:21:58.537059+0000 mon.a (mon.0) 1317 : cluster 3 Health check update: Degraded data redundancy: 20194/160014 objects degraded (12.620%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:21:59.354 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:59 smithi133 bash[105589]: cluster 2023-10-09T05:21:58.055273+0000 mgr.y (mgr.25277) 1010 : cluster 0 pgmap v1058: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 20194/160014 objects degraded (12.620%); 11590/160014 objects misplaced (7.243%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T05:21:59.354 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:59 smithi133 bash[105589]: audit 2023-10-09T05:21:58.457107+0000 mon.a (mon.0) 1316 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:21:59.354 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:21:59 smithi133 bash[105589]: cluster 2023-10-09T05:21:58.537059+0000 mon.a (mon.0) 1317 : cluster 3 Health check update: Degraded data redundancy: 20194/160014 objects degraded (12.620%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:21:59.374 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:21:59.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:59.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:21:59.479 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:59.523 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:21:59.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:59.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:21:59.630 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:59.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:59 smithi194 bash[92666]: cluster 2023-10-09T05:21:58.055273+0000 mgr.y (mgr.25277) 1010 : cluster 0 pgmap v1058: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 20194/160014 objects degraded (12.620%); 11590/160014 objects misplaced (7.243%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T05:21:59.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:59 smithi194 bash[92666]: audit 2023-10-09T05:21:58.457107+0000 mon.a (mon.0) 1316 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:21:59.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:21:59 smithi194 bash[92666]: cluster 2023-10-09T05:21:58.537059+0000 mon.a (mon.0) 1317 : cluster 3 Health check update: Degraded data redundancy: 20194/160014 objects degraded (12.620%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:21:59.653 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:21:59.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:59.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:21:59.756 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:59.775 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:21:59.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:59.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:21:59.852 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:21:59.878 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:59.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:21:59.918 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:21:59.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:00.019 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:00.048 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:00.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:00.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:00.151 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:00.175 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:00.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:00.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:00.278 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:00.314 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:00.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:00.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:00.421 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:00.455 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:00.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:00.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:00.559 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:00.593 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:00.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:00.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:00.695 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:00.724 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:00.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:00.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:00.828 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:01.177 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:01.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:01.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:01.280 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:01.309 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:01.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:01.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:01.416 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:01.440 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:01.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:01.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:01.543 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:01.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:01 smithi133 bash[102858]: cluster 2023-10-09T05:22:00.056015+0000 mgr.y (mgr.25277) 1011 : cluster 0 pgmap v1059: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 20194/156159 objects degraded (12.932%); 11425/156159 objects misplaced (7.316%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T05:22:01.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:01 smithi133 bash[105589]: cluster 2023-10-09T05:22:00.056015+0000 mgr.y (mgr.25277) 1011 : cluster 0 pgmap v1059: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 20194/156159 objects degraded (12.932%); 11425/156159 objects misplaced (7.316%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T05:22:01.552 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:01 smithi194 bash[92666]: cluster 2023-10-09T05:22:00.056015+0000 mgr.y (mgr.25277) 1011 : cluster 0 pgmap v1059: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 20194/156159 objects degraded (12.932%); 11425/156159 objects misplaced (7.316%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T05:22:01.578 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:01.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:01.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:01.681 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:01.711 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:01.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:01.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:01.814 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:01.891 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:22:02.029 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:02.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:02.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:02.138 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:02.163 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:02.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:02.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:02.267 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:02.296 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:02.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:02.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:02.400 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:02.430 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:02.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:02.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:02.538 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:02.566 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:02.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:02.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:02.673 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:02.712 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:02.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:02.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:02.816 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:03.004 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:03.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:03.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:03.109 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:03.140 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:03.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:03.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:03.242 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:03 smithi133 bash[102858]: cluster 2023-10-09T05:22:02.056726+0000 mgr.y (mgr.25277) 1012 : cluster 0 pgmap v1060: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 20248/152640 objects degraded (13.265%); 11252/152640 objects misplaced (7.372%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T05:22:03.243 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:03.273 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:03.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:03.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:03.376 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:03.404 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:03.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:03.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:03.506 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:03.516 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:03 smithi133 bash[105589]: cluster 2023-10-09T05:22:02.056726+0000 mgr.y (mgr.25277) 1012 : cluster 0 pgmap v1060: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 20248/152640 objects degraded (13.265%); 11252/152640 objects misplaced (7.372%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T05:22:03.530 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:03.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:03.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:03.635 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:03.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:03 smithi194 bash[92666]: cluster 2023-10-09T05:22:02.056726+0000 mgr.y (mgr.25277) 1012 : cluster 0 pgmap v1060: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 20248/152640 objects degraded (13.265%); 11252/152640 objects misplaced (7.372%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T05:22:03.662 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:03.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:03.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:03.741 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:22:03.767 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:03.817 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:03.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:03.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:03.923 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:03.929 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:22:03.948 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:03.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:03.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:04.053 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:04.094 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:04.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:04.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:04.199 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:04.227 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:04.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:04.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:04.335 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:04.362 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:04.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:04.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:04.465 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:04.474 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:04 smithi133 bash[102858]: cluster 2023-10-09T05:22:03.538189+0000 mon.a (mon.0) 1318 : cluster 3 Health check update: Degraded data redundancy: 20248/152640 objects degraded (13.265%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:04.475 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:22:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:22:04] "GET /metrics HTTP/1.1" 200 36530 "" "Prometheus/2.43.0" 2023-10-09T05:22:04.475 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:04 smithi133 bash[105589]: cluster 2023-10-09T05:22:03.538189+0000 mon.a (mon.0) 1318 : cluster 3 Health check update: Degraded data redundancy: 20248/152640 objects degraded (13.265%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:04.599 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:04.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:04.732 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:04 smithi194 bash[92666]: cluster 2023-10-09T05:22:03.538189+0000 mon.a (mon.0) 1318 : cluster 3 Health check update: Degraded data redundancy: 20248/152640 objects degraded (13.265%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:04.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:04.833 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:04.912 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:04.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:04.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:05.016 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:05.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:05 smithi133 bash[102858]: cluster 2023-10-09T05:22:04.057425+0000 mgr.y (mgr.25277) 1013 : cluster 0 pgmap v1061: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 20248/150150 objects degraded (13.485%); 11102/150150 objects misplaced (7.394%); 3.4 MiB/s, 55 objects/s recovering 2023-10-09T05:22:05.302 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:05 smithi133 bash[102858]: audit 2023-10-09T05:22:04.767524+0000 mon.a (mon.0) 1319 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:22:05.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:05 smithi133 bash[105589]: cluster 2023-10-09T05:22:04.057425+0000 mgr.y (mgr.25277) 1013 : cluster 0 pgmap v1061: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 20248/150150 objects degraded (13.485%); 11102/150150 objects misplaced (7.394%); 3.4 MiB/s, 55 objects/s recovering 2023-10-09T05:22:05.303 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:05 smithi133 bash[105589]: audit 2023-10-09T05:22:04.767524+0000 mon.a (mon.0) 1319 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:22:05.312 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:05.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:05.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:05.422 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:05.446 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:05.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:05.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:05.550 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:05.558 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:05 smithi194 bash[92666]: cluster 2023-10-09T05:22:04.057425+0000 mgr.y (mgr.25277) 1013 : cluster 0 pgmap v1061: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 20248/150150 objects degraded (13.485%); 11102/150150 objects misplaced (7.394%); 3.4 MiB/s, 55 objects/s recovering 2023-10-09T05:22:05.558 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:05 smithi194 bash[92666]: audit 2023-10-09T05:22:04.767524+0000 mon.a (mon.0) 1319 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:22:05.577 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:05.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:05.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:05.682 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:05.712 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:05.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:05.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:05.819 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:05.850 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:05.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:05.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:05.955 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:05.978 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:05.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:05.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:06.084 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:06.198 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:06.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:06.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:06.304 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:06.346 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:06.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:06.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:06.452 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:06.491 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:06.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:06.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:06.598 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:06.623 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:06.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:06.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:06.729 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:06.755 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:06.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:06.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:06.859 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:06.895 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:06.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:06.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:07.000 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:07.020 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:07.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:07.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:07.123 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:07.193 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:07.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:07.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:07.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:07 smithi133 bash[105589]: cluster 2023-10-09T05:22:06.061685+0000 mgr.y (mgr.25277) 1014 : cluster 0 pgmap v1062: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 20321/146580 objects degraded (13.863%); 10909/146580 objects misplaced (7.442%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T05:22:07.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:07 smithi133 bash[102858]: cluster 2023-10-09T05:22:06.061685+0000 mgr.y (mgr.25277) 1014 : cluster 0 pgmap v1062: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 20321/146580 objects degraded (13.863%); 10909/146580 objects misplaced (7.442%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T05:22:07.300 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:07.324 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:07.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:07.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:07.430 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:07.448 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:07.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:07.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:07.551 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:07.560 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:07 smithi194 bash[92666]: cluster 2023-10-09T05:22:06.061685+0000 mgr.y (mgr.25277) 1014 : cluster 0 pgmap v1062: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 20321/146580 objects degraded (13.863%); 10909/146580 objects misplaced (7.442%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T05:22:07.575 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:07.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:07.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:07.681 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:07.717 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:07.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:07.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:07.824 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:07.846 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:07.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:07.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:07.949 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:07.992 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:07.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:07.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:08.096 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:08.124 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:08.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:08.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:08.229 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:08.252 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:08.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:08.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:08.357 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:08.387 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:08.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:08.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:08.492 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:08.525 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:08.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:08.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:08.632 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:08.674 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:08.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:08.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:08.779 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:08.801 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:08.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:08.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:08.904 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:08.933 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:08.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:08.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:09.036 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:09.084 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:09.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:09.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:09.173 INFO:teuthology.orchestra.run.smithi194.stdout:{ 2023-10-09T05:22:09.173 INFO:teuthology.orchestra.run.smithi194.stdout: "ops": [], 2023-10-09T05:22:09.174 INFO:teuthology.orchestra.run.smithi194.stdout: "complaint_time": 30, 2023-10-09T05:22:09.174 INFO:teuthology.orchestra.run.smithi194.stdout: "num_blocked_ops": 0 2023-10-09T05:22:09.174 INFO:teuthology.orchestra.run.smithi194.stdout:} 2023-10-09T05:22:09.188 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:09.213 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:09.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:09.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:09.316 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:09.342 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:09.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:09.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:09.449 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:09.480 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:09.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:09.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:09.584 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:09.613 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:09.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:09.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:09.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:09 smithi194 bash[92666]: cluster 2023-10-09T05:22:08.062312+0000 mgr.y (mgr.25277) 1015 : cluster 0 pgmap v1063: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 20321/144096 objects degraded (14.102%); 10909/144096 objects misplaced (7.571%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:22:09.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:09 smithi194 bash[92666]: cluster 2023-10-09T05:22:08.539334+0000 mon.a (mon.0) 1320 : cluster 3 Health check update: Degraded data redundancy: 20321/144096 objects degraded (14.102%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:09.718 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:09.756 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:09.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:09.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:09.861 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:09.886 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:09.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:09.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:09.992 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:10.024 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:10.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:10.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:10.127 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:10.145 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:09 smithi133 bash[105589]: cluster 2023-10-09T05:22:08.062312+0000 mgr.y (mgr.25277) 1015 : cluster 0 pgmap v1063: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 20321/144096 objects degraded (14.102%); 10909/144096 objects misplaced (7.571%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:22:10.146 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:09 smithi133 bash[105589]: cluster 2023-10-09T05:22:08.539334+0000 mon.a (mon.0) 1320 : cluster 3 Health check update: Degraded data redundancy: 20321/144096 objects degraded (14.102%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:10.151 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:10.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:10.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:10.257 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:10.288 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T05:22:10.290 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:10.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:10.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:10.394 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:10.403 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:10 smithi133 bash[102858]: cluster 2023-10-09T05:22:08.062312+0000 mgr.y (mgr.25277) 1015 : cluster 0 pgmap v1063: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 20321/144096 objects degraded (14.102%); 10909/144096 objects misplaced (7.571%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:22:10.404 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:10 smithi133 bash[102858]: cluster 2023-10-09T05:22:08.539334+0000 mon.a (mon.0) 1320 : cluster 3 Health check update: Degraded data redundancy: 20321/144096 objects degraded (14.102%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:10.650 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:10.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:10.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:10.756 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:10.792 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:10.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:10.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:10.897 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:10.925 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:10.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:10.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:11.029 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:11.055 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:11.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:11.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:11.159 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:11.214 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:11.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:11.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:11.322 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:11.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:11 smithi194 bash[92666]: cluster 2023-10-09T05:22:10.063057+0000 mgr.y (mgr.25277) 1016 : cluster 0 pgmap v1064: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 20321/140334 objects degraded (14.480%); 10738/140334 objects misplaced (7.652%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T05:22:11.387 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:11.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:11.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:11.497 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:11.506 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:11 smithi133 bash[102858]: cluster 2023-10-09T05:22:10.063057+0000 mgr.y (mgr.25277) 1016 : cluster 0 pgmap v1064: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 20321/140334 objects degraded (14.480%); 10738/140334 objects misplaced (7.652%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T05:22:11.507 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:11 smithi133 bash[105589]: cluster 2023-10-09T05:22:10.063057+0000 mgr.y (mgr.25277) 1016 : cluster 0 pgmap v1064: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 20321/140334 objects degraded (14.480%); 10738/140334 objects misplaced (7.652%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T05:22:11.521 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:11.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:11.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:11.627 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:11.661 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:11.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:11.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:11.768 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:11.787 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:11.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:11.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:11.892 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:11.933 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:11.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:11.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:12.038 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:12.073 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:12.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:12.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:12.175 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:12.206 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:12.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:12.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:12.311 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:12.356 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:12.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:12.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:12.460 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:12.513 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:12.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:12.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:12.620 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:12.649 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:12.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:12.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:12.754 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:12.847 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:12.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:12.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:12.950 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:12.976 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:12.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:12.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:13.080 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:13.115 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:13.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:13.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:13.218 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:13.245 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:13.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:13.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:13.349 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:13.380 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:13.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:13.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:13.485 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:13.495 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:13 smithi194 bash[92666]: cluster 2023-10-09T05:22:12.063744+0000 mgr.y (mgr.25277) 1017 : cluster 0 pgmap v1065: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 531 op/s; 20340/137022 objects degraded (14.844%); 10585/137022 objects misplaced (7.725%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T05:22:13.495 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:13 smithi194 bash[92666]: audit 2023-10-09T05:22:12.475539+0000 mon.a (mon.0) 1321 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:22:13.495 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:13 smithi194 bash[92666]: audit 2023-10-09T05:22:12.487900+0000 mon.a (mon.0) 1322 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:22:13.514 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:13.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:13.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:13.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:13 smithi133 bash[105589]: cluster 2023-10-09T05:22:12.063744+0000 mgr.y (mgr.25277) 1017 : cluster 0 pgmap v1065: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 531 op/s; 20340/137022 objects degraded (14.844%); 10585/137022 objects misplaced (7.725%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T05:22:13.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:13 smithi133 bash[105589]: audit 2023-10-09T05:22:12.475539+0000 mon.a (mon.0) 1321 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:22:13.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:13 smithi133 bash[105589]: audit 2023-10-09T05:22:12.487900+0000 mon.a (mon.0) 1322 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:22:13.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:13 smithi133 bash[102858]: cluster 2023-10-09T05:22:12.063744+0000 mgr.y (mgr.25277) 1017 : cluster 0 pgmap v1065: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 531 op/s; 20340/137022 objects degraded (14.844%); 10585/137022 objects misplaced (7.725%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T05:22:13.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:13 smithi133 bash[102858]: audit 2023-10-09T05:22:12.475539+0000 mon.a (mon.0) 1321 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:22:13.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:13 smithi133 bash[102858]: audit 2023-10-09T05:22:12.487900+0000 mon.a (mon.0) 1322 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:22:13.621 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:13.663 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:13.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:13.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:13.767 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:13.794 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:13.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:13.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:13.901 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:13.925 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:13.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:13.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:14.034 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:14.064 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:14.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:14.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:14.167 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:14.205 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:14.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:14.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:14.259 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:14 smithi133 bash[102858]: audit 2023-10-09T05:22:13.457739+0000 mon.a (mon.0) 1323 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:22:14.260 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:14 smithi133 bash[102858]: cluster 2023-10-09T05:22:13.541137+0000 mon.a (mon.0) 1324 : cluster 3 Health check update: Degraded data redundancy: 20340/137022 objects degraded (14.844%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:14.260 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:14 smithi133 bash[105589]: audit 2023-10-09T05:22:13.457739+0000 mon.a (mon.0) 1323 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:22:14.260 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:14 smithi133 bash[105589]: cluster 2023-10-09T05:22:13.541137+0000 mon.a (mon.0) 1324 : cluster 3 Health check update: Degraded data redundancy: 20340/137022 objects degraded (14.844%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:14.308 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:14.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:14 smithi194 bash[92666]: audit 2023-10-09T05:22:13.457739+0000 mon.a (mon.0) 1323 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:22:14.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:14 smithi194 bash[92666]: cluster 2023-10-09T05:22:13.541137+0000 mon.a (mon.0) 1324 : cluster 3 Health check update: Degraded data redundancy: 20340/137022 objects degraded (14.844%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:14.500 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:14.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:14.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:14.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:22:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:22:14] "GET /metrics HTTP/1.1" 200 36529 "" "Prometheus/2.43.0" 2023-10-09T05:22:14.605 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:14.632 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:14.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:14.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:14.735 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:14.760 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:14.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:14.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:14.815 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:22:14.865 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:14.892 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:14.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:14.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:15.003 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:15.034 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:15.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:15.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:15.140 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:15.165 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:15.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:15.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:15.268 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:15.296 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:15.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:15.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:15.401 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:15.409 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:15 smithi194 bash[92666]: cluster 2023-10-09T05:22:14.064445+0000 mgr.y (mgr.25277) 1018 : cluster 0 pgmap v1066: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 500 op/s; 20340/134595 objects degraded (15.112%); 10419/134595 objects misplaced (7.741%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T05:22:15.423 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:15.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:15.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:15.529 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:15.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:15 smithi133 bash[102858]: cluster 2023-10-09T05:22:14.064445+0000 mgr.y (mgr.25277) 1018 : cluster 0 pgmap v1066: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 500 op/s; 20340/134595 objects degraded (15.112%); 10419/134595 objects misplaced (7.741%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T05:22:15.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:15 smithi133 bash[105589]: cluster 2023-10-09T05:22:14.064445+0000 mgr.y (mgr.25277) 1018 : cluster 0 pgmap v1066: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 500 op/s; 20340/134595 objects degraded (15.112%); 10419/134595 objects misplaced (7.741%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T05:22:15.557 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:15.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:15.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:15.660 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:15.681 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:15.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:15.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:15.784 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:15.788 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:22:15.812 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:15.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:15.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:15.832 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:22:15.868 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:22:15.910 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:22:15.916 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:15.953 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:22:15.983 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:22:16.029 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:22:16.063 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:22:16.073 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:22:16.074 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:22:16.074 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:22:16.074 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:22:16.074 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:22:16.074 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:22:16.074 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:22:16.074 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:22:16.150 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:16.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:16.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:16.258 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:16.274 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:16 smithi133 bash[105589]: audit 2023-10-09T05:22:15.753057+0000 mon.a (mon.0) 1325 : audit 0 from='client.? 172.21.15.133:0/94312594' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:22:16.274 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:16 smithi133 bash[102858]: audit 2023-10-09T05:22:15.753057+0000 mon.a (mon.0) 1325 : audit 0 from='client.? 172.21.15.133:0/94312594' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:22:16.283 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:16.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:16.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:16.392 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:16.423 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:16.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:16.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:16.528 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:16.535 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:16 smithi194 bash[92666]: audit 2023-10-09T05:22:15.753057+0000 mon.a (mon.0) 1325 : audit 0 from='client.? 172.21.15.133:0/94312594' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:22:16.567 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:16.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:16.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:16.675 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:16.699 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:16.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:16.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:16.806 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:16.832 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:16.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:16.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:16.936 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:17.012 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:17.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:17.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:17.119 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:17.161 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:17.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:17.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:17.267 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:17.286 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:17 smithi133 bash[102858]: cluster 2023-10-09T05:22:16.065181+0000 mgr.y (mgr.25277) 1019 : cluster 0 pgmap v1067: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 516 op/s; 20358/131535 objects degraded (15.477%); 10283/131535 objects misplaced (7.818%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T05:22:17.286 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:17 smithi133 bash[102858]: audit 2023-10-09T05:22:17.065504+0000 mon.c (mon.2) 96 : audit 1 from='client.? 172.21.15.133:0/3298426835' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:22:17.287 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:17 smithi133 bash[102858]: audit 2023-10-09T05:22:17.066697+0000 mon.a (mon.0) 1326 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:22:17.291 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:17.296 INFO:teuthology.orchestra.run.smithi133.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T05:22:17.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:17.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:17.398 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:17.420 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:17.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:17.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:17.423 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:17 smithi194 bash[92666]: cluster 2023-10-09T05:22:16.065181+0000 mgr.y (mgr.25277) 1019 : cluster 0 pgmap v1067: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 516 op/s; 20358/131535 objects degraded (15.477%); 10283/131535 objects misplaced (7.818%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T05:22:17.423 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:17 smithi194 bash[92666]: audit 2023-10-09T05:22:17.065504+0000 mon.c (mon.2) 96 : audit 1 from='client.? 172.21.15.133:0/3298426835' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:22:17.423 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:17 smithi194 bash[92666]: audit 2023-10-09T05:22:17.066697+0000 mon.a (mon.0) 1326 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:22:17.524 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:17.546 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:17.547 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:17 smithi133 bash[105589]: cluster 2023-10-09T05:22:16.065181+0000 mgr.y (mgr.25277) 1019 : cluster 0 pgmap v1067: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 516 op/s; 20358/131535 objects degraded (15.477%); 10283/131535 objects misplaced (7.818%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T05:22:17.547 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:17 smithi133 bash[105589]: audit 2023-10-09T05:22:17.065504+0000 mon.c (mon.2) 96 : audit 1 from='client.? 172.21.15.133:0/3298426835' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:22:17.547 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:17 smithi133 bash[105589]: audit 2023-10-09T05:22:17.066697+0000 mon.a (mon.0) 1326 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:22:17.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:17.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:17.649 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:17.673 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:17.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:17.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:17.777 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:17.823 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:17.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:17.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:17.928 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:17.963 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:17.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:17.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:18.067 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:18.091 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:18.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:18.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:18.196 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:18.229 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:18.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:18.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:18.314 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is unset 2023-10-09T05:22:18.335 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:18.365 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:18.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:18.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:18.472 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:18.481 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:18 smithi133 bash[102858]: cluster 2023-10-09T05:22:17.275943+0000 osd.4 (osd.4) 210 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T05:22:18.482 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:18 smithi133 bash[102858]: audit 2023-10-09T05:22:17.278826+0000 mon.a (mon.0) 1327 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:22:18.482 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:18 smithi133 bash[102858]: cluster 2023-10-09T05:22:17.279022+0000 mon.a (mon.0) 1328 : cluster 0 osdmap e246: 8 total, 8 up, 7 in 2023-10-09T05:22:18.482 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:18 smithi133 bash[102858]: cluster 2023-10-09T05:22:17.287266+0000 osd.6 (osd.6) 187 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T05:22:18.482 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:18 smithi133 bash[102858]: cluster 2023-10-09T05:22:17.297198+0000 osd.7 (osd.7) 193 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T05:22:18.482 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:18 smithi133 bash[102858]: audit 2023-10-09T05:22:18.002445+0000 mon.b (mon.1) 22 : audit 1 from='client.? 172.21.15.133:0/2788956042' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:22:18.482 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:18 smithi133 bash[102858]: audit 2023-10-09T05:22:18.002824+0000 mon.a (mon.0) 1329 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:22:18.482 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:18 smithi133 bash[105589]: cluster 2023-10-09T05:22:17.275943+0000 osd.4 (osd.4) 210 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T05:22:18.482 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:18 smithi133 bash[105589]: audit 2023-10-09T05:22:17.278826+0000 mon.a (mon.0) 1327 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:22:18.483 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:18 smithi133 bash[105589]: cluster 2023-10-09T05:22:17.279022+0000 mon.a (mon.0) 1328 : cluster 0 osdmap e246: 8 total, 8 up, 7 in 2023-10-09T05:22:18.483 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:18 smithi133 bash[105589]: cluster 2023-10-09T05:22:17.287266+0000 osd.6 (osd.6) 187 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T05:22:18.483 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:18 smithi133 bash[105589]: cluster 2023-10-09T05:22:17.297198+0000 osd.7 (osd.7) 193 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T05:22:18.483 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:18 smithi133 bash[105589]: audit 2023-10-09T05:22:18.002445+0000 mon.b (mon.1) 22 : audit 1 from='client.? 172.21.15.133:0/2788956042' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:22:18.483 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:18 smithi133 bash[105589]: audit 2023-10-09T05:22:18.002824+0000 mon.a (mon.0) 1329 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:22:18.512 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:18.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:18.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:18.617 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:18.626 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:18 smithi194 bash[92666]: cluster 2023-10-09T05:22:17.275943+0000 osd.4 (osd.4) 210 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T05:22:18.626 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:18 smithi194 bash[92666]: audit 2023-10-09T05:22:17.278826+0000 mon.a (mon.0) 1327 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:22:18.626 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:18 smithi194 bash[92666]: cluster 2023-10-09T05:22:17.279022+0000 mon.a (mon.0) 1328 : cluster 0 osdmap e246: 8 total, 8 up, 7 in 2023-10-09T05:22:18.626 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:18 smithi194 bash[92666]: cluster 2023-10-09T05:22:17.287266+0000 osd.6 (osd.6) 187 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T05:22:18.626 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:18 smithi194 bash[92666]: cluster 2023-10-09T05:22:17.297198+0000 osd.7 (osd.7) 193 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T05:22:18.626 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:18 smithi194 bash[92666]: audit 2023-10-09T05:22:18.002445+0000 mon.b (mon.1) 22 : audit 1 from='client.? 172.21.15.133:0/2788956042' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:22:18.626 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:18 smithi194 bash[92666]: audit 2023-10-09T05:22:18.002824+0000 mon.a (mon.0) 1329 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:22:18.643 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:18.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:18.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:18.746 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:18.772 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:18.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:18.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:18.875 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:18.911 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:18.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:18.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:19.018 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:19.043 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:19.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:19.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:19.147 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:19.178 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:19.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:19.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:19.285 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:19.314 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:19.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:19.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:19.421 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:19.433 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:19 smithi133 bash[102858]: cluster 2023-10-09T05:22:18.065802+0000 mgr.y (mgr.25277) 1020 : cluster 0 pgmap v1069: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 499 op/s; 20358/129099 objects degraded (15.769%); 10283/129099 objects misplaced (7.965%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T05:22:19.434 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:19 smithi133 bash[102858]: cluster 2023-10-09T05:22:18.285708+0000 mon.a (mon.0) 1330 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:22:19.434 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:19 smithi133 bash[102858]: cluster 2023-10-09T05:22:18.299153+0000 osd.4 (osd.4) 211 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T05:22:19.434 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:19 smithi133 bash[102858]: audit 2023-10-09T05:22:18.305488+0000 mon.a (mon.0) 1331 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:22:19.434 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:19 smithi133 bash[102858]: cluster 2023-10-09T05:22:18.306370+0000 mon.a (mon.0) 1332 : cluster 0 osdmap e247: 8 total, 8 up, 7 in 2023-10-09T05:22:19.434 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:19 smithi133 bash[102858]: cluster 2023-10-09T05:22:18.307528+0000 osd.4 (osd.4) 212 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T05:22:19.434 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:19 smithi133 bash[102858]: cluster 2023-10-09T05:22:18.312268+0000 osd.7 (osd.7) 194 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T05:22:19.434 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:19 smithi133 bash[102858]: cluster 2023-10-09T05:22:18.314779+0000 osd.7 (osd.7) 195 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T05:22:19.434 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:19 smithi133 bash[102858]: cluster 2023-10-09T05:22:18.314945+0000 osd.6 (osd.6) 188 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T05:22:19.434 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:19 smithi133 bash[102858]: cluster 2023-10-09T05:22:18.316851+0000 osd.6 (osd.6) 189 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T05:22:19.434 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:19 smithi133 bash[102858]: cluster 2023-10-09T05:22:18.542827+0000 mon.a (mon.0) 1333 : cluster 3 Health check update: Degraded data redundancy: 20358/129099 objects degraded (15.769%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:19.434 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:19 smithi133 bash[102858]: cluster 2023-10-09T05:22:18.866169+0000 osd.5 (osd.5) 231 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T05:22:19.435 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:19 smithi133 bash[102858]: cluster 2023-10-09T05:22:18.868925+0000 osd.5 (osd.5) 232 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T05:22:19.435 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:19 smithi133 bash[105589]: cluster 2023-10-09T05:22:18.065802+0000 mgr.y (mgr.25277) 1020 : cluster 0 pgmap v1069: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 499 op/s; 20358/129099 objects degraded (15.769%); 10283/129099 objects misplaced (7.965%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T05:22:19.435 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:19 smithi133 bash[105589]: cluster 2023-10-09T05:22:18.285708+0000 mon.a (mon.0) 1330 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:22:19.435 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:19 smithi133 bash[105589]: cluster 2023-10-09T05:22:18.299153+0000 osd.4 (osd.4) 211 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T05:22:19.435 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:19 smithi133 bash[105589]: audit 2023-10-09T05:22:18.305488+0000 mon.a (mon.0) 1331 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:22:19.435 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:19 smithi133 bash[105589]: cluster 2023-10-09T05:22:18.306370+0000 mon.a (mon.0) 1332 : cluster 0 osdmap e247: 8 total, 8 up, 7 in 2023-10-09T05:22:19.435 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:19 smithi133 bash[105589]: cluster 2023-10-09T05:22:18.307528+0000 osd.4 (osd.4) 212 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T05:22:19.435 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:19 smithi133 bash[105589]: cluster 2023-10-09T05:22:18.312268+0000 osd.7 (osd.7) 194 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T05:22:19.435 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:19 smithi133 bash[105589]: cluster 2023-10-09T05:22:18.314779+0000 osd.7 (osd.7) 195 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T05:22:19.435 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:19 smithi133 bash[105589]: cluster 2023-10-09T05:22:18.314945+0000 osd.6 (osd.6) 188 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T05:22:19.436 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:19 smithi133 bash[105589]: cluster 2023-10-09T05:22:18.316851+0000 osd.6 (osd.6) 189 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T05:22:19.436 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:19 smithi133 bash[105589]: cluster 2023-10-09T05:22:18.542827+0000 mon.a (mon.0) 1333 : cluster 3 Health check update: Degraded data redundancy: 20358/129099 objects degraded (15.769%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:19.436 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:19 smithi133 bash[105589]: cluster 2023-10-09T05:22:18.866169+0000 osd.5 (osd.5) 231 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T05:22:19.436 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:19 smithi133 bash[105589]: cluster 2023-10-09T05:22:18.868925+0000 osd.5 (osd.5) 232 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T05:22:19.450 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:19.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:19.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:19.559 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:19.590 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:19.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:19.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:19.611 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:22:19.611 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:22:19.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:19 smithi194 bash[92666]: cluster 2023-10-09T05:22:18.065802+0000 mgr.y (mgr.25277) 1020 : cluster 0 pgmap v1069: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 499 op/s; 20358/129099 objects degraded (15.769%); 10283/129099 objects misplaced (7.965%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T05:22:19.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:19 smithi194 bash[92666]: cluster 2023-10-09T05:22:18.285708+0000 mon.a (mon.0) 1330 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:22:19.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:19 smithi194 bash[92666]: cluster 2023-10-09T05:22:18.299153+0000 osd.4 (osd.4) 211 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T05:22:19.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:19 smithi194 bash[92666]: audit 2023-10-09T05:22:18.305488+0000 mon.a (mon.0) 1331 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:22:19.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:19 smithi194 bash[92666]: cluster 2023-10-09T05:22:18.306370+0000 mon.a (mon.0) 1332 : cluster 0 osdmap e247: 8 total, 8 up, 7 in 2023-10-09T05:22:19.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:19 smithi194 bash[92666]: cluster 2023-10-09T05:22:18.307528+0000 osd.4 (osd.4) 212 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T05:22:19.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:19 smithi194 bash[92666]: cluster 2023-10-09T05:22:18.312268+0000 osd.7 (osd.7) 194 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T05:22:19.639 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:19 smithi194 bash[92666]: cluster 2023-10-09T05:22:18.314779+0000 osd.7 (osd.7) 195 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T05:22:19.639 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:19 smithi194 bash[92666]: cluster 2023-10-09T05:22:18.314945+0000 osd.6 (osd.6) 188 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T05:22:19.639 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:19 smithi194 bash[92666]: cluster 2023-10-09T05:22:18.316851+0000 osd.6 (osd.6) 189 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T05:22:19.639 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:19 smithi194 bash[92666]: cluster 2023-10-09T05:22:18.542827+0000 mon.a (mon.0) 1333 : cluster 3 Health check update: Degraded data redundancy: 20358/129099 objects degraded (15.769%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:19.639 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:19 smithi194 bash[92666]: cluster 2023-10-09T05:22:18.866169+0000 osd.5 (osd.5) 231 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T05:22:19.639 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:19 smithi194 bash[92666]: cluster 2023-10-09T05:22:18.868925+0000 osd.5 (osd.5) 232 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T05:22:19.698 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:19.730 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:19.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:19.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:19.837 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:19.860 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:19.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:19.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:19.963 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:19.999 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:20.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:20.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:20.106 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:20.142 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:20.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:20.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:20.247 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:20.271 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:20.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:20.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:20.378 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:20.402 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:20.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:20.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:20.507 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:20.550 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:20.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:20.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:20.576 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T05:22:20.656 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:20.719 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:20.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:20.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:20.809 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T05:22:20.824 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:20.848 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:20.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:20.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:20.948 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:22:20.954 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:20.979 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:20.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:20.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:21.083 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:21.147 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:21.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:21.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:21.252 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:21.276 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:21.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:21.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:21.379 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:21.392 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:21 smithi133 bash[102858]: cluster 2023-10-09T05:22:20.066473+0000 mgr.y (mgr.25277) 1021 : cluster 0 pgmap v1071: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 520 op/s; 20362/124521 objects degraded (16.352%); 9975/124521 objects misplaced (8.011%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T05:22:21.392 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:21 smithi133 bash[105589]: cluster 2023-10-09T05:22:20.066473+0000 mgr.y (mgr.25277) 1021 : cluster 0 pgmap v1071: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 520 op/s; 20362/124521 objects degraded (16.352%); 9975/124521 objects misplaced (8.011%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T05:22:21.420 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:21.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:21.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:21.524 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:21.532 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:21 smithi194 bash[92666]: cluster 2023-10-09T05:22:20.066473+0000 mgr.y (mgr.25277) 1021 : cluster 0 pgmap v1071: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 520 op/s; 20362/124521 objects degraded (16.352%); 9975/124521 objects misplaced (8.011%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T05:22:21.548 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:21.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:21.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:21.652 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:21.682 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:21.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:21.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:21.790 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:21.845 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:21.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:21.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:21.954 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:21.985 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:21.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:21.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:22.090 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:22.152 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:22.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:22.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:22.261 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:22.310 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:22.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:22.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:22.418 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:22.448 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:22.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:22.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:22.552 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:22.576 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:22.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:22.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:22.594 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:22:22.679 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:22.737 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:22.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:22.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:22.845 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:22.897 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:22.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:22.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:23.007 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:23.035 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:23.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:23.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:23.141 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:23.175 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:23.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:23.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:23.278 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:23.308 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:23.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:23.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:23.411 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:23.455 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:23.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:23.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:23.560 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:23.595 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:23.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:23.603 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:23 smithi133 bash[102858]: cluster 2023-10-09T05:22:22.067144+0000 mgr.y (mgr.25277) 1022 : cluster 0 pgmap v1072: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 517 op/s; 20362/122169 objects degraded (16.667%); 9975/122169 objects misplaced (8.165%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T05:22:23.603 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:23 smithi133 bash[102858]: audit 2023-10-09T05:22:22.656724+0000 mon.a (mon.0) 1334 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:22:23.603 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:23 smithi133 bash[102858]: audit 2023-10-09T05:22:22.670371+0000 mon.a (mon.0) 1335 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:22:23.603 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:23 smithi133 bash[105589]: cluster 2023-10-09T05:22:22.067144+0000 mgr.y (mgr.25277) 1022 : cluster 0 pgmap v1072: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 517 op/s; 20362/122169 objects degraded (16.667%); 9975/122169 objects misplaced (8.165%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T05:22:23.604 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:23 smithi133 bash[105589]: audit 2023-10-09T05:22:22.656724+0000 mon.a (mon.0) 1334 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:22:23.604 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:23 smithi133 bash[105589]: audit 2023-10-09T05:22:22.670371+0000 mon.a (mon.0) 1335 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:22:23.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:23.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:23 smithi194 bash[92666]: cluster 2023-10-09T05:22:22.067144+0000 mgr.y (mgr.25277) 1022 : cluster 0 pgmap v1072: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 517 op/s; 20362/122169 objects degraded (16.667%); 9975/122169 objects misplaced (8.165%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T05:22:23.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:23 smithi194 bash[92666]: audit 2023-10-09T05:22:22.656724+0000 mon.a (mon.0) 1334 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:22:23.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:23 smithi194 bash[92666]: audit 2023-10-09T05:22:22.670371+0000 mon.a (mon.0) 1335 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:22:23.705 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:23.730 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:23.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:23.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:23.835 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:23.870 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:23.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:23.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:23.976 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:23.998 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:24.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:24.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:24.102 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:24.152 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:24.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:24.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:24.257 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:24.282 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:24.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:24.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:24.386 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:24.412 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:24.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:24.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:24.515 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:24.525 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:24 smithi133 bash[102858]: audit 2023-10-09T05:22:23.357347+0000 mon.a (mon.0) 1336 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:22:24.525 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:24 smithi133 bash[102858]: audit 2023-10-09T05:22:23.358643+0000 mon.a (mon.0) 1337 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:22:24.525 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:24 smithi133 bash[102858]: audit 2023-10-09T05:22:23.368896+0000 mon.a (mon.0) 1338 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:22:24.525 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:24 smithi133 bash[102858]: cluster 2023-10-09T05:22:23.543886+0000 mon.a (mon.0) 1339 : cluster 3 Health check update: Degraded data redundancy: 20362/122169 objects degraded (16.667%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:24.526 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:22:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:22:24] "GET /metrics HTTP/1.1" 200 36529 "" "Prometheus/2.43.0" 2023-10-09T05:22:24.526 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:24 smithi133 bash[105589]: audit 2023-10-09T05:22:23.357347+0000 mon.a (mon.0) 1336 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:22:24.526 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:24 smithi133 bash[105589]: audit 2023-10-09T05:22:23.358643+0000 mon.a (mon.0) 1337 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:22:24.526 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:24 smithi133 bash[105589]: audit 2023-10-09T05:22:23.368896+0000 mon.a (mon.0) 1338 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:22:24.526 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:24 smithi133 bash[105589]: cluster 2023-10-09T05:22:23.543886+0000 mon.a (mon.0) 1339 : cluster 3 Health check update: Degraded data redundancy: 20362/122169 objects degraded (16.667%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:24.542 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:24.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:24.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:24.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:24 smithi194 bash[92666]: audit 2023-10-09T05:22:23.357347+0000 mon.a (mon.0) 1336 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:22:24.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:24 smithi194 bash[92666]: audit 2023-10-09T05:22:23.358643+0000 mon.a (mon.0) 1337 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:22:24.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:24 smithi194 bash[92666]: audit 2023-10-09T05:22:23.368896+0000 mon.a (mon.0) 1338 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:22:24.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:24 smithi194 bash[92666]: cluster 2023-10-09T05:22:23.543886+0000 mon.a (mon.0) 1339 : cluster 3 Health check update: Degraded data redundancy: 20362/122169 objects degraded (16.667%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:24.647 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:24.673 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:24.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:24.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:24.779 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:24.808 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:24.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:24.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:24.914 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:24.941 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:24.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:24.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:25.044 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:25.075 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:25.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:25.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:25.143 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:22:25.144 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:22:25.144 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:22:25.181 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:25.233 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:25.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:25.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:25.343 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:25.377 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:25.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:25.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:25.480 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:25.489 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:25 smithi133 bash[102858]: cluster 2023-10-09T05:22:24.067922+0000 mgr.y (mgr.25277) 1023 : cluster 0 pgmap v1073: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 490 op/s; 20362/119760 objects degraded (17.002%); 9815/119760 objects misplaced (8.196%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T05:22:25.490 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:25 smithi133 bash[105589]: cluster 2023-10-09T05:22:24.067922+0000 mgr.y (mgr.25277) 1023 : cluster 0 pgmap v1073: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 490 op/s; 20362/119760 objects degraded (17.002%); 9815/119760 objects misplaced (8.196%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T05:22:25.520 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:25.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:25.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:25.630 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:25.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:25 smithi194 bash[92666]: cluster 2023-10-09T05:22:24.067922+0000 mgr.y (mgr.25277) 1023 : cluster 0 pgmap v1073: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 490 op/s; 20362/119760 objects degraded (17.002%); 9815/119760 objects misplaced (8.196%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T05:22:25.673 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:25.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:25.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:25.779 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:25.811 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:25.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:25.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:25.917 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:25.963 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:25.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:25.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:26.071 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:26.092 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:26.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:26.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:26.200 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:26.225 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:26.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:26.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:26.332 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:26.380 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:26.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:26.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:26.485 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:26.524 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:26.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:26.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:26.628 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:26.662 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:26.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:26.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:26.766 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:26.796 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:26.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:26.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:26.902 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:26.930 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:26.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:26.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:27.034 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:27.067 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:27.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:27.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:27.172 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:27.182 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:22:27.192 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:27.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:27.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:27.301 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:27.329 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:27.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:27.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:27.430 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:27.439 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:27 smithi194 bash[92666]: cluster 2023-10-09T05:22:26.068816+0000 mgr.y (mgr.25277) 1024 : cluster 0 pgmap v1074: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 496 op/s; 20394/116469 objects degraded (17.510%); 9654/116469 objects misplaced (8.289%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T05:22:27.459 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:27.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:27.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:27.562 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:27.590 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:27.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:27.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:27.694 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:27.725 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:27.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:27.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:27.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:27 smithi133 bash[105589]: cluster 2023-10-09T05:22:26.068816+0000 mgr.y (mgr.25277) 1024 : cluster 0 pgmap v1074: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 496 op/s; 20394/116469 objects degraded (17.510%); 9654/116469 objects misplaced (8.289%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T05:22:27.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:27 smithi133 bash[102858]: cluster 2023-10-09T05:22:26.068816+0000 mgr.y (mgr.25277) 1024 : cluster 0 pgmap v1074: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 496 op/s; 20394/116469 objects degraded (17.510%); 9654/116469 objects misplaced (8.289%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T05:22:27.830 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:27.881 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:27.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:27.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:27.984 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:28.011 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:28.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:28.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:28.114 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:28.146 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:28.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:28.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:28.249 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:28.272 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:28.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:28.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:28.377 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:28.407 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:28.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:28.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:28.516 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:28.550 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:28.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:28.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:28.653 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:28.681 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:28.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:28.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:28.784 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:28.807 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:28.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:28.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:28.913 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:28.958 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:28.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:28.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:29.063 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:29.099 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:29.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:29.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:29.203 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:29.253 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:29.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:29.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:29.358 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:29.369 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:29 smithi133 bash[102858]: cluster 2023-10-09T05:22:28.069481+0000 mgr.y (mgr.25277) 1025 : cluster 0 pgmap v1075: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 20394/114054 objects degraded (17.881%); 9654/114054 objects misplaced (8.464%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T05:22:29.369 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:29 smithi133 bash[102858]: audit 2023-10-09T05:22:28.458121+0000 mon.a (mon.0) 1340 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:22:29.369 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:29 smithi133 bash[102858]: cluster 2023-10-09T05:22:28.546584+0000 mon.a (mon.0) 1341 : cluster 3 Health check update: Degraded data redundancy: 20394/114054 objects degraded (17.881%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:29.370 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:29 smithi133 bash[105589]: cluster 2023-10-09T05:22:28.069481+0000 mgr.y (mgr.25277) 1025 : cluster 0 pgmap v1075: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 20394/114054 objects degraded (17.881%); 9654/114054 objects misplaced (8.464%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T05:22:29.370 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:29 smithi133 bash[105589]: audit 2023-10-09T05:22:28.458121+0000 mon.a (mon.0) 1340 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:22:29.370 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:29 smithi133 bash[105589]: cluster 2023-10-09T05:22:28.546584+0000 mon.a (mon.0) 1341 : cluster 3 Health check update: Degraded data redundancy: 20394/114054 objects degraded (17.881%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:29.385 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:29.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:29.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:29.490 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:29.516 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:29.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:29.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:29.619 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:29.627 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:29 smithi194 bash[92666]: cluster 2023-10-09T05:22:28.069481+0000 mgr.y (mgr.25277) 1025 : cluster 0 pgmap v1075: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 20394/114054 objects degraded (17.881%); 9654/114054 objects misplaced (8.464%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T05:22:29.627 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:29 smithi194 bash[92666]: audit 2023-10-09T05:22:28.458121+0000 mon.a (mon.0) 1340 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:22:29.628 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:29 smithi194 bash[92666]: cluster 2023-10-09T05:22:28.546584+0000 mon.a (mon.0) 1341 : cluster 3 Health check update: Degraded data redundancy: 20394/114054 objects degraded (17.881%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:29.651 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:29.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:29.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:29.754 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:29.784 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:29.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:29.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:29.891 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:29.915 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:29.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:29.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:30.019 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:30.048 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:30.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:30.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:30.154 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:30.188 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:30.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:30.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:30.292 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:30.349 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:30.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:30.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:30.456 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:30.497 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:30.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:30.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:30.601 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:30.626 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:30.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:30.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:30.729 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:30.754 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:30.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:30.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:30.860 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:30.892 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:30.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:30.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:30.997 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:31.027 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:31.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:31.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:31.136 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:31.162 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:31.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:31.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:31.266 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:31.311 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:31.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:31.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:31.416 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:31.442 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:31.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:31.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:31.486 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:31 smithi133 bash[102858]: cluster 2023-10-09T05:22:30.070334+0000 mgr.y (mgr.25277) 1026 : cluster 0 pgmap v1076: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 20433/108759 objects degraded (18.787%); 9297/108759 objects misplaced (8.548%); 4.8 MiB/s, 77 objects/s recovering 2023-10-09T05:22:31.545 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:31.574 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:31.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:31.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:31.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:31 smithi194 bash[92666]: cluster 2023-10-09T05:22:30.070334+0000 mgr.y (mgr.25277) 1026 : cluster 0 pgmap v1076: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 20433/108759 objects degraded (18.787%); 9297/108759 objects misplaced (8.548%); 4.8 MiB/s, 77 objects/s recovering 2023-10-09T05:22:31.678 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:31.712 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:31.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:31.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:31.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:31 smithi133 bash[105589]: cluster 2023-10-09T05:22:30.070334+0000 mgr.y (mgr.25277) 1026 : cluster 0 pgmap v1076: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 20433/108759 objects degraded (18.787%); 9297/108759 objects misplaced (8.548%); 4.8 MiB/s, 77 objects/s recovering 2023-10-09T05:22:31.815 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:31.842 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:31.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:31.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:31.951 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:31.978 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:31.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:31.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:32.085 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:32.109 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:32.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:32.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:32.212 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:32.235 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:32.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:32.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:32.338 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:32.366 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:32.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:32.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:32.473 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:32.502 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:32.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:32.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:32.604 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:32.631 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:32.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:32.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:32.735 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:32.769 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:32.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:32.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:32.874 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:32.903 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:32.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:32.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:33.012 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:33.051 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:33.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:33.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:33.156 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:33.183 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:33.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:33.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:33.291 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:33.316 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:33.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:33.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:33.420 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:33.428 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:33 smithi194 bash[92666]: cluster 2023-10-09T05:22:32.070995+0000 mgr.y (mgr.25277) 1027 : cluster 0 pgmap v1077: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 505 op/s; 20433/106314 objects degraded (19.219%); 9297/106314 objects misplaced (8.745%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T05:22:33.447 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:33.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:33.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:33.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:33 smithi133 bash[105589]: cluster 2023-10-09T05:22:32.070995+0000 mgr.y (mgr.25277) 1027 : cluster 0 pgmap v1077: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 505 op/s; 20433/106314 objects degraded (19.219%); 9297/106314 objects misplaced (8.745%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T05:22:33.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:33 smithi133 bash[102858]: cluster 2023-10-09T05:22:32.070995+0000 mgr.y (mgr.25277) 1027 : cluster 0 pgmap v1077: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 505 op/s; 20433/106314 objects degraded (19.219%); 9297/106314 objects misplaced (8.745%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T05:22:33.553 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:33.582 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:33.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:33.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:33.689 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:33.726 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:33.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:33.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:33.831 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:33.865 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:33.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:33.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:33.969 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:34.011 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:34.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:34.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:34.114 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:34.145 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:34.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:34.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:34.252 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:34.296 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:34.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:34.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:34.401 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:34.428 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:34.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:34.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:34.532 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:34.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:22:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:22:34] "GET /metrics HTTP/1.1" 200 36521 "" "Prometheus/2.43.0" 2023-10-09T05:22:34.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:34 smithi133 bash[105589]: cluster 2023-10-09T05:22:33.547826+0000 mon.a (mon.0) 1342 : cluster 3 Health check update: Degraded data redundancy: 20433/106314 objects degraded (19.219%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:34.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:34 smithi133 bash[102858]: cluster 2023-10-09T05:22:33.547826+0000 mon.a (mon.0) 1342 : cluster 3 Health check update: Degraded data redundancy: 20433/106314 objects degraded (19.219%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:34.556 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:34.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:34.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:34.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:34 smithi194 bash[92666]: cluster 2023-10-09T05:22:33.547826+0000 mon.a (mon.0) 1342 : cluster 3 Health check update: Degraded data redundancy: 20433/106314 objects degraded (19.219%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:34.661 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:34.683 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:34.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:34.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:34.790 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:34.826 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:34.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:34.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:34.929 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:34.967 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:34.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:34.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:35.073 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:35.099 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:35.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:35.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:35.205 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:35.233 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:35.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:35.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:35.337 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:35.363 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:35.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:35.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:35.466 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:35.491 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:35.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:35.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:35.596 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:35.623 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:35.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:35.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:35.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:35 smithi194 bash[92666]: cluster 2023-10-09T05:22:34.071554+0000 mgr.y (mgr.25277) 1028 : cluster 0 pgmap v1078: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 454 op/s; 20433/105813 objects degraded (19.310%); 9297/105813 objects misplaced (8.786%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T05:22:35.672 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:35 smithi133 bash[105589]: cluster 2023-10-09T05:22:34.071554+0000 mgr.y (mgr.25277) 1028 : cluster 0 pgmap v1078: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 454 op/s; 20433/105813 objects degraded (19.310%); 9297/105813 objects misplaced (8.786%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T05:22:35.672 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:35 smithi133 bash[102858]: cluster 2023-10-09T05:22:34.071554+0000 mgr.y (mgr.25277) 1028 : cluster 0 pgmap v1078: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 454 op/s; 20433/105813 objects degraded (19.310%); 9297/105813 objects misplaced (8.786%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T05:22:35.730 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:35.782 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:35.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:35.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:35.889 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:35.920 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:35.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:35.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:36.023 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:36.061 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:36.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:36.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:36.167 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:36.184 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:36.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:36.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:36.286 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:36.314 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:36.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:36.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:36.417 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:36.449 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:36.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:36.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:36.557 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:36.582 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:36.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:36.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:36.685 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:36.717 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:36.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:36.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:36.821 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:36.848 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:36.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:36.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:36.953 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:36.996 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:37.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:37.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:37.104 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:37.130 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:37.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:37.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:37.233 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:37.257 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:37.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:37.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:37.364 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:37.372 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:37 smithi194 bash[92666]: cluster 2023-10-09T05:22:36.072356+0000 mgr.y (mgr.25277) 1029 : cluster 0 pgmap v1079: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 20434/100401 objects degraded (20.352%); 8950/100401 objects misplaced (8.914%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T05:22:37.388 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:37.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:37.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:37.494 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:37.518 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:37.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:37.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:37.624 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:37.662 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:37.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:37.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:37.767 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:37.775 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:37 smithi133 bash[102858]: cluster 2023-10-09T05:22:36.072356+0000 mgr.y (mgr.25277) 1029 : cluster 0 pgmap v1079: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 20434/100401 objects degraded (20.352%); 8950/100401 objects misplaced (8.914%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T05:22:37.775 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:37 smithi133 bash[105589]: cluster 2023-10-09T05:22:36.072356+0000 mgr.y (mgr.25277) 1029 : cluster 0 pgmap v1079: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 20434/100401 objects degraded (20.352%); 8950/100401 objects misplaced (8.914%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T05:22:37.802 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:37.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:37.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:37.909 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:37.952 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:37.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:37.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:38.057 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:38.087 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:38.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:38.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:38.192 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:38.238 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:38.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:38.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:38.344 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:38.374 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:38.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:38.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:38.477 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:38.517 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:38.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:38.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:38.626 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:38.653 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:38.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:38.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:38.757 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:38.785 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:38.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:38.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:38.892 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:38.913 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:38.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:38.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:39.017 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:39.047 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:39.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:39.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:39.150 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:39.180 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:39.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:39.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:39.286 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:39.324 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:39.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:39.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:39.429 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:39.439 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:39 smithi194 bash[92666]: cluster 2023-10-09T05:22:38.072989+0000 mgr.y (mgr.25277) 1030 : cluster 0 pgmap v1080: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 20434/97932 objects degraded (20.865%); 8950/97932 objects misplaced (9.139%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T05:22:39.439 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:39 smithi194 bash[92666]: cluster 2023-10-09T05:22:38.549037+0000 mon.a (mon.0) 1343 : cluster 3 Health check update: Degraded data redundancy: 20434/97932 objects degraded (20.865%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:39.456 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:39.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:39.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:39.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:39 smithi133 bash[102858]: cluster 2023-10-09T05:22:38.072989+0000 mgr.y (mgr.25277) 1030 : cluster 0 pgmap v1080: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 20434/97932 objects degraded (20.865%); 8950/97932 objects misplaced (9.139%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T05:22:39.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:39 smithi133 bash[102858]: cluster 2023-10-09T05:22:38.549037+0000 mon.a (mon.0) 1343 : cluster 3 Health check update: Degraded data redundancy: 20434/97932 objects degraded (20.865%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:39.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:39 smithi133 bash[105589]: cluster 2023-10-09T05:22:38.072989+0000 mgr.y (mgr.25277) 1030 : cluster 0 pgmap v1080: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 20434/97932 objects degraded (20.865%); 8950/97932 objects misplaced (9.139%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T05:22:39.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:39 smithi133 bash[105589]: cluster 2023-10-09T05:22:38.549037+0000 mon.a (mon.0) 1343 : cluster 3 Health check update: Degraded data redundancy: 20434/97932 objects degraded (20.865%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:39.561 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:39.596 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:39.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:39.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:39.704 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:39.732 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:39.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:39.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:39.837 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:39.857 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:39.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:39.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:39.961 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:39.988 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:39.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:39.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:40.092 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:40.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:40.132 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:40.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:40.233 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:40.265 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:40.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:40.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:40.369 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:40.402 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:40.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:40.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:40.505 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:40.531 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:40.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:40.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:40.634 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:40.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:40 smithi194 bash[92666]: cluster 2023-10-09T05:22:40.073814+0000 mgr.y (mgr.25277) 1031 : cluster 0 pgmap v1081: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 585 op/s; 20434/92955 objects degraded (21.983%); 8588/92955 objects misplaced (9.239%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T05:22:40.663 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:40.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:40.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:40.768 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:40.793 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:40.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:40.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:40.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:40 smithi133 bash[105589]: cluster 2023-10-09T05:22:40.073814+0000 mgr.y (mgr.25277) 1031 : cluster 0 pgmap v1081: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 585 op/s; 20434/92955 objects degraded (21.983%); 8588/92955 objects misplaced (9.239%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T05:22:40.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:40 smithi133 bash[102858]: cluster 2023-10-09T05:22:40.073814+0000 mgr.y (mgr.25277) 1031 : cluster 0 pgmap v1081: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 585 op/s; 20434/92955 objects degraded (21.983%); 8588/92955 objects misplaced (9.239%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T05:22:40.898 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:40.922 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:40.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:40.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:41.028 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:41.056 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:41.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:41.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:41.159 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:41.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:41.209 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:41.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:41.310 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:41.340 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:41.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:41.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:41.445 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:41.475 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:41.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:41.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:41.577 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:41.604 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:41.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:41.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:41.712 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:41.736 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:41.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:41.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:41.840 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:41.872 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:41.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:41.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:41.976 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:42.008 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:42.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:42.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:42.112 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:42.135 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:42.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:42.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:42.238 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:42.296 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:42.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:42.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:42.403 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:42.428 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:42.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:42.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:42.532 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:42.554 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:42.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:42.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:42.657 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:42.691 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:42.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:42.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:42.796 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:42.822 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:42.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:42.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:42.929 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:42.951 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:42.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:42.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:43.053 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:43.076 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:43.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:43.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:43.129 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:43 smithi194 bash[92666]: cluster 2023-10-09T05:22:42.074556+0000 mgr.y (mgr.25277) 1032 : cluster 0 pgmap v1082: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 20434/90393 objects degraded (22.606%); 8588/90393 objects misplaced (9.501%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T05:22:43.185 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:43.191 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:43 smithi133 bash[102858]: cluster 2023-10-09T05:22:42.074556+0000 mgr.y (mgr.25277) 1032 : cluster 0 pgmap v1082: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 20434/90393 objects degraded (22.606%); 8588/90393 objects misplaced (9.501%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T05:22:43.192 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:43 smithi133 bash[105589]: cluster 2023-10-09T05:22:42.074556+0000 mgr.y (mgr.25277) 1032 : cluster 0 pgmap v1082: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 20434/90393 objects degraded (22.606%); 8588/90393 objects misplaced (9.501%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T05:22:43.206 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:43.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:43.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:43.309 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:43.338 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:43.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:43.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:43.444 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:43.467 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:43.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:43.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:43.572 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:43.625 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:43.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:43.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:43.731 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:43.774 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:43.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:43.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:43.880 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:43.917 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:43.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:43.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:44.021 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:44.046 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:44.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:44.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:44.149 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:44.157 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:44 smithi133 bash[102858]: audit 2023-10-09T05:22:43.458190+0000 mon.a (mon.0) 1344 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:22:44.157 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:44 smithi133 bash[102858]: cluster 2023-10-09T05:22:43.550121+0000 mon.a (mon.0) 1345 : cluster 3 Health check update: Degraded data redundancy: 20434/90393 objects degraded (22.606%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:44.158 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:44 smithi133 bash[105589]: audit 2023-10-09T05:22:43.458190+0000 mon.a (mon.0) 1344 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:22:44.158 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:44 smithi133 bash[105589]: cluster 2023-10-09T05:22:43.550121+0000 mon.a (mon.0) 1345 : cluster 3 Health check update: Degraded data redundancy: 20434/90393 objects degraded (22.606%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:44.185 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:44.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:44.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:44.290 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:44.325 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:44.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:44.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:44.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:44 smithi194 bash[92666]: audit 2023-10-09T05:22:43.458190+0000 mon.a (mon.0) 1344 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:22:44.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:44 smithi194 bash[92666]: cluster 2023-10-09T05:22:43.550121+0000 mon.a (mon.0) 1345 : cluster 3 Health check update: Degraded data redundancy: 20434/90393 objects degraded (22.606%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:44.433 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:44.442 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:22:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:22:44] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.43.0" 2023-10-09T05:22:44.480 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:44.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:44.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:44.584 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:44.618 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:44.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:44.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:44.721 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:44.753 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:44.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:44.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:44.860 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:44.897 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:44.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:44.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:44.929 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:22:44.929 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:22:45.002 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:45.028 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:45.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:45.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:45.133 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:45.182 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:45.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:45.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:45.287 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:45.297 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:45 smithi194 bash[92666]: cluster 2023-10-09T05:22:44.075113+0000 mgr.y (mgr.25277) 1033 : cluster 0 pgmap v1083: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 457 op/s; 20434/89847 objects degraded (22.743%); 8588/89847 objects misplaced (9.558%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T05:22:45.312 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:45.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:45.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:45.407 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:45 smithi133 bash[105589]: cluster 2023-10-09T05:22:44.075113+0000 mgr.y (mgr.25277) 1033 : cluster 0 pgmap v1083: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 457 op/s; 20434/89847 objects degraded (22.743%); 8588/89847 objects misplaced (9.558%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T05:22:45.408 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:45 smithi133 bash[102858]: cluster 2023-10-09T05:22:44.075113+0000 mgr.y (mgr.25277) 1033 : cluster 0 pgmap v1083: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 457 op/s; 20434/89847 objects degraded (22.743%); 8588/89847 objects misplaced (9.558%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T05:22:45.415 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:45.443 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:45.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:45.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:45.549 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:45.571 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:45.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:45.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:45.675 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:45.703 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:45.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:45.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:45.707 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:22:45.788 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:22:45.805 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:45.837 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:22:45.852 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:45.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:45.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:45.885 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:22:45.925 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:22:45.962 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:45.965 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:22:45.988 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:45.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:45.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:46.009 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:22:46.053 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:22:46.055 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:22:46.055 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:22:46.055 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:22:46.055 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:22:46.055 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:22:46.055 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:22:46.056 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:22:46.056 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:22:46.091 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:46.122 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:46.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:46.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:46.186 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is unset 2023-10-09T05:22:46.190 INFO:teuthology.orchestra.run.smithi133.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T05:22:46.227 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:46.248 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:46.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:46.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:46.351 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:46.363 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:46 smithi194 bash[92666]: audit 2023-10-09T05:22:45.648126+0000 mon.a (mon.0) 1346 : audit 0 from='client.? 172.21.15.133:0/227781163' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:22:46.364 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:46 smithi194 bash[92666]: audit 2023-10-09T05:22:45.857992+0000 mon.a (mon.0) 1347 : audit 1 from='client.? 172.21.15.133:0/3762745908' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:22:46.364 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:46 smithi194 bash[92666]: audit 2023-10-09T05:22:46.058392+0000 mon.a (mon.0) 1348 : audit 1 from='client.? 172.21.15.133:0/1899371668' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:22:46.382 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:46.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:46.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:46.430 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:46 smithi133 bash[105589]: audit 2023-10-09T05:22:45.648126+0000 mon.a (mon.0) 1346 : audit 0 from='client.? 172.21.15.133:0/227781163' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:22:46.430 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:46 smithi133 bash[105589]: audit 2023-10-09T05:22:45.857992+0000 mon.a (mon.0) 1347 : audit 1 from='client.? 172.21.15.133:0/3762745908' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:22:46.430 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:46 smithi133 bash[105589]: audit 2023-10-09T05:22:46.058392+0000 mon.a (mon.0) 1348 : audit 1 from='client.? 172.21.15.133:0/1899371668' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:22:46.430 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:46 smithi133 bash[102858]: audit 2023-10-09T05:22:45.648126+0000 mon.a (mon.0) 1346 : audit 0 from='client.? 172.21.15.133:0/227781163' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:22:46.430 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:46 smithi133 bash[102858]: audit 2023-10-09T05:22:45.857992+0000 mon.a (mon.0) 1347 : audit 1 from='client.? 172.21.15.133:0/3762745908' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:22:46.430 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:46 smithi133 bash[102858]: audit 2023-10-09T05:22:46.058392+0000 mon.a (mon.0) 1348 : audit 1 from='client.? 172.21.15.133:0/1899371668' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:22:46.485 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:46.507 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:46.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:46.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:46.613 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:46.646 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:46.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:46.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:46.754 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:46.793 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:46.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:46.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:46.903 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:46.937 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:46.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:46.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:47.041 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:47.068 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:47.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:47.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:47.171 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:47.195 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:47.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:47.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:47.299 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:47.324 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:47.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:47.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:47.369 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:47 smithi133 bash[102858]: cluster 2023-10-09T05:22:46.075905+0000 mgr.y (mgr.25277) 1034 : cluster 0 pgmap v1084: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 596 op/s; 20436/84327 objects degraded (24.234%); 8170/84327 objects misplaced (9.688%); 5.9 MiB/s, 93 objects/s recovering 2023-10-09T05:22:47.369 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:47 smithi133 bash[102858]: cluster 2023-10-09T05:22:46.148914+0000 mon.a (mon.0) 1349 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:22:47.369 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:47 smithi133 bash[102858]: audit 2023-10-09T05:22:46.176382+0000 mon.a (mon.0) 1350 : audit 1 from='client.? 172.21.15.133:0/3762745908' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:22:47.369 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:47 smithi133 bash[102858]: audit 2023-10-09T05:22:46.176603+0000 mon.a (mon.0) 1351 : audit 1 from='client.? 172.21.15.133:0/1899371668' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:22:47.369 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:47 smithi133 bash[102858]: cluster 2023-10-09T05:22:46.176777+0000 mon.a (mon.0) 1352 : cluster 0 osdmap e248: 8 total, 8 up, 7 in 2023-10-09T05:22:47.369 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:47 smithi133 bash[102858]: cluster 2023-10-09T05:22:46.182908+0000 osd.6 (osd.6) 190 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T05:22:47.369 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:47 smithi133 bash[102858]: cluster 2023-10-09T05:22:46.184048+0000 osd.4 (osd.4) 213 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T05:22:47.369 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:47 smithi133 bash[102858]: cluster 2023-10-09T05:22:46.186101+0000 osd.6 (osd.6) 191 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T05:22:47.370 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:47 smithi133 bash[102858]: cluster 2023-10-09T05:22:46.188126+0000 osd.6 (osd.6) 192 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T05:22:47.370 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:47 smithi133 bash[102858]: cluster 2023-10-09T05:22:46.188620+0000 osd.7 (osd.7) 196 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T05:22:47.370 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:47 smithi133 bash[102858]: cluster 2023-10-09T05:22:46.189898+0000 osd.4 (osd.4) 214 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T05:22:47.370 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:47 smithi133 bash[105589]: cluster 2023-10-09T05:22:46.075905+0000 mgr.y (mgr.25277) 1034 : cluster 0 pgmap v1084: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 596 op/s; 20436/84327 objects degraded (24.234%); 8170/84327 objects misplaced (9.688%); 5.9 MiB/s, 93 objects/s recovering 2023-10-09T05:22:47.370 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:47 smithi133 bash[105589]: cluster 2023-10-09T05:22:46.148914+0000 mon.a (mon.0) 1349 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:22:47.371 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:47 smithi133 bash[105589]: audit 2023-10-09T05:22:46.176382+0000 mon.a (mon.0) 1350 : audit 1 from='client.? 172.21.15.133:0/3762745908' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:22:47.371 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:47 smithi133 bash[105589]: audit 2023-10-09T05:22:46.176603+0000 mon.a (mon.0) 1351 : audit 1 from='client.? 172.21.15.133:0/1899371668' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:22:47.371 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:47 smithi133 bash[105589]: cluster 2023-10-09T05:22:46.176777+0000 mon.a (mon.0) 1352 : cluster 0 osdmap e248: 8 total, 8 up, 7 in 2023-10-09T05:22:47.371 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:47 smithi133 bash[105589]: cluster 2023-10-09T05:22:46.182908+0000 osd.6 (osd.6) 190 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T05:22:47.371 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:47 smithi133 bash[105589]: cluster 2023-10-09T05:22:46.184048+0000 osd.4 (osd.4) 213 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T05:22:47.371 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:47 smithi133 bash[105589]: cluster 2023-10-09T05:22:46.186101+0000 osd.6 (osd.6) 191 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T05:22:47.371 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:47 smithi133 bash[105589]: cluster 2023-10-09T05:22:46.188126+0000 osd.6 (osd.6) 192 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T05:22:47.371 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:47 smithi133 bash[105589]: cluster 2023-10-09T05:22:46.188620+0000 osd.7 (osd.7) 196 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T05:22:47.371 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:47 smithi133 bash[105589]: cluster 2023-10-09T05:22:46.189898+0000 osd.4 (osd.4) 214 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T05:22:47.408 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T05:22:47.428 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:47.491 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:47.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:47.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:47.596 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:47.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:47 smithi194 bash[92666]: cluster 2023-10-09T05:22:46.075905+0000 mgr.y (mgr.25277) 1034 : cluster 0 pgmap v1084: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 596 op/s; 20436/84327 objects degraded (24.234%); 8170/84327 objects misplaced (9.688%); 5.9 MiB/s, 93 objects/s recovering 2023-10-09T05:22:47.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:47 smithi194 bash[92666]: cluster 2023-10-09T05:22:46.148914+0000 mon.a (mon.0) 1349 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:22:47.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:47 smithi194 bash[92666]: audit 2023-10-09T05:22:46.176382+0000 mon.a (mon.0) 1350 : audit 1 from='client.? 172.21.15.133:0/3762745908' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:22:47.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:47 smithi194 bash[92666]: audit 2023-10-09T05:22:46.176603+0000 mon.a (mon.0) 1351 : audit 1 from='client.? 172.21.15.133:0/1899371668' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:22:47.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:47 smithi194 bash[92666]: cluster 2023-10-09T05:22:46.176777+0000 mon.a (mon.0) 1352 : cluster 0 osdmap e248: 8 total, 8 up, 7 in 2023-10-09T05:22:47.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:47 smithi194 bash[92666]: cluster 2023-10-09T05:22:46.182908+0000 osd.6 (osd.6) 190 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T05:22:47.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:47 smithi194 bash[92666]: cluster 2023-10-09T05:22:46.184048+0000 osd.4 (osd.4) 213 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T05:22:47.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:47 smithi194 bash[92666]: cluster 2023-10-09T05:22:46.186101+0000 osd.6 (osd.6) 191 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T05:22:47.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:47 smithi194 bash[92666]: cluster 2023-10-09T05:22:46.188126+0000 osd.6 (osd.6) 192 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T05:22:47.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:47 smithi194 bash[92666]: cluster 2023-10-09T05:22:46.188620+0000 osd.7 (osd.7) 196 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T05:22:47.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:47 smithi194 bash[92666]: cluster 2023-10-09T05:22:46.189898+0000 osd.4 (osd.4) 214 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T05:22:47.642 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:47.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:47.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:47.756 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:47.791 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:47.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:47.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:47.897 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:47.933 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:47.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:47.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:48.040 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:48.080 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:48.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:48.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:48.185 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:48.214 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:48.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:48.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:48.317 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:48.348 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:48.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:48.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:48.361 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:48 smithi133 bash[105589]: cluster 2023-10-09T05:22:47.171370+0000 mon.a (mon.0) 1353 : cluster 0 osdmap e249: 8 total, 8 up, 7 in 2023-10-09T05:22:48.361 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:48 smithi133 bash[105589]: cluster 2023-10-09T05:22:47.173474+0000 osd.6 (osd.6) 193 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T05:22:48.361 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:48 smithi133 bash[105589]: cluster 2023-10-09T05:22:47.186921+0000 osd.7 (osd.7) 197 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T05:22:48.361 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:48 smithi133 bash[105589]: cluster 2023-10-09T05:22:47.188051+0000 osd.4 (osd.4) 215 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T05:22:48.361 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:48 smithi133 bash[105589]: cluster 2023-10-09T05:22:47.189888+0000 osd.4 (osd.4) 216 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T05:22:48.361 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:48 smithi133 bash[102858]: cluster 2023-10-09T05:22:47.171370+0000 mon.a (mon.0) 1353 : cluster 0 osdmap e249: 8 total, 8 up, 7 in 2023-10-09T05:22:48.361 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:48 smithi133 bash[102858]: cluster 2023-10-09T05:22:47.173474+0000 osd.6 (osd.6) 193 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T05:22:48.362 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:48 smithi133 bash[102858]: cluster 2023-10-09T05:22:47.186921+0000 osd.7 (osd.7) 197 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T05:22:48.362 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:48 smithi133 bash[102858]: cluster 2023-10-09T05:22:47.188051+0000 osd.4 (osd.4) 215 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T05:22:48.362 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:48 smithi133 bash[102858]: cluster 2023-10-09T05:22:47.189888+0000 osd.4 (osd.4) 216 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T05:22:48.453 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:48.465 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:48 smithi194 bash[92666]: cluster 2023-10-09T05:22:47.171370+0000 mon.a (mon.0) 1353 : cluster 0 osdmap e249: 8 total, 8 up, 7 in 2023-10-09T05:22:48.465 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:48 smithi194 bash[92666]: cluster 2023-10-09T05:22:47.173474+0000 osd.6 (osd.6) 193 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T05:22:48.465 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:48 smithi194 bash[92666]: cluster 2023-10-09T05:22:47.186921+0000 osd.7 (osd.7) 197 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T05:22:48.465 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:48 smithi194 bash[92666]: cluster 2023-10-09T05:22:47.188051+0000 osd.4 (osd.4) 215 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T05:22:48.465 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:48 smithi194 bash[92666]: cluster 2023-10-09T05:22:47.189888+0000 osd.4 (osd.4) 216 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T05:22:48.481 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:48.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:48.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:48.585 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:48.612 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:48.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:48.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:48.715 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:48.743 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:48.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:48.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:48.849 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:48.884 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:48.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:48.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:48.990 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:49.018 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:49.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:49.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:49.122 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:49.150 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:49.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:49.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:49.188 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:49 smithi133 bash[102858]: cluster 2023-10-09T05:22:48.076656+0000 mgr.y (mgr.25277) 1035 : cluster 0 pgmap v1087: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 447 op/s; 20436/82209 objects degraded (24.859%); 8170/82209 objects misplaced (9.938%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T05:22:49.188 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:49 smithi133 bash[102858]: cluster 2023-10-09T05:22:48.551262+0000 mon.a (mon.0) 1354 : cluster 3 Health check update: Degraded data redundancy: 20436/82209 objects degraded (24.859%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:49.257 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:49.265 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:49 smithi194 bash[92666]: cluster 2023-10-09T05:22:48.076656+0000 mgr.y (mgr.25277) 1035 : cluster 0 pgmap v1087: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 447 op/s; 20436/82209 objects degraded (24.859%); 8170/82209 objects misplaced (9.938%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T05:22:49.266 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:49 smithi194 bash[92666]: cluster 2023-10-09T05:22:48.551262+0000 mon.a (mon.0) 1354 : cluster 3 Health check update: Degraded data redundancy: 20436/82209 objects degraded (24.859%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:49.280 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:49.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:49.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:49.385 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:49.420 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:49.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:49.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:49.527 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:49.536 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:49 smithi133 bash[105589]: cluster 2023-10-09T05:22:48.076656+0000 mgr.y (mgr.25277) 1035 : cluster 0 pgmap v1087: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 447 op/s; 20436/82209 objects degraded (24.859%); 8170/82209 objects misplaced (9.938%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T05:22:49.536 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:49 smithi133 bash[105589]: cluster 2023-10-09T05:22:48.551262+0000 mon.a (mon.0) 1354 : cluster 3 Health check update: Degraded data redundancy: 20436/82209 objects degraded (24.859%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:49.571 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:49.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:49.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:49.676 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:49.708 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:49.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:49.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:49.815 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:49.843 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:49.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:49.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:49.949 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:49.995 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:50.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:50.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:50.101 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:50.129 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:50.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:50.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:50.240 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:50.281 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:50.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:50.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:50.318 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T05:22:50.392 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:50.414 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:50.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:50.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:50.518 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:50.563 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:50.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:50.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:50.667 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:50.708 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:50.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:50.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:50.812 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:50.846 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:50.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:50.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:50.953 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:50.981 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:50.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:50.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:51.084 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:51.110 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:51.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:51.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:51.213 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:51.223 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:51 smithi194 bash[92666]: cluster 2023-10-09T05:22:50.077429+0000 mgr.y (mgr.25277) 1036 : cluster 0 pgmap v1088: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 20436/77718 objects degraded (26.295%); 7774/77718 objects misplaced (10.003%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T05:22:51.244 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:51.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:51.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:51.347 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:51.385 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:51.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:51.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:51.492 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:51.501 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:51 smithi133 bash[102858]: cluster 2023-10-09T05:22:50.077429+0000 mgr.y (mgr.25277) 1036 : cluster 0 pgmap v1088: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 20436/77718 objects degraded (26.295%); 7774/77718 objects misplaced (10.003%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T05:22:51.501 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:51 smithi133 bash[105589]: cluster 2023-10-09T05:22:50.077429+0000 mgr.y (mgr.25277) 1036 : cluster 0 pgmap v1088: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 20436/77718 objects degraded (26.295%); 7774/77718 objects misplaced (10.003%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T05:22:51.523 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:51.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:51.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:51.630 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:51.666 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:51.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:51.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:51.770 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:51.798 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:51.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:51.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:51.905 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:51.940 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:51.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:51.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:52.046 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:52.078 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:52.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:52.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:52.181 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:52.194 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:22:52.206 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:52.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:52.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:52.309 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:22:52.313 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:52.338 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:52.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:52.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:52.445 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:52.478 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:52.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:52.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:52.584 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:52.633 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:52.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:52.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:52.738 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:52.769 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:52.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:52.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:52.879 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:52.908 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:52.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:52.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:53.016 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:53.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:53.044 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:53.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:53.146 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:53.187 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:53.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:53.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:53.282 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:22:53.293 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:53.317 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:53.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:53.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:53.420 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:53.451 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:53.454 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:53 smithi194 bash[92666]: cluster 2023-10-09T05:22:52.077936+0000 mgr.y (mgr.25277) 1037 : cluster 0 pgmap v1089: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 20436/75294 objects degraded (27.142%); 7774/75294 objects misplaced (10.325%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T05:22:53.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:53.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:53.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:53 smithi133 bash[105589]: cluster 2023-10-09T05:22:52.077936+0000 mgr.y (mgr.25277) 1037 : cluster 0 pgmap v1089: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 20436/75294 objects degraded (27.142%); 7774/75294 objects misplaced (10.325%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T05:22:53.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:53 smithi133 bash[102858]: cluster 2023-10-09T05:22:52.077936+0000 mgr.y (mgr.25277) 1037 : cluster 0 pgmap v1089: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 20436/75294 objects degraded (27.142%); 7774/75294 objects misplaced (10.325%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T05:22:53.555 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:53.581 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:53.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:53.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:53.684 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:53.717 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:53.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:53.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:53.828 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:53.858 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:53.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:53.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:53.961 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:53.987 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:53.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:53.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:54.091 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:54.122 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:54.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:54.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:54.230 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:54.238 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:54 smithi133 bash[102858]: cluster 2023-10-09T05:22:53.552724+0000 mon.a (mon.0) 1355 : cluster 3 Health check update: Degraded data redundancy: 20436/75294 objects degraded (27.142%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:54.239 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:54 smithi133 bash[105589]: cluster 2023-10-09T05:22:53.552724+0000 mon.a (mon.0) 1355 : cluster 3 Health check update: Degraded data redundancy: 20436/75294 objects degraded (27.142%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:54.260 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:54.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:54.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:54.304 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:22:54.366 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:54.402 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:54.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:54.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:54.509 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:54.517 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:22:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:22:54] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.43.0" 2023-10-09T05:22:54.543 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:54.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:54.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:54.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:54 smithi194 bash[92666]: cluster 2023-10-09T05:22:53.552724+0000 mon.a (mon.0) 1355 : cluster 3 Health check update: Degraded data redundancy: 20436/75294 objects degraded (27.142%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:54.647 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:54.671 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:54.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:54.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:54.778 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:54.808 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:54.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:54.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:54.913 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:54.942 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:54.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:54.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:55.045 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:55.069 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:55.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:55.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:55.173 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:55.201 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:55.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:55.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:55.306 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:55.314 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:55 smithi194 bash[92666]: cluster 2023-10-09T05:22:54.078485+0000 mgr.y (mgr.25277) 1038 : cluster 0 pgmap v1090: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 397 op/s; 20436/74775 objects degraded (27.330%); 7774/74775 objects misplaced (10.397%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T05:22:55.333 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:55.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:55.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:55.436 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:55.465 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:55.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:55.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:55.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:55 smithi133 bash[105589]: cluster 2023-10-09T05:22:54.078485+0000 mgr.y (mgr.25277) 1038 : cluster 0 pgmap v1090: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 397 op/s; 20436/74775 objects degraded (27.330%); 7774/74775 objects misplaced (10.397%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T05:22:55.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:55 smithi133 bash[102858]: cluster 2023-10-09T05:22:54.078485+0000 mgr.y (mgr.25277) 1038 : cluster 0 pgmap v1090: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 397 op/s; 20436/74775 objects degraded (27.330%); 7774/74775 objects misplaced (10.397%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T05:22:55.571 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:55.595 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:55.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:55.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:55.701 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:55.728 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:55.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:55.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:55.832 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:55.861 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:55.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:55.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:55.966 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:56.002 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:56.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:56.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:56.108 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:56.133 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:56.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:56.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:56.239 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:56.266 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:56.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:56.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:56.340 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:22:56.341 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:22:56.369 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:56.411 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:56.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:56.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:56.518 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:56.555 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:56.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:56.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:56.661 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:56.685 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:56.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:56.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:56.790 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:56.811 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:56.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:56.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:56.914 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:56.952 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:56.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:56.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:57.059 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:57.091 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:57.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:57.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:57.195 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:57.205 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:57 smithi133 bash[102858]: cluster 2023-10-09T05:22:56.079289+0000 mgr.y (mgr.25277) 1039 : cluster 0 pgmap v1091: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 20436/69090 objects degraded (29.579%); 7354/69090 objects misplaced (10.644%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T05:22:57.224 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:57.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:57.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:57.330 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:57.370 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:57.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:57.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:57.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:57 smithi194 bash[92666]: cluster 2023-10-09T05:22:56.079289+0000 mgr.y (mgr.25277) 1039 : cluster 0 pgmap v1091: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 20436/69090 objects degraded (29.579%); 7354/69090 objects misplaced (10.644%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T05:22:57.475 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:57.502 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:57.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:57.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:57.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:57 smithi133 bash[105589]: cluster 2023-10-09T05:22:56.079289+0000 mgr.y (mgr.25277) 1039 : cluster 0 pgmap v1091: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 20436/69090 objects degraded (29.579%); 7354/69090 objects misplaced (10.644%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T05:22:57.607 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:57.652 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:57.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:57.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:57.759 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:57.793 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:22:57.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:57.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:22:57.902 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:57.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:57.937 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:57.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:58.038 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:58.078 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:58.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:58.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:58.182 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:58.213 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:58.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:58.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:58.318 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:58.342 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:22:58.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:58.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:22:58.449 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:58.469 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:58.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:58.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:58.572 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:58.609 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:58.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:58.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:58.715 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:58.742 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:58.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:58.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:58.848 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:58.885 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:58.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:58.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:58.988 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:59.023 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:22:59.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:59.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:22:59.130 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:59.175 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:59.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:59.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:22:59.215 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:59 smithi133 bash[102858]: cluster 2023-10-09T05:22:58.079938+0000 mgr.y (mgr.25277) 1040 : cluster 0 pgmap v1092: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 20436/66594 objects degraded (30.687%); 7354/66594 objects misplaced (11.043%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T05:22:59.215 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:59 smithi133 bash[102858]: audit 2023-10-09T05:22:58.461240+0000 mon.a (mon.0) 1356 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:22:59.281 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:59.304 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:22:59.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:59.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:22:59.409 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:59.445 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:22:59.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:59.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:22:59.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:59 smithi133 bash[102858]: audit 2023-10-09T05:22:58.462007+0000 mon.a (mon.0) 1357 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:22:59.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:22:59 smithi133 bash[102858]: cluster 2023-10-09T05:22:58.553765+0000 mon.a (mon.0) 1358 : cluster 3 Health check update: Degraded data redundancy: 20436/66594 objects degraded (30.687%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:59.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:59 smithi133 bash[105589]: cluster 2023-10-09T05:22:58.079938+0000 mgr.y (mgr.25277) 1040 : cluster 0 pgmap v1092: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 20436/66594 objects degraded (30.687%); 7354/66594 objects misplaced (11.043%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T05:22:59.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:59 smithi133 bash[105589]: audit 2023-10-09T05:22:58.461240+0000 mon.a (mon.0) 1356 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:22:59.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:59 smithi133 bash[105589]: audit 2023-10-09T05:22:58.462007+0000 mon.a (mon.0) 1357 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:22:59.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:22:59 smithi133 bash[105589]: cluster 2023-10-09T05:22:58.553765+0000 mon.a (mon.0) 1358 : cluster 3 Health check update: Degraded data redundancy: 20436/66594 objects degraded (30.687%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:59.550 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:59.578 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:22:59.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:59.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:22:59.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:59 smithi194 bash[92666]: cluster 2023-10-09T05:22:58.079938+0000 mgr.y (mgr.25277) 1040 : cluster 0 pgmap v1092: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 20436/66594 objects degraded (30.687%); 7354/66594 objects misplaced (11.043%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T05:22:59.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:59 smithi194 bash[92666]: audit 2023-10-09T05:22:58.461240+0000 mon.a (mon.0) 1356 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:22:59.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:59 smithi194 bash[92666]: audit 2023-10-09T05:22:58.462007+0000 mon.a (mon.0) 1357 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:22:59.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:22:59 smithi194 bash[92666]: cluster 2023-10-09T05:22:58.553765+0000 mon.a (mon.0) 1358 : cluster 3 Health check update: Degraded data redundancy: 20436/66594 objects degraded (30.687%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:22:59.683 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:59.712 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:59.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:59.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:59.815 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:59.849 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:22:59.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:59.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:22:59.955 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:59.989 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:22:59.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:59.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:00.092 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:00.114 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:00.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:00.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:00.219 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:00.244 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:00.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:00.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:00.347 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:00.372 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:00.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:00.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:00.480 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:00.506 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:00.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:00.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:00.610 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:00.648 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:00.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:00.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:00.755 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:00.807 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:00.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:00.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:00.910 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:00.932 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:00.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:00.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:01.036 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:01.073 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:01.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:01.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:01.177 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:01.204 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:01.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:01.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:01.218 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:01 smithi133 bash[102858]: cluster 2023-10-09T05:23:00.080716+0000 mgr.y (mgr.25277) 1041 : cluster 0 pgmap v1093: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 20436/61749 objects degraded (33.095%); 6895/61749 objects misplaced (11.166%); 6.6 MiB/s, 106 objects/s recovering 2023-10-09T05:23:01.219 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:01 smithi133 bash[105589]: cluster 2023-10-09T05:23:00.080716+0000 mgr.y (mgr.25277) 1041 : cluster 0 pgmap v1093: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 20436/61749 objects degraded (33.095%); 6895/61749 objects misplaced (11.166%); 6.6 MiB/s, 106 objects/s recovering 2023-10-09T05:23:01.312 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:01.350 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:01.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:01.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:01.456 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:01.499 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:01.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:01.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:01.606 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:01.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:01 smithi194 bash[92666]: cluster 2023-10-09T05:23:00.080716+0000 mgr.y (mgr.25277) 1041 : cluster 0 pgmap v1093: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 20436/61749 objects degraded (33.095%); 6895/61749 objects misplaced (11.166%); 6.6 MiB/s, 106 objects/s recovering 2023-10-09T05:23:01.659 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:01.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:01.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:01.764 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:01.805 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:01.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:01.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:01.911 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:01.941 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:01.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:01.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:02.045 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:02.072 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:02.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:02.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:02.177 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:02.204 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:02.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:02.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:02.307 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:02.336 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:02.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:02.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:02.440 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:02.466 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:02.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:02.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:02.572 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:02.601 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:02.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:02.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:02.706 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:02.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:02.744 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:02.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:02.845 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:02.884 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:02.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:02.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:02.986 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:03.007 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:03.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:03.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:03.110 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:03.153 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:03.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:03.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:03.257 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:03.274 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:03.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:03.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:03.376 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:03.397 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:03.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:03.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:03.501 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:03.511 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:03 smithi194 bash[92666]: cluster 2023-10-09T05:23:02.081399+0000 mgr.y (mgr.25277) 1042 : cluster 0 pgmap v1094: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 513 op/s; 20436/59208 objects degraded (34.516%); 6895/59208 objects misplaced (11.645%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T05:23:03.537 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:03.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:03.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:03.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:03 smithi133 bash[105589]: cluster 2023-10-09T05:23:02.081399+0000 mgr.y (mgr.25277) 1042 : cluster 0 pgmap v1094: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 513 op/s; 20436/59208 objects degraded (34.516%); 6895/59208 objects misplaced (11.645%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T05:23:03.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:03 smithi133 bash[102858]: cluster 2023-10-09T05:23:02.081399+0000 mgr.y (mgr.25277) 1042 : cluster 0 pgmap v1094: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 513 op/s; 20436/59208 objects degraded (34.516%); 6895/59208 objects misplaced (11.645%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T05:23:03.640 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:03.672 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:03.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:03.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:03.777 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:03.806 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:03.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:03.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:03.910 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:03.951 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:03.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:03.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:04.054 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:04.080 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:04.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:04.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:04.183 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:04.208 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:04.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:04.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:04.313 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:04.320 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:04 smithi194 bash[92666]: cluster 2023-10-09T05:23:03.554845+0000 mon.a (mon.0) 1359 : cluster 3 Health check update: Degraded data redundancy: 20436/59208 objects degraded (34.516%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:04.340 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:04.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:04.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:04.443 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:04.462 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:04.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:04.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:04.508 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:04 smithi133 bash[102858]: cluster 2023-10-09T05:23:03.554845+0000 mon.a (mon.0) 1359 : cluster 3 Health check update: Degraded data redundancy: 20436/59208 objects degraded (34.516%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:04.509 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:23:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:23:04] "GET /metrics HTTP/1.1" 200 36514 "" "Prometheus/2.43.0" 2023-10-09T05:23:04.509 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:04 smithi133 bash[105589]: cluster 2023-10-09T05:23:03.554845+0000 mon.a (mon.0) 1359 : cluster 3 Health check update: Degraded data redundancy: 20436/59208 objects degraded (34.516%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:04.564 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:04.597 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:04.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:04.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:04.705 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:04.737 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:04.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:04.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:04.843 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:04.878 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:04.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:04.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:04.984 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:05.014 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:05.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:05.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:05.125 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:05.150 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:05.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:05.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:05.254 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:05.284 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:05.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:05.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:05.305 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:05 smithi133 bash[105589]: cluster 2023-10-09T05:23:04.081969+0000 mgr.y (mgr.25277) 1043 : cluster 0 pgmap v1095: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 460 op/s; 20436/58716 objects degraded (34.805%); 6895/58716 objects misplaced (11.743%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T05:23:05.306 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:05 smithi133 bash[102858]: cluster 2023-10-09T05:23:04.081969+0000 mgr.y (mgr.25277) 1043 : cluster 0 pgmap v1095: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 460 op/s; 20436/58716 objects degraded (34.805%); 6895/58716 objects misplaced (11.743%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T05:23:05.388 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:05.416 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:05.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:05.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:05.520 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:05.554 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:05.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:05.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:05.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:05 smithi194 bash[92666]: cluster 2023-10-09T05:23:04.081969+0000 mgr.y (mgr.25277) 1043 : cluster 0 pgmap v1095: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 460 op/s; 20436/58716 objects degraded (34.805%); 6895/58716 objects misplaced (11.743%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T05:23:05.665 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:05.692 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:05.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:05.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:05.797 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:05.828 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:05.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:05.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:05.931 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:05.973 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:05.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:05.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:06.079 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:06.107 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:06.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:06.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:06.213 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:06.247 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:06.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:06.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:06.352 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:06.377 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:06.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:06.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:06.482 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:06.510 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:06.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:06.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:06.614 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:06.665 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:06.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:06.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:06.768 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:06.795 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:06.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:06.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:06.900 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:06.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:06.940 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:06.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:07.041 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:07.067 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:07.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:07.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:07.171 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:07.189 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:07.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:07.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:07.293 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:07.323 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:07.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:07.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:07.428 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:07.452 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:07.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:07.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:07.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:07 smithi133 bash[102858]: cluster 2023-10-09T05:23:06.082775+0000 mgr.y (mgr.25277) 1044 : cluster 0 pgmap v1096: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 595 op/s; 20436/53328 objects degraded (38.321%); 6353/53328 objects misplaced (11.913%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T05:23:07.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:07 smithi133 bash[105589]: cluster 2023-10-09T05:23:06.082775+0000 mgr.y (mgr.25277) 1044 : cluster 0 pgmap v1096: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 595 op/s; 20436/53328 objects degraded (38.321%); 6353/53328 objects misplaced (11.913%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T05:23:07.555 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:07.564 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:07 smithi194 bash[92666]: cluster 2023-10-09T05:23:06.082775+0000 mgr.y (mgr.25277) 1044 : cluster 0 pgmap v1096: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 595 op/s; 20436/53328 objects degraded (38.321%); 6353/53328 objects misplaced (11.913%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T05:23:07.580 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:07.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:07.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:07.687 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:07.710 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:07.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:07.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:07.813 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:07.839 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:07.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:07.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:07.944 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:07.994 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:07.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:08.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:08.100 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:08.128 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:08.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:08.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:08.235 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:08.261 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:08.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:08.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:08.365 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:08.393 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:08.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:08.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:08.496 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:08.543 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:08.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:08.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:08.648 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:08.674 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:08.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:08.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:08.777 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:08.813 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:08.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:08.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:08.916 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:08.942 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:08.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:08.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:09.045 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:09.112 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:09.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:09.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:09.217 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:09.253 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:09.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:09.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:09.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:09 smithi133 bash[102858]: cluster 2023-10-09T05:23:08.085848+0000 mgr.y (mgr.25277) 1045 : cluster 0 pgmap v1097: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 504 op/s; 20436/50910 objects degraded (40.141%); 6353/50910 objects misplaced (12.479%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T05:23:09.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:09 smithi133 bash[102858]: cluster 2023-10-09T05:23:08.556117+0000 mon.a (mon.0) 1360 : cluster 3 Health check update: Degraded data redundancy: 20436/50910 objects degraded (40.141%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:09.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:09 smithi133 bash[105589]: cluster 2023-10-09T05:23:08.085848+0000 mgr.y (mgr.25277) 1045 : cluster 0 pgmap v1097: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 504 op/s; 20436/50910 objects degraded (40.141%); 6353/50910 objects misplaced (12.479%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T05:23:09.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:09 smithi133 bash[105589]: cluster 2023-10-09T05:23:08.556117+0000 mon.a (mon.0) 1360 : cluster 3 Health check update: Degraded data redundancy: 20436/50910 objects degraded (40.141%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:09.358 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:09.388 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:09.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:09.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:09.492 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:09.500 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:09 smithi194 bash[92666]: cluster 2023-10-09T05:23:08.085848+0000 mgr.y (mgr.25277) 1045 : cluster 0 pgmap v1097: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 504 op/s; 20436/50910 objects degraded (40.141%); 6353/50910 objects misplaced (12.479%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T05:23:09.500 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:09 smithi194 bash[92666]: cluster 2023-10-09T05:23:08.556117+0000 mon.a (mon.0) 1360 : cluster 3 Health check update: Degraded data redundancy: 20436/50910 objects degraded (40.141%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:09.516 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:09.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:09.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:09.621 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:09.648 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:09.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:09.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:09.755 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:09.782 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:09.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:09.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:09.889 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:09.918 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:09.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:09.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:10.021 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:10.049 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:10.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:10.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:10.152 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:10.179 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:10.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:10.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:10.282 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:10.310 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:10.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:10.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:10.414 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:10.432 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:10.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:10.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:10.535 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:10.558 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:10.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:10.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:10.661 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:10.691 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:10.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:10.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:10.736 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:10 smithi133 bash[105589]: cluster 2023-10-09T05:23:09.567727+0000 mon.a (mon.0) 1361 : cluster 0 osdmap e250: 8 total, 8 up, 7 in 2023-10-09T05:23:10.736 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:10 smithi133 bash[105589]: cluster 2023-10-09T05:23:09.574337+0000 osd.6 (osd.6) 194 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T05:23:10.736 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:10 smithi133 bash[105589]: cluster 2023-10-09T05:23:09.574345+0000 osd.4 (osd.4) 217 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T05:23:10.736 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:10 smithi133 bash[105589]: cluster 2023-10-09T05:23:09.581200+0000 osd.7 (osd.7) 198 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T05:23:10.736 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:10 smithi133 bash[105589]: cluster 2023-10-09T05:23:09.584559+0000 osd.7 (osd.7) 199 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T05:23:10.736 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:10 smithi133 bash[105589]: cluster 2023-10-09T05:23:09.644713+0000 osd.5 (osd.5) 233 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T05:23:10.736 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:10 smithi133 bash[105589]: cluster 2023-10-09T05:23:10.086358+0000 mgr.y (mgr.25277) 1046 : cluster 0 pgmap v1099: 89 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 954 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 20171/45828 objects degraded (44.015%); 6007/45828 objects misplaced (13.108%); 5.5 MiB/s, 115 objects/s recovering 2023-10-09T05:23:10.737 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:10 smithi133 bash[102858]: cluster 2023-10-09T05:23:09.567727+0000 mon.a (mon.0) 1361 : cluster 0 osdmap e250: 8 total, 8 up, 7 in 2023-10-09T05:23:10.737 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:10 smithi133 bash[102858]: cluster 2023-10-09T05:23:09.574337+0000 osd.6 (osd.6) 194 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T05:23:10.737 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:10 smithi133 bash[102858]: cluster 2023-10-09T05:23:09.574345+0000 osd.4 (osd.4) 217 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T05:23:10.737 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:10 smithi133 bash[102858]: cluster 2023-10-09T05:23:09.581200+0000 osd.7 (osd.7) 198 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T05:23:10.737 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:10 smithi133 bash[102858]: cluster 2023-10-09T05:23:09.584559+0000 osd.7 (osd.7) 199 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T05:23:10.737 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:10 smithi133 bash[102858]: cluster 2023-10-09T05:23:09.644713+0000 osd.5 (osd.5) 233 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T05:23:10.737 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:10 smithi133 bash[102858]: cluster 2023-10-09T05:23:10.086358+0000 mgr.y (mgr.25277) 1046 : cluster 0 pgmap v1099: 89 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 954 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 20171/45828 objects degraded (44.015%); 6007/45828 objects misplaced (13.108%); 5.5 MiB/s, 115 objects/s recovering 2023-10-09T05:23:10.795 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:10.821 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:10.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:10.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:10.826 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:10 smithi194 bash[92666]: cluster 2023-10-09T05:23:09.567727+0000 mon.a (mon.0) 1361 : cluster 0 osdmap e250: 8 total, 8 up, 7 in 2023-10-09T05:23:10.826 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:10 smithi194 bash[92666]: cluster 2023-10-09T05:23:09.574337+0000 osd.6 (osd.6) 194 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T05:23:10.826 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:10 smithi194 bash[92666]: cluster 2023-10-09T05:23:09.574345+0000 osd.4 (osd.4) 217 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T05:23:10.826 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:10 smithi194 bash[92666]: cluster 2023-10-09T05:23:09.581200+0000 osd.7 (osd.7) 198 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T05:23:10.826 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:10 smithi194 bash[92666]: cluster 2023-10-09T05:23:09.584559+0000 osd.7 (osd.7) 199 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T05:23:10.826 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:10 smithi194 bash[92666]: cluster 2023-10-09T05:23:09.644713+0000 osd.5 (osd.5) 233 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T05:23:10.826 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:10 smithi194 bash[92666]: cluster 2023-10-09T05:23:10.086358+0000 mgr.y (mgr.25277) 1046 : cluster 0 pgmap v1099: 89 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 954 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 20171/45828 objects degraded (44.015%); 6007/45828 objects misplaced (13.108%); 5.5 MiB/s, 115 objects/s recovering 2023-10-09T05:23:10.925 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:10.983 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:10.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:10.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:11.089 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:11.121 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:11.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:11.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:11.224 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:11.225 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:23:11.225 INFO:teuthology.orchestra.run.smithi133.stdout: "ops": [], 2023-10-09T05:23:11.225 INFO:teuthology.orchestra.run.smithi133.stdout: "complaint_time": 30, 2023-10-09T05:23:11.225 INFO:teuthology.orchestra.run.smithi133.stdout: "num_blocked_ops": 0 2023-10-09T05:23:11.225 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:23:11.256 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:11.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:11.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:11.359 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:11.392 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:11.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:11.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:11.498 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:11.524 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:11.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:11.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:11.627 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:11.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:11 smithi194 bash[92666]: cluster 2023-10-09T05:23:10.568351+0000 osd.4 (osd.4) 218 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T05:23:11.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:11 smithi194 bash[92666]: cluster 2023-10-09T05:23:10.568608+0000 osd.5 (osd.5) 234 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T05:23:11.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:11 smithi194 bash[92666]: cluster 2023-10-09T05:23:10.570509+0000 mon.a (mon.0) 1362 : cluster 0 osdmap e251: 8 total, 8 up, 7 in 2023-10-09T05:23:11.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:11 smithi194 bash[92666]: cluster 2023-10-09T05:23:10.586749+0000 osd.6 (osd.6) 195 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T05:23:11.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:11 smithi194 bash[92666]: cluster 2023-10-09T05:23:10.763260+0000 osd.7 (osd.7) 200 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T05:23:11.670 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:11.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:11.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:11.776 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:11.810 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:11.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:11.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:11.913 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:11.923 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:11 smithi133 bash[105589]: cluster 2023-10-09T05:23:10.568351+0000 osd.4 (osd.4) 218 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T05:23:11.923 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:11 smithi133 bash[105589]: cluster 2023-10-09T05:23:10.568608+0000 osd.5 (osd.5) 234 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T05:23:11.923 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:11 smithi133 bash[105589]: cluster 2023-10-09T05:23:10.570509+0000 mon.a (mon.0) 1362 : cluster 0 osdmap e251: 8 total, 8 up, 7 in 2023-10-09T05:23:11.923 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:11 smithi133 bash[105589]: cluster 2023-10-09T05:23:10.586749+0000 osd.6 (osd.6) 195 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T05:23:11.923 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:11 smithi133 bash[105589]: cluster 2023-10-09T05:23:10.763260+0000 osd.7 (osd.7) 200 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T05:23:11.924 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:11 smithi133 bash[102858]: cluster 2023-10-09T05:23:10.568351+0000 osd.4 (osd.4) 218 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T05:23:11.924 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:11 smithi133 bash[102858]: cluster 2023-10-09T05:23:10.568608+0000 osd.5 (osd.5) 234 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T05:23:11.924 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:11 smithi133 bash[102858]: cluster 2023-10-09T05:23:10.570509+0000 mon.a (mon.0) 1362 : cluster 0 osdmap e251: 8 total, 8 up, 7 in 2023-10-09T05:23:11.924 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:11 smithi133 bash[102858]: cluster 2023-10-09T05:23:10.586749+0000 osd.6 (osd.6) 195 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T05:23:11.924 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:11 smithi133 bash[102858]: cluster 2023-10-09T05:23:10.763260+0000 osd.7 (osd.7) 200 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T05:23:11.955 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:11.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:11.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:12.059 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:12.085 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:12.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:12.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:12.192 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:12.219 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:12.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:12.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:12.326 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:12.351 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:12.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:12.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:12.456 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:12.490 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:12.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:12.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:12.596 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:12.604 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:12 smithi194 bash[92666]: cluster 2023-10-09T05:23:12.087017+0000 mgr.y (mgr.25277) 1047 : cluster 0 pgmap v1101: 89 pgs: 1 active+recovering+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 905 MiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 20171/43470 objects degraded (46.402%); 5922/43470 objects misplaced (13.623%); 7.6 MiB/s, 154 objects/s recovering 2023-10-09T05:23:12.629 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:12.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:12.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:12.735 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:12.758 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:12.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:12.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:12.843 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:12 smithi133 bash[105589]: cluster 2023-10-09T05:23:12.087017+0000 mgr.y (mgr.25277) 1047 : cluster 0 pgmap v1101: 89 pgs: 1 active+recovering+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 905 MiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 20171/43470 objects degraded (46.402%); 5922/43470 objects misplaced (13.623%); 7.6 MiB/s, 154 objects/s recovering 2023-10-09T05:23:12.843 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:12 smithi133 bash[102858]: cluster 2023-10-09T05:23:12.087017+0000 mgr.y (mgr.25277) 1047 : cluster 0 pgmap v1101: 89 pgs: 1 active+recovering+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 905 MiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 20171/43470 objects degraded (46.402%); 5922/43470 objects misplaced (13.623%); 7.6 MiB/s, 154 objects/s recovering 2023-10-09T05:23:12.865 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:12.889 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:12.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:12.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:12.997 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:13.025 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:13.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:13.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:13.130 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:13.152 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:13.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:13.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:13.256 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:13.279 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:13.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:13.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:13.384 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:13.416 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:13.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:13.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:13.520 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:13.549 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:13.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:13.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:13.652 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:13.675 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:13.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:13.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:13.779 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:13.788 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:13 smithi194 bash[92666]: audit 2023-10-09T05:23:13.458596+0000 mon.a (mon.0) 1363 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:23:13.788 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:13 smithi194 bash[92666]: cluster 2023-10-09T05:23:13.557157+0000 mon.a (mon.0) 1364 : cluster 3 Health check update: Degraded data redundancy: 20171/43470 objects degraded (46.402%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:13.805 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:13.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:13.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:13.908 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:13.917 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:13 smithi133 bash[102858]: audit 2023-10-09T05:23:13.458596+0000 mon.a (mon.0) 1363 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:23:13.917 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:13 smithi133 bash[102858]: cluster 2023-10-09T05:23:13.557157+0000 mon.a (mon.0) 1364 : cluster 3 Health check update: Degraded data redundancy: 20171/43470 objects degraded (46.402%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:13.918 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:13 smithi133 bash[105589]: audit 2023-10-09T05:23:13.458596+0000 mon.a (mon.0) 1363 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:23:13.918 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:13 smithi133 bash[105589]: cluster 2023-10-09T05:23:13.557157+0000 mon.a (mon.0) 1364 : cluster 3 Health check update: Degraded data redundancy: 20171/43470 objects degraded (46.402%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:13.933 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:13.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:13.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:14.038 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:14.061 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:14.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:14.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:14.170 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:14.204 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:14.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:14.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:14.307 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:14.344 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:14.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:14.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:14.449 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:14.458 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:23:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:23:14] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.43.0" 2023-10-09T05:23:14.473 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:14.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:14.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:14.583 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:14.607 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:14.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:14.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:14.711 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:14.720 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:14 smithi133 bash[105589]: cluster 2023-10-09T05:23:14.087714+0000 mgr.y (mgr.25277) 1048 : cluster 0 pgmap v1102: 89 pgs: 1 active+recovering+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 896 MiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 428 op/s; 20171/43035 objects degraded (46.871%); 5922/43035 objects misplaced (13.761%); 3.4 MiB/s, 86 objects/s recovering 2023-10-09T05:23:14.720 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:14 smithi133 bash[102858]: cluster 2023-10-09T05:23:14.087714+0000 mgr.y (mgr.25277) 1048 : cluster 0 pgmap v1102: 89 pgs: 1 active+recovering+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 896 MiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 428 op/s; 20171/43035 objects degraded (46.871%); 5922/43035 objects misplaced (13.761%); 3.4 MiB/s, 86 objects/s recovering 2023-10-09T05:23:14.746 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:14.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:14.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:14.852 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:14.876 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:14.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:14.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:14.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:14 smithi194 bash[92666]: cluster 2023-10-09T05:23:14.087714+0000 mgr.y (mgr.25277) 1048 : cluster 0 pgmap v1102: 89 pgs: 1 active+recovering+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 896 MiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 428 op/s; 20171/43035 objects degraded (46.871%); 5922/43035 objects misplaced (13.761%); 3.4 MiB/s, 86 objects/s recovering 2023-10-09T05:23:14.986 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:15.013 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:15.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:15.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:15.121 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:15.145 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:15.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:15.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:15.248 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:15.274 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:15.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:15.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:15.379 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:15.406 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:15.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:15.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:15.510 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:15.539 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:15.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:15.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:15.644 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:15.677 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:15.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:15.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:15.781 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:15.809 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:15.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:15.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:15.916 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:15.956 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:15.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:15.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:16.060 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:16.088 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:16.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:16.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:16.196 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:16.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:16.226 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:16.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:16.327 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:16.372 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:16.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:16.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:16.482 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:16.505 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:16.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:16.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:16.609 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:16.632 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:16.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:16.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:16.737 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:16.765 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:16.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:16.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:16.870 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:16.896 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:16.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:16.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:17.000 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:17.027 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:17.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:17.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:17.136 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:17.160 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:17.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:17.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:17.169 INFO:teuthology.orchestra.run.smithi133.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T05:23:17.267 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:17.275 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:17 smithi133 bash[105589]: cluster 2023-10-09T05:23:16.089334+0000 mgr.y (mgr.25277) 1049 : cluster 0 pgmap v1103: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 796 MiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 526 op/s; 20171/38265 objects degraded (52.714%); 5680/38265 objects misplaced (14.844%); 5.3 MiB/s, 117 objects/s recovering 2023-10-09T05:23:17.276 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:17 smithi133 bash[105589]: audit 2023-10-09T05:23:16.253765+0000 mon.c (mon.2) 97 : audit 1 from='client.? 172.21.15.133:0/2756723414' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:23:17.276 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:17 smithi133 bash[105589]: audit 2023-10-09T05:23:16.255029+0000 mon.a (mon.0) 1365 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:23:17.276 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:17 smithi133 bash[102858]: cluster 2023-10-09T05:23:16.089334+0000 mgr.y (mgr.25277) 1049 : cluster 0 pgmap v1103: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 796 MiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 526 op/s; 20171/38265 objects degraded (52.714%); 5680/38265 objects misplaced (14.844%); 5.3 MiB/s, 117 objects/s recovering 2023-10-09T05:23:17.276 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:17 smithi133 bash[102858]: audit 2023-10-09T05:23:16.253765+0000 mon.c (mon.2) 97 : audit 1 from='client.? 172.21.15.133:0/2756723414' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:23:17.276 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:17 smithi133 bash[102858]: audit 2023-10-09T05:23:16.255029+0000 mon.a (mon.0) 1365 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:23:17.289 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:17.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:17.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:17.392 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:17.403 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:17 smithi194 bash[92666]: cluster 2023-10-09T05:23:16.089334+0000 mgr.y (mgr.25277) 1049 : cluster 0 pgmap v1103: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 796 MiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 526 op/s; 20171/38265 objects degraded (52.714%); 5680/38265 objects misplaced (14.844%); 5.3 MiB/s, 117 objects/s recovering 2023-10-09T05:23:17.403 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:17 smithi194 bash[92666]: audit 2023-10-09T05:23:16.253765+0000 mon.c (mon.2) 97 : audit 1 from='client.? 172.21.15.133:0/2756723414' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:23:17.403 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:17 smithi194 bash[92666]: audit 2023-10-09T05:23:16.255029+0000 mon.a (mon.0) 1365 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:23:17.425 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:17.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:17.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:17.529 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:17.553 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:17.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:17.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:17.656 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:17.678 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:17.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:17.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:17.781 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:17.810 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:17.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:17.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:17.825 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T05:23:17.914 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:17.936 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:17.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:17.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:18.039 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:18.074 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:18.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:18.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:18.170 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:18 smithi133 bash[102858]: audit 2023-10-09T05:23:17.151545+0000 mon.a (mon.0) 1366 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:23:18.171 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:18 smithi133 bash[102858]: cluster 2023-10-09T05:23:17.151610+0000 mon.a (mon.0) 1367 : cluster 0 osdmap e252: 8 total, 8 up, 7 in 2023-10-09T05:23:18.171 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:18 smithi133 bash[102858]: cluster 2023-10-09T05:23:17.155372+0000 osd.6 (osd.6) 196 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T05:23:18.171 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:18 smithi133 bash[102858]: cluster 2023-10-09T05:23:17.158586+0000 osd.4 (osd.4) 219 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T05:23:18.171 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:18 smithi133 bash[102858]: cluster 2023-10-09T05:23:17.159097+0000 osd.7 (osd.7) 201 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T05:23:18.171 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:18 smithi133 bash[102858]: cluster 2023-10-09T05:23:17.159537+0000 osd.6 (osd.6) 197 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T05:23:18.171 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:18 smithi133 bash[102858]: cluster 2023-10-09T05:23:17.163911+0000 osd.7 (osd.7) 202 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T05:23:18.177 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:18.208 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:18.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:18.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:18.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:18 smithi133 bash[105589]: audit 2023-10-09T05:23:17.151545+0000 mon.a (mon.0) 1366 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:23:18.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:18 smithi133 bash[105589]: cluster 2023-10-09T05:23:17.151610+0000 mon.a (mon.0) 1367 : cluster 0 osdmap e252: 8 total, 8 up, 7 in 2023-10-09T05:23:18.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:18 smithi133 bash[105589]: cluster 2023-10-09T05:23:17.155372+0000 osd.6 (osd.6) 196 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T05:23:18.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:18 smithi133 bash[105589]: cluster 2023-10-09T05:23:17.158586+0000 osd.4 (osd.4) 219 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T05:23:18.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:18 smithi133 bash[105589]: cluster 2023-10-09T05:23:17.159097+0000 osd.7 (osd.7) 201 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T05:23:18.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:18 smithi133 bash[105589]: cluster 2023-10-09T05:23:17.159537+0000 osd.6 (osd.6) 197 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T05:23:18.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:18 smithi133 bash[105589]: cluster 2023-10-09T05:23:17.163911+0000 osd.7 (osd.7) 202 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T05:23:18.311 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:18.322 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:18 smithi194 bash[92666]: audit 2023-10-09T05:23:17.151545+0000 mon.a (mon.0) 1366 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:23:18.322 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:18 smithi194 bash[92666]: cluster 2023-10-09T05:23:17.151610+0000 mon.a (mon.0) 1367 : cluster 0 osdmap e252: 8 total, 8 up, 7 in 2023-10-09T05:23:18.322 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:18 smithi194 bash[92666]: cluster 2023-10-09T05:23:17.155372+0000 osd.6 (osd.6) 196 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T05:23:18.322 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:18 smithi194 bash[92666]: cluster 2023-10-09T05:23:17.158586+0000 osd.4 (osd.4) 219 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T05:23:18.322 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:18 smithi194 bash[92666]: cluster 2023-10-09T05:23:17.159097+0000 osd.7 (osd.7) 201 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T05:23:18.322 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:18 smithi194 bash[92666]: cluster 2023-10-09T05:23:17.159537+0000 osd.6 (osd.6) 197 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T05:23:18.322 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:18 smithi194 bash[92666]: cluster 2023-10-09T05:23:17.163911+0000 osd.7 (osd.7) 202 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T05:23:18.339 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:18.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:18.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:18.443 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:18.468 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:18.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:18.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:18.571 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:18.597 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:18.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:18.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:18.702 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:18.734 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:18.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:18.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:18.838 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:18.868 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:18.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:18.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:18.972 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:19.002 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:19.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:19.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:19.105 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:19.142 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:19.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:19.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:19.246 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:19.270 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:19.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:19.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:19.374 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:19.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:19 smithi194 bash[92666]: cluster 2023-10-09T05:23:18.089854+0000 mgr.y (mgr.25277) 1050 : cluster 0 pgmap v1105: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 752 MiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 403 op/s; 20171/36150 objects degraded (55.798%); 5597/36150 objects misplaced (15.483%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T05:23:19.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:19 smithi194 bash[92666]: cluster 2023-10-09T05:23:18.169757+0000 osd.4 (osd.4) 220 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T05:23:19.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:19 smithi194 bash[92666]: cluster 2023-10-09T05:23:18.170372+0000 osd.5 (osd.5) 235 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T05:23:19.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:19 smithi194 bash[92666]: cluster 2023-10-09T05:23:18.173448+0000 mon.a (mon.0) 1368 : cluster 0 osdmap e253: 8 total, 8 up, 7 in 2023-10-09T05:23:19.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:19 smithi194 bash[92666]: cluster 2023-10-09T05:23:18.173717+0000 osd.6 (osd.6) 198 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T05:23:19.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:19 smithi194 bash[92666]: cluster 2023-10-09T05:23:18.174645+0000 osd.4 (osd.4) 221 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T05:23:19.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:19 smithi194 bash[92666]: cluster 2023-10-09T05:23:18.174647+0000 osd.7 (osd.7) 203 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T05:23:19.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:19 smithi194 bash[92666]: cluster 2023-10-09T05:23:18.175628+0000 osd.6 (osd.6) 199 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T05:23:19.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:19 smithi194 bash[92666]: cluster 2023-10-09T05:23:18.176432+0000 osd.7 (osd.7) 204 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T05:23:19.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:19 smithi194 bash[92666]: cluster 2023-10-09T05:23:18.176861+0000 osd.4 (osd.4) 222 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T05:23:19.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:19 smithi194 bash[92666]: cluster 2023-10-09T05:23:18.561402+0000 mon.a (mon.0) 1369 : cluster 3 Health check update: Degraded data redundancy: 20171/36150 objects degraded (55.798%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:19.395 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:19.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:19.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:19.498 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:19.506 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:19 smithi133 bash[105589]: cluster 2023-10-09T05:23:18.089854+0000 mgr.y (mgr.25277) 1050 : cluster 0 pgmap v1105: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 752 MiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 403 op/s; 20171/36150 objects degraded (55.798%); 5597/36150 objects misplaced (15.483%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T05:23:19.507 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:19 smithi133 bash[105589]: cluster 2023-10-09T05:23:18.169757+0000 osd.4 (osd.4) 220 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T05:23:19.507 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:19 smithi133 bash[105589]: cluster 2023-10-09T05:23:18.170372+0000 osd.5 (osd.5) 235 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T05:23:19.507 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:19 smithi133 bash[105589]: cluster 2023-10-09T05:23:18.173448+0000 mon.a (mon.0) 1368 : cluster 0 osdmap e253: 8 total, 8 up, 7 in 2023-10-09T05:23:19.507 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:19 smithi133 bash[105589]: cluster 2023-10-09T05:23:18.173717+0000 osd.6 (osd.6) 198 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T05:23:19.507 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:19 smithi133 bash[105589]: cluster 2023-10-09T05:23:18.174645+0000 osd.4 (osd.4) 221 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T05:23:19.507 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:19 smithi133 bash[105589]: cluster 2023-10-09T05:23:18.174647+0000 osd.7 (osd.7) 203 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T05:23:19.507 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:19 smithi133 bash[105589]: cluster 2023-10-09T05:23:18.175628+0000 osd.6 (osd.6) 199 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T05:23:19.507 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:19 smithi133 bash[105589]: cluster 2023-10-09T05:23:18.176432+0000 osd.7 (osd.7) 204 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T05:23:19.507 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:19 smithi133 bash[105589]: cluster 2023-10-09T05:23:18.176861+0000 osd.4 (osd.4) 222 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T05:23:19.507 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:19 smithi133 bash[105589]: cluster 2023-10-09T05:23:18.561402+0000 mon.a (mon.0) 1369 : cluster 3 Health check update: Degraded data redundancy: 20171/36150 objects degraded (55.798%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:19.508 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:19 smithi133 bash[102858]: cluster 2023-10-09T05:23:18.089854+0000 mgr.y (mgr.25277) 1050 : cluster 0 pgmap v1105: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 752 MiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 403 op/s; 20171/36150 objects degraded (55.798%); 5597/36150 objects misplaced (15.483%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T05:23:19.508 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:19 smithi133 bash[102858]: cluster 2023-10-09T05:23:18.169757+0000 osd.4 (osd.4) 220 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T05:23:19.508 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:19 smithi133 bash[102858]: cluster 2023-10-09T05:23:18.170372+0000 osd.5 (osd.5) 235 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T05:23:19.508 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:19 smithi133 bash[102858]: cluster 2023-10-09T05:23:18.173448+0000 mon.a (mon.0) 1368 : cluster 0 osdmap e253: 8 total, 8 up, 7 in 2023-10-09T05:23:19.508 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:19 smithi133 bash[102858]: cluster 2023-10-09T05:23:18.173717+0000 osd.6 (osd.6) 198 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T05:23:19.508 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:19 smithi133 bash[102858]: cluster 2023-10-09T05:23:18.174645+0000 osd.4 (osd.4) 221 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T05:23:19.508 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:19 smithi133 bash[102858]: cluster 2023-10-09T05:23:18.174647+0000 osd.7 (osd.7) 203 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T05:23:19.508 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:19 smithi133 bash[102858]: cluster 2023-10-09T05:23:18.175628+0000 osd.6 (osd.6) 199 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T05:23:19.509 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:19 smithi133 bash[102858]: cluster 2023-10-09T05:23:18.176432+0000 osd.7 (osd.7) 204 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T05:23:19.509 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:19 smithi133 bash[102858]: cluster 2023-10-09T05:23:18.176861+0000 osd.4 (osd.4) 222 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T05:23:19.509 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:19 smithi133 bash[102858]: cluster 2023-10-09T05:23:18.561402+0000 mon.a (mon.0) 1369 : cluster 3 Health check update: Degraded data redundancy: 20171/36150 objects degraded (55.798%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:19.528 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:19.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:19.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:19.632 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:19.656 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:19.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:19.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:19.762 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:19.794 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:19.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:19.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:19.900 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:19.929 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:19.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:19.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:20.032 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:20.060 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:20.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:20.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:20.163 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:20.190 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:20.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:20.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:20.296 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:20.322 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:20.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:20.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:20.427 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:20.438 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:20 smithi194 bash[92666]: cluster 2023-10-09T05:23:19.177281+0000 osd.4 (osd.4) 223 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T05:23:20.438 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:20 smithi194 bash[92666]: cluster 2023-10-09T05:23:19.178859+0000 mon.a (mon.0) 1370 : cluster 0 osdmap e254: 8 total, 8 up, 7 in 2023-10-09T05:23:20.438 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:20 smithi194 bash[92666]: cluster 2023-10-09T05:23:19.462604+0000 osd.6 (osd.6) 200 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T05:23:20.438 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:20 smithi194 bash[92666]: cluster 2023-10-09T05:23:19.915171+0000 osd.5 (osd.5) 236 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T05:23:20.450 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:20.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:20.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:20.527 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:23:20.554 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:20.586 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:23:20.587 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:20.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:20.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:20.603 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:20 smithi133 bash[105589]: cluster 2023-10-09T05:23:19.177281+0000 osd.4 (osd.4) 223 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T05:23:20.603 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:20 smithi133 bash[105589]: cluster 2023-10-09T05:23:19.178859+0000 mon.a (mon.0) 1370 : cluster 0 osdmap e254: 8 total, 8 up, 7 in 2023-10-09T05:23:20.603 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:20 smithi133 bash[105589]: cluster 2023-10-09T05:23:19.462604+0000 osd.6 (osd.6) 200 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T05:23:20.603 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:20 smithi133 bash[105589]: cluster 2023-10-09T05:23:19.915171+0000 osd.5 (osd.5) 236 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T05:23:20.603 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:20 smithi133 bash[102858]: cluster 2023-10-09T05:23:19.177281+0000 osd.4 (osd.4) 223 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T05:23:20.603 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:20 smithi133 bash[102858]: cluster 2023-10-09T05:23:19.178859+0000 mon.a (mon.0) 1370 : cluster 0 osdmap e254: 8 total, 8 up, 7 in 2023-10-09T05:23:20.604 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:20 smithi133 bash[102858]: cluster 2023-10-09T05:23:19.462604+0000 osd.6 (osd.6) 200 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T05:23:20.604 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:20 smithi133 bash[102858]: cluster 2023-10-09T05:23:19.915171+0000 osd.5 (osd.5) 236 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T05:23:20.631 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:23:20.666 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:23:20.694 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:20.701 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:23:20.721 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:20.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:20.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:20.747 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:23:20.787 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:23:20.824 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:20.833 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:23:20.837 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:23:20.837 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:23:20.837 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:23:20.838 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:23:20.838 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:23:20.838 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:23:20.838 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:23:20.838 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:23:20.849 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:20.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:20.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:20.952 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:20.975 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:20.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:20.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:21.078 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:21.098 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:21.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:21.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:21.209 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:21.236 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:21.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:21.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:21.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:21 smithi133 bash[105589]: cluster 2023-10-09T05:23:20.090598+0000 mgr.y (mgr.25277) 1051 : cluster 0 pgmap v1108: 89 pgs: 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 678 MiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 29913/32571 objects degraded (91.839%); 5510/32571 objects misplaced (16.917%); 4.3 MiB/s, 72 objects/s recovering 2023-10-09T05:23:21.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:21 smithi133 bash[105589]: audit 2023-10-09T05:23:20.469690+0000 mon.a (mon.0) 1371 : audit 0 from='client.? 172.21.15.133:0/1202305659' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:23:21.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:21 smithi133 bash[102858]: cluster 2023-10-09T05:23:20.090598+0000 mgr.y (mgr.25277) 1051 : cluster 0 pgmap v1108: 89 pgs: 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 678 MiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 29913/32571 objects degraded (91.839%); 5510/32571 objects misplaced (16.917%); 4.3 MiB/s, 72 objects/s recovering 2023-10-09T05:23:21.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:21 smithi133 bash[102858]: audit 2023-10-09T05:23:20.469690+0000 mon.a (mon.0) 1371 : audit 0 from='client.? 172.21.15.133:0/1202305659' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:23:21.340 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:21.368 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:21.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:21.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:21.471 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:21.479 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:21 smithi194 bash[92666]: cluster 2023-10-09T05:23:20.090598+0000 mgr.y (mgr.25277) 1051 : cluster 0 pgmap v1108: 89 pgs: 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 678 MiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 29913/32571 objects degraded (91.839%); 5510/32571 objects misplaced (16.917%); 4.3 MiB/s, 72 objects/s recovering 2023-10-09T05:23:21.479 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:21 smithi194 bash[92666]: audit 2023-10-09T05:23:20.469690+0000 mon.a (mon.0) 1371 : audit 0 from='client.? 172.21.15.133:0/1202305659' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:23:21.496 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:21.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:21.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:21.599 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:21.634 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:21.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:21.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:21.742 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:21.766 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:21.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:21.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:21.870 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:21.906 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:21.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:21.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:22.016 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:22.054 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:22.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:22.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:22.124 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T05:23:22.164 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:22.190 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:22.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:22.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:22.201 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:22 smithi133 bash[102858]: audit 2023-10-09T05:23:22.149660+0000 mon.c (mon.2) 98 : audit 1 from='client.? 172.21.15.133:0/3630161757' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:23:22.201 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:22 smithi133 bash[102858]: audit 2023-10-09T05:23:22.150837+0000 mon.a (mon.0) 1372 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:23:22.229 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is set 2023-10-09T05:23:22.294 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:22.322 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:22.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:22.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:22.426 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:22.437 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:22 smithi133 bash[105589]: audit 2023-10-09T05:23:22.149660+0000 mon.c (mon.2) 98 : audit 1 from='client.? 172.21.15.133:0/3630161757' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:23:22.437 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:22 smithi133 bash[105589]: audit 2023-10-09T05:23:22.150837+0000 mon.a (mon.0) 1372 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:23:22.450 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:22.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:22.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:22.558 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:22.590 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:22.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:22.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:22.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:22 smithi194 bash[92666]: audit 2023-10-09T05:23:22.149660+0000 mon.c (mon.2) 98 : audit 1 from='client.? 172.21.15.133:0/3630161757' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:23:22.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:22 smithi194 bash[92666]: audit 2023-10-09T05:23:22.150837+0000 mon.a (mon.0) 1372 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:23:22.695 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:22.722 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:22.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:22.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:22.827 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:22.860 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:22.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:22.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:22.965 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:22.995 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:22.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:22.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:23.098 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:23.125 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:23.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:23.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:23.232 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:23.241 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:23 smithi194 bash[92666]: cluster 2023-10-09T05:23:22.098064+0000 mgr.y (mgr.25277) 1052 : cluster 0 pgmap v1109: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 645 MiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 401 op/s; 29624/31029 objects degraded (95.472%); 5463/31029 objects misplaced (17.606%); 5.3 MiB/s, 88 objects/s recovering 2023-10-09T05:23:23.241 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:23 smithi194 bash[92666]: cluster 2023-10-09T05:23:22.198920+0000 mon.a (mon.0) 1373 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:23:23.241 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:23 smithi194 bash[92666]: audit 2023-10-09T05:23:22.216604+0000 mon.a (mon.0) 1374 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:23:23.241 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:23 smithi194 bash[92666]: cluster 2023-10-09T05:23:22.216982+0000 mon.a (mon.0) 1375 : cluster 0 osdmap e255: 8 total, 8 up, 7 in 2023-10-09T05:23:23.241 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:23 smithi194 bash[92666]: cluster 2023-10-09T05:23:22.218611+0000 osd.5 (osd.5) 237 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T05:23:23.241 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:23 smithi194 bash[92666]: cluster 2023-10-09T05:23:22.227019+0000 osd.7 (osd.7) 205 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T05:23:23.241 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:23 smithi194 bash[92666]: cluster 2023-10-09T05:23:22.248393+0000 osd.6 (osd.6) 201 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T05:23:23.242 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:23 smithi194 bash[92666]: cluster 2023-10-09T05:23:22.252802+0000 osd.6 (osd.6) 202 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T05:23:23.257 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:23.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:23.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:23.360 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:23.387 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:23.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:23.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:23.490 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:23.499 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:23 smithi133 bash[102858]: cluster 2023-10-09T05:23:22.098064+0000 mgr.y (mgr.25277) 1052 : cluster 0 pgmap v1109: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 645 MiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 401 op/s; 29624/31029 objects degraded (95.472%); 5463/31029 objects misplaced (17.606%); 5.3 MiB/s, 88 objects/s recovering 2023-10-09T05:23:23.499 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:23 smithi133 bash[102858]: cluster 2023-10-09T05:23:22.198920+0000 mon.a (mon.0) 1373 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:23:23.499 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:23 smithi133 bash[102858]: audit 2023-10-09T05:23:22.216604+0000 mon.a (mon.0) 1374 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:23:23.499 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:23 smithi133 bash[102858]: cluster 2023-10-09T05:23:22.216982+0000 mon.a (mon.0) 1375 : cluster 0 osdmap e255: 8 total, 8 up, 7 in 2023-10-09T05:23:23.499 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:23 smithi133 bash[102858]: cluster 2023-10-09T05:23:22.218611+0000 osd.5 (osd.5) 237 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T05:23:23.500 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:23 smithi133 bash[102858]: cluster 2023-10-09T05:23:22.227019+0000 osd.7 (osd.7) 205 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T05:23:23.500 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:23 smithi133 bash[102858]: cluster 2023-10-09T05:23:22.248393+0000 osd.6 (osd.6) 201 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T05:23:23.500 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:23 smithi133 bash[102858]: cluster 2023-10-09T05:23:22.252802+0000 osd.6 (osd.6) 202 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T05:23:23.501 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:23 smithi133 bash[105589]: cluster 2023-10-09T05:23:22.098064+0000 mgr.y (mgr.25277) 1052 : cluster 0 pgmap v1109: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 645 MiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 401 op/s; 29624/31029 objects degraded (95.472%); 5463/31029 objects misplaced (17.606%); 5.3 MiB/s, 88 objects/s recovering 2023-10-09T05:23:23.501 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:23 smithi133 bash[105589]: cluster 2023-10-09T05:23:22.198920+0000 mon.a (mon.0) 1373 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:23:23.501 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:23 smithi133 bash[105589]: audit 2023-10-09T05:23:22.216604+0000 mon.a (mon.0) 1374 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:23:23.501 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:23 smithi133 bash[105589]: cluster 2023-10-09T05:23:22.216982+0000 mon.a (mon.0) 1375 : cluster 0 osdmap e255: 8 total, 8 up, 7 in 2023-10-09T05:23:23.501 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:23 smithi133 bash[105589]: cluster 2023-10-09T05:23:22.218611+0000 osd.5 (osd.5) 237 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T05:23:23.501 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:23 smithi133 bash[105589]: cluster 2023-10-09T05:23:22.227019+0000 osd.7 (osd.7) 205 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T05:23:23.501 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:23 smithi133 bash[105589]: cluster 2023-10-09T05:23:22.248393+0000 osd.6 (osd.6) 201 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T05:23:23.502 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:23 smithi133 bash[105589]: cluster 2023-10-09T05:23:22.252802+0000 osd.6 (osd.6) 202 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T05:23:23.522 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:23.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:23.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:23.628 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:23.667 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:23.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:23.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:23.777 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:23.800 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:23.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:23.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:23.903 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:23.925 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:23.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:23.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:24.029 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:24.052 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:24.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:24.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:24.155 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:24.201 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:24.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:24.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:24.309 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:24.335 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:24.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:24.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:24.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:24 smithi194 bash[92666]: cluster 2023-10-09T05:23:22.248368+0000 osd.4 (osd.4) 224 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T05:23:24.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:24 smithi194 bash[92666]: cluster 2023-10-09T05:23:22.252035+0000 osd.4 (osd.4) 225 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T05:23:24.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:24 smithi194 bash[92666]: audit 2023-10-09T05:23:23.462648+0000 mon.a (mon.0) 1376 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:23:24.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:24 smithi194 bash[92666]: cluster 2023-10-09T05:23:23.562623+0000 mon.a (mon.0) 1377 : cluster 3 Health check update: Degraded data redundancy: 29624/31029 objects degraded (95.472%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:24.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:24 smithi194 bash[92666]: audit 2023-10-09T05:23:23.999392+0000 mon.a (mon.0) 1378 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:23:24.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:24 smithi194 bash[92666]: audit 2023-10-09T05:23:24.010808+0000 mon.a (mon.0) 1379 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:23:24.443 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:24.457 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:24 smithi133 bash[102858]: cluster 2023-10-09T05:23:22.248368+0000 osd.4 (osd.4) 224 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T05:23:24.457 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:24 smithi133 bash[102858]: cluster 2023-10-09T05:23:22.252035+0000 osd.4 (osd.4) 225 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T05:23:24.457 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:24 smithi133 bash[102858]: audit 2023-10-09T05:23:23.462648+0000 mon.a (mon.0) 1376 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:23:24.457 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:24 smithi133 bash[102858]: cluster 2023-10-09T05:23:23.562623+0000 mon.a (mon.0) 1377 : cluster 3 Health check update: Degraded data redundancy: 29624/31029 objects degraded (95.472%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:24.457 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:24 smithi133 bash[102858]: audit 2023-10-09T05:23:23.999392+0000 mon.a (mon.0) 1378 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:23:24.457 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:24 smithi133 bash[102858]: audit 2023-10-09T05:23:24.010808+0000 mon.a (mon.0) 1379 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:23:24.458 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:23:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:23:24] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.43.0" 2023-10-09T05:23:24.458 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:24 smithi133 bash[105589]: cluster 2023-10-09T05:23:22.248368+0000 osd.4 (osd.4) 224 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T05:23:24.458 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:24 smithi133 bash[105589]: cluster 2023-10-09T05:23:22.252035+0000 osd.4 (osd.4) 225 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T05:23:24.458 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:24 smithi133 bash[105589]: audit 2023-10-09T05:23:23.462648+0000 mon.a (mon.0) 1376 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:23:24.458 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:24 smithi133 bash[105589]: cluster 2023-10-09T05:23:23.562623+0000 mon.a (mon.0) 1377 : cluster 3 Health check update: Degraded data redundancy: 29624/31029 objects degraded (95.472%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:24.458 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:24 smithi133 bash[105589]: audit 2023-10-09T05:23:23.999392+0000 mon.a (mon.0) 1378 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:23:24.458 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:24 smithi133 bash[105589]: audit 2023-10-09T05:23:24.010808+0000 mon.a (mon.0) 1379 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:23:24.486 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:24.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:24.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:24.592 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:24.624 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:24.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:24.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:24.732 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:24.753 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:24.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:24.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:24.856 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:24.883 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:24.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:24.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:24.987 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:25.013 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:25.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:25.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:25.116 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:25.142 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:25.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:25.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:25.251 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:25.289 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:25.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:25.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:25.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:25 smithi194 bash[92666]: cluster 2023-10-09T05:23:24.100424+0000 mgr.y (mgr.25277) 1053 : cluster 0 pgmap v1111: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 638 MiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 303 op/s; 29624/30669 objects degraded (96.593%); 5463/30669 objects misplaced (17.813%); 4.4 MiB/s, 74 objects/s recovering 2023-10-09T05:23:25.397 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:25.424 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:25.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:25.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:25.528 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:25.536 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:25 smithi133 bash[102858]: cluster 2023-10-09T05:23:24.100424+0000 mgr.y (mgr.25277) 1053 : cluster 0 pgmap v1111: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 638 MiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 303 op/s; 29624/30669 objects degraded (96.593%); 5463/30669 objects misplaced (17.813%); 4.4 MiB/s, 74 objects/s recovering 2023-10-09T05:23:25.536 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:25 smithi133 bash[105589]: cluster 2023-10-09T05:23:24.100424+0000 mgr.y (mgr.25277) 1053 : cluster 0 pgmap v1111: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 638 MiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 303 op/s; 29624/30669 objects degraded (96.593%); 5463/30669 objects misplaced (17.813%); 4.4 MiB/s, 74 objects/s recovering 2023-10-09T05:23:25.554 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:25.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:25.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:25.664 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:25.696 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:25.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:25.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:25.799 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:25.833 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:25.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:25.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:25.944 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:25.970 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:25.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:25.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:26.073 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:26.113 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:26.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:26.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:26.219 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:26.244 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:26.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:26.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:26.348 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:26.382 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:26.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:26.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:26.489 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:26.517 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:26.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:26.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:26.621 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:26.655 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:26.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:26.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:26.762 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:26.788 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:26.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:26.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:26.865 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:23:26.882 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:23:26.892 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:26.970 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:26.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:26.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:27.076 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:27.124 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:27.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:27.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:27.232 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:27.276 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:27.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:27.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:27.388 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:27.399 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:23:27.414 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:27.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:27.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:27.518 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:27.528 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:27 smithi194 bash[92666]: cluster 2023-10-09T05:23:26.101202+0000 mgr.y (mgr.25277) 1054 : cluster 0 pgmap v1112: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 541 MiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 406 op/s; 29624/26013 objects degraded (113.882%); 5363/26013 objects misplaced (20.617%); 4.1 MiB/s, 68 objects/s recovering 2023-10-09T05:23:27.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:27 smithi133 bash[102858]: cluster 2023-10-09T05:23:26.101202+0000 mgr.y (mgr.25277) 1054 : cluster 0 pgmap v1112: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 541 MiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 406 op/s; 29624/26013 objects degraded (113.882%); 5363/26013 objects misplaced (20.617%); 4.1 MiB/s, 68 objects/s recovering 2023-10-09T05:23:27.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:27 smithi133 bash[105589]: cluster 2023-10-09T05:23:26.101202+0000 mgr.y (mgr.25277) 1054 : cluster 0 pgmap v1112: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 541 MiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 406 op/s; 29624/26013 objects degraded (113.882%); 5363/26013 objects misplaced (20.617%); 4.1 MiB/s, 68 objects/s recovering 2023-10-09T05:23:27.549 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:27.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:27.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:27.652 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:27.686 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:27.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:27.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:27.793 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:27.824 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:27.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:27.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:27.932 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:27.959 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:27.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:27.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:28.066 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:28.094 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:28.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:28.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:28.197 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:28.227 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:28.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:28.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:28.331 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:28.363 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:28.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:28.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:28.419 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:23:28.469 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:28.491 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:28.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:28.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:28.621 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:28.645 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:28.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:28.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:28.753 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:28.807 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:28.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:28.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:28.914 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:28.937 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:28.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:28.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:29.041 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:29.068 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:29.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:29.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:29.175 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:29.225 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:29.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:29.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:29.327 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:29.352 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:29.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:29.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:29.455 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:29.486 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:29 smithi133 bash[102858]: cluster 2023-10-09T05:23:28.101837+0000 mgr.y (mgr.25277) 1055 : cluster 0 pgmap v1113: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 503 MiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 430 op/s; 29624/24168 objects degraded (122.575%); 5256/24168 objects misplaced (21.748%); 4.2 MiB/s, 73 objects/s recovering 2023-10-09T05:23:29.487 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:29 smithi133 bash[102858]: audit 2023-10-09T05:23:28.465819+0000 mon.a (mon.0) 1380 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:23:29.487 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:29 smithi133 bash[102858]: audit 2023-10-09T05:23:28.470627+0000 mon.a (mon.0) 1381 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:23:29.487 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:29 smithi133 bash[102858]: cluster 2023-10-09T05:23:28.572669+0000 mon.a (mon.0) 1382 : cluster 3 Health check update: Degraded data redundancy: 29624/24168 objects degraded (122.575%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:29.487 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:29 smithi133 bash[105589]: cluster 2023-10-09T05:23:28.101837+0000 mgr.y (mgr.25277) 1055 : cluster 0 pgmap v1113: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 503 MiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 430 op/s; 29624/24168 objects degraded (122.575%); 5256/24168 objects misplaced (21.748%); 4.2 MiB/s, 73 objects/s recovering 2023-10-09T05:23:29.487 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:29 smithi133 bash[105589]: audit 2023-10-09T05:23:28.465819+0000 mon.a (mon.0) 1380 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:23:29.487 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:29 smithi133 bash[105589]: audit 2023-10-09T05:23:28.470627+0000 mon.a (mon.0) 1381 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:23:29.487 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:29 smithi133 bash[105589]: cluster 2023-10-09T05:23:28.572669+0000 mon.a (mon.0) 1382 : cluster 3 Health check update: Degraded data redundancy: 29624/24168 objects degraded (122.575%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:29.513 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:29.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:29.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:29.621 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:29.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:29 smithi194 bash[92666]: cluster 2023-10-09T05:23:28.101837+0000 mgr.y (mgr.25277) 1055 : cluster 0 pgmap v1113: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 503 MiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 430 op/s; 29624/24168 objects degraded (122.575%); 5256/24168 objects misplaced (21.748%); 4.2 MiB/s, 73 objects/s recovering 2023-10-09T05:23:29.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:29 smithi194 bash[92666]: audit 2023-10-09T05:23:28.465819+0000 mon.a (mon.0) 1380 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:23:29.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:29 smithi194 bash[92666]: audit 2023-10-09T05:23:28.470627+0000 mon.a (mon.0) 1381 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:23:29.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:29 smithi194 bash[92666]: cluster 2023-10-09T05:23:28.572669+0000 mon.a (mon.0) 1382 : cluster 3 Health check update: Degraded data redundancy: 29624/24168 objects degraded (122.575%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:29.692 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:29.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:29.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:29.796 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:29.832 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:29.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:29.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:29.936 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:29.959 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:29.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:29.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:30.062 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:30.087 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:30.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:30.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:30.196 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:30.231 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:30.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:30.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:30.335 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:30.358 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:30.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:30.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:30.461 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:30.507 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:30.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:30.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:30.612 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:30.641 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:30.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:30.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:30.745 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:30.785 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:30.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:30.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:30.889 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:30.914 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:30.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:30.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:31.018 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:31.043 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:31.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:31.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:31.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:30 smithi133 bash[102858]: audit 2023-10-09T05:23:30.000040+0000 mon.a (mon.0) 1383 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:23:31.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:30 smithi133 bash[102858]: audit 2023-10-09T05:23:30.019244+0000 mon.a (mon.0) 1384 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:23:31.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:30 smithi133 bash[102858]: cluster 2023-10-09T05:23:30.102505+0000 mgr.y (mgr.25277) 1056 : cluster 0 pgmap v1114: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 398 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 446 op/s; 29624/19152 objects degraded (154.678%); 5256/19152 objects misplaced (27.444%); 3.2 MiB/s, 54 objects/s recovering 2023-10-09T05:23:31.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:31 smithi133 bash[105589]: audit 2023-10-09T05:23:30.000040+0000 mon.a (mon.0) 1383 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:23:31.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:31 smithi133 bash[105589]: audit 2023-10-09T05:23:30.019244+0000 mon.a (mon.0) 1384 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:23:31.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:31 smithi133 bash[105589]: cluster 2023-10-09T05:23:30.102505+0000 mgr.y (mgr.25277) 1056 : cluster 0 pgmap v1114: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 398 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 446 op/s; 29624/19152 objects degraded (154.678%); 5256/19152 objects misplaced (27.444%); 3.2 MiB/s, 54 objects/s recovering 2023-10-09T05:23:31.148 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:31.188 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:31.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:31.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:31.292 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:31.302 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:30 smithi194 bash[92666]: audit 2023-10-09T05:23:30.000040+0000 mon.a (mon.0) 1383 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:23:31.302 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:30 smithi194 bash[92666]: audit 2023-10-09T05:23:30.019244+0000 mon.a (mon.0) 1384 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:23:31.302 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:30 smithi194 bash[92666]: cluster 2023-10-09T05:23:30.102505+0000 mgr.y (mgr.25277) 1056 : cluster 0 pgmap v1114: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 398 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 446 op/s; 29624/19152 objects degraded (154.678%); 5256/19152 objects misplaced (27.444%); 3.2 MiB/s, 54 objects/s recovering 2023-10-09T05:23:31.315 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:31.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:31.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:31.418 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:31.456 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:31.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:31.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:31.477 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:23:31.477 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:23:31.563 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:31.592 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:31.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:31.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:31.697 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:31.724 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:31.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:31.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:31.828 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:31.856 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:31.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:31.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:31.959 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:31.990 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:31.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:31.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:32.094 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:32.141 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:32.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:32.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:32.245 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:32.254 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:32 smithi133 bash[102858]: audit 2023-10-09T05:23:31.165263+0000 mon.a (mon.0) 1385 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:23:32.254 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:32 smithi133 bash[102858]: audit 2023-10-09T05:23:31.178165+0000 mon.a (mon.0) 1386 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:23:32.254 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:32 smithi133 bash[102858]: audit 2023-10-09T05:23:31.805781+0000 mon.a (mon.0) 1387 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:23:32.255 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:32 smithi133 bash[102858]: audit 2023-10-09T05:23:31.816277+0000 mon.a (mon.0) 1388 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:23:32.255 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:32 smithi133 bash[102858]: audit 2023-10-09T05:23:31.828822+0000 mon.a (mon.0) 1389 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:23:32.255 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:32 smithi133 bash[105589]: audit 2023-10-09T05:23:31.165263+0000 mon.a (mon.0) 1385 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:23:32.255 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:32 smithi133 bash[105589]: audit 2023-10-09T05:23:31.178165+0000 mon.a (mon.0) 1386 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:23:32.255 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:32 smithi133 bash[105589]: audit 2023-10-09T05:23:31.805781+0000 mon.a (mon.0) 1387 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:23:32.255 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:32 smithi133 bash[105589]: audit 2023-10-09T05:23:31.816277+0000 mon.a (mon.0) 1388 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:23:32.255 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:32 smithi133 bash[105589]: audit 2023-10-09T05:23:31.828822+0000 mon.a (mon.0) 1389 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:23:32.273 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:32.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:32.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:32.376 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:32.404 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:32.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:32.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:32.507 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:32.533 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:32.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:32.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:32.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:32 smithi194 bash[92666]: audit 2023-10-09T05:23:31.165263+0000 mon.a (mon.0) 1385 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:23:32.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:32 smithi194 bash[92666]: audit 2023-10-09T05:23:31.178165+0000 mon.a (mon.0) 1386 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:23:32.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:32 smithi194 bash[92666]: audit 2023-10-09T05:23:31.805781+0000 mon.a (mon.0) 1387 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:23:32.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:32 smithi194 bash[92666]: audit 2023-10-09T05:23:31.816277+0000 mon.a (mon.0) 1388 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:23:32.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:32 smithi194 bash[92666]: audit 2023-10-09T05:23:31.828822+0000 mon.a (mon.0) 1389 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:23:32.642 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:32.670 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:32.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:32.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:32.776 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:32.811 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:32.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:32.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:32.917 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:32.950 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:32.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:32.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:33.054 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:33.088 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:33.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:33.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:33.194 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:33.203 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:33 smithi133 bash[105589]: cluster 2023-10-09T05:23:32.103189+0000 mgr.y (mgr.25277) 1057 : cluster 0 pgmap v1115: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 351 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 470 op/s; 29565/16914 objects degraded (174.796%); 5102/16914 objects misplaced (30.164%); 1.4 MiB/s, 41 objects/s recovering 2023-10-09T05:23:33.203 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:33 smithi133 bash[102858]: cluster 2023-10-09T05:23:32.103189+0000 mgr.y (mgr.25277) 1057 : cluster 0 pgmap v1115: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 351 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 470 op/s; 29565/16914 objects degraded (174.796%); 5102/16914 objects misplaced (30.164%); 1.4 MiB/s, 41 objects/s recovering 2023-10-09T05:23:33.232 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:33.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:33.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:33.336 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:33.366 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:33.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:33.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:33.472 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:33.500 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:33.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:33.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:33.607 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:33.615 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:33 smithi194 bash[92666]: cluster 2023-10-09T05:23:32.103189+0000 mgr.y (mgr.25277) 1057 : cluster 0 pgmap v1115: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 351 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 470 op/s; 29565/16914 objects degraded (174.796%); 5102/16914 objects misplaced (30.164%); 1.4 MiB/s, 41 objects/s recovering 2023-10-09T05:23:33.634 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:33.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:33.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:33.742 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:33.767 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:33.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:33.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:33.874 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:33.910 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:33.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:33.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:34.015 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:34.042 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:34.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:34.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:34.148 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:34.166 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:34.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:34.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:34.270 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:34.279 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:34 smithi133 bash[102858]: cluster 2023-10-09T05:23:33.574703+0000 mon.a (mon.0) 1390 : cluster 3 Health check update: Degraded data redundancy: 29565/16914 objects degraded (174.796%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:34.279 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:34 smithi133 bash[105589]: cluster 2023-10-09T05:23:33.574703+0000 mon.a (mon.0) 1390 : cluster 3 Health check update: Degraded data redundancy: 29565/16914 objects degraded (174.796%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:34.320 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:34.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:34.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:34.428 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:34.436 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:34 smithi194 bash[92666]: cluster 2023-10-09T05:23:33.574703+0000 mon.a (mon.0) 1390 : cluster 3 Health check update: Degraded data redundancy: 29565/16914 objects degraded (174.796%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:34.453 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:34.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:34.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:34.555 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:34.585 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:34.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:34.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:34.688 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:34.719 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:23:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:23:34] "GET /metrics HTTP/1.1" 200 36513 "" "Prometheus/2.43.0" 2023-10-09T05:23:34.729 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:34.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:34.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:34.837 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:34.861 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:34.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:34.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:34.967 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:34.997 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:34.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:34.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:35.100 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:35.130 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:35.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:35.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:35.235 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:35.259 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:35.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:35.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:35.362 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:35.372 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:35 smithi194 bash[92666]: cluster 2023-10-09T05:23:34.103725+0000 mgr.y (mgr.25277) 1058 : cluster 0 pgmap v1116: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 341 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 409 op/s; 29565/16428 objects degraded (179.967%); 5102/16428 objects misplaced (31.057%); 1.2 MiB/s, 35 objects/s recovering 2023-10-09T05:23:35.390 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:35.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:35.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:35.497 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:35.508 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:35 smithi133 bash[102858]: cluster 2023-10-09T05:23:34.103725+0000 mgr.y (mgr.25277) 1058 : cluster 0 pgmap v1116: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 341 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 409 op/s; 29565/16428 objects degraded (179.967%); 5102/16428 objects misplaced (31.057%); 1.2 MiB/s, 35 objects/s recovering 2023-10-09T05:23:35.509 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:35 smithi133 bash[105589]: cluster 2023-10-09T05:23:34.103725+0000 mgr.y (mgr.25277) 1058 : cluster 0 pgmap v1116: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 341 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 409 op/s; 29565/16428 objects degraded (179.967%); 5102/16428 objects misplaced (31.057%); 1.2 MiB/s, 35 objects/s recovering 2023-10-09T05:23:35.537 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:35.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:35.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:35.644 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:35.669 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:35.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:35.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:35.772 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:35.801 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:35.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:35.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:35.904 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:35.931 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:35.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:35.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:36.034 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:36.058 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:36.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:36.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:36.162 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:36.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:36.207 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:36.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:36.308 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:36.328 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:36.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:36.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:36.431 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:36.466 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:36.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:36.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:36.571 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:36.603 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:36.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:36.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:36.707 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:36.735 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:36.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:36.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:36.839 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:36.865 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:36.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:36.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:36.970 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:36.997 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:37.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:37.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:37.101 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:37.129 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:37.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:37.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:37.185 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:37 smithi133 bash[102858]: cluster 2023-10-09T05:23:36.104527+0000 mgr.y (mgr.25277) 1059 : cluster 0 pgmap v1117: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 237 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 29476/11430 objects degraded (257.883%); 5102/11430 objects misplaced (44.637%); 1.2 MiB/s, 42 objects/s recovering 2023-10-09T05:23:37.185 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:37 smithi133 bash[105589]: cluster 2023-10-09T05:23:36.104527+0000 mgr.y (mgr.25277) 1059 : cluster 0 pgmap v1117: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 237 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 29476/11430 objects degraded (257.883%); 5102/11430 objects misplaced (44.637%); 1.2 MiB/s, 42 objects/s recovering 2023-10-09T05:23:37.233 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:37.252 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:37.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:37.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:37.355 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:37.366 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:37 smithi194 bash[92666]: cluster 2023-10-09T05:23:36.104527+0000 mgr.y (mgr.25277) 1059 : cluster 0 pgmap v1117: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 237 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 29476/11430 objects degraded (257.883%); 5102/11430 objects misplaced (44.637%); 1.2 MiB/s, 42 objects/s recovering 2023-10-09T05:23:37.387 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:37.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:37.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:37.491 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:37.520 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:37.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:37.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:37.624 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:37.649 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:37.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:37.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:37.752 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:37.779 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:37.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:37.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:37.882 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:37.911 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:37.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:37.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:38.013 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:38.043 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:38.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:38.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:38.150 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:38.177 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:38.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:38.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:38.284 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:38.314 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:38.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:38.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:38.417 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:38.453 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:38.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:38.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:38.557 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:38.581 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:38.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:38.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:38.684 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:38.717 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:38.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:38.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:38.822 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:38.851 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:38.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:38.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:38.957 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:38.980 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:38.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:38.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:39.087 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:39.109 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:39.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:39.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:39.175 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:39 smithi133 bash[102858]: cluster 2023-10-09T05:23:38.105141+0000 mgr.y (mgr.25277) 1060 : cluster 0 pgmap v1118: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 197 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 458 op/s; 29476/9501 objects degraded (310.241%); 4993/9501 objects misplaced (52.552%); 677 KiB/s, 43 objects/s recovering 2023-10-09T05:23:39.176 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:39 smithi133 bash[102858]: cluster 2023-10-09T05:23:38.575843+0000 mon.a (mon.0) 1391 : cluster 3 Health check update: Degraded data redundancy: 29476/9501 objects degraded (310.241%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:39.211 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:39.242 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:39.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:39.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:39.348 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:39.383 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:39.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:39.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:39.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:39 smithi194 bash[92666]: cluster 2023-10-09T05:23:38.105141+0000 mgr.y (mgr.25277) 1060 : cluster 0 pgmap v1118: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 197 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 458 op/s; 29476/9501 objects degraded (310.241%); 4993/9501 objects misplaced (52.552%); 677 KiB/s, 43 objects/s recovering 2023-10-09T05:23:39.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:39 smithi194 bash[92666]: cluster 2023-10-09T05:23:38.575843+0000 mon.a (mon.0) 1391 : cluster 3 Health check update: Degraded data redundancy: 29476/9501 objects degraded (310.241%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:39.487 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:39.495 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:39 smithi133 bash[105589]: cluster 2023-10-09T05:23:38.105141+0000 mgr.y (mgr.25277) 1060 : cluster 0 pgmap v1118: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 197 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 458 op/s; 29476/9501 objects degraded (310.241%); 4993/9501 objects misplaced (52.552%); 677 KiB/s, 43 objects/s recovering 2023-10-09T05:23:39.496 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:39 smithi133 bash[105589]: cluster 2023-10-09T05:23:38.575843+0000 mon.a (mon.0) 1391 : cluster 3 Health check update: Degraded data redundancy: 29476/9501 objects degraded (310.241%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:39.542 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:39.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:39.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:39.648 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:39.676 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:39.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:39.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:39.780 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:39.810 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:39.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:39.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:39.917 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:39.949 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:39.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:39.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:40.053 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:40.078 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:40.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:40.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:40.182 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:40.206 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:40.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:40.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:40.312 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:40.336 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:40.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:40.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:40.439 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:40.461 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:40.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:40.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:40.568 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:40.601 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:40.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:40.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:40.705 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:40.732 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:40.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:40.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:40.837 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:40.877 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:40.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:40.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:40.984 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:41.007 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:41.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:41.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:41.112 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:41.141 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:41.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:41.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:41.244 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:41.254 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:41 smithi133 bash[102858]: cluster 2023-10-09T05:23:40.105927+0000 mgr.y (mgr.25277) 1061 : cluster 0 pgmap v1119: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 101 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 535 op/s; 29476/4872 objects degraded (605.008%); 4993/4872 objects misplaced (102.484%); 235 KiB/s, 34 objects/s recovering 2023-10-09T05:23:41.293 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:41.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:41.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:41.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:41 smithi194 bash[92666]: cluster 2023-10-09T05:23:40.105927+0000 mgr.y (mgr.25277) 1061 : cluster 0 pgmap v1119: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 101 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 535 op/s; 29476/4872 objects degraded (605.008%); 4993/4872 objects misplaced (102.484%); 235 KiB/s, 34 objects/s recovering 2023-10-09T05:23:41.397 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:41.424 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:41.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:41.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:41.529 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:41.540 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:41 smithi133 bash[105589]: cluster 2023-10-09T05:23:40.105927+0000 mgr.y (mgr.25277) 1061 : cluster 0 pgmap v1119: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 101 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 535 op/s; 29476/4872 objects degraded (605.008%); 4993/4872 objects misplaced (102.484%); 235 KiB/s, 34 objects/s recovering 2023-10-09T05:23:41.567 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:41.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:41.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:41.670 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:41.697 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:41.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:41.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:41.765 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Removed 95917 objects 2023-10-09T05:23:41.765 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Clean up completed and total clean up time :187.815 2023-10-09T05:23:41.802 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:41.835 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:41.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:41.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:41.939 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:41.959 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:41.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:41.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:42.063 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:42.099 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:42.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:42.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:42.204 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:42.236 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:42.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:42.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:42.340 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:42.376 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:42.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:42.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:42.480 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:42.500 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:42.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:42.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:42.604 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:42.622 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:42.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:42.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:42.725 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:42.757 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:42.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:42.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:42.861 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:42.896 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:42.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:42.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:42.999 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:43.026 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:43.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:43.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:43.130 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:43.162 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:43.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:43.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:43.266 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:43.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:43 smithi133 bash[105589]: cluster 2023-10-09T05:23:42.106445+0000 mgr.y (mgr.25277) 1062 : cluster 0 pgmap v1120: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 56 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 455 op/s; 29336/2757 objects degraded (1064.055%); 4857/2757 objects misplaced (176.170%); 235 KiB/s, 57 objects/s recovering 2023-10-09T05:23:43.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:43 smithi133 bash[102858]: cluster 2023-10-09T05:23:42.106445+0000 mgr.y (mgr.25277) 1062 : cluster 0 pgmap v1120: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 56 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 455 op/s; 29336/2757 objects degraded (1064.055%); 4857/2757 objects misplaced (176.170%); 235 KiB/s, 57 objects/s recovering 2023-10-09T05:23:43.302 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:43.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:43.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:43.405 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:43.418 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:43 smithi194 bash[92666]: cluster 2023-10-09T05:23:42.106445+0000 mgr.y (mgr.25277) 1062 : cluster 0 pgmap v1120: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 56 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 455 op/s; 29336/2757 objects degraded (1064.055%); 4857/2757 objects misplaced (176.170%); 235 KiB/s, 57 objects/s recovering 2023-10-09T05:23:43.433 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:43.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:43.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:43.536 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:43.536 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_6 2023-10-09T05:23:43.537 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_6 unique_pool_6 --yes-i-really-really-mean-it 2023-10-09T05:23:43.557 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:43.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:43.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:43.661 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:43.683 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:43.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:43.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:43.787 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:43.814 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:43.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:43.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:43.919 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:43.938 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:43.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:43.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:44.041 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:44.064 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:44.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:44.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:44.168 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:44.195 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:44.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:44.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:44.299 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:44.323 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:44.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:44.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:44.426 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:44.437 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:44 smithi194 bash[92666]: audit 2023-10-09T05:23:43.459342+0000 mon.a (mon.0) 1392 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:23:44.438 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:44 smithi194 bash[92666]: cluster 2023-10-09T05:23:43.577037+0000 mon.a (mon.0) 1393 : cluster 3 Health check update: Degraded data redundancy: 29336/2757 objects degraded (1064.055%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:44.455 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:44.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:44.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:44.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:23:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:23:44] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.43.0" 2023-10-09T05:23:44.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:44 smithi133 bash[105589]: audit 2023-10-09T05:23:43.459342+0000 mon.a (mon.0) 1392 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:23:44.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:44 smithi133 bash[105589]: cluster 2023-10-09T05:23:43.577037+0000 mon.a (mon.0) 1393 : cluster 3 Health check update: Degraded data redundancy: 29336/2757 objects degraded (1064.055%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:44.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:44 smithi133 bash[102858]: audit 2023-10-09T05:23:43.459342+0000 mon.a (mon.0) 1392 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:23:44.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:44 smithi133 bash[102858]: cluster 2023-10-09T05:23:43.577037+0000 mon.a (mon.0) 1393 : cluster 3 Health check update: Degraded data redundancy: 29336/2757 objects degraded (1064.055%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:44.559 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:44.581 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:44.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:44.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:44.687 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:44.724 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:44.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:44.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:44.828 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:44.862 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:44.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:44.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:44.966 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:45.001 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:45.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:45.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:45.104 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:45.125 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:45.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:45.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:45.228 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:45.242 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:45 smithi194 bash[92666]: cluster 2023-10-09T05:23:44.106929+0000 mgr.y (mgr.25277) 1063 : cluster 0 pgmap v1121: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 50 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 402 op/s; 29336/2433 objects degraded (1205.754%); 4857/2433 objects misplaced (199.630%); 0 B/s, 39 objects/s recovering 2023-10-09T05:23:45.259 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:45.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:45.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:45.363 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:45.389 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:45.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:45.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:45.493 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:45.526 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:45.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:45.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:45.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:45 smithi133 bash[102858]: cluster 2023-10-09T05:23:44.106929+0000 mgr.y (mgr.25277) 1063 : cluster 0 pgmap v1121: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 50 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 402 op/s; 29336/2433 objects degraded (1205.754%); 4857/2433 objects misplaced (199.630%); 0 B/s, 39 objects/s recovering 2023-10-09T05:23:45.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:45 smithi133 bash[105589]: cluster 2023-10-09T05:23:44.106929+0000 mgr.y (mgr.25277) 1063 : cluster 0 pgmap v1121: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 50 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 402 op/s; 29336/2433 objects degraded (1205.754%); 4857/2433 objects misplaced (199.630%); 0 B/s, 39 objects/s recovering 2023-10-09T05:23:45.629 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:45.664 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:45.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:45.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:45.768 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:45.800 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:45.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:45.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:45.903 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:45.935 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:45.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:45.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:46.068 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:46.099 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:46.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:46.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:46.203 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:46.224 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:46.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:46.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:46.327 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:46.360 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:46.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:46.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:46.463 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:46.495 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:46.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:46.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:46.598 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:46.620 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:46.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:46.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:46.723 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:46.754 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:46.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:46.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:46.858 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:46.882 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:46.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:46.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:46.986 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:47.011 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:47.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:47.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:47.115 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:47.151 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:47.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:47.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:47.255 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:47.284 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:47.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:47.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:47.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:47 smithi194 bash[92666]: cluster 2023-10-09T05:23:46.107706+0000 mgr.y (mgr.25277) 1064 : cluster 0 pgmap v1122: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 451 op/s; 29021/153 objects degraded (18967.974%); 4857/153 objects misplaced (3174.510%); 0 B/s, 65 objects/s recovering 2023-10-09T05:23:47.389 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:47.419 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:47.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:47.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:47.523 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:47.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:47 smithi133 bash[105589]: cluster 2023-10-09T05:23:46.107706+0000 mgr.y (mgr.25277) 1064 : cluster 0 pgmap v1122: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 451 op/s; 29021/153 objects degraded (18967.974%); 4857/153 objects misplaced (3174.510%); 0 B/s, 65 objects/s recovering 2023-10-09T05:23:47.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:47 smithi133 bash[102858]: cluster 2023-10-09T05:23:46.107706+0000 mgr.y (mgr.25277) 1064 : cluster 0 pgmap v1122: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 451 op/s; 29021/153 objects degraded (18967.974%); 4857/153 objects misplaced (3174.510%); 0 B/s, 65 objects/s recovering 2023-10-09T05:23:47.553 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:47.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:47.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:47.656 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:47.680 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:47.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:47.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:47.783 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:47.817 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:47.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:47.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:47.921 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:47.958 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:47.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:47.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:48.061 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:48.087 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:48.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:48.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:48.191 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:48.221 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:48.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:48.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:48.325 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:48.347 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:48.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:48.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:48.451 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:48.485 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:48.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:48.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:48.588 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:48.611 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:48.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:48.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:48.717 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:48.744 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:48.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:48.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:48.848 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:48.883 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:48.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:48.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:48.988 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:49.014 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:49.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:49.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:49.118 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:49.140 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:49.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:49.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:49.244 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:49.279 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:49.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:49.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:49.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:49 smithi133 bash[102858]: cluster 2023-10-09T05:23:48.108526+0000 mgr.y (mgr.25277) 1065 : cluster 0 pgmap v1123: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 315 op/s; 29021/72 objects degraded (40306.944%); 4463/72 objects misplaced (6198.611%); 0 B/s, 91 objects/s recovering 2023-10-09T05:23:49.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:49 smithi133 bash[102858]: cluster 2023-10-09T05:23:48.578061+0000 mon.a (mon.0) 1394 : cluster 3 Health check update: Degraded data redundancy: 29021/72 objects degraded (40306.944%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:49.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:49 smithi133 bash[105589]: cluster 2023-10-09T05:23:48.108526+0000 mgr.y (mgr.25277) 1065 : cluster 0 pgmap v1123: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 315 op/s; 29021/72 objects degraded (40306.944%); 4463/72 objects misplaced (6198.611%); 0 B/s, 91 objects/s recovering 2023-10-09T05:23:49.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:49 smithi133 bash[105589]: cluster 2023-10-09T05:23:48.578061+0000 mon.a (mon.0) 1394 : cluster 3 Health check update: Degraded data redundancy: 29021/72 objects degraded (40306.944%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:49.383 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:49.403 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:49.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:49.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:49.506 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:49.531 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:49.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:49.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:49.635 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:49.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:49 smithi194 bash[92666]: cluster 2023-10-09T05:23:48.108526+0000 mgr.y (mgr.25277) 1065 : cluster 0 pgmap v1123: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 315 op/s; 29021/72 objects degraded (40306.944%); 4463/72 objects misplaced (6198.611%); 0 B/s, 91 objects/s recovering 2023-10-09T05:23:49.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:49 smithi194 bash[92666]: cluster 2023-10-09T05:23:48.578061+0000 mon.a (mon.0) 1394 : cluster 3 Health check update: Degraded data redundancy: 29021/72 objects degraded (40306.944%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:49.657 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:49.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:49.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:49.764 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:49.798 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:49.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:49.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:49.902 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:49.935 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:49.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:49.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:50.039 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:50.070 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:50.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:50.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:50.174 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:50.207 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:50.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:50.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:50.311 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:50.339 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:50.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:50.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:50.443 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:50.467 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:50.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:50.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:50.570 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:50.603 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:50.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:50.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:50.707 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:50.728 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:50.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:50.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:50.831 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:50.861 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:50.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:50.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:50.966 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:50.989 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:50.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:50.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:51.094 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:51.125 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:51.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:51.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:51.228 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:51.239 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:51 smithi133 bash[105589]: cluster 2023-10-09T05:23:50.109624+0000 mgr.y (mgr.25277) 1066 : cluster 0 pgmap v1124: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 261 op/s; 29021/72 objects degraded (40306.944%); 4463/72 objects misplaced (6198.611%); 0 B/s, 82 objects/s recovering 2023-10-09T05:23:51.239 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:51 smithi133 bash[102858]: cluster 2023-10-09T05:23:50.109624+0000 mgr.y (mgr.25277) 1066 : cluster 0 pgmap v1124: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 261 op/s; 29021/72 objects degraded (40306.944%); 4463/72 objects misplaced (6198.611%); 0 B/s, 82 objects/s recovering 2023-10-09T05:23:51.258 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:51.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:51.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:51.361 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:51.387 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:51.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:51.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:51.490 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:51.517 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:51.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:51.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:51.619 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:51.634 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:51 smithi194 bash[92666]: cluster 2023-10-09T05:23:50.109624+0000 mgr.y (mgr.25277) 1066 : cluster 0 pgmap v1124: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 261 op/s; 29021/72 objects degraded (40306.944%); 4463/72 objects misplaced (6198.611%); 0 B/s, 82 objects/s recovering 2023-10-09T05:23:51.657 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:51.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:51.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:51.761 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:51.787 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:51.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:51.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:51.891 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:51.924 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:51.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:51.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:52.028 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:52.057 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:52.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:52.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:52.161 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:52.193 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:52.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:52.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:52.297 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:52.327 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:52.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:52.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:52.430 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:52.451 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:52.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:52.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:52.554 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:52.576 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:52.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:52.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:52.681 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:52.706 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:52.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:52.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:52.809 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:52.840 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:52.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:52.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:52.944 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:52.965 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:52.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:52.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:53.069 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:53.103 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:53.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:53.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:53.206 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:53.236 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:53.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:53.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:53.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:53 smithi133 bash[102858]: cluster 2023-10-09T05:23:52.110546+0000 mgr.y (mgr.25277) 1067 : cluster 0 pgmap v1125: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 133 op/s; 28646/72 objects degraded (39786.111%); 4056/72 objects misplaced (5633.333%); 0 B/s, 147 objects/s recovering 2023-10-09T05:23:53.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:53 smithi133 bash[105589]: cluster 2023-10-09T05:23:52.110546+0000 mgr.y (mgr.25277) 1067 : cluster 0 pgmap v1125: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 133 op/s; 28646/72 objects degraded (39786.111%); 4056/72 objects misplaced (5633.333%); 0 B/s, 147 objects/s recovering 2023-10-09T05:23:53.340 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:53.352 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:53 smithi194 bash[92666]: cluster 2023-10-09T05:23:52.110546+0000 mgr.y (mgr.25277) 1067 : cluster 0 pgmap v1125: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 133 op/s; 28646/72 objects degraded (39786.111%); 4056/72 objects misplaced (5633.333%); 0 B/s, 147 objects/s recovering 2023-10-09T05:23:53.370 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:53.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:53.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:53.474 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:53.497 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:53.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:53.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:53.600 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:53.628 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:53.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:53.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:53.731 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:53.761 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:53.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:53.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:53.865 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:53.898 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:53.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:53.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:54.001 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:54.023 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:54.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:54.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:54.127 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:54.157 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:54.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:54.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:54.261 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:54.269 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:54 smithi133 bash[102858]: cluster 2023-10-09T05:23:53.578898+0000 mon.a (mon.0) 1395 : cluster 3 Health check update: Degraded data redundancy: 28646/72 objects degraded (39786.111%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:54.269 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:54 smithi133 bash[105589]: cluster 2023-10-09T05:23:53.578898+0000 mon.a (mon.0) 1395 : cluster 3 Health check update: Degraded data redundancy: 28646/72 objects degraded (39786.111%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:54.286 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:54.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:54.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:54.391 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:54.412 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:54.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:54.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:54.515 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:54.529 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:54 smithi194 bash[92666]: cluster 2023-10-09T05:23:53.578898+0000 mon.a (mon.0) 1395 : cluster 3 Health check update: Degraded data redundancy: 28646/72 objects degraded (39786.111%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:54.547 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:23:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:23:54] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.43.0" 2023-10-09T05:23:54.548 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:54.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:54.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:54.652 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:54.675 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:54.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:54.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:54.778 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:54.810 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:54.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:54.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:54.913 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:54.947 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:54.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:54.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:55.051 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:55.076 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:55.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:55.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:55.179 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:55.212 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:55.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:55.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:55.316 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:55.351 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:55.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:55.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:55.455 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:55.466 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:55 smithi133 bash[105589]: cluster 2023-10-09T05:23:54.111126+0000 mgr.y (mgr.25277) 1068 : cluster 0 pgmap v1126: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 74 op/s; 28646/72 objects degraded (39786.111%); 4056/72 objects misplaced (5633.333%); 0 B/s, 124 objects/s recovering 2023-10-09T05:23:55.467 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:55 smithi133 bash[102858]: cluster 2023-10-09T05:23:54.111126+0000 mgr.y (mgr.25277) 1068 : cluster 0 pgmap v1126: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 74 op/s; 28646/72 objects degraded (39786.111%); 4056/72 objects misplaced (5633.333%); 0 B/s, 124 objects/s recovering 2023-10-09T05:23:55.485 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:55.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:55.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:55.588 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:55.610 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:55.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:55.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:55.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:55 smithi194 bash[92666]: cluster 2023-10-09T05:23:54.111126+0000 mgr.y (mgr.25277) 1068 : cluster 0 pgmap v1126: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 74 op/s; 28646/72 objects degraded (39786.111%); 4056/72 objects misplaced (5633.333%); 0 B/s, 124 objects/s recovering 2023-10-09T05:23:55.714 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:55.736 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:55.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:55.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:55.840 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:55.874 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:55.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:55.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:55.977 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:56.001 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:56.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:56.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:56.104 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:56.136 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:56.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:56.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:56.239 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:56.263 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:56.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:56.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:56.366 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:56.392 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:56.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:56.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:56.495 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:56.530 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:56.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:56.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:56.633 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:56.660 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:56.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:56.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:56.762 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:56.795 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:56.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:56.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:56.899 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:56.921 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:56.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:56.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:57.024 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:57.056 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:57.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:57.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:57.161 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:57.184 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:57.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:57.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:57.288 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:57.301 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:57 smithi194 bash[92666]: cluster 2023-10-09T05:23:56.111979+0000 mgr.y (mgr.25277) 1069 : cluster 0 pgmap v1127: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 65 op/s; 28257/72 objects degraded (39245.833%); 4056/72 objects misplaced (5633.333%); 0 B/s, 156 objects/s recovering 2023-10-09T05:23:57.322 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:57.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:57.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:57.426 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:57.460 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:57.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:57.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:57.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:57 smithi133 bash[105589]: cluster 2023-10-09T05:23:56.111979+0000 mgr.y (mgr.25277) 1069 : cluster 0 pgmap v1127: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 65 op/s; 28257/72 objects degraded (39245.833%); 4056/72 objects misplaced (5633.333%); 0 B/s, 156 objects/s recovering 2023-10-09T05:23:57.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:57 smithi133 bash[102858]: cluster 2023-10-09T05:23:56.111979+0000 mgr.y (mgr.25277) 1069 : cluster 0 pgmap v1127: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 65 op/s; 28257/72 objects degraded (39245.833%); 4056/72 objects misplaced (5633.333%); 0 B/s, 156 objects/s recovering 2023-10-09T05:23:57.563 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:57.594 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:57.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:57.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:57.698 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:57.730 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:57.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:57.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:57.834 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:57.854 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:57.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:57.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:23:57.957 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:57.980 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:57.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:57.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:58.084 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:58.115 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:58.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:58.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:58.219 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:58.239 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:58.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:58.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:58.342 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:58.374 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:23:58.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:58.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:23:58.478 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:58.502 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:58.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:58.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:58.605 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:58.630 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:58.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:58.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:58.734 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:58.767 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:58.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:58.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:58.871 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:58.904 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:23:58.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:58.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:23:59.008 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:59.034 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:23:59.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:59.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:23:59.137 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:59.164 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:23:59.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:59.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:23:59.268 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:59.290 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:59.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:59.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:59.393 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:59.396 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:23:59.425 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:59.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:59.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:59.521 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:59 smithi133 bash[102858]: cluster 2023-10-09T05:23:58.112798+0000 mgr.y (mgr.25277) 1070 : cluster 0 pgmap v1128: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 28257/72 objects degraded (39245.833%); 3630/72 objects misplaced (5041.667%); 0 B/s, 165 objects/s recovering 2023-10-09T05:23:59.521 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:59 smithi133 bash[102858]: audit 2023-10-09T05:23:58.459642+0000 mon.a (mon.0) 1396 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:23:59.521 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:23:59 smithi133 bash[102858]: cluster 2023-10-09T05:23:58.579760+0000 mon.a (mon.0) 1397 : cluster 3 Health check update: Degraded data redundancy: 28257/72 objects degraded (39245.833%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:59.521 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:59 smithi133 bash[105589]: cluster 2023-10-09T05:23:58.112798+0000 mgr.y (mgr.25277) 1070 : cluster 0 pgmap v1128: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 28257/72 objects degraded (39245.833%); 3630/72 objects misplaced (5041.667%); 0 B/s, 165 objects/s recovering 2023-10-09T05:23:59.522 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:59 smithi133 bash[105589]: audit 2023-10-09T05:23:58.459642+0000 mon.a (mon.0) 1396 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:23:59.522 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:23:59 smithi133 bash[105589]: cluster 2023-10-09T05:23:58.579760+0000 mon.a (mon.0) 1397 : cluster 3 Health check update: Degraded data redundancy: 28257/72 objects degraded (39245.833%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:59.529 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:59.541 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:59 smithi194 bash[92666]: cluster 2023-10-09T05:23:58.112798+0000 mgr.y (mgr.25277) 1070 : cluster 0 pgmap v1128: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 28257/72 objects degraded (39245.833%); 3630/72 objects misplaced (5041.667%); 0 B/s, 165 objects/s recovering 2023-10-09T05:23:59.541 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:59 smithi194 bash[92666]: audit 2023-10-09T05:23:58.459642+0000 mon.a (mon.0) 1396 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:23:59.541 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:23:59 smithi194 bash[92666]: cluster 2023-10-09T05:23:58.579760+0000 mon.a (mon.0) 1397 : cluster 3 Health check update: Degraded data redundancy: 28257/72 objects degraded (39245.833%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:23:59.559 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:23:59.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:59.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:23:59.662 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:59.688 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:23:59.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:59.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:23:59.791 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:59.820 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:23:59.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:59.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:23:59.924 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:59.949 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:23:59.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:23:59.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:00.052 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:00.075 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:00.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:00.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:00.178 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:00.211 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:00.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:00.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:00.315 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:00.338 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:00.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:00.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:00.441 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:00.473 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:00.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:00.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:00.577 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:00.609 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:00.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:00.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:00.713 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:00.747 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:00.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:00.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:00.851 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:00.875 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:00.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:00.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:00.980 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:01.005 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:01.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:01.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:01.109 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:01.142 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:01.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:01.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:01.211 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:01 smithi133 bash[102858]: cluster 2023-10-09T05:24:00.113649+0000 mgr.y (mgr.25277) 1071 : cluster 0 pgmap v1129: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 28257/72 objects degraded (39245.833%); 3630/72 objects misplaced (5041.667%); 0 B/s, 133 objects/s recovering 2023-10-09T05:24:01.245 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:01.275 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:01.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:01.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:01.379 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:01.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:01 smithi194 bash[92666]: cluster 2023-10-09T05:24:00.113649+0000 mgr.y (mgr.25277) 1071 : cluster 0 pgmap v1129: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 28257/72 objects degraded (39245.833%); 3630/72 objects misplaced (5041.667%); 0 B/s, 133 objects/s recovering 2023-10-09T05:24:01.412 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:01.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:01.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:01.517 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:01.524 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:01 smithi133 bash[105589]: cluster 2023-10-09T05:24:00.113649+0000 mgr.y (mgr.25277) 1071 : cluster 0 pgmap v1129: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 28257/72 objects degraded (39245.833%); 3630/72 objects misplaced (5041.667%); 0 B/s, 133 objects/s recovering 2023-10-09T05:24:01.541 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:01.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:01.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:01.644 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:01.666 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:01.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:01.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:01.769 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:01.800 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:01.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:01.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:01.904 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:01.939 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:01.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:01.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:02.046 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:02.068 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:02.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:02.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:02.171 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:02.201 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:02.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:02.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:02.305 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:02.337 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:02.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:02.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:02.440 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:02.464 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:02.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:02.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:02.567 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:02.596 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:02.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:02.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:02.699 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:02.725 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:02.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:02.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:02.829 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:02.863 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:02.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:02.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:02.943 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:24:02.943 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:24:02.967 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:03.003 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:03.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:03.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:03.108 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:03.141 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:03.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:03.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:03.245 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:03.271 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:03.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:03.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:03.375 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:03.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:03 smithi194 bash[92666]: cluster 2023-10-09T05:24:02.114313+0000 mgr.y (mgr.25277) 1072 : cluster 0 pgmap v1130: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 27836/72 objects degraded (38661.111%); 3163/72 objects misplaced (4393.056%); 0 B/s, 206 objects/s recovering 2023-10-09T05:24:03.401 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:03.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:03.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:03.505 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:03.512 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:03 smithi133 bash[105589]: cluster 2023-10-09T05:24:02.114313+0000 mgr.y (mgr.25277) 1072 : cluster 0 pgmap v1130: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 27836/72 objects degraded (38661.111%); 3163/72 objects misplaced (4393.056%); 0 B/s, 206 objects/s recovering 2023-10-09T05:24:03.512 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:03 smithi133 bash[102858]: cluster 2023-10-09T05:24:02.114313+0000 mgr.y (mgr.25277) 1072 : cluster 0 pgmap v1130: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 27836/72 objects degraded (38661.111%); 3163/72 objects misplaced (4393.056%); 0 B/s, 206 objects/s recovering 2023-10-09T05:24:03.526 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:03.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:03.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:03.629 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:03.660 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:03.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:03.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:03.764 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:03.795 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:03.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:03.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:03.900 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:03.921 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:03.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:03.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:04.024 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:04.042 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:04.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:04.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:04.147 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:04.169 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:04.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:04.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:04.272 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:04.297 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:04.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:04.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:04.399 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:04.425 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:04.429 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:04 smithi133 bash[102858]: cluster 2023-10-09T05:24:03.580821+0000 mon.a (mon.0) 1398 : cluster 3 Health check update: Degraded data redundancy: 27836/72 objects degraded (38661.111%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:24:04.429 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:24:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:24:04] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.43.0" 2023-10-09T05:24:04.430 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:04 smithi133 bash[105589]: cluster 2023-10-09T05:24:03.580821+0000 mon.a (mon.0) 1398 : cluster 3 Health check update: Degraded data redundancy: 27836/72 objects degraded (38661.111%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:24:04.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:04.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:04.531 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:04.554 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:04.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:04.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:04.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:04 smithi194 bash[92666]: cluster 2023-10-09T05:24:03.580821+0000 mon.a (mon.0) 1398 : cluster 3 Health check update: Degraded data redundancy: 27836/72 objects degraded (38661.111%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:24:04.657 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:04.689 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:04.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:04.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:04.793 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:04.828 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:04.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:04.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:04.930 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:04.962 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:04.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:04.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:05.066 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:05.090 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:05.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:05.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:05.193 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:05.217 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:05 smithi133 bash[102858]: cluster 2023-10-09T05:24:04.114854+0000 mgr.y (mgr.25277) 1073 : cluster 0 pgmap v1131: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 27836/72 objects degraded (38661.111%); 3163/72 objects misplaced (4393.056%); 0 B/s, 141 objects/s recovering 2023-10-09T05:24:05.227 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:05.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:05.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:05.330 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:05.351 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:05.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:05.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:05.453 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:05.466 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:05 smithi194 bash[92666]: cluster 2023-10-09T05:24:04.114854+0000 mgr.y (mgr.25277) 1073 : cluster 0 pgmap v1131: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 27836/72 objects degraded (38661.111%); 3163/72 objects misplaced (4393.056%); 0 B/s, 141 objects/s recovering 2023-10-09T05:24:05.489 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:05.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:05.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:05.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:05 smithi133 bash[105589]: cluster 2023-10-09T05:24:04.114854+0000 mgr.y (mgr.25277) 1073 : cluster 0 pgmap v1131: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 27836/72 objects degraded (38661.111%); 3163/72 objects misplaced (4393.056%); 0 B/s, 141 objects/s recovering 2023-10-09T05:24:05.593 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:05.626 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:05.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:05.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:05.731 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:05.758 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:05.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:05.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:05.861 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:05.884 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:05.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:05.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:05.988 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:06.021 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:06.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:06.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:06.125 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:06.147 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:06.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:06.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:06.252 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:06.277 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:06.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:06.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:06.380 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:06.406 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:06.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:06.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:06.509 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:06.532 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:06.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:06.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:06.636 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:06.666 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:06.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:06.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:06.770 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:06.802 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:06.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:06.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:06.905 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:06.942 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:06.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:06.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:07.046 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:07.071 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:07.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:07.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:07.174 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:07.201 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:07.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:07.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:07.305 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:07.327 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:07 smithi133 bash[105589]: cluster 2023-10-09T05:24:06.115459+0000 mgr.y (mgr.25277) 1074 : cluster 0 pgmap v1132: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 27379/72 objects degraded (38026.389%); 3163/72 objects misplaced (4393.056%); 0 B/s, 179 objects/s recovering 2023-10-09T05:24:07.327 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:07 smithi133 bash[102858]: cluster 2023-10-09T05:24:06.115459+0000 mgr.y (mgr.25277) 1074 : cluster 0 pgmap v1132: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 27379/72 objects degraded (38026.389%); 3163/72 objects misplaced (4393.056%); 0 B/s, 179 objects/s recovering 2023-10-09T05:24:07.343 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:07.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:07.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:07.446 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:07.470 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:07.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:07.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:07.573 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:07.597 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:07.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:07.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:07.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:07 smithi194 bash[92666]: cluster 2023-10-09T05:24:06.115459+0000 mgr.y (mgr.25277) 1074 : cluster 0 pgmap v1132: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 27379/72 objects degraded (38026.389%); 3163/72 objects misplaced (4393.056%); 0 B/s, 179 objects/s recovering 2023-10-09T05:24:07.700 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:07.723 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:07.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:07.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:07.826 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:07.847 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:07.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:07.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:07.951 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:07.982 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:07.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:07.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:08.085 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:08.118 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:08.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:08.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:08.222 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:08.252 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:08.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:08.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:08.356 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:08.379 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:08.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:08.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:08.482 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:08.513 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:08.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:08.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:08.617 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:08.650 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:08.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:08.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:08.754 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:08.790 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:08.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:08.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:08.895 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:08.928 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:08.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:08.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:09.032 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:09.052 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:09.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:09.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:09.155 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:09.182 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:09.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:09.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:09.285 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:09.297 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:09 smithi133 bash[105589]: cluster 2023-10-09T05:24:08.116063+0000 mgr.y (mgr.25277) 1075 : cluster 0 pgmap v1133: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 27379/72 objects degraded (38026.389%); 2644/72 objects misplaced (3672.222%); 0 B/s, 190 objects/s recovering 2023-10-09T05:24:09.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:09 smithi133 bash[105589]: cluster 2023-10-09T05:24:08.581747+0000 mon.a (mon.0) 1399 : cluster 3 Health check update: Degraded data redundancy: 27379/72 objects degraded (38026.389%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:24:09.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:09 smithi133 bash[102858]: cluster 2023-10-09T05:24:08.116063+0000 mgr.y (mgr.25277) 1075 : cluster 0 pgmap v1133: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 27379/72 objects degraded (38026.389%); 2644/72 objects misplaced (3672.222%); 0 B/s, 190 objects/s recovering 2023-10-09T05:24:09.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:09 smithi133 bash[102858]: cluster 2023-10-09T05:24:08.581747+0000 mon.a (mon.0) 1399 : cluster 3 Health check update: Degraded data redundancy: 27379/72 objects degraded (38026.389%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:24:09.311 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:09.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:09.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:09.415 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:09.438 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:09.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:09.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:09.541 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:09.554 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:09 smithi194 bash[92666]: cluster 2023-10-09T05:24:08.116063+0000 mgr.y (mgr.25277) 1075 : cluster 0 pgmap v1133: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 27379/72 objects degraded (38026.389%); 2644/72 objects misplaced (3672.222%); 0 B/s, 190 objects/s recovering 2023-10-09T05:24:09.555 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:09 smithi194 bash[92666]: cluster 2023-10-09T05:24:08.581747+0000 mon.a (mon.0) 1399 : cluster 3 Health check update: Degraded data redundancy: 27379/72 objects degraded (38026.389%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:24:09.574 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:09.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:09.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:09.677 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:09.701 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:09.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:09.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:09.804 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:09.836 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:09.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:09.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:09.939 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:09.971 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:09.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:09.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:10.075 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:10.100 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:10.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:10.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:10.204 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:10.227 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:10.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:10.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:10.331 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:10.358 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:10.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:10.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:10.462 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:10.483 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:10.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:10.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:10.587 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:10.607 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:10.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:10.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:10.711 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:10.747 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:10.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:10.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:10.850 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:10.871 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:10.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:10.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:10.975 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:11.007 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:11.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:11.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:11.111 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:11.134 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:11.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:11.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:11.237 INFO:teuthology.orchestra.run.smithi133.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T05:24:11.241 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:11.254 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:11 smithi194 bash[92666]: cluster 2023-10-09T05:24:10.116705+0000 mgr.y (mgr.25277) 1076 : cluster 0 pgmap v1134: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 27379/72 objects degraded (38026.389%); 2644/72 objects misplaced (3672.222%); 0 B/s, 155 objects/s recovering 2023-10-09T05:24:11.254 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:11 smithi194 bash[92666]: audit 2023-10-09T05:24:11.172584+0000 mon.c (mon.2) 99 : audit 1 from='client.? 172.21.15.133:0/3690270402' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:24:11.254 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:11 smithi194 bash[92666]: audit 2023-10-09T05:24:11.173020+0000 mon.a (mon.0) 1400 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:24:11.270 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:11.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:11.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:11.374 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:11.406 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:11.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:11.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:11.510 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:11.543 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:11.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:11.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:11.547 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:11 smithi133 bash[105589]: cluster 2023-10-09T05:24:10.116705+0000 mgr.y (mgr.25277) 1076 : cluster 0 pgmap v1134: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 27379/72 objects degraded (38026.389%); 2644/72 objects misplaced (3672.222%); 0 B/s, 155 objects/s recovering 2023-10-09T05:24:11.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:11 smithi133 bash[105589]: audit 2023-10-09T05:24:11.172584+0000 mon.c (mon.2) 99 : audit 1 from='client.? 172.21.15.133:0/3690270402' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:24:11.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:11 smithi133 bash[105589]: audit 2023-10-09T05:24:11.173020+0000 mon.a (mon.0) 1400 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:24:11.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:11 smithi133 bash[102858]: cluster 2023-10-09T05:24:10.116705+0000 mgr.y (mgr.25277) 1076 : cluster 0 pgmap v1134: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 27379/72 objects degraded (38026.389%); 2644/72 objects misplaced (3672.222%); 0 B/s, 155 objects/s recovering 2023-10-09T05:24:11.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:11 smithi133 bash[102858]: audit 2023-10-09T05:24:11.172584+0000 mon.c (mon.2) 99 : audit 1 from='client.? 172.21.15.133:0/3690270402' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:24:11.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:11 smithi133 bash[102858]: audit 2023-10-09T05:24:11.173020+0000 mon.a (mon.0) 1400 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:24:11.647 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:11.668 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:11.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:11.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:11.771 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:11.803 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:11.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:11.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:11.906 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:11.933 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:11.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:11.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:12.037 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:12.070 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:12.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:12.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:12.175 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:12.196 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:12.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:12.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:12.299 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:12.311 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:12 smithi194 bash[92666]: cluster 2023-10-09T05:24:11.228649+0000 osd.5 (osd.5) 238 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T05:24:12.311 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:12 smithi194 bash[92666]: audit 2023-10-09T05:24:11.229971+0000 mon.a (mon.0) 1401 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:24:12.311 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:12 smithi194 bash[92666]: cluster 2023-10-09T05:24:11.230060+0000 mon.a (mon.0) 1402 : cluster 0 osdmap e256: 8 total, 8 up, 7 in 2023-10-09T05:24:12.312 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:12 smithi194 bash[92666]: cluster 2023-10-09T05:24:11.671232+0000 osd.7 (osd.7) 206 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T05:24:12.312 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:12 smithi194 bash[92666]: cluster 2023-10-09T05:24:11.671255+0000 osd.4 (osd.4) 226 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T05:24:12.312 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:12 smithi194 bash[92666]: cluster 2023-10-09T05:24:11.671324+0000 osd.6 (osd.6) 203 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T05:24:12.329 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:12.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:12.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:12.433 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:12.443 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:12 smithi133 bash[105589]: cluster 2023-10-09T05:24:11.228649+0000 osd.5 (osd.5) 238 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T05:24:12.443 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:12 smithi133 bash[105589]: audit 2023-10-09T05:24:11.229971+0000 mon.a (mon.0) 1401 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:24:12.443 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:12 smithi133 bash[105589]: cluster 2023-10-09T05:24:11.230060+0000 mon.a (mon.0) 1402 : cluster 0 osdmap e256: 8 total, 8 up, 7 in 2023-10-09T05:24:12.443 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:12 smithi133 bash[105589]: cluster 2023-10-09T05:24:11.671232+0000 osd.7 (osd.7) 206 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T05:24:12.443 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:12 smithi133 bash[105589]: cluster 2023-10-09T05:24:11.671255+0000 osd.4 (osd.4) 226 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T05:24:12.443 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:12 smithi133 bash[105589]: cluster 2023-10-09T05:24:11.671324+0000 osd.6 (osd.6) 203 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T05:24:12.444 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:12 smithi133 bash[102858]: cluster 2023-10-09T05:24:11.228649+0000 osd.5 (osd.5) 238 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T05:24:12.444 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:12 smithi133 bash[102858]: audit 2023-10-09T05:24:11.229971+0000 mon.a (mon.0) 1401 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:24:12.444 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:12 smithi133 bash[102858]: cluster 2023-10-09T05:24:11.230060+0000 mon.a (mon.0) 1402 : cluster 0 osdmap e256: 8 total, 8 up, 7 in 2023-10-09T05:24:12.444 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:12 smithi133 bash[102858]: cluster 2023-10-09T05:24:11.671232+0000 osd.7 (osd.7) 206 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T05:24:12.444 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:12 smithi133 bash[102858]: cluster 2023-10-09T05:24:11.671255+0000 osd.4 (osd.4) 226 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T05:24:12.444 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:12 smithi133 bash[102858]: cluster 2023-10-09T05:24:11.671324+0000 osd.6 (osd.6) 203 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T05:24:12.461 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:12.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:12.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:12.567 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:12.592 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:12.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:12.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:12.695 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:12.730 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:12.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:12.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:12.833 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:12.865 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:12.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:12.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:12.968 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:13.002 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:13.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:13.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:13.106 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:13.138 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:13.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:13.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:13.242 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:13.275 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:13.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:13.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:13.378 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:13.405 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:13.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:13.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:13.508 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:13.519 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:13 smithi133 bash[105589]: cluster 2023-10-09T05:24:12.117349+0000 mgr.y (mgr.25277) 1077 : cluster 0 pgmap v1136: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 26872/72 objects degraded (37322.222%); 2034/72 objects misplaced (2825.000%); 0 B/s, 209 objects/s recovering 2023-10-09T05:24:13.520 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:13 smithi133 bash[102858]: cluster 2023-10-09T05:24:12.117349+0000 mgr.y (mgr.25277) 1077 : cluster 0 pgmap v1136: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 26872/72 objects degraded (37322.222%); 2034/72 objects misplaced (2825.000%); 0 B/s, 209 objects/s recovering 2023-10-09T05:24:13.532 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:13.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:13.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:13.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:13 smithi194 bash[92666]: cluster 2023-10-09T05:24:12.117349+0000 mgr.y (mgr.25277) 1077 : cluster 0 pgmap v1136: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 26872/72 objects degraded (37322.222%); 2034/72 objects misplaced (2825.000%); 0 B/s, 209 objects/s recovering 2023-10-09T05:24:13.636 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:13.661 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:13.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:13.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:13.764 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:13.796 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:13.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:13.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:13.900 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:13.922 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:13.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:13.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:14.027 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:14.058 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:14.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:14.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:14.162 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:14.190 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:14.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:14.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:14.247 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:14 smithi133 bash[102858]: audit 2023-10-09T05:24:13.460262+0000 mon.a (mon.0) 1403 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:24:14.247 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:14 smithi133 bash[102858]: cluster 2023-10-09T05:24:13.582873+0000 mon.a (mon.0) 1404 : cluster 3 Health check update: Degraded data redundancy: 26872/72 objects degraded (37322.222%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:24:14.294 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:14.307 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:14 smithi194 bash[92666]: audit 2023-10-09T05:24:13.460262+0000 mon.a (mon.0) 1403 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:24:14.307 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:14 smithi194 bash[92666]: cluster 2023-10-09T05:24:13.582873+0000 mon.a (mon.0) 1404 : cluster 3 Health check update: Degraded data redundancy: 26872/72 objects degraded (37322.222%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:24:14.327 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:14.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:14.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:14.430 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:14.465 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:14.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:14.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:14.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:14 smithi133 bash[105589]: audit 2023-10-09T05:24:13.460262+0000 mon.a (mon.0) 1403 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:24:14.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:14 smithi133 bash[105589]: cluster 2023-10-09T05:24:13.582873+0000 mon.a (mon.0) 1404 : cluster 3 Health check update: Degraded data redundancy: 26872/72 objects degraded (37322.222%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:24:14.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:24:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:24:14] "GET /metrics HTTP/1.1" 200 36479 "" "Prometheus/2.43.0" 2023-10-09T05:24:14.568 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:14.602 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:14.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:14.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:14.705 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:14.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:14.743 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:14.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:14.843 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:14.875 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:14.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:14.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:14.978 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:14.999 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:15.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:15.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:15.102 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:15.128 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:15.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:15.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:15.231 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:15.261 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:15.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:15.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:15.364 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:15.392 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:15.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:15.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:15.495 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:15.508 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:15 smithi194 bash[92666]: cluster 2023-10-09T05:24:14.117844+0000 mgr.y (mgr.25277) 1078 : cluster 0 pgmap v1137: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 26872/72 objects degraded (37322.222%); 2034/72 objects misplaced (2825.000%); 0 B/s, 209 objects/s recovering 2023-10-09T05:24:15.526 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:15.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:15.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:15.547 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:15 smithi133 bash[105589]: cluster 2023-10-09T05:24:14.117844+0000 mgr.y (mgr.25277) 1078 : cluster 0 pgmap v1137: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 26872/72 objects degraded (37322.222%); 2034/72 objects misplaced (2825.000%); 0 B/s, 209 objects/s recovering 2023-10-09T05:24:15.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:15 smithi133 bash[102858]: cluster 2023-10-09T05:24:14.117844+0000 mgr.y (mgr.25277) 1078 : cluster 0 pgmap v1137: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 26872/72 objects degraded (37322.222%); 2034/72 objects misplaced (2825.000%); 0 B/s, 209 objects/s recovering 2023-10-09T05:24:15.629 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:15.660 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:15.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:15.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:15.764 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:15.794 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:15.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:15.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:15.898 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:15.922 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:15.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:15.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:16.025 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:16.051 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:16.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:16.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:16.155 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:16.186 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:16.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:16.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:16.270 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is set 2023-10-09T05:24:16.290 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:16.323 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:16.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:16.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:16.427 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:16.466 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:16.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:16.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:16.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:16 smithi133 bash[105589]: audit 2023-10-09T05:24:15.866296+0000 mon.a (mon.0) 1405 : audit 1 from='client.? 172.21.15.133:0/3506524525' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:24:16.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:16 smithi133 bash[102858]: audit 2023-10-09T05:24:15.866296+0000 mon.a (mon.0) 1405 : audit 1 from='client.? 172.21.15.133:0/3506524525' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:24:16.571 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:16.594 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:16.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:16.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:16.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:16 smithi194 bash[92666]: audit 2023-10-09T05:24:15.866296+0000 mon.a (mon.0) 1405 : audit 1 from='client.? 172.21.15.133:0/3506524525' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:24:16.699 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:16.732 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:16.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:16.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:16.835 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:16.869 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:16.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:16.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:16.973 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:17.004 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:17.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:17.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:17.108 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:17.130 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:17.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:17.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:17.233 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:17.261 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:17.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:17.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:17.366 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:17.399 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:17.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:17.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:17.503 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:17.513 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:17 smithi133 bash[105589]: cluster 2023-10-09T05:24:16.118729+0000 mgr.y (mgr.25277) 1079 : cluster 0 pgmap v1138: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 25876/72 objects degraded (35938.889%); 2034/72 objects misplaced (2825.000%); 0 B/s, 263 objects/s recovering 2023-10-09T05:24:17.513 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:17 smithi133 bash[105589]: cluster 2023-10-09T05:24:16.249914+0000 mon.a (mon.0) 1406 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:24:17.513 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:17 smithi133 bash[105589]: audit 2023-10-09T05:24:16.263462+0000 mon.a (mon.0) 1407 : audit 1 from='client.? 172.21.15.133:0/3506524525' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:24:17.513 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:17 smithi133 bash[105589]: cluster 2023-10-09T05:24:16.263531+0000 mon.a (mon.0) 1408 : cluster 0 osdmap e257: 8 total, 8 up, 7 in 2023-10-09T05:24:17.513 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:17 smithi133 bash[105589]: cluster 2023-10-09T05:24:16.539512+0000 osd.6 (osd.6) 204 : cluster 3 failed to encode map e257 with expected crc 2023-10-09T05:24:17.513 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:17 smithi133 bash[105589]: cluster 2023-10-09T05:24:16.542283+0000 osd.6 (osd.6) 205 : cluster 3 failed to encode map e257 with expected crc 2023-10-09T05:24:17.513 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:17 smithi133 bash[105589]: cluster 2023-10-09T05:24:16.923658+0000 osd.4 (osd.4) 227 : cluster 3 failed to encode map e257 with expected crc 2023-10-09T05:24:17.514 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:17 smithi133 bash[105589]: cluster 2023-10-09T05:24:16.926319+0000 osd.4 (osd.4) 228 : cluster 3 failed to encode map e257 with expected crc 2023-10-09T05:24:17.514 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:17 smithi133 bash[102858]: cluster 2023-10-09T05:24:16.118729+0000 mgr.y (mgr.25277) 1079 : cluster 0 pgmap v1138: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 25876/72 objects degraded (35938.889%); 2034/72 objects misplaced (2825.000%); 0 B/s, 263 objects/s recovering 2023-10-09T05:24:17.514 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:17 smithi133 bash[102858]: cluster 2023-10-09T05:24:16.249914+0000 mon.a (mon.0) 1406 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:24:17.514 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:17 smithi133 bash[102858]: audit 2023-10-09T05:24:16.263462+0000 mon.a (mon.0) 1407 : audit 1 from='client.? 172.21.15.133:0/3506524525' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:24:17.514 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:17 smithi133 bash[102858]: cluster 2023-10-09T05:24:16.263531+0000 mon.a (mon.0) 1408 : cluster 0 osdmap e257: 8 total, 8 up, 7 in 2023-10-09T05:24:17.514 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:17 smithi133 bash[102858]: cluster 2023-10-09T05:24:16.539512+0000 osd.6 (osd.6) 204 : cluster 3 failed to encode map e257 with expected crc 2023-10-09T05:24:17.514 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:17 smithi133 bash[102858]: cluster 2023-10-09T05:24:16.542283+0000 osd.6 (osd.6) 205 : cluster 3 failed to encode map e257 with expected crc 2023-10-09T05:24:17.515 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:17 smithi133 bash[102858]: cluster 2023-10-09T05:24:16.923658+0000 osd.4 (osd.4) 227 : cluster 3 failed to encode map e257 with expected crc 2023-10-09T05:24:17.515 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:17 smithi133 bash[102858]: cluster 2023-10-09T05:24:16.926319+0000 osd.4 (osd.4) 228 : cluster 3 failed to encode map e257 with expected crc 2023-10-09T05:24:17.528 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:17.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:17.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:17.631 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:17.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:17 smithi194 bash[92666]: cluster 2023-10-09T05:24:16.118729+0000 mgr.y (mgr.25277) 1079 : cluster 0 pgmap v1138: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 25876/72 objects degraded (35938.889%); 2034/72 objects misplaced (2825.000%); 0 B/s, 263 objects/s recovering 2023-10-09T05:24:17.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:17 smithi194 bash[92666]: cluster 2023-10-09T05:24:16.249914+0000 mon.a (mon.0) 1406 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:24:17.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:17 smithi194 bash[92666]: audit 2023-10-09T05:24:16.263462+0000 mon.a (mon.0) 1407 : audit 1 from='client.? 172.21.15.133:0/3506524525' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:24:17.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:17 smithi194 bash[92666]: cluster 2023-10-09T05:24:16.263531+0000 mon.a (mon.0) 1408 : cluster 0 osdmap e257: 8 total, 8 up, 7 in 2023-10-09T05:24:17.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:17 smithi194 bash[92666]: cluster 2023-10-09T05:24:16.539512+0000 osd.6 (osd.6) 204 : cluster 3 failed to encode map e257 with expected crc 2023-10-09T05:24:17.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:17 smithi194 bash[92666]: cluster 2023-10-09T05:24:16.542283+0000 osd.6 (osd.6) 205 : cluster 3 failed to encode map e257 with expected crc 2023-10-09T05:24:17.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:17 smithi194 bash[92666]: cluster 2023-10-09T05:24:16.923658+0000 osd.4 (osd.4) 227 : cluster 3 failed to encode map e257 with expected crc 2023-10-09T05:24:17.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:17 smithi194 bash[92666]: cluster 2023-10-09T05:24:16.926319+0000 osd.4 (osd.4) 228 : cluster 3 failed to encode map e257 with expected crc 2023-10-09T05:24:17.653 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:17.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:17.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:17.757 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:17.790 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:17.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:17.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:17.894 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:17.928 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:17.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:17.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:18.032 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:18.066 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:18.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:18.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:18.171 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:18.193 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:18.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:18.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:18.297 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:18.321 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:18.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:18.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:18.424 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:18.457 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:18.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:18.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:18.561 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:18.584 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:18.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:18.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:18.689 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:18.726 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:18.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:18.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:18.830 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:18.857 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:18.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:18.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:18.960 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:18.994 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:18.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:18.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:19.098 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:19.132 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:19.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:19.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:19.235 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:19.245 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T05:24:19.263 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:19.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:19.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:19.369 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:19.395 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:19.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:19.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:19.498 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:19.531 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:19.533 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:19 smithi194 bash[92666]: cluster 2023-10-09T05:24:18.119290+0000 mgr.y (mgr.25277) 1080 : cluster 0 pgmap v1140: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 25876/72 objects degraded (35938.889%); 1281/72 objects misplaced (1779.167%); 0 B/s, 358 objects/s recovering 2023-10-09T05:24:19.533 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:19 smithi194 bash[92666]: cluster 2023-10-09T05:24:18.583947+0000 mon.a (mon.0) 1409 : cluster 3 Health check update: Degraded data redundancy: 25876/72 objects degraded (35938.889%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:24:19.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:19.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:19.604 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:19 smithi133 bash[105589]: cluster 2023-10-09T05:24:18.119290+0000 mgr.y (mgr.25277) 1080 : cluster 0 pgmap v1140: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 25876/72 objects degraded (35938.889%); 1281/72 objects misplaced (1779.167%); 0 B/s, 358 objects/s recovering 2023-10-09T05:24:19.605 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:19 smithi133 bash[105589]: cluster 2023-10-09T05:24:18.583947+0000 mon.a (mon.0) 1409 : cluster 3 Health check update: Degraded data redundancy: 25876/72 objects degraded (35938.889%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:24:19.605 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:19 smithi133 bash[102858]: cluster 2023-10-09T05:24:18.119290+0000 mgr.y (mgr.25277) 1080 : cluster 0 pgmap v1140: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 25876/72 objects degraded (35938.889%); 1281/72 objects misplaced (1779.167%); 0 B/s, 358 objects/s recovering 2023-10-09T05:24:19.605 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:19 smithi133 bash[102858]: cluster 2023-10-09T05:24:18.583947+0000 mon.a (mon.0) 1409 : cluster 3 Health check update: Degraded data redundancy: 25876/72 objects degraded (35938.889%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:24:19.634 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:19.659 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:19.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:19.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:19.763 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:19.789 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:19.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:19.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:19.892 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:19.928 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:19.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:19.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:20.032 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:20.063 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:20.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:20.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:20.167 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:20.199 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:20.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:20.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:20.303 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:20.338 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:20.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:20.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:20.443 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:20.474 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:20.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:20.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:20.578 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:20.607 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:20.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:20.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:20.711 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:20.736 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:20.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:20.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:20.840 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:20.874 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:20.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:20.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:20.978 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:21.010 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:21.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:21.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:21.114 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:21.141 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T05:24:21.141 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:24:21.141 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:21.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:21.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:21.244 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:21.275 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:21.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:21.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:21.378 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:21.409 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:21.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:21.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:21.494 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:21 smithi133 bash[102858]: cluster 2023-10-09T05:24:20.119975+0000 mgr.y (mgr.25277) 1081 : cluster 0 pgmap v1141: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 25876/72 objects degraded (35938.889%); 1281/72 objects misplaced (1779.167%); 0 B/s, 265 objects/s recovering 2023-10-09T05:24:21.494 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:21 smithi133 bash[105589]: cluster 2023-10-09T05:24:20.119975+0000 mgr.y (mgr.25277) 1081 : cluster 0 pgmap v1141: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 25876/72 objects degraded (35938.889%); 1281/72 objects misplaced (1779.167%); 0 B/s, 265 objects/s recovering 2023-10-09T05:24:21.514 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:21.539 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:21.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:21.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:21.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:21 smithi194 bash[92666]: cluster 2023-10-09T05:24:20.119975+0000 mgr.y (mgr.25277) 1081 : cluster 0 pgmap v1141: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 25876/72 objects degraded (35938.889%); 1281/72 objects misplaced (1779.167%); 0 B/s, 265 objects/s recovering 2023-10-09T05:24:21.643 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:21.668 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:21.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:21.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:21.757 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:24:21.772 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:21.788 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:24:21.806 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:21.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:21.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:21.814 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:24:21.838 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:24:21.866 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:24:21.896 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:24:21.909 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:21.927 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:24:21.935 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:21.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:21.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:21.959 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:24:21.961 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:24:21.961 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:24:21.961 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:24:21.961 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:24:21.961 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:24:21.962 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:24:21.962 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:24:21.962 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:24:22.038 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:22.070 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:22.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:22.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:22.175 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:22.210 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:22.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:22.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:22.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:22 smithi133 bash[102858]: audit 2023-10-09T05:24:21.729269+0000 mon.a (mon.0) 1410 : audit 0 from='client.? 172.21.15.133:0/4264623880' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:24:22.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:22 smithi133 bash[105589]: audit 2023-10-09T05:24:21.729269+0000 mon.a (mon.0) 1410 : audit 0 from='client.? 172.21.15.133:0/4264623880' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:24:22.313 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:22.336 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:22.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:22.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:22.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:22 smithi194 bash[92666]: audit 2023-10-09T05:24:21.729269+0000 mon.a (mon.0) 1410 : audit 0 from='client.? 172.21.15.133:0/4264623880' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:24:22.439 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:22.470 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:22.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:22.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:22.573 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:22.606 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:22.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:22.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:22.709 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:22.737 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:22.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:22.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:22.840 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:22.865 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:22.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:22.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:22.969 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:23.001 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:23.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:23.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:23.105 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:23.132 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:23.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:23.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:23.169 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:24:23.236 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:23.259 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:23.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:23.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:23.362 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:23.389 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:23.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:23.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:23.493 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:23.502 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:23 smithi133 bash[105589]: cluster 2023-10-09T05:24:22.120729+0000 mgr.y (mgr.25277) 1082 : cluster 0 pgmap v1142: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 24801/72 objects degraded (34445.833%); 363/72 objects misplaced (504.167%); 0 B/s, 374 objects/s recovering 2023-10-09T05:24:23.503 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:23 smithi133 bash[102858]: cluster 2023-10-09T05:24:22.120729+0000 mgr.y (mgr.25277) 1082 : cluster 0 pgmap v1142: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 24801/72 objects degraded (34445.833%); 363/72 objects misplaced (504.167%); 0 B/s, 374 objects/s recovering 2023-10-09T05:24:23.520 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:23.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:23.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:23.624 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:23.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:23 smithi194 bash[92666]: cluster 2023-10-09T05:24:22.120729+0000 mgr.y (mgr.25277) 1082 : cluster 0 pgmap v1142: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 24801/72 objects degraded (34445.833%); 363/72 objects misplaced (504.167%); 0 B/s, 374 objects/s recovering 2023-10-09T05:24:23.657 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:23.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:23.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:23.761 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:23.783 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:23.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:23.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:23.887 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:23.916 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:23.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:23.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:24.019 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:24.050 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:24.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:24.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:24.153 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:24.186 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:24.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:24.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:24.289 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:24.315 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:24.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:24.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:24.420 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:24.442 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:24.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:24.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:24.545 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:24.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:24:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:24:24] "GET /metrics HTTP/1.1" 200 36479 "" "Prometheus/2.43.0" 2023-10-09T05:24:24.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:24 smithi133 bash[105589]: cluster 2023-10-09T05:24:23.283878+0000 osd.7 (osd.7) 207 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T05:24:24.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:24 smithi133 bash[105589]: cluster 2023-10-09T05:24:23.285174+0000 mon.a (mon.0) 1411 : cluster 0 osdmap e258: 8 total, 8 up, 7 in 2023-10-09T05:24:24.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:24 smithi133 bash[105589]: cluster 2023-10-09T05:24:23.287267+0000 osd.5 (osd.5) 239 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T05:24:24.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:24 smithi133 bash[105589]: cluster 2023-10-09T05:24:23.295813+0000 osd.4 (osd.4) 229 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T05:24:24.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:24 smithi133 bash[105589]: cluster 2023-10-09T05:24:23.295834+0000 osd.6 (osd.6) 206 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T05:24:24.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:24 smithi133 bash[105589]: cluster 2023-10-09T05:24:23.585147+0000 mon.a (mon.0) 1412 : cluster 3 Health check update: Degraded data redundancy: 24801/72 objects degraded (34445.833%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:24:24.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:24 smithi133 bash[102858]: cluster 2023-10-09T05:24:23.283878+0000 osd.7 (osd.7) 207 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T05:24:24.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:24 smithi133 bash[102858]: cluster 2023-10-09T05:24:23.285174+0000 mon.a (mon.0) 1411 : cluster 0 osdmap e258: 8 total, 8 up, 7 in 2023-10-09T05:24:24.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:24 smithi133 bash[102858]: cluster 2023-10-09T05:24:23.287267+0000 osd.5 (osd.5) 239 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T05:24:24.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:24 smithi133 bash[102858]: cluster 2023-10-09T05:24:23.295813+0000 osd.4 (osd.4) 229 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T05:24:24.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:24 smithi133 bash[102858]: cluster 2023-10-09T05:24:23.295834+0000 osd.6 (osd.6) 206 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T05:24:24.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:24 smithi133 bash[102858]: cluster 2023-10-09T05:24:23.585147+0000 mon.a (mon.0) 1412 : cluster 3 Health check update: Degraded data redundancy: 24801/72 objects degraded (34445.833%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:24:24.567 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:24.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:24.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:24.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:24 smithi194 bash[92666]: cluster 2023-10-09T05:24:23.283878+0000 osd.7 (osd.7) 207 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T05:24:24.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:24 smithi194 bash[92666]: cluster 2023-10-09T05:24:23.285174+0000 mon.a (mon.0) 1411 : cluster 0 osdmap e258: 8 total, 8 up, 7 in 2023-10-09T05:24:24.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:24 smithi194 bash[92666]: cluster 2023-10-09T05:24:23.287267+0000 osd.5 (osd.5) 239 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T05:24:24.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:24 smithi194 bash[92666]: cluster 2023-10-09T05:24:23.295813+0000 osd.4 (osd.4) 229 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T05:24:24.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:24 smithi194 bash[92666]: cluster 2023-10-09T05:24:23.295834+0000 osd.6 (osd.6) 206 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T05:24:24.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:24 smithi194 bash[92666]: cluster 2023-10-09T05:24:23.585147+0000 mon.a (mon.0) 1412 : cluster 3 Health check update: Degraded data redundancy: 24801/72 objects degraded (34445.833%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:24:24.672 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:24.694 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:24.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:24.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:24.797 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:24.828 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:24.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:24.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:24.932 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:24.959 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:24.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:24.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:25.062 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:25.092 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:25.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:25.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:25.195 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:25.230 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:25.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:25.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:25.334 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:25.346 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:25 smithi194 bash[92666]: cluster 2023-10-09T05:24:24.121377+0000 mgr.y (mgr.25277) 1083 : cluster 0 pgmap v1144: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 24801/72 objects degraded (34445.833%); 363/72 objects misplaced (504.167%); 0 B/s, 343 objects/s recovering 2023-10-09T05:24:25.346 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:25 smithi194 bash[92666]: cluster 2023-10-09T05:24:24.284548+0000 osd.5 (osd.5) 240 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T05:24:25.346 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:25 smithi194 bash[92666]: cluster 2023-10-09T05:24:24.286505+0000 mon.a (mon.0) 1413 : cluster 0 osdmap e259: 8 total, 8 up, 7 in 2023-10-09T05:24:25.346 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:25 smithi194 bash[92666]: cluster 2023-10-09T05:24:24.291583+0000 osd.4 (osd.4) 230 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T05:24:25.346 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:25 smithi194 bash[92666]: cluster 2023-10-09T05:24:24.291653+0000 osd.6 (osd.6) 207 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T05:24:25.347 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:25 smithi194 bash[92666]: cluster 2023-10-09T05:24:24.312916+0000 osd.7 (osd.7) 208 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T05:24:25.368 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:25.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:25.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:25.471 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:25.495 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:25.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:25.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:25.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:25 smithi133 bash[105589]: cluster 2023-10-09T05:24:24.121377+0000 mgr.y (mgr.25277) 1083 : cluster 0 pgmap v1144: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 24801/72 objects degraded (34445.833%); 363/72 objects misplaced (504.167%); 0 B/s, 343 objects/s recovering 2023-10-09T05:24:25.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:25 smithi133 bash[105589]: cluster 2023-10-09T05:24:24.284548+0000 osd.5 (osd.5) 240 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T05:24:25.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:25 smithi133 bash[105589]: cluster 2023-10-09T05:24:24.286505+0000 mon.a (mon.0) 1413 : cluster 0 osdmap e259: 8 total, 8 up, 7 in 2023-10-09T05:24:25.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:25 smithi133 bash[105589]: cluster 2023-10-09T05:24:24.291583+0000 osd.4 (osd.4) 230 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T05:24:25.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:25 smithi133 bash[105589]: cluster 2023-10-09T05:24:24.291653+0000 osd.6 (osd.6) 207 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T05:24:25.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:25 smithi133 bash[105589]: cluster 2023-10-09T05:24:24.312916+0000 osd.7 (osd.7) 208 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T05:24:25.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:25 smithi133 bash[102858]: cluster 2023-10-09T05:24:24.121377+0000 mgr.y (mgr.25277) 1083 : cluster 0 pgmap v1144: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 24801/72 objects degraded (34445.833%); 363/72 objects misplaced (504.167%); 0 B/s, 343 objects/s recovering 2023-10-09T05:24:25.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:25 smithi133 bash[102858]: cluster 2023-10-09T05:24:24.284548+0000 osd.5 (osd.5) 240 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T05:24:25.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:25 smithi133 bash[102858]: cluster 2023-10-09T05:24:24.286505+0000 mon.a (mon.0) 1413 : cluster 0 osdmap e259: 8 total, 8 up, 7 in 2023-10-09T05:24:25.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:25 smithi133 bash[102858]: cluster 2023-10-09T05:24:24.291583+0000 osd.4 (osd.4) 230 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T05:24:25.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:25 smithi133 bash[102858]: cluster 2023-10-09T05:24:24.291653+0000 osd.6 (osd.6) 207 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T05:24:25.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:25 smithi133 bash[102858]: cluster 2023-10-09T05:24:24.312916+0000 osd.7 (osd.7) 208 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T05:24:25.600 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:25.628 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:25.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:25.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:25.731 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:25.766 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:25.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:25.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:25.873 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:25.895 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:25.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:25.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:25.999 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:26.032 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:26.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:26.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:26.135 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:26.162 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:26.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:26.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:26.266 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:26.297 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:26.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:26.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:26.401 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:26.438 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:26.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:26.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:26.541 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:26.572 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:26.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:26.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:26.676 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:26.701 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:26.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:26.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:26.792 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:24:26.805 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:26.831 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:26.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:26.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:26.934 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:26.966 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:26.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:26.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:27.070 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:27.094 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:27.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:27.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:27.197 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:27.228 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:27.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:27.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:27.304 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:24:27.332 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:27.361 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:27.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:27.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:27.464 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:27.474 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:27 smithi133 bash[105589]: cluster 2023-10-09T05:24:26.122263+0000 mgr.y (mgr.25277) 1084 : cluster 0 pgmap v1146: 89 pgs: 1 active+recovering+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 23104/72 objects degraded (32088.889%); 363/72 objects misplaced (504.167%); 0 B/s, 387 objects/s recovering 2023-10-09T05:24:27.477 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:27 smithi133 bash[102858]: cluster 2023-10-09T05:24:26.122263+0000 mgr.y (mgr.25277) 1084 : cluster 0 pgmap v1146: 89 pgs: 1 active+recovering+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 23104/72 objects degraded (32088.889%); 363/72 objects misplaced (504.167%); 0 B/s, 387 objects/s recovering 2023-10-09T05:24:27.488 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:27.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:27.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:27.592 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:27.603 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:27 smithi194 bash[92666]: cluster 2023-10-09T05:24:26.122263+0000 mgr.y (mgr.25277) 1084 : cluster 0 pgmap v1146: 89 pgs: 1 active+recovering+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 23104/72 objects degraded (32088.889%); 363/72 objects misplaced (504.167%); 0 B/s, 387 objects/s recovering 2023-10-09T05:24:27.624 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:27.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:27.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:27.728 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:27.752 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:27.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:27.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:27.809 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:24:27.856 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:27.876 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:27.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:27.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:27.979 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:28.004 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:28.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:28.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:28.106 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:28.136 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:28.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:28.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:28.239 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:28.261 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:28.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:28.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:28.365 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:28.396 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:28.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:28.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:28.500 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:28.530 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:28.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:28.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:28.633 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:28.659 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:28.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:28.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:28.763 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:28.790 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:28.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:28.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:28.894 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:28.919 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:28.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:28.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:29.022 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:29.051 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:29.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:29.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:29.155 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:29.186 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:29.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:29.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:29.289 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:29.316 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:29.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:29.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:29.421 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:29.432 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:29 smithi133 bash[102858]: cluster 2023-10-09T05:24:28.123212+0000 mgr.y (mgr.25277) 1085 : cluster 0 pgmap v1147: 89 pgs: 1 active+recovering+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 23104/72 objects degraded (32088.889%); 363/72 objects misplaced (504.167%); 0 B/s, 387 objects/s recovering 2023-10-09T05:24:29.432 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:29 smithi133 bash[102858]: cluster 2023-10-09T05:24:28.310380+0000 mon.a (mon.0) 1414 : cluster 0 osdmap e260: 8 total, 8 up, 7 in 2023-10-09T05:24:29.432 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:29 smithi133 bash[102858]: cluster 2023-10-09T05:24:28.311179+0000 osd.5 (osd.5) 241 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T05:24:29.432 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:29 smithi133 bash[102858]: cluster 2023-10-09T05:24:28.318106+0000 osd.6 (osd.6) 208 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T05:24:29.432 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:29 smithi133 bash[102858]: cluster 2023-10-09T05:24:28.336837+0000 osd.7 (osd.7) 209 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T05:24:29.432 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:29 smithi133 bash[102858]: audit 2023-10-09T05:24:28.460819+0000 mon.a (mon.0) 1415 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:24:29.433 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:29 smithi133 bash[102858]: cluster 2023-10-09T05:24:28.586439+0000 mon.a (mon.0) 1416 : cluster 3 Health check update: Degraded data redundancy: 23104/72 objects degraded (32088.889%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:24:29.433 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:29 smithi133 bash[102858]: cluster 2023-10-09T05:24:28.622018+0000 osd.4 (osd.4) 231 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T05:24:29.433 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:29 smithi133 bash[105589]: cluster 2023-10-09T05:24:28.123212+0000 mgr.y (mgr.25277) 1085 : cluster 0 pgmap v1147: 89 pgs: 1 active+recovering+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 23104/72 objects degraded (32088.889%); 363/72 objects misplaced (504.167%); 0 B/s, 387 objects/s recovering 2023-10-09T05:24:29.433 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:29 smithi133 bash[105589]: cluster 2023-10-09T05:24:28.310380+0000 mon.a (mon.0) 1414 : cluster 0 osdmap e260: 8 total, 8 up, 7 in 2023-10-09T05:24:29.433 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:29 smithi133 bash[105589]: cluster 2023-10-09T05:24:28.311179+0000 osd.5 (osd.5) 241 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T05:24:29.433 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:29 smithi133 bash[105589]: cluster 2023-10-09T05:24:28.318106+0000 osd.6 (osd.6) 208 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T05:24:29.433 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:29 smithi133 bash[105589]: cluster 2023-10-09T05:24:28.336837+0000 osd.7 (osd.7) 209 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T05:24:29.433 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:29 smithi133 bash[105589]: audit 2023-10-09T05:24:28.460819+0000 mon.a (mon.0) 1415 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:24:29.434 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:29 smithi133 bash[105589]: cluster 2023-10-09T05:24:28.586439+0000 mon.a (mon.0) 1416 : cluster 3 Health check update: Degraded data redundancy: 23104/72 objects degraded (32088.889%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:24:29.434 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:29 smithi133 bash[105589]: cluster 2023-10-09T05:24:28.622018+0000 osd.4 (osd.4) 231 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T05:24:29.445 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:29.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:29.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:29.549 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:29.571 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:29.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:29.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:29 smithi194 bash[92666]: cluster 2023-10-09T05:24:28.123212+0000 mgr.y (mgr.25277) 1085 : cluster 0 pgmap v1147: 89 pgs: 1 active+recovering+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 23104/72 objects degraded (32088.889%); 363/72 objects misplaced (504.167%); 0 B/s, 387 objects/s recovering 2023-10-09T05:24:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:29 smithi194 bash[92666]: cluster 2023-10-09T05:24:28.310380+0000 mon.a (mon.0) 1414 : cluster 0 osdmap e260: 8 total, 8 up, 7 in 2023-10-09T05:24:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:29 smithi194 bash[92666]: cluster 2023-10-09T05:24:28.311179+0000 osd.5 (osd.5) 241 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T05:24:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:29 smithi194 bash[92666]: cluster 2023-10-09T05:24:28.318106+0000 osd.6 (osd.6) 208 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T05:24:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:29 smithi194 bash[92666]: cluster 2023-10-09T05:24:28.336837+0000 osd.7 (osd.7) 209 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T05:24:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:29 smithi194 bash[92666]: audit 2023-10-09T05:24:28.460819+0000 mon.a (mon.0) 1415 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:24:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:29 smithi194 bash[92666]: cluster 2023-10-09T05:24:28.586439+0000 mon.a (mon.0) 1416 : cluster 3 Health check update: Degraded data redundancy: 23104/72 objects degraded (32088.889%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:24:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:29 smithi194 bash[92666]: cluster 2023-10-09T05:24:28.622018+0000 osd.4 (osd.4) 231 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T05:24:29.675 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:29.701 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:29.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:29.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:29.805 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:29.832 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:29.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:29.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:29.841 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:24:29.935 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:29.972 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:29.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:29.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:30.076 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:30.110 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:30.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:30.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:30.214 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:30.244 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:30.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:30.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:30.347 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:30.380 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:30.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:30.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:30.483 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:30.502 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:30.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:30.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:30.606 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:30.613 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:30 smithi133 bash[102858]: cluster 2023-10-09T05:24:29.308048+0000 osd.6 (osd.6) 209 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T05:24:30.614 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:30 smithi133 bash[102858]: cluster 2023-10-09T05:24:29.310047+0000 mon.a (mon.0) 1417 : cluster 0 osdmap e261: 8 total, 8 up, 7 in 2023-10-09T05:24:30.614 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:30 smithi133 bash[102858]: cluster 2023-10-09T05:24:29.332392+0000 osd.7 (osd.7) 210 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T05:24:30.614 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:30 smithi133 bash[102858]: cluster 2023-10-09T05:24:29.726633+0000 osd.4 (osd.4) 232 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T05:24:30.614 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:30 smithi133 bash[102858]: cluster 2023-10-09T05:24:29.729435+0000 osd.4 (osd.4) 233 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T05:24:30.614 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:30 smithi133 bash[105589]: cluster 2023-10-09T05:24:29.308048+0000 osd.6 (osd.6) 209 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T05:24:30.614 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:30 smithi133 bash[105589]: cluster 2023-10-09T05:24:29.310047+0000 mon.a (mon.0) 1417 : cluster 0 osdmap e261: 8 total, 8 up, 7 in 2023-10-09T05:24:30.614 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:30 smithi133 bash[105589]: cluster 2023-10-09T05:24:29.332392+0000 osd.7 (osd.7) 210 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T05:24:30.614 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:30 smithi133 bash[105589]: cluster 2023-10-09T05:24:29.726633+0000 osd.4 (osd.4) 232 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T05:24:30.614 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:30 smithi133 bash[105589]: cluster 2023-10-09T05:24:29.729435+0000 osd.4 (osd.4) 233 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T05:24:30.627 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:30.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:30.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:30.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:30 smithi194 bash[92666]: cluster 2023-10-09T05:24:29.308048+0000 osd.6 (osd.6) 209 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T05:24:30.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:30 smithi194 bash[92666]: cluster 2023-10-09T05:24:29.310047+0000 mon.a (mon.0) 1417 : cluster 0 osdmap e261: 8 total, 8 up, 7 in 2023-10-09T05:24:30.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:30 smithi194 bash[92666]: cluster 2023-10-09T05:24:29.332392+0000 osd.7 (osd.7) 210 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T05:24:30.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:30 smithi194 bash[92666]: cluster 2023-10-09T05:24:29.726633+0000 osd.4 (osd.4) 232 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T05:24:30.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:30 smithi194 bash[92666]: cluster 2023-10-09T05:24:29.729435+0000 osd.4 (osd.4) 233 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T05:24:30.730 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:30.760 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:30.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:30.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:30.863 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:30.895 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:30.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:30.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:30.998 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:31.030 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:31.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:31.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:31.134 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:31.163 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:31.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:31.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:31.267 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:31.292 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:31.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:31.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:31.354 INFO:teuthology.orchestra.run.smithi133.stderr:pool 'unique_pool_6' removed 2023-10-09T05:24:31.396 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:31.455 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:31.455 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:31 smithi133 bash[102858]: cluster 2023-10-09T05:24:30.123964+0000 mgr.y (mgr.25277) 1086 : cluster 0 pgmap v1150: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 22428/72 objects degraded (31150.000%); 0 B/s, 455 objects/s recovering 2023-10-09T05:24:31.455 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:31 smithi133 bash[102858]: audit 2023-10-09T05:24:30.478269+0000 mon.a (mon.0) 1418 : audit 1 from='client.? 172.21.15.133:0/1493525876' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:24:31.456 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:31 smithi133 bash[105589]: cluster 2023-10-09T05:24:30.123964+0000 mgr.y (mgr.25277) 1086 : cluster 0 pgmap v1150: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 22428/72 objects degraded (31150.000%); 0 B/s, 455 objects/s recovering 2023-10-09T05:24:31.456 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:31 smithi133 bash[105589]: audit 2023-10-09T05:24:30.478269+0000 mon.a (mon.0) 1418 : audit 1 from='client.? 172.21.15.133:0/1493525876' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:24:31.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:31.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:31.558 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:31.579 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:31.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:31.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:31.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:31 smithi194 bash[92666]: cluster 2023-10-09T05:24:30.123964+0000 mgr.y (mgr.25277) 1086 : cluster 0 pgmap v1150: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 22428/72 objects degraded (31150.000%); 0 B/s, 455 objects/s recovering 2023-10-09T05:24:31.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:31 smithi194 bash[92666]: audit 2023-10-09T05:24:30.478269+0000 mon.a (mon.0) 1418 : audit 1 from='client.? 172.21.15.133:0/1493525876' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:24:31.682 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:31.711 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:31.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:31.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:31.816 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:31.835 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:31.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:31.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:31.938 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:31.967 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:31.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:31.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:32.071 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:32.098 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:32.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:32.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:32.201 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:32.219 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:32.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:32.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:32.321 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:32.345 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:32.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:32.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:32.448 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:32.482 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:32.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:32.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:32.523 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:32 smithi133 bash[105589]: cluster 2023-10-09T05:24:31.329867+0000 osd.5 (osd.5) 242 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T05:24:32.523 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:32 smithi133 bash[105589]: audit 2023-10-09T05:24:31.332212+0000 mon.a (mon.0) 1419 : audit 1 from='client.? 172.21.15.133:0/1493525876' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:24:32.523 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:32 smithi133 bash[105589]: cluster 2023-10-09T05:24:31.332299+0000 mon.a (mon.0) 1420 : cluster 0 osdmap e262: 8 total, 8 up, 7 in 2023-10-09T05:24:32.524 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:32 smithi133 bash[105589]: cluster 2023-10-09T05:24:31.376074+0000 osd.7 (osd.7) 211 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T05:24:32.524 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:32 smithi133 bash[105589]: cluster 2023-10-09T05:24:31.376090+0000 osd.6 (osd.6) 210 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T05:24:32.524 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:32 smithi133 bash[105589]: cluster 2023-10-09T05:24:31.377212+0000 osd.4 (osd.4) 234 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T05:24:32.524 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:32 smithi133 bash[105589]: audit 2023-10-09T05:24:31.917584+0000 mon.a (mon.0) 1421 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:24:32.524 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:32 smithi133 bash[102858]: cluster 2023-10-09T05:24:31.329867+0000 osd.5 (osd.5) 242 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T05:24:32.524 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:32 smithi133 bash[102858]: audit 2023-10-09T05:24:31.332212+0000 mon.a (mon.0) 1419 : audit 1 from='client.? 172.21.15.133:0/1493525876' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:24:32.524 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:32 smithi133 bash[102858]: cluster 2023-10-09T05:24:31.332299+0000 mon.a (mon.0) 1420 : cluster 0 osdmap e262: 8 total, 8 up, 7 in 2023-10-09T05:24:32.524 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:32 smithi133 bash[102858]: cluster 2023-10-09T05:24:31.376074+0000 osd.7 (osd.7) 211 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T05:24:32.524 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:32 smithi133 bash[102858]: cluster 2023-10-09T05:24:31.376090+0000 osd.6 (osd.6) 210 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T05:24:32.524 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:32 smithi133 bash[102858]: cluster 2023-10-09T05:24:31.377212+0000 osd.4 (osd.4) 234 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T05:24:32.525 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:32 smithi133 bash[102858]: audit 2023-10-09T05:24:31.917584+0000 mon.a (mon.0) 1421 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:24:32.586 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:32.616 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:32.618 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:32 smithi194 bash[92666]: cluster 2023-10-09T05:24:31.329867+0000 osd.5 (osd.5) 242 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T05:24:32.618 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:32 smithi194 bash[92666]: audit 2023-10-09T05:24:31.332212+0000 mon.a (mon.0) 1419 : audit 1 from='client.? 172.21.15.133:0/1493525876' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:24:32.619 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:32 smithi194 bash[92666]: cluster 2023-10-09T05:24:31.332299+0000 mon.a (mon.0) 1420 : cluster 0 osdmap e262: 8 total, 8 up, 7 in 2023-10-09T05:24:32.619 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:32 smithi194 bash[92666]: cluster 2023-10-09T05:24:31.376074+0000 osd.7 (osd.7) 211 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T05:24:32.619 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:32 smithi194 bash[92666]: cluster 2023-10-09T05:24:31.376090+0000 osd.6 (osd.6) 210 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T05:24:32.619 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:32 smithi194 bash[92666]: cluster 2023-10-09T05:24:31.377212+0000 osd.4 (osd.4) 234 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T05:24:32.619 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:32 smithi194 bash[92666]: audit 2023-10-09T05:24:31.917584+0000 mon.a (mon.0) 1421 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:24:32.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:32.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:32.721 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:32.753 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:32.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:32.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:32.858 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:32.898 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:32.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:32.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:33.003 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:33.043 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:33.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:33.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:33.148 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:33.179 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:33.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:33.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:33.282 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:33.316 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:33.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:33.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:33.420 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:33.445 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:33.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:33.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:33.549 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:33.573 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:33.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:33.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:33.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:33 smithi194 bash[92666]: cluster 2023-10-09T05:24:32.124337+0000 mgr.y (mgr.25277) 1087 : cluster 0 pgmap v1152: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail 2023-10-09T05:24:33.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:33 smithi194 bash[92666]: cluster 2023-10-09T05:24:32.342183+0000 mon.a (mon.0) 1422 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 22428/72 objects degraded (31150.000%), 4 pgs degraded, 4 pgs undersized) 2023-10-09T05:24:33.677 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:33.709 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:33.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:33.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:33.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:33 smithi133 bash[105589]: cluster 2023-10-09T05:24:32.124337+0000 mgr.y (mgr.25277) 1087 : cluster 0 pgmap v1152: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail 2023-10-09T05:24:33.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:33 smithi133 bash[105589]: cluster 2023-10-09T05:24:32.342183+0000 mon.a (mon.0) 1422 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 22428/72 objects degraded (31150.000%), 4 pgs degraded, 4 pgs undersized) 2023-10-09T05:24:33.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:33 smithi133 bash[102858]: cluster 2023-10-09T05:24:32.124337+0000 mgr.y (mgr.25277) 1087 : cluster 0 pgmap v1152: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail 2023-10-09T05:24:33.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:33 smithi133 bash[102858]: cluster 2023-10-09T05:24:32.342183+0000 mon.a (mon.0) 1422 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 22428/72 objects degraded (31150.000%), 4 pgs degraded, 4 pgs undersized) 2023-10-09T05:24:33.813 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:33.835 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:33.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:33.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:33.939 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:33.971 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:33.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:33.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:34.076 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:34.101 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:34.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:34.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:34.204 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:34.229 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:34.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:34.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:34.332 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:34.340 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:24:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:24:34] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.43.0" 2023-10-09T05:24:34.354 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:34.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:34.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:34.457 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:34.488 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:34.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:34.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:34.591 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:34.599 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:34 smithi133 bash[105589]: cluster 2023-10-09T05:24:34.124958+0000 mgr.y (mgr.25277) 1088 : cluster 0 pgmap v1153: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail 2023-10-09T05:24:34.617 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:34.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:34.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:34.720 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:34.743 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:34.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:34.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:34.846 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:34.854 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:34 smithi194 bash[92666]: cluster 2023-10-09T05:24:34.124958+0000 mgr.y (mgr.25277) 1088 : cluster 0 pgmap v1153: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail 2023-10-09T05:24:34.866 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:34 smithi133 bash[102858]: cluster 2023-10-09T05:24:34.124958+0000 mgr.y (mgr.25277) 1088 : cluster 0 pgmap v1153: 73 pgs: 73 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail 2023-10-09T05:24:34.867 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T05:24:34.868 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T05:24:34.868 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_7 2023-10-09T05:24:34.868 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_7 16 2023-10-09T05:24:34.869 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:34.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:34.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:34.972 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:34.998 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:35.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:35.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:35.103 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:35.126 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:35.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:35.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:35.229 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:35.251 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:35.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:35.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:35.354 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:35.390 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:35.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:35.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:35.494 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:35.521 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:35.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:35.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:35.626 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:35.648 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:35.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:35.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:35.751 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:35.773 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:35.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:35.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:35.876 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:35.901 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:35.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:35.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:36.005 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:36.045 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:36.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:36.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:36.149 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:36.173 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:36.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:36.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:36.277 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:36.306 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:36.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:36.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:36.410 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:36.437 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:36.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:36.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:36.540 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:36.575 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:36.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:36.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:36.679 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:36.708 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:36.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:36.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:36.812 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:36.841 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:36.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:36.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:36.945 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:36.972 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:36.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:36.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:37.075 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:37.100 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:37.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:37.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:37.204 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:37.251 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:37.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:37.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:37.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:37 smithi133 bash[102858]: cluster 2023-10-09T05:24:36.125748+0000 mgr.y (mgr.25277) 1089 : cluster 0 pgmap v1154: 73 pgs: 73 active+clean; 582 KiB data, 7.3 GiB used, 618 GiB / 626 GiB avail 2023-10-09T05:24:37.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:37 smithi133 bash[105589]: cluster 2023-10-09T05:24:36.125748+0000 mgr.y (mgr.25277) 1089 : cluster 0 pgmap v1154: 73 pgs: 73 active+clean; 582 KiB data, 7.3 GiB used, 618 GiB / 626 GiB avail 2023-10-09T05:24:37.355 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:37.365 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:37 smithi194 bash[92666]: cluster 2023-10-09T05:24:36.125748+0000 mgr.y (mgr.25277) 1089 : cluster 0 pgmap v1154: 73 pgs: 73 active+clean; 582 KiB data, 7.3 GiB used, 618 GiB / 626 GiB avail 2023-10-09T05:24:37.380 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:37.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:37.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:37.483 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:37.512 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:37.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:37.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:37.616 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:37.647 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:37.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:37.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:37.751 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:37.780 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:37.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:37.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:37.854 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:24:37.912 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:37.932 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:37.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:37.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:38.036 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:38.065 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:38.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:38.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:38.169 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:38.193 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:38.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:38.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:38.296 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:38.318 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:38.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:38.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:38.418 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:24:38.419 INFO:teuthology.orchestra.run.smithi133.stdout: "ops": [], 2023-10-09T05:24:38.419 INFO:teuthology.orchestra.run.smithi133.stdout: "num_ops": 0 2023-10-09T05:24:38.419 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:24:38.421 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:38.452 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:38.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:38.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:38.556 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:38.582 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:38.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:38.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:38.685 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:38.717 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:38.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:38.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:38.821 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:38.854 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:38.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:38.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:38.958 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:38.983 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:38.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:38.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:39.086 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:39.117 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:39.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:39.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:39.220 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:39.256 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:39.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:39.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:39.360 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:39.383 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:39.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:39.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:39.486 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:39.510 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:39.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:39.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:39.614 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:39.625 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:39 smithi194 bash[92666]: cluster 2023-10-09T05:24:38.126259+0000 mgr.y (mgr.25277) 1090 : cluster 0 pgmap v1155: 73 pgs: 73 active+clean; 582 KiB data, 7.3 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:39.625 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:39 smithi194 bash[92666]: audit 2023-10-09T05:24:38.352911+0000 mon.a (mon.0) 1423 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:24:39.625 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:39 smithi194 bash[92666]: audit 2023-10-09T05:24:38.358337+0000 mon.a (mon.0) 1424 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:24:39.625 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:39 smithi194 bash[92666]: audit 2023-10-09T05:24:39.043727+0000 mon.a (mon.0) 1425 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:24:39.642 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:39.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:39.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:39.747 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:39.773 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:39.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:39.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:39.792 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:39 smithi133 bash[105589]: cluster 2023-10-09T05:24:38.126259+0000 mgr.y (mgr.25277) 1090 : cluster 0 pgmap v1155: 73 pgs: 73 active+clean; 582 KiB data, 7.3 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:39.792 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:39 smithi133 bash[105589]: audit 2023-10-09T05:24:38.352911+0000 mon.a (mon.0) 1423 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:24:39.792 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:39 smithi133 bash[105589]: audit 2023-10-09T05:24:38.358337+0000 mon.a (mon.0) 1424 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:24:39.792 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:39 smithi133 bash[105589]: audit 2023-10-09T05:24:39.043727+0000 mon.a (mon.0) 1425 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:24:39.792 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:39 smithi133 bash[102858]: cluster 2023-10-09T05:24:38.126259+0000 mgr.y (mgr.25277) 1090 : cluster 0 pgmap v1155: 73 pgs: 73 active+clean; 582 KiB data, 7.3 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:39.792 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:39 smithi133 bash[102858]: audit 2023-10-09T05:24:38.352911+0000 mon.a (mon.0) 1423 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:24:39.792 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:39 smithi133 bash[102858]: audit 2023-10-09T05:24:38.358337+0000 mon.a (mon.0) 1424 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:24:39.792 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:39 smithi133 bash[102858]: audit 2023-10-09T05:24:39.043727+0000 mon.a (mon.0) 1425 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:24:39.877 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:39.911 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:39.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:39.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:40.015 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:40.044 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:40.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:40.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:40.148 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:40.181 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:40.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:40.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:40.286 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:40.313 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:40.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:40.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:40.416 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:40.446 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:40.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:40.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:40.550 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:40.581 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:40.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:40.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:40.679 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T05:24:40.684 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:40.716 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:40.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:40.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:40.819 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:40.845 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:40.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:40.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:40.949 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:40.974 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:40.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:40.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:41.079 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:41.106 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:41.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:41.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:41.150 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:24:41.211 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:41.238 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:41.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:41.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:41.341 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:41.371 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:41.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:41.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:41.474 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:41.485 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:41 smithi133 bash[105589]: cluster 2023-10-09T05:24:40.127401+0000 mgr.y (mgr.25277) 1091 : cluster 0 pgmap v1156: 73 pgs: 73 active+clean; 582 KiB data, 7.3 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:41.485 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:41 smithi133 bash[102858]: cluster 2023-10-09T05:24:40.127401+0000 mgr.y (mgr.25277) 1091 : cluster 0 pgmap v1156: 73 pgs: 73 active+clean; 582 KiB data, 7.3 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:41.499 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:41.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:41.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:41.602 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:41.615 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:41 smithi194 bash[92666]: cluster 2023-10-09T05:24:40.127401+0000 mgr.y (mgr.25277) 1091 : cluster 0 pgmap v1156: 73 pgs: 73 active+clean; 582 KiB data, 7.3 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:41.634 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:41.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:41.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:41.738 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:41.769 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:41.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:41.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:41.873 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:41.906 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:41.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:41.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:42.010 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:42.039 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:42.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:42.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:42.074 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:24:42.075 INFO:teuthology.orchestra.run.smithi133.stderr:dumped all 2023-10-09T05:24:42.142 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:42.161 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:42.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:42.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:42.265 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:42.287 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:42.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:42.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:42.375 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is unset 2023-10-09T05:24:42.391 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:42.413 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:42.414 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:42 smithi133 bash[105589]: audit 2023-10-09T05:24:42.140917+0000 mon.c (mon.2) 100 : audit 1 from='client.? 172.21.15.133:0/2983575283' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:24:42.415 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:42 smithi133 bash[105589]: audit 2023-10-09T05:24:42.141287+0000 mon.a (mon.0) 1426 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:24:42.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:42.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:42.416 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:42 smithi133 bash[102858]: audit 2023-10-09T05:24:42.140917+0000 mon.c (mon.2) 100 : audit 1 from='client.? 172.21.15.133:0/2983575283' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:24:42.416 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:42 smithi133 bash[102858]: audit 2023-10-09T05:24:42.141287+0000 mon.a (mon.0) 1426 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:24:42.517 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:42.534 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:42.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:42.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:42.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:42 smithi194 bash[92666]: audit 2023-10-09T05:24:42.140917+0000 mon.c (mon.2) 100 : audit 1 from='client.? 172.21.15.133:0/2983575283' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:24:42.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:42 smithi194 bash[92666]: audit 2023-10-09T05:24:42.141287+0000 mon.a (mon.0) 1426 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:24:42.638 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:42.670 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:42.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:42.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:42.774 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:42.792 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:42.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:42.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:42.895 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:42.914 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:42.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:42.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:43.017 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:43.039 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:43.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:43.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:43.142 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:43.167 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:43.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:43.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:43.270 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:43.305 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:43.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:43.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:43.408 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:43.443 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:43.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:43.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:43.547 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:43.572 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:43 smithi194 bash[92666]: audit 2023-10-09T05:24:42.061793+0000 mgr.y (mgr.25277) 1092 : audit 0 from='client.45323 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:24:43.572 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:43 smithi194 bash[92666]: cluster 2023-10-09T05:24:42.128027+0000 mgr.y (mgr.25277) 1093 : cluster 0 pgmap v1157: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:43.572 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:43 smithi194 bash[92666]: cluster 2023-10-09T05:24:42.357099+0000 mon.a (mon.0) 1427 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:24:43.572 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:43 smithi194 bash[92666]: cluster 2023-10-09T05:24:42.368299+0000 osd.5 (osd.5) 243 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T05:24:43.572 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:43 smithi194 bash[92666]: audit 2023-10-09T05:24:42.370624+0000 mon.a (mon.0) 1428 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:24:43.572 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:43 smithi194 bash[92666]: cluster 2023-10-09T05:24:42.370702+0000 mon.a (mon.0) 1429 : cluster 0 osdmap e263: 8 total, 8 up, 7 in 2023-10-09T05:24:43.572 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:43 smithi194 bash[92666]: audit 2023-10-09T05:24:42.417135+0000 mon.a (mon.0) 1430 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:24:43.572 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:43 smithi194 bash[92666]: audit 2023-10-09T05:24:42.422917+0000 mon.a (mon.0) 1431 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:24:43.572 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:43 smithi194 bash[92666]: audit 2023-10-09T05:24:42.841186+0000 mon.a (mon.0) 1432 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:24:43.572 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:43 smithi194 bash[92666]: audit 2023-10-09T05:24:42.842216+0000 mon.a (mon.0) 1433 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:24:43.573 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:43 smithi194 bash[92666]: audit 2023-10-09T05:24:42.847458+0000 mon.a (mon.0) 1434 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:24:43.587 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:43.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:43.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:43.691 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:43.710 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:43 smithi133 bash[102858]: audit 2023-10-09T05:24:42.061793+0000 mgr.y (mgr.25277) 1092 : audit 0 from='client.45323 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:24:43.710 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:43 smithi133 bash[102858]: cluster 2023-10-09T05:24:42.128027+0000 mgr.y (mgr.25277) 1093 : cluster 0 pgmap v1157: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:43.710 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:43 smithi133 bash[102858]: cluster 2023-10-09T05:24:42.357099+0000 mon.a (mon.0) 1427 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:24:43.710 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:43 smithi133 bash[102858]: cluster 2023-10-09T05:24:42.368299+0000 osd.5 (osd.5) 243 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T05:24:43.710 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:43 smithi133 bash[102858]: audit 2023-10-09T05:24:42.370624+0000 mon.a (mon.0) 1428 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:24:43.710 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:43 smithi133 bash[102858]: cluster 2023-10-09T05:24:42.370702+0000 mon.a (mon.0) 1429 : cluster 0 osdmap e263: 8 total, 8 up, 7 in 2023-10-09T05:24:43.711 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:43 smithi133 bash[102858]: audit 2023-10-09T05:24:42.417135+0000 mon.a (mon.0) 1430 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:24:43.711 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:43 smithi133 bash[102858]: audit 2023-10-09T05:24:42.422917+0000 mon.a (mon.0) 1431 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:24:43.711 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:43 smithi133 bash[102858]: audit 2023-10-09T05:24:42.841186+0000 mon.a (mon.0) 1432 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:24:43.711 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:43 smithi133 bash[102858]: audit 2023-10-09T05:24:42.842216+0000 mon.a (mon.0) 1433 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:24:43.711 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:43 smithi133 bash[102858]: audit 2023-10-09T05:24:42.847458+0000 mon.a (mon.0) 1434 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:24:43.711 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:43 smithi133 bash[105589]: audit 2023-10-09T05:24:42.061793+0000 mgr.y (mgr.25277) 1092 : audit 0 from='client.45323 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:24:43.711 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:43 smithi133 bash[105589]: cluster 2023-10-09T05:24:42.128027+0000 mgr.y (mgr.25277) 1093 : cluster 0 pgmap v1157: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:43.711 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:43 smithi133 bash[105589]: cluster 2023-10-09T05:24:42.357099+0000 mon.a (mon.0) 1427 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:24:43.712 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:43 smithi133 bash[105589]: cluster 2023-10-09T05:24:42.368299+0000 osd.5 (osd.5) 243 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T05:24:43.712 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:43 smithi133 bash[105589]: audit 2023-10-09T05:24:42.370624+0000 mon.a (mon.0) 1428 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:24:43.712 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:43 smithi133 bash[105589]: cluster 2023-10-09T05:24:42.370702+0000 mon.a (mon.0) 1429 : cluster 0 osdmap e263: 8 total, 8 up, 7 in 2023-10-09T05:24:43.712 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:43 smithi133 bash[105589]: audit 2023-10-09T05:24:42.417135+0000 mon.a (mon.0) 1430 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:24:43.712 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:43 smithi133 bash[105589]: audit 2023-10-09T05:24:42.422917+0000 mon.a (mon.0) 1431 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:24:43.712 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:43 smithi133 bash[105589]: audit 2023-10-09T05:24:42.841186+0000 mon.a (mon.0) 1432 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:24:43.712 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:43 smithi133 bash[105589]: audit 2023-10-09T05:24:42.842216+0000 mon.a (mon.0) 1433 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:24:43.712 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:43 smithi133 bash[105589]: audit 2023-10-09T05:24:42.847458+0000 mon.a (mon.0) 1434 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:24:43.726 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:43.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:43.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:43.829 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:43.861 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:43.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:43.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:43.965 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:43.995 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:43.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:43.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:44.098 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:44.125 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:44.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:44.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:44.229 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:44.261 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:44.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:44.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:44.365 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:44.398 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:44.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:44.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:44.501 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:44.516 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:44 smithi194 bash[92666]: audit 2023-10-09T05:24:43.466469+0000 mon.a (mon.0) 1435 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:24:44.516 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:44 smithi194 bash[92666]: audit 2023-10-09T05:24:43.467512+0000 mon.a (mon.0) 1436 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:24:44.516 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:44 smithi194 bash[92666]: cluster 2023-10-09T05:24:44.128668+0000 mgr.y (mgr.25277) 1094 : cluster 0 pgmap v1159: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:44.534 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:44.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:44.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:44.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:24:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:24:44] "GET /metrics HTTP/1.1" 200 34060 "" "Prometheus/2.43.0" 2023-10-09T05:24:44.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:44 smithi133 bash[102858]: audit 2023-10-09T05:24:43.466469+0000 mon.a (mon.0) 1435 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:24:44.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:44 smithi133 bash[102858]: audit 2023-10-09T05:24:43.467512+0000 mon.a (mon.0) 1436 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:24:44.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:44 smithi133 bash[102858]: cluster 2023-10-09T05:24:44.128668+0000 mgr.y (mgr.25277) 1094 : cluster 0 pgmap v1159: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:44.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:44 smithi133 bash[105589]: audit 2023-10-09T05:24:43.466469+0000 mon.a (mon.0) 1435 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:24:44.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:44 smithi133 bash[105589]: audit 2023-10-09T05:24:43.467512+0000 mon.a (mon.0) 1436 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:24:44.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:44 smithi133 bash[105589]: cluster 2023-10-09T05:24:44.128668+0000 mgr.y (mgr.25277) 1094 : cluster 0 pgmap v1159: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:44.637 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:44.670 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:44.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:44.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:44.775 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:44.815 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:44.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:44.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:44.920 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:44.947 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:44.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:44.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:45.051 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:45.083 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:45.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:45.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:45.186 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:45.219 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:45.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:45.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:45.323 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:45.358 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:45.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:45.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:45.462 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:45.488 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:45.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:45.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:45.591 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:45.617 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:45.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:45.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:45.720 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:45.754 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:45.756 INFO:teuthology.orchestra.run.smithi133.stdout:{"pg_ready":true,"pg_map":{"version":1156,"stamp":"2023-10-09T05:24:40.126640+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":506,"num_read_kb":931,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":4154034,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":222,"ondisk_log_size":222,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":235,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":7620580,"kb_used_data":1956652,"kb_used_omap":20,"kb_used_meta":5663851,"kb_avail":648595484,"statfs":{"total":671965249536,"available":664161775616,"internally_reserved":0,"allocated":2003611648,"data_stored":1995320303,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":5799784078},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":17,"apply_latency_ms":17,"commit_latency_ns":17000000,"apply_latency_ns":17000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-22428,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-191836,"num_write_kb":-6138689,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-25681,"num_bytes_recovered":-849412096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003182"},"pg_stats":[{"pgid":"3.1a","version":"69'2","reported_seq":262,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.732937+0000","last_change":"2023-10-09T05:09:57.873320+0000","last_active":"2023-10-09T05:24:29.732937+0000","last_peered":"2023-10-09T05:24:29.732937+0000","last_clean":"2023-10-09T05:24:29.732937+0000","last_became_active":"2023-10-09T05:09:57.868376+0000","last_became_peered":"2023-10-09T05:09:57.868376+0000","last_unstale":"2023-10-09T05:24:29.732937+0000","last_undegraded":"2023-10-09T05:24:29.732937+0000","last_fullsized":"2023-10-09T05:24:29.732937+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:27.186610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":244,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-09T05:24:31.332424+0000","last_change":"2023-10-09T05:14:15.345991+0000","last_active":"2023-10-09T05:24:31.332424+0000","last_peered":"2023-10-09T05:24:31.332424+0000","last_clean":"2023-10-09T05:24:31.332424+0000","last_became_active":"2023-10-09T05:14:15.345511+0000","last_became_peered":"2023-10-09T05:14:15.345511+0000","last_unstale":"2023-10-09T05:24:31.332424+0000","last_undegraded":"2023-10-09T05:24:31.332424+0000","last_fullsized":"2023-10-09T05:24:31.332424+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:49.234025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":127,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-09T05:24:31.334379+0000","last_change":"2023-10-09T05:09:57.857511+0000","last_active":"2023-10-09T05:24:31.334379+0000","last_peered":"2023-10-09T05:24:31.334379+0000","last_clean":"2023-10-09T05:24:31.334379+0000","last_became_active":"2023-10-09T05:09:57.857193+0000","last_became_peered":"2023-10-09T05:09:57.857193+0000","last_unstale":"2023-10-09T05:24:31.334379+0000","last_undegraded":"2023-10-09T05:24:31.334379+0000","last_fullsized":"2023-10-09T05:24:31.334379+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:32.842980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":260,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.317283+0000","last_change":"2023-10-09T05:18:26.422923+0000","last_active":"2023-10-09T05:24:29.317283+0000","last_peered":"2023-10-09T05:24:29.317283+0000","last_clean":"2023-10-09T05:24:29.317283+0000","last_became_active":"2023-10-09T05:18:26.422508+0000","last_became_peered":"2023-10-09T05:18:26.422508+0000","last_unstale":"2023-10-09T05:24:29.317283+0000","last_undegraded":"2023-10-09T05:24:29.317283+0000","last_fullsized":"2023-10-09T05:24:29.317283+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:00.876037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":280,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.312547+0000","last_change":"2023-10-09T05:23:18.188378+0000","last_active":"2023-10-09T05:24:29.312547+0000","last_peered":"2023-10-09T05:24:29.312547+0000","last_clean":"2023-10-09T05:24:29.312547+0000","last_became_active":"2023-10-09T05:23:18.186629+0000","last_became_peered":"2023-10-09T05:23:18.186629+0000","last_unstale":"2023-10-09T05:24:29.312547+0000","last_undegraded":"2023-10-09T05:24:29.312547+0000","last_fullsized":"2023-10-09T05:24:29.312547+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:16.444225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":252,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.314013+0000","last_change":"2023-10-09T05:13:03.039637+0000","last_active":"2023-10-09T05:24:29.314013+0000","last_peered":"2023-10-09T05:24:29.314013+0000","last_clean":"2023-10-09T05:24:29.314013+0000","last_became_active":"2023-10-09T05:13:03.039268+0000","last_became_peered":"2023-10-09T05:13:03.039268+0000","last_unstale":"2023-10-09T05:24:29.314013+0000","last_undegraded":"2023-10-09T05:24:29.314013+0000","last_fullsized":"2023-10-09T05:24:29.314013+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:19.875767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":248,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.317765+0000","last_change":"2023-10-09T05:22:47.216769+0000","last_active":"2023-10-09T05:24:29.317765+0000","last_peered":"2023-10-09T05:24:29.317765+0000","last_clean":"2023-10-09T05:24:29.317765+0000","last_became_active":"2023-10-09T05:22:47.215847+0000","last_became_peered":"2023-10-09T05:22:47.215847+0000","last_unstale":"2023-10-09T05:24:29.317765+0000","last_undegraded":"2023-10-09T05:24:29.317765+0000","last_fullsized":"2023-10-09T05:24:29.317765+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:51.392675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":235,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.314160+0000","last_change":"2023-10-09T05:09:57.849695+0000","last_active":"2023-10-09T05:24:29.314160+0000","last_peered":"2023-10-09T05:24:29.314160+0000","last_clean":"2023-10-09T05:24:29.314160+0000","last_became_active":"2023-10-09T05:09:57.848478+0000","last_became_peered":"2023-10-09T05:09:57.848478+0000","last_unstale":"2023-10-09T05:24:29.314160+0000","last_undegraded":"2023-10-09T05:24:29.314160+0000","last_fullsized":"2023-10-09T05:24:29.314160+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:26.115312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":71,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.338280+0000","last_change":"2023-10-09T05:23:18.194144+0000","last_active":"2023-10-09T05:24:29.338280+0000","last_peered":"2023-10-09T05:24:29.338280+0000","last_clean":"2023-10-09T05:24:29.338280+0000","last_became_active":"2023-10-09T05:23:18.193390+0000","last_became_peered":"2023-10-09T05:23:18.193390+0000","last_unstale":"2023-10-09T05:24:29.338280+0000","last_undegraded":"2023-10-09T05:24:29.338280+0000","last_fullsized":"2023-10-09T05:24:29.338280+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:22.549362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":248,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.313604+0000","last_change":"2023-10-09T05:18:26.418431+0000","last_active":"2023-10-09T05:24:29.313604+0000","last_peered":"2023-10-09T05:24:29.313604+0000","last_clean":"2023-10-09T05:24:29.313604+0000","last_became_active":"2023-10-09T05:18:26.418100+0000","last_became_peered":"2023-10-09T05:18:26.418100+0000","last_unstale":"2023-10-09T05:24:29.313604+0000","last_undegraded":"2023-10-09T05:24:29.313604+0000","last_fullsized":"2023-10-09T05:24:29.313604+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:21.241286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":57,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.338104+0000","last_change":"2023-10-09T05:18:26.452552+0000","last_active":"2023-10-09T05:24:29.338104+0000","last_peered":"2023-10-09T05:24:29.338104+0000","last_clean":"2023-10-09T05:24:29.338104+0000","last_became_active":"2023-10-09T05:18:26.428454+0000","last_became_peered":"2023-10-09T05:18:26.428454+0000","last_unstale":"2023-10-09T05:24:29.338104+0000","last_undegraded":"2023-10-09T05:24:29.338104+0000","last_fullsized":"2023-10-09T05:24:29.338104+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:15:52.203861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":20,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.311623+0000","last_change":"2023-10-09T05:23:18.199181+0000","last_active":"2023-10-09T05:24:29.311623+0000","last_peered":"2023-10-09T05:24:29.311623+0000","last_clean":"2023-10-09T05:24:29.311623+0000","last_became_active":"2023-10-09T05:23:18.198401+0000","last_became_peered":"2023-10-09T05:23:18.198401+0000","last_unstale":"2023-10-09T05:24:29.311623+0000","last_undegraded":"2023-10-09T05:24:29.311623+0000","last_fullsized":"2023-10-09T05:24:29.311623+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:55:49.891420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'23","reported_seq":314,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.733444+0000","last_change":"2023-10-09T05:18:26.436165+0000","last_active":"2023-10-09T05:24:29.733444+0000","last_peered":"2023-10-09T05:24:29.733444+0000","last_clean":"2023-10-09T05:24:29.733444+0000","last_became_active":"2023-10-09T05:18:26.430078+0000","last_became_peered":"2023-10-09T05:18:26.430078+0000","last_unstale":"2023-10-09T05:24:29.733444+0000","last_undegraded":"2023-10-09T05:24:29.733444+0000","last_fullsized":"2023-10-09T05:24:29.733444+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:04:48.797166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":242,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.732941+0000","last_change":"2023-10-09T05:09:57.868761+0000","last_active":"2023-10-09T05:24:29.732941+0000","last_peered":"2023-10-09T05:24:29.732941+0000","last_clean":"2023-10-09T05:24:29.732941+0000","last_became_active":"2023-10-09T05:09:57.868121+0000","last_became_peered":"2023-10-09T05:09:57.868121+0000","last_unstale":"2023-10-09T05:24:29.732941+0000","last_undegraded":"2023-10-09T05:24:29.732941+0000","last_fullsized":"2023-10-09T05:24:29.732941+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:43.412235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":57,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.337633+0000","last_change":"2023-10-09T05:18:26.443719+0000","last_active":"2023-10-09T05:24:29.337633+0000","last_peered":"2023-10-09T05:24:29.337633+0000","last_clean":"2023-10-09T05:24:29.337633+0000","last_became_active":"2023-10-09T05:18:26.426718+0000","last_became_peered":"2023-10-09T05:18:26.426718+0000","last_unstale":"2023-10-09T05:24:29.337633+0000","last_undegraded":"2023-10-09T05:24:29.337633+0000","last_fullsized":"2023-10-09T05:24:29.337633+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:49:12.090862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":289,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.313735+0000","last_change":"2023-10-09T05:23:18.187593+0000","last_active":"2023-10-09T05:24:29.313735+0000","last_peered":"2023-10-09T05:24:29.313735+0000","last_clean":"2023-10-09T05:24:29.313735+0000","last_became_active":"2023-10-09T05:23:18.186445+0000","last_became_peered":"2023-10-09T05:23:18.186445+0000","last_unstale":"2023-10-09T05:24:29.313735+0000","last_undegraded":"2023-10-09T05:24:29.313735+0000","last_fullsized":"2023-10-09T05:24:29.313735+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:11.327635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":257,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.316719+0000","last_change":"2023-10-09T05:18:26.424399+0000","last_active":"2023-10-09T05:24:29.316719+0000","last_peered":"2023-10-09T05:24:29.316719+0000","last_clean":"2023-10-09T05:24:29.316719+0000","last_became_active":"2023-10-09T05:18:26.422122+0000","last_became_peered":"2023-10-09T05:18:26.422122+0000","last_unstale":"2023-10-09T05:24:29.316719+0000","last_undegraded":"2023-10-09T05:24:29.316719+0000","last_fullsized":"2023-10-09T05:24:29.316719+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:39.393756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":51,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.337943+0000","last_change":"2023-10-09T05:18:26.422950+0000","last_active":"2023-10-09T05:24:29.337943+0000","last_peered":"2023-10-09T05:24:29.337943+0000","last_clean":"2023-10-09T05:24:29.337943+0000","last_became_active":"2023-10-09T05:18:26.422592+0000","last_became_peered":"2023-10-09T05:18:26.422592+0000","last_unstale":"2023-10-09T05:24:29.337943+0000","last_undegraded":"2023-10-09T05:24:29.337943+0000","last_fullsized":"2023-10-09T05:24:29.337943+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:20.701107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":51,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.337436+0000","last_change":"2023-10-09T05:18:26.429482+0000","last_active":"2023-10-09T05:24:29.337436+0000","last_peered":"2023-10-09T05:24:29.337436+0000","last_clean":"2023-10-09T05:24:29.337436+0000","last_became_active":"2023-10-09T05:18:26.429122+0000","last_became_peered":"2023-10-09T05:18:26.429122+0000","last_unstale":"2023-10-09T05:24:29.337436+0000","last_undegraded":"2023-10-09T05:24:29.337436+0000","last_fullsized":"2023-10-09T05:24:29.337436+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:47.301906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":258,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.317338+0000","last_change":"2023-10-09T05:18:26.421678+0000","last_active":"2023-10-09T05:24:29.317338+0000","last_peered":"2023-10-09T05:24:29.317338+0000","last_clean":"2023-10-09T05:24:29.317338+0000","last_became_active":"2023-10-09T05:18:26.419673+0000","last_became_peered":"2023-10-09T05:18:26.419673+0000","last_unstale":"2023-10-09T05:24:29.317338+0000","last_undegraded":"2023-10-09T05:24:29.317338+0000","last_fullsized":"2023-10-09T05:24:29.317338+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:38.876021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":255,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.312651+0000","last_change":"2023-10-09T05:13:03.047339+0000","last_active":"2023-10-09T05:24:29.312651+0000","last_peered":"2023-10-09T05:24:29.312651+0000","last_clean":"2023-10-09T05:24:29.312651+0000","last_became_active":"2023-10-09T05:13:03.040038+0000","last_became_peered":"2023-10-09T05:13:03.040038+0000","last_unstale":"2023-10-09T05:24:29.312651+0000","last_undegraded":"2023-10-09T05:24:29.312651+0000","last_fullsized":"2023-10-09T05:24:29.312651+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:56.726249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":271,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.314330+0000","last_change":"2023-10-09T05:18:26.415173+0000","last_active":"2023-10-09T05:24:29.314330+0000","last_peered":"2023-10-09T05:24:29.314330+0000","last_clean":"2023-10-09T05:24:29.314330+0000","last_became_active":"2023-10-09T05:18:26.414452+0000","last_became_peered":"2023-10-09T05:18:26.414452+0000","last_unstale":"2023-10-09T05:24:29.314330+0000","last_undegraded":"2023-10-09T05:24:29.314330+0000","last_fullsized":"2023-10-09T05:24:29.314330+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:34.379767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":55,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.341865+0000","last_change":"2023-10-09T05:18:26.446608+0000","last_active":"2023-10-09T05:24:29.341865+0000","last_peered":"2023-10-09T05:24:29.341865+0000","last_clean":"2023-10-09T05:24:29.341865+0000","last_became_active":"2023-10-09T05:18:26.428420+0000","last_became_peered":"2023-10-09T05:18:26.428420+0000","last_unstale":"2023-10-09T05:24:29.341865+0000","last_undegraded":"2023-10-09T05:24:29.341865+0000","last_fullsized":"2023-10-09T05:24:29.341865+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:23.693908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":51,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.337464+0000","last_change":"2023-10-09T05:18:26.427461+0000","last_active":"2023-10-09T05:24:29.337464+0000","last_peered":"2023-10-09T05:24:29.337464+0000","last_clean":"2023-10-09T05:24:29.337464+0000","last_became_active":"2023-10-09T05:18:26.427118+0000","last_became_peered":"2023-10-09T05:18:26.427118+0000","last_unstale":"2023-10-09T05:24:29.337464+0000","last_undegraded":"2023-10-09T05:24:29.337464+0000","last_fullsized":"2023-10-09T05:24:29.337464+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:05:01.868197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":55,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.338275+0000","last_change":"2023-10-09T05:18:26.440042+0000","last_active":"2023-10-09T05:24:29.338275+0000","last_peered":"2023-10-09T05:24:29.338275+0000","last_clean":"2023-10-09T05:24:29.338275+0000","last_became_active":"2023-10-09T05:18:26.426726+0000","last_became_peered":"2023-10-09T05:18:26.426726+0000","last_unstale":"2023-10-09T05:24:29.338275+0000","last_undegraded":"2023-10-09T05:24:29.338275+0000","last_fullsized":"2023-10-09T05:24:29.338275+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:23.985650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":259,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.732774+0000","last_change":"2023-10-09T05:09:57.868031+0000","last_active":"2023-10-09T05:24:29.732774+0000","last_peered":"2023-10-09T05:24:29.732774+0000","last_clean":"2023-10-09T05:24:29.732774+0000","last_became_active":"2023-10-09T05:09:57.866477+0000","last_became_peered":"2023-10-09T05:09:57.866477+0000","last_unstale":"2023-10-09T05:24:29.732774+0000","last_undegraded":"2023-10-09T05:24:29.732774+0000","last_fullsized":"2023-10-09T05:24:29.732774+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:27.102583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":55,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.338116+0000","last_change":"2023-10-09T05:18:26.437953+0000","last_active":"2023-10-09T05:24:29.338116+0000","last_peered":"2023-10-09T05:24:29.338116+0000","last_clean":"2023-10-09T05:24:29.338116+0000","last_became_active":"2023-10-09T05:18:26.422703+0000","last_became_peered":"2023-10-09T05:18:26.422703+0000","last_unstale":"2023-10-09T05:24:29.338116+0000","last_undegraded":"2023-10-09T05:24:29.338116+0000","last_fullsized":"2023-10-09T05:24:29.338116+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:13.926811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":292,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.313134+0000","last_change":"2023-10-09T05:18:26.423526+0000","last_active":"2023-10-09T05:24:29.313134+0000","last_peered":"2023-10-09T05:24:29.313134+0000","last_clean":"2023-10-09T05:24:29.313134+0000","last_became_active":"2023-10-09T05:18:26.423092+0000","last_became_peered":"2023-10-09T05:18:26.423092+0000","last_unstale":"2023-10-09T05:24:29.313134+0000","last_undegraded":"2023-10-09T05:24:29.313134+0000","last_fullsized":"2023-10-09T05:24:29.313134+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:54.877664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":182,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.313934+0000","last_change":"2023-10-09T05:13:03.037926+0000","last_active":"2023-10-09T05:24:29.313934+0000","last_peered":"2023-10-09T05:24:29.313934+0000","last_clean":"2023-10-09T05:24:29.313934+0000","last_became_active":"2023-10-09T05:13:03.037639+0000","last_became_peered":"2023-10-09T05:13:03.037639+0000","last_unstale":"2023-10-09T05:24:29.313934+0000","last_undegraded":"2023-10-09T05:24:29.313934+0000","last_fullsized":"2023-10-09T05:24:29.313934+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:52.977143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":241,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.316875+0000","last_change":"2023-10-09T05:09:57.852914+0000","last_active":"2023-10-09T05:24:29.316875+0000","last_peered":"2023-10-09T05:24:29.316875+0000","last_clean":"2023-10-09T05:24:29.316875+0000","last_became_active":"2023-10-09T05:09:57.852522+0000","last_became_peered":"2023-10-09T05:09:57.852522+0000","last_unstale":"2023-10-09T05:24:29.316875+0000","last_undegraded":"2023-10-09T05:24:29.316875+0000","last_fullsized":"2023-10-09T05:24:29.316875+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:24.848494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'3","reported_seq":215,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-09T05:24:31.332715+0000","last_change":"2023-10-09T05:02:09.650579+0000","last_active":"2023-10-09T05:24:31.332715+0000","last_peered":"2023-10-09T05:24:31.332715+0000","last_clean":"2023-10-09T05:24:31.332715+0000","last_became_active":"2023-10-09T05:02:09.650160+0000","last_became_peered":"2023-10-09T05:02:09.650160+0000","last_unstale":"2023-10-09T05:24:31.332715+0000","last_undegraded":"2023-10-09T05:24:31.332715+0000","last_fullsized":"2023-10-09T05:24:31.332715+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:18.292670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":222,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.733312+0000","last_change":"2023-10-09T05:02:09.652449+0000","last_active":"2023-10-09T05:24:29.733312+0000","last_peered":"2023-10-09T05:24:29.733312+0000","last_clean":"2023-10-09T05:24:29.733312+0000","last_became_active":"2023-10-09T05:02:09.651994+0000","last_became_peered":"2023-10-09T05:02:09.651994+0000","last_unstale":"2023-10-09T05:24:29.733312+0000","last_undegraded":"2023-10-09T05:24:29.733312+0000","last_fullsized":"2023-10-09T05:24:29.733312+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:14.480513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":263,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.313227+0000","last_change":"2023-10-09T05:23:18.199172+0000","last_active":"2023-10-09T05:24:29.313227+0000","last_peered":"2023-10-09T05:24:29.313227+0000","last_clean":"2023-10-09T05:24:29.313227+0000","last_became_active":"2023-10-09T05:23:18.198762+0000","last_became_peered":"2023-10-09T05:23:18.198762+0000","last_unstale":"2023-10-09T05:24:29.313227+0000","last_undegraded":"2023-10-09T05:24:29.313227+0000","last_fullsized":"2023-10-09T05:24:29.313227+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:44:23.239788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":279,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.733405+0000","last_change":"2023-10-09T05:20:18.129228+0000","last_active":"2023-10-09T05:24:29.733405+0000","last_peered":"2023-10-09T05:24:29.733405+0000","last_clean":"2023-10-09T05:24:29.733405+0000","last_became_active":"2023-10-09T05:20:18.128769+0000","last_became_peered":"2023-10-09T05:20:18.128769+0000","last_unstale":"2023-10-09T05:24:29.733405+0000","last_undegraded":"2023-10-09T05:24:29.733405+0000","last_fullsized":"2023-10-09T05:24:29.733405+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:50.427100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":263,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.732647+0000","last_change":"2023-10-09T05:18:26.431286+0000","last_active":"2023-10-09T05:24:29.732647+0000","last_peered":"2023-10-09T05:24:29.732647+0000","last_clean":"2023-10-09T05:24:29.732647+0000","last_became_active":"2023-10-09T05:18:26.430127+0000","last_became_peered":"2023-10-09T05:18:26.430127+0000","last_unstale":"2023-10-09T05:24:29.732647+0000","last_undegraded":"2023-10-09T05:24:29.732647+0000","last_fullsized":"2023-10-09T05:24:29.732647+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:42.982259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":262,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.733433+0000","last_change":"2023-10-09T05:23:18.193800+0000","last_active":"2023-10-09T05:24:29.733433+0000","last_peered":"2023-10-09T05:24:29.733433+0000","last_clean":"2023-10-09T05:24:29.733433+0000","last_became_active":"2023-10-09T05:23:18.193338+0000","last_became_peered":"2023-10-09T05:23:18.193338+0000","last_unstale":"2023-10-09T05:24:29.733433+0000","last_undegraded":"2023-10-09T05:24:29.733433+0000","last_fullsized":"2023-10-09T05:24:29.733433+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:29.109566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":273,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.314291+0000","last_change":"2023-10-09T05:18:26.413106+0000","last_active":"2023-10-09T05:24:29.314291+0000","last_peered":"2023-10-09T05:24:29.314291+0000","last_clean":"2023-10-09T05:24:29.314291+0000","last_became_active":"2023-10-09T05:18:26.412334+0000","last_became_peered":"2023-10-09T05:18:26.412334+0000","last_unstale":"2023-10-09T05:24:29.314291+0000","last_undegraded":"2023-10-09T05:24:29.314291+0000","last_fullsized":"2023-10-09T05:24:29.314291+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:18.058016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":127,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.312982+0000","last_change":"2023-10-09T05:09:57.856764+0000","last_active":"2023-10-09T05:24:29.312982+0000","last_peered":"2023-10-09T05:24:29.312982+0000","last_clean":"2023-10-09T05:24:29.312982+0000","last_became_active":"2023-10-09T05:09:57.856461+0000","last_became_peered":"2023-10-09T05:09:57.856461+0000","last_unstale":"2023-10-09T05:24:29.312982+0000","last_undegraded":"2023-10-09T05:24:29.312982+0000","last_fullsized":"2023-10-09T05:24:29.312982+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:06.848082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":163,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.312299+0000","last_change":"2023-10-09T05:18:26.422150+0000","last_active":"2023-10-09T05:24:29.312299+0000","last_peered":"2023-10-09T05:24:29.312299+0000","last_clean":"2023-10-09T05:24:29.312299+0000","last_became_active":"2023-10-09T05:18:26.421853+0000","last_became_peered":"2023-10-09T05:18:26.421853+0000","last_unstale":"2023-10-09T05:24:29.312299+0000","last_undegraded":"2023-10-09T05:24:29.312299+0000","last_fullsized":"2023-10-09T05:24:29.312299+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:37:47.000556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":199,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.315221+0000","last_change":"2023-10-09T05:13:03.031243+0000","last_active":"2023-10-09T05:24:29.315221+0000","last_peered":"2023-10-09T05:24:29.315221+0000","last_clean":"2023-10-09T05:24:29.315221+0000","last_became_active":"2023-10-09T05:13:03.028769+0000","last_became_peered":"2023-10-09T05:13:03.028769+0000","last_unstale":"2023-10-09T05:24:29.315221+0000","last_undegraded":"2023-10-09T05:24:29.315221+0000","last_fullsized":"2023-10-09T05:24:29.315221+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:25:25.239799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":20,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.311520+0000","last_change":"2023-10-09T05:23:18.197927+0000","last_active":"2023-10-09T05:24:29.311520+0000","last_peered":"2023-10-09T05:24:29.311520+0000","last_clean":"2023-10-09T05:24:29.311520+0000","last_became_active":"2023-10-09T05:23:18.197431+0000","last_became_peered":"2023-10-09T05:23:18.197431+0000","last_unstale":"2023-10-09T05:24:29.311520+0000","last_undegraded":"2023-10-09T05:24:29.311520+0000","last_fullsized":"2023-10-09T05:24:29.311520+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:51:41.376456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":233,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.311710+0000","last_change":"2023-10-09T05:13:03.031386+0000","last_active":"2023-10-09T05:24:29.311710+0000","last_peered":"2023-10-09T05:24:29.311710+0000","last_clean":"2023-10-09T05:24:29.311710+0000","last_became_active":"2023-10-09T05:13:03.030816+0000","last_became_peered":"2023-10-09T05:13:03.030816+0000","last_unstale":"2023-10-09T05:24:29.311710+0000","last_undegraded":"2023-10-09T05:24:29.311710+0000","last_fullsized":"2023-10-09T05:24:29.311710+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:28.088661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":223,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.732921+0000","last_change":"2023-10-09T04:56:56.087320+0000","last_active":"2023-10-09T05:24:29.732921+0000","last_peered":"2023-10-09T05:24:29.732921+0000","last_clean":"2023-10-09T05:24:29.732921+0000","last_became_active":"2023-10-09T04:56:56.086552+0000","last_became_peered":"2023-10-09T04:56:56.086552+0000","last_unstale":"2023-10-09T05:24:29.732921+0000","last_undegraded":"2023-10-09T05:24:29.732921+0000","last_fullsized":"2023-10-09T05:24:29.732921+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:14.205288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":20,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.311993+0000","last_change":"2023-10-09T05:23:18.196260+0000","last_active":"2023-10-09T05:24:29.311993+0000","last_peered":"2023-10-09T05:24:29.311993+0000","last_clean":"2023-10-09T05:24:29.311993+0000","last_became_active":"2023-10-09T05:23:18.195907+0000","last_became_peered":"2023-10-09T05:23:18.195907+0000","last_unstale":"2023-10-09T05:24:29.311993+0000","last_undegraded":"2023-10-09T05:24:29.311993+0000","last_fullsized":"2023-10-09T05:24:29.311993+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:36.243432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":39,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.316047+0000","last_change":"2023-10-09T05:20:18.123706+0000","last_active":"2023-10-09T05:24:29.316047+0000","last_peered":"2023-10-09T05:24:29.316047+0000","last_clean":"2023-10-09T05:24:29.316047+0000","last_became_active":"2023-10-09T05:20:18.122859+0000","last_became_peered":"2023-10-09T05:20:18.122859+0000","last_unstale":"2023-10-09T05:24:29.316047+0000","last_undegraded":"2023-10-09T05:24:29.316047+0000","last_fullsized":"2023-10-09T05:24:29.316047+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.906228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":128,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.311788+0000","last_change":"2023-10-09T05:09:57.854064+0000","last_active":"2023-10-09T05:24:29.311788+0000","last_peered":"2023-10-09T05:24:29.311788+0000","last_clean":"2023-10-09T05:24:29.311788+0000","last_became_active":"2023-10-09T05:09:57.847953+0000","last_became_peered":"2023-10-09T05:09:57.847953+0000","last_unstale":"2023-10-09T05:24:29.311788+0000","last_undegraded":"2023-10-09T05:24:29.311788+0000","last_fullsized":"2023-10-09T05:24:29.311788+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:45.914976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":20,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.313282+0000","last_change":"2023-10-09T05:23:18.200437+0000","last_active":"2023-10-09T05:24:29.313282+0000","last_peered":"2023-10-09T05:24:29.313282+0000","last_clean":"2023-10-09T05:24:29.313282+0000","last_became_active":"2023-10-09T05:23:18.200002+0000","last_became_peered":"2023-10-09T05:23:18.200002+0000","last_unstale":"2023-10-09T05:24:29.313282+0000","last_undegraded":"2023-10-09T05:24:29.313282+0000","last_fullsized":"2023-10-09T05:24:29.313282+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:16.926545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":205,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.316673+0000","last_change":"2023-10-09T05:18:26.416511+0000","last_active":"2023-10-09T05:24:29.316673+0000","last_peered":"2023-10-09T05:24:29.316673+0000","last_clean":"2023-10-09T05:24:29.316673+0000","last_became_active":"2023-10-09T05:18:26.416101+0000","last_became_peered":"2023-10-09T05:18:26.416101+0000","last_unstale":"2023-10-09T05:24:29.316673+0000","last_undegraded":"2023-10-09T05:24:29.316673+0000","last_fullsized":"2023-10-09T05:24:29.316673+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:10.243472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":547,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.337752+0000","last_change":"2023-10-09T05:18:29.419527+0000","last_active":"2023-10-09T05:24:29.337752+0000","last_peered":"2023-10-09T05:24:29.337752+0000","last_clean":"2023-10-09T05:24:29.337752+0000","last_became_active":"2023-10-09T05:18:29.418852+0000","last_became_peered":"2023-10-09T05:18:29.418852+0000","last_unstale":"2023-10-09T05:24:29.337752+0000","last_undegraded":"2023-10-09T05:24:29.337752+0000","last_fullsized":"2023-10-09T05:24:29.337752+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_clean_scrub_stamp":"2023-10-09T04:36:44.261013+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:02.899170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":222,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.314040+0000","last_change":"2023-10-09T05:13:03.027260+0000","last_active":"2023-10-09T05:24:29.314040+0000","last_peered":"2023-10-09T05:24:29.314040+0000","last_clean":"2023-10-09T05:24:29.314040+0000","last_became_active":"2023-10-09T05:13:03.026416+0000","last_became_peered":"2023-10-09T05:13:03.026416+0000","last_unstale":"2023-10-09T05:24:29.314040+0000","last_undegraded":"2023-10-09T05:24:29.314040+0000","last_fullsized":"2023-10-09T05:24:29.314040+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:05.057338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":245,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.313461+0000","last_change":"2023-10-09T05:18:26.426126+0000","last_active":"2023-10-09T05:24:29.313461+0000","last_peered":"2023-10-09T05:24:29.313461+0000","last_clean":"2023-10-09T05:24:29.313461+0000","last_became_active":"2023-10-09T05:18:26.425721+0000","last_became_peered":"2023-10-09T05:18:26.425721+0000","last_unstale":"2023-10-09T05:24:29.313461+0000","last_undegraded":"2023-10-09T05:24:29.313461+0000","last_fullsized":"2023-10-09T05:24:29.313461+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:48.229218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":71,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.337703+0000","last_change":"2023-10-09T05:23:18.193825+0000","last_active":"2023-10-09T05:24:29.337703+0000","last_peered":"2023-10-09T05:24:29.337703+0000","last_clean":"2023-10-09T05:24:29.337703+0000","last_became_active":"2023-10-09T05:23:18.193394+0000","last_became_peered":"2023-10-09T05:23:18.193394+0000","last_unstale":"2023-10-09T05:24:29.337703+0000","last_undegraded":"2023-10-09T05:24:29.337703+0000","last_fullsized":"2023-10-09T05:24:29.337703+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:40.594469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":223,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.317577+0000","last_change":"2023-10-09T05:02:09.648781+0000","last_active":"2023-10-09T05:24:29.317577+0000","last_peered":"2023-10-09T05:24:29.317577+0000","last_clean":"2023-10-09T05:24:29.317577+0000","last_became_active":"2023-10-09T05:02:09.648078+0000","last_became_peered":"2023-10-09T05:02:09.648078+0000","last_unstale":"2023-10-09T05:24:29.317577+0000","last_undegraded":"2023-10-09T05:24:29.317577+0000","last_fullsized":"2023-10-09T05:24:29.317577+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:35.287091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":301,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.317178+0000","last_change":"2023-10-09T05:20:18.132750+0000","last_active":"2023-10-09T05:24:29.317178+0000","last_peered":"2023-10-09T05:24:29.317178+0000","last_clean":"2023-10-09T05:24:29.317178+0000","last_became_active":"2023-10-09T05:20:18.132396+0000","last_became_peered":"2023-10-09T05:20:18.132396+0000","last_unstale":"2023-10-09T05:24:29.317178+0000","last_undegraded":"2023-10-09T05:24:29.317178+0000","last_fullsized":"2023-10-09T05:24:29.317178+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:00:55.731045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":51,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.338176+0000","last_change":"2023-10-09T05:18:26.429468+0000","last_active":"2023-10-09T05:24:29.338176+0000","last_peered":"2023-10-09T05:24:29.338176+0000","last_clean":"2023-10-09T05:24:29.338176+0000","last_became_active":"2023-10-09T05:18:26.429098+0000","last_became_peered":"2023-10-09T05:18:26.429098+0000","last_unstale":"2023-10-09T05:24:29.338176+0000","last_undegraded":"2023-10-09T05:24:29.338176+0000","last_fullsized":"2023-10-09T05:24:29.338176+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:47.798226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":154,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.312551+0000","last_change":"2023-10-09T05:20:18.126997+0000","last_active":"2023-10-09T05:24:29.312551+0000","last_peered":"2023-10-09T05:24:29.312551+0000","last_clean":"2023-10-09T05:24:29.312551+0000","last_became_active":"2023-10-09T05:20:18.126641+0000","last_became_peered":"2023-10-09T05:20:18.126641+0000","last_unstale":"2023-10-09T05:24:29.312551+0000","last_undegraded":"2023-10-09T05:24:29.312551+0000","last_fullsized":"2023-10-09T05:24:29.312551+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:42.112245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":263,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.317057+0000","last_change":"2023-10-09T05:18:26.418524+0000","last_active":"2023-10-09T05:24:29.317057+0000","last_peered":"2023-10-09T05:24:29.317057+0000","last_clean":"2023-10-09T05:24:29.317057+0000","last_became_active":"2023-10-09T05:18:26.418058+0000","last_became_peered":"2023-10-09T05:18:26.418058+0000","last_unstale":"2023-10-09T05:24:29.317057+0000","last_undegraded":"2023-10-09T05:24:29.317057+0000","last_fullsized":"2023-10-09T05:24:29.317057+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:40.594449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":266,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.314247+0000","last_change":"2023-10-09T05:18:26.431563+0000","last_active":"2023-10-09T05:24:29.314247+0000","last_peered":"2023-10-09T05:24:29.314247+0000","last_clean":"2023-10-09T05:24:29.314247+0000","last_became_active":"2023-10-09T05:18:26.421025+0000","last_became_peered":"2023-10-09T05:18:26.421025+0000","last_unstale":"2023-10-09T05:24:29.314247+0000","last_undegraded":"2023-10-09T05:24:29.314247+0000","last_fullsized":"2023-10-09T05:24:29.314247+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:30.333585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.312788+0000","last_change":"2023-10-09T05:23:18.196610+0000","last_active":"2023-10-09T05:24:29.312788+0000","last_peered":"2023-10-09T05:24:29.312788+0000","last_clean":"2023-10-09T05:24:29.312788+0000","last_became_active":"2023-10-09T05:23:18.196279+0000","last_became_peered":"2023-10-09T05:23:18.196279+0000","last_unstale":"2023-10-09T05:24:29.312788+0000","last_undegraded":"2023-10-09T05:24:29.312788+0000","last_fullsized":"2023-10-09T05:24:29.312788+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:52.868384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":265,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.312336+0000","last_change":"2023-10-09T05:21:08.179657+0000","last_active":"2023-10-09T05:24:29.312336+0000","last_peered":"2023-10-09T05:24:29.312336+0000","last_clean":"2023-10-09T05:24:29.312336+0000","last_became_active":"2023-10-09T05:21:08.178958+0000","last_became_peered":"2023-10-09T05:21:08.178958+0000","last_unstale":"2023-10-09T05:24:29.312336+0000","last_undegraded":"2023-10-09T05:24:29.312336+0000","last_fullsized":"2023-10-09T05:24:29.312336+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:20.815986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":267,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.313943+0000","last_change":"2023-10-09T05:18:26.424429+0000","last_active":"2023-10-09T05:24:29.313943+0000","last_peered":"2023-10-09T05:24:29.313943+0000","last_clean":"2023-10-09T05:24:29.313943+0000","last_became_active":"2023-10-09T05:18:26.423034+0000","last_became_peered":"2023-10-09T05:18:26.423034+0000","last_unstale":"2023-10-09T05:24:29.313943+0000","last_undegraded":"2023-10-09T05:24:29.313943+0000","last_fullsized":"2023-10-09T05:24:29.313943+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:16.251181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":253,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.317232+0000","last_change":"2023-10-09T05:09:57.878002+0000","last_active":"2023-10-09T05:24:29.317232+0000","last_peered":"2023-10-09T05:24:29.317232+0000","last_clean":"2023-10-09T05:24:29.317232+0000","last_became_active":"2023-10-09T05:09:57.852792+0000","last_became_peered":"2023-10-09T05:09:57.852792+0000","last_unstale":"2023-10-09T05:24:29.317232+0000","last_undegraded":"2023-10-09T05:24:29.317232+0000","last_fullsized":"2023-10-09T05:24:29.317232+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:18.763275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":215,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.732529+0000","last_change":"2023-10-09T05:13:03.026541+0000","last_active":"2023-10-09T05:24:29.732529+0000","last_peered":"2023-10-09T05:24:29.732529+0000","last_clean":"2023-10-09T05:24:29.732529+0000","last_became_active":"2023-10-09T05:13:03.025729+0000","last_became_peered":"2023-10-09T05:13:03.025729+0000","last_unstale":"2023-10-09T05:24:29.732529+0000","last_undegraded":"2023-10-09T05:24:29.732529+0000","last_fullsized":"2023-10-09T05:24:29.732529+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:22.938235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'2","reported_seq":31,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.317011+0000","last_change":"2023-10-09T05:21:08.173845+0000","last_active":"2023-10-09T05:24:29.317011+0000","last_peered":"2023-10-09T05:24:29.317011+0000","last_clean":"2023-10-09T05:24:29.317011+0000","last_became_active":"2023-10-09T05:21:08.173508+0000","last_became_peered":"2023-10-09T05:21:08.173508+0000","last_unstale":"2023-10-09T05:24:29.317011+0000","last_undegraded":"2023-10-09T05:24:29.317011+0000","last_fullsized":"2023-10-09T05:24:29.317011+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:03.628428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":38,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.316818+0000","last_change":"2023-10-09T05:20:18.125172+0000","last_active":"2023-10-09T05:24:29.316818+0000","last_peered":"2023-10-09T05:24:29.316818+0000","last_clean":"2023-10-09T05:24:29.316818+0000","last_became_active":"2023-10-09T05:20:18.123624+0000","last_became_peered":"2023-10-09T05:20:18.123624+0000","last_unstale":"2023-10-09T05:24:29.316818+0000","last_undegraded":"2023-10-09T05:24:29.316818+0000","last_fullsized":"2023-10-09T05:24:29.316818+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:11.106483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":117,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.733526+0000","last_change":"2023-10-09T05:13:03.055887+0000","last_active":"2023-10-09T05:24:29.733526+0000","last_peered":"2023-10-09T05:24:29.733526+0000","last_clean":"2023-10-09T05:24:29.733526+0000","last_became_active":"2023-10-09T05:13:03.028782+0000","last_became_peered":"2023-10-09T05:13:03.028782+0000","last_unstale":"2023-10-09T05:24:29.733526+0000","last_undegraded":"2023-10-09T05:24:29.733526+0000","last_fullsized":"2023-10-09T05:24:29.733526+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:38.828205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":279,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.733161+0000","last_change":"2023-10-09T05:23:18.207836+0000","last_active":"2023-10-09T05:24:29.733161+0000","last_peered":"2023-10-09T05:24:29.733161+0000","last_clean":"2023-10-09T05:24:29.733161+0000","last_became_active":"2023-10-09T05:23:18.207404+0000","last_became_peered":"2023-10-09T05:23:18.207404+0000","last_unstale":"2023-10-09T05:24:29.733161+0000","last_undegraded":"2023-10-09T05:24:29.733161+0000","last_fullsized":"2023-10-09T05:24:29.733161+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:36:22.236093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":117,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.733553+0000","last_change":"2023-10-09T05:13:03.049546+0000","last_active":"2023-10-09T05:24:29.733553+0000","last_peered":"2023-10-09T05:24:29.733553+0000","last_clean":"2023-10-09T05:24:29.733553+0000","last_became_active":"2023-10-09T05:13:03.028676+0000","last_became_peered":"2023-10-09T05:13:03.028676+0000","last_unstale":"2023-10-09T05:24:29.733553+0000","last_undegraded":"2023-10-09T05:24:29.733553+0000","last_fullsized":"2023-10-09T05:24:29.733553+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:55.089350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":239,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.733038+0000","last_change":"2023-10-09T05:02:09.649748+0000","last_active":"2023-10-09T05:24:29.733038+0000","last_peered":"2023-10-09T05:24:29.733038+0000","last_clean":"2023-10-09T05:24:29.733038+0000","last_became_active":"2023-10-09T05:02:09.649444+0000","last_became_peered":"2023-10-09T05:02:09.649444+0000","last_unstale":"2023-10-09T05:24:29.733038+0000","last_undegraded":"2023-10-09T05:24:29.733038+0000","last_fullsized":"2023-10-09T05:24:29.733038+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:07.436820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'2","reported_seq":259,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.314573+0000","last_change":"2023-10-09T05:22:47.197002+0000","last_active":"2023-10-09T05:24:29.314573+0000","last_peered":"2023-10-09T05:24:29.314573+0000","last_clean":"2023-10-09T05:24:29.314573+0000","last_became_active":"2023-10-09T05:22:47.196685+0000","last_became_peered":"2023-10-09T05:22:47.196685+0000","last_unstale":"2023-10-09T05:24:29.314573+0000","last_undegraded":"2023-10-09T05:24:29.314573+0000","last_fullsized":"2023-10-09T05:24:29.314573+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:47:32.594447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":244,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.313528+0000","last_change":"2023-10-09T05:02:09.649674+0000","last_active":"2023-10-09T05:24:29.313528+0000","last_peered":"2023-10-09T05:24:29.313528+0000","last_clean":"2023-10-09T05:24:29.313528+0000","last_became_active":"2023-10-09T05:02:09.649041+0000","last_became_peered":"2023-10-09T05:02:09.649041+0000","last_unstale":"2023-10-09T05:24:29.313528+0000","last_undegraded":"2023-10-09T05:24:29.313528+0000","last_fullsized":"2023-10-09T05:24:29.313528+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:18.524071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'5","reported_seq":37,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.317444+0000","last_change":"2023-10-09T05:21:08.181594+0000","last_active":"2023-10-09T05:24:29.317444+0000","last_peered":"2023-10-09T05:24:29.317444+0000","last_clean":"2023-10-09T05:24:29.317444+0000","last_became_active":"2023-10-09T05:21:08.176895+0000","last_became_peered":"2023-10-09T05:21:08.176895+0000","last_unstale":"2023-10-09T05:24:29.317444+0000","last_undegraded":"2023-10-09T05:24:29.317444+0000","last_fullsized":"2023-10-09T05:24:29.317444+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:48:45.569544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":242,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-09T05:24:29.732666+0000","last_change":"2023-10-09T05:09:57.866732+0000","last_active":"2023-10-09T05:24:29.732666+0000","last_peered":"2023-10-09T05:24:29.732666+0000","last_clean":"2023-10-09T05:24:29.732666+0000","last_became_active":"2023-10-09T05:09:57.866117+0000","last_became_peered":"2023-10-09T05:09:57.866117+0000","last_unstale":"2023-10-09T05:24:29.732666+0000","last_undegraded":"2023-10-09T05:24:29.732666+0000","last_fullsized":"2023-10-09T05:24:29.732666+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:49:14.815342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":20,"num_read_kb":19,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":46,"num_bytes_recovered":21458,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234509,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":918628,"kb_used_data":3292,"kb_used_omap":0,"kb_used_meta":915328,"kb_avail":92826524,"statfs":{"total":95995035648,"available":95054360576,"internally_reserved":0,"allocated":3371008,"data_stored":2483699,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":937295872},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:23:53 2023","interfaces":[{"interface":"back","average":{"1min":1.329,"5min":1.26,"15min":1.7130000000000001},"min":{"1min":0.35399999999999998,"5min":0.28399999999999997,"15min":0.24299999999999999},"max":{"1min":7.6130000000000004,"5min":8.8000000000000007,"15min":36.018000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.359,"5min":1.464,"15min":1.8939999999999999},"min":{"1min":0.42199999999999999,"5min":0.307,"15min":0.28599999999999998},"max":{"1min":7.633,"5min":8.8170000000000002,"15min":36.067999999999998},"last":0.65800000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:24:30 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":1.8740000000000001,"15min":1.5800000000000001},"min":{"1min":0.41899999999999998,"5min":0.28199999999999997,"15min":0.26200000000000001},"max":{"1min":1.294,"5min":23.530999999999999,"15min":35.134},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.77900000000000003,"5min":1.776,"15min":1.7709999999999999},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.27100000000000002},"max":{"1min":1.7170000000000001,"5min":23.443000000000001,"15min":35.116999999999997},"last":0.76800000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:23:53 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":1.446,"15min":1.681},"min":{"1min":0.31900000000000001,"5min":0.31,"15min":0.29999999999999999},"max":{"1min":3.3079999999999998,"5min":7.0609999999999999,"15min":66.858000000000004},"last":0.72599999999999998},{"interface":"front","average":{"1min":1.605,"5min":1.3620000000000001,"15min":1.5209999999999999},"min":{"1min":0.48399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.1040000000000001,"5min":5.9130000000000003,"15min":68.998999999999995},"last":0.60199999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:23:53 2023","interfaces":[{"interface":"back","average":{"1min":1.776,"5min":2.1960000000000002,"15min":2.577},"min":{"1min":0.42199999999999999,"5min":0.35899999999999999,"15min":0.34899999999999998},"max":{"1min":9.5739999999999998,"5min":23.885000000000002,"15min":99.683999999999997},"last":0.745},{"interface":"front","average":{"1min":3.4279999999999999,"5min":2.468,"15min":2.6749999999999998},"min":{"1min":0.46800000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":40.494999999999997,"5min":40.494999999999997,"15min":99.411000000000001},"last":0.48999999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:24:30 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.77900000000000003,"15min":0.94699999999999995},"min":{"1min":0.41899999999999998,"5min":0.35699999999999998,"15min":0.313},"max":{"1min":1.337,"5min":2.9169999999999998,"15min":9.7850000000000001},"last":0.68100000000000005},{"interface":"front","average":{"1min":1.0580000000000001,"5min":0.97499999999999998,"15min":1.1559999999999999},"min":{"1min":0.46400000000000002,"5min":0.38,"15min":0.34799999999999998},"max":{"1min":1.3440000000000001,"5min":3.8799999999999999,"15min":11.428000000000001},"last":0.83099999999999996}]},{"osd":5,"last update":"Mon Oct 9 05:24:30 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":1.071,"15min":1.046},"min":{"1min":0.56599999999999995,"5min":0.36799999999999999,"15min":0.35999999999999999},"max":{"1min":1.2609999999999999,"5min":4.8689999999999998,"15min":5.9649999999999999},"last":0.69299999999999995},{"interface":"front","average":{"1min":1.0269999999999999,"5min":0.86899999999999999,"15min":1.1040000000000001},"min":{"1min":0.60699999999999998,"5min":0.39200000000000002,"15min":0.37},"max":{"1min":1.3660000000000001,"5min":3.0289999999999999,"15min":10.103},"last":0.70899999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:24:30 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":0.99199999999999999,"15min":1.0860000000000001},"min":{"1min":0.59599999999999997,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.395,"5min":5.9379999999999997,"15min":8.4130000000000003},"last":0.64500000000000002},{"interface":"front","average":{"1min":1.127,"5min":1.165,"15min":1.3029999999999999},"min":{"1min":0.58999999999999997,"5min":0.43099999999999999,"15min":0.38200000000000001},"max":{"1min":1.5509999999999999,"5min":6.7910000000000004,"15min":11.797000000000001},"last":0.80300000000000005}]}]},{"osd":6,"up_from":46,"seq":197568496151,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":903008,"kb_used_data":3288,"kb_used_omap":0,"kb_used_meta":899712,"kb_avail":92842144,"statfs":{"total":95995035648,"available":95070355456,"internally_reserved":0,"allocated":3366912,"data_stored":2484430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921305088},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:23:48 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":2.0270000000000001,"15min":2.8109999999999999},"min":{"1min":0.28499999999999998,"5min":0.26300000000000001,"15min":0.23599999999999999},"max":{"1min":2.4550000000000001,"5min":88.929000000000002,"15min":98.275000000000006},"last":0.92600000000000005},{"interface":"front","average":{"1min":0.78800000000000003,"5min":2.1259999999999999,"15min":2.9199999999999999},"min":{"1min":0.36899999999999999,"5min":0.33700000000000002,"15min":0.309},"max":{"1min":2.4860000000000002,"5min":88.995999999999995,"15min":98.334999999999994},"last":0.97699999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:23:48 2023","interfaces":[{"interface":"back","average":{"1min":2.4500000000000002,"5min":3.9380000000000002,"15min":2.411},"min":{"1min":0.39200000000000002,"5min":0.27500000000000002,"15min":0.26700000000000002},"max":{"1min":9.3759999999999994,"5min":120.873,"15min":120.873},"last":0.79000000000000004},{"interface":"front","average":{"1min":1.607,"5min":3.4769999999999999,"15min":2.2989999999999999},"min":{"1min":0.33600000000000002,"5min":0.30199999999999999,"15min":0.28899999999999998},"max":{"1min":4.2160000000000002,"5min":119.248,"15min":119.248},"last":1.0309999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:23:48 2023","interfaces":[{"interface":"back","average":{"1min":1.621,"5min":1.4690000000000001,"15min":1.1879999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.28699999999999998},"max":{"1min":7.7999999999999998,"5min":8.718,"15min":13.568},"last":0.83399999999999996},{"interface":"front","average":{"1min":2.347,"5min":1.8859999999999999,"15min":1.4179999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.32400000000000001},"max":{"1min":10.214,"5min":13.712,"15min":13.712},"last":1.052}]},{"osd":3,"last update":"Mon Oct 9 05:23:48 2023","interfaces":[{"interface":"back","average":{"1min":10.731999999999999,"5min":6.7009999999999996,"15min":4.4530000000000003},"min":{"1min":0.35999999999999999,"5min":0.34799999999999998,"15min":0.32400000000000001},"max":{"1min":113.922,"5min":115.658,"15min":115.658},"last":1.0880000000000001},{"interface":"front","average":{"1min":10.565,"5min":6.3680000000000003,"15min":4.4219999999999997},"min":{"1min":0.376,"5min":0.36799999999999999,"15min":0.30199999999999999},"max":{"1min":114.071,"5min":115.489,"15min":115.489},"last":1.159}]},{"osd":4,"last update":"Mon Oct 9 05:23:48 2023","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":1.0269999999999999,"15min":1.048},"min":{"1min":0.29999999999999999,"5min":0.27600000000000002,"15min":0.27000000000000002},"max":{"1min":3.7930000000000001,"5min":7.7000000000000002,"15min":8.3670000000000009},"last":0.89900000000000002},{"interface":"front","average":{"1min":0.82399999999999995,"5min":0.85699999999999998,"15min":1.044},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":3.1429999999999998,"5min":4.7279999999999998,"15min":13.473000000000001},"last":1.1100000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:23:48 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":1.03,"15min":1.0609999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.8650000000000002,"5min":7.5899999999999999,"15min":7.5899999999999999},"last":1.01},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.92800000000000005,"15min":1.099},"min":{"1min":0.44,"5min":0.38,"15min":0.35899999999999999},"max":{"1min":3.3799999999999999,"5min":4.8220000000000001,"15min":13.292999999999999},"last":1.208}]},{"osd":7,"last update":"Mon Oct 9 05:24:16 2023","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":1.2749999999999999,"15min":1.294},"min":{"1min":0.39400000000000002,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.3,"5min":6.6210000000000004,"15min":8.1120000000000001},"last":1.1919999999999999},{"interface":"front","average":{"1min":1.0740000000000001,"5min":1.367,"15min":1.4330000000000001},"min":{"1min":0.47099999999999997,"5min":0.46500000000000002,"15min":0.433},"max":{"1min":1.3919999999999999,"5min":6.6630000000000003,"15min":8.1859999999999999},"last":1.371}]}]},{"osd":1,"up_from":96,"seq":412316860741,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2057008,"kb_used_data":1174760,"kb_used_omap":4,"kb_used_meta":882235,"kb_avail":91688144,"statfs":{"total":95995035648,"available":93888659456,"internally_reserved":0,"allocated":1202954240,"data_stored":1201473261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":903409550},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:23:40 2023","interfaces":[{"interface":"back","average":{"1min":6.7190000000000003,"5min":2.6789999999999998,"15min":1.9930000000000001},"min":{"1min":0.26300000000000001,"5min":0.19500000000000001,"15min":0.17299999999999999},"max":{"1min":83.194999999999993,"5min":83.194999999999993,"15min":83.194999999999993},"last":3.3650000000000002},{"interface":"front","average":{"1min":7.0940000000000003,"5min":3.411,"15min":2.335},"min":{"1min":0.26800000000000002,"5min":0.186,"15min":0.186},"max":{"1min":83.972999999999999,"5min":83.972999999999999,"15min":83.972999999999999},"last":0.624}]},{"osd":2,"last update":"Mon Oct 9 05:23:44 2023","interfaces":[{"interface":"back","average":{"1min":7.4790000000000001,"5min":3.496,"15min":1.921},"min":{"1min":0.221,"5min":0.215,"15min":0.21099999999999999},"max":{"1min":83.337000000000003,"5min":83.337000000000003,"15min":83.337000000000003},"last":2.996},{"interface":"front","average":{"1min":8.5259999999999998,"5min":3.5089999999999999,"15min":2.024},"min":{"1min":0.29299999999999998,"5min":0.26000000000000001,"15min":0.23599999999999999},"max":{"1min":83.143000000000001,"5min":83.143000000000001,"15min":83.143000000000001},"last":2.7810000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:23:44 2023","interfaces":[{"interface":"back","average":{"1min":7.3010000000000002,"5min":3.6970000000000001,"15min":2.6469999999999998},"min":{"1min":0.26700000000000002,"5min":0.215,"15min":0.215},"max":{"1min":84.105999999999995,"5min":84.105999999999995,"15min":84.105999999999995},"last":0.93999999999999995},{"interface":"front","average":{"1min":9.3049999999999997,"5min":3.9860000000000002,"15min":2.8180000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26800000000000002},"max":{"1min":83.566000000000003,"5min":83.566000000000003,"15min":83.566000000000003},"last":2.6230000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:23:40 2023","interfaces":[{"interface":"back","average":{"1min":7.4539999999999997,"5min":3.1669999999999998,"15min":1.9339999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":83.453999999999994,"5min":83.453999999999994,"15min":83.453999999999994},"last":3.0289999999999999},{"interface":"front","average":{"1min":5.8520000000000003,"5min":2.4359999999999999,"15min":1.7230000000000001},"min":{"1min":0.40300000000000002,"5min":0.32400000000000001,"15min":0.28100000000000003},"max":{"1min":83.375,"5min":83.375,"15min":83.375},"last":3.4039999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:23:40 2023","interfaces":[{"interface":"back","average":{"1min":1.7010000000000001,"5min":1.8260000000000001,"15min":1.663},"min":{"1min":0.40600000000000003,"5min":0.378,"15min":0.28899999999999998},"max":{"1min":7.6639999999999997,"5min":8.1129999999999995,"15min":9.0530000000000008},"last":0.97299999999999998},{"interface":"front","average":{"1min":7.367,"5min":2.9700000000000002,"15min":1.8959999999999999},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.38200000000000001},"max":{"1min":83.268000000000001,"5min":83.268000000000001,"15min":83.268000000000001},"last":3.0710000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:23:40 2023","interfaces":[{"interface":"back","average":{"1min":1.9199999999999999,"5min":1.962,"15min":1.6220000000000001},"min":{"1min":0.42599999999999999,"5min":0.34899999999999998,"15min":0.34699999999999998},"max":{"1min":7.6040000000000001,"5min":8.1509999999999998,"15min":8.5630000000000006},"last":1.006},{"interface":"front","average":{"1min":7.3810000000000002,"5min":3.0190000000000001,"15min":1.9510000000000001},"min":{"1min":0.376,"5min":0.376,"15min":0.373},"max":{"1min":83.400000000000006,"5min":83.400000000000006,"15min":83.400000000000006},"last":3.097}]},{"osd":7,"last update":"Mon Oct 9 05:23:40 2023","interfaces":[{"interface":"back","average":{"1min":7.5039999999999996,"5min":3.0209999999999999,"15min":2.0579999999999998},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":83.510000000000005,"5min":83.510000000000005,"15min":83.510000000000005},"last":2.6659999999999999},{"interface":"front","average":{"1min":6.484,"5min":2.4529999999999998,"15min":1.8129999999999999},"min":{"1min":0.435,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":83.480999999999995,"5min":83.480999999999995,"15min":83.480999999999995},"last":3.4300000000000002}]}]},{"osd":0,"up_from":88,"seq":377957122382,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1126144,"kb_used_data":353976,"kb_used_omap":5,"kb_used_meta":772154,"kb_avail":92619008,"statfs":{"total":95995035648,"available":94841864192,"internally_reserved":0,"allocated":362471424,"data_stored":361033915,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":790686330},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":17,"apply_latency_ms":17,"commit_latency_ns":17000000,"apply_latency_ns":17000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:23:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0510000000000002,"5min":3.4359999999999999,"15min":2.669},"min":{"1min":0.24399999999999999,"5min":0.191,"15min":0.17799999999999999},"max":{"1min":8.3800000000000008,"5min":80.843999999999994,"15min":109.88},"last":0.49099999999999999},{"interface":"front","average":{"1min":2.593,"5min":4.1390000000000002,"15min":3.048},"min":{"1min":0.217,"5min":0.214,"15min":0.21299999999999999},"max":{"1min":8.2490000000000006,"5min":81.159999999999997,"15min":109.435},"last":0.54400000000000004}]},{"osd":2,"last update":"Mon Oct 9 05:23:48 2023","interfaces":[{"interface":"back","average":{"1min":1.9910000000000001,"5min":2.3889999999999998,"15min":1.7509999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.24199999999999999},"max":{"1min":7.2779999999999996,"5min":15.901999999999999,"15min":17.797000000000001},"last":0.51000000000000001},{"interface":"front","average":{"1min":2.722,"5min":2.3029999999999999,"15min":1.7609999999999999},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":9.2919999999999998,"5min":12.962,"15min":17.129999999999999},"last":0.56399999999999995}]},{"osd":3,"last update":"Mon Oct 9 05:23:48 2023","interfaces":[{"interface":"back","average":{"1min":1.883,"5min":3.3679999999999999,"15min":2.4900000000000002},"min":{"1min":0.23999999999999999,"5min":0.22800000000000001,"15min":0.218},"max":{"1min":6.6210000000000004,"5min":82.286000000000001,"15min":82.286000000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":3.1589999999999998,"5min":3.8820000000000001,"15min":2.8860000000000001},"min":{"1min":0.376,"5min":0.32100000000000001,"15min":0.25800000000000001},"max":{"1min":9.3729999999999993,"5min":75.969999999999999,"15min":75.969999999999999},"last":0.71599999999999997}]},{"osd":4,"last update":"Mon Oct 9 05:23:48 2023","interfaces":[{"interface":"back","average":{"1min":1.677,"5min":1.9079999999999999,"15min":1.601},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":5.5010000000000003,"5min":9.7460000000000004,"15min":12.255000000000001},"last":1.0169999999999999},{"interface":"front","average":{"1min":2.1099999999999999,"5min":1.611,"15min":1.508},"min":{"1min":0.41799999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":8.3369999999999997,"5min":8.3580000000000005,"15min":8.3580000000000005},"last":1.2050000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:23:48 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.5349999999999999,"15min":1.4590000000000001},"min":{"1min":0.39400000000000002,"5min":0.372,"15min":0.29899999999999999},"max":{"1min":3.988,"5min":11.916,"15min":11.916},"last":0.97099999999999997},{"interface":"front","average":{"1min":1.9239999999999999,"5min":1.976,"15min":1.7969999999999999},"min":{"1min":0.49099999999999999,"5min":0.39500000000000002,"15min":0.36699999999999999},"max":{"1min":5.6660000000000004,"5min":9.8010000000000002,"15min":12.143000000000001},"last":1.1950000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:23:48 2023","interfaces":[{"interface":"back","average":{"1min":1.7090000000000001,"5min":1.7070000000000001,"15min":1.6359999999999999},"min":{"1min":0.45400000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":8.3629999999999995,"5min":12.125,"15min":12.125},"last":1.004},{"interface":"front","average":{"1min":1.3420000000000001,"5min":1.651,"15min":1.583},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":5.8209999999999997,"5min":12.007999999999999,"15min":12.007999999999999},"last":1.177}]},{"osd":7,"last update":"Mon Oct 9 05:23:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":1.681,"15min":1.6830000000000001},"min":{"1min":0.41599999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":4.1660000000000004,"5min":11.977,"15min":11.977},"last":1.0289999999999999},{"interface":"front","average":{"1min":1.909,"5min":2.1000000000000001,"15min":1.865},"min":{"1min":0.53000000000000003,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":5.7039999999999997,"5min":9.7750000000000004,"15min":12.1},"last":1.054}]}]},{"osd":2,"up_from":106,"seq":455266533661,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":619788,"kb_used_data":57284,"kb_used_omap":7,"kb_used_meta":562488,"kb_avail":93125364,"statfs":{"total":95995035648,"available":95360372736,"internally_reserved":0,"allocated":58658816,"data_stored":57267007,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":575987793},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:23:39 2023","interfaces":[{"interface":"back","average":{"1min":3.0550000000000002,"5min":2.0089999999999999,"15min":1.917},"min":{"1min":0.20100000000000001,"5min":0.193,"15min":0.193},"max":{"1min":16.442,"5min":18.079999999999998,"15min":53.088000000000001},"last":0.64500000000000002},{"interface":"front","average":{"1min":3.4630000000000001,"5min":2.0680000000000001,"15min":1.946},"min":{"1min":0.35299999999999998,"5min":0.224,"15min":0.215},"max":{"1min":16.870000000000001,"5min":16.870000000000001,"15min":46.762},"last":0.96699999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:23:39 2023","interfaces":[{"interface":"back","average":{"1min":4.0529999999999999,"5min":2.4289999999999998,"15min":2.1200000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.224},"max":{"1min":23.251000000000001,"5min":23.251000000000001,"15min":68.933999999999997},"last":0.69399999999999995},{"interface":"front","average":{"1min":3.0019999999999998,"5min":2.573,"15min":2.177},"min":{"1min":0.25900000000000001,"5min":0.249,"15min":0.20399999999999999},"max":{"1min":17.632999999999999,"5min":17.632999999999999,"15min":68.846000000000004},"last":0.59199999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:23:53 2023","interfaces":[{"interface":"back","average":{"1min":2.7050000000000001,"5min":2.9159999999999999,"15min":2.2280000000000002},"min":{"1min":0.36699999999999999,"5min":0.27500000000000002,"15min":0.23699999999999999},"max":{"1min":13.933999999999999,"5min":13.933999999999999,"15min":68.968000000000004},"last":0.79900000000000004},{"interface":"front","average":{"1min":2.71,"5min":2.7200000000000002,"15min":1.986},"min":{"1min":0.35099999999999998,"5min":0.28899999999999998,"15min":0.23899999999999999},"max":{"1min":10.6,"5min":13.441000000000001,"15min":68.891999999999996},"last":0.75900000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:23:39 2023","interfaces":[{"interface":"back","average":{"1min":1.9550000000000001,"5min":1.782,"15min":1.4430000000000001},"min":{"1min":0.40200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":6.2220000000000004,"5min":11.157999999999999,"15min":11.157999999999999},"last":1.0169999999999999},{"interface":"front","average":{"1min":2.2330000000000001,"5min":1.8979999999999999,"15min":1.405},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.29699999999999999},"max":{"1min":10.648,"5min":12.460000000000001,"15min":12.460000000000001},"last":0.83999999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:23:39 2023","interfaces":[{"interface":"back","average":{"1min":1.9930000000000001,"5min":1.484,"15min":1.385},"min":{"1min":0.35199999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":7.4710000000000001,"5min":8.2910000000000004,"15min":9.9190000000000005},"last":0.88200000000000001},{"interface":"front","average":{"1min":1.8999999999999999,"5min":1.819,"15min":1.663},"min":{"1min":0.48499999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":6.2750000000000004,"5min":11.205,"15min":11.205},"last":1.278}]},{"osd":6,"last update":"Mon Oct 9 05:23:39 2023","interfaces":[{"interface":"back","average":{"1min":1.6910000000000001,"5min":2.0510000000000002,"15min":1.595},"min":{"1min":0.40200000000000002,"5min":0.33400000000000002,"15min":0.32500000000000001},"max":{"1min":8.4819999999999993,"5min":12.503,"15min":16.337},"last":1.101},{"interface":"front","average":{"1min":2.125,"5min":1.5309999999999999,"15min":1.514},"min":{"1min":0.41799999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":7.4349999999999996,"5min":7.4349999999999996,"15min":16.399999999999999},"last":1.3129999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:23:39 2023","interfaces":[{"interface":"back","average":{"1min":2.5800000000000001,"5min":1.597,"15min":1.458},"min":{"1min":0.44500000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":9.6340000000000003,"5min":9.6340000000000003,"15min":9.9410000000000007},"last":1.1619999999999999},{"interface":"front","average":{"1min":1.849,"5min":1.9430000000000001,"15min":1.6299999999999999},"min":{"1min":0.48099999999999998,"5min":0.379,"15min":0.379},"max":{"1min":6.9850000000000003,"5min":11.236000000000001,"15min":11.236000000000001},"last":1.359}]}]},{"osd":3,"up_from":123,"seq":528280977680,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":742592,"kb_used_data":3192,"kb_used_omap":2,"kb_used_meta":739389,"kb_avail":93002560,"statfs":{"total":95995035648,"available":95234621440,"internally_reserved":0,"allocated":3268608,"data_stored":1944686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":757134389},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:23:51 2023","interfaces":[{"interface":"back","average":{"1min":2.2410000000000001,"5min":3.8639999999999999,"15min":3.093},"min":{"1min":0.224,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":10.993,"5min":199.84999999999999,"15min":199.84999999999999},"last":0.71199999999999997},{"interface":"front","average":{"1min":1.4790000000000001,"5min":1.9950000000000001,"15min":2.6240000000000001},"min":{"1min":0.251,"5min":0.22900000000000001,"15min":0.188},"max":{"1min":5.218,"5min":10.045999999999999,"15min":83.234999999999999},"last":0.66500000000000004}]},{"osd":1,"last update":"Mon Oct 9 05:23:51 2023","interfaces":[{"interface":"back","average":{"1min":2.4350000000000001,"5min":2.335,"15min":2.2839999999999998},"min":{"1min":0.34000000000000002,"5min":0.22,"15min":0.22},"max":{"1min":10.249000000000001,"5min":16.096,"15min":44.206000000000003},"last":0.98499999999999999},{"interface":"front","average":{"1min":2.222,"5min":2.2879999999999998,"15min":2.0960000000000001},"min":{"1min":0.29999999999999999,"5min":0.26200000000000001,"15min":0.218},"max":{"1min":10.944000000000001,"5min":13.141,"15min":44.496000000000002},"last":0.82099999999999995}]},{"osd":2,"last update":"Mon Oct 9 05:23:51 2023","interfaces":[{"interface":"back","average":{"1min":2.2370000000000001,"5min":2.0510000000000002,"15min":1.7410000000000001},"min":{"1min":0.33100000000000002,"5min":0.253,"15min":0.22700000000000001},"max":{"1min":9.1679999999999993,"5min":9.9960000000000004,"15min":12.794},"last":0.86399999999999999},{"interface":"front","average":{"1min":2.2440000000000002,"5min":2.4620000000000002,"15min":2.1070000000000002},"min":{"1min":0.35899999999999999,"5min":0.251,"15min":0.22800000000000001},"max":{"1min":8.5329999999999995,"5min":16.009,"15min":16.009},"last":0.77100000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:23:51 2023","interfaces":[{"interface":"back","average":{"1min":2.3010000000000002,"5min":1.956,"15min":1.772},"min":{"1min":0.442,"5min":0.314,"15min":0.314},"max":{"1min":8.4760000000000009,"5min":8.4760000000000009,"15min":14.833},"last":1.0840000000000001},{"interface":"front","average":{"1min":1.577,"5min":3.5790000000000002,"15min":2.1579999999999999},"min":{"1min":0.40699999999999997,"5min":0.33300000000000002,"15min":0.316},"max":{"1min":6.782,"5min":199.50800000000001,"15min":199.50800000000001},"last":0.95299999999999996}]},{"osd":5,"last update":"Mon Oct 9 05:23:51 2023","interfaces":[{"interface":"back","average":{"1min":1.5980000000000001,"5min":3.5739999999999998,"15min":2.2050000000000001},"min":{"1min":0.48699999999999999,"5min":0.35999999999999999,"15min":0.33500000000000002},"max":{"1min":6.8550000000000004,"5min":199.93799999999999,"15min":199.93799999999999},"last":1.0569999999999999},{"interface":"front","average":{"1min":1.032,"5min":1.405,"15min":1.623},"min":{"1min":0.45300000000000001,"5min":0.34899999999999998,"15min":0.33400000000000002},"max":{"1min":2.552,"5min":10.452999999999999,"15min":11.135},"last":1.1619999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:23:51 2023","interfaces":[{"interface":"back","average":{"1min":1.53,"5min":1.5620000000000001,"15min":1.627},"min":{"1min":0.41599999999999998,"5min":0.375,"15min":0.36099999999999999},"max":{"1min":6.7699999999999996,"5min":10.411,"15min":11.173999999999999},"last":1.1000000000000001},{"interface":"front","average":{"1min":2.2480000000000002,"5min":2.0070000000000001,"15min":1.9770000000000001},"min":{"1min":0.46800000000000003,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":8.4440000000000008,"5min":8.4440000000000008,"15min":14.881},"last":1.272}]},{"osd":7,"last update":"Mon Oct 9 05:23:51 2023","interfaces":[{"interface":"back","average":{"1min":2.0680000000000001,"5min":1.8520000000000001,"15min":1.865},"min":{"1min":0.41299999999999998,"5min":0.38800000000000001,"15min":0.378},"max":{"1min":8.5039999999999996,"5min":8.5039999999999996,"15min":14.856},"last":1.2210000000000001},{"interface":"front","average":{"1min":2.073,"5min":3.7970000000000002,"15min":2.4159999999999999},"min":{"1min":0.45600000000000002,"5min":0.43099999999999999,"15min":0.38500000000000001},"max":{"1min":7.6879999999999997,"5min":199.96799999999999,"15min":199.96799999999999},"last":1.236}]}]},{"osd":4,"up_from":30,"seq":128849019437,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1253412,"kb_used_data":360860,"kb_used_omap":0,"kb_used_meta":892544,"kb_avail":92491740,"statfs":{"total":95995035648,"available":94711541760,"internally_reserved":0,"allocated":369520640,"data_stored":368633305,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":913965056},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:23:41 2023","interfaces":[{"interface":"back","average":{"1min":1.52,"5min":1.2969999999999999,"15min":2.0920000000000001},"min":{"1min":0.30599999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":5.7190000000000003,"5min":12.468999999999999,"15min":102.669},"last":1.3460000000000001},{"interface":"front","average":{"1min":1.6850000000000001,"5min":1.4399999999999999,"15min":2.113},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.308},"max":{"1min":5.6970000000000001,"5min":12.407999999999999,"15min":102.72499999999999},"last":1.123}]},{"osd":1,"last update":"Mon Oct 9 05:24:34 2023","interfaces":[{"interface":"back","average":{"1min":0.83999999999999997,"5min":3.8700000000000001,"15min":2.1600000000000001},"min":{"1min":0.34799999999999998,"5min":0.30399999999999999,"15min":0.25700000000000001},"max":{"1min":3.5379999999999998,"5min":142.297,"15min":142.297},"last":0.85299999999999998},{"interface":"front","average":{"1min":1.0509999999999999,"5min":3.8100000000000001,"15min":2.0310000000000001},"min":{"1min":0.42799999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":3.4609999999999999,"5min":135.11600000000001,"15min":135.11600000000001},"last":1.591}]},{"osd":2,"last update":"Mon Oct 9 05:23:41 2023","interfaces":[{"interface":"back","average":{"1min":2.3559999999999999,"5min":1.679,"15min":1.46},"min":{"1min":0.38500000000000001,"5min":0.318,"15min":0.29499999999999998},"max":{"1min":7.6529999999999996,"5min":18.076000000000001,"15min":66.201999999999998},"last":1.2410000000000001},{"interface":"front","average":{"1min":2.0510000000000002,"5min":1.593,"15min":1.5820000000000001},"min":{"1min":0.32700000000000001,"5min":0.308,"15min":0.29599999999999999},"max":{"1min":8.6440000000000001,"5min":18.68,"15min":66.361000000000004},"last":0.747}]},{"osd":3,"last update":"Mon Oct 9 05:23:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3180000000000001,"5min":3.6349999999999998,"15min":2.8109999999999999},"min":{"1min":0.40500000000000003,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":4.3600000000000003,"5min":131.726,"15min":131.726},"last":1.4299999999999999},{"interface":"front","average":{"1min":2.0289999999999999,"5min":4.1799999999999997,"15min":2.8530000000000002},"min":{"1min":0.439,"5min":0.36199999999999999,"15min":0.35699999999999998},"max":{"1min":8.077,"5min":131.57300000000001,"15min":131.57300000000001},"last":0.99299999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:23:48 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.75600000000000001,"15min":0.89600000000000002},"min":{"1min":0.373,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.544,"5min":3.544,"15min":7.7549999999999999},"last":1.512},{"interface":"front","average":{"1min":0.85499999999999998,"5min":0.872,"15min":1.0589999999999999},"min":{"1min":0.44500000000000001,"5min":0.36799999999999999,"15min":0.34799999999999998},"max":{"1min":1.532,"5min":6.1079999999999997,"15min":6.1079999999999997},"last":1.228}]},{"osd":6,"last update":"Mon Oct 9 05:24:34 2023","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":1.107,"15min":1.121},"min":{"1min":0.53100000000000003,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.0910000000000002,"5min":5.5049999999999999,"15min":6.3760000000000003},"last":1.8340000000000001},{"interface":"front","average":{"1min":1.1399999999999999,"5min":0.94799999999999995,"15min":1.052},"min":{"1min":0.53200000000000003,"5min":0.36599999999999999,"15min":0.35999999999999999},"max":{"1min":1.645,"5min":3.6600000000000001,"15min":8.4930000000000003},"last":1.669}]},{"osd":7,"last update":"Mon Oct 9 05:24:34 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.0840000000000001,"15min":1.089},"min":{"1min":0.58199999999999996,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.518,"5min":7.7690000000000001,"15min":7.7690000000000001},"last":1.1870000000000001},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.129,"15min":1.264},"min":{"1min":0.73299999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.4239999999999999,"5min":8.0269999999999992,"15min":8.9689999999999994},"last":1.7130000000000001}]}]},{"osd":5,"up_from":0,"seq":163208757793,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:24:45.761 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:24:45.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:45.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:45.862 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:45.894 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:45.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:45.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:45.997 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:46.022 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:46.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:46.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:46.124 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:46.151 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:46.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:46.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:46.255 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:46.279 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:46.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:46.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:46.382 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:46.414 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:46.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:46.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:46.518 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:46.554 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:46.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:46.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:46.658 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:46.685 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:46.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:46.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:46.789 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:46.819 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:46.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:46.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:46.840 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:24:46.923 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:46.956 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:46.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:46.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:47.060 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:47.081 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:47.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:47.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:47.185 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:47.218 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:47.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:47.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:47.322 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:47.355 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:47.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:47.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:47.459 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:47.484 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:47.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:47.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:47.587 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:47.599 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:47 smithi194 bash[92666]: cluster 2023-10-09T05:24:46.129980+0000 mgr.y (mgr.25277) 1095 : cluster 0 pgmap v1160: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:47.604 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:47 smithi133 bash[102858]: cluster 2023-10-09T05:24:46.129980+0000 mgr.y (mgr.25277) 1095 : cluster 0 pgmap v1160: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:47.604 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:47 smithi133 bash[105589]: cluster 2023-10-09T05:24:46.129980+0000 mgr.y (mgr.25277) 1095 : cluster 0 pgmap v1160: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:47.618 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:47.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:47.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:47.722 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:47.754 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:47.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:47.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:47.858 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:47.888 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:47.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:47.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:47.991 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:48.029 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:48.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:48.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:48.133 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:48.167 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:48.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:48.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:48.271 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:48.303 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:48.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:48.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:48.365 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:24:48.408 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:48.442 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:48.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:48.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:48.545 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:48.565 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:48.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:48.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:48.669 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:48.730 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:48.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:48.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:48.833 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:48.865 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:48.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:48.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:48.970 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:49.002 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:49.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:49.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:49.105 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:49.137 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:49.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:49.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:49.241 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:49.262 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:49.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:49.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:49.365 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:49.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:49 smithi194 bash[92666]: cluster 2023-10-09T05:24:48.130598+0000 mgr.y (mgr.25277) 1096 : cluster 0 pgmap v1161: 73 pgs: 73 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:49.399 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:49.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:49.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:49.503 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:49.534 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:49.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:49.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:49.637 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:49.669 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:49.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:49.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:49.773 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:49.781 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:49 smithi133 bash[105589]: cluster 2023-10-09T05:24:48.130598+0000 mgr.y (mgr.25277) 1096 : cluster 0 pgmap v1161: 73 pgs: 73 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:49.781 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:49 smithi133 bash[102858]: cluster 2023-10-09T05:24:48.130598+0000 mgr.y (mgr.25277) 1096 : cluster 0 pgmap v1161: 73 pgs: 73 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:49.798 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:49.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:49.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:49.901 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:49.936 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:49.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:49.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:50.040 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:50.065 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:50.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:50.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:50.169 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:50.202 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:50.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:50.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:50.305 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:50.332 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:50.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:50.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:50.435 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:50.464 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:50.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:50.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:50.567 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:50.603 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:50.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:50.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:50.618 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:24:50.642 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:24:50.664 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:24:50.686 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:24:50.707 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:50.713 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:24:50.731 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:50.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:50.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:50.740 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:24:50.763 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:24:50.785 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:24:50.788 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:24:50.788 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:24:50.788 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:24:50.788 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:24:50.788 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:24:50.788 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:24:50.788 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:24:50.788 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:24:50.834 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:50.864 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:50.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:50.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:50.968 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:51.002 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:51.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:51.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:51.106 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:51.131 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:51.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:51.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:51.235 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:51.267 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:51.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:51.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:51.371 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:51.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:51 smithi194 bash[92666]: cluster 2023-10-09T05:24:50.131542+0000 mgr.y (mgr.25277) 1097 : cluster 0 pgmap v1162: 73 pgs: 73 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:51.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:51 smithi194 bash[92666]: audit 2023-10-09T05:24:50.596270+0000 mon.a (mon.0) 1437 : audit 0 from='client.? 172.21.15.133:0/976199594' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:24:51.394 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:51.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:51.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:51.497 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:51.505 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:51 smithi133 bash[102858]: cluster 2023-10-09T05:24:50.131542+0000 mgr.y (mgr.25277) 1097 : cluster 0 pgmap v1162: 73 pgs: 73 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:51.505 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:51 smithi133 bash[102858]: audit 2023-10-09T05:24:50.596270+0000 mon.a (mon.0) 1437 : audit 0 from='client.? 172.21.15.133:0/976199594' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:24:51.506 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:51 smithi133 bash[105589]: cluster 2023-10-09T05:24:50.131542+0000 mgr.y (mgr.25277) 1097 : cluster 0 pgmap v1162: 73 pgs: 73 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:51.506 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:51 smithi133 bash[105589]: audit 2023-10-09T05:24:50.596270+0000 mon.a (mon.0) 1437 : audit 0 from='client.? 172.21.15.133:0/976199594' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:24:51.524 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:51.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:51.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:51.628 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:51.659 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:51.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:51.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:51.763 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:51.785 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:51.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:51.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:51.888 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:51.915 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:51.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:51.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:52.018 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:52.051 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:52.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:52.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:52.155 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:52.180 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:52.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:52.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:52.283 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:52.318 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:52.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:52.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:52.422 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:52.443 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:52.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:52.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:52.547 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:52.582 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:52.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:52.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:52.685 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:52.711 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:52.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:52.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:52.815 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:52.841 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:52.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:52.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:52.945 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:52.975 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:52.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:52.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:53.079 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:53.106 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:53.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:53.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:53.209 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:53.218 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:53 smithi133 bash[105589]: cluster 2023-10-09T05:24:52.132277+0000 mgr.y (mgr.25277) 1098 : cluster 0 pgmap v1163: 73 pgs: 73 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:53.218 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:53 smithi133 bash[102858]: cluster 2023-10-09T05:24:52.132277+0000 mgr.y (mgr.25277) 1098 : cluster 0 pgmap v1163: 73 pgs: 73 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:53.233 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:53.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:53.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:53.337 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:53.374 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:53.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:53.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:53.477 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:53.490 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:53 smithi194 bash[92666]: cluster 2023-10-09T05:24:52.132277+0000 mgr.y (mgr.25277) 1098 : cluster 0 pgmap v1163: 73 pgs: 73 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:53.506 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:53.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:53.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:53.611 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:53.635 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:53.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:53.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:53.738 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:53.771 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:53.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:53.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:53.876 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:53.907 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:53.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:53.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:54.012 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:54.038 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:54.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:54.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:54.142 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:54.171 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:54.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:54.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:54.276 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:54.305 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:54.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:54.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:54.410 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:54.436 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:54.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:54.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:54.540 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:54.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:24:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:24:54] "GET /metrics HTTP/1.1" 200 34060 "" "Prometheus/2.43.0" 2023-10-09T05:24:54.570 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:24:54.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:54.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:24:54.673 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:54.696 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:54.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:54.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:54.799 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:54.830 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:54.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:54.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:54.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:54 smithi194 bash[92666]: cluster 2023-10-09T05:24:54.132917+0000 mgr.y (mgr.25277) 1099 : cluster 0 pgmap v1164: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:54.898 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:54 smithi133 bash[105589]: cluster 2023-10-09T05:24:54.132917+0000 mgr.y (mgr.25277) 1099 : cluster 0 pgmap v1164: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:54.898 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:54 smithi133 bash[102858]: cluster 2023-10-09T05:24:54.132917+0000 mgr.y (mgr.25277) 1099 : cluster 0 pgmap v1164: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:54.934 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:54.955 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:54.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:54.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:54.965 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:24:54.965 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:24:55.059 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:55.095 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:55.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:55.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:55.198 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:55.219 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:55.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:55.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:55.323 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:55.347 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:55.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:55.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:55.450 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:55.485 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:55.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:55.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:55.589 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:55.625 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:55.625 INFO:teuthology.orchestra.run.smithi133.stderr:pool 'unique_pool_7' created 2023-10-09T05:24:55.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:55.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:55.729 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:55.741 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:55 smithi194 bash[92666]: audit 2023-10-09T05:24:54.933186+0000 mon.c (mon.2) 101 : audit 1 from='client.? 172.21.15.133:0/1326471424' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-10-09T05:24:55.741 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:55 smithi194 bash[92666]: audit 2023-10-09T05:24:54.933723+0000 mon.a (mon.0) 1438 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-10-09T05:24:55.761 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:55.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:55.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:55.865 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:55.897 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:55.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:55.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:56.002 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:56.029 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:56.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:56.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:56.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:55 smithi133 bash[102858]: audit 2023-10-09T05:24:54.933186+0000 mon.c (mon.2) 101 : audit 1 from='client.? 172.21.15.133:0/1326471424' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-10-09T05:24:56.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:55 smithi133 bash[102858]: audit 2023-10-09T05:24:54.933723+0000 mon.a (mon.0) 1438 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-10-09T05:24:56.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:55 smithi133 bash[105589]: audit 2023-10-09T05:24:54.933186+0000 mon.c (mon.2) 101 : audit 1 from='client.? 172.21.15.133:0/1326471424' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-10-09T05:24:56.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:55 smithi133 bash[105589]: audit 2023-10-09T05:24:54.933723+0000 mon.a (mon.0) 1438 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-10-09T05:24:56.135 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:56.159 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:56.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:56.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:56.262 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:56.297 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:24:56.299 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:56.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:56.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:56.403 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:56.421 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:56.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:56.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:56.524 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:56.551 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:56.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:56.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:56.654 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:56.662 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:56 smithi133 bash[102858]: cluster 2023-10-09T05:24:55.614076+0000 osd.7 (osd.7) 212 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T05:24:56.662 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:56 smithi133 bash[102858]: audit 2023-10-09T05:24:55.614934+0000 mon.a (mon.0) 1439 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2023-10-09T05:24:56.662 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:56 smithi133 bash[102858]: cluster 2023-10-09T05:24:55.615047+0000 mon.a (mon.0) 1440 : cluster 0 osdmap e264: 8 total, 8 up, 7 in 2023-10-09T05:24:56.663 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:56 smithi133 bash[102858]: cluster 2023-10-09T05:24:55.616115+0000 osd.5 (osd.5) 244 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T05:24:56.663 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:56 smithi133 bash[102858]: cluster 2023-10-09T05:24:55.618287+0000 osd.4 (osd.4) 235 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T05:24:56.663 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:56 smithi133 bash[102858]: cluster 2023-10-09T05:24:55.619752+0000 osd.6 (osd.6) 211 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T05:24:56.663 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:56 smithi133 bash[102858]: cluster 2023-10-09T05:24:55.622497+0000 osd.6 (osd.6) 212 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T05:24:56.663 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:56 smithi133 bash[102858]: cluster 2023-10-09T05:24:56.133644+0000 mgr.y (mgr.25277) 1100 : cluster 0 pgmap v1166: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:56.663 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:56 smithi133 bash[105589]: cluster 2023-10-09T05:24:55.614076+0000 osd.7 (osd.7) 212 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T05:24:56.663 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:56 smithi133 bash[105589]: audit 2023-10-09T05:24:55.614934+0000 mon.a (mon.0) 1439 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2023-10-09T05:24:56.664 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:56 smithi133 bash[105589]: cluster 2023-10-09T05:24:55.615047+0000 mon.a (mon.0) 1440 : cluster 0 osdmap e264: 8 total, 8 up, 7 in 2023-10-09T05:24:56.664 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:56 smithi133 bash[105589]: cluster 2023-10-09T05:24:55.616115+0000 osd.5 (osd.5) 244 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T05:24:56.664 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:56 smithi133 bash[105589]: cluster 2023-10-09T05:24:55.618287+0000 osd.4 (osd.4) 235 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T05:24:56.664 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:56 smithi133 bash[105589]: cluster 2023-10-09T05:24:55.619752+0000 osd.6 (osd.6) 211 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T05:24:56.664 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:56 smithi133 bash[105589]: cluster 2023-10-09T05:24:55.622497+0000 osd.6 (osd.6) 212 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T05:24:56.664 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:56 smithi133 bash[105589]: cluster 2023-10-09T05:24:56.133644+0000 mgr.y (mgr.25277) 1100 : cluster 0 pgmap v1166: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:56.675 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:56.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:56.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:56.778 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:56.811 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:56.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:56.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:56.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:56 smithi194 bash[92666]: cluster 2023-10-09T05:24:55.614076+0000 osd.7 (osd.7) 212 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T05:24:56.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:56 smithi194 bash[92666]: audit 2023-10-09T05:24:55.614934+0000 mon.a (mon.0) 1439 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2023-10-09T05:24:56.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:56 smithi194 bash[92666]: cluster 2023-10-09T05:24:55.615047+0000 mon.a (mon.0) 1440 : cluster 0 osdmap e264: 8 total, 8 up, 7 in 2023-10-09T05:24:56.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:56 smithi194 bash[92666]: cluster 2023-10-09T05:24:55.616115+0000 osd.5 (osd.5) 244 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T05:24:56.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:56 smithi194 bash[92666]: cluster 2023-10-09T05:24:55.618287+0000 osd.4 (osd.4) 235 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T05:24:56.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:56 smithi194 bash[92666]: cluster 2023-10-09T05:24:55.619752+0000 osd.6 (osd.6) 211 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T05:24:56.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:56 smithi194 bash[92666]: cluster 2023-10-09T05:24:55.622497+0000 osd.6 (osd.6) 212 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T05:24:56.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:56 smithi194 bash[92666]: cluster 2023-10-09T05:24:56.133644+0000 mgr.y (mgr.25277) 1100 : cluster 0 pgmap v1166: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:56.916 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:56.944 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:56.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:56.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:57.048 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:57.076 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:57.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:57.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:57.179 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:57.205 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:57.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:57.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:57.309 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:57.345 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:57.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:57.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:57.448 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:57.478 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:57.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:57.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:57.582 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:57.610 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:57.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:57.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:57.713 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:57.748 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:57.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:57.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:57.851 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:57.885 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:57.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:57 smithi194 bash[92666]: cluster 2023-10-09T05:24:56.619269+0000 osd.7 (osd.7) 213 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T05:24:57.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:57 smithi194 bash[92666]: cluster 2023-10-09T05:24:56.622232+0000 osd.5 (osd.5) 245 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T05:24:57.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:57 smithi194 bash[92666]: cluster 2023-10-09T05:24:56.624519+0000 mon.a (mon.0) 1441 : cluster 0 osdmap e265: 8 total, 8 up, 7 in 2023-10-09T05:24:57.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:57 smithi194 bash[92666]: cluster 2023-10-09T05:24:56.625133+0000 osd.7 (osd.7) 214 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T05:24:57.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:57 smithi194 bash[92666]: cluster 2023-10-09T05:24:56.625323+0000 osd.4 (osd.4) 236 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T05:24:57.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:57 smithi194 bash[92666]: cluster 2023-10-09T05:24:56.625739+0000 osd.6 (osd.6) 213 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T05:24:57.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:57.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:57.990 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:58.024 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:58.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:58.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:58.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:57 smithi133 bash[105589]: cluster 2023-10-09T05:24:56.619269+0000 osd.7 (osd.7) 213 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T05:24:58.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:57 smithi133 bash[105589]: cluster 2023-10-09T05:24:56.622232+0000 osd.5 (osd.5) 245 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T05:24:58.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:57 smithi133 bash[105589]: cluster 2023-10-09T05:24:56.624519+0000 mon.a (mon.0) 1441 : cluster 0 osdmap e265: 8 total, 8 up, 7 in 2023-10-09T05:24:58.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:57 smithi133 bash[105589]: cluster 2023-10-09T05:24:56.625133+0000 osd.7 (osd.7) 214 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T05:24:58.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:57 smithi133 bash[105589]: cluster 2023-10-09T05:24:56.625323+0000 osd.4 (osd.4) 236 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T05:24:58.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:57 smithi133 bash[105589]: cluster 2023-10-09T05:24:56.625739+0000 osd.6 (osd.6) 213 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T05:24:58.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:57 smithi133 bash[102858]: cluster 2023-10-09T05:24:56.619269+0000 osd.7 (osd.7) 213 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T05:24:58.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:57 smithi133 bash[102858]: cluster 2023-10-09T05:24:56.622232+0000 osd.5 (osd.5) 245 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T05:24:58.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:57 smithi133 bash[102858]: cluster 2023-10-09T05:24:56.624519+0000 mon.a (mon.0) 1441 : cluster 0 osdmap e265: 8 total, 8 up, 7 in 2023-10-09T05:24:58.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:57 smithi133 bash[102858]: cluster 2023-10-09T05:24:56.625133+0000 osd.7 (osd.7) 214 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T05:24:58.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:57 smithi133 bash[102858]: cluster 2023-10-09T05:24:56.625323+0000 osd.4 (osd.4) 236 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T05:24:58.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:57 smithi133 bash[102858]: cluster 2023-10-09T05:24:56.625739+0000 osd.6 (osd.6) 213 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T05:24:58.128 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:58.151 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:58.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:58.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:24:58.255 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:58.281 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:58.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:58.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:58.386 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:58.416 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:58.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:58.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:58.520 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:58.551 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:24:58.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:58.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:24:58.654 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:58.682 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:58.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:58.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:58.785 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:58.798 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:58 smithi194 bash[92666]: cluster 2023-10-09T05:24:57.628848+0000 osd.6 (osd.6) 214 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T05:24:58.798 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:58 smithi194 bash[92666]: cluster 2023-10-09T05:24:57.629433+0000 osd.5 (osd.5) 246 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T05:24:58.798 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:58 smithi194 bash[92666]: cluster 2023-10-09T05:24:57.629955+0000 mon.a (mon.0) 1442 : cluster 0 osdmap e266: 8 total, 8 up, 7 in 2023-10-09T05:24:58.798 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:58 smithi194 bash[92666]: cluster 2023-10-09T05:24:57.903851+0000 osd.4 (osd.4) 237 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T05:24:58.799 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:58 smithi194 bash[92666]: cluster 2023-10-09T05:24:58.134239+0000 mgr.y (mgr.25277) 1101 : cluster 0 pgmap v1169: 89 pgs: 5 creating+peering, 11 unknown, 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:58.799 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:24:58 smithi194 bash[92666]: audit 2023-10-09T05:24:58.461970+0000 mon.a (mon.0) 1443 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:24:58.817 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:58.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:58.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:58.922 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:58.954 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:24:58.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:58.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:24:58.999 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:58 smithi133 bash[102858]: cluster 2023-10-09T05:24:57.628848+0000 osd.6 (osd.6) 214 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T05:24:58.999 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:58 smithi133 bash[102858]: cluster 2023-10-09T05:24:57.629433+0000 osd.5 (osd.5) 246 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T05:24:58.999 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:58 smithi133 bash[102858]: cluster 2023-10-09T05:24:57.629955+0000 mon.a (mon.0) 1442 : cluster 0 osdmap e266: 8 total, 8 up, 7 in 2023-10-09T05:24:58.999 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:58 smithi133 bash[102858]: cluster 2023-10-09T05:24:57.903851+0000 osd.4 (osd.4) 237 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T05:24:58.999 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:58 smithi133 bash[102858]: cluster 2023-10-09T05:24:58.134239+0000 mgr.y (mgr.25277) 1101 : cluster 0 pgmap v1169: 89 pgs: 5 creating+peering, 11 unknown, 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:58.999 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:24:58 smithi133 bash[102858]: audit 2023-10-09T05:24:58.461970+0000 mon.a (mon.0) 1443 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:24:59.000 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:58 smithi133 bash[105589]: cluster 2023-10-09T05:24:57.628848+0000 osd.6 (osd.6) 214 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T05:24:59.000 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:58 smithi133 bash[105589]: cluster 2023-10-09T05:24:57.629433+0000 osd.5 (osd.5) 246 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T05:24:59.000 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:58 smithi133 bash[105589]: cluster 2023-10-09T05:24:57.629955+0000 mon.a (mon.0) 1442 : cluster 0 osdmap e266: 8 total, 8 up, 7 in 2023-10-09T05:24:59.000 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:58 smithi133 bash[105589]: cluster 2023-10-09T05:24:57.903851+0000 osd.4 (osd.4) 237 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T05:24:59.000 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:58 smithi133 bash[105589]: cluster 2023-10-09T05:24:58.134239+0000 mgr.y (mgr.25277) 1101 : cluster 0 pgmap v1169: 89 pgs: 5 creating+peering, 11 unknown, 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:24:59.000 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:24:58 smithi133 bash[105589]: audit 2023-10-09T05:24:58.461970+0000 mon.a (mon.0) 1443 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:24:59.058 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:59.087 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:59.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:59.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:59.124 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_7 rados --yes-i-really-mean-it || true 2023-10-09T05:24:59.190 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:59.212 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:24:59.212 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:24:59.227 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:24:59.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:59.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:24:59.331 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:59.360 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:59.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:59.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:59.463 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:59.493 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:24:59.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:59.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:24:59.598 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:59.625 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:24:59.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:59.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:24:59.730 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:59.761 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:24:59.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:59.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:24:59.866 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:59.899 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:24:59.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:24:59.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:00.002 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:00.025 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:00.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:00.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:00.128 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:00.158 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:00.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:00.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:00.262 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:00.296 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:00.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:00.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:00.400 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:00.432 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:00.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:00.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:00.536 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:00.564 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:00.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:00.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:00.668 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:00.692 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:00.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:00.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:00.796 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:00.822 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:00.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:00.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:00.926 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:00.961 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:00.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:00.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:01.065 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:01.090 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:01.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:01.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:01.195 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:01.212 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T05:25:01.230 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:01.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:01.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:01.334 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:01.341 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:01 smithi133 bash[102858]: cluster 2023-10-09T05:25:00.135243+0000 mgr.y (mgr.25277) 1102 : cluster 0 pgmap v1170: 89 pgs: 5 creating+peering, 84 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:01.341 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:01 smithi133 bash[105589]: cluster 2023-10-09T05:25:00.135243+0000 mgr.y (mgr.25277) 1102 : cluster 0 pgmap v1170: 89 pgs: 5 creating+peering, 84 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:01.352 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:01.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:01.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:01.456 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:01.476 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:01.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:01.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:01.579 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:01.596 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:01 smithi194 bash[92666]: cluster 2023-10-09T05:25:00.135243+0000 mgr.y (mgr.25277) 1102 : cluster 0 pgmap v1170: 89 pgs: 5 creating+peering, 84 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:01.604 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:01.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:01.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:01.708 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:01.743 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:01.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:01.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:01.846 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:01.872 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:01.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:01.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:01.975 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:02.006 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:02.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:02.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:02.109 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:02.143 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:02.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:02.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:02.247 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:02.282 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:02.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:02.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:02.386 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:02.415 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:02.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:02.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:02.519 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:02.541 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:02.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:02.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:02.646 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:02.677 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:02.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:02.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:02.781 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:02.812 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:02.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:02.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:02.916 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:02.941 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:02.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:02.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:03.044 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:03.066 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:03.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:03.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:03.170 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:03.204 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:03.205 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:25:03.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:03.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:03.308 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:03.338 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:03.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:03.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:03.443 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:03.454 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:03 smithi194 bash[92666]: cluster 2023-10-09T05:25:02.136038+0000 mgr.y (mgr.25277) 1103 : cluster 0 pgmap v1171: 89 pgs: 89 active+clean; 582 KiB data, 6.9 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:03.474 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:03.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:03.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:03.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:03 smithi133 bash[102858]: cluster 2023-10-09T05:25:02.136038+0000 mgr.y (mgr.25277) 1103 : cluster 0 pgmap v1171: 89 pgs: 89 active+clean; 582 KiB data, 6.9 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:03.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:03 smithi133 bash[105589]: cluster 2023-10-09T05:25:02.136038+0000 mgr.y (mgr.25277) 1103 : cluster 0 pgmap v1171: 89 pgs: 89 active+clean; 582 KiB data, 6.9 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:03.577 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:03.608 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:03.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:03.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:03.711 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:03.737 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:03.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:03.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:03.841 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:03.877 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:03.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:03.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:03.982 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:04.013 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:04.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:04.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:04.117 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:04.148 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:04.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:04.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:04.251 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:04.283 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:04.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:04.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:04.390 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:04.422 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:04.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:04.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:04.526 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:04.551 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:25:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:25:04] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.43.0" 2023-10-09T05:25:04.557 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:04.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:04.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:04.661 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:04.674 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:04 smithi194 bash[92666]: cluster 2023-10-09T05:25:04.136502+0000 mgr.y (mgr.25277) 1104 : cluster 0 pgmap v1172: 89 pgs: 89 active+clean; 582 KiB data, 6.9 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:04.692 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:04.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:04.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:04.796 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:04.819 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:04.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:04.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:04.922 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:04.955 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:04.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:04.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:05.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:04 smithi133 bash[102858]: cluster 2023-10-09T05:25:04.136502+0000 mgr.y (mgr.25277) 1104 : cluster 0 pgmap v1172: 89 pgs: 89 active+clean; 582 KiB data, 6.9 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:05.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:04 smithi133 bash[105589]: cluster 2023-10-09T05:25:04.136502+0000 mgr.y (mgr.25277) 1104 : cluster 0 pgmap v1172: 89 pgs: 89 active+clean; 582 KiB data, 6.9 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:05.059 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:05.092 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:05.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:05.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:05.112 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:25:05.113 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:25:05.197 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:05.219 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:05.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:05.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:05.322 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:05.354 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:05.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:05.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:05.459 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:05.488 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:05.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:05.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:05.592 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:05.625 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:05.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:05.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:05.729 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:05.760 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:05.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:05.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:05.864 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:05.889 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:05.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:05.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:05.992 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:06.024 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:06.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:06.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:06.129 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:06.160 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:06.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:06.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:06.264 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:06.293 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:06.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:06.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:06.397 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:06.425 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:06.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:06.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:06.528 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:06.562 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:06.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:06.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:06.665 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:06.687 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:06.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:06.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:06.791 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:06.822 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:06.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:06.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:06.925 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:06.947 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:06.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:06.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:07.051 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:07.075 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:07.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:07.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:07.178 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:07.209 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:07.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:07.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:07.313 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:07.332 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:07.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:07.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:07.435 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:07.452 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:07.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:07.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:07.481 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:25:07.483 INFO:teuthology.orchestra.run.smithi133.stderr:dumped all 2023-10-09T05:25:07.556 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:07.570 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:07 smithi194 bash[92666]: cluster 2023-10-09T05:25:06.137235+0000 mgr.y (mgr.25277) 1105 : cluster 0 pgmap v1173: 89 pgs: 89 active+clean; 582 KiB data, 6.9 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:07.592 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:07.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:07.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:07.623 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:07 smithi133 bash[102858]: cluster 2023-10-09T05:25:06.137235+0000 mgr.y (mgr.25277) 1105 : cluster 0 pgmap v1173: 89 pgs: 89 active+clean; 582 KiB data, 6.9 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:07.623 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:07 smithi133 bash[105589]: cluster 2023-10-09T05:25:06.137235+0000 mgr.y (mgr.25277) 1105 : cluster 0 pgmap v1173: 89 pgs: 89 active+clean; 582 KiB data, 6.9 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:07.696 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:07.722 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:07.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:07.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:07.826 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:07.860 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:07.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:07.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:07.965 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:07.996 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:07.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:07.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:08.100 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:08.129 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:08.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:08.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:08.234 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:08.265 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:08.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:08.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:08.369 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:08.395 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:08.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:08.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:08.499 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:08.521 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:08.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:08.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:08.624 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:08.653 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:08.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:08.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:08.757 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:08.785 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:08.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:08.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:08.890 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:08.912 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:08.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:08.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:09.015 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:09.051 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:09.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:09.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:09.155 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:09.182 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:09.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:09.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:09.285 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:09.310 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:09.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:09.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:09.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:09 smithi194 bash[92666]: audit 2023-10-09T05:25:07.467821+0000 mgr.y (mgr.25277) 1106 : audit 0 from='client.35376 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:25:09.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:09 smithi194 bash[92666]: cluster 2023-10-09T05:25:08.137727+0000 mgr.y (mgr.25277) 1107 : cluster 0 pgmap v1174: 89 pgs: 89 active+clean; 582 KiB data, 6.9 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:09.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:09 smithi194 bash[92666]: audit 2023-10-09T05:25:09.064238+0000 mon.a (mon.0) 1444 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:09.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:09 smithi194 bash[92666]: audit 2023-10-09T05:25:09.064522+0000 mon.a (mon.0) 1445 : audit 1 from='client.? 172.21.15.133:0/1549287280' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:25:09.413 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:09.445 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:09.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:09.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:09.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:09 smithi133 bash[102858]: audit 2023-10-09T05:25:07.467821+0000 mgr.y (mgr.25277) 1106 : audit 0 from='client.35376 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:25:09.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:09 smithi133 bash[102858]: cluster 2023-10-09T05:25:08.137727+0000 mgr.y (mgr.25277) 1107 : cluster 0 pgmap v1174: 89 pgs: 89 active+clean; 582 KiB data, 6.9 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:09.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:09 smithi133 bash[102858]: audit 2023-10-09T05:25:09.064238+0000 mon.a (mon.0) 1444 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:09.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:09 smithi133 bash[102858]: audit 2023-10-09T05:25:09.064522+0000 mon.a (mon.0) 1445 : audit 1 from='client.? 172.21.15.133:0/1549287280' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:25:09.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:09 smithi133 bash[105589]: audit 2023-10-09T05:25:07.467821+0000 mgr.y (mgr.25277) 1106 : audit 0 from='client.35376 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:25:09.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:09 smithi133 bash[105589]: cluster 2023-10-09T05:25:08.137727+0000 mgr.y (mgr.25277) 1107 : cluster 0 pgmap v1174: 89 pgs: 89 active+clean; 582 KiB data, 6.9 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:09.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:09 smithi133 bash[105589]: audit 2023-10-09T05:25:09.064238+0000 mon.a (mon.0) 1444 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:09.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:09 smithi133 bash[105589]: audit 2023-10-09T05:25:09.064522+0000 mon.a (mon.0) 1445 : audit 1 from='client.? 172.21.15.133:0/1549287280' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:25:09.549 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:09.582 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:09.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:09.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:09.686 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:09.711 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:09.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:09.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:09.814 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:09.845 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:09.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:09.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:09.948 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:09.980 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:09.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:09.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:10.074 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is unset 2023-10-09T05:25:10.084 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:10.113 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:10.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:10.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:10.217 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:10.248 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:10.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:10.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:10.352 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:10.374 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:10.376 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:10 smithi133 bash[105589]: cluster 2023-10-09T05:25:10.061592+0000 mon.a (mon.0) 1446 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:25:10.376 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:10 smithi133 bash[105589]: cluster 2023-10-09T05:25:10.061626+0000 mon.a (mon.0) 1447 : cluster 1 Cluster is now healthy 2023-10-09T05:25:10.376 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:10 smithi133 bash[105589]: cluster 2023-10-09T05:25:10.067530+0000 osd.5 (osd.5) 247 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T05:25:10.376 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:10 smithi133 bash[105589]: audit 2023-10-09T05:25:10.070182+0000 mon.a (mon.0) 1448 : audit 1 from='client.? 172.21.15.133:0/1549287280' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:25:10.377 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:10 smithi133 bash[105589]: cluster 2023-10-09T05:25:10.070293+0000 mon.a (mon.0) 1449 : cluster 0 osdmap e267: 8 total, 8 up, 7 in 2023-10-09T05:25:10.377 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:10 smithi133 bash[102858]: cluster 2023-10-09T05:25:10.061592+0000 mon.a (mon.0) 1446 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:25:10.377 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:10 smithi133 bash[102858]: cluster 2023-10-09T05:25:10.061626+0000 mon.a (mon.0) 1447 : cluster 1 Cluster is now healthy 2023-10-09T05:25:10.377 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:10 smithi133 bash[102858]: cluster 2023-10-09T05:25:10.067530+0000 osd.5 (osd.5) 247 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T05:25:10.377 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:10 smithi133 bash[102858]: audit 2023-10-09T05:25:10.070182+0000 mon.a (mon.0) 1448 : audit 1 from='client.? 172.21.15.133:0/1549287280' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:25:10.378 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:10 smithi133 bash[102858]: cluster 2023-10-09T05:25:10.070293+0000 mon.a (mon.0) 1449 : cluster 0 osdmap e267: 8 total, 8 up, 7 in 2023-10-09T05:25:10.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:10.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:10.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:10 smithi194 bash[92666]: cluster 2023-10-09T05:25:10.061592+0000 mon.a (mon.0) 1446 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:25:10.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:10 smithi194 bash[92666]: cluster 2023-10-09T05:25:10.061626+0000 mon.a (mon.0) 1447 : cluster 1 Cluster is now healthy 2023-10-09T05:25:10.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:10 smithi194 bash[92666]: cluster 2023-10-09T05:25:10.067530+0000 osd.5 (osd.5) 247 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T05:25:10.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:10 smithi194 bash[92666]: audit 2023-10-09T05:25:10.070182+0000 mon.a (mon.0) 1448 : audit 1 from='client.? 172.21.15.133:0/1549287280' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:25:10.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:10 smithi194 bash[92666]: cluster 2023-10-09T05:25:10.070293+0000 mon.a (mon.0) 1449 : cluster 0 osdmap e267: 8 total, 8 up, 7 in 2023-10-09T05:25:10.435 INFO:teuthology.orchestra.run.smithi133.stdout:{"pg_ready":true,"pg_map":{"version":1173,"stamp":"2023-10-09T05:25:06.136784+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":506,"num_read_kb":931,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":4154034,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":222,"ondisk_log_size":222,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":273,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":7241524,"kb_used_data":1577596,"kb_used_omap":20,"kb_used_meta":5663851,"kb_avail":648974540,"statfs":{"total":671965249536,"available":664549928960,"internally_reserved":0,"allocated":1615458304,"data_stored":1607118882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":5799784078},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.518494"},"pg_stats":[{"pgid":"3.1a","version":"69'2","reported_seq":266,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.637131+0000","last_change":"2023-10-09T05:09:57.873320+0000","last_active":"2023-10-09T05:24:56.637131+0000","last_peered":"2023-10-09T05:24:56.637131+0000","last_clean":"2023-10-09T05:24:56.637131+0000","last_became_active":"2023-10-09T05:09:57.868376+0000","last_became_peered":"2023-10-09T05:09:57.868376+0000","last_unstale":"2023-10-09T05:24:56.637131+0000","last_undegraded":"2023-10-09T05:24:56.637131+0000","last_fullsized":"2023-10-09T05:24:56.637131+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:27.186610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":248,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.626659+0000","last_change":"2023-10-09T05:14:15.345991+0000","last_active":"2023-10-09T05:24:56.626659+0000","last_peered":"2023-10-09T05:24:56.626659+0000","last_clean":"2023-10-09T05:24:56.626659+0000","last_became_active":"2023-10-09T05:14:15.345511+0000","last_became_peered":"2023-10-09T05:14:15.345511+0000","last_unstale":"2023-10-09T05:24:56.626659+0000","last_undegraded":"2023-10-09T05:24:56.626659+0000","last_fullsized":"2023-10-09T05:24:56.626659+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:49.234025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":131,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.622752+0000","last_change":"2023-10-09T05:09:57.857511+0000","last_active":"2023-10-09T05:24:56.622752+0000","last_peered":"2023-10-09T05:24:56.622752+0000","last_clean":"2023-10-09T05:24:56.622752+0000","last_became_active":"2023-10-09T05:09:57.857193+0000","last_became_peered":"2023-10-09T05:09:57.857193+0000","last_unstale":"2023-10-09T05:24:56.622752+0000","last_undegraded":"2023-10-09T05:24:56.622752+0000","last_fullsized":"2023-10-09T05:24:56.622752+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:32.842980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":265,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-10-09T05:24:57.952560+0000","last_change":"2023-10-09T05:18:26.422923+0000","last_active":"2023-10-09T05:24:57.952560+0000","last_peered":"2023-10-09T05:24:57.952560+0000","last_clean":"2023-10-09T05:24:57.952560+0000","last_became_active":"2023-10-09T05:18:26.422508+0000","last_became_peered":"2023-10-09T05:18:26.422508+0000","last_unstale":"2023-10-09T05:24:57.952560+0000","last_undegraded":"2023-10-09T05:24:57.952560+0000","last_fullsized":"2023-10-09T05:24:57.952560+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:00.876037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":284,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.626858+0000","last_change":"2023-10-09T05:23:18.188378+0000","last_active":"2023-10-09T05:24:56.626858+0000","last_peered":"2023-10-09T05:24:56.626858+0000","last_clean":"2023-10-09T05:24:56.626858+0000","last_became_active":"2023-10-09T05:23:18.186629+0000","last_became_peered":"2023-10-09T05:23:18.186629+0000","last_unstale":"2023-10-09T05:24:56.626858+0000","last_undegraded":"2023-10-09T05:24:56.626858+0000","last_fullsized":"2023-10-09T05:24:56.626858+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:16.444225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":256,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.642609+0000","last_change":"2023-10-09T05:13:03.039637+0000","last_active":"2023-10-09T05:24:56.642609+0000","last_peered":"2023-10-09T05:24:56.642609+0000","last_clean":"2023-10-09T05:24:56.642609+0000","last_became_active":"2023-10-09T05:13:03.039268+0000","last_became_peered":"2023-10-09T05:13:03.039268+0000","last_unstale":"2023-10-09T05:24:56.642609+0000","last_undegraded":"2023-10-09T05:24:56.642609+0000","last_fullsized":"2023-10-09T05:24:56.642609+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:19.875767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":252,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.629507+0000","last_change":"2023-10-09T05:22:47.216769+0000","last_active":"2023-10-09T05:24:56.629507+0000","last_peered":"2023-10-09T05:24:56.629507+0000","last_clean":"2023-10-09T05:24:56.629507+0000","last_became_active":"2023-10-09T05:22:47.215847+0000","last_became_peered":"2023-10-09T05:22:47.215847+0000","last_unstale":"2023-10-09T05:24:56.629507+0000","last_undegraded":"2023-10-09T05:24:56.629507+0000","last_fullsized":"2023-10-09T05:24:56.629507+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:51.392675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":239,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.626951+0000","last_change":"2023-10-09T05:09:57.849695+0000","last_active":"2023-10-09T05:24:56.626951+0000","last_peered":"2023-10-09T05:24:56.626951+0000","last_clean":"2023-10-09T05:24:56.626951+0000","last_became_active":"2023-10-09T05:09:57.848478+0000","last_became_peered":"2023-10-09T05:09:57.848478+0000","last_unstale":"2023-10-09T05:24:56.626951+0000","last_undegraded":"2023-10-09T05:24:56.626951+0000","last_fullsized":"2023-10-09T05:24:56.626951+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:26.115312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":75,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.630254+0000","last_change":"2023-10-09T05:23:18.194144+0000","last_active":"2023-10-09T05:24:56.630254+0000","last_peered":"2023-10-09T05:24:56.630254+0000","last_clean":"2023-10-09T05:24:56.630254+0000","last_became_active":"2023-10-09T05:23:18.193390+0000","last_became_peered":"2023-10-09T05:23:18.193390+0000","last_unstale":"2023-10-09T05:24:56.630254+0000","last_undegraded":"2023-10-09T05:24:56.630254+0000","last_fullsized":"2023-10-09T05:24:56.630254+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:22.549362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":252,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.628242+0000","last_change":"2023-10-09T05:18:26.418431+0000","last_active":"2023-10-09T05:24:56.628242+0000","last_peered":"2023-10-09T05:24:56.628242+0000","last_clean":"2023-10-09T05:24:56.628242+0000","last_became_active":"2023-10-09T05:18:26.418100+0000","last_became_peered":"2023-10-09T05:18:26.418100+0000","last_unstale":"2023-10-09T05:24:56.628242+0000","last_undegraded":"2023-10-09T05:24:56.628242+0000","last_fullsized":"2023-10-09T05:24:56.628242+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:21.241286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":61,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.630612+0000","last_change":"2023-10-09T05:18:26.452552+0000","last_active":"2023-10-09T05:24:56.630612+0000","last_peered":"2023-10-09T05:24:56.630612+0000","last_clean":"2023-10-09T05:24:56.630612+0000","last_became_active":"2023-10-09T05:18:26.428454+0000","last_became_peered":"2023-10-09T05:18:26.428454+0000","last_unstale":"2023-10-09T05:24:56.630612+0000","last_undegraded":"2023-10-09T05:24:56.630612+0000","last_fullsized":"2023-10-09T05:24:56.630612+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:15:52.203861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":24,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.622596+0000","last_change":"2023-10-09T05:23:18.199181+0000","last_active":"2023-10-09T05:24:56.622596+0000","last_peered":"2023-10-09T05:24:56.622596+0000","last_clean":"2023-10-09T05:24:56.622596+0000","last_became_active":"2023-10-09T05:23:18.198401+0000","last_became_peered":"2023-10-09T05:23:18.198401+0000","last_unstale":"2023-10-09T05:24:56.622596+0000","last_undegraded":"2023-10-09T05:24:56.622596+0000","last_fullsized":"2023-10-09T05:24:56.622596+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:55:49.891420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'23","reported_seq":318,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.636926+0000","last_change":"2023-10-09T05:18:26.436165+0000","last_active":"2023-10-09T05:24:56.636926+0000","last_peered":"2023-10-09T05:24:56.636926+0000","last_clean":"2023-10-09T05:24:56.636926+0000","last_became_active":"2023-10-09T05:18:26.430078+0000","last_became_peered":"2023-10-09T05:18:26.430078+0000","last_unstale":"2023-10-09T05:24:56.636926+0000","last_undegraded":"2023-10-09T05:24:56.636926+0000","last_fullsized":"2023-10-09T05:24:56.636926+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:04:48.797166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":246,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.638174+0000","last_change":"2023-10-09T05:09:57.868761+0000","last_active":"2023-10-09T05:24:56.638174+0000","last_peered":"2023-10-09T05:24:56.638174+0000","last_clean":"2023-10-09T05:24:56.638174+0000","last_became_active":"2023-10-09T05:09:57.868121+0000","last_became_peered":"2023-10-09T05:09:57.868121+0000","last_unstale":"2023-10-09T05:24:56.638174+0000","last_undegraded":"2023-10-09T05:24:56.638174+0000","last_fullsized":"2023-10-09T05:24:56.638174+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:43.412235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":61,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.630009+0000","last_change":"2023-10-09T05:18:26.443719+0000","last_active":"2023-10-09T05:24:56.630009+0000","last_peered":"2023-10-09T05:24:56.630009+0000","last_clean":"2023-10-09T05:24:56.630009+0000","last_became_active":"2023-10-09T05:18:26.426718+0000","last_became_peered":"2023-10-09T05:18:26.426718+0000","last_unstale":"2023-10-09T05:24:56.630009+0000","last_undegraded":"2023-10-09T05:24:56.630009+0000","last_fullsized":"2023-10-09T05:24:56.630009+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:49:12.090862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":293,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.627344+0000","last_change":"2023-10-09T05:23:18.187593+0000","last_active":"2023-10-09T05:24:56.627344+0000","last_peered":"2023-10-09T05:24:56.627344+0000","last_clean":"2023-10-09T05:24:56.627344+0000","last_became_active":"2023-10-09T05:23:18.186445+0000","last_became_peered":"2023-10-09T05:23:18.186445+0000","last_unstale":"2023-10-09T05:24:56.627344+0000","last_undegraded":"2023-10-09T05:24:56.627344+0000","last_fullsized":"2023-10-09T05:24:56.627344+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:11.327635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":262,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.626537+0000","last_change":"2023-10-09T05:18:26.424399+0000","last_active":"2023-10-09T05:24:56.626537+0000","last_peered":"2023-10-09T05:24:56.626537+0000","last_clean":"2023-10-09T05:24:56.626537+0000","last_became_active":"2023-10-09T05:18:26.422122+0000","last_became_peered":"2023-10-09T05:18:26.422122+0000","last_unstale":"2023-10-09T05:24:56.626537+0000","last_undegraded":"2023-10-09T05:24:56.626537+0000","last_fullsized":"2023-10-09T05:24:56.626537+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:39.393756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":55,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.630585+0000","last_change":"2023-10-09T05:18:26.422950+0000","last_active":"2023-10-09T05:24:56.630585+0000","last_peered":"2023-10-09T05:24:56.630585+0000","last_clean":"2023-10-09T05:24:56.630585+0000","last_became_active":"2023-10-09T05:18:26.422592+0000","last_became_peered":"2023-10-09T05:18:26.422592+0000","last_unstale":"2023-10-09T05:24:56.630585+0000","last_undegraded":"2023-10-09T05:24:56.630585+0000","last_fullsized":"2023-10-09T05:24:56.630585+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:20.701107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":55,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.629793+0000","last_change":"2023-10-09T05:18:26.429482+0000","last_active":"2023-10-09T05:24:56.629793+0000","last_peered":"2023-10-09T05:24:56.629793+0000","last_clean":"2023-10-09T05:24:56.629793+0000","last_became_active":"2023-10-09T05:18:26.429122+0000","last_became_peered":"2023-10-09T05:18:26.429122+0000","last_unstale":"2023-10-09T05:24:56.629793+0000","last_undegraded":"2023-10-09T05:24:56.629793+0000","last_fullsized":"2023-10-09T05:24:56.629793+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:47.301906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":262,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.627536+0000","last_change":"2023-10-09T05:18:26.421678+0000","last_active":"2023-10-09T05:24:56.627536+0000","last_peered":"2023-10-09T05:24:56.627536+0000","last_clean":"2023-10-09T05:24:56.627536+0000","last_became_active":"2023-10-09T05:18:26.419673+0000","last_became_peered":"2023-10-09T05:18:26.419673+0000","last_unstale":"2023-10-09T05:24:56.627536+0000","last_undegraded":"2023-10-09T05:24:56.627536+0000","last_fullsized":"2023-10-09T05:24:56.627536+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:38.876021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":259,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.639134+0000","last_change":"2023-10-09T05:13:03.047339+0000","last_active":"2023-10-09T05:24:56.639134+0000","last_peered":"2023-10-09T05:24:56.639134+0000","last_clean":"2023-10-09T05:24:56.639134+0000","last_became_active":"2023-10-09T05:13:03.040038+0000","last_became_peered":"2023-10-09T05:13:03.040038+0000","last_unstale":"2023-10-09T05:24:56.639134+0000","last_undegraded":"2023-10-09T05:24:56.639134+0000","last_fullsized":"2023-10-09T05:24:56.639134+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:56.726249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":275,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.628677+0000","last_change":"2023-10-09T05:18:26.415173+0000","last_active":"2023-10-09T05:24:56.628677+0000","last_peered":"2023-10-09T05:24:56.628677+0000","last_clean":"2023-10-09T05:24:56.628677+0000","last_became_active":"2023-10-09T05:18:26.414452+0000","last_became_peered":"2023-10-09T05:18:26.414452+0000","last_unstale":"2023-10-09T05:24:56.628677+0000","last_undegraded":"2023-10-09T05:24:56.628677+0000","last_fullsized":"2023-10-09T05:24:56.628677+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:34.379767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"0'0","reported_seq":10,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.644548+0000","last_change":"2023-10-09T05:24:56.644144+0000","last_active":"2023-10-09T05:24:56.644548+0000","last_peered":"2023-10-09T05:24:56.644548+0000","last_clean":"2023-10-09T05:24:56.644548+0000","last_became_active":"2023-10-09T05:24:56.643459+0000","last_became_peered":"2023-10-09T05:24:56.643459+0000","last_unstale":"2023-10-09T05:24:56.644548+0000","last_undegraded":"2023-10-09T05:24:56.644548+0000","last_fullsized":"2023-10-09T05:24:56.644548+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:39.899186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":59,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.630546+0000","last_change":"2023-10-09T05:18:26.446608+0000","last_active":"2023-10-09T05:24:56.630546+0000","last_peered":"2023-10-09T05:24:56.630546+0000","last_clean":"2023-10-09T05:24:56.630546+0000","last_became_active":"2023-10-09T05:18:26.428420+0000","last_became_peered":"2023-10-09T05:18:26.428420+0000","last_unstale":"2023-10-09T05:24:56.630546+0000","last_undegraded":"2023-10-09T05:24:56.630546+0000","last_fullsized":"2023-10-09T05:24:56.630546+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:23.693908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":55,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.629474+0000","last_change":"2023-10-09T05:18:26.427461+0000","last_active":"2023-10-09T05:24:56.629474+0000","last_peered":"2023-10-09T05:24:56.629474+0000","last_clean":"2023-10-09T05:24:56.629474+0000","last_became_active":"2023-10-09T05:18:26.427118+0000","last_became_peered":"2023-10-09T05:18:26.427118+0000","last_unstale":"2023-10-09T05:24:56.629474+0000","last_undegraded":"2023-10-09T05:24:56.629474+0000","last_fullsized":"2023-10-09T05:24:56.629474+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:05:01.868197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.1","version":"0'0","reported_seq":10,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.644280+0000","last_change":"2023-10-09T05:24:56.643202+0000","last_active":"2023-10-09T05:24:56.644280+0000","last_peered":"2023-10-09T05:24:56.644280+0000","last_clean":"2023-10-09T05:24:56.644280+0000","last_became_active":"2023-10-09T05:24:56.642700+0000","last_became_peered":"2023-10-09T05:24:56.642700+0000","last_unstale":"2023-10-09T05:24:56.644280+0000","last_undegraded":"2023-10-09T05:24:56.644280+0000","last_fullsized":"2023-10-09T05:24:56.644280+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:25.444496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":59,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.630259+0000","last_change":"2023-10-09T05:18:26.440042+0000","last_active":"2023-10-09T05:24:56.630259+0000","last_peered":"2023-10-09T05:24:56.630259+0000","last_clean":"2023-10-09T05:24:56.630259+0000","last_became_active":"2023-10-09T05:18:26.426726+0000","last_became_peered":"2023-10-09T05:18:26.426726+0000","last_unstale":"2023-10-09T05:24:56.630259+0000","last_undegraded":"2023-10-09T05:24:56.630259+0000","last_fullsized":"2023-10-09T05:24:56.630259+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:23.985650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":263,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.637159+0000","last_change":"2023-10-09T05:09:57.868031+0000","last_active":"2023-10-09T05:24:56.637159+0000","last_peered":"2023-10-09T05:24:56.637159+0000","last_clean":"2023-10-09T05:24:56.637159+0000","last_became_active":"2023-10-09T05:09:57.866477+0000","last_became_peered":"2023-10-09T05:09:57.866477+0000","last_unstale":"2023-10-09T05:24:56.637159+0000","last_undegraded":"2023-10-09T05:24:56.637159+0000","last_fullsized":"2023-10-09T05:24:56.637159+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:27.102583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"0'0","reported_seq":10,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.641048+0000","last_change":"2023-10-09T05:24:56.640700+0000","last_active":"2023-10-09T05:24:56.641048+0000","last_peered":"2023-10-09T05:24:56.641048+0000","last_clean":"2023-10-09T05:24:56.641048+0000","last_became_active":"2023-10-09T05:24:56.640352+0000","last_became_peered":"2023-10-09T05:24:56.640352+0000","last_unstale":"2023-10-09T05:24:56.641048+0000","last_undegraded":"2023-10-09T05:24:56.641048+0000","last_fullsized":"2023-10-09T05:24:56.641048+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:55:08.218749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":59,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.630695+0000","last_change":"2023-10-09T05:18:26.437953+0000","last_active":"2023-10-09T05:24:56.630695+0000","last_peered":"2023-10-09T05:24:56.630695+0000","last_clean":"2023-10-09T05:24:56.630695+0000","last_became_active":"2023-10-09T05:18:26.422703+0000","last_became_peered":"2023-10-09T05:18:26.422703+0000","last_unstale":"2023-10-09T05:24:56.630695+0000","last_undegraded":"2023-10-09T05:24:56.630695+0000","last_fullsized":"2023-10-09T05:24:56.630695+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:13.926811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":296,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.639621+0000","last_change":"2023-10-09T05:18:26.423526+0000","last_active":"2023-10-09T05:24:56.639621+0000","last_peered":"2023-10-09T05:24:56.639621+0000","last_clean":"2023-10-09T05:24:56.639621+0000","last_became_active":"2023-10-09T05:18:26.423092+0000","last_became_peered":"2023-10-09T05:18:26.423092+0000","last_unstale":"2023-10-09T05:24:56.639621+0000","last_undegraded":"2023-10-09T05:24:56.639621+0000","last_fullsized":"2023-10-09T05:24:56.639621+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:54.877664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"0'0","reported_seq":11,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.635001+0000","last_change":"2023-10-09T05:24:56.634648+0000","last_active":"2023-10-09T05:24:56.635001+0000","last_peered":"2023-10-09T05:24:56.635001+0000","last_clean":"2023-10-09T05:24:56.635001+0000","last_became_active":"2023-10-09T05:24:56.633771+0000","last_became_peered":"2023-10-09T05:24:56.633771+0000","last_unstale":"2023-10-09T05:24:56.635001+0000","last_undegraded":"2023-10-09T05:24:56.635001+0000","last_fullsized":"2023-10-09T05:24:56.635001+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:20:25.848591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":186,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.627600+0000","last_change":"2023-10-09T05:13:03.037926+0000","last_active":"2023-10-09T05:24:56.627600+0000","last_peered":"2023-10-09T05:24:56.627600+0000","last_clean":"2023-10-09T05:24:56.627600+0000","last_became_active":"2023-10-09T05:13:03.037639+0000","last_became_peered":"2023-10-09T05:13:03.037639+0000","last_unstale":"2023-10-09T05:24:56.627600+0000","last_undegraded":"2023-10-09T05:24:56.627600+0000","last_fullsized":"2023-10-09T05:24:56.627600+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:52.977143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":245,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.626542+0000","last_change":"2023-10-09T05:09:57.852914+0000","last_active":"2023-10-09T05:24:56.626542+0000","last_peered":"2023-10-09T05:24:56.626542+0000","last_clean":"2023-10-09T05:24:56.626542+0000","last_became_active":"2023-10-09T05:09:57.852522+0000","last_became_peered":"2023-10-09T05:09:57.852522+0000","last_unstale":"2023-10-09T05:24:56.626542+0000","last_undegraded":"2023-10-09T05:24:56.626542+0000","last_fullsized":"2023-10-09T05:24:56.626542+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:24.848494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"0'0","reported_seq":10,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.640360+0000","last_change":"2023-10-09T05:24:56.639909+0000","last_active":"2023-10-09T05:24:56.640360+0000","last_peered":"2023-10-09T05:24:56.640360+0000","last_clean":"2023-10-09T05:24:56.640360+0000","last_became_active":"2023-10-09T05:24:56.639434+0000","last_became_peered":"2023-10-09T05:24:56.639434+0000","last_unstale":"2023-10-09T05:24:56.640360+0000","last_undegraded":"2023-10-09T05:24:56.640360+0000","last_fullsized":"2023-10-09T05:24:56.640360+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:11.231733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"69'3","reported_seq":219,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.627372+0000","last_change":"2023-10-09T05:02:09.650579+0000","last_active":"2023-10-09T05:24:56.627372+0000","last_peered":"2023-10-09T05:24:56.627372+0000","last_clean":"2023-10-09T05:24:56.627372+0000","last_became_active":"2023-10-09T05:02:09.650160+0000","last_became_peered":"2023-10-09T05:02:09.650160+0000","last_unstale":"2023-10-09T05:24:56.627372+0000","last_undegraded":"2023-10-09T05:24:56.627372+0000","last_fullsized":"2023-10-09T05:24:56.627372+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:18.292670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":226,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.636791+0000","last_change":"2023-10-09T05:02:09.652449+0000","last_active":"2023-10-09T05:24:56.636791+0000","last_peered":"2023-10-09T05:24:56.636791+0000","last_clean":"2023-10-09T05:24:56.636791+0000","last_became_active":"2023-10-09T05:02:09.651994+0000","last_became_peered":"2023-10-09T05:02:09.651994+0000","last_unstale":"2023-10-09T05:24:56.636791+0000","last_undegraded":"2023-10-09T05:24:56.636791+0000","last_fullsized":"2023-10-09T05:24:56.636791+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:14.480513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"0'0","reported_seq":10,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.644252+0000","last_change":"2023-10-09T05:24:56.643292+0000","last_active":"2023-10-09T05:24:56.644252+0000","last_peered":"2023-10-09T05:24:56.644252+0000","last_clean":"2023-10-09T05:24:56.644252+0000","last_became_active":"2023-10-09T05:24:56.642863+0000","last_became_peered":"2023-10-09T05:24:56.642863+0000","last_unstale":"2023-10-09T05:24:56.644252+0000","last_undegraded":"2023-10-09T05:24:56.644252+0000","last_fullsized":"2023-10-09T05:24:56.644252+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:09.977734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":267,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.639846+0000","last_change":"2023-10-09T05:23:18.199172+0000","last_active":"2023-10-09T05:24:56.639846+0000","last_peered":"2023-10-09T05:24:56.639846+0000","last_clean":"2023-10-09T05:24:56.639846+0000","last_became_active":"2023-10-09T05:23:18.198762+0000","last_became_peered":"2023-10-09T05:23:18.198762+0000","last_unstale":"2023-10-09T05:24:56.639846+0000","last_undegraded":"2023-10-09T05:24:56.639846+0000","last_fullsized":"2023-10-09T05:24:56.639846+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:44:23.239788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":283,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.637558+0000","last_change":"2023-10-09T05:20:18.129228+0000","last_active":"2023-10-09T05:24:56.637558+0000","last_peered":"2023-10-09T05:24:56.637558+0000","last_clean":"2023-10-09T05:24:56.637558+0000","last_became_active":"2023-10-09T05:20:18.128769+0000","last_became_peered":"2023-10-09T05:20:18.128769+0000","last_unstale":"2023-10-09T05:24:56.637558+0000","last_undegraded":"2023-10-09T05:24:56.637558+0000","last_fullsized":"2023-10-09T05:24:56.637558+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:50.427100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"0'0","reported_seq":11,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.632389+0000","last_change":"2023-10-09T05:24:56.631754+0000","last_active":"2023-10-09T05:24:56.632389+0000","last_peered":"2023-10-09T05:24:56.632389+0000","last_clean":"2023-10-09T05:24:56.632389+0000","last_became_active":"2023-10-09T05:24:56.631413+0000","last_became_peered":"2023-10-09T05:24:56.631413+0000","last_unstale":"2023-10-09T05:24:56.632389+0000","last_undegraded":"2023-10-09T05:24:56.632389+0000","last_fullsized":"2023-10-09T05:24:56.632389+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:06.416230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":267,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.637012+0000","last_change":"2023-10-09T05:18:26.431286+0000","last_active":"2023-10-09T05:24:56.637012+0000","last_peered":"2023-10-09T05:24:56.637012+0000","last_clean":"2023-10-09T05:24:56.637012+0000","last_became_active":"2023-10-09T05:18:26.430127+0000","last_became_peered":"2023-10-09T05:18:26.430127+0000","last_unstale":"2023-10-09T05:24:56.637012+0000","last_undegraded":"2023-10-09T05:24:56.637012+0000","last_fullsized":"2023-10-09T05:24:56.637012+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:42.982259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":266,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.639412+0000","last_change":"2023-10-09T05:23:18.193800+0000","last_active":"2023-10-09T05:24:56.639412+0000","last_peered":"2023-10-09T05:24:56.639412+0000","last_clean":"2023-10-09T05:24:56.639412+0000","last_became_active":"2023-10-09T05:23:18.193338+0000","last_became_peered":"2023-10-09T05:23:18.193338+0000","last_unstale":"2023-10-09T05:24:56.639412+0000","last_undegraded":"2023-10-09T05:24:56.639412+0000","last_fullsized":"2023-10-09T05:24:56.639412+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:29.109566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"0'0","reported_seq":11,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.634034+0000","last_change":"2023-10-09T05:24:56.632416+0000","last_active":"2023-10-09T05:24:56.634034+0000","last_peered":"2023-10-09T05:24:56.634034+0000","last_clean":"2023-10-09T05:24:56.634034+0000","last_became_active":"2023-10-09T05:24:56.632033+0000","last_became_peered":"2023-10-09T05:24:56.632033+0000","last_unstale":"2023-10-09T05:24:56.634034+0000","last_undegraded":"2023-10-09T05:24:56.634034+0000","last_fullsized":"2023-10-09T05:24:56.634034+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:08.729587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":277,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.626607+0000","last_change":"2023-10-09T05:18:26.413106+0000","last_active":"2023-10-09T05:24:56.626607+0000","last_peered":"2023-10-09T05:24:56.626607+0000","last_clean":"2023-10-09T05:24:56.626607+0000","last_became_active":"2023-10-09T05:18:26.412334+0000","last_became_peered":"2023-10-09T05:18:26.412334+0000","last_unstale":"2023-10-09T05:24:56.626607+0000","last_undegraded":"2023-10-09T05:24:56.626607+0000","last_fullsized":"2023-10-09T05:24:56.626607+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:18.058016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":131,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.623908+0000","last_change":"2023-10-09T05:09:57.856764+0000","last_active":"2023-10-09T05:24:56.623908+0000","last_peered":"2023-10-09T05:24:56.623908+0000","last_clean":"2023-10-09T05:24:56.623908+0000","last_became_active":"2023-10-09T05:09:57.856461+0000","last_became_peered":"2023-10-09T05:09:57.856461+0000","last_unstale":"2023-10-09T05:24:56.623908+0000","last_undegraded":"2023-10-09T05:24:56.623908+0000","last_fullsized":"2023-10-09T05:24:56.623908+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:06.848082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.a","version":"0'0","reported_seq":11,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-10-09T05:24:57.951269+0000","last_change":"2023-10-09T05:24:56.632715+0000","last_active":"2023-10-09T05:24:57.951269+0000","last_peered":"2023-10-09T05:24:57.951269+0000","last_clean":"2023-10-09T05:24:57.951269+0000","last_became_active":"2023-10-09T05:24:56.632411+0000","last_became_peered":"2023-10-09T05:24:56.632411+0000","last_unstale":"2023-10-09T05:24:57.951269+0000","last_undegraded":"2023-10-09T05:24:57.951269+0000","last_fullsized":"2023-10-09T05:24:57.951269+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:25.311429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":168,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.623507+0000","last_change":"2023-10-09T05:18:26.422150+0000","last_active":"2023-10-09T05:24:56.623507+0000","last_peered":"2023-10-09T05:24:56.623507+0000","last_clean":"2023-10-09T05:24:56.623507+0000","last_became_active":"2023-10-09T05:18:26.421853+0000","last_became_peered":"2023-10-09T05:18:26.421853+0000","last_unstale":"2023-10-09T05:24:56.623507+0000","last_undegraded":"2023-10-09T05:24:56.623507+0000","last_fullsized":"2023-10-09T05:24:56.623507+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:37:47.000556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":203,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.626490+0000","last_change":"2023-10-09T05:13:03.031243+0000","last_active":"2023-10-09T05:24:56.626490+0000","last_peered":"2023-10-09T05:24:56.626490+0000","last_clean":"2023-10-09T05:24:56.626490+0000","last_became_active":"2023-10-09T05:13:03.028769+0000","last_became_peered":"2023-10-09T05:13:03.028769+0000","last_unstale":"2023-10-09T05:24:56.626490+0000","last_undegraded":"2023-10-09T05:24:56.626490+0000","last_fullsized":"2023-10-09T05:24:56.626490+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:25:25.239799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":24,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.622579+0000","last_change":"2023-10-09T05:23:18.197927+0000","last_active":"2023-10-09T05:24:56.622579+0000","last_peered":"2023-10-09T05:24:56.622579+0000","last_clean":"2023-10-09T05:24:56.622579+0000","last_became_active":"2023-10-09T05:23:18.197431+0000","last_became_peered":"2023-10-09T05:23:18.197431+0000","last_unstale":"2023-10-09T05:24:56.622579+0000","last_undegraded":"2023-10-09T05:24:56.622579+0000","last_fullsized":"2023-10-09T05:24:56.622579+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:51:41.376456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"0'0","reported_seq":11,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.634143+0000","last_change":"2023-10-09T05:24:56.631792+0000","last_active":"2023-10-09T05:24:56.634143+0000","last_peered":"2023-10-09T05:24:56.634143+0000","last_clean":"2023-10-09T05:24:56.634143+0000","last_became_active":"2023-10-09T05:24:56.631473+0000","last_became_peered":"2023-10-09T05:24:56.631473+0000","last_unstale":"2023-10-09T05:24:56.634143+0000","last_undegraded":"2023-10-09T05:24:56.634143+0000","last_fullsized":"2023-10-09T05:24:56.634143+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:17.170510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":237,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.622685+0000","last_change":"2023-10-09T05:13:03.031386+0000","last_active":"2023-10-09T05:24:56.622685+0000","last_peered":"2023-10-09T05:24:56.622685+0000","last_clean":"2023-10-09T05:24:56.622685+0000","last_became_active":"2023-10-09T05:13:03.030816+0000","last_became_peered":"2023-10-09T05:13:03.030816+0000","last_unstale":"2023-10-09T05:24:56.622685+0000","last_undegraded":"2023-10-09T05:24:56.622685+0000","last_fullsized":"2023-10-09T05:24:56.622685+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:28.088661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":227,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.638060+0000","last_change":"2023-10-09T04:56:56.087320+0000","last_active":"2023-10-09T05:24:56.638060+0000","last_peered":"2023-10-09T05:24:56.638060+0000","last_clean":"2023-10-09T05:24:56.638060+0000","last_became_active":"2023-10-09T04:56:56.086552+0000","last_became_peered":"2023-10-09T04:56:56.086552+0000","last_unstale":"2023-10-09T05:24:56.638060+0000","last_undegraded":"2023-10-09T05:24:56.638060+0000","last_fullsized":"2023-10-09T05:24:56.638060+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:14.205288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":24,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.623377+0000","last_change":"2023-10-09T05:23:18.196260+0000","last_active":"2023-10-09T05:24:56.623377+0000","last_peered":"2023-10-09T05:24:56.623377+0000","last_clean":"2023-10-09T05:24:56.623377+0000","last_became_active":"2023-10-09T05:23:18.195907+0000","last_became_peered":"2023-10-09T05:23:18.195907+0000","last_unstale":"2023-10-09T05:24:56.623377+0000","last_undegraded":"2023-10-09T05:24:56.623377+0000","last_fullsized":"2023-10-09T05:24:56.623377+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:36.243432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.f","version":"0'0","reported_seq":10,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.644727+0000","last_change":"2023-10-09T05:24:56.644098+0000","last_active":"2023-10-09T05:24:56.644727+0000","last_peered":"2023-10-09T05:24:56.644727+0000","last_clean":"2023-10-09T05:24:56.644727+0000","last_became_active":"2023-10-09T05:24:56.643666+0000","last_became_peered":"2023-10-09T05:24:56.643666+0000","last_unstale":"2023-10-09T05:24:56.644727+0000","last_undegraded":"2023-10-09T05:24:56.644727+0000","last_fullsized":"2023-10-09T05:24:56.644727+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:39:36.431548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":43,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.629026+0000","last_change":"2023-10-09T05:20:18.123706+0000","last_active":"2023-10-09T05:24:56.629026+0000","last_peered":"2023-10-09T05:24:56.629026+0000","last_clean":"2023-10-09T05:24:56.629026+0000","last_became_active":"2023-10-09T05:20:18.122859+0000","last_became_peered":"2023-10-09T05:20:18.122859+0000","last_unstale":"2023-10-09T05:24:56.629026+0000","last_undegraded":"2023-10-09T05:24:56.629026+0000","last_fullsized":"2023-10-09T05:24:56.629026+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.906228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":132,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.622451+0000","last_change":"2023-10-09T05:09:57.854064+0000","last_active":"2023-10-09T05:24:56.622451+0000","last_peered":"2023-10-09T05:24:56.622451+0000","last_clean":"2023-10-09T05:24:56.622451+0000","last_became_active":"2023-10-09T05:09:57.847953+0000","last_became_peered":"2023-10-09T05:09:57.847953+0000","last_unstale":"2023-10-09T05:24:56.622451+0000","last_undegraded":"2023-10-09T05:24:56.622451+0000","last_fullsized":"2023-10-09T05:24:56.622451+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:45.914976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":24,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.623885+0000","last_change":"2023-10-09T05:23:18.200437+0000","last_active":"2023-10-09T05:24:56.623885+0000","last_peered":"2023-10-09T05:24:56.623885+0000","last_clean":"2023-10-09T05:24:56.623885+0000","last_became_active":"2023-10-09T05:23:18.200002+0000","last_became_peered":"2023-10-09T05:23:18.200002+0000","last_unstale":"2023-10-09T05:24:56.623885+0000","last_undegraded":"2023-10-09T05:24:56.623885+0000","last_fullsized":"2023-10-09T05:24:56.623885+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:16.926545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"0'0","reported_seq":10,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.633955+0000","last_change":"2023-10-09T05:24:56.633695+0000","last_active":"2023-10-09T05:24:56.633955+0000","last_peered":"2023-10-09T05:24:56.633955+0000","last_clean":"2023-10-09T05:24:56.633955+0000","last_became_active":"2023-10-09T05:24:56.633393+0000","last_became_peered":"2023-10-09T05:24:56.633393+0000","last_unstale":"2023-10-09T05:24:56.633955+0000","last_undegraded":"2023-10-09T05:24:56.633955+0000","last_fullsized":"2023-10-09T05:24:56.633955+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:04.072215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":209,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.629929+0000","last_change":"2023-10-09T05:18:26.416511+0000","last_active":"2023-10-09T05:24:56.629929+0000","last_peered":"2023-10-09T05:24:56.629929+0000","last_clean":"2023-10-09T05:24:56.629929+0000","last_became_active":"2023-10-09T05:18:26.416101+0000","last_became_peered":"2023-10-09T05:18:26.416101+0000","last_unstale":"2023-10-09T05:24:56.629929+0000","last_undegraded":"2023-10-09T05:24:56.629929+0000","last_fullsized":"2023-10-09T05:24:56.629929+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:10.243472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":551,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.629533+0000","last_change":"2023-10-09T05:18:29.419527+0000","last_active":"2023-10-09T05:24:56.629533+0000","last_peered":"2023-10-09T05:24:56.629533+0000","last_clean":"2023-10-09T05:24:56.629533+0000","last_became_active":"2023-10-09T05:18:29.418852+0000","last_became_peered":"2023-10-09T05:18:29.418852+0000","last_unstale":"2023-10-09T05:24:56.629533+0000","last_undegraded":"2023-10-09T05:24:56.629533+0000","last_fullsized":"2023-10-09T05:24:56.629533+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_clean_scrub_stamp":"2023-10-09T04:36:44.261013+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:02.899170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":227,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-10-09T05:24:57.952203+0000","last_change":"2023-10-09T05:13:03.027260+0000","last_active":"2023-10-09T05:24:57.952203+0000","last_peered":"2023-10-09T05:24:57.952203+0000","last_clean":"2023-10-09T05:24:57.952203+0000","last_became_active":"2023-10-09T05:13:03.026416+0000","last_became_peered":"2023-10-09T05:13:03.026416+0000","last_unstale":"2023-10-09T05:24:57.952203+0000","last_undegraded":"2023-10-09T05:24:57.952203+0000","last_fullsized":"2023-10-09T05:24:57.952203+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:05.057338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":249,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.641260+0000","last_change":"2023-10-09T05:18:26.426126+0000","last_active":"2023-10-09T05:24:56.641260+0000","last_peered":"2023-10-09T05:24:56.641260+0000","last_clean":"2023-10-09T05:24:56.641260+0000","last_became_active":"2023-10-09T05:18:26.425721+0000","last_became_peered":"2023-10-09T05:18:26.425721+0000","last_unstale":"2023-10-09T05:24:56.641260+0000","last_undegraded":"2023-10-09T05:24:56.641260+0000","last_fullsized":"2023-10-09T05:24:56.641260+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:48.229218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"0'0","reported_seq":11,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.641933+0000","last_change":"2023-10-09T05:24:56.641295+0000","last_active":"2023-10-09T05:24:56.641933+0000","last_peered":"2023-10-09T05:24:56.641933+0000","last_clean":"2023-10-09T05:24:56.641933+0000","last_became_active":"2023-10-09T05:24:56.640909+0000","last_became_peered":"2023-10-09T05:24:56.640909+0000","last_unstale":"2023-10-09T05:24:56.641933+0000","last_undegraded":"2023-10-09T05:24:56.641933+0000","last_fullsized":"2023-10-09T05:24:56.641933+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:26.475478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":75,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.629596+0000","last_change":"2023-10-09T05:23:18.193825+0000","last_active":"2023-10-09T05:24:56.629596+0000","last_peered":"2023-10-09T05:24:56.629596+0000","last_clean":"2023-10-09T05:24:56.629596+0000","last_became_active":"2023-10-09T05:23:18.193394+0000","last_became_peered":"2023-10-09T05:23:18.193394+0000","last_unstale":"2023-10-09T05:24:56.629596+0000","last_undegraded":"2023-10-09T05:24:56.629596+0000","last_fullsized":"2023-10-09T05:24:56.629596+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:40.594469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":227,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.627728+0000","last_change":"2023-10-09T05:02:09.648781+0000","last_active":"2023-10-09T05:24:56.627728+0000","last_peered":"2023-10-09T05:24:56.627728+0000","last_clean":"2023-10-09T05:24:56.627728+0000","last_became_active":"2023-10-09T05:02:09.648078+0000","last_became_peered":"2023-10-09T05:02:09.648078+0000","last_unstale":"2023-10-09T05:24:56.627728+0000","last_undegraded":"2023-10-09T05:24:56.627728+0000","last_fullsized":"2023-10-09T05:24:56.627728+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:35.287091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":306,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-10-09T05:24:57.952484+0000","last_change":"2023-10-09T05:20:18.132750+0000","last_active":"2023-10-09T05:24:57.952484+0000","last_peered":"2023-10-09T05:24:57.952484+0000","last_clean":"2023-10-09T05:24:57.952484+0000","last_became_active":"2023-10-09T05:20:18.132396+0000","last_became_peered":"2023-10-09T05:20:18.132396+0000","last_unstale":"2023-10-09T05:24:57.952484+0000","last_undegraded":"2023-10-09T05:24:57.952484+0000","last_fullsized":"2023-10-09T05:24:57.952484+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:00:55.731045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"0'0","reported_seq":11,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-10-09T05:24:57.951166+0000","last_change":"2023-10-09T05:24:56.642285+0000","last_active":"2023-10-09T05:24:57.951166+0000","last_peered":"2023-10-09T05:24:57.951166+0000","last_clean":"2023-10-09T05:24:57.951166+0000","last_became_active":"2023-10-09T05:24:56.641521+0000","last_became_peered":"2023-10-09T05:24:56.641521+0000","last_unstale":"2023-10-09T05:24:57.951166+0000","last_undegraded":"2023-10-09T05:24:57.951166+0000","last_fullsized":"2023-10-09T05:24:57.951166+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:06:01.013563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":55,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.630683+0000","last_change":"2023-10-09T05:18:26.429468+0000","last_active":"2023-10-09T05:24:56.630683+0000","last_peered":"2023-10-09T05:24:56.630683+0000","last_clean":"2023-10-09T05:24:56.630683+0000","last_became_active":"2023-10-09T05:18:26.429098+0000","last_became_peered":"2023-10-09T05:18:26.429098+0000","last_unstale":"2023-10-09T05:24:56.630683+0000","last_undegraded":"2023-10-09T05:24:56.630683+0000","last_fullsized":"2023-10-09T05:24:56.630683+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:47.798226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":159,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.623678+0000","last_change":"2023-10-09T05:20:18.126997+0000","last_active":"2023-10-09T05:24:56.623678+0000","last_peered":"2023-10-09T05:24:56.623678+0000","last_clean":"2023-10-09T05:24:56.623678+0000","last_became_active":"2023-10-09T05:20:18.126641+0000","last_became_peered":"2023-10-09T05:20:18.126641+0000","last_unstale":"2023-10-09T05:24:56.623678+0000","last_undegraded":"2023-10-09T05:24:56.623678+0000","last_fullsized":"2023-10-09T05:24:56.623678+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:42.112245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":267,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.628354+0000","last_change":"2023-10-09T05:18:26.418524+0000","last_active":"2023-10-09T05:24:56.628354+0000","last_peered":"2023-10-09T05:24:56.628354+0000","last_clean":"2023-10-09T05:24:56.628354+0000","last_became_active":"2023-10-09T05:18:26.418058+0000","last_became_peered":"2023-10-09T05:18:26.418058+0000","last_unstale":"2023-10-09T05:24:56.628354+0000","last_undegraded":"2023-10-09T05:24:56.628354+0000","last_fullsized":"2023-10-09T05:24:56.628354+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:40.594449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"0'0","reported_seq":11,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-10-09T05:24:57.950677+0000","last_change":"2023-10-09T05:24:56.633539+0000","last_active":"2023-10-09T05:24:57.950677+0000","last_peered":"2023-10-09T05:24:57.950677+0000","last_clean":"2023-10-09T05:24:57.950677+0000","last_became_active":"2023-10-09T05:24:56.633197+0000","last_became_peered":"2023-10-09T05:24:56.633197+0000","last_unstale":"2023-10-09T05:24:57.950677+0000","last_undegraded":"2023-10-09T05:24:57.950677+0000","last_fullsized":"2023-10-09T05:24:57.950677+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:53.863072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":270,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.628645+0000","last_change":"2023-10-09T05:18:26.431563+0000","last_active":"2023-10-09T05:24:56.628645+0000","last_peered":"2023-10-09T05:24:56.628645+0000","last_clean":"2023-10-09T05:24:56.628645+0000","last_became_active":"2023-10-09T05:18:26.421025+0000","last_became_peered":"2023-10-09T05:18:26.421025+0000","last_unstale":"2023-10-09T05:24:56.628645+0000","last_undegraded":"2023-10-09T05:24:56.628645+0000","last_fullsized":"2023-10-09T05:24:56.628645+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:30.333585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":24,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.623283+0000","last_change":"2023-10-09T05:23:18.196610+0000","last_active":"2023-10-09T05:24:56.623283+0000","last_peered":"2023-10-09T05:24:56.623283+0000","last_clean":"2023-10-09T05:24:56.623283+0000","last_became_active":"2023-10-09T05:23:18.196279+0000","last_became_peered":"2023-10-09T05:23:18.196279+0000","last_unstale":"2023-10-09T05:24:56.623283+0000","last_undegraded":"2023-10-09T05:24:56.623283+0000","last_fullsized":"2023-10-09T05:24:56.623283+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:52.868384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":269,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.626715+0000","last_change":"2023-10-09T05:21:08.179657+0000","last_active":"2023-10-09T05:24:56.626715+0000","last_peered":"2023-10-09T05:24:56.626715+0000","last_clean":"2023-10-09T05:24:56.626715+0000","last_became_active":"2023-10-09T05:21:08.178958+0000","last_became_peered":"2023-10-09T05:21:08.178958+0000","last_unstale":"2023-10-09T05:24:56.626715+0000","last_undegraded":"2023-10-09T05:24:56.626715+0000","last_fullsized":"2023-10-09T05:24:56.626715+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:20.815986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"0'0","reported_seq":11,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-10-09T05:24:57.950467+0000","last_change":"2023-10-09T05:24:56.641633+0000","last_active":"2023-10-09T05:24:57.950467+0000","last_peered":"2023-10-09T05:24:57.950467+0000","last_clean":"2023-10-09T05:24:57.950467+0000","last_became_active":"2023-10-09T05:24:56.641288+0000","last_became_peered":"2023-10-09T05:24:56.641288+0000","last_unstale":"2023-10-09T05:24:57.950467+0000","last_undegraded":"2023-10-09T05:24:57.950467+0000","last_fullsized":"2023-10-09T05:24:57.950467+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:19.463846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":271,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.642589+0000","last_change":"2023-10-09T05:18:26.424429+0000","last_active":"2023-10-09T05:24:56.642589+0000","last_peered":"2023-10-09T05:24:56.642589+0000","last_clean":"2023-10-09T05:24:56.642589+0000","last_became_active":"2023-10-09T05:18:26.423034+0000","last_became_peered":"2023-10-09T05:18:26.423034+0000","last_unstale":"2023-10-09T05:24:56.642589+0000","last_undegraded":"2023-10-09T05:24:56.642589+0000","last_fullsized":"2023-10-09T05:24:56.642589+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:16.251181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":258,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-10-09T05:24:57.952537+0000","last_change":"2023-10-09T05:09:57.878002+0000","last_active":"2023-10-09T05:24:57.952537+0000","last_peered":"2023-10-09T05:24:57.952537+0000","last_clean":"2023-10-09T05:24:57.952537+0000","last_became_active":"2023-10-09T05:09:57.852792+0000","last_became_peered":"2023-10-09T05:09:57.852792+0000","last_unstale":"2023-10-09T05:24:57.952537+0000","last_undegraded":"2023-10-09T05:24:57.952537+0000","last_fullsized":"2023-10-09T05:24:57.952537+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:18.763275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":219,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.636756+0000","last_change":"2023-10-09T05:13:03.026541+0000","last_active":"2023-10-09T05:24:56.636756+0000","last_peered":"2023-10-09T05:24:56.636756+0000","last_clean":"2023-10-09T05:24:56.636756+0000","last_became_active":"2023-10-09T05:13:03.025729+0000","last_became_peered":"2023-10-09T05:13:03.025729+0000","last_unstale":"2023-10-09T05:24:56.636756+0000","last_undegraded":"2023-10-09T05:24:56.636756+0000","last_fullsized":"2023-10-09T05:24:56.636756+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:22.938235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'2","reported_seq":35,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.628252+0000","last_change":"2023-10-09T05:21:08.173845+0000","last_active":"2023-10-09T05:24:56.628252+0000","last_peered":"2023-10-09T05:24:56.628252+0000","last_clean":"2023-10-09T05:24:56.628252+0000","last_became_active":"2023-10-09T05:21:08.173508+0000","last_became_peered":"2023-10-09T05:21:08.173508+0000","last_unstale":"2023-10-09T05:24:56.628252+0000","last_undegraded":"2023-10-09T05:24:56.628252+0000","last_fullsized":"2023-10-09T05:24:56.628252+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:03.628428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":42,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.629514+0000","last_change":"2023-10-09T05:20:18.125172+0000","last_active":"2023-10-09T05:24:56.629514+0000","last_peered":"2023-10-09T05:24:56.629514+0000","last_clean":"2023-10-09T05:24:56.629514+0000","last_became_active":"2023-10-09T05:20:18.123624+0000","last_became_peered":"2023-10-09T05:20:18.123624+0000","last_unstale":"2023-10-09T05:24:56.629514+0000","last_undegraded":"2023-10-09T05:24:56.629514+0000","last_fullsized":"2023-10-09T05:24:56.629514+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:11.106483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":121,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.637065+0000","last_change":"2023-10-09T05:13:03.055887+0000","last_active":"2023-10-09T05:24:56.637065+0000","last_peered":"2023-10-09T05:24:56.637065+0000","last_clean":"2023-10-09T05:24:56.637065+0000","last_became_active":"2023-10-09T05:13:03.028782+0000","last_became_peered":"2023-10-09T05:13:03.028782+0000","last_unstale":"2023-10-09T05:24:56.637065+0000","last_undegraded":"2023-10-09T05:24:56.637065+0000","last_fullsized":"2023-10-09T05:24:56.637065+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:38.828205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":283,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.638389+0000","last_change":"2023-10-09T05:23:18.207836+0000","last_active":"2023-10-09T05:24:56.638389+0000","last_peered":"2023-10-09T05:24:56.638389+0000","last_clean":"2023-10-09T05:24:56.638389+0000","last_became_active":"2023-10-09T05:23:18.207404+0000","last_became_peered":"2023-10-09T05:23:18.207404+0000","last_unstale":"2023-10-09T05:24:56.638389+0000","last_undegraded":"2023-10-09T05:24:56.638389+0000","last_fullsized":"2023-10-09T05:24:56.638389+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:36:22.236093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":121,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.637716+0000","last_change":"2023-10-09T05:13:03.049546+0000","last_active":"2023-10-09T05:24:56.637716+0000","last_peered":"2023-10-09T05:24:56.637716+0000","last_clean":"2023-10-09T05:24:56.637716+0000","last_became_active":"2023-10-09T05:13:03.028676+0000","last_became_peered":"2023-10-09T05:13:03.028676+0000","last_unstale":"2023-10-09T05:24:56.637716+0000","last_undegraded":"2023-10-09T05:24:56.637716+0000","last_fullsized":"2023-10-09T05:24:56.637716+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:55.089350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":243,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.637259+0000","last_change":"2023-10-09T05:02:09.649748+0000","last_active":"2023-10-09T05:24:56.637259+0000","last_peered":"2023-10-09T05:24:56.637259+0000","last_clean":"2023-10-09T05:24:56.637259+0000","last_became_active":"2023-10-09T05:02:09.649444+0000","last_became_peered":"2023-10-09T05:02:09.649444+0000","last_unstale":"2023-10-09T05:24:56.637259+0000","last_undegraded":"2023-10-09T05:24:56.637259+0000","last_fullsized":"2023-10-09T05:24:56.637259+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:07.436820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'2","reported_seq":263,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.626487+0000","last_change":"2023-10-09T05:22:47.197002+0000","last_active":"2023-10-09T05:24:56.626487+0000","last_peered":"2023-10-09T05:24:56.626487+0000","last_clean":"2023-10-09T05:24:56.626487+0000","last_became_active":"2023-10-09T05:22:47.196685+0000","last_became_peered":"2023-10-09T05:22:47.196685+0000","last_unstale":"2023-10-09T05:24:56.626487+0000","last_undegraded":"2023-10-09T05:24:56.626487+0000","last_fullsized":"2023-10-09T05:24:56.626487+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:47:32.594447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":248,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.628154+0000","last_change":"2023-10-09T05:02:09.649674+0000","last_active":"2023-10-09T05:24:56.628154+0000","last_peered":"2023-10-09T05:24:56.628154+0000","last_clean":"2023-10-09T05:24:56.628154+0000","last_became_active":"2023-10-09T05:02:09.649041+0000","last_became_peered":"2023-10-09T05:02:09.649041+0000","last_unstale":"2023-10-09T05:24:56.628154+0000","last_undegraded":"2023-10-09T05:24:56.628154+0000","last_fullsized":"2023-10-09T05:24:56.628154+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:18.524071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'5","reported_seq":41,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.629146+0000","last_change":"2023-10-09T05:21:08.181594+0000","last_active":"2023-10-09T05:24:56.629146+0000","last_peered":"2023-10-09T05:24:56.629146+0000","last_clean":"2023-10-09T05:24:56.629146+0000","last_became_active":"2023-10-09T05:21:08.176895+0000","last_became_peered":"2023-10-09T05:21:08.176895+0000","last_unstale":"2023-10-09T05:24:56.629146+0000","last_undegraded":"2023-10-09T05:24:56.629146+0000","last_fullsized":"2023-10-09T05:24:56.629146+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:48:45.569544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":246,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:24:56.637761+0000","last_change":"2023-10-09T05:09:57.866732+0000","last_active":"2023-10-09T05:24:56.637761+0000","last_peered":"2023-10-09T05:24:56.637761+0000","last_clean":"2023-10-09T05:24:56.637761+0000","last_became_active":"2023-10-09T05:09:57.866117+0000","last_became_peered":"2023-10-09T05:09:57.866117+0000","last_unstale":"2023-10-09T05:24:56.637761+0000","last_undegraded":"2023-10-09T05:24:56.637761+0000","last_fullsized":"2023-10-09T05:24:56.637761+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:49:14.815342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":20,"num_read_kb":19,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":46,"num_bytes_recovered":21458,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234514,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":918660,"kb_used_data":3324,"kb_used_omap":0,"kb_used_meta":915328,"kb_avail":92826492,"statfs":{"total":95995035648,"available":95054327808,"internally_reserved":0,"allocated":3403776,"data_stored":2515823,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":937295872},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:24:54 2023","interfaces":[{"interface":"back","average":{"1min":0.78900000000000003,"5min":1.2050000000000001,"15min":1.6990000000000001},"min":{"1min":0.34100000000000003,"5min":0.28399999999999997,"15min":0.24299999999999999},"max":{"1min":1.349,"5min":8.8000000000000007,"15min":36.018000000000001},"last":0.98599999999999999},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.4059999999999999,"15min":1.895},"min":{"1min":0.61799999999999999,"5min":0.307,"15min":0.28599999999999998},"max":{"1min":3.758,"5min":8.8170000000000002,"15min":36.067999999999998},"last":0.74199999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:24:30 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":1.8740000000000001,"15min":1.5800000000000001},"min":{"1min":0.41899999999999998,"5min":0.28199999999999997,"15min":0.26200000000000001},"max":{"1min":1.294,"5min":23.530999999999999,"15min":35.134},"last":1.0860000000000001},{"interface":"front","average":{"1min":0.77900000000000003,"5min":1.776,"15min":1.7709999999999999},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.27100000000000002},"max":{"1min":1.7170000000000001,"5min":23.443000000000001,"15min":35.116999999999997},"last":0.63800000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:24:54 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.492,"15min":1.6930000000000001},"min":{"1min":0.57399999999999995,"5min":0.31900000000000001,"15min":0.29999999999999999},"max":{"1min":4.7610000000000001,"5min":7.0609999999999999,"15min":66.858000000000004},"last":0.94099999999999995},{"interface":"front","average":{"1min":0.95099999999999996,"5min":1.2749999999999999,"15min":1.524},"min":{"1min":0.60199999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.1639999999999997,"5min":5.9130000000000003,"15min":68.998999999999995},"last":0.69299999999999995}]},{"osd":3,"last update":"Mon Oct 9 05:24:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":2.0590000000000002,"15min":2.585},"min":{"1min":0.72699999999999998,"5min":0.35899999999999999,"15min":0.34899999999999998},"max":{"1min":4.1970000000000001,"5min":23.885000000000002,"15min":99.683999999999997},"last":0.80200000000000005},{"interface":"front","average":{"1min":1.1259999999999999,"5min":2.3959999999999999,"15min":2.6760000000000002},"min":{"1min":0.48999999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":3.879,"5min":40.494999999999997,"15min":99.411000000000001},"last":0.82999999999999996}]},{"osd":4,"last update":"Mon Oct 9 05:24:30 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.77900000000000003,"15min":0.94699999999999995},"min":{"1min":0.41899999999999998,"5min":0.35699999999999998,"15min":0.313},"max":{"1min":1.337,"5min":2.9169999999999998,"15min":9.7850000000000001},"last":1.1619999999999999},{"interface":"front","average":{"1min":1.0580000000000001,"5min":0.97499999999999998,"15min":1.1559999999999999},"min":{"1min":0.46400000000000002,"5min":0.38,"15min":0.34799999999999998},"max":{"1min":1.3440000000000001,"5min":3.8799999999999999,"15min":11.428000000000001},"last":1.056}]},{"osd":5,"last update":"Mon Oct 9 05:24:30 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":1.071,"15min":1.046},"min":{"1min":0.56599999999999995,"5min":0.36799999999999999,"15min":0.35999999999999999},"max":{"1min":1.2609999999999999,"5min":4.8689999999999998,"15min":5.9649999999999999},"last":0.88100000000000001},{"interface":"front","average":{"1min":1.0269999999999999,"5min":0.86899999999999999,"15min":1.1040000000000001},"min":{"1min":0.60699999999999998,"5min":0.39200000000000002,"15min":0.37},"max":{"1min":1.3660000000000001,"5min":3.0289999999999999,"15min":10.103},"last":1.2649999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:24:30 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":0.99199999999999999,"15min":1.0860000000000001},"min":{"1min":0.59599999999999997,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.395,"5min":5.9379999999999997,"15min":8.4130000000000003},"last":1.214},{"interface":"front","average":{"1min":1.127,"5min":1.165,"15min":1.3029999999999999},"min":{"1min":0.58999999999999997,"5min":0.43099999999999999,"15min":0.38200000000000001},"max":{"1min":1.5509999999999999,"5min":6.7910000000000004,"15min":11.797000000000001},"last":1.133}]}]},{"osd":6,"up_from":46,"seq":197568496156,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":903040,"kb_used_data":3320,"kb_used_omap":0,"kb_used_meta":899712,"kb_avail":92842112,"statfs":{"total":95995035648,"available":95070322688,"internally_reserved":0,"allocated":3399680,"data_stored":2516554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921305088},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:24:48 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.95899999999999996,"15min":2.7839999999999998},"min":{"1min":0.41999999999999998,"5min":0.26300000000000001,"15min":0.23599999999999999},"max":{"1min":1.0960000000000001,"5min":9.0239999999999991,"15min":98.275000000000006},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.93100000000000005,"5min":1.093,"15min":2.9039999999999999},"min":{"1min":0.68600000000000005,"5min":0.33700000000000002,"15min":0.309},"max":{"1min":1.284,"5min":9.1110000000000007,"15min":98.334999999999994},"last":0.754}]},{"osd":1,"last update":"Mon Oct 9 05:24:48 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":3.8380000000000001,"15min":2.3980000000000001},"min":{"1min":0.55200000000000005,"5min":0.27500000000000002,"15min":0.26700000000000002},"max":{"1min":0.83699999999999997,"5min":120.873,"15min":120.873},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.93200000000000005,"5min":3.4239999999999999,"15min":2.2930000000000001},"min":{"1min":0.59599999999999997,"5min":0.30199999999999999,"15min":0.28899999999999998},"max":{"1min":1.3720000000000001,"5min":119.248,"15min":119.248},"last":1.03}]},{"osd":2,"last update":"Mon Oct 9 05:24:48 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":1.5069999999999999,"15min":1.1830000000000001},"min":{"1min":0.76300000000000001,"5min":0.30199999999999999,"15min":0.28699999999999998},"max":{"1min":1.399,"5min":8.718,"15min":13.568},"last":0.84899999999999998},{"interface":"front","average":{"1min":0.85399999999999998,"5min":1.877,"15min":1.4179999999999999},"min":{"1min":0.63900000000000001,"5min":0.34399999999999997,"15min":0.32400000000000001},"max":{"1min":1.7210000000000001,"5min":13.712,"15min":13.712},"last":0.79700000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:24:48 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":5.5410000000000004,"15min":4.452},"min":{"1min":0.48799999999999999,"5min":0.34799999999999998,"15min":0.32400000000000001},"max":{"1min":1.7549999999999999,"5min":115.658,"15min":115.658},"last":0.72499999999999998},{"interface":"front","average":{"1min":1.167,"5min":5.2949999999999999,"15min":4.4240000000000004},"min":{"1min":0.78100000000000003,"5min":0.36799999999999999,"15min":0.30199999999999999},"max":{"1min":1.857,"5min":115.489,"15min":115.489},"last":0.93700000000000006}]},{"osd":4,"last update":"Mon Oct 9 05:24:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":1.046,"15min":1.056},"min":{"1min":0.63500000000000001,"5min":0.27600000000000002,"15min":0.27000000000000002},"max":{"1min":1.663,"5min":7.7000000000000002,"15min":8.3670000000000009},"last":0.99199999999999999},{"interface":"front","average":{"1min":1.236,"5min":0.91300000000000003,"15min":1.048},"min":{"1min":1.032,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.6299999999999999,"5min":4.7279999999999998,"15min":13.473000000000001},"last":1.079}]},{"osd":5,"last update":"Mon Oct 9 05:24:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":1.087,"15min":1.0660000000000001},"min":{"1min":0.59899999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.579,"5min":7.5899999999999999,"15min":7.5899999999999999},"last":0.91200000000000003},{"interface":"front","average":{"1min":1.2769999999999999,"5min":0.97499999999999998,"15min":1.1000000000000001},"min":{"1min":1.0029999999999999,"5min":0.38,"15min":0.35899999999999999},"max":{"1min":1.716,"5min":4.8220000000000001,"15min":13.292999999999999},"last":1.1479999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:24:16 2023","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":1.2749999999999999,"15min":1.294},"min":{"1min":0.39400000000000002,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.3,"5min":6.6210000000000004,"15min":8.1120000000000001},"last":1.0509999999999999},{"interface":"front","average":{"1min":1.0740000000000001,"5min":1.367,"15min":1.4330000000000001},"min":{"1min":0.47099999999999997,"5min":0.46500000000000002,"15min":0.433},"max":{"1min":1.3919999999999999,"5min":6.6630000000000003,"15min":8.1859999999999999},"last":1.119}]}]},{"osd":1,"up_from":96,"seq":412316860746,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1865056,"kb_used_data":982808,"kb_used_omap":4,"kb_used_meta":882235,"kb_avail":91880096,"statfs":{"total":95995035648,"available":94085218304,"internally_reserved":0,"allocated":1006395392,"data_stored":1004899328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":903409550},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:24:43 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":2.597,"15min":1.986},"min":{"1min":0.251,"5min":0.19500000000000001,"15min":0.17299999999999999},"max":{"1min":3.3650000000000002,"5min":83.194999999999993,"15min":83.194999999999993},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.42499999999999999,"5min":3.1960000000000002,"15min":2.3119999999999998},"min":{"1min":0.27400000000000002,"5min":0.186,"15min":0.186},"max":{"1min":0.63100000000000001,"5min":83.972999999999999,"15min":83.972999999999999},"last":0.65400000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:24:46 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":3.4750000000000001,"15min":1.917},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":2.996,"5min":83.337000000000003,"15min":83.337000000000003},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.78900000000000003,"5min":3.5049999999999999,"15min":2.0219999999999998},"min":{"1min":0.316,"5min":0.29299999999999998,"15min":0.23599999999999999},"max":{"1min":2.7810000000000001,"5min":83.143000000000001,"15min":83.143000000000001},"last":0.997}]},{"osd":3,"last update":"Mon Oct 9 05:24:46 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":3.4660000000000002,"15min":2.6389999999999998},"min":{"1min":0.316,"5min":0.215,"15min":0.215},"max":{"1min":0.93999999999999995,"5min":84.105999999999995,"15min":84.105999999999995},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.73599999999999999,"5min":3.8580000000000001,"15min":2.8130000000000002},"min":{"1min":0.34200000000000003,"5min":0.26900000000000002,"15min":0.26800000000000002},"max":{"1min":2.6230000000000002,"5min":83.566000000000003,"15min":83.566000000000003},"last":0.93200000000000005}]},{"osd":4,"last update":"Mon Oct 9 05:24:43 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":3.1600000000000001,"15min":1.925},"min":{"1min":0.56100000000000005,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":3.0289999999999999,"5min":83.453999999999994,"15min":83.453999999999994},"last":1.0429999999999999},{"interface":"front","average":{"1min":1.0069999999999999,"5min":2.4129999999999998,"15min":1.7230000000000001},"min":{"1min":0.622,"5min":0.32400000000000001,"15min":0.28100000000000003},"max":{"1min":3.4039999999999999,"5min":83.375,"15min":83.375},"last":1.0620000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:24:43 2023","interfaces":[{"interface":"back","average":{"1min":0.83599999999999997,"5min":1.669,"15min":1.6479999999999999},"min":{"1min":0.56499999999999995,"5min":0.378,"15min":0.28899999999999998},"max":{"1min":1.1859999999999999,"5min":8.1129999999999995,"15min":9.0530000000000008},"last":1.1359999999999999},{"interface":"front","average":{"1min":0.91600000000000004,"5min":2.9420000000000002,"15min":1.879},"min":{"1min":0.54200000000000004,"5min":0.40600000000000003,"15min":0.38200000000000001},"max":{"1min":3.0710000000000002,"5min":83.268000000000001,"15min":83.268000000000001},"last":1.101}]},{"osd":6,"last update":"Mon Oct 9 05:24:43 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":1.76,"15min":1.6120000000000001},"min":{"1min":0.68500000000000005,"5min":0.34899999999999998,"15min":0.34699999999999998},"max":{"1min":1.268,"5min":8.1509999999999998,"15min":8.5630000000000006},"last":1.198},{"interface":"front","average":{"1min":0.96499999999999997,"5min":2.9980000000000002,"15min":1.9359999999999999},"min":{"1min":0.60699999999999998,"5min":0.376,"15min":0.373},"max":{"1min":3.097,"5min":83.400000000000006,"15min":83.400000000000006},"last":1.2909999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:24:43 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":2.9849999999999999,"15min":2.0419999999999998},"min":{"1min":0.57699999999999996,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":2.6659999999999999,"5min":83.510000000000005,"15min":83.510000000000005},"last":1.3460000000000001},{"interface":"front","average":{"1min":1.0569999999999999,"5min":2.407,"15min":1.8089999999999999},"min":{"1min":0.70399999999999996,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":3.4300000000000002,"5min":83.480999999999995,"15min":83.480999999999995},"last":1.238}]}]},{"osd":0,"up_from":88,"seq":377957122388,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1066668,"kb_used_data":294500,"kb_used_omap":5,"kb_used_meta":772154,"kb_avail":92678484,"statfs":{"total":95995035648,"available":94902767616,"internally_reserved":0,"allocated":301568000,"data_stored":300118873,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":790686330},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:24:37 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":3.3700000000000001,"15min":2.6699999999999999},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.17799999999999999},"max":{"1min":4.3300000000000001,"5min":80.843999999999994,"15min":109.88},"last":0.84299999999999997},{"interface":"front","average":{"1min":0.85399999999999998,"5min":4.1340000000000003,"15min":3.0550000000000002},"min":{"1min":0.248,"5min":0.214,"15min":0.21299999999999999},"max":{"1min":7.3239999999999998,"5min":81.159999999999997,"15min":109.435},"last":0.76800000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:24:48 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":2.383,"15min":1.7450000000000001},"min":{"1min":0.34100000000000003,"5min":0.25600000000000001,"15min":0.24199999999999999},"max":{"1min":1.752,"5min":15.901999999999999,"15min":17.797000000000001},"last":0.90900000000000003},{"interface":"front","average":{"1min":0.52300000000000002,"5min":2.246,"15min":1.7470000000000001},"min":{"1min":0.28000000000000003,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.89200000000000002,"5min":12.962,"15min":17.129999999999999},"last":0.78900000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:24:48 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":3.2509999999999999,"15min":2.4809999999999999},"min":{"1min":0.38600000000000001,"5min":0.22800000000000001,"15min":0.218},"max":{"1min":1.4099999999999999,"5min":82.286000000000001,"15min":82.286000000000001},"last":0.81000000000000005},{"interface":"front","average":{"1min":0.67400000000000004,"5min":3.8239999999999998,"15min":2.8769999999999998},"min":{"1min":0.33900000000000002,"5min":0.32100000000000001,"15min":0.25800000000000001},"max":{"1min":1.8149999999999999,"5min":75.969999999999999,"15min":75.969999999999999},"last":0.871}]},{"osd":4,"last update":"Mon Oct 9 05:24:48 2023","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":1.9119999999999999,"15min":1.5960000000000001},"min":{"1min":0.53400000000000003,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":2.0750000000000002,"5min":9.7460000000000004,"15min":12.255000000000001},"last":0.94099999999999995},{"interface":"front","average":{"1min":0.85199999999999998,"5min":1.583,"15min":1.498},"min":{"1min":0.58299999999999996,"5min":0.34699999999999998,"15min":0.28199999999999997},"max":{"1min":1.25,"5min":8.3580000000000005,"15min":8.3580000000000005},"last":0.92100000000000004}]},{"osd":5,"last update":"Mon Oct 9 05:24:48 2023","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":1.5129999999999999,"15min":1.4490000000000001},"min":{"1min":0.61599999999999999,"5min":0.372,"15min":0.29899999999999999},"max":{"1min":1.276,"5min":11.916,"15min":11.916},"last":1.012},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.9970000000000001,"15min":1.7949999999999999},"min":{"1min":0.55700000000000005,"5min":0.39500000000000002,"15min":0.36699999999999999},"max":{"1min":2.0150000000000001,"5min":9.8010000000000002,"15min":12.143000000000001},"last":1.1759999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:24:48 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":1.629,"15min":1.6240000000000001},"min":{"1min":0.629,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.0620000000000001,"5min":12.125,"15min":12.125},"last":0.99099999999999999},{"interface":"front","average":{"1min":0.88300000000000001,"5min":1.6319999999999999,"15min":1.573},"min":{"1min":0.58399999999999996,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.2010000000000001,"5min":12.007999999999999,"15min":12.007999999999999},"last":1.1060000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:24:48 2023","interfaces":[{"interface":"back","average":{"1min":0.86099999999999999,"5min":1.6060000000000001,"15min":1.669},"min":{"1min":0.68700000000000006,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.2370000000000001,"5min":11.977,"15min":11.977},"last":1.3029999999999999},{"interface":"front","average":{"1min":1.0169999999999999,"5min":2.0659999999999998,"15min":1.861},"min":{"1min":0.68300000000000005,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.9610000000000001,"5min":9.7750000000000004,"15min":12.1},"last":1.206}]}]},{"osd":2,"up_from":106,"seq":455266533666,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":565816,"kb_used_data":3312,"kb_used_omap":7,"kb_used_meta":562488,"kb_avail":93179336,"statfs":{"total":95995035648,"available":95415640064,"internally_reserved":0,"allocated":3391488,"data_stored":1988061,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":575987793},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:24:43 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":2.048,"15min":1.9330000000000001},"min":{"1min":0.224,"5min":0.193,"15min":0.193},"max":{"1min":5.6180000000000003,"5min":18.079999999999998,"15min":53.088000000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.57699999999999996,"5min":2.0760000000000001,"15min":1.9330000000000001},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":2.0659999999999998,"5min":16.870000000000001,"15min":46.762},"last":0.32700000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:24:43 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":2.359,"15min":2.117},"min":{"1min":0.26900000000000002,"5min":0.255,"15min":0.224},"max":{"1min":3.3119999999999998,"5min":23.251000000000001,"15min":68.933999999999997},"last":0.77400000000000002},{"interface":"front","average":{"1min":1.0489999999999999,"5min":2.617,"15min":2.2010000000000001},"min":{"1min":0.26600000000000001,"5min":0.25900000000000001,"15min":0.20399999999999999},"max":{"1min":7.9189999999999996,"5min":17.632999999999999,"15min":68.846000000000004},"last":0.64200000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:24:57 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":2.7589999999999999,"15min":2.2200000000000002},"min":{"1min":0.34300000000000003,"5min":0.33700000000000002,"15min":0.23699999999999999},"max":{"1min":2.2010000000000001,"5min":13.933999999999999,"15min":68.968000000000004},"last":0.88300000000000001},{"interface":"front","average":{"1min":0.70299999999999996,"5min":2.54,"15min":1.9870000000000001},"min":{"1min":0.311,"5min":0.29299999999999998,"15min":0.23899999999999999},"max":{"1min":3.5539999999999998,"5min":13.441000000000001,"15min":68.891999999999996},"last":0.72499999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:24:43 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":1.7889999999999999,"15min":1.4279999999999999},"min":{"1min":0.498,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.9059999999999999,"5min":11.157999999999999,"15min":11.157999999999999},"last":0.96099999999999997},{"interface":"front","average":{"1min":0.83299999999999996,"5min":1.8779999999999999,"15min":1.3919999999999999},"min":{"1min":0.40400000000000003,"5min":0.34200000000000003,"15min":0.29699999999999999},"max":{"1min":2.008,"5min":12.460000000000001,"15min":12.460000000000001},"last":0.85199999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:24:43 2023","interfaces":[{"interface":"back","average":{"1min":1.403,"5min":1.6020000000000001,"15min":1.4079999999999999},"min":{"1min":0.60199999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":5.7149999999999999,"5min":8.2910000000000004,"15min":9.9190000000000005},"last":1.071},{"interface":"front","average":{"1min":0.875,"5min":1.8300000000000001,"15min":1.649},"min":{"1min":0.40899999999999997,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.9299999999999999,"5min":11.205,"15min":11.205},"last":1.105}]},{"osd":6,"last update":"Mon Oct 9 05:24:43 2023","interfaces":[{"interface":"back","average":{"1min":1.2330000000000001,"5min":2.1139999999999999,"15min":1.6080000000000001},"min":{"1min":0.35299999999999998,"5min":0.33400000000000002,"15min":0.32500000000000001},"max":{"1min":7.9610000000000003,"5min":12.503,"15min":16.337},"last":0.93400000000000005},{"interface":"front","average":{"1min":1.5329999999999999,"5min":1.6659999999999999,"15min":1.5429999999999999},"min":{"1min":0.63100000000000001,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":5.6760000000000002,"5min":7.4349999999999996,"15min":16.399999999999999},"last":0.91200000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:24:43 2023","interfaces":[{"interface":"back","average":{"1min":1.3049999999999999,"5min":1.671,"15min":1.472},"min":{"1min":0.61499999999999999,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":5.5880000000000001,"5min":9.6340000000000003,"15min":9.9410000000000007},"last":0.98799999999999999},{"interface":"front","average":{"1min":0.95199999999999996,"5min":1.944,"15min":1.615},"min":{"1min":0.53000000000000003,"5min":0.379,"15min":0.379},"max":{"1min":1.8680000000000001,"5min":11.236000000000001,"15min":11.236000000000001},"last":1.0389999999999999}]}]},{"osd":3,"up_from":123,"seq":528280977685,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":742632,"kb_used_data":3232,"kb_used_omap":2,"kb_used_meta":739389,"kb_avail":93002520,"statfs":{"total":95995035648,"available":95234580480,"internally_reserved":0,"allocated":3309568,"data_stored":1977822,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":757134389},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:24:53 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":1.988,"15min":3.0760000000000001},"min":{"1min":0.27200000000000002,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.79200000000000004,"5min":13.199,"15min":199.84999999999999},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.47299999999999998,"5min":1.8580000000000001,"15min":2.597},"min":{"1min":0.32400000000000001,"5min":0.23999999999999999,"15min":0.188},"max":{"1min":0.76000000000000001,"5min":7.609,"15min":83.234999999999999},"last":0.59999999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:24:53 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":2.258,"15min":2.2679999999999998},"min":{"1min":0.24399999999999999,"5min":0.22,"15min":0.22},"max":{"1min":0.98499999999999999,"5min":16.096,"15min":44.206000000000003},"last":0.628},{"interface":"front","average":{"1min":0.56399999999999995,"5min":2.105,"15min":2.0819999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.218},"max":{"1min":1.2010000000000001,"5min":13.141,"15min":44.496000000000002},"last":0.93700000000000006}]},{"osd":2,"last update":"Mon Oct 9 05:24:53 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":1.9339999999999999,"15min":1.7190000000000001},"min":{"1min":0.318,"5min":0.28000000000000003,"15min":0.22700000000000001},"max":{"1min":0.86399999999999999,"5min":9.1679999999999993,"15min":12.794},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.59399999999999997,"5min":2.4289999999999998,"15min":2.0910000000000002},"min":{"1min":0.308,"5min":0.28299999999999997,"15min":0.22800000000000001},"max":{"1min":1.073,"5min":16.009,"15min":16.009},"last":0.73899999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:24:53 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":1.907,"15min":1.754},"min":{"1min":0.51400000000000001,"5min":0.35599999999999998,"15min":0.314},"max":{"1min":1.1080000000000001,"5min":8.4760000000000009,"15min":14.833},"last":0.88},{"interface":"front","average":{"1min":0.91800000000000004,"5min":1.758,"15min":2.1469999999999998},"min":{"1min":0.59299999999999997,"5min":0.33300000000000002,"15min":0.316},"max":{"1min":1.3200000000000001,"5min":13.234999999999999,"15min":199.50800000000001},"last":1.3180000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:24:53 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":1.7490000000000001,"15min":2.1899999999999999},"min":{"1min":0.502,"5min":0.35999999999999999,"15min":0.33500000000000002},"max":{"1min":1.248,"5min":13.287000000000001,"15min":199.93799999999999},"last":1.2110000000000001},{"interface":"front","average":{"1min":0.876,"5min":1.2849999999999999,"15min":1.605},"min":{"1min":0.67600000000000005,"5min":0.34899999999999998,"15min":0.33400000000000002},"max":{"1min":1.1619999999999999,"5min":7.7140000000000004,"15min":11.135},"last":1.0700000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:24:53 2023","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":1.468,"15min":1.607},"min":{"1min":0.59399999999999997,"5min":0.375,"15min":0.36099999999999999},"max":{"1min":1.1779999999999999,"5min":7.6280000000000001,"15min":11.173999999999999},"last":1.1160000000000001},{"interface":"front","average":{"1min":0.93100000000000005,"5min":1.9259999999999999,"15min":1.9630000000000001},"min":{"1min":0.59099999999999997,"5min":0.39500000000000002,"15min":0.32800000000000001},"max":{"1min":1.272,"5min":8.4440000000000008,"15min":14.881},"last":1.25}]},{"osd":7,"last update":"Mon Oct 9 05:24:53 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":1.8129999999999999,"15min":1.847},"min":{"1min":0.58399999999999996,"5min":0.38800000000000001,"15min":0.378},"max":{"1min":1.24,"5min":8.5039999999999996,"15min":14.856},"last":1.1659999999999999},{"interface":"front","average":{"1min":0.92400000000000004,"5min":1.948,"15min":2.4009999999999998},"min":{"1min":0.57699999999999996,"5min":0.43099999999999999,"15min":0.38500000000000001},"max":{"1min":1.286,"5min":13.262,"15min":199.96799999999999},"last":1.401}]}]},{"osd":4,"up_from":30,"seq":128849019442,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1179652,"kb_used_data":287100,"kb_used_omap":0,"kb_used_meta":892544,"kb_avail":92565500,"statfs":{"total":95995035648,"available":94787072000,"internally_reserved":0,"allocated":293990400,"data_stored":293102421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":913965056},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:24:47 2023","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":1.266,"15min":2.0710000000000002},"min":{"1min":0.47999999999999998,"5min":0.28899999999999998,"15min":0.24199999999999999},"max":{"1min":1.3460000000000001,"5min":12.468999999999999,"15min":102.669},"last":1.3200000000000001},{"interface":"front","average":{"1min":0.85399999999999998,"5min":1.4039999999999999,"15min":2.0939999999999999},"min":{"1min":0.39600000000000002,"5min":0.33700000000000002,"15min":0.308},"max":{"1min":1.3700000000000001,"5min":12.407999999999999,"15min":102.72499999999999},"last":1.1910000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:24:34 2023","interfaces":[{"interface":"back","average":{"1min":0.83999999999999997,"5min":3.8700000000000001,"15min":2.1600000000000001},"min":{"1min":0.34799999999999998,"5min":0.30399999999999999,"15min":0.25700000000000001},"max":{"1min":3.5379999999999998,"5min":142.297,"15min":142.297},"last":1.0329999999999999},{"interface":"front","average":{"1min":1.0509999999999999,"5min":3.8100000000000001,"15min":2.0310000000000001},"min":{"1min":0.42799999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":3.4609999999999999,"5min":135.11600000000001,"15min":135.11600000000001},"last":1.1220000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:24:47 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":1.716,"15min":1.4570000000000001},"min":{"1min":0.60199999999999998,"5min":0.37,"15min":0.29499999999999998},"max":{"1min":1.367,"5min":18.076000000000001,"15min":66.201999999999998},"last":0.76400000000000001},{"interface":"front","average":{"1min":0.86299999999999999,"5min":1.569,"15min":1.5700000000000001},"min":{"1min":0.46400000000000002,"5min":0.308,"15min":0.29599999999999999},"max":{"1min":1.248,"5min":18.68,"15min":66.361000000000004},"last":0.872}]},{"osd":3,"last update":"Mon Oct 9 05:24:52 2023","interfaces":[{"interface":"back","average":{"1min":1.2250000000000001,"5min":3.5219999999999998,"15min":2.819},"min":{"1min":0.81799999999999995,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":3.6829999999999998,"5min":131.726,"15min":131.726},"last":0.98499999999999999},{"interface":"front","average":{"1min":0.90000000000000002,"5min":4.0339999999999998,"15min":2.8399999999999999},"min":{"1min":0.57999999999999996,"5min":0.36199999999999999,"15min":0.35699999999999998},"max":{"1min":1.302,"5min":131.57300000000001,"15min":131.57300000000001},"last":1.2709999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:24:52 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":0.78800000000000003,"15min":0.90000000000000002},"min":{"1min":0.72799999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.538,"5min":3.544,"15min":7.7549999999999999},"last":1.2529999999999999},{"interface":"front","average":{"1min":1.0649999999999999,"5min":0.872,"15min":1.0529999999999999},"min":{"1min":0.59799999999999998,"5min":0.36799999999999999,"15min":0.34799999999999998},"max":{"1min":1.5609999999999999,"5min":6.1079999999999997,"15min":6.1079999999999997},"last":1.353}]},{"osd":6,"last update":"Mon Oct 9 05:24:34 2023","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":1.107,"15min":1.121},"min":{"1min":0.53100000000000003,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.0910000000000002,"5min":5.5049999999999999,"15min":6.3760000000000003},"last":1.1619999999999999},{"interface":"front","average":{"1min":1.1399999999999999,"5min":0.94799999999999995,"15min":1.052},"min":{"1min":0.53200000000000003,"5min":0.36599999999999999,"15min":0.35999999999999999},"max":{"1min":1.645,"5min":3.6600000000000001,"15min":8.4930000000000003},"last":1.3839999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:24:34 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.0840000000000001,"15min":1.089},"min":{"1min":0.58199999999999996,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.518,"5min":7.7690000000000001,"15min":7.7690000000000001},"last":1.087},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.129,"15min":1.264},"min":{"1min":0.73299999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.4239999999999999,"5min":8.0269999999999992,"15min":8.9689999999999994},"last":1.224}]}]},{"osd":5,"up_from":0,"seq":163208757798,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:25:10.442 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T05:25:10.442 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:25:10.442 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2023-10-09T05:25:10.442 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T05:25:10.442 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T05:25:10.442 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 config set filestore_inject_stall 3 2023-10-09T05:25:10.479 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:10.517 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:10.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:10.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:10.621 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:10.656 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:10.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:10.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:10.761 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:10.794 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:10.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:10.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:10.897 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:10.923 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:10.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:10.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:11.026 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:11.067 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:11.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:11.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:11.171 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:11.199 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:11.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:11.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:11.305 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:11.328 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:11.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:11.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:11.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:11 smithi194 bash[92666]: cluster 2023-10-09T05:25:10.067562+0000 osd.4 (osd.4) 238 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T05:25:11.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:11 smithi194 bash[92666]: cluster 2023-10-09T05:25:10.138404+0000 mgr.y (mgr.25277) 1108 : cluster 0 pgmap v1176: 89 pgs: 89 active+clean; 582 KiB data, 6.9 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:11.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:11 smithi194 bash[92666]: cluster 2023-10-09T05:25:10.664024+0000 osd.7 (osd.7) 215 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T05:25:11.431 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:11.445 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:11 smithi133 bash[102858]: cluster 2023-10-09T05:25:10.067562+0000 osd.4 (osd.4) 238 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T05:25:11.445 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:11 smithi133 bash[102858]: cluster 2023-10-09T05:25:10.138404+0000 mgr.y (mgr.25277) 1108 : cluster 0 pgmap v1176: 89 pgs: 89 active+clean; 582 KiB data, 6.9 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:11.445 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:11 smithi133 bash[102858]: cluster 2023-10-09T05:25:10.664024+0000 osd.7 (osd.7) 215 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T05:25:11.445 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:11 smithi133 bash[105589]: cluster 2023-10-09T05:25:10.067562+0000 osd.4 (osd.4) 238 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T05:25:11.445 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:11 smithi133 bash[105589]: cluster 2023-10-09T05:25:10.138404+0000 mgr.y (mgr.25277) 1108 : cluster 0 pgmap v1176: 89 pgs: 89 active+clean; 582 KiB data, 6.9 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:11.445 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:11 smithi133 bash[105589]: cluster 2023-10-09T05:25:10.664024+0000 osd.7 (osd.7) 215 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T05:25:11.459 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:11.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:11.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:11.562 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:11.587 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:11.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:11.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:11.691 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:11.721 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:11.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:11.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:11.824 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:11.858 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:11.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:11.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:11.962 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:11.987 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:11.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:11.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:12.090 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:12.117 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:12.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:12.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:12.220 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:12.249 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:12.251 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:12 smithi133 bash[105589]: cluster 2023-10-09T05:25:11.075476+0000 osd.6 (osd.6) 215 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T05:25:12.251 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:12 smithi133 bash[102858]: cluster 2023-10-09T05:25:11.075476+0000 osd.6 (osd.6) 215 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T05:25:12.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:12.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:12.353 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:12.382 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:12.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:12.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:12.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:12 smithi194 bash[92666]: cluster 2023-10-09T05:25:11.075476+0000 osd.6 (osd.6) 215 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T05:25:12.486 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:12.517 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:12.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:12.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:12.621 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:12.647 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:12.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:12.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:12.750 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:12.777 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:12.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:12.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:12.880 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:12.915 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:12.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:12.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:13.020 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:13.051 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:13.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:13.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:13.155 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:13.180 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:13.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:13.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:13.284 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:13.310 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:13.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:13.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:13.414 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:13.425 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:13 smithi133 bash[105589]: cluster 2023-10-09T05:25:12.139177+0000 mgr.y (mgr.25277) 1109 : cluster 0 pgmap v1177: 89 pgs: 89 active+clean; 582 KiB data, 6.8 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:13.425 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:13 smithi133 bash[102858]: cluster 2023-10-09T05:25:12.139177+0000 mgr.y (mgr.25277) 1109 : cluster 0 pgmap v1177: 89 pgs: 89 active+clean; 582 KiB data, 6.8 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:13.438 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:13.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:13.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:13.542 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:13.566 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:13.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:13.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:13.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:13 smithi194 bash[92666]: cluster 2023-10-09T05:25:12.139177+0000 mgr.y (mgr.25277) 1109 : cluster 0 pgmap v1177: 89 pgs: 89 active+clean; 582 KiB data, 6.8 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:13.671 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:13.704 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:13.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:13.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:13.808 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:13.835 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:13.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:13.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:13.939 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:13.964 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:13.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:13.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:14.068 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:14.090 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:14.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:14.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:14.193 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:14.225 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:14.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:14.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:14.329 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:14.342 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:25:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:25:14] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.43.0" 2023-10-09T05:25:14.359 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:14.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:14.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:14.462 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:14.485 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:14.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:14.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:14.588 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:14.628 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:14.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:14.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:14.732 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:14.746 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:14 smithi194 bash[92666]: audit 2023-10-09T05:25:13.468404+0000 mon.a (mon.0) 1450 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:14.746 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:14 smithi194 bash[92666]: audit 2023-10-09T05:25:13.469788+0000 mon.a (mon.0) 1451 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:25:14.746 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:14 smithi194 bash[92666]: cluster 2023-10-09T05:25:14.139818+0000 mgr.y (mgr.25277) 1110 : cluster 0 pgmap v1178: 89 pgs: 89 active+clean; 582 KiB data, 6.8 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:14.746 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:14 smithi194 bash[92666]: audit 2023-10-09T05:25:14.358165+0000 mon.a (mon.0) 1452 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2023-10-09T05:25:14.746 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:14 smithi194 bash[92666]: audit 2023-10-09T05:25:14.358730+0000 mon.a (mon.0) 1453 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-10-09T05:25:14.763 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:14.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:14.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:14.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:14 smithi133 bash[102858]: audit 2023-10-09T05:25:13.468404+0000 mon.a (mon.0) 1450 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:14.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:14 smithi133 bash[102858]: audit 2023-10-09T05:25:13.469788+0000 mon.a (mon.0) 1451 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:25:14.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:14 smithi133 bash[102858]: cluster 2023-10-09T05:25:14.139818+0000 mgr.y (mgr.25277) 1110 : cluster 0 pgmap v1178: 89 pgs: 89 active+clean; 582 KiB data, 6.8 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:14.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:14 smithi133 bash[102858]: audit 2023-10-09T05:25:14.358165+0000 mon.a (mon.0) 1452 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2023-10-09T05:25:14.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:14 smithi133 bash[102858]: audit 2023-10-09T05:25:14.358730+0000 mon.a (mon.0) 1453 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-10-09T05:25:14.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:14 smithi133 bash[105589]: audit 2023-10-09T05:25:13.468404+0000 mon.a (mon.0) 1450 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:14.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:14 smithi133 bash[105589]: audit 2023-10-09T05:25:13.469788+0000 mon.a (mon.0) 1451 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:25:14.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:14 smithi133 bash[105589]: cluster 2023-10-09T05:25:14.139818+0000 mgr.y (mgr.25277) 1110 : cluster 0 pgmap v1178: 89 pgs: 89 active+clean; 582 KiB data, 6.8 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:14.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:14 smithi133 bash[105589]: audit 2023-10-09T05:25:14.358165+0000 mon.a (mon.0) 1452 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2023-10-09T05:25:14.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:14 smithi133 bash[105589]: audit 2023-10-09T05:25:14.358730+0000 mon.a (mon.0) 1453 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-10-09T05:25:14.867 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:14.898 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:14.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:14.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:15.003 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:15.033 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:15.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:15.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:15.136 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:15.158 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:15.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:15.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:15.261 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:15.288 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:15.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:15.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:15.391 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:15.413 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:15.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:15.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:15.516 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:15.545 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:15.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:15.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:15.628 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:15 smithi133 bash[102858]: cluster 2023-10-09T05:25:14.615689+0000 osd.5 (osd.5) 248 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T05:25:15.628 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:15 smithi133 bash[102858]: audit 2023-10-09T05:25:14.617172+0000 mon.a (mon.0) 1454 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2023-10-09T05:25:15.628 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:15 smithi133 bash[102858]: audit 2023-10-09T05:25:14.617380+0000 mon.a (mon.0) 1455 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-10-09T05:25:15.628 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:15 smithi133 bash[102858]: cluster 2023-10-09T05:25:14.617497+0000 mon.a (mon.0) 1456 : cluster 0 osdmap e268: 8 total, 8 up, 7 in 2023-10-09T05:25:15.628 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:15 smithi133 bash[102858]: cluster 2023-10-09T05:25:14.618073+0000 osd.4 (osd.4) 239 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T05:25:15.628 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:15 smithi133 bash[102858]: cluster 2023-10-09T05:25:14.621869+0000 osd.7 (osd.7) 216 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T05:25:15.630 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:25:15.648 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:15.676 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:15.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:15.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:15.779 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:15.810 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:15.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:15.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:15.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:15 smithi194 bash[92666]: cluster 2023-10-09T05:25:14.615689+0000 osd.5 (osd.5) 248 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T05:25:15.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:15 smithi194 bash[92666]: audit 2023-10-09T05:25:14.617172+0000 mon.a (mon.0) 1454 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2023-10-09T05:25:15.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:15 smithi194 bash[92666]: audit 2023-10-09T05:25:14.617380+0000 mon.a (mon.0) 1455 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-10-09T05:25:15.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:15 smithi194 bash[92666]: cluster 2023-10-09T05:25:14.617497+0000 mon.a (mon.0) 1456 : cluster 0 osdmap e268: 8 total, 8 up, 7 in 2023-10-09T05:25:15.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:15 smithi194 bash[92666]: cluster 2023-10-09T05:25:14.618073+0000 osd.4 (osd.4) 239 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T05:25:15.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:15 smithi194 bash[92666]: cluster 2023-10-09T05:25:14.621869+0000 osd.7 (osd.7) 216 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T05:25:15.915 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:15.949 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:15.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:15.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:16.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:15 smithi133 bash[105589]: cluster 2023-10-09T05:25:14.615689+0000 osd.5 (osd.5) 248 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T05:25:16.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:15 smithi133 bash[105589]: audit 2023-10-09T05:25:14.617172+0000 mon.a (mon.0) 1454 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2023-10-09T05:25:16.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:15 smithi133 bash[105589]: audit 2023-10-09T05:25:14.617380+0000 mon.a (mon.0) 1455 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-10-09T05:25:16.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:15 smithi133 bash[105589]: cluster 2023-10-09T05:25:14.617497+0000 mon.a (mon.0) 1456 : cluster 0 osdmap e268: 8 total, 8 up, 7 in 2023-10-09T05:25:16.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:15 smithi133 bash[105589]: cluster 2023-10-09T05:25:14.618073+0000 osd.4 (osd.4) 239 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T05:25:16.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:15 smithi133 bash[105589]: cluster 2023-10-09T05:25:14.621869+0000 osd.7 (osd.7) 216 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T05:25:16.053 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:16.092 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:16.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:16.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:16.195 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:16.228 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:16.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:16.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:16.332 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:16.364 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:16.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:16.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:16.467 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:16.499 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:16.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:16.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:16.603 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:16.635 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:16.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:16.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:16.739 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:16.762 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:16.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:16.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:16.865 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:16.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:16 smithi194 bash[92666]: cluster 2023-10-09T05:25:15.619148+0000 osd.7 (osd.7) 217 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T05:25:16.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:16 smithi194 bash[92666]: cluster 2023-10-09T05:25:15.620593+0000 osd.4 (osd.4) 240 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T05:25:16.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:16 smithi194 bash[92666]: cluster 2023-10-09T05:25:15.621504+0000 mon.a (mon.0) 1457 : cluster 0 osdmap e269: 8 total, 8 up, 7 in 2023-10-09T05:25:16.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:16 smithi194 bash[92666]: cluster 2023-10-09T05:25:15.621593+0000 osd.5 (osd.5) 249 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T05:25:16.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:16 smithi194 bash[92666]: cluster 2023-10-09T05:25:15.625236+0000 osd.4 (osd.4) 241 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T05:25:16.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:16 smithi194 bash[92666]: cluster 2023-10-09T05:25:15.655553+0000 osd.6 (osd.6) 216 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T05:25:16.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:16 smithi194 bash[92666]: cluster 2023-10-09T05:25:16.140596+0000 mgr.y (mgr.25277) 1111 : cluster 0 pgmap v1181: 89 pgs: 89 active+clean; 582 KiB data, 6.8 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:16.896 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:16.897 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:16 smithi133 bash[105589]: cluster 2023-10-09T05:25:15.619148+0000 osd.7 (osd.7) 217 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T05:25:16.898 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:16 smithi133 bash[105589]: cluster 2023-10-09T05:25:15.620593+0000 osd.4 (osd.4) 240 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T05:25:16.898 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:16 smithi133 bash[105589]: cluster 2023-10-09T05:25:15.621504+0000 mon.a (mon.0) 1457 : cluster 0 osdmap e269: 8 total, 8 up, 7 in 2023-10-09T05:25:16.898 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:16 smithi133 bash[105589]: cluster 2023-10-09T05:25:15.621593+0000 osd.5 (osd.5) 249 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T05:25:16.898 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:16 smithi133 bash[105589]: cluster 2023-10-09T05:25:15.625236+0000 osd.4 (osd.4) 241 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T05:25:16.898 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:16 smithi133 bash[105589]: cluster 2023-10-09T05:25:15.655553+0000 osd.6 (osd.6) 216 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T05:25:16.898 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:16 smithi133 bash[105589]: cluster 2023-10-09T05:25:16.140596+0000 mgr.y (mgr.25277) 1111 : cluster 0 pgmap v1181: 89 pgs: 89 active+clean; 582 KiB data, 6.8 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:16.898 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:16 smithi133 bash[102858]: cluster 2023-10-09T05:25:15.619148+0000 osd.7 (osd.7) 217 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T05:25:16.898 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:16 smithi133 bash[102858]: cluster 2023-10-09T05:25:15.620593+0000 osd.4 (osd.4) 240 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T05:25:16.898 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:16 smithi133 bash[102858]: cluster 2023-10-09T05:25:15.621504+0000 mon.a (mon.0) 1457 : cluster 0 osdmap e269: 8 total, 8 up, 7 in 2023-10-09T05:25:16.899 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:16 smithi133 bash[102858]: cluster 2023-10-09T05:25:15.621593+0000 osd.5 (osd.5) 249 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T05:25:16.899 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:16 smithi133 bash[102858]: cluster 2023-10-09T05:25:15.625236+0000 osd.4 (osd.4) 241 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T05:25:16.899 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:16 smithi133 bash[102858]: cluster 2023-10-09T05:25:15.655553+0000 osd.6 (osd.6) 216 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T05:25:16.899 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:16 smithi133 bash[102858]: cluster 2023-10-09T05:25:16.140596+0000 mgr.y (mgr.25277) 1111 : cluster 0 pgmap v1181: 89 pgs: 89 active+clean; 582 KiB data, 6.8 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:16.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:16.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:17.000 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:17.026 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:17.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:17.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:17.130 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:17.162 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:17.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:17.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:17.266 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:17.297 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:17.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:17.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:17.401 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:17.423 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:17.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:17.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:17.527 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:17.552 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:17.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:17.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:17.656 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:17.678 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:17.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:17.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:17.782 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:17.807 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:17.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:17.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:17.910 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:17.915 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:25:17.934 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:25:17.934 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:25:17.944 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:17.945 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:25:17.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:17.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:17.970 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:25:17.994 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:25:18.020 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:25:18.048 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:18.049 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:25:18.073 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:25:18.074 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:18.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:18.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:18.100 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:25:18.101 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:25:18.101 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:25:18.102 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:25:18.102 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:25:18.102 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:25:18.102 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:25:18.102 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:25:18.102 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:25:18.177 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:17 smithi133 bash[102858]: audit 2023-10-09T05:25:17.884544+0000 mon.a (mon.0) 1458 : audit 0 from='client.? 172.21.15.133:0/3102382134' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:25:18.178 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:17 smithi133 bash[105589]: audit 2023-10-09T05:25:17.884544+0000 mon.a (mon.0) 1458 : audit 0 from='client.? 172.21.15.133:0/3102382134' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:25:18.178 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:18.210 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:18.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:18.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:18.313 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:18.325 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:17 smithi194 bash[92666]: audit 2023-10-09T05:25:17.884544+0000 mon.a (mon.0) 1458 : audit 0 from='client.? 172.21.15.133:0/3102382134' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:25:18.345 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:18.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:18.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:18.352 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:25:18.449 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:18.478 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:18.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:18.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:18.581 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:18.612 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:18.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:18.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:18.716 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:18.738 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:18.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:18.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:18.840 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:18.874 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:18.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:18.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:18.946 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:18 smithi133 bash[102858]: cluster 2023-10-09T05:25:18.141041+0000 mgr.y (mgr.25277) 1112 : cluster 0 pgmap v1182: 89 pgs: 89 active+clean; 582 KiB data, 6.8 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:18.946 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:18 smithi133 bash[102858]: audit 2023-10-09T05:25:18.330464+0000 mon.a (mon.0) 1459 : audit 1 from='client.? 172.21.15.133:0/665551449' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:25:18.956 INFO:teuthology.orchestra.run.smithi133.stderr:enabled application 'rados' on pool 'unique_pool_7' 2023-10-09T05:25:18.978 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:19.012 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:19.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:19.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:19.115 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:19.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:18 smithi194 bash[92666]: cluster 2023-10-09T05:25:18.141041+0000 mgr.y (mgr.25277) 1112 : cluster 0 pgmap v1182: 89 pgs: 89 active+clean; 582 KiB data, 6.8 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:19.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:18 smithi194 bash[92666]: audit 2023-10-09T05:25:18.330464+0000 mon.a (mon.0) 1459 : audit 1 from='client.? 172.21.15.133:0/665551449' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:25:19.141 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:19.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:19.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:19.244 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:19.280 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:19.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:19.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:19.323 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:18 smithi133 bash[105589]: cluster 2023-10-09T05:25:18.141041+0000 mgr.y (mgr.25277) 1112 : cluster 0 pgmap v1182: 89 pgs: 89 active+clean; 582 KiB data, 6.8 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:19.323 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:18 smithi133 bash[105589]: audit 2023-10-09T05:25:18.330464+0000 mon.a (mon.0) 1459 : audit 1 from='client.? 172.21.15.133:0/665551449' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:25:19.384 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:19.415 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:19.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:19.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:19.519 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:19.542 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:19.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:19.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:19.646 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:19.678 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:19.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:19.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:19.782 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:19.808 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:19.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:19.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:19.912 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:19.999 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:20.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:20.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:20.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:19 smithi133 bash[105589]: audit 2023-10-09T05:25:18.949288+0000 mon.a (mon.0) 1460 : audit 1 from='client.? 172.21.15.133:0/665551449' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:25:20.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:19 smithi133 bash[105589]: cluster 2023-10-09T05:25:18.949388+0000 mon.a (mon.0) 1461 : cluster 0 osdmap e270: 8 total, 8 up, 7 in 2023-10-09T05:25:20.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:19 smithi133 bash[105589]: cluster 2023-10-09T05:25:19.756480+0000 osd.6 (osd.6) 217 : cluster 3 failed to encode map e270 with expected crc 2023-10-09T05:25:20.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:19 smithi133 bash[102858]: audit 2023-10-09T05:25:18.949288+0000 mon.a (mon.0) 1460 : audit 1 from='client.? 172.21.15.133:0/665551449' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:25:20.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:19 smithi133 bash[102858]: cluster 2023-10-09T05:25:18.949388+0000 mon.a (mon.0) 1461 : cluster 0 osdmap e270: 8 total, 8 up, 7 in 2023-10-09T05:25:20.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:19 smithi133 bash[102858]: cluster 2023-10-09T05:25:19.756480+0000 osd.6 (osd.6) 217 : cluster 3 failed to encode map e270 with expected crc 2023-10-09T05:25:20.102 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:20.137 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:20.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:20.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:20.241 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:20.253 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:19 smithi194 bash[92666]: audit 2023-10-09T05:25:18.949288+0000 mon.a (mon.0) 1460 : audit 1 from='client.? 172.21.15.133:0/665551449' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:25:20.253 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:19 smithi194 bash[92666]: cluster 2023-10-09T05:25:18.949388+0000 mon.a (mon.0) 1461 : cluster 0 osdmap e270: 8 total, 8 up, 7 in 2023-10-09T05:25:20.253 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:19 smithi194 bash[92666]: cluster 2023-10-09T05:25:19.756480+0000 osd.6 (osd.6) 217 : cluster 3 failed to encode map e270 with expected crc 2023-10-09T05:25:20.271 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:20.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:20.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:20.375 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:20.407 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:20.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:20.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:20.511 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:20.543 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:20.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:20.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:20.647 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:20.675 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:20.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:20.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:20.779 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:20.809 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:20.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:20.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:20.913 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:20.946 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:20.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:20.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:21.049 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:21.073 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:21.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:21.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:21.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:20 smithi194 bash[92666]: cluster 2023-10-09T05:25:19.756648+0000 osd.4 (osd.4) 242 : cluster 3 failed to encode map e270 with expected crc 2023-10-09T05:25:21.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:20 smithi194 bash[92666]: cluster 2023-10-09T05:25:19.756650+0000 osd.5 (osd.5) 250 : cluster 3 failed to encode map e270 with expected crc 2023-10-09T05:25:21.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:20 smithi194 bash[92666]: cluster 2023-10-09T05:25:19.756661+0000 osd.7 (osd.7) 218 : cluster 3 failed to encode map e270 with expected crc 2023-10-09T05:25:21.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:20 smithi194 bash[92666]: cluster 2023-10-09T05:25:20.141880+0000 mgr.y (mgr.25277) 1113 : cluster 0 pgmap v1184: 89 pgs: 89 active+clean; 582 KiB data, 6.8 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:21.150 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T05:25:21.177 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:21.200 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:21 smithi133 bash[105589]: cluster 2023-10-09T05:25:19.756648+0000 osd.4 (osd.4) 242 : cluster 3 failed to encode map e270 with expected crc 2023-10-09T05:25:21.200 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:21 smithi133 bash[105589]: cluster 2023-10-09T05:25:19.756650+0000 osd.5 (osd.5) 250 : cluster 3 failed to encode map e270 with expected crc 2023-10-09T05:25:21.200 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:21 smithi133 bash[105589]: cluster 2023-10-09T05:25:19.756661+0000 osd.7 (osd.7) 218 : cluster 3 failed to encode map e270 with expected crc 2023-10-09T05:25:21.201 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:21 smithi133 bash[105589]: cluster 2023-10-09T05:25:20.141880+0000 mgr.y (mgr.25277) 1113 : cluster 0 pgmap v1184: 89 pgs: 89 active+clean; 582 KiB data, 6.8 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:21.201 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:20 smithi133 bash[102858]: cluster 2023-10-09T05:25:19.756648+0000 osd.4 (osd.4) 242 : cluster 3 failed to encode map e270 with expected crc 2023-10-09T05:25:21.201 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:20 smithi133 bash[102858]: cluster 2023-10-09T05:25:19.756650+0000 osd.5 (osd.5) 250 : cluster 3 failed to encode map e270 with expected crc 2023-10-09T05:25:21.201 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:20 smithi133 bash[102858]: cluster 2023-10-09T05:25:19.756661+0000 osd.7 (osd.7) 218 : cluster 3 failed to encode map e270 with expected crc 2023-10-09T05:25:21.201 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:20 smithi133 bash[102858]: cluster 2023-10-09T05:25:20.141880+0000 mgr.y (mgr.25277) 1113 : cluster 0 pgmap v1184: 89 pgs: 89 active+clean; 582 KiB data, 6.8 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:21.212 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:21.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:21.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:21.315 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:21.352 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:21.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:21.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:21.456 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:21.490 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:21.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:21.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:21.593 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:21.622 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:21.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:21.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:21.725 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:21.759 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:21.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:21.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:21.864 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:21.891 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:21.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:21.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:21.995 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:22.029 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:22.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:22.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:22.132 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:22.169 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:22.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:22.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:22.274 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:22.306 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:22.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:22.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:22.410 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:22.442 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:22.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:22.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:22.548 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:22.576 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:22.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:22.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:22.680 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:22.690 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:25:22.718 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:22.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:22.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:22.819 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:22.849 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:22.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:22.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:22.952 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:22.987 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:22.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:22.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:23.091 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:23.123 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:23.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:23.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:23.220 DEBUG:teuthology.orchestra.run.smithi133:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_7 bench 90 write' 2023-10-09T05:25:23.222 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T05:25:23.222 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T05:25:23.228 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:23.262 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:23.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:23.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:23.287 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:25:23.366 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:23.379 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:23 smithi194 bash[92666]: cluster 2023-10-09T05:25:22.142438+0000 mgr.y (mgr.25277) 1114 : cluster 0 pgmap v1185: 89 pgs: 89 active+clean; 582 KiB data, 6.7 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:23.393 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:23.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:23.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:23.496 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:23.506 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:23 smithi133 bash[105589]: cluster 2023-10-09T05:25:22.142438+0000 mgr.y (mgr.25277) 1114 : cluster 0 pgmap v1185: 89 pgs: 89 active+clean; 582 KiB data, 6.7 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:23.506 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:23 smithi133 bash[102858]: cluster 2023-10-09T05:25:22.142438+0000 mgr.y (mgr.25277) 1114 : cluster 0 pgmap v1185: 89 pgs: 89 active+clean; 582 KiB data, 6.7 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:23.522 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:23.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:23.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:23.625 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:23.658 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:hints = 1 2023-10-09T05:25:23.658 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T05:25:23.658 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Object prefix: benchmark_data_smithi133_218956 2023-10-09T05:25:23.658 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:25:23.658 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 0 0 0 0 0 0 - 0 2023-10-09T05:25:23.660 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:23.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:23.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:23.765 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:23.793 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:23.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:23.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:23.897 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:23.919 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:23.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:23.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:24.022 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:24.057 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:24.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:24.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:24.159 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:24.189 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:24.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:24.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:24.293 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:24.318 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:24.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:24.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:24.421 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:24.453 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:24.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:24.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:24.557 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:24.570 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:25:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:25:24] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.43.0" 2023-10-09T05:25:24.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:24.590 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:24.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:24.658 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 1 16 1973 1957 122.306 122.312 0.00354572 0.00815091 2023-10-09T05:25:24.691 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:24.701 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:24 smithi194 bash[92666]: cluster 2023-10-09T05:25:24.142913+0000 mgr.y (mgr.25277) 1115 : cluster 0 pgmap v1186: 89 pgs: 89 active+clean; 582 KiB data, 6.7 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:24.719 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:24.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:24.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:24.822 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:24.849 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:24.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:24.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:24.953 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:24.962 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:24 smithi133 bash[105589]: cluster 2023-10-09T05:25:24.142913+0000 mgr.y (mgr.25277) 1115 : cluster 0 pgmap v1186: 89 pgs: 89 active+clean; 582 KiB data, 6.7 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:24.963 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:24 smithi133 bash[102858]: cluster 2023-10-09T05:25:24.142913+0000 mgr.y (mgr.25277) 1115 : cluster 0 pgmap v1186: 89 pgs: 89 active+clean; 582 KiB data, 6.7 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:25:24.981 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:24.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:24.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:25.085 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:25.112 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:25.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:25.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:25.215 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:25.243 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:25.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:25.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:25.347 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:25.385 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:25.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:25.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:25.490 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:25.519 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:25.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:25.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:25.625 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:25.649 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:25.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:25.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:25.658 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 2 16 4795 4779 149.333 176.375 0.00633964 0.0066855 2023-10-09T05:25:25.753 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:25.780 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:25.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:25.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:25.884 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:25.913 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:25.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:25.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:26.017 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:26.041 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:26.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:26.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:26.145 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:26.172 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:26.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:26.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:26.276 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:26.310 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:26.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:26.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:26.415 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:26.444 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:26.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:26.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:26.548 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:26.573 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:26.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:26.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:26.659 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 3 16 7008 6992 145.655 138.312 0.00508012 0.00685737 2023-10-09T05:25:26.675 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:26.705 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:26.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:26.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:26.808 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:26.843 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:26.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:26.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:26.946 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:26.972 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:26.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:26.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:27.076 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:27.102 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:27.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:27.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:27.203 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:27 smithi133 bash[102858]: cluster 2023-10-09T05:25:26.143465+0000 mgr.y (mgr.25277) 1116 : cluster 0 pgmap v1187: 89 pgs: 89 active+clean; 116 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 12 MiB/s wr, 185 op/s 2023-10-09T05:25:27.205 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:27.214 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:27 smithi194 bash[92666]: cluster 2023-10-09T05:25:26.143465+0000 mgr.y (mgr.25277) 1116 : cluster 0 pgmap v1187: 89 pgs: 89 active+clean; 116 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 12 MiB/s wr, 185 op/s 2023-10-09T05:25:27.237 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:27.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:27.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:27.340 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:27.367 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:27.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:27.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:27.471 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:27.499 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:27.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:27.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:27.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:27 smithi133 bash[105589]: cluster 2023-10-09T05:25:26.143465+0000 mgr.y (mgr.25277) 1116 : cluster 0 pgmap v1187: 89 pgs: 89 active+clean; 116 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 12 MiB/s wr, 185 op/s 2023-10-09T05:25:27.603 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:27.627 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:27.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:27.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:27.659 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 4 16 9602 9586 149.768 162.125 0.00945805 0.00666563 2023-10-09T05:25:27.732 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:27.761 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:27.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:27.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:27.787 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:25:27.788 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:25:27.867 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:27.898 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:27.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:27.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:28.001 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:28.029 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:28.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:28.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:28.133 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:28.162 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:28.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:28.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:28.266 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:28.294 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:28.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:28.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:28.398 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:28.435 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:28.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:28.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:28.539 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:28.565 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:28.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:28.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:28.659 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 5 16 11726 11710 146.362 132.75 0.00506991 0.00682262 2023-10-09T05:25:28.669 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:28.696 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:28.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:28.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:28.799 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:28.831 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:28.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:28.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:28.933 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:28.961 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:28.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:28.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:29.065 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:29.092 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:29.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:29.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:29.196 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:29.227 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:29.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:29.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:29.330 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:29.360 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:29.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:29.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:29.466 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:29.475 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:29 smithi194 bash[92666]: cluster 2023-10-09T05:25:28.144090+0000 mgr.y (mgr.25277) 1117 : cluster 0 pgmap v1188: 89 pgs: 89 active+clean; 169 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 17 MiB/s wr, 269 op/s 2023-10-09T05:25:29.475 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:29 smithi194 bash[92666]: audit 2023-10-09T05:25:28.461911+0000 mon.a (mon.0) 1462 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:25:29.492 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:29.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:29.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:29.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:29 smithi133 bash[105589]: cluster 2023-10-09T05:25:28.144090+0000 mgr.y (mgr.25277) 1117 : cluster 0 pgmap v1188: 89 pgs: 89 active+clean; 169 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 17 MiB/s wr, 269 op/s 2023-10-09T05:25:29.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:29 smithi133 bash[105589]: audit 2023-10-09T05:25:28.461911+0000 mon.a (mon.0) 1462 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:25:29.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:29 smithi133 bash[102858]: cluster 2023-10-09T05:25:28.144090+0000 mgr.y (mgr.25277) 1117 : cluster 0 pgmap v1188: 89 pgs: 89 active+clean; 169 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 17 MiB/s wr, 269 op/s 2023-10-09T05:25:29.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:29 smithi133 bash[102858]: audit 2023-10-09T05:25:28.461911+0000 mon.a (mon.0) 1462 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:25:29.596 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:29.620 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:29.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:29.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:29.659 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 6 16 13959 13943 145.227 139.562 0.00318832 0.00687659 2023-10-09T05:25:29.724 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:29.748 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:29.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:29.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:29.851 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:29.873 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:29.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:29.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:29.976 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:30.005 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:30.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:30.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:30.111 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:30.139 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:30.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:30.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:30.244 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:30.276 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:30.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:30.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:30.381 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:30.407 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:30.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:30.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:30.513 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:30.538 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:30.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:30.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:30.641 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:30.659 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 7 16 16157 16141 144.103 137.375 0.00703149 0.00693287 2023-10-09T05:25:30.671 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:30.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:30.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:30.775 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:30.802 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:30.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:30.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:30.907 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:30.941 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:30.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:30.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:31.046 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:31.077 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:31.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:31.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:31.180 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:31.207 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:31.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:31.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:31.215 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:31 smithi133 bash[102858]: cluster 2023-10-09T05:25:30.144772+0000 mgr.y (mgr.25277) 1118 : cluster 0 pgmap v1189: 89 pgs: 89 active+clean; 671 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 60 MiB/s wr, 957 op/s 2023-10-09T05:25:31.216 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:31 smithi133 bash[105589]: cluster 2023-10-09T05:25:30.144772+0000 mgr.y (mgr.25277) 1118 : cluster 0 pgmap v1189: 89 pgs: 89 active+clean; 671 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 60 MiB/s wr, 957 op/s 2023-10-09T05:25:31.310 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:31.339 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:31.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:31.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:31.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:31 smithi194 bash[92666]: cluster 2023-10-09T05:25:30.144772+0000 mgr.y (mgr.25277) 1118 : cluster 0 pgmap v1189: 89 pgs: 89 active+clean; 671 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 60 MiB/s wr, 957 op/s 2023-10-09T05:25:31.443 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:31.478 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:31.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:31.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:31.584 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:31.609 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:31.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:31.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:31.659 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 8 16 18078 18062 141.097 120.062 0.00514717 0.007081 2023-10-09T05:25:31.713 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:31.744 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:31.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:31.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:31.848 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:31.876 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:31.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:31.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:31.980 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:32.014 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:32.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:32.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:32.119 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:32.152 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:32.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:32.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:32.256 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:32.289 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:32.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:32.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:32.394 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:32.424 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:32.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:32.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:32.529 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:32.563 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:32.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:32.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:32.659 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 9 15 19905 19890 138.113 114.25 0.0106311 0.00723322 2023-10-09T05:25:32.668 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:32.694 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:32.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:32.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:32.800 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:32.826 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:32.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:32.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:32.930 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:32.952 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:32.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:32.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:33.055 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:33.083 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:33.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:33.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:33.188 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:33.209 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:33.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:33.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:33.312 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:33.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:33.340 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:33.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:33.441 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:33.453 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:33 smithi133 bash[102858]: cluster 2023-10-09T05:25:32.145394+0000 mgr.y (mgr.25277) 1119 : cluster 0 pgmap v1190: 89 pgs: 89 active+clean; 887 MiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-09T05:25:33.453 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:33 smithi133 bash[105589]: cluster 2023-10-09T05:25:32.145394+0000 mgr.y (mgr.25277) 1119 : cluster 0 pgmap v1190: 89 pgs: 89 active+clean; 887 MiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-09T05:25:33.472 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:33.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:33.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:33.576 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:33.585 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:33 smithi194 bash[92666]: cluster 2023-10-09T05:25:32.145394+0000 mgr.y (mgr.25277) 1119 : cluster 0 pgmap v1190: 89 pgs: 89 active+clean; 887 MiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-09T05:25:33.604 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:33.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:33.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:33.659 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 10 16 21558 21542 134.626 103.25 0.00631137 0.00742235 2023-10-09T05:25:33.707 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:33.731 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:33.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:33.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:33.836 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:33.855 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:33.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:33.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:33.958 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:33.987 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:33.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:33.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:34.091 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:34.120 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:34.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:34.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:34.225 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:34.251 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:34.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:34.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:34.354 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:34.382 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:34.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:34.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:34.485 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:34.510 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:34.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:34.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:34.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:25:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:25:34] "GET /metrics HTTP/1.1" 200 36467 "" "Prometheus/2.43.0" 2023-10-09T05:25:34.583 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:25:34.583 INFO:teuthology.orchestra.run.smithi133.stdout: "success": "" 2023-10-09T05:25:34.583 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:25:34.612 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:34.643 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:34.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:34.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:34.659 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 11 16 23258 23242 132.045 106.25 0.00553284 0.00756547 2023-10-09T05:25:34.749 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:34.773 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:34.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:34.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:34.876 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:34.884 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:34 smithi194 bash[92666]: cluster 2023-10-09T05:25:34.145947+0000 mgr.y (mgr.25277) 1120 : cluster 0 pgmap v1191: 89 pgs: 89 active+clean; 965 MiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T05:25:34.905 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:34.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:34.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:35.009 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:35.018 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:34 smithi133 bash[105589]: cluster 2023-10-09T05:25:34.145947+0000 mgr.y (mgr.25277) 1120 : cluster 0 pgmap v1191: 89 pgs: 89 active+clean; 965 MiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T05:25:35.018 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:34 smithi133 bash[102858]: cluster 2023-10-09T05:25:34.145947+0000 mgr.y (mgr.25277) 1120 : cluster 0 pgmap v1191: 89 pgs: 89 active+clean; 965 MiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T05:25:35.035 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:35.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:35.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:35.139 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:35.167 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:35.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:35.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:35.270 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:35.287 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:35.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:35.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:35.390 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:35.414 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:35.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:35.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:35.517 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:35.553 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:35.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:35.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:35.658 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:35.661 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 12 16 24457 24441 127.285 74.9375 0.00270716 0.00778671 2023-10-09T05:25:35.676 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:35.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:35.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:35.783 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:35.812 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:35.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:35.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:35.916 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:35.950 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:35.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:35.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:36.056 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:36.085 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:36.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:36.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:36.188 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:36.227 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:36.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:36.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:36.331 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:36.359 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:36.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:36.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:36.462 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:36.488 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:36.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:36.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:36.592 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:36.619 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:36.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:36.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:36.660 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 13 16 26200 26184 125.873 108.938 0.0101388 0.00793883 2023-10-09T05:25:36.723 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:36.748 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:36.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:36.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:36.854 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:36.878 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:36.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:36.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:36.982 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:37.009 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:37.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:37.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:37.114 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:37.144 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:37.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:37.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:37.247 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:37.276 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:37.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:37.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:37.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:37 smithi133 bash[102858]: cluster 2023-10-09T05:25:36.146735+0000 mgr.y (mgr.25277) 1121 : cluster 0 pgmap v1192: 89 pgs: 89 active+clean; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-10-09T05:25:37.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:37 smithi133 bash[105589]: cluster 2023-10-09T05:25:36.146735+0000 mgr.y (mgr.25277) 1121 : cluster 0 pgmap v1192: 89 pgs: 89 active+clean; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-10-09T05:25:37.381 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:37.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:37 smithi194 bash[92666]: cluster 2023-10-09T05:25:36.146735+0000 mgr.y (mgr.25277) 1121 : cluster 0 pgmap v1192: 89 pgs: 89 active+clean; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-10-09T05:25:37.407 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:37.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:37.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:37.510 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:37.534 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:37.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:37.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:37.640 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:37.662 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 14 16 27843 27827 124.216 102.688 0.00488937 0.00804296 2023-10-09T05:25:37.663 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:37.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:37.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:37.768 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:37.802 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:37.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:37.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:37.906 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:37.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:37.936 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:37.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:38.037 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:38.069 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:38.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:38.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:38.172 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:38.204 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:38.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:38.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:38.308 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:38.331 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:38.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:38.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:38.433 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:38.465 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:38.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:38.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:38.569 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:38.595 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:38.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:38.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:38.660 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 15 16 29265 29249 121.86 88.875 0.00803105 0.0081996 2023-10-09T05:25:38.698 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:38.727 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:38.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:38.729 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:38.830 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:38.866 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:38.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:38.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:38.968 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:38.996 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:38.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:38.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:39.100 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:39.126 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:39.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:39.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:39.230 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:39.255 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:39.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:39.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:39.359 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:39.369 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:39 smithi194 bash[92666]: cluster 2023-10-09T05:25:38.147234+0000 mgr.y (mgr.25277) 1122 : cluster 0 pgmap v1193: 89 pgs: 89 active+clean; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 112 MiB/s wr, 1.78k op/s 2023-10-09T05:25:39.389 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:39.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:39.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:39.494 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:39.517 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:39.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:39.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:39.563 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:39 smithi133 bash[105589]: cluster 2023-10-09T05:25:38.147234+0000 mgr.y (mgr.25277) 1122 : cluster 0 pgmap v1193: 89 pgs: 89 active+clean; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 112 MiB/s wr, 1.78k op/s 2023-10-09T05:25:39.563 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:39 smithi133 bash[102858]: cluster 2023-10-09T05:25:38.147234+0000 mgr.y (mgr.25277) 1122 : cluster 0 pgmap v1193: 89 pgs: 89 active+clean; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 112 MiB/s wr, 1.78k op/s 2023-10-09T05:25:39.621 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:39.652 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:39.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:39.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:39.660 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 16 16 30522 30506 119.153 78.5625 0.00998791 0.00838629 2023-10-09T05:25:39.755 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:39.783 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:39.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:39.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:39.886 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:39.919 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:39.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:39.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:40.023 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:40.047 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:40.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:40.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:40.152 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:40.177 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:40.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:40.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:40.282 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:40.310 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:40.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:40.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:40.415 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:40.441 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:40.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:40.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:40.544 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:40.566 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:40.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:40.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:40.663 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 17 16 31645 31629 116.272 70.1875 0.0136578 0.00859507 2023-10-09T05:25:40.670 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:40.719 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:40.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:40.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:40.822 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:40.855 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:40.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:40.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:40.960 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:40.989 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:40.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:40.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:41.092 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:41.124 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:41.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:41.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:41.229 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:41.256 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:41.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:41.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:41.359 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:41.384 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:41 smithi194 bash[92666]: cluster 2023-10-09T05:25:40.147925+0000 mgr.y (mgr.25277) 1123 : cluster 0 pgmap v1194: 89 pgs: 89 active+clean; 1.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 134 MiB/s wr, 2.15k op/s 2023-10-09T05:25:41.390 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:41.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:41.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:41.492 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:41.523 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:41.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:41.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:41.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:41 smithi133 bash[102858]: cluster 2023-10-09T05:25:40.147925+0000 mgr.y (mgr.25277) 1123 : cluster 0 pgmap v1194: 89 pgs: 89 active+clean; 1.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 134 MiB/s wr, 2.15k op/s 2023-10-09T05:25:41.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:41 smithi133 bash[105589]: cluster 2023-10-09T05:25:40.147925+0000 mgr.y (mgr.25277) 1123 : cluster 0 pgmap v1194: 89 pgs: 89 active+clean; 1.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 134 MiB/s wr, 2.15k op/s 2023-10-09T05:25:41.628 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:41.659 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:41.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:41.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:41.664 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 18 15 33275 33260 115.462 101.938 0.00927157 0.00865554 2023-10-09T05:25:41.765 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:41.797 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:41.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:41.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:41.901 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:41.929 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:41.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:41.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:42.034 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:42.066 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:42.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:42.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:42.170 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:42.197 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:42.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:42.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:42.299 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:42.327 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:42.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:42.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:42.433 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:42.462 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:42.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:42.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:42.566 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:42.595 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:42.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:42.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:42.662 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 19 16 34897 34881 114.717 101.312 0.0110215 0.00871095 2023-10-09T05:25:42.698 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:42.729 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:42.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:42.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:42.834 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:42.864 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:42.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:42.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:42.969 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:43.008 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:43.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:43.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:43.111 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:43.138 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:43.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:43.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:43.241 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:43.267 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:43.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:43.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:43.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:43 smithi133 bash[105589]: cluster 2023-10-09T05:25:42.148488+0000 mgr.y (mgr.25277) 1124 : cluster 0 pgmap v1195: 89 pgs: 89 active+clean; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T05:25:43.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:43 smithi133 bash[105589]: audit 2023-10-09T05:25:42.906126+0000 mon.a (mon.0) 1463 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:25:43.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:43 smithi133 bash[102858]: cluster 2023-10-09T05:25:42.148488+0000 mgr.y (mgr.25277) 1124 : cluster 0 pgmap v1195: 89 pgs: 89 active+clean; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T05:25:43.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:43 smithi133 bash[102858]: audit 2023-10-09T05:25:42.906126+0000 mon.a (mon.0) 1463 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:25:43.370 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:43.379 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:43 smithi194 bash[92666]: cluster 2023-10-09T05:25:42.148488+0000 mgr.y (mgr.25277) 1124 : cluster 0 pgmap v1195: 89 pgs: 89 active+clean; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T05:25:43.379 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:43 smithi194 bash[92666]: audit 2023-10-09T05:25:42.906126+0000 mon.a (mon.0) 1463 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:25:43.400 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:43.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:43.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:43.507 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:43.546 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:43.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:43.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:43.649 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:43.662 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T05:25:43.662922+0000 min lat: 0.00116799 max lat: 0.191628 avg lat: 0.00881221 2023-10-09T05:25:43.663 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:25:43.663 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 20 16 36307 36291 113.387 88.125 0.00505864 0.00881221 2023-10-09T05:25:43.675 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:43.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:43.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:43.779 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:43.809 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:43.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:43.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:43.916 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:43.942 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:43.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:43.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:44.046 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:44.077 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:44.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:44.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:44.180 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:44.216 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:44.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:44.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:44.320 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:44.349 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:44.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:44.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:44.385 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:44 smithi133 bash[102858]: audit 2023-10-09T05:25:43.383873+0000 mon.a (mon.0) 1464 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:44.386 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:44 smithi133 bash[102858]: audit 2023-10-09T05:25:43.407833+0000 mon.a (mon.0) 1465 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:44.386 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:44 smithi133 bash[102858]: audit 2023-10-09T05:25:43.462326+0000 mon.a (mon.0) 1466 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:25:44.386 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:25:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:25:44] "GET /metrics HTTP/1.1" 200 36471 "" "Prometheus/2.43.0" 2023-10-09T05:25:44.455 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:44.484 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:44.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:44.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:44.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:44 smithi133 bash[105589]: audit 2023-10-09T05:25:43.383873+0000 mon.a (mon.0) 1464 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:44.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:44 smithi133 bash[105589]: audit 2023-10-09T05:25:43.407833+0000 mon.a (mon.0) 1465 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:44.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:44 smithi133 bash[105589]: audit 2023-10-09T05:25:43.462326+0000 mon.a (mon.0) 1466 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:25:44.588 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:44.598 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:44 smithi194 bash[92666]: audit 2023-10-09T05:25:43.383873+0000 mon.a (mon.0) 1464 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:44.598 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:44 smithi194 bash[92666]: audit 2023-10-09T05:25:43.407833+0000 mon.a (mon.0) 1465 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:44.598 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:44 smithi194 bash[92666]: audit 2023-10-09T05:25:43.462326+0000 mon.a (mon.0) 1466 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:25:44.617 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:44.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:44.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:44.663 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 21 16 37632 37616 111.931 82.8125 0.0128846 0.00892775 2023-10-09T05:25:44.724 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:44.751 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:44.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:44.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:44.854 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:44.882 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:44.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:44.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:44.986 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:45.023 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:45.025 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 6, 1, 7] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:25:45.025 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T05:25:45.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:45.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:45.128 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:45.157 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:45.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:45.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:45.261 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:45.298 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:45.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:45.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:45.407 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:45.441 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:45.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:45.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:45.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:45 smithi133 bash[102858]: cluster 2023-10-09T05:25:44.149115+0000 mgr.y (mgr.25277) 1125 : cluster 0 pgmap v1196: 89 pgs: 89 active+clean; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-09T05:25:45.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:45 smithi133 bash[102858]: audit 2023-10-09T05:25:44.697069+0000 mon.a (mon.0) 1467 : audit 1 from='client.? 172.21.15.133:0/1609571433' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:25:45.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:45 smithi133 bash[105589]: cluster 2023-10-09T05:25:44.149115+0000 mgr.y (mgr.25277) 1125 : cluster 0 pgmap v1196: 89 pgs: 89 active+clean; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-09T05:25:45.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:45 smithi133 bash[105589]: audit 2023-10-09T05:25:44.697069+0000 mon.a (mon.0) 1467 : audit 1 from='client.? 172.21.15.133:0/1609571433' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:25:45.550 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:45.573 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:45.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:45.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:45.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:45 smithi194 bash[92666]: cluster 2023-10-09T05:25:44.149115+0000 mgr.y (mgr.25277) 1125 : cluster 0 pgmap v1196: 89 pgs: 89 active+clean; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-09T05:25:45.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:45 smithi194 bash[92666]: audit 2023-10-09T05:25:44.697069+0000 mon.a (mon.0) 1467 : audit 1 from='client.? 172.21.15.133:0/1609571433' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:25:45.638 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is set 2023-10-09T05:25:45.663 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 22 16 38802 38786 110.167 73.125 0.018794 0.00906882 2023-10-09T05:25:45.677 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:45.709 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:45.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:45.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:45.813 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:45.830 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:45.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:45.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:45.933 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:45.982 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:45.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:45.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:46.087 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:46.120 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:46.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:46.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:46.228 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:46.258 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:46.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:46.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:46.363 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:46.389 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:46 smithi133 bash[105589]: cluster 2023-10-09T05:25:45.616858+0000 mon.a (mon.0) 1468 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:25:46.389 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:46 smithi133 bash[105589]: cluster 2023-10-09T05:25:45.629855+0000 osd.6 (osd.6) 218 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T05:25:46.390 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:46 smithi133 bash[105589]: audit 2023-10-09T05:25:45.630853+0000 mon.a (mon.0) 1469 : audit 1 from='client.? 172.21.15.133:0/1609571433' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:25:46.390 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:46 smithi133 bash[105589]: cluster 2023-10-09T05:25:45.632053+0000 osd.6 (osd.6) 219 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T05:25:46.390 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:46 smithi133 bash[105589]: cluster 2023-10-09T05:25:45.632093+0000 mon.a (mon.0) 1470 : cluster 0 osdmap e271: 8 total, 8 up, 7 in 2023-10-09T05:25:46.390 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:46 smithi133 bash[105589]: cluster 2023-10-09T05:25:45.633316+0000 osd.6 (osd.6) 220 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T05:25:46.390 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:46 smithi133 bash[105589]: cluster 2023-10-09T05:25:45.633764+0000 osd.4 (osd.4) 243 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T05:25:46.390 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:46 smithi133 bash[105589]: cluster 2023-10-09T05:25:45.636774+0000 osd.7 (osd.7) 219 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T05:25:46.390 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:46 smithi133 bash[105589]: cluster 2023-10-09T05:25:45.639166+0000 osd.7 (osd.7) 220 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T05:25:46.390 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:46 smithi133 bash[105589]: cluster 2023-10-09T05:25:45.948615+0000 osd.5 (osd.5) 251 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T05:25:46.391 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:46 smithi133 bash[102858]: cluster 2023-10-09T05:25:45.616858+0000 mon.a (mon.0) 1468 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:25:46.391 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:46 smithi133 bash[102858]: cluster 2023-10-09T05:25:45.629855+0000 osd.6 (osd.6) 218 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T05:25:46.391 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:46 smithi133 bash[102858]: audit 2023-10-09T05:25:45.630853+0000 mon.a (mon.0) 1469 : audit 1 from='client.? 172.21.15.133:0/1609571433' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:25:46.391 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:46 smithi133 bash[102858]: cluster 2023-10-09T05:25:45.632053+0000 osd.6 (osd.6) 219 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T05:25:46.391 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:46 smithi133 bash[102858]: cluster 2023-10-09T05:25:45.632093+0000 mon.a (mon.0) 1470 : cluster 0 osdmap e271: 8 total, 8 up, 7 in 2023-10-09T05:25:46.391 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:46 smithi133 bash[102858]: cluster 2023-10-09T05:25:45.633316+0000 osd.6 (osd.6) 220 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T05:25:46.391 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:46 smithi133 bash[102858]: cluster 2023-10-09T05:25:45.633764+0000 osd.4 (osd.4) 243 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T05:25:46.391 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:46 smithi133 bash[102858]: cluster 2023-10-09T05:25:45.636774+0000 osd.7 (osd.7) 219 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T05:25:46.395 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:46.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:46.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:46.500 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:46.533 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:46.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:46.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:46.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:46 smithi194 bash[92666]: cluster 2023-10-09T05:25:45.616858+0000 mon.a (mon.0) 1468 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:25:46.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:46 smithi194 bash[92666]: cluster 2023-10-09T05:25:45.629855+0000 osd.6 (osd.6) 218 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T05:25:46.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:46 smithi194 bash[92666]: audit 2023-10-09T05:25:45.630853+0000 mon.a (mon.0) 1469 : audit 1 from='client.? 172.21.15.133:0/1609571433' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:25:46.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:46 smithi194 bash[92666]: cluster 2023-10-09T05:25:45.632053+0000 osd.6 (osd.6) 219 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T05:25:46.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:46 smithi194 bash[92666]: cluster 2023-10-09T05:25:45.632093+0000 mon.a (mon.0) 1470 : cluster 0 osdmap e271: 8 total, 8 up, 7 in 2023-10-09T05:25:46.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:46 smithi194 bash[92666]: cluster 2023-10-09T05:25:45.633316+0000 osd.6 (osd.6) 220 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T05:25:46.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:46 smithi194 bash[92666]: cluster 2023-10-09T05:25:45.633764+0000 osd.4 (osd.4) 243 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T05:25:46.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:46 smithi194 bash[92666]: cluster 2023-10-09T05:25:45.636774+0000 osd.7 (osd.7) 219 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T05:25:46.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:46 smithi194 bash[92666]: cluster 2023-10-09T05:25:45.639166+0000 osd.7 (osd.7) 220 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T05:25:46.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:46 smithi194 bash[92666]: cluster 2023-10-09T05:25:45.948615+0000 osd.5 (osd.5) 251 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T05:25:46.638 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:46.650 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:46 smithi133 bash[102858]: cluster 2023-10-09T05:25:45.639166+0000 osd.7 (osd.7) 220 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T05:25:46.650 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:46 smithi133 bash[102858]: cluster 2023-10-09T05:25:45.948615+0000 osd.5 (osd.5) 251 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T05:25:46.663 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 23 16 39980 39964 108.578 73.625 0.0042261 0.00916487 2023-10-09T05:25:46.665 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:46.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:46.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:46.769 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:46.798 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:46.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:46.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:46.904 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:46.948 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:46.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:46.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:47.053 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:47.087 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:47.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:47.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:47.191 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:47.228 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:47.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:47.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:47.332 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:47.359 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:47.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:47.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:47.464 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:47.500 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:47.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:47.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:47.556 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:47 smithi133 bash[105589]: cluster 2023-10-09T05:25:46.149680+0000 mgr.y (mgr.25277) 1126 : cluster 0 pgmap v1198: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T05:25:47.556 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:47 smithi133 bash[102858]: cluster 2023-10-09T05:25:46.149680+0000 mgr.y (mgr.25277) 1126 : cluster 0 pgmap v1198: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T05:25:47.604 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:47.629 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:47.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:47.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:47.663 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 24 16 41253 41237 107.368 79.5625 0.00316807 0.00928843 2023-10-09T05:25:47.732 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:47.742 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:47 smithi194 bash[92666]: cluster 2023-10-09T05:25:46.149680+0000 mgr.y (mgr.25277) 1126 : cluster 0 pgmap v1198: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T05:25:47.762 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:47.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:47.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:47.865 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:47.888 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:47.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:47.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:47.992 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:48.017 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:48.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:48.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:48.122 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:48.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:48.166 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:48.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:48.267 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:48.297 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:48.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:48.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:48.402 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:48.428 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:48.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:48.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:48.532 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:48.573 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:48.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:48.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:48.663 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 25 16 42633 42617 106.523 86.25 0.00434382 0.00938139 2023-10-09T05:25:48.677 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:48.706 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:48.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:48.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:48.813 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:48.837 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:48.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:48.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:48.968 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:49.000 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:49.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:49.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:49.103 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:49.137 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:49.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:49.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:49.240 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:49.269 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:49.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:49.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:49.372 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:49.403 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:49.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:49.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:49.507 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:49.549 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:49.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:49.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:49.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:49 smithi194 bash[92666]: cluster 2023-10-09T05:25:48.150202+0000 mgr.y (mgr.25277) 1127 : cluster 0 pgmap v1199: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-10-09T05:25:49.651 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:49.658 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:49 smithi133 bash[105589]: cluster 2023-10-09T05:25:48.150202+0000 mgr.y (mgr.25277) 1127 : cluster 0 pgmap v1199: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-10-09T05:25:49.659 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:49 smithi133 bash[102858]: cluster 2023-10-09T05:25:48.150202+0000 mgr.y (mgr.25277) 1127 : cluster 0 pgmap v1199: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-10-09T05:25:49.663 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 26 16 43896 43880 105.462 78.9375 0.0155359 0.00947294 2023-10-09T05:25:49.679 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:49.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:49.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:49.785 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:49.811 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:49.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:49.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:49.915 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:49.949 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:49.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:49.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:50.051 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:50.083 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:50.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:50.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:50.185 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:50.220 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:50.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:50.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:50.325 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:50.369 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:50.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:50.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:50.472 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:50.497 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:50.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:50.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:50.600 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:50.663 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 27 16 45049 45033 104.225 72.0625 0.0163101 0.00958924 2023-10-09T05:25:50.673 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:50.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:50.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:50.762 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:25:50.762 INFO:teuthology.orchestra.run.smithi133.stdout: "ops": [ 2023-10-09T05:25:50.762 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:25:50.762 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_repop(client.45374.0:45136 12.b e271/264)", 2023-10-09T05:25:50.762 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:25:50.758670+0000", 2023-10-09T05:25:50.762 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.001649883, 2023-10-09T05:25:50.762 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0016764010000000001, 2023-10-09T05:25:50.762 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:25:50.763 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "queued for pg", 2023-10-09T05:25:50.763 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:25:50.763 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:25:50.763 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:25:50.763 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:25:50.758670+0000", 2023-10-09T05:25:50.763 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:25:50.763 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:25:50.763 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:25:50.763 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:25:50.763 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:25:50.758670+0000", 2023-10-09T05:25:50.763 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:25:50.763 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:25:50.764 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:25:50.764 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:25:50.764 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:25:50.758670+0000", 2023-10-09T05:25:50.764 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:25:50.764 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:25:50.764 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:25:50.764 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:25:50.764 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:25:50.758695+0000", 2023-10-09T05:25:50.764 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.5282999999999998e-05 2023-10-09T05:25:50.764 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:25:50.764 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:25:50.764 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:25:50.765 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:25:50.758708+0000", 2023-10-09T05:25:50.765 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.3366e-05 2023-10-09T05:25:50.765 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:25:50.765 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:25:50.765 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:25:50.765 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:25:50.758717+0000", 2023-10-09T05:25:50.765 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.8040000000000007e-06 2023-10-09T05:25:50.765 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:25:50.765 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:25:50.765 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:25:50.765 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:25:50.765 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:25:50.766 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_repop(client.45374.0:45137 12.d e271/264 12:beb90bd3:::benchmark_data_smithi133_218956_object45136:head v 271'2805, mlcod=271'2801)", 2023-10-09T05:25:50.766 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:25:50.760022+0000", 2023-10-09T05:25:50.766 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.00029812699999999997, 2023-10-09T05:25:50.766 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00038984099999999998, 2023-10-09T05:25:50.766 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:25:50.766 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "started", 2023-10-09T05:25:50.766 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:25:50.766 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:25:50.766 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:25:50.766 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:25:50.760022+0000", 2023-10-09T05:25:50.766 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:25:50.767 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:25:50.767 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:25:50.767 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:25:50.767 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:25:50.760022+0000", 2023-10-09T05:25:50.767 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:25:50.767 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:25:50.767 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:25:50.767 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:25:50.767 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:25:50.760021+0000", 2023-10-09T05:25:50.767 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:25:50.767 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:25:50.767 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:25:50.768 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:25:50.768 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:25:50.760061+0000", 2023-10-09T05:25:50.768 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.9857000000000001e-05 2023-10-09T05:25:50.768 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:25:50.768 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:25:50.768 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:25:50.768 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:25:50.760085+0000", 2023-10-09T05:25:50.768 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.4102999999999999e-05 2023-10-09T05:25:50.768 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:25:50.768 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:25:50.768 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:25:50.768 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:25:50.760092+0000", 2023-10-09T05:25:50.769 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.0509999999999999e-06 2023-10-09T05:25:50.769 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:25:50.769 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:25:50.769 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:25:50.769 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:25:50.760138+0000", 2023-10-09T05:25:50.769 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.5226999999999999e-05 2023-10-09T05:25:50.769 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:25:50.769 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:25:50.769 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:25:50.770 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:25:50.760161+0000", 2023-10-09T05:25:50.770 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.3782999999999999e-05 2023-10-09T05:25:50.770 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:25:50.770 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:25:50.770 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:25:50.770 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:25:50.770 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:25:50.770 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45374.0:45142 12.a 12:59ee4e96:::benchmark_data_smithi133_218956_object45141:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-10-09T05:25:50.770 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:25:50.760125+0000", 2023-10-09T05:25:50.770 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.000195393, 2023-10-09T05:25:50.770 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00035710000000000001, 2023-10-09T05:25:50.770 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:25:50.771 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "started", 2023-10-09T05:25:50.771 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:25:50.771 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45374", 2023-10-09T05:25:50.771 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2988063385", 2023-10-09T05:25:50.771 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 45142 2023-10-09T05:25:50.771 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:25:50.771 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:25:50.771 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:25:50.771 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:25:50.771 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:25:50.760125+0000", 2023-10-09T05:25:50.771 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:25:50.771 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:25:50.772 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:25:50.772 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:25:50.772 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:25:50.760125+0000", 2023-10-09T05:25:50.772 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:25:50.772 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:25:50.772 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:25:50.772 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:25:50.772 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:25:50.760123+0000", 2023-10-09T05:25:50.772 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999981 2023-10-09T05:25:50.772 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:25:50.772 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:25:50.772 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:25:50.773 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:25:50.760155+0000", 2023-10-09T05:25:50.773 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.1967000000000001e-05 2023-10-09T05:25:50.773 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:25:50.773 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:25:50.773 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:25:50.773 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:25:50.760167+0000", 2023-10-09T05:25:50.773 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.1671000000000001e-05 2023-10-09T05:25:50.773 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:25:50.773 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:25:50.773 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:25:50.773 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:25:50.760174+0000", 2023-10-09T05:25:50.773 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.6120000000000001e-06 2023-10-09T05:25:50.774 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:25:50.774 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:25:50.774 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:25:50.774 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:25:50.760228+0000", 2023-10-09T05:25:50.774 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.3439000000000001e-05 2023-10-09T05:25:50.774 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:25:50.774 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:25:50.774 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:25:50.774 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:25:50.760431+0000", 2023-10-09T05:25:50.774 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00020310400000000001 2023-10-09T05:25:50.774 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:25:50.774 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:25:50.775 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:25:50.775 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:25:50.775 INFO:teuthology.orchestra.run.smithi133.stdout: ], 2023-10-09T05:25:50.775 INFO:teuthology.orchestra.run.smithi133.stdout: "num_ops": 3 2023-10-09T05:25:50.775 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:25:50.777 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:50.800 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:50.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:50.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:50.824 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:25:50.865 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:25:50.906 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:50.913 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:25:50.937 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:50.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:50.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:50.955 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:25:50.992 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:25:51.030 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:25:51.043 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:51.075 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:51.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:51.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:51.081 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:25:51.118 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:25:51.123 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:25:51.123 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:25:51.123 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:25:51.123 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:25:51.123 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:25:51.123 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:25:51.123 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:25:51.123 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:25:51.179 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:51.214 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:51.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:51.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:51.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:51 smithi133 bash[102858]: audit 2023-10-09T05:25:50.133166+0000 mon.a (mon.0) 1471 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:51.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:51 smithi133 bash[102858]: audit 2023-10-09T05:25:50.146955+0000 mon.a (mon.0) 1472 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:51.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:51 smithi133 bash[102858]: cluster 2023-10-09T05:25:50.150928+0000 mgr.y (mgr.25277) 1128 : cluster 0 pgmap v1200: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T05:25:51.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:51 smithi133 bash[102858]: audit 2023-10-09T05:25:50.728884+0000 mon.c (mon.2) 102 : audit 0 from='client.? 172.21.15.133:0/250977404' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:25:51.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:51 smithi133 bash[105589]: audit 2023-10-09T05:25:50.133166+0000 mon.a (mon.0) 1471 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:51.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:51 smithi133 bash[105589]: audit 2023-10-09T05:25:50.146955+0000 mon.a (mon.0) 1472 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:51.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:51 smithi133 bash[105589]: cluster 2023-10-09T05:25:50.150928+0000 mgr.y (mgr.25277) 1128 : cluster 0 pgmap v1200: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T05:25:51.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:51 smithi133 bash[105589]: audit 2023-10-09T05:25:50.728884+0000 mon.c (mon.2) 102 : audit 0 from='client.? 172.21.15.133:0/250977404' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:25:51.319 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:51.330 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:51 smithi194 bash[92666]: audit 2023-10-09T05:25:50.133166+0000 mon.a (mon.0) 1471 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:51.330 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:51 smithi194 bash[92666]: audit 2023-10-09T05:25:50.146955+0000 mon.a (mon.0) 1472 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:51.330 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:51 smithi194 bash[92666]: cluster 2023-10-09T05:25:50.150928+0000 mgr.y (mgr.25277) 1128 : cluster 0 pgmap v1200: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T05:25:51.330 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:51 smithi194 bash[92666]: audit 2023-10-09T05:25:50.728884+0000 mon.c (mon.2) 102 : audit 0 from='client.? 172.21.15.133:0/250977404' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:25:51.349 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:51.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:51.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:51.452 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:51.486 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:51.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:51.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:51.591 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:51.621 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:51.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:51.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:51.663 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 28 16 46269 46253 103.226 76.25 0.00438763 0.00968123 2023-10-09T05:25:51.725 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:51.760 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:51.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:51.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:51.866 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:51.902 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:51.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:51.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:51.933 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:25:52.008 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:52.032 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:52.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:52.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:52.137 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:52.162 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:52.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:52.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:52.265 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:52.307 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:52.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:52.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:52.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:52 smithi194 bash[92666]: audit 2023-10-09T05:25:51.266809+0000 mon.a (mon.0) 1473 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:52.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:52 smithi194 bash[92666]: audit 2023-10-09T05:25:51.278204+0000 mon.a (mon.0) 1474 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:52.413 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:52.450 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:52.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:52.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:52.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:52 smithi133 bash[105589]: audit 2023-10-09T05:25:51.266809+0000 mon.a (mon.0) 1473 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:52.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:52 smithi133 bash[105589]: audit 2023-10-09T05:25:51.278204+0000 mon.a (mon.0) 1474 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:52.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:52 smithi133 bash[102858]: audit 2023-10-09T05:25:51.266809+0000 mon.a (mon.0) 1473 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:52.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:52 smithi133 bash[102858]: audit 2023-10-09T05:25:51.278204+0000 mon.a (mon.0) 1474 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:52.556 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:52.585 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:52.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:52.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:52.663 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 29 16 47461 47445 102.235 74.5 0.00515045 0.00977431 2023-10-09T05:25:52.689 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:52.726 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:52.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:52.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:52.832 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:52.859 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:52.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:52.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:52.963 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:52.989 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:52.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:52.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:53.092 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:53.129 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:53.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:53.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:53.232 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:53.261 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:53.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:53.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:53.291 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:53 smithi133 bash[102858]: cluster 2023-10-09T05:25:52.151541+0000 mgr.y (mgr.25277) 1129 : cluster 0 pgmap v1201: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T05:25:53.320 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T05:25:53.365 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:53.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:53 smithi194 bash[92666]: cluster 2023-10-09T05:25:52.151541+0000 mgr.y (mgr.25277) 1129 : cluster 0 pgmap v1201: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T05:25:53.432 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:53.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:53.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:53.538 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:53.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:53 smithi133 bash[105589]: cluster 2023-10-09T05:25:52.151541+0000 mgr.y (mgr.25277) 1129 : cluster 0 pgmap v1201: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T05:25:53.565 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:53.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:53.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:53.665 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 30 16 48763 48747 101.539 81.375 0.0154766 0.00984273 2023-10-09T05:25:53.671 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:53.703 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:53.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:53.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:53.808 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:53.836 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:53.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:53.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:53.939 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:53.967 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:53.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:53.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:54.072 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:54.099 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:54.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:54.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:54.203 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:54.230 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:54.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:54.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:54.335 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:54.364 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:54.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:54.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:54.467 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:54.477 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:25:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:25:54] "GET /metrics HTTP/1.1" 200 36471 "" "Prometheus/2.43.0" 2023-10-09T05:25:54.491 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:54.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:54.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:54.595 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:54.621 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:54.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:54.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:54.664 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 31 16 49995 49979 100.748 77 0.0176449 0.00992012 2023-10-09T05:25:54.725 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:54.753 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:54.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:54.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:54.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:54 smithi133 bash[105589]: cluster 2023-10-09T05:25:54.152052+0000 mgr.y (mgr.25277) 1130 : cluster 0 pgmap v1202: 89 pgs: 89 active+clean; 2.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T05:25:54.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:54 smithi133 bash[102858]: cluster 2023-10-09T05:25:54.152052+0000 mgr.y (mgr.25277) 1130 : cluster 0 pgmap v1202: 89 pgs: 89 active+clean; 2.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T05:25:54.817 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:25:54.858 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:54.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:54 smithi194 bash[92666]: cluster 2023-10-09T05:25:54.152052+0000 mgr.y (mgr.25277) 1130 : cluster 0 pgmap v1202: 89 pgs: 89 active+clean; 2.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T05:25:54.898 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:54.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:54.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:55.001 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:55.028 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:55.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:55.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:55.131 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:55.168 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:55.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:55.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:55.273 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:55.300 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:25:55.315 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:55.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:55.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:55.420 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:55.445 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:55.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:55.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:55.551 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:55.577 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:55.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:55.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:55.664 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 32 16 51008 50992 99.5776 63.3125 0.0178089 0.0100348 2023-10-09T05:25:55.681 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:55.713 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:55.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:55.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:55.817 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:55.848 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:55.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:55.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:55.961 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:55.987 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:55.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:55.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:56.091 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:56.116 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:56.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:56.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:56.222 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:56.252 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:56.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:56.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:56.359 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:56.383 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:56.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:56.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:56.489 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:56.528 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:56.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:56.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:56.632 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:56.656 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:56.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:56.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:56.664 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 33 16 52076 52060 98.5825 66.75 0.00793021 0.0101382 2023-10-09T05:25:56.760 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:56.792 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:56.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:56.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:56.896 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:56.922 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:56.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:56.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:57.027 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:57.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:57.064 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:57.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:57.165 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:57.190 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:57.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:57.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:57.278 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:57 smithi133 bash[102858]: cluster 2023-10-09T05:25:56.152787+0000 mgr.y (mgr.25277) 1131 : cluster 0 pgmap v1203: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T05:25:57.278 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:57 smithi133 bash[102858]: audit 2023-10-09T05:25:56.277598+0000 mon.a (mon.0) 1475 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:57.278 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:57 smithi133 bash[102858]: audit 2023-10-09T05:25:56.287951+0000 mon.a (mon.0) 1476 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:57.294 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:57.319 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:57.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:57.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:57.422 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:57.449 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:57.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:57.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:57.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:57 smithi133 bash[105589]: cluster 2023-10-09T05:25:56.152787+0000 mgr.y (mgr.25277) 1131 : cluster 0 pgmap v1203: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T05:25:57.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:57 smithi133 bash[105589]: audit 2023-10-09T05:25:56.277598+0000 mon.a (mon.0) 1475 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:57.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:57 smithi133 bash[105589]: audit 2023-10-09T05:25:56.287951+0000 mon.a (mon.0) 1476 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:57.554 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:57.563 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:57 smithi194 bash[92666]: cluster 2023-10-09T05:25:56.152787+0000 mgr.y (mgr.25277) 1131 : cluster 0 pgmap v1203: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T05:25:57.563 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:57 smithi194 bash[92666]: audit 2023-10-09T05:25:56.277598+0000 mon.a (mon.0) 1475 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:57.563 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:57 smithi194 bash[92666]: audit 2023-10-09T05:25:56.287951+0000 mon.a (mon.0) 1476 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:57.588 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:57.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:57.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:57.664 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 34 16 53255 53239 97.8502 73.6875 0.0157542 0.0102139 2023-10-09T05:25:57.692 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:57.717 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:57.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:57.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:57.820 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:57.848 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:57.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:57.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:57.951 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:57.976 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:57.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:57.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:58.080 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:58.116 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:25:58.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:58.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:25:58.223 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:58.256 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:58.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:58.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:58.361 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:58.399 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:58.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:58.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:58.502 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:58.538 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:58.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:58.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:58.643 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:58.664 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 35 16 54413 54397 97.1222 72.375 0.00425172 0.0102883 2023-10-09T05:25:58.674 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:58.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:58.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:58.785 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:58.809 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:25:58.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:58.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:25:58.916 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:58.945 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:25:58.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:58.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:25:59.048 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:59.050 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:25:59.050 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:25:59.076 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:59.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:59.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:59.182 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:59.214 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:59.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:59.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:59.285 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:59 smithi133 bash[105589]: cluster 2023-10-09T05:25:58.153291+0000 mgr.y (mgr.25277) 1132 : cluster 0 pgmap v1204: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T05:25:59.285 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:59 smithi133 bash[105589]: audit 2023-10-09T05:25:58.473742+0000 mon.a (mon.0) 1477 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:59.285 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:25:59 smithi133 bash[105589]: audit 2023-10-09T05:25:58.475089+0000 mon.a (mon.0) 1478 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:25:59.286 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:59 smithi133 bash[102858]: cluster 2023-10-09T05:25:58.153291+0000 mgr.y (mgr.25277) 1132 : cluster 0 pgmap v1204: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T05:25:59.286 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:59 smithi133 bash[102858]: audit 2023-10-09T05:25:58.473742+0000 mon.a (mon.0) 1477 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:59.286 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:25:59 smithi133 bash[102858]: audit 2023-10-09T05:25:58.475089+0000 mon.a (mon.0) 1478 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:25:59.323 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:59.363 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:25:59.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:59.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:25:59.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:59 smithi194 bash[92666]: cluster 2023-10-09T05:25:58.153291+0000 mgr.y (mgr.25277) 1132 : cluster 0 pgmap v1204: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T05:25:59.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:59 smithi194 bash[92666]: audit 2023-10-09T05:25:58.473742+0000 mon.a (mon.0) 1477 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:25:59.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:25:59 smithi194 bash[92666]: audit 2023-10-09T05:25:58.475089+0000 mon.a (mon.0) 1478 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:25:59.470 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:59.496 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:25:59.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:59.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:25:59.601 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:59.630 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:25:59.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:59.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:25:59.664 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 36 16 55469 55453 96.2576 66 0.0184966 0.0103829 2023-10-09T05:25:59.733 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:59.768 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:25:59.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:59.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:25:59.872 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:59.896 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:25:59.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:25:59.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:25:59.999 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:00.032 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:00.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:00.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:00.137 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:00.161 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:00.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:00.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:00.265 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:00.294 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:00.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:00.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:00.398 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:00.426 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:00.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:00.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:00.532 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:00.561 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:00.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:00.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:00.664 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 37 16 56615 56599 95.5917 71.625 0.0126726 0.0104555 2023-10-09T05:26:00.664 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:00.693 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:00.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:00.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:00.797 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:00.827 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:00.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:00.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:00.931 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:00.952 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:00.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:00.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:01.054 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:01.085 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:01.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:01.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:01.188 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:01.218 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:01.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:01.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:01.321 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:01.348 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:01.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:01.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:01.451 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:01.460 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:01 smithi133 bash[102858]: cluster 2023-10-09T05:26:00.156016+0000 mgr.y (mgr.25277) 1133 : cluster 0 pgmap v1205: 89 pgs: 89 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T05:26:01.460 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:01 smithi133 bash[105589]: cluster 2023-10-09T05:26:00.156016+0000 mgr.y (mgr.25277) 1133 : cluster 0 pgmap v1205: 89 pgs: 89 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T05:26:01.477 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:01.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:01.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:01.582 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:01.599 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:01.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:01.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:01.631 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:26:01.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:01 smithi194 bash[92666]: cluster 2023-10-09T05:26:00.156016+0000 mgr.y (mgr.25277) 1133 : cluster 0 pgmap v1205: 89 pgs: 89 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T05:26:01.664 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 38 16 57487 57471 94.5102 54.5 0.0078 0.0105443 2023-10-09T05:26:01.702 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:01.730 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:01.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:01.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:01.833 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:01.862 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:01.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:01.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:01.967 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:01.994 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:01.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:01.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:02.100 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:02.125 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:02.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:02.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:02.228 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:02.259 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:02.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:02.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:02.365 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:02.390 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:02.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:02.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:02.496 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:02.537 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:02.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:02.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:02.642 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:02.665 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 39 16 58662 58646 93.9685 73.4375 0.0109208 0.0106363 2023-10-09T05:26:02.668 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:02.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:02.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:02.772 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:02.800 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:02.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:02.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:02.904 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:02.926 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:02.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:02.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:03.034 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:03.077 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:03.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:03.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:03.181 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:03.208 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:03.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:03.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:03.295 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:03 smithi133 bash[102858]: cluster 2023-10-09T05:26:02.156576+0000 mgr.y (mgr.25277) 1134 : cluster 0 pgmap v1206: 89 pgs: 89 active+clean; 3.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T05:26:03.312 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:03.336 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:03.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:03.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:03.442 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:03.452 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:03 smithi194 bash[92666]: cluster 2023-10-09T05:26:02.156576+0000 mgr.y (mgr.25277) 1134 : cluster 0 pgmap v1206: 89 pgs: 89 active+clean; 3.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T05:26:03.467 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:03.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:03.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:03.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:03 smithi133 bash[105589]: cluster 2023-10-09T05:26:02.156576+0000 mgr.y (mgr.25277) 1134 : cluster 0 pgmap v1206: 89 pgs: 89 active+clean; 3.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T05:26:03.571 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:03.595 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:03.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:03.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:03.665 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T05:26:03.665446+0000 min lat: 0.00116799 max lat: 0.192733 avg lat: 0.0107155 2023-10-09T05:26:03.665 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:26:03.665 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 40 16 59721 59705 93.2739 66.1875 0.00987665 0.0107155 2023-10-09T05:26:03.698 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:03.723 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:03.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:03.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:03.829 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:03.864 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:03.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:03.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:03.969 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:03.990 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:03.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:03.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:04.094 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:04.119 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:04.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:04.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:04.224 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:04.251 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:04.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:04.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:04.355 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:04.379 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:04.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:04.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:04.482 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:04.491 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:26:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:26:04] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.43.0" 2023-10-09T05:26:04.515 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:04.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:04.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:04.623 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:04.630 INFO:teuthology.orchestra.run.smithi133.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T05:26:04.649 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:04.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:04.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:04.665 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 41 16 60454 60438 92.1162 45.8125 0.00298678 0.010813 2023-10-09T05:26:04.752 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:04.762 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:04 smithi133 bash[105589]: audit 2023-10-09T05:26:03.785939+0000 mon.c (mon.2) 103 : audit 1 from='client.? 172.21.15.133:0/4159205097' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:26:04.762 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:04 smithi133 bash[105589]: audit 2023-10-09T05:26:03.787037+0000 mon.a (mon.0) 1479 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:26:04.762 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:04 smithi133 bash[105589]: cluster 2023-10-09T05:26:04.157030+0000 mgr.y (mgr.25277) 1135 : cluster 0 pgmap v1207: 89 pgs: 89 active+clean; 3.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T05:26:04.763 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:04 smithi133 bash[102858]: audit 2023-10-09T05:26:03.785939+0000 mon.c (mon.2) 103 : audit 1 from='client.? 172.21.15.133:0/4159205097' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:26:04.763 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:04 smithi133 bash[102858]: audit 2023-10-09T05:26:03.787037+0000 mon.a (mon.0) 1479 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:26:04.763 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:04 smithi133 bash[102858]: cluster 2023-10-09T05:26:04.157030+0000 mgr.y (mgr.25277) 1135 : cluster 0 pgmap v1207: 89 pgs: 89 active+clean; 3.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T05:26:04.781 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:04.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:04.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:04.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:04 smithi194 bash[92666]: audit 2023-10-09T05:26:03.785939+0000 mon.c (mon.2) 103 : audit 1 from='client.? 172.21.15.133:0/4159205097' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:26:04.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:04 smithi194 bash[92666]: audit 2023-10-09T05:26:03.787037+0000 mon.a (mon.0) 1479 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:26:04.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:04 smithi194 bash[92666]: cluster 2023-10-09T05:26:04.157030+0000 mgr.y (mgr.25277) 1135 : cluster 0 pgmap v1207: 89 pgs: 89 active+clean; 3.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T05:26:04.886 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:04.913 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:04.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:04.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:05.016 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:05.040 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:05.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:05.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:05.142 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:05.170 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:05.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:05.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:05.277 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:05.307 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:05.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:05.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:05.411 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:05.440 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:05.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:05.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:05.545 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:05.567 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:05.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:05.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:05.665 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 42 16 61488 61472 91.4616 64.625 0.00577744 0.0109273 2023-10-09T05:26:05.672 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:05.696 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:05.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:05.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:05.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:05 smithi133 bash[105589]: audit 2023-10-09T05:26:04.625827+0000 mon.a (mon.0) 1480 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:26:05.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:05 smithi133 bash[105589]: cluster 2023-10-09T05:26:04.626071+0000 mon.a (mon.0) 1481 : cluster 0 osdmap e272: 8 total, 8 up, 7 in 2023-10-09T05:26:05.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:05 smithi133 bash[105589]: cluster 2023-10-09T05:26:04.670601+0000 osd.7 (osd.7) 221 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T05:26:05.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:05 smithi133 bash[105589]: cluster 2023-10-09T05:26:04.677938+0000 osd.6 (osd.6) 221 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T05:26:05.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:05 smithi133 bash[105589]: cluster 2023-10-09T05:26:04.678398+0000 osd.4 (osd.4) 244 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T05:26:05.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:05 smithi133 bash[105589]: cluster 2023-10-09T05:26:04.680091+0000 osd.6 (osd.6) 222 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T05:26:05.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:05 smithi133 bash[102858]: audit 2023-10-09T05:26:04.625827+0000 mon.a (mon.0) 1480 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:26:05.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:05 smithi133 bash[102858]: cluster 2023-10-09T05:26:04.626071+0000 mon.a (mon.0) 1481 : cluster 0 osdmap e272: 8 total, 8 up, 7 in 2023-10-09T05:26:05.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:05 smithi133 bash[102858]: cluster 2023-10-09T05:26:04.670601+0000 osd.7 (osd.7) 221 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T05:26:05.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:05 smithi133 bash[102858]: cluster 2023-10-09T05:26:04.677938+0000 osd.6 (osd.6) 221 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T05:26:05.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:05 smithi133 bash[102858]: cluster 2023-10-09T05:26:04.678398+0000 osd.4 (osd.4) 244 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T05:26:05.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:05 smithi133 bash[102858]: cluster 2023-10-09T05:26:04.680091+0000 osd.6 (osd.6) 222 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T05:26:05.800 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:05.809 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:05 smithi194 bash[92666]: audit 2023-10-09T05:26:04.625827+0000 mon.a (mon.0) 1480 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:26:05.809 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:05 smithi194 bash[92666]: cluster 2023-10-09T05:26:04.626071+0000 mon.a (mon.0) 1481 : cluster 0 osdmap e272: 8 total, 8 up, 7 in 2023-10-09T05:26:05.809 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:05 smithi194 bash[92666]: cluster 2023-10-09T05:26:04.670601+0000 osd.7 (osd.7) 221 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T05:26:05.809 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:05 smithi194 bash[92666]: cluster 2023-10-09T05:26:04.677938+0000 osd.6 (osd.6) 221 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T05:26:05.809 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:05 smithi194 bash[92666]: cluster 2023-10-09T05:26:04.678398+0000 osd.4 (osd.4) 244 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T05:26:05.809 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:05 smithi194 bash[92666]: cluster 2023-10-09T05:26:04.680091+0000 osd.6 (osd.6) 222 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T05:26:05.824 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:05.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:05.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:05.928 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:05.968 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:05.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:05.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:06.074 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:06.103 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:06.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:06.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:06.207 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:06.233 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:06.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:06.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:06.339 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:06.366 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:06.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:06.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:06.469 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:06.504 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:06.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:06.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:06.608 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:06.633 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:06.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:06.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:06.665 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 43 16 62591 62575 90.9377 68.9375 0.00585609 0.0109907 2023-10-09T05:26:06.737 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:06.772 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:06.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:06.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:06.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:06 smithi133 bash[102858]: cluster 2023-10-09T05:26:06.161712+0000 mgr.y (mgr.25277) 1136 : cluster 0 pgmap v1209: 89 pgs: 89 active+clean; 3.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T05:26:06.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:06 smithi133 bash[102858]: audit 2023-10-09T05:26:06.627951+0000 mon.a (mon.0) 1482 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:26:06.802 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:06 smithi133 bash[105589]: cluster 2023-10-09T05:26:06.161712+0000 mgr.y (mgr.25277) 1136 : cluster 0 pgmap v1209: 89 pgs: 89 active+clean; 3.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T05:26:06.803 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:06 smithi133 bash[105589]: audit 2023-10-09T05:26:06.627951+0000 mon.a (mon.0) 1482 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:26:06.880 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:06.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:06 smithi194 bash[92666]: cluster 2023-10-09T05:26:06.161712+0000 mgr.y (mgr.25277) 1136 : cluster 0 pgmap v1209: 89 pgs: 89 active+clean; 3.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T05:26:06.888 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:06 smithi194 bash[92666]: audit 2023-10-09T05:26:06.627951+0000 mon.a (mon.0) 1482 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:26:06.914 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:06.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:06.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:07.018 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:07.048 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:07.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:07.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:07.151 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:07.179 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:07.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:07.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:07.284 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:07.308 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:07.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:07.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:07.411 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:07.442 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:07.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:07.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:07.546 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:07.581 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:07.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:07.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:07.666 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 44 16 63672 63656 90.4063 67.5625 0.00693586 0.0110545 2023-10-09T05:26:07.686 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:07.714 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:07.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:07.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:07.817 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:07.828 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:07 smithi194 bash[92666]: audit 2023-10-09T05:26:06.635436+0000 mon.a (mon.0) 1483 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:26:07.828 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:07 smithi194 bash[92666]: audit 2023-10-09T05:26:06.639295+0000 mon.a (mon.0) 1484 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:26:07.828 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:07 smithi194 bash[92666]: audit 2023-10-09T05:26:06.640576+0000 mon.a (mon.0) 1485 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:26:07.828 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:07 smithi194 bash[92666]: audit 2023-10-09T05:26:06.655855+0000 mon.a (mon.0) 1486 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:26:07.843 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:07 smithi133 bash[105589]: audit 2023-10-09T05:26:06.635436+0000 mon.a (mon.0) 1483 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:26:07.843 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:07 smithi133 bash[105589]: audit 2023-10-09T05:26:06.639295+0000 mon.a (mon.0) 1484 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:26:07.843 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:07 smithi133 bash[105589]: audit 2023-10-09T05:26:06.640576+0000 mon.a (mon.0) 1485 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:26:07.843 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:07 smithi133 bash[105589]: audit 2023-10-09T05:26:06.655855+0000 mon.a (mon.0) 1486 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:26:07.844 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:07 smithi133 bash[102858]: audit 2023-10-09T05:26:06.635436+0000 mon.a (mon.0) 1483 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:26:07.844 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:07 smithi133 bash[102858]: audit 2023-10-09T05:26:06.639295+0000 mon.a (mon.0) 1484 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:26:07.844 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:07 smithi133 bash[102858]: audit 2023-10-09T05:26:06.640576+0000 mon.a (mon.0) 1485 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:26:07.844 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:07 smithi133 bash[102858]: audit 2023-10-09T05:26:06.655855+0000 mon.a (mon.0) 1486 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:26:07.847 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:07.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:07.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:07.951 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:07.971 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:07.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:07.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:08.074 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:08.099 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:08.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:08.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:08.204 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:08.234 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:08.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:08.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:08.338 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:08.367 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:08.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:08.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:08.474 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:08.503 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:08.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:08.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:08.607 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:08.644 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:08.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:08.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:08.665 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 45 16 64637 64621 89.7375 60.3125 0.0132286 0.0111377 2023-10-09T05:26:08.748 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:08.756 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:08 smithi133 bash[105589]: cluster 2023-10-09T05:26:08.162196+0000 mgr.y (mgr.25277) 1137 : cluster 0 pgmap v1210: 89 pgs: 89 active+clean; 3.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T05:26:08.757 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:08 smithi133 bash[102858]: cluster 2023-10-09T05:26:08.162196+0000 mgr.y (mgr.25277) 1137 : cluster 0 pgmap v1210: 89 pgs: 89 active+clean; 3.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T05:26:08.774 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:08.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:08.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:08.879 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:08.907 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:08.910 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:08 smithi194 bash[92666]: cluster 2023-10-09T05:26:08.162196+0000 mgr.y (mgr.25277) 1137 : cluster 0 pgmap v1210: 89 pgs: 89 active+clean; 3.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T05:26:08.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:08.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:09.012 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:09.033 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:09.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:09.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:09.136 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:09.167 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:09.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:09.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:09.177 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T05:26:09.272 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:09.308 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:09.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:09.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:09.412 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:09.448 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:09.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:09.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:09.553 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:09.580 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:09.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:09.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:09.665 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 46 16 65590 65574 89.0814 59.5625 0.0161841 0.0112201 2023-10-09T05:26:09.685 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:09.710 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:09.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:09.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:09.817 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:09.856 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:09.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:09.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:09.960 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:09.990 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:09.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:09.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:10.094 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:10.118 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:10.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:10.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:10.222 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:10.250 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:10.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:10.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:10.355 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:10.381 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:10.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:10.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:10.489 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:10.517 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:10.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:10.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:10.621 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:10.645 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:10.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:10.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:10.665 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 47 16 66691 66675 88.6501 68.8125 0.0125205 0.011275 2023-10-09T05:26:10.749 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:10.783 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:10.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:10.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:10.888 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:10.917 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:10.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:10.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:11.020 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:11.045 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:11.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:11.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:11.151 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:11.180 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:11.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:11.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:11.284 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:11.309 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:11.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:11.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:11.414 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:11.422 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:11 smithi133 bash[105589]: cluster 2023-10-09T05:26:10.162874+0000 mgr.y (mgr.25277) 1138 : cluster 0 pgmap v1211: 89 pgs: 89 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T05:26:11.423 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:11 smithi133 bash[102858]: cluster 2023-10-09T05:26:10.162874+0000 mgr.y (mgr.25277) 1138 : cluster 0 pgmap v1211: 89 pgs: 89 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T05:26:11.455 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:11.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:11.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:11.560 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:11.569 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:11 smithi194 bash[92666]: cluster 2023-10-09T05:26:10.162874+0000 mgr.y (mgr.25277) 1138 : cluster 0 pgmap v1211: 89 pgs: 89 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T05:26:11.583 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:11.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:11.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:11.666 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 48 16 67652 67636 88.0533 60.0625 0.104864 0.0113501 2023-10-09T05:26:11.686 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:11.726 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:11.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:11.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:11.829 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:11.856 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:11.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:11.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:11.964 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:11.992 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:11.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:11.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:12.098 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:12.124 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:12.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:12.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:12.228 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:12.263 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:12.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:12.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:12.367 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:12.413 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:12.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:12.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:12.517 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:12.548 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:12.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:12.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:12.652 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:12.666 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 49 16 68693 68677 87.584 65.0625 0.0096544 0.0114123 2023-10-09T05:26:12.691 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:12.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:12.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:12.797 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:12.824 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:12.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:12.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:12.930 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:12.965 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:12.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:12.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:13.068 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:13.098 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:13.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:13.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:13.201 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:13.231 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:13.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:13.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:13.336 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:13.362 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:13.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:13.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:13.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:13 smithi194 bash[92666]: cluster 2023-10-09T05:26:12.163430+0000 mgr.y (mgr.25277) 1139 : cluster 0 pgmap v1212: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T05:26:13.466 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:13.502 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:13.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:13.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:13.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:13 smithi133 bash[105589]: cluster 2023-10-09T05:26:12.163430+0000 mgr.y (mgr.25277) 1139 : cluster 0 pgmap v1212: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T05:26:13.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:13 smithi133 bash[102858]: cluster 2023-10-09T05:26:12.163430+0000 mgr.y (mgr.25277) 1139 : cluster 0 pgmap v1212: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T05:26:13.606 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:13.629 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:13.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:13.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:13.666 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 50 16 69674 69658 87.0585 61.3125 0.00770902 0.0114812 2023-10-09T05:26:13.733 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:13.764 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:13.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:13.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:13.869 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:13.899 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:13.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:13.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:14.004 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:14.031 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:14.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:14.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:14.072 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:26:14.134 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:14.156 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:14.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:14.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:14.259 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:14.295 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:14.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:14.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:14.401 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:14.410 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:14 smithi194 bash[92666]: audit 2023-10-09T05:26:13.463100+0000 mon.a (mon.0) 1487 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:26:14.432 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:14.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:14.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:14.536 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:14.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:26:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:26:14] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.43.0" 2023-10-09T05:26:14.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:14 smithi133 bash[105589]: audit 2023-10-09T05:26:13.463100+0000 mon.a (mon.0) 1487 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:26:14.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:14 smithi133 bash[102858]: audit 2023-10-09T05:26:13.463100+0000 mon.a (mon.0) 1487 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:26:14.563 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:14.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:14.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:14.668 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:14.669 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 51 16 70703 70687 86.6124 64.3125 0.012706 0.0115392 2023-10-09T05:26:14.701 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:14.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:14.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:14.808 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:14.834 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:14.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:14.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:14.939 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:14.967 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:14.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:14.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:15.070 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:15.101 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:15.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:15.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:15.205 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:15.227 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:15 smithi133 bash[105589]: cluster 2023-10-09T05:26:14.164114+0000 mgr.y (mgr.25277) 1140 : cluster 0 pgmap v1213: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T05:26:15.229 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:15 smithi133 bash[102858]: cluster 2023-10-09T05:26:14.164114+0000 mgr.y (mgr.25277) 1140 : cluster 0 pgmap v1213: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T05:26:15.248 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:15.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:15.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:15.355 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:15.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:15 smithi194 bash[92666]: cluster 2023-10-09T05:26:14.164114+0000 mgr.y (mgr.25277) 1140 : cluster 0 pgmap v1213: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T05:26:15.390 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:15.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:15.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:15.495 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:15.531 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:15.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:15.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:15.636 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:15.656 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:15.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:15.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:15.667 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 52 16 71537 71521 85.9491 52.125 0.00711739 0.0116236 2023-10-09T05:26:15.759 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:15.789 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:15.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:15.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:15.895 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:15.920 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:15.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:15.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:16.025 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:16.051 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:16.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:16.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:16.156 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:16.182 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:16.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:16.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:16.288 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:16.318 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:16.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:16.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:16.423 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:16.446 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:16.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:16.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:16.549 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:16.575 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:16.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:16.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:16.667 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 53 16 72447 72431 85.4004 56.875 0.0155193 0.011704 2023-10-09T05:26:16.678 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:16.697 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:16.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:16.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:16.800 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:16.828 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:16.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:16.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:16.935 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:16.961 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:16.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:16.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:17.067 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:17.091 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:17.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:17.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:17.196 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:17.230 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:17.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:17.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:17.336 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:17.368 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:17.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:17.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:17.472 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:17.483 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:17 smithi194 bash[92666]: cluster 2023-10-09T05:26:16.166375+0000 mgr.y (mgr.25277) 1141 : cluster 0 pgmap v1214: 89 pgs: 89 active+clean; 4.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 60 MiB/s wr, 953 op/s 2023-10-09T05:26:17.507 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:17.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:17.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:17.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:17 smithi133 bash[105589]: cluster 2023-10-09T05:26:16.166375+0000 mgr.y (mgr.25277) 1141 : cluster 0 pgmap v1214: 89 pgs: 89 active+clean; 4.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 60 MiB/s wr, 953 op/s 2023-10-09T05:26:17.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:17 smithi133 bash[102858]: cluster 2023-10-09T05:26:16.166375+0000 mgr.y (mgr.25277) 1141 : cluster 0 pgmap v1214: 89 pgs: 89 active+clean; 4.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 60 MiB/s wr, 953 op/s 2023-10-09T05:26:17.611 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:17.637 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:17.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:17.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:17.667 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 54 16 73259 73243 84.7588 50.75 0.00979373 0.0117925 2023-10-09T05:26:17.742 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:17.769 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:17.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:17.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:17.872 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:17.899 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:17.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:17.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:18.003 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:18.006 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:26:18.006 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:26:18.030 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:18.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:18.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:18.135 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:18.166 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:18.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:18.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:18.269 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:18.301 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:18.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:18.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:18.405 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:18.433 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:26:18.435 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:18.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:18.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:18.493 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:26:18.538 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:18.540 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:26:18.560 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:18.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:18.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:18.664 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:18.667 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 55 16 73981 73965 84.0381 45.125 0.0143683 0.0118894 2023-10-09T05:26:18.683 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:26:18.693 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:18.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:18.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:18.722 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:26:18.761 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:26:18.799 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:18.807 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:26:18.829 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:18.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:18.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:18.845 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:26:18.848 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:26:18.848 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:26:18.849 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:26:18.849 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:26:18.849 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:26:18.849 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:26:18.849 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:26:18.849 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:26:18.932 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:18.963 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:18.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:18.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:19.067 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:19.093 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:19.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:19.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:19.196 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:19.219 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:19.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:19.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:19.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:19 smithi133 bash[102858]: cluster 2023-10-09T05:26:18.166870+0000 mgr.y (mgr.25277) 1142 : cluster 0 pgmap v1215: 89 pgs: 89 active+clean; 4.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 56 MiB/s wr, 888 op/s 2023-10-09T05:26:19.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:19 smithi133 bash[102858]: audit 2023-10-09T05:26:18.353696+0000 mon.b (mon.1) 23 : audit 0 from='client.? 172.21.15.133:0/3043084670' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:26:19.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:19 smithi133 bash[102858]: audit 2023-10-09T05:26:19.048629+0000 mon.c (mon.2) 104 : audit 1 from='client.? 172.21.15.133:0/600252395' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:26:19.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:19 smithi133 bash[102858]: audit 2023-10-09T05:26:19.049090+0000 mon.a (mon.0) 1488 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:26:19.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:19 smithi133 bash[105589]: cluster 2023-10-09T05:26:18.166870+0000 mgr.y (mgr.25277) 1142 : cluster 0 pgmap v1215: 89 pgs: 89 active+clean; 4.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 56 MiB/s wr, 888 op/s 2023-10-09T05:26:19.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:19 smithi133 bash[105589]: audit 2023-10-09T05:26:18.353696+0000 mon.b (mon.1) 23 : audit 0 from='client.? 172.21.15.133:0/3043084670' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:26:19.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:19 smithi133 bash[105589]: audit 2023-10-09T05:26:19.048629+0000 mon.c (mon.2) 104 : audit 1 from='client.? 172.21.15.133:0/600252395' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:26:19.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:19 smithi133 bash[105589]: audit 2023-10-09T05:26:19.049090+0000 mon.a (mon.0) 1488 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:26:19.324 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:19.346 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:19.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:19.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:19.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:19 smithi194 bash[92666]: cluster 2023-10-09T05:26:18.166870+0000 mgr.y (mgr.25277) 1142 : cluster 0 pgmap v1215: 89 pgs: 89 active+clean; 4.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 56 MiB/s wr, 888 op/s 2023-10-09T05:26:19.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:19 smithi194 bash[92666]: audit 2023-10-09T05:26:18.353696+0000 mon.b (mon.1) 23 : audit 0 from='client.? 172.21.15.133:0/3043084670' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:26:19.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:19 smithi194 bash[92666]: audit 2023-10-09T05:26:19.048629+0000 mon.c (mon.2) 104 : audit 1 from='client.? 172.21.15.133:0/600252395' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:26:19.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:19 smithi194 bash[92666]: audit 2023-10-09T05:26:19.049090+0000 mon.a (mon.0) 1488 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:26:19.449 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:19.479 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:19.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:19.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:19.588 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:19.619 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:19.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:19.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:19.640 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is set 2023-10-09T05:26:19.667 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 56 16 74869 74853 83.5285 55.5 0.00337032 0.0119602 2023-10-09T05:26:19.724 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:19.742 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:19.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:19.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:19.845 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:19.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:19.871 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:19.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:19.972 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:20.005 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:20.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:20.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:20.111 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:20.137 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:20.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:20.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:20.236 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:20 smithi133 bash[105589]: cluster 2023-10-09T05:26:19.624311+0000 mon.a (mon.0) 1489 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:26:20.237 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:20 smithi133 bash[105589]: audit 2023-10-09T05:26:19.634323+0000 mon.a (mon.0) 1490 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:26:20.237 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:20 smithi133 bash[105589]: cluster 2023-10-09T05:26:19.634695+0000 mon.a (mon.0) 1491 : cluster 0 osdmap e273: 8 total, 8 up, 7 in 2023-10-09T05:26:20.237 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:20 smithi133 bash[105589]: cluster 2023-10-09T05:26:19.647415+0000 osd.6 (osd.6) 223 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T05:26:20.237 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:20 smithi133 bash[105589]: cluster 2023-10-09T05:26:19.648092+0000 osd.7 (osd.7) 222 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T05:26:20.237 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:20 smithi133 bash[105589]: cluster 2023-10-09T05:26:19.873973+0000 osd.5 (osd.5) 252 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T05:26:20.237 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:20 smithi133 bash[102858]: cluster 2023-10-09T05:26:19.624311+0000 mon.a (mon.0) 1489 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:26:20.238 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:20 smithi133 bash[102858]: audit 2023-10-09T05:26:19.634323+0000 mon.a (mon.0) 1490 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:26:20.238 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:20 smithi133 bash[102858]: cluster 2023-10-09T05:26:19.634695+0000 mon.a (mon.0) 1491 : cluster 0 osdmap e273: 8 total, 8 up, 7 in 2023-10-09T05:26:20.238 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:20 smithi133 bash[102858]: cluster 2023-10-09T05:26:19.647415+0000 osd.6 (osd.6) 223 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T05:26:20.238 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:20 smithi133 bash[102858]: cluster 2023-10-09T05:26:19.648092+0000 osd.7 (osd.7) 222 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T05:26:20.238 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:20 smithi133 bash[102858]: cluster 2023-10-09T05:26:19.873973+0000 osd.5 (osd.5) 252 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T05:26:20.245 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:20.282 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:20.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:20.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:20.387 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:20.407 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:20.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:20.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:20.510 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:20.537 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:20.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:20.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:20.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:20 smithi194 bash[92666]: cluster 2023-10-09T05:26:19.624311+0000 mon.a (mon.0) 1489 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:26:20.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:20 smithi194 bash[92666]: audit 2023-10-09T05:26:19.634323+0000 mon.a (mon.0) 1490 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:26:20.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:20 smithi194 bash[92666]: cluster 2023-10-09T05:26:19.634695+0000 mon.a (mon.0) 1491 : cluster 0 osdmap e273: 8 total, 8 up, 7 in 2023-10-09T05:26:20.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:20 smithi194 bash[92666]: cluster 2023-10-09T05:26:19.647415+0000 osd.6 (osd.6) 223 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T05:26:20.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:20 smithi194 bash[92666]: cluster 2023-10-09T05:26:19.648092+0000 osd.7 (osd.7) 222 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T05:26:20.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:20 smithi194 bash[92666]: cluster 2023-10-09T05:26:19.873973+0000 osd.5 (osd.5) 252 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T05:26:20.640 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:20.667 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:20.667 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 57 16 75644 75628 82.9128 48.4375 0.0205007 0.012055 2023-10-09T05:26:20.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:20.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:20.772 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:20.802 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:20.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:20.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:20.909 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:20.935 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:20.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:20.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:21.043 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:21.075 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:21.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:21.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:21.180 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:21.205 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:21.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:21.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:21.237 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:21 smithi133 bash[102858]: cluster 2023-10-09T05:26:19.649404+0000 osd.4 (osd.4) 245 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T05:26:21.237 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:21 smithi133 bash[102858]: cluster 2023-10-09T05:26:20.167357+0000 mgr.y (mgr.25277) 1143 : cluster 0 pgmap v1217: 89 pgs: 89 active+clean; 4.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 60 MiB/s wr, 957 op/s 2023-10-09T05:26:21.310 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:21.336 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:21.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:21.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:21.437 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:21.468 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:21.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:21.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:21.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:21 smithi133 bash[105589]: cluster 2023-10-09T05:26:19.649404+0000 osd.4 (osd.4) 245 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T05:26:21.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:21 smithi133 bash[105589]: cluster 2023-10-09T05:26:20.167357+0000 mgr.y (mgr.25277) 1143 : cluster 0 pgmap v1217: 89 pgs: 89 active+clean; 4.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 60 MiB/s wr, 957 op/s 2023-10-09T05:26:21.574 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:21.604 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:21.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:21.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:21.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:21 smithi194 bash[92666]: cluster 2023-10-09T05:26:19.649404+0000 osd.4 (osd.4) 245 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T05:26:21.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:21 smithi194 bash[92666]: cluster 2023-10-09T05:26:20.167357+0000 mgr.y (mgr.25277) 1143 : cluster 0 pgmap v1217: 89 pgs: 89 active+clean; 4.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 60 MiB/s wr, 957 op/s 2023-10-09T05:26:21.667 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 58 16 76618 76602 82.5327 60.875 0.00369622 0.0120992 2023-10-09T05:26:21.707 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:21.731 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:21.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:21.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:21.833 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:21.870 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:21.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:21.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:21.976 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:22.011 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:22.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:22.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:22.114 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:22.139 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:22.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:22.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:22.243 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:22.271 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:22.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:22.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:22.374 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:22.399 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:22.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:22.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:22.502 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:22.539 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:22.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:22.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:22.644 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:22.671 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 59 16 77419 77403 81.9823 50.0625 0.0191053 0.0121915 2023-10-09T05:26:22.672 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:22.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:22.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:22.777 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:22.813 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:22.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:22.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:22.916 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:22.943 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:22.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:22.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:23.046 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:23.070 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:23.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:23.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:23.174 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:23.199 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:23.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:23.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:23.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:23 smithi133 bash[102858]: cluster 2023-10-09T05:26:22.168015+0000 mgr.y (mgr.25277) 1144 : cluster 0 pgmap v1218: 89 pgs: 89 active+clean; 4.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 58 MiB/s wr, 924 op/s 2023-10-09T05:26:23.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:23 smithi133 bash[105589]: cluster 2023-10-09T05:26:22.168015+0000 mgr.y (mgr.25277) 1144 : cluster 0 pgmap v1218: 89 pgs: 89 active+clean; 4.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 58 MiB/s wr, 924 op/s 2023-10-09T05:26:23.304 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:23.331 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:23.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:23.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:23.416 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T05:26:23.437 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:23.498 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:23.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:23.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:23.604 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:23.613 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:23 smithi194 bash[92666]: cluster 2023-10-09T05:26:22.168015+0000 mgr.y (mgr.25277) 1144 : cluster 0 pgmap v1218: 89 pgs: 89 active+clean; 4.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 58 MiB/s wr, 924 op/s 2023-10-09T05:26:23.633 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:23.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:23.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:23.667 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T05:26:23.668017+0000 min lat: 0.00116799 max lat: 0.27335 avg lat: 0.0122481 2023-10-09T05:26:23.667 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:26:23.668 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 60 16 78360 78344 81.596 58.8125 0.0102972 0.0122481 2023-10-09T05:26:23.741 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:23.780 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:23.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:23.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:23.885 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:23.914 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:23.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:23.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:24.018 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:24.048 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:24.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:24.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:24.154 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:24.180 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:24.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:24.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:24.283 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:24.284 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:26:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:26:24] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.43.0" 2023-10-09T05:26:24.313 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:24.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:24.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:24.418 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:24.441 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:24.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:24.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:24.544 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:24.573 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:24.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:24.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:24.668 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 61 16 79290 79274 81.2112 58.125 0.0109178 0.0123068 2023-10-09T05:26:24.679 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:24.719 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:24.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:24.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:24.823 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:24.832 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:24 smithi133 bash[105589]: cluster 2023-10-09T05:26:24.168472+0000 mgr.y (mgr.25277) 1145 : cluster 0 pgmap v1219: 89 pgs: 89 active+clean; 4.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 57 MiB/s wr, 908 op/s 2023-10-09T05:26:24.832 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:24 smithi133 bash[102858]: cluster 2023-10-09T05:26:24.168472+0000 mgr.y (mgr.25277) 1145 : cluster 0 pgmap v1219: 89 pgs: 89 active+clean; 4.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 57 MiB/s wr, 908 op/s 2023-10-09T05:26:24.852 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:24.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:24.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:24.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:24 smithi194 bash[92666]: cluster 2023-10-09T05:26:24.168472+0000 mgr.y (mgr.25277) 1145 : cluster 0 pgmap v1219: 89 pgs: 89 active+clean; 4.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 57 MiB/s wr, 908 op/s 2023-10-09T05:26:24.958 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:24.983 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:24.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:24.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:25.086 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:25.116 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:25.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:25.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:25.225 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:25.250 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:25.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:25.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:25.353 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:25.386 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:25.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:25.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:25.492 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:25.521 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:25.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:25.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:25.624 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:25.649 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:25.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:25.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:25.668 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 62 16 80235 80219 80.8538 59.0625 0.0144496 0.0123621 2023-10-09T05:26:25.699 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:26:25.715 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:26:25.753 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:25.836 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:25.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:25.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:25.941 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:25.972 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:25.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:25.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:26.079 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:26.107 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:26.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:26.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:26.210 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:26.244 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:26.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:26.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:26.273 INFO:teuthology.orchestra.run.smithi133.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T05:26:26.349 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:26.379 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:26.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:26.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:26.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:26 smithi194 bash[92666]: audit 2023-10-09T05:26:26.198291+0000 mon.c (mon.2) 105 : audit 1 from='client.? 172.21.15.133:0/1125774248' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:26:26.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:26 smithi194 bash[92666]: audit 2023-10-09T05:26:26.198763+0000 mon.a (mon.0) 1492 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:26:26.483 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:26.509 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:26.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:26.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:26.516 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:26 smithi133 bash[105589]: audit 2023-10-09T05:26:26.198291+0000 mon.c (mon.2) 105 : audit 1 from='client.? 172.21.15.133:0/1125774248' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:26:26.516 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:26 smithi133 bash[105589]: audit 2023-10-09T05:26:26.198763+0000 mon.a (mon.0) 1492 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:26:26.516 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:26 smithi133 bash[102858]: audit 2023-10-09T05:26:26.198291+0000 mon.c (mon.2) 105 : audit 1 from='client.? 172.21.15.133:0/1125774248' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:26:26.516 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:26 smithi133 bash[102858]: audit 2023-10-09T05:26:26.198763+0000 mon.a (mon.0) 1492 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:26:26.612 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:26.636 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:26.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:26.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:26.668 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 63 16 81003 80987 80.3323 48 0.0124816 0.0124417 2023-10-09T05:26:26.741 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:26.784 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:26.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:26.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:26.888 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:26.917 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:26.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:26.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:27.020 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:27.052 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:27.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:27.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:27.156 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:27.189 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:27.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:27.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:27.267 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:27 smithi133 bash[102858]: cluster 2023-10-09T05:26:26.169180+0000 mgr.y (mgr.25277) 1146 : cluster 0 pgmap v1220: 89 pgs: 89 active+clean; 4.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 868 op/s 2023-10-09T05:26:27.267 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:27 smithi133 bash[102858]: cluster 2023-10-09T05:26:26.257868+0000 osd.6 (osd.6) 224 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T05:26:27.268 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:27 smithi133 bash[102858]: cluster 2023-10-09T05:26:26.259818+0000 osd.5 (osd.5) 253 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T05:26:27.293 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:27.324 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:27.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:27.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:27.433 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:27.440 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:27 smithi194 bash[92666]: cluster 2023-10-09T05:26:26.169180+0000 mgr.y (mgr.25277) 1146 : cluster 0 pgmap v1220: 89 pgs: 89 active+clean; 4.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 868 op/s 2023-10-09T05:26:27.440 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:27 smithi194 bash[92666]: cluster 2023-10-09T05:26:26.257868+0000 osd.6 (osd.6) 224 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T05:26:27.440 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:27 smithi194 bash[92666]: cluster 2023-10-09T05:26:26.259818+0000 osd.5 (osd.5) 253 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T05:26:27.440 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:27 smithi194 bash[92666]: audit 2023-10-09T05:26:26.260046+0000 mon.a (mon.0) 1493 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:26:27.440 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:27 smithi194 bash[92666]: cluster 2023-10-09T05:26:26.260101+0000 mon.a (mon.0) 1494 : cluster 0 osdmap e274: 8 total, 8 up, 7 in 2023-10-09T05:26:27.459 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:27.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:27.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:27.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:27 smithi133 bash[102858]: audit 2023-10-09T05:26:26.260046+0000 mon.a (mon.0) 1493 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:26:27.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:27 smithi133 bash[102858]: cluster 2023-10-09T05:26:26.260101+0000 mon.a (mon.0) 1494 : cluster 0 osdmap e274: 8 total, 8 up, 7 in 2023-10-09T05:26:27.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:27 smithi133 bash[105589]: cluster 2023-10-09T05:26:26.169180+0000 mgr.y (mgr.25277) 1146 : cluster 0 pgmap v1220: 89 pgs: 89 active+clean; 4.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 868 op/s 2023-10-09T05:26:27.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:27 smithi133 bash[105589]: cluster 2023-10-09T05:26:26.257868+0000 osd.6 (osd.6) 224 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T05:26:27.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:27 smithi133 bash[105589]: cluster 2023-10-09T05:26:26.259818+0000 osd.5 (osd.5) 253 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T05:26:27.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:27 smithi133 bash[105589]: audit 2023-10-09T05:26:26.260046+0000 mon.a (mon.0) 1493 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:26:27.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:27 smithi133 bash[105589]: cluster 2023-10-09T05:26:26.260101+0000 mon.a (mon.0) 1494 : cluster 0 osdmap e274: 8 total, 8 up, 7 in 2023-10-09T05:26:27.562 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:27.590 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:27.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:27.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:27.668 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 64 16 81910 81894 79.9628 56.6875 0.00998349 0.0125005 2023-10-09T05:26:27.695 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:27.722 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:27.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:27.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:27.828 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:27.854 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:27.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:27.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:27.890 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T05:26:27.956 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:27.981 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:27.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:27.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:28.085 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:28.117 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:28.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:28.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:28.220 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:28.248 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:28.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:28.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:28.352 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:28.382 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:28.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:28.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:28.488 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:28.514 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:28.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:28.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:28.620 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:28.652 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:28.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:28.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:28.668 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 65 16 82772 82756 79.5614 53.875 0.0130451 0.0125628 2023-10-09T05:26:28.757 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:28.784 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:28.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:28.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:28.890 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:28.923 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:28.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:28.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:29.029 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:29.056 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:29.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:29.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:29.159 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:29.183 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:29.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:29.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:29.278 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:29 smithi133 bash[102858]: cluster 2023-10-09T05:26:28.169713+0000 mgr.y (mgr.25277) 1147 : cluster 0 pgmap v1222: 89 pgs: 89 active+clean; 4.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 46 MiB/s wr, 732 op/s 2023-10-09T05:26:29.278 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:29 smithi133 bash[102858]: audit 2023-10-09T05:26:28.463325+0000 mon.a (mon.0) 1495 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:26:29.290 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:29.298 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:29 smithi194 bash[92666]: cluster 2023-10-09T05:26:28.169713+0000 mgr.y (mgr.25277) 1147 : cluster 0 pgmap v1222: 89 pgs: 89 active+clean; 4.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 46 MiB/s wr, 732 op/s 2023-10-09T05:26:29.298 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:29 smithi194 bash[92666]: audit 2023-10-09T05:26:28.463325+0000 mon.a (mon.0) 1495 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:26:29.324 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:29.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:29.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:29.430 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:29.456 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:29.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:29.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:29.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:29 smithi133 bash[105589]: cluster 2023-10-09T05:26:28.169713+0000 mgr.y (mgr.25277) 1147 : cluster 0 pgmap v1222: 89 pgs: 89 active+clean; 4.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 46 MiB/s wr, 732 op/s 2023-10-09T05:26:29.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:29 smithi133 bash[105589]: audit 2023-10-09T05:26:28.463325+0000 mon.a (mon.0) 1495 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:26:29.559 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:29.582 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:29.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:29.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:29.668 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 66 16 83666 83650 79.2025 55.875 0.0378165 0.0126206 2023-10-09T05:26:29.688 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:29.722 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:29.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:29.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:29.829 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:29.859 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:29.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:29.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:29.963 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:29.994 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:30.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:30.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:30.101 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:30.133 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:30.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:30.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:30.239 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:30.270 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:30.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:30.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:30.375 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:30.406 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:30.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:30.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:30.415 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:26:30.510 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:30.540 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:30.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:30.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:30.644 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:30.668 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:30.669 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 67 16 84502 84486 78.8001 52.25 0.0277974 0.0126809 2023-10-09T05:26:30.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:30.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:30.772 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:30.802 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:30.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:30.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:30.908 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:30.935 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:30.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:30.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:31.041 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:31.073 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:31.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:31.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:31.178 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:31.206 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:31.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:31.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:31.311 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:31.338 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:31.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:31.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:31.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:31 smithi194 bash[92666]: cluster 2023-10-09T05:26:30.170447+0000 mgr.y (mgr.25277) 1148 : cluster 0 pgmap v1223: 89 pgs: 89 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 55 MiB/s wr, 877 op/s 2023-10-09T05:26:31.437 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:26:31.437 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:26:31.442 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:31.471 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:31.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:31.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:31.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:31 smithi133 bash[102858]: cluster 2023-10-09T05:26:30.170447+0000 mgr.y (mgr.25277) 1148 : cluster 0 pgmap v1223: 89 pgs: 89 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 55 MiB/s wr, 877 op/s 2023-10-09T05:26:31.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:31 smithi133 bash[105589]: cluster 2023-10-09T05:26:30.170447+0000 mgr.y (mgr.25277) 1148 : cluster 0 pgmap v1223: 89 pgs: 89 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 55 MiB/s wr, 877 op/s 2023-10-09T05:26:31.574 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:31.604 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:31.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:31.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:31.668 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 68 16 85375 85359 78.4436 54.5625 0.01192 0.012743 2023-10-09T05:26:31.712 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:31.737 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:31.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:31.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:31.841 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:31.869 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:31.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:31.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:31.973 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:31.999 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:32.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:32.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:32.107 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:32.137 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:32.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:32.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:32.239 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:32.274 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:32.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:32.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:32.377 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:32.408 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:32.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:32.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:32.452 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:26:32.511 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:32.546 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:32.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:32.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:32.653 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:32.668 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 69 16 86285 86269 78.131 56.875 0.0212825 0.0127919 2023-10-09T05:26:32.677 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:32.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:32.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:32.781 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:32.811 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:32.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:32.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:32.915 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:32.941 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:32.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:32.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:33.045 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:33.077 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:33.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:33.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:33.184 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:33.215 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:33.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:33.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:33.319 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:33.343 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:33.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:33.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:33.448 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:33.476 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:33 smithi133 bash[102858]: cluster 2023-10-09T05:26:32.172156+0000 mgr.y (mgr.25277) 1149 : cluster 0 pgmap v1224: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 55 MiB/s wr, 878 op/s 2023-10-09T05:26:33.477 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:33 smithi133 bash[105589]: cluster 2023-10-09T05:26:32.172156+0000 mgr.y (mgr.25277) 1149 : cluster 0 pgmap v1224: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 55 MiB/s wr, 878 op/s 2023-10-09T05:26:33.481 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:33.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:33.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:33.590 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:33.621 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:33.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:33.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:33.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:33 smithi194 bash[92666]: cluster 2023-10-09T05:26:32.172156+0000 mgr.y (mgr.25277) 1149 : cluster 0 pgmap v1224: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 55 MiB/s wr, 878 op/s 2023-10-09T05:26:33.668 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 70 16 87103 87087 77.7452 51.125 0.0204511 0.0128572 2023-10-09T05:26:33.725 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:33.751 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:33.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:33.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:33.856 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:33.880 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:33.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:33.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:33.985 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:34.007 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:34.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:34.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:34.110 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:34.142 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:34.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:34.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:34.245 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:34.270 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:34.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:34.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:34.378 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:34.412 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:34.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:34.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:34.516 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:34.545 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:34.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:34.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:34.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:26:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:26:34] "GET /metrics HTTP/1.1" 200 36478 "" "Prometheus/2.43.0" 2023-10-09T05:26:34.648 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:34.668 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 71 16 87836 87820 77.2954 45.8125 0.0107605 0.0129317 2023-10-09T05:26:34.676 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:34.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:34.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:34.780 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:34.790 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:34 smithi194 bash[92666]: cluster 2023-10-09T05:26:34.172606+0000 mgr.y (mgr.25277) 1150 : cluster 0 pgmap v1225: 89 pgs: 89 active+clean; 5.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 55 MiB/s wr, 879 op/s 2023-10-09T05:26:34.808 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:34.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:34.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:34.912 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:34.930 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:34.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:34.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:35.007 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:34 smithi133 bash[105589]: cluster 2023-10-09T05:26:34.172606+0000 mgr.y (mgr.25277) 1150 : cluster 0 pgmap v1225: 89 pgs: 89 active+clean; 5.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 55 MiB/s wr, 879 op/s 2023-10-09T05:26:35.008 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:34 smithi133 bash[102858]: cluster 2023-10-09T05:26:34.172606+0000 mgr.y (mgr.25277) 1150 : cluster 0 pgmap v1225: 89 pgs: 89 active+clean; 5.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 55 MiB/s wr, 879 op/s 2023-10-09T05:26:35.033 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:35.069 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:35.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:35.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:35.178 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:35.201 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:35.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:35.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:35.304 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:35.333 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:35.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:35.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:35.436 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:35.467 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:35.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:35.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:35.573 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:35.596 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:35.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:35.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:35.669 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 72 16 88524 88508 76.819 43 0.0121458 0.0130116 2023-10-09T05:26:35.703 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:35.732 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:35.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:35.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:35.835 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:35.864 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:35.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:35.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:35.967 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:35.996 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:35.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:36.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:36.101 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:36.127 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:36.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:36.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:36.231 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:36.257 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:36.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:36.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:36.360 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:36.398 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:36.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:36.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:36.504 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:36.534 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:36.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:36.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:36.637 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:36.660 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:36.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:36.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:36.668 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 73 16 89378 89362 76.4978 53.375 0.00538719 0.0130658 2023-10-09T05:26:36.764 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:36.791 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:36.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:36.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:36.896 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:36.921 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:36.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:36.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:37.024 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:37.050 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:37.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:37.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:37.153 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:37.177 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:37.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:37.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:37.229 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:37 smithi133 bash[102858]: cluster 2023-10-09T05:26:36.173277+0000 mgr.y (mgr.25277) 1151 : cluster 0 pgmap v1226: 89 pgs: 89 active+clean; 5.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 54 MiB/s wr, 858 op/s 2023-10-09T05:26:37.281 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:37.311 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:37.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:37.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:37.414 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:37.423 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:37 smithi194 bash[92666]: cluster 2023-10-09T05:26:36.173277+0000 mgr.y (mgr.25277) 1151 : cluster 0 pgmap v1226: 89 pgs: 89 active+clean; 5.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 54 MiB/s wr, 858 op/s 2023-10-09T05:26:37.443 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:37.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:37.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:37.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:37 smithi133 bash[105589]: cluster 2023-10-09T05:26:36.173277+0000 mgr.y (mgr.25277) 1151 : cluster 0 pgmap v1226: 89 pgs: 89 active+clean; 5.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 54 MiB/s wr, 858 op/s 2023-10-09T05:26:37.548 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:37.576 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:37.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:37.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:37.669 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 74 16 90218 90202 76.1735 52.5 0.014926 0.0131219 2023-10-09T05:26:37.681 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:37.702 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:37.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:37.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:37.809 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:37.833 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:37.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:37.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:37.939 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:37.975 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:37.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:37.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:38.083 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:38.105 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:38.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:38.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:38.209 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:38.238 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:38.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:38.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:38.343 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:38.373 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:38.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:38.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:38.478 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:38.508 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:38.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:38.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:38.611 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:38.635 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:38.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:38.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:38.669 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 75 16 91072 91056 75.8695 53.375 0.00778218 0.0131749 2023-10-09T05:26:38.739 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:38.765 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:38.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:38.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:38.868 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:38.911 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:38.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:38.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:39.019 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:39.052 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:39.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:39.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:39.156 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:39.179 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:39.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:39.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:39.283 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:39.310 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:39.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:39.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:39.419 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:39.456 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:39.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:39.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:39.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:39 smithi133 bash[102858]: cluster 2023-10-09T05:26:38.173859+0000 mgr.y (mgr.25277) 1152 : cluster 0 pgmap v1227: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 48 MiB/s wr, 762 op/s 2023-10-09T05:26:39.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:39 smithi133 bash[105589]: cluster 2023-10-09T05:26:38.173859+0000 mgr.y (mgr.25277) 1152 : cluster 0 pgmap v1227: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 48 MiB/s wr, 762 op/s 2023-10-09T05:26:39.561 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:39.594 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:39.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:39.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:39.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:39 smithi194 bash[92666]: cluster 2023-10-09T05:26:38.173859+0000 mgr.y (mgr.25277) 1152 : cluster 0 pgmap v1227: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 48 MiB/s wr, 762 op/s 2023-10-09T05:26:39.669 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 76 16 91963 91947 75.6039 55.6875 0.0123985 0.0132205 2023-10-09T05:26:39.701 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:39.729 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:39.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:39.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:39.832 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:39.857 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:39.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:39.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:39.965 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:39.989 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:39.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:39.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:40.094 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:40.121 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:40.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:40.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:40.227 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:40.254 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:40.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:40.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:40.361 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:40.401 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:40.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:40.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:40.509 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:40.536 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:40.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:40.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:40.641 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:40.669 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 77 16 92848 92832 75.3402 55.3125 0.026125 0.0132676 2023-10-09T05:26:40.673 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:40.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:40.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:40.779 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:40.809 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:40.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:40.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:40.913 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:40.948 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:40.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:40.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:41.052 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:41.076 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:41.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:41.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:41.182 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:41.210 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:41.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:41.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:41.316 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:41.325 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:41 smithi194 bash[92666]: cluster 2023-10-09T05:26:40.174503+0000 mgr.y (mgr.25277) 1153 : cluster 0 pgmap v1228: 89 pgs: 89 active+clean; 5.5 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 59 MiB/s wr, 946 op/s 2023-10-09T05:26:41.348 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:41.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:41.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:41.451 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:41.461 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:41 smithi133 bash[102858]: cluster 2023-10-09T05:26:40.174503+0000 mgr.y (mgr.25277) 1153 : cluster 0 pgmap v1228: 89 pgs: 89 active+clean; 5.5 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 59 MiB/s wr, 946 op/s 2023-10-09T05:26:41.461 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:41 smithi133 bash[105589]: cluster 2023-10-09T05:26:40.174503+0000 mgr.y (mgr.25277) 1153 : cluster 0 pgmap v1228: 89 pgs: 89 active+clean; 5.5 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 59 MiB/s wr, 946 op/s 2023-10-09T05:26:41.479 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:41.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:41.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:41.585 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:41.633 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:41.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:41.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:41.669 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 78 16 93690 93674 75.0489 52.625 0.0107705 0.0133194 2023-10-09T05:26:41.742 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:41.771 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:41.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:41.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:41.878 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:41.905 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:41.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:41.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:42.008 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:42.041 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:42.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:42.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:42.148 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:42.177 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:42.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:42.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:42.285 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:42.309 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:42.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:42.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:42.415 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:42.447 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:42.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:42.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:42.550 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:42.580 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:42.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:42.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:42.669 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 79 16 94427 94411 74.682 46.0625 0.00843874 0.0133841 2023-10-09T05:26:42.686 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:42.711 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:42.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:42.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:42.817 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:42.856 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:42.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:42.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:42.960 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:42.990 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:42.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:42.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:43.096 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:43.120 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:43.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:43.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:43.228 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:43.270 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:43.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:43.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:43.374 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:43.382 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:43 smithi133 bash[105589]: cluster 2023-10-09T05:26:42.175033+0000 mgr.y (mgr.25277) 1154 : cluster 0 pgmap v1229: 89 pgs: 89 active+clean; 5.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 50 MiB/s wr, 805 op/s 2023-10-09T05:26:43.383 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:43 smithi133 bash[102858]: cluster 2023-10-09T05:26:42.175033+0000 mgr.y (mgr.25277) 1154 : cluster 0 pgmap v1229: 89 pgs: 89 active+clean; 5.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 50 MiB/s wr, 805 op/s 2023-10-09T05:26:43.404 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:43.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:43.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:43.507 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:43.516 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:43 smithi194 bash[92666]: cluster 2023-10-09T05:26:42.175033+0000 mgr.y (mgr.25277) 1154 : cluster 0 pgmap v1229: 89 pgs: 89 active+clean; 5.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 50 MiB/s wr, 805 op/s 2023-10-09T05:26:43.541 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:43.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:43.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:43.645 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:43.669 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T05:26:43.669884+0000 min lat: 0.00116799 max lat: 0.27335 avg lat: 0.0134379 2023-10-09T05:26:43.670 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:26:43.670 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 80 16 95240 95224 74.3836 50.8125 0.0137345 0.0134379 2023-10-09T05:26:43.675 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:43.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:43.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:43.783 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:43.806 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:43.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:43.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:43.912 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:43.948 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:43.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:43.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:44.051 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:44.078 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:44.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:44.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:44.184 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:44.219 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:44.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:44.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:44.323 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:44.330 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:44 smithi194 bash[92666]: audit 2023-10-09T05:26:43.470154+0000 mon.a (mon.0) 1496 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:26:44.348 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:44.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:44.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:44.455 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:44.464 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:26:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:26:44] "GET /metrics HTTP/1.1" 200 36476 "" "Prometheus/2.43.0" 2023-10-09T05:26:44.465 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:44 smithi133 bash[105589]: audit 2023-10-09T05:26:43.470154+0000 mon.a (mon.0) 1496 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:26:44.465 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:44 smithi133 bash[102858]: audit 2023-10-09T05:26:43.470154+0000 mon.a (mon.0) 1496 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:26:44.483 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:44.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:44.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:44.590 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:44.616 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:44.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:44.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:44.670 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 81 16 96025 96009 74.0709 49.0625 0.016948 0.0134943 2023-10-09T05:26:44.721 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:44.748 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:44.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:44.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:44.837 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:26:44.854 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:44.881 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:44.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:44.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:44.893 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:26:44.953 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:26:44.986 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:45.009 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:26:45.013 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:45.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:45.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:45.052 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:26:45.093 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:26:45.117 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:45.135 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:26:45.143 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:45.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:45.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:45.178 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:26:45.186 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:26:45.186 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:26:45.186 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:26:45.186 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:26:45.187 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:26:45.187 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:26:45.187 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:26:45.187 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:26:45.248 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:45.275 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:45.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:45.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:45.379 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:45.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:45 smithi194 bash[92666]: cluster 2023-10-09T05:26:44.175487+0000 mgr.y (mgr.25277) 1155 : cluster 0 pgmap v1230: 89 pgs: 89 active+clean; 5.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 46 MiB/s wr, 737 op/s 2023-10-09T05:26:45.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:45 smithi194 bash[92666]: audit 2023-10-09T05:26:44.783715+0000 mon.a (mon.0) 1497 : audit 0 from='client.? 172.21.15.133:0/2078704972' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:26:45.422 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:45.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:45.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:45.531 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:45.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:45 smithi133 bash[102858]: cluster 2023-10-09T05:26:44.175487+0000 mgr.y (mgr.25277) 1155 : cluster 0 pgmap v1230: 89 pgs: 89 active+clean; 5.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 46 MiB/s wr, 737 op/s 2023-10-09T05:26:45.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:45 smithi133 bash[102858]: audit 2023-10-09T05:26:44.783715+0000 mon.a (mon.0) 1497 : audit 0 from='client.? 172.21.15.133:0/2078704972' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:26:45.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:45 smithi133 bash[105589]: cluster 2023-10-09T05:26:44.175487+0000 mgr.y (mgr.25277) 1155 : cluster 0 pgmap v1230: 89 pgs: 89 active+clean; 5.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 46 MiB/s wr, 737 op/s 2023-10-09T05:26:45.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:45 smithi133 bash[105589]: audit 2023-10-09T05:26:44.783715+0000 mon.a (mon.0) 1497 : audit 0 from='client.? 172.21.15.133:0/2078704972' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:26:45.555 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:45.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:45.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:45.611 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:26:45.611 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:26:45.659 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:45.670 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 82 16 96653 96637 73.6462 39.25 0.0371293 0.0135714 2023-10-09T05:26:45.700 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:45.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:45.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:45.806 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:45.834 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:45.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:45.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:45.939 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:45.986 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:45.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:45.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:46.092 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:46.120 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:46.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:46.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:46.224 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:46.249 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:46 smithi133 bash[102858]: audit 2023-10-09T05:26:45.785479+0000 mon.a (mon.0) 1498 : audit 1 from='client.? 172.21.15.133:0/3911027243' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:26:46.250 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:46 smithi133 bash[102858]: audit 2023-10-09T05:26:45.969839+0000 mon.c (mon.2) 106 : audit 1 from='client.? 172.21.15.133:0/3212254150' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:26:46.250 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:46 smithi133 bash[102858]: audit 2023-10-09T05:26:45.971500+0000 mon.a (mon.0) 1499 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:26:46.252 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:46.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:46.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:46.280 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is unset 2023-10-09T05:26:46.282 INFO:teuthology.orchestra.run.smithi133.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T05:26:46.356 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:46.398 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:46.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:46.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:46.502 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:46.511 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:46 smithi194 bash[92666]: audit 2023-10-09T05:26:45.785479+0000 mon.a (mon.0) 1498 : audit 1 from='client.? 172.21.15.133:0/3911027243' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:26:46.511 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:46 smithi194 bash[92666]: audit 2023-10-09T05:26:45.969839+0000 mon.c (mon.2) 106 : audit 1 from='client.? 172.21.15.133:0/3212254150' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:26:46.512 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:46 smithi194 bash[92666]: audit 2023-10-09T05:26:45.971500+0000 mon.a (mon.0) 1499 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:26:46.531 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:46.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:46.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:46.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:46 smithi133 bash[105589]: audit 2023-10-09T05:26:45.785479+0000 mon.a (mon.0) 1498 : audit 1 from='client.? 172.21.15.133:0/3911027243' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:26:46.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:46 smithi133 bash[105589]: audit 2023-10-09T05:26:45.969839+0000 mon.c (mon.2) 106 : audit 1 from='client.? 172.21.15.133:0/3212254150' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:26:46.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:46 smithi133 bash[105589]: audit 2023-10-09T05:26:45.971500+0000 mon.a (mon.0) 1499 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:26:46.634 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:46.663 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:46.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:46.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:46.670 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 83 16 97429 97413 73.3432 48.5 0.0139623 0.0136288 2023-10-09T05:26:46.769 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:46.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:46.793 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:46.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:46.894 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:46.923 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:46.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:46.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:47.030 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:47.065 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:47.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:47.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:47.171 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:47.207 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:47.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:47.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:47.251 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:47 smithi194 bash[92666]: cluster 2023-10-09T05:26:46.176181+0000 mgr.y (mgr.25277) 1156 : cluster 0 pgmap v1231: 89 pgs: 89 active+clean; 5.8 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 59 MiB/s wr, 943 op/s 2023-10-09T05:26:47.252 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:47 smithi194 bash[92666]: cluster 2023-10-09T05:26:46.246620+0000 mon.a (mon.0) 1500 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:26:47.252 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:47 smithi194 bash[92666]: cluster 2023-10-09T05:26:46.262503+0000 osd.6 (osd.6) 225 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T05:26:47.252 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:47 smithi194 bash[92666]: audit 2023-10-09T05:26:46.265342+0000 mon.a (mon.0) 1501 : audit 1 from='client.? 172.21.15.133:0/3911027243' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:26:47.252 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:47 smithi194 bash[92666]: audit 2023-10-09T05:26:46.265543+0000 mon.a (mon.0) 1502 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:26:47.252 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:47 smithi194 bash[92666]: cluster 2023-10-09T05:26:46.265741+0000 mon.a (mon.0) 1503 : cluster 0 osdmap e275: 8 total, 8 up, 7 in 2023-10-09T05:26:47.252 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:47 smithi194 bash[92666]: cluster 2023-10-09T05:26:46.275982+0000 osd.7 (osd.7) 223 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T05:26:47.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:47 smithi133 bash[102858]: cluster 2023-10-09T05:26:46.176181+0000 mgr.y (mgr.25277) 1156 : cluster 0 pgmap v1231: 89 pgs: 89 active+clean; 5.8 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 59 MiB/s wr, 943 op/s 2023-10-09T05:26:47.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:47 smithi133 bash[102858]: cluster 2023-10-09T05:26:46.246620+0000 mon.a (mon.0) 1500 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:26:47.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:47 smithi133 bash[102858]: cluster 2023-10-09T05:26:46.262503+0000 osd.6 (osd.6) 225 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T05:26:47.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:47 smithi133 bash[102858]: audit 2023-10-09T05:26:46.265342+0000 mon.a (mon.0) 1501 : audit 1 from='client.? 172.21.15.133:0/3911027243' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:26:47.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:47 smithi133 bash[102858]: audit 2023-10-09T05:26:46.265543+0000 mon.a (mon.0) 1502 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:26:47.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:47 smithi133 bash[102858]: cluster 2023-10-09T05:26:46.265741+0000 mon.a (mon.0) 1503 : cluster 0 osdmap e275: 8 total, 8 up, 7 in 2023-10-09T05:26:47.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:47 smithi133 bash[102858]: cluster 2023-10-09T05:26:46.275982+0000 osd.7 (osd.7) 223 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T05:26:47.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:47 smithi133 bash[105589]: cluster 2023-10-09T05:26:46.176181+0000 mgr.y (mgr.25277) 1156 : cluster 0 pgmap v1231: 89 pgs: 89 active+clean; 5.8 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 59 MiB/s wr, 943 op/s 2023-10-09T05:26:47.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:47 smithi133 bash[105589]: cluster 2023-10-09T05:26:46.246620+0000 mon.a (mon.0) 1500 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:26:47.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:47 smithi133 bash[105589]: cluster 2023-10-09T05:26:46.262503+0000 osd.6 (osd.6) 225 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T05:26:47.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:47 smithi133 bash[105589]: audit 2023-10-09T05:26:46.265342+0000 mon.a (mon.0) 1501 : audit 1 from='client.? 172.21.15.133:0/3911027243' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:26:47.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:47 smithi133 bash[105589]: audit 2023-10-09T05:26:46.265543+0000 mon.a (mon.0) 1502 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:26:47.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:47 smithi133 bash[105589]: cluster 2023-10-09T05:26:46.265741+0000 mon.a (mon.0) 1503 : cluster 0 osdmap e275: 8 total, 8 up, 7 in 2023-10-09T05:26:47.301 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:47 smithi133 bash[105589]: cluster 2023-10-09T05:26:46.275982+0000 osd.7 (osd.7) 223 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T05:26:47.314 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:47.344 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:47.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:47.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:47.449 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:47.475 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:47.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:47.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:47.578 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:47.607 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:47.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:47.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:47.670 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 84 16 98123 98107 72.9864 43.375 0.0185196 0.0136962 2023-10-09T05:26:47.710 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:47.740 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:47.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:47.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:47.846 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:47.873 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:47.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:47.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:47.976 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:48.006 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:48.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:48.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:48.114 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:48.140 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:48.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:48.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:48.244 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:48.279 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:48.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:48.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:48.381 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:48.405 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:48.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:48.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:48.508 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:48.549 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:48.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:48.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:48.654 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:48.671 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 85 16 98951 98935 72.7352 51.75 0.0456729 0.0137425 2023-10-09T05:26:48.690 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:48.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:48.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:48.795 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:48.821 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:48.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:48.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:48.929 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:48.963 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:48.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:48.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:49.065 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:49.085 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:49.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:49.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:49.188 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:49.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:49.242 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:49.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:49.344 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:49.377 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:49.384 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:49 smithi133 bash[105589]: cluster 2023-10-09T05:26:48.176801+0000 mgr.y (mgr.25277) 1157 : cluster 0 pgmap v1233: 89 pgs: 89 active+clean; 5.9 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 51 MiB/s wr, 813 op/s 2023-10-09T05:26:49.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:49.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:49.386 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:49 smithi133 bash[102858]: cluster 2023-10-09T05:26:48.176801+0000 mgr.y (mgr.25277) 1157 : cluster 0 pgmap v1233: 89 pgs: 89 active+clean; 5.9 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 51 MiB/s wr, 813 op/s 2023-10-09T05:26:49.486 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:49.520 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:49.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:49.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:49.624 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:49.632 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:49 smithi194 bash[92666]: cluster 2023-10-09T05:26:48.176801+0000 mgr.y (mgr.25277) 1157 : cluster 0 pgmap v1233: 89 pgs: 89 active+clean; 5.9 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 51 MiB/s wr, 813 op/s 2023-10-09T05:26:49.648 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:49.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:49.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:49.671 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 86 16 99691 99675 72.4272 46.25 0.0104543 0.0138016 2023-10-09T05:26:49.752 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:49.778 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:49.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:49.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:49.883 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:49.915 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:49.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:49.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:50.020 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:50.048 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:50.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:50.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:50.154 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:50.177 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:50.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:50.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:50.282 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:50.318 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:50.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:50.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:50.421 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:50.457 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:50.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:50.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:50.564 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:50.601 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:50.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:50.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:50.651 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T05:26:50.672 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 87 16 100502 100486 72.1772 50.6875 0.0107255 0.0138481 2023-10-09T05:26:50.707 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:50.734 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:50.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:50.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:50.768 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T05:26:50.837 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:50.889 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:50.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:50.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:50.995 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:51.017 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:51.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:51.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:51.121 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:51.145 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:51.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:51.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:51.252 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:51.276 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:26:51.278 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:51.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:51.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:51.381 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:51.415 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:51.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:51.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:51.523 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:51.536 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:51 smithi133 bash[102858]: cluster 2023-10-09T05:26:50.177600+0000 mgr.y (mgr.25277) 1158 : cluster 0 pgmap v1234: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 49 MiB/s wr, 785 op/s 2023-10-09T05:26:51.536 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:51 smithi133 bash[105589]: cluster 2023-10-09T05:26:50.177600+0000 mgr.y (mgr.25277) 1158 : cluster 0 pgmap v1234: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 49 MiB/s wr, 785 op/s 2023-10-09T05:26:51.553 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:51.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:51.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:51.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:51 smithi194 bash[92666]: cluster 2023-10-09T05:26:50.177600+0000 mgr.y (mgr.25277) 1158 : cluster 0 pgmap v1234: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 49 MiB/s wr, 785 op/s 2023-10-09T05:26:51.657 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:51.672 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 88 16 101222 101206 71.8684 45 0.0236367 0.013909 2023-10-09T05:26:51.682 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:51.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:51.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:51.788 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:51.817 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:51.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:51.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:51.922 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:51.951 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:51.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:51.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:52.057 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:52.094 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:52.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:52.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:52.200 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:52.229 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:52.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:52.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:52.335 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:52.358 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:52.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:52.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:52.462 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:52.496 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:52.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:52.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:52.603 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:52.640 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:52.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:52.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:52.672 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 89 16 102005 101989 71.6107 48.9375 0.026887 0.0139591 2023-10-09T05:26:52.746 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:52.749 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:26:52.781 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:52.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:52.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:52.887 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:52.910 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:52.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:52.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:53.016 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:53.043 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:53.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:53.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:53.147 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:53.173 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:53.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:53.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:53.277 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:53 smithi194 bash[92666]: cluster 2023-10-09T05:26:52.178093+0000 mgr.y (mgr.25277) 1159 : cluster 0 pgmap v1235: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 48 MiB/s wr, 771 op/s 2023-10-09T05:26:53.281 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:53.307 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:53.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:53.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:53.417 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:53.450 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:53.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:53.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:53.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:53 smithi133 bash[105589]: cluster 2023-10-09T05:26:52.178093+0000 mgr.y (mgr.25277) 1159 : cluster 0 pgmap v1235: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 48 MiB/s wr, 771 op/s 2023-10-09T05:26:53.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:53 smithi133 bash[102858]: cluster 2023-10-09T05:26:52.178093+0000 mgr.y (mgr.25277) 1159 : cluster 0 pgmap v1235: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 48 MiB/s wr, 771 op/s 2023-10-09T05:26:53.553 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:53.585 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:53.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:53.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:53.672 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Total time run: 90.0085 2023-10-09T05:26:53.673 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Total writes made: 102739 2023-10-09T05:26:53.673 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Write size: 65536 2023-10-09T05:26:53.673 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Object size: 65536 2023-10-09T05:26:53.673 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Bandwidth (MB/sec): 71.3398 2023-10-09T05:26:53.673 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Stddev Bandwidth: 27.9561 2023-10-09T05:26:53.673 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Max bandwidth (MB/sec): 176.375 2023-10-09T05:26:53.673 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Min bandwidth (MB/sec): 39.25 2023-10-09T05:26:53.674 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Average IOPS: 1141 2023-10-09T05:26:53.674 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Stddev IOPS: 447.297 2023-10-09T05:26:53.674 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Max IOPS: 2822 2023-10-09T05:26:53.674 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Min IOPS: 628 2023-10-09T05:26:53.674 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Average Latency(s): 0.0140131 2023-10-09T05:26:53.674 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Stddev Latency(s): 0.0136018 2023-10-09T05:26:53.674 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Max latency(s): 0.27335 2023-10-09T05:26:53.674 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Min latency(s): 0.00116799 2023-10-09T05:26:53.678 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T05:26:53.691 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:53.725 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:53.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:53.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:53.832 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:53.866 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:53.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:53.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:53.971 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:53.998 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:54.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:54.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:54.101 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:54.142 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:54.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:54.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:54.250 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:54.277 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:54.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:54.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:54.381 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:54.406 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:54.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:54.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:54.516 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:54.541 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:54.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:54.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:54.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:26:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:26:54] "GET /metrics HTTP/1.1" 200 36476 "" "Prometheus/2.43.0" 2023-10-09T05:26:54.648 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:54.673 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:54.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:54.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:54.777 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:54.785 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:54 smithi194 bash[92666]: cluster 2023-10-09T05:26:54.178601+0000 mgr.y (mgr.25277) 1160 : cluster 0 pgmap v1236: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 51 MiB/s wr, 814 op/s 2023-10-09T05:26:54.801 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:54.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:54.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:54.908 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:54.918 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:54 smithi133 bash[102858]: cluster 2023-10-09T05:26:54.178601+0000 mgr.y (mgr.25277) 1160 : cluster 0 pgmap v1236: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 51 MiB/s wr, 814 op/s 2023-10-09T05:26:54.918 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:54 smithi133 bash[105589]: cluster 2023-10-09T05:26:54.178601+0000 mgr.y (mgr.25277) 1160 : cluster 0 pgmap v1236: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 51 MiB/s wr, 814 op/s 2023-10-09T05:26:54.943 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:54.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:54.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:55.048 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:55.083 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:55.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:55.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:55.192 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:55.220 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:55.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:55.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:55.325 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:55.362 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:55.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:55.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:55.467 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:55.508 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:55.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:55.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:55.537 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:26:55.537 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:26:55.537 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:26:55.615 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:55.656 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:55.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:55.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:55.760 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:55.785 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:55.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:55.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:55.889 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:55.918 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:55.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:55.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:56.022 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:56.055 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:56.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:56.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:56.158 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:56.204 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:56.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:56.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:56.309 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:56.337 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:56.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:56.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:56.441 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:56.470 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:56.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:56.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:56.576 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:56.606 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:56.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:56.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:56.710 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:56.743 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:56.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:56.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:56.849 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:56.870 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:56.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:56.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:56.973 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:57.007 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:57.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:57.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:57.110 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:57.132 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:57.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:57.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:57.234 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:57 smithi133 bash[102858]: cluster 2023-10-09T05:26:56.179374+0000 mgr.y (mgr.25277) 1161 : cluster 0 pgmap v1237: 89 pgs: 89 active+clean; 6.2 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 43 MiB/s wr, 733 op/s 2023-10-09T05:26:57.235 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:57.264 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:57.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:57.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:57.367 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:57.376 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:57 smithi194 bash[92666]: cluster 2023-10-09T05:26:56.179374+0000 mgr.y (mgr.25277) 1161 : cluster 0 pgmap v1237: 89 pgs: 89 active+clean; 6.2 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 43 MiB/s wr, 733 op/s 2023-10-09T05:26:57.392 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:57.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:57.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:57.500 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:57.507 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:57 smithi133 bash[105589]: cluster 2023-10-09T05:26:56.179374+0000 mgr.y (mgr.25277) 1161 : cluster 0 pgmap v1237: 89 pgs: 89 active+clean; 6.2 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 43 MiB/s wr, 733 op/s 2023-10-09T05:26:57.527 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:26:57.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:57.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:26:57.591 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:26:57.636 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:57.662 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:57.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:57.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:57.765 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:57.789 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:57.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:57.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:57.898 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:57.934 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:26:57.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:57.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:26:58.037 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:58.082 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:58.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:58.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:58.189 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:58.225 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:58.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:58.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:58.329 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:58.352 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:58.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:58.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:58.458 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:58.498 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:26:58.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:58.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:26:58.602 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:58.632 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:58.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:58.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:58.737 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:58.763 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:58.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:58.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:58.868 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:58.905 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:58.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:58.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:59.009 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:59.046 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:59.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:59.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:59.152 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:59.185 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:59.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:59.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:26:59.289 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:59.315 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:59.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:59.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:59.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:59 smithi194 bash[92666]: cluster 2023-10-09T05:26:58.179849+0000 mgr.y (mgr.25277) 1162 : cluster 0 pgmap v1238: 89 pgs: 89 active+clean; 6.2 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 85 B/s rd, 37 MiB/s wr, 656 op/s 2023-10-09T05:26:59.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:26:59 smithi194 bash[92666]: audit 2023-10-09T05:26:58.473843+0000 mon.a (mon.0) 1504 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:26:59.419 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:59.443 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:26:59.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:59.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:26:59.546 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:59.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:59 smithi133 bash[102858]: cluster 2023-10-09T05:26:58.179849+0000 mgr.y (mgr.25277) 1162 : cluster 0 pgmap v1238: 89 pgs: 89 active+clean; 6.2 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 85 B/s rd, 37 MiB/s wr, 656 op/s 2023-10-09T05:26:59.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:26:59 smithi133 bash[102858]: audit 2023-10-09T05:26:58.473843+0000 mon.a (mon.0) 1504 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:26:59.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:59 smithi133 bash[105589]: cluster 2023-10-09T05:26:58.179849+0000 mgr.y (mgr.25277) 1162 : cluster 0 pgmap v1238: 89 pgs: 89 active+clean; 6.2 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 85 B/s rd, 37 MiB/s wr, 656 op/s 2023-10-09T05:26:59.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:26:59 smithi133 bash[105589]: audit 2023-10-09T05:26:58.473843+0000 mon.a (mon.0) 1504 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:26:59.573 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:26:59.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:59.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:26:59.679 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:59.702 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:26:59.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:59.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:26:59.808 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:59.840 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:26:59.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:59.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:26:59.943 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:59.969 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:26:59.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:26:59.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:00.072 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:00.095 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:00.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:00.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:00.198 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:00.224 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:00.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:00.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:00.331 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:00.361 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:00.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:00.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:00.464 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:00.490 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:00.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:00.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:00.595 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:00.620 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:00.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:00.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:00.724 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:00.756 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:00.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:00.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:00.861 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:00.889 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:00.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:00.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:00.996 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:01.025 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:01.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:01.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:01.129 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:01.165 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:01.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:01.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:01.274 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:01.311 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:01.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:01.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:01.414 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:01.418 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:01 smithi133 bash[105589]: cluster 2023-10-09T05:27:00.180750+0000 mgr.y (mgr.25277) 1163 : cluster 0 pgmap v1239: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 85 B/s rd, 35 MiB/s wr, 897 op/s 2023-10-09T05:27:01.418 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:01 smithi133 bash[102858]: cluster 2023-10-09T05:27:00.180750+0000 mgr.y (mgr.25277) 1163 : cluster 0 pgmap v1239: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 85 B/s rd, 35 MiB/s wr, 897 op/s 2023-10-09T05:27:01.442 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:01.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:01.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:01.546 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:01.554 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:01 smithi194 bash[92666]: cluster 2023-10-09T05:27:00.180750+0000 mgr.y (mgr.25277) 1163 : cluster 0 pgmap v1239: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 85 B/s rd, 35 MiB/s wr, 897 op/s 2023-10-09T05:27:01.572 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:01.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:01.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:01.677 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:01.701 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:01.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:01.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:01.804 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:01.835 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:01.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:01.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:01.940 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:01.970 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:01.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:01.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:02.073 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:02.101 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:02.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:02.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:02.207 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:02.239 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:02.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:02.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:02.345 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:02.379 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:02.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:02.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:02.485 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:02.508 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:02.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:02.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:02.612 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:02.637 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:02.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:02.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:02.745 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:02.769 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:02.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:02.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:02.874 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:02.902 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:02.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:02.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:03.006 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:03.041 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:03.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:03.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:03.148 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:03.187 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:03.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:03.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:03.291 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:03.318 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:03.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:03.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:03.424 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:03.432 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:03 smithi133 bash[102858]: cluster 2023-10-09T05:27:02.181182+0000 mgr.y (mgr.25277) 1164 : cluster 0 pgmap v1240: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 85 B/s rd, 21 MiB/s wr, 726 op/s 2023-10-09T05:27:03.432 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:03 smithi133 bash[105589]: cluster 2023-10-09T05:27:02.181182+0000 mgr.y (mgr.25277) 1164 : cluster 0 pgmap v1240: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 85 B/s rd, 21 MiB/s wr, 726 op/s 2023-10-09T05:27:03.448 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:03.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:03.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:03.551 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:03.580 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:03.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:03.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:03.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:03 smithi194 bash[92666]: cluster 2023-10-09T05:27:02.181182+0000 mgr.y (mgr.25277) 1164 : cluster 0 pgmap v1240: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 85 B/s rd, 21 MiB/s wr, 726 op/s 2023-10-09T05:27:03.688 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:03.711 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:03.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:03.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:03.814 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:03.849 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:03.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:03.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:03.952 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:03.983 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:03.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:03.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:04.091 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:04.119 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:04.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:04.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:04.224 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:04.263 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:04.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:04.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:04.368 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:04.405 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:04.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:04.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:04.509 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:04.546 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:04.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:04.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:04.617 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:27:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:27:04] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.43.0" 2023-10-09T05:27:04.650 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:04.697 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:04.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:04.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:04.803 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:04.832 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:04.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:04.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:04.940 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:04.962 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:04 smithi133 bash[102858]: cluster 2023-10-09T05:27:04.181811+0000 mgr.y (mgr.25277) 1165 : cluster 0 pgmap v1241: 89 pgs: 89 active+clean; 5.9 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 85 B/s rd, 15 MiB/s wr, 731 op/s 2023-10-09T05:27:04.963 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:04 smithi133 bash[105589]: cluster 2023-10-09T05:27:04.181811+0000 mgr.y (mgr.25277) 1165 : cluster 0 pgmap v1241: 89 pgs: 89 active+clean; 5.9 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 85 B/s rd, 15 MiB/s wr, 731 op/s 2023-10-09T05:27:04.968 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:04.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:04.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:05.072 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:05.079 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:04 smithi194 bash[92666]: cluster 2023-10-09T05:27:04.181811+0000 mgr.y (mgr.25277) 1165 : cluster 0 pgmap v1241: 89 pgs: 89 active+clean; 5.9 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 85 B/s rd, 15 MiB/s wr, 731 op/s 2023-10-09T05:27:05.097 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:05.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:05.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:05.203 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:05.236 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:05.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:05.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:05.339 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:05.368 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:05.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:05.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:05.472 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:05.612 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:05.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:05.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:05.719 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:05.753 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:05.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:05.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:05.856 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:05.876 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:05.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:05.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:05.979 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:06.006 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:06.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:06.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:06.112 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:06.138 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:06.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:06.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:06.247 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:06.267 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:06.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:06.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:06.370 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:06.397 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:06.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:06.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:06.500 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:06.524 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:06.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:06.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:06.628 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:06.652 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:06.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:06.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:06.762 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:06.790 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:06.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:06.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:06.893 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:06.936 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:06.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:06.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:07.045 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:07.082 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:07.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:07.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:07.186 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:07.224 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:07.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:07.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:07.329 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:07.355 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:07.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:07.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:07.458 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:07.466 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:07 smithi133 bash[102858]: cluster 2023-10-09T05:27:06.182375+0000 mgr.y (mgr.25277) 1166 : cluster 0 pgmap v1242: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 85 B/s rd, 10 MiB/s wr, 835 op/s 2023-10-09T05:27:07.467 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:07 smithi133 bash[102858]: audit 2023-10-09T05:27:06.736643+0000 mon.a (mon.0) 1505 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:27:07.467 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:07 smithi133 bash[105589]: cluster 2023-10-09T05:27:06.182375+0000 mgr.y (mgr.25277) 1166 : cluster 0 pgmap v1242: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 85 B/s rd, 10 MiB/s wr, 835 op/s 2023-10-09T05:27:07.467 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:07 smithi133 bash[105589]: audit 2023-10-09T05:27:06.736643+0000 mon.a (mon.0) 1505 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:27:07.482 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:07.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:07.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:07.585 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:07.618 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:07.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:07.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:07.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:07 smithi194 bash[92666]: cluster 2023-10-09T05:27:06.182375+0000 mgr.y (mgr.25277) 1166 : cluster 0 pgmap v1242: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 85 B/s rd, 10 MiB/s wr, 835 op/s 2023-10-09T05:27:07.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:07 smithi194 bash[92666]: audit 2023-10-09T05:27:06.736643+0000 mon.a (mon.0) 1505 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:27:07.726 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:07.762 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:07.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:07.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:07.867 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:07.899 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:07.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:07.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:08.009 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:08.037 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:08.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:08.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:08.140 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:08.166 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:08.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:08.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:08.274 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:08.305 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:08.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:08.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:08.413 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:08.444 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:08.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:08.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:08.547 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:08.577 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:08.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:08.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:08.682 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:08.729 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:08.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:08.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:08.834 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:08.860 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:08.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:08.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:08.964 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:08.994 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:08.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:08.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:09.100 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:09.129 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:09.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:09.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:09.236 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:09.269 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:09.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:09.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:09.373 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:09.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:09.410 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:09.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:09.511 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:09.523 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:09 smithi133 bash[102858]: cluster 2023-10-09T05:27:08.182821+0000 mgr.y (mgr.25277) 1167 : cluster 0 pgmap v1243: 89 pgs: 89 active+clean; 5.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 1.1 MiB/s wr, 693 op/s 2023-10-09T05:27:09.537 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:09.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:09.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:09.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:09 smithi194 bash[92666]: cluster 2023-10-09T05:27:08.182821+0000 mgr.y (mgr.25277) 1167 : cluster 0 pgmap v1243: 89 pgs: 89 active+clean; 5.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 1.1 MiB/s wr, 693 op/s 2023-10-09T05:27:09.641 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:09.675 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:09.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:09.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:09.779 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:09.788 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:09 smithi133 bash[105589]: cluster 2023-10-09T05:27:08.182821+0000 mgr.y (mgr.25277) 1167 : cluster 0 pgmap v1243: 89 pgs: 89 active+clean; 5.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 1.1 MiB/s wr, 693 op/s 2023-10-09T05:27:09.807 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:09.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:09.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:09.915 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:09.964 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:09.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:09.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:10.074 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:10.119 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:10.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:10.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:10.224 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:10.263 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:10.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:10.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:10.367 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:10.415 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:10.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:10.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:10.521 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:10.553 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:10.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:10.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:10.659 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:10.690 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:10.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:10.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:10.793 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:10.825 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:10.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:10.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:10.930 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:10.960 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:10.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:10.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:11.068 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:11.101 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:11.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:11.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:11.206 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:11.233 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:11.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:11.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:11.250 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:11 smithi133 bash[102858]: cluster 2023-10-09T05:27:10.183505+0000 mgr.y (mgr.25277) 1168 : cluster 0 pgmap v1244: 89 pgs: 89 active+clean; 5.5 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 149 KiB/s wr, 926 op/s 2023-10-09T05:27:11.339 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:11.363 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:11.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:11.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:11.468 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:11.481 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:11 smithi194 bash[92666]: cluster 2023-10-09T05:27:10.183505+0000 mgr.y (mgr.25277) 1168 : cluster 0 pgmap v1244: 89 pgs: 89 active+clean; 5.5 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 149 KiB/s wr, 926 op/s 2023-10-09T05:27:11.506 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:11.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:11.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:11.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:11 smithi133 bash[105589]: cluster 2023-10-09T05:27:10.183505+0000 mgr.y (mgr.25277) 1168 : cluster 0 pgmap v1244: 89 pgs: 89 active+clean; 5.5 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 149 KiB/s wr, 926 op/s 2023-10-09T05:27:11.608 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:11.635 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:11.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:11.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:11.739 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:11.770 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:11.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:11.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:11.875 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:11.927 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:11.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:11.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:12.033 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:12.065 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:12.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:12.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:12.174 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:12.220 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:12.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:12.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:12.324 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:12.350 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:12.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:12.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:12.454 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:12.486 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:12.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:12.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:12.594 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:12.621 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:12.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:12.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:12.724 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:12.756 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:12.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:12.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:12.827 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:27:12.860 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:12.872 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:27:12.910 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:12.915 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:27:12.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:12.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:12.969 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:27:13.002 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:27:13.020 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:13.040 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:27:13.051 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:13.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:13.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:13.075 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:27:13.137 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:27:13.145 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:27:13.145 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:27:13.145 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:27:13.145 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:27:13.145 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:27:13.145 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:27:13.145 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:27:13.145 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:27:13.155 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:13.184 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:13.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:13.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:13.290 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:13.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:13 smithi133 bash[102858]: cluster 2023-10-09T05:27:12.183976+0000 mgr.y (mgr.25277) 1169 : cluster 0 pgmap v1245: 89 pgs: 89 active+clean; 5.5 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 680 op/s 2023-10-09T05:27:13.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:13 smithi133 bash[102858]: audit 2023-10-09T05:27:12.771181+0000 mon.b (mon.1) 24 : audit 0 from='client.? 172.21.15.133:0/876085951' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:27:13.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:13 smithi133 bash[105589]: cluster 2023-10-09T05:27:12.183976+0000 mgr.y (mgr.25277) 1169 : cluster 0 pgmap v1245: 89 pgs: 89 active+clean; 5.5 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 680 op/s 2023-10-09T05:27:13.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:13 smithi133 bash[105589]: audit 2023-10-09T05:27:12.771181+0000 mon.b (mon.1) 24 : audit 0 from='client.? 172.21.15.133:0/876085951' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:27:13.317 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:13.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:13.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:13.423 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:13.454 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:13.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:13.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:13.557 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:13.595 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:13.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:13.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:13.639 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:13 smithi194 bash[92666]: cluster 2023-10-09T05:27:12.183976+0000 mgr.y (mgr.25277) 1169 : cluster 0 pgmap v1245: 89 pgs: 89 active+clean; 5.5 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 680 op/s 2023-10-09T05:27:13.639 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:13 smithi194 bash[92666]: audit 2023-10-09T05:27:12.771181+0000 mon.b (mon.1) 24 : audit 0 from='client.? 172.21.15.133:0/876085951' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:27:13.703 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:13.731 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:13.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:13.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:13.838 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:13.862 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:13.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:13.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:13.967 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:13.994 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:13.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:13.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:14.099 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:14.132 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:14.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:14.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:14.245 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:14.275 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:14.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:14.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:14.383 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:14.404 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:14.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:14.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:14.433 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:27:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:27:14] "GET /metrics HTTP/1.1" 200 36479 "" "Prometheus/2.43.0" 2023-10-09T05:27:14.492 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:27:14.492 INFO:teuthology.orchestra.run.smithi133.stdout: "ops": [ 2023-10-09T05:27:14.493 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.493 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_repop(client.45374.0:118530 12.3 e275/264 12:c5657507:::benchmark_data_smithi133_218956_object15788:head v 275'7345, mlcod=275'7342)", 2023-10-09T05:27:14.493 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:27:14.453120+0000", 2023-10-09T05:27:14.493 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.035882997, 2023-10-09T05:27:14.493 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.035909313999999998, 2023-10-09T05:27:14.493 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:27:14.493 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "started", 2023-10-09T05:27:14.493 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:27:14.493 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.493 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:27:14.494 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.453120+0000", 2023-10-09T05:27:14.494 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:27:14.494 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.494 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.494 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:27:14.494 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.453120+0000", 2023-10-09T05:27:14.494 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:27:14.494 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.494 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.495 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:27:14.495 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.453119+0000", 2023-10-09T05:27:14.495 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:27:14.495 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.495 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.495 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:27:14.495 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.453128+0000", 2023-10-09T05:27:14.495 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.3330000000000003e-06 2023-10-09T05:27:14.495 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.495 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.496 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:27:14.496 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.453147+0000", 2023-10-09T05:27:14.496 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.9360000000000001e-05 2023-10-09T05:27:14.496 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.496 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.496 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:27:14.496 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.453157+0000", 2023-10-09T05:27:14.496 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.3109999999999995e-06 2023-10-09T05:27:14.496 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.497 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.497 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:27:14.497 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.460312+0000", 2023-10-09T05:27:14.497 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0071551649999999998 2023-10-09T05:27:14.497 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.497 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.497 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:27:14.497 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.460468+0000", 2023-10-09T05:27:14.498 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000156098 2023-10-09T05:27:14.498 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:27:14.498 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:27:14.498 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:27:14.498 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.498 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.498 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45374.0:118540 12.b 12.1ad7dceb (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e275)", 2023-10-09T05:27:14.498 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:27:14.460047+0000", 2023-10-09T05:27:14.498 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.028955705000000002, 2023-10-09T05:27:14.498 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.029030174999999998, 2023-10-09T05:27:14.498 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:27:14.499 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "queued for pg", 2023-10-09T05:27:14.499 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:27:14.499 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45374", 2023-10-09T05:27:14.499 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2988063385", 2023-10-09T05:27:14.499 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 118540 2023-10-09T05:27:14.499 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.499 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:27:14.500 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.500 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:27:14.500 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.460047+0000", 2023-10-09T05:27:14.500 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:27:14.501 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.501 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.501 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:27:14.501 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.460047+0000", 2023-10-09T05:27:14.501 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:27:14.501 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.502 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.502 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:27:14.502 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.460045+0000", 2023-10-09T05:27:14.502 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:27:14.502 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.503 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.503 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:27:14.503 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.460052+0000", 2023-10-09T05:27:14.503 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.6200000000000001e-06 2023-10-09T05:27:14.503 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.503 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.503 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:27:14.504 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.460065+0000", 2023-10-09T05:27:14.504 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.2573e-05 2023-10-09T05:27:14.504 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.504 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.504 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:27:14.504 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.460074+0000", 2023-10-09T05:27:14.505 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.2359999999999993e-06 2023-10-09T05:27:14.505 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:27:14.505 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:27:14.505 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:27:14.505 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.505 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.506 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_repop(client.45374.0:118539 12.3 e275/264 12:c4410484:::benchmark_data_smithi133_218956_object15797:head v 275'7346, mlcod=275'7343)", 2023-10-09T05:27:14.506 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:27:14.463092+0000", 2023-10-09T05:27:14.506 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.025910737, 2023-10-09T05:27:14.506 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.026026462, 2023-10-09T05:27:14.506 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:27:14.506 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "started", 2023-10-09T05:27:14.507 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:27:14.507 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.507 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:27:14.507 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.463092+0000", 2023-10-09T05:27:14.507 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:27:14.507 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.507 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.508 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:27:14.508 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.463092+0000", 2023-10-09T05:27:14.508 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:27:14.508 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.508 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.508 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:27:14.508 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.463091+0000", 2023-10-09T05:27:14.508 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:27:14.509 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.509 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.509 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:27:14.509 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.463099+0000", 2023-10-09T05:27:14.509 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.9850000000000006e-06 2023-10-09T05:27:14.509 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.509 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.509 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:27:14.510 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.463115+0000", 2023-10-09T05:27:14.510 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.5577999999999999e-05 2023-10-09T05:27:14.510 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.510 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.510 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:27:14.510 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.463122+0000", 2023-10-09T05:27:14.510 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.4109999999999999e-06 2023-10-09T05:27:14.511 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.511 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.511 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:27:14.511 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.472092+0000", 2023-10-09T05:27:14.511 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0089700370000000001 2023-10-09T05:27:14.511 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.511 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.512 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:27:14.512 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.472156+0000", 2023-10-09T05:27:14.512 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.4196999999999995e-05 2023-10-09T05:27:14.512 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:27:14.512 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:27:14.513 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:27:14.513 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.513 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.513 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45374.0:118497 12.b 12:d8ec4bc8:::benchmark_data_smithi133_218956_object15755:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e275)", 2023-10-09T05:27:14.513 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:27:14.384557+0000", 2023-10-09T05:27:14.513 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.10444574500000001, 2023-10-09T05:27:14.513 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.104612661, 2023-10-09T05:27:14.513 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:27:14.513 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:27:14.514 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:27:14.514 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45374", 2023-10-09T05:27:14.514 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2988063385", 2023-10-09T05:27:14.514 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 118497 2023-10-09T05:27:14.514 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.514 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:27:14.514 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.514 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:27:14.514 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.384557+0000", 2023-10-09T05:27:14.514 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:27:14.514 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.515 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.515 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:27:14.515 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.384557+0000", 2023-10-09T05:27:14.515 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:27:14.515 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.515 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.515 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:27:14.515 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.384555+0000", 2023-10-09T05:27:14.515 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:27:14.515 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.515 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.515 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:27:14.516 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.384563+0000", 2023-10-09T05:27:14.516 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.4070000000000004e-06 2023-10-09T05:27:14.516 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.516 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.516 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:27:14.516 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.384584+0000", 2023-10-09T05:27:14.516 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.0876000000000001e-05 2023-10-09T05:27:14.516 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.516 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.516 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:27:14.516 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.384592+0000", 2023-10-09T05:27:14.516 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.3769999999999996e-06 2023-10-09T05:27:14.517 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.517 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.517 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:27:14.517 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.448166+0000", 2023-10-09T05:27:14.517 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.063573928000000002 2023-10-09T05:27:14.517 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.517 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.517 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:27:14.517 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.448459+0000", 2023-10-09T05:27:14.517 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00029313700000000001 2023-10-09T05:27:14.518 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.518 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.518 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:27:14.518 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.448923+0000", 2023-10-09T05:27:14.518 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00046329900000000002 2023-10-09T05:27:14.518 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.518 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.518 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:27:14.518 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.468371+0000", 2023-10-09T05:27:14.518 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.019448706 2023-10-09T05:27:14.519 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.519 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.519 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:27:14.519 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.477392+0000", 2023-10-09T05:27:14.519 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0090210670000000007 2023-10-09T05:27:14.519 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:27:14.519 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:27:14.519 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:27:14.519 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.519 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.520 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_repop_reply(client.45374.0:118497 12.b e275/264)", 2023-10-09T05:27:14.520 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:27:14.481064+0000", 2023-10-09T05:27:14.520 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.0079389950000000008, 2023-10-09T05:27:14.520 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0081613740000000008, 2023-10-09T05:27:14.520 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:27:14.520 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "queued for pg", 2023-10-09T05:27:14.520 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:27:14.520 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.520 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:27:14.520 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.481064+0000", 2023-10-09T05:27:14.520 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:27:14.520 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.521 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.521 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:27:14.521 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.481064+0000", 2023-10-09T05:27:14.521 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:27:14.521 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.521 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.521 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:27:14.521 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.481063+0000", 2023-10-09T05:27:14.521 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:27:14.521 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.521 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.521 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:27:14.522 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.481071+0000", 2023-10-09T05:27:14.522 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.0139999999999998e-06 2023-10-09T05:27:14.522 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.522 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.522 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:27:14.522 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.481089+0000", 2023-10-09T05:27:14.522 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.8788e-05 2023-10-09T05:27:14.522 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.522 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.522 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:27:14.522 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.481100+0000", 2023-10-09T05:27:14.522 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.0659000000000001e-05 2023-10-09T05:27:14.523 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:27:14.523 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:27:14.523 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:27:14.523 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.523 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.523 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_repop_reply(client.45374.0:118500 12.b e275/264)", 2023-10-09T05:27:14.523 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:27:14.481110+0000", 2023-10-09T05:27:14.523 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.007892764, 2023-10-09T05:27:14.523 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0081415819999999996, 2023-10-09T05:27:14.523 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:27:14.523 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "queued for pg", 2023-10-09T05:27:14.523 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:27:14.524 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.524 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:27:14.524 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.481110+0000", 2023-10-09T05:27:14.524 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:27:14.524 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.524 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.524 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:27:14.524 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.481110+0000", 2023-10-09T05:27:14.524 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:27:14.524 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.524 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.524 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:27:14.524 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.481110+0000", 2023-10-09T05:27:14.525 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999995 2023-10-09T05:27:14.525 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.525 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.525 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:27:14.525 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.481113+0000", 2023-10-09T05:27:14.525 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.1580000000000001e-06 2023-10-09T05:27:14.525 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.525 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.526 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:27:14.526 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.481120+0000", 2023-10-09T05:27:14.526 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.0809999999999996e-06 2023-10-09T05:27:14.526 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.526 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.526 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:27:14.526 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.481125+0000", 2023-10-09T05:27:14.526 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.8280000000000002e-06 2023-10-09T05:27:14.526 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:27:14.527 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:27:14.527 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:27:14.527 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.527 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.527 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_repop(client.45374.0:118543 12.1 e275/264 12:89953c4b:::benchmark_data_smithi133_218956_object15801:head v 275'7346, mlcod=275'7344)", 2023-10-09T05:27:14.527 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:27:14.481186+0000", 2023-10-09T05:27:14.527 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.0078161910000000001, 2023-10-09T05:27:14.527 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0080971740000000004, 2023-10-09T05:27:14.527 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:27:14.527 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "started", 2023-10-09T05:27:14.528 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:27:14.528 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.528 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:27:14.528 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.481186+0000", 2023-10-09T05:27:14.528 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:27:14.528 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.528 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.528 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:27:14.528 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.481186+0000", 2023-10-09T05:27:14.528 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:27:14.529 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.529 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.529 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:27:14.529 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.481186+0000", 2023-10-09T05:27:14.529 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999995 2023-10-09T05:27:14.529 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.529 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.529 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:27:14.529 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.481193+0000", 2023-10-09T05:27:14.529 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.0999999999999998e-06 2023-10-09T05:27:14.530 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.530 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.530 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:27:14.530 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.481202+0000", 2023-10-09T05:27:14.530 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.9269999999999994e-06 2023-10-09T05:27:14.530 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.530 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.530 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:27:14.530 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.481208+0000", 2023-10-09T05:27:14.531 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.6520000000000003e-06 2023-10-09T05:27:14.531 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.531 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.531 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:27:14.531 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.484935+0000", 2023-10-09T05:27:14.531 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0037270630000000001 2023-10-09T05:27:14.531 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.531 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.531 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:27:14.531 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.484960+0000", 2023-10-09T05:27:14.532 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.5536000000000001e-05 2023-10-09T05:27:14.532 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:27:14.532 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:27:14.532 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:27:14.532 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.532 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.532 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45374.0:118546 12.8 12:1f50bc51:::benchmark_data_smithi133_218956_object15804:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e275)", 2023-10-09T05:27:14.532 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:27:14.481238+0000", 2023-10-09T05:27:14.532 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.0077649759999999998, 2023-10-09T05:27:14.532 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0080830620000000002, 2023-10-09T05:27:14.533 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:27:14.533 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:27:14.533 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:27:14.533 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45374", 2023-10-09T05:27:14.533 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2988063385", 2023-10-09T05:27:14.533 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 118546 2023-10-09T05:27:14.533 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.533 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:27:14.533 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.533 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:27:14.533 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.481238+0000", 2023-10-09T05:27:14.533 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:27:14.534 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.534 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.534 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:27:14.534 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.481238+0000", 2023-10-09T05:27:14.534 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:27:14.534 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.534 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.534 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:27:14.534 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.481236+0000", 2023-10-09T05:27:14.534 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:27:14.535 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.535 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.535 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:27:14.535 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.481241+0000", 2023-10-09T05:27:14.535 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.7509999999999999e-06 2023-10-09T05:27:14.535 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.535 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.535 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:27:14.535 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.481250+0000", 2023-10-09T05:27:14.536 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.1900000000000001e-06 2023-10-09T05:27:14.536 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.536 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.536 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:27:14.536 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.481257+0000", 2023-10-09T05:27:14.537 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.5370000000000003e-06 2023-10-09T05:27:14.537 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.537 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.537 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:27:14.537 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.481392+0000", 2023-10-09T05:27:14.537 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000135057 2023-10-09T05:27:14.537 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.537 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.538 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:27:14.538 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.481531+0000", 2023-10-09T05:27:14.538 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000139696 2023-10-09T05:27:14.538 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.538 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.538 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:27:14.539 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.481660+0000", 2023-10-09T05:27:14.539 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00012833000000000001 2023-10-09T05:27:14.539 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:27:14.539 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:27:14.539 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:27:14.539 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.540 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.540 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45374.0:118500 12.b 12:d809c0e7:::benchmark_data_smithi133_218956_object15758:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e275)", 2023-10-09T05:27:14.540 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:27:14.394557+0000", 2023-10-09T05:27:14.540 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.094445235000000002, 2023-10-09T05:27:14.540 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.094829697000000004, 2023-10-09T05:27:14.541 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:27:14.541 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:27:14.541 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:27:14.541 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45374", 2023-10-09T05:27:14.541 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2988063385", 2023-10-09T05:27:14.542 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 118500 2023-10-09T05:27:14.542 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.542 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:27:14.542 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.543 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:27:14.543 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.394557+0000", 2023-10-09T05:27:14.543 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:27:14.543 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.543 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.544 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:27:14.544 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.394557+0000", 2023-10-09T05:27:14.544 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:27:14.544 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.545 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.545 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:27:14.545 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.394556+0000", 2023-10-09T05:27:14.545 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:27:14.546 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.546 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.546 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:27:14.547 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.394564+0000", 2023-10-09T05:27:14.547 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.8099999999999998e-06 2023-10-09T05:27:14.547 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.547 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.547 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:27:14.548 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.394576+0000", 2023-10-09T05:27:14.548 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.2224e-05 2023-10-09T05:27:14.548 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.548 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.549 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:27:14.549 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.394584+0000", 2023-10-09T05:27:14.549 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.9689999999999994e-06 2023-10-09T05:27:14.549 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.549 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.550 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:27:14.550 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.458183+0000", 2023-10-09T05:27:14.550 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.063599618999999996 2023-10-09T05:27:14.550 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.550 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.550 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:27:14.551 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.458383+0000", 2023-10-09T05:27:14.551 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000199996 2023-10-09T05:27:14.551 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.551 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.551 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:27:14.551 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.458556+0000", 2023-10-09T05:27:14.552 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00017274599999999999 2023-10-09T05:27:14.552 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.552 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.552 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:27:14.552 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.468559+0000", 2023-10-09T05:27:14.552 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.010002624999999999 2023-10-09T05:27:14.552 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.553 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.553 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:27:14.553 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.489236+0000", 2023-10-09T05:27:14.553 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.020676752 2023-10-09T05:27:14.553 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:27:14.553 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:27:14.553 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:27:14.554 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.554 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.554 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_repop_reply(client.45374.0:118504 12.b e275/264)", 2023-10-09T05:27:14.554 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:27:14.485177+0000", 2023-10-09T05:27:14.554 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.0038257159999999998, 2023-10-09T05:27:14.554 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.004260977, 2023-10-09T05:27:14.554 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:27:14.555 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "queued for pg", 2023-10-09T05:27:14.555 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:27:14.555 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.555 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:27:14.555 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.485177+0000", 2023-10-09T05:27:14.555 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:27:14.555 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.556 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.556 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:27:14.556 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.485177+0000", 2023-10-09T05:27:14.556 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:27:14.556 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.556 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.556 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:27:14.556 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.485176+0000", 2023-10-09T05:27:14.556 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:27:14.556 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.557 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.557 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:27:14.557 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.485183+0000", 2023-10-09T05:27:14.557 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.1849999999999998e-06 2023-10-09T05:27:14.557 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.557 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.557 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:27:14.557 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.485196+0000", 2023-10-09T05:27:14.557 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.3084e-05 2023-10-09T05:27:14.557 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.557 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.557 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:27:14.558 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.485205+0000", 2023-10-09T05:27:14.558 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.5299999999999996e-06 2023-10-09T05:27:14.558 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:27:14.558 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:27:14.559 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:27:14.559 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.559 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.560 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_repop_reply(client.45374.0:118546 12.8 e275/264)", 2023-10-09T05:27:14.560 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:27:14.486350+0000", 2023-10-09T05:27:14.560 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.0026523610000000002, 2023-10-09T05:27:14.560 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0031135109999999998, 2023-10-09T05:27:14.560 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:27:14.560 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "queued for pg", 2023-10-09T05:27:14.560 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:27:14.560 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.560 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:27:14.560 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.486350+0000", 2023-10-09T05:27:14.560 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:27:14.560 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.561 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.561 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:27:14.561 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.486350+0000", 2023-10-09T05:27:14.561 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:27:14.561 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.561 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.561 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:27:14.561 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.486349+0000", 2023-10-09T05:27:14.561 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:27:14.561 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.561 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.562 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:27:14.562 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.486356+0000", 2023-10-09T05:27:14.562 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.0249999999999997e-06 2023-10-09T05:27:14.562 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.562 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.562 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:27:14.562 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.486370+0000", 2023-10-09T05:27:14.563 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.3898e-05 2023-10-09T05:27:14.563 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.563 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.563 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:27:14.563 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.486379+0000", 2023-10-09T05:27:14.563 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.3430000000000002e-06 2023-10-09T05:27:14.563 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:27:14.564 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:27:14.564 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:27:14.564 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.564 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.564 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45374.0:118504 12.b 12:d96250aa:::benchmark_data_smithi133_218956_object15762:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e275)", 2023-10-09T05:27:14.564 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:27:14.395282+0000", 2023-10-09T05:27:14.564 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.093720226000000004, 2023-10-09T05:27:14.564 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.094212276999999997, 2023-10-09T05:27:14.564 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:27:14.565 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:27:14.565 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:27:14.565 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45374", 2023-10-09T05:27:14.565 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2988063385", 2023-10-09T05:27:14.565 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 118504 2023-10-09T05:27:14.565 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.565 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:27:14.565 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.565 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:27:14.565 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.395282+0000", 2023-10-09T05:27:14.565 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:27:14.565 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.566 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.566 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:27:14.566 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.395282+0000", 2023-10-09T05:27:14.566 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:27:14.566 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.566 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.566 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:27:14.566 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.395282+0000", 2023-10-09T05:27:14.566 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:27:14.566 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.566 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.567 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:27:14.567 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.395286+0000", 2023-10-09T05:27:14.567 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.4669999999999998e-06 2023-10-09T05:27:14.567 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.567 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.567 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:27:14.567 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.395295+0000", 2023-10-09T05:27:14.567 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.8899999999999996e-06 2023-10-09T05:27:14.567 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.567 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.567 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:27:14.567 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.395301+0000", 2023-10-09T05:27:14.568 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.9750000000000004e-06 2023-10-09T05:27:14.568 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.568 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.568 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:27:14.568 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.477545+0000", 2023-10-09T05:27:14.568 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.082243978999999995 2023-10-09T05:27:14.568 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.568 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.569 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:27:14.569 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.477969+0000", 2023-10-09T05:27:14.569 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00042419499999999999 2023-10-09T05:27:14.569 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:27:14.569 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:27:14.569 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:27:14.569 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:27:14.478453+0000", 2023-10-09T05:27:14.569 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000483707 2023-10-09T05:27:14.569 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:27:14.570 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:27:14.570 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:27:14.570 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:27:14.570 INFO:teuthology.orchestra.run.smithi133.stdout: ], 2023-10-09T05:27:14.570 INFO:teuthology.orchestra.run.smithi133.stdout: "num_ops": 12 2023-10-09T05:27:14.570 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:27:14.571 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:14.571 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:14 smithi133 bash[105589]: audit 2023-10-09T05:27:13.309498+0000 mon.a (mon.0) 1506 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:27:14.572 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:14 smithi133 bash[105589]: audit 2023-10-09T05:27:13.325603+0000 mon.a (mon.0) 1507 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:27:14.572 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:14 smithi133 bash[105589]: audit 2023-10-09T05:27:13.474308+0000 mon.a (mon.0) 1508 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:27:14.572 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:14 smithi133 bash[105589]: audit 2023-10-09T05:27:13.862601+0000 mon.a (mon.0) 1509 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:27:14.572 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:14 smithi133 bash[105589]: audit 2023-10-09T05:27:13.862611+0000 mon.b (mon.1) 25 : audit 1 from='client.? 172.21.15.133:0/3305418511' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:27:14.572 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:14 smithi133 bash[102858]: audit 2023-10-09T05:27:13.309498+0000 mon.a (mon.0) 1506 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:27:14.572 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:14 smithi133 bash[102858]: audit 2023-10-09T05:27:13.325603+0000 mon.a (mon.0) 1507 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:27:14.572 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:14 smithi133 bash[102858]: audit 2023-10-09T05:27:13.474308+0000 mon.a (mon.0) 1508 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:27:14.573 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:14 smithi133 bash[102858]: audit 2023-10-09T05:27:13.862601+0000 mon.a (mon.0) 1509 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:27:14.573 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:14 smithi133 bash[102858]: audit 2023-10-09T05:27:13.862611+0000 mon.b (mon.1) 25 : audit 1 from='client.? 172.21.15.133:0/3305418511' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:27:14.604 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:14.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:14.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:14.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:14 smithi194 bash[92666]: audit 2023-10-09T05:27:13.309498+0000 mon.a (mon.0) 1506 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:27:14.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:14 smithi194 bash[92666]: audit 2023-10-09T05:27:13.325603+0000 mon.a (mon.0) 1507 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:27:14.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:14 smithi194 bash[92666]: audit 2023-10-09T05:27:13.474308+0000 mon.a (mon.0) 1508 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:27:14.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:14 smithi194 bash[92666]: audit 2023-10-09T05:27:13.862601+0000 mon.a (mon.0) 1509 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:27:14.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:14 smithi194 bash[92666]: audit 2023-10-09T05:27:13.862611+0000 mon.b (mon.1) 25 : audit 1 from='client.? 172.21.15.133:0/3305418511' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:27:14.686 INFO:teuthology.orchestra.run.smithi133.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T05:27:14.710 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:14.733 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:14.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:14.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:14.837 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:14.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:14.875 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:14.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:14.976 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:15.005 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:15.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:15.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:15.110 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:15.164 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:15.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:15.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:15.268 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:15.302 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:15.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:15.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:15.408 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:15.440 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:15.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:15.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:15.546 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:15.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:15 smithi133 bash[102858]: cluster 2023-10-09T05:27:14.184447+0000 mgr.y (mgr.25277) 1170 : cluster 0 pgmap v1246: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 740 op/s 2023-10-09T05:27:15.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:15 smithi133 bash[102858]: audit 2023-10-09T05:27:14.538696+0000 mon.a (mon.0) 1510 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:27:15.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:15 smithi133 bash[102858]: audit 2023-10-09T05:27:14.549612+0000 mon.a (mon.0) 1511 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:27:15.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:15 smithi133 bash[102858]: cluster 2023-10-09T05:27:14.661336+0000 osd.5 (osd.5) 254 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T05:27:15.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:15 smithi133 bash[102858]: audit 2023-10-09T05:27:14.665434+0000 mon.a (mon.0) 1512 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:27:15.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:15 smithi133 bash[102858]: cluster 2023-10-09T05:27:14.665826+0000 mon.a (mon.0) 1513 : cluster 0 osdmap e276: 8 total, 8 up, 7 in 2023-10-09T05:27:15.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:15 smithi133 bash[102858]: cluster 2023-10-09T05:27:14.668629+0000 osd.7 (osd.7) 224 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T05:27:15.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:15 smithi133 bash[102858]: cluster 2023-10-09T05:27:14.672319+0000 osd.6 (osd.6) 226 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T05:27:15.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:15 smithi133 bash[102858]: cluster 2023-10-09T05:27:14.695340+0000 osd.4 (osd.4) 246 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T05:27:15.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:15 smithi133 bash[102858]: cluster 2023-10-09T05:27:14.698455+0000 osd.4 (osd.4) 247 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T05:27:15.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:15 smithi133 bash[102858]: audit 2023-10-09T05:27:15.176115+0000 mon.a (mon.0) 1514 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:27:15.551 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:15 smithi133 bash[102858]: audit 2023-10-09T05:27:15.177219+0000 mon.a (mon.0) 1515 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:27:15.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:15 smithi133 bash[102858]: audit 2023-10-09T05:27:15.189562+0000 mon.a (mon.0) 1516 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:27:15.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:15 smithi133 bash[102858]: audit 2023-10-09T05:27:15.209838+0000 mon.c (mon.2) 107 : audit 1 from='client.? 172.21.15.133:0/2021519745' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:27:15.552 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:15 smithi133 bash[102858]: audit 2023-10-09T05:27:15.211946+0000 mon.a (mon.0) 1517 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:27:15.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:15 smithi133 bash[105589]: cluster 2023-10-09T05:27:14.184447+0000 mgr.y (mgr.25277) 1170 : cluster 0 pgmap v1246: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 740 op/s 2023-10-09T05:27:15.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:15 smithi133 bash[105589]: audit 2023-10-09T05:27:14.538696+0000 mon.a (mon.0) 1510 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:27:15.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:15 smithi133 bash[105589]: audit 2023-10-09T05:27:14.549612+0000 mon.a (mon.0) 1511 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:27:15.552 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:15 smithi133 bash[105589]: cluster 2023-10-09T05:27:14.661336+0000 osd.5 (osd.5) 254 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T05:27:15.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:15 smithi133 bash[105589]: audit 2023-10-09T05:27:14.665434+0000 mon.a (mon.0) 1512 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:27:15.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:15 smithi133 bash[105589]: cluster 2023-10-09T05:27:14.665826+0000 mon.a (mon.0) 1513 : cluster 0 osdmap e276: 8 total, 8 up, 7 in 2023-10-09T05:27:15.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:15 smithi133 bash[105589]: cluster 2023-10-09T05:27:14.668629+0000 osd.7 (osd.7) 224 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T05:27:15.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:15 smithi133 bash[105589]: cluster 2023-10-09T05:27:14.672319+0000 osd.6 (osd.6) 226 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T05:27:15.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:15 smithi133 bash[105589]: cluster 2023-10-09T05:27:14.695340+0000 osd.4 (osd.4) 246 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T05:27:15.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:15 smithi133 bash[105589]: cluster 2023-10-09T05:27:14.698455+0000 osd.4 (osd.4) 247 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T05:27:15.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:15 smithi133 bash[105589]: audit 2023-10-09T05:27:15.176115+0000 mon.a (mon.0) 1514 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:27:15.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:15 smithi133 bash[105589]: audit 2023-10-09T05:27:15.177219+0000 mon.a (mon.0) 1515 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:27:15.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:15 smithi133 bash[105589]: audit 2023-10-09T05:27:15.189562+0000 mon.a (mon.0) 1516 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:27:15.553 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:15 smithi133 bash[105589]: audit 2023-10-09T05:27:15.209838+0000 mon.c (mon.2) 107 : audit 1 from='client.? 172.21.15.133:0/2021519745' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:27:15.554 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:15 smithi133 bash[105589]: audit 2023-10-09T05:27:15.211946+0000 mon.a (mon.0) 1517 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:27:15.565 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:15.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:15.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:15.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:15 smithi194 bash[92666]: cluster 2023-10-09T05:27:14.184447+0000 mgr.y (mgr.25277) 1170 : cluster 0 pgmap v1246: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 740 op/s 2023-10-09T05:27:15.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:15 smithi194 bash[92666]: audit 2023-10-09T05:27:14.538696+0000 mon.a (mon.0) 1510 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:27:15.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:15 smithi194 bash[92666]: audit 2023-10-09T05:27:14.549612+0000 mon.a (mon.0) 1511 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:27:15.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:15 smithi194 bash[92666]: cluster 2023-10-09T05:27:14.661336+0000 osd.5 (osd.5) 254 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T05:27:15.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:15 smithi194 bash[92666]: audit 2023-10-09T05:27:14.665434+0000 mon.a (mon.0) 1512 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:27:15.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:15 smithi194 bash[92666]: cluster 2023-10-09T05:27:14.665826+0000 mon.a (mon.0) 1513 : cluster 0 osdmap e276: 8 total, 8 up, 7 in 2023-10-09T05:27:15.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:15 smithi194 bash[92666]: cluster 2023-10-09T05:27:14.668629+0000 osd.7 (osd.7) 224 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T05:27:15.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:15 smithi194 bash[92666]: cluster 2023-10-09T05:27:14.672319+0000 osd.6 (osd.6) 226 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T05:27:15.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:15 smithi194 bash[92666]: cluster 2023-10-09T05:27:14.695340+0000 osd.4 (osd.4) 246 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T05:27:15.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:15 smithi194 bash[92666]: cluster 2023-10-09T05:27:14.698455+0000 osd.4 (osd.4) 247 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T05:27:15.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:15 smithi194 bash[92666]: audit 2023-10-09T05:27:15.176115+0000 mon.a (mon.0) 1514 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:27:15.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:15 smithi194 bash[92666]: audit 2023-10-09T05:27:15.177219+0000 mon.a (mon.0) 1515 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:27:15.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:15 smithi194 bash[92666]: audit 2023-10-09T05:27:15.189562+0000 mon.a (mon.0) 1516 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:27:15.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:15 smithi194 bash[92666]: audit 2023-10-09T05:27:15.209838+0000 mon.c (mon.2) 107 : audit 1 from='client.? 172.21.15.133:0/2021519745' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:27:15.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:15 smithi194 bash[92666]: audit 2023-10-09T05:27:15.211946+0000 mon.a (mon.0) 1517 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:27:15.671 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:15.690 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:15.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:15.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:15.793 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:15.838 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:15.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:15.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:15.946 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:15.971 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:15.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:15.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:16.077 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:16.096 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:16.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:16.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:16.199 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:16.201 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is unset 2023-10-09T05:27:16.231 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:16.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:16.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:16.335 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:16.357 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:16.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:16.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:16.426 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:16 smithi133 bash[105589]: cluster 2023-10-09T05:27:16.181796+0000 mon.a (mon.0) 1518 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:27:16.427 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:16 smithi133 bash[105589]: cluster 2023-10-09T05:27:16.181834+0000 mon.a (mon.0) 1519 : cluster 1 Cluster is now healthy 2023-10-09T05:27:16.427 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:16 smithi133 bash[105589]: audit 2023-10-09T05:27:16.188533+0000 mon.a (mon.0) 1520 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:27:16.427 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:16 smithi133 bash[105589]: cluster 2023-10-09T05:27:16.188775+0000 mon.a (mon.0) 1521 : cluster 0 osdmap e277: 8 total, 8 up, 7 in 2023-10-09T05:27:16.427 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:16 smithi133 bash[105589]: cluster 2023-10-09T05:27:16.192553+0000 osd.4 (osd.4) 248 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T05:27:16.427 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:16 smithi133 bash[102858]: cluster 2023-10-09T05:27:16.181796+0000 mon.a (mon.0) 1518 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:27:16.428 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:16 smithi133 bash[102858]: cluster 2023-10-09T05:27:16.181834+0000 mon.a (mon.0) 1519 : cluster 1 Cluster is now healthy 2023-10-09T05:27:16.428 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:16 smithi133 bash[102858]: audit 2023-10-09T05:27:16.188533+0000 mon.a (mon.0) 1520 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:27:16.428 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:16 smithi133 bash[102858]: cluster 2023-10-09T05:27:16.188775+0000 mon.a (mon.0) 1521 : cluster 0 osdmap e277: 8 total, 8 up, 7 in 2023-10-09T05:27:16.428 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:16 smithi133 bash[102858]: cluster 2023-10-09T05:27:16.192553+0000 osd.4 (osd.4) 248 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T05:27:16.463 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:16.497 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:16.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:16.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:16.604 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:16.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:16 smithi194 bash[92666]: cluster 2023-10-09T05:27:16.181796+0000 mon.a (mon.0) 1518 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:27:16.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:16 smithi194 bash[92666]: cluster 2023-10-09T05:27:16.181834+0000 mon.a (mon.0) 1519 : cluster 1 Cluster is now healthy 2023-10-09T05:27:16.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:16 smithi194 bash[92666]: audit 2023-10-09T05:27:16.188533+0000 mon.a (mon.0) 1520 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:27:16.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:16 smithi194 bash[92666]: cluster 2023-10-09T05:27:16.188775+0000 mon.a (mon.0) 1521 : cluster 0 osdmap e277: 8 total, 8 up, 7 in 2023-10-09T05:27:16.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:16 smithi194 bash[92666]: cluster 2023-10-09T05:27:16.192553+0000 osd.4 (osd.4) 248 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T05:27:16.637 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:16.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:16.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:16.739 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:16.771 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:16.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:16.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:16.875 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:16.905 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:16.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:16.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:17.011 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:17.034 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:17.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:17.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:17.137 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:17.170 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:17.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:17.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:17.275 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:17.302 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:17.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:17.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:17.405 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:17.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:17.434 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:17.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:17.536 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:17.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:17 smithi133 bash[102858]: cluster 2023-10-09T05:27:16.184908+0000 mgr.y (mgr.25277) 1171 : cluster 0 pgmap v1248: 89 pgs: 89 active+clean; 5.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 738 op/s 2023-10-09T05:27:17.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:17 smithi133 bash[102858]: cluster 2023-10-09T05:27:16.194915+0000 osd.6 (osd.6) 227 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T05:27:17.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:17 smithi133 bash[102858]: cluster 2023-10-09T05:27:16.197863+0000 osd.7 (osd.7) 225 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T05:27:17.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:17 smithi133 bash[105589]: cluster 2023-10-09T05:27:16.184908+0000 mgr.y (mgr.25277) 1171 : cluster 0 pgmap v1248: 89 pgs: 89 active+clean; 5.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 738 op/s 2023-10-09T05:27:17.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:17 smithi133 bash[105589]: cluster 2023-10-09T05:27:16.194915+0000 osd.6 (osd.6) 227 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T05:27:17.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:17 smithi133 bash[105589]: cluster 2023-10-09T05:27:16.197863+0000 osd.7 (osd.7) 225 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T05:27:17.565 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:17.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:17.567 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:17 smithi194 bash[92666]: cluster 2023-10-09T05:27:16.184908+0000 mgr.y (mgr.25277) 1171 : cluster 0 pgmap v1248: 89 pgs: 89 active+clean; 5.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 738 op/s 2023-10-09T05:27:17.568 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:17 smithi194 bash[92666]: cluster 2023-10-09T05:27:16.194915+0000 osd.6 (osd.6) 227 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T05:27:17.568 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:17 smithi194 bash[92666]: cluster 2023-10-09T05:27:16.197863+0000 osd.7 (osd.7) 225 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T05:27:17.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:17.669 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:17.701 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:17.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:17.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:17.805 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:17.837 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:17.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:17.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:17.941 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:17.974 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:17.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:17.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:18.081 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:18.109 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:18.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:18.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:18.216 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:18.239 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:18.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:18.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:18.343 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:18.371 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:18.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:18.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:18.474 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:18.484 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:18 smithi133 bash[105589]: cluster 2023-10-09T05:27:17.891287+0000 osd.5 (osd.5) 255 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T05:27:18.484 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:18 smithi133 bash[102858]: cluster 2023-10-09T05:27:17.891287+0000 osd.5 (osd.5) 255 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T05:27:18.510 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:18.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:18.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:18.596 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:27:18.613 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:18.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:18 smithi194 bash[92666]: cluster 2023-10-09T05:27:17.891287+0000 osd.5 (osd.5) 255 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T05:27:18.684 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:18.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:18.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:18.788 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:18.815 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:18.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:18.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:18.920 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:18.953 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:18.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:18.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:19.057 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:19.085 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:19.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:19.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:19.163 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T05:27:19.188 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:19.248 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T05:27:19.266 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:19.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:19.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:19.371 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:19.420 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:19.426 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:19 smithi133 bash[105589]: cluster 2023-10-09T05:27:18.185344+0000 mgr.y (mgr.25277) 1172 : cluster 0 pgmap v1250: 89 pgs: 89 active+clean; 5.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 501 op/s 2023-10-09T05:27:19.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:19.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:19.427 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:19 smithi133 bash[102858]: cluster 2023-10-09T05:27:18.185344+0000 mgr.y (mgr.25277) 1172 : cluster 0 pgmap v1250: 89 pgs: 89 active+clean; 5.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 501 op/s 2023-10-09T05:27:19.527 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:19.562 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:19.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:19.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:19.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:19 smithi194 bash[92666]: cluster 2023-10-09T05:27:18.185344+0000 mgr.y (mgr.25277) 1172 : cluster 0 pgmap v1250: 89 pgs: 89 active+clean; 5.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 501 op/s 2023-10-09T05:27:19.670 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:19.695 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:19.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:19.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:19.801 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:19.831 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:19.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:19.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:19.937 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:19.972 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:19.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:19.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:20.077 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:20.110 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:20.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:20.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:20.216 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:20.244 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:20.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:20.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:20.349 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:20.376 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:20.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:20.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:20.481 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:20.525 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:20.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:20.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:20.632 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:20.660 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:20.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:20.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:20.768 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:20.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:20.818 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:20.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:20.919 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:20.943 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:20.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:20.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:21.048 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:21.084 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:21.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:21.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:21.189 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:21.218 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:21.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:21.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:21.277 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:27:21.326 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:21.332 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:21 smithi133 bash[102858]: cluster 2023-10-09T05:27:20.185972+0000 mgr.y (mgr.25277) 1173 : cluster 0 pgmap v1251: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 819 op/s 2023-10-09T05:27:21.351 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:21.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:21.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:21.461 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:21.468 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:21 smithi194 bash[92666]: cluster 2023-10-09T05:27:20.185972+0000 mgr.y (mgr.25277) 1173 : cluster 0 pgmap v1251: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 819 op/s 2023-10-09T05:27:21.488 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:21.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:21.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:21.592 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:21.600 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:21 smithi133 bash[105589]: cluster 2023-10-09T05:27:20.185972+0000 mgr.y (mgr.25277) 1173 : cluster 0 pgmap v1251: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 819 op/s 2023-10-09T05:27:21.633 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:21.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:21.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:21.737 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:21.764 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:21.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:21.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:21.868 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:21.900 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:21.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:21.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:22.007 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:22.031 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:22.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:22.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:22.137 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:22.170 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:22.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:22.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:22.273 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:22.302 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:22.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:22.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:22.409 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:22.442 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:22.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:22.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:22.548 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:22.576 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:22.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:22.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:22.680 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:22.713 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:22.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:22.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:22.819 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:22.840 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:22.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:22.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:22.945 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:22.993 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:22.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:22.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:23.097 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:23.124 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:23.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:23.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:23.231 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:23.261 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:23.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:23.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:23.364 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:23.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:23 smithi194 bash[92666]: cluster 2023-10-09T05:27:22.186525+0000 mgr.y (mgr.25277) 1174 : cluster 0 pgmap v1252: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 727 op/s 2023-10-09T05:27:23.398 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:23.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:23.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:23.453 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:27:23.453 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:27:23.454 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:27:23.503 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:23.552 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:23.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:23.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:23.658 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:23.667 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:23 smithi133 bash[105589]: cluster 2023-10-09T05:27:22.186525+0000 mgr.y (mgr.25277) 1174 : cluster 0 pgmap v1252: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 727 op/s 2023-10-09T05:27:23.668 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:23 smithi133 bash[102858]: cluster 2023-10-09T05:27:22.186525+0000 mgr.y (mgr.25277) 1174 : cluster 0 pgmap v1252: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 727 op/s 2023-10-09T05:27:23.691 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:23.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:23.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:23.798 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:23.830 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:23.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:23.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:23.934 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:23.957 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:23.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:23.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:24.060 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:24.086 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:24.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:24.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:24.190 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:24.219 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:24.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:24.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:24.323 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:24.347 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:24.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:24.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:24.456 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:24.484 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:24.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:24.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:24.569 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:27:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:27:24] "GET /metrics HTTP/1.1" 200 36479 "" "Prometheus/2.43.0" 2023-10-09T05:27:24.588 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:24.613 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:24.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:24.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:24.717 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:24.743 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:24.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:24.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:24.849 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:24.859 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:24 smithi194 bash[92666]: cluster 2023-10-09T05:27:24.187061+0000 mgr.y (mgr.25277) 1175 : cluster 0 pgmap v1253: 89 pgs: 89 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 571 op/s 2023-10-09T05:27:24.873 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:24.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:24.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:24.977 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:25.007 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:25.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:25.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:25.074 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:24 smithi133 bash[102858]: cluster 2023-10-09T05:27:24.187061+0000 mgr.y (mgr.25277) 1175 : cluster 0 pgmap v1253: 89 pgs: 89 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 571 op/s 2023-10-09T05:27:25.074 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:24 smithi133 bash[105589]: cluster 2023-10-09T05:27:24.187061+0000 mgr.y (mgr.25277) 1175 : cluster 0 pgmap v1253: 89 pgs: 89 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 571 op/s 2023-10-09T05:27:25.112 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:25.147 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:25.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:25.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:25.253 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:25.289 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:25.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:25.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:25.396 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:25.434 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:25.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:25.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:25.487 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:27:25.540 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:25.589 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:25.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:25.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:25.693 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:25.734 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:25.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:25.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:25.838 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:25.877 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:25.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:25.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:25.982 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:26.019 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:26.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:26.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:26.123 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:26.149 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:26.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:26.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:26.259 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:26.301 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:26.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:26.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:26.407 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:26.435 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:26.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:26.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:26.540 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:26.577 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:26.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:26.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:26.682 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:26.711 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:26.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:26.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:26.814 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:26.838 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:26.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:26.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:26.946 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:26.971 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:26.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:26.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:27.074 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:27.099 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:27.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:27.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:27.202 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:27.230 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:27.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:27.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:27.334 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:27.343 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:27 smithi133 bash[105589]: cluster 2023-10-09T05:27:26.187701+0000 mgr.y (mgr.25277) 1176 : cluster 0 pgmap v1254: 89 pgs: 89 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 703 op/s 2023-10-09T05:27:27.343 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:27 smithi133 bash[102858]: cluster 2023-10-09T05:27:26.187701+0000 mgr.y (mgr.25277) 1176 : cluster 0 pgmap v1254: 89 pgs: 89 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 703 op/s 2023-10-09T05:27:27.364 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:27.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:27.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:27.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:27 smithi194 bash[92666]: cluster 2023-10-09T05:27:26.187701+0000 mgr.y (mgr.25277) 1176 : cluster 0 pgmap v1254: 89 pgs: 89 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 703 op/s 2023-10-09T05:27:27.468 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:27.498 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:27.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:27.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:27.604 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:27.635 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:27.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:27.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:27.740 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:27.766 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:27.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:27.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:27.872 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:27.903 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:27.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:27.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:28.007 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:28.080 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:28.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:28.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:28.184 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:28.214 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:28.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:28.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:28.316 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:28.348 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:28.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:28.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:28.453 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:29.202 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:29.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:29.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:29.305 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:29.389 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:29.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:29.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:29.493 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:29.502 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:29 smithi133 bash[102858]: cluster 2023-10-09T05:27:28.188189+0000 mgr.y (mgr.25277) 1177 : cluster 0 pgmap v1255: 89 pgs: 89 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 624 op/s 2023-10-09T05:27:29.502 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:29 smithi133 bash[102858]: audit 2023-10-09T05:27:28.481256+0000 mon.a (mon.0) 1522 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:27:29.502 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:29 smithi133 bash[102858]: audit 2023-10-09T05:27:28.482549+0000 mon.a (mon.0) 1523 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:27:29.502 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:29 smithi133 bash[105589]: cluster 2023-10-09T05:27:28.188189+0000 mgr.y (mgr.25277) 1177 : cluster 0 pgmap v1255: 89 pgs: 89 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 624 op/s 2023-10-09T05:27:29.503 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:29 smithi133 bash[105589]: audit 2023-10-09T05:27:28.481256+0000 mon.a (mon.0) 1522 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:27:29.503 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:29 smithi133 bash[105589]: audit 2023-10-09T05:27:28.482549+0000 mon.a (mon.0) 1523 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:27:29.522 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:29.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:29.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:29.627 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:29.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:29 smithi194 bash[92666]: cluster 2023-10-09T05:27:28.188189+0000 mgr.y (mgr.25277) 1177 : cluster 0 pgmap v1255: 89 pgs: 89 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 624 op/s 2023-10-09T05:27:29.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:29 smithi194 bash[92666]: audit 2023-10-09T05:27:28.481256+0000 mon.a (mon.0) 1522 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:27:29.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:29 smithi194 bash[92666]: audit 2023-10-09T05:27:28.482549+0000 mon.a (mon.0) 1523 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:27:29.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:29.683 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:29.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:29.784 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:29.969 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:29.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:29.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:30.072 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:30.102 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:30.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:30.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:30.207 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:30.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:30.241 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:30.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:30.342 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:30.429 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:30.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:30.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:30.532 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:30.633 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:30.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:30.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:30.737 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:30.768 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:30.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:30.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:30.876 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:30.904 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:30.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:30.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:31.008 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:31.047 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:31.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:31.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:31.152 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:31.172 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:31.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:31.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:31.276 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:31.303 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:31.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:31.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:31.406 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:31.440 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:31.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:31.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:31.546 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:31.573 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:31.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:31.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:31.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:31 smithi194 bash[92666]: cluster 2023-10-09T05:27:30.188811+0000 mgr.y (mgr.25277) 1178 : cluster 0 pgmap v1256: 89 pgs: 89 active+clean; 4.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 840 op/s 2023-10-09T05:27:31.677 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:31.689 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:31 smithi133 bash[105589]: cluster 2023-10-09T05:27:30.188811+0000 mgr.y (mgr.25277) 1178 : cluster 0 pgmap v1256: 89 pgs: 89 active+clean; 4.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 840 op/s 2023-10-09T05:27:31.690 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:31 smithi133 bash[102858]: cluster 2023-10-09T05:27:30.188811+0000 mgr.y (mgr.25277) 1178 : cluster 0 pgmap v1256: 89 pgs: 89 active+clean; 4.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 840 op/s 2023-10-09T05:27:31.718 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:31.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:31.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:31.823 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:31.852 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:31.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:31.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:31.961 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:31.994 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:31.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:31.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:32.100 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:32.139 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:32.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:32.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:32.249 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:32.270 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:32.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:32.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:32.376 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:32.408 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:32.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:32.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:32.512 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:32.541 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:32.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:32.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:32.645 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:32.672 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:32.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:32.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:32.777 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:32.813 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:32.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:32.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:32.922 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:32.962 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:32.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:32.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:33.065 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:33.100 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:33.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:33.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:33.204 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:33.229 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:33.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:33.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:33.337 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:33.347 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:33 smithi133 bash[105589]: cluster 2023-10-09T05:27:32.189292+0000 mgr.y (mgr.25277) 1179 : cluster 0 pgmap v1257: 89 pgs: 89 active+clean; 4.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 622 op/s 2023-10-09T05:27:33.347 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:33 smithi133 bash[102858]: cluster 2023-10-09T05:27:32.189292+0000 mgr.y (mgr.25277) 1179 : cluster 0 pgmap v1257: 89 pgs: 89 active+clean; 4.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 622 op/s 2023-10-09T05:27:33.371 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:33.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:33.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:33.479 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:33.491 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:33 smithi194 bash[92666]: cluster 2023-10-09T05:27:32.189292+0000 mgr.y (mgr.25277) 1179 : cluster 0 pgmap v1257: 89 pgs: 89 active+clean; 4.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 622 op/s 2023-10-09T05:27:33.508 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:33.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:33.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:33.612 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:33.646 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:33.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:33.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:33.752 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:33.786 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:33.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:33.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:33.893 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:33.929 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:33.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:33.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:34.037 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:34.069 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:34.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:34.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:34.172 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:34.199 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:34.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:34.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:34.304 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:34.330 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:34.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:34.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:34.437 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:34.464 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:34.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:34.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:34.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:27:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:27:34] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.43.0" 2023-10-09T05:27:34.568 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:34.610 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:34.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:34.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:34.713 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:34.743 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:34.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:34.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:34.847 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:34 smithi133 bash[102858]: cluster 2023-10-09T05:27:34.189975+0000 mgr.y (mgr.25277) 1180 : cluster 0 pgmap v1258: 89 pgs: 89 active+clean; 4.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 671 op/s 2023-10-09T05:27:34.847 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:34 smithi133 bash[105589]: cluster 2023-10-09T05:27:34.189975+0000 mgr.y (mgr.25277) 1180 : cluster 0 pgmap v1258: 89 pgs: 89 active+clean; 4.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 671 op/s 2023-10-09T05:27:34.849 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:34.878 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:34.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:34.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:34.981 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:34.988 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:34 smithi194 bash[92666]: cluster 2023-10-09T05:27:34.189975+0000 mgr.y (mgr.25277) 1180 : cluster 0 pgmap v1258: 89 pgs: 89 active+clean; 4.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 671 op/s 2023-10-09T05:27:35.005 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:35.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:35.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:35.111 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:35.148 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:35.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:35.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:35.251 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:35.285 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:35.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:35.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:35.389 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:35.413 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:35.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:35.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:35.521 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:35.553 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:35.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:35.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:35.656 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:35.692 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:35.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:35.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:35.798 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:35.837 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:35.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:35.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:35.939 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:35.973 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:35.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:35.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:36.080 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:36.111 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:36.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:36.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:36.217 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:36.243 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:36.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:36.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:36.346 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:36.378 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:36.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:36.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:36.485 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:36.520 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:36.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:36.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:36.623 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:36.647 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:36.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:36.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:36.750 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:36.822 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:36.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:36.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:36.925 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:36.959 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:36.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:36.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:37.067 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:37.092 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:37.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:37.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:37.198 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:37.224 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:37.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:37.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:37.246 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:37 smithi133 bash[102858]: cluster 2023-10-09T05:27:36.190463+0000 mgr.y (mgr.25277) 1181 : cluster 0 pgmap v1259: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 740 op/s 2023-10-09T05:27:37.327 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:37.336 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:37 smithi194 bash[92666]: cluster 2023-10-09T05:27:36.190463+0000 mgr.y (mgr.25277) 1181 : cluster 0 pgmap v1259: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 740 op/s 2023-10-09T05:27:37.353 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:37.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:37.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:37.457 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:37.491 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:37.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:37.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:37.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:37 smithi133 bash[105589]: cluster 2023-10-09T05:27:36.190463+0000 mgr.y (mgr.25277) 1181 : cluster 0 pgmap v1259: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 740 op/s 2023-10-09T05:27:37.596 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:37.635 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:37.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:37.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:37.736 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:37.774 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:37.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:37.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:37.878 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:37.901 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:37.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:37.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:38.006 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:38.032 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:38.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:38.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:38.137 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:38.159 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:38.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:38.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:38.263 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:38.289 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:38.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:38.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:38.393 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:38.417 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:38.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:38.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:38.523 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:38.564 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:38.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:38.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:38.668 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:38.693 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:38.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:38.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:38.803 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:38.840 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:38.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:38.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:38.946 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:38.982 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:38.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:38.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:39.083 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:39.114 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:39.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:39.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:39.221 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:39.248 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:39.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:39.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:39.352 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:39.375 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:39.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:39.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:39.389 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:39 smithi133 bash[102858]: cluster 2023-10-09T05:27:38.190923+0000 mgr.y (mgr.25277) 1182 : cluster 0 pgmap v1260: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 608 op/s 2023-10-09T05:27:39.390 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:39 smithi133 bash[105589]: cluster 2023-10-09T05:27:38.190923+0000 mgr.y (mgr.25277) 1182 : cluster 0 pgmap v1260: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 608 op/s 2023-10-09T05:27:39.477 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:39.480 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:27:39.480 INFO:teuthology.orchestra.run.smithi133.stdout: "ops": [], 2023-10-09T05:27:39.480 INFO:teuthology.orchestra.run.smithi133.stdout: "complaint_time": 30, 2023-10-09T05:27:39.480 INFO:teuthology.orchestra.run.smithi133.stdout: "num_blocked_ops": 0 2023-10-09T05:27:39.480 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:27:39.486 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:39 smithi194 bash[92666]: cluster 2023-10-09T05:27:38.190923+0000 mgr.y (mgr.25277) 1182 : cluster 0 pgmap v1260: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 608 op/s 2023-10-09T05:27:39.502 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:39.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:39.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:39.606 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:39.638 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:39.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:39.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:39.734 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:27:39.742 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:39.767 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:39.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:39.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:39.783 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:27:39.838 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:27:39.875 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:39.900 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:39.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:39.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:39.909 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:27:39.955 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:27:40.005 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:27:40.007 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:40.031 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:40.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:40.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:40.045 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:27:40.094 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:27:40.113 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:27:40.113 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:27:40.113 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:27:40.114 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:27:40.114 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:27:40.114 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:27:40.114 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:27:40.114 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:27:40.133 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:40.174 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:40.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:40.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:40.280 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:40.311 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:40 smithi133 bash[105589]: audit 2023-10-09T05:27:39.681643+0000 mon.a (mon.0) 1524 : audit 0 from='client.? 172.21.15.133:0/3674872547' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:27:40.311 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:40 smithi133 bash[102858]: audit 2023-10-09T05:27:39.681643+0000 mon.a (mon.0) 1524 : audit 0 from='client.? 172.21.15.133:0/3674872547' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:27:40.313 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:40.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:40.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:40.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:40 smithi194 bash[92666]: audit 2023-10-09T05:27:39.681643+0000 mon.a (mon.0) 1524 : audit 0 from='client.? 172.21.15.133:0/3674872547' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:27:40.423 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:40.451 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:40.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:40.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:40.555 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:40.585 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:40.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:40.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:40.688 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:40.720 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:40.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:40.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:40.823 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:40.848 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:40.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:40.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:40.951 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:40.978 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:40.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:40.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:41.083 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:41.110 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:41.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:41.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:41.213 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:41.236 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:41.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:41.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:41.281 INFO:teuthology.orchestra.run.smithi133.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T05:27:41.281 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is set 2023-10-09T05:27:41.282 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:41 smithi133 bash[102858]: cluster 2023-10-09T05:27:40.191682+0000 mgr.y (mgr.25277) 1183 : cluster 0 pgmap v1261: 89 pgs: 89 active+clean; 4.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 857 op/s 2023-10-09T05:27:41.343 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:41.382 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:41.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:41.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:41.488 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:41.519 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:41.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:41.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:41.624 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:41.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:41 smithi194 bash[92666]: cluster 2023-10-09T05:27:40.191682+0000 mgr.y (mgr.25277) 1183 : cluster 0 pgmap v1261: 89 pgs: 89 active+clean; 4.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 857 op/s 2023-10-09T05:27:41.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:41 smithi194 bash[92666]: audit 2023-10-09T05:27:40.645224+0000 mon.a (mon.0) 1525 : audit 1 from='client.? 172.21.15.133:0/3868713572' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:27:41.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:41 smithi194 bash[92666]: audit 2023-10-09T05:27:40.714971+0000 mon.c (mon.2) 108 : audit 1 from='client.? 172.21.15.133:0/2312473036' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:27:41.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:41 smithi194 bash[92666]: audit 2023-10-09T05:27:40.716625+0000 mon.a (mon.0) 1526 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:27:41.648 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:41.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:41.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:41.663 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:41 smithi133 bash[105589]: cluster 2023-10-09T05:27:40.191682+0000 mgr.y (mgr.25277) 1183 : cluster 0 pgmap v1261: 89 pgs: 89 active+clean; 4.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 857 op/s 2023-10-09T05:27:41.664 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:41 smithi133 bash[105589]: audit 2023-10-09T05:27:40.645224+0000 mon.a (mon.0) 1525 : audit 1 from='client.? 172.21.15.133:0/3868713572' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:27:41.664 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:41 smithi133 bash[105589]: audit 2023-10-09T05:27:40.714971+0000 mon.c (mon.2) 108 : audit 1 from='client.? 172.21.15.133:0/2312473036' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:27:41.664 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:41 smithi133 bash[105589]: audit 2023-10-09T05:27:40.716625+0000 mon.a (mon.0) 1526 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:27:41.664 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:41 smithi133 bash[102858]: audit 2023-10-09T05:27:40.645224+0000 mon.a (mon.0) 1525 : audit 1 from='client.? 172.21.15.133:0/3868713572' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:27:41.664 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:41 smithi133 bash[102858]: audit 2023-10-09T05:27:40.714971+0000 mon.c (mon.2) 108 : audit 1 from='client.? 172.21.15.133:0/2312473036' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:27:41.664 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:41 smithi133 bash[102858]: audit 2023-10-09T05:27:40.716625+0000 mon.a (mon.0) 1526 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:27:41.754 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:41.780 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:41.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:41.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:41.886 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:41.910 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:41.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:41.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:42.017 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:42.058 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:42.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:42.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:42.163 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:42.192 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:42.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:42.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:42.293 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:42 smithi133 bash[102858]: cluster 2023-10-09T05:27:41.258187+0000 mon.a (mon.0) 1527 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:27:42.293 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:42 smithi133 bash[102858]: cluster 2023-10-09T05:27:41.267378+0000 osd.7 (osd.7) 226 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T05:27:42.293 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:42 smithi133 bash[102858]: cluster 2023-10-09T05:27:41.269439+0000 osd.6 (osd.6) 228 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T05:27:42.293 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:42 smithi133 bash[102858]: cluster 2023-10-09T05:27:41.269529+0000 osd.7 (osd.7) 227 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T05:27:42.293 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:42 smithi133 bash[102858]: cluster 2023-10-09T05:27:41.271314+0000 osd.6 (osd.6) 229 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T05:27:42.293 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:42 smithi133 bash[102858]: cluster 2023-10-09T05:27:41.272684+0000 osd.6 (osd.6) 230 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T05:27:42.297 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:42.329 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:42.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:42.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:42.432 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:42.465 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:42.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:42.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:42.573 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:42.587 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:42 smithi194 bash[92666]: cluster 2023-10-09T05:27:41.258187+0000 mon.a (mon.0) 1527 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:27:42.587 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:42 smithi194 bash[92666]: cluster 2023-10-09T05:27:41.267378+0000 osd.7 (osd.7) 226 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T05:27:42.587 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:42 smithi194 bash[92666]: cluster 2023-10-09T05:27:41.269439+0000 osd.6 (osd.6) 228 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T05:27:42.587 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:42 smithi194 bash[92666]: cluster 2023-10-09T05:27:41.269529+0000 osd.7 (osd.7) 227 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T05:27:42.588 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:42 smithi194 bash[92666]: cluster 2023-10-09T05:27:41.271314+0000 osd.6 (osd.6) 229 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T05:27:42.588 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:42 smithi194 bash[92666]: cluster 2023-10-09T05:27:41.272684+0000 osd.6 (osd.6) 230 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T05:27:42.588 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:42 smithi194 bash[92666]: audit 2023-10-09T05:27:41.273561+0000 mon.a (mon.0) 1528 : audit 1 from='client.? 172.21.15.133:0/3868713572' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:27:42.588 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:42 smithi194 bash[92666]: audit 2023-10-09T05:27:41.274000+0000 mon.a (mon.0) 1529 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:27:42.588 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:42 smithi194 bash[92666]: cluster 2023-10-09T05:27:41.274238+0000 mon.a (mon.0) 1530 : cluster 0 osdmap e278: 8 total, 8 up, 7 in 2023-10-09T05:27:42.588 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:42 smithi194 bash[92666]: cluster 2023-10-09T05:27:41.297619+0000 osd.4 (osd.4) 249 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T05:27:42.588 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:42 smithi194 bash[92666]: cluster 2023-10-09T05:27:41.301986+0000 osd.4 (osd.4) 250 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T05:27:42.588 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:42 smithi194 bash[92666]: cluster 2023-10-09T05:27:41.918036+0000 osd.5 (osd.5) 256 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T05:27:42.593 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:42 smithi133 bash[102858]: audit 2023-10-09T05:27:41.273561+0000 mon.a (mon.0) 1528 : audit 1 from='client.? 172.21.15.133:0/3868713572' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:27:42.593 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:42 smithi133 bash[102858]: audit 2023-10-09T05:27:41.274000+0000 mon.a (mon.0) 1529 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:27:42.593 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:42 smithi133 bash[102858]: cluster 2023-10-09T05:27:41.274238+0000 mon.a (mon.0) 1530 : cluster 0 osdmap e278: 8 total, 8 up, 7 in 2023-10-09T05:27:42.594 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:42 smithi133 bash[102858]: cluster 2023-10-09T05:27:41.297619+0000 osd.4 (osd.4) 249 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T05:27:42.594 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:42 smithi133 bash[102858]: cluster 2023-10-09T05:27:41.301986+0000 osd.4 (osd.4) 250 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T05:27:42.594 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:42 smithi133 bash[102858]: cluster 2023-10-09T05:27:41.918036+0000 osd.5 (osd.5) 256 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T05:27:42.594 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:42 smithi133 bash[105589]: cluster 2023-10-09T05:27:41.258187+0000 mon.a (mon.0) 1527 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:27:42.594 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:42 smithi133 bash[105589]: cluster 2023-10-09T05:27:41.267378+0000 osd.7 (osd.7) 226 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T05:27:42.594 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:42 smithi133 bash[105589]: cluster 2023-10-09T05:27:41.269439+0000 osd.6 (osd.6) 228 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T05:27:42.594 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:42 smithi133 bash[105589]: cluster 2023-10-09T05:27:41.269529+0000 osd.7 (osd.7) 227 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T05:27:42.594 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:42 smithi133 bash[105589]: cluster 2023-10-09T05:27:41.271314+0000 osd.6 (osd.6) 229 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T05:27:42.594 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:42 smithi133 bash[105589]: cluster 2023-10-09T05:27:41.272684+0000 osd.6 (osd.6) 230 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T05:27:42.594 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:42 smithi133 bash[105589]: audit 2023-10-09T05:27:41.273561+0000 mon.a (mon.0) 1528 : audit 1 from='client.? 172.21.15.133:0/3868713572' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:27:42.595 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:42 smithi133 bash[105589]: audit 2023-10-09T05:27:41.274000+0000 mon.a (mon.0) 1529 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:27:42.595 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:42 smithi133 bash[105589]: cluster 2023-10-09T05:27:41.274238+0000 mon.a (mon.0) 1530 : cluster 0 osdmap e278: 8 total, 8 up, 7 in 2023-10-09T05:27:42.595 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:42 smithi133 bash[105589]: cluster 2023-10-09T05:27:41.297619+0000 osd.4 (osd.4) 249 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T05:27:42.595 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:42 smithi133 bash[105589]: cluster 2023-10-09T05:27:41.301986+0000 osd.4 (osd.4) 250 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T05:27:42.595 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:42 smithi133 bash[105589]: cluster 2023-10-09T05:27:41.918036+0000 osd.5 (osd.5) 256 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T05:27:42.613 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:42.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:42.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:42.716 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:42.746 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:42.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:42.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:42.853 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:42.880 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:42.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:42.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:42.986 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:43.018 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:43.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:43.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:43.124 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:43.157 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:43.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:43.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:43.261 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:43.294 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:43.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:43.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:43.402 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:43.455 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:43.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:43.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:43.482 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T05:27:43.533 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:43 smithi133 bash[105589]: cluster 2023-10-09T05:27:42.192124+0000 mgr.y (mgr.25277) 1184 : cluster 0 pgmap v1263: 89 pgs: 89 active+clean; 4.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 691 op/s 2023-10-09T05:27:43.533 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:43 smithi133 bash[102858]: cluster 2023-10-09T05:27:42.192124+0000 mgr.y (mgr.25277) 1184 : cluster 0 pgmap v1263: 89 pgs: 89 active+clean; 4.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 691 op/s 2023-10-09T05:27:43.561 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:43.569 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:43 smithi194 bash[92666]: cluster 2023-10-09T05:27:42.192124+0000 mgr.y (mgr.25277) 1184 : cluster 0 pgmap v1263: 89 pgs: 89 active+clean; 4.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 691 op/s 2023-10-09T05:27:43.587 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:43.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:43.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:43.693 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:43.720 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:43.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:43.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:43.826 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:43.866 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:43.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:43.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:43.970 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:43.999 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:44.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:44.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:44.103 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:44.133 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:44.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:44.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:44.236 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:44.269 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:44.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:44.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:44.281 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:27:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:27:44] "GET /metrics HTTP/1.1" 200 36478 "" "Prometheus/2.43.0" 2023-10-09T05:27:44.375 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:44.404 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:44.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:44.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:44.508 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:44.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:44.539 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:44 smithi133 bash[102858]: audit 2023-10-09T05:27:43.480660+0000 mon.a (mon.0) 1531 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:27:44.539 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:44 smithi133 bash[102858]: audit 2023-10-09T05:27:43.482070+0000 mon.a (mon.0) 1532 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:27:44.540 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:44 smithi133 bash[102858]: cluster 2023-10-09T05:27:44.196051+0000 mgr.y (mgr.25277) 1185 : cluster 0 pgmap v1264: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 690 op/s 2023-10-09T05:27:44.540 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:44 smithi133 bash[105589]: audit 2023-10-09T05:27:43.480660+0000 mon.a (mon.0) 1531 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:27:44.540 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:44 smithi133 bash[105589]: audit 2023-10-09T05:27:43.482070+0000 mon.a (mon.0) 1532 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:27:44.540 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:44 smithi133 bash[105589]: cluster 2023-10-09T05:27:44.196051+0000 mgr.y (mgr.25277) 1185 : cluster 0 pgmap v1264: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 690 op/s 2023-10-09T05:27:44.541 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:44.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:44.642 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:44.667 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:44.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:44.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:44.770 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T05:27:44.771 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:44.811 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:44.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:44.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:44.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:44 smithi194 bash[92666]: audit 2023-10-09T05:27:43.480660+0000 mon.a (mon.0) 1531 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:27:44.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:44 smithi194 bash[92666]: audit 2023-10-09T05:27:43.482070+0000 mon.a (mon.0) 1532 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:27:44.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:44 smithi194 bash[92666]: cluster 2023-10-09T05:27:44.196051+0000 mgr.y (mgr.25277) 1185 : cluster 0 pgmap v1264: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 690 op/s 2023-10-09T05:27:44.918 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:44.960 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:44.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:44.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:45.063 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:45.104 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:45.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:45.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:45.212 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:45.238 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:45.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:45.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:45.344 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:45.374 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:45.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:45.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:45.481 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:45.507 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:45.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:45.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:45.567 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:27:45.614 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:45.641 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:45.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:45.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:45.751 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:45.776 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:45.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:45.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:45.880 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:45.919 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:45.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:45.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:46.023 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:46.063 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:46.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:46.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:46.167 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:46.199 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:46.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:46.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:46.304 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:46.330 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:46.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:46.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:46.434 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:46.487 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:46.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:46.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:46.596 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:46.633 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:46.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:46.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:46.735 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:46.759 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:46.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:46.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:46.785 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:27:46.867 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:46.891 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:46.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:46.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:46.994 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:47.031 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:47.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:47.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:47.135 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:47.166 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:47.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:47.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:47.269 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:47.301 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:47.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:47.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:47.405 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:47.419 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:47 smithi133 bash[102858]: cluster 2023-10-09T05:27:46.196716+0000 mgr.y (mgr.25277) 1186 : cluster 0 pgmap v1265: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 727 op/s 2023-10-09T05:27:47.419 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:47 smithi133 bash[105589]: cluster 2023-10-09T05:27:46.196716+0000 mgr.y (mgr.25277) 1186 : cluster 0 pgmap v1265: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 727 op/s 2023-10-09T05:27:47.444 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:47.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:47.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:47.546 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:47.578 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:47.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:47.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:47.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:47 smithi194 bash[92666]: cluster 2023-10-09T05:27:46.196716+0000 mgr.y (mgr.25277) 1186 : cluster 0 pgmap v1265: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 727 op/s 2023-10-09T05:27:47.684 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:47.724 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:47.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:47.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:47.828 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:47.856 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:47.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:47.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:47.960 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:47.987 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:47.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:47.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:48.091 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:48.116 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:48.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:48.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:48.220 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:48.247 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:48.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:48.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:48.352 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:48.384 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:27:48.388 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:48.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:48.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:48.493 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:48.527 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:48.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:48.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:48.633 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:48.662 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:48.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:48.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:48.765 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:48.805 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:48.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:48.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:48.910 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:48.932 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:48.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:48.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:49.036 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:49.076 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:49.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:49.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:49.180 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:49.208 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:49.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:49.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:49.312 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:49.336 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:49.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:49.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:49.406 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:27:49.440 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:49.455 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:49 smithi133 bash[102858]: cluster 2023-10-09T05:27:48.197126+0000 mgr.y (mgr.25277) 1187 : cluster 0 pgmap v1266: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 681 op/s 2023-10-09T05:27:49.456 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:49 smithi133 bash[105589]: cluster 2023-10-09T05:27:48.197126+0000 mgr.y (mgr.25277) 1187 : cluster 0 pgmap v1266: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 681 op/s 2023-10-09T05:27:49.483 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:49.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:49.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:49.588 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:49.620 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:49.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:49.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:49.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:49 smithi194 bash[92666]: cluster 2023-10-09T05:27:48.197126+0000 mgr.y (mgr.25277) 1187 : cluster 0 pgmap v1266: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 681 op/s 2023-10-09T05:27:49.727 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:49.755 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:49.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:49.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:49.860 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:49.886 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:49.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:49.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:49.989 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:50.019 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:50.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:50.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:50.125 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:50.152 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:50.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:50.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:50.255 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:50.291 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:50.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:50.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:50.394 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:50.426 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:50.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:50.432 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:27:50.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:50.534 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:50.560 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:50.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:50.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:50.669 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:50.695 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:50.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:50.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:50.801 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:50.837 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:50.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:50.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:50.944 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:50.970 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:50.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:50.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:51.075 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:51.108 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:51.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:51.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:51.211 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:51.237 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:51.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:51.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:51.340 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:51.351 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:51 smithi194 bash[92666]: cluster 2023-10-09T05:27:50.197814+0000 mgr.y (mgr.25277) 1188 : cluster 0 pgmap v1267: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 690 op/s 2023-10-09T05:27:51.367 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:51.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:51.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:51.443 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:27:51.473 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:51.490 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:51 smithi133 bash[105589]: cluster 2023-10-09T05:27:50.197814+0000 mgr.y (mgr.25277) 1188 : cluster 0 pgmap v1267: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 690 op/s 2023-10-09T05:27:51.490 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:51 smithi133 bash[102858]: cluster 2023-10-09T05:27:50.197814+0000 mgr.y (mgr.25277) 1188 : cluster 0 pgmap v1267: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 690 op/s 2023-10-09T05:27:51.507 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:51.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:51.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:51.612 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:51.645 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:51.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:51.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:51.754 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:51.781 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:51.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:51.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:51.885 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:51.916 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:51.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:51.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:52.021 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:52.046 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:52.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:52.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:52.152 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:52.182 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:52.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:52.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:52.289 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:52.317 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:52.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:52.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:52.424 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:52.466 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:52.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:52.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:52.569 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:52.596 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:52.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:52.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:52.703 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:52.736 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:52.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:52.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:52.841 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:52.872 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:52.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:52.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:52.977 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:53.196 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:53.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:53.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:53.268 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:53 smithi133 bash[102858]: cluster 2023-10-09T05:27:52.198173+0000 mgr.y (mgr.25277) 1189 : cluster 0 pgmap v1268: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 631 op/s 2023-10-09T05:27:53.304 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:53.344 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:53.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:53.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:53.450 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:53.486 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:53.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:53.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:53.589 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:53.592 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:53 smithi133 bash[105589]: cluster 2023-10-09T05:27:52.198173+0000 mgr.y (mgr.25277) 1189 : cluster 0 pgmap v1268: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 631 op/s 2023-10-09T05:27:53.617 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:53.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:53.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:53.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:53 smithi194 bash[92666]: cluster 2023-10-09T05:27:52.198173+0000 mgr.y (mgr.25277) 1189 : cluster 0 pgmap v1268: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 631 op/s 2023-10-09T05:27:53.722 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:53.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:53.754 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:53.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:53.855 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:53.887 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:53.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:53.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:53.993 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:54.031 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:54.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:54.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:54.137 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:54.162 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:54.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:54.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:54.267 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:54.309 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:54.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:54.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:54.412 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:54.419 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:27:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:27:54] "GET /metrics HTTP/1.1" 200 36478 "" "Prometheus/2.43.0" 2023-10-09T05:27:54.441 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:54.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:54.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:54.546 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:54.632 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:54.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:54.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:54.717 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:54 smithi133 bash[102858]: cluster 2023-10-09T05:27:54.198862+0000 mgr.y (mgr.25277) 1190 : cluster 0 pgmap v1269: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 611 op/s 2023-10-09T05:27:54.738 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:54.747 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:54 smithi133 bash[105589]: cluster 2023-10-09T05:27:54.198862+0000 mgr.y (mgr.25277) 1190 : cluster 0 pgmap v1269: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 611 op/s 2023-10-09T05:27:54.770 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:54.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:54.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:54.877 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:54.906 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:54.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:54.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:55.010 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:55.018 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:54 smithi194 bash[92666]: cluster 2023-10-09T05:27:54.198862+0000 mgr.y (mgr.25277) 1190 : cluster 0 pgmap v1269: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 611 op/s 2023-10-09T05:27:55.033 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:55.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:55.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:55.137 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:55.165 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:55.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:55.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:55.268 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:55.300 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:55.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:55.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:55.406 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:55.428 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:55.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:55.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:55.534 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:55.557 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:55.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:55.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:55.660 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:55.697 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:55.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:55.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:55.801 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:55.827 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:55.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:55.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:55.932 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:55.962 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:55.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:55.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:56.066 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:56.090 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:56.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:56.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:56.194 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:56.220 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:56.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:56.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:56.322 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:56.355 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:56.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:56.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:56.466 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:56.491 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:56.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:56.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:56.598 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:56.629 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:56.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:56.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:56.734 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:56.765 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:56.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:56.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:56.872 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:56.905 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:56.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:56.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:57.008 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:57.033 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:27:57.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:57.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:27:57.137 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:57.171 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:57.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:57.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:57.274 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:57.282 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:57 smithi133 bash[105589]: cluster 2023-10-09T05:27:56.199542+0000 mgr.y (mgr.25277) 1191 : cluster 0 pgmap v1270: 89 pgs: 89 active+clean; 3.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 778 op/s 2023-10-09T05:27:57.283 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:57 smithi133 bash[102858]: cluster 2023-10-09T05:27:56.199542+0000 mgr.y (mgr.25277) 1191 : cluster 0 pgmap v1270: 89 pgs: 89 active+clean; 3.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 778 op/s 2023-10-09T05:27:57.319 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:57.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:57.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:57.425 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:57.458 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:57.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:57.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:57.561 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:57.570 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:57 smithi194 bash[92666]: cluster 2023-10-09T05:27:56.199542+0000 mgr.y (mgr.25277) 1191 : cluster 0 pgmap v1270: 89 pgs: 89 active+clean; 3.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 778 op/s 2023-10-09T05:27:57.601 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:57.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:57.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:57.704 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:57.728 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:57.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:57.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:57.831 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:57.854 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:57.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:57.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:57.957 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:57.985 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:57.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:57.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:58.092 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:58.126 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:58.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:58.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:58.233 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:58.260 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:58.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:58.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:27:58.367 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:58.400 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:58.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:58.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:58.509 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:58.536 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:27:58.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:58.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:27:58.640 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:58.667 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:27:58.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:58.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:27:58.774 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:58.797 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:58.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:58.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:58.904 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:58.931 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:58.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:58.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:59.038 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:59.079 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:27:59.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:59.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:27:59.186 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:59.217 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:27:59.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:59.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:27:59.264 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:59 smithi194 bash[92666]: cluster 2023-10-09T05:27:58.199955+0000 mgr.y (mgr.25277) 1192 : cluster 0 pgmap v1271: 89 pgs: 89 active+clean; 3.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 584 op/s 2023-10-09T05:27:59.264 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:27:59 smithi194 bash[92666]: audit 2023-10-09T05:27:58.475061+0000 mon.a (mon.0) 1533 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:27:59.323 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:59.349 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:27:59.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:59.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:27:59.453 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:59.718 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:59.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:59.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:59.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:59 smithi133 bash[102858]: cluster 2023-10-09T05:27:58.199955+0000 mgr.y (mgr.25277) 1192 : cluster 0 pgmap v1271: 89 pgs: 89 active+clean; 3.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 584 op/s 2023-10-09T05:27:59.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:27:59 smithi133 bash[102858]: audit 2023-10-09T05:27:58.475061+0000 mon.a (mon.0) 1533 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:27:59.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:59 smithi133 bash[105589]: cluster 2023-10-09T05:27:58.199955+0000 mgr.y (mgr.25277) 1192 : cluster 0 pgmap v1271: 89 pgs: 89 active+clean; 3.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 584 op/s 2023-10-09T05:27:59.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:27:59 smithi133 bash[105589]: audit 2023-10-09T05:27:58.475061+0000 mon.a (mon.0) 1533 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:27:59.821 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:59.849 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:27:59.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:59.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:27:59.953 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:59.980 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:27:59.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:27:59.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:00.084 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:00.116 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:00.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:00.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:00.219 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:00.255 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:00.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:00.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:00.358 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:00.386 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:00.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:00.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:00.490 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:00.527 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:00.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:00.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:00.633 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:00.660 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:00.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:00.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:00.763 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:00.798 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:00.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:00.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:00.902 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:00.963 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:00.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:00.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:01.072 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:01.106 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:01.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:01.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:01.213 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:01.246 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:01.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:01.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:01.267 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:01 smithi133 bash[102858]: cluster 2023-10-09T05:28:00.200797+0000 mgr.y (mgr.25277) 1193 : cluster 0 pgmap v1272: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 863 op/s 2023-10-09T05:28:01.355 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:01.392 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:01.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:01.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:01.498 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:01.535 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:01.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:01.537 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:01 smithi133 bash[105589]: cluster 2023-10-09T05:28:00.200797+0000 mgr.y (mgr.25277) 1193 : cluster 0 pgmap v1272: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 863 op/s 2023-10-09T05:28:01.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:01.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:01 smithi194 bash[92666]: cluster 2023-10-09T05:28:00.200797+0000 mgr.y (mgr.25277) 1193 : cluster 0 pgmap v1272: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 863 op/s 2023-10-09T05:28:01.639 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:01.677 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:01.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:01.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:01.781 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:01.808 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:01.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:01.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:01.913 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:01.946 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:01.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:01.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:02.057 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:02.084 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:02.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:02.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:02.189 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:02.233 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:02.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:02.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:02.336 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:02.369 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:02.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:02.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:02.477 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:02.506 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:02.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:02.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:02.612 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:02.650 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:02.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:02.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:02.753 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:02.789 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:02.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:02.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:02.897 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:02.931 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:02.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:02.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:03.034 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:03.078 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:03.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:03.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:03.185 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:03.232 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:03.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:03.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:03.337 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:03.365 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:03.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:03.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:03.468 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:03.477 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:03 smithi133 bash[105589]: cluster 2023-10-09T05:28:02.201222+0000 mgr.y (mgr.25277) 1194 : cluster 0 pgmap v1273: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 568 op/s 2023-10-09T05:28:03.477 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:03 smithi133 bash[102858]: cluster 2023-10-09T05:28:02.201222+0000 mgr.y (mgr.25277) 1194 : cluster 0 pgmap v1273: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 568 op/s 2023-10-09T05:28:03.500 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:03.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:03.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:03.605 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:03.614 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:03 smithi194 bash[92666]: cluster 2023-10-09T05:28:02.201222+0000 mgr.y (mgr.25277) 1194 : cluster 0 pgmap v1273: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 568 op/s 2023-10-09T05:28:03.631 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:03.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:03.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:03.737 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:03.770 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:03.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:03.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:03.875 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:03.902 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:03.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:03.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:04.008 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:04.044 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:04.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:04.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:04.152 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:04.180 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:04.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:04.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:04.283 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:04.311 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:04.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:04.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:04.416 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:04.425 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:28:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:28:04] "GET /metrics HTTP/1.1" 200 36476 "" "Prometheus/2.43.0" 2023-10-09T05:28:04.445 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:04.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:04.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:04.552 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:04.579 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:04.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:04.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:04.687 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:04.696 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:04 smithi133 bash[102858]: cluster 2023-10-09T05:28:04.201582+0000 mgr.y (mgr.25277) 1195 : cluster 0 pgmap v1274: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 605 op/s 2023-10-09T05:28:04.696 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:04 smithi133 bash[105589]: cluster 2023-10-09T05:28:04.201582+0000 mgr.y (mgr.25277) 1195 : cluster 0 pgmap v1274: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 605 op/s 2023-10-09T05:28:04.712 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:04.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:04.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:04.818 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:04.827 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:04 smithi194 bash[92666]: cluster 2023-10-09T05:28:04.201582+0000 mgr.y (mgr.25277) 1195 : cluster 0 pgmap v1274: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 605 op/s 2023-10-09T05:28:04.842 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:04.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:04.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:04.951 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:04.974 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:04.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:04.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:05.077 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:05.104 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:05.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:05.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:05.211 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:05.241 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:05.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:05.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:05.348 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:05.375 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:05.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:05.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:05.481 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:05.501 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:05.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:05.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:05.604 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:05.634 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:05.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:05.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:05.738 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:05.767 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:05.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:05.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:05.871 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:05.895 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:05.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:05.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:06.001 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:06.037 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:06.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:06.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:06.141 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:06.175 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:06.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:06.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:06.281 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:06.314 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:06.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:06.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:06.418 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:06.442 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:06.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:06.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:06.546 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:06.570 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:06.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:06.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:06.675 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:06.710 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:06.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:06.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:06.814 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:06.847 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:06.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:06.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:06.955 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:06.970 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:28:06.970 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:28:07.002 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:07.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:07.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:07.105 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:07.145 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:07.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:07.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:07.248 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:07.276 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:07.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:07.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:07.285 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:28:07.323 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:28:07.349 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:28:07.369 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:28:07.384 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:07.391 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:28:07.413 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:28:07.418 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:07.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:07.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:07.456 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:28:07.496 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:28:07.505 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:28:07.505 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:28:07.505 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:28:07.506 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:28:07.506 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:28:07.506 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:28:07.506 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:28:07.506 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:28:07.522 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:07.532 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:07 smithi194 bash[92666]: cluster 2023-10-09T05:28:06.202301+0000 mgr.y (mgr.25277) 1196 : cluster 0 pgmap v1275: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 819 op/s 2023-10-09T05:28:07.532 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:07 smithi194 bash[92666]: audit 2023-10-09T05:28:07.239090+0000 mon.a (mon.0) 1534 : audit 0 from='client.? 172.21.15.133:0/421868206' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:28:07.542 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:07 smithi133 bash[102858]: cluster 2023-10-09T05:28:06.202301+0000 mgr.y (mgr.25277) 1196 : cluster 0 pgmap v1275: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 819 op/s 2023-10-09T05:28:07.543 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:07 smithi133 bash[102858]: audit 2023-10-09T05:28:07.239090+0000 mon.a (mon.0) 1534 : audit 0 from='client.? 172.21.15.133:0/421868206' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:28:07.543 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:07 smithi133 bash[105589]: cluster 2023-10-09T05:28:06.202301+0000 mgr.y (mgr.25277) 1196 : cluster 0 pgmap v1275: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 819 op/s 2023-10-09T05:28:07.543 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:07 smithi133 bash[105589]: audit 2023-10-09T05:28:07.239090+0000 mon.a (mon.0) 1534 : audit 0 from='client.? 172.21.15.133:0/421868206' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:28:07.549 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:07.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:07.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:07.655 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:07.681 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:07.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:07.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:07.784 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:07.813 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:07.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:07.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:07.921 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:07.949 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:07.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:07.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:08.052 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:08.077 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:08.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:08.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:08.180 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:08.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:08.217 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:08.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:08.318 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:08.347 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:08.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:08.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:08.454 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:08.491 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:08.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:08.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:08.595 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:08.621 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:08.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:08.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:08.724 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:08.750 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:08.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:08.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:08.855 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:08.885 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:08.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:08.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:08.988 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:09.022 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:09.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:09.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:09.125 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:09.150 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:09.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:09.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:09.254 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:09.284 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:09.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:09.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:09.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:09 smithi194 bash[92666]: cluster 2023-10-09T05:28:08.203867+0000 mgr.y (mgr.25277) 1197 : cluster 0 pgmap v1276: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 565 op/s 2023-10-09T05:28:09.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:09 smithi194 bash[92666]: audit 2023-10-09T05:28:08.895621+0000 mon.b (mon.1) 26 : audit 1 from='client.? 172.21.15.133:0/3263397383' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:28:09.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:09 smithi194 bash[92666]: audit 2023-10-09T05:28:08.895845+0000 mon.a (mon.0) 1535 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:28:09.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:09 smithi194 bash[92666]: audit 2023-10-09T05:28:09.074176+0000 mon.b (mon.1) 27 : audit 1 from='client.? 172.21.15.133:0/1664510546' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:28:09.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:09 smithi194 bash[92666]: audit 2023-10-09T05:28:09.074196+0000 mon.a (mon.0) 1536 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:28:09.389 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:09.415 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:09.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:09.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:09.520 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:09.529 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:09 smithi133 bash[102858]: cluster 2023-10-09T05:28:08.203867+0000 mgr.y (mgr.25277) 1197 : cluster 0 pgmap v1276: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 565 op/s 2023-10-09T05:28:09.529 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:09 smithi133 bash[102858]: audit 2023-10-09T05:28:08.895621+0000 mon.b (mon.1) 26 : audit 1 from='client.? 172.21.15.133:0/3263397383' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:28:09.529 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:09 smithi133 bash[102858]: audit 2023-10-09T05:28:08.895845+0000 mon.a (mon.0) 1535 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:28:09.529 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:09 smithi133 bash[102858]: audit 2023-10-09T05:28:09.074176+0000 mon.b (mon.1) 27 : audit 1 from='client.? 172.21.15.133:0/1664510546' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:28:09.530 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:09 smithi133 bash[102858]: audit 2023-10-09T05:28:09.074196+0000 mon.a (mon.0) 1536 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:28:09.530 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:09 smithi133 bash[105589]: cluster 2023-10-09T05:28:08.203867+0000 mgr.y (mgr.25277) 1197 : cluster 0 pgmap v1276: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 565 op/s 2023-10-09T05:28:09.530 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:09 smithi133 bash[105589]: audit 2023-10-09T05:28:08.895621+0000 mon.b (mon.1) 26 : audit 1 from='client.? 172.21.15.133:0/3263397383' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:28:09.530 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:09 smithi133 bash[105589]: audit 2023-10-09T05:28:08.895845+0000 mon.a (mon.0) 1535 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:28:09.530 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:09 smithi133 bash[105589]: audit 2023-10-09T05:28:09.074176+0000 mon.b (mon.1) 27 : audit 1 from='client.? 172.21.15.133:0/1664510546' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:28:09.530 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:09 smithi133 bash[105589]: audit 2023-10-09T05:28:09.074196+0000 mon.a (mon.0) 1536 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:28:09.548 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:09.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:09.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:09.651 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:09.674 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is set 2023-10-09T05:28:09.678 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:09.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:09.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:09.687 INFO:teuthology.orchestra.run.smithi133.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T05:28:09.783 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:09.808 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:09.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:09.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:09.914 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:09.938 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:09.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:09.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:10.042 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:10.069 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:10.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:10.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:10.179 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:10.185 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T05:28:10.202 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:10.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:10.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:10.306 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:10.315 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:10 smithi133 bash[105589]: cluster 2023-10-09T05:28:09.660187+0000 mon.a (mon.0) 1537 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:28:10.315 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:10 smithi133 bash[105589]: cluster 2023-10-09T05:28:09.665035+0000 osd.4 (osd.4) 251 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T05:28:10.315 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:10 smithi133 bash[105589]: audit 2023-10-09T05:28:09.667556+0000 mon.a (mon.0) 1538 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:28:10.316 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:10 smithi133 bash[105589]: audit 2023-10-09T05:28:09.667670+0000 mon.a (mon.0) 1539 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:28:10.316 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:10 smithi133 bash[105589]: cluster 2023-10-09T05:28:09.667755+0000 mon.a (mon.0) 1540 : cluster 0 osdmap e279: 8 total, 8 up, 7 in 2023-10-09T05:28:10.316 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:10 smithi133 bash[105589]: cluster 2023-10-09T05:28:09.675752+0000 osd.6 (osd.6) 231 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T05:28:10.316 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:10 smithi133 bash[105589]: cluster 2023-10-09T05:28:09.680462+0000 osd.7 (osd.7) 228 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T05:28:10.316 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:10 smithi133 bash[102858]: cluster 2023-10-09T05:28:09.660187+0000 mon.a (mon.0) 1537 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:28:10.316 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:10 smithi133 bash[102858]: cluster 2023-10-09T05:28:09.665035+0000 osd.4 (osd.4) 251 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T05:28:10.316 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:10 smithi133 bash[102858]: audit 2023-10-09T05:28:09.667556+0000 mon.a (mon.0) 1538 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:28:10.317 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:10 smithi133 bash[102858]: audit 2023-10-09T05:28:09.667670+0000 mon.a (mon.0) 1539 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:28:10.317 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:10 smithi133 bash[102858]: cluster 2023-10-09T05:28:09.667755+0000 mon.a (mon.0) 1540 : cluster 0 osdmap e279: 8 total, 8 up, 7 in 2023-10-09T05:28:10.317 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:10 smithi133 bash[102858]: cluster 2023-10-09T05:28:09.675752+0000 osd.6 (osd.6) 231 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T05:28:10.317 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:10 smithi133 bash[102858]: cluster 2023-10-09T05:28:09.680462+0000 osd.7 (osd.7) 228 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T05:28:10.348 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:10.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:10.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:10.453 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:10.460 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:10 smithi194 bash[92666]: cluster 2023-10-09T05:28:09.660187+0000 mon.a (mon.0) 1537 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:28:10.460 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:10 smithi194 bash[92666]: cluster 2023-10-09T05:28:09.665035+0000 osd.4 (osd.4) 251 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T05:28:10.461 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:10 smithi194 bash[92666]: audit 2023-10-09T05:28:09.667556+0000 mon.a (mon.0) 1538 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:28:10.461 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:10 smithi194 bash[92666]: audit 2023-10-09T05:28:09.667670+0000 mon.a (mon.0) 1539 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:28:10.461 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:10 smithi194 bash[92666]: cluster 2023-10-09T05:28:09.667755+0000 mon.a (mon.0) 1540 : cluster 0 osdmap e279: 8 total, 8 up, 7 in 2023-10-09T05:28:10.461 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:10 smithi194 bash[92666]: cluster 2023-10-09T05:28:09.675752+0000 osd.6 (osd.6) 231 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T05:28:10.461 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:10 smithi194 bash[92666]: cluster 2023-10-09T05:28:09.680462+0000 osd.7 (osd.7) 228 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T05:28:10.476 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:10.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:10.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:10.581 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:10.606 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:10.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:10.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:10.709 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:10.739 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:10.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:10.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:10.843 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:10.872 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:10.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:10.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:10.981 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:11.010 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:11.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:11.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:11.117 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:11.139 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:11.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:11.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:11.242 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:11.267 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:11.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:11.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:11.373 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:11.400 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:11.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:11.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:11.503 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:11.512 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:11 smithi133 bash[102858]: cluster 2023-10-09T05:28:10.205779+0000 mgr.y (mgr.25277) 1198 : cluster 0 pgmap v1278: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 675 op/s 2023-10-09T05:28:11.512 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:11 smithi133 bash[102858]: cluster 2023-10-09T05:28:10.327382+0000 osd.5 (osd.5) 257 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T05:28:11.512 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:11 smithi133 bash[105589]: cluster 2023-10-09T05:28:10.205779+0000 mgr.y (mgr.25277) 1198 : cluster 0 pgmap v1278: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 675 op/s 2023-10-09T05:28:11.512 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:11 smithi133 bash[105589]: cluster 2023-10-09T05:28:10.327382+0000 osd.5 (osd.5) 257 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T05:28:11.545 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:11.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:11.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:11.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:11 smithi194 bash[92666]: cluster 2023-10-09T05:28:10.205779+0000 mgr.y (mgr.25277) 1198 : cluster 0 pgmap v1278: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 675 op/s 2023-10-09T05:28:11.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:11 smithi194 bash[92666]: cluster 2023-10-09T05:28:10.327382+0000 osd.5 (osd.5) 257 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T05:28:11.651 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:11.678 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:11.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:11.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:11.783 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:11.818 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:11.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:11.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:11.924 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:11.953 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:11.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:11.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:12.056 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:12.082 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:12.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:12.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:12.189 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:12.226 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:28:12.233 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:12.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:12.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:12.340 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:12.361 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T05:28:12.384 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:12.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:12.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:12.487 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:12.512 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:12.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:12.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:12.615 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:12.657 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:12.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:12.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:12.763 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:12.806 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:12.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:12.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:12.908 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:12.934 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:12.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:12.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:13.041 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:13.071 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:13.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:13.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:13.179 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:13.204 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:13.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:13.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:13.308 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:13.346 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:13.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:13.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:13.449 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:13.477 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:13.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:13.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:13.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:13 smithi133 bash[105589]: cluster 2023-10-09T05:28:12.206184+0000 mgr.y (mgr.25277) 1199 : cluster 0 pgmap v1279: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 675 op/s 2023-10-09T05:28:13.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:13 smithi133 bash[102858]: cluster 2023-10-09T05:28:12.206184+0000 mgr.y (mgr.25277) 1199 : cluster 0 pgmap v1279: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 675 op/s 2023-10-09T05:28:13.581 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:13.616 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:13.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:13.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:13.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:13 smithi194 bash[92666]: cluster 2023-10-09T05:28:12.206184+0000 mgr.y (mgr.25277) 1199 : cluster 0 pgmap v1279: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 675 op/s 2023-10-09T05:28:13.721 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:13.747 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:13.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:13.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:13.854 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:13.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:13.894 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:13.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:13.994 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:14.020 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:14.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:14.022 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:14.123 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:14.151 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:14.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:14.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:14.258 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:14.291 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:14.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:14.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:14.397 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:14.418 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:28:14.429 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:14.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:14.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:14.535 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:14.544 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:14 smithi194 bash[92666]: audit 2023-10-09T05:28:13.475283+0000 mon.a (mon.0) 1541 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:14.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:14 smithi133 bash[105589]: audit 2023-10-09T05:28:13.475283+0000 mon.a (mon.0) 1541 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:14.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:14 smithi133 bash[102858]: audit 2023-10-09T05:28:13.475283+0000 mon.a (mon.0) 1541 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:14.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:28:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:28:14] "GET /metrics HTTP/1.1" 200 36474 "" "Prometheus/2.43.0" 2023-10-09T05:28:14.564 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:14.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:14.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:14.668 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:14.707 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:14.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:14.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:14.814 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:14.839 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:14.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:14.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:14.943 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:14.976 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:14.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:14.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:15.023 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:28:15.080 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:15.126 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:15.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:15.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:15.231 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:15.260 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:15.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:15.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:15.365 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:15.395 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:15.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:15.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:15.498 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:15.506 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:15 smithi194 bash[92666]: cluster 2023-10-09T05:28:14.209652+0000 mgr.y (mgr.25277) 1200 : cluster 0 pgmap v1280: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 674 op/s 2023-10-09T05:28:15.506 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:15 smithi194 bash[92666]: audit 2023-10-09T05:28:15.278968+0000 mon.a (mon.0) 1542 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:28:15.527 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:15.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:15.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:15.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:15 smithi133 bash[102858]: cluster 2023-10-09T05:28:14.209652+0000 mgr.y (mgr.25277) 1200 : cluster 0 pgmap v1280: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 674 op/s 2023-10-09T05:28:15.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:15 smithi133 bash[102858]: audit 2023-10-09T05:28:15.278968+0000 mon.a (mon.0) 1542 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:28:15.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:15 smithi133 bash[105589]: cluster 2023-10-09T05:28:14.209652+0000 mgr.y (mgr.25277) 1200 : cluster 0 pgmap v1280: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 674 op/s 2023-10-09T05:28:15.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:15 smithi133 bash[105589]: audit 2023-10-09T05:28:15.278968+0000 mon.a (mon.0) 1542 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:28:15.635 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:15.656 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:15.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:15.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:15.759 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:15.791 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:15.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:15.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:15.893 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:15.915 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:15.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:15.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:16.018 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:16.057 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:16.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:16.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:16.164 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:16.187 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:16.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:16.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:16.293 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:16.323 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:16.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:16.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:16.429 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:16.461 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:16.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:16.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:16.564 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:16.592 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:16.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:16.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:16.697 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:16.720 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:16.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:16.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:16.823 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:16.845 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:16.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:16.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:16.948 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:16.972 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:16.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:16.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:17.075 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:17.109 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:28:17.109 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:28:17.117 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:17.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:17.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:17.223 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:17.260 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:17.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:17.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:17.368 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:17.389 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:17.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:17.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:17.492 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:17.502 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:17 smithi133 bash[105589]: cluster 2023-10-09T05:28:16.210403+0000 mgr.y (mgr.25277) 1201 : cluster 0 pgmap v1281: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 669 op/s 2023-10-09T05:28:17.502 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:17 smithi133 bash[102858]: cluster 2023-10-09T05:28:16.210403+0000 mgr.y (mgr.25277) 1201 : cluster 0 pgmap v1281: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 669 op/s 2023-10-09T05:28:17.529 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:17.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:17.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:17.614 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:17 smithi194 bash[92666]: cluster 2023-10-09T05:28:16.210403+0000 mgr.y (mgr.25277) 1201 : cluster 0 pgmap v1281: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 669 op/s 2023-10-09T05:28:17.638 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:17.665 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:17.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:17.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:17.768 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:17.793 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:17.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:17.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:17.896 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:17.919 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:17.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:17.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:18.026 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:18.048 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:18.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:18.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:18.151 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:18.170 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:18.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:18.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:18.273 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:18.317 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:18.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:18.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:18.424 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:18.446 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:18.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:18.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:18.552 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:18.583 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:18.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:18.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:18.688 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:18.724 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:18.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:18.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:18.829 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:18.855 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:18.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:18.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:18.960 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:18.994 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:18.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:18.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:19.098 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:19.141 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:19.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:19.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:19.145 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:28:19.245 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:19.272 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:19.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:19.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:19.377 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:19.403 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:19.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:19.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:19.436 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:19 smithi194 bash[92666]: cluster 2023-10-09T05:28:18.210854+0000 mgr.y (mgr.25277) 1202 : cluster 0 pgmap v1282: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 669 op/s 2023-10-09T05:28:19.510 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:19.537 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:19.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:19.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:19.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:19 smithi133 bash[102858]: cluster 2023-10-09T05:28:18.210854+0000 mgr.y (mgr.25277) 1202 : cluster 0 pgmap v1282: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 669 op/s 2023-10-09T05:28:19.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:19 smithi133 bash[105589]: cluster 2023-10-09T05:28:18.210854+0000 mgr.y (mgr.25277) 1202 : cluster 0 pgmap v1282: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 669 op/s 2023-10-09T05:28:19.645 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:19.671 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:19.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:19.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:19.780 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:19.807 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:19.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:19.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:19.913 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:19.961 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:19.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:19.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:20.069 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:20.099 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:20.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:20.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:20.206 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:20.238 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:20.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:20.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:20.345 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:20.382 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:20.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:20.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:20.490 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:20.518 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:20.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:20.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:20.623 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:20.661 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:20.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:20.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:20.759 INFO:teuthology.orchestra.run.smithi194.stdout:{ 2023-10-09T05:28:20.759 INFO:teuthology.orchestra.run.smithi194.stdout: "ops": [ 2023-10-09T05:28:20.759 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.759 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.45374.0:164070 12.0 12:0ae4d06f:::benchmark_data_smithi133_218956_object61328:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e279)", 2023-10-09T05:28:20.759 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:28:20.738127+0000", 2023-10-09T05:28:20.760 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 0.019449317000000001, 2023-10-09T05:28:20.760 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.019485652999999999, 2023-10-09T05:28:20.760 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:28:20.760 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:28:20.760 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:28:20.760 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.45374", 2023-10-09T05:28:20.760 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2988063385", 2023-10-09T05:28:20.760 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 164070 2023-10-09T05:28:20.760 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.760 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:28:20.760 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.760 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:28:20.761 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.738127+0000", 2023-10-09T05:28:20.761 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:28:20.761 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.761 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.761 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:28:20.761 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.738127+0000", 2023-10-09T05:28:20.761 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:28:20.761 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.761 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.761 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:28:20.761 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.738126+0000", 2023-10-09T05:28:20.761 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:28:20.762 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.762 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.762 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:28:20.762 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.738133+0000", 2023-10-09T05:28:20.762 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.0180000000000004e-06 2023-10-09T05:28:20.762 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.762 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.762 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:28:20.762 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.738144+0000", 2023-10-09T05:28:20.762 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0857e-05 2023-10-09T05:28:20.762 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.762 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.763 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:28:20.763 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.738150+0000", 2023-10-09T05:28:20.763 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.5809999999999996e-06 2023-10-09T05:28:20.763 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.763 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.763 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:28:20.763 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.743203+0000", 2023-10-09T05:28:20.763 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0050535780000000004 2023-10-09T05:28:20.764 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.764 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.764 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:28:20.764 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.743366+0000", 2023-10-09T05:28:20.764 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000163098 2023-10-09T05:28:20.764 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.764 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.764 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:28:20.764 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.743527+0000", 2023-10-09T05:28:20.764 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00016078800000000001 2023-10-09T05:28:20.764 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.764 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.765 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:28:20.765 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.750165+0000", 2023-10-09T05:28:20.765 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0066374989999999998 2023-10-09T05:28:20.765 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.765 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.765 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:28:20.765 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.756859+0000", 2023-10-09T05:28:20.765 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0066944559999999997 2023-10-09T05:28:20.765 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.765 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.765 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:28:20.765 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.757027+0000", 2023-10-09T05:28:20.766 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00016804100000000001 2023-10-09T05:28:20.766 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.766 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.766 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:28:20.766 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.757260+0000", 2023-10-09T05:28:20.766 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000232527 2023-10-09T05:28:20.766 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:28:20.766 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:28:20.766 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:28:20.766 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.766 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.767 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.45374.0:164072 12.0 12:0b0a1778:::benchmark_data_smithi133_218956_object61330:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e279)", 2023-10-09T05:28:20.767 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:28:20.739366+0000", 2023-10-09T05:28:20.767 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 0.018210739, 2023-10-09T05:28:20.767 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.018381763999999998, 2023-10-09T05:28:20.767 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:28:20.767 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:28:20.767 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:28:20.767 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.45374", 2023-10-09T05:28:20.767 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2988063385", 2023-10-09T05:28:20.767 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 164072 2023-10-09T05:28:20.767 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.767 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:28:20.768 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.768 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:28:20.768 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.739366+0000", 2023-10-09T05:28:20.768 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:28:20.768 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.768 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.768 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:28:20.768 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.739366+0000", 2023-10-09T05:28:20.768 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:28:20.768 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.768 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.768 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:28:20.769 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.739364+0000", 2023-10-09T05:28:20.769 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:28:20.769 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.769 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.769 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:28:20.769 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.739372+0000", 2023-10-09T05:28:20.769 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.1240000000000005e-06 2023-10-09T05:28:20.769 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.769 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.769 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:28:20.769 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.739407+0000", 2023-10-09T05:28:20.769 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.4598000000000003e-05 2023-10-09T05:28:20.770 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.770 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.770 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:28:20.770 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.739415+0000", 2023-10-09T05:28:20.770 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.5529999999999993e-06 2023-10-09T05:28:20.770 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.770 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.770 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:28:20.770 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.750282+0000", 2023-10-09T05:28:20.770 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.01086662 2023-10-09T05:28:20.770 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.771 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.771 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:28:20.771 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.750685+0000", 2023-10-09T05:28:20.771 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00040250799999999999 2023-10-09T05:28:20.771 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.771 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.771 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:28:20.771 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.751165+0000", 2023-10-09T05:28:20.771 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00048066599999999998 2023-10-09T05:28:20.771 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:28:20.771 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:28:20.771 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:28:20.772 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.772 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.772 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_repop_reply(client.45374.0:164070 12.0 e279/264 ondisk, result = 0)", 2023-10-09T05:28:20.772 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:28:20.754666+0000", 2023-10-09T05:28:20.772 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 0.0029107080000000001, 2023-10-09T05:28:20.772 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0031872129999999999, 2023-10-09T05:28:20.772 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:28:20.772 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "started", 2023-10-09T05:28:20.772 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:28:20.772 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.772 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:28:20.772 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.754666+0000", 2023-10-09T05:28:20.773 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:28:20.773 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.773 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.773 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:28:20.773 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.754666+0000", 2023-10-09T05:28:20.773 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:28:20.773 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.773 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.773 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:28:20.773 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.754664+0000", 2023-10-09T05:28:20.773 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:28:20.773 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.774 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.774 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:28:20.774 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.754672+0000", 2023-10-09T05:28:20.774 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.5870000000000004e-06 2023-10-09T05:28:20.774 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.774 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.774 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:28:20.774 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.754690+0000", 2023-10-09T05:28:20.774 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.7770000000000001e-05 2023-10-09T05:28:20.774 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.774 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.775 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:28:20.775 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.754698+0000", 2023-10-09T05:28:20.775 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.2239999999999995e-06 2023-10-09T05:28:20.775 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.775 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.775 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:28:20.775 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.756975+0000", 2023-10-09T05:28:20.775 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0022772719999999999 2023-10-09T05:28:20.775 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.775 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.775 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:28:20.775 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.757003+0000", 2023-10-09T05:28:20.776 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.7498e-05 2023-10-09T05:28:20.776 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:28:20.776 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:28:20.776 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:28:20.776 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.776 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.776 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_repop(client.45374.0:164084 12.f e279/264 12:feba3640:::benchmark_data_smithi133_218956_object61342:head v 279'10293, mlcod=279'10291)", 2023-10-09T05:28:20.776 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:28:20.757352+0000", 2023-10-09T05:28:20.776 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 0.00022388900000000001, 2023-10-09T05:28:20.776 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00058489900000000005, 2023-10-09T05:28:20.776 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:28:20.776 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "started", 2023-10-09T05:28:20.777 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:28:20.777 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.777 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:28:20.777 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.757352+0000", 2023-10-09T05:28:20.777 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:28:20.777 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.777 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.777 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:28:20.777 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.757352+0000", 2023-10-09T05:28:20.777 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:28:20.777 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.778 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.778 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:28:20.778 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.757351+0000", 2023-10-09T05:28:20.778 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:28:20.778 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.778 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.778 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:28:20.778 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.757361+0000", 2023-10-09T05:28:20.778 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.9180000000000006e-06 2023-10-09T05:28:20.778 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.778 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.778 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:28:20.779 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.757398+0000", 2023-10-09T05:28:20.779 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.6711999999999997e-05 2023-10-09T05:28:20.779 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.779 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.779 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:28:20.779 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.757406+0000", 2023-10-09T05:28:20.779 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.7880000000000007e-06 2023-10-09T05:28:20.779 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.779 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.779 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:28:20.779 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.757469+0000", 2023-10-09T05:28:20.779 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.3398000000000005e-05 2023-10-09T05:28:20.780 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:28:20.780 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:28:20.780 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:28:20.780 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:28:20.757498+0000", 2023-10-09T05:28:20.780 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.8391e-05 2023-10-09T05:28:20.780 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:28:20.780 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:28:20.780 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:28:20.780 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:28:20.780 INFO:teuthology.orchestra.run.smithi194.stdout: ], 2023-10-09T05:28:20.780 INFO:teuthology.orchestra.run.smithi194.stdout: "num_ops": 4 2023-10-09T05:28:20.781 INFO:teuthology.orchestra.run.smithi194.stdout:} 2023-10-09T05:28:20.781 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:20.815 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:20.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:20.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:20.919 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:20.958 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:20.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:20.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:21.064 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:21.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:21.128 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:21.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:21.229 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:21.255 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:21.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:21.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:21.357 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:21.399 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:21.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:21.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:21.507 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:21.514 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:21 smithi194 bash[92666]: cluster 2023-10-09T05:28:20.211665+0000 mgr.y (mgr.25277) 1203 : cluster 0 pgmap v1283: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 712 op/s 2023-10-09T05:28:21.540 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:21.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:21.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:21.647 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:21.655 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:21 smithi133 bash[105589]: cluster 2023-10-09T05:28:20.211665+0000 mgr.y (mgr.25277) 1203 : cluster 0 pgmap v1283: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 712 op/s 2023-10-09T05:28:21.655 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:21 smithi133 bash[102858]: cluster 2023-10-09T05:28:20.211665+0000 mgr.y (mgr.25277) 1203 : cluster 0 pgmap v1283: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 712 op/s 2023-10-09T05:28:21.680 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:21.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:21.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:21.786 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:21.823 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:21.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:21.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:21.928 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:21.958 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:21.967 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T05:28:21.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:21.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:22.070 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:22.108 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:22.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:22.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:22.219 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:22.251 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:22.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:22.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:22.354 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:22.380 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:22.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:22.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:22.488 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:22.521 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:22.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:22.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:22.631 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:22.655 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:22.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:22.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:22.760 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:22.795 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:22.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:22.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:22.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:22 smithi133 bash[105589]: audit 2023-10-09T05:28:21.791629+0000 mon.a (mon.0) 1543 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:28:22.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:22 smithi133 bash[105589]: audit 2023-10-09T05:28:21.809333+0000 mon.a (mon.0) 1544 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:28:22.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:22 smithi133 bash[105589]: audit 2023-10-09T05:28:21.857114+0000 mon.a (mon.0) 1545 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:28:22.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:22 smithi133 bash[105589]: audit 2023-10-09T05:28:21.870779+0000 mon.a (mon.0) 1546 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:28:22.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:22 smithi133 bash[105589]: cluster 2023-10-09T05:28:22.212229+0000 mgr.y (mgr.25277) 1204 : cluster 0 pgmap v1284: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 558 op/s 2023-10-09T05:28:22.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:22 smithi133 bash[105589]: audit 2023-10-09T05:28:22.513891+0000 mon.a (mon.0) 1547 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:22.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:22 smithi133 bash[105589]: audit 2023-10-09T05:28:22.515018+0000 mon.a (mon.0) 1548 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:28:22.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:22 smithi133 bash[105589]: audit 2023-10-09T05:28:22.526480+0000 mon.a (mon.0) 1549 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:28:22.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:22 smithi133 bash[102858]: audit 2023-10-09T05:28:21.791629+0000 mon.a (mon.0) 1543 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:28:22.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:22 smithi133 bash[102858]: audit 2023-10-09T05:28:21.809333+0000 mon.a (mon.0) 1544 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:28:22.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:22 smithi133 bash[102858]: audit 2023-10-09T05:28:21.857114+0000 mon.a (mon.0) 1545 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:28:22.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:22 smithi133 bash[102858]: audit 2023-10-09T05:28:21.870779+0000 mon.a (mon.0) 1546 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:28:22.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:22 smithi133 bash[102858]: cluster 2023-10-09T05:28:22.212229+0000 mgr.y (mgr.25277) 1204 : cluster 0 pgmap v1284: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 558 op/s 2023-10-09T05:28:22.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:22 smithi133 bash[102858]: audit 2023-10-09T05:28:22.513891+0000 mon.a (mon.0) 1547 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:22.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:22 smithi133 bash[102858]: audit 2023-10-09T05:28:22.515018+0000 mon.a (mon.0) 1548 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:28:22.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:22 smithi133 bash[102858]: audit 2023-10-09T05:28:22.526480+0000 mon.a (mon.0) 1549 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:28:22.900 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:22.935 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:22.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:22.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:23.039 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:23.062 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:23.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:23.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:23.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:22 smithi194 bash[92666]: audit 2023-10-09T05:28:21.791629+0000 mon.a (mon.0) 1543 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:28:23.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:22 smithi194 bash[92666]: audit 2023-10-09T05:28:21.809333+0000 mon.a (mon.0) 1544 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:28:23.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:22 smithi194 bash[92666]: audit 2023-10-09T05:28:21.857114+0000 mon.a (mon.0) 1545 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:28:23.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:22 smithi194 bash[92666]: audit 2023-10-09T05:28:21.870779+0000 mon.a (mon.0) 1546 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:28:23.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:22 smithi194 bash[92666]: cluster 2023-10-09T05:28:22.212229+0000 mgr.y (mgr.25277) 1204 : cluster 0 pgmap v1284: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 558 op/s 2023-10-09T05:28:23.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:22 smithi194 bash[92666]: audit 2023-10-09T05:28:22.513891+0000 mon.a (mon.0) 1547 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:23.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:22 smithi194 bash[92666]: audit 2023-10-09T05:28:22.515018+0000 mon.a (mon.0) 1548 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:28:23.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:22 smithi194 bash[92666]: audit 2023-10-09T05:28:22.526480+0000 mon.a (mon.0) 1549 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:28:23.166 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:23.193 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:23.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:23.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:23.296 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:23.321 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:23.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:23.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:23.427 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:23.464 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:23.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:23.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:23.568 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:23.593 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:23.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:23.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:23.696 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:23.721 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:23.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:23.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:23.827 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:23.851 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:23.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:23.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:23.954 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:23.981 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:23.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:23.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:24.085 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:24.109 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:24.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:24.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:24.214 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:24.251 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:24.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:24.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:24.360 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:24.395 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:24.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:24.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:24.500 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:24.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:24.534 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:24.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:24.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:28:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:28:24] "GET /metrics HTTP/1.1" 200 36474 "" "Prometheus/2.43.0" 2023-10-09T05:28:24.635 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:24.661 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:24.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:24.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:24.764 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:24.773 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:24 smithi194 bash[92666]: cluster 2023-10-09T05:28:24.212789+0000 mgr.y (mgr.25277) 1205 : cluster 0 pgmap v1285: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 592 op/s 2023-10-09T05:28:24.797 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:24.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:24.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:24.906 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:24.916 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:24 smithi133 bash[102858]: cluster 2023-10-09T05:28:24.212789+0000 mgr.y (mgr.25277) 1205 : cluster 0 pgmap v1285: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 592 op/s 2023-10-09T05:28:24.916 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:24 smithi133 bash[105589]: cluster 2023-10-09T05:28:24.212789+0000 mgr.y (mgr.25277) 1205 : cluster 0 pgmap v1285: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 592 op/s 2023-10-09T05:28:24.937 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:24.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:24.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:25.041 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:25.068 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:25.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:25.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:25.176 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:25.211 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:25.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:25.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:25.319 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:25.357 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:25.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:25.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:25.466 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:25.489 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:25.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:25.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:25.592 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:25.623 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:25.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:25.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:25.727 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:25.757 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:25.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:25.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:25.860 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:25.890 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:25.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:25.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:25.994 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:26.028 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:26.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:26.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:26.132 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:26.158 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:26.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:26.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:26.268 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:26.297 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:26.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:26.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:26.404 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:26.431 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:26.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:26.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:26.536 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:26.576 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:26.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:26.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:26.684 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:26.709 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:26.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:26.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:26.815 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:28:26.819 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:26.845 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:26.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:26.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:26.951 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:26.975 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:26.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:26.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:27.081 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:27.118 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:27.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:27.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:27.224 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:27.256 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:27.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:27.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:27.361 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:27.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:27 smithi194 bash[92666]: cluster 2023-10-09T05:28:26.213549+0000 mgr.y (mgr.25277) 1206 : cluster 0 pgmap v1286: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 795 op/s 2023-10-09T05:28:27.389 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:27.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:27.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:27.427 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:27 smithi133 bash[105589]: cluster 2023-10-09T05:28:26.213549+0000 mgr.y (mgr.25277) 1206 : cluster 0 pgmap v1286: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 795 op/s 2023-10-09T05:28:27.428 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:27 smithi133 bash[102858]: cluster 2023-10-09T05:28:26.213549+0000 mgr.y (mgr.25277) 1206 : cluster 0 pgmap v1286: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 795 op/s 2023-10-09T05:28:27.500 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:27.531 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:27.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:27.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:27.636 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:27.661 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:27.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:27.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:27.769 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:27.801 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:27.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:27.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:27.903 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:27.928 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:27.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:27.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:28.031 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:28.069 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:28.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:28.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:28.173 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:28.199 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:28.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:28.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:28.306 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:28.333 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:28.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:28.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:28.436 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:28.460 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:28.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:28.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:28.566 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:28.598 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:28.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:28.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:28.702 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:28.725 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:28.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:28.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:28.828 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:28.863 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:28.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:28.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:28.969 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:28.995 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:28.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:28.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:29.100 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:29.125 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:29.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:29.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:29.228 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:29.251 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:29.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:29.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:29.356 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:29.363 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:29 smithi194 bash[92666]: cluster 2023-10-09T05:28:28.217644+0000 mgr.y (mgr.25277) 1207 : cluster 0 pgmap v1287: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 548 op/s 2023-10-09T05:28:29.364 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:29 smithi194 bash[92666]: audit 2023-10-09T05:28:28.475999+0000 mon.a (mon.0) 1550 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:29.384 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:29.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:29.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:29.413 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:29 smithi133 bash[102858]: cluster 2023-10-09T05:28:28.217644+0000 mgr.y (mgr.25277) 1207 : cluster 0 pgmap v1287: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 548 op/s 2023-10-09T05:28:29.413 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:29 smithi133 bash[102858]: audit 2023-10-09T05:28:28.475999+0000 mon.a (mon.0) 1550 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:29.413 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:29 smithi133 bash[105589]: cluster 2023-10-09T05:28:28.217644+0000 mgr.y (mgr.25277) 1207 : cluster 0 pgmap v1287: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 548 op/s 2023-10-09T05:28:29.413 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:29 smithi133 bash[105589]: audit 2023-10-09T05:28:28.475999+0000 mon.a (mon.0) 1550 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:29.489 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:29.509 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:29.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:29.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:29.615 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:29.639 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:29.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:29.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:29.743 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:29.781 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:29.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:29.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:29.884 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:29.910 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:29.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:29.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:30.016 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:30.050 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:30.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:30.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:30.156 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:30.175 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:30.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:30.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:30.277 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:30.305 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:30.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:30.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:30.408 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:30.432 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:30.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:30.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:30.536 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:30.577 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:30.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:30.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:30.682 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:30.713 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:30.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:30.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:30.817 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:30.850 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:30.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:30.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:30.956 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:30.990 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:30.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:30.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:31.097 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:31.128 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:31.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:31.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:31.231 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:31.269 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:31.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:31.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:31.372 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:31.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:31 smithi194 bash[92666]: cluster 2023-10-09T05:28:30.218484+0000 mgr.y (mgr.25277) 1208 : cluster 0 pgmap v1288: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 845 op/s 2023-10-09T05:28:31.414 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:31.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:31.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:31.491 INFO:teuthology.orchestra.run.smithi194.stdout:{ 2023-10-09T05:28:31.492 INFO:teuthology.orchestra.run.smithi194.stdout: "ops": [], 2023-10-09T05:28:31.492 INFO:teuthology.orchestra.run.smithi194.stdout: "complaint_time": 30, 2023-10-09T05:28:31.492 INFO:teuthology.orchestra.run.smithi194.stdout: "num_blocked_ops": 0 2023-10-09T05:28:31.492 INFO:teuthology.orchestra.run.smithi194.stdout:} 2023-10-09T05:28:31.518 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:31.527 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:31 smithi133 bash[105589]: cluster 2023-10-09T05:28:30.218484+0000 mgr.y (mgr.25277) 1208 : cluster 0 pgmap v1288: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 845 op/s 2023-10-09T05:28:31.527 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:31 smithi133 bash[102858]: cluster 2023-10-09T05:28:30.218484+0000 mgr.y (mgr.25277) 1208 : cluster 0 pgmap v1288: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 845 op/s 2023-10-09T05:28:31.545 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:31.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:31.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:31.582 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:28:31.636 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:28:31.652 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:31.674 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:31.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:31.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:31.692 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:28:31.732 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:28:31.773 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:28:31.777 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:31.817 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:31.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:31.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:31.827 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:28:31.860 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:28:31.904 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:28:31.914 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:28:31.914 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:28:31.915 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:28:31.915 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:28:31.915 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:28:31.915 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:28:31.915 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:28:31.915 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:28:31.923 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:31.964 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:31.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:31.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:32.068 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:32.093 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:32.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:32.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:32.196 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:32.219 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:32.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:32.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:32.294 INFO:teuthology.orchestra.run.smithi133.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T05:28:32.304 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is unset 2023-10-09T05:28:32.325 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:32.332 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:32 smithi194 bash[92666]: audit 2023-10-09T05:28:31.490767+0000 mon.b (mon.1) 28 : audit 1 from='client.? 172.21.15.133:0/1139475979' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:28:32.333 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:32 smithi194 bash[92666]: audit 2023-10-09T05:28:31.490974+0000 mon.a (mon.0) 1551 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:28:32.333 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:32 smithi194 bash[92666]: audit 2023-10-09T05:28:31.525277+0000 mon.c (mon.2) 109 : audit 0 from='client.? 172.21.15.133:0/2154260213' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:28:32.333 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:32 smithi194 bash[92666]: audit 2023-10-09T05:28:31.974193+0000 mon.a (mon.0) 1552 : audit 1 from='client.? 172.21.15.133:0/3456862136' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:28:32.351 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:32.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:32.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:32.455 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T05:28:32.455 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:32.464 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:32 smithi133 bash[105589]: audit 2023-10-09T05:28:31.490767+0000 mon.b (mon.1) 28 : audit 1 from='client.? 172.21.15.133:0/1139475979' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:28:32.464 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:32 smithi133 bash[105589]: audit 2023-10-09T05:28:31.490974+0000 mon.a (mon.0) 1551 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:28:32.464 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:32 smithi133 bash[105589]: audit 2023-10-09T05:28:31.525277+0000 mon.c (mon.2) 109 : audit 0 from='client.? 172.21.15.133:0/2154260213' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:28:32.464 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:32 smithi133 bash[105589]: audit 2023-10-09T05:28:31.974193+0000 mon.a (mon.0) 1552 : audit 1 from='client.? 172.21.15.133:0/3456862136' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:28:32.464 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:32 smithi133 bash[102858]: audit 2023-10-09T05:28:31.490767+0000 mon.b (mon.1) 28 : audit 1 from='client.? 172.21.15.133:0/1139475979' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:28:32.464 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:32 smithi133 bash[102858]: audit 2023-10-09T05:28:31.490974+0000 mon.a (mon.0) 1551 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:28:32.464 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:32 smithi133 bash[102858]: audit 2023-10-09T05:28:31.525277+0000 mon.c (mon.2) 109 : audit 0 from='client.? 172.21.15.133:0/2154260213' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:28:32.464 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:32 smithi133 bash[102858]: audit 2023-10-09T05:28:31.974193+0000 mon.a (mon.0) 1552 : audit 1 from='client.? 172.21.15.133:0/3456862136' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:28:32.485 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:32.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:32.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:32.592 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:32.620 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:32.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:32.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:32.724 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:32.762 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:32.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:32.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:32.868 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:32.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:32.904 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:32.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:33.004 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:33.036 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:33.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:33.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:33.140 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:33.167 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:33.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:33.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:33.271 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:33.294 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:33.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:33.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:33.399 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:33.422 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:33.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:33.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:33.527 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:33.537 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:33 smithi133 bash[102858]: cluster 2023-10-09T05:28:32.218843+0000 mgr.y (mgr.25277) 1209 : cluster 0 pgmap v1289: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 567 op/s 2023-10-09T05:28:33.537 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:33 smithi133 bash[102858]: cluster 2023-10-09T05:28:32.279231+0000 mon.a (mon.0) 1553 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:28:33.538 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:33 smithi133 bash[102858]: cluster 2023-10-09T05:28:32.282898+0000 osd.5 (osd.5) 258 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T05:28:33.538 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:33 smithi133 bash[102858]: cluster 2023-10-09T05:28:32.283034+0000 osd.7 (osd.7) 229 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T05:28:33.538 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:33 smithi133 bash[102858]: audit 2023-10-09T05:28:32.286688+0000 mon.a (mon.0) 1554 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:28:33.538 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:33 smithi133 bash[102858]: audit 2023-10-09T05:28:32.286891+0000 mon.a (mon.0) 1555 : audit 1 from='client.? 172.21.15.133:0/3456862136' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:28:33.538 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:33 smithi133 bash[102858]: cluster 2023-10-09T05:28:32.287110+0000 mon.a (mon.0) 1556 : cluster 0 osdmap e280: 8 total, 8 up, 7 in 2023-10-09T05:28:33.538 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:33 smithi133 bash[102858]: cluster 2023-10-09T05:28:32.287239+0000 osd.6 (osd.6) 232 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T05:28:33.538 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:33 smithi133 bash[105589]: cluster 2023-10-09T05:28:32.218843+0000 mgr.y (mgr.25277) 1209 : cluster 0 pgmap v1289: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 567 op/s 2023-10-09T05:28:33.538 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:33 smithi133 bash[105589]: cluster 2023-10-09T05:28:32.279231+0000 mon.a (mon.0) 1553 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:28:33.538 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:33 smithi133 bash[105589]: cluster 2023-10-09T05:28:32.282898+0000 osd.5 (osd.5) 258 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T05:28:33.539 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:33 smithi133 bash[105589]: cluster 2023-10-09T05:28:32.283034+0000 osd.7 (osd.7) 229 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T05:28:33.539 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:33 smithi133 bash[105589]: audit 2023-10-09T05:28:32.286688+0000 mon.a (mon.0) 1554 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:28:33.539 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:33 smithi133 bash[105589]: audit 2023-10-09T05:28:32.286891+0000 mon.a (mon.0) 1555 : audit 1 from='client.? 172.21.15.133:0/3456862136' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:28:33.539 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:33 smithi133 bash[105589]: cluster 2023-10-09T05:28:32.287110+0000 mon.a (mon.0) 1556 : cluster 0 osdmap e280: 8 total, 8 up, 7 in 2023-10-09T05:28:33.539 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:33 smithi133 bash[105589]: cluster 2023-10-09T05:28:32.287239+0000 osd.6 (osd.6) 232 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T05:28:33.563 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:33.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:33.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:33.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:33 smithi194 bash[92666]: cluster 2023-10-09T05:28:32.218843+0000 mgr.y (mgr.25277) 1209 : cluster 0 pgmap v1289: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 567 op/s 2023-10-09T05:28:33.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:33 smithi194 bash[92666]: cluster 2023-10-09T05:28:32.279231+0000 mon.a (mon.0) 1553 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:28:33.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:33 smithi194 bash[92666]: cluster 2023-10-09T05:28:32.282898+0000 osd.5 (osd.5) 258 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T05:28:33.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:33 smithi194 bash[92666]: cluster 2023-10-09T05:28:32.283034+0000 osd.7 (osd.7) 229 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T05:28:33.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:33 smithi194 bash[92666]: audit 2023-10-09T05:28:32.286688+0000 mon.a (mon.0) 1554 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:28:33.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:33 smithi194 bash[92666]: audit 2023-10-09T05:28:32.286891+0000 mon.a (mon.0) 1555 : audit 1 from='client.? 172.21.15.133:0/3456862136' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:28:33.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:33 smithi194 bash[92666]: cluster 2023-10-09T05:28:32.287110+0000 mon.a (mon.0) 1556 : cluster 0 osdmap e280: 8 total, 8 up, 7 in 2023-10-09T05:28:33.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:33 smithi194 bash[92666]: cluster 2023-10-09T05:28:32.287239+0000 osd.6 (osd.6) 232 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T05:28:33.670 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:33.700 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:33.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:33.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:33.808 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:33.838 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:33.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:33.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:33.943 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:33.969 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:33.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:33.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:34.071 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:34.102 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:34.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:34.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:34.209 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:34.235 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:34.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:34.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:34.341 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:34.367 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:34.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:34.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:34.470 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:34.493 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:34.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:34.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:34.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:28:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:28:34] "GET /metrics HTTP/1.1" 200 36478 "" "Prometheus/2.43.0" 2023-10-09T05:28:34.595 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:34.628 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:34.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:34.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:34.735 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:34.744 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:34 smithi194 bash[92666]: cluster 2023-10-09T05:28:34.219276+0000 mgr.y (mgr.25277) 1210 : cluster 0 pgmap v1291: 89 pgs: 89 active+clean; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 682 op/s 2023-10-09T05:28:34.762 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:34.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:34.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:34.869 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:34.896 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:34.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:34.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:34.999 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:35.008 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:34 smithi133 bash[102858]: cluster 2023-10-09T05:28:34.219276+0000 mgr.y (mgr.25277) 1210 : cluster 0 pgmap v1291: 89 pgs: 89 active+clean; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 682 op/s 2023-10-09T05:28:35.008 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:34 smithi133 bash[105589]: cluster 2023-10-09T05:28:34.219276+0000 mgr.y (mgr.25277) 1210 : cluster 0 pgmap v1291: 89 pgs: 89 active+clean; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 682 op/s 2023-10-09T05:28:35.025 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:35.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:35.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:35.130 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:35.169 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:35.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:35.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:35.272 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:35.295 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:35.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:35.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:35.332 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T05:28:35.332 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:28:35.398 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:35.451 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:35.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:35.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:35.558 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:35.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:35.594 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:35.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:35.695 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:35.736 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:35.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:35.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:35.841 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:35.879 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:35.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:35.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:35.988 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:36.005 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:28:36.012 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:36.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:36.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:36.117 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:36.146 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:36.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:36.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:36.251 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:36.291 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:36.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:36.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:36.395 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:36.431 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:36.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:36.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:36.536 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:36.568 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:36.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:36.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:36.671 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:36.705 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:36.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:36.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:36.810 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:36.853 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:36.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:36.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:36.960 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:37.000 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:37.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:37.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:37.109 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:37.132 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:37.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:37.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:37.236 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:37.266 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:37.267 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:28:37.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:37.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:37.337 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:28:37.371 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:37.392 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:37.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:37.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:37.495 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:37.504 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:37 smithi133 bash[102858]: cluster 2023-10-09T05:28:36.220013+0000 mgr.y (mgr.25277) 1211 : cluster 0 pgmap v1292: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 679 op/s 2023-10-09T05:28:37.505 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:37 smithi133 bash[105589]: cluster 2023-10-09T05:28:36.220013+0000 mgr.y (mgr.25277) 1211 : cluster 0 pgmap v1292: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 679 op/s 2023-10-09T05:28:37.522 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:37.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:37.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:37.624 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:37.632 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:37 smithi194 bash[92666]: cluster 2023-10-09T05:28:36.220013+0000 mgr.y (mgr.25277) 1211 : cluster 0 pgmap v1292: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 679 op/s 2023-10-09T05:28:37.657 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:37.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:37.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:37.761 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:37.788 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:37.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:37.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:37.893 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:37.921 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:37.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:37.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:38.024 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:38.057 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:38.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:38.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:38.160 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:38.183 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:38.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:38.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:38.286 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:38.310 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:38.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:38.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:38.416 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:38.447 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:38.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:38.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:38.556 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:38.583 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:38.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:38.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:38.688 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:38.712 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:38.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:38.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:38.818 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:38.839 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:38.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:38.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:38.948 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:38.972 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:38.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:38.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:39.076 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:39.111 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:39.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:39.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:39.219 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:39.265 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:39.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:39.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:39.375 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:39.403 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:39.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:39.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:39.506 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:39.515 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:39 smithi133 bash[105589]: cluster 2023-10-09T05:28:38.220398+0000 mgr.y (mgr.25277) 1212 : cluster 0 pgmap v1293: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 679 op/s 2023-10-09T05:28:39.515 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:39 smithi133 bash[102858]: cluster 2023-10-09T05:28:38.220398+0000 mgr.y (mgr.25277) 1212 : cluster 0 pgmap v1293: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 679 op/s 2023-10-09T05:28:39.558 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:39.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:39.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:39.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:39 smithi194 bash[92666]: cluster 2023-10-09T05:28:38.220398+0000 mgr.y (mgr.25277) 1212 : cluster 0 pgmap v1293: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 679 op/s 2023-10-09T05:28:39.663 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:39.686 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:39.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:39.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:39.793 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:39.848 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:39.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:39.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:39.955 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:39.980 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:39.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:39.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:40.084 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:40.106 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:40.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:40.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:40.211 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:40.216 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:28:40.217 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:28:40.243 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:40.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:40.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:40.354 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:40.383 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:40.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:40.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:40.485 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:40.521 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:40.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:40.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:40.628 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:40.655 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:40.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:40.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:40.761 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:40.784 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:40.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:40.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:40.888 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:40.913 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:40.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:40.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:41.022 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:41.048 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:41.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:41.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:41.156 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:41.186 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:41.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:41.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:41.240 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:28:41.284 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:41 smithi194 bash[92666]: cluster 2023-10-09T05:28:40.222985+0000 mgr.y (mgr.25277) 1213 : cluster 0 pgmap v1294: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 675 op/s 2023-10-09T05:28:41.292 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:41.325 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:41.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:41.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:41.430 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:41.453 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:41.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:41.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:41.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:41 smithi133 bash[105589]: cluster 2023-10-09T05:28:40.222985+0000 mgr.y (mgr.25277) 1213 : cluster 0 pgmap v1294: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 675 op/s 2023-10-09T05:28:41.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:41 smithi133 bash[102858]: cluster 2023-10-09T05:28:40.222985+0000 mgr.y (mgr.25277) 1213 : cluster 0 pgmap v1294: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 675 op/s 2023-10-09T05:28:41.557 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:41.590 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:41.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:41.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:41.693 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:41.735 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:41.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:41.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:41.838 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:41.875 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:41.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:41.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:41.968 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:28:41.968 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:28:41.980 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:42.005 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:42.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:42.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:42.114 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:42.140 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:42.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:42.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:42.248 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:42.276 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:42.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:42.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:42.382 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:42.412 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:42.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:42.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:42.517 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:42.549 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:42.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:42.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:42.659 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:42.703 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:42.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:42.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:42.808 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:42.832 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:42.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:42.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:42.935 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:42.968 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:42.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:42.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:42.989 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T05:28:43.075 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:43.101 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:43.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:43.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:43.204 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:43.241 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:43.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:43.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:43.352 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:43.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:43 smithi194 bash[92666]: cluster 2023-10-09T05:28:42.223392+0000 mgr.y (mgr.25277) 1214 : cluster 0 pgmap v1295: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 675 op/s 2023-10-09T05:28:43.391 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:43.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:43.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:43.492 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:43 smithi133 bash[105589]: cluster 2023-10-09T05:28:42.223392+0000 mgr.y (mgr.25277) 1214 : cluster 0 pgmap v1295: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 675 op/s 2023-10-09T05:28:43.493 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:43 smithi133 bash[102858]: cluster 2023-10-09T05:28:42.223392+0000 mgr.y (mgr.25277) 1214 : cluster 0 pgmap v1295: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 675 op/s 2023-10-09T05:28:43.496 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:43.523 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:43.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:43.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:43.628 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:43.671 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:43.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:43.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:43.775 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:43.808 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:43.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:43.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:43.915 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:43.947 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:43.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:43.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:44.050 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:44.082 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:44.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:44.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:44.188 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:44.212 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:44.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:44.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:44.318 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:44.351 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:44.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:44.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:44.455 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:44.483 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:44.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:44.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:44.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:28:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:28:44] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.43.0" 2023-10-09T05:28:44.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:44 smithi133 bash[105589]: audit 2023-10-09T05:28:43.477973+0000 mon.a (mon.0) 1557 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:44.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:44 smithi133 bash[102858]: audit 2023-10-09T05:28:43.477973+0000 mon.a (mon.0) 1557 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:44.588 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:44.616 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:44.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:44.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:44.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:44 smithi194 bash[92666]: audit 2023-10-09T05:28:43.477973+0000 mon.a (mon.0) 1557 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:44.724 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:44.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:44.751 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:44.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:44.852 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:44.889 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:44.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:44.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:44.991 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:45.020 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:45.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:45.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:45.129 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:45.161 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:45.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:45.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:45.265 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:45.296 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:45.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:45.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:45.400 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:45.409 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:45 smithi194 bash[92666]: cluster 2023-10-09T05:28:44.223814+0000 mgr.y (mgr.25277) 1215 : cluster 0 pgmap v1296: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 601 op/s 2023-10-09T05:28:45.426 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:45.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:45.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:45.515 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:45 smithi133 bash[105589]: cluster 2023-10-09T05:28:44.223814+0000 mgr.y (mgr.25277) 1215 : cluster 0 pgmap v1296: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 601 op/s 2023-10-09T05:28:45.516 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:45 smithi133 bash[102858]: cluster 2023-10-09T05:28:44.223814+0000 mgr.y (mgr.25277) 1215 : cluster 0 pgmap v1296: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 601 op/s 2023-10-09T05:28:45.533 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:45.561 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:45.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:45.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:45.668 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:45.695 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:45.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:45.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:45.801 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:45.830 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:45.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:45.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:45.938 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:45.966 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:45.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:45.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:46.075 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:46.102 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:46.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:46.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:46.212 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:46.238 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:46.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:46.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:46.347 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:46.371 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:46.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:46.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:46.474 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:46.503 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:46.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:46.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:46.607 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:46.634 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:46.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:46.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:46.741 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:46.773 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:46.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:46.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:46.881 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:46.912 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:46.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:46.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:47.023 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:47.068 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:47.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:47.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:47.172 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:47.207 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:47.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:47.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:47.311 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:47.352 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:47.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:47.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:47.459 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:47.485 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:47.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:47.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:47.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:47 smithi133 bash[105589]: cluster 2023-10-09T05:28:46.224544+0000 mgr.y (mgr.25277) 1216 : cluster 0 pgmap v1297: 89 pgs: 89 active+clean; 1.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 814 op/s 2023-10-09T05:28:47.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:47 smithi133 bash[102858]: cluster 2023-10-09T05:28:46.224544+0000 mgr.y (mgr.25277) 1216 : cluster 0 pgmap v1297: 89 pgs: 89 active+clean; 1.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 814 op/s 2023-10-09T05:28:47.592 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:47.603 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:47 smithi194 bash[92666]: cluster 2023-10-09T05:28:46.224544+0000 mgr.y (mgr.25277) 1216 : cluster 0 pgmap v1297: 89 pgs: 89 active+clean; 1.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 814 op/s 2023-10-09T05:28:47.620 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:47.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:47.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:47.726 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:47.763 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:47.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:47.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:47.858 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:28:47.866 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:47.911 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:47.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:47.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:48.018 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:48.070 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:48.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:48.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:48.173 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:48.246 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:48.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:48.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:48.355 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:48.384 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:48.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:48.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:48.487 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:48.506 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:48.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:48.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:48.609 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:48.638 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:48.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:48.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:48.741 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:48.779 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:48.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:48.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:48.887 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:48.924 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:48.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:48.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:49.028 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:49.059 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:49.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:49.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:49.169 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:49.196 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:49.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:49.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:49.302 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:49.329 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:49.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:49.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:49.436 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:49.468 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:49.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:49.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:49.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:49 smithi133 bash[102858]: cluster 2023-10-09T05:28:48.224950+0000 mgr.y (mgr.25277) 1217 : cluster 0 pgmap v1298: 89 pgs: 89 active+clean; 1.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 578 op/s 2023-10-09T05:28:49.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:49 smithi133 bash[105589]: cluster 2023-10-09T05:28:48.224950+0000 mgr.y (mgr.25277) 1217 : cluster 0 pgmap v1298: 89 pgs: 89 active+clean; 1.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 578 op/s 2023-10-09T05:28:49.572 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:49.580 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:49 smithi194 bash[92666]: cluster 2023-10-09T05:28:48.224950+0000 mgr.y (mgr.25277) 1217 : cluster 0 pgmap v1298: 89 pgs: 89 active+clean; 1.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 578 op/s 2023-10-09T05:28:49.597 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:49.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:49.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:49.703 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:49.725 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:49.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:49.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:49.828 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:49.860 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:49.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:49.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:49.963 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:49.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:49.994 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:49.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:50.095 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:50.122 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:50.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:50.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:50.227 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:50.249 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:50.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:50.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:50.352 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:50.380 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:50.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:50.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:50.484 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:50.517 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:50.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:50.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:50.621 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:50.659 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:50.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:50.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:50.740 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:28:50.764 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:50.809 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:50.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:50.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:50.818 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:28:50.885 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:28:50.918 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:50.949 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:28:50.949 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:50.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:50.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:50.996 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:28:51.038 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:28:51.056 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:51.082 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:51.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:51.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:51.085 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:28:51.086 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:28:51.094 INFO:teuthology.orchestra.run.smithi133.stderr:dumped all 2023-10-09T05:28:51.137 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:28:51.143 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:28:51.143 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:28:51.143 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:28:51.144 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:28:51.144 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:28:51.144 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:28:51.144 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:28:51.144 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:28:51.185 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:51.222 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:51.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:51.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:51.330 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:51.342 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is unset 2023-10-09T05:28:51.354 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:51.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:51.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:51.458 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:51.467 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:51 smithi133 bash[102858]: cluster 2023-10-09T05:28:50.225853+0000 mgr.y (mgr.25277) 1218 : cluster 0 pgmap v1299: 89 pgs: 89 active+clean; 1.3 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 862 op/s 2023-10-09T05:28:51.468 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:51 smithi133 bash[102858]: audit 2023-10-09T05:28:50.678786+0000 mon.c (mon.2) 110 : audit 0 from='client.? 172.21.15.133:0/2243700769' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:28:51.468 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:51 smithi133 bash[102858]: audit 2023-10-09T05:28:51.273611+0000 mon.a (mon.0) 1558 : audit 1 from='client.? 172.21.15.133:0/3301018994' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:28:51.468 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:51 smithi133 bash[105589]: cluster 2023-10-09T05:28:50.225853+0000 mgr.y (mgr.25277) 1218 : cluster 0 pgmap v1299: 89 pgs: 89 active+clean; 1.3 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 862 op/s 2023-10-09T05:28:51.468 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:51 smithi133 bash[105589]: audit 2023-10-09T05:28:50.678786+0000 mon.c (mon.2) 110 : audit 0 from='client.? 172.21.15.133:0/2243700769' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:28:51.468 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:51 smithi133 bash[105589]: audit 2023-10-09T05:28:51.273611+0000 mon.a (mon.0) 1558 : audit 1 from='client.? 172.21.15.133:0/3301018994' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:28:51.494 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:51.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:51.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:51.598 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:51.625 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:51.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:51.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:51.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:51 smithi194 bash[92666]: cluster 2023-10-09T05:28:50.225853+0000 mgr.y (mgr.25277) 1218 : cluster 0 pgmap v1299: 89 pgs: 89 active+clean; 1.3 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 862 op/s 2023-10-09T05:28:51.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:51 smithi194 bash[92666]: audit 2023-10-09T05:28:50.678786+0000 mon.c (mon.2) 110 : audit 0 from='client.? 172.21.15.133:0/2243700769' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:28:51.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:51 smithi194 bash[92666]: audit 2023-10-09T05:28:51.273611+0000 mon.a (mon.0) 1558 : audit 1 from='client.? 172.21.15.133:0/3301018994' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:28:51.732 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:51.768 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:51.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:51.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:51.872 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:51.911 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:51.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:51.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:52.016 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:52.058 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:52.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:52.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:52.164 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:52.195 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:52.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:52.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:52.298 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:52.338 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:52.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:52.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:52.444 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:52.473 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:52 smithi133 bash[102858]: audit 2023-10-09T05:28:51.061686+0000 mgr.y (mgr.25277) 1219 : audit 0 from='client.55126 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:28:52.473 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:52 smithi133 bash[102858]: cluster 2023-10-09T05:28:51.312459+0000 mon.a (mon.0) 1559 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:28:52.473 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:52 smithi133 bash[102858]: cluster 2023-10-09T05:28:51.312625+0000 mon.a (mon.0) 1560 : cluster 1 Cluster is now healthy 2023-10-09T05:28:52.473 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:52 smithi133 bash[102858]: cluster 2023-10-09T05:28:51.329749+0000 osd.5 (osd.5) 259 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T05:28:52.474 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:52 smithi133 bash[102858]: audit 2023-10-09T05:28:51.333003+0000 mon.a (mon.0) 1561 : audit 1 from='client.? 172.21.15.133:0/3301018994' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:28:52.474 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:52 smithi133 bash[102858]: cluster 2023-10-09T05:28:51.333224+0000 mon.a (mon.0) 1562 : cluster 0 osdmap e281: 8 total, 8 up, 7 in 2023-10-09T05:28:52.474 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:52 smithi133 bash[102858]: cluster 2023-10-09T05:28:51.334760+0000 osd.7 (osd.7) 230 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T05:28:52.474 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:52 smithi133 bash[102858]: cluster 2023-10-09T05:28:51.338774+0000 osd.7 (osd.7) 231 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T05:28:52.474 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:52 smithi133 bash[102858]: cluster 2023-10-09T05:28:51.340824+0000 osd.4 (osd.4) 252 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T05:28:52.474 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:52 smithi133 bash[102858]: cluster 2023-10-09T05:28:51.358727+0000 osd.6 (osd.6) 233 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T05:28:52.474 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:52 smithi133 bash[105589]: audit 2023-10-09T05:28:51.061686+0000 mgr.y (mgr.25277) 1219 : audit 0 from='client.55126 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:28:52.474 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:52 smithi133 bash[105589]: cluster 2023-10-09T05:28:51.312459+0000 mon.a (mon.0) 1559 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:28:52.474 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:52 smithi133 bash[105589]: cluster 2023-10-09T05:28:51.312625+0000 mon.a (mon.0) 1560 : cluster 1 Cluster is now healthy 2023-10-09T05:28:52.475 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:52 smithi133 bash[105589]: cluster 2023-10-09T05:28:51.329749+0000 osd.5 (osd.5) 259 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T05:28:52.475 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:52 smithi133 bash[105589]: audit 2023-10-09T05:28:51.333003+0000 mon.a (mon.0) 1561 : audit 1 from='client.? 172.21.15.133:0/3301018994' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:28:52.475 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:52 smithi133 bash[105589]: cluster 2023-10-09T05:28:51.333224+0000 mon.a (mon.0) 1562 : cluster 0 osdmap e281: 8 total, 8 up, 7 in 2023-10-09T05:28:52.475 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:52 smithi133 bash[105589]: cluster 2023-10-09T05:28:51.334760+0000 osd.7 (osd.7) 230 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T05:28:52.475 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:52 smithi133 bash[105589]: cluster 2023-10-09T05:28:51.338774+0000 osd.7 (osd.7) 231 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T05:28:52.475 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:52 smithi133 bash[105589]: cluster 2023-10-09T05:28:51.340824+0000 osd.4 (osd.4) 252 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T05:28:52.475 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:52 smithi133 bash[105589]: cluster 2023-10-09T05:28:51.358727+0000 osd.6 (osd.6) 233 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T05:28:52.475 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:52.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:52.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:52.578 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:52.587 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:52 smithi194 bash[92666]: audit 2023-10-09T05:28:51.061686+0000 mgr.y (mgr.25277) 1219 : audit 0 from='client.55126 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:28:52.587 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:52 smithi194 bash[92666]: cluster 2023-10-09T05:28:51.312459+0000 mon.a (mon.0) 1559 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:28:52.587 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:52 smithi194 bash[92666]: cluster 2023-10-09T05:28:51.312625+0000 mon.a (mon.0) 1560 : cluster 1 Cluster is now healthy 2023-10-09T05:28:52.587 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:52 smithi194 bash[92666]: cluster 2023-10-09T05:28:51.329749+0000 osd.5 (osd.5) 259 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T05:28:52.587 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:52 smithi194 bash[92666]: audit 2023-10-09T05:28:51.333003+0000 mon.a (mon.0) 1561 : audit 1 from='client.? 172.21.15.133:0/3301018994' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:28:52.587 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:52 smithi194 bash[92666]: cluster 2023-10-09T05:28:51.333224+0000 mon.a (mon.0) 1562 : cluster 0 osdmap e281: 8 total, 8 up, 7 in 2023-10-09T05:28:52.587 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:52 smithi194 bash[92666]: cluster 2023-10-09T05:28:51.334760+0000 osd.7 (osd.7) 230 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T05:28:52.587 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:52 smithi194 bash[92666]: cluster 2023-10-09T05:28:51.338774+0000 osd.7 (osd.7) 231 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T05:28:52.587 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:52 smithi194 bash[92666]: cluster 2023-10-09T05:28:51.340824+0000 osd.4 (osd.4) 252 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T05:28:52.588 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:52 smithi194 bash[92666]: cluster 2023-10-09T05:28:51.358727+0000 osd.6 (osd.6) 233 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T05:28:52.603 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:52.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:52.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:52.708 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:52.743 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:52.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:52.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:52.769 INFO:teuthology.orchestra.run.smithi194.stdout:{ 2023-10-09T05:28:52.769 INFO:teuthology.orchestra.run.smithi194.stdout: "ops": [], 2023-10-09T05:28:52.770 INFO:teuthology.orchestra.run.smithi194.stdout: "num_ops": 0 2023-10-09T05:28:52.770 INFO:teuthology.orchestra.run.smithi194.stdout:} 2023-10-09T05:28:52.849 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:52.872 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:52.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:52.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:52.980 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:53.003 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:53.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:53.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:53.109 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:53.136 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:53.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:53.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:53.239 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:53.262 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:53.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:53.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:53.335 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:53 smithi194 bash[92666]: cluster 2023-10-09T05:28:52.229635+0000 mgr.y (mgr.25277) 1220 : cluster 0 pgmap v1301: 89 pgs: 89 active+clean; 1.3 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 682 op/s 2023-10-09T05:28:53.369 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:53.393 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:53.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:53.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:53.496 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:53.532 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:53.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:53.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:53.636 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:53.645 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:53 smithi133 bash[102858]: cluster 2023-10-09T05:28:52.229635+0000 mgr.y (mgr.25277) 1220 : cluster 0 pgmap v1301: 89 pgs: 89 active+clean; 1.3 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 682 op/s 2023-10-09T05:28:53.646 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:53 smithi133 bash[105589]: cluster 2023-10-09T05:28:52.229635+0000 mgr.y (mgr.25277) 1220 : cluster 0 pgmap v1301: 89 pgs: 89 active+clean; 1.3 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 682 op/s 2023-10-09T05:28:53.672 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:53.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:53.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:53.774 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T05:28:53.777 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:53.844 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:53.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:53.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:53.859 INFO:teuthology.orchestra.run.smithi133.stdout:{"pg_ready":true,"pg_map":{"version":1299,"stamp":"2023-10-09T05:28:50.225137+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1433606072,"num_objects":21891,"num_object_clones":0,"num_object_copies":65673,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21891,"num_whiteouts":0,"num_read":507,"num_read_kb":932,"num_write":183913,"num_write_kb":6577742,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":4154034,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127835,"ondisk_log_size":127835,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":10404184,"kb_used_data":4276640,"kb_used_omap":20,"kb_used_meta":6127403,"kb_avail":645811880,"statfs":{"total":671965249536,"available":661311365120,"internally_reserved":0,"allocated":4379279360,"data_stored":4370740925,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":6274461326},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-678232064,"num_objects":-10349,"num_object_clones":0,"num_object_copies":-31047,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-10349,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10349,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004942"},"pg_stats":[{"pgid":"3.1a","version":"69'2","reported_seq":280,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.670125+0000","last_change":"2023-10-09T05:09:57.873320+0000","last_active":"2023-10-09T05:28:09.670125+0000","last_peered":"2023-10-09T05:28:09.670125+0000","last_clean":"2023-10-09T05:28:09.670125+0000","last_became_active":"2023-10-09T05:09:57.868376+0000","last_became_peered":"2023-10-09T05:09:57.868376+0000","last_unstale":"2023-10-09T05:28:09.670125+0000","last_undegraded":"2023-10-09T05:28:09.670125+0000","last_fullsized":"2023-10-09T05:28:09.670125+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:27.186610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":263,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.697003+0000","last_change":"2023-10-09T05:14:15.345991+0000","last_active":"2023-10-09T05:28:09.697003+0000","last_peered":"2023-10-09T05:28:09.697003+0000","last_clean":"2023-10-09T05:28:09.697003+0000","last_became_active":"2023-10-09T05:14:15.345511+0000","last_became_peered":"2023-10-09T05:14:15.345511+0000","last_unstale":"2023-10-09T05:28:09.697003+0000","last_undegraded":"2023-10-09T05:28:09.697003+0000","last_fullsized":"2023-10-09T05:28:09.697003+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:49.234025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":146,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.666584+0000","last_change":"2023-10-09T05:09:57.857511+0000","last_active":"2023-10-09T05:28:09.666584+0000","last_peered":"2023-10-09T05:28:09.666584+0000","last_clean":"2023-10-09T05:28:09.666584+0000","last_became_active":"2023-10-09T05:09:57.857193+0000","last_became_peered":"2023-10-09T05:09:57.857193+0000","last_unstale":"2023-10-09T05:28:09.666584+0000","last_undegraded":"2023-10-09T05:28:09.666584+0000","last_fullsized":"2023-10-09T05:28:09.666584+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:32.842980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":280,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.684905+0000","last_change":"2023-10-09T05:18:26.422923+0000","last_active":"2023-10-09T05:28:09.684905+0000","last_peered":"2023-10-09T05:28:09.684905+0000","last_clean":"2023-10-09T05:28:09.684905+0000","last_became_active":"2023-10-09T05:18:26.422508+0000","last_became_peered":"2023-10-09T05:18:26.422508+0000","last_unstale":"2023-10-09T05:28:09.684905+0000","last_undegraded":"2023-10-09T05:28:09.684905+0000","last_fullsized":"2023-10-09T05:28:09.684905+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:00.876037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":300,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.307362+0000","last_change":"2023-10-09T05:23:18.188378+0000","last_active":"2023-10-09T05:28:32.307362+0000","last_peered":"2023-10-09T05:28:32.307362+0000","last_clean":"2023-10-09T05:28:32.307362+0000","last_became_active":"2023-10-09T05:23:18.186629+0000","last_became_peered":"2023-10-09T05:23:18.186629+0000","last_unstale":"2023-10-09T05:28:32.307362+0000","last_undegraded":"2023-10-09T05:28:32.307362+0000","last_fullsized":"2023-10-09T05:28:32.307362+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:16.444225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":270,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.680038+0000","last_change":"2023-10-09T05:13:03.039637+0000","last_active":"2023-10-09T05:28:09.680038+0000","last_peered":"2023-10-09T05:28:09.680038+0000","last_clean":"2023-10-09T05:28:09.680038+0000","last_became_active":"2023-10-09T05:13:03.039268+0000","last_became_peered":"2023-10-09T05:13:03.039268+0000","last_unstale":"2023-10-09T05:28:09.680038+0000","last_undegraded":"2023-10-09T05:28:09.680038+0000","last_fullsized":"2023-10-09T05:28:09.680038+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:19.875767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":268,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:19.224274+0000","last_change":"2023-10-09T05:22:47.216769+0000","last_active":"2023-10-09T05:28:19.224274+0000","last_peered":"2023-10-09T05:28:19.224274+0000","last_clean":"2023-10-09T05:28:19.224274+0000","last_became_active":"2023-10-09T05:22:47.215847+0000","last_became_peered":"2023-10-09T05:22:47.215847+0000","last_unstale":"2023-10-09T05:28:19.224274+0000","last_undegraded":"2023-10-09T05:28:19.224274+0000","last_fullsized":"2023-10-09T05:28:19.224274+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:51.392675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":255,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.696403+0000","last_change":"2023-10-09T05:09:57.849695+0000","last_active":"2023-10-09T05:28:09.696403+0000","last_peered":"2023-10-09T05:28:09.696403+0000","last_clean":"2023-10-09T05:28:09.696403+0000","last_became_active":"2023-10-09T05:09:57.848478+0000","last_became_peered":"2023-10-09T05:09:57.848478+0000","last_unstale":"2023-10-09T05:28:09.696403+0000","last_undegraded":"2023-10-09T05:28:09.696403+0000","last_fullsized":"2023-10-09T05:28:09.696403+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:26.115312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":89,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.685358+0000","last_change":"2023-10-09T05:23:18.194144+0000","last_active":"2023-10-09T05:28:09.685358+0000","last_peered":"2023-10-09T05:28:09.685358+0000","last_clean":"2023-10-09T05:28:09.685358+0000","last_became_active":"2023-10-09T05:23:18.193390+0000","last_became_peered":"2023-10-09T05:23:18.193390+0000","last_unstale":"2023-10-09T05:28:09.685358+0000","last_undegraded":"2023-10-09T05:28:09.685358+0000","last_fullsized":"2023-10-09T05:28:09.685358+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:22.549362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":268,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.703680+0000","last_change":"2023-10-09T05:18:26.418431+0000","last_active":"2023-10-09T05:28:09.703680+0000","last_peered":"2023-10-09T05:28:09.703680+0000","last_clean":"2023-10-09T05:28:09.703680+0000","last_became_active":"2023-10-09T05:18:26.418100+0000","last_became_peered":"2023-10-09T05:18:26.418100+0000","last_unstale":"2023-10-09T05:28:09.703680+0000","last_undegraded":"2023-10-09T05:28:09.703680+0000","last_fullsized":"2023-10-09T05:28:09.703680+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:21.241286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":75,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.684965+0000","last_change":"2023-10-09T05:18:26.452552+0000","last_active":"2023-10-09T05:28:09.684965+0000","last_peered":"2023-10-09T05:28:09.684965+0000","last_clean":"2023-10-09T05:28:09.684965+0000","last_became_active":"2023-10-09T05:18:26.428454+0000","last_became_peered":"2023-10-09T05:18:26.428454+0000","last_unstale":"2023-10-09T05:28:09.684965+0000","last_undegraded":"2023-10-09T05:28:09.684965+0000","last_fullsized":"2023-10-09T05:28:09.684965+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:15:52.203861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":40,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.666735+0000","last_change":"2023-10-09T05:23:18.199181+0000","last_active":"2023-10-09T05:28:09.666735+0000","last_peered":"2023-10-09T05:28:09.666735+0000","last_clean":"2023-10-09T05:28:09.666735+0000","last_became_active":"2023-10-09T05:23:18.198401+0000","last_became_peered":"2023-10-09T05:23:18.198401+0000","last_unstale":"2023-10-09T05:28:09.666735+0000","last_undegraded":"2023-10-09T05:28:09.666735+0000","last_fullsized":"2023-10-09T05:28:09.666735+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:55:49.891420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'23","reported_seq":332,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.670220+0000","last_change":"2023-10-09T05:18:26.436165+0000","last_active":"2023-10-09T05:28:09.670220+0000","last_peered":"2023-10-09T05:28:09.670220+0000","last_clean":"2023-10-09T05:28:09.670220+0000","last_became_active":"2023-10-09T05:18:26.430078+0000","last_became_peered":"2023-10-09T05:18:26.430078+0000","last_unstale":"2023-10-09T05:28:09.670220+0000","last_undegraded":"2023-10-09T05:28:09.670220+0000","last_fullsized":"2023-10-09T05:28:09.670220+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:04:48.797166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":260,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.670193+0000","last_change":"2023-10-09T05:09:57.868761+0000","last_active":"2023-10-09T05:28:09.670193+0000","last_peered":"2023-10-09T05:28:09.670193+0000","last_clean":"2023-10-09T05:28:09.670193+0000","last_became_active":"2023-10-09T05:09:57.868121+0000","last_became_peered":"2023-10-09T05:09:57.868121+0000","last_unstale":"2023-10-09T05:28:09.670193+0000","last_undegraded":"2023-10-09T05:28:09.670193+0000","last_fullsized":"2023-10-09T05:28:09.670193+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:43.412235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":75,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.684599+0000","last_change":"2023-10-09T05:18:26.443719+0000","last_active":"2023-10-09T05:28:09.684599+0000","last_peered":"2023-10-09T05:28:09.684599+0000","last_clean":"2023-10-09T05:28:09.684599+0000","last_became_active":"2023-10-09T05:18:26.426718+0000","last_became_peered":"2023-10-09T05:18:26.426718+0000","last_unstale":"2023-10-09T05:28:09.684599+0000","last_undegraded":"2023-10-09T05:28:09.684599+0000","last_fullsized":"2023-10-09T05:28:09.684599+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:49:12.090862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":309,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.697133+0000","last_change":"2023-10-09T05:23:18.187593+0000","last_active":"2023-10-09T05:28:09.697133+0000","last_peered":"2023-10-09T05:28:09.697133+0000","last_clean":"2023-10-09T05:28:09.697133+0000","last_became_active":"2023-10-09T05:23:18.186445+0000","last_became_peered":"2023-10-09T05:23:18.186445+0000","last_unstale":"2023-10-09T05:28:09.697133+0000","last_undegraded":"2023-10-09T05:28:09.697133+0000","last_fullsized":"2023-10-09T05:28:09.697133+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:11.327635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":277,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.680152+0000","last_change":"2023-10-09T05:18:26.424399+0000","last_active":"2023-10-09T05:28:09.680152+0000","last_peered":"2023-10-09T05:28:09.680152+0000","last_clean":"2023-10-09T05:28:09.680152+0000","last_became_active":"2023-10-09T05:18:26.422122+0000","last_became_peered":"2023-10-09T05:18:26.422122+0000","last_unstale":"2023-10-09T05:28:09.680152+0000","last_undegraded":"2023-10-09T05:28:09.680152+0000","last_fullsized":"2023-10-09T05:28:09.680152+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:39.393756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":69,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.685178+0000","last_change":"2023-10-09T05:18:26.422950+0000","last_active":"2023-10-09T05:28:09.685178+0000","last_peered":"2023-10-09T05:28:09.685178+0000","last_clean":"2023-10-09T05:28:09.685178+0000","last_became_active":"2023-10-09T05:18:26.422592+0000","last_became_peered":"2023-10-09T05:18:26.422592+0000","last_unstale":"2023-10-09T05:28:09.685178+0000","last_undegraded":"2023-10-09T05:28:09.685178+0000","last_fullsized":"2023-10-09T05:28:09.685178+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:20.701107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":69,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.684719+0000","last_change":"2023-10-09T05:18:26.429482+0000","last_active":"2023-10-09T05:28:09.684719+0000","last_peered":"2023-10-09T05:28:09.684719+0000","last_clean":"2023-10-09T05:28:09.684719+0000","last_became_active":"2023-10-09T05:18:26.429122+0000","last_became_peered":"2023-10-09T05:18:26.429122+0000","last_unstale":"2023-10-09T05:28:09.684719+0000","last_undegraded":"2023-10-09T05:28:09.684719+0000","last_fullsized":"2023-10-09T05:28:09.684719+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:47.301906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":278,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:19.224236+0000","last_change":"2023-10-09T05:18:26.421678+0000","last_active":"2023-10-09T05:28:19.224236+0000","last_peered":"2023-10-09T05:28:19.224236+0000","last_clean":"2023-10-09T05:28:19.224236+0000","last_became_active":"2023-10-09T05:18:26.419673+0000","last_became_peered":"2023-10-09T05:18:26.419673+0000","last_unstale":"2023-10-09T05:28:19.224236+0000","last_undegraded":"2023-10-09T05:28:19.224236+0000","last_fullsized":"2023-10-09T05:28:19.224236+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:38.876021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":273,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.678969+0000","last_change":"2023-10-09T05:13:03.047339+0000","last_active":"2023-10-09T05:28:09.678969+0000","last_peered":"2023-10-09T05:28:09.678969+0000","last_clean":"2023-10-09T05:28:09.678969+0000","last_became_active":"2023-10-09T05:13:03.040038+0000","last_became_peered":"2023-10-09T05:13:03.040038+0000","last_unstale":"2023-10-09T05:28:09.678969+0000","last_undegraded":"2023-10-09T05:28:09.678969+0000","last_fullsized":"2023-10-09T05:28:09.678969+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:56.726249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":291,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.309885+0000","last_change":"2023-10-09T05:18:26.415173+0000","last_active":"2023-10-09T05:28:32.309885+0000","last_peered":"2023-10-09T05:28:32.309885+0000","last_clean":"2023-10-09T05:28:32.309885+0000","last_became_active":"2023-10-09T05:18:26.414452+0000","last_became_peered":"2023-10-09T05:18:26.414452+0000","last_unstale":"2023-10-09T05:28:32.309885+0000","last_undegraded":"2023-10-09T05:28:32.309885+0000","last_fullsized":"2023-10-09T05:28:32.309885+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:34.379767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"280'11446","reported_seq":11470,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:49.471064+0000","last_change":"2023-10-09T05:24:56.644144+0000","last_active":"2023-10-09T05:28:49.471064+0000","last_peered":"2023-10-09T05:28:49.471064+0000","last_clean":"2023-10-09T05:28:49.471064+0000","last_became_active":"2023-10-09T05:24:56.643459+0000","last_became_peered":"2023-10-09T05:24:56.643459+0000","last_unstale":"2023-10-09T05:28:49.471064+0000","last_undegraded":"2023-10-09T05:28:49.471064+0000","last_fullsized":"2023-10-09T05:28:49.471064+0000","mapping_epoch":264,"log_start":"271'3500","ondisk_log_start":"271'3500","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":7946,"log_dups_size":0,"ondisk_log_size":7946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:39.899186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87556096,"num_objects":1336,"num_object_clones":0,"num_object_copies":4008,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1336,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11446,"num_write_kb":409024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":73,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.684982+0000","last_change":"2023-10-09T05:18:26.446608+0000","last_active":"2023-10-09T05:28:09.684982+0000","last_peered":"2023-10-09T05:28:09.684982+0000","last_clean":"2023-10-09T05:28:09.684982+0000","last_became_active":"2023-10-09T05:18:26.428420+0000","last_became_peered":"2023-10-09T05:18:26.428420+0000","last_unstale":"2023-10-09T05:28:09.684982+0000","last_undegraded":"2023-10-09T05:28:09.684982+0000","last_fullsized":"2023-10-09T05:28:09.684982+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:23.693908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":69,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.684550+0000","last_change":"2023-10-09T05:18:26.427461+0000","last_active":"2023-10-09T05:28:09.684550+0000","last_peered":"2023-10-09T05:28:09.684550+0000","last_clean":"2023-10-09T05:28:09.684550+0000","last_became_active":"2023-10-09T05:18:26.427118+0000","last_became_peered":"2023-10-09T05:18:26.427118+0000","last_unstale":"2023-10-09T05:28:09.684550+0000","last_undegraded":"2023-10-09T05:28:09.684550+0000","last_fullsized":"2023-10-09T05:28:09.684550+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:05:01.868197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.1","version":"280'11422","reported_seq":11446,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:49.475007+0000","last_change":"2023-10-09T05:24:56.643202+0000","last_active":"2023-10-09T05:28:49.475007+0000","last_peered":"2023-10-09T05:28:49.475007+0000","last_clean":"2023-10-09T05:28:49.475007+0000","last_became_active":"2023-10-09T05:24:56.642700+0000","last_became_peered":"2023-10-09T05:24:56.642700+0000","last_unstale":"2023-10-09T05:28:49.475007+0000","last_undegraded":"2023-10-09T05:28:49.475007+0000","last_fullsized":"2023-10-09T05:28:49.475007+0000","mapping_epoch":264,"log_start":"271'3700","ondisk_log_start":"271'3700","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":7722,"log_dups_size":0,"ondisk_log_size":7722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:25.444496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89784320,"num_objects":1370,"num_object_clones":0,"num_object_copies":4110,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1370,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11422,"num_write_kb":409344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":73,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.685322+0000","last_change":"2023-10-09T05:18:26.440042+0000","last_active":"2023-10-09T05:28:09.685322+0000","last_peered":"2023-10-09T05:28:09.685322+0000","last_clean":"2023-10-09T05:28:09.685322+0000","last_became_active":"2023-10-09T05:18:26.426726+0000","last_became_peered":"2023-10-09T05:18:26.426726+0000","last_unstale":"2023-10-09T05:28:09.685322+0000","last_undegraded":"2023-10-09T05:28:09.685322+0000","last_fullsized":"2023-10-09T05:28:09.685322+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:23.985650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":277,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.669984+0000","last_change":"2023-10-09T05:09:57.868031+0000","last_active":"2023-10-09T05:28:09.669984+0000","last_peered":"2023-10-09T05:28:09.669984+0000","last_clean":"2023-10-09T05:28:09.669984+0000","last_became_active":"2023-10-09T05:09:57.866477+0000","last_became_peered":"2023-10-09T05:09:57.866477+0000","last_unstale":"2023-10-09T05:28:09.669984+0000","last_undegraded":"2023-10-09T05:28:09.669984+0000","last_fullsized":"2023-10-09T05:28:09.669984+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:27.102583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"280'11436","reported_seq":11460,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:49.427387+0000","last_change":"2023-10-09T05:24:56.640700+0000","last_active":"2023-10-09T05:28:49.427387+0000","last_peered":"2023-10-09T05:28:49.427387+0000","last_clean":"2023-10-09T05:28:49.427387+0000","last_became_active":"2023-10-09T05:24:56.640352+0000","last_became_peered":"2023-10-09T05:24:56.640352+0000","last_unstale":"2023-10-09T05:28:49.427387+0000","last_undegraded":"2023-10-09T05:28:49.427387+0000","last_fullsized":"2023-10-09T05:28:49.427387+0000","mapping_epoch":264,"log_start":"271'3500","ondisk_log_start":"271'3500","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":7936,"log_dups_size":0,"ondisk_log_size":7936,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:55:08.218749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89653248,"num_objects":1368,"num_object_clones":0,"num_object_copies":4104,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1368,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11436,"num_write_kb":409728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":73,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.685055+0000","last_change":"2023-10-09T05:18:26.437953+0000","last_active":"2023-10-09T05:28:09.685055+0000","last_peered":"2023-10-09T05:28:09.685055+0000","last_clean":"2023-10-09T05:28:09.685055+0000","last_became_active":"2023-10-09T05:18:26.422703+0000","last_became_peered":"2023-10-09T05:18:26.422703+0000","last_unstale":"2023-10-09T05:28:09.685055+0000","last_undegraded":"2023-10-09T05:28:09.685055+0000","last_fullsized":"2023-10-09T05:28:09.685055+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:13.926811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":310,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.679399+0000","last_change":"2023-10-09T05:18:26.423526+0000","last_active":"2023-10-09T05:28:09.679399+0000","last_peered":"2023-10-09T05:28:09.679399+0000","last_clean":"2023-10-09T05:28:09.679399+0000","last_became_active":"2023-10-09T05:18:26.423092+0000","last_became_peered":"2023-10-09T05:18:26.423092+0000","last_unstale":"2023-10-09T05:28:09.679399+0000","last_undegraded":"2023-10-09T05:28:09.679399+0000","last_fullsized":"2023-10-09T05:28:09.679399+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:54.877664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"280'11440","reported_seq":11465,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:49.272430+0000","last_change":"2023-10-09T05:24:56.634648+0000","last_active":"2023-10-09T05:28:49.272430+0000","last_peered":"2023-10-09T05:28:49.272430+0000","last_clean":"2023-10-09T05:28:49.272430+0000","last_became_active":"2023-10-09T05:24:56.633771+0000","last_became_peered":"2023-10-09T05:24:56.633771+0000","last_unstale":"2023-10-09T05:28:49.272430+0000","last_undegraded":"2023-10-09T05:28:49.272430+0000","last_fullsized":"2023-10-09T05:28:49.272430+0000","mapping_epoch":264,"log_start":"271'3300","ondisk_log_start":"271'3300","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":8140,"log_dups_size":0,"ondisk_log_size":8140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:20:25.848591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89522176,"num_objects":1366,"num_object_clones":0,"num_object_copies":4098,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1366,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11440,"num_write_kb":409792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":202,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.309742+0000","last_change":"2023-10-09T05:13:03.037926+0000","last_active":"2023-10-09T05:28:32.309742+0000","last_peered":"2023-10-09T05:28:32.309742+0000","last_clean":"2023-10-09T05:28:32.309742+0000","last_became_active":"2023-10-09T05:13:03.037639+0000","last_became_peered":"2023-10-09T05:13:03.037639+0000","last_unstale":"2023-10-09T05:28:32.309742+0000","last_undegraded":"2023-10-09T05:28:32.309742+0000","last_fullsized":"2023-10-09T05:28:32.309742+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:52.977143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":261,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:24.225811+0000","last_change":"2023-10-09T05:09:57.852914+0000","last_active":"2023-10-09T05:28:24.225811+0000","last_peered":"2023-10-09T05:28:24.225811+0000","last_clean":"2023-10-09T05:28:24.225811+0000","last_became_active":"2023-10-09T05:09:57.852522+0000","last_became_peered":"2023-10-09T05:09:57.852522+0000","last_unstale":"2023-10-09T05:28:24.225811+0000","last_undegraded":"2023-10-09T05:28:24.225811+0000","last_fullsized":"2023-10-09T05:28:24.225811+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:24.848494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"280'11517","reported_seq":11541,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:49.476228+0000","last_change":"2023-10-09T05:24:56.639909+0000","last_active":"2023-10-09T05:28:49.476228+0000","last_peered":"2023-10-09T05:28:49.476228+0000","last_clean":"2023-10-09T05:28:49.476228+0000","last_became_active":"2023-10-09T05:24:56.639434+0000","last_became_peered":"2023-10-09T05:24:56.639434+0000","last_unstale":"2023-10-09T05:28:49.476228+0000","last_undegraded":"2023-10-09T05:28:49.476228+0000","last_fullsized":"2023-10-09T05:28:49.476228+0000","mapping_epoch":264,"log_start":"272'3800","ondisk_log_start":"272'3800","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":7717,"log_dups_size":0,"ondisk_log_size":7717,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:11.231733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90767360,"num_objects":1385,"num_object_clones":0,"num_object_copies":4155,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1385,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11517,"num_write_kb":412864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"69'3","reported_seq":234,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.699564+0000","last_change":"2023-10-09T05:02:09.650579+0000","last_active":"2023-10-09T05:28:09.699564+0000","last_peered":"2023-10-09T05:28:09.699564+0000","last_clean":"2023-10-09T05:28:09.699564+0000","last_became_active":"2023-10-09T05:02:09.650160+0000","last_became_peered":"2023-10-09T05:02:09.650160+0000","last_unstale":"2023-10-09T05:28:09.699564+0000","last_undegraded":"2023-10-09T05:28:09.699564+0000","last_fullsized":"2023-10-09T05:28:09.699564+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:18.292670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":240,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.670400+0000","last_change":"2023-10-09T05:02:09.652449+0000","last_active":"2023-10-09T05:28:09.670400+0000","last_peered":"2023-10-09T05:28:09.670400+0000","last_clean":"2023-10-09T05:28:09.670400+0000","last_became_active":"2023-10-09T05:02:09.651994+0000","last_became_peered":"2023-10-09T05:02:09.651994+0000","last_unstale":"2023-10-09T05:28:09.670400+0000","last_undegraded":"2023-10-09T05:28:09.670400+0000","last_fullsized":"2023-10-09T05:28:09.670400+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:14.480513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"280'11436","reported_seq":11460,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:49.470071+0000","last_change":"2023-10-09T05:24:56.643292+0000","last_active":"2023-10-09T05:28:49.470071+0000","last_peered":"2023-10-09T05:28:49.470071+0000","last_clean":"2023-10-09T05:28:49.470071+0000","last_became_active":"2023-10-09T05:24:56.642863+0000","last_became_peered":"2023-10-09T05:24:56.642863+0000","last_unstale":"2023-10-09T05:28:49.470071+0000","last_undegraded":"2023-10-09T05:28:49.470071+0000","last_fullsized":"2023-10-09T05:28:49.470071+0000","mapping_epoch":264,"log_start":"271'3700","ondisk_log_start":"271'3700","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":7736,"log_dups_size":0,"ondisk_log_size":7736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:09.977734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89653248,"num_objects":1368,"num_object_clones":0,"num_object_copies":4104,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1368,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11436,"num_write_kb":409728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":281,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.679570+0000","last_change":"2023-10-09T05:23:18.199172+0000","last_active":"2023-10-09T05:28:09.679570+0000","last_peered":"2023-10-09T05:28:09.679570+0000","last_clean":"2023-10-09T05:28:09.679570+0000","last_became_active":"2023-10-09T05:23:18.198762+0000","last_became_peered":"2023-10-09T05:23:18.198762+0000","last_unstale":"2023-10-09T05:28:09.679570+0000","last_undegraded":"2023-10-09T05:28:09.679570+0000","last_fullsized":"2023-10-09T05:28:09.679570+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:44:23.239788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":297,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.670296+0000","last_change":"2023-10-09T05:20:18.129228+0000","last_active":"2023-10-09T05:28:09.670296+0000","last_peered":"2023-10-09T05:28:09.670296+0000","last_clean":"2023-10-09T05:28:09.670296+0000","last_became_active":"2023-10-09T05:20:18.128769+0000","last_became_peered":"2023-10-09T05:20:18.128769+0000","last_unstale":"2023-10-09T05:28:09.670296+0000","last_undegraded":"2023-10-09T05:28:09.670296+0000","last_fullsized":"2023-10-09T05:28:09.670296+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:50.427100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"280'11436","reported_seq":11462,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:50.108698+0000","last_change":"2023-10-09T05:24:56.631754+0000","last_active":"2023-10-09T05:28:50.108698+0000","last_peered":"2023-10-09T05:28:50.108698+0000","last_clean":"2023-10-09T05:28:50.108698+0000","last_became_active":"2023-10-09T05:24:56.631413+0000","last_became_peered":"2023-10-09T05:24:56.631413+0000","last_unstale":"2023-10-09T05:28:50.108698+0000","last_undegraded":"2023-10-09T05:28:50.108698+0000","last_fullsized":"2023-10-09T05:28:50.108698+0000","mapping_epoch":264,"log_start":"272'3900","ondisk_log_start":"272'3900","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":7536,"log_dups_size":0,"ondisk_log_size":7536,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:06.416230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89128960,"num_objects":1360,"num_object_clones":0,"num_object_copies":4080,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1360,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11436,"num_write_kb":409472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":281,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.669973+0000","last_change":"2023-10-09T05:18:26.431286+0000","last_active":"2023-10-09T05:28:09.669973+0000","last_peered":"2023-10-09T05:28:09.669973+0000","last_clean":"2023-10-09T05:28:09.669973+0000","last_became_active":"2023-10-09T05:18:26.430127+0000","last_became_peered":"2023-10-09T05:18:26.430127+0000","last_unstale":"2023-10-09T05:28:09.669973+0000","last_undegraded":"2023-10-09T05:28:09.669973+0000","last_fullsized":"2023-10-09T05:28:09.669973+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:42.982259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":280,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.670667+0000","last_change":"2023-10-09T05:23:18.193800+0000","last_active":"2023-10-09T05:28:09.670667+0000","last_peered":"2023-10-09T05:28:09.670667+0000","last_clean":"2023-10-09T05:28:09.670667+0000","last_became_active":"2023-10-09T05:23:18.193338+0000","last_became_peered":"2023-10-09T05:23:18.193338+0000","last_unstale":"2023-10-09T05:28:09.670667+0000","last_undegraded":"2023-10-09T05:28:09.670667+0000","last_fullsized":"2023-10-09T05:28:09.670667+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:29.109566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"280'11512","reported_seq":11538,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:50.071989+0000","last_change":"2023-10-09T05:24:56.632416+0000","last_active":"2023-10-09T05:28:50.071989+0000","last_peered":"2023-10-09T05:28:50.071989+0000","last_clean":"2023-10-09T05:28:50.071989+0000","last_became_active":"2023-10-09T05:24:56.632033+0000","last_became_peered":"2023-10-09T05:24:56.632033+0000","last_unstale":"2023-10-09T05:28:50.071989+0000","last_undegraded":"2023-10-09T05:28:50.071989+0000","last_fullsized":"2023-10-09T05:28:50.071989+0000","mapping_epoch":264,"log_start":"272'4000","ondisk_log_start":"272'4000","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":7512,"log_dups_size":0,"ondisk_log_size":7512,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:08.729587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83230720,"num_objects":1270,"num_object_clones":0,"num_object_copies":3810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11512,"num_write_kb":409024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":293,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.997960+0000","last_change":"2023-10-09T05:18:26.413106+0000","last_active":"2023-10-09T05:28:09.997960+0000","last_peered":"2023-10-09T05:28:09.997960+0000","last_clean":"2023-10-09T05:28:09.997960+0000","last_became_active":"2023-10-09T05:18:26.412334+0000","last_became_peered":"2023-10-09T05:18:26.412334+0000","last_unstale":"2023-10-09T05:28:09.997960+0000","last_undegraded":"2023-10-09T05:28:09.997960+0000","last_fullsized":"2023-10-09T05:28:09.997960+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:18.058016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":147,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:10.065435+0000","last_change":"2023-10-09T05:09:57.856764+0000","last_active":"2023-10-09T05:28:10.065435+0000","last_peered":"2023-10-09T05:28:10.065435+0000","last_clean":"2023-10-09T05:28:10.065435+0000","last_became_active":"2023-10-09T05:09:57.856461+0000","last_became_peered":"2023-10-09T05:09:57.856461+0000","last_unstale":"2023-10-09T05:28:10.065435+0000","last_undegraded":"2023-10-09T05:28:10.065435+0000","last_fullsized":"2023-10-09T05:28:10.065435+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:06.848082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.a","version":"280'11631","reported_seq":11657,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:48.393567+0000","last_change":"2023-10-09T05:24:56.632715+0000","last_active":"2023-10-09T05:28:48.393567+0000","last_peered":"2023-10-09T05:28:48.393567+0000","last_clean":"2023-10-09T05:28:48.393567+0000","last_became_active":"2023-10-09T05:24:56.632411+0000","last_became_peered":"2023-10-09T05:24:56.632411+0000","last_unstale":"2023-10-09T05:28:48.393567+0000","last_undegraded":"2023-10-09T05:28:48.393567+0000","last_fullsized":"2023-10-09T05:28:48.393567+0000","mapping_epoch":264,"log_start":"270'1900","ondisk_log_start":"270'1900","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":9731,"log_dups_size":0,"ondisk_log_size":9731,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:25.311429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89325568,"num_objects":1363,"num_object_clones":0,"num_object_copies":4089,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1363,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11631,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":183,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.667366+0000","last_change":"2023-10-09T05:18:26.422150+0000","last_active":"2023-10-09T05:28:09.667366+0000","last_peered":"2023-10-09T05:28:09.667366+0000","last_clean":"2023-10-09T05:28:09.667366+0000","last_became_active":"2023-10-09T05:18:26.421853+0000","last_became_peered":"2023-10-09T05:18:26.421853+0000","last_unstale":"2023-10-09T05:28:09.667366+0000","last_undegraded":"2023-10-09T05:28:09.667366+0000","last_fullsized":"2023-10-09T05:28:09.667366+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:37:47.000556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":219,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.696882+0000","last_change":"2023-10-09T05:13:03.031243+0000","last_active":"2023-10-09T05:28:09.696882+0000","last_peered":"2023-10-09T05:28:09.696882+0000","last_clean":"2023-10-09T05:28:09.696882+0000","last_became_active":"2023-10-09T05:13:03.028769+0000","last_became_peered":"2023-10-09T05:13:03.028769+0000","last_unstale":"2023-10-09T05:28:09.696882+0000","last_undegraded":"2023-10-09T05:28:09.696882+0000","last_fullsized":"2023-10-09T05:28:09.696882+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:25:25.239799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":40,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.666648+0000","last_change":"2023-10-09T05:23:18.197927+0000","last_active":"2023-10-09T05:28:09.666648+0000","last_peered":"2023-10-09T05:28:09.666648+0000","last_clean":"2023-10-09T05:28:09.666648+0000","last_became_active":"2023-10-09T05:23:18.197431+0000","last_became_peered":"2023-10-09T05:23:18.197431+0000","last_unstale":"2023-10-09T05:28:09.666648+0000","last_undegraded":"2023-10-09T05:28:09.666648+0000","last_fullsized":"2023-10-09T05:28:09.666648+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:51:41.376456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"280'11353","reported_seq":11379,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:49.218802+0000","last_change":"2023-10-09T05:24:56.631792+0000","last_active":"2023-10-09T05:28:49.218802+0000","last_peered":"2023-10-09T05:28:49.218802+0000","last_clean":"2023-10-09T05:28:49.218802+0000","last_became_active":"2023-10-09T05:24:56.631473+0000","last_became_peered":"2023-10-09T05:24:56.631473+0000","last_unstale":"2023-10-09T05:28:49.218802+0000","last_undegraded":"2023-10-09T05:28:49.218802+0000","last_fullsized":"2023-10-09T05:28:49.218802+0000","mapping_epoch":264,"log_start":"271'3200","ondisk_log_start":"271'3200","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":8153,"log_dups_size":0,"ondisk_log_size":8153,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:17.170510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92209152,"num_objects":1407,"num_object_clones":0,"num_object_copies":4221,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1407,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11353,"num_write_kb":408320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":253,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.666823+0000","last_change":"2023-10-09T05:13:03.031386+0000","last_active":"2023-10-09T05:28:09.666823+0000","last_peered":"2023-10-09T05:28:09.666823+0000","last_clean":"2023-10-09T05:28:09.666823+0000","last_became_active":"2023-10-09T05:13:03.030816+0000","last_became_peered":"2023-10-09T05:13:03.030816+0000","last_unstale":"2023-10-09T05:28:09.666823+0000","last_undegraded":"2023-10-09T05:28:09.666823+0000","last_fullsized":"2023-10-09T05:28:09.666823+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:28.088661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":241,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.670127+0000","last_change":"2023-10-09T04:56:56.087320+0000","last_active":"2023-10-09T05:28:09.670127+0000","last_peered":"2023-10-09T05:28:09.670127+0000","last_clean":"2023-10-09T05:28:09.670127+0000","last_became_active":"2023-10-09T04:56:56.086552+0000","last_became_peered":"2023-10-09T04:56:56.086552+0000","last_unstale":"2023-10-09T05:28:09.670127+0000","last_undegraded":"2023-10-09T05:28:09.670127+0000","last_fullsized":"2023-10-09T05:28:09.670127+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:14.205288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":40,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.667226+0000","last_change":"2023-10-09T05:23:18.196260+0000","last_active":"2023-10-09T05:28:09.667226+0000","last_peered":"2023-10-09T05:28:09.667226+0000","last_clean":"2023-10-09T05:28:09.667226+0000","last_became_active":"2023-10-09T05:23:18.195907+0000","last_became_peered":"2023-10-09T05:23:18.195907+0000","last_unstale":"2023-10-09T05:28:09.667226+0000","last_undegraded":"2023-10-09T05:28:09.667226+0000","last_fullsized":"2023-10-09T05:28:09.667226+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:36.243432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.f","version":"280'11522","reported_seq":11546,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:49.502838+0000","last_change":"2023-10-09T05:24:56.644098+0000","last_active":"2023-10-09T05:28:49.502838+0000","last_peered":"2023-10-09T05:28:49.502838+0000","last_clean":"2023-10-09T05:28:49.502838+0000","last_became_active":"2023-10-09T05:24:56.643666+0000","last_became_peered":"2023-10-09T05:24:56.643666+0000","last_unstale":"2023-10-09T05:28:49.502838+0000","last_undegraded":"2023-10-09T05:28:49.502838+0000","last_fullsized":"2023-10-09T05:28:49.502838+0000","mapping_epoch":264,"log_start":"271'3800","ondisk_log_start":"271'3800","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":7722,"log_dups_size":0,"ondisk_log_size":7722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:39:36.431548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87425024,"num_objects":1334,"num_object_clones":0,"num_object_copies":4002,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1334,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11522,"num_write_kb":411392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":59,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.306319+0000","last_change":"2023-10-09T05:20:18.123706+0000","last_active":"2023-10-09T05:28:32.306319+0000","last_peered":"2023-10-09T05:28:32.306319+0000","last_clean":"2023-10-09T05:28:32.306319+0000","last_became_active":"2023-10-09T05:20:18.122859+0000","last_became_peered":"2023-10-09T05:20:18.122859+0000","last_unstale":"2023-10-09T05:28:32.306319+0000","last_undegraded":"2023-10-09T05:28:32.306319+0000","last_fullsized":"2023-10-09T05:28:32.306319+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.906228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":148,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:10.065282+0000","last_change":"2023-10-09T05:09:57.854064+0000","last_active":"2023-10-09T05:28:10.065282+0000","last_peered":"2023-10-09T05:28:10.065282+0000","last_clean":"2023-10-09T05:28:10.065282+0000","last_became_active":"2023-10-09T05:09:57.847953+0000","last_became_peered":"2023-10-09T05:09:57.847953+0000","last_unstale":"2023-10-09T05:28:10.065282+0000","last_undegraded":"2023-10-09T05:28:10.065282+0000","last_fullsized":"2023-10-09T05:28:10.065282+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:45.914976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":23,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.713098+0000","last_change":"2023-10-09T05:25:15.659153+0000","last_active":"2023-10-09T05:28:09.713098+0000","last_peered":"2023-10-09T05:28:09.713098+0000","last_clean":"2023-10-09T05:28:09.713098+0000","last_became_active":"2023-10-09T05:25:15.658750+0000","last_became_peered":"2023-10-09T05:25:15.658750+0000","last_unstale":"2023-10-09T05:28:09.713098+0000","last_undegraded":"2023-10-09T05:28:09.713098+0000","last_fullsized":"2023-10-09T05:28:09.713098+0000","mapping_epoch":268,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":269,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:10.370264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.d","version":"280'11432","reported_seq":11457,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:49.542877+0000","last_change":"2023-10-09T05:24:56.633695+0000","last_active":"2023-10-09T05:28:49.542877+0000","last_peered":"2023-10-09T05:28:49.542877+0000","last_clean":"2023-10-09T05:28:49.542877+0000","last_became_active":"2023-10-09T05:24:56.633393+0000","last_became_peered":"2023-10-09T05:24:56.633393+0000","last_unstale":"2023-10-09T05:28:49.542877+0000","last_undegraded":"2023-10-09T05:28:49.542877+0000","last_fullsized":"2023-10-09T05:28:49.542877+0000","mapping_epoch":264,"log_start":"271'3700","ondisk_log_start":"271'3700","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":7732,"log_dups_size":0,"ondisk_log_size":7732,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:04.072215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89587736,"num_objects":1368,"num_object_clones":0,"num_object_copies":4104,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1368,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11432,"num_write_kb":409537,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":225,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.674437+0000","last_change":"2023-10-09T05:18:26.416511+0000","last_active":"2023-10-09T05:28:09.674437+0000","last_peered":"2023-10-09T05:28:09.674437+0000","last_clean":"2023-10-09T05:28:09.674437+0000","last_became_active":"2023-10-09T05:18:26.416101+0000","last_became_peered":"2023-10-09T05:18:26.416101+0000","last_unstale":"2023-10-09T05:28:09.674437+0000","last_undegraded":"2023-10-09T05:28:09.674437+0000","last_fullsized":"2023-10-09T05:28:09.674437+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:10.243472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":565,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.684359+0000","last_change":"2023-10-09T05:18:29.419527+0000","last_active":"2023-10-09T05:28:09.684359+0000","last_peered":"2023-10-09T05:28:09.684359+0000","last_clean":"2023-10-09T05:28:09.684359+0000","last_became_active":"2023-10-09T05:18:29.418852+0000","last_became_peered":"2023-10-09T05:18:29.418852+0000","last_unstale":"2023-10-09T05:28:09.684359+0000","last_undegraded":"2023-10-09T05:28:09.684359+0000","last_fullsized":"2023-10-09T05:28:09.684359+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_clean_scrub_stamp":"2023-10-09T04:36:44.261013+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:02.899170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":242,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.730609+0000","last_change":"2023-10-09T05:13:03.027260+0000","last_active":"2023-10-09T05:28:09.730609+0000","last_peered":"2023-10-09T05:28:09.730609+0000","last_clean":"2023-10-09T05:28:09.730609+0000","last_became_active":"2023-10-09T05:13:03.026416+0000","last_became_peered":"2023-10-09T05:13:03.026416+0000","last_unstale":"2023-10-09T05:28:09.730609+0000","last_undegraded":"2023-10-09T05:28:09.730609+0000","last_fullsized":"2023-10-09T05:28:09.730609+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:05.057338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":263,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.684548+0000","last_change":"2023-10-09T05:18:26.426126+0000","last_active":"2023-10-09T05:28:09.684548+0000","last_peered":"2023-10-09T05:28:09.684548+0000","last_clean":"2023-10-09T05:28:09.684548+0000","last_became_active":"2023-10-09T05:18:26.425721+0000","last_became_peered":"2023-10-09T05:18:26.425721+0000","last_unstale":"2023-10-09T05:28:09.684548+0000","last_undegraded":"2023-10-09T05:28:09.684548+0000","last_fullsized":"2023-10-09T05:28:09.684548+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:48.229218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"280'11605","reported_seq":11631,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:49.173295+0000","last_change":"2023-10-09T05:24:56.641295+0000","last_active":"2023-10-09T05:28:49.173295+0000","last_peered":"2023-10-09T05:28:49.173295+0000","last_clean":"2023-10-09T05:28:49.173295+0000","last_became_active":"2023-10-09T05:24:56.640909+0000","last_became_peered":"2023-10-09T05:24:56.640909+0000","last_unstale":"2023-10-09T05:28:49.173295+0000","last_undegraded":"2023-10-09T05:28:49.173295+0000","last_fullsized":"2023-10-09T05:28:49.173295+0000","mapping_epoch":264,"log_start":"271'3400","ondisk_log_start":"271'3400","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":8205,"log_dups_size":0,"ondisk_log_size":8205,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:26.475478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94830592,"num_objects":1447,"num_object_clones":0,"num_object_copies":4341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1447,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11605,"num_write_kb":417664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":89,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.684464+0000","last_change":"2023-10-09T05:23:18.193825+0000","last_active":"2023-10-09T05:28:09.684464+0000","last_peered":"2023-10-09T05:28:09.684464+0000","last_clean":"2023-10-09T05:28:09.684464+0000","last_became_active":"2023-10-09T05:23:18.193394+0000","last_became_peered":"2023-10-09T05:23:18.193394+0000","last_unstale":"2023-10-09T05:28:09.684464+0000","last_undegraded":"2023-10-09T05:28:09.684464+0000","last_fullsized":"2023-10-09T05:28:09.684464+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:40.594469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":243,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.680126+0000","last_change":"2023-10-09T05:02:09.648781+0000","last_active":"2023-10-09T05:28:09.680126+0000","last_peered":"2023-10-09T05:28:09.680126+0000","last_clean":"2023-10-09T05:28:09.680126+0000","last_became_active":"2023-10-09T05:02:09.648078+0000","last_became_peered":"2023-10-09T05:02:09.648078+0000","last_unstale":"2023-10-09T05:28:09.680126+0000","last_undegraded":"2023-10-09T05:28:09.680126+0000","last_fullsized":"2023-10-09T05:28:09.680126+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:35.287091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":321,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.681633+0000","last_change":"2023-10-09T05:20:18.132750+0000","last_active":"2023-10-09T05:28:09.681633+0000","last_peered":"2023-10-09T05:28:09.681633+0000","last_clean":"2023-10-09T05:28:09.681633+0000","last_became_active":"2023-10-09T05:20:18.132396+0000","last_became_peered":"2023-10-09T05:20:18.132396+0000","last_unstale":"2023-10-09T05:28:09.681633+0000","last_undegraded":"2023-10-09T05:28:09.681633+0000","last_fullsized":"2023-10-09T05:28:09.681633+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:00:55.731045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"280'11449","reported_seq":11475,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:50.121283+0000","last_change":"2023-10-09T05:24:56.642285+0000","last_active":"2023-10-09T05:28:50.121283+0000","last_peered":"2023-10-09T05:28:50.121283+0000","last_clean":"2023-10-09T05:28:50.121283+0000","last_became_active":"2023-10-09T05:24:56.641521+0000","last_became_peered":"2023-10-09T05:24:56.641521+0000","last_unstale":"2023-10-09T05:28:50.121283+0000","last_undegraded":"2023-10-09T05:28:50.121283+0000","last_fullsized":"2023-10-09T05:28:50.121283+0000","mapping_epoch":264,"log_start":"272'4400","ondisk_log_start":"272'4400","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":7049,"log_dups_size":0,"ondisk_log_size":7049,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:06:01.013563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89456640,"num_objects":1365,"num_object_clones":0,"num_object_copies":4095,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1365,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11449,"num_write_kb":410048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":69,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.685130+0000","last_change":"2023-10-09T05:18:26.429468+0000","last_active":"2023-10-09T05:28:09.685130+0000","last_peered":"2023-10-09T05:28:09.685130+0000","last_clean":"2023-10-09T05:28:09.685130+0000","last_became_active":"2023-10-09T05:18:26.429098+0000","last_became_peered":"2023-10-09T05:18:26.429098+0000","last_unstale":"2023-10-09T05:28:09.685130+0000","last_undegraded":"2023-10-09T05:28:09.685130+0000","last_fullsized":"2023-10-09T05:28:09.685130+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:47.798226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":174,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.667465+0000","last_change":"2023-10-09T05:20:18.126997+0000","last_active":"2023-10-09T05:28:09.667465+0000","last_peered":"2023-10-09T05:28:09.667465+0000","last_clean":"2023-10-09T05:28:09.667465+0000","last_became_active":"2023-10-09T05:20:18.126641+0000","last_became_peered":"2023-10-09T05:20:18.126641+0000","last_unstale":"2023-10-09T05:28:09.667465+0000","last_undegraded":"2023-10-09T05:28:09.667465+0000","last_fullsized":"2023-10-09T05:28:09.667465+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:42.112245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":283,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.680398+0000","last_change":"2023-10-09T05:18:26.418524+0000","last_active":"2023-10-09T05:28:09.680398+0000","last_peered":"2023-10-09T05:28:09.680398+0000","last_clean":"2023-10-09T05:28:09.680398+0000","last_became_active":"2023-10-09T05:18:26.418058+0000","last_became_peered":"2023-10-09T05:18:26.418058+0000","last_unstale":"2023-10-09T05:28:09.680398+0000","last_undegraded":"2023-10-09T05:28:09.680398+0000","last_fullsized":"2023-10-09T05:28:09.680398+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:40.594449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"280'11526","reported_seq":11552,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:50.094693+0000","last_change":"2023-10-09T05:24:56.633539+0000","last_active":"2023-10-09T05:28:50.094693+0000","last_peered":"2023-10-09T05:28:50.094693+0000","last_clean":"2023-10-09T05:28:50.094693+0000","last_became_active":"2023-10-09T05:24:56.633197+0000","last_became_peered":"2023-10-09T05:24:56.633197+0000","last_unstale":"2023-10-09T05:28:50.094693+0000","last_undegraded":"2023-10-09T05:28:50.094693+0000","last_fullsized":"2023-10-09T05:28:50.094693+0000","mapping_epoch":264,"log_start":"272'4500","ondisk_log_start":"272'4500","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":7026,"log_dups_size":0,"ondisk_log_size":7026,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:53.863072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88997888,"num_objects":1358,"num_object_clones":0,"num_object_copies":4074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1358,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11526,"num_write_kb":412288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":286,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.998128+0000","last_change":"2023-10-09T05:18:26.431563+0000","last_active":"2023-10-09T05:28:09.998128+0000","last_peered":"2023-10-09T05:28:09.998128+0000","last_clean":"2023-10-09T05:28:09.998128+0000","last_became_active":"2023-10-09T05:18:26.421025+0000","last_became_peered":"2023-10-09T05:18:26.421025+0000","last_unstale":"2023-10-09T05:28:09.998128+0000","last_undegraded":"2023-10-09T05:28:09.998128+0000","last_fullsized":"2023-10-09T05:28:09.998128+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:30.333585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":40,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.667354+0000","last_change":"2023-10-09T05:23:18.196610+0000","last_active":"2023-10-09T05:28:09.667354+0000","last_peered":"2023-10-09T05:28:09.667354+0000","last_clean":"2023-10-09T05:28:09.667354+0000","last_became_active":"2023-10-09T05:23:18.196279+0000","last_became_peered":"2023-10-09T05:23:18.196279+0000","last_unstale":"2023-10-09T05:28:09.667354+0000","last_undegraded":"2023-10-09T05:28:09.667354+0000","last_fullsized":"2023-10-09T05:28:09.667354+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:52.868384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":285,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:14.998821+0000","last_change":"2023-10-09T05:21:08.179657+0000","last_active":"2023-10-09T05:28:14.998821+0000","last_peered":"2023-10-09T05:28:14.998821+0000","last_clean":"2023-10-09T05:28:14.998821+0000","last_became_active":"2023-10-09T05:21:08.178958+0000","last_became_peered":"2023-10-09T05:21:08.178958+0000","last_unstale":"2023-10-09T05:28:14.998821+0000","last_undegraded":"2023-10-09T05:28:14.998821+0000","last_fullsized":"2023-10-09T05:28:14.998821+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:20.815986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"280'11450","reported_seq":11475,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:48.768678+0000","last_change":"2023-10-09T05:24:56.641633+0000","last_active":"2023-10-09T05:28:48.768678+0000","last_peered":"2023-10-09T05:28:48.768678+0000","last_clean":"2023-10-09T05:28:48.768678+0000","last_became_active":"2023-10-09T05:24:56.641288+0000","last_became_peered":"2023-10-09T05:24:56.641288+0000","last_unstale":"2023-10-09T05:28:48.768678+0000","last_undegraded":"2023-10-09T05:28:48.768678+0000","last_fullsized":"2023-10-09T05:28:48.768678+0000","mapping_epoch":264,"log_start":"270'1700","ondisk_log_start":"270'1700","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":9750,"log_dups_size":0,"ondisk_log_size":9750,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:19.463846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91881472,"num_objects":1402,"num_object_clones":0,"num_object_copies":4206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11450,"num_write_kb":411264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":285,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.679924+0000","last_change":"2023-10-09T05:18:26.424429+0000","last_active":"2023-10-09T05:28:09.679924+0000","last_peered":"2023-10-09T05:28:09.679924+0000","last_clean":"2023-10-09T05:28:09.679924+0000","last_became_active":"2023-10-09T05:18:26.423034+0000","last_became_peered":"2023-10-09T05:18:26.423034+0000","last_unstale":"2023-10-09T05:28:09.679924+0000","last_undegraded":"2023-10-09T05:28:09.679924+0000","last_fullsized":"2023-10-09T05:28:09.679924+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:16.251181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":273,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.681725+0000","last_change":"2023-10-09T05:09:57.878002+0000","last_active":"2023-10-09T05:28:09.681725+0000","last_peered":"2023-10-09T05:28:09.681725+0000","last_clean":"2023-10-09T05:28:09.681725+0000","last_became_active":"2023-10-09T05:09:57.852792+0000","last_became_peered":"2023-10-09T05:09:57.852792+0000","last_unstale":"2023-10-09T05:28:09.681725+0000","last_undegraded":"2023-10-09T05:28:09.681725+0000","last_fullsized":"2023-10-09T05:28:09.681725+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:18.763275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":233,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.669859+0000","last_change":"2023-10-09T05:13:03.026541+0000","last_active":"2023-10-09T05:28:09.669859+0000","last_peered":"2023-10-09T05:28:09.669859+0000","last_clean":"2023-10-09T05:28:09.669859+0000","last_became_active":"2023-10-09T05:13:03.025729+0000","last_became_peered":"2023-10-09T05:13:03.025729+0000","last_unstale":"2023-10-09T05:28:09.669859+0000","last_undegraded":"2023-10-09T05:28:09.669859+0000","last_fullsized":"2023-10-09T05:28:09.669859+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:22.938235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'2","reported_seq":51,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.306793+0000","last_change":"2023-10-09T05:21:08.173845+0000","last_active":"2023-10-09T05:28:32.306793+0000","last_peered":"2023-10-09T05:28:32.306793+0000","last_clean":"2023-10-09T05:28:32.306793+0000","last_became_active":"2023-10-09T05:21:08.173508+0000","last_became_peered":"2023-10-09T05:21:08.173508+0000","last_unstale":"2023-10-09T05:28:32.306793+0000","last_undegraded":"2023-10-09T05:28:32.306793+0000","last_fullsized":"2023-10-09T05:28:32.306793+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:03.628428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":58,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.695970+0000","last_change":"2023-10-09T05:20:18.125172+0000","last_active":"2023-10-09T05:28:09.695970+0000","last_peered":"2023-10-09T05:28:09.695970+0000","last_clean":"2023-10-09T05:28:09.695970+0000","last_became_active":"2023-10-09T05:20:18.123624+0000","last_became_peered":"2023-10-09T05:20:18.123624+0000","last_unstale":"2023-10-09T05:28:09.695970+0000","last_undegraded":"2023-10-09T05:28:09.695970+0000","last_fullsized":"2023-10-09T05:28:09.695970+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:11.106483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":135,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.670345+0000","last_change":"2023-10-09T05:13:03.055887+0000","last_active":"2023-10-09T05:28:09.670345+0000","last_peered":"2023-10-09T05:28:09.670345+0000","last_clean":"2023-10-09T05:28:09.670345+0000","last_became_active":"2023-10-09T05:13:03.028782+0000","last_became_peered":"2023-10-09T05:13:03.028782+0000","last_unstale":"2023-10-09T05:28:09.670345+0000","last_undegraded":"2023-10-09T05:28:09.670345+0000","last_fullsized":"2023-10-09T05:28:09.670345+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:38.828205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":297,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.670336+0000","last_change":"2023-10-09T05:23:18.207836+0000","last_active":"2023-10-09T05:28:09.670336+0000","last_peered":"2023-10-09T05:28:09.670336+0000","last_clean":"2023-10-09T05:28:09.670336+0000","last_became_active":"2023-10-09T05:23:18.207404+0000","last_became_peered":"2023-10-09T05:23:18.207404+0000","last_unstale":"2023-10-09T05:28:09.670336+0000","last_undegraded":"2023-10-09T05:28:09.670336+0000","last_fullsized":"2023-10-09T05:28:09.670336+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:36:22.236093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":135,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.670416+0000","last_change":"2023-10-09T05:13:03.049546+0000","last_active":"2023-10-09T05:28:09.670416+0000","last_peered":"2023-10-09T05:28:09.670416+0000","last_clean":"2023-10-09T05:28:09.670416+0000","last_became_active":"2023-10-09T05:13:03.028676+0000","last_became_peered":"2023-10-09T05:13:03.028676+0000","last_unstale":"2023-10-09T05:28:09.670416+0000","last_undegraded":"2023-10-09T05:28:09.670416+0000","last_fullsized":"2023-10-09T05:28:09.670416+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:55.089350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":257,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.670271+0000","last_change":"2023-10-09T05:02:09.649748+0000","last_active":"2023-10-09T05:28:09.670271+0000","last_peered":"2023-10-09T05:28:09.670271+0000","last_clean":"2023-10-09T05:28:09.670271+0000","last_became_active":"2023-10-09T05:02:09.649444+0000","last_became_peered":"2023-10-09T05:02:09.649444+0000","last_unstale":"2023-10-09T05:28:09.670271+0000","last_undegraded":"2023-10-09T05:28:09.670271+0000","last_fullsized":"2023-10-09T05:28:09.670271+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:07.436820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'2","reported_seq":279,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:14.999133+0000","last_change":"2023-10-09T05:22:47.197002+0000","last_active":"2023-10-09T05:28:14.999133+0000","last_peered":"2023-10-09T05:28:14.999133+0000","last_clean":"2023-10-09T05:28:14.999133+0000","last_became_active":"2023-10-09T05:22:47.196685+0000","last_became_peered":"2023-10-09T05:22:47.196685+0000","last_unstale":"2023-10-09T05:28:14.999133+0000","last_undegraded":"2023-10-09T05:28:14.999133+0000","last_fullsized":"2023-10-09T05:28:14.999133+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:47:32.594447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":264,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.307228+0000","last_change":"2023-10-09T05:02:09.649674+0000","last_active":"2023-10-09T05:28:32.307228+0000","last_peered":"2023-10-09T05:28:32.307228+0000","last_clean":"2023-10-09T05:28:32.307228+0000","last_became_active":"2023-10-09T05:02:09.649041+0000","last_became_peered":"2023-10-09T05:02:09.649041+0000","last_unstale":"2023-10-09T05:28:32.307228+0000","last_undegraded":"2023-10-09T05:28:32.307228+0000","last_fullsized":"2023-10-09T05:28:32.307228+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:18.524071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'5","reported_seq":57,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.681538+0000","last_change":"2023-10-09T05:21:08.181594+0000","last_active":"2023-10-09T05:28:09.681538+0000","last_peered":"2023-10-09T05:28:09.681538+0000","last_clean":"2023-10-09T05:28:09.681538+0000","last_became_active":"2023-10-09T05:21:08.176895+0000","last_became_peered":"2023-10-09T05:21:08.176895+0000","last_unstale":"2023-10-09T05:28:09.681538+0000","last_undegraded":"2023-10-09T05:28:09.681538+0000","last_fullsized":"2023-10-09T05:28:09.681538+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:48:45.569544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":270,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:28:09.669946+0000","last_change":"2023-10-09T05:25:15.663537+0000","last_active":"2023-10-09T05:28:09.669946+0000","last_peered":"2023-10-09T05:28:09.669946+0000","last_clean":"2023-10-09T05:28:09.669946+0000","last_became_active":"2023-10-09T05:25:15.662771+0000","last_became_peered":"2023-10-09T05:25:15.662771+0000","last_unstale":"2023-10-09T05:28:09.669946+0000","last_undegraded":"2023-10-09T05:28:09.669946+0000","last_fullsized":"2023-10-09T05:28:09.669946+0000","mapping_epoch":268,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":269,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:53.588294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":1433010200,"num_objects":21867,"num_object_clones":0,"num_object_copies":65601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21867,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":183613,"num_write_kb":6575297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4297535488,"data_stored":4297523272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127613,"ondisk_log_size":127613,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":20,"num_read_kb":19,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":46,"num_bytes_recovered":21458,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234559,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1634936,"kb_used_data":723696,"kb_used_omap":0,"kb_used_meta":911232,"kb_avail":92110216,"statfs":{"total":95995035648,"available":94320861184,"internally_reserved":0,"allocated":741064704,"data_stored":740172484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":933101568},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:28:00 2023","interfaces":[{"interface":"back","average":{"1min":1.7769999999999999,"5min":3.2690000000000001,"15min":2.1440000000000001},"min":{"1min":0.34399999999999997,"5min":0.248,"15min":0.248},"max":{"1min":7.6609999999999996,"5min":122.248,"15min":122.248},"last":0.58299999999999996},{"interface":"front","average":{"1min":1.8149999999999999,"5min":3.2290000000000001,"15min":2.2069999999999999},"min":{"1min":0.34000000000000002,"5min":0.29499999999999998,"15min":0.29099999999999998},"max":{"1min":7.6929999999999996,"5min":122.22199999999999,"15min":122.22199999999999},"last":0.51600000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:28:38 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":1.21,"15min":1.353},"min":{"1min":0.30499999999999999,"5min":0.29199999999999998,"15min":0.28000000000000003},"max":{"1min":3.9239999999999999,"5min":6.1109999999999998,"15min":23.530999999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.593,"5min":1.321,"15min":1.4339999999999999},"min":{"1min":0.313,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":6.1699999999999999,"5min":6.8710000000000004,"15min":23.443000000000001},"last":0.40799999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:28:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":2.9279999999999999,"15min":2.1880000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.222,"5min":122.065,"15min":122.065},"last":0.56299999999999994},{"interface":"front","average":{"1min":2.2309999999999999,"5min":3.4279999999999999,"15min":2.2690000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.30399999999999999},"max":{"1min":8.0790000000000006,"5min":122.026,"15min":122.026},"last":1.5229999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:28:00 2023","interfaces":[{"interface":"back","average":{"1min":1.6100000000000001,"5min":3.133,"15min":2.3719999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.35299999999999998},"max":{"1min":7.6740000000000004,"5min":138.667,"15min":138.667},"last":1.5569999999999999},{"interface":"front","average":{"1min":1.74,"5min":3.484,"15min":2.4889999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34300000000000003},"max":{"1min":4.9720000000000004,"5min":138.54400000000001,"15min":138.54400000000001},"last":4.1680000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:28:38 2023","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":1.01,"15min":0.98299999999999998},"min":{"1min":0.34899999999999998,"5min":0.311,"15min":0.311},"max":{"1min":4.548,"5min":6.0309999999999997,"15min":9.7850000000000001},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.90900000000000003,"5min":1.0680000000000001,"15min":1.0209999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":4.5780000000000003,"5min":5.4219999999999997,"15min":5.4219999999999997},"last":0.45000000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:28:38 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":0.98599999999999999,"15min":0.92700000000000005},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.101,"5min":4.6870000000000003,"15min":4.8689999999999998},"last":0.433},{"interface":"front","average":{"1min":0.90500000000000003,"5min":1.0720000000000001,"15min":0.92700000000000005},"min":{"1min":0.40899999999999997,"5min":0.373,"15min":0.373},"max":{"1min":4.0529999999999999,"5min":5.9779999999999998,"15min":5.9779999999999998},"last":0.622}]},{"osd":6,"last update":"Mon Oct 9 05:28:38 2023","interfaces":[{"interface":"back","average":{"1min":1.2869999999999999,"5min":1.333,"15min":1.1679999999999999},"min":{"1min":0.42699999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":6.282,"5min":7.5519999999999996,"15min":7.5519999999999996},"last":0.66000000000000003},{"interface":"front","average":{"1min":1.2310000000000001,"5min":1.1160000000000001,"15min":1.1479999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.38200000000000001},"max":{"1min":5.0030000000000001,"5min":5.0030000000000001,"15min":6.7910000000000004},"last":0.747}]}]},{"osd":6,"up_from":46,"seq":197568496201,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1523312,"kb_used_data":619496,"kb_used_omap":0,"kb_used_meta":903808,"kb_avail":92221840,"statfs":{"total":95995035648,"available":94435164160,"internally_reserved":0,"allocated":634363904,"data_stored":633480583,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925499392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:27:58 2023","interfaces":[{"interface":"back","average":{"1min":2.7919999999999998,"5min":2.9239999999999999,"15min":3.2010000000000001},"min":{"1min":0.26200000000000001,"5min":0.26100000000000001,"15min":0.23599999999999999},"max":{"1min":18.574000000000002,"5min":154.29300000000001,"15min":154.29300000000001},"last":0.745},{"interface":"front","average":{"1min":2.871,"5min":3.0569999999999999,"15min":3.3170000000000002},"min":{"1min":0.35899999999999999,"5min":0.33100000000000002,"15min":0.309},"max":{"1min":18.631,"5min":154.35300000000001,"15min":154.35300000000001},"last":0.84299999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:27:58 2023","interfaces":[{"interface":"back","average":{"1min":1.841,"5min":1.6699999999999999,"15min":2.3399999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.3490000000000002,"5min":23.318000000000001,"15min":120.873},"last":6.5789999999999997},{"interface":"front","average":{"1min":1.409,"5min":1.5269999999999999,"15min":2.2349999999999999},"min":{"1min":0.30499999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":4.1219999999999999,"5min":23.140000000000001,"15min":119.248},"last":0.875}]},{"osd":2,"last update":"Mon Oct 9 05:27:58 2023","interfaces":[{"interface":"back","average":{"1min":13.545,"5min":5.1159999999999997,"15min":2.5710000000000002},"min":{"1min":0.36599999999999999,"5min":0.30199999999999999,"15min":0.28699999999999998},"max":{"1min":112.649,"5min":112.649,"15min":112.649},"last":0.78400000000000003},{"interface":"front","average":{"1min":13.554,"5min":5.2450000000000001,"15min":2.8109999999999999},"min":{"1min":0.38300000000000001,"5min":0.31,"15min":0.31},"max":{"1min":112.834,"5min":112.834,"15min":112.834},"last":0.56899999999999995}]},{"osd":3,"last update":"Mon Oct 9 05:27:58 2023","interfaces":[{"interface":"back","average":{"1min":2.3479999999999999,"5min":6.9279999999999999,"15min":5.2309999999999999},"min":{"1min":0.377,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":17.117999999999999,"5min":169.01599999999999,"15min":169.01599999999999},"last":9.5640000000000001},{"interface":"front","average":{"1min":2.1829999999999998,"5min":6.806,"15min":5.1820000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.32200000000000001},"max":{"1min":15.866,"5min":169.40799999999999,"15min":169.40799999999999},"last":0.94399999999999995}]},{"osd":4,"last update":"Mon Oct 9 05:27:58 2023","interfaces":[{"interface":"back","average":{"1min":1.325,"5min":0.98399999999999999,"15min":1.0940000000000001},"min":{"1min":0.33200000000000002,"5min":0.29999999999999999,"15min":0.27000000000000002},"max":{"1min":4.5119999999999996,"5min":4.5119999999999996,"15min":8.3670000000000009},"last":1.196},{"interface":"front","average":{"1min":1.04,"5min":1.004,"15min":1.0589999999999999},"min":{"1min":0.38100000000000001,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":4.069,"5min":4.069,"15min":13.473000000000001},"last":1.2390000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:27:58 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":0.97999999999999998,"15min":0.98599999999999999},"min":{"1min":0.37,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":4.5410000000000004,"5min":4.5410000000000004,"15min":7.5899999999999999},"last":0.91100000000000003},{"interface":"front","average":{"1min":1.056,"5min":1.071,"15min":1.0700000000000001},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.378},"max":{"1min":4.0369999999999999,"5min":4.0369999999999999,"15min":13.292999999999999},"last":1.2769999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:28:18 2023","interfaces":[{"interface":"back","average":{"1min":1.6779999999999999,"5min":1.3580000000000001,"15min":1.3069999999999999},"min":{"1min":0.40699999999999997,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":8.5350000000000001,"5min":8.8780000000000001,"15min":8.8780000000000001},"last":3.0009999999999999},{"interface":"front","average":{"1min":1.734,"5min":1.452,"15min":1.4350000000000001},"min":{"1min":0.48699999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":8.5749999999999993,"5min":8.923,"15min":8.923},"last":3.0329999999999999}]}]},{"osd":1,"up_from":96,"seq":412316860792,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1389452,"kb_used_data":435012,"kb_used_omap":4,"kb_used_meta":954427,"kb_avail":92355700,"statfs":{"total":95995035648,"available":94572236800,"internally_reserved":0,"allocated":445452288,"data_stored":443900007,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":977334158},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:27:50 2023","interfaces":[{"interface":"back","average":{"1min":1.52,"5min":3.0790000000000002,"15min":2.2629999999999999},"min":{"1min":0.245,"5min":0.217,"15min":0.19},"max":{"1min":4.3259999999999996,"5min":83.194999999999993,"15min":83.194999999999993},"last":5.3250000000000002},{"interface":"front","average":{"1min":1.641,"5min":3.262,"15min":2.5779999999999998},"min":{"1min":0.246,"5min":0.192,"15min":0.186},"max":{"1min":6.7039999999999997,"5min":83.972999999999999,"15min":83.972999999999999},"last":1.175}]},{"osd":2,"last update":"Mon Oct 9 05:27:53 2023","interfaces":[{"interface":"back","average":{"1min":2.7869999999999999,"5min":2.754,"15min":2.0390000000000001},"min":{"1min":0.30599999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":10.516999999999999,"5min":83.337000000000003,"15min":83.337000000000003},"last":1.02},{"interface":"front","average":{"1min":2.1469999999999998,"5min":2.8479999999999999,"15min":2.0899999999999999},"min":{"1min":0.254,"5min":0.254,"15min":0.23599999999999999},"max":{"1min":7.2290000000000001,"5min":83.143000000000001,"15min":83.143000000000001},"last":1.1439999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:27:53 2023","interfaces":[{"interface":"back","average":{"1min":1.792,"5min":2.9889999999999999,"15min":2.4470000000000001},"min":{"1min":0.38100000000000001,"5min":0.221,"15min":0.215},"max":{"1min":5.2779999999999996,"5min":84.105999999999995,"15min":84.105999999999995},"last":7.5709999999999997},{"interface":"front","average":{"1min":2.6299999999999999,"5min":3.9399999999999999,"15min":2.714},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.4930000000000003,"5min":83.566000000000003,"15min":83.566000000000003},"last":8.5109999999999992}]},{"osd":4,"last update":"Mon Oct 9 05:27:50 2023","interfaces":[{"interface":"back","average":{"1min":2.1419999999999999,"5min":2.6680000000000001,"15min":1.9630000000000001},"min":{"1min":0.48299999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":7.0389999999999997,"5min":83.453999999999994,"15min":83.453999999999994},"last":4.7370000000000001},{"interface":"front","average":{"1min":1.6299999999999999,"5min":2.2109999999999999,"15min":1.738},"min":{"1min":0.40500000000000003,"5min":0.34599999999999997,"15min":0.28100000000000003},"max":{"1min":7.0099999999999998,"5min":83.375,"15min":83.375},"last":5.4299999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:27:50 2023","interfaces":[{"interface":"back","average":{"1min":1.2869999999999999,"5min":1.2390000000000001,"15min":1.4630000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.28899999999999998},"max":{"1min":3.6869999999999998,"5min":7.6639999999999997,"15min":8.1129999999999995},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.9199999999999999,"5min":2.597,"15min":1.841},"min":{"1min":0.52500000000000002,"5min":0.39900000000000002,"15min":0.38200000000000001},"max":{"1min":6.9370000000000003,"5min":83.268000000000001,"15min":83.268000000000001},"last":4.6600000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:27:50 2023","interfaces":[{"interface":"back","average":{"1min":1.6779999999999999,"5min":1.4450000000000001,"15min":1.5840000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":3.6549999999999998,"5min":7.6040000000000001,"15min":8.5630000000000006},"last":0.435},{"interface":"front","average":{"1min":2.5529999999999999,"5min":2.7480000000000002,"15min":1.9910000000000001},"min":{"1min":0.47999999999999998,"5min":0.376,"15min":0.373},"max":{"1min":7.9660000000000002,"5min":83.400000000000006,"15min":83.400000000000006},"last":4.6970000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:27:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4239999999999999,"5min":2.7959999999999998,"15min":1.998},"min":{"1min":0.504,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":6.8550000000000004,"5min":83.510000000000005,"15min":83.510000000000005},"last":4.7199999999999998},{"interface":"front","average":{"1min":1.5329999999999999,"5min":2.3610000000000002,"15min":1.726},"min":{"1min":0.39800000000000002,"5min":0.377,"15min":0.36899999999999999},"max":{"1min":3.7949999999999999,"5min":83.480999999999995,"15min":83.480999999999995},"last":5.4080000000000004}]}]},{"osd":0,"up_from":88,"seq":377957122433,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1423884,"kb_used_data":547524,"kb_used_omap":5,"kb_used_meta":876346,"kb_avail":92321268,"statfs":{"total":95995035648,"available":94536978432,"internally_reserved":0,"allocated":560664576,"data_stored":559170760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":897378938},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:28:45 2023","interfaces":[{"interface":"back","average":{"1min":1.9379999999999999,"5min":1.425,"15min":2.0899999999999999},"min":{"1min":0.28499999999999998,"5min":0.21199999999999999,"15min":0.17799999999999999},"max":{"1min":7.0099999999999998,"5min":7.0099999999999998,"15min":80.843999999999994},"last":4.5190000000000001},{"interface":"front","average":{"1min":1.179,"5min":1.0980000000000001,"15min":2.302},"min":{"1min":0.28100000000000003,"5min":0.221,"15min":0.21299999999999999},"max":{"1min":6.0339999999999998,"5min":8.9849999999999994,"15min":81.159999999999997},"last":1.5569999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:27:49 2023","interfaces":[{"interface":"back","average":{"1min":7.4249999999999998,"5min":3.2000000000000002,"15min":2.4820000000000002},"min":{"1min":0.29599999999999999,"5min":0.25600000000000001,"15min":0.24199999999999999},"max":{"1min":92.745000000000005,"5min":92.745000000000005,"15min":92.745000000000005},"last":1.6970000000000001},{"interface":"front","average":{"1min":7.968,"5min":3.5419999999999998,"15min":2.52},"min":{"1min":0.36199999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":97.816000000000003,"5min":97.816000000000003,"15min":97.816000000000003},"last":2.0230000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:27:49 2023","interfaces":[{"interface":"back","average":{"1min":3.7440000000000002,"5min":1.893,"15min":2.3540000000000001},"min":{"1min":0.218,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":37.883000000000003,"5min":37.883000000000003,"15min":82.286000000000001},"last":6.4619999999999997},{"interface":"front","average":{"1min":4.8159999999999998,"5min":2.2010000000000001,"15min":2.8130000000000002},"min":{"1min":0.45700000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":45.279000000000003,"5min":45.279000000000003,"15min":75.969999999999999},"last":2.7450000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:27:49 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":1.1579999999999999,"15min":1.603},"min":{"1min":0.376,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":4.9379999999999997,"5min":5.5010000000000003,"15min":12.255000000000001},"last":2.141},{"interface":"front","average":{"1min":1.8540000000000001,"5min":1.5469999999999999,"15min":1.5569999999999999},"min":{"1min":0.40200000000000002,"5min":0.34100000000000003,"15min":0.28199999999999997},"max":{"1min":5.0910000000000002,"5min":8.3369999999999997,"15min":8.3580000000000005},"last":1.966}]},{"osd":5,"last update":"Mon Oct 9 05:27:49 2023","interfaces":[{"interface":"back","average":{"1min":1.4279999999999999,"5min":1.131,"15min":1.3799999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":4.6269999999999998,"5min":4.6269999999999998,"15min":11.916},"last":0.54000000000000004},{"interface":"front","average":{"1min":1.292,"5min":1.264,"15min":1.6839999999999999},"min":{"1min":0.45100000000000001,"5min":0.434,"15min":0.36699999999999999},"max":{"1min":2.9590000000000001,"5min":5.6660000000000004,"15min":12.143000000000001},"last":2.1880000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:27:49 2023","interfaces":[{"interface":"back","average":{"1min":1.9650000000000001,"5min":1.5880000000000001,"15min":1.7250000000000001},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":5.141,"5min":8.3629999999999995,"15min":12.125},"last":1.9930000000000001},{"interface":"front","average":{"1min":1.796,"5min":1.3680000000000001,"15min":1.595},"min":{"1min":0.35999999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":8.4659999999999993,"5min":8.4659999999999993,"15min":12.007999999999999},"last":0.51800000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:27:49 2023","interfaces":[{"interface":"back","average":{"1min":2.069,"5min":1.373,"15min":1.575},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":8.6590000000000007,"5min":8.6590000000000007,"15min":11.977},"last":0.48199999999999998},{"interface":"front","average":{"1min":2.1400000000000001,"5min":1.5249999999999999,"15min":1.847},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.40100000000000002},"max":{"1min":10.795,"5min":10.795,"15min":12.1},"last":2.1659999999999999}]}]},{"osd":2,"up_from":106,"seq":455266533711,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1635164,"kb_used_data":893780,"kb_used_omap":7,"kb_used_meta":741368,"kb_avail":92109988,"statfs":{"total":95995035648,"available":94320627712,"internally_reserved":0,"allocated":915230720,"data_stored":913778532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":759160913},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:27:47 2023","interfaces":[{"interface":"back","average":{"1min":2.4180000000000001,"5min":2.1160000000000001,"15min":2.0259999999999998},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":5.3230000000000004,"5min":16.442,"15min":40.204999999999998},"last":0.52800000000000002},{"interface":"front","average":{"1min":2.6520000000000001,"5min":1.8720000000000001,"15min":1.954},"min":{"1min":0.28499999999999998,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":7.1369999999999996,"5min":16.870000000000001,"15min":40.170000000000002},"last":4.7389999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:27:47 2023","interfaces":[{"interface":"back","average":{"1min":2.6440000000000001,"5min":2.0259999999999998,"15min":2.2149999999999999},"min":{"1min":0.29099999999999998,"5min":0.22700000000000001,"15min":0.224},"max":{"1min":6.7949999999999999,"5min":23.251000000000001,"15min":68.933999999999997},"last":5.2279999999999998},{"interface":"front","average":{"1min":2.5640000000000001,"5min":2.0710000000000002,"15min":2.3140000000000001},"min":{"1min":0.32000000000000001,"5min":0.25900000000000001,"15min":0.20399999999999999},"max":{"1min":8.8439999999999994,"5min":17.632999999999999,"15min":68.846000000000004},"last":0.32300000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:27:59 2023","interfaces":[{"interface":"back","average":{"1min":3.0150000000000001,"5min":2.032,"15min":2.4569999999999999},"min":{"1min":0.32900000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":15.125,"5min":15.125,"15min":68.968000000000004},"last":2.6429999999999998},{"interface":"front","average":{"1min":3.6880000000000002,"5min":2.504,"15min":2.3780000000000001},"min":{"1min":0.36699999999999999,"5min":0.311,"15min":0.23899999999999999},"max":{"1min":8.8140000000000001,"5min":14.619999999999999,"15min":68.891999999999996},"last":5.0430000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:27:47 2023","interfaces":[{"interface":"back","average":{"1min":2.7519999999999998,"5min":1.5880000000000001,"15min":1.597},"min":{"1min":0.374,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":7.3570000000000002,"5min":7.3570000000000002,"15min":11.157999999999999},"last":2.6749999999999998},{"interface":"front","average":{"1min":2.0699999999999998,"5min":1.6639999999999999,"15min":1.5580000000000001},"min":{"1min":0.45300000000000001,"5min":0.34200000000000003,"15min":0.29699999999999999},"max":{"1min":7.1929999999999996,"5min":10.648,"15min":12.460000000000001},"last":0.45500000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:27:47 2023","interfaces":[{"interface":"back","average":{"1min":1.6910000000000001,"5min":1.778,"15min":1.53},"min":{"1min":0.39500000000000002,"5min":0.34100000000000003,"15min":0.33000000000000002},"max":{"1min":4.1799999999999997,"5min":7.4710000000000001,"15min":9.9190000000000005},"last":0.50800000000000001},{"interface":"front","average":{"1min":2.1899999999999999,"5min":1.5169999999999999,"15min":1.673},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.34699999999999998},"max":{"1min":7.4020000000000001,"5min":7.4020000000000001,"15min":11.205},"last":2.7090000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:27:47 2023","interfaces":[{"interface":"back","average":{"1min":2.0880000000000001,"5min":1.6819999999999999,"15min":1.7529999999999999},"min":{"1min":0.44700000000000001,"5min":0.35299999999999998,"15min":0.32500000000000001},"max":{"1min":7.1589999999999998,"5min":10.279999999999999,"15min":16.337},"last":0.66900000000000004},{"interface":"front","average":{"1min":2.0819999999999999,"5min":2.0219999999999998,"15min":1.776},"min":{"1min":0.42799999999999999,"5min":0.38100000000000001,"15min":0.35199999999999998},"max":{"1min":5.6470000000000002,"5min":11.118,"15min":16.399999999999999},"last":0.70699999999999996}]},{"osd":7,"last update":"Mon Oct 9 05:27:47 2023","interfaces":[{"interface":"back","average":{"1min":2.2589999999999999,"5min":2.2280000000000002,"15min":1.7150000000000001},"min":{"1min":0.436,"5min":0.40899999999999997,"15min":0.38300000000000001},"max":{"1min":6.6050000000000004,"5min":13.571999999999999,"15min":13.571999999999999},"last":0.80800000000000005},{"interface":"front","average":{"1min":3.48,"5min":1.9259999999999999,"15min":1.8049999999999999},"min":{"1min":0.56699999999999995,"5min":0.46800000000000003,"15min":0.379},"max":{"1min":7.4340000000000002,"5min":9.6530000000000005,"15min":11.236000000000001},"last":6.7220000000000004}]}]},{"osd":3,"up_from":123,"seq":528280977730,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1545928,"kb_used_data":698176,"kb_used_omap":2,"kb_used_meta":847677,"kb_avail":92199224,"statfs":{"total":95995035648,"available":94412005376,"internally_reserved":0,"allocated":714932224,"data_stored":713555789,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":868021301},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:27:58 2023","interfaces":[{"interface":"back","average":{"1min":4.274,"5min":2.2480000000000002,"15min":3.1669999999999998},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.19500000000000001},"max":{"1min":46.765999999999998,"5min":46.765999999999998,"15min":199.84999999999999},"last":0.34399999999999997},{"interface":"front","average":{"1min":4.1520000000000001,"5min":2.0710000000000002,"15min":2.8460000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.20200000000000001},"max":{"1min":47.069000000000003,"5min":47.069000000000003,"15min":83.234999999999999},"last":11.249000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:27:58 2023","interfaces":[{"interface":"back","average":{"1min":4.1440000000000001,"5min":2.1190000000000002,"15min":2.294},"min":{"1min":0.221,"5min":0.22,"15min":0.22},"max":{"1min":26.372,"5min":26.372,"15min":44.206000000000003},"last":2.9700000000000002},{"interface":"front","average":{"1min":3.754,"5min":2.02,"15min":2.056},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.218},"max":{"1min":28.007999999999999,"5min":28.007999999999999,"15min":44.496000000000002},"last":0.39900000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:27:58 2023","interfaces":[{"interface":"back","average":{"1min":5.0460000000000003,"5min":2.3799999999999999,"15min":2.016},"min":{"1min":0.434,"5min":0.27300000000000002,"15min":0.22700000000000001},"max":{"1min":29.945,"5min":29.945,"15min":29.945},"last":4.2549999999999999},{"interface":"front","average":{"1min":3.9129999999999998,"5min":2.1499999999999999,"15min":2.306},"min":{"1min":0.251,"5min":0.251,"15min":0.22800000000000001},"max":{"1min":26.452999999999999,"5min":26.452999999999999,"15min":26.452999999999999},"last":3.0139999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:27:58 2023","interfaces":[{"interface":"back","average":{"1min":1.702,"5min":1.6259999999999999,"15min":1.7649999999999999},"min":{"1min":0.36899999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.4560000000000004,"5min":8.4760000000000009,"15min":12.438000000000001},"last":3.1110000000000002},{"interface":"front","average":{"1min":2.0289999999999999,"5min":1.53,"15min":2.1379999999999999},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.316},"max":{"1min":8.2309999999999999,"5min":8.468,"15min":199.50800000000001},"last":0.47499999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:27:58 2023","interfaces":[{"interface":"back","average":{"1min":1.8720000000000001,"5min":1.5509999999999999,"15min":2.1749999999999998},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.33500000000000002},"max":{"1min":8.2579999999999991,"5min":8.5269999999999992,"15min":199.93799999999999},"last":2.923},{"interface":"front","average":{"1min":1.9470000000000001,"5min":1.4850000000000001,"15min":1.508},"min":{"1min":0.48499999999999999,"5min":0.40000000000000002,"15min":0.33900000000000002},"max":{"1min":6.3739999999999997,"5min":7.5289999999999999,"15min":11.135},"last":3.589}]},{"osd":6,"last update":"Mon Oct 9 05:27:58 2023","interfaces":[{"interface":"back","average":{"1min":2.3679999999999999,"5min":1.615,"15min":1.607},"min":{"1min":0.46899999999999997,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":6.3200000000000003,"5min":7.4740000000000002,"15min":11.173999999999999},"last":3.5470000000000002},{"interface":"front","average":{"1min":2.2370000000000001,"5min":1.8300000000000001,"15min":2.0019999999999998},"min":{"1min":0.441,"5min":0.36699999999999999,"15min":0.32800000000000001},"max":{"1min":7.4939999999999998,"5min":8.5500000000000007,"15min":12.407},"last":3.04}]},{"osd":7,"last update":"Mon Oct 9 05:27:58 2023","interfaces":[{"interface":"back","average":{"1min":1.879,"5min":1.8240000000000001,"15min":1.8600000000000001},"min":{"1min":0.51500000000000001,"5min":0.39100000000000001,"15min":0.378},"max":{"1min":7.5410000000000004,"5min":8.6039999999999992,"15min":12.468},"last":3.0750000000000002},{"interface":"front","average":{"1min":2.2360000000000002,"5min":1.853,"15min":2.3359999999999999},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":8.2070000000000007,"5min":8.4979999999999993,"15min":199.96799999999999},"last":2.9580000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019487,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1251508,"kb_used_data":358956,"kb_used_omap":0,"kb_used_meta":892544,"kb_avail":92493644,"statfs":{"total":95995035648,"available":94713491456,"internally_reserved":0,"allocated":367570944,"data_stored":366682770,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":913965056},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:27:52 2023","interfaces":[{"interface":"back","average":{"1min":9.7789999999999999,"5min":3.77,"15min":2.3399999999999999},"min":{"1min":0.26300000000000001,"5min":0.25,"15min":0.24199999999999999},"max":{"1min":151.209,"5min":151.209,"15min":151.209},"last":3.4399999999999999},{"interface":"front","average":{"1min":9.1899999999999995,"5min":3.6930000000000001,"15min":2.3170000000000002},"min":{"1min":0.34100000000000003,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":151.303,"5min":151.303,"15min":151.303},"last":3.4580000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:28:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1359999999999999,"5min":1.48,"15min":2.2240000000000002},"min":{"1min":0.33200000000000002,"5min":0.28999999999999998,"15min":0.25700000000000001},"max":{"1min":3.274,"5min":53.911000000000001,"15min":142.297},"last":0.44700000000000001},{"interface":"front","average":{"1min":1.107,"5min":1.6240000000000001,"15min":2.1480000000000001},"min":{"1min":0.30099999999999999,"5min":0.29699999999999999,"15min":0.27800000000000002},"max":{"1min":4.4240000000000004,"5min":54.892000000000003,"15min":135.11600000000001},"last":0.38600000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:27:52 2023","interfaces":[{"interface":"back","average":{"1min":10.105,"5min":3.1659999999999999,"15min":2.1179999999999999},"min":{"1min":0.40200000000000002,"5min":0.311,"15min":0.29499999999999998},"max":{"1min":152.643,"5min":152.643,"15min":152.643},"last":5.0839999999999996},{"interface":"front","average":{"1min":9.8140000000000001,"5min":3.056,"15min":2.2210000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":152.554,"5min":152.554,"15min":152.554},"last":5.2080000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:27:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4710000000000001,"5min":1.492,"15min":2.1869999999999998},"min":{"1min":0.40899999999999997,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":6.0999999999999996,"5min":8.7650000000000006,"15min":131.726},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.605,"5min":1.5029999999999999,"15min":2.2759999999999998},"min":{"1min":0.41999999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":10.324999999999999,"5min":10.324999999999999,"15min":131.57300000000001},"last":0.47299999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:27:59 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":1.0329999999999999,"15min":0.90400000000000003},"min":{"1min":0.45600000000000002,"5min":0.373,"15min":0.33100000000000002},"max":{"1min":6.1399999999999997,"5min":6.1399999999999997,"15min":7.7549999999999999},"last":0.5},{"interface":"front","average":{"1min":1.1950000000000001,"5min":0.98699999999999999,"15min":0.96799999999999997},"min":{"1min":0.36399999999999999,"5min":0.35499999999999998,"15min":0.34799999999999998},"max":{"1min":10.255000000000001,"5min":10.255000000000001,"15min":10.255000000000001},"last":0.58499999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:28:41 2023","interfaces":[{"interface":"back","average":{"1min":1.341,"5min":1.486,"15min":1.296},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.33900000000000002},"max":{"1min":3.4460000000000002,"5min":20.818000000000001,"15min":20.818000000000001},"last":0.72499999999999998},{"interface":"front","average":{"1min":1.2150000000000001,"5min":1.222,"15min":1.131},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":3.621,"5min":6.3079999999999998,"15min":8.4930000000000003},"last":0.56799999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:28:41 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":1.212,"15min":1.123},"min":{"1min":0.372,"5min":0.372,"15min":0.33900000000000002},"max":{"1min":5.1079999999999997,"5min":10.289,"15min":10.289},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.0660000000000001,"5min":1.1899999999999999,"15min":1.1899999999999999},"min":{"1min":0.39400000000000002,"5min":0.379,"15min":0.34799999999999998},"max":{"1min":3.2090000000000001,"5min":4.7489999999999997,"15min":8.0269999999999992},"last":0.69199999999999995}]}]},{"osd":5,"up_from":0,"seq":163208757843,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":546832384,"data_stored":546832384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":436473856,"data_stored":436469784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":910692352,"data_stored":910688280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":701956096,"data_stored":701956096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":358350848,"data_stored":358350848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":625475584,"data_stored":625475584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":717754368,"data_stored":717750296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:28:53.866 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:28:53.947 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:53.983 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:53.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:53.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:54.090 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:54.119 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:54.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:54.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:54.222 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:54.257 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:54.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:54.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:54.363 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:54.388 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:54.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:54.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:54.491 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:54.517 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:54.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:54.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:54.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:28:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:28:54] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.43.0" 2023-10-09T05:28:54.620 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:54.649 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:54.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:54.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:54.754 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:54.792 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:54.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:54.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:54.897 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:54.906 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:54 smithi133 bash[105589]: cluster 2023-10-09T05:28:54.230081+0000 mgr.y (mgr.25277) 1221 : cluster 0 pgmap v1302: 89 pgs: 89 active+clean; 1.3 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 682 op/s 2023-10-09T05:28:54.907 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:54 smithi133 bash[102858]: cluster 2023-10-09T05:28:54.230081+0000 mgr.y (mgr.25277) 1221 : cluster 0 pgmap v1302: 89 pgs: 89 active+clean; 1.3 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 682 op/s 2023-10-09T05:28:54.927 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:54.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:54.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:55.033 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:55.060 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:55.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:55.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:55.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:54 smithi194 bash[92666]: cluster 2023-10-09T05:28:54.230081+0000 mgr.y (mgr.25277) 1221 : cluster 0 pgmap v1302: 89 pgs: 89 active+clean; 1.3 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 682 op/s 2023-10-09T05:28:55.165 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:55.196 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:55.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:55.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:55.300 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:55.334 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:55.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:55.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:55.441 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:55.479 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:55.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:55.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:55.581 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:55.609 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:55.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:55.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:55.712 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:55.750 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:55.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:55.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:55.857 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:28:55.857 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:55.871 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:28:55.885 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:55.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:55.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:55.992 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:56.018 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:56.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:56.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:56.126 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:56.157 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:56.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:56.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:56.260 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:56.302 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:56.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:56.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:56.409 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:56.439 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:56.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:56.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:56.546 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:56.574 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:56.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:56.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:56.680 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:56.711 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:56.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:56.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:56.815 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:56.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:56.857 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:56.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:56.958 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:56.987 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:56.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:56.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:57.091 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:57.139 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:57.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:57.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:57.243 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:57.279 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:57.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:57.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:57.381 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:57.391 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:57 smithi133 bash[102858]: cluster 2023-10-09T05:28:56.230773+0000 mgr.y (mgr.25277) 1222 : cluster 0 pgmap v1303: 89 pgs: 89 active+clean; 1.1 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 671 op/s 2023-10-09T05:28:57.391 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:57 smithi133 bash[105589]: cluster 2023-10-09T05:28:56.230773+0000 mgr.y (mgr.25277) 1222 : cluster 0 pgmap v1303: 89 pgs: 89 active+clean; 1.1 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 671 op/s 2023-10-09T05:28:57.410 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:57.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:57.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:57.516 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:57.550 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:28:57.552 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:57 smithi194 bash[92666]: cluster 2023-10-09T05:28:56.230773+0000 mgr.y (mgr.25277) 1222 : cluster 0 pgmap v1303: 89 pgs: 89 active+clean; 1.1 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 671 op/s 2023-10-09T05:28:57.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:57.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:28:57.654 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:57.685 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:57.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:57.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:57.791 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:57.845 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:57.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:57.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:57.950 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:57.980 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:57.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:57.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:58.083 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:58.126 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:58.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:58.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:58.232 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:58.258 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:58.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:58.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:58.366 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:58.417 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:58.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:58.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:58.523 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:58.552 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:58.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:58.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:58.564 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:28:58.657 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:58.694 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:28:58.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:58.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:28:58.753 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:28:58.799 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:58.823 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:58.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:58.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:58.925 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:58.964 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:28:58.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:58.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:28:59.072 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:59.097 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:28:59.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:59.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:28:59.202 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:59.243 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:28:59.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:59.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:28:59.295 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:59 smithi133 bash[102858]: cluster 2023-10-09T05:28:58.231194+0000 mgr.y (mgr.25277) 1223 : cluster 0 pgmap v1304: 89 pgs: 89 active+clean; 1.1 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 671 op/s 2023-10-09T05:28:59.295 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:59 smithi133 bash[102858]: audit 2023-10-09T05:28:58.486042+0000 mon.a (mon.0) 1563 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:28:59.296 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:28:59 smithi133 bash[102858]: audit 2023-10-09T05:28:58.489389+0000 mon.a (mon.0) 1564 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:59.349 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:59.384 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:28:59.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:59 smithi194 bash[92666]: cluster 2023-10-09T05:28:58.231194+0000 mgr.y (mgr.25277) 1223 : cluster 0 pgmap v1304: 89 pgs: 89 active+clean; 1.1 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 671 op/s 2023-10-09T05:28:59.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:59 smithi194 bash[92666]: audit 2023-10-09T05:28:58.486042+0000 mon.a (mon.0) 1563 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:28:59.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:28:59 smithi194 bash[92666]: audit 2023-10-09T05:28:58.489389+0000 mon.a (mon.0) 1564 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:59.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:59.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:28:59.488 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:59.524 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:59.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:59.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:59.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:59 smithi133 bash[105589]: cluster 2023-10-09T05:28:58.231194+0000 mgr.y (mgr.25277) 1223 : cluster 0 pgmap v1304: 89 pgs: 89 active+clean; 1.1 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 671 op/s 2023-10-09T05:28:59.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:59 smithi133 bash[105589]: audit 2023-10-09T05:28:58.486042+0000 mon.a (mon.0) 1563 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:28:59.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:28:59 smithi133 bash[105589]: audit 2023-10-09T05:28:58.489389+0000 mon.a (mon.0) 1564 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:59.631 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:59.662 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:28:59.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:59.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:28:59.768 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:59.774 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:28:59.775 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:28:59.798 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:59.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:59.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:28:59.901 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:59.932 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:28:59.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:28:59.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:00.039 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:00.065 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:00.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:00.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:00.170 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:00.196 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:00.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:00.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:00.301 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:00.331 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:00.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:00.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:00.436 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:00.457 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:00.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:00.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:00.564 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:00.595 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:00.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:00.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:00.701 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:00.722 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:00.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:00.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:00.832 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:00.860 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:00.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:00.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:00.962 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:00.994 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:00.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:00.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:01.097 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:01.122 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:01.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:01.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:01.225 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:01.257 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:01.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:01.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:01.363 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:01.371 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:01 smithi194 bash[92666]: cluster 2023-10-09T05:29:00.231901+0000 mgr.y (mgr.25277) 1224 : cluster 0 pgmap v1305: 89 pgs: 89 active+clean; 944 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 676 op/s 2023-10-09T05:29:01.386 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:01.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:01.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:01.491 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:01.523 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:01.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:01.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:01.628 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:01.656 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:01.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:01.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:01.758 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:01.768 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:01 smithi133 bash[102858]: cluster 2023-10-09T05:29:00.231901+0000 mgr.y (mgr.25277) 1224 : cluster 0 pgmap v1305: 89 pgs: 89 active+clean; 944 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 676 op/s 2023-10-09T05:29:01.768 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:01 smithi133 bash[105589]: cluster 2023-10-09T05:29:00.231901+0000 mgr.y (mgr.25277) 1224 : cluster 0 pgmap v1305: 89 pgs: 89 active+clean; 944 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 676 op/s 2023-10-09T05:29:01.800 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:01.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:01.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:01.904 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:01.940 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:01.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:01.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:02.043 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:02.069 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:02.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:02.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:02.172 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:02.199 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:02.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:02.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:02.304 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:02.336 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:02.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:02.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:02.498 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:02.539 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:02.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:02.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:02.642 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:02.674 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:02.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:02.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:02.780 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:02.808 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:02.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:02.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:02.911 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:02.940 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:02.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:02.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:03.044 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:03.082 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:03.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:03.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:03.189 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:03.205 INFO:teuthology.orchestra.run.smithi194.stdout:{ 2023-10-09T05:29:03.205 INFO:teuthology.orchestra.run.smithi194.stdout: "ops": [], 2023-10-09T05:29:03.205 INFO:teuthology.orchestra.run.smithi194.stdout: "complaint_time": 30, 2023-10-09T05:29:03.205 INFO:teuthology.orchestra.run.smithi194.stdout: "num_blocked_ops": 0 2023-10-09T05:29:03.206 INFO:teuthology.orchestra.run.smithi194.stdout:} 2023-10-09T05:29:03.214 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:03.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:03.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:03.320 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:03.349 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:03.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:03.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:03.458 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:03.485 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:03.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:03.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:03.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:03 smithi133 bash[105589]: cluster 2023-10-09T05:29:02.232257+0000 mgr.y (mgr.25277) 1225 : cluster 0 pgmap v1306: 89 pgs: 89 active+clean; 944 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 620 op/s 2023-10-09T05:29:03.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:03 smithi133 bash[102858]: cluster 2023-10-09T05:29:02.232257+0000 mgr.y (mgr.25277) 1225 : cluster 0 pgmap v1306: 89 pgs: 89 active+clean; 944 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 620 op/s 2023-10-09T05:29:03.591 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:03.626 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:03.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:03.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:03.654 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:03 smithi194 bash[92666]: cluster 2023-10-09T05:29:02.232257+0000 mgr.y (mgr.25277) 1225 : cluster 0 pgmap v1306: 89 pgs: 89 active+clean; 944 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 620 op/s 2023-10-09T05:29:03.729 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:03.765 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:03.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:03.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:03.871 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:03.902 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:03.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:03.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:04.012 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:04.041 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:04.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:04.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:04.144 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:04.169 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:04.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:04.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:04.277 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:04.281 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:29:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:29:04] "GET /metrics HTTP/1.1" 200 36478 "" "Prometheus/2.43.0" 2023-10-09T05:29:04.289 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T05:29:04.305 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:04.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:04.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:04.412 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:04.440 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:04.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:04.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:04.544 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:04.569 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:04.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:04.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:04.676 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:04.700 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:04.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:04.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:04.804 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:04.812 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:04 smithi133 bash[102858]: cluster 2023-10-09T05:29:04.232760+0000 mgr.y (mgr.25277) 1226 : cluster 0 pgmap v1307: 89 pgs: 89 active+clean; 918 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 599 op/s 2023-10-09T05:29:04.813 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:04 smithi133 bash[105589]: cluster 2023-10-09T05:29:04.232760+0000 mgr.y (mgr.25277) 1226 : cluster 0 pgmap v1307: 89 pgs: 89 active+clean; 918 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 599 op/s 2023-10-09T05:29:04.843 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:04.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:04.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:04.949 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:04.957 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:04 smithi194 bash[92666]: cluster 2023-10-09T05:29:04.232760+0000 mgr.y (mgr.25277) 1226 : cluster 0 pgmap v1307: 89 pgs: 89 active+clean; 918 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 599 op/s 2023-10-09T05:29:04.972 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:04.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:04.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:05.081 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:05.109 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:05.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:05.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:05.212 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:05.248 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:05.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:05.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:05.352 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:05.381 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:05.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:05.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:05.488 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:05.526 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:05.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:05.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:05.632 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:05.658 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:05.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:05.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:05.763 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:05.793 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:05.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:05.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:05.900 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:05.925 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:05.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:05.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:06.032 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:06.057 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:06.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:06.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:06.162 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:06.187 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:06.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:06.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:06.290 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:06.319 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:06.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:06.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:06.421 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:06.453 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:06.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:06.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:06.561 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:06.589 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:06.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:06.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:06.695 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:06.727 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:06.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:06.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:06.831 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:06.874 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:06.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:06.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:06.984 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:07.014 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:07.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:07.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:07.121 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:07.149 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:07.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:07.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:07.257 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:07.282 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:07.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:07.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:07.387 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:07.413 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:07.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:07.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:07.516 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:07.524 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:07 smithi194 bash[92666]: cluster 2023-10-09T05:29:06.233512+0000 mgr.y (mgr.25277) 1227 : cluster 0 pgmap v1308: 89 pgs: 89 active+clean; 742 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 798 op/s 2023-10-09T05:29:07.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:07 smithi133 bash[105589]: cluster 2023-10-09T05:29:06.233512+0000 mgr.y (mgr.25277) 1227 : cluster 0 pgmap v1308: 89 pgs: 89 active+clean; 742 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 798 op/s 2023-10-09T05:29:07.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:07 smithi133 bash[102858]: cluster 2023-10-09T05:29:06.233512+0000 mgr.y (mgr.25277) 1227 : cluster 0 pgmap v1308: 89 pgs: 89 active+clean; 742 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 798 op/s 2023-10-09T05:29:07.553 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:07.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:07.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:07.656 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:07.684 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:07.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:07.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:07.789 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:07.814 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:07.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:07.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:07.923 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:07.951 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:07.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:07.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:08.054 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:08.093 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:08.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:08.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:08.197 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:08.229 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:08.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:08.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:08.333 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:08.365 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:08.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:08.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:08.469 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:08.517 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:08.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:08.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:08.622 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:08.651 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:08.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:08.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:08.758 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:08.786 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:08.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:08.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:08.889 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:08.928 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:08.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:08.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:09.034 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:09.064 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:09.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:09.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:09.076 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:29:09.168 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:09.200 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:09.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:09.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:09.304 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:09.335 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:09.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:09.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:09.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:09 smithi194 bash[92666]: cluster 2023-10-09T05:29:08.234094+0000 mgr.y (mgr.25277) 1228 : cluster 0 pgmap v1309: 89 pgs: 89 active+clean; 742 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 556 op/s 2023-10-09T05:29:09.438 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:09.480 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:09.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:09.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:09.507 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:09 smithi133 bash[102858]: cluster 2023-10-09T05:29:08.234094+0000 mgr.y (mgr.25277) 1228 : cluster 0 pgmap v1309: 89 pgs: 89 active+clean; 742 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 556 op/s 2023-10-09T05:29:09.584 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:09.612 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:09.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:09.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:09.717 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:09.752 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:09.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:09.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:09.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:09 smithi133 bash[105589]: cluster 2023-10-09T05:29:08.234094+0000 mgr.y (mgr.25277) 1228 : cluster 0 pgmap v1309: 89 pgs: 89 active+clean; 742 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 556 op/s 2023-10-09T05:29:09.857 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:09.880 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:09.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:09.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:09.983 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:10.010 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:10.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:10.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:10.116 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:10.145 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:10.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:10.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:10.249 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:10.273 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:10.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:10.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:10.375 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:10.407 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:10.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:10.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:10.512 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:10.539 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:10.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:10.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:10.643 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:10.682 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:10.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:10.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:10.784 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:10.811 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:10.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:10.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:10.919 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:10.942 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:10.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:10.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:11.049 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:11.086 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:11.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:11.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:11.195 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:11.221 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:11.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:11.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:11.310 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:11 smithi133 bash[102858]: cluster 2023-10-09T05:29:10.234855+0000 mgr.y (mgr.25277) 1229 : cluster 0 pgmap v1310: 89 pgs: 89 active+clean; 528 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 841 op/s 2023-10-09T05:29:11.325 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:11.356 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:11.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:11.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:11.460 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:11.468 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:11 smithi194 bash[92666]: cluster 2023-10-09T05:29:10.234855+0000 mgr.y (mgr.25277) 1229 : cluster 0 pgmap v1310: 89 pgs: 89 active+clean; 528 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 841 op/s 2023-10-09T05:29:11.498 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:11.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:11.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:11.602 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:11.625 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:11.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:11.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:11.733 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:11.765 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:11.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:11.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:11.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:11 smithi133 bash[105589]: cluster 2023-10-09T05:29:10.234855+0000 mgr.y (mgr.25277) 1229 : cluster 0 pgmap v1310: 89 pgs: 89 active+clean; 528 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 841 op/s 2023-10-09T05:29:11.875 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:11.898 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:11.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:11.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:12.005 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:12.031 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:12.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:12.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:12.134 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:12.156 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:12.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:12.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:12.264 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:12.287 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:12.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:12.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:12.390 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:12.419 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:12.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:12.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:12.525 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:12.558 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:12.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:12.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:12.661 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:12.689 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:12.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:12.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:12.792 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:12.816 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:12.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:12.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:12.919 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:12.949 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:12.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:12.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:13.056 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:13.083 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:13.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:13.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:13.189 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:13.217 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:13.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:13.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:13.321 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:13.346 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:13.350 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:29:13.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:13.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:13.360 INFO:teuthology.orchestra.run.smithi133.stderr:dumped all 2023-10-09T05:29:13.403 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:13 smithi133 bash[102858]: cluster 2023-10-09T05:29:12.235242+0000 mgr.y (mgr.25277) 1230 : cluster 0 pgmap v1311: 89 pgs: 89 active+clean; 528 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 554 op/s 2023-10-09T05:29:13.404 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:13 smithi133 bash[105589]: cluster 2023-10-09T05:29:12.235242+0000 mgr.y (mgr.25277) 1230 : cluster 0 pgmap v1311: 89 pgs: 89 active+clean; 528 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 554 op/s 2023-10-09T05:29:13.444 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:29:13.453 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:13.506 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:29:13.507 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:13.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:13.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:13.571 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:29:13.610 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:13.619 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:13 smithi194 bash[92666]: cluster 2023-10-09T05:29:12.235242+0000 mgr.y (mgr.25277) 1230 : cluster 0 pgmap v1311: 89 pgs: 89 active+clean; 528 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 554 op/s 2023-10-09T05:29:13.627 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:29:13.644 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:13.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:13.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:13.665 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:29:13.707 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:29:13.744 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:29:13.749 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:13.783 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:13.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:13.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:13.793 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:29:13.797 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:29:13.797 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:29:13.797 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:29:13.797 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:29:13.797 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:29:13.797 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:29:13.797 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:29:13.797 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:29:13.891 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:13.897 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:29:13.897 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:29:13.914 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:13.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:13.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:14.017 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:14.055 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:14.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:14.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:14.161 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:14.186 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:14.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:14.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:14.292 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:14.323 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:14.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:14.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:14.431 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:14.466 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:14.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:14.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:14.537 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:14 smithi194 bash[92666]: audit 2023-10-09T05:29:13.327329+0000 mgr.y (mgr.25277) 1231 : audit 0 from='client.45491 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:29:14.538 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:14 smithi194 bash[92666]: audit 2023-10-09T05:29:13.372960+0000 mon.a (mon.0) 1565 : audit 0 from='client.? 172.21.15.133:0/2145158920' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:29:14.538 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:14 smithi194 bash[92666]: audit 2023-10-09T05:29:13.478795+0000 mon.a (mon.0) 1566 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:29:14.538 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:14 smithi194 bash[92666]: audit 2023-10-09T05:29:13.803432+0000 mon.c (mon.2) 111 : audit 1 from='client.? 172.21.15.133:0/2447974972' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:29:14.538 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:14 smithi194 bash[92666]: audit 2023-10-09T05:29:13.804080+0000 mon.a (mon.0) 1567 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:29:14.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:14 smithi133 bash[102858]: audit 2023-10-09T05:29:13.327329+0000 mgr.y (mgr.25277) 1231 : audit 0 from='client.45491 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:29:14.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:14 smithi133 bash[102858]: audit 2023-10-09T05:29:13.372960+0000 mon.a (mon.0) 1565 : audit 0 from='client.? 172.21.15.133:0/2145158920' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:29:14.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:14 smithi133 bash[102858]: audit 2023-10-09T05:29:13.478795+0000 mon.a (mon.0) 1566 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:29:14.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:14 smithi133 bash[102858]: audit 2023-10-09T05:29:13.803432+0000 mon.c (mon.2) 111 : audit 1 from='client.? 172.21.15.133:0/2447974972' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:29:14.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:14 smithi133 bash[102858]: audit 2023-10-09T05:29:13.804080+0000 mon.a (mon.0) 1567 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:29:14.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:29:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:29:14] "GET /metrics HTTP/1.1" 200 36470 "" "Prometheus/2.43.0" 2023-10-09T05:29:14.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:14 smithi133 bash[105589]: audit 2023-10-09T05:29:13.327329+0000 mgr.y (mgr.25277) 1231 : audit 0 from='client.45491 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:29:14.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:14 smithi133 bash[105589]: audit 2023-10-09T05:29:13.372960+0000 mon.a (mon.0) 1565 : audit 0 from='client.? 172.21.15.133:0/2145158920' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:29:14.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:14 smithi133 bash[105589]: audit 2023-10-09T05:29:13.478795+0000 mon.a (mon.0) 1566 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:29:14.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:14 smithi133 bash[105589]: audit 2023-10-09T05:29:13.803432+0000 mon.c (mon.2) 111 : audit 1 from='client.? 172.21.15.133:0/2447974972' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:29:14.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:14 smithi133 bash[105589]: audit 2023-10-09T05:29:13.804080+0000 mon.a (mon.0) 1567 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:29:14.568 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:14.611 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:14.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:14.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:14.696 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is set 2023-10-09T05:29:14.718 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:14.745 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:14.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:14.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:14.848 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:14.879 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:14.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:14.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:14.983 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:15.017 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:15.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:15.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:15.031 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T05:29:15.120 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:15.137 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:15.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:15.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:15.240 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:15.294 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:15.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:15.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:15.400 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:15.426 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:15.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:15.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:15.529 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:15.539 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:15 smithi133 bash[102858]: cluster 2023-10-09T05:29:14.235670+0000 mgr.y (mgr.25277) 1232 : cluster 0 pgmap v1312: 89 pgs: 89 active+clean; 502 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 589 op/s 2023-10-09T05:29:15.539 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:15 smithi133 bash[102858]: cluster 2023-10-09T05:29:14.673653+0000 mon.a (mon.0) 1568 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:29:15.539 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:15 smithi133 bash[102858]: audit 2023-10-09T05:29:14.689547+0000 mon.a (mon.0) 1569 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:29:15.539 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:15 smithi133 bash[102858]: cluster 2023-10-09T05:29:14.689915+0000 mon.a (mon.0) 1570 : cluster 0 osdmap e282: 8 total, 8 up, 7 in 2023-10-09T05:29:15.539 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:15 smithi133 bash[102858]: cluster 2023-10-09T05:29:14.698878+0000 osd.7 (osd.7) 232 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T05:29:15.539 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:15 smithi133 bash[102858]: cluster 2023-10-09T05:29:14.703829+0000 osd.4 (osd.4) 253 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T05:29:15.539 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:15 smithi133 bash[102858]: cluster 2023-10-09T05:29:14.704155+0000 osd.6 (osd.6) 234 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T05:29:15.539 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:15 smithi133 bash[102858]: cluster 2023-10-09T05:29:14.705626+0000 osd.6 (osd.6) 235 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T05:29:15.540 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:15 smithi133 bash[105589]: cluster 2023-10-09T05:29:14.235670+0000 mgr.y (mgr.25277) 1232 : cluster 0 pgmap v1312: 89 pgs: 89 active+clean; 502 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 589 op/s 2023-10-09T05:29:15.540 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:15 smithi133 bash[105589]: cluster 2023-10-09T05:29:14.673653+0000 mon.a (mon.0) 1568 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:29:15.540 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:15 smithi133 bash[105589]: audit 2023-10-09T05:29:14.689547+0000 mon.a (mon.0) 1569 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:29:15.540 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:15 smithi133 bash[105589]: cluster 2023-10-09T05:29:14.689915+0000 mon.a (mon.0) 1570 : cluster 0 osdmap e282: 8 total, 8 up, 7 in 2023-10-09T05:29:15.540 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:15 smithi133 bash[105589]: cluster 2023-10-09T05:29:14.698878+0000 osd.7 (osd.7) 232 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T05:29:15.540 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:15 smithi133 bash[105589]: cluster 2023-10-09T05:29:14.703829+0000 osd.4 (osd.4) 253 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T05:29:15.540 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:15 smithi133 bash[105589]: cluster 2023-10-09T05:29:14.704155+0000 osd.6 (osd.6) 234 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T05:29:15.540 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:15 smithi133 bash[105589]: cluster 2023-10-09T05:29:14.705626+0000 osd.6 (osd.6) 235 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T05:29:15.562 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:15.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:15.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:15.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:15 smithi194 bash[92666]: cluster 2023-10-09T05:29:14.235670+0000 mgr.y (mgr.25277) 1232 : cluster 0 pgmap v1312: 89 pgs: 89 active+clean; 502 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 589 op/s 2023-10-09T05:29:15.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:15 smithi194 bash[92666]: cluster 2023-10-09T05:29:14.673653+0000 mon.a (mon.0) 1568 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:29:15.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:15 smithi194 bash[92666]: audit 2023-10-09T05:29:14.689547+0000 mon.a (mon.0) 1569 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:29:15.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:15 smithi194 bash[92666]: cluster 2023-10-09T05:29:14.689915+0000 mon.a (mon.0) 1570 : cluster 0 osdmap e282: 8 total, 8 up, 7 in 2023-10-09T05:29:15.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:15 smithi194 bash[92666]: cluster 2023-10-09T05:29:14.698878+0000 osd.7 (osd.7) 232 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T05:29:15.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:15 smithi194 bash[92666]: cluster 2023-10-09T05:29:14.703829+0000 osd.4 (osd.4) 253 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T05:29:15.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:15 smithi194 bash[92666]: cluster 2023-10-09T05:29:14.704155+0000 osd.6 (osd.6) 234 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T05:29:15.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:15 smithi194 bash[92666]: cluster 2023-10-09T05:29:14.705626+0000 osd.6 (osd.6) 235 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T05:29:15.667 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:15.701 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:15.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:15.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:15.805 INFO:teuthology.orchestra.run.smithi133.stdout:{"pg_ready":true,"pg_map":{"version":1311,"stamp":"2023-10-09T05:29:12.235040+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":554112952,"num_objects":8471,"num_object_clones":0,"num_object_copies":25413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8471,"num_whiteouts":0,"num_read":507,"num_read_kb":932,"num_write":197333,"num_write_kb":6577742,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":4154034,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128155,"ondisk_log_size":128155,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":7877792,"kb_used_data":1719016,"kb_used_omap":20,"kb_used_meta":6158635,"kb_avail":648338272,"statfs":{"total":671965249536,"available":663898390528,"internally_reserved":0,"allocated":1760272384,"data_stored":1751718956,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":6306442894},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,2,8,1,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-436076544,"num_objects":-6654,"num_object_clones":0,"num_object_copies":-19962,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6654,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6654,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003665"},"pg_stats":[{"pgid":"3.1a","version":"69'2","reported_seq":281,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.301471+0000","last_change":"2023-10-09T05:09:57.873320+0000","last_active":"2023-10-09T05:28:32.301471+0000","last_peered":"2023-10-09T05:28:32.301471+0000","last_clean":"2023-10-09T05:28:32.301471+0000","last_became_active":"2023-10-09T05:09:57.868376+0000","last_became_peered":"2023-10-09T05:09:57.868376+0000","last_unstale":"2023-10-09T05:28:32.301471+0000","last_undegraded":"2023-10-09T05:28:32.301471+0000","last_fullsized":"2023-10-09T05:28:32.301471+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:27.186610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":264,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.308070+0000","last_change":"2023-10-09T05:14:15.345991+0000","last_active":"2023-10-09T05:28:32.308070+0000","last_peered":"2023-10-09T05:28:32.308070+0000","last_clean":"2023-10-09T05:28:32.308070+0000","last_became_active":"2023-10-09T05:14:15.345511+0000","last_became_peered":"2023-10-09T05:14:15.345511+0000","last_unstale":"2023-10-09T05:28:32.308070+0000","last_undegraded":"2023-10-09T05:28:32.308070+0000","last_fullsized":"2023-10-09T05:28:32.308070+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:49.234025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":147,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.286070+0000","last_change":"2023-10-09T05:09:57.857511+0000","last_active":"2023-10-09T05:28:32.286070+0000","last_peered":"2023-10-09T05:28:32.286070+0000","last_clean":"2023-10-09T05:28:32.286070+0000","last_became_active":"2023-10-09T05:09:57.857193+0000","last_became_peered":"2023-10-09T05:09:57.857193+0000","last_unstale":"2023-10-09T05:28:32.286070+0000","last_undegraded":"2023-10-09T05:28:32.286070+0000","last_fullsized":"2023-10-09T05:28:32.286070+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:32.842980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":282,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.351040+0000","last_change":"2023-10-09T05:18:26.422923+0000","last_active":"2023-10-09T05:28:51.351040+0000","last_peered":"2023-10-09T05:28:51.351040+0000","last_clean":"2023-10-09T05:28:51.351040+0000","last_became_active":"2023-10-09T05:18:26.422508+0000","last_became_peered":"2023-10-09T05:18:26.422508+0000","last_unstale":"2023-10-09T05:28:51.351040+0000","last_undegraded":"2023-10-09T05:28:51.351040+0000","last_fullsized":"2023-10-09T05:28:51.351040+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:00.876037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":301,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:50.151550+0000","last_change":"2023-10-09T05:23:18.188378+0000","last_active":"2023-10-09T05:28:50.151550+0000","last_peered":"2023-10-09T05:28:50.151550+0000","last_clean":"2023-10-09T05:28:50.151550+0000","last_became_active":"2023-10-09T05:23:18.186629+0000","last_became_peered":"2023-10-09T05:23:18.186629+0000","last_unstale":"2023-10-09T05:28:50.151550+0000","last_undegraded":"2023-10-09T05:28:50.151550+0000","last_fullsized":"2023-10-09T05:28:50.151550+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:16.444225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":271,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.293850+0000","last_change":"2023-10-09T05:13:03.039637+0000","last_active":"2023-10-09T05:28:32.293850+0000","last_peered":"2023-10-09T05:28:32.293850+0000","last_clean":"2023-10-09T05:28:32.293850+0000","last_became_active":"2023-10-09T05:13:03.039268+0000","last_became_peered":"2023-10-09T05:13:03.039268+0000","last_unstale":"2023-10-09T05:28:32.293850+0000","last_undegraded":"2023-10-09T05:28:32.293850+0000","last_fullsized":"2023-10-09T05:28:32.293850+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:19.875767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":269,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.310395+0000","last_change":"2023-10-09T05:22:47.216769+0000","last_active":"2023-10-09T05:28:32.310395+0000","last_peered":"2023-10-09T05:28:32.310395+0000","last_clean":"2023-10-09T05:28:32.310395+0000","last_became_active":"2023-10-09T05:22:47.215847+0000","last_became_peered":"2023-10-09T05:22:47.215847+0000","last_unstale":"2023-10-09T05:28:32.310395+0000","last_undegraded":"2023-10-09T05:28:32.310395+0000","last_fullsized":"2023-10-09T05:28:32.310395+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:51.392675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":256,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.307405+0000","last_change":"2023-10-09T05:09:57.849695+0000","last_active":"2023-10-09T05:28:32.307405+0000","last_peered":"2023-10-09T05:28:32.307405+0000","last_clean":"2023-10-09T05:28:32.307405+0000","last_became_active":"2023-10-09T05:09:57.848478+0000","last_became_peered":"2023-10-09T05:09:57.848478+0000","last_unstale":"2023-10-09T05:28:32.307405+0000","last_undegraded":"2023-10-09T05:28:32.307405+0000","last_fullsized":"2023-10-09T05:28:32.307405+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:26.115312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":90,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.294008+0000","last_change":"2023-10-09T05:23:18.194144+0000","last_active":"2023-10-09T05:28:32.294008+0000","last_peered":"2023-10-09T05:28:32.294008+0000","last_clean":"2023-10-09T05:28:32.294008+0000","last_became_active":"2023-10-09T05:23:18.193390+0000","last_became_peered":"2023-10-09T05:23:18.193390+0000","last_unstale":"2023-10-09T05:28:32.294008+0000","last_undegraded":"2023-10-09T05:28:32.294008+0000","last_fullsized":"2023-10-09T05:28:32.294008+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:22.549362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":269,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.307306+0000","last_change":"2023-10-09T05:18:26.418431+0000","last_active":"2023-10-09T05:28:32.307306+0000","last_peered":"2023-10-09T05:28:32.307306+0000","last_clean":"2023-10-09T05:28:32.307306+0000","last_became_active":"2023-10-09T05:18:26.418100+0000","last_became_peered":"2023-10-09T05:18:26.418100+0000","last_unstale":"2023-10-09T05:28:32.307306+0000","last_undegraded":"2023-10-09T05:28:32.307306+0000","last_fullsized":"2023-10-09T05:28:32.307306+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:21.241286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":76,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.293502+0000","last_change":"2023-10-09T05:18:26.452552+0000","last_active":"2023-10-09T05:28:32.293502+0000","last_peered":"2023-10-09T05:28:32.293502+0000","last_clean":"2023-10-09T05:28:32.293502+0000","last_became_active":"2023-10-09T05:18:26.428454+0000","last_became_peered":"2023-10-09T05:18:26.428454+0000","last_unstale":"2023-10-09T05:28:32.293502+0000","last_undegraded":"2023-10-09T05:28:32.293502+0000","last_fullsized":"2023-10-09T05:28:32.293502+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:15:52.203861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":41,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.285856+0000","last_change":"2023-10-09T05:23:18.199181+0000","last_active":"2023-10-09T05:28:32.285856+0000","last_peered":"2023-10-09T05:28:32.285856+0000","last_clean":"2023-10-09T05:28:32.285856+0000","last_became_active":"2023-10-09T05:23:18.198401+0000","last_became_peered":"2023-10-09T05:23:18.198401+0000","last_unstale":"2023-10-09T05:28:32.285856+0000","last_undegraded":"2023-10-09T05:28:32.285856+0000","last_fullsized":"2023-10-09T05:28:32.285856+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:55:49.891420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'23","reported_seq":333,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.301497+0000","last_change":"2023-10-09T05:18:26.436165+0000","last_active":"2023-10-09T05:28:32.301497+0000","last_peered":"2023-10-09T05:28:32.301497+0000","last_clean":"2023-10-09T05:28:32.301497+0000","last_became_active":"2023-10-09T05:18:26.430078+0000","last_became_peered":"2023-10-09T05:18:26.430078+0000","last_unstale":"2023-10-09T05:28:32.301497+0000","last_undegraded":"2023-10-09T05:28:32.301497+0000","last_fullsized":"2023-10-09T05:28:32.301497+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:04:48.797166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":261,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.301735+0000","last_change":"2023-10-09T05:09:57.868761+0000","last_active":"2023-10-09T05:28:32.301735+0000","last_peered":"2023-10-09T05:28:32.301735+0000","last_clean":"2023-10-09T05:28:32.301735+0000","last_became_active":"2023-10-09T05:09:57.868121+0000","last_became_peered":"2023-10-09T05:09:57.868121+0000","last_unstale":"2023-10-09T05:28:32.301735+0000","last_undegraded":"2023-10-09T05:28:32.301735+0000","last_fullsized":"2023-10-09T05:28:32.301735+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:43.412235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":76,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.293459+0000","last_change":"2023-10-09T05:18:26.443719+0000","last_active":"2023-10-09T05:28:32.293459+0000","last_peered":"2023-10-09T05:28:32.293459+0000","last_clean":"2023-10-09T05:28:32.293459+0000","last_became_active":"2023-10-09T05:18:26.426718+0000","last_became_peered":"2023-10-09T05:18:26.426718+0000","last_unstale":"2023-10-09T05:28:32.293459+0000","last_undegraded":"2023-10-09T05:28:32.293459+0000","last_fullsized":"2023-10-09T05:28:32.293459+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:49:12.090862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":310,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.309457+0000","last_change":"2023-10-09T05:23:18.187593+0000","last_active":"2023-10-09T05:28:32.309457+0000","last_peered":"2023-10-09T05:28:32.309457+0000","last_clean":"2023-10-09T05:28:32.309457+0000","last_became_active":"2023-10-09T05:23:18.186445+0000","last_became_peered":"2023-10-09T05:23:18.186445+0000","last_unstale":"2023-10-09T05:28:32.309457+0000","last_undegraded":"2023-10-09T05:28:32.309457+0000","last_fullsized":"2023-10-09T05:28:32.309457+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:11.327635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":278,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.305712+0000","last_change":"2023-10-09T05:18:26.424399+0000","last_active":"2023-10-09T05:28:32.305712+0000","last_peered":"2023-10-09T05:28:32.305712+0000","last_clean":"2023-10-09T05:28:32.305712+0000","last_became_active":"2023-10-09T05:18:26.422122+0000","last_became_peered":"2023-10-09T05:18:26.422122+0000","last_unstale":"2023-10-09T05:28:32.305712+0000","last_undegraded":"2023-10-09T05:28:32.305712+0000","last_fullsized":"2023-10-09T05:28:32.305712+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:39.393756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":70,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.293448+0000","last_change":"2023-10-09T05:18:26.422950+0000","last_active":"2023-10-09T05:28:32.293448+0000","last_peered":"2023-10-09T05:28:32.293448+0000","last_clean":"2023-10-09T05:28:32.293448+0000","last_became_active":"2023-10-09T05:18:26.422592+0000","last_became_peered":"2023-10-09T05:18:26.422592+0000","last_unstale":"2023-10-09T05:28:32.293448+0000","last_undegraded":"2023-10-09T05:28:32.293448+0000","last_fullsized":"2023-10-09T05:28:32.293448+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:20.701107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":70,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.292741+0000","last_change":"2023-10-09T05:18:26.429482+0000","last_active":"2023-10-09T05:28:32.292741+0000","last_peered":"2023-10-09T05:28:32.292741+0000","last_clean":"2023-10-09T05:28:32.292741+0000","last_became_active":"2023-10-09T05:18:26.429122+0000","last_became_peered":"2023-10-09T05:18:26.429122+0000","last_unstale":"2023-10-09T05:28:32.292741+0000","last_undegraded":"2023-10-09T05:28:32.292741+0000","last_fullsized":"2023-10-09T05:28:32.292741+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:47.301906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":279,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.317231+0000","last_change":"2023-10-09T05:18:26.421678+0000","last_active":"2023-10-09T05:28:32.317231+0000","last_peered":"2023-10-09T05:28:32.317231+0000","last_clean":"2023-10-09T05:28:32.317231+0000","last_became_active":"2023-10-09T05:18:26.419673+0000","last_became_peered":"2023-10-09T05:18:26.419673+0000","last_unstale":"2023-10-09T05:28:32.317231+0000","last_undegraded":"2023-10-09T05:28:32.317231+0000","last_fullsized":"2023-10-09T05:28:32.317231+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:38.876021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":274,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.292893+0000","last_change":"2023-10-09T05:13:03.047339+0000","last_active":"2023-10-09T05:28:32.292893+0000","last_peered":"2023-10-09T05:28:32.292893+0000","last_clean":"2023-10-09T05:28:32.292893+0000","last_became_active":"2023-10-09T05:13:03.040038+0000","last_became_peered":"2023-10-09T05:13:03.040038+0000","last_unstale":"2023-10-09T05:28:32.292893+0000","last_undegraded":"2023-10-09T05:28:32.292893+0000","last_fullsized":"2023-10-09T05:28:32.292893+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:56.726249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":292,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:45.138776+0000","last_change":"2023-10-09T05:18:26.415173+0000","last_active":"2023-10-09T05:28:45.138776+0000","last_peered":"2023-10-09T05:28:45.138776+0000","last_clean":"2023-10-09T05:28:45.138776+0000","last_became_active":"2023-10-09T05:18:26.414452+0000","last_became_peered":"2023-10-09T05:18:26.414452+0000","last_unstale":"2023-10-09T05:28:45.138776+0000","last_undegraded":"2023-10-09T05:28:45.138776+0000","last_fullsized":"2023-10-09T05:28:45.138776+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:34.379767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"281'12273","reported_seq":12298,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:29:09.477420+0000","last_change":"2023-10-09T05:24:56.644144+0000","last_active":"2023-10-09T05:29:09.477420+0000","last_peered":"2023-10-09T05:29:09.477420+0000","last_clean":"2023-10-09T05:29:09.477420+0000","last_became_active":"2023-10-09T05:24:56.643459+0000","last_became_peered":"2023-10-09T05:24:56.643459+0000","last_unstale":"2023-10-09T05:29:09.477420+0000","last_undegraded":"2023-10-09T05:29:09.477420+0000","last_fullsized":"2023-10-09T05:29:09.477420+0000","mapping_epoch":264,"log_start":"272'4300","ondisk_log_start":"272'4300","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":7973,"log_dups_size":0,"ondisk_log_size":7973,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:39.899186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33357824,"num_objects":509,"num_object_clones":0,"num_object_copies":1527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12273,"num_write_kb":409024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":74,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.294061+0000","last_change":"2023-10-09T05:18:26.446608+0000","last_active":"2023-10-09T05:28:32.294061+0000","last_peered":"2023-10-09T05:28:32.294061+0000","last_clean":"2023-10-09T05:28:32.294061+0000","last_became_active":"2023-10-09T05:18:26.428420+0000","last_became_peered":"2023-10-09T05:18:26.428420+0000","last_unstale":"2023-10-09T05:28:32.294061+0000","last_undegraded":"2023-10-09T05:28:32.294061+0000","last_fullsized":"2023-10-09T05:28:32.294061+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:23.693908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":70,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.292649+0000","last_change":"2023-10-09T05:18:26.427461+0000","last_active":"2023-10-09T05:28:32.292649+0000","last_peered":"2023-10-09T05:28:32.292649+0000","last_clean":"2023-10-09T05:28:32.292649+0000","last_became_active":"2023-10-09T05:18:26.427118+0000","last_became_peered":"2023-10-09T05:18:26.427118+0000","last_unstale":"2023-10-09T05:28:32.292649+0000","last_undegraded":"2023-10-09T05:28:32.292649+0000","last_fullsized":"2023-10-09T05:28:32.292649+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:05:01.868197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.1","version":"281'12279","reported_seq":12304,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:29:09.503080+0000","last_change":"2023-10-09T05:24:56.643202+0000","last_active":"2023-10-09T05:29:09.503080+0000","last_peered":"2023-10-09T05:29:09.503080+0000","last_clean":"2023-10-09T05:29:09.503080+0000","last_became_active":"2023-10-09T05:24:56.642700+0000","last_became_peered":"2023-10-09T05:24:56.642700+0000","last_unstale":"2023-10-09T05:29:09.503080+0000","last_undegraded":"2023-10-09T05:29:09.503080+0000","last_fullsized":"2023-10-09T05:29:09.503080+0000","mapping_epoch":264,"log_start":"272'4500","ondisk_log_start":"272'4500","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":7779,"log_dups_size":0,"ondisk_log_size":7779,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:25.444496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33619968,"num_objects":513,"num_object_clones":0,"num_object_copies":1539,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":513,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12279,"num_write_kb":409344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":74,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.293963+0000","last_change":"2023-10-09T05:18:26.440042+0000","last_active":"2023-10-09T05:28:32.293963+0000","last_peered":"2023-10-09T05:28:32.293963+0000","last_clean":"2023-10-09T05:28:32.293963+0000","last_became_active":"2023-10-09T05:18:26.426726+0000","last_became_peered":"2023-10-09T05:18:26.426726+0000","last_unstale":"2023-10-09T05:28:32.293963+0000","last_undegraded":"2023-10-09T05:28:32.293963+0000","last_fullsized":"2023-10-09T05:28:32.293963+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:23.985650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":278,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.301483+0000","last_change":"2023-10-09T05:09:57.868031+0000","last_active":"2023-10-09T05:28:32.301483+0000","last_peered":"2023-10-09T05:28:32.301483+0000","last_clean":"2023-10-09T05:28:32.301483+0000","last_became_active":"2023-10-09T05:09:57.866477+0000","last_became_peered":"2023-10-09T05:09:57.866477+0000","last_unstale":"2023-10-09T05:28:32.301483+0000","last_undegraded":"2023-10-09T05:28:32.301483+0000","last_fullsized":"2023-10-09T05:28:32.301483+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:27.102583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"281'12273","reported_seq":12298,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:29:09.442723+0000","last_change":"2023-10-09T05:24:56.640700+0000","last_active":"2023-10-09T05:29:09.442723+0000","last_peered":"2023-10-09T05:29:09.442723+0000","last_clean":"2023-10-09T05:29:09.442723+0000","last_became_active":"2023-10-09T05:24:56.640352+0000","last_became_peered":"2023-10-09T05:24:56.640352+0000","last_unstale":"2023-10-09T05:29:09.442723+0000","last_undegraded":"2023-10-09T05:29:09.442723+0000","last_fullsized":"2023-10-09T05:29:09.442723+0000","mapping_epoch":264,"log_start":"272'4300","ondisk_log_start":"272'4300","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":7973,"log_dups_size":0,"ondisk_log_size":7973,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:55:08.218749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34799616,"num_objects":531,"num_object_clones":0,"num_object_copies":1593,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":531,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12273,"num_write_kb":409728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":74,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.293725+0000","last_change":"2023-10-09T05:18:26.437953+0000","last_active":"2023-10-09T05:28:32.293725+0000","last_peered":"2023-10-09T05:28:32.293725+0000","last_clean":"2023-10-09T05:28:32.293725+0000","last_became_active":"2023-10-09T05:18:26.422703+0000","last_became_peered":"2023-10-09T05:18:26.422703+0000","last_unstale":"2023-10-09T05:28:32.293725+0000","last_undegraded":"2023-10-09T05:28:32.293725+0000","last_fullsized":"2023-10-09T05:28:32.293725+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:13.926811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":311,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.293215+0000","last_change":"2023-10-09T05:18:26.423526+0000","last_active":"2023-10-09T05:28:32.293215+0000","last_peered":"2023-10-09T05:28:32.293215+0000","last_clean":"2023-10-09T05:28:32.293215+0000","last_became_active":"2023-10-09T05:18:26.423092+0000","last_became_peered":"2023-10-09T05:18:26.423092+0000","last_unstale":"2023-10-09T05:28:32.293215+0000","last_undegraded":"2023-10-09T05:28:32.293215+0000","last_fullsized":"2023-10-09T05:28:32.293215+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:54.877664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"281'12264","reported_seq":12292,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:29:09.317538+0000","last_change":"2023-10-09T05:24:56.634648+0000","last_active":"2023-10-09T05:29:09.317538+0000","last_peered":"2023-10-09T05:29:09.317538+0000","last_clean":"2023-10-09T05:29:09.317538+0000","last_became_active":"2023-10-09T05:24:56.633771+0000","last_became_peered":"2023-10-09T05:24:56.633771+0000","last_unstale":"2023-10-09T05:29:09.317538+0000","last_undegraded":"2023-10-09T05:29:09.317538+0000","last_fullsized":"2023-10-09T05:29:09.317538+0000","mapping_epoch":264,"log_start":"272'4100","ondisk_log_start":"272'4100","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":8164,"log_dups_size":0,"ondisk_log_size":8164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:20:25.848591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35520512,"num_objects":542,"num_object_clones":0,"num_object_copies":1626,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":542,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12264,"num_write_kb":409792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":203,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:50.151654+0000","last_change":"2023-10-09T05:13:03.037926+0000","last_active":"2023-10-09T05:28:50.151654+0000","last_peered":"2023-10-09T05:28:50.151654+0000","last_clean":"2023-10-09T05:28:50.151654+0000","last_became_active":"2023-10-09T05:13:03.037639+0000","last_became_peered":"2023-10-09T05:13:03.037639+0000","last_unstale":"2023-10-09T05:28:50.151654+0000","last_undegraded":"2023-10-09T05:28:50.151654+0000","last_fullsized":"2023-10-09T05:28:50.151654+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:52.977143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":262,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.305727+0000","last_change":"2023-10-09T05:09:57.852914+0000","last_active":"2023-10-09T05:28:32.305727+0000","last_peered":"2023-10-09T05:28:32.305727+0000","last_clean":"2023-10-09T05:28:32.305727+0000","last_became_active":"2023-10-09T05:09:57.852522+0000","last_became_peered":"2023-10-09T05:09:57.852522+0000","last_unstale":"2023-10-09T05:28:32.305727+0000","last_undegraded":"2023-10-09T05:28:32.305727+0000","last_fullsized":"2023-10-09T05:28:32.305727+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:24.848494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"281'12350","reported_seq":12375,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:29:09.530364+0000","last_change":"2023-10-09T05:24:56.639909+0000","last_active":"2023-10-09T05:29:09.530364+0000","last_peered":"2023-10-09T05:29:09.530364+0000","last_clean":"2023-10-09T05:29:09.530364+0000","last_became_active":"2023-10-09T05:24:56.639434+0000","last_became_peered":"2023-10-09T05:24:56.639434+0000","last_unstale":"2023-10-09T05:29:09.530364+0000","last_undegraded":"2023-10-09T05:29:09.530364+0000","last_fullsized":"2023-10-09T05:29:09.530364+0000","mapping_epoch":264,"log_start":"272'4600","ondisk_log_start":"272'4600","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":7750,"log_dups_size":0,"ondisk_log_size":7750,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:11.231733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36175872,"num_objects":552,"num_object_clones":0,"num_object_copies":1656,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":552,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12350,"num_write_kb":412864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"69'3","reported_seq":236,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.338704+0000","last_change":"2023-10-09T05:02:09.650579+0000","last_active":"2023-10-09T05:28:51.338704+0000","last_peered":"2023-10-09T05:28:51.338704+0000","last_clean":"2023-10-09T05:28:51.338704+0000","last_became_active":"2023-10-09T05:02:09.650160+0000","last_became_peered":"2023-10-09T05:02:09.650160+0000","last_unstale":"2023-10-09T05:28:51.338704+0000","last_undegraded":"2023-10-09T05:28:51.338704+0000","last_fullsized":"2023-10-09T05:28:51.338704+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:18.292670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":241,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.301390+0000","last_change":"2023-10-09T05:02:09.652449+0000","last_active":"2023-10-09T05:28:32.301390+0000","last_peered":"2023-10-09T05:28:32.301390+0000","last_clean":"2023-10-09T05:28:32.301390+0000","last_became_active":"2023-10-09T05:02:09.651994+0000","last_became_peered":"2023-10-09T05:02:09.651994+0000","last_unstale":"2023-10-09T05:28:32.301390+0000","last_undegraded":"2023-10-09T05:28:32.301390+0000","last_fullsized":"2023-10-09T05:28:32.301390+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:14.480513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"281'12251","reported_seq":12274,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:29:09.519052+0000","last_change":"2023-10-09T05:24:56.643292+0000","last_active":"2023-10-09T05:29:09.519052+0000","last_peered":"2023-10-09T05:29:09.519052+0000","last_clean":"2023-10-09T05:29:09.519052+0000","last_became_active":"2023-10-09T05:24:56.642863+0000","last_became_peered":"2023-10-09T05:24:56.642863+0000","last_unstale":"2023-10-09T05:29:09.519052+0000","last_undegraded":"2023-10-09T05:29:09.519052+0000","last_fullsized":"2023-10-09T05:29:09.519052+0000","mapping_epoch":264,"log_start":"272'4500","ondisk_log_start":"272'4500","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":7751,"log_dups_size":0,"ondisk_log_size":7751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:09.977734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36241408,"num_objects":553,"num_object_clones":0,"num_object_copies":1659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":553,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12251,"num_write_kb":409728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":282,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.293272+0000","last_change":"2023-10-09T05:23:18.199172+0000","last_active":"2023-10-09T05:28:32.293272+0000","last_peered":"2023-10-09T05:28:32.293272+0000","last_clean":"2023-10-09T05:28:32.293272+0000","last_became_active":"2023-10-09T05:23:18.198762+0000","last_became_peered":"2023-10-09T05:23:18.198762+0000","last_unstale":"2023-10-09T05:28:32.293272+0000","last_undegraded":"2023-10-09T05:28:32.293272+0000","last_fullsized":"2023-10-09T05:28:32.293272+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:44:23.239788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":298,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.301371+0000","last_change":"2023-10-09T05:20:18.129228+0000","last_active":"2023-10-09T05:28:32.301371+0000","last_peered":"2023-10-09T05:28:32.301371+0000","last_clean":"2023-10-09T05:28:32.301371+0000","last_became_active":"2023-10-09T05:20:18.128769+0000","last_became_peered":"2023-10-09T05:20:18.128769+0000","last_unstale":"2023-10-09T05:28:32.301371+0000","last_undegraded":"2023-10-09T05:28:32.301371+0000","last_fullsized":"2023-10-09T05:28:32.301371+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:50.427100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"281'12259","reported_seq":12285,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:29:10.130617+0000","last_change":"2023-10-09T05:24:56.631754+0000","last_active":"2023-10-09T05:29:10.130617+0000","last_peered":"2023-10-09T05:29:10.130617+0000","last_clean":"2023-10-09T05:29:10.130617+0000","last_became_active":"2023-10-09T05:24:56.631413+0000","last_became_peered":"2023-10-09T05:24:56.631413+0000","last_unstale":"2023-10-09T05:29:10.130617+0000","last_undegraded":"2023-10-09T05:29:10.130617+0000","last_fullsized":"2023-10-09T05:29:10.130617+0000","mapping_epoch":264,"log_start":"273'4700","ondisk_log_start":"273'4700","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":7559,"log_dups_size":0,"ondisk_log_size":7559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:06.416230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35192832,"num_objects":537,"num_object_clones":0,"num_object_copies":1611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12259,"num_write_kb":409472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":282,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.301380+0000","last_change":"2023-10-09T05:18:26.431286+0000","last_active":"2023-10-09T05:28:32.301380+0000","last_peered":"2023-10-09T05:28:32.301380+0000","last_clean":"2023-10-09T05:28:32.301380+0000","last_became_active":"2023-10-09T05:18:26.430127+0000","last_became_peered":"2023-10-09T05:18:26.430127+0000","last_unstale":"2023-10-09T05:28:32.301380+0000","last_undegraded":"2023-10-09T05:28:32.301380+0000","last_fullsized":"2023-10-09T05:28:32.301380+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:42.982259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":281,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.301980+0000","last_change":"2023-10-09T05:23:18.193800+0000","last_active":"2023-10-09T05:28:32.301980+0000","last_peered":"2023-10-09T05:28:32.301980+0000","last_clean":"2023-10-09T05:28:32.301980+0000","last_became_active":"2023-10-09T05:23:18.193338+0000","last_became_peered":"2023-10-09T05:23:18.193338+0000","last_unstale":"2023-10-09T05:28:32.301980+0000","last_undegraded":"2023-10-09T05:28:32.301980+0000","last_fullsized":"2023-10-09T05:28:32.301980+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:29.109566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"281'12297","reported_seq":12324,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:29:10.102473+0000","last_change":"2023-10-09T05:24:56.632416+0000","last_active":"2023-10-09T05:29:10.102473+0000","last_peered":"2023-10-09T05:29:10.102473+0000","last_clean":"2023-10-09T05:29:10.102473+0000","last_became_active":"2023-10-09T05:24:56.632033+0000","last_became_peered":"2023-10-09T05:24:56.632033+0000","last_unstale":"2023-10-09T05:29:10.102473+0000","last_undegraded":"2023-10-09T05:29:10.102473+0000","last_fullsized":"2023-10-09T05:29:10.102473+0000","mapping_epoch":264,"log_start":"272'4700","ondisk_log_start":"272'4700","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":7597,"log_dups_size":0,"ondisk_log_size":7597,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:08.729587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31784960,"num_objects":485,"num_object_clones":0,"num_object_copies":1455,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":485,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12297,"num_write_kb":409024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":294,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.307235+0000","last_change":"2023-10-09T05:18:26.413106+0000","last_active":"2023-10-09T05:28:32.307235+0000","last_peered":"2023-10-09T05:28:32.307235+0000","last_clean":"2023-10-09T05:28:32.307235+0000","last_became_active":"2023-10-09T05:18:26.412334+0000","last_became_peered":"2023-10-09T05:18:26.412334+0000","last_unstale":"2023-10-09T05:28:32.307235+0000","last_undegraded":"2023-10-09T05:28:32.307235+0000","last_fullsized":"2023-10-09T05:28:32.307235+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:18.058016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":148,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.289493+0000","last_change":"2023-10-09T05:09:57.856764+0000","last_active":"2023-10-09T05:28:32.289493+0000","last_peered":"2023-10-09T05:28:32.289493+0000","last_clean":"2023-10-09T05:28:32.289493+0000","last_became_active":"2023-10-09T05:09:57.856461+0000","last_became_peered":"2023-10-09T05:09:57.856461+0000","last_unstale":"2023-10-09T05:28:32.289493+0000","last_undegraded":"2023-10-09T05:28:32.289493+0000","last_fullsized":"2023-10-09T05:28:32.289493+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:06.848082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.a","version":"281'12473","reported_seq":12499,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:29:08.880044+0000","last_change":"2023-10-09T05:24:56.632715+0000","last_active":"2023-10-09T05:29:08.880044+0000","last_peered":"2023-10-09T05:29:08.880044+0000","last_clean":"2023-10-09T05:29:08.880044+0000","last_became_active":"2023-10-09T05:24:56.632411+0000","last_became_peered":"2023-10-09T05:24:56.632411+0000","last_unstale":"2023-10-09T05:29:08.880044+0000","last_undegraded":"2023-10-09T05:29:08.880044+0000","last_fullsized":"2023-10-09T05:29:08.880044+0000","mapping_epoch":264,"log_start":"271'2700","ondisk_log_start":"271'2700","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":9773,"log_dups_size":0,"ondisk_log_size":9773,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:25.311429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34144256,"num_objects":521,"num_object_clones":0,"num_object_copies":1563,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":521,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12473,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":185,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.332209+0000","last_change":"2023-10-09T05:18:26.422150+0000","last_active":"2023-10-09T05:28:51.332209+0000","last_peered":"2023-10-09T05:28:51.332209+0000","last_clean":"2023-10-09T05:28:51.332209+0000","last_became_active":"2023-10-09T05:18:26.421853+0000","last_became_peered":"2023-10-09T05:18:26.421853+0000","last_unstale":"2023-10-09T05:28:51.332209+0000","last_undegraded":"2023-10-09T05:28:51.332209+0000","last_fullsized":"2023-10-09T05:28:51.332209+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:37:47.000556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":220,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.307934+0000","last_change":"2023-10-09T05:13:03.031243+0000","last_active":"2023-10-09T05:28:32.307934+0000","last_peered":"2023-10-09T05:28:32.307934+0000","last_clean":"2023-10-09T05:28:32.307934+0000","last_became_active":"2023-10-09T05:13:03.028769+0000","last_became_peered":"2023-10-09T05:13:03.028769+0000","last_unstale":"2023-10-09T05:28:32.307934+0000","last_undegraded":"2023-10-09T05:28:32.307934+0000","last_fullsized":"2023-10-09T05:28:32.307934+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:25:25.239799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":41,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.285771+0000","last_change":"2023-10-09T05:23:18.197927+0000","last_active":"2023-10-09T05:28:32.285771+0000","last_peered":"2023-10-09T05:28:32.285771+0000","last_clean":"2023-10-09T05:28:32.285771+0000","last_became_active":"2023-10-09T05:23:18.197431+0000","last_became_peered":"2023-10-09T05:23:18.197431+0000","last_unstale":"2023-10-09T05:28:32.285771+0000","last_undegraded":"2023-10-09T05:28:32.285771+0000","last_fullsized":"2023-10-09T05:28:32.285771+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:51:41.376456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"281'12235","reported_seq":12258,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:29:09.318347+0000","last_change":"2023-10-09T05:24:56.631792+0000","last_active":"2023-10-09T05:29:09.318347+0000","last_peered":"2023-10-09T05:29:09.318347+0000","last_clean":"2023-10-09T05:29:09.318347+0000","last_became_active":"2023-10-09T05:24:56.631473+0000","last_became_peered":"2023-10-09T05:24:56.631473+0000","last_unstale":"2023-10-09T05:29:09.318347+0000","last_undegraded":"2023-10-09T05:29:09.318347+0000","last_fullsized":"2023-10-09T05:29:09.318347+0000","mapping_epoch":264,"log_start":"272'4100","ondisk_log_start":"272'4100","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":8135,"log_dups_size":0,"ondisk_log_size":8135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:17.170510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34406400,"num_objects":525,"num_object_clones":0,"num_object_copies":1575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12235,"num_write_kb":408320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":254,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.286003+0000","last_change":"2023-10-09T05:13:03.031386+0000","last_active":"2023-10-09T05:28:32.286003+0000","last_peered":"2023-10-09T05:28:32.286003+0000","last_clean":"2023-10-09T05:28:32.286003+0000","last_became_active":"2023-10-09T05:13:03.030816+0000","last_became_peered":"2023-10-09T05:13:03.030816+0000","last_unstale":"2023-10-09T05:28:32.286003+0000","last_undegraded":"2023-10-09T05:28:32.286003+0000","last_fullsized":"2023-10-09T05:28:32.286003+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:28.088661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":242,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.301434+0000","last_change":"2023-10-09T04:56:56.087320+0000","last_active":"2023-10-09T05:28:32.301434+0000","last_peered":"2023-10-09T05:28:32.301434+0000","last_clean":"2023-10-09T05:28:32.301434+0000","last_became_active":"2023-10-09T04:56:56.086552+0000","last_became_peered":"2023-10-09T04:56:56.086552+0000","last_unstale":"2023-10-09T05:28:32.301434+0000","last_undegraded":"2023-10-09T05:28:32.301434+0000","last_fullsized":"2023-10-09T05:28:32.301434+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:14.205288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":41,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.287626+0000","last_change":"2023-10-09T05:23:18.196260+0000","last_active":"2023-10-09T05:28:32.287626+0000","last_peered":"2023-10-09T05:28:32.287626+0000","last_clean":"2023-10-09T05:28:32.287626+0000","last_became_active":"2023-10-09T05:23:18.195907+0000","last_became_peered":"2023-10-09T05:23:18.195907+0000","last_unstale":"2023-10-09T05:28:32.287626+0000","last_undegraded":"2023-10-09T05:28:32.287626+0000","last_fullsized":"2023-10-09T05:28:32.287626+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:36.243432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.f","version":"281'12345","reported_seq":12370,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:29:09.533661+0000","last_change":"2023-10-09T05:24:56.644098+0000","last_active":"2023-10-09T05:29:09.533661+0000","last_peered":"2023-10-09T05:29:09.533661+0000","last_clean":"2023-10-09T05:29:09.533661+0000","last_became_active":"2023-10-09T05:24:56.643666+0000","last_became_peered":"2023-10-09T05:24:56.643666+0000","last_unstale":"2023-10-09T05:29:09.533661+0000","last_undegraded":"2023-10-09T05:29:09.533661+0000","last_fullsized":"2023-10-09T05:29:09.533661+0000","mapping_epoch":264,"log_start":"272'4600","ondisk_log_start":"272'4600","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":7745,"log_dups_size":0,"ondisk_log_size":7745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:39:36.431548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33488896,"num_objects":511,"num_object_clones":0,"num_object_copies":1533,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":511,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12345,"num_write_kb":411392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":60,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:38.713533+0000","last_change":"2023-10-09T05:20:18.123706+0000","last_active":"2023-10-09T05:28:38.713533+0000","last_peered":"2023-10-09T05:28:38.713533+0000","last_clean":"2023-10-09T05:28:38.713533+0000","last_became_active":"2023-10-09T05:20:18.122859+0000","last_became_peered":"2023-10-09T05:20:18.122859+0000","last_unstale":"2023-10-09T05:28:38.713533+0000","last_undegraded":"2023-10-09T05:28:38.713533+0000","last_fullsized":"2023-10-09T05:28:38.713533+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.906228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":149,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.285866+0000","last_change":"2023-10-09T05:09:57.854064+0000","last_active":"2023-10-09T05:28:32.285866+0000","last_peered":"2023-10-09T05:28:32.285866+0000","last_clean":"2023-10-09T05:28:32.285866+0000","last_became_active":"2023-10-09T05:09:57.847953+0000","last_became_peered":"2023-10-09T05:09:57.847953+0000","last_unstale":"2023-10-09T05:28:32.285866+0000","last_undegraded":"2023-10-09T05:28:32.285866+0000","last_fullsized":"2023-10-09T05:28:32.285866+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:45.914976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":24,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.310922+0000","last_change":"2023-10-09T05:25:15.659153+0000","last_active":"2023-10-09T05:28:32.310922+0000","last_peered":"2023-10-09T05:28:32.310922+0000","last_clean":"2023-10-09T05:28:32.310922+0000","last_became_active":"2023-10-09T05:25:15.658750+0000","last_became_peered":"2023-10-09T05:25:15.658750+0000","last_unstale":"2023-10-09T05:28:32.310922+0000","last_undegraded":"2023-10-09T05:28:32.310922+0000","last_fullsized":"2023-10-09T05:28:32.310922+0000","mapping_epoch":268,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":269,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:10.370264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.d","version":"281'12289","reported_seq":12314,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:29:09.564905+0000","last_change":"2023-10-09T05:24:56.633695+0000","last_active":"2023-10-09T05:29:09.564905+0000","last_peered":"2023-10-09T05:29:09.564905+0000","last_clean":"2023-10-09T05:29:09.564905+0000","last_became_active":"2023-10-09T05:24:56.633393+0000","last_became_peered":"2023-10-09T05:24:56.633393+0000","last_unstale":"2023-10-09T05:29:09.564905+0000","last_undegraded":"2023-10-09T05:29:09.564905+0000","last_fullsized":"2023-10-09T05:29:09.564905+0000","mapping_epoch":264,"log_start":"272'4500","ondisk_log_start":"272'4500","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":7789,"log_dups_size":0,"ondisk_log_size":7789,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:04.072215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33423384,"num_objects":511,"num_object_clones":0,"num_object_copies":1533,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":511,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12289,"num_write_kb":409537,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":226,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.306377+0000","last_change":"2023-10-09T05:18:26.416511+0000","last_active":"2023-10-09T05:28:32.306377+0000","last_peered":"2023-10-09T05:28:32.306377+0000","last_clean":"2023-10-09T05:28:32.306377+0000","last_became_active":"2023-10-09T05:18:26.416101+0000","last_became_peered":"2023-10-09T05:18:26.416101+0000","last_unstale":"2023-10-09T05:28:32.306377+0000","last_undegraded":"2023-10-09T05:28:32.306377+0000","last_fullsized":"2023-10-09T05:28:32.306377+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:10.243472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":566,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.292709+0000","last_change":"2023-10-09T05:18:29.419527+0000","last_active":"2023-10-09T05:28:32.292709+0000","last_peered":"2023-10-09T05:28:32.292709+0000","last_clean":"2023-10-09T05:28:32.292709+0000","last_became_active":"2023-10-09T05:18:29.418852+0000","last_became_peered":"2023-10-09T05:18:29.418852+0000","last_unstale":"2023-10-09T05:28:32.292709+0000","last_undegraded":"2023-10-09T05:28:32.292709+0000","last_fullsized":"2023-10-09T05:28:32.292709+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_clean_scrub_stamp":"2023-10-09T04:36:44.261013+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:02.899170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":243,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.315815+0000","last_change":"2023-10-09T05:13:03.027260+0000","last_active":"2023-10-09T05:28:32.315815+0000","last_peered":"2023-10-09T05:28:32.315815+0000","last_clean":"2023-10-09T05:28:32.315815+0000","last_became_active":"2023-10-09T05:13:03.026416+0000","last_became_peered":"2023-10-09T05:13:03.026416+0000","last_unstale":"2023-10-09T05:28:32.315815+0000","last_undegraded":"2023-10-09T05:28:32.315815+0000","last_fullsized":"2023-10-09T05:28:32.315815+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:05.057338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":264,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.293585+0000","last_change":"2023-10-09T05:18:26.426126+0000","last_active":"2023-10-09T05:28:32.293585+0000","last_peered":"2023-10-09T05:28:32.293585+0000","last_clean":"2023-10-09T05:28:32.293585+0000","last_became_active":"2023-10-09T05:18:26.425721+0000","last_became_peered":"2023-10-09T05:18:26.425721+0000","last_unstale":"2023-10-09T05:28:32.293585+0000","last_undegraded":"2023-10-09T05:28:32.293585+0000","last_fullsized":"2023-10-09T05:28:32.293585+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:48.229218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"281'12486","reported_seq":12513,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:29:09.300772+0000","last_change":"2023-10-09T05:24:56.641295+0000","last_active":"2023-10-09T05:29:09.300772+0000","last_peered":"2023-10-09T05:29:09.300772+0000","last_clean":"2023-10-09T05:29:09.300772+0000","last_became_active":"2023-10-09T05:24:56.640909+0000","last_became_peered":"2023-10-09T05:24:56.640909+0000","last_unstale":"2023-10-09T05:29:09.300772+0000","last_undegraded":"2023-10-09T05:29:09.300772+0000","last_fullsized":"2023-10-09T05:29:09.300772+0000","mapping_epoch":264,"log_start":"272'4300","ondisk_log_start":"272'4300","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":8186,"log_dups_size":0,"ondisk_log_size":8186,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:26.475478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37093376,"num_objects":566,"num_object_clones":0,"num_object_copies":1698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12486,"num_write_kb":417664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":90,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.292768+0000","last_change":"2023-10-09T05:23:18.193825+0000","last_active":"2023-10-09T05:28:32.292768+0000","last_peered":"2023-10-09T05:28:32.292768+0000","last_clean":"2023-10-09T05:28:32.292768+0000","last_became_active":"2023-10-09T05:23:18.193394+0000","last_became_peered":"2023-10-09T05:23:18.193394+0000","last_unstale":"2023-10-09T05:28:32.292768+0000","last_undegraded":"2023-10-09T05:28:32.292768+0000","last_fullsized":"2023-10-09T05:28:32.292768+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:40.594469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":244,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.305758+0000","last_change":"2023-10-09T05:02:09.648781+0000","last_active":"2023-10-09T05:28:32.305758+0000","last_peered":"2023-10-09T05:28:32.305758+0000","last_clean":"2023-10-09T05:28:32.305758+0000","last_became_active":"2023-10-09T05:02:09.648078+0000","last_became_peered":"2023-10-09T05:02:09.648078+0000","last_unstale":"2023-10-09T05:28:32.305758+0000","last_undegraded":"2023-10-09T05:28:32.305758+0000","last_fullsized":"2023-10-09T05:28:32.305758+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:35.287091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":322,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.316960+0000","last_change":"2023-10-09T05:20:18.132750+0000","last_active":"2023-10-09T05:28:32.316960+0000","last_peered":"2023-10-09T05:28:32.316960+0000","last_clean":"2023-10-09T05:28:32.316960+0000","last_became_active":"2023-10-09T05:20:18.132396+0000","last_became_peered":"2023-10-09T05:20:18.132396+0000","last_unstale":"2023-10-09T05:28:32.316960+0000","last_undegraded":"2023-10-09T05:28:32.316960+0000","last_fullsized":"2023-10-09T05:28:32.316960+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:00:55.731045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"281'12297","reported_seq":12324,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:29:10.198284+0000","last_change":"2023-10-09T05:24:56.642285+0000","last_active":"2023-10-09T05:29:10.198284+0000","last_peered":"2023-10-09T05:29:10.198284+0000","last_clean":"2023-10-09T05:29:10.198284+0000","last_became_active":"2023-10-09T05:24:56.641521+0000","last_became_peered":"2023-10-09T05:24:56.641521+0000","last_unstale":"2023-10-09T05:29:10.198284+0000","last_undegraded":"2023-10-09T05:29:10.198284+0000","last_fullsized":"2023-10-09T05:29:10.198284+0000","mapping_epoch":264,"log_start":"274'5300","ondisk_log_start":"274'5300","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":6997,"log_dups_size":0,"ondisk_log_size":6997,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:06:01.013563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33882112,"num_objects":517,"num_object_clones":0,"num_object_copies":1551,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":517,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12297,"num_write_kb":410048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":70,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.293802+0000","last_change":"2023-10-09T05:18:26.429468+0000","last_active":"2023-10-09T05:28:32.293802+0000","last_peered":"2023-10-09T05:28:32.293802+0000","last_clean":"2023-10-09T05:28:32.293802+0000","last_became_active":"2023-10-09T05:18:26.429098+0000","last_became_peered":"2023-10-09T05:18:26.429098+0000","last_unstale":"2023-10-09T05:28:32.293802+0000","last_undegraded":"2023-10-09T05:28:32.293802+0000","last_fullsized":"2023-10-09T05:28:32.293802+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:47.798226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":176,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.332413+0000","last_change":"2023-10-09T05:20:18.126997+0000","last_active":"2023-10-09T05:28:51.332413+0000","last_peered":"2023-10-09T05:28:51.332413+0000","last_clean":"2023-10-09T05:28:51.332413+0000","last_became_active":"2023-10-09T05:20:18.126641+0000","last_became_peered":"2023-10-09T05:20:18.126641+0000","last_unstale":"2023-10-09T05:28:51.332413+0000","last_undegraded":"2023-10-09T05:28:51.332413+0000","last_fullsized":"2023-10-09T05:28:51.332413+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:42.112245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":284,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.310006+0000","last_change":"2023-10-09T05:18:26.418524+0000","last_active":"2023-10-09T05:28:32.310006+0000","last_peered":"2023-10-09T05:28:32.310006+0000","last_clean":"2023-10-09T05:28:32.310006+0000","last_became_active":"2023-10-09T05:18:26.418058+0000","last_became_peered":"2023-10-09T05:18:26.418058+0000","last_unstale":"2023-10-09T05:28:32.310006+0000","last_undegraded":"2023-10-09T05:28:32.310006+0000","last_fullsized":"2023-10-09T05:28:32.310006+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:40.594449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"281'12369","reported_seq":12396,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:29:10.187978+0000","last_change":"2023-10-09T05:24:56.633539+0000","last_active":"2023-10-09T05:29:10.187978+0000","last_peered":"2023-10-09T05:29:10.187978+0000","last_clean":"2023-10-09T05:29:10.187978+0000","last_became_active":"2023-10-09T05:24:56.633197+0000","last_became_peered":"2023-10-09T05:24:56.633197+0000","last_unstale":"2023-10-09T05:29:10.187978+0000","last_undegraded":"2023-10-09T05:29:10.187978+0000","last_fullsized":"2023-10-09T05:29:10.187978+0000","mapping_epoch":264,"log_start":"274'5300","ondisk_log_start":"274'5300","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":7069,"log_dups_size":0,"ondisk_log_size":7069,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:53.863072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33751040,"num_objects":515,"num_object_clones":0,"num_object_copies":1545,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":515,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12369,"num_write_kb":412288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":287,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.309812+0000","last_change":"2023-10-09T05:18:26.431563+0000","last_active":"2023-10-09T05:28:32.309812+0000","last_peered":"2023-10-09T05:28:32.309812+0000","last_clean":"2023-10-09T05:28:32.309812+0000","last_became_active":"2023-10-09T05:18:26.421025+0000","last_became_peered":"2023-10-09T05:18:26.421025+0000","last_unstale":"2023-10-09T05:28:32.309812+0000","last_undegraded":"2023-10-09T05:28:32.309812+0000","last_fullsized":"2023-10-09T05:28:32.309812+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:30.333585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":41,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.289660+0000","last_change":"2023-10-09T05:23:18.196610+0000","last_active":"2023-10-09T05:28:32.289660+0000","last_peered":"2023-10-09T05:28:32.289660+0000","last_clean":"2023-10-09T05:28:32.289660+0000","last_became_active":"2023-10-09T05:23:18.196279+0000","last_became_peered":"2023-10-09T05:23:18.196279+0000","last_unstale":"2023-10-09T05:28:32.289660+0000","last_undegraded":"2023-10-09T05:28:32.289660+0000","last_fullsized":"2023-10-09T05:28:32.289660+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:52.868384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":286,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.307318+0000","last_change":"2023-10-09T05:21:08.179657+0000","last_active":"2023-10-09T05:28:32.307318+0000","last_peered":"2023-10-09T05:28:32.307318+0000","last_clean":"2023-10-09T05:28:32.307318+0000","last_became_active":"2023-10-09T05:21:08.178958+0000","last_became_peered":"2023-10-09T05:21:08.178958+0000","last_unstale":"2023-10-09T05:28:32.307318+0000","last_undegraded":"2023-10-09T05:28:32.307318+0000","last_fullsized":"2023-10-09T05:28:32.307318+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:20.815986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"281'12293","reported_seq":12320,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:29:08.855864+0000","last_change":"2023-10-09T05:24:56.641633+0000","last_active":"2023-10-09T05:29:08.855864+0000","last_peered":"2023-10-09T05:29:08.855864+0000","last_clean":"2023-10-09T05:29:08.855864+0000","last_became_active":"2023-10-09T05:24:56.641288+0000","last_became_peered":"2023-10-09T05:24:56.641288+0000","last_unstale":"2023-10-09T05:29:08.855864+0000","last_undegraded":"2023-10-09T05:29:08.855864+0000","last_fullsized":"2023-10-09T05:29:08.855864+0000","mapping_epoch":264,"log_start":"271'2600","ondisk_log_start":"271'2600","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":9693,"log_dups_size":0,"ondisk_log_size":9693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:19.463846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36634624,"num_objects":559,"num_object_clones":0,"num_object_copies":1677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":559,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12293,"num_write_kb":411264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":286,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.293927+0000","last_change":"2023-10-09T05:18:26.424429+0000","last_active":"2023-10-09T05:28:32.293927+0000","last_peered":"2023-10-09T05:28:32.293927+0000","last_clean":"2023-10-09T05:28:32.293927+0000","last_became_active":"2023-10-09T05:18:26.423034+0000","last_became_peered":"2023-10-09T05:18:26.423034+0000","last_unstale":"2023-10-09T05:28:32.293927+0000","last_undegraded":"2023-10-09T05:28:32.293927+0000","last_fullsized":"2023-10-09T05:28:32.293927+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:16.251181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":274,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.317065+0000","last_change":"2023-10-09T05:09:57.878002+0000","last_active":"2023-10-09T05:28:32.317065+0000","last_peered":"2023-10-09T05:28:32.317065+0000","last_clean":"2023-10-09T05:28:32.317065+0000","last_became_active":"2023-10-09T05:09:57.852792+0000","last_became_peered":"2023-10-09T05:09:57.852792+0000","last_unstale":"2023-10-09T05:28:32.317065+0000","last_undegraded":"2023-10-09T05:28:32.317065+0000","last_fullsized":"2023-10-09T05:28:32.317065+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:18.763275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":234,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.301150+0000","last_change":"2023-10-09T05:13:03.026541+0000","last_active":"2023-10-09T05:28:32.301150+0000","last_peered":"2023-10-09T05:28:32.301150+0000","last_clean":"2023-10-09T05:28:32.301150+0000","last_became_active":"2023-10-09T05:13:03.025729+0000","last_became_peered":"2023-10-09T05:13:03.025729+0000","last_unstale":"2023-10-09T05:28:32.301150+0000","last_undegraded":"2023-10-09T05:28:32.301150+0000","last_fullsized":"2023-10-09T05:28:32.301150+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:22.938235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'2","reported_seq":52,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:39.252986+0000","last_change":"2023-10-09T05:21:08.173845+0000","last_active":"2023-10-09T05:28:39.252986+0000","last_peered":"2023-10-09T05:28:39.252986+0000","last_clean":"2023-10-09T05:28:39.252986+0000","last_became_active":"2023-10-09T05:21:08.173508+0000","last_became_peered":"2023-10-09T05:21:08.173508+0000","last_unstale":"2023-10-09T05:28:39.252986+0000","last_undegraded":"2023-10-09T05:28:39.252986+0000","last_fullsized":"2023-10-09T05:28:39.252986+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:03.628428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":59,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.308023+0000","last_change":"2023-10-09T05:20:18.125172+0000","last_active":"2023-10-09T05:28:32.308023+0000","last_peered":"2023-10-09T05:28:32.308023+0000","last_clean":"2023-10-09T05:28:32.308023+0000","last_became_active":"2023-10-09T05:20:18.123624+0000","last_became_peered":"2023-10-09T05:20:18.123624+0000","last_unstale":"2023-10-09T05:28:32.308023+0000","last_undegraded":"2023-10-09T05:28:32.308023+0000","last_fullsized":"2023-10-09T05:28:32.308023+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:11.106483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":136,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.301592+0000","last_change":"2023-10-09T05:13:03.055887+0000","last_active":"2023-10-09T05:28:32.301592+0000","last_peered":"2023-10-09T05:28:32.301592+0000","last_clean":"2023-10-09T05:28:32.301592+0000","last_became_active":"2023-10-09T05:13:03.028782+0000","last_became_peered":"2023-10-09T05:13:03.028782+0000","last_unstale":"2023-10-09T05:28:32.301592+0000","last_undegraded":"2023-10-09T05:28:32.301592+0000","last_fullsized":"2023-10-09T05:28:32.301592+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:38.828205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":298,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.301821+0000","last_change":"2023-10-09T05:23:18.207836+0000","last_active":"2023-10-09T05:28:32.301821+0000","last_peered":"2023-10-09T05:28:32.301821+0000","last_clean":"2023-10-09T05:28:32.301821+0000","last_became_active":"2023-10-09T05:23:18.207404+0000","last_became_peered":"2023-10-09T05:23:18.207404+0000","last_unstale":"2023-10-09T05:28:32.301821+0000","last_undegraded":"2023-10-09T05:28:32.301821+0000","last_fullsized":"2023-10-09T05:28:32.301821+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:36:22.236093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":136,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.301504+0000","last_change":"2023-10-09T05:13:03.049546+0000","last_active":"2023-10-09T05:28:32.301504+0000","last_peered":"2023-10-09T05:28:32.301504+0000","last_clean":"2023-10-09T05:28:32.301504+0000","last_became_active":"2023-10-09T05:13:03.028676+0000","last_became_peered":"2023-10-09T05:13:03.028676+0000","last_unstale":"2023-10-09T05:28:32.301504+0000","last_undegraded":"2023-10-09T05:28:32.301504+0000","last_fullsized":"2023-10-09T05:28:32.301504+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:55.089350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":258,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.301620+0000","last_change":"2023-10-09T05:02:09.649748+0000","last_active":"2023-10-09T05:28:32.301620+0000","last_peered":"2023-10-09T05:28:32.301620+0000","last_clean":"2023-10-09T05:28:32.301620+0000","last_became_active":"2023-10-09T05:02:09.649444+0000","last_became_peered":"2023-10-09T05:02:09.649444+0000","last_unstale":"2023-10-09T05:28:32.301620+0000","last_undegraded":"2023-10-09T05:28:32.301620+0000","last_fullsized":"2023-10-09T05:28:32.301620+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:07.436820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'2","reported_seq":280,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.307983+0000","last_change":"2023-10-09T05:22:47.197002+0000","last_active":"2023-10-09T05:28:32.307983+0000","last_peered":"2023-10-09T05:28:32.307983+0000","last_clean":"2023-10-09T05:28:32.307983+0000","last_became_active":"2023-10-09T05:22:47.196685+0000","last_became_peered":"2023-10-09T05:22:47.196685+0000","last_unstale":"2023-10-09T05:28:32.307983+0000","last_undegraded":"2023-10-09T05:28:32.307983+0000","last_fullsized":"2023-10-09T05:28:32.307983+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:47:32.594447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":265,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:50.151576+0000","last_change":"2023-10-09T05:02:09.649674+0000","last_active":"2023-10-09T05:28:50.151576+0000","last_peered":"2023-10-09T05:28:50.151576+0000","last_clean":"2023-10-09T05:28:50.151576+0000","last_became_active":"2023-10-09T05:02:09.649041+0000","last_became_peered":"2023-10-09T05:02:09.649041+0000","last_unstale":"2023-10-09T05:28:50.151576+0000","last_undegraded":"2023-10-09T05:28:50.151576+0000","last_fullsized":"2023-10-09T05:28:50.151576+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:18.524071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'5","reported_seq":58,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.310122+0000","last_change":"2023-10-09T05:21:08.181594+0000","last_active":"2023-10-09T05:28:32.310122+0000","last_peered":"2023-10-09T05:28:32.310122+0000","last_clean":"2023-10-09T05:28:32.310122+0000","last_became_active":"2023-10-09T05:21:08.176895+0000","last_became_peered":"2023-10-09T05:21:08.176895+0000","last_unstale":"2023-10-09T05:28:32.310122+0000","last_undegraded":"2023-10-09T05:28:32.310122+0000","last_fullsized":"2023-10-09T05:28:32.310122+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:48:45.569544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":271,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T05:28:32.301278+0000","last_change":"2023-10-09T05:25:15.663537+0000","last_active":"2023-10-09T05:28:32.301278+0000","last_peered":"2023-10-09T05:28:32.301278+0000","last_clean":"2023-10-09T05:28:32.301278+0000","last_became_active":"2023-10-09T05:25:15.662771+0000","last_became_peered":"2023-10-09T05:25:15.662771+0000","last_unstale":"2023-10-09T05:28:32.301278+0000","last_undegraded":"2023-10-09T05:28:32.301278+0000","last_fullsized":"2023-10-09T05:28:32.301278+0000","mapping_epoch":268,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":269,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:53.588294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":553517080,"num_objects":8447,"num_object_clones":0,"num_object_copies":25341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8447,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":197033,"num_write_kb":6575297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1663119360,"data_stored":1663107144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127933,"ondisk_log_size":127933,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":20,"num_read_kb":19,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":46,"num_bytes_recovered":21458,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234563,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1198336,"kb_used_data":295288,"kb_used_omap":0,"kb_used_meta":903040,"kb_avail":92546816,"statfs":{"total":95995035648,"available":94767939584,"internally_reserved":0,"allocated":302374912,"data_stored":301482685,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924712960},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":3.2149999999999999,"15min":2.1269999999999998},"min":{"1min":0.26500000000000001,"5min":0.248,"15min":0.248},"max":{"1min":3.4990000000000001,"5min":122.248,"15min":122.248},"last":5.4980000000000002},{"interface":"front","average":{"1min":1.304,"5min":3.218,"15min":2.1920000000000002},"min":{"1min":0.30199999999999999,"5min":0.29499999999999998,"15min":0.29099999999999998},"max":{"1min":6.1280000000000001,"5min":122.22199999999999,"15min":122.22199999999999},"last":2.2850000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:28:38 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":1.21,"15min":1.353},"min":{"1min":0.30499999999999999,"5min":0.29199999999999998,"15min":0.28000000000000003},"max":{"1min":3.9239999999999999,"5min":6.1109999999999998,"15min":23.530999999999999},"last":5.5199999999999996},{"interface":"front","average":{"1min":1.593,"5min":1.321,"15min":1.4339999999999999},"min":{"1min":0.313,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":6.1699999999999999,"5min":6.8710000000000004,"15min":23.443000000000001},"last":2.246}]},{"osd":2,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":3.4249999999999998,"5min":3.3730000000000002,"15min":2.3029999999999999},"min":{"1min":0.314,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":37.945999999999998,"5min":122.065,"15min":122.065},"last":2.403},{"interface":"front","average":{"1min":3.9470000000000001,"5min":3.8959999999999999,"15min":2.4449999999999998},"min":{"1min":0.34300000000000003,"5min":0.32700000000000001,"15min":0.30399999999999999},"max":{"1min":38.043999999999997,"5min":122.026,"15min":122.026},"last":4.0220000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":2.8599999999999999,"5min":3.3500000000000001,"15min":2.4380000000000002},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":38.636000000000003,"5min":138.667,"15min":138.667},"last":8.0090000000000003},{"interface":"front","average":{"1min":3.4950000000000001,"5min":3.4969999999999999,"15min":2.6080000000000001},"min":{"1min":0.35099999999999998,"5min":0.34699999999999998,"15min":0.34300000000000003},"max":{"1min":41.343000000000004,"5min":138.54400000000001,"15min":138.54400000000001},"last":2.6970000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:28:38 2023","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":1.01,"15min":0.98299999999999998},"min":{"1min":0.34899999999999998,"5min":0.311,"15min":0.311},"max":{"1min":4.548,"5min":6.0309999999999997,"15min":9.7850000000000001},"last":5.4589999999999996},{"interface":"front","average":{"1min":0.90900000000000003,"5min":1.0680000000000001,"15min":1.0209999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":4.5780000000000003,"5min":5.4219999999999997,"15min":5.4219999999999997},"last":2.625}]},{"osd":5,"last update":"Mon Oct 9 05:28:38 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":0.98599999999999999,"15min":0.92700000000000005},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.101,"5min":4.6870000000000003,"15min":4.8689999999999998},"last":4.0819999999999999},{"interface":"front","average":{"1min":0.90500000000000003,"5min":1.0720000000000001,"15min":0.92700000000000005},"min":{"1min":0.40899999999999997,"5min":0.373,"15min":0.373},"max":{"1min":4.0529999999999999,"5min":5.9779999999999998,"15min":5.9779999999999998},"last":5.548}]},{"osd":6,"last update":"Mon Oct 9 05:28:38 2023","interfaces":[{"interface":"back","average":{"1min":1.2869999999999999,"5min":1.333,"15min":1.1679999999999999},"min":{"1min":0.42699999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":6.282,"5min":7.5519999999999996,"15min":7.5519999999999996},"last":8.5609999999999999},{"interface":"front","average":{"1min":1.2310000000000001,"5min":1.1160000000000001,"15min":1.1479999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.38200000000000001},"max":{"1min":5.0030000000000001,"5min":5.0030000000000001,"15min":6.7910000000000004},"last":2.6640000000000001}]}]},{"osd":6,"up_from":46,"seq":197568496205,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1146616,"kb_used_data":250992,"kb_used_omap":0,"kb_used_meta":895616,"kb_avail":92598536,"statfs":{"total":95995035648,"available":94820900864,"internally_reserved":0,"allocated":257015808,"data_stored":256132480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917110784},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":3.089,"15min":3.2389999999999999},"min":{"1min":0.28999999999999998,"5min":0.26100000000000001,"15min":0.23599999999999999},"max":{"1min":4.6520000000000001,"5min":154.29300000000001,"15min":154.29300000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":1.581,"5min":3.2160000000000002,"15min":3.3519999999999999},"min":{"1min":0.38400000000000001,"5min":0.33100000000000002,"15min":0.309},"max":{"1min":4.6820000000000004,"5min":154.35300000000001,"15min":154.35300000000001},"last":0.55900000000000005}]},{"osd":1,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":2.4079999999999999,"5min":1.6619999999999999,"15min":2.2240000000000002},"min":{"1min":0.38300000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.5789999999999997,"5min":23.318000000000001,"15min":120.873},"last":0.39800000000000002},{"interface":"front","average":{"1min":1.724,"5min":1.5509999999999999,"15min":2.081},"min":{"1min":0.34100000000000003,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":5.7350000000000003,"5min":23.140000000000001,"15min":119.248},"last":0.73299999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":2.415,"5min":5.2750000000000004,"15min":2.6789999999999998},"min":{"1min":0.38300000000000001,"5min":0.35599999999999998,"15min":0.29399999999999998},"max":{"1min":7.8799999999999999,"5min":112.649,"15min":112.649},"last":0.621},{"interface":"front","average":{"1min":2.1030000000000002,"5min":5.1959999999999997,"15min":2.8999999999999999},"min":{"1min":0.38700000000000001,"5min":0.31,"15min":0.31},"max":{"1min":7.0339999999999998,"5min":112.834,"15min":112.834},"last":0.77500000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":2.298,"5min":5.2409999999999997,"15min":4.7910000000000004},"min":{"1min":0.36499999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":9.5640000000000001,"5min":169.01599999999999,"15min":169.01599999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":1.8260000000000001,"5min":5.0579999999999998,"15min":4.6890000000000001},"min":{"1min":0.42399999999999999,"5min":0.33800000000000002,"15min":0.32800000000000001},"max":{"1min":8.4600000000000009,"5min":169.40799999999999,"15min":169.40799999999999},"last":0.67300000000000004}]},{"osd":4,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.008,"15min":1.111},"min":{"1min":0.40500000000000003,"5min":0.315,"15min":0.27000000000000002},"max":{"1min":4.5099999999999998,"5min":4.5119999999999996,"15min":8.3670000000000009},"last":0.503},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.96699999999999997,"15min":1.042},"min":{"1min":0.39100000000000001,"5min":0.38100000000000001,"15min":0.34100000000000003},"max":{"1min":1.2390000000000001,"5min":4.069,"15min":13.473000000000001},"last":0.59099999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":0.99099999999999999,"15min":0.97199999999999998},"min":{"1min":0.38900000000000001,"5min":0.35999999999999999,"15min":0.33300000000000002},"max":{"1min":4.4699999999999998,"5min":4.5410000000000004,"15min":7.5899999999999999},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.91800000000000004,"5min":1.0589999999999999,"15min":1.0700000000000001},"min":{"1min":0.47599999999999998,"5min":0.40400000000000003,"15min":0.378},"max":{"1min":4.3230000000000004,"5min":4.3230000000000004,"15min":13.292999999999999},"last":0.85999999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:28:18 2023","interfaces":[{"interface":"back","average":{"1min":1.6779999999999999,"5min":1.3580000000000001,"15min":1.3069999999999999},"min":{"1min":0.40699999999999997,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":8.5350000000000001,"5min":8.8780000000000001,"15min":8.8780000000000001},"last":1.0069999999999999},{"interface":"front","average":{"1min":1.734,"5min":1.452,"15min":1.4350000000000001},"min":{"1min":0.48699999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":8.5749999999999993,"5min":8.923,"15min":8.923},"last":1.05}]}]},{"osd":1,"up_from":96,"seq":412316860796,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1127320,"kb_used_data":172880,"kb_used_omap":4,"kb_used_meta":954427,"kb_avail":92617832,"statfs":{"total":95995035648,"available":94840659968,"internally_reserved":0,"allocated":177029120,"data_stored":175473096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":977334158},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:28:50 2023","interfaces":[{"interface":"back","average":{"1min":3.7469999999999999,"5min":2.4849999999999999,"15min":2.4580000000000002},"min":{"1min":0.33800000000000002,"5min":0.217,"15min":0.19},"max":{"1min":19.172999999999998,"5min":76.007999999999996,"15min":83.194999999999993},"last":0.376},{"interface":"front","average":{"1min":4.2670000000000003,"5min":2.6960000000000002,"15min":2.7839999999999998},"min":{"1min":0.27200000000000002,"5min":0.192,"15min":0.186},"max":{"1min":28.113,"5min":75.168999999999997,"15min":83.972999999999999},"last":9.3179999999999996}]},{"osd":2,"last update":"Mon Oct 9 05:28:56 2023","interfaces":[{"interface":"back","average":{"1min":2.657,"5min":1.79,"15min":2.1789999999999998},"min":{"1min":0.27000000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":14.295,"5min":14.295,"15min":83.337000000000003},"last":4.0529999999999999},{"interface":"front","average":{"1min":3.044,"5min":1.752,"15min":2.2570000000000001},"min":{"1min":0.26000000000000001,"5min":0.254,"15min":0.23599999999999999},"max":{"1min":14.326000000000001,"5min":14.326000000000001,"15min":83.143000000000001},"last":4.8339999999999996}]},{"osd":3,"last update":"Mon Oct 9 05:28:56 2023","interfaces":[{"interface":"back","average":{"1min":3.4460000000000002,"5min":2.218,"15min":2.444},"min":{"1min":0.30199999999999999,"5min":0.221,"15min":0.215},"max":{"1min":9.798,"5min":51.780999999999999,"15min":84.105999999999995},"last":4.5810000000000004},{"interface":"front","average":{"1min":2.9529999999999998,"5min":2.669,"15min":2.6869999999999998},"min":{"1min":0.28899999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":8.5109999999999992,"5min":51.813000000000002,"15min":83.566000000000003},"last":4.4450000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:28:50 2023","interfaces":[{"interface":"back","average":{"1min":1.9470000000000001,"5min":1.5669999999999999,"15min":2.0249999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":9.8200000000000003,"5min":9.8200000000000003,"15min":83.453999999999994},"last":4.3200000000000003},{"interface":"front","average":{"1min":1.8779999999999999,"5min":1.4159999999999999,"15min":1.742},"min":{"1min":0.35599999999999998,"5min":0.34599999999999997,"15min":0.28100000000000003},"max":{"1min":9.7850000000000001,"5min":9.7850000000000001,"15min":83.375},"last":0.55300000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:28:50 2023","interfaces":[{"interface":"back","average":{"1min":2.5129999999999999,"5min":1.4019999999999999,"15min":1.5009999999999999},"min":{"1min":0.39700000000000002,"5min":0.373,"15min":0.373},"max":{"1min":9.7520000000000007,"5min":9.7520000000000007,"15min":9.7520000000000007},"last":4.548},{"interface":"front","average":{"1min":2.5329999999999999,"5min":1.6299999999999999,"15min":1.9339999999999999},"min":{"1min":0.42399999999999999,"5min":0.39900000000000002,"15min":0.38200000000000001},"max":{"1min":14.263999999999999,"5min":14.263999999999999,"15min":83.268000000000001},"last":4.3639999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:28:50 2023","interfaces":[{"interface":"back","average":{"1min":2.9790000000000001,"5min":1.657,"15min":1.6739999999999999},"min":{"1min":0.435,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":9.8520000000000003,"5min":9.8520000000000003,"15min":9.8520000000000003},"last":4.5099999999999998},{"interface":"front","average":{"1min":2.077,"5min":1.6879999999999999,"15min":2.0299999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.373},"max":{"1min":14.231,"5min":14.231,"15min":83.400000000000006},"last":4.4189999999999996}]},{"osd":7,"last update":"Mon Oct 9 05:28:50 2023","interfaces":[{"interface":"back","average":{"1min":2.125,"5min":1.72,"15min":2.0019999999999998},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":14.166,"5min":14.166,"15min":83.510000000000005},"last":4.3899999999999997},{"interface":"front","average":{"1min":1.921,"5min":1.4490000000000001,"15min":1.7110000000000001},"min":{"1min":0.40300000000000002,"5min":0.377,"15min":0.36899999999999999},"max":{"1min":9.8819999999999997,"5min":9.8819999999999997,"15min":83.480999999999995},"last":0.59199999999999997}]}]},{"osd":0,"up_from":88,"seq":377957122437,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1130840,"kb_used_data":224912,"kb_used_omap":5,"kb_used_meta":905914,"kb_avail":92614312,"statfs":{"total":95995035648,"available":94837055488,"internally_reserved":0,"allocated":230309888,"data_stored":228812329,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":927656570},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:28:45 2023","interfaces":[{"interface":"back","average":{"1min":1.9379999999999999,"5min":1.425,"15min":2.0899999999999999},"min":{"1min":0.28499999999999998,"5min":0.21199999999999999,"15min":0.17799999999999999},"max":{"1min":7.0099999999999998,"5min":7.0099999999999998,"15min":80.843999999999994},"last":6.5519999999999996},{"interface":"front","average":{"1min":1.179,"5min":1.0980000000000001,"15min":2.302},"min":{"1min":0.28100000000000003,"5min":0.221,"15min":0.21299999999999999},"max":{"1min":6.0339999999999998,"5min":8.9849999999999994,"15min":81.159999999999997},"last":0.61599999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:28:50 2023","interfaces":[{"interface":"back","average":{"1min":2.3159999999999998,"5min":3.2650000000000001,"15min":2.5619999999999998},"min":{"1min":0.51900000000000002,"5min":0.29599999999999999,"15min":0.25600000000000001},"max":{"1min":9.3879999999999999,"5min":92.745000000000005,"15min":92.745000000000005},"last":1.1040000000000001},{"interface":"front","average":{"1min":2.3340000000000001,"5min":3.4649999999999999,"15min":2.6269999999999998},"min":{"1min":0.20699999999999999,"5min":0.20399999999999999,"15min":0.19700000000000001},"max":{"1min":7.5629999999999997,"5min":97.816000000000003,"15min":97.816000000000003},"last":6.5970000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:28:50 2023","interfaces":[{"interface":"back","average":{"1min":2.5939999999999999,"5min":2.0350000000000001,"15min":2.4590000000000001},"min":{"1min":0.27600000000000002,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":9.532,"5min":37.883000000000003,"15min":82.286000000000001},"last":9.5039999999999996},{"interface":"front","average":{"1min":2.4830000000000001,"5min":2.0659999999999998,"15min":2.895},"min":{"1min":0.42799999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":9.6289999999999996,"5min":45.279000000000003,"15min":75.969999999999999},"last":1.4399999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:28:50 2023","interfaces":[{"interface":"back","average":{"1min":1.7549999999999999,"5min":1.173,"15min":1.637},"min":{"1min":0.45800000000000002,"5min":0.36799999999999999,"15min":0.33800000000000002},"max":{"1min":8.4629999999999992,"5min":8.4629999999999992,"15min":12.255000000000001},"last":1.353},{"interface":"front","average":{"1min":1.8879999999999999,"5min":1.502,"15min":1.617},"min":{"1min":0.39900000000000002,"5min":0.34100000000000003,"15min":0.28199999999999997},"max":{"1min":6.8700000000000001,"5min":6.8700000000000001,"15min":8.3580000000000005},"last":9.8320000000000007}]},{"osd":5,"last update":"Mon Oct 9 05:28:50 2023","interfaces":[{"interface":"back","average":{"1min":1.321,"5min":1.1859999999999999,"15min":1.3959999999999999},"min":{"1min":0.38300000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":5.758,"5min":5.758,"15min":11.916},"last":9.5869999999999997},{"interface":"front","average":{"1min":1.661,"5min":1.2110000000000001,"15min":1.6899999999999999},"min":{"1min":0.53500000000000003,"5min":0.434,"15min":0.375},"max":{"1min":8.5299999999999994,"5min":8.5299999999999994,"15min":12.143000000000001},"last":1.4099999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:28:50 2023","interfaces":[{"interface":"back","average":{"1min":2.1360000000000001,"5min":1.673,"15min":1.8},"min":{"1min":0.40300000000000002,"5min":0.374,"15min":0.374},"max":{"1min":6.9470000000000001,"5min":6.9470000000000001,"15min":12.125},"last":9.7949999999999999},{"interface":"front","average":{"1min":1.3180000000000001,"5min":1.363,"15min":1.6200000000000001},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35899999999999999},"max":{"1min":5.7960000000000003,"5min":8.4659999999999993,"15min":12.007999999999999},"last":9.6180000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:28:50 2023","interfaces":[{"interface":"back","average":{"1min":1.863,"5min":1.5249999999999999,"15min":1.6100000000000001},"min":{"1min":0.45300000000000001,"5min":0.308,"15min":0.308},"max":{"1min":6.0609999999999999,"5min":8.6590000000000007,"15min":11.977},"last":9.5530000000000008},{"interface":"front","average":{"1min":1.8129999999999999,"5min":1.506,"15min":1.871},"min":{"1min":0.46600000000000003,"5min":0.41599999999999998,"15min":0.40100000000000002},"max":{"1min":8.5020000000000007,"5min":10.795,"15min":12.1},"last":1.385}]}]},{"osd":2,"up_from":106,"seq":455266533715,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1117480,"kb_used_data":362144,"kb_used_omap":7,"kb_used_meta":755320,"kb_avail":92627672,"statfs":{"total":95995035648,"available":94850736128,"internally_reserved":0,"allocated":370835456,"data_stored":369379525,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":773447761},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,3,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:28:49 2023","interfaces":[{"interface":"back","average":{"1min":4.0019999999999998,"5min":2.306,"15min":2.25},"min":{"1min":0.24099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":27.010999999999999,"5min":27.010999999999999,"15min":40.204999999999998},"last":2.6960000000000002},{"interface":"front","average":{"1min":3.7250000000000001,"5min":1.925,"15min":2.1480000000000001},"min":{"1min":0.23699999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":18.922000000000001,"5min":18.922000000000001,"15min":40.170000000000002},"last":4.2699999999999996}]},{"osd":1,"last update":"Mon Oct 9 05:28:49 2023","interfaces":[{"interface":"back","average":{"1min":2.0339999999999998,"5min":1.6220000000000001,"15min":1.893},"min":{"1min":0.248,"5min":0.22700000000000001,"15min":0.224},"max":{"1min":7.133,"5min":7.2089999999999996,"15min":23.251000000000001},"last":0.98299999999999998},{"interface":"front","average":{"1min":3.1779999999999999,"5min":2.1059999999999999,"15min":2.0430000000000001},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.20399999999999999},"max":{"1min":8.1920000000000002,"5min":14.034000000000001,"15min":17.632999999999999},"last":5.5970000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":5.2839999999999998,"5min":2.548,"15min":2.5129999999999999},"min":{"1min":0.307,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":15.334,"5min":15.334,"15min":15.334},"last":1.0029999999999999},{"interface":"front","average":{"1min":3.6960000000000002,"5min":2.7010000000000001,"15min":2.3540000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.23899999999999999},"max":{"1min":18.515000000000001,"5min":18.515000000000001,"15min":18.515000000000001},"last":0.44}]},{"osd":4,"last update":"Mon Oct 9 05:28:49 2023","interfaces":[{"interface":"back","average":{"1min":1.952,"5min":1.587,"15min":1.6859999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":10.597,"5min":10.597,"15min":11.157999999999999},"last":0.90800000000000003},{"interface":"front","average":{"1min":2.7210000000000001,"5min":1.7609999999999999,"15min":1.6919999999999999},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.29699999999999999},"max":{"1min":8.2289999999999992,"5min":10.242000000000001,"15min":12.460000000000001},"last":0.624}]},{"osd":5,"last update":"Mon Oct 9 05:28:49 2023","interfaces":[{"interface":"back","average":{"1min":1.3899999999999999,"5min":1.6579999999999999,"15min":1.5600000000000001},"min":{"1min":0.35599999999999998,"5min":0.34100000000000003,"15min":0.33000000000000002},"max":{"1min":4.3490000000000002,"5min":6.6890000000000001,"15min":9.9190000000000005},"last":1.034},{"interface":"front","average":{"1min":2.0579999999999998,"5min":1.5489999999999999,"15min":1.716},"min":{"1min":0.40500000000000003,"5min":0.39700000000000002,"15min":0.34699999999999998},"max":{"1min":10.686,"5min":10.686,"15min":11.205},"last":0.96299999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:28:49 2023","interfaces":[{"interface":"back","average":{"1min":2.569,"5min":1.8580000000000001,"15min":1.871},"min":{"1min":0.42999999999999999,"5min":0.35299999999999998,"15min":0.32500000000000001},"max":{"1min":8.2590000000000003,"5min":10.279999999999999,"15min":16.337},"last":0.58999999999999997},{"interface":"front","average":{"1min":1.4550000000000001,"5min":1.8879999999999999,"15min":1.8160000000000001},"min":{"1min":0.38,"5min":0.38,"15min":0.35199999999999998},"max":{"1min":4.3170000000000002,"5min":11.118,"15min":16.399999999999999},"last":1.0580000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:28:49 2023","interfaces":[{"interface":"back","average":{"1min":2.2949999999999999,"5min":2.1709999999999998,"15min":1.7969999999999999},"min":{"1min":0.54200000000000004,"5min":0.40899999999999997,"15min":0.38300000000000001},"max":{"1min":8.9469999999999992,"5min":13.571999999999999,"15min":13.571999999999999},"last":1.079},{"interface":"front","average":{"1min":3.0640000000000001,"5min":2.169,"15min":1.9399999999999999},"min":{"1min":0.55400000000000005,"5min":0.46800000000000003,"15min":0.379},"max":{"1min":10.659000000000001,"5min":10.659000000000001,"15min":11.236000000000001},"last":0.93999999999999995}]}]},{"osd":3,"up_from":123,"seq":528280977734,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1127252,"kb_used_data":271308,"kb_used_omap":2,"kb_used_meta":855869,"kb_avail":92617900,"statfs":{"total":95995035648,"available":94840729600,"internally_reserved":0,"allocated":277819392,"data_stored":276439214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":876409909},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":2.323,"5min":2.2650000000000001,"15min":3.27},"min":{"1min":0.251,"5min":0.20499999999999999,"15min":0.19500000000000001},"max":{"1min":13.698,"5min":46.765999999999998,"15min":199.84999999999999},"last":7.6769999999999996},{"interface":"front","average":{"1min":3.3210000000000002,"5min":2.4390000000000001,"15min":3.0139999999999998},"min":{"1min":0.435,"5min":0.224,"15min":0.20200000000000001},"max":{"1min":11.249000000000001,"5min":47.069000000000003,"15min":83.234999999999999},"last":8.4499999999999993}]},{"osd":1,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":2.8220000000000001,"5min":2.1960000000000002,"15min":2.4220000000000002},"min":{"1min":0.253,"5min":0.22,"15min":0.22},"max":{"1min":8.9429999999999996,"5min":26.372,"15min":44.206000000000003},"last":8.3209999999999997},{"interface":"front","average":{"1min":3.3090000000000002,"5min":2.2370000000000001,"15min":2.2229999999999999},"min":{"1min":0.28000000000000003,"5min":0.24399999999999999,"15min":0.218},"max":{"1min":16.809000000000001,"5min":28.007999999999999,"15min":44.496000000000002},"last":7.899}]},{"osd":2,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":3.1589999999999998,"5min":2.5640000000000001,"15min":2.1669999999999998},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.22700000000000001},"max":{"1min":19.597999999999999,"5min":29.945,"15min":29.945},"last":8.1500000000000004},{"interface":"front","average":{"1min":2.6989999999999998,"5min":2.2410000000000001,"15min":2.4369999999999998},"min":{"1min":0.28000000000000003,"5min":0.251,"15min":0.22800000000000001},"max":{"1min":7.2670000000000003,"5min":26.452999999999999,"15min":26.452999999999999},"last":7.9180000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":2.3969999999999998,"5min":1.645,"15min":1.875},"min":{"1min":0.443,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.0119999999999996,"5min":8.3629999999999995,"15min":12.438000000000001},"last":8.0199999999999996},{"interface":"front","average":{"1min":2.266,"5min":1.6679999999999999,"15min":2.222},"min":{"1min":0.39600000000000002,"5min":0.36299999999999999,"15min":0.316},"max":{"1min":13.037000000000001,"5min":13.037000000000001,"15min":199.50800000000001},"last":8.173}]},{"osd":5,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":2.3580000000000001,"5min":1.7030000000000001,"15min":2.2589999999999999},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.33500000000000002},"max":{"1min":13.079000000000001,"5min":13.079000000000001,"15min":199.93799999999999},"last":8.2200000000000006},{"interface":"front","average":{"1min":2.524,"5min":1.784,"15min":1.607},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.34899999999999998},"max":{"1min":9.8109999999999999,"5min":9.8109999999999999,"15min":11.135},"last":9.4410000000000007}]},{"osd":6,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":2.3690000000000002,"5min":1.782,"15min":1.6879999999999999},"min":{"1min":0.36099999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":9.7789999999999999,"5min":9.7789999999999999,"15min":11.173999999999999},"last":9.4120000000000008},{"interface":"front","average":{"1min":2.6970000000000001,"5min":1.919,"15min":2.1120000000000001},"min":{"1min":0.55200000000000005,"5min":0.36699999999999999,"15min":0.32800000000000001},"max":{"1min":7.056,"5min":8.5500000000000007,"15min":12.407},"last":8.3889999999999993}]},{"osd":7,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7080000000000002,"5min":1.952,"15min":1.9570000000000001},"min":{"1min":0.51400000000000001,"5min":0.39100000000000001,"15min":0.378},"max":{"1min":8.1110000000000007,"5min":8.6039999999999992,"15min":12.468},"last":8.4169999999999998},{"interface":"front","average":{"1min":2.605,"5min":1.9590000000000001,"15min":2.4340000000000002},"min":{"1min":0.42799999999999999,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":12.989000000000001,"5min":12.989000000000001,"15min":199.96799999999999},"last":8.2539999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019491,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1029948,"kb_used_data":141492,"kb_used_omap":0,"kb_used_meta":888448,"kb_avail":92715204,"statfs":{"total":95995035648,"available":94940368896,"internally_reserved":0,"allocated":144887808,"data_stored":143999627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":909770752},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:28:52 2023","interfaces":[{"interface":"back","average":{"1min":5.343,"5min":4.5350000000000001,"15min":2.637},"min":{"1min":0.248,"5min":0.248,"15min":0.24199999999999999},"max":{"1min":59.783000000000001,"5min":151.209,"15min":151.209},"last":0.35999999999999999},{"interface":"front","average":{"1min":5.2359999999999998,"5min":4.4029999999999996,"15min":2.6120000000000001},"min":{"1min":0.30499999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":59.823,"5min":151.303,"15min":151.303},"last":0.55000000000000004}]},{"osd":1,"last update":"Mon Oct 9 05:28:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1359999999999999,"5min":1.48,"15min":2.2240000000000002},"min":{"1min":0.33200000000000002,"5min":0.28999999999999998,"15min":0.25700000000000001},"max":{"1min":3.274,"5min":53.911000000000001,"15min":142.297},"last":0.51700000000000002},{"interface":"front","average":{"1min":1.107,"5min":1.6240000000000001,"15min":2.1480000000000001},"min":{"1min":0.30099999999999999,"5min":0.29699999999999999,"15min":0.27800000000000002},"max":{"1min":4.4240000000000004,"5min":54.892000000000003,"15min":135.11600000000001},"last":0.40000000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:28:52 2023","interfaces":[{"interface":"back","average":{"1min":5.7960000000000003,"5min":3.8540000000000001,"15min":2.4119999999999999},"min":{"1min":0.35899999999999999,"5min":0.311,"15min":0.311},"max":{"1min":78.260999999999996,"5min":152.643,"15min":152.643},"last":0.69799999999999995},{"interface":"front","average":{"1min":5.9409999999999998,"5min":3.8340000000000001,"15min":2.5339999999999998},"min":{"1min":0.33700000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":73.174000000000007,"5min":152.554,"15min":152.554},"last":0.73599999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:29:02 2023","interfaces":[{"interface":"back","average":{"1min":4.992,"5min":2.2269999999999999,"15min":2.403},"min":{"1min":0.38300000000000001,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":73.004000000000005,"5min":73.004000000000005,"15min":131.726},"last":0.42399999999999999},{"interface":"front","average":{"1min":5.2309999999999999,"5min":2.1440000000000001,"15min":2.5369999999999999},"min":{"1min":0.33900000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":75.013000000000005,"5min":75.013000000000005,"15min":131.57300000000001},"last":4.0270000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:29:02 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":1.0269999999999999,"15min":0.90700000000000003},"min":{"1min":0.38100000000000001,"5min":0.376,"15min":0.33100000000000002},"max":{"1min":2.899,"5min":6.1399999999999997,"15min":7.7549999999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.0229999999999999,"15min":0.93700000000000006},"min":{"1min":0.371,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":8.2799999999999994,"5min":10.255000000000001,"15min":10.255000000000001},"last":0.59499999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:28:41 2023","interfaces":[{"interface":"back","average":{"1min":1.341,"5min":1.486,"15min":1.296},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.33900000000000002},"max":{"1min":3.4460000000000002,"5min":20.818000000000001,"15min":20.818000000000001},"last":1.329},{"interface":"front","average":{"1min":1.2150000000000001,"5min":1.222,"15min":1.131},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":3.621,"5min":6.3079999999999998,"15min":8.4930000000000003},"last":0.47199999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:28:41 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":1.212,"15min":1.123},"min":{"1min":0.372,"5min":0.372,"15min":0.33900000000000002},"max":{"1min":5.1079999999999997,"5min":10.289,"15min":10.289},"last":0.84399999999999997},{"interface":"front","average":{"1min":1.0660000000000001,"5min":1.1899999999999999,"15min":1.1899999999999999},"min":{"1min":0.39400000000000002,"5min":0.379,"15min":0.34799999999999998},"max":{"1min":3.2090000000000001,"5min":4.7489999999999997,"15min":8.0269999999999992},"last":0.78900000000000003}]}]},{"osd":5,"up_from":0,"seq":163208757847,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":213647360,"data_stored":213647360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":167251968,"data_stored":167247896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":361435136,"data_stored":361431064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":262012928,"data_stored":262012928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":138805248,"data_stored":138805248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":243269632,"data_stored":243269632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":276697088,"data_stored":276693016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:29:15.812 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T05:29:15.812 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:29:15.812 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_7 2023-10-09T05:29:15.813 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:29:15.813 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:15.849 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:15.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:15.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:15.953 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:16.009 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:16.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:16.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:16.114 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:16.154 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:16.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:16.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:16.261 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:16.286 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:16.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:16.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:16.390 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:16.420 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:16.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:16.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:16.529 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:16.554 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:16.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:16.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:16.660 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:16.694 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:16.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:16.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:16.803 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:16.829 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:16.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:16.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:16.936 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:16.962 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:16.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:16.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:17.067 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:17.099 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:17.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:17.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:17.203 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:17.229 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:17.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:17.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:17.333 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:17.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:17.361 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:17.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:17.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:17 smithi194 bash[92666]: cluster 2023-10-09T05:29:16.236172+0000 mgr.y (mgr.25277) 1233 : cluster 0 pgmap v1314: 89 pgs: 89 active+clean; 316 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 681 op/s 2023-10-09T05:29:17.463 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:17.494 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:17.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:17.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:17.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:17 smithi133 bash[105589]: cluster 2023-10-09T05:29:16.236172+0000 mgr.y (mgr.25277) 1233 : cluster 0 pgmap v1314: 89 pgs: 89 active+clean; 316 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 681 op/s 2023-10-09T05:29:17.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:17 smithi133 bash[102858]: cluster 2023-10-09T05:29:16.236172+0000 mgr.y (mgr.25277) 1233 : cluster 0 pgmap v1314: 89 pgs: 89 active+clean; 316 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 681 op/s 2023-10-09T05:29:17.598 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:17.624 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:17.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:17.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:17.728 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:17.756 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:17.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:17.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:17.864 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:17.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:17.896 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:17.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:17.997 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:18.044 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:18.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:18.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:18.152 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:18.206 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:18.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:18.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:18.310 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:18.350 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:18.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:18.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:18.460 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:18.500 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:18.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:18.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:18.609 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:18.647 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:18.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:18.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:18.764 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:18.794 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:18.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:18.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:18.901 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:18.931 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:18.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:18.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:19.040 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:19.079 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:19.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:19.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:19.185 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:19.211 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:19.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:19.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:19.275 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:29:19.292 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:29:19.316 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:19.344 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:19.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:19.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:19.447 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:19.485 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:19.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:19.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:19.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:19 smithi133 bash[105589]: cluster 2023-10-09T05:29:18.236605+0000 mgr.y (mgr.25277) 1234 : cluster 0 pgmap v1315: 89 pgs: 89 active+clean; 316 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 681 op/s 2023-10-09T05:29:19.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:19 smithi133 bash[102858]: cluster 2023-10-09T05:29:18.236605+0000 mgr.y (mgr.25277) 1234 : cluster 0 pgmap v1315: 89 pgs: 89 active+clean; 316 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 681 op/s 2023-10-09T05:29:19.589 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:19.620 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:19.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:19.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:19.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:19 smithi194 bash[92666]: cluster 2023-10-09T05:29:18.236605+0000 mgr.y (mgr.25277) 1234 : cluster 0 pgmap v1315: 89 pgs: 89 active+clean; 316 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 681 op/s 2023-10-09T05:29:19.724 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:19.756 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:19.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:19.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:19.799 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:29:19.863 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:19.900 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:19.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:19.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:20.007 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:20.048 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:20.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:20.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:20.154 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:20.185 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:20.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:20.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:20.289 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:20.317 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:20.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:20.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:20.423 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:20.446 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:20.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:20.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:20.552 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:20.585 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:20.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:20.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:20.655 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:29:20.689 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:20.712 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:20.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:20.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:20.814 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:20.841 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:20.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:20.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:20.943 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:20.967 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:20.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:20.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:21.071 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:21.114 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:21.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:21.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:21.219 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:21.245 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:21.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:21.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:21.349 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:21.371 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:21.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:21.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:21.474 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:21.498 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:21.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:21.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:21.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:21 smithi133 bash[105589]: cluster 2023-10-09T05:29:20.237346+0000 mgr.y (mgr.25277) 1235 : cluster 0 pgmap v1316: 89 pgs: 89 active+clean; 134 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 631 op/s 2023-10-09T05:29:21.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:21 smithi133 bash[102858]: cluster 2023-10-09T05:29:20.237346+0000 mgr.y (mgr.25277) 1235 : cluster 0 pgmap v1316: 89 pgs: 89 active+clean; 134 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 631 op/s 2023-10-09T05:29:21.606 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:21.614 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:21 smithi194 bash[92666]: cluster 2023-10-09T05:29:20.237346+0000 mgr.y (mgr.25277) 1235 : cluster 0 pgmap v1316: 89 pgs: 89 active+clean; 134 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 631 op/s 2023-10-09T05:29:21.630 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:21.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:21.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:21.734 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:21.761 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:21.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:21.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:21.869 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:21.905 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:21.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:21.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:22.013 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:22.044 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:22.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:22.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:22.075 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Removed 102739 objects 2023-10-09T05:29:22.075 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Clean up completed and total clean up time :148.397 2023-10-09T05:29:22.152 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:22.177 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:22.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:22.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:22.279 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:22.302 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:22.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:22.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:22.405 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:22.433 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:22.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:22.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:22.537 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:22.568 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:22.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:22.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:22.672 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:22.697 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:22.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:22.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:22.801 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:22.822 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:22.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:22.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:22.925 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:22.958 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:22.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:22.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:23.063 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:23.091 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:23.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:23.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:23.195 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:23.223 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:23.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:23.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:23.327 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:23.356 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:23.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:23.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:23.460 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:23.492 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:23.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:23.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:23.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:23 smithi133 bash[102858]: cluster 2023-10-09T05:29:22.238016+0000 mgr.y (mgr.25277) 1236 : cluster 0 pgmap v1317: 89 pgs: 89 active+clean; 106 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 676 op/s 2023-10-09T05:29:23.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:23 smithi133 bash[102858]: audit 2023-10-09T05:29:22.609928+0000 mon.a (mon.0) 1571 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:29:23.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:23 smithi133 bash[105589]: cluster 2023-10-09T05:29:22.238016+0000 mgr.y (mgr.25277) 1236 : cluster 0 pgmap v1317: 89 pgs: 89 active+clean; 106 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 676 op/s 2023-10-09T05:29:23.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:23 smithi133 bash[105589]: audit 2023-10-09T05:29:22.609928+0000 mon.a (mon.0) 1571 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:29:23.595 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:23.629 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:23.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:23.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:23.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:23 smithi194 bash[92666]: cluster 2023-10-09T05:29:22.238016+0000 mgr.y (mgr.25277) 1236 : cluster 0 pgmap v1317: 89 pgs: 89 active+clean; 106 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 676 op/s 2023-10-09T05:29:23.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:23 smithi194 bash[92666]: audit 2023-10-09T05:29:22.609928+0000 mon.a (mon.0) 1571 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:29:23.733 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:23.761 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:23.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:23.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:23.866 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:23.894 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:23.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:23.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:23.991 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:29:23.991 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:29:23.998 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:24.028 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:24.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:24.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:24.132 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:24.159 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:24.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:24.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:24.262 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:24.287 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:24.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:24.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:24.298 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:29:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:29:24] "GET /metrics HTTP/1.1" 200 36470 "" "Prometheus/2.43.0" 2023-10-09T05:29:24.390 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:24.422 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:24.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:24.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:24.448 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:29:24.449 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:29:24.526 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:24.556 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:24.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:24.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:24.660 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:24.689 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:24.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:24.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:24.792 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:24.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:24 smithi133 bash[105589]: cluster 2023-10-09T05:29:24.238682+0000 mgr.y (mgr.25277) 1237 : cluster 0 pgmap v1318: 89 pgs: 89 active+clean; 90 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 659 op/s 2023-10-09T05:29:24.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:24 smithi133 bash[102858]: cluster 2023-10-09T05:29:24.238682+0000 mgr.y (mgr.25277) 1237 : cluster 0 pgmap v1318: 89 pgs: 89 active+clean; 90 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 659 op/s 2023-10-09T05:29:24.826 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:24.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:24.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:24.895 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T05:29:24.929 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:24.953 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:24 smithi194 bash[92666]: cluster 2023-10-09T05:29:24.238682+0000 mgr.y (mgr.25277) 1237 : cluster 0 pgmap v1318: 89 pgs: 89 active+clean; 90 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 659 op/s 2023-10-09T05:29:24.971 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:24.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:24.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:25.074 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:25.096 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:25.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:25.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:25.199 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:25.231 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:25.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:25.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:25.334 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:25.366 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:25.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:25.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:25.469 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:25.498 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:25.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:25.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:25.600 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:25.630 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:25.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:25.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:25.734 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:25.761 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:25.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:25.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:25.864 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:25.895 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:25.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:25.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:25.998 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:26.023 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:26.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:26.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:26.126 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:26.148 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:26.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:26.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:26.252 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:26.276 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:26.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:26.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:26.379 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:26.402 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:26.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:26.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:26.505 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:26.530 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:26.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:26.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:26.633 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:26.654 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:26.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:26.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:26.757 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:26.778 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:26.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:26.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:26.881 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:26.915 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:26.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:26.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:27.019 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:27.053 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:27.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:27.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:27.156 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:27.181 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:27.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:27.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:27.285 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:27.306 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:27.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:27.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:27.410 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:27.419 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:27 smithi133 bash[102858]: cluster 2023-10-09T05:29:26.239412+0000 mgr.y (mgr.25277) 1238 : cluster 0 pgmap v1319: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 508 op/s 2023-10-09T05:29:27.419 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:27 smithi133 bash[105589]: cluster 2023-10-09T05:29:26.239412+0000 mgr.y (mgr.25277) 1238 : cluster 0 pgmap v1319: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 508 op/s 2023-10-09T05:29:27.432 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:27.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:27.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:27.536 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:27.561 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:27.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:27.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:27.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:27 smithi194 bash[92666]: cluster 2023-10-09T05:29:26.239412+0000 mgr.y (mgr.25277) 1238 : cluster 0 pgmap v1319: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 508 op/s 2023-10-09T05:29:27.665 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:27.696 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:27.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:27.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:27.800 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:27.821 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:27.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:27.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:27.890 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:29:27.892 INFO:teuthology.orchestra.run.smithi133.stderr:dumped all 2023-10-09T05:29:27.923 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:27.946 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:27.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:27.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:28.049 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:28.074 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:28.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:28.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:28.177 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:28.200 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:28.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:28.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:28.303 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:28.358 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:28.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:28.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:28.462 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:28.489 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:28.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:28.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:28.592 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:28.622 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:28.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:28.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:28.726 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:28.752 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:28.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:28.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:28.855 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:28.881 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:28.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:28.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:28.985 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:29.014 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:29.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:29.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:29.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:28 smithi133 bash[105589]: audit 2023-10-09T05:29:27.875348+0000 mgr.y (mgr.25277) 1239 : audit 0 from='client.45509 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:29:29.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:28 smithi133 bash[105589]: audit 2023-10-09T05:29:27.901198+0000 mon.a (mon.0) 1572 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:29:29.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:28 smithi133 bash[105589]: audit 2023-10-09T05:29:27.909458+0000 mon.a (mon.0) 1573 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:29:29.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:28 smithi133 bash[105589]: cluster 2023-10-09T05:29:28.240002+0000 mgr.y (mgr.25277) 1240 : cluster 0 pgmap v1320: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 420 op/s 2023-10-09T05:29:29.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:28 smithi133 bash[105589]: audit 2023-10-09T05:29:28.482938+0000 mon.a (mon.0) 1574 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:29:29.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:28 smithi133 bash[105589]: audit 2023-10-09T05:29:28.484403+0000 mon.a (mon.0) 1575 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:29:29.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:28 smithi133 bash[102858]: audit 2023-10-09T05:29:27.875348+0000 mgr.y (mgr.25277) 1239 : audit 0 from='client.45509 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:29:29.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:28 smithi133 bash[102858]: audit 2023-10-09T05:29:27.901198+0000 mon.a (mon.0) 1572 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:29:29.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:28 smithi133 bash[102858]: audit 2023-10-09T05:29:27.909458+0000 mon.a (mon.0) 1573 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:29:29.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:28 smithi133 bash[102858]: cluster 2023-10-09T05:29:28.240002+0000 mgr.y (mgr.25277) 1240 : cluster 0 pgmap v1320: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 420 op/s 2023-10-09T05:29:29.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:28 smithi133 bash[102858]: audit 2023-10-09T05:29:28.482938+0000 mon.a (mon.0) 1574 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:29:29.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:28 smithi133 bash[102858]: audit 2023-10-09T05:29:28.484403+0000 mon.a (mon.0) 1575 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:29:29.117 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:29.146 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:29.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:29.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:29.249 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:29.262 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:28 smithi194 bash[92666]: audit 2023-10-09T05:29:27.875348+0000 mgr.y (mgr.25277) 1239 : audit 0 from='client.45509 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:29:29.262 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:28 smithi194 bash[92666]: audit 2023-10-09T05:29:27.901198+0000 mon.a (mon.0) 1572 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:29:29.262 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:28 smithi194 bash[92666]: audit 2023-10-09T05:29:27.909458+0000 mon.a (mon.0) 1573 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:29:29.262 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:28 smithi194 bash[92666]: cluster 2023-10-09T05:29:28.240002+0000 mgr.y (mgr.25277) 1240 : cluster 0 pgmap v1320: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 420 op/s 2023-10-09T05:29:29.262 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:28 smithi194 bash[92666]: audit 2023-10-09T05:29:28.482938+0000 mon.a (mon.0) 1574 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:29:29.263 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:28 smithi194 bash[92666]: audit 2023-10-09T05:29:28.484403+0000 mon.a (mon.0) 1575 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:29:29.279 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:29.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:29.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:29.383 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:29.407 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:29.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:29.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:29.511 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:29.522 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:29:29.542 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:29.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:29.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:29.645 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:29.676 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:29.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:29.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:29.780 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:29.812 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:29.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:29.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:29.916 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:29.944 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:29.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:29.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:30.047 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:30.076 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:30.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:30.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:30.180 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:30.211 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:30.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:30.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:30.314 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:30.315 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:30 smithi133 bash[102858]: audit 2023-10-09T05:29:29.065612+0000 mon.a (mon.0) 1576 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:29:30.315 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:30 smithi133 bash[102858]: audit 2023-10-09T05:29:29.072998+0000 mon.a (mon.0) 1577 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:29:30.315 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:30 smithi133 bash[102858]: audit 2023-10-09T05:29:29.540492+0000 mon.a (mon.0) 1578 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:29:30.316 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:30 smithi133 bash[102858]: audit 2023-10-09T05:29:29.542195+0000 mon.a (mon.0) 1579 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:29:30.316 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:30 smithi133 bash[102858]: audit 2023-10-09T05:29:29.550021+0000 mon.a (mon.0) 1580 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:29:30.316 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:30 smithi133 bash[105589]: audit 2023-10-09T05:29:29.065612+0000 mon.a (mon.0) 1576 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:29:30.316 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:30 smithi133 bash[105589]: audit 2023-10-09T05:29:29.072998+0000 mon.a (mon.0) 1577 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:29:30.316 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:30 smithi133 bash[105589]: audit 2023-10-09T05:29:29.540492+0000 mon.a (mon.0) 1578 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:29:30.316 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:30 smithi133 bash[105589]: audit 2023-10-09T05:29:29.542195+0000 mon.a (mon.0) 1579 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:29:30.316 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:30 smithi133 bash[105589]: audit 2023-10-09T05:29:29.550021+0000 mon.a (mon.0) 1580 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:29:30.317 INFO:teuthology.orchestra.run.smithi133.stdout:{"pg_ready":true,"pg_map":{"version":1319,"stamp":"2023-10-09T05:29:26.238879+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":507,"num_read_kb":932,"num_write":205780,"num_write_kb":6577742,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":4154034,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127902,"ondisk_log_size":127902,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":6192988,"kb_used_data":24420,"kb_used_omap":20,"kb_used_meta":6168491,"kb_avail":650023076,"statfs":{"total":671965249536,"available":665623629824,"internally_reserved":0,"allocated":25006080,"data_stored":16449875,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":6316535438},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-385286168,"num_objects":-5880,"num_object_clones":0,"num_object_copies":-17640,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5880,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5880,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"11.553393"},"pg_stats":[{"pgid":"3.1a","version":"69'2","reported_seq":282,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.344554+0000","last_change":"2023-10-09T05:09:57.873320+0000","last_active":"2023-10-09T05:28:51.344554+0000","last_peered":"2023-10-09T05:28:51.344554+0000","last_clean":"2023-10-09T05:28:51.344554+0000","last_became_active":"2023-10-09T05:09:57.868376+0000","last_became_peered":"2023-10-09T05:09:57.868376+0000","last_unstale":"2023-10-09T05:28:51.344554+0000","last_undegraded":"2023-10-09T05:28:51.344554+0000","last_fullsized":"2023-10-09T05:28:51.344554+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:27.186610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":265,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.338591+0000","last_change":"2023-10-09T05:14:15.345991+0000","last_active":"2023-10-09T05:28:51.338591+0000","last_peered":"2023-10-09T05:28:51.338591+0000","last_clean":"2023-10-09T05:28:51.338591+0000","last_became_active":"2023-10-09T05:14:15.345511+0000","last_became_peered":"2023-10-09T05:14:15.345511+0000","last_unstale":"2023-10-09T05:28:51.338591+0000","last_undegraded":"2023-10-09T05:28:51.338591+0000","last_fullsized":"2023-10-09T05:28:51.338591+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:49.234025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":148,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.331189+0000","last_change":"2023-10-09T05:09:57.857511+0000","last_active":"2023-10-09T05:28:51.331189+0000","last_peered":"2023-10-09T05:28:51.331189+0000","last_clean":"2023-10-09T05:28:51.331189+0000","last_became_active":"2023-10-09T05:09:57.857193+0000","last_became_peered":"2023-10-09T05:09:57.857193+0000","last_unstale":"2023-10-09T05:28:51.331189+0000","last_undegraded":"2023-10-09T05:28:51.331189+0000","last_fullsized":"2023-10-09T05:28:51.331189+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:32.842980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":283,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:29:04.329406+0000","last_change":"2023-10-09T05:18:26.422923+0000","last_active":"2023-10-09T05:29:04.329406+0000","last_peered":"2023-10-09T05:29:04.329406+0000","last_clean":"2023-10-09T05:29:04.329406+0000","last_became_active":"2023-10-09T05:18:26.422508+0000","last_became_peered":"2023-10-09T05:18:26.422508+0000","last_unstale":"2023-10-09T05:29:04.329406+0000","last_undegraded":"2023-10-09T05:29:04.329406+0000","last_fullsized":"2023-10-09T05:29:04.329406+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:00.876037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":302,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.339437+0000","last_change":"2023-10-09T05:23:18.188378+0000","last_active":"2023-10-09T05:28:51.339437+0000","last_peered":"2023-10-09T05:28:51.339437+0000","last_clean":"2023-10-09T05:28:51.339437+0000","last_became_active":"2023-10-09T05:23:18.186629+0000","last_became_peered":"2023-10-09T05:23:18.186629+0000","last_unstale":"2023-10-09T05:28:51.339437+0000","last_undegraded":"2023-10-09T05:28:51.339437+0000","last_fullsized":"2023-10-09T05:28:51.339437+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:16.444225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":272,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.366109+0000","last_change":"2023-10-09T05:13:03.039637+0000","last_active":"2023-10-09T05:28:51.366109+0000","last_peered":"2023-10-09T05:28:51.366109+0000","last_clean":"2023-10-09T05:28:51.366109+0000","last_became_active":"2023-10-09T05:13:03.039268+0000","last_became_peered":"2023-10-09T05:13:03.039268+0000","last_unstale":"2023-10-09T05:28:51.366109+0000","last_undegraded":"2023-10-09T05:28:51.366109+0000","last_fullsized":"2023-10-09T05:28:51.366109+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:19.875767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":270,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.346853+0000","last_change":"2023-10-09T05:22:47.216769+0000","last_active":"2023-10-09T05:28:51.346853+0000","last_peered":"2023-10-09T05:28:51.346853+0000","last_clean":"2023-10-09T05:28:51.346853+0000","last_became_active":"2023-10-09T05:22:47.215847+0000","last_became_peered":"2023-10-09T05:22:47.215847+0000","last_unstale":"2023-10-09T05:28:51.346853+0000","last_undegraded":"2023-10-09T05:28:51.346853+0000","last_fullsized":"2023-10-09T05:28:51.346853+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:51.392675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":257,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.339505+0000","last_change":"2023-10-09T05:09:57.849695+0000","last_active":"2023-10-09T05:28:51.339505+0000","last_peered":"2023-10-09T05:28:51.339505+0000","last_clean":"2023-10-09T05:28:51.339505+0000","last_became_active":"2023-10-09T05:09:57.848478+0000","last_became_peered":"2023-10-09T05:09:57.848478+0000","last_unstale":"2023-10-09T05:28:51.339505+0000","last_undegraded":"2023-10-09T05:28:51.339505+0000","last_fullsized":"2023-10-09T05:28:51.339505+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:26.115312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":91,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.346115+0000","last_change":"2023-10-09T05:23:18.194144+0000","last_active":"2023-10-09T05:28:51.346115+0000","last_peered":"2023-10-09T05:28:51.346115+0000","last_clean":"2023-10-09T05:28:51.346115+0000","last_became_active":"2023-10-09T05:23:18.193390+0000","last_became_peered":"2023-10-09T05:23:18.193390+0000","last_unstale":"2023-10-09T05:28:51.346115+0000","last_undegraded":"2023-10-09T05:28:51.346115+0000","last_fullsized":"2023-10-09T05:28:51.346115+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:22.549362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":270,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.333412+0000","last_change":"2023-10-09T05:18:26.418431+0000","last_active":"2023-10-09T05:28:51.333412+0000","last_peered":"2023-10-09T05:28:51.333412+0000","last_clean":"2023-10-09T05:28:51.333412+0000","last_became_active":"2023-10-09T05:18:26.418100+0000","last_became_peered":"2023-10-09T05:18:26.418100+0000","last_unstale":"2023-10-09T05:28:51.333412+0000","last_undegraded":"2023-10-09T05:28:51.333412+0000","last_fullsized":"2023-10-09T05:28:51.333412+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:21.241286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":77,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.346043+0000","last_change":"2023-10-09T05:18:26.452552+0000","last_active":"2023-10-09T05:28:51.346043+0000","last_peered":"2023-10-09T05:28:51.346043+0000","last_clean":"2023-10-09T05:28:51.346043+0000","last_became_active":"2023-10-09T05:18:26.428454+0000","last_became_peered":"2023-10-09T05:18:26.428454+0000","last_unstale":"2023-10-09T05:28:51.346043+0000","last_undegraded":"2023-10-09T05:28:51.346043+0000","last_fullsized":"2023-10-09T05:28:51.346043+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:15:52.203861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":42,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.331294+0000","last_change":"2023-10-09T05:23:18.199181+0000","last_active":"2023-10-09T05:28:51.331294+0000","last_peered":"2023-10-09T05:28:51.331294+0000","last_clean":"2023-10-09T05:28:51.331294+0000","last_became_active":"2023-10-09T05:23:18.198401+0000","last_became_peered":"2023-10-09T05:23:18.198401+0000","last_unstale":"2023-10-09T05:28:51.331294+0000","last_undegraded":"2023-10-09T05:28:51.331294+0000","last_fullsized":"2023-10-09T05:28:51.331294+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:55:49.891420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'23","reported_seq":334,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.343946+0000","last_change":"2023-10-09T05:18:26.436165+0000","last_active":"2023-10-09T05:28:51.343946+0000","last_peered":"2023-10-09T05:28:51.343946+0000","last_clean":"2023-10-09T05:28:51.343946+0000","last_became_active":"2023-10-09T05:18:26.430078+0000","last_became_peered":"2023-10-09T05:18:26.430078+0000","last_unstale":"2023-10-09T05:28:51.343946+0000","last_undegraded":"2023-10-09T05:28:51.343946+0000","last_fullsized":"2023-10-09T05:28:51.343946+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:04:48.797166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":262,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.343959+0000","last_change":"2023-10-09T05:09:57.868761+0000","last_active":"2023-10-09T05:28:51.343959+0000","last_peered":"2023-10-09T05:28:51.343959+0000","last_clean":"2023-10-09T05:28:51.343959+0000","last_became_active":"2023-10-09T05:09:57.868121+0000","last_became_peered":"2023-10-09T05:09:57.868121+0000","last_unstale":"2023-10-09T05:28:51.343959+0000","last_undegraded":"2023-10-09T05:28:51.343959+0000","last_fullsized":"2023-10-09T05:28:51.343959+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:43.412235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":77,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.344964+0000","last_change":"2023-10-09T05:18:26.443719+0000","last_active":"2023-10-09T05:28:51.344964+0000","last_peered":"2023-10-09T05:28:51.344964+0000","last_clean":"2023-10-09T05:28:51.344964+0000","last_became_active":"2023-10-09T05:18:26.426718+0000","last_became_peered":"2023-10-09T05:18:26.426718+0000","last_unstale":"2023-10-09T05:28:51.344964+0000","last_undegraded":"2023-10-09T05:28:51.344964+0000","last_fullsized":"2023-10-09T05:28:51.344964+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:49:12.090862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":311,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.339076+0000","last_change":"2023-10-09T05:23:18.187593+0000","last_active":"2023-10-09T05:28:51.339076+0000","last_peered":"2023-10-09T05:28:51.339076+0000","last_clean":"2023-10-09T05:28:51.339076+0000","last_became_active":"2023-10-09T05:23:18.186445+0000","last_became_peered":"2023-10-09T05:23:18.186445+0000","last_unstale":"2023-10-09T05:28:51.339076+0000","last_undegraded":"2023-10-09T05:28:51.339076+0000","last_fullsized":"2023-10-09T05:28:51.339076+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:11.327635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":279,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.347095+0000","last_change":"2023-10-09T05:18:26.424399+0000","last_active":"2023-10-09T05:28:51.347095+0000","last_peered":"2023-10-09T05:28:51.347095+0000","last_clean":"2023-10-09T05:28:51.347095+0000","last_became_active":"2023-10-09T05:18:26.422122+0000","last_became_peered":"2023-10-09T05:18:26.422122+0000","last_unstale":"2023-10-09T05:28:51.347095+0000","last_undegraded":"2023-10-09T05:28:51.347095+0000","last_fullsized":"2023-10-09T05:28:51.347095+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:39.393756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":71,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.345947+0000","last_change":"2023-10-09T05:18:26.422950+0000","last_active":"2023-10-09T05:28:51.345947+0000","last_peered":"2023-10-09T05:28:51.345947+0000","last_clean":"2023-10-09T05:28:51.345947+0000","last_became_active":"2023-10-09T05:18:26.422592+0000","last_became_peered":"2023-10-09T05:18:26.422592+0000","last_unstale":"2023-10-09T05:28:51.345947+0000","last_undegraded":"2023-10-09T05:28:51.345947+0000","last_fullsized":"2023-10-09T05:28:51.345947+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:20.701107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":71,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.344786+0000","last_change":"2023-10-09T05:18:26.429482+0000","last_active":"2023-10-09T05:28:51.344786+0000","last_peered":"2023-10-09T05:28:51.344786+0000","last_clean":"2023-10-09T05:28:51.344786+0000","last_became_active":"2023-10-09T05:18:26.429122+0000","last_became_peered":"2023-10-09T05:18:26.429122+0000","last_unstale":"2023-10-09T05:28:51.344786+0000","last_undegraded":"2023-10-09T05:28:51.344786+0000","last_fullsized":"2023-10-09T05:28:51.344786+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:47.301906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":280,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.351132+0000","last_change":"2023-10-09T05:18:26.421678+0000","last_active":"2023-10-09T05:28:51.351132+0000","last_peered":"2023-10-09T05:28:51.351132+0000","last_clean":"2023-10-09T05:28:51.351132+0000","last_became_active":"2023-10-09T05:18:26.419673+0000","last_became_peered":"2023-10-09T05:18:26.419673+0000","last_unstale":"2023-10-09T05:28:51.351132+0000","last_undegraded":"2023-10-09T05:28:51.351132+0000","last_fullsized":"2023-10-09T05:28:51.351132+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:38.876021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":275,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.361140+0000","last_change":"2023-10-09T05:13:03.047339+0000","last_active":"2023-10-09T05:28:51.361140+0000","last_peered":"2023-10-09T05:28:51.361140+0000","last_clean":"2023-10-09T05:28:51.361140+0000","last_became_active":"2023-10-09T05:13:03.040038+0000","last_became_peered":"2023-10-09T05:13:03.040038+0000","last_unstale":"2023-10-09T05:28:51.361140+0000","last_undegraded":"2023-10-09T05:28:51.361140+0000","last_fullsized":"2023-10-09T05:28:51.361140+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:56.726249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":293,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.338899+0000","last_change":"2023-10-09T05:18:26.415173+0000","last_active":"2023-10-09T05:28:51.338899+0000","last_peered":"2023-10-09T05:28:51.338899+0000","last_clean":"2023-10-09T05:28:51.338899+0000","last_became_active":"2023-10-09T05:18:26.414452+0000","last_became_peered":"2023-10-09T05:18:26.414452+0000","last_unstale":"2023-10-09T05:28:51.338899+0000","last_undegraded":"2023-10-09T05:28:51.338899+0000","last_fullsized":"2023-10-09T05:28:51.338899+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:34.379767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"282'12782","reported_seq":12808,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T05:29:22.071398+0000","last_change":"2023-10-09T05:24:56.644144+0000","last_active":"2023-10-09T05:29:22.071398+0000","last_peered":"2023-10-09T05:29:22.071398+0000","last_clean":"2023-10-09T05:29:22.071398+0000","last_became_active":"2023-10-09T05:24:56.643459+0000","last_became_peered":"2023-10-09T05:24:56.643459+0000","last_unstale":"2023-10-09T05:29:22.071398+0000","last_undegraded":"2023-10-09T05:29:22.071398+0000","last_fullsized":"2023-10-09T05:29:22.071398+0000","mapping_epoch":264,"log_start":"273'4800","ondisk_log_start":"273'4800","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":7982,"log_dups_size":0,"ondisk_log_size":7982,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:39.899186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12782,"num_write_kb":409024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":75,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.346081+0000","last_change":"2023-10-09T05:18:26.446608+0000","last_active":"2023-10-09T05:28:51.346081+0000","last_peered":"2023-10-09T05:28:51.346081+0000","last_clean":"2023-10-09T05:28:51.346081+0000","last_became_active":"2023-10-09T05:18:26.428420+0000","last_became_peered":"2023-10-09T05:18:26.428420+0000","last_unstale":"2023-10-09T05:28:51.346081+0000","last_undegraded":"2023-10-09T05:28:51.346081+0000","last_fullsized":"2023-10-09T05:28:51.346081+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:23.693908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":71,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.344845+0000","last_change":"2023-10-09T05:18:26.427461+0000","last_active":"2023-10-09T05:28:51.344845+0000","last_peered":"2023-10-09T05:28:51.344845+0000","last_clean":"2023-10-09T05:28:51.344845+0000","last_became_active":"2023-10-09T05:18:26.427118+0000","last_became_peered":"2023-10-09T05:18:26.427118+0000","last_unstale":"2023-10-09T05:28:51.344845+0000","last_undegraded":"2023-10-09T05:28:51.344845+0000","last_fullsized":"2023-10-09T05:28:51.344845+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:05:01.868197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.1","version":"282'12792","reported_seq":12818,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T05:29:22.024702+0000","last_change":"2023-10-09T05:24:56.643202+0000","last_active":"2023-10-09T05:29:22.024702+0000","last_peered":"2023-10-09T05:29:22.024702+0000","last_clean":"2023-10-09T05:29:22.024702+0000","last_became_active":"2023-10-09T05:24:56.642700+0000","last_became_peered":"2023-10-09T05:24:56.642700+0000","last_unstale":"2023-10-09T05:29:22.024702+0000","last_undegraded":"2023-10-09T05:29:22.024702+0000","last_fullsized":"2023-10-09T05:29:22.024702+0000","mapping_epoch":264,"log_start":"273'5000","ondisk_log_start":"273'5000","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":7792,"log_dups_size":0,"ondisk_log_size":7792,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:25.444496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12792,"num_write_kb":409344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":75,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.346089+0000","last_change":"2023-10-09T05:18:26.440042+0000","last_active":"2023-10-09T05:28:51.346089+0000","last_peered":"2023-10-09T05:28:51.346089+0000","last_clean":"2023-10-09T05:28:51.346089+0000","last_became_active":"2023-10-09T05:18:26.426726+0000","last_became_peered":"2023-10-09T05:18:26.426726+0000","last_unstale":"2023-10-09T05:28:51.346089+0000","last_undegraded":"2023-10-09T05:28:51.346089+0000","last_fullsized":"2023-10-09T05:28:51.346089+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:23.985650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":279,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.343733+0000","last_change":"2023-10-09T05:09:57.868031+0000","last_active":"2023-10-09T05:28:51.343733+0000","last_peered":"2023-10-09T05:28:51.343733+0000","last_clean":"2023-10-09T05:28:51.343733+0000","last_became_active":"2023-10-09T05:09:57.866477+0000","last_became_peered":"2023-10-09T05:09:57.866477+0000","last_unstale":"2023-10-09T05:28:51.343733+0000","last_undegraded":"2023-10-09T05:28:51.343733+0000","last_fullsized":"2023-10-09T05:28:51.343733+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:27.102583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"282'12804","reported_seq":12830,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T05:29:22.051469+0000","last_change":"2023-10-09T05:24:56.640700+0000","last_active":"2023-10-09T05:29:22.051469+0000","last_peered":"2023-10-09T05:29:22.051469+0000","last_clean":"2023-10-09T05:29:22.051469+0000","last_became_active":"2023-10-09T05:24:56.640352+0000","last_became_peered":"2023-10-09T05:24:56.640352+0000","last_unstale":"2023-10-09T05:29:22.051469+0000","last_undegraded":"2023-10-09T05:29:22.051469+0000","last_fullsized":"2023-10-09T05:29:22.051469+0000","mapping_epoch":264,"log_start":"273'4900","ondisk_log_start":"273'4900","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":7904,"log_dups_size":0,"ondisk_log_size":7904,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:55:08.218749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12804,"num_write_kb":409728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":75,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.346142+0000","last_change":"2023-10-09T05:18:26.437953+0000","last_active":"2023-10-09T05:28:51.346142+0000","last_peered":"2023-10-09T05:28:51.346142+0000","last_clean":"2023-10-09T05:28:51.346142+0000","last_became_active":"2023-10-09T05:18:26.422703+0000","last_became_peered":"2023-10-09T05:18:26.422703+0000","last_unstale":"2023-10-09T05:28:51.346142+0000","last_undegraded":"2023-10-09T05:28:51.346142+0000","last_fullsized":"2023-10-09T05:28:51.346142+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:13.926811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":312,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.361706+0000","last_change":"2023-10-09T05:18:26.423526+0000","last_active":"2023-10-09T05:28:51.361706+0000","last_peered":"2023-10-09T05:28:51.361706+0000","last_clean":"2023-10-09T05:28:51.361706+0000","last_became_active":"2023-10-09T05:18:26.423092+0000","last_became_peered":"2023-10-09T05:18:26.423092+0000","last_unstale":"2023-10-09T05:28:51.361706+0000","last_undegraded":"2023-10-09T05:28:51.361706+0000","last_fullsized":"2023-10-09T05:28:51.361706+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:54.877664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"282'12806","reported_seq":12835,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T05:29:22.017227+0000","last_change":"2023-10-09T05:24:56.634648+0000","last_active":"2023-10-09T05:29:22.017227+0000","last_peered":"2023-10-09T05:29:22.017227+0000","last_clean":"2023-10-09T05:29:22.017227+0000","last_became_active":"2023-10-09T05:24:56.633771+0000","last_became_peered":"2023-10-09T05:24:56.633771+0000","last_unstale":"2023-10-09T05:29:22.017227+0000","last_undegraded":"2023-10-09T05:29:22.017227+0000","last_fullsized":"2023-10-09T05:29:22.017227+0000","mapping_epoch":264,"log_start":"272'4600","ondisk_log_start":"272'4600","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":8206,"log_dups_size":0,"ondisk_log_size":8206,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:20:25.848591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12806,"num_write_kb":409792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":204,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.339337+0000","last_change":"2023-10-09T05:13:03.037926+0000","last_active":"2023-10-09T05:28:51.339337+0000","last_peered":"2023-10-09T05:28:51.339337+0000","last_clean":"2023-10-09T05:28:51.339337+0000","last_became_active":"2023-10-09T05:13:03.037639+0000","last_became_peered":"2023-10-09T05:13:03.037639+0000","last_unstale":"2023-10-09T05:28:51.339337+0000","last_undegraded":"2023-10-09T05:28:51.339337+0000","last_fullsized":"2023-10-09T05:28:51.339337+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:52.977143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":263,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.346325+0000","last_change":"2023-10-09T05:09:57.852914+0000","last_active":"2023-10-09T05:28:51.346325+0000","last_peered":"2023-10-09T05:28:51.346325+0000","last_clean":"2023-10-09T05:28:51.346325+0000","last_became_active":"2023-10-09T05:09:57.852522+0000","last_became_peered":"2023-10-09T05:09:57.852522+0000","last_unstale":"2023-10-09T05:28:51.346325+0000","last_undegraded":"2023-10-09T05:28:51.346325+0000","last_fullsized":"2023-10-09T05:28:51.346325+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:24.848494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"282'12902","reported_seq":12928,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T05:29:21.982548+0000","last_change":"2023-10-09T05:24:56.639909+0000","last_active":"2023-10-09T05:29:21.982548+0000","last_peered":"2023-10-09T05:29:21.982548+0000","last_clean":"2023-10-09T05:29:21.982548+0000","last_became_active":"2023-10-09T05:24:56.639434+0000","last_became_peered":"2023-10-09T05:24:56.639434+0000","last_unstale":"2023-10-09T05:29:21.982548+0000","last_undegraded":"2023-10-09T05:29:21.982548+0000","last_fullsized":"2023-10-09T05:29:21.982548+0000","mapping_epoch":264,"log_start":"274'5200","ondisk_log_start":"274'5200","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":7702,"log_dups_size":0,"ondisk_log_size":7702,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:11.231733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12902,"num_write_kb":412864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"69'3","reported_seq":237,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:29:10.216408+0000","last_change":"2023-10-09T05:02:09.650579+0000","last_active":"2023-10-09T05:29:10.216408+0000","last_peered":"2023-10-09T05:29:10.216408+0000","last_clean":"2023-10-09T05:29:10.216408+0000","last_became_active":"2023-10-09T05:02:09.650160+0000","last_became_peered":"2023-10-09T05:02:09.650160+0000","last_unstale":"2023-10-09T05:29:10.216408+0000","last_undegraded":"2023-10-09T05:29:10.216408+0000","last_fullsized":"2023-10-09T05:29:10.216408+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:18.292670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":242,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.343840+0000","last_change":"2023-10-09T05:02:09.652449+0000","last_active":"2023-10-09T05:28:51.343840+0000","last_peered":"2023-10-09T05:28:51.343840+0000","last_clean":"2023-10-09T05:28:51.343840+0000","last_became_active":"2023-10-09T05:02:09.651994+0000","last_became_peered":"2023-10-09T05:02:09.651994+0000","last_unstale":"2023-10-09T05:28:51.343840+0000","last_undegraded":"2023-10-09T05:28:51.343840+0000","last_fullsized":"2023-10-09T05:28:51.343840+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:14.480513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"282'12804","reported_seq":12830,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T05:29:22.034828+0000","last_change":"2023-10-09T05:24:56.643292+0000","last_active":"2023-10-09T05:29:22.034828+0000","last_peered":"2023-10-09T05:29:22.034828+0000","last_clean":"2023-10-09T05:29:22.034828+0000","last_became_active":"2023-10-09T05:24:56.642863+0000","last_became_peered":"2023-10-09T05:24:56.642863+0000","last_unstale":"2023-10-09T05:29:22.034828+0000","last_undegraded":"2023-10-09T05:29:22.034828+0000","last_fullsized":"2023-10-09T05:29:22.034828+0000","mapping_epoch":264,"log_start":"274'5100","ondisk_log_start":"274'5100","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":7704,"log_dups_size":0,"ondisk_log_size":7704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:09.977734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12804,"num_write_kb":409728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":283,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.361643+0000","last_change":"2023-10-09T05:23:18.199172+0000","last_active":"2023-10-09T05:28:51.361643+0000","last_peered":"2023-10-09T05:28:51.361643+0000","last_clean":"2023-10-09T05:28:51.361643+0000","last_became_active":"2023-10-09T05:23:18.198762+0000","last_became_peered":"2023-10-09T05:23:18.198762+0000","last_unstale":"2023-10-09T05:28:51.361643+0000","last_undegraded":"2023-10-09T05:28:51.361643+0000","last_fullsized":"2023-10-09T05:28:51.361643+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:44:23.239788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":299,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.344415+0000","last_change":"2023-10-09T05:20:18.129228+0000","last_active":"2023-10-09T05:28:51.344415+0000","last_peered":"2023-10-09T05:28:51.344415+0000","last_clean":"2023-10-09T05:28:51.344415+0000","last_became_active":"2023-10-09T05:20:18.128769+0000","last_became_peered":"2023-10-09T05:20:18.128769+0000","last_unstale":"2023-10-09T05:28:51.344415+0000","last_undegraded":"2023-10-09T05:28:51.344415+0000","last_fullsized":"2023-10-09T05:28:51.344415+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:50.427100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"282'12796","reported_seq":12825,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T05:29:22.031338+0000","last_change":"2023-10-09T05:24:56.631754+0000","last_active":"2023-10-09T05:29:22.031338+0000","last_peered":"2023-10-09T05:29:22.031338+0000","last_clean":"2023-10-09T05:29:22.031338+0000","last_became_active":"2023-10-09T05:24:56.631413+0000","last_became_peered":"2023-10-09T05:24:56.631413+0000","last_unstale":"2023-10-09T05:29:22.031338+0000","last_undegraded":"2023-10-09T05:29:22.031338+0000","last_fullsized":"2023-10-09T05:29:22.031338+0000","mapping_epoch":264,"log_start":"274'5200","ondisk_log_start":"274'5200","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":7596,"log_dups_size":0,"ondisk_log_size":7596,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:06.416230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12796,"num_write_kb":409472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":283,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.343608+0000","last_change":"2023-10-09T05:18:26.431286+0000","last_active":"2023-10-09T05:28:51.343608+0000","last_peered":"2023-10-09T05:28:51.343608+0000","last_clean":"2023-10-09T05:28:51.343608+0000","last_became_active":"2023-10-09T05:18:26.430127+0000","last_became_peered":"2023-10-09T05:18:26.430127+0000","last_unstale":"2023-10-09T05:28:51.343608+0000","last_undegraded":"2023-10-09T05:28:51.343608+0000","last_fullsized":"2023-10-09T05:28:51.343608+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:42.982259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":282,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.345866+0000","last_change":"2023-10-09T05:23:18.193800+0000","last_active":"2023-10-09T05:28:51.345866+0000","last_peered":"2023-10-09T05:28:51.345866+0000","last_clean":"2023-10-09T05:28:51.345866+0000","last_became_active":"2023-10-09T05:23:18.193338+0000","last_became_peered":"2023-10-09T05:23:18.193338+0000","last_unstale":"2023-10-09T05:28:51.345866+0000","last_undegraded":"2023-10-09T05:28:51.345866+0000","last_fullsized":"2023-10-09T05:28:51.345866+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:29.109566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"282'12782","reported_seq":12811,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T05:29:22.075526+0000","last_change":"2023-10-09T05:24:56.632416+0000","last_active":"2023-10-09T05:29:22.075526+0000","last_peered":"2023-10-09T05:29:22.075526+0000","last_clean":"2023-10-09T05:29:22.075526+0000","last_became_active":"2023-10-09T05:24:56.632033+0000","last_became_peered":"2023-10-09T05:24:56.632033+0000","last_unstale":"2023-10-09T05:29:22.075526+0000","last_undegraded":"2023-10-09T05:29:22.075526+0000","last_fullsized":"2023-10-09T05:29:22.075526+0000","mapping_epoch":264,"log_start":"274'5200","ondisk_log_start":"274'5200","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":7582,"log_dups_size":0,"ondisk_log_size":7582,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:08.729587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12782,"num_write_kb":409024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":295,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.336135+0000","last_change":"2023-10-09T05:18:26.413106+0000","last_active":"2023-10-09T05:28:51.336135+0000","last_peered":"2023-10-09T05:28:51.336135+0000","last_clean":"2023-10-09T05:28:51.336135+0000","last_became_active":"2023-10-09T05:18:26.412334+0000","last_became_peered":"2023-10-09T05:18:26.412334+0000","last_unstale":"2023-10-09T05:28:51.336135+0000","last_undegraded":"2023-10-09T05:28:51.336135+0000","last_fullsized":"2023-10-09T05:28:51.336135+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:18.058016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":149,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.333039+0000","last_change":"2023-10-09T05:09:57.856764+0000","last_active":"2023-10-09T05:28:51.333039+0000","last_peered":"2023-10-09T05:28:51.333039+0000","last_clean":"2023-10-09T05:28:51.333039+0000","last_became_active":"2023-10-09T05:09:57.856461+0000","last_became_peered":"2023-10-09T05:09:57.856461+0000","last_unstale":"2023-10-09T05:28:51.333039+0000","last_undegraded":"2023-10-09T05:28:51.333039+0000","last_fullsized":"2023-10-09T05:28:51.333039+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:06.848082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.a","version":"282'12994","reported_seq":13023,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T05:29:22.028768+0000","last_change":"2023-10-09T05:24:56.632715+0000","last_active":"2023-10-09T05:29:22.028768+0000","last_peered":"2023-10-09T05:29:22.028768+0000","last_clean":"2023-10-09T05:29:22.028768+0000","last_became_active":"2023-10-09T05:24:56.632411+0000","last_became_peered":"2023-10-09T05:24:56.632411+0000","last_unstale":"2023-10-09T05:29:22.028768+0000","last_undegraded":"2023-10-09T05:29:22.028768+0000","last_fullsized":"2023-10-09T05:29:22.028768+0000","mapping_epoch":264,"log_start":"271'3300","ondisk_log_start":"271'3300","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":9694,"log_dups_size":0,"ondisk_log_size":9694,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:25.311429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12994,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":186,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:55.124930+0000","last_change":"2023-10-09T05:18:26.422150+0000","last_active":"2023-10-09T05:28:55.124930+0000","last_peered":"2023-10-09T05:28:55.124930+0000","last_clean":"2023-10-09T05:28:55.124930+0000","last_became_active":"2023-10-09T05:18:26.421853+0000","last_became_peered":"2023-10-09T05:18:26.421853+0000","last_unstale":"2023-10-09T05:28:55.124930+0000","last_undegraded":"2023-10-09T05:28:55.124930+0000","last_fullsized":"2023-10-09T05:28:55.124930+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:37:47.000556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":221,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.338459+0000","last_change":"2023-10-09T05:13:03.031243+0000","last_active":"2023-10-09T05:28:51.338459+0000","last_peered":"2023-10-09T05:28:51.338459+0000","last_clean":"2023-10-09T05:28:51.338459+0000","last_became_active":"2023-10-09T05:13:03.028769+0000","last_became_peered":"2023-10-09T05:13:03.028769+0000","last_unstale":"2023-10-09T05:28:51.338459+0000","last_undegraded":"2023-10-09T05:28:51.338459+0000","last_fullsized":"2023-10-09T05:28:51.338459+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:25:25.239799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":42,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.331124+0000","last_change":"2023-10-09T05:23:18.197927+0000","last_active":"2023-10-09T05:28:51.331124+0000","last_peered":"2023-10-09T05:28:51.331124+0000","last_clean":"2023-10-09T05:28:51.331124+0000","last_became_active":"2023-10-09T05:23:18.197431+0000","last_became_peered":"2023-10-09T05:23:18.197431+0000","last_unstale":"2023-10-09T05:28:51.331124+0000","last_undegraded":"2023-10-09T05:28:51.331124+0000","last_fullsized":"2023-10-09T05:28:51.331124+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:51:41.376456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"282'12760","reported_seq":12789,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T05:29:22.018204+0000","last_change":"2023-10-09T05:24:56.631792+0000","last_active":"2023-10-09T05:29:22.018204+0000","last_peered":"2023-10-09T05:29:22.018204+0000","last_clean":"2023-10-09T05:29:22.018204+0000","last_became_active":"2023-10-09T05:24:56.631473+0000","last_became_peered":"2023-10-09T05:24:56.631473+0000","last_unstale":"2023-10-09T05:29:22.018204+0000","last_undegraded":"2023-10-09T05:29:22.018204+0000","last_fullsized":"2023-10-09T05:29:22.018204+0000","mapping_epoch":264,"log_start":"272'4600","ondisk_log_start":"272'4600","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":8160,"log_dups_size":0,"ondisk_log_size":8160,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:17.170510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12760,"num_write_kb":408320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":255,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.331368+0000","last_change":"2023-10-09T05:13:03.031386+0000","last_active":"2023-10-09T05:28:51.331368+0000","last_peered":"2023-10-09T05:28:51.331368+0000","last_clean":"2023-10-09T05:28:51.331368+0000","last_became_active":"2023-10-09T05:13:03.030816+0000","last_became_peered":"2023-10-09T05:13:03.030816+0000","last_unstale":"2023-10-09T05:28:51.331368+0000","last_undegraded":"2023-10-09T05:28:51.331368+0000","last_fullsized":"2023-10-09T05:28:51.331368+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:28.088661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":243,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.343880+0000","last_change":"2023-10-09T04:56:56.087320+0000","last_active":"2023-10-09T05:28:51.343880+0000","last_peered":"2023-10-09T05:28:51.343880+0000","last_clean":"2023-10-09T05:28:51.343880+0000","last_became_active":"2023-10-09T04:56:56.086552+0000","last_became_peered":"2023-10-09T04:56:56.086552+0000","last_unstale":"2023-10-09T05:28:51.343880+0000","last_undegraded":"2023-10-09T05:28:51.343880+0000","last_fullsized":"2023-10-09T05:28:51.343880+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:14.205288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":42,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.332009+0000","last_change":"2023-10-09T05:23:18.196260+0000","last_active":"2023-10-09T05:28:51.332009+0000","last_peered":"2023-10-09T05:28:51.332009+0000","last_clean":"2023-10-09T05:28:51.332009+0000","last_became_active":"2023-10-09T05:23:18.195907+0000","last_became_peered":"2023-10-09T05:23:18.195907+0000","last_unstale":"2023-10-09T05:28:51.332009+0000","last_undegraded":"2023-10-09T05:28:51.332009+0000","last_fullsized":"2023-10-09T05:28:51.332009+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:36.243432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.f","version":"282'12856","reported_seq":12882,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T05:29:22.075998+0000","last_change":"2023-10-09T05:24:56.644098+0000","last_active":"2023-10-09T05:29:22.075998+0000","last_peered":"2023-10-09T05:29:22.075998+0000","last_clean":"2023-10-09T05:29:22.075998+0000","last_became_active":"2023-10-09T05:24:56.643666+0000","last_became_peered":"2023-10-09T05:24:56.643666+0000","last_unstale":"2023-10-09T05:29:22.075998+0000","last_undegraded":"2023-10-09T05:29:22.075998+0000","last_fullsized":"2023-10-09T05:29:22.075998+0000","mapping_epoch":264,"log_start":"274'5100","ondisk_log_start":"274'5100","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":7756,"log_dups_size":0,"ondisk_log_size":7756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:39:36.431548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12856,"num_write_kb":411392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":61,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.342815+0000","last_change":"2023-10-09T05:20:18.123706+0000","last_active":"2023-10-09T05:28:51.342815+0000","last_peered":"2023-10-09T05:28:51.342815+0000","last_clean":"2023-10-09T05:28:51.342815+0000","last_became_active":"2023-10-09T05:20:18.122859+0000","last_became_peered":"2023-10-09T05:20:18.122859+0000","last_unstale":"2023-10-09T05:28:51.342815+0000","last_undegraded":"2023-10-09T05:28:51.342815+0000","last_fullsized":"2023-10-09T05:28:51.342815+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.906228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":150,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.330706+0000","last_change":"2023-10-09T05:09:57.854064+0000","last_active":"2023-10-09T05:28:51.330706+0000","last_peered":"2023-10-09T05:28:51.330706+0000","last_clean":"2023-10-09T05:28:51.330706+0000","last_became_active":"2023-10-09T05:09:57.847953+0000","last_became_peered":"2023-10-09T05:09:57.847953+0000","last_unstale":"2023-10-09T05:28:51.330706+0000","last_undegraded":"2023-10-09T05:28:51.330706+0000","last_fullsized":"2023-10-09T05:28:51.330706+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:45.914976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":25,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.347774+0000","last_change":"2023-10-09T05:25:15.659153+0000","last_active":"2023-10-09T05:28:51.347774+0000","last_peered":"2023-10-09T05:28:51.347774+0000","last_clean":"2023-10-09T05:28:51.347774+0000","last_became_active":"2023-10-09T05:25:15.658750+0000","last_became_peered":"2023-10-09T05:25:15.658750+0000","last_unstale":"2023-10-09T05:28:51.347774+0000","last_undegraded":"2023-10-09T05:28:51.347774+0000","last_fullsized":"2023-10-09T05:28:51.347774+0000","mapping_epoch":268,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":269,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:10.370264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.d","version":"282'12800","reported_seq":12827,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T05:29:22.065594+0000","last_change":"2023-10-09T05:24:56.633695+0000","last_active":"2023-10-09T05:29:22.065594+0000","last_peered":"2023-10-09T05:29:22.065594+0000","last_clean":"2023-10-09T05:29:22.065594+0000","last_became_active":"2023-10-09T05:24:56.633393+0000","last_became_peered":"2023-10-09T05:24:56.633393+0000","last_unstale":"2023-10-09T05:29:22.065594+0000","last_undegraded":"2023-10-09T05:29:22.065594+0000","last_fullsized":"2023-10-09T05:29:22.065594+0000","mapping_epoch":264,"log_start":"274'5100","ondisk_log_start":"274'5100","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":7700,"log_dups_size":0,"ondisk_log_size":7700,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:04.072215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12800,"num_write_kb":409537,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":227,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.345417+0000","last_change":"2023-10-09T05:18:26.416511+0000","last_active":"2023-10-09T05:28:51.345417+0000","last_peered":"2023-10-09T05:28:51.345417+0000","last_clean":"2023-10-09T05:28:51.345417+0000","last_became_active":"2023-10-09T05:18:26.416101+0000","last_became_peered":"2023-10-09T05:18:26.416101+0000","last_unstale":"2023-10-09T05:28:51.345417+0000","last_undegraded":"2023-10-09T05:28:51.345417+0000","last_fullsized":"2023-10-09T05:28:51.345417+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:10.243472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":567,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.344935+0000","last_change":"2023-10-09T05:18:29.419527+0000","last_active":"2023-10-09T05:28:51.344935+0000","last_peered":"2023-10-09T05:28:51.344935+0000","last_clean":"2023-10-09T05:28:51.344935+0000","last_became_active":"2023-10-09T05:18:29.418852+0000","last_became_peered":"2023-10-09T05:18:29.418852+0000","last_unstale":"2023-10-09T05:28:51.344935+0000","last_undegraded":"2023-10-09T05:28:51.344935+0000","last_fullsized":"2023-10-09T05:28:51.344935+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_clean_scrub_stamp":"2023-10-09T04:36:44.261013+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:02.899170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":245,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T05:29:14.723948+0000","last_change":"2023-10-09T05:13:03.027260+0000","last_active":"2023-10-09T05:29:14.723948+0000","last_peered":"2023-10-09T05:29:14.723948+0000","last_clean":"2023-10-09T05:29:14.723948+0000","last_became_active":"2023-10-09T05:13:03.026416+0000","last_became_peered":"2023-10-09T05:13:03.026416+0000","last_unstale":"2023-10-09T05:29:14.723948+0000","last_undegraded":"2023-10-09T05:29:14.723948+0000","last_fullsized":"2023-10-09T05:29:14.723948+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:05.057338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":265,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.362576+0000","last_change":"2023-10-09T05:18:26.426126+0000","last_active":"2023-10-09T05:28:51.362576+0000","last_peered":"2023-10-09T05:28:51.362576+0000","last_clean":"2023-10-09T05:28:51.362576+0000","last_became_active":"2023-10-09T05:18:26.425721+0000","last_became_peered":"2023-10-09T05:18:26.425721+0000","last_unstale":"2023-10-09T05:28:51.362576+0000","last_undegraded":"2023-10-09T05:28:51.362576+0000","last_fullsized":"2023-10-09T05:28:51.362576+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:48.229218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"282'13052","reported_seq":13080,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T05:29:22.057887+0000","last_change":"2023-10-09T05:24:56.641295+0000","last_active":"2023-10-09T05:29:22.057887+0000","last_peered":"2023-10-09T05:29:22.057887+0000","last_clean":"2023-10-09T05:29:22.057887+0000","last_became_active":"2023-10-09T05:24:56.640909+0000","last_became_peered":"2023-10-09T05:24:56.640909+0000","last_unstale":"2023-10-09T05:29:22.057887+0000","last_undegraded":"2023-10-09T05:29:22.057887+0000","last_fullsized":"2023-10-09T05:29:22.057887+0000","mapping_epoch":264,"log_start":"273'4900","ondisk_log_start":"273'4900","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":8152,"log_dups_size":0,"ondisk_log_size":8152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:26.475478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13052,"num_write_kb":417664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":91,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.344965+0000","last_change":"2023-10-09T05:23:18.193825+0000","last_active":"2023-10-09T05:28:51.344965+0000","last_peered":"2023-10-09T05:28:51.344965+0000","last_clean":"2023-10-09T05:28:51.344965+0000","last_became_active":"2023-10-09T05:23:18.193394+0000","last_became_peered":"2023-10-09T05:23:18.193394+0000","last_unstale":"2023-10-09T05:28:51.344965+0000","last_undegraded":"2023-10-09T05:28:51.344965+0000","last_fullsized":"2023-10-09T05:28:51.344965+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:40.594469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":245,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.346248+0000","last_change":"2023-10-09T05:02:09.648781+0000","last_active":"2023-10-09T05:28:51.346248+0000","last_peered":"2023-10-09T05:28:51.346248+0000","last_clean":"2023-10-09T05:28:51.346248+0000","last_became_active":"2023-10-09T05:02:09.648078+0000","last_became_peered":"2023-10-09T05:02:09.648078+0000","last_unstale":"2023-10-09T05:28:51.346248+0000","last_undegraded":"2023-10-09T05:28:51.346248+0000","last_fullsized":"2023-10-09T05:28:51.346248+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:35.287091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":324,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T05:29:14.703365+0000","last_change":"2023-10-09T05:20:18.132750+0000","last_active":"2023-10-09T05:29:14.703365+0000","last_peered":"2023-10-09T05:29:14.703365+0000","last_clean":"2023-10-09T05:29:14.703365+0000","last_became_active":"2023-10-09T05:20:18.132396+0000","last_became_peered":"2023-10-09T05:20:18.132396+0000","last_unstale":"2023-10-09T05:29:14.703365+0000","last_undegraded":"2023-10-09T05:29:14.703365+0000","last_fullsized":"2023-10-09T05:29:14.703365+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:00:55.731045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"282'12814","reported_seq":12842,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T05:29:22.069649+0000","last_change":"2023-10-09T05:24:56.642285+0000","last_active":"2023-10-09T05:29:22.069649+0000","last_peered":"2023-10-09T05:29:22.069649+0000","last_clean":"2023-10-09T05:29:22.069649+0000","last_became_active":"2023-10-09T05:24:56.641521+0000","last_became_peered":"2023-10-09T05:24:56.641521+0000","last_unstale":"2023-10-09T05:29:22.069649+0000","last_undegraded":"2023-10-09T05:29:22.069649+0000","last_fullsized":"2023-10-09T05:29:22.069649+0000","mapping_epoch":264,"log_start":"274'5800","ondisk_log_start":"274'5800","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":7014,"log_dups_size":0,"ondisk_log_size":7014,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:06:01.013563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12814,"num_write_kb":410048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":71,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.346176+0000","last_change":"2023-10-09T05:18:26.429468+0000","last_active":"2023-10-09T05:28:51.346176+0000","last_peered":"2023-10-09T05:28:51.346176+0000","last_clean":"2023-10-09T05:28:51.346176+0000","last_became_active":"2023-10-09T05:18:26.429098+0000","last_became_peered":"2023-10-09T05:18:26.429098+0000","last_unstale":"2023-10-09T05:28:51.346176+0000","last_undegraded":"2023-10-09T05:28:51.346176+0000","last_fullsized":"2023-10-09T05:28:51.346176+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:47.798226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":177,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:55.124951+0000","last_change":"2023-10-09T05:20:18.126997+0000","last_active":"2023-10-09T05:28:55.124951+0000","last_peered":"2023-10-09T05:28:55.124951+0000","last_clean":"2023-10-09T05:28:55.124951+0000","last_became_active":"2023-10-09T05:20:18.126641+0000","last_became_peered":"2023-10-09T05:20:18.126641+0000","last_unstale":"2023-10-09T05:28:55.124951+0000","last_undegraded":"2023-10-09T05:28:55.124951+0000","last_fullsized":"2023-10-09T05:28:55.124951+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:42.112245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":285,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.346556+0000","last_change":"2023-10-09T05:18:26.418524+0000","last_active":"2023-10-09T05:28:51.346556+0000","last_peered":"2023-10-09T05:28:51.346556+0000","last_clean":"2023-10-09T05:28:51.346556+0000","last_became_active":"2023-10-09T05:18:26.418058+0000","last_became_peered":"2023-10-09T05:18:26.418058+0000","last_unstale":"2023-10-09T05:28:51.346556+0000","last_undegraded":"2023-10-09T05:28:51.346556+0000","last_fullsized":"2023-10-09T05:28:51.346556+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:40.594449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"282'12884","reported_seq":12913,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T05:29:22.046466+0000","last_change":"2023-10-09T05:24:56.633539+0000","last_active":"2023-10-09T05:29:22.046466+0000","last_peered":"2023-10-09T05:29:22.046466+0000","last_clean":"2023-10-09T05:29:22.046466+0000","last_became_active":"2023-10-09T05:24:56.633197+0000","last_became_peered":"2023-10-09T05:24:56.633197+0000","last_unstale":"2023-10-09T05:29:22.046466+0000","last_undegraded":"2023-10-09T05:29:22.046466+0000","last_fullsized":"2023-10-09T05:29:22.046466+0000","mapping_epoch":264,"log_start":"274'5900","ondisk_log_start":"274'5900","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":6984,"log_dups_size":0,"ondisk_log_size":6984,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:53.863072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12884,"num_write_kb":412288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":288,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.338830+0000","last_change":"2023-10-09T05:18:26.431563+0000","last_active":"2023-10-09T05:28:51.338830+0000","last_peered":"2023-10-09T05:28:51.338830+0000","last_clean":"2023-10-09T05:28:51.338830+0000","last_became_active":"2023-10-09T05:18:26.421025+0000","last_became_peered":"2023-10-09T05:18:26.421025+0000","last_unstale":"2023-10-09T05:28:51.338830+0000","last_undegraded":"2023-10-09T05:28:51.338830+0000","last_fullsized":"2023-10-09T05:28:51.338830+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:30.333585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":42,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.332153+0000","last_change":"2023-10-09T05:23:18.196610+0000","last_active":"2023-10-09T05:28:51.332153+0000","last_peered":"2023-10-09T05:28:51.332153+0000","last_clean":"2023-10-09T05:28:51.332153+0000","last_became_active":"2023-10-09T05:23:18.196279+0000","last_became_peered":"2023-10-09T05:23:18.196279+0000","last_unstale":"2023-10-09T05:28:51.332153+0000","last_undegraded":"2023-10-09T05:28:51.332153+0000","last_fullsized":"2023-10-09T05:28:51.332153+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:52.868384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":287,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.339350+0000","last_change":"2023-10-09T05:21:08.179657+0000","last_active":"2023-10-09T05:28:51.339350+0000","last_peered":"2023-10-09T05:28:51.339350+0000","last_clean":"2023-10-09T05:28:51.339350+0000","last_became_active":"2023-10-09T05:21:08.178958+0000","last_became_peered":"2023-10-09T05:21:08.178958+0000","last_unstale":"2023-10-09T05:28:51.339350+0000","last_undegraded":"2023-10-09T05:28:51.339350+0000","last_fullsized":"2023-10-09T05:28:51.339350+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:20.815986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"282'12852","reported_seq":12881,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T05:29:22.005388+0000","last_change":"2023-10-09T05:24:56.641633+0000","last_active":"2023-10-09T05:29:22.005388+0000","last_peered":"2023-10-09T05:29:22.005388+0000","last_clean":"2023-10-09T05:29:22.005388+0000","last_became_active":"2023-10-09T05:24:56.641288+0000","last_became_peered":"2023-10-09T05:24:56.641288+0000","last_unstale":"2023-10-09T05:29:22.005388+0000","last_undegraded":"2023-10-09T05:29:22.005388+0000","last_fullsized":"2023-10-09T05:29:22.005388+0000","mapping_epoch":264,"log_start":"271'3100","ondisk_log_start":"271'3100","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:24:55.604050+0000","last_clean_scrub_stamp":"2023-10-09T05:24:55.604050+0000","objects_scrubbed":0,"log_size":9752,"log_dups_size":0,"ondisk_log_size":9752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:19.463846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12852,"num_write_kb":411264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":287,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.365988+0000","last_change":"2023-10-09T05:18:26.424429+0000","last_active":"2023-10-09T05:28:51.365988+0000","last_peered":"2023-10-09T05:28:51.365988+0000","last_clean":"2023-10-09T05:28:51.365988+0000","last_became_active":"2023-10-09T05:18:26.423034+0000","last_became_peered":"2023-10-09T05:18:26.423034+0000","last_unstale":"2023-10-09T05:28:51.365988+0000","last_undegraded":"2023-10-09T05:28:51.365988+0000","last_fullsized":"2023-10-09T05:28:51.365988+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:16.251181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":276,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T05:29:14.703459+0000","last_change":"2023-10-09T05:09:57.878002+0000","last_active":"2023-10-09T05:29:14.703459+0000","last_peered":"2023-10-09T05:29:14.703459+0000","last_clean":"2023-10-09T05:29:14.703459+0000","last_became_active":"2023-10-09T05:09:57.852792+0000","last_became_peered":"2023-10-09T05:09:57.852792+0000","last_unstale":"2023-10-09T05:29:14.703459+0000","last_undegraded":"2023-10-09T05:29:14.703459+0000","last_fullsized":"2023-10-09T05:29:14.703459+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:18.763275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":235,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.343518+0000","last_change":"2023-10-09T05:13:03.026541+0000","last_active":"2023-10-09T05:28:51.343518+0000","last_peered":"2023-10-09T05:28:51.343518+0000","last_clean":"2023-10-09T05:28:51.343518+0000","last_became_active":"2023-10-09T05:13:03.025729+0000","last_became_peered":"2023-10-09T05:13:03.025729+0000","last_unstale":"2023-10-09T05:28:51.343518+0000","last_undegraded":"2023-10-09T05:28:51.343518+0000","last_fullsized":"2023-10-09T05:28:51.343518+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:22.938235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'2","reported_seq":53,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.346481+0000","last_change":"2023-10-09T05:21:08.173845+0000","last_active":"2023-10-09T05:28:51.346481+0000","last_peered":"2023-10-09T05:28:51.346481+0000","last_clean":"2023-10-09T05:28:51.346481+0000","last_became_active":"2023-10-09T05:21:08.173508+0000","last_became_peered":"2023-10-09T05:21:08.173508+0000","last_unstale":"2023-10-09T05:28:51.346481+0000","last_undegraded":"2023-10-09T05:28:51.346481+0000","last_fullsized":"2023-10-09T05:28:51.346481+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:03.628428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":60,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.340104+0000","last_change":"2023-10-09T05:20:18.125172+0000","last_active":"2023-10-09T05:28:51.340104+0000","last_peered":"2023-10-09T05:28:51.340104+0000","last_clean":"2023-10-09T05:28:51.340104+0000","last_became_active":"2023-10-09T05:20:18.123624+0000","last_became_peered":"2023-10-09T05:20:18.123624+0000","last_unstale":"2023-10-09T05:28:51.340104+0000","last_undegraded":"2023-10-09T05:28:51.340104+0000","last_fullsized":"2023-10-09T05:28:51.340104+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:11.106483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":137,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.344295+0000","last_change":"2023-10-09T05:13:03.055887+0000","last_active":"2023-10-09T05:28:51.344295+0000","last_peered":"2023-10-09T05:28:51.344295+0000","last_clean":"2023-10-09T05:28:51.344295+0000","last_became_active":"2023-10-09T05:13:03.028782+0000","last_became_peered":"2023-10-09T05:13:03.028782+0000","last_unstale":"2023-10-09T05:28:51.344295+0000","last_undegraded":"2023-10-09T05:28:51.344295+0000","last_fullsized":"2023-10-09T05:28:51.344295+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:38.828205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":299,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.344090+0000","last_change":"2023-10-09T05:23:18.207836+0000","last_active":"2023-10-09T05:28:51.344090+0000","last_peered":"2023-10-09T05:28:51.344090+0000","last_clean":"2023-10-09T05:28:51.344090+0000","last_became_active":"2023-10-09T05:23:18.207404+0000","last_became_peered":"2023-10-09T05:23:18.207404+0000","last_unstale":"2023-10-09T05:28:51.344090+0000","last_undegraded":"2023-10-09T05:28:51.344090+0000","last_fullsized":"2023-10-09T05:28:51.344090+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:36:22.236093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":137,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.344469+0000","last_change":"2023-10-09T05:13:03.049546+0000","last_active":"2023-10-09T05:28:51.344469+0000","last_peered":"2023-10-09T05:28:51.344469+0000","last_clean":"2023-10-09T05:28:51.344469+0000","last_became_active":"2023-10-09T05:13:03.028676+0000","last_became_peered":"2023-10-09T05:13:03.028676+0000","last_unstale":"2023-10-09T05:28:51.344469+0000","last_undegraded":"2023-10-09T05:28:51.344469+0000","last_fullsized":"2023-10-09T05:28:51.344469+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:55.089350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":259,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.344704+0000","last_change":"2023-10-09T05:02:09.649748+0000","last_active":"2023-10-09T05:28:51.344704+0000","last_peered":"2023-10-09T05:28:51.344704+0000","last_clean":"2023-10-09T05:28:51.344704+0000","last_became_active":"2023-10-09T05:02:09.649444+0000","last_became_peered":"2023-10-09T05:02:09.649444+0000","last_unstale":"2023-10-09T05:28:51.344704+0000","last_undegraded":"2023-10-09T05:28:51.344704+0000","last_fullsized":"2023-10-09T05:28:51.344704+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:07.436820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'2","reported_seq":281,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.338528+0000","last_change":"2023-10-09T05:22:47.197002+0000","last_active":"2023-10-09T05:28:51.338528+0000","last_peered":"2023-10-09T05:28:51.338528+0000","last_clean":"2023-10-09T05:28:51.338528+0000","last_became_active":"2023-10-09T05:22:47.196685+0000","last_became_peered":"2023-10-09T05:22:47.196685+0000","last_unstale":"2023-10-09T05:28:51.338528+0000","last_undegraded":"2023-10-09T05:28:51.338528+0000","last_fullsized":"2023-10-09T05:28:51.338528+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:47:32.594447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":266,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.333342+0000","last_change":"2023-10-09T05:02:09.649674+0000","last_active":"2023-10-09T05:28:51.333342+0000","last_peered":"2023-10-09T05:28:51.333342+0000","last_clean":"2023-10-09T05:28:51.333342+0000","last_became_active":"2023-10-09T05:02:09.649041+0000","last_became_peered":"2023-10-09T05:02:09.649041+0000","last_unstale":"2023-10-09T05:28:51.333342+0000","last_undegraded":"2023-10-09T05:28:51.333342+0000","last_fullsized":"2023-10-09T05:28:51.333342+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:18.524071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'5","reported_seq":59,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.346610+0000","last_change":"2023-10-09T05:21:08.181594+0000","last_active":"2023-10-09T05:28:51.346610+0000","last_peered":"2023-10-09T05:28:51.346610+0000","last_clean":"2023-10-09T05:28:51.346610+0000","last_became_active":"2023-10-09T05:21:08.176895+0000","last_became_peered":"2023-10-09T05:21:08.176895+0000","last_unstale":"2023-10-09T05:28:51.346610+0000","last_undegraded":"2023-10-09T05:28:51.346610+0000","last_fullsized":"2023-10-09T05:28:51.346610+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:48:45.569544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":272,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T05:28:51.343693+0000","last_change":"2023-10-09T05:25:15.663537+0000","last_active":"2023-10-09T05:28:51.343693+0000","last_peered":"2023-10-09T05:28:51.343693+0000","last_clean":"2023-10-09T05:28:51.343693+0000","last_became_active":"2023-10-09T05:25:15.662771+0000","last_became_peered":"2023-10-09T05:25:15.662771+0000","last_unstale":"2023-10-09T05:28:51.343693+0000","last_undegraded":"2023-10-09T05:28:51.343693+0000","last_fullsized":"2023-10-09T05:28:51.343693+0000","mapping_epoch":268,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":269,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:53.588294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":205480,"num_write_kb":6575297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127680,"ondisk_log_size":127680,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":20,"num_read_kb":19,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":46,"num_bytes_recovered":21458,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234566,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":910596,"kb_used_data":3452,"kb_used_omap":0,"kb_used_meta":907136,"kb_avail":92834556,"statfs":{"total":95995035648,"available":95062585344,"internally_reserved":0,"allocated":3534848,"data_stored":2646686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928907264},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":3.2149999999999999,"15min":2.1269999999999998},"min":{"1min":0.26500000000000001,"5min":0.248,"15min":0.248},"max":{"1min":3.4990000000000001,"5min":122.248,"15min":122.248},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.304,"5min":3.218,"15min":2.1920000000000002},"min":{"1min":0.30199999999999999,"5min":0.29499999999999998,"15min":0.29099999999999998},"max":{"1min":6.1280000000000001,"5min":122.22199999999999,"15min":122.22199999999999},"last":0.65600000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:28:38 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":1.21,"15min":1.353},"min":{"1min":0.30499999999999999,"5min":0.29199999999999998,"15min":0.28000000000000003},"max":{"1min":3.9239999999999999,"5min":6.1109999999999998,"15min":23.530999999999999},"last":0.61799999999999999},{"interface":"front","average":{"1min":1.593,"5min":1.321,"15min":1.4339999999999999},"min":{"1min":0.313,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":6.1699999999999999,"5min":6.8710000000000004,"15min":23.443000000000001},"last":0.55500000000000005}]},{"osd":2,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":3.4249999999999998,"5min":3.3730000000000002,"15min":2.3029999999999999},"min":{"1min":0.314,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":37.945999999999998,"5min":122.065,"15min":122.065},"last":0.751},{"interface":"front","average":{"1min":3.9470000000000001,"5min":3.8959999999999999,"15min":2.4449999999999998},"min":{"1min":0.34300000000000003,"5min":0.32700000000000001,"15min":0.30399999999999999},"max":{"1min":38.043999999999997,"5min":122.026,"15min":122.026},"last":0.69299999999999995}]},{"osd":3,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":2.8599999999999999,"5min":3.3500000000000001,"15min":2.4380000000000002},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":38.636000000000003,"5min":138.667,"15min":138.667},"last":0.47699999999999998},{"interface":"front","average":{"1min":3.4950000000000001,"5min":3.4969999999999999,"15min":2.6080000000000001},"min":{"1min":0.35099999999999998,"5min":0.34699999999999998,"15min":0.34300000000000003},"max":{"1min":41.343000000000004,"5min":138.54400000000001,"15min":138.54400000000001},"last":0.82099999999999995}]},{"osd":4,"last update":"Mon Oct 9 05:28:38 2023","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":1.01,"15min":0.98299999999999998},"min":{"1min":0.34899999999999998,"5min":0.311,"15min":0.311},"max":{"1min":4.548,"5min":6.0309999999999997,"15min":9.7850000000000001},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.90900000000000003,"5min":1.0680000000000001,"15min":1.0209999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":4.5780000000000003,"5min":5.4219999999999997,"15min":5.4219999999999997},"last":0.91000000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:28:38 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":0.98599999999999999,"15min":0.92700000000000005},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.101,"5min":4.6870000000000003,"15min":4.8689999999999998},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.90500000000000003,"5min":1.0720000000000001,"15min":0.92700000000000005},"min":{"1min":0.40899999999999997,"5min":0.373,"15min":0.373},"max":{"1min":4.0529999999999999,"5min":5.9779999999999998,"15min":5.9779999999999998},"last":0.70599999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:28:38 2023","interfaces":[{"interface":"back","average":{"1min":1.2869999999999999,"5min":1.333,"15min":1.1679999999999999},"min":{"1min":0.42699999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":6.282,"5min":7.5519999999999996,"15min":7.5519999999999996},"last":1.0700000000000001},{"interface":"front","average":{"1min":1.2310000000000001,"5min":1.1160000000000001,"15min":1.1479999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.38200000000000001},"max":{"1min":5.0030000000000001,"5min":5.0030000000000001,"15min":6.7910000000000004},"last":1.143}]}]},{"osd":6,"up_from":46,"seq":197568496208,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":894976,"kb_used_data":3448,"kb_used_omap":0,"kb_used_meta":891520,"kb_avail":92850176,"statfs":{"total":95995035648,"available":95078580224,"internally_reserved":0,"allocated":3530752,"data_stored":2647417,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":912916480},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":3.089,"15min":3.2389999999999999},"min":{"1min":0.28999999999999998,"5min":0.26100000000000001,"15min":0.23599999999999999},"max":{"1min":4.6520000000000001,"5min":154.29300000000001,"15min":154.29300000000001},"last":0.60799999999999998},{"interface":"front","average":{"1min":1.581,"5min":3.2160000000000002,"15min":3.3519999999999999},"min":{"1min":0.38400000000000001,"5min":0.33100000000000002,"15min":0.309},"max":{"1min":4.6820000000000004,"5min":154.35300000000001,"15min":154.35300000000001},"last":0.64200000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":2.4079999999999999,"5min":1.6619999999999999,"15min":2.2240000000000002},"min":{"1min":0.38300000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.5789999999999997,"5min":23.318000000000001,"15min":120.873},"last":4.0730000000000004},{"interface":"front","average":{"1min":1.724,"5min":1.5509999999999999,"15min":2.081},"min":{"1min":0.34100000000000003,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":5.7350000000000003,"5min":23.140000000000001,"15min":119.248},"last":4.1559999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":2.415,"5min":5.2750000000000004,"15min":2.6789999999999998},"min":{"1min":0.38300000000000001,"5min":0.35599999999999998,"15min":0.29399999999999998},"max":{"1min":7.8799999999999999,"5min":112.649,"15min":112.649},"last":4.7789999999999999},{"interface":"front","average":{"1min":2.1030000000000002,"5min":5.1959999999999997,"15min":2.8999999999999999},"min":{"1min":0.38700000000000001,"5min":0.31,"15min":0.31},"max":{"1min":7.0339999999999998,"5min":112.834,"15min":112.834},"last":0.38400000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":2.298,"5min":5.2409999999999997,"15min":4.7910000000000004},"min":{"1min":0.36499999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":9.5640000000000001,"5min":169.01599999999999,"15min":169.01599999999999},"last":0.92500000000000004},{"interface":"front","average":{"1min":1.8260000000000001,"5min":5.0579999999999998,"15min":4.6890000000000001},"min":{"1min":0.42399999999999999,"5min":0.33800000000000002,"15min":0.32800000000000001},"max":{"1min":8.4600000000000009,"5min":169.40799999999999,"15min":169.40799999999999},"last":0.36299999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.008,"15min":1.111},"min":{"1min":0.40500000000000003,"5min":0.315,"15min":0.27000000000000002},"max":{"1min":4.5099999999999998,"5min":4.5119999999999996,"15min":8.3670000000000009},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.96699999999999997,"15min":1.042},"min":{"1min":0.39100000000000001,"5min":0.38100000000000001,"15min":0.34100000000000003},"max":{"1min":1.2390000000000001,"5min":4.069,"15min":13.473000000000001},"last":0.40600000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":0.99099999999999999,"15min":0.97199999999999998},"min":{"1min":0.38900000000000001,"5min":0.35999999999999999,"15min":0.33300000000000002},"max":{"1min":4.4699999999999998,"5min":4.5410000000000004,"15min":7.5899999999999999},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.91800000000000004,"5min":1.0589999999999999,"15min":1.0700000000000001},"min":{"1min":0.47599999999999998,"5min":0.40400000000000003,"15min":0.378},"max":{"1min":4.3230000000000004,"5min":4.3230000000000004,"15min":13.292999999999999},"last":0.44600000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:29:21 2023","interfaces":[{"interface":"back","average":{"1min":1.3540000000000001,"5min":1.4399999999999999,"15min":1.3280000000000001},"min":{"1min":0.40699999999999997,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":8.6530000000000005,"5min":8.8780000000000001,"15min":8.8780000000000001},"last":0.57299999999999995},{"interface":"front","average":{"1min":1.4139999999999999,"5min":1.52,"15min":1.4530000000000001},"min":{"1min":0.47799999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":8.6859999999999999,"5min":8.923,"15min":8.923},"last":0.65400000000000003}]}]},{"osd":1,"up_from":96,"seq":412316860799,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":949856,"kb_used_data":3608,"kb_used_omap":4,"kb_used_meta":946235,"kb_avail":92795296,"statfs":{"total":95995035648,"available":95022383104,"internally_reserved":0,"allocated":3694592,"data_stored":2138897,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":968945550},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:28:50 2023","interfaces":[{"interface":"back","average":{"1min":3.7469999999999999,"5min":2.4849999999999999,"15min":2.4580000000000002},"min":{"1min":0.33800000000000002,"5min":0.217,"15min":0.19},"max":{"1min":19.172999999999998,"5min":76.007999999999996,"15min":83.194999999999993},"last":2.331},{"interface":"front","average":{"1min":4.2670000000000003,"5min":2.6960000000000002,"15min":2.7839999999999998},"min":{"1min":0.27200000000000002,"5min":0.192,"15min":0.186},"max":{"1min":28.113,"5min":75.168999999999997,"15min":83.972999999999999},"last":0.58399999999999996}]},{"osd":2,"last update":"Mon Oct 9 05:28:56 2023","interfaces":[{"interface":"back","average":{"1min":2.657,"5min":1.79,"15min":2.1789999999999998},"min":{"1min":0.27000000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":14.295,"5min":14.295,"15min":83.337000000000003},"last":0.94999999999999996},{"interface":"front","average":{"1min":3.044,"5min":1.752,"15min":2.2570000000000001},"min":{"1min":0.26000000000000001,"5min":0.254,"15min":0.23599999999999999},"max":{"1min":14.326000000000001,"5min":14.326000000000001,"15min":83.143000000000001},"last":0.66800000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:28:56 2023","interfaces":[{"interface":"back","average":{"1min":3.4460000000000002,"5min":2.218,"15min":2.444},"min":{"1min":0.30199999999999999,"5min":0.221,"15min":0.215},"max":{"1min":9.798,"5min":51.780999999999999,"15min":84.105999999999995},"last":0.28100000000000003},{"interface":"front","average":{"1min":2.9529999999999998,"5min":2.669,"15min":2.6869999999999998},"min":{"1min":0.28899999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":8.5109999999999992,"5min":51.813000000000002,"15min":83.566000000000003},"last":0.42399999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:28:50 2023","interfaces":[{"interface":"back","average":{"1min":1.9470000000000001,"5min":1.5669999999999999,"15min":2.0249999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":9.8200000000000003,"5min":9.8200000000000003,"15min":83.453999999999994},"last":0.52400000000000002},{"interface":"front","average":{"1min":1.8779999999999999,"5min":1.4159999999999999,"15min":1.742},"min":{"1min":0.35599999999999998,"5min":0.34599999999999997,"15min":0.28100000000000003},"max":{"1min":9.7850000000000001,"5min":9.7850000000000001,"15min":83.375},"last":2.4049999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:28:50 2023","interfaces":[{"interface":"back","average":{"1min":2.5129999999999999,"5min":1.4019999999999999,"15min":1.5009999999999999},"min":{"1min":0.39700000000000002,"5min":0.373,"15min":0.373},"max":{"1min":9.7520000000000007,"5min":9.7520000000000007,"15min":9.7520000000000007},"last":0.56499999999999995},{"interface":"front","average":{"1min":2.5329999999999999,"5min":1.6299999999999999,"15min":1.9339999999999999},"min":{"1min":0.42399999999999999,"5min":0.39900000000000002,"15min":0.38200000000000001},"max":{"1min":14.263999999999999,"5min":14.263999999999999,"15min":83.268000000000001},"last":0.63400000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:28:50 2023","interfaces":[{"interface":"back","average":{"1min":2.9790000000000001,"5min":1.657,"15min":1.6739999999999999},"min":{"1min":0.435,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":9.8520000000000003,"5min":9.8520000000000003,"15min":9.8520000000000003},"last":0.53900000000000003},{"interface":"front","average":{"1min":2.077,"5min":1.6879999999999999,"15min":2.0299999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.373},"max":{"1min":14.231,"5min":14.231,"15min":83.400000000000006},"last":0.60199999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:28:50 2023","interfaces":[{"interface":"back","average":{"1min":2.125,"5min":1.72,"15min":2.0019999999999998},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":14.166,"5min":14.166,"15min":83.510000000000005},"last":0.57099999999999995},{"interface":"front","average":{"1min":1.921,"5min":1.4490000000000001,"15min":1.7110000000000001},"min":{"1min":0.40300000000000002,"5min":0.377,"15min":0.36899999999999999},"max":{"1min":9.8819999999999997,"5min":9.8819999999999997,"15min":83.480999999999995},"last":2.4449999999999998}]}]},{"osd":0,"up_from":88,"seq":377957122440,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":905956,"kb_used_data":4124,"kb_used_omap":5,"kb_used_meta":901818,"kb_avail":92839196,"statfs":{"total":95995035648,"available":95067336704,"internally_reserved":0,"allocated":4222976,"data_stored":2721674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":923462266},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:28:45 2023","interfaces":[{"interface":"back","average":{"1min":1.9379999999999999,"5min":1.425,"15min":2.0899999999999999},"min":{"1min":0.28499999999999998,"5min":0.21199999999999999,"15min":0.17799999999999999},"max":{"1min":7.0099999999999998,"5min":7.0099999999999998,"15min":80.843999999999994},"last":0.39400000000000002},{"interface":"front","average":{"1min":1.179,"5min":1.0980000000000001,"15min":2.302},"min":{"1min":0.28100000000000003,"5min":0.221,"15min":0.21299999999999999},"max":{"1min":6.0339999999999998,"5min":8.9849999999999994,"15min":81.159999999999997},"last":0.65800000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:28:50 2023","interfaces":[{"interface":"back","average":{"1min":2.3159999999999998,"5min":3.2650000000000001,"15min":2.5619999999999998},"min":{"1min":0.51900000000000002,"5min":0.29599999999999999,"15min":0.25600000000000001},"max":{"1min":9.3879999999999999,"5min":92.745000000000005,"15min":92.745000000000005},"last":0.755},{"interface":"front","average":{"1min":2.3340000000000001,"5min":3.4649999999999999,"15min":2.6269999999999998},"min":{"1min":0.20699999999999999,"5min":0.20399999999999999,"15min":0.19700000000000001},"max":{"1min":7.5629999999999997,"5min":97.816000000000003,"15min":97.816000000000003},"last":0.54300000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:28:50 2023","interfaces":[{"interface":"back","average":{"1min":2.5939999999999999,"5min":2.0350000000000001,"15min":2.4590000000000001},"min":{"1min":0.27600000000000002,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":9.532,"5min":37.883000000000003,"15min":82.286000000000001},"last":0.434},{"interface":"front","average":{"1min":2.4830000000000001,"5min":2.0659999999999998,"15min":2.895},"min":{"1min":0.42799999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":9.6289999999999996,"5min":45.279000000000003,"15min":75.969999999999999},"last":0.88900000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:28:50 2023","interfaces":[{"interface":"back","average":{"1min":1.7549999999999999,"5min":1.173,"15min":1.637},"min":{"1min":0.45800000000000002,"5min":0.36799999999999999,"15min":0.33800000000000002},"max":{"1min":8.4629999999999992,"5min":8.4629999999999992,"15min":12.255000000000001},"last":1.0449999999999999},{"interface":"front","average":{"1min":1.8879999999999999,"5min":1.502,"15min":1.617},"min":{"1min":0.39900000000000002,"5min":0.34100000000000003,"15min":0.28199999999999997},"max":{"1min":6.8700000000000001,"5min":6.8700000000000001,"15min":8.3580000000000005},"last":0.59399999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:28:50 2023","interfaces":[{"interface":"back","average":{"1min":1.321,"5min":1.1859999999999999,"15min":1.3959999999999999},"min":{"1min":0.38300000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":5.758,"5min":5.758,"15min":11.916},"last":0.61199999999999999},{"interface":"front","average":{"1min":1.661,"5min":1.2110000000000001,"15min":1.6899999999999999},"min":{"1min":0.53500000000000003,"5min":0.434,"15min":0.375},"max":{"1min":8.5299999999999994,"5min":8.5299999999999994,"15min":12.143000000000001},"last":0.96999999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:28:50 2023","interfaces":[{"interface":"back","average":{"1min":2.1360000000000001,"5min":1.673,"15min":1.8},"min":{"1min":0.40300000000000002,"5min":0.374,"15min":0.374},"max":{"1min":6.9470000000000001,"5min":6.9470000000000001,"15min":12.125},"last":0.63300000000000001},{"interface":"front","average":{"1min":1.3180000000000001,"5min":1.363,"15min":1.6200000000000001},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35899999999999999},"max":{"1min":5.7960000000000003,"5min":8.4659999999999993,"15min":12.007999999999999},"last":0.73199999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:28:50 2023","interfaces":[{"interface":"back","average":{"1min":1.863,"5min":1.5249999999999999,"15min":1.6100000000000001},"min":{"1min":0.45300000000000001,"5min":0.308,"15min":0.308},"max":{"1min":6.0609999999999999,"5min":8.6590000000000007,"15min":11.977},"last":0.92000000000000004},{"interface":"front","average":{"1min":1.8129999999999999,"5min":1.506,"15min":1.871},"min":{"1min":0.46600000000000003,"5min":0.41599999999999998,"15min":0.40100000000000002},"max":{"1min":8.5020000000000007,"5min":10.795,"15min":12.1},"last":1.1240000000000001}]}]},{"osd":2,"up_from":106,"seq":455266533718,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":758832,"kb_used_data":3496,"kb_used_omap":7,"kb_used_meta":755320,"kb_avail":92986320,"statfs":{"total":95995035648,"available":95217991680,"internally_reserved":0,"allocated":3579904,"data_stored":2124302,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":773447761},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:28:49 2023","interfaces":[{"interface":"back","average":{"1min":4.0019999999999998,"5min":2.306,"15min":2.25},"min":{"1min":0.24099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":27.010999999999999,"5min":27.010999999999999,"15min":40.204999999999998},"last":0.55500000000000005},{"interface":"front","average":{"1min":3.7250000000000001,"5min":1.925,"15min":2.1480000000000001},"min":{"1min":0.23699999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":18.922000000000001,"5min":18.922000000000001,"15min":40.170000000000002},"last":0.48299999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:28:49 2023","interfaces":[{"interface":"back","average":{"1min":2.0339999999999998,"5min":1.6220000000000001,"15min":1.893},"min":{"1min":0.248,"5min":0.22700000000000001,"15min":0.224},"max":{"1min":7.133,"5min":7.2089999999999996,"15min":23.251000000000001},"last":0.51300000000000001},{"interface":"front","average":{"1min":3.1779999999999999,"5min":2.1059999999999999,"15min":2.0430000000000001},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.20399999999999999},"max":{"1min":8.1920000000000002,"5min":14.034000000000001,"15min":17.632999999999999},"last":0.54500000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":5.2839999999999998,"5min":2.548,"15min":2.5129999999999999},"min":{"1min":0.307,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":15.334,"5min":15.334,"15min":15.334},"last":0.58499999999999996},{"interface":"front","average":{"1min":3.6960000000000002,"5min":2.7010000000000001,"15min":2.3540000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.23899999999999999},"max":{"1min":18.515000000000001,"5min":18.515000000000001,"15min":18.515000000000001},"last":0.46700000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:28:49 2023","interfaces":[{"interface":"back","average":{"1min":1.952,"5min":1.587,"15min":1.6859999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":10.597,"5min":10.597,"15min":11.157999999999999},"last":0.89500000000000002},{"interface":"front","average":{"1min":2.7210000000000001,"5min":1.7609999999999999,"15min":1.6919999999999999},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.29699999999999999},"max":{"1min":8.2289999999999992,"5min":10.242000000000001,"15min":12.460000000000001},"last":0.74099999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:28:49 2023","interfaces":[{"interface":"back","average":{"1min":1.3899999999999999,"5min":1.6579999999999999,"15min":1.5600000000000001},"min":{"1min":0.35599999999999998,"5min":0.34100000000000003,"15min":0.33000000000000002},"max":{"1min":4.3490000000000002,"5min":6.6890000000000001,"15min":9.9190000000000005},"last":0.95199999999999996},{"interface":"front","average":{"1min":2.0579999999999998,"5min":1.5489999999999999,"15min":1.716},"min":{"1min":0.40500000000000003,"5min":0.39700000000000002,"15min":0.34699999999999998},"max":{"1min":10.686,"5min":10.686,"15min":11.205},"last":0.879}]},{"osd":6,"last update":"Mon Oct 9 05:28:49 2023","interfaces":[{"interface":"back","average":{"1min":2.569,"5min":1.8580000000000001,"15min":1.871},"min":{"1min":0.42999999999999999,"5min":0.35299999999999998,"15min":0.32500000000000001},"max":{"1min":8.2590000000000003,"5min":10.279999999999999,"15min":16.337},"last":0.77000000000000002},{"interface":"front","average":{"1min":1.4550000000000001,"5min":1.8879999999999999,"15min":1.8160000000000001},"min":{"1min":0.38,"5min":0.38,"15min":0.35199999999999998},"max":{"1min":4.3170000000000002,"5min":11.118,"15min":16.399999999999999},"last":0.91200000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:28:49 2023","interfaces":[{"interface":"back","average":{"1min":2.2949999999999999,"5min":2.1709999999999998,"15min":1.7969999999999999},"min":{"1min":0.54200000000000004,"5min":0.40899999999999997,"15min":0.38300000000000001},"max":{"1min":8.9469999999999992,"5min":13.571999999999999,"15min":13.571999999999999},"last":0.98599999999999999},{"interface":"front","average":{"1min":3.0640000000000001,"5min":2.169,"15min":1.9399999999999999},"min":{"1min":0.55400000000000005,"5min":0.46800000000000003,"15min":0.379},"max":{"1min":10.659000000000001,"5min":10.659000000000001,"15min":11.236000000000001},"last":0.85999999999999999}]}]},{"osd":3,"up_from":123,"seq":528280977737,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":873248,"kb_used_data":3416,"kb_used_omap":2,"kb_used_meta":869821,"kb_avail":92871904,"statfs":{"total":95995035648,"available":95100829696,"internally_reserved":0,"allocated":3497984,"data_stored":2114063,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":890696757},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":2.323,"5min":2.2650000000000001,"15min":3.27},"min":{"1min":0.251,"5min":0.20499999999999999,"15min":0.19500000000000001},"max":{"1min":13.698,"5min":46.765999999999998,"15min":199.84999999999999},"last":0.93999999999999995},{"interface":"front","average":{"1min":3.3210000000000002,"5min":2.4390000000000001,"15min":3.0139999999999998},"min":{"1min":0.435,"5min":0.224,"15min":0.20200000000000001},"max":{"1min":11.249000000000001,"5min":47.069000000000003,"15min":83.234999999999999},"last":0.63100000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":2.8220000000000001,"5min":2.1960000000000002,"15min":2.4220000000000002},"min":{"1min":0.253,"5min":0.22,"15min":0.22},"max":{"1min":8.9429999999999996,"5min":26.372,"15min":44.206000000000003},"last":0.80200000000000005},{"interface":"front","average":{"1min":3.3090000000000002,"5min":2.2370000000000001,"15min":2.2229999999999999},"min":{"1min":0.28000000000000003,"5min":0.24399999999999999,"15min":0.218},"max":{"1min":16.809000000000001,"5min":28.007999999999999,"15min":44.496000000000002},"last":0.66100000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":3.1589999999999998,"5min":2.5640000000000001,"15min":2.1669999999999998},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.22700000000000001},"max":{"1min":19.597999999999999,"5min":29.945,"15min":29.945},"last":0.496},{"interface":"front","average":{"1min":2.6989999999999998,"5min":2.2410000000000001,"15min":2.4369999999999998},"min":{"1min":0.28000000000000003,"5min":0.251,"15min":0.22800000000000001},"max":{"1min":7.2670000000000003,"5min":26.452999999999999,"15min":26.452999999999999},"last":0.52800000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":2.3969999999999998,"5min":1.645,"15min":1.875},"min":{"1min":0.443,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.0119999999999996,"5min":8.3629999999999995,"15min":12.438000000000001},"last":0.91200000000000003},{"interface":"front","average":{"1min":2.266,"5min":1.6679999999999999,"15min":2.222},"min":{"1min":0.39600000000000002,"5min":0.36299999999999999,"15min":0.316},"max":{"1min":13.037000000000001,"5min":13.037000000000001,"15min":199.50800000000001},"last":0.83599999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":2.3580000000000001,"5min":1.7030000000000001,"15min":2.2589999999999999},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.33500000000000002},"max":{"1min":13.079000000000001,"5min":13.079000000000001,"15min":199.93799999999999},"last":0.88800000000000001},{"interface":"front","average":{"1min":2.524,"5min":1.784,"15min":1.607},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.34899999999999998},"max":{"1min":9.8109999999999999,"5min":9.8109999999999999,"15min":11.135},"last":0.754}]},{"osd":6,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":2.3690000000000002,"5min":1.782,"15min":1.6879999999999999},"min":{"1min":0.36099999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":9.7789999999999999,"5min":9.7789999999999999,"15min":11.173999999999999},"last":0.68200000000000005},{"interface":"front","average":{"1min":2.6970000000000001,"5min":1.919,"15min":2.1120000000000001},"min":{"1min":0.55200000000000005,"5min":0.36699999999999999,"15min":0.32800000000000001},"max":{"1min":7.056,"5min":8.5500000000000007,"15min":12.407},"last":1.0820000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:29:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7080000000000002,"5min":1.952,"15min":1.9570000000000001},"min":{"1min":0.51400000000000001,"5min":0.39100000000000001,"15min":0.378},"max":{"1min":8.1110000000000007,"5min":8.6039999999999992,"15min":12.468},"last":1.167},{"interface":"front","average":{"1min":2.605,"5min":1.9590000000000001,"15min":2.4340000000000002},"min":{"1min":0.42799999999999999,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":12.989000000000001,"5min":12.989000000000001,"15min":199.96799999999999},"last":0.71299999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019494,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":899524,"kb_used_data":2876,"kb_used_omap":0,"kb_used_meta":896640,"kb_avail":92845628,"statfs":{"total":95995035648,"available":95073923072,"internally_reserved":0,"allocated":2945024,"data_stored":2056836,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":918159360},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:28:52 2023","interfaces":[{"interface":"back","average":{"1min":5.343,"5min":4.5350000000000001,"15min":2.637},"min":{"1min":0.248,"5min":0.248,"15min":0.24199999999999999},"max":{"1min":59.783000000000001,"5min":151.209,"15min":151.209},"last":0.57399999999999995},{"interface":"front","average":{"1min":5.2359999999999998,"5min":4.4029999999999996,"15min":2.6120000000000001},"min":{"1min":0.30499999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":59.823,"5min":151.303,"15min":151.303},"last":0.45000000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:28:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1359999999999999,"5min":1.48,"15min":2.2240000000000002},"min":{"1min":0.33200000000000002,"5min":0.28999999999999998,"15min":0.25700000000000001},"max":{"1min":3.274,"5min":53.911000000000001,"15min":142.297},"last":0.29099999999999998},{"interface":"front","average":{"1min":1.107,"5min":1.6240000000000001,"15min":2.1480000000000001},"min":{"1min":0.30099999999999999,"5min":0.29699999999999999,"15min":0.27800000000000002},"max":{"1min":4.4240000000000004,"5min":54.892000000000003,"15min":135.11600000000001},"last":0.64700000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:28:52 2023","interfaces":[{"interface":"back","average":{"1min":5.7960000000000003,"5min":3.8540000000000001,"15min":2.4119999999999999},"min":{"1min":0.35899999999999999,"5min":0.311,"15min":0.311},"max":{"1min":78.260999999999996,"5min":152.643,"15min":152.643},"last":0.53900000000000003},{"interface":"front","average":{"1min":5.9409999999999998,"5min":3.8340000000000001,"15min":2.5339999999999998},"min":{"1min":0.33700000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":73.174000000000007,"5min":152.554,"15min":152.554},"last":6.516}]},{"osd":3,"last update":"Mon Oct 9 05:29:02 2023","interfaces":[{"interface":"back","average":{"1min":4.992,"5min":2.2269999999999999,"15min":2.403},"min":{"1min":0.38300000000000001,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":73.004000000000005,"5min":73.004000000000005,"15min":131.726},"last":0.60499999999999998},{"interface":"front","average":{"1min":5.2309999999999999,"5min":2.1440000000000001,"15min":2.5369999999999999},"min":{"1min":0.33900000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":75.013000000000005,"5min":75.013000000000005,"15min":131.57300000000001},"last":0.71699999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:29:02 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":1.0269999999999999,"15min":0.90700000000000003},"min":{"1min":0.38100000000000001,"5min":0.376,"15min":0.33100000000000002},"max":{"1min":2.899,"5min":6.1399999999999997,"15min":7.7549999999999999},"last":0.66700000000000004},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.0229999999999999,"15min":0.93700000000000006},"min":{"1min":0.371,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":8.2799999999999994,"5min":10.255000000000001,"15min":10.255000000000001},"last":0.47799999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:28:41 2023","interfaces":[{"interface":"back","average":{"1min":1.341,"5min":1.486,"15min":1.296},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.33900000000000002},"max":{"1min":3.4460000000000002,"5min":20.818000000000001,"15min":20.818000000000001},"last":0.94399999999999995},{"interface":"front","average":{"1min":1.2150000000000001,"5min":1.222,"15min":1.131},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":3.621,"5min":6.3079999999999998,"15min":8.4930000000000003},"last":0.63100000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:28:41 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":1.212,"15min":1.123},"min":{"1min":0.372,"5min":0.372,"15min":0.33900000000000002},"max":{"1min":5.1079999999999997,"5min":10.289,"15min":10.289},"last":0.73799999999999999},{"interface":"front","average":{"1min":1.0660000000000001,"5min":1.1899999999999999,"15min":1.1899999999999999},"min":{"1min":0.39400000000000002,"5min":0.379,"15min":0.34799999999999998},"max":{"1min":3.2090000000000001,"5min":4.7489999999999997,"15min":8.0269999999999992},"last":5.2569999999999997}]}]},{"osd":5,"up_from":0,"seq":163208757850,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:29:30.324 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_7 pgp_num 16 2023-10-09T05:29:30.332 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:30 smithi194 bash[92666]: audit 2023-10-09T05:29:29.065612+0000 mon.a (mon.0) 1576 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:29:30.332 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:30 smithi194 bash[92666]: audit 2023-10-09T05:29:29.072998+0000 mon.a (mon.0) 1577 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:29:30.332 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:30 smithi194 bash[92666]: audit 2023-10-09T05:29:29.540492+0000 mon.a (mon.0) 1578 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:29:30.333 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:30 smithi194 bash[92666]: audit 2023-10-09T05:29:29.542195+0000 mon.a (mon.0) 1579 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:29:30.333 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:30 smithi194 bash[92666]: audit 2023-10-09T05:29:29.550021+0000 mon.a (mon.0) 1580 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:29:30.350 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:30.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:30.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:30.453 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:30.473 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:30.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:30.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:30.576 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:30.608 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:30.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:30.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:30.711 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:30.737 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:30.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:30.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:30.841 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:30.876 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:30.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:30.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:30.980 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:31.012 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:31.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:31.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:31.115 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:31.121 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:31 smithi194 bash[92666]: cluster 2023-10-09T05:29:30.241008+0000 mgr.y (mgr.25277) 1241 : cluster 0 pgmap v1321: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 420 op/s 2023-10-09T05:29:31.142 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:31.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:31.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:31.246 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:31.272 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:31.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:31.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:31.377 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:31.385 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:31 smithi133 bash[102858]: cluster 2023-10-09T05:29:30.241008+0000 mgr.y (mgr.25277) 1241 : cluster 0 pgmap v1321: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 420 op/s 2023-10-09T05:29:31.385 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:31 smithi133 bash[105589]: cluster 2023-10-09T05:29:30.241008+0000 mgr.y (mgr.25277) 1241 : cluster 0 pgmap v1321: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 420 op/s 2023-10-09T05:29:31.403 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:31.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:31.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:31.506 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:31.536 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:31.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:31.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:31.639 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:31.668 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:31.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:31.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:31.771 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:31.796 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:31.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:31.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:31.899 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:31.926 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:31.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:31.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:32.029 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:32.062 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:32.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:32.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:32.166 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:32.195 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:32.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:32.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:32.298 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:32.324 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:32.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:32.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:32.428 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:32.450 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:32.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:32.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:32.553 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:32.582 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:32.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:32.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:32.686 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:32.720 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:32.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:32.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:32.823 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:32.852 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:32.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:32.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:32.955 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:32.985 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:32.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:32.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:33.089 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:33.115 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:33.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:33.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:33.219 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:33.254 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:33.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:33.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:33.358 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:33.380 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:33 smithi194 bash[92666]: cluster 2023-10-09T05:29:32.241894+0000 mgr.y (mgr.25277) 1242 : cluster 0 pgmap v1322: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 177 op/s 2023-10-09T05:29:33.402 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:33.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:33.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:33.455 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:29:33.480 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:29:33.505 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:33.508 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:29:33.518 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:33 smithi133 bash[105589]: cluster 2023-10-09T05:29:32.241894+0000 mgr.y (mgr.25277) 1242 : cluster 0 pgmap v1322: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 177 op/s 2023-10-09T05:29:33.518 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:33 smithi133 bash[102858]: cluster 2023-10-09T05:29:32.241894+0000 mgr.y (mgr.25277) 1242 : cluster 0 pgmap v1322: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 177 op/s 2023-10-09T05:29:33.533 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:33.534 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:29:33.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:33.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:33.560 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:29:33.589 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:29:33.614 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:29:33.623 INFO:teuthology.orchestra.run.smithi194.stdout:{ 2023-10-09T05:29:33.624 INFO:teuthology.orchestra.run.smithi194.stdout: "ops": [], 2023-10-09T05:29:33.624 INFO:teuthology.orchestra.run.smithi194.stdout: "complaint_time": 30, 2023-10-09T05:29:33.624 INFO:teuthology.orchestra.run.smithi194.stdout: "num_blocked_ops": 0 2023-10-09T05:29:33.624 INFO:teuthology.orchestra.run.smithi194.stdout:} 2023-10-09T05:29:33.637 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:33.641 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:29:33.644 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:29:33.644 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:29:33.644 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:29:33.644 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:29:33.644 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:29:33.644 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:29:33.644 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:29:33.644 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:29:33.669 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:33.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:33.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:33.772 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:33.803 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:33.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:33.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:33.907 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:33.933 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:33.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:33.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:34.037 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:34.066 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:34.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:34.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:34.169 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:34.199 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:34.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:34.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:34.303 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:34.318 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is set 2023-10-09T05:29:34.327 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:34.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:34.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:34.430 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:34.444 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:34 smithi194 bash[92666]: audit 2023-10-09T05:29:33.425025+0000 mon.c (mon.2) 112 : audit 0 from='client.? 172.21.15.133:0/93865697' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:29:34.445 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:34 smithi194 bash[92666]: audit 2023-10-09T05:29:33.655617+0000 mon.c (mon.2) 113 : audit 1 from='client.? 172.21.15.133:0/3638722114' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:29:34.445 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:34 smithi194 bash[92666]: audit 2023-10-09T05:29:33.656327+0000 mon.a (mon.0) 1581 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:29:34.462 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:34.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:34.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:34.512 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T05:29:34.567 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:34.569 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:29:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:29:34] "GET /metrics HTTP/1.1" 200 36425 "" "Prometheus/2.43.0" 2023-10-09T05:29:34.569 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:34 smithi133 bash[105589]: audit 2023-10-09T05:29:33.425025+0000 mon.c (mon.2) 112 : audit 0 from='client.? 172.21.15.133:0/93865697' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:29:34.569 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:34 smithi133 bash[105589]: audit 2023-10-09T05:29:33.655617+0000 mon.c (mon.2) 113 : audit 1 from='client.? 172.21.15.133:0/3638722114' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:29:34.569 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:34 smithi133 bash[105589]: audit 2023-10-09T05:29:33.656327+0000 mon.a (mon.0) 1581 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:29:34.570 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:34 smithi133 bash[102858]: audit 2023-10-09T05:29:33.425025+0000 mon.c (mon.2) 112 : audit 0 from='client.? 172.21.15.133:0/93865697' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:29:34.570 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:34 smithi133 bash[102858]: audit 2023-10-09T05:29:33.655617+0000 mon.c (mon.2) 113 : audit 1 from='client.? 172.21.15.133:0/3638722114' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:29:34.570 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:34 smithi133 bash[102858]: audit 2023-10-09T05:29:33.656327+0000 mon.a (mon.0) 1581 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:29:34.594 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:34.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:34.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:34.698 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:34.728 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:34.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:34.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:34.831 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:34.866 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:34.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:34.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:34.969 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:35.003 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:35.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:35.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:35.107 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:35.139 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:35.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:35.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:35.242 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:35.274 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:35.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:35.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:35.378 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:35.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:35 smithi194 bash[92666]: cluster 2023-10-09T05:29:34.242629+0000 mgr.y (mgr.25277) 1243 : cluster 0 pgmap v1323: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 140 op/s 2023-10-09T05:29:35.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:35 smithi194 bash[92666]: cluster 2023-10-09T05:29:34.299529+0000 mon.a (mon.0) 1582 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:29:35.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:35 smithi194 bash[92666]: cluster 2023-10-09T05:29:34.311017+0000 osd.5 (osd.5) 260 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T05:29:35.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:35 smithi194 bash[92666]: audit 2023-10-09T05:29:34.313218+0000 mon.a (mon.0) 1583 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:29:35.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:35 smithi194 bash[92666]: cluster 2023-10-09T05:29:34.313317+0000 mon.a (mon.0) 1584 : cluster 0 osdmap e283: 8 total, 8 up, 7 in 2023-10-09T05:29:35.409 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:35.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:35.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:35.513 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:35.541 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:35.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:35.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:35.645 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:35.658 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:35 smithi133 bash[105589]: cluster 2023-10-09T05:29:34.242629+0000 mgr.y (mgr.25277) 1243 : cluster 0 pgmap v1323: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 140 op/s 2023-10-09T05:29:35.658 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:35 smithi133 bash[105589]: cluster 2023-10-09T05:29:34.299529+0000 mon.a (mon.0) 1582 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:29:35.659 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:35 smithi133 bash[105589]: cluster 2023-10-09T05:29:34.311017+0000 osd.5 (osd.5) 260 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T05:29:35.659 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:35 smithi133 bash[105589]: audit 2023-10-09T05:29:34.313218+0000 mon.a (mon.0) 1583 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:29:35.659 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:35 smithi133 bash[105589]: cluster 2023-10-09T05:29:34.313317+0000 mon.a (mon.0) 1584 : cluster 0 osdmap e283: 8 total, 8 up, 7 in 2023-10-09T05:29:35.661 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:35 smithi133 bash[102858]: cluster 2023-10-09T05:29:34.242629+0000 mgr.y (mgr.25277) 1243 : cluster 0 pgmap v1323: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 140 op/s 2023-10-09T05:29:35.661 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:35 smithi133 bash[102858]: cluster 2023-10-09T05:29:34.299529+0000 mon.a (mon.0) 1582 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:29:35.662 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:35 smithi133 bash[102858]: cluster 2023-10-09T05:29:34.311017+0000 osd.5 (osd.5) 260 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T05:29:35.662 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:35 smithi133 bash[102858]: audit 2023-10-09T05:29:34.313218+0000 mon.a (mon.0) 1583 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:29:35.662 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:35 smithi133 bash[102858]: cluster 2023-10-09T05:29:34.313317+0000 mon.a (mon.0) 1584 : cluster 0 osdmap e283: 8 total, 8 up, 7 in 2023-10-09T05:29:35.677 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:35.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:35.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:35.781 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:35.809 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:35.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:35.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:35.914 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:35.941 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:35.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:35.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:36.045 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:36.079 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:36.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:36.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:36.183 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:36.217 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:36.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:36.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:36.298 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:29:36.321 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:36.349 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:36.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:36.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:36.452 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:36.482 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:36.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:36.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:36.586 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:36.613 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:36.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:36.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:36.717 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:36.750 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:36.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:36.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:36.853 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:36.889 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:36.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:36.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:36.991 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:37.025 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:37.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:37.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:37.129 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:37.135 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:29:37.163 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:37.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:37.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:37.266 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:37.293 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:37.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:37.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:37.396 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:37.431 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:37.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:37.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:37.535 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:37.547 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:37 smithi194 bash[92666]: cluster 2023-10-09T05:29:36.243863+0000 mgr.y (mgr.25277) 1244 : cluster 0 pgmap v1325: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:37.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:37 smithi133 bash[102858]: cluster 2023-10-09T05:29:36.243863+0000 mgr.y (mgr.25277) 1244 : cluster 0 pgmap v1325: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:37.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:37 smithi133 bash[105589]: cluster 2023-10-09T05:29:36.243863+0000 mgr.y (mgr.25277) 1244 : cluster 0 pgmap v1325: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:37.565 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:37.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:37.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:37.668 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:37.699 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:37.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:37.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:37.804 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:37.818 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:29:37.834 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:37.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:37.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:37.938 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:37.968 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:37.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:37.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:38.071 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:38.103 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:38.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:38.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:38.207 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:38.238 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:38.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:38.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:38.342 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:38.375 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:38.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:38.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:38.479 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:38.509 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:38.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:38.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:38.613 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:38.640 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:38.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:38.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:38.744 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:38.774 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:38.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:38.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:38.877 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:38.906 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:38.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:38.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:39.011 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:39.045 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:39.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:39.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:39.149 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:39.179 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:39.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:39.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:39.283 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:39.288 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:29:39.318 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:39.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:39.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:39.422 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:39.433 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:39 smithi133 bash[105589]: cluster 2023-10-09T05:29:38.244538+0000 mgr.y (mgr.25277) 1245 : cluster 0 pgmap v1326: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:39.433 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:39 smithi133 bash[102858]: cluster 2023-10-09T05:29:38.244538+0000 mgr.y (mgr.25277) 1245 : cluster 0 pgmap v1326: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:39.450 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:39.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:39.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:39.555 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:39.563 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:39 smithi194 bash[92666]: cluster 2023-10-09T05:29:38.244538+0000 mgr.y (mgr.25277) 1245 : cluster 0 pgmap v1326: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:39.582 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:39.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:39.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:39.685 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:39.713 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:39.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:39.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:39.816 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:39.849 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:39.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:39.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:39.953 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:39.979 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:39.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:39.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:40.083 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:40.114 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:40.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:40.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:40.217 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:40.249 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:40.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:40.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:40.352 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:40.383 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:40.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:40.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:40.487 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:40.514 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:40.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:40.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:40.617 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:40.648 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:40.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:40.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:40.752 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:40.783 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:40.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:40.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:40.886 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:40.912 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:40.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:40.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:41.015 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:41.043 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:41.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:41.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:41.146 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:41.182 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:41.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:41.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:41.286 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:41.318 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:41.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:41.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:41.329 INFO:teuthology.orchestra.run.smithi133.stderr:set pool 12 pgp_num to 16 2023-10-09T05:29:41.421 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:41.434 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:41 smithi133 bash[105589]: cluster 2023-10-09T05:29:40.245231+0000 mgr.y (mgr.25277) 1246 : cluster 0 pgmap v1327: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:41.434 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:41 smithi133 bash[105589]: audit 2023-10-09T05:29:40.987642+0000 mon.a (mon.0) 1585 : audit 1 from='client.? 172.21.15.133:0/3616597603' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:29:41.435 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:41 smithi133 bash[102858]: cluster 2023-10-09T05:29:40.245231+0000 mgr.y (mgr.25277) 1246 : cluster 0 pgmap v1327: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:41.435 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:41 smithi133 bash[102858]: audit 2023-10-09T05:29:40.987642+0000 mon.a (mon.0) 1585 : audit 1 from='client.? 172.21.15.133:0/3616597603' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:29:41.452 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:41.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:41.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:41.555 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:41.569 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:41 smithi194 bash[92666]: cluster 2023-10-09T05:29:40.245231+0000 mgr.y (mgr.25277) 1246 : cluster 0 pgmap v1327: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:41.569 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:41 smithi194 bash[92666]: audit 2023-10-09T05:29:40.987642+0000 mon.a (mon.0) 1585 : audit 1 from='client.? 172.21.15.133:0/3616597603' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:29:41.588 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:41.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:41.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:41.693 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:41.722 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:41.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:41.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:41.826 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:41.856 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:41.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:41.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:41.960 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:41.993 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:41.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:41.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:42.097 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:42.128 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:42.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:42.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:42.232 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:42.262 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:42.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:42.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:42.367 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:42.391 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:42 smithi133 bash[102858]: audit 2023-10-09T05:29:41.323243+0000 mon.a (mon.0) 1586 : audit 1 from='client.? 172.21.15.133:0/3616597603' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T05:29:42.391 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:42 smithi133 bash[102858]: cluster 2023-10-09T05:29:41.323337+0000 mon.a (mon.0) 1587 : cluster 0 osdmap e284: 8 total, 8 up, 7 in 2023-10-09T05:29:42.391 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:42 smithi133 bash[102858]: cluster 2023-10-09T05:29:41.371034+0000 osd.4 (osd.4) 254 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T05:29:42.391 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:42 smithi133 bash[102858]: cluster 2023-10-09T05:29:41.371195+0000 osd.5 (osd.5) 261 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T05:29:42.391 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:42 smithi133 bash[102858]: cluster 2023-10-09T05:29:41.371492+0000 osd.7 (osd.7) 233 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T05:29:42.391 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:42 smithi133 bash[102858]: cluster 2023-10-09T05:29:41.371596+0000 osd.6 (osd.6) 236 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T05:29:42.392 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:42 smithi133 bash[105589]: audit 2023-10-09T05:29:41.323243+0000 mon.a (mon.0) 1586 : audit 1 from='client.? 172.21.15.133:0/3616597603' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T05:29:42.392 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:42 smithi133 bash[105589]: cluster 2023-10-09T05:29:41.323337+0000 mon.a (mon.0) 1587 : cluster 0 osdmap e284: 8 total, 8 up, 7 in 2023-10-09T05:29:42.392 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:42 smithi133 bash[105589]: cluster 2023-10-09T05:29:41.371034+0000 osd.4 (osd.4) 254 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T05:29:42.392 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:42 smithi133 bash[105589]: cluster 2023-10-09T05:29:41.371195+0000 osd.5 (osd.5) 261 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T05:29:42.392 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:42 smithi133 bash[105589]: cluster 2023-10-09T05:29:41.371492+0000 osd.7 (osd.7) 233 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T05:29:42.392 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:42 smithi133 bash[105589]: cluster 2023-10-09T05:29:41.371596+0000 osd.6 (osd.6) 236 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T05:29:42.393 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_7 2023-10-09T05:29:42.393 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_7 unique_pool_7 --yes-i-really-really-mean-it 2023-10-09T05:29:42.401 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:42.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:42.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:42.505 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:42.536 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:42.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:42.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:42.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:42 smithi194 bash[92666]: audit 2023-10-09T05:29:41.323243+0000 mon.a (mon.0) 1586 : audit 1 from='client.? 172.21.15.133:0/3616597603' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T05:29:42.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:42 smithi194 bash[92666]: cluster 2023-10-09T05:29:41.323337+0000 mon.a (mon.0) 1587 : cluster 0 osdmap e284: 8 total, 8 up, 7 in 2023-10-09T05:29:42.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:42 smithi194 bash[92666]: cluster 2023-10-09T05:29:41.371034+0000 osd.4 (osd.4) 254 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T05:29:42.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:42 smithi194 bash[92666]: cluster 2023-10-09T05:29:41.371195+0000 osd.5 (osd.5) 261 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T05:29:42.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:42 smithi194 bash[92666]: cluster 2023-10-09T05:29:41.371492+0000 osd.7 (osd.7) 233 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T05:29:42.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:42 smithi194 bash[92666]: cluster 2023-10-09T05:29:41.371596+0000 osd.6 (osd.6) 236 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T05:29:42.640 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:42.670 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:42.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:42.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:42.774 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:42.806 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:42.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:42.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:42.811 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:29:42.812 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:29:42.910 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:42.937 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:42.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:42.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:43.040 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:43.057 INFO:teuthology.orchestra.run.smithi194.stdout:{ 2023-10-09T05:29:43.057 INFO:teuthology.orchestra.run.smithi194.stdout: "size": 20, 2023-10-09T05:29:43.057 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 600, 2023-10-09T05:29:43.058 INFO:teuthology.orchestra.run.smithi194.stdout: "ops": [ 2023-10-09T05:29:43.058 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.058 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35187.0:89383 11.3 11:cfb860b6:::benchmark_data_smithi133_194104_object89382:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e234)", 2023-10-09T05:29:43.058 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:20:17.112347+0000", 2023-10-09T05:29:43.058 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 565.94240945299998, 2023-10-09T05:29:43.058 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.5934816070000002, 2023-10-09T05:29:43.058 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:29:43.058 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:29:43.058 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:29:43.058 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35187", 2023-10-09T05:29:43.058 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2118344251", 2023-10-09T05:29:43.059 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 89383 2023-10-09T05:29:43.059 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.059 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:29:43.059 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.059 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:29:43.059 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.112347+0000", 2023-10-09T05:29:43.059 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.059 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.059 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.059 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:29:43.059 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.112347+0000", 2023-10-09T05:29:43.059 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.060 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.060 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.060 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:29:43.060 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.112346+0000", 2023-10-09T05:29:43.060 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:29:43.060 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.060 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.060 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:29:43.060 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.112388+0000", 2023-10-09T05:29:43.060 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.1934999999999997e-05 2023-10-09T05:29:43.060 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.060 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.060 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:29:43.061 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.112402+0000", 2023-10-09T05:29:43.061 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.4610000000000001e-05 2023-10-09T05:29:43.061 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.061 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.061 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:29:43.061 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.112412+0000", 2023-10-09T05:29:43.061 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.5140000000000007e-06 2023-10-09T05:29:43.061 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.061 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.061 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:29:43.061 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.113742+0000", 2023-10-09T05:29:43.061 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0013305649999999999 2023-10-09T05:29:43.062 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.062 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.062 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:29:43.062 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.113754+0000", 2023-10-09T05:29:43.062 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.2227e-05 2023-10-09T05:29:43.062 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.062 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.062 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:29:43.062 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:18.121466+0000", 2023-10-09T05:29:43.062 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0077117360000001 2023-10-09T05:29:43.062 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.063 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.063 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:29:43.063 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:18.121480+0000", 2023-10-09T05:29:43.063 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.362e-05 2023-10-09T05:29:43.063 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.063 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.063 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:29:43.063 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.614249+0000", 2023-10-09T05:29:43.063 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.4927689310000001 2023-10-09T05:29:43.063 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.063 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.063 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:29:43.063 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.614422+0000", 2023-10-09T05:29:43.064 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00017280900000000001 2023-10-09T05:29:43.064 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.064 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.064 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 2,7", 2023-10-09T05:29:43.064 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.614619+0000", 2023-10-09T05:29:43.064 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00019773800000000001 2023-10-09T05:29:43.064 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.064 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.064 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.064 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.623930+0000", 2023-10-09T05:29:43.064 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0093107180000000008 2023-10-09T05:29:43.064 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.065 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.065 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:29:43.065 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.623948+0000", 2023-10-09T05:29:43.065 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.8176e-05 2023-10-09T05:29:43.065 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.065 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.065 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.065 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.700971+0000", 2023-10-09T05:29:43.065 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.077022485000000002 2023-10-09T05:29:43.065 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.065 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.065 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:29:43.065 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.701014+0000", 2023-10-09T05:29:43.066 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.3659999999999999e-05 2023-10-09T05:29:43.066 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.066 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.066 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:29:43.066 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.705829+0000", 2023-10-09T05:29:43.066 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00481453 2023-10-09T05:29:43.066 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.066 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:29:43.066 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.066 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.066 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.066 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35187.0:181020 11.3 11:c1499b72:::benchmark_data_smithi133_194104_object85100:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e252)", 2023-10-09T05:29:43.066 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:23:17.191031+0000", 2023-10-09T05:29:43.067 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 385.86372531199999, 2023-10-09T05:29:43.067 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.295921874, 2023-10-09T05:29:43.067 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:29:43.067 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:29:43.067 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:29:43.067 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35187", 2023-10-09T05:29:43.067 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2118344251", 2023-10-09T05:29:43.067 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 181020 2023-10-09T05:29:43.067 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.067 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:29:43.067 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.067 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:29:43.068 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.191031+0000", 2023-10-09T05:29:43.068 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.068 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.068 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.068 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:29:43.068 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.191031+0000", 2023-10-09T05:29:43.068 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.068 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.068 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.068 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:29:43.068 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.191029+0000", 2023-10-09T05:29:43.068 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999981 2023-10-09T05:29:43.068 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.069 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.069 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:29:43.069 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.191040+0000", 2023-10-09T05:29:43.069 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0823e-05 2023-10-09T05:29:43.069 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.069 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.069 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:29:43.069 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.191070+0000", 2023-10-09T05:29:43.069 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.9971000000000002e-05 2023-10-09T05:29:43.069 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.069 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.069 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:29:43.069 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.191086+0000", 2023-10-09T05:29:43.070 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.5756999999999998e-05 2023-10-09T05:29:43.070 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.070 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.070 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:29:43.070 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.191177+0000", 2023-10-09T05:29:43.070 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.0807e-05 2023-10-09T05:29:43.070 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.070 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.070 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:29:43.070 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.191198+0000", 2023-10-09T05:29:43.070 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.1308e-05 2023-10-09T05:29:43.070 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.071 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.071 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:29:43.071 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.453298+0000", 2023-10-09T05:29:43.071 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.262099855 2023-10-09T05:29:43.071 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.071 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.071 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:29:43.071 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.453456+0000", 2023-10-09T05:29:43.071 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00015837499999999999 2023-10-09T05:29:43.071 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.071 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.071 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:29:43.072 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.453583+0000", 2023-10-09T05:29:43.072 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000127102 2023-10-09T05:29:43.072 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.072 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.072 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.072 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.463713+0000", 2023-10-09T05:29:43.072 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.010129943000000001 2023-10-09T05:29:43.072 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.072 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.072 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:29:43.072 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.463724+0000", 2023-10-09T05:29:43.072 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0732e-05 2023-10-09T05:29:43.072 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.073 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.073 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.073 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.485882+0000", 2023-10-09T05:29:43.073 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.022157679 2023-10-09T05:29:43.073 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.073 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.073 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:29:43.073 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.485962+0000", 2023-10-09T05:29:43.073 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.0029000000000002e-05 2023-10-09T05:29:43.073 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.073 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.073 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:29:43.073 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.486953+0000", 2023-10-09T05:29:43.074 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00099152500000000005 2023-10-09T05:29:43.074 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.074 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:29:43.074 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.074 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.074 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.074 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35187.0:181023 11.3 11:c908c260:::benchmark_data_smithi133_194104_object85103:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e252)", 2023-10-09T05:29:43.074 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:23:17.195937+0000", 2023-10-09T05:29:43.074 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 385.85881948399998, 2023-10-09T05:29:43.074 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.300794593, 2023-10-09T05:29:43.074 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:29:43.074 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:29:43.075 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:29:43.075 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35187", 2023-10-09T05:29:43.075 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2118344251", 2023-10-09T05:29:43.075 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 181023 2023-10-09T05:29:43.075 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.075 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:29:43.075 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.075 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:29:43.075 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.195937+0000", 2023-10-09T05:29:43.075 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.075 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.075 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.076 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:29:43.076 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.195937+0000", 2023-10-09T05:29:43.076 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.076 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.076 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.076 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:29:43.076 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.195935+0000", 2023-10-09T05:29:43.076 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999976 2023-10-09T05:29:43.076 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.076 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.076 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:29:43.076 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.195948+0000", 2023-10-09T05:29:43.076 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.348e-05 2023-10-09T05:29:43.076 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.077 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.077 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:29:43.077 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.195982+0000", 2023-10-09T05:29:43.077 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.3399000000000003e-05 2023-10-09T05:29:43.077 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.077 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.077 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:29:43.077 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.195999+0000", 2023-10-09T05:29:43.077 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.6906e-05 2023-10-09T05:29:43.077 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.077 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.077 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:29:43.077 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.196149+0000", 2023-10-09T05:29:43.078 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000150638 2023-10-09T05:29:43.078 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.078 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.078 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:29:43.078 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.196158+0000", 2023-10-09T05:29:43.078 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.4999999999999999e-06 2023-10-09T05:29:43.078 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.078 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.078 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:29:43.078 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.459806+0000", 2023-10-09T05:29:43.078 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.263647999 2023-10-09T05:29:43.079 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.079 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.079 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:29:43.079 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.459918+0000", 2023-10-09T05:29:43.079 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000111827 2023-10-09T05:29:43.079 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.079 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.079 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:29:43.079 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.460011+0000", 2023-10-09T05:29:43.079 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.2893000000000007e-05 2023-10-09T05:29:43.079 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.079 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.079 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:29:43.080 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.473401+0000", 2023-10-09T05:29:43.080 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.013390613000000001 2023-10-09T05:29:43.080 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.080 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.080 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.080 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.473490+0000", 2023-10-09T05:29:43.080 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.8744000000000003e-05 2023-10-09T05:29:43.080 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.080 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.080 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.080 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.496021+0000", 2023-10-09T05:29:43.080 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.022530669 2023-10-09T05:29:43.081 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.081 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.081 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:29:43.081 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.496082+0000", 2023-10-09T05:29:43.081 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.1457000000000005e-05 2023-10-09T05:29:43.081 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.081 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.081 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:29:43.081 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.496732+0000", 2023-10-09T05:29:43.081 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00064975700000000005 2023-10-09T05:29:43.081 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.081 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:29:43.081 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.082 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.082 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.082 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35187.0:181063 11.3 11:c9612b26:::benchmark_data_smithi133_194104_object85143:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e252)", 2023-10-09T05:29:43.082 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:23:17.298089+0000", 2023-10-09T05:29:43.082 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 385.75666793900001, 2023-10-09T05:29:43.082 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.1998125449999999, 2023-10-09T05:29:43.082 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:29:43.082 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:29:43.082 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:29:43.082 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35187", 2023-10-09T05:29:43.082 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2118344251", 2023-10-09T05:29:43.083 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 181063 2023-10-09T05:29:43.083 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.083 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:29:43.083 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.083 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:29:43.083 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.298089+0000", 2023-10-09T05:29:43.083 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.083 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.083 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.083 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:29:43.083 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.298089+0000", 2023-10-09T05:29:43.083 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.084 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.084 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.084 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:29:43.084 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.298087+0000", 2023-10-09T05:29:43.084 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:29:43.084 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.084 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.084 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:29:43.084 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.298095+0000", 2023-10-09T05:29:43.084 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.8229999999999995e-06 2023-10-09T05:29:43.084 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.084 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.085 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:29:43.085 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.298110+0000", 2023-10-09T05:29:43.085 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.4759999999999999e-05 2023-10-09T05:29:43.085 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.085 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.085 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:29:43.085 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.298117+0000", 2023-10-09T05:29:43.085 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.8499999999999996e-06 2023-10-09T05:29:43.085 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.085 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.085 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:29:43.085 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.298190+0000", 2023-10-09T05:29:43.086 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.2937999999999999e-05 2023-10-09T05:29:43.086 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.086 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.086 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:29:43.086 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.298203+0000", 2023-10-09T05:29:43.086 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.3441000000000001e-05 2023-10-09T05:29:43.086 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.086 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.086 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:29:43.086 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.463807+0000", 2023-10-09T05:29:43.086 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.1656035739999999 2023-10-09T05:29:43.086 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.087 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.087 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:29:43.087 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.464093+0000", 2023-10-09T05:29:43.087 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00028631199999999999 2023-10-09T05:29:43.087 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.087 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.087 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:29:43.087 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.464363+0000", 2023-10-09T05:29:43.087 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00027000199999999998 2023-10-09T05:29:43.087 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.087 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.087 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:29:43.087 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.473415+0000", 2023-10-09T05:29:43.088 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0090517529999999992 2023-10-09T05:29:43.088 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.088 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.088 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.088 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.473577+0000", 2023-10-09T05:29:43.088 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00016210000000000001 2023-10-09T05:29:43.088 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.088 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.088 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.088 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.496857+0000", 2023-10-09T05:29:43.088 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.023280556000000001 2023-10-09T05:29:43.089 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.089 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.089 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:29:43.089 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.496949+0000", 2023-10-09T05:29:43.089 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.1374999999999998e-05 2023-10-09T05:29:43.089 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.089 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.089 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:29:43.089 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.497901+0000", 2023-10-09T05:29:43.089 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00095263399999999995 2023-10-09T05:29:43.089 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.089 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:29:43.089 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.090 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.090 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.090 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35187.0:181064 11.3 11:cb921749:::benchmark_data_smithi133_194104_object85144:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e252)", 2023-10-09T05:29:43.090 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:23:17.303849+0000", 2023-10-09T05:29:43.090 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 385.75090729499999, 2023-10-09T05:29:43.090 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.2146236049999999, 2023-10-09T05:29:43.090 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:29:43.090 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:29:43.090 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:29:43.090 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35187", 2023-10-09T05:29:43.090 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2118344251", 2023-10-09T05:29:43.091 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 181064 2023-10-09T05:29:43.091 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.091 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:29:43.091 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.091 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:29:43.091 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.303849+0000", 2023-10-09T05:29:43.091 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.091 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.091 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.091 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:29:43.091 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.303849+0000", 2023-10-09T05:29:43.091 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.091 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.092 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.092 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:29:43.092 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.303848+0000", 2023-10-09T05:29:43.092 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:29:43.092 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.092 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.092 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:29:43.092 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.303855+0000", 2023-10-09T05:29:43.092 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.9940000000000003e-06 2023-10-09T05:29:43.092 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.092 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.092 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:29:43.093 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.303870+0000", 2023-10-09T05:29:43.093 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.5106000000000001e-05 2023-10-09T05:29:43.093 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.093 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.093 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:29:43.093 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.303876+0000", 2023-10-09T05:29:43.093 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.5459999999999999e-06 2023-10-09T05:29:43.093 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.093 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.093 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:29:43.093 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.303928+0000", 2023-10-09T05:29:43.093 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.1866000000000001e-05 2023-10-09T05:29:43.094 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.094 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.094 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:29:43.094 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.303936+0000", 2023-10-09T05:29:43.094 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.7870000000000007e-06 2023-10-09T05:29:43.094 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.094 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.094 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:29:43.094 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.469164+0000", 2023-10-09T05:29:43.094 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.1652273470000001 2023-10-09T05:29:43.094 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.095 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.095 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:29:43.095 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.469325+0000", 2023-10-09T05:29:43.095 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00016107400000000001 2023-10-09T05:29:43.095 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.095 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.095 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:29:43.095 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.469453+0000", 2023-10-09T05:29:43.095 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00012778299999999999 2023-10-09T05:29:43.095 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.095 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.095 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:29:43.096 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.480688+0000", 2023-10-09T05:29:43.096 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.011235541 2023-10-09T05:29:43.096 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.096 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.096 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.096 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.480855+0000", 2023-10-09T05:29:43.096 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00016681000000000001 2023-10-09T05:29:43.096 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.096 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.096 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.096 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.517398+0000", 2023-10-09T05:29:43.096 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.036543125000000003 2023-10-09T05:29:43.096 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.097 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.097 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:29:43.097 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.517450+0000", 2023-10-09T05:29:43.097 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.2266999999999999e-05 2023-10-09T05:29:43.097 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.097 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.097 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:29:43.097 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.518473+0000", 2023-10-09T05:29:43.097 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.001022703 2023-10-09T05:29:43.097 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.097 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:29:43.097 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.098 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.098 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.098 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35187.0:181069 11.3 11:c397534b:::benchmark_data_smithi133_194104_object85149:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e252)", 2023-10-09T05:29:43.098 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:23:17.309959+0000", 2023-10-09T05:29:43.098 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 385.74479806800002, 2023-10-09T05:29:43.098 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.2090776999999999, 2023-10-09T05:29:43.098 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:29:43.098 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:29:43.098 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:29:43.098 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35187", 2023-10-09T05:29:43.098 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2118344251", 2023-10-09T05:29:43.098 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 181069 2023-10-09T05:29:43.099 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.099 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:29:43.099 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.099 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:29:43.099 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.309959+0000", 2023-10-09T05:29:43.099 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.099 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.099 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.099 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:29:43.099 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.309959+0000", 2023-10-09T05:29:43.099 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.099 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.100 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.100 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:29:43.100 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.309958+0000", 2023-10-09T05:29:43.100 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:29:43.100 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.100 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.100 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:29:43.100 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.309965+0000", 2023-10-09T05:29:43.100 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.1749999999999999e-06 2023-10-09T05:29:43.100 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.100 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.100 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:29:43.100 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.309973+0000", 2023-10-09T05:29:43.101 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.9829999999999996e-06 2023-10-09T05:29:43.101 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.101 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.101 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:29:43.101 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.309977+0000", 2023-10-09T05:29:43.101 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.3499999999999999e-06 2023-10-09T05:29:43.101 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.101 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.101 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:29:43.101 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.310049+0000", 2023-10-09T05:29:43.101 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.2281999999999997e-05 2023-10-09T05:29:43.101 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.101 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.102 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:29:43.102 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.310081+0000", 2023-10-09T05:29:43.102 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.1628000000000003e-05 2023-10-09T05:29:43.102 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.102 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.102 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:29:43.102 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.473662+0000", 2023-10-09T05:29:43.102 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.1635810710000001 2023-10-09T05:29:43.102 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.102 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.102 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:29:43.103 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.473770+0000", 2023-10-09T05:29:43.103 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000108198 2023-10-09T05:29:43.103 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.103 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.103 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:29:43.103 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.473870+0000", 2023-10-09T05:29:43.103 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000100072 2023-10-09T05:29:43.103 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.103 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.103 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:29:43.103 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.484003+0000", 2023-10-09T05:29:43.103 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.010132966 2023-10-09T05:29:43.103 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.104 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.104 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.104 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.484042+0000", 2023-10-09T05:29:43.104 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.8173999999999997e-05 2023-10-09T05:29:43.104 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.104 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.104 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.104 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.518540+0000", 2023-10-09T05:29:43.104 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.034498230999999997 2023-10-09T05:29:43.104 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.104 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.104 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:29:43.104 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.518595+0000", 2023-10-09T05:29:43.105 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.5399999999999998e-05 2023-10-09T05:29:43.105 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.105 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.105 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:29:43.105 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.519036+0000", 2023-10-09T05:29:43.105 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00044115199999999999 2023-10-09T05:29:43.105 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.105 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:29:43.105 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.105 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.105 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.105 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35187.0:181070 11.3 11:c717a75a:::benchmark_data_smithi133_194104_object85150:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e252)", 2023-10-09T05:29:43.106 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:23:17.313078+0000", 2023-10-09T05:29:43.106 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 385.74167907200001, 2023-10-09T05:29:43.106 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.2070205620000001, 2023-10-09T05:29:43.106 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:29:43.106 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:29:43.106 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:29:43.106 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35187", 2023-10-09T05:29:43.106 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2118344251", 2023-10-09T05:29:43.106 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 181070 2023-10-09T05:29:43.106 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.106 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:29:43.106 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.107 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:29:43.107 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.313078+0000", 2023-10-09T05:29:43.107 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.107 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.107 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.107 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:29:43.107 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.313078+0000", 2023-10-09T05:29:43.107 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.107 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.107 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.107 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:29:43.107 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.313061+0000", 2023-10-09T05:29:43.107 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999838 2023-10-09T05:29:43.107 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.108 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.108 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:29:43.108 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.313085+0000", 2023-10-09T05:29:43.108 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.3198e-05 2023-10-09T05:29:43.108 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.108 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.108 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:29:43.108 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.313104+0000", 2023-10-09T05:29:43.108 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.9378999999999999e-05 2023-10-09T05:29:43.108 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.108 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.108 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:29:43.108 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.313112+0000", 2023-10-09T05:29:43.108 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.5889999999999997e-06 2023-10-09T05:29:43.109 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.109 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.109 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:29:43.109 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.313193+0000", 2023-10-09T05:29:43.109 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.1528999999999998e-05 2023-10-09T05:29:43.109 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.109 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.109 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:29:43.109 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.313207+0000", 2023-10-09T05:29:43.109 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.4344e-05 2023-10-09T05:29:43.109 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.109 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.109 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:29:43.110 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.477017+0000", 2023-10-09T05:29:43.110 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.163809348 2023-10-09T05:29:43.110 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.110 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.110 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:29:43.110 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.477138+0000", 2023-10-09T05:29:43.110 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000120826 2023-10-09T05:29:43.110 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.110 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.110 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:29:43.110 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.477265+0000", 2023-10-09T05:29:43.110 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00012727999999999999 2023-10-09T05:29:43.110 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.111 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.111 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.111 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.484071+0000", 2023-10-09T05:29:43.111 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0068062579999999999 2023-10-09T05:29:43.111 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.111 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.111 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:29:43.111 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.484076+0000", 2023-10-09T05:29:43.111 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.169e-06 2023-10-09T05:29:43.111 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.111 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.111 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.111 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.519131+0000", 2023-10-09T05:29:43.112 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.035055168999999997 2023-10-09T05:29:43.112 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.112 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.112 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:29:43.112 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.519236+0000", 2023-10-09T05:29:43.112 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00010482199999999999 2023-10-09T05:29:43.112 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.112 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.112 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:29:43.112 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.520098+0000", 2023-10-09T05:29:43.112 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00086192799999999998 2023-10-09T05:29:43.112 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.112 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:29:43.113 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.113 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.113 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.113 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35187.0:181101 11.3 11:c0a5b0b0:::benchmark_data_smithi133_194104_object85181:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e252)", 2023-10-09T05:29:43.113 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:23:17.402652+0000", 2023-10-09T05:29:43.113 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 385.65210498699997, 2023-10-09T05:29:43.113 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.155877942, 2023-10-09T05:29:43.113 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:29:43.113 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:29:43.113 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:29:43.113 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35187", 2023-10-09T05:29:43.113 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2118344251", 2023-10-09T05:29:43.114 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 181101 2023-10-09T05:29:43.114 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.114 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:29:43.114 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.114 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:29:43.114 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.402652+0000", 2023-10-09T05:29:43.114 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.114 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.114 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.114 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:29:43.114 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.402652+0000", 2023-10-09T05:29:43.114 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.115 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.115 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.115 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:29:43.115 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.402650+0000", 2023-10-09T05:29:43.115 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999981 2023-10-09T05:29:43.115 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.115 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.115 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:29:43.115 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.402659+0000", 2023-10-09T05:29:43.115 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.8079999999999994e-06 2023-10-09T05:29:43.115 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.115 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.116 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:29:43.116 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.402676+0000", 2023-10-09T05:29:43.116 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.7608000000000001e-05 2023-10-09T05:29:43.116 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.116 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.116 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:29:43.116 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.402684+0000", 2023-10-09T05:29:43.116 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.0690000000000002e-06 2023-10-09T05:29:43.116 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.116 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.116 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:29:43.116 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.402760+0000", 2023-10-09T05:29:43.116 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.5796000000000004e-05 2023-10-09T05:29:43.117 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.117 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.117 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:29:43.117 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:17.402790+0000", 2023-10-09T05:29:43.117 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.9862e-05 2023-10-09T05:29:43.117 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.117 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.117 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:29:43.117 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.480915+0000", 2023-10-09T05:29:43.117 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0781247810000001 2023-10-09T05:29:43.117 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.118 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.118 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:29:43.118 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.481041+0000", 2023-10-09T05:29:43.118 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000126451 2023-10-09T05:29:43.118 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.118 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.118 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:29:43.118 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.481143+0000", 2023-10-09T05:29:43.118 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000101702 2023-10-09T05:29:43.118 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.118 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.119 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:29:43.119 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.484295+0000", 2023-10-09T05:29:43.119 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0031516320000000001 2023-10-09T05:29:43.119 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.119 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.119 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.119 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.484849+0000", 2023-10-09T05:29:43.119 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00055466300000000005 2023-10-09T05:29:43.119 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.119 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.119 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.119 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.557289+0000", 2023-10-09T05:29:43.119 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.072439999000000005 2023-10-09T05:29:43.120 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.120 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.120 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:29:43.120 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.557368+0000", 2023-10-09T05:29:43.120 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.8825000000000005e-05 2023-10-09T05:29:43.120 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.120 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.120 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:29:43.120 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:23:18.558530+0000", 2023-10-09T05:29:43.120 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0011616549999999999 2023-10-09T05:29:43.120 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.120 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:29:43.121 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.121 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.121 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.121 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.45374.0:74845 12.5 12:ae0c8e17:::benchmark_data_smithi133_218956_object74844:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e272)", 2023-10-09T05:29:43.121 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:26:19.622295+0000", 2023-10-09T05:29:43.121 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 203.432461969, 2023-10-09T05:29:43.121 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.25575165, 2023-10-09T05:29:43.121 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:29:43.121 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:29:43.121 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:29:43.121 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.45374", 2023-10-09T05:29:43.121 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2988063385", 2023-10-09T05:29:43.121 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 74845 2023-10-09T05:29:43.122 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.122 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:29:43.122 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.122 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:29:43.122 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:26:19.622295+0000", 2023-10-09T05:29:43.122 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.122 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.122 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.122 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:29:43.122 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:26:19.622295+0000", 2023-10-09T05:29:43.122 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.123 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.123 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.123 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:29:43.123 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:26:19.622293+0000", 2023-10-09T05:29:43.123 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:29:43.123 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.123 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.123 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:29:43.123 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:26:19.622370+0000", 2023-10-09T05:29:43.123 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.7157999999999995e-05 2023-10-09T05:29:43.123 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.123 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.123 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:29:43.124 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:26:19.622383+0000", 2023-10-09T05:29:43.124 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.2281e-05 2023-10-09T05:29:43.124 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.124 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.124 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:29:43.124 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:26:19.622390+0000", 2023-10-09T05:29:43.124 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.8589999999999999e-06 2023-10-09T05:29:43.124 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.124 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.124 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:29:43.124 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:26:19.622422+0000", 2023-10-09T05:29:43.124 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.2685999999999998e-05 2023-10-09T05:29:43.125 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.125 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.125 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:29:43.125 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:26:19.622542+0000", 2023-10-09T05:29:43.125 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000120161 2023-10-09T05:29:43.125 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.125 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.125 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:29:43.125 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:26:19.622650+0000", 2023-10-09T05:29:43.125 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00010739 2023-10-09T05:29:43.125 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.125 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.125 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:29:43.126 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:26:19.626010+0000", 2023-10-09T05:29:43.126 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0033601009999999999 2023-10-09T05:29:43.126 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.126 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.126 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.126 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:26:19.626411+0000", 2023-10-09T05:29:43.126 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00040086599999999999 2023-10-09T05:29:43.126 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.126 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.126 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.126 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:26:19.877937+0000", 2023-10-09T05:29:43.127 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.25152670900000001 2023-10-09T05:29:43.127 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.127 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.127 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:29:43.127 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:26:19.878012+0000", 2023-10-09T05:29:43.127 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.4428e-05 2023-10-09T05:29:43.127 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.127 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.127 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:29:43.127 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:26:19.878046+0000", 2023-10-09T05:29:43.127 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.4545999999999998e-05 2023-10-09T05:29:43.127 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.127 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:29:43.128 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.128 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.128 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.128 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.45374.0:74867 12.5 12:a2e8e55f:::benchmark_data_smithi133_218956_object74866:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e272)", 2023-10-09T05:29:43.128 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:26:19.649389+0000", 2023-10-09T05:29:43.128 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 203.405367802, 2023-10-09T05:29:43.128 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.228784457, 2023-10-09T05:29:43.128 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:29:43.128 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:29:43.128 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:29:43.128 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.45374", 2023-10-09T05:29:43.128 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2988063385", 2023-10-09T05:29:43.128 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 74867 2023-10-09T05:29:43.129 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.129 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:29:43.129 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.129 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:29:43.129 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:26:19.649389+0000", 2023-10-09T05:29:43.129 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.129 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.129 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.129 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:29:43.129 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:26:19.649389+0000", 2023-10-09T05:29:43.129 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.129 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.130 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.130 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:29:43.130 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:26:19.649388+0000", 2023-10-09T05:29:43.130 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:29:43.130 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.130 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.130 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:29:43.130 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:26:19.649425+0000", 2023-10-09T05:29:43.130 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.7175000000000002e-05 2023-10-09T05:29:43.130 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.130 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.130 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:29:43.130 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:26:19.649442+0000", 2023-10-09T05:29:43.131 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.7135000000000001e-05 2023-10-09T05:29:43.131 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.131 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.131 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:29:43.131 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:26:19.649449+0000", 2023-10-09T05:29:43.131 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.6810000000000001e-06 2023-10-09T05:29:43.131 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.131 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.131 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:29:43.131 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:26:19.649521+0000", 2023-10-09T05:29:43.131 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.2538000000000002e-05 2023-10-09T05:29:43.131 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.131 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.132 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:29:43.132 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:26:19.649738+0000", 2023-10-09T05:29:43.132 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00021621100000000001 2023-10-09T05:29:43.132 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.132 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.132 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:29:43.132 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:26:19.649941+0000", 2023-10-09T05:29:43.132 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00020353799999999999 2023-10-09T05:29:43.132 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.132 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.132 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:29:43.132 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:26:19.654551+0000", 2023-10-09T05:29:43.132 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0046099950000000004 2023-10-09T05:29:43.133 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.133 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.133 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.133 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:26:19.657631+0000", 2023-10-09T05:29:43.133 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0030802640000000001 2023-10-09T05:29:43.133 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.133 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.133 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.133 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:26:19.878084+0000", 2023-10-09T05:29:43.133 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.220452749 2023-10-09T05:29:43.133 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.133 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.133 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:29:43.134 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:26:19.878133+0000", 2023-10-09T05:29:43.134 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.9258000000000002e-05 2023-10-09T05:29:43.134 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.134 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.134 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:29:43.134 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:26:19.878173+0000", 2023-10-09T05:29:43.134 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.9910999999999998e-05 2023-10-09T05:29:43.134 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.134 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:29:43.134 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.134 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.135 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.135 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.45374.0:128791 12.5 12:a7df30dd:::benchmark_data_smithi133_218956_object26049:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-10-09T05:29:43.135 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:27:29.043781+0000", 2023-10-09T05:29:43.135 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 134.010976048, 2023-10-09T05:29:43.135 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.27514119599999998, 2023-10-09T05:29:43.135 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:29:43.135 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:29:43.135 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:29:43.135 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.45374", 2023-10-09T05:29:43.135 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2988063385", 2023-10-09T05:29:43.135 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 128791 2023-10-09T05:29:43.135 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.135 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:29:43.136 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.136 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:29:43.136 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.043781+0000", 2023-10-09T05:29:43.136 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.136 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.136 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.136 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:29:43.136 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.043781+0000", 2023-10-09T05:29:43.136 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.136 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.136 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.136 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:29:43.137 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.043780+0000", 2023-10-09T05:29:43.137 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999995 2023-10-09T05:29:43.137 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.137 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.137 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:29:43.137 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.043783+0000", 2023-10-09T05:29:43.137 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.3020000000000002e-06 2023-10-09T05:29:43.137 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.137 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.137 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:29:43.137 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.043788+0000", 2023-10-09T05:29:43.137 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.259e-06 2023-10-09T05:29:43.137 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.138 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.138 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:29:43.138 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.043791+0000", 2023-10-09T05:29:43.138 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.9040000000000001e-06 2023-10-09T05:29:43.138 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.138 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.138 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:29:43.138 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.043852+0000", 2023-10-09T05:29:43.138 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.1141000000000003e-05 2023-10-09T05:29:43.138 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.138 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.138 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:29:43.139 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.044013+0000", 2023-10-09T05:29:43.139 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00016104300000000001 2023-10-09T05:29:43.139 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.139 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.139 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:29:43.139 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.044159+0000", 2023-10-09T05:29:43.139 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00014595799999999999 2023-10-09T05:29:43.139 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.139 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.139 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:29:43.139 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.052090+0000", 2023-10-09T05:29:43.139 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0079310939999999996 2023-10-09T05:29:43.139 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.140 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.140 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.140 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.125773+0000", 2023-10-09T05:29:43.140 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.073683070000000003 2023-10-09T05:29:43.140 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.140 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.140 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.140 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.318761+0000", 2023-10-09T05:29:43.140 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.192988401 2023-10-09T05:29:43.140 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.140 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.140 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:29:43.140 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.318851+0000", 2023-10-09T05:29:43.141 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.9759000000000006e-05 2023-10-09T05:29:43.141 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.141 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.141 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:29:43.141 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.318922+0000", 2023-10-09T05:29:43.141 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.0636000000000006e-05 2023-10-09T05:29:43.141 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.141 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:29:43.141 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.141 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.141 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.141 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.45374.0:128794 12.5 12:a35c773e:::benchmark_data_smithi133_218956_object26052:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-10-09T05:29:43.141 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:27:29.043979+0000", 2023-10-09T05:29:43.142 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 134.01077788200001, 2023-10-09T05:29:43.142 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.29330039699999999, 2023-10-09T05:29:43.142 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:29:43.142 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:29:43.142 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:29:43.142 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.45374", 2023-10-09T05:29:43.142 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2988063385", 2023-10-09T05:29:43.142 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 128794 2023-10-09T05:29:43.142 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.142 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:29:43.142 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.142 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:29:43.142 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.043979+0000", 2023-10-09T05:29:43.143 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.143 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.143 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.143 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:29:43.143 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.043979+0000", 2023-10-09T05:29:43.143 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.143 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.143 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.143 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:29:43.143 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.043978+0000", 2023-10-09T05:29:43.143 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:29:43.143 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.143 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.144 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:29:43.144 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.043989+0000", 2023-10-09T05:29:43.144 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.1411e-05 2023-10-09T05:29:43.144 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.144 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.144 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:29:43.144 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.043998+0000", 2023-10-09T05:29:43.144 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.5029999999999997e-06 2023-10-09T05:29:43.144 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.144 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.144 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:29:43.144 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.044005+0000", 2023-10-09T05:29:43.144 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.1679999999999998e-06 2023-10-09T05:29:43.145 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.145 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.145 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:29:43.145 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.048052+0000", 2023-10-09T05:29:43.145 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0040472240000000003 2023-10-09T05:29:43.145 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.145 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.145 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:29:43.145 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.048202+0000", 2023-10-09T05:29:43.145 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00015012399999999999 2023-10-09T05:29:43.145 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.145 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.145 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:29:43.145 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.048346+0000", 2023-10-09T05:29:43.146 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000143378 2023-10-09T05:29:43.146 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.146 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.146 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:29:43.146 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.052368+0000", 2023-10-09T05:29:43.146 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0040220009999999999 2023-10-09T05:29:43.146 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.146 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.146 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.146 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.126200+0000", 2023-10-09T05:29:43.146 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.073831820000000006 2023-10-09T05:29:43.146 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.147 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.147 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.147 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.337177+0000", 2023-10-09T05:29:43.147 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.21097736 2023-10-09T05:29:43.147 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.147 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.147 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:29:43.147 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.337235+0000", 2023-10-09T05:29:43.147 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.8139e-05 2023-10-09T05:29:43.147 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.147 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.147 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:29:43.147 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.337279+0000", 2023-10-09T05:29:43.147 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.4132999999999999e-05 2023-10-09T05:29:43.148 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.148 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:29:43.148 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.148 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.148 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.148 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.45374.0:128797 12.4 12:281bf68d:::benchmark_data_smithi133_218956_object26055:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-10-09T05:29:43.148 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:27:29.048154+0000", 2023-10-09T05:29:43.148 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 134.00660245500001, 2023-10-09T05:29:43.148 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.26531340399999997, 2023-10-09T05:29:43.148 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:29:43.148 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:29:43.148 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:29:43.148 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.45374", 2023-10-09T05:29:43.149 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2988063385", 2023-10-09T05:29:43.149 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 128797 2023-10-09T05:29:43.149 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.149 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:29:43.149 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.149 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:29:43.149 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.048154+0000", 2023-10-09T05:29:43.149 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.149 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.149 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.149 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:29:43.149 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.048154+0000", 2023-10-09T05:29:43.150 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.150 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.150 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.150 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:29:43.150 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.048153+0000", 2023-10-09T05:29:43.150 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:29:43.150 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.150 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.150 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:29:43.150 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.048159+0000", 2023-10-09T05:29:43.150 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.2280000000000003e-06 2023-10-09T05:29:43.150 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.151 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.151 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:29:43.151 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.048171+0000", 2023-10-09T05:29:43.151 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.1803e-05 2023-10-09T05:29:43.151 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.151 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.151 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:29:43.151 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.048177+0000", 2023-10-09T05:29:43.151 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.1890000000000004e-06 2023-10-09T05:29:43.151 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.151 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.151 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:29:43.151 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.048222+0000", 2023-10-09T05:29:43.152 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.4536000000000002e-05 2023-10-09T05:29:43.152 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.152 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.152 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:29:43.152 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.048331+0000", 2023-10-09T05:29:43.152 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000108856 2023-10-09T05:29:43.152 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.152 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.152 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:29:43.152 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.048436+0000", 2023-10-09T05:29:43.152 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000105123 2023-10-09T05:29:43.152 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.152 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.152 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.153 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.052804+0000", 2023-10-09T05:29:43.153 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0043685779999999997 2023-10-09T05:29:43.153 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.153 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.153 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:29:43.153 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.052857+0000", 2023-10-09T05:29:43.153 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.3015e-05 2023-10-09T05:29:43.153 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.153 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.153 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.153 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.313343+0000", 2023-10-09T05:29:43.153 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.26048575699999998 2023-10-09T05:29:43.153 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.154 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.154 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:29:43.154 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.313412+0000", 2023-10-09T05:29:43.154 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.9036999999999996e-05 2023-10-09T05:29:43.154 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.154 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.154 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:29:43.154 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.313468+0000", 2023-10-09T05:29:43.154 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.5597000000000003e-05 2023-10-09T05:29:43.154 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.154 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:29:43.155 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.155 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.155 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.155 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.45374.0:128801 12.5 12:a7f0e3aa:::benchmark_data_smithi133_218956_object26059:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-10-09T05:29:43.155 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:27:29.126285+0000", 2023-10-09T05:29:43.155 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 133.928472148, 2023-10-09T05:29:43.155 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.211118363, 2023-10-09T05:29:43.155 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:29:43.155 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:29:43.155 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:29:43.155 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.45374", 2023-10-09T05:29:43.155 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2988063385", 2023-10-09T05:29:43.155 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 128801 2023-10-09T05:29:43.156 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.156 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:29:43.156 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.156 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:29:43.156 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.126285+0000", 2023-10-09T05:29:43.156 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.156 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.156 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.156 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:29:43.156 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.126285+0000", 2023-10-09T05:29:43.156 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.156 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.157 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.157 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:29:43.157 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.126284+0000", 2023-10-09T05:29:43.157 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999995 2023-10-09T05:29:43.157 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.157 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.157 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:29:43.157 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.126286+0000", 2023-10-09T05:29:43.157 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.103e-06 2023-10-09T05:29:43.157 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.157 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.157 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:29:43.158 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.126290+0000", 2023-10-09T05:29:43.158 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.8240000000000001e-06 2023-10-09T05:29:43.158 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.158 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.158 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:29:43.158 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.126293+0000", 2023-10-09T05:29:43.158 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.5289999999999998e-06 2023-10-09T05:29:43.158 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.158 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.158 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:29:43.158 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.126400+0000", 2023-10-09T05:29:43.159 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000107537 2023-10-09T05:29:43.159 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.159 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.159 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:29:43.159 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.126695+0000", 2023-10-09T05:29:43.159 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000294875 2023-10-09T05:29:43.159 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.159 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.159 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:29:43.159 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.126977+0000", 2023-10-09T05:29:43.159 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00028134999999999998 2023-10-09T05:29:43.159 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.159 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.160 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:29:43.160 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.140889+0000", 2023-10-09T05:29:43.160 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.013912493999999999 2023-10-09T05:29:43.160 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.160 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.160 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.160 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.145041+0000", 2023-10-09T05:29:43.160 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0041521839999999997 2023-10-09T05:29:43.160 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.160 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.160 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.160 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.337320+0000", 2023-10-09T05:29:43.161 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.19227865399999999 2023-10-09T05:29:43.161 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.161 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.161 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:29:43.161 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.337363+0000", 2023-10-09T05:29:43.161 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.3448999999999997e-05 2023-10-09T05:29:43.161 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.161 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.161 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:29:43.161 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.337403+0000", 2023-10-09T05:29:43.161 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.9650000000000002e-05 2023-10-09T05:29:43.161 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.162 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:29:43.162 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.162 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.162 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.162 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.45374.0:128802 12.5 12:af1ef82a:::benchmark_data_smithi133_218956_object26060:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-10-09T05:29:43.162 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:27:29.126317+0000", 2023-10-09T05:29:43.162 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 133.92843966699999, 2023-10-09T05:29:43.162 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.21120854, 2023-10-09T05:29:43.162 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:29:43.162 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:29:43.162 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:29:43.162 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.45374", 2023-10-09T05:29:43.163 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2988063385", 2023-10-09T05:29:43.163 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 128802 2023-10-09T05:29:43.163 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.163 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:29:43.163 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.163 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:29:43.163 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.126317+0000", 2023-10-09T05:29:43.163 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.163 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.163 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.163 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:29:43.163 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.126317+0000", 2023-10-09T05:29:43.164 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.164 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.164 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.164 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:29:43.164 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.126316+0000", 2023-10-09T05:29:43.164 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:29:43.164 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.164 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.164 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:29:43.164 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.126320+0000", 2023-10-09T05:29:43.165 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.76e-06 2023-10-09T05:29:43.165 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.165 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.165 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:29:43.165 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.126328+0000", 2023-10-09T05:29:43.165 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.9959999999999993e-06 2023-10-09T05:29:43.165 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.165 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.165 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:29:43.165 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.126333+0000", 2023-10-09T05:29:43.165 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.3310000000000003e-06 2023-10-09T05:29:43.165 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.166 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.166 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:29:43.166 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.132512+0000", 2023-10-09T05:29:43.166 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0061789219999999999 2023-10-09T05:29:43.166 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.166 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.166 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:29:43.166 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.132677+0000", 2023-10-09T05:29:43.166 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000165069 2023-10-09T05:29:43.166 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.166 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.167 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:29:43.167 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.132854+0000", 2023-10-09T05:29:43.167 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00017674300000000001 2023-10-09T05:29:43.167 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.167 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.167 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:29:43.167 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.144998+0000", 2023-10-09T05:29:43.167 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.012144339000000001 2023-10-09T05:29:43.167 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.167 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.167 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.167 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.145069+0000", 2023-10-09T05:29:43.168 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.0196000000000004e-05 2023-10-09T05:29:43.168 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.168 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.168 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.168 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.337439+0000", 2023-10-09T05:29:43.168 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.19237079800000001 2023-10-09T05:29:43.168 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.168 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.168 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:29:43.168 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.337490+0000", 2023-10-09T05:29:43.168 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.0374999999999998e-05 2023-10-09T05:29:43.168 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.169 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.169 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:29:43.169 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.337526+0000", 2023-10-09T05:29:43.169 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.5809000000000002e-05 2023-10-09T05:29:43.169 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.169 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:29:43.169 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.169 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.169 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.169 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.45374.0:128810 12.5 12:ab14e938:::benchmark_data_smithi133_218956_object26068:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-10-09T05:29:43.169 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:27:29.134986+0000", 2023-10-09T05:29:43.169 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 133.91977095999999, 2023-10-09T05:29:43.169 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.20264454900000001, 2023-10-09T05:29:43.170 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:29:43.170 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:29:43.170 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:29:43.170 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.45374", 2023-10-09T05:29:43.170 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2988063385", 2023-10-09T05:29:43.170 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 128810 2023-10-09T05:29:43.170 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.170 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:29:43.170 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.170 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:29:43.170 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.134986+0000", 2023-10-09T05:29:43.170 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.171 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.171 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.171 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:29:43.171 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.134986+0000", 2023-10-09T05:29:43.171 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.171 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.171 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.171 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:29:43.171 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.134984+0000", 2023-10-09T05:29:43.171 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:29:43.171 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.171 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.171 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:29:43.172 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.134993+0000", 2023-10-09T05:29:43.172 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.0070000000000003e-06 2023-10-09T05:29:43.172 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.172 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.172 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:29:43.172 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.135012+0000", 2023-10-09T05:29:43.172 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.8604e-05 2023-10-09T05:29:43.172 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.172 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.172 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:29:43.172 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.135023+0000", 2023-10-09T05:29:43.172 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.1028e-05 2023-10-09T05:29:43.173 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.173 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.173 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:29:43.173 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.140919+0000", 2023-10-09T05:29:43.173 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0058963369999999998 2023-10-09T05:29:43.173 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.173 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.173 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:29:43.173 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.141044+0000", 2023-10-09T05:29:43.173 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00012440199999999999 2023-10-09T05:29:43.173 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.173 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.173 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:29:43.174 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.141161+0000", 2023-10-09T05:29:43.174 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000117302 2023-10-09T05:29:43.174 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.174 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.174 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:29:43.174 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.145134+0000", 2023-10-09T05:29:43.174 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0039730629999999998 2023-10-09T05:29:43.174 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.174 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.174 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.174 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.146961+0000", 2023-10-09T05:29:43.174 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.001827424 2023-10-09T05:29:43.175 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.175 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.175 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.175 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.337559+0000", 2023-10-09T05:29:43.175 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.19059736499999999 2023-10-09T05:29:43.175 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.175 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.175 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:29:43.175 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.337597+0000", 2023-10-09T05:29:43.175 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.8541999999999998e-05 2023-10-09T05:29:43.175 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.175 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.175 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:29:43.176 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.337630+0000", 2023-10-09T05:29:43.176 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.3030000000000001e-05 2023-10-09T05:29:43.176 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.176 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:29:43.176 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.176 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.176 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.176 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.45374.0:193546 12.1 12:8eddfeba:::benchmark_data_smithi133_218956_object90804:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-10-09T05:29:43.176 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:29:04.373328+0000", 2023-10-09T05:29:43.176 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 38.681428979000003, 2023-10-09T05:29:43.176 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.200572469, 2023-10-09T05:29:43.176 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:29:43.177 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:29:43.177 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:29:43.177 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.45374", 2023-10-09T05:29:43.177 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2988063385", 2023-10-09T05:29:43.177 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 193546 2023-10-09T05:29:43.177 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.177 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:29:43.177 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.177 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:29:43.177 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:04.373328+0000", 2023-10-09T05:29:43.177 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.177 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.178 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.178 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:29:43.178 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:04.373328+0000", 2023-10-09T05:29:43.178 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.178 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.178 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.178 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:29:43.178 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:04.373325+0000", 2023-10-09T05:29:43.178 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999976 2023-10-09T05:29:43.178 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.178 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.179 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:29:43.179 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:04.373338+0000", 2023-10-09T05:29:43.179 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.2644e-05 2023-10-09T05:29:43.179 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.179 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.179 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:29:43.179 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:04.373364+0000", 2023-10-09T05:29:43.179 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.5948999999999998e-05 2023-10-09T05:29:43.179 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.179 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.179 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:29:43.179 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:04.373378+0000", 2023-10-09T05:29:43.179 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.4632e-05 2023-10-09T05:29:43.180 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.180 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.180 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:29:43.180 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:04.373480+0000", 2023-10-09T05:29:43.180 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00010121800000000001 2023-10-09T05:29:43.180 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.180 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.180 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:29:43.180 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:04.373717+0000", 2023-10-09T05:29:43.180 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000237364 2023-10-09T05:29:43.180 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.180 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.181 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:29:43.181 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:04.373949+0000", 2023-10-09T05:29:43.181 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00023193500000000001 2023-10-09T05:29:43.181 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.181 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.181 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:29:43.181 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:04.382600+0000", 2023-10-09T05:29:43.181 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0086509340000000007 2023-10-09T05:29:43.181 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.181 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.181 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.181 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:04.387129+0000", 2023-10-09T05:29:43.182 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0045288960000000001 2023-10-09T05:29:43.182 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.182 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.182 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.182 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:04.573779+0000", 2023-10-09T05:29:43.182 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.18665042800000001 2023-10-09T05:29:43.182 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.182 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.182 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:29:43.182 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:04.573848+0000", 2023-10-09T05:29:43.182 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.8817000000000002e-05 2023-10-09T05:29:43.182 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.182 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.183 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:29:43.183 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:04.573900+0000", 2023-10-09T05:29:43.183 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.2212000000000001e-05 2023-10-09T05:29:43.183 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.183 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:29:43.183 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.183 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.183 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.183 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.45374.0:196095 12.1 12:824034a0:::benchmark_data_smithi133_218956_object93353:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-10-09T05:29:43.183 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:29:08.100080+0000", 2023-10-09T05:29:43.183 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 34.954676609000003, 2023-10-09T05:29:43.183 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.20820061400000001, 2023-10-09T05:29:43.184 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:29:43.184 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:29:43.184 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:29:43.184 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.45374", 2023-10-09T05:29:43.184 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2988063385", 2023-10-09T05:29:43.184 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 196095 2023-10-09T05:29:43.184 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.184 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:29:43.184 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.184 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:29:43.184 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.100080+0000", 2023-10-09T05:29:43.184 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.185 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.185 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.185 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:29:43.185 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.100080+0000", 2023-10-09T05:29:43.185 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.185 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.185 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.185 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:29:43.185 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.100080+0000", 2023-10-09T05:29:43.185 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999995 2023-10-09T05:29:43.185 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.185 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.185 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:29:43.186 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.100082+0000", 2023-10-09T05:29:43.186 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.4499999999999998e-06 2023-10-09T05:29:43.186 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.186 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.186 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:29:43.186 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.100088+0000", 2023-10-09T05:29:43.186 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.626e-06 2023-10-09T05:29:43.186 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.186 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.186 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:29:43.186 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.100092+0000", 2023-10-09T05:29:43.186 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.8120000000000001e-06 2023-10-09T05:29:43.187 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.187 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.187 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:29:43.187 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.100155+0000", 2023-10-09T05:29:43.187 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.3717e-05 2023-10-09T05:29:43.187 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.187 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.187 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:29:43.187 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.100281+0000", 2023-10-09T05:29:43.187 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00012551700000000001 2023-10-09T05:29:43.187 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.187 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.187 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:29:43.188 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.100378+0000", 2023-10-09T05:29:43.188 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.6868999999999997e-05 2023-10-09T05:29:43.188 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.188 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.188 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:29:43.188 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.105684+0000", 2023-10-09T05:29:43.188 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0053067319999999998 2023-10-09T05:29:43.188 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.188 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.188 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.188 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.105893+0000", 2023-10-09T05:29:43.188 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00020829900000000001 2023-10-09T05:29:43.188 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.189 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.189 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.189 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.308175+0000", 2023-10-09T05:29:43.189 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.20228254000000001 2023-10-09T05:29:43.189 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.189 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.189 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:29:43.189 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.308233+0000", 2023-10-09T05:29:43.189 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.8152999999999997e-05 2023-10-09T05:29:43.189 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.189 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.189 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:29:43.190 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.308281+0000", 2023-10-09T05:29:43.190 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.7379999999999997e-05 2023-10-09T05:29:43.190 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.190 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:29:43.190 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.190 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.190 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.190 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.45374.0:196097 12.4 12:2d825edd:::benchmark_data_smithi133_218956_object93355:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-10-09T05:29:43.190 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:29:08.100101+0000", 2023-10-09T05:29:43.190 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 34.954655742, 2023-10-09T05:29:43.190 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.20273697500000001, 2023-10-09T05:29:43.190 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:29:43.191 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:29:43.191 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:29:43.191 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.45374", 2023-10-09T05:29:43.191 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2988063385", 2023-10-09T05:29:43.191 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 196097 2023-10-09T05:29:43.191 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.191 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:29:43.191 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.191 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:29:43.191 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.100101+0000", 2023-10-09T05:29:43.191 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.191 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.191 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.192 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:29:43.192 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.100101+0000", 2023-10-09T05:29:43.192 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.192 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.192 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.192 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:29:43.192 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.100101+0000", 2023-10-09T05:29:43.192 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999995 2023-10-09T05:29:43.192 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.192 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.192 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:29:43.192 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.100103+0000", 2023-10-09T05:29:43.193 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.2029999999999999e-06 2023-10-09T05:29:43.193 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.193 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.193 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:29:43.193 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.100109+0000", 2023-10-09T05:29:43.193 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.3289999999999999e-06 2023-10-09T05:29:43.193 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.193 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.193 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:29:43.193 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.100113+0000", 2023-10-09T05:29:43.193 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.5319999999999998e-06 2023-10-09T05:29:43.193 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.193 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.194 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:29:43.194 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.100151+0000", 2023-10-09T05:29:43.194 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.7882000000000001e-05 2023-10-09T05:29:43.194 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.194 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.194 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:29:43.194 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.100311+0000", 2023-10-09T05:29:43.194 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00015994100000000001 2023-10-09T05:29:43.194 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.194 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.194 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:29:43.194 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.100458+0000", 2023-10-09T05:29:43.195 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00014742999999999999 2023-10-09T05:29:43.195 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.195 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.195 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:29:43.195 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.104601+0000", 2023-10-09T05:29:43.195 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0041426989999999997 2023-10-09T05:29:43.195 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.195 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.195 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.195 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.106673+0000", 2023-10-09T05:29:43.195 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0020722219999999999 2023-10-09T05:29:43.195 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.195 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.196 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.196 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.302566+0000", 2023-10-09T05:29:43.196 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.195893118 2023-10-09T05:29:43.196 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.196 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.196 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:29:43.196 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.302720+0000", 2023-10-09T05:29:43.196 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00015429199999999999 2023-10-09T05:29:43.196 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.196 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.196 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:29:43.196 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.302838+0000", 2023-10-09T05:29:43.197 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000117683 2023-10-09T05:29:43.197 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.197 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:29:43.197 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.197 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.197 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.197 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.45374.0:196115 12.1 12:8e847496:::benchmark_data_smithi133_218956_object93373:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-10-09T05:29:43.197 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:29:08.127731+0000", 2023-10-09T05:29:43.197 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 34.927025741000001, 2023-10-09T05:29:43.197 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.19577218800000001, 2023-10-09T05:29:43.197 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:29:43.197 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:29:43.197 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:29:43.198 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.45374", 2023-10-09T05:29:43.198 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2988063385", 2023-10-09T05:29:43.198 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 196115 2023-10-09T05:29:43.198 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.198 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:29:43.198 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.198 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:29:43.198 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.127731+0000", 2023-10-09T05:29:43.198 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.198 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.198 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.199 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:29:43.199 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.127731+0000", 2023-10-09T05:29:43.199 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:29:43.199 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.199 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.199 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:29:43.199 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.127729+0000", 2023-10-09T05:29:43.199 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:29:43.199 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.199 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.199 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:29:43.199 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.127737+0000", 2023-10-09T05:29:43.200 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.4410000000000004e-06 2023-10-09T05:29:43.200 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.200 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.200 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:29:43.200 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.127751+0000", 2023-10-09T05:29:43.200 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.3672999999999999e-05 2023-10-09T05:29:43.200 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.200 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.200 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:29:43.200 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.127757+0000", 2023-10-09T05:29:43.200 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.6599999999999998e-06 2023-10-09T05:29:43.200 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.200 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.201 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:29:43.201 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.127794+0000", 2023-10-09T05:29:43.201 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.6535999999999997e-05 2023-10-09T05:29:43.201 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.201 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.201 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:29:43.201 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.127911+0000", 2023-10-09T05:29:43.201 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000117713 2023-10-09T05:29:43.201 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.201 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.201 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:29:43.201 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.127998+0000", 2023-10-09T05:29:43.201 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.7064999999999994e-05 2023-10-09T05:29:43.202 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.202 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.202 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:29:43.202 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.131807+0000", 2023-10-09T05:29:43.202 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.003808573 2023-10-09T05:29:43.202 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.202 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.202 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.202 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.133657+0000", 2023-10-09T05:29:43.202 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0018498950000000001 2023-10-09T05:29:43.202 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.202 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.203 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:29:43.203 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.323394+0000", 2023-10-09T05:29:43.203 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.18973753300000001 2023-10-09T05:29:43.203 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.203 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.203 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:29:43.203 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.323455+0000", 2023-10-09T05:29:43.203 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.0270999999999999e-05 2023-10-09T05:29:43.203 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:29:43.203 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:29:43.203 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:29:43.203 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:29:08.323503+0000", 2023-10-09T05:29:43.203 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.8432e-05 2023-10-09T05:29:43.204 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.204 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:29:43.204 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.204 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:29:43.204 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:29:43.204 INFO:teuthology.orchestra.run.smithi194.stdout:} 2023-10-09T05:29:43.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:43.205 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:43.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:43.306 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:43.339 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:43.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:43.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:43.442 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:43.469 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:43.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:43.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:43.572 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:43.606 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:43.608 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:43 smithi133 bash[105589]: cluster 2023-10-09T05:29:42.245835+0000 mgr.y (mgr.25277) 1247 : cluster 0 pgmap v1329: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:43.608 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:43 smithi133 bash[102858]: cluster 2023-10-09T05:29:42.245835+0000 mgr.y (mgr.25277) 1247 : cluster 0 pgmap v1329: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:43.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:43.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:43.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:43 smithi194 bash[92666]: cluster 2023-10-09T05:29:42.245835+0000 mgr.y (mgr.25277) 1247 : cluster 0 pgmap v1329: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:43.710 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:43.734 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:43.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:43.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:43.836 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:43.864 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:43.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:43.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:43.967 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:43.995 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:43.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:43.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:44.099 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:44.108 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T05:29:44.126 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:44.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:44.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:44.230 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:44.261 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:44.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:44.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:44.365 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:44.376 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:44 smithi133 bash[105589]: audit 2023-10-09T05:29:43.480187+0000 mon.a (mon.0) 1588 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:29:44.377 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:29:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:29:44] "GET /metrics HTTP/1.1" 200 36425 "" "Prometheus/2.43.0" 2023-10-09T05:29:44.377 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:44 smithi133 bash[102858]: audit 2023-10-09T05:29:43.480187+0000 mon.a (mon.0) 1588 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:29:44.394 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:44.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:44.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:44.497 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:44.529 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:44.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:44.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:44.632 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:44.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:44 smithi194 bash[92666]: audit 2023-10-09T05:29:43.480187+0000 mon.a (mon.0) 1588 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:29:44.662 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:44.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:44.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:44.768 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:44.800 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:44.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:44.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:44.904 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:44.937 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:44.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:44.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:45.041 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:45.074 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:45.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:45.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:45.177 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:45.203 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:45.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:45.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:45.307 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:45.336 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:45.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:45.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:45.440 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:45.452 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:45 smithi133 bash[102858]: cluster 2023-10-09T05:29:44.246332+0000 mgr.y (mgr.25277) 1248 : cluster 0 pgmap v1330: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:45.452 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:45 smithi133 bash[105589]: cluster 2023-10-09T05:29:44.246332+0000 mgr.y (mgr.25277) 1248 : cluster 0 pgmap v1330: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:45.466 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:45.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:45.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:45.570 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:45.605 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:45.606 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:45 smithi194 bash[92666]: cluster 2023-10-09T05:29:44.246332+0000 mgr.y (mgr.25277) 1248 : cluster 0 pgmap v1330: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:45.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:45.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:45.708 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:45.736 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:45.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:45.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:45.841 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:45.866 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:45.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:45.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:45.969 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:46.002 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:46.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:46.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:46.105 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:46.131 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:46.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:46.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:46.234 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:46.262 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:46.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:46.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:46.365 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:46.386 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:46.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:46.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:46.490 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:46.511 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:46.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:46.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:46.614 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:46.642 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:46.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:46.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:46.746 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:46.781 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:46.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:46.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:46.885 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:46.918 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:46.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:46.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:47.021 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:47.049 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:47.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:47.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:47.153 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:47.185 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:47.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:47.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:47.289 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:47.320 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:47.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:47.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:47.393 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 6, 1, 7] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:29:47.393 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:29:47.394 INFO:tasks.thrashosds.thrasher:Removing osd 4, in_osds are: [0, 2, 3, 4, 6, 1, 7] 2023-10-09T05:29:47.394 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd out 4 2023-10-09T05:29:47.424 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:47.428 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:29:47.438 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:47 smithi194 bash[92666]: cluster 2023-10-09T05:29:46.247001+0000 mgr.y (mgr.25277) 1249 : cluster 0 pgmap v1331: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:47.458 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:47.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:47.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:47.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:47 smithi133 bash[102858]: cluster 2023-10-09T05:29:46.247001+0000 mgr.y (mgr.25277) 1249 : cluster 0 pgmap v1331: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:47.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:47 smithi133 bash[105589]: cluster 2023-10-09T05:29:46.247001+0000 mgr.y (mgr.25277) 1249 : cluster 0 pgmap v1331: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:47.561 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:47.592 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:47.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:47.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:47.696 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:47.728 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:47.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:47.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:47.832 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:47.862 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:47.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:47.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:47.966 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:47.998 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:47.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:48.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:48.101 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:48.133 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:48.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:48.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:48.237 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:48.267 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:48.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:48.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:48.370 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:48.396 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:48.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:48.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:48.500 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:48.532 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:48.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:48.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:48.636 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:48.664 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:48.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:48.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:48.768 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:48.800 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:48.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:48.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:48.888 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:29:48.903 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:48.933 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:48.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:48.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:49.038 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:49.067 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:49.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:49.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:49.171 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:49.197 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:49.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:49.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:49.300 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:49.326 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:49.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:49.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:49.429 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:49.439 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:49 smithi194 bash[92666]: cluster 2023-10-09T05:29:48.247624+0000 mgr.y (mgr.25277) 1250 : cluster 0 pgmap v1332: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:49.458 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:49.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:49.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:49.562 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:49.589 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:49.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:49.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:49.650 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:29:49.674 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:29:49.694 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:49.699 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:29:49.722 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:49.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:49.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:49.726 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:29:49.751 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:29:49.775 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:29:49.797 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:49 smithi133 bash[102858]: cluster 2023-10-09T05:29:48.247624+0000 mgr.y (mgr.25277) 1250 : cluster 0 pgmap v1332: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:49.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:49 smithi133 bash[105589]: cluster 2023-10-09T05:29:48.247624+0000 mgr.y (mgr.25277) 1250 : cluster 0 pgmap v1332: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:49.804 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:29:49.825 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:49.826 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:29:49.827 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:29:49.827 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:29:49.827 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:29:49.827 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:29:49.828 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:29:49.828 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:29:49.828 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:29:49.828 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:29:49.851 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:49.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:49.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:49.954 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:49.976 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:49.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:49.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:50.079 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:50.102 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:50.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:50.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:50.206 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:50.234 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:50.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:50.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:50.338 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:50.367 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:50.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:50.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:50.470 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:50.478 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:50 smithi133 bash[102858]: audit 2023-10-09T05:29:49.623638+0000 mon.c (mon.2) 114 : audit 0 from='client.? 172.21.15.133:0/1067917702' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:29:50.478 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:50 smithi133 bash[105589]: audit 2023-10-09T05:29:49.623638+0000 mon.c (mon.2) 114 : audit 0 from='client.? 172.21.15.133:0/1067917702' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:29:50.496 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:50.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:50.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:50.599 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:50.629 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:50.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:50.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:50.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:50 smithi194 bash[92666]: audit 2023-10-09T05:29:49.623638+0000 mon.c (mon.2) 114 : audit 0 from='client.? 172.21.15.133:0/1067917702' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:29:50.732 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:50.758 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:50.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:50.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:50.861 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:50.887 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:50.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:50.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:50.989 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:51.017 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:51.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:51.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:51.121 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:51.151 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:51.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:51.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:51.255 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:51.285 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:51.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:51.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:51.382 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is unset 2023-10-09T05:29:51.388 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:51.415 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:51.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:51.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:51.425 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:51 smithi133 bash[105589]: cluster 2023-10-09T05:29:50.248393+0000 mgr.y (mgr.25277) 1251 : cluster 0 pgmap v1333: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:51.425 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:51 smithi133 bash[105589]: audit 2023-10-09T05:29:50.390090+0000 mon.a (mon.0) 1589 : audit 1 from='client.? 172.21.15.133:0/494950949' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:29:51.426 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:51 smithi133 bash[102858]: cluster 2023-10-09T05:29:50.248393+0000 mgr.y (mgr.25277) 1251 : cluster 0 pgmap v1333: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:51.426 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:51 smithi133 bash[102858]: audit 2023-10-09T05:29:50.390090+0000 mon.a (mon.0) 1589 : audit 1 from='client.? 172.21.15.133:0/494950949' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:29:51.518 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:51.539 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:51.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:51.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:51.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:51 smithi194 bash[92666]: cluster 2023-10-09T05:29:50.248393+0000 mgr.y (mgr.25277) 1251 : cluster 0 pgmap v1333: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:51.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:51 smithi194 bash[92666]: audit 2023-10-09T05:29:50.390090+0000 mon.a (mon.0) 1589 : audit 1 from='client.? 172.21.15.133:0/494950949' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:29:51.643 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:51.666 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:51.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:51.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:51.769 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:51.797 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:51.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:51.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:51.900 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:51.928 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:51.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:51.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:52.031 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:52.054 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:52.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:52.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:52.157 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:52.187 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:52.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:52.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:52.290 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:52.312 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:52.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:52.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:52.415 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:52.447 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:52.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:52.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:52.550 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:52.580 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:52.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:52.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:52.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:52 smithi194 bash[92666]: cluster 2023-10-09T05:29:51.363472+0000 mon.a (mon.0) 1590 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:29:52.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:52 smithi194 bash[92666]: cluster 2023-10-09T05:29:51.373937+0000 osd.4 (osd.4) 255 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T05:29:52.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:52 smithi194 bash[92666]: cluster 2023-10-09T05:29:51.374783+0000 osd.5 (osd.5) 262 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T05:29:52.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:52 smithi194 bash[92666]: audit 2023-10-09T05:29:51.375661+0000 mon.a (mon.0) 1591 : audit 1 from='client.? 172.21.15.133:0/494950949' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:29:52.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:52 smithi194 bash[92666]: cluster 2023-10-09T05:29:51.375749+0000 mon.a (mon.0) 1592 : cluster 0 osdmap e285: 8 total, 8 up, 7 in 2023-10-09T05:29:52.684 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:52.712 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:52.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:52.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:52.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:52 smithi133 bash[102858]: cluster 2023-10-09T05:29:51.363472+0000 mon.a (mon.0) 1590 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:29:52.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:52 smithi133 bash[102858]: cluster 2023-10-09T05:29:51.373937+0000 osd.4 (osd.4) 255 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T05:29:52.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:52 smithi133 bash[102858]: cluster 2023-10-09T05:29:51.374783+0000 osd.5 (osd.5) 262 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T05:29:52.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:52 smithi133 bash[102858]: audit 2023-10-09T05:29:51.375661+0000 mon.a (mon.0) 1591 : audit 1 from='client.? 172.21.15.133:0/494950949' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:29:52.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:52 smithi133 bash[102858]: cluster 2023-10-09T05:29:51.375749+0000 mon.a (mon.0) 1592 : cluster 0 osdmap e285: 8 total, 8 up, 7 in 2023-10-09T05:29:52.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:52 smithi133 bash[105589]: cluster 2023-10-09T05:29:51.363472+0000 mon.a (mon.0) 1590 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:29:52.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:52 smithi133 bash[105589]: cluster 2023-10-09T05:29:51.373937+0000 osd.4 (osd.4) 255 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T05:29:52.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:52 smithi133 bash[105589]: cluster 2023-10-09T05:29:51.374783+0000 osd.5 (osd.5) 262 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T05:29:52.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:52 smithi133 bash[105589]: audit 2023-10-09T05:29:51.375661+0000 mon.a (mon.0) 1591 : audit 1 from='client.? 172.21.15.133:0/494950949' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:29:52.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:52 smithi133 bash[105589]: cluster 2023-10-09T05:29:51.375749+0000 mon.a (mon.0) 1592 : cluster 0 osdmap e285: 8 total, 8 up, 7 in 2023-10-09T05:29:52.816 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:52.841 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:52.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:52.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:52.919 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:29:52.919 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:29:52.945 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:52.970 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:52.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:52.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:53.073 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:53.102 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:53.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:53.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:53.207 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:53.227 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:53.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:53.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:53.331 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:53.353 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:53.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:53.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:53.456 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:53.465 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:53 smithi194 bash[92666]: cluster 2023-10-09T05:29:52.249010+0000 mgr.y (mgr.25277) 1252 : cluster 0 pgmap v1335: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:53.480 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:53.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:53.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:53.584 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:53.610 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:53.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:53.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:53.713 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:53.741 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:53.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:53.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:53.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:53 smithi133 bash[102858]: cluster 2023-10-09T05:29:52.249010+0000 mgr.y (mgr.25277) 1252 : cluster 0 pgmap v1335: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:53.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:53 smithi133 bash[105589]: cluster 2023-10-09T05:29:52.249010+0000 mgr.y (mgr.25277) 1252 : cluster 0 pgmap v1335: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:53.820 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T05:29:53.845 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:53.875 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:53.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:53.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:53.979 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:54.008 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:54.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:54.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:54.115 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:54.145 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:54.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:54.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:54.249 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:54.278 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:54.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:54.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:54.382 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:54.413 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:54.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:54.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:54.430 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:29:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:29:54] "GET /metrics HTTP/1.1" 200 36425 "" "Prometheus/2.43.0" 2023-10-09T05:29:54.444 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:29:54.516 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:54.536 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:54.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:54.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:54.639 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:54.664 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:54.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:54.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:54.768 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:54.792 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:54.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:54.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:54.896 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:54.908 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:29:54.924 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:54.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:54.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:55.028 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:55.040 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:54 smithi194 bash[92666]: cluster 2023-10-09T05:29:54.249584+0000 mgr.y (mgr.25277) 1253 : cluster 0 pgmap v1336: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:55.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:54 smithi133 bash[102858]: cluster 2023-10-09T05:29:54.249584+0000 mgr.y (mgr.25277) 1253 : cluster 0 pgmap v1336: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:55.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:54 smithi133 bash[105589]: cluster 2023-10-09T05:29:54.249584+0000 mgr.y (mgr.25277) 1253 : cluster 0 pgmap v1336: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:55.057 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:55.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:55.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:55.161 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:55.191 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:55.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:55.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:55.295 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:55.325 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:55.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:55.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:55.428 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:55.459 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:55.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:55.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:55.563 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:55.593 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:55.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:55.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:29:55.697 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:55.728 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:55.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:55.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:55.832 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:55.862 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:55.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:55.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:55.965 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:55.994 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:55.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:55.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:56.097 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:56.125 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:56.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:56.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:56.230 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:56.255 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:56.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:56.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:56.359 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:56.384 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:56.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:56.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:56.432 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:29:56.488 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:56.512 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:56.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:56.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:56.615 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:56.642 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:56.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:56.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:56.745 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:56.772 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:56.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:56.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:56.875 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:56.902 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:56.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:56.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:57.005 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:57.033 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:57.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:57.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:57.136 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:57.166 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:57.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:57.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:57.270 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:57.301 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:57.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:57.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:57.336 INFO:teuthology.orchestra.run.smithi133.stderr:pool 'unique_pool_7' removed 2023-10-09T05:29:57.405 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:57.431 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:57 smithi133 bash[102858]: cluster 2023-10-09T05:29:56.250327+0000 mgr.y (mgr.25277) 1254 : cluster 0 pgmap v1337: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:57.431 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:57 smithi133 bash[102858]: audit 2023-10-09T05:29:57.124780+0000 mon.c (mon.2) 115 : audit 1 from='client.? 172.21.15.133:0/4282821365' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:29:57.432 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:57 smithi133 bash[102858]: audit 2023-10-09T05:29:57.125535+0000 mon.a (mon.0) 1593 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:29:57.432 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:57 smithi133 bash[105589]: cluster 2023-10-09T05:29:56.250327+0000 mgr.y (mgr.25277) 1254 : cluster 0 pgmap v1337: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:57.432 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:57 smithi133 bash[105589]: audit 2023-10-09T05:29:57.124780+0000 mon.c (mon.2) 115 : audit 1 from='client.? 172.21.15.133:0/4282821365' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:29:57.432 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:57 smithi133 bash[105589]: audit 2023-10-09T05:29:57.125535+0000 mon.a (mon.0) 1593 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:29:57.438 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:57.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:57.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:57.541 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:57.568 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:57.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:57.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:57.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:57 smithi194 bash[92666]: cluster 2023-10-09T05:29:56.250327+0000 mgr.y (mgr.25277) 1254 : cluster 0 pgmap v1337: 89 pgs: 89 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:57.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:57 smithi194 bash[92666]: audit 2023-10-09T05:29:57.124780+0000 mon.c (mon.2) 115 : audit 1 from='client.? 172.21.15.133:0/4282821365' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:29:57.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:57 smithi194 bash[92666]: audit 2023-10-09T05:29:57.125535+0000 mon.a (mon.0) 1593 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:29:57.672 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:57.693 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:57.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:57.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:57.796 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:57.823 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:57.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:57.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:57.926 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:57.955 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:29:57.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:57.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:29:58.059 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:58.089 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:58.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:58.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:58.193 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:58.225 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:58.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:58.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:58.329 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:58.357 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:58.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:58.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:58.460 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:58.486 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:29:58.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:58.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:29:58.589 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:58.598 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:29:58.601 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:58 smithi133 bash[105589]: audit 2023-10-09T05:29:57.312957+0000 mon.a (mon.0) 1594 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:29:58.601 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:58 smithi133 bash[105589]: cluster 2023-10-09T05:29:57.313099+0000 mon.a (mon.0) 1595 : cluster 0 osdmap e286: 8 total, 8 up, 7 in 2023-10-09T05:29:58.601 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:58 smithi133 bash[105589]: cluster 2023-10-09T05:29:57.537564+0000 osd.7 (osd.7) 234 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T05:29:58.601 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:58 smithi133 bash[105589]: cluster 2023-10-09T05:29:57.537583+0000 osd.6 (osd.6) 237 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T05:29:58.601 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:58 smithi133 bash[105589]: cluster 2023-10-09T05:29:57.537645+0000 osd.5 (osd.5) 263 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T05:29:58.601 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:58 smithi133 bash[105589]: cluster 2023-10-09T05:29:57.537870+0000 osd.4 (osd.4) 256 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T05:29:58.601 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:58 smithi133 bash[105589]: cluster 2023-10-09T05:29:57.540705+0000 osd.7 (osd.7) 235 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T05:29:58.601 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:58 smithi133 bash[105589]: cluster 2023-10-09T05:29:57.542778+0000 osd.7 (osd.7) 236 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T05:29:58.601 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:58 smithi133 bash[105589]: cluster 2023-10-09T05:29:57.545585+0000 osd.7 (osd.7) 237 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T05:29:58.602 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:58 smithi133 bash[102858]: audit 2023-10-09T05:29:57.312957+0000 mon.a (mon.0) 1594 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:29:58.602 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:58 smithi133 bash[102858]: cluster 2023-10-09T05:29:57.313099+0000 mon.a (mon.0) 1595 : cluster 0 osdmap e286: 8 total, 8 up, 7 in 2023-10-09T05:29:58.602 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:58 smithi133 bash[102858]: cluster 2023-10-09T05:29:57.537564+0000 osd.7 (osd.7) 234 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T05:29:58.602 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:58 smithi133 bash[102858]: cluster 2023-10-09T05:29:57.537583+0000 osd.6 (osd.6) 237 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T05:29:58.602 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:58 smithi133 bash[102858]: cluster 2023-10-09T05:29:57.537645+0000 osd.5 (osd.5) 263 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T05:29:58.602 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:58 smithi133 bash[102858]: cluster 2023-10-09T05:29:57.537870+0000 osd.4 (osd.4) 256 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T05:29:58.602 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:58 smithi133 bash[102858]: cluster 2023-10-09T05:29:57.540705+0000 osd.7 (osd.7) 235 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T05:29:58.602 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:58 smithi133 bash[102858]: cluster 2023-10-09T05:29:57.542778+0000 osd.7 (osd.7) 236 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T05:29:58.602 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:58 smithi133 bash[102858]: cluster 2023-10-09T05:29:57.545585+0000 osd.7 (osd.7) 237 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T05:29:58.617 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:29:58.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:58.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:29:58.628 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:58 smithi194 bash[92666]: audit 2023-10-09T05:29:57.312957+0000 mon.a (mon.0) 1594 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:29:58.628 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:58 smithi194 bash[92666]: cluster 2023-10-09T05:29:57.313099+0000 mon.a (mon.0) 1595 : cluster 0 osdmap e286: 8 total, 8 up, 7 in 2023-10-09T05:29:58.628 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:58 smithi194 bash[92666]: cluster 2023-10-09T05:29:57.537564+0000 osd.7 (osd.7) 234 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T05:29:58.628 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:58 smithi194 bash[92666]: cluster 2023-10-09T05:29:57.537583+0000 osd.6 (osd.6) 237 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T05:29:58.629 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:58 smithi194 bash[92666]: cluster 2023-10-09T05:29:57.537645+0000 osd.5 (osd.5) 263 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T05:29:58.629 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:58 smithi194 bash[92666]: cluster 2023-10-09T05:29:57.537870+0000 osd.4 (osd.4) 256 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T05:29:58.629 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:58 smithi194 bash[92666]: cluster 2023-10-09T05:29:57.540705+0000 osd.7 (osd.7) 235 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T05:29:58.629 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:58 smithi194 bash[92666]: cluster 2023-10-09T05:29:57.542778+0000 osd.7 (osd.7) 236 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T05:29:58.629 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:58 smithi194 bash[92666]: cluster 2023-10-09T05:29:57.545585+0000 osd.7 (osd.7) 237 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T05:29:58.720 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:58.745 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:58.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:58.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:58.848 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:58.870 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:58.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:58.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:58.973 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:58.998 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:29:59.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:59.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:29:59.101 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:59.130 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:59.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:59.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:59.233 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:59.265 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:59.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:59.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:59.368 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:59.397 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:29:59.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:59.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:29:59.500 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:59.524 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:59 smithi133 bash[105589]: cluster 2023-10-09T05:29:58.250979+0000 mgr.y (mgr.25277) 1255 : cluster 0 pgmap v1339: 73 pgs: 73 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:59.525 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:29:59 smithi133 bash[105589]: audit 2023-10-09T05:29:58.480483+0000 mon.a (mon.0) 1596 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:29:59.525 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:59 smithi133 bash[102858]: cluster 2023-10-09T05:29:58.250979+0000 mgr.y (mgr.25277) 1255 : cluster 0 pgmap v1339: 73 pgs: 73 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:59.525 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:29:59 smithi133 bash[102858]: audit 2023-10-09T05:29:58.480483+0000 mon.a (mon.0) 1596 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:29:59.526 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T05:29:59.526 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T05:29:59.527 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_8 2023-10-09T05:29:59.527 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_8 16 2023-10-09T05:29:59.533 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:59.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:59.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:59.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:59 smithi194 bash[92666]: cluster 2023-10-09T05:29:58.250979+0000 mgr.y (mgr.25277) 1255 : cluster 0 pgmap v1339: 73 pgs: 73 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:29:59.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:29:59 smithi194 bash[92666]: audit 2023-10-09T05:29:58.480483+0000 mon.a (mon.0) 1596 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:29:59.637 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:59.667 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:29:59.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:59.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:29:59.770 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:59.794 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:29:59.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:59.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:29:59.898 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:59.928 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:29:59.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:29:59.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:00.031 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:00.056 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:00.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:00.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:00.161 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:00.190 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:00.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:00.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:00.294 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:00.321 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:00.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:00.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:00.425 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:00.438 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:00 smithi194 bash[92666]: cluster 2023-10-09T05:30:00.000178+0000 mon.a (mon.0) 1597 : cluster 3 Health detail: HEALTH_WARN nodeep-scrub flag(s) set 2023-10-09T05:30:00.438 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:00 smithi194 bash[92666]: cluster 2023-10-09T05:30:00.000224+0000 mon.a (mon.0) 1598 : cluster 3 [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-10-09T05:30:00.452 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:00.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:00.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:00.555 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:00.580 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:00.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:00.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:00.684 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:00.709 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:00.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:00.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:00.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:00 smithi133 bash[105589]: cluster 2023-10-09T05:30:00.000178+0000 mon.a (mon.0) 1597 : cluster 3 Health detail: HEALTH_WARN nodeep-scrub flag(s) set 2023-10-09T05:30:00.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:00 smithi133 bash[105589]: cluster 2023-10-09T05:30:00.000224+0000 mon.a (mon.0) 1598 : cluster 3 [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-10-09T05:30:00.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:00 smithi133 bash[102858]: cluster 2023-10-09T05:30:00.000178+0000 mon.a (mon.0) 1597 : cluster 3 Health detail: HEALTH_WARN nodeep-scrub flag(s) set 2023-10-09T05:30:00.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:00 smithi133 bash[102858]: cluster 2023-10-09T05:30:00.000224+0000 mon.a (mon.0) 1598 : cluster 3 [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-10-09T05:30:00.812 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:00.842 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:00.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:00.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:00.945 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:00.976 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:00.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:00.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:01.006 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:30:01.080 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:01.103 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:01.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:01.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:01.206 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:01.232 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:01.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:01.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:01.335 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:01.364 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:01.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:01.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:01.467 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:01.491 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:01.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:01.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:01.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:01 smithi133 bash[102858]: cluster 2023-10-09T05:30:00.252049+0000 mgr.y (mgr.25277) 1256 : cluster 0 pgmap v1340: 73 pgs: 73 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:30:01.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:01 smithi133 bash[105589]: cluster 2023-10-09T05:30:00.252049+0000 mgr.y (mgr.25277) 1256 : cluster 0 pgmap v1340: 73 pgs: 73 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:30:01.594 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:01.607 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:01 smithi194 bash[92666]: cluster 2023-10-09T05:30:00.252049+0000 mgr.y (mgr.25277) 1256 : cluster 0 pgmap v1340: 73 pgs: 73 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:30:01.627 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:01.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:01.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:01.730 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:01.764 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:01.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:01.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:01.868 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:01.897 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:01.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:01.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:02.001 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:02.031 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:02.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:02.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:02.047 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:01 smithi133 bash[102858]: debug 2023-10-09T05:30:01.711+0000 7f5eb4d1d700 -1 mon.a@0(leader).osd e286 definitely_dead 0 2023-10-09T05:30:02.135 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:02.167 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:02.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:02.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:02.271 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:02.301 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:02.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:02.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:02.372 INFO:teuthology.orchestra.run.smithi133.stderr:marked out osd.4. 2023-10-09T05:30:02.405 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:02.424 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:02.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:02.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:02.512 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:30:02.512 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:30:02.528 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:02.539 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:02 smithi194 bash[92666]: audit 2023-10-09T05:30:01.714353+0000 mon.c (mon.2) 116 : audit 1 from='client.? 172.21.15.133:0/3429689002' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T05:30:02.539 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:02 smithi194 bash[92666]: audit 2023-10-09T05:30:01.714855+0000 mon.a (mon.0) 1599 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T05:30:02.539 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:02 smithi194 bash[92666]: cluster 2023-10-09T05:30:01.715002+0000 mon.a (mon.0) 1600 : cluster 1 Client client.admin marked osd.4 out, while it was still marked up 2023-10-09T05:30:02.554 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:02.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:02.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:02.658 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:02.691 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:02.692 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:02 smithi133 bash[105589]: audit 2023-10-09T05:30:01.714353+0000 mon.c (mon.2) 116 : audit 1 from='client.? 172.21.15.133:0/3429689002' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T05:30:02.693 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:02 smithi133 bash[105589]: audit 2023-10-09T05:30:01.714855+0000 mon.a (mon.0) 1599 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T05:30:02.693 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:02 smithi133 bash[105589]: cluster 2023-10-09T05:30:01.715002+0000 mon.a (mon.0) 1600 : cluster 1 Client client.admin marked osd.4 out, while it was still marked up 2023-10-09T05:30:02.693 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:02 smithi133 bash[102858]: audit 2023-10-09T05:30:01.714353+0000 mon.c (mon.2) 116 : audit 1 from='client.? 172.21.15.133:0/3429689002' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T05:30:02.693 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:02 smithi133 bash[102858]: audit 2023-10-09T05:30:01.714855+0000 mon.a (mon.0) 1599 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T05:30:02.693 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:02 smithi133 bash[102858]: cluster 2023-10-09T05:30:01.715002+0000 mon.a (mon.0) 1600 : cluster 1 Client client.admin marked osd.4 out, while it was still marked up 2023-10-09T05:30:02.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:02.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:02.795 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:02.823 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:02.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:02.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:02.926 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:02.952 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:02.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:02.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:03.056 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:03.082 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:03.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:03.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:03.186 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:03.207 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:03.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:03.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:03.310 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:03.338 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:03.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:03.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:03.371 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:03 smithi194 bash[92666]: cluster 2023-10-09T05:30:02.252799+0000 mgr.y (mgr.25277) 1257 : cluster 0 pgmap v1341: 73 pgs: 73 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:30:03.372 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:03 smithi194 bash[92666]: audit 2023-10-09T05:30:02.355323+0000 mon.a (mon.0) 1601 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-10-09T05:30:03.372 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:03 smithi194 bash[92666]: cluster 2023-10-09T05:30:02.355382+0000 mon.a (mon.0) 1602 : cluster 0 osdmap e287: 8 total, 8 up, 6 in 2023-10-09T05:30:03.372 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:03 smithi194 bash[92666]: cluster 2023-10-09T05:30:02.358731+0000 osd.7 (osd.7) 238 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T05:30:03.372 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:03 smithi194 bash[92666]: cluster 2023-10-09T05:30:02.359063+0000 osd.4 (osd.4) 257 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T05:30:03.372 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:03 smithi194 bash[92666]: cluster 2023-10-09T05:30:02.359650+0000 osd.6 (osd.6) 238 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T05:30:03.372 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:03 smithi194 bash[92666]: cluster 2023-10-09T05:30:02.361274+0000 osd.7 (osd.7) 239 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T05:30:03.372 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:03 smithi194 bash[92666]: cluster 2023-10-09T05:30:02.361555+0000 osd.4 (osd.4) 258 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T05:30:03.372 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:03 smithi194 bash[92666]: cluster 2023-10-09T05:30:02.362649+0000 osd.6 (osd.6) 239 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T05:30:03.442 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:03.463 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T05:30:03.472 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:03.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:03.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:03.575 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:03.588 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:03 smithi133 bash[105589]: cluster 2023-10-09T05:30:02.252799+0000 mgr.y (mgr.25277) 1257 : cluster 0 pgmap v1341: 73 pgs: 73 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:30:03.588 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:03 smithi133 bash[105589]: audit 2023-10-09T05:30:02.355323+0000 mon.a (mon.0) 1601 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-10-09T05:30:03.588 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:03 smithi133 bash[105589]: cluster 2023-10-09T05:30:02.355382+0000 mon.a (mon.0) 1602 : cluster 0 osdmap e287: 8 total, 8 up, 6 in 2023-10-09T05:30:03.588 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:03 smithi133 bash[105589]: cluster 2023-10-09T05:30:02.358731+0000 osd.7 (osd.7) 238 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T05:30:03.588 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:03 smithi133 bash[105589]: cluster 2023-10-09T05:30:02.359063+0000 osd.4 (osd.4) 257 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T05:30:03.588 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:03 smithi133 bash[105589]: cluster 2023-10-09T05:30:02.359650+0000 osd.6 (osd.6) 238 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T05:30:03.589 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:03 smithi133 bash[105589]: cluster 2023-10-09T05:30:02.361274+0000 osd.7 (osd.7) 239 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T05:30:03.589 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:03 smithi133 bash[105589]: cluster 2023-10-09T05:30:02.361555+0000 osd.4 (osd.4) 258 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T05:30:03.589 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:03 smithi133 bash[105589]: cluster 2023-10-09T05:30:02.362649+0000 osd.6 (osd.6) 239 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T05:30:03.589 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:03 smithi133 bash[102858]: cluster 2023-10-09T05:30:02.252799+0000 mgr.y (mgr.25277) 1257 : cluster 0 pgmap v1341: 73 pgs: 73 active+clean; 582 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T05:30:03.589 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:03 smithi133 bash[102858]: audit 2023-10-09T05:30:02.355323+0000 mon.a (mon.0) 1601 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-10-09T05:30:03.589 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:03 smithi133 bash[102858]: cluster 2023-10-09T05:30:02.355382+0000 mon.a (mon.0) 1602 : cluster 0 osdmap e287: 8 total, 8 up, 6 in 2023-10-09T05:30:03.589 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:03 smithi133 bash[102858]: cluster 2023-10-09T05:30:02.358731+0000 osd.7 (osd.7) 238 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T05:30:03.589 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:03 smithi133 bash[102858]: cluster 2023-10-09T05:30:02.359063+0000 osd.4 (osd.4) 257 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T05:30:03.590 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:03 smithi133 bash[102858]: cluster 2023-10-09T05:30:02.359650+0000 osd.6 (osd.6) 238 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T05:30:03.590 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:03 smithi133 bash[102858]: cluster 2023-10-09T05:30:02.361274+0000 osd.7 (osd.7) 239 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T05:30:03.590 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:03 smithi133 bash[102858]: cluster 2023-10-09T05:30:02.361555+0000 osd.4 (osd.4) 258 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T05:30:03.590 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:03 smithi133 bash[102858]: cluster 2023-10-09T05:30:02.362649+0000 osd.6 (osd.6) 239 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T05:30:03.607 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:03.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:03.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:03.710 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:03.740 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:03.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:03.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:03.844 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:03.872 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:03.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:03.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:03.975 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:04.003 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:04.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:04.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:04.107 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:04.138 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:04.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:04.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:04.243 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:04.273 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:04.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:04.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:04.298 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:30:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:30:04] "GET /metrics HTTP/1.1" 200 34061 "" "Prometheus/2.43.0" 2023-10-09T05:30:04.381 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:04.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:04 smithi194 bash[92666]: cluster 2023-10-09T05:30:03.356604+0000 osd.6 (osd.6) 240 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T05:30:04.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:04 smithi194 bash[92666]: cluster 2023-10-09T05:30:03.357487+0000 osd.5 (osd.5) 264 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T05:30:04.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:04 smithi194 bash[92666]: cluster 2023-10-09T05:30:03.361284+0000 mon.a (mon.0) 1603 : cluster 0 osdmap e288: 8 total, 8 up, 6 in 2023-10-09T05:30:04.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:04 smithi194 bash[92666]: cluster 2023-10-09T05:30:03.361635+0000 osd.6 (osd.6) 241 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T05:30:04.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:04 smithi194 bash[92666]: cluster 2023-10-09T05:30:03.362015+0000 osd.4 (osd.4) 259 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T05:30:04.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:04 smithi194 bash[92666]: cluster 2023-10-09T05:30:03.362233+0000 osd.7 (osd.7) 240 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T05:30:04.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:04 smithi194 bash[92666]: cluster 2023-10-09T05:30:03.364288+0000 osd.7 (osd.7) 241 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T05:30:04.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:04 smithi194 bash[92666]: cluster 2023-10-09T05:30:03.364693+0000 osd.6 (osd.6) 242 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T05:30:04.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:04 smithi194 bash[92666]: cluster 2023-10-09T05:30:03.365888+0000 osd.6 (osd.6) 243 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T05:30:04.413 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:04.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:04.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:04.517 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:04.549 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:04.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:04.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:04.654 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:04.667 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:04 smithi133 bash[105589]: cluster 2023-10-09T05:30:03.356604+0000 osd.6 (osd.6) 240 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T05:30:04.667 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:04 smithi133 bash[105589]: cluster 2023-10-09T05:30:03.357487+0000 osd.5 (osd.5) 264 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T05:30:04.667 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:04 smithi133 bash[105589]: cluster 2023-10-09T05:30:03.361284+0000 mon.a (mon.0) 1603 : cluster 0 osdmap e288: 8 total, 8 up, 6 in 2023-10-09T05:30:04.667 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:04 smithi133 bash[105589]: cluster 2023-10-09T05:30:03.361635+0000 osd.6 (osd.6) 241 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T05:30:04.667 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:04 smithi133 bash[105589]: cluster 2023-10-09T05:30:03.362015+0000 osd.4 (osd.4) 259 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T05:30:04.667 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:04 smithi133 bash[105589]: cluster 2023-10-09T05:30:03.362233+0000 osd.7 (osd.7) 240 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T05:30:04.667 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:04 smithi133 bash[105589]: cluster 2023-10-09T05:30:03.364288+0000 osd.7 (osd.7) 241 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T05:30:04.667 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:04 smithi133 bash[105589]: cluster 2023-10-09T05:30:03.364693+0000 osd.6 (osd.6) 242 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T05:30:04.668 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:04 smithi133 bash[105589]: cluster 2023-10-09T05:30:03.365888+0000 osd.6 (osd.6) 243 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T05:30:04.668 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:04 smithi133 bash[102858]: cluster 2023-10-09T05:30:03.356604+0000 osd.6 (osd.6) 240 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T05:30:04.668 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:04 smithi133 bash[102858]: cluster 2023-10-09T05:30:03.357487+0000 osd.5 (osd.5) 264 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T05:30:04.668 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:04 smithi133 bash[102858]: cluster 2023-10-09T05:30:03.361284+0000 mon.a (mon.0) 1603 : cluster 0 osdmap e288: 8 total, 8 up, 6 in 2023-10-09T05:30:04.668 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:04 smithi133 bash[102858]: cluster 2023-10-09T05:30:03.361635+0000 osd.6 (osd.6) 241 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T05:30:04.668 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:04 smithi133 bash[102858]: cluster 2023-10-09T05:30:03.362015+0000 osd.4 (osd.4) 259 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T05:30:04.668 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:04 smithi133 bash[102858]: cluster 2023-10-09T05:30:03.362233+0000 osd.7 (osd.7) 240 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T05:30:04.668 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:04 smithi133 bash[102858]: cluster 2023-10-09T05:30:03.364288+0000 osd.7 (osd.7) 241 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T05:30:04.668 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:04 smithi133 bash[102858]: cluster 2023-10-09T05:30:03.364693+0000 osd.6 (osd.6) 242 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T05:30:04.669 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:04 smithi133 bash[102858]: cluster 2023-10-09T05:30:03.365888+0000 osd.6 (osd.6) 243 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T05:30:04.684 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:04.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:04.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:04.787 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:04.817 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:04.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:04.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:04.920 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:04.955 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:04.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:04.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:05.059 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:05.088 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:05.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:05.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:05.193 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:05.222 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:05.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:05.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:05.301 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:30:05.327 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:05.357 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:05.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:05.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:05.461 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:05.492 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:05.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:05.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:05.537 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:30:05.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:05 smithi133 bash[102858]: cluster 2023-10-09T05:30:04.253431+0000 mgr.y (mgr.25277) 1258 : cluster 0 pgmap v1344: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:05.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:05 smithi133 bash[105589]: cluster 2023-10-09T05:30:04.253431+0000 mgr.y (mgr.25277) 1258 : cluster 0 pgmap v1344: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:05.596 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:05.622 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:05.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:05.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:05.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:05 smithi194 bash[92666]: cluster 2023-10-09T05:30:04.253431+0000 mgr.y (mgr.25277) 1258 : cluster 0 pgmap v1344: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:05.725 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:05.759 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:05.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:05.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:05.865 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:05.891 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:05.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:05.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:05.995 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:06.023 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:06.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:06.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:06.126 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:06.157 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:06.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:06.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:06.261 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:06.283 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:06.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:06.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:06.386 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:06.417 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:06.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:06.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:06.521 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:06.557 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:06.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:06.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:06.661 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:06.694 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:06.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:06.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:06.797 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:06.829 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:06.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:06.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:06.934 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:06.964 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:06.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:06.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:07.068 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:07.106 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:07.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:07.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:07.213 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:07.255 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:07.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:07.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:07.359 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:07.390 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:07.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:07.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:07.493 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:07.507 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:07 smithi194 bash[92666]: cluster 2023-10-09T05:30:06.254408+0000 mgr.y (mgr.25277) 1259 : cluster 0 pgmap v1345: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 383 B/s rd, 0 op/s; 18 B/s, 1 keys/s, 1 objects/s recovering 2023-10-09T05:30:07.530 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:07.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:07.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:07.633 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:07.670 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:07.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:07.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:07.774 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:07.789 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:07 smithi133 bash[102858]: cluster 2023-10-09T05:30:06.254408+0000 mgr.y (mgr.25277) 1259 : cluster 0 pgmap v1345: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 383 B/s rd, 0 op/s; 18 B/s, 1 keys/s, 1 objects/s recovering 2023-10-09T05:30:07.789 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:07 smithi133 bash[105589]: cluster 2023-10-09T05:30:06.254408+0000 mgr.y (mgr.25277) 1259 : cluster 0 pgmap v1345: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 383 B/s rd, 0 op/s; 18 B/s, 1 keys/s, 1 objects/s recovering 2023-10-09T05:30:07.805 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:07.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:07.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:07.909 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:07.934 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:07.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:07.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:08.039 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:08.044 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:30:08.068 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:30:08.074 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:08.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:08.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:08.096 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:30:08.117 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:30:08.140 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:30:08.164 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:30:08.178 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:08.187 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:30:08.209 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:08.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:08.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:08.218 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:30:08.220 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:30:08.221 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:30:08.221 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:30:08.221 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:30:08.221 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:30:08.221 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:30:08.221 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:30:08.221 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:30:08.236 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:30:08.313 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:08.344 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:08.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:08.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:08.447 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:08.472 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:08.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:08.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:08.575 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:08.600 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:08.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:08.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:08.702 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:08 smithi194 bash[92666]: audit 2023-10-09T05:30:08.015578+0000 mon.c (mon.2) 117 : audit 0 from='client.? 172.21.15.133:0/4145833569' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:30:08.703 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:08.729 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:08.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:08.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:08.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:08 smithi133 bash[105589]: audit 2023-10-09T05:30:08.015578+0000 mon.c (mon.2) 117 : audit 0 from='client.? 172.21.15.133:0/4145833569' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:30:08.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:08 smithi133 bash[102858]: audit 2023-10-09T05:30:08.015578+0000 mon.c (mon.2) 117 : audit 0 from='client.? 172.21.15.133:0/4145833569' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:30:08.832 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:08.862 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:08.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:08.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:08.966 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:08.994 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:08.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:08.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:09.037 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 6, 1, 7] out_osds: [5, 4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:30:09.037 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:30:09.098 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:09.131 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:09.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:09.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:09.235 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:09.269 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:09.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:09.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:09.373 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:09.404 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:09.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:09.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:09.508 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:09.538 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:09.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:09.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:09.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:09 smithi194 bash[92666]: cluster 2023-10-09T05:30:08.254892+0000 mgr.y (mgr.25277) 1260 : cluster 0 pgmap v1346: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 383 B/s rd, 0 op/s; 18 B/s, 1 keys/s, 1 objects/s recovering 2023-10-09T05:30:09.641 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:09.669 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:09.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:09.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:09.774 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:09.805 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:09.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:09.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:09.910 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:09.937 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:09.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:09.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:10.023 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:09 smithi133 bash[102858]: cluster 2023-10-09T05:30:08.254892+0000 mgr.y (mgr.25277) 1260 : cluster 0 pgmap v1346: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 383 B/s rd, 0 op/s; 18 B/s, 1 keys/s, 1 objects/s recovering 2023-10-09T05:30:10.024 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:09 smithi133 bash[105589]: cluster 2023-10-09T05:30:08.254892+0000 mgr.y (mgr.25277) 1260 : cluster 0 pgmap v1346: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 383 B/s rd, 0 op/s; 18 B/s, 1 keys/s, 1 objects/s recovering 2023-10-09T05:30:10.040 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:10.066 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:10.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:10.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:10.170 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:10.195 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:10.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:10.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:10.299 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:10.325 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:10.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:10.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:10.429 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:10.455 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:10.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:10.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:10.558 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:10.586 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:10.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:10.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:10.690 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:10.720 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:10.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:10.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:10.823 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:10.850 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:10.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:10.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:10.956 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:10.980 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:10.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:10.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:11.084 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:11.111 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:11.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:11.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:11.214 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:11.241 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:11.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:11.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:11.345 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:11.371 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:11.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:11.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:11.475 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:11.487 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:11 smithi133 bash[105589]: cluster 2023-10-09T05:30:10.255855+0000 mgr.y (mgr.25277) 1261 : cluster 0 pgmap v1347: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 383 B/s rd, 0 op/s; 18 B/s, 1 keys/s, 1 objects/s recovering 2023-10-09T05:30:11.488 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:11 smithi133 bash[102858]: cluster 2023-10-09T05:30:10.255855+0000 mgr.y (mgr.25277) 1261 : cluster 0 pgmap v1347: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 383 B/s rd, 0 op/s; 18 B/s, 1 keys/s, 1 objects/s recovering 2023-10-09T05:30:11.503 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:11.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:11.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:11.608 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:11.643 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:11.644 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:11 smithi194 bash[92666]: cluster 2023-10-09T05:30:10.255855+0000 mgr.y (mgr.25277) 1261 : cluster 0 pgmap v1347: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 383 B/s rd, 0 op/s; 18 B/s, 1 keys/s, 1 objects/s recovering 2023-10-09T05:30:11.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:11.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:11.746 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:11.771 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:11.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:11.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:11.877 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:11.905 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:11.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:11.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:12.008 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:12.038 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:12.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:12.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:12.143 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:12.174 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:12.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:12.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:12.278 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:12.303 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:12.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:12.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:12.358 INFO:teuthology.orchestra.run.smithi194.stdout:{ 2023-10-09T05:30:12.358 INFO:teuthology.orchestra.run.smithi194.stdout: "size": 20, 2023-10-09T05:30:12.359 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 600, 2023-10-09T05:30:12.359 INFO:teuthology.orchestra.run.smithi194.stdout: "ops": [ 2023-10-09T05:30:12.359 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.359 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35187.0:89379 11.f 11:feacd460:::benchmark_data_smithi133_194104_object89378:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e234)", 2023-10-09T05:30:12.359 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:20:17.110706+0000", 2023-10-09T05:30:12.359 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 595.24584293400005, 2023-10-09T05:30:12.359 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.4808422380000001, 2023-10-09T05:30:12.359 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:30:12.360 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "reached pg", 2023-10-09T05:30:12.360 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:30:12.360 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35187", 2023-10-09T05:30:12.360 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2118344251", 2023-10-09T05:30:12.360 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 89379 2023-10-09T05:30:12.360 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.360 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:30:12.360 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.360 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:30:12.361 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.110706+0000", 2023-10-09T05:30:12.361 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:30:12.361 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.361 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.361 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:30:12.361 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.110706+0000", 2023-10-09T05:30:12.361 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:30:12.361 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.362 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.362 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:30:12.362 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.110704+0000", 2023-10-09T05:30:12.362 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999976 2023-10-09T05:30:12.362 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.362 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.362 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:30:12.363 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.110746+0000", 2023-10-09T05:30:12.363 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.2802000000000001e-05 2023-10-09T05:30:12.363 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.363 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.363 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:30:12.363 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.110771+0000", 2023-10-09T05:30:12.363 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.4318e-05 2023-10-09T05:30:12.363 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.363 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.363 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:30:12.363 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.110780+0000", 2023-10-09T05:30:12.364 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.1749999999999994e-06 2023-10-09T05:30:12.364 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.364 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.364 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:30:12.364 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.110849+0000", 2023-10-09T05:30:12.364 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.9616000000000006e-05 2023-10-09T05:30:12.364 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.364 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.364 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:30:12.364 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.111010+0000", 2023-10-09T05:30:12.364 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00016021699999999999 2023-10-09T05:30:12.364 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.365 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.365 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:30:12.365 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.111183+0000", 2023-10-09T05:30:12.365 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00017347900000000001 2023-10-09T05:30:12.365 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.365 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.365 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:30:12.365 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.120515+0000", 2023-10-09T05:30:12.365 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0093314250000000008 2023-10-09T05:30:12.365 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.365 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.366 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:30:12.366 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.127771+0000", 2023-10-09T05:30:12.366 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0072562670000000003 2023-10-09T05:30:12.366 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.366 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.366 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:30:12.366 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.127784+0000", 2023-10-09T05:30:12.366 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.3318e-05 2023-10-09T05:30:12.366 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.367 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.367 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:30:12.367 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:18.123701+0000", 2023-10-09T05:30:12.367 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.99591639799999998 2023-10-09T05:30:12.367 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.367 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.367 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:30:12.367 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:18.123715+0000", 2023-10-09T05:30:12.367 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.4705999999999999e-05 2023-10-09T05:30:12.367 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.367 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.367 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:30:12.368 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.591488+0000", 2023-10-09T05:30:12.368 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.4677724910000001 2023-10-09T05:30:12.368 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.368 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.368 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:30:12.368 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.591548+0000", 2023-10-09T05:30:12.368 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.0390000000000003e-05 2023-10-09T05:30:12.368 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:30:12.368 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:30:12.368 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:30:12.368 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.368 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.369 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35187.0:89384 11.f 11:f64b8ea5:::benchmark_data_smithi133_194104_object89383:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-10-09T05:30:12.369 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:20:17.115450+0000", 2023-10-09T05:30:12.369 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 595.24109908299999, 2023-10-09T05:30:12.369 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.5167833079999999, 2023-10-09T05:30:12.369 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:30:12.369 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:30:12.369 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:30:12.370 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35187", 2023-10-09T05:30:12.370 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2118344251", 2023-10-09T05:30:12.370 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 89384 2023-10-09T05:30:12.370 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.370 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:30:12.371 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.371 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:30:12.371 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.115450+0000", 2023-10-09T05:30:12.371 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:30:12.371 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.371 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.371 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:30:12.372 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.115450+0000", 2023-10-09T05:30:12.372 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:30:12.372 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.372 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.372 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:30:12.372 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.115448+0000", 2023-10-09T05:30:12.372 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:30:12.373 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.373 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.373 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:30:12.373 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.115489+0000", 2023-10-09T05:30:12.373 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.0728e-05 2023-10-09T05:30:12.373 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.373 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.374 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:30:12.374 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.115509+0000", 2023-10-09T05:30:12.374 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.0126e-05 2023-10-09T05:30:12.374 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.374 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.374 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:30:12.375 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.115518+0000", 2023-10-09T05:30:12.375 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.6950000000000006e-06 2023-10-09T05:30:12.375 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.375 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.375 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:30:12.375 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.115567+0000", 2023-10-09T05:30:12.375 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.8894999999999999e-05 2023-10-09T05:30:12.375 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.375 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.376 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op must wait for map", 2023-10-09T05:30:12.376 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.115574+0000", 2023-10-09T05:30:12.376 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.4529999999999996e-06 2023-10-09T05:30:12.376 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.376 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.376 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:30:12.376 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.127967+0000", 2023-10-09T05:30:12.376 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.012392454000000001 2023-10-09T05:30:12.376 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.376 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.376 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:30:12.376 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.127972+0000", 2023-10-09T05:30:12.376 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.6409999999999999e-06 2023-10-09T05:30:12.377 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.377 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.377 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:30:12.377 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:18.123987+0000", 2023-10-09T05:30:12.377 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.99601491799999997 2023-10-09T05:30:12.377 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.377 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.377 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:30:12.377 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:18.123996+0000", 2023-10-09T05:30:12.377 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.5930000000000006e-06 2023-10-09T05:30:12.377 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.377 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.378 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:30:12.378 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.593946+0000", 2023-10-09T05:30:12.378 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.469950573 2023-10-09T05:30:12.378 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.378 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.378 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:30:12.378 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.594054+0000", 2023-10-09T05:30:12.378 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00010771400000000001 2023-10-09T05:30:12.378 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.378 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.378 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:30:12.378 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.594177+0000", 2023-10-09T05:30:12.379 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00012306200000000001 2023-10-09T05:30:12.379 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.379 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.379 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:30:12.379 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.599292+0000", 2023-10-09T05:30:12.379 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0051149450000000001 2023-10-09T05:30:12.379 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.379 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.379 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:30:12.379 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.615064+0000", 2023-10-09T05:30:12.379 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.015772372 2023-10-09T05:30:12.380 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.380 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.380 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:30:12.380 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.628313+0000", 2023-10-09T05:30:12.380 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.013248887000000001 2023-10-09T05:30:12.380 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.380 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.380 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:30:12.380 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.628359+0000", 2023-10-09T05:30:12.380 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.5864e-05 2023-10-09T05:30:12.380 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.380 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.381 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:30:12.381 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.632233+0000", 2023-10-09T05:30:12.381 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0038739069999999998 2023-10-09T05:30:12.381 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:30:12.381 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:30:12.381 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:30:12.381 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.381 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.381 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35187.0:89389 11.f 11:f97ca0b9:::benchmark_data_smithi133_194104_object89388:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-10-09T05:30:12.381 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:20:17.151108+0000", 2023-10-09T05:30:12.381 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 595.20544045600002, 2023-10-09T05:30:12.382 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.4836755200000002, 2023-10-09T05:30:12.382 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:30:12.382 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:30:12.382 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:30:12.382 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35187", 2023-10-09T05:30:12.382 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2118344251", 2023-10-09T05:30:12.382 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 89389 2023-10-09T05:30:12.382 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.382 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:30:12.382 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.383 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:30:12.383 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.151108+0000", 2023-10-09T05:30:12.383 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:30:12.383 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.383 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.383 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:30:12.383 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.151108+0000", 2023-10-09T05:30:12.383 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:30:12.383 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.384 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.384 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:30:12.384 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.151105+0000", 2023-10-09T05:30:12.384 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999971 2023-10-09T05:30:12.384 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.384 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.384 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:30:12.384 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.151210+0000", 2023-10-09T05:30:12.384 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00010412399999999999 2023-10-09T05:30:12.384 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.384 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.385 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:30:12.385 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.151242+0000", 2023-10-09T05:30:12.385 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.2651000000000003e-05 2023-10-09T05:30:12.385 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.385 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.385 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:30:12.385 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.151272+0000", 2023-10-09T05:30:12.385 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.0176e-05 2023-10-09T05:30:12.385 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.386 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.386 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:30:12.386 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.151348+0000", 2023-10-09T05:30:12.386 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.5857000000000002e-05 2023-10-09T05:30:12.386 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.386 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.386 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:30:12.386 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.151357+0000", 2023-10-09T05:30:12.386 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.9900000000000003e-06 2023-10-09T05:30:12.387 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.387 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.387 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:30:12.387 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:18.124022+0000", 2023-10-09T05:30:12.387 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.97266477799999995 2023-10-09T05:30:12.387 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.387 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.388 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:30:12.388 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:18.124029+0000", 2023-10-09T05:30:12.388 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.1609999999999997e-06 2023-10-09T05:30:12.388 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.388 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.388 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:30:12.388 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.597005+0000", 2023-10-09T05:30:12.388 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.472975623 2023-10-09T05:30:12.388 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.389 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.389 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:30:12.389 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.597123+0000", 2023-10-09T05:30:12.389 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000117822 2023-10-09T05:30:12.389 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.389 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.389 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:30:12.390 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.597229+0000", 2023-10-09T05:30:12.390 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00010666399999999999 2023-10-09T05:30:12.390 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.390 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.390 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:30:12.390 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.603929+0000", 2023-10-09T05:30:12.390 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.006699254 2023-10-09T05:30:12.390 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.390 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.390 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:30:12.391 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.615151+0000", 2023-10-09T05:30:12.391 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.011222555 2023-10-09T05:30:12.391 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.391 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.391 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:30:12.391 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.632308+0000", 2023-10-09T05:30:12.391 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.017157199000000001 2023-10-09T05:30:12.391 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.391 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.392 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:30:12.392 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.632352+0000", 2023-10-09T05:30:12.392 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.3389000000000001e-05 2023-10-09T05:30:12.392 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.392 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.392 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:30:12.392 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.634784+0000", 2023-10-09T05:30:12.392 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.002432203 2023-10-09T05:30:12.392 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:30:12.393 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:30:12.393 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:30:12.393 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.393 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.393 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35187.0:89392 11.f 11:f145a35f:::benchmark_data_smithi133_194104_object89391:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-10-09T05:30:12.393 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:20:17.165746+0000", 2023-10-09T05:30:12.393 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 595.19080289700003, 2023-10-09T05:30:12.393 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.471708011, 2023-10-09T05:30:12.393 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:30:12.393 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:30:12.394 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:30:12.394 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35187", 2023-10-09T05:30:12.394 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2118344251", 2023-10-09T05:30:12.394 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 89392 2023-10-09T05:30:12.394 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.394 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:30:12.394 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.394 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:30:12.395 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.165746+0000", 2023-10-09T05:30:12.395 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:30:12.395 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.395 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.395 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:30:12.395 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.165746+0000", 2023-10-09T05:30:12.395 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:30:12.395 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.395 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.396 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:30:12.396 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.165744+0000", 2023-10-09T05:30:12.396 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999981 2023-10-09T05:30:12.396 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.396 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.396 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:30:12.396 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.165793+0000", 2023-10-09T05:30:12.396 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.9424e-05 2023-10-09T05:30:12.396 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.396 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.397 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:30:12.397 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.165811+0000", 2023-10-09T05:30:12.397 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.7779999999999999e-05 2023-10-09T05:30:12.397 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.397 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.397 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:30:12.397 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.165818+0000", 2023-10-09T05:30:12.397 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.2609999999999996e-06 2023-10-09T05:30:12.397 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.398 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.398 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:30:12.398 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.165871+0000", 2023-10-09T05:30:12.398 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.2945999999999998e-05 2023-10-09T05:30:12.398 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.398 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.398 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:30:12.398 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:17.165880+0000", 2023-10-09T05:30:12.399 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.5799999999999992e-06 2023-10-09T05:30:12.399 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.399 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.399 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:30:12.399 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:18.124211+0000", 2023-10-09T05:30:12.399 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.95833152799999999 2023-10-09T05:30:12.399 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.399 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.399 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for peered", 2023-10-09T05:30:12.400 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:18.124219+0000", 2023-10-09T05:30:12.400 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.1259999999999998e-06 2023-10-09T05:30:12.400 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.400 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.400 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:30:12.400 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.599351+0000", 2023-10-09T05:30:12.400 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.475131282 2023-10-09T05:30:12.400 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.400 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.400 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:30:12.400 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.599632+0000", 2023-10-09T05:30:12.401 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00028102900000000002 2023-10-09T05:30:12.401 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.401 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.401 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:30:12.401 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.599999+0000", 2023-10-09T05:30:12.401 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00036702300000000001 2023-10-09T05:30:12.401 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.401 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.401 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:30:12.401 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.604123+0000", 2023-10-09T05:30:12.401 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0041238730000000001 2023-10-09T05:30:12.401 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.402 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.402 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:30:12.402 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.615240+0000", 2023-10-09T05:30:12.402 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.011117201 2023-10-09T05:30:12.402 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.402 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.402 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:30:12.402 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.634920+0000", 2023-10-09T05:30:12.402 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.019680092 2023-10-09T05:30:12.402 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.403 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.403 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:30:12.403 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.634972+0000", 2023-10-09T05:30:12.403 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.2265999999999997e-05 2023-10-09T05:30:12.403 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:30:12.403 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:30:12.403 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:30:12.403 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:20:19.637454+0000", 2023-10-09T05:30:12.403 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.002481689 2023-10-09T05:30:12.403 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:30:12.403 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:30:12.404 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:30:12.404 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:30:12.404 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:30:12.404 INFO:teuthology.orchestra.run.smithi194.stdout:} 2023-10-09T05:30:12.407 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:12.437 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:12.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:12.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:12.526 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:30:12.540 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:12.601 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:12.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:12.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:12.705 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:12.732 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:12.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:12.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:12.835 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:12.867 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:12.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:12.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:12.971 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:13.004 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:13.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:13.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:13.108 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:13.137 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:13.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:13.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:13.241 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:13.259 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T05:30:13.275 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:13.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:13.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:13.379 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:13.415 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:13.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:13.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:13.519 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:13.555 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:13.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:13.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:13.659 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:13.695 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:13.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:13.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:13.762 INFO:teuthology.orchestra.run.smithi133.stderr:pool 'unique_pool_8' created 2023-10-09T05:30:13.765 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is unset 2023-10-09T05:30:13.799 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:13.828 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:13.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:13.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:13.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:13 smithi194 bash[92666]: cluster 2023-10-09T05:30:12.256572+0000 mgr.y (mgr.25277) 1262 : cluster 0 pgmap v1348: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 310 B/s rd, 0 op/s; 14 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T05:30:13.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:13 smithi194 bash[92666]: audit 2023-10-09T05:30:13.236683+0000 mon.c (mon.2) 118 : audit 1 from='client.? 172.21.15.133:0/2072667548' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:30:13.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:13 smithi194 bash[92666]: audit 2023-10-09T05:30:13.237484+0000 mon.a (mon.0) 1604 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:30:13.931 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:13.962 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:13.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:13.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:14.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:13 smithi133 bash[102858]: cluster 2023-10-09T05:30:12.256572+0000 mgr.y (mgr.25277) 1262 : cluster 0 pgmap v1348: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 310 B/s rd, 0 op/s; 14 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T05:30:14.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:13 smithi133 bash[102858]: audit 2023-10-09T05:30:13.236683+0000 mon.c (mon.2) 118 : audit 1 from='client.? 172.21.15.133:0/2072667548' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:30:14.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:13 smithi133 bash[102858]: audit 2023-10-09T05:30:13.237484+0000 mon.a (mon.0) 1604 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:30:14.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:13 smithi133 bash[105589]: cluster 2023-10-09T05:30:12.256572+0000 mgr.y (mgr.25277) 1262 : cluster 0 pgmap v1348: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 310 B/s rd, 0 op/s; 14 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T05:30:14.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:13 smithi133 bash[105589]: audit 2023-10-09T05:30:13.236683+0000 mon.c (mon.2) 118 : audit 1 from='client.? 172.21.15.133:0/2072667548' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:30:14.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:13 smithi133 bash[105589]: audit 2023-10-09T05:30:13.237484+0000 mon.a (mon.0) 1604 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:30:14.067 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:14.100 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:14.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:14.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:14.204 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:14.235 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:14.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:14.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:14.339 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:14.375 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:14.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:14.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:14.458 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:30:14] "GET /metrics HTTP/1.1" 200 35512 "" "Prometheus/2.43.0" 2023-10-09T05:30:14.480 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:14.506 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:14.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:14.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:14.609 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:14.639 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:14.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:14.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:14.744 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:14.781 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:14.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:14.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:14.884 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:14.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:14 smithi194 bash[92666]: audit 2023-10-09T05:30:13.283446+0000 mon.c (mon.2) 119 : audit 1 from='client.? 172.21.15.133:0/2638399888' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-10-09T05:30:14.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:14 smithi194 bash[92666]: audit 2023-10-09T05:30:13.284087+0000 mon.a (mon.0) 1605 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-10-09T05:30:14.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:14 smithi194 bash[92666]: cluster 2023-10-09T05:30:13.732419+0000 mon.a (mon.0) 1606 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:30:14.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:14 smithi194 bash[92666]: cluster 2023-10-09T05:30:13.732521+0000 mon.a (mon.0) 1607 : cluster 1 Cluster is now healthy 2023-10-09T05:30:14.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:14 smithi194 bash[92666]: audit 2023-10-09T05:30:13.734943+0000 mon.a (mon.0) 1608 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:30:14.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:14 smithi194 bash[92666]: audit 2023-10-09T05:30:13.752357+0000 mon.a (mon.0) 1609 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:30:14.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:14 smithi194 bash[92666]: audit 2023-10-09T05:30:13.752460+0000 mon.a (mon.0) 1610 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2023-10-09T05:30:14.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:14 smithi194 bash[92666]: cluster 2023-10-09T05:30:13.752530+0000 mon.a (mon.0) 1611 : cluster 0 osdmap e289: 8 total, 8 up, 6 in 2023-10-09T05:30:14.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:14 smithi194 bash[92666]: cluster 2023-10-09T05:30:13.754595+0000 osd.6 (osd.6) 244 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T05:30:14.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:14 smithi194 bash[92666]: cluster 2023-10-09T05:30:13.754634+0000 osd.7 (osd.7) 242 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T05:30:14.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:14 smithi194 bash[92666]: cluster 2023-10-09T05:30:13.757451+0000 osd.7 (osd.7) 243 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T05:30:14.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:14 smithi194 bash[92666]: cluster 2023-10-09T05:30:13.757743+0000 osd.6 (osd.6) 245 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T05:30:14.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:14 smithi194 bash[92666]: cluster 2023-10-09T05:30:14.097989+0000 osd.4 (osd.4) 260 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T05:30:14.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:14 smithi194 bash[92666]: cluster 2023-10-09T05:30:14.101069+0000 osd.4 (osd.4) 261 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T05:30:14.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:14 smithi194 bash[92666]: cluster 2023-10-09T05:30:14.104121+0000 osd.4 (osd.4) 262 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T05:30:14.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:14 smithi194 bash[92666]: cluster 2023-10-09T05:30:14.257343+0000 mgr.y (mgr.25277) 1263 : cluster 0 pgmap v1350: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 307 B/s rd, 0 op/s; 14 B/s, 0 keys/s, 0 objects/s recovering 2023-10-09T05:30:14.912 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:14.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:14.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:15.015 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:15.047 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:15.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[105589]: audit 2023-10-09T05:30:13.283446+0000 mon.c (mon.2) 119 : audit 1 from='client.? 172.21.15.133:0/2638399888' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-10-09T05:30:15.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[105589]: audit 2023-10-09T05:30:13.284087+0000 mon.a (mon.0) 1605 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-10-09T05:30:15.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[105589]: cluster 2023-10-09T05:30:13.732419+0000 mon.a (mon.0) 1606 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:30:15.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[105589]: cluster 2023-10-09T05:30:13.732521+0000 mon.a (mon.0) 1607 : cluster 1 Cluster is now healthy 2023-10-09T05:30:15.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[105589]: audit 2023-10-09T05:30:13.734943+0000 mon.a (mon.0) 1608 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:30:15.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[105589]: audit 2023-10-09T05:30:13.752357+0000 mon.a (mon.0) 1609 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:30:15.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[105589]: audit 2023-10-09T05:30:13.752460+0000 mon.a (mon.0) 1610 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2023-10-09T05:30:15.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[105589]: cluster 2023-10-09T05:30:13.752530+0000 mon.a (mon.0) 1611 : cluster 0 osdmap e289: 8 total, 8 up, 6 in 2023-10-09T05:30:15.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[105589]: cluster 2023-10-09T05:30:13.754595+0000 osd.6 (osd.6) 244 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T05:30:15.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[105589]: cluster 2023-10-09T05:30:13.754634+0000 osd.7 (osd.7) 242 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T05:30:15.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[105589]: cluster 2023-10-09T05:30:13.757451+0000 osd.7 (osd.7) 243 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T05:30:15.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[105589]: cluster 2023-10-09T05:30:13.757743+0000 osd.6 (osd.6) 245 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T05:30:15.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[105589]: cluster 2023-10-09T05:30:14.097989+0000 osd.4 (osd.4) 260 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T05:30:15.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[105589]: cluster 2023-10-09T05:30:14.101069+0000 osd.4 (osd.4) 261 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T05:30:15.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[105589]: cluster 2023-10-09T05:30:14.104121+0000 osd.4 (osd.4) 262 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T05:30:15.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[105589]: cluster 2023-10-09T05:30:14.257343+0000 mgr.y (mgr.25277) 1263 : cluster 0 pgmap v1350: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 307 B/s rd, 0 op/s; 14 B/s, 0 keys/s, 0 objects/s recovering 2023-10-09T05:30:15.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:15.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[102858]: audit 2023-10-09T05:30:13.283446+0000 mon.c (mon.2) 119 : audit 1 from='client.? 172.21.15.133:0/2638399888' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-10-09T05:30:15.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[102858]: audit 2023-10-09T05:30:13.284087+0000 mon.a (mon.0) 1605 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-10-09T05:30:15.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[102858]: cluster 2023-10-09T05:30:13.732419+0000 mon.a (mon.0) 1606 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:30:15.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[102858]: cluster 2023-10-09T05:30:13.732521+0000 mon.a (mon.0) 1607 : cluster 1 Cluster is now healthy 2023-10-09T05:30:15.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[102858]: audit 2023-10-09T05:30:13.734943+0000 mon.a (mon.0) 1608 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:30:15.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[102858]: audit 2023-10-09T05:30:13.752357+0000 mon.a (mon.0) 1609 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:30:15.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[102858]: audit 2023-10-09T05:30:13.752460+0000 mon.a (mon.0) 1610 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2023-10-09T05:30:15.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[102858]: cluster 2023-10-09T05:30:13.752530+0000 mon.a (mon.0) 1611 : cluster 0 osdmap e289: 8 total, 8 up, 6 in 2023-10-09T05:30:15.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[102858]: cluster 2023-10-09T05:30:13.754595+0000 osd.6 (osd.6) 244 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T05:30:15.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[102858]: cluster 2023-10-09T05:30:13.754634+0000 osd.7 (osd.7) 242 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T05:30:15.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[102858]: cluster 2023-10-09T05:30:13.757451+0000 osd.7 (osd.7) 243 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T05:30:15.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[102858]: cluster 2023-10-09T05:30:13.757743+0000 osd.6 (osd.6) 245 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T05:30:15.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[102858]: cluster 2023-10-09T05:30:14.097989+0000 osd.4 (osd.4) 260 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T05:30:15.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[102858]: cluster 2023-10-09T05:30:14.101069+0000 osd.4 (osd.4) 261 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T05:30:15.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[102858]: cluster 2023-10-09T05:30:14.104121+0000 osd.4 (osd.4) 262 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T05:30:15.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:14 smithi133 bash[102858]: cluster 2023-10-09T05:30:14.257343+0000 mgr.y (mgr.25277) 1263 : cluster 0 pgmap v1350: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 307 B/s rd, 0 op/s; 14 B/s, 0 keys/s, 0 objects/s recovering 2023-10-09T05:30:15.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:15.153 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:15.184 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:15.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:15.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:15.289 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:15.324 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:15.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:15.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:15.428 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:15.463 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:15.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:15.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:15.567 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:15.596 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:15.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:15.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:15.699 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:15.729 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:15.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:15.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:15.834 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:15.842 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:15 smithi133 bash[105589]: cluster 2023-10-09T05:30:14.758812+0000 osd.6 (osd.6) 246 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T05:30:15.842 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:15 smithi133 bash[105589]: cluster 2023-10-09T05:30:14.763606+0000 mon.a (mon.0) 1612 : cluster 0 osdmap e290: 8 total, 8 up, 6 in 2023-10-09T05:30:15.843 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:15 smithi133 bash[105589]: cluster 2023-10-09T05:30:14.764559+0000 osd.7 (osd.7) 244 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T05:30:15.843 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:15 smithi133 bash[105589]: cluster 2023-10-09T05:30:14.764619+0000 osd.6 (osd.6) 247 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T05:30:15.843 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:15 smithi133 bash[102858]: cluster 2023-10-09T05:30:14.758812+0000 osd.6 (osd.6) 246 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T05:30:15.843 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:15 smithi133 bash[102858]: cluster 2023-10-09T05:30:14.763606+0000 mon.a (mon.0) 1612 : cluster 0 osdmap e290: 8 total, 8 up, 6 in 2023-10-09T05:30:15.843 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:15 smithi133 bash[102858]: cluster 2023-10-09T05:30:14.764559+0000 osd.7 (osd.7) 244 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T05:30:15.844 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:15 smithi133 bash[102858]: cluster 2023-10-09T05:30:14.764619+0000 osd.6 (osd.6) 247 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T05:30:15.863 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:15.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:15.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:15.874 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_8 rados --yes-i-really-mean-it || true 2023-10-09T05:30:15.970 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:15.986 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:15 smithi194 bash[92666]: cluster 2023-10-09T05:30:14.758812+0000 osd.6 (osd.6) 246 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T05:30:15.986 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:15 smithi194 bash[92666]: cluster 2023-10-09T05:30:14.763606+0000 mon.a (mon.0) 1612 : cluster 0 osdmap e290: 8 total, 8 up, 6 in 2023-10-09T05:30:15.986 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:15 smithi194 bash[92666]: cluster 2023-10-09T05:30:14.764559+0000 osd.7 (osd.7) 244 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T05:30:15.986 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:15 smithi194 bash[92666]: cluster 2023-10-09T05:30:14.764619+0000 osd.6 (osd.6) 247 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T05:30:16.006 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:16.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:16.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:16.109 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:16.136 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:16.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:16.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:16.241 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:16.274 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:16.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:16.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:16.378 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:16.411 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:16.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:16.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:16.514 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:16.546 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:16.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:16.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:16.650 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:16.686 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:16.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:16.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:16.790 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:16.824 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:16.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:16.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:16.928 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:16.963 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:16.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:16.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:17.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:16 smithi133 bash[102858]: cluster 2023-10-09T05:30:15.765709+0000 osd.5 (osd.5) 265 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T05:30:17.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:16 smithi133 bash[102858]: cluster 2023-10-09T05:30:15.767138+0000 mon.a (mon.0) 1613 : cluster 0 osdmap e291: 8 total, 8 up, 6 in 2023-10-09T05:30:17.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:16 smithi133 bash[102858]: cluster 2023-10-09T05:30:15.777536+0000 osd.4 (osd.4) 263 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T05:30:17.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:16 smithi133 bash[102858]: cluster 2023-10-09T05:30:15.777551+0000 osd.7 (osd.7) 245 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T05:30:17.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:16 smithi133 bash[102858]: cluster 2023-10-09T05:30:15.777790+0000 osd.6 (osd.6) 248 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T05:30:17.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:16 smithi133 bash[102858]: cluster 2023-10-09T05:30:16.258011+0000 mgr.y (mgr.25277) 1264 : cluster 0 pgmap v1353: 89 pgs: 11 creating+peering, 78 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:17.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:16 smithi133 bash[105589]: cluster 2023-10-09T05:30:15.765709+0000 osd.5 (osd.5) 265 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T05:30:17.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:16 smithi133 bash[105589]: cluster 2023-10-09T05:30:15.767138+0000 mon.a (mon.0) 1613 : cluster 0 osdmap e291: 8 total, 8 up, 6 in 2023-10-09T05:30:17.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:16 smithi133 bash[105589]: cluster 2023-10-09T05:30:15.777536+0000 osd.4 (osd.4) 263 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T05:30:17.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:16 smithi133 bash[105589]: cluster 2023-10-09T05:30:15.777551+0000 osd.7 (osd.7) 245 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T05:30:17.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:16 smithi133 bash[105589]: cluster 2023-10-09T05:30:15.777790+0000 osd.6 (osd.6) 248 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T05:30:17.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:16 smithi133 bash[105589]: cluster 2023-10-09T05:30:16.258011+0000 mgr.y (mgr.25277) 1264 : cluster 0 pgmap v1353: 89 pgs: 11 creating+peering, 78 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:17.067 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:17.100 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:17.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:17.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:17.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:16 smithi194 bash[92666]: cluster 2023-10-09T05:30:15.765709+0000 osd.5 (osd.5) 265 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T05:30:17.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:16 smithi194 bash[92666]: cluster 2023-10-09T05:30:15.767138+0000 mon.a (mon.0) 1613 : cluster 0 osdmap e291: 8 total, 8 up, 6 in 2023-10-09T05:30:17.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:16 smithi194 bash[92666]: cluster 2023-10-09T05:30:15.777536+0000 osd.4 (osd.4) 263 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T05:30:17.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:16 smithi194 bash[92666]: cluster 2023-10-09T05:30:15.777551+0000 osd.7 (osd.7) 245 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T05:30:17.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:16 smithi194 bash[92666]: cluster 2023-10-09T05:30:15.777790+0000 osd.6 (osd.6) 248 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T05:30:17.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:16 smithi194 bash[92666]: cluster 2023-10-09T05:30:16.258011+0000 mgr.y (mgr.25277) 1264 : cluster 0 pgmap v1353: 89 pgs: 11 creating+peering, 78 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:17.205 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:17.236 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:17.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:17.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:17.339 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:17.367 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:17.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:17.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:17.471 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:17.505 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:17.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:17.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:17.610 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:17.646 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:17.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:17.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:17.750 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:17.783 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:17.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:17.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:17.887 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:30:17.887 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:17.915 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:17.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:17.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:18.019 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:18.050 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:18.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:18.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:18.154 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:18.184 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:18.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:18.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:18.288 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:18.319 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:18.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:18.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:18.411 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:30:18.413 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:30:18.422 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:18.452 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:18.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:18.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:18.557 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:18.589 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:18.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:18.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:18.691 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:18.718 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:18.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:18.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:18.822 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:18.855 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:18.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:18.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:18.959 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:18.993 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:18.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:18.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:19.097 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:19.127 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:19.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:19.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:19.232 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:19.263 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:19.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:19.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:19.367 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:19.397 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:19.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:19.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:19.501 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:19.531 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:19.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:19.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:19.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:19 smithi133 bash[102858]: cluster 2023-10-09T05:30:18.258611+0000 mgr.y (mgr.25277) 1265 : cluster 0 pgmap v1354: 89 pgs: 11 creating+peering, 78 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:19.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:19 smithi133 bash[105589]: cluster 2023-10-09T05:30:18.258611+0000 mgr.y (mgr.25277) 1265 : cluster 0 pgmap v1354: 89 pgs: 11 creating+peering, 78 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:19.634 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:19.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:19 smithi194 bash[92666]: cluster 2023-10-09T05:30:18.258611+0000 mgr.y (mgr.25277) 1265 : cluster 0 pgmap v1354: 89 pgs: 11 creating+peering, 78 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:19.666 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:19.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:19.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:19.769 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:19.801 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:19.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:19.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:19.903 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:19.992 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:19.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:19.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:20.096 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:20.145 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:20.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:20.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:20.249 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:20.281 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:20.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:20.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:20.385 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:20.417 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:20.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:20.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:20.442 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:30:20.521 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:20.556 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:20.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:20.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:20.660 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:20.693 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:20.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:20.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:20.797 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:20.828 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:20.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:20.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:20.933 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:20.961 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:20.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:20.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:21.065 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:21.096 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:21.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:21.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:21.201 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:21.234 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:21.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:21.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:21.337 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:21.359 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:21 smithi194 bash[92666]: cluster 2023-10-09T05:30:20.259252+0000 mgr.y (mgr.25277) 1266 : cluster 0 pgmap v1355: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:21.381 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:21.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:21.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:21.485 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:21.493 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:21 smithi133 bash[102858]: cluster 2023-10-09T05:30:20.259252+0000 mgr.y (mgr.25277) 1266 : cluster 0 pgmap v1355: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:21.493 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:21 smithi133 bash[105589]: cluster 2023-10-09T05:30:20.259252+0000 mgr.y (mgr.25277) 1266 : cluster 0 pgmap v1355: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:21.509 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:21.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:21.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:21.612 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:21.642 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:21.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:21.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:21.746 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:21.779 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:21.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:21.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:21.883 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:21.922 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:21.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:21.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:22.026 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:22.057 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:22.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:22.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:22.160 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:22.194 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:22.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:22.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:22.298 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:22.326 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:22.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:22.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:22.430 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:22.465 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:22.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:22.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:22.472 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:30:22.570 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:22.595 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:22.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:22.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:22.698 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:22.725 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:22.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:22.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:22.828 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:22.856 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:22.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:22.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:22.959 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:22.994 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:22.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:22.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:23.098 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:23.132 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:23.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:23.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:23.235 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:23.263 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:23.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:23.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:23.366 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:23.397 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:23.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:23.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:23.501 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:23.514 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:23 smithi133 bash[102858]: cluster 2023-10-09T05:30:22.259758+0000 mgr.y (mgr.25277) 1267 : cluster 0 pgmap v1356: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:23.514 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:23 smithi133 bash[105589]: cluster 2023-10-09T05:30:22.259758+0000 mgr.y (mgr.25277) 1267 : cluster 0 pgmap v1356: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:23.535 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:23.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:23.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:23.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:23 smithi194 bash[92666]: cluster 2023-10-09T05:30:22.259758+0000 mgr.y (mgr.25277) 1267 : cluster 0 pgmap v1356: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:23.638 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:23.664 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:23.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:23.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:23.768 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:23.799 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:23.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:23.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:23.904 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:23.932 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:23.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:23.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:24.035 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:24.071 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:24.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:24.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:24.174 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:24.203 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:24.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:24.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:24.306 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:24.331 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:24.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:24.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:24.435 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:24.443 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:30:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:30:24] "GET /metrics HTTP/1.1" 200 35512 "" "Prometheus/2.43.0" 2023-10-09T05:30:24.461 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:24.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:24.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:24.565 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:24.591 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:24.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:24.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:24.695 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:24.730 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:24.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:24.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:24.834 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:24.867 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:24.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:24.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:24.971 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:24.994 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:24.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:24.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:25.097 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:25.127 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:25.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:25.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:25.232 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:25.240 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:25 smithi133 bash[105589]: audit 2023-10-09T05:30:24.122822+0000 mon.a (mon.0) 1614 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:30:25.240 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:25 smithi133 bash[105589]: cluster 2023-10-09T05:30:24.260303+0000 mgr.y (mgr.25277) 1268 : cluster 0 pgmap v1357: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:25.240 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:25 smithi133 bash[102858]: audit 2023-10-09T05:30:24.122822+0000 mon.a (mon.0) 1614 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:30:25.240 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:25 smithi133 bash[102858]: cluster 2023-10-09T05:30:24.260303+0000 mgr.y (mgr.25277) 1268 : cluster 0 pgmap v1357: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:25.253 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:25.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:25.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:25.356 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:25.385 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:25.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:25 smithi194 bash[92666]: audit 2023-10-09T05:30:24.122822+0000 mon.a (mon.0) 1614 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:30:25.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:25 smithi194 bash[92666]: cluster 2023-10-09T05:30:24.260303+0000 mgr.y (mgr.25277) 1268 : cluster 0 pgmap v1357: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:25.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:25.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:25.489 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:25.514 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:25.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:25.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:25.619 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:25.649 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:25.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:25.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:25.753 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:25.778 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:25.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:25.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:25.881 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:25.904 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:25.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:25.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:26.007 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:26.029 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:26.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:26.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:26.133 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:26.168 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:26.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:26.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:26.273 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:26.307 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:26.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:26.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:26.411 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:26.441 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:26.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:26.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:26.544 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:26.569 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:26.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:26.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:26.673 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:26.703 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:26.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:26.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:26.807 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:26.827 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:26.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:26.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:26.930 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:26.960 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:26.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:26.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:27.064 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:27.086 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:27.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:27.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:27.188 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:27.222 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:27.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:27.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:27.327 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:27.348 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:27.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:27.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:27.452 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:27.465 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:27 smithi194 bash[92666]: cluster 2023-10-09T05:30:26.261521+0000 mgr.y (mgr.25277) 1269 : cluster 0 pgmap v1358: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:27.486 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:27.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:27.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:27.590 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:27.609 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:27.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:27.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:27.644 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:27 smithi133 bash[102858]: cluster 2023-10-09T05:30:26.261521+0000 mgr.y (mgr.25277) 1269 : cluster 0 pgmap v1358: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:27.645 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:27 smithi133 bash[105589]: cluster 2023-10-09T05:30:26.261521+0000 mgr.y (mgr.25277) 1269 : cluster 0 pgmap v1358: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:27.713 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:27.744 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:27.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:27.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:27.848 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:27.874 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:27.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:27.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:27.977 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:27.989 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 05:30:27 smithi194 bash[80638]: logger=cleanup t=2023-10-09T05:30:27.952516618Z level=info msg="Completed cleanup jobs" duration=45.710016ms 2023-10-09T05:30:28.011 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:28.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:28.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:28.116 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:28.141 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:28.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:28.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:28.244 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:28.267 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:28.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:28.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:28.370 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:28.401 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:28.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:28.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:28.505 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:28.525 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:28.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:28.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:28.629 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:28.651 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:28.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:28.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:28.755 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:28.776 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:28.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:28.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:28.880 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:28.905 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:28.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:28.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:29.008 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:29.040 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:29.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:29.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:29.143 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:29.164 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:29.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:29.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:29.267 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:29.294 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:29.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:29.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:29.397 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:29.428 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:29.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:29.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:29.532 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:29.566 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:29.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:29.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:29.670 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:29.677 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:29 smithi133 bash[102858]: cluster 2023-10-09T05:30:28.261969+0000 mgr.y (mgr.25277) 1270 : cluster 0 pgmap v1359: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:29.677 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:29 smithi133 bash[102858]: audit 2023-10-09T05:30:28.484647+0000 mon.a (mon.0) 1615 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:30:29.677 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:29 smithi133 bash[102858]: audit 2023-10-09T05:30:28.485679+0000 mon.a (mon.0) 1616 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:30:29.677 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:29 smithi133 bash[105589]: cluster 2023-10-09T05:30:28.261969+0000 mgr.y (mgr.25277) 1270 : cluster 0 pgmap v1359: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:29.677 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:29 smithi133 bash[105589]: audit 2023-10-09T05:30:28.484647+0000 mon.a (mon.0) 1615 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:30:29.677 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:29 smithi133 bash[105589]: audit 2023-10-09T05:30:28.485679+0000 mon.a (mon.0) 1616 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:30:29.689 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:29.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:29.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:29.795 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:29.825 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:29.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:29.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:29.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:29 smithi194 bash[92666]: cluster 2023-10-09T05:30:28.261969+0000 mgr.y (mgr.25277) 1270 : cluster 0 pgmap v1359: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:29.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:29 smithi194 bash[92666]: audit 2023-10-09T05:30:28.484647+0000 mon.a (mon.0) 1615 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:30:29.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:29 smithi194 bash[92666]: audit 2023-10-09T05:30:28.485679+0000 mon.a (mon.0) 1616 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:30:29.928 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:29.952 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:29.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:29.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:30.055 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:30.078 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:30.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:30.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:30.182 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:30.208 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:30.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:30.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:30.311 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:30.341 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:30.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:30.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:30.444 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:30.473 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:30.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:30.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:30.576 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:30.587 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:30 smithi194 bash[92666]: audit 2023-10-09T05:30:29.631601+0000 mon.a (mon.0) 1617 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:30:30.607 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:30.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:30.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:30.712 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:30.724 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:30 smithi133 bash[105589]: audit 2023-10-09T05:30:29.631601+0000 mon.a (mon.0) 1617 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:30:30.724 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:30 smithi133 bash[102858]: audit 2023-10-09T05:30:29.631601+0000 mon.a (mon.0) 1617 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:30:30.739 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:30.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:30.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:30.842 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:30.874 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:30.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:30.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:30.979 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:31.010 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:31.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:31.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:31.114 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:31.140 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:31.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:31.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:31.243 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:31.272 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:31.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:31.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:31.376 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:31.404 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:31.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:31.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:31.507 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:31.540 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:31.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:31.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:31.643 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:31.675 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:31.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:31.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:31.778 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:31.791 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:31 smithi194 bash[92666]: cluster 2023-10-09T05:30:30.262969+0000 mgr.y (mgr.25277) 1271 : cluster 0 pgmap v1360: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:31.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:31 smithi133 bash[105589]: cluster 2023-10-09T05:30:30.262969+0000 mgr.y (mgr.25277) 1271 : cluster 0 pgmap v1360: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:31.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:31 smithi133 bash[102858]: cluster 2023-10-09T05:30:30.262969+0000 mgr.y (mgr.25277) 1271 : cluster 0 pgmap v1360: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:31.811 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:31.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:31.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:31.915 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:31.945 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:31.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:31.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:32.049 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:32.071 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:32.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:32.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:32.174 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:32.208 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:32.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:32.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:32.312 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:32.347 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:32.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:32.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:32.455 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:32.487 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:32.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:32.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:32.591 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:32.622 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:32.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:32.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:32.726 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:32.760 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:32.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:32.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:32.863 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:32.895 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:32.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:32.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:32.999 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:33.023 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:33.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:33.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:33.126 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:33.162 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:33.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:33.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:33.266 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:33.296 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:33.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:33.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:33.400 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:33.427 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:33.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:33.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:33.531 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:33.557 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:33.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:33.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:33.661 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:33.690 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:33.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:33.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:33.793 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:33.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:33 smithi133 bash[102858]: cluster 2023-10-09T05:30:32.263521+0000 mgr.y (mgr.25277) 1272 : cluster 0 pgmap v1361: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:33.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:33 smithi133 bash[105589]: cluster 2023-10-09T05:30:32.263521+0000 mgr.y (mgr.25277) 1272 : cluster 0 pgmap v1361: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:33.818 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:33.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:33.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:33.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:33 smithi194 bash[92666]: cluster 2023-10-09T05:30:32.263521+0000 mgr.y (mgr.25277) 1272 : cluster 0 pgmap v1361: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:33.897 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:30:33.898 INFO:teuthology.orchestra.run.smithi133.stdout: "ops": [], 2023-10-09T05:30:33.898 INFO:teuthology.orchestra.run.smithi133.stdout: "num_ops": 0 2023-10-09T05:30:33.898 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:30:33.922 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:33.956 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:33.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:33.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:34.060 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:34.090 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:34.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:34.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:34.194 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:34.213 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:34.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:34.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:34.317 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:34.328 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:30:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:30:34] "GET /metrics HTTP/1.1" 200 36402 "" "Prometheus/2.43.0" 2023-10-09T05:30:34.344 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:34.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:34.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:34.395 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:30:34.422 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:30:34.447 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:30:34.448 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:34.471 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:30:34.473 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:34.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:34.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:34.494 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:30:34.529 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:30:34.566 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:30:34.577 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:34.589 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:30:34.590 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:30:34.591 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:30:34.591 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:30:34.591 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:30:34.591 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:30:34.591 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:30:34.591 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:30:34.591 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:30:34.605 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:34.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:34.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:34.708 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:34.740 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:34.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:34.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:34.761 INFO:teuthology.orchestra.run.smithi133.stderr:enabled application 'rados' on pool 'unique_pool_8' 2023-10-09T05:30:34.844 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:34.867 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:34.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:34.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:34.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:34 smithi194 bash[92666]: cluster 2023-10-09T05:30:34.263938+0000 mgr.y (mgr.25277) 1273 : cluster 0 pgmap v1362: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:34.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:34 smithi194 bash[92666]: audit 2023-10-09T05:30:34.372675+0000 mon.c (mon.2) 120 : audit 0 from='client.? 172.21.15.133:0/110060090' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:30:34.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:34 smithi194 bash[92666]: audit 2023-10-09T05:30:34.512576+0000 mon.a (mon.0) 1618 : audit 1 from='client.? 172.21.15.133:0/2827205720' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:30:34.970 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:34.980 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:34 smithi133 bash[105589]: cluster 2023-10-09T05:30:34.263938+0000 mgr.y (mgr.25277) 1273 : cluster 0 pgmap v1362: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:34.980 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:34 smithi133 bash[105589]: audit 2023-10-09T05:30:34.372675+0000 mon.c (mon.2) 120 : audit 0 from='client.? 172.21.15.133:0/110060090' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:30:34.980 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:34 smithi133 bash[105589]: audit 2023-10-09T05:30:34.512576+0000 mon.a (mon.0) 1618 : audit 1 from='client.? 172.21.15.133:0/2827205720' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:30:34.981 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:34 smithi133 bash[102858]: cluster 2023-10-09T05:30:34.263938+0000 mgr.y (mgr.25277) 1273 : cluster 0 pgmap v1362: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:34.981 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:34 smithi133 bash[102858]: audit 2023-10-09T05:30:34.372675+0000 mon.c (mon.2) 120 : audit 0 from='client.? 172.21.15.133:0/110060090' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:30:34.981 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:34 smithi133 bash[102858]: audit 2023-10-09T05:30:34.512576+0000 mon.a (mon.0) 1618 : audit 1 from='client.? 172.21.15.133:0/2827205720' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:30:34.993 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:34.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:34.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:35.096 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:35.122 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:35.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:35.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:35.227 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:35.252 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:35.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:35.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:35.357 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:35.387 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:35.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:35.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:35.491 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:35.524 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:35.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:35.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:35.627 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:35.652 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:35.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:35.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:35.756 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:35.766 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:35 smithi133 bash[102858]: cluster 2023-10-09T05:30:34.756521+0000 osd.5 (osd.5) 266 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T05:30:35.766 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:35 smithi133 bash[102858]: audit 2023-10-09T05:30:34.757509+0000 mon.a (mon.0) 1619 : audit 1 from='client.? 172.21.15.133:0/2827205720' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:30:35.766 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:35 smithi133 bash[102858]: cluster 2023-10-09T05:30:34.757589+0000 mon.a (mon.0) 1620 : cluster 0 osdmap e292: 8 total, 8 up, 6 in 2023-10-09T05:30:35.766 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:35 smithi133 bash[102858]: audit 2023-10-09T05:30:34.853975+0000 mon.c (mon.2) 121 : audit 1 from='client.? 172.21.15.133:0/26654310' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:30:35.766 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:35 smithi133 bash[102858]: audit 2023-10-09T05:30:34.854270+0000 mon.a (mon.0) 1621 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:30:35.767 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:35 smithi133 bash[102858]: cluster 2023-10-09T05:30:35.002971+0000 osd.4 (osd.4) 264 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T05:30:35.767 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:35 smithi133 bash[102858]: cluster 2023-10-09T05:30:35.005874+0000 osd.4 (osd.4) 265 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T05:30:35.768 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is set 2023-10-09T05:30:35.782 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:35.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:35.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:35.885 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:35.909 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:35.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:35.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:36.012 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:36.021 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:35 smithi133 bash[105589]: cluster 2023-10-09T05:30:34.756521+0000 osd.5 (osd.5) 266 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T05:30:36.021 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:35 smithi133 bash[105589]: audit 2023-10-09T05:30:34.757509+0000 mon.a (mon.0) 1619 : audit 1 from='client.? 172.21.15.133:0/2827205720' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:30:36.021 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:35 smithi133 bash[105589]: cluster 2023-10-09T05:30:34.757589+0000 mon.a (mon.0) 1620 : cluster 0 osdmap e292: 8 total, 8 up, 6 in 2023-10-09T05:30:36.021 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:35 smithi133 bash[105589]: audit 2023-10-09T05:30:34.853975+0000 mon.c (mon.2) 121 : audit 1 from='client.? 172.21.15.133:0/26654310' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:30:36.021 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:35 smithi133 bash[105589]: audit 2023-10-09T05:30:34.854270+0000 mon.a (mon.0) 1621 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:30:36.022 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:35 smithi133 bash[105589]: cluster 2023-10-09T05:30:35.002971+0000 osd.4 (osd.4) 264 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T05:30:36.022 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:35 smithi133 bash[105589]: cluster 2023-10-09T05:30:35.005874+0000 osd.4 (osd.4) 265 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T05:30:36.036 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:36.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:36.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:36.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:35 smithi194 bash[92666]: cluster 2023-10-09T05:30:34.756521+0000 osd.5 (osd.5) 266 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T05:30:36.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:35 smithi194 bash[92666]: audit 2023-10-09T05:30:34.757509+0000 mon.a (mon.0) 1619 : audit 1 from='client.? 172.21.15.133:0/2827205720' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:30:36.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:35 smithi194 bash[92666]: cluster 2023-10-09T05:30:34.757589+0000 mon.a (mon.0) 1620 : cluster 0 osdmap e292: 8 total, 8 up, 6 in 2023-10-09T05:30:36.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:35 smithi194 bash[92666]: audit 2023-10-09T05:30:34.853975+0000 mon.c (mon.2) 121 : audit 1 from='client.? 172.21.15.133:0/26654310' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:30:36.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:35 smithi194 bash[92666]: audit 2023-10-09T05:30:34.854270+0000 mon.a (mon.0) 1621 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:30:36.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:35 smithi194 bash[92666]: cluster 2023-10-09T05:30:35.002971+0000 osd.4 (osd.4) 264 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T05:30:36.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:35 smithi194 bash[92666]: cluster 2023-10-09T05:30:35.005874+0000 osd.4 (osd.4) 265 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T05:30:36.140 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:36.159 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:36.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:36.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:36.263 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:36.291 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:36.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:36.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:36.394 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:36.421 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:36.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:36.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:36.524 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:36.548 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:36.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:36.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:36.615 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T05:30:36.651 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:36.689 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:36.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:36.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:36.792 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:36.814 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:36.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:36.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:36.917 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:36.928 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:36 smithi194 bash[92666]: cluster 2023-10-09T05:30:35.459942+0000 osd.6 (osd.6) 249 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T05:30:36.929 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:36 smithi194 bash[92666]: cluster 2023-10-09T05:30:35.683982+0000 osd.7 (osd.7) 246 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T05:30:36.929 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:36 smithi194 bash[92666]: cluster 2023-10-09T05:30:35.754344+0000 mon.a (mon.0) 1622 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:30:36.929 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:36 smithi194 bash[92666]: cluster 2023-10-09T05:30:35.758834+0000 osd.5 (osd.5) 267 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T05:30:36.929 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:36 smithi194 bash[92666]: audit 2023-10-09T05:30:35.762940+0000 mon.a (mon.0) 1623 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:30:36.929 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:36 smithi194 bash[92666]: cluster 2023-10-09T05:30:35.763044+0000 mon.a (mon.0) 1624 : cluster 0 osdmap e293: 8 total, 8 up, 6 in 2023-10-09T05:30:36.930 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:36 smithi194 bash[92666]: audit 2023-10-09T05:30:36.019971+0000 mon.a (mon.0) 1625 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:30:36.930 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:36 smithi194 bash[92666]: audit 2023-10-09T05:30:36.027898+0000 mon.a (mon.0) 1626 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:30:36.930 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:36 smithi194 bash[92666]: cluster 2023-10-09T05:30:36.264429+0000 mgr.y (mgr.25277) 1274 : cluster 0 pgmap v1365: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:36.947 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:36.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:36.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:37.052 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:37.079 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:37.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:37.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:37.083 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:36 smithi133 bash[105589]: cluster 2023-10-09T05:30:35.459942+0000 osd.6 (osd.6) 249 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T05:30:37.083 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:36 smithi133 bash[105589]: cluster 2023-10-09T05:30:35.683982+0000 osd.7 (osd.7) 246 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T05:30:37.083 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:36 smithi133 bash[105589]: cluster 2023-10-09T05:30:35.754344+0000 mon.a (mon.0) 1622 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:30:37.083 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:36 smithi133 bash[105589]: cluster 2023-10-09T05:30:35.758834+0000 osd.5 (osd.5) 267 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T05:30:37.083 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:36 smithi133 bash[105589]: audit 2023-10-09T05:30:35.762940+0000 mon.a (mon.0) 1623 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:30:37.083 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:36 smithi133 bash[105589]: cluster 2023-10-09T05:30:35.763044+0000 mon.a (mon.0) 1624 : cluster 0 osdmap e293: 8 total, 8 up, 6 in 2023-10-09T05:30:37.083 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:36 smithi133 bash[105589]: audit 2023-10-09T05:30:36.019971+0000 mon.a (mon.0) 1625 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:30:37.083 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:36 smithi133 bash[105589]: audit 2023-10-09T05:30:36.027898+0000 mon.a (mon.0) 1626 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:30:37.083 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:36 smithi133 bash[105589]: cluster 2023-10-09T05:30:36.264429+0000 mgr.y (mgr.25277) 1274 : cluster 0 pgmap v1365: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:37.084 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:36 smithi133 bash[102858]: cluster 2023-10-09T05:30:35.459942+0000 osd.6 (osd.6) 249 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T05:30:37.084 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:36 smithi133 bash[102858]: cluster 2023-10-09T05:30:35.683982+0000 osd.7 (osd.7) 246 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T05:30:37.084 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:36 smithi133 bash[102858]: cluster 2023-10-09T05:30:35.754344+0000 mon.a (mon.0) 1622 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:30:37.084 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:36 smithi133 bash[102858]: cluster 2023-10-09T05:30:35.758834+0000 osd.5 (osd.5) 267 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T05:30:37.084 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:36 smithi133 bash[102858]: audit 2023-10-09T05:30:35.762940+0000 mon.a (mon.0) 1623 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:30:37.084 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:36 smithi133 bash[102858]: cluster 2023-10-09T05:30:35.763044+0000 mon.a (mon.0) 1624 : cluster 0 osdmap e293: 8 total, 8 up, 6 in 2023-10-09T05:30:37.084 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:36 smithi133 bash[102858]: audit 2023-10-09T05:30:36.019971+0000 mon.a (mon.0) 1625 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:30:37.084 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:36 smithi133 bash[102858]: audit 2023-10-09T05:30:36.027898+0000 mon.a (mon.0) 1626 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:30:37.084 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:36 smithi133 bash[102858]: cluster 2023-10-09T05:30:36.264429+0000 mgr.y (mgr.25277) 1274 : cluster 0 pgmap v1365: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:37.182 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:37.203 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:37.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:37.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:37.306 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:37.338 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:37.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:37.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:37.443 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:37.475 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:37.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:37.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:37.578 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:37.607 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:37.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:37.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:37.711 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:37.744 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:37.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:37.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:37.848 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:37.877 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:37.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:37.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:37.981 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:37.994 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:37 smithi133 bash[102858]: cluster 2023-10-09T05:30:36.673456+0000 osd.4 (osd.4) 266 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T05:30:37.995 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:37 smithi133 bash[102858]: cluster 2023-10-09T05:30:36.673779+0000 osd.7 (osd.7) 247 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T05:30:37.995 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:37 smithi133 bash[102858]: cluster 2023-10-09T05:30:36.674014+0000 osd.6 (osd.6) 250 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T05:30:37.995 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:37 smithi133 bash[105589]: cluster 2023-10-09T05:30:36.673456+0000 osd.4 (osd.4) 266 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T05:30:37.995 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:37 smithi133 bash[105589]: cluster 2023-10-09T05:30:36.673779+0000 osd.7 (osd.7) 247 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T05:30:37.995 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:37 smithi133 bash[105589]: cluster 2023-10-09T05:30:36.674014+0000 osd.6 (osd.6) 250 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T05:30:38.012 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:38.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:38.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:38.116 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:38.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:37 smithi194 bash[92666]: cluster 2023-10-09T05:30:36.673456+0000 osd.4 (osd.4) 266 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T05:30:38.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:37 smithi194 bash[92666]: cluster 2023-10-09T05:30:36.673779+0000 osd.7 (osd.7) 247 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T05:30:38.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:37 smithi194 bash[92666]: cluster 2023-10-09T05:30:36.674014+0000 osd.6 (osd.6) 250 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T05:30:38.142 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:38.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:38.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:38.245 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:38.275 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:38.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:38.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:38.379 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:38.411 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:38.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:38.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:38.514 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:38.544 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:38.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:38.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:38.648 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:38.682 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:38.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:38.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:38.787 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:38.819 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:38.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:38.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:38.922 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:38.953 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:38.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:38.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:39.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:38 smithi133 bash[102858]: cluster 2023-10-09T05:30:38.264888+0000 mgr.y (mgr.25277) 1275 : cluster 0 pgmap v1366: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:39.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:38 smithi133 bash[105589]: cluster 2023-10-09T05:30:38.264888+0000 mgr.y (mgr.25277) 1275 : cluster 0 pgmap v1366: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:39.058 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:39.086 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:39.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:39.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:39.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:38 smithi194 bash[92666]: cluster 2023-10-09T05:30:38.264888+0000 mgr.y (mgr.25277) 1275 : cluster 0 pgmap v1366: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-10-09T05:30:39.190 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:39.222 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:39.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:39.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:39.325 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:39.357 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:39.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:39.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:39.450 DEBUG:teuthology.orchestra.run.smithi133:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_8 bench 90 write' 2023-10-09T05:30:39.450 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_8 2023-10-09T05:30:39.451 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:30:39.453 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T05:30:39.453 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T05:30:39.461 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:39.514 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:hints = 1 2023-10-09T05:30:39.515 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T05:30:39.515 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Object prefix: benchmark_data_smithi133_241067 2023-10-09T05:30:39.515 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:30:39.515 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 0 0 0 0 0 0 - 0 2023-10-09T05:30:39.528 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:39.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:39.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:39.633 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:39.669 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:39.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:39.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:39.790 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:39.836 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:39.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:39.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:39.940 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:39.966 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:39.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:39.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:40.071 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:40.099 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:40.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:40.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:40.203 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:40.232 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:40.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:40.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:40.337 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:40.341 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:30:40.378 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:40.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:40.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:40.462 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:30:40.482 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:40.512 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:40.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:40.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:40.515 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 1 16 2185 2169 135.554 135.562 0.011214 0.00732508 2023-10-09T05:30:40.615 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:40.672 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:40.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:40.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:40.777 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:40.808 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:40.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:40.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:40.912 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:40.953 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:40.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:40.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:41.058 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:41.087 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:41.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:41.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:41.192 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:41.219 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:41.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:41.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:41.237 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:41 smithi133 bash[102858]: audit 2023-10-09T05:30:40.237193+0000 mon.a (mon.0) 1627 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:30:41.237 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:41 smithi133 bash[102858]: audit 2023-10-09T05:30:40.246351+0000 mon.a (mon.0) 1628 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:30:41.237 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:41 smithi133 bash[102858]: cluster 2023-10-09T05:30:40.265683+0000 mgr.y (mgr.25277) 1276 : cluster 0 pgmap v1367: 89 pgs: 89 active+clean; 15 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 1.8 MiB/s wr, 28 op/s 2023-10-09T05:30:41.237 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:41 smithi133 bash[102858]: audit 2023-10-09T05:30:40.974700+0000 mon.a (mon.0) 1629 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:30:41.237 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:41 smithi133 bash[102858]: audit 2023-10-09T05:30:40.977136+0000 mon.a (mon.0) 1630 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:30:41.238 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:41 smithi133 bash[102858]: audit 2023-10-09T05:30:40.985510+0000 mon.a (mon.0) 1631 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:30:41.322 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:41.354 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:41.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:41.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:41.460 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:41.471 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:41 smithi194 bash[92666]: audit 2023-10-09T05:30:40.237193+0000 mon.a (mon.0) 1627 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:30:41.471 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:41 smithi194 bash[92666]: audit 2023-10-09T05:30:40.246351+0000 mon.a (mon.0) 1628 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:30:41.471 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:41 smithi194 bash[92666]: cluster 2023-10-09T05:30:40.265683+0000 mgr.y (mgr.25277) 1276 : cluster 0 pgmap v1367: 89 pgs: 89 active+clean; 15 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 1.8 MiB/s wr, 28 op/s 2023-10-09T05:30:41.471 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:41 smithi194 bash[92666]: audit 2023-10-09T05:30:40.974700+0000 mon.a (mon.0) 1629 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:30:41.471 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:41 smithi194 bash[92666]: audit 2023-10-09T05:30:40.977136+0000 mon.a (mon.0) 1630 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:30:41.471 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:41 smithi194 bash[92666]: audit 2023-10-09T05:30:40.985510+0000 mon.a (mon.0) 1631 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:30:41.489 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:41.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:41.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:41.516 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 2 16 4427 4411 137.833 140.125 0.00883333 0.00723969 2023-10-09T05:30:41.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:41 smithi133 bash[105589]: audit 2023-10-09T05:30:40.237193+0000 mon.a (mon.0) 1627 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:30:41.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:41 smithi133 bash[105589]: audit 2023-10-09T05:30:40.246351+0000 mon.a (mon.0) 1628 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:30:41.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:41 smithi133 bash[105589]: cluster 2023-10-09T05:30:40.265683+0000 mgr.y (mgr.25277) 1276 : cluster 0 pgmap v1367: 89 pgs: 89 active+clean; 15 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 1.8 MiB/s wr, 28 op/s 2023-10-09T05:30:41.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:41 smithi133 bash[105589]: audit 2023-10-09T05:30:40.974700+0000 mon.a (mon.0) 1629 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:30:41.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:41 smithi133 bash[105589]: audit 2023-10-09T05:30:40.977136+0000 mon.a (mon.0) 1630 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:30:41.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:41 smithi133 bash[105589]: audit 2023-10-09T05:30:40.985510+0000 mon.a (mon.0) 1631 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:30:41.591 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:41.619 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:41.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:41.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:41.723 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:41.752 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:41.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:41.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:41.855 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:41.885 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:41.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:41.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:41.989 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:42.016 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:30:42.018 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:42.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:42.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:42.122 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:42.146 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:42.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:42.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:42.250 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:42.281 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:42.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:42.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:42.385 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:42.411 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:42.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:42.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:42.515 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:42.516 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 3 16 6887 6871 143.126 153.75 0.00505726 0.00697244 2023-10-09T05:30:42.543 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:42.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:42.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:42.648 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:42.674 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:42.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:42.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:42.779 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:42.807 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:42.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:42.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:42.911 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:42.942 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:42.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:42.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:43.045 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:43.074 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:43.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:43.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:43.179 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:43.206 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:43.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:43.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:43.310 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:43.338 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:43.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:43.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:43.442 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:43.468 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:43.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:43.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:43.516 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 4 16 9189 9173 143.31 143.875 0.0124058 0.00696696 2023-10-09T05:30:43.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:43 smithi133 bash[102858]: cluster 2023-10-09T05:30:42.269931+0000 mgr.y (mgr.25277) 1277 : cluster 0 pgmap v1368: 89 pgs: 89 active+clean; 34 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 4.2 MiB/s wr, 66 op/s 2023-10-09T05:30:43.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:43 smithi133 bash[105589]: cluster 2023-10-09T05:30:42.269931+0000 mgr.y (mgr.25277) 1277 : cluster 0 pgmap v1368: 89 pgs: 89 active+clean; 34 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 4.2 MiB/s wr, 66 op/s 2023-10-09T05:30:43.572 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:43.582 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:43 smithi194 bash[92666]: cluster 2023-10-09T05:30:42.269931+0000 mgr.y (mgr.25277) 1277 : cluster 0 pgmap v1368: 89 pgs: 89 active+clean; 34 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 4.2 MiB/s wr, 66 op/s 2023-10-09T05:30:43.600 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:43.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:43.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:43.703 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:43.732 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:43.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:43.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:43.835 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:43.865 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:43.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:43.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:43.967 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:43.996 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:43.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:44.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:44.068 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:30:44.101 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:44.132 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:44.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:44.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:44.236 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:44.267 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:44.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:44.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:44.373 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:44.405 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:44.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:44.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:44.508 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:44.516 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 5 15 11401 11386 142.307 138.312 0.0104894 0.00701599 2023-10-09T05:30:44.517 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:30:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:30:44] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.43.0" 2023-10-09T05:30:44.518 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:44 smithi133 bash[102858]: audit 2023-10-09T05:30:43.492358+0000 mon.a (mon.0) 1632 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:30:44.519 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:44 smithi133 bash[102858]: audit 2023-10-09T05:30:43.494407+0000 mon.a (mon.0) 1633 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:30:44.519 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:44 smithi133 bash[105589]: audit 2023-10-09T05:30:43.492358+0000 mon.a (mon.0) 1632 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:30:44.519 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:44 smithi133 bash[105589]: audit 2023-10-09T05:30:43.494407+0000 mon.a (mon.0) 1633 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:30:44.533 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:44.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:44.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:44.637 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:44.667 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:44.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:44.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:44.771 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:44.781 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:44 smithi194 bash[92666]: audit 2023-10-09T05:30:43.492358+0000 mon.a (mon.0) 1632 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:30:44.781 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:44 smithi194 bash[92666]: audit 2023-10-09T05:30:43.494407+0000 mon.a (mon.0) 1633 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:30:44.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:44.801 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:44.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:44.902 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:44.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:44.930 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:44.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:45.031 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:45.056 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:45.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:45.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:45.092 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:30:45.092 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:30:45.161 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:45.189 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:45.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:45.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:45.294 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:45.322 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:45.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:45.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:45.425 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:45.454 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:45.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:45.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:45.488 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:45 smithi133 bash[102858]: cluster 2023-10-09T05:30:44.270498+0000 mgr.y (mgr.25277) 1278 : cluster 0 pgmap v1369: 89 pgs: 89 active+clean; 112 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 12 MiB/s wr, 187 op/s 2023-10-09T05:30:45.516 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 6 16 13636 13620 141.858 139.625 0.00438255 0.00704104 2023-10-09T05:30:45.558 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:45.587 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:45.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:45.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:45.692 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:45.716 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:45.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:45.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:45.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:45 smithi133 bash[105589]: cluster 2023-10-09T05:30:44.270498+0000 mgr.y (mgr.25277) 1278 : cluster 0 pgmap v1369: 89 pgs: 89 active+clean; 112 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 12 MiB/s wr, 187 op/s 2023-10-09T05:30:45.820 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:45.847 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:45.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:45.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:45.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:45 smithi194 bash[92666]: cluster 2023-10-09T05:30:44.270498+0000 mgr.y (mgr.25277) 1278 : cluster 0 pgmap v1369: 89 pgs: 89 active+clean; 112 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 12 MiB/s wr, 187 op/s 2023-10-09T05:30:45.950 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:45.980 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:45.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:45.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:46.083 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:46.114 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:46.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:46.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:46.219 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:46.248 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:46.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:46.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:46.352 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:46.378 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:46.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:46.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:46.481 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:46.511 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:46.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:46.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:46.516 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 7 16 15796 15780 140.876 135 0.00728071 0.00709074 2023-10-09T05:30:46.615 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:46.642 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:46.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:46.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:46.746 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:46.775 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:46.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:46.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:46.879 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:46.908 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:46.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:46.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:47.013 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:47.040 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:47.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:47.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:47.144 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:47.169 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:47.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:47.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:47.274 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:47.300 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:47.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:47.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:47.403 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:47.429 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:47.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:47.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:47.516 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 8 16 17797 17781 138.898 125.062 0.00839158 0.00719247 2023-10-09T05:30:47.533 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:47.562 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:47.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:47.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:47.667 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:47.691 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:47.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:47.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:47.747 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:47 smithi133 bash[105589]: cluster 2023-10-09T05:30:46.271270+0000 mgr.y (mgr.25277) 1279 : cluster 0 pgmap v1370: 89 pgs: 89 active+clean; 734 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T05:30:47.747 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:47 smithi133 bash[102858]: cluster 2023-10-09T05:30:46.271270+0000 mgr.y (mgr.25277) 1279 : cluster 0 pgmap v1370: 89 pgs: 89 active+clean; 734 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T05:30:47.794 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:47.820 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:47.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:47.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:47.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:47 smithi194 bash[92666]: cluster 2023-10-09T05:30:46.271270+0000 mgr.y (mgr.25277) 1279 : cluster 0 pgmap v1370: 89 pgs: 89 active+clean; 734 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T05:30:47.924 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:47.955 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:47.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:47.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:48.058 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:48.086 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:48.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:48.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:48.193 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:48.223 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:48.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:48.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:48.327 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:48.362 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:48.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:48.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:48.465 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:48.490 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:48.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:48.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:48.516 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 9 16 19355 19339 134.284 97.375 0.00709018 0.00743929 2023-10-09T05:30:48.594 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:48.619 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:48.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:48.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:48.723 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:48.752 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:48.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:48.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:48.855 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:48.884 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:48.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:48.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:48.989 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:49.012 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:49.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:49.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:49.117 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:49.151 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:49.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:49.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:49.255 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:49.279 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:49.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:49.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:49.382 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:49.409 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:49.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:49.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:49.513 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:49.517 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 10 16 21193 21177 132.342 114.875 0.00709802 0.00754842 2023-10-09T05:30:49.522 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:49 smithi133 bash[105589]: cluster 2023-10-09T05:30:48.271694+0000 mgr.y (mgr.25277) 1280 : cluster 0 pgmap v1371: 89 pgs: 89 active+clean; 734 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 61 MiB/s wr, 977 op/s 2023-10-09T05:30:49.523 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:49 smithi133 bash[102858]: cluster 2023-10-09T05:30:48.271694+0000 mgr.y (mgr.25277) 1280 : cluster 0 pgmap v1371: 89 pgs: 89 active+clean; 734 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 61 MiB/s wr, 977 op/s 2023-10-09T05:30:49.541 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:49.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:49.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:49.645 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:49.672 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:49.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:49.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:49.775 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:49.797 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:49.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:49.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:49.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:49 smithi194 bash[92666]: cluster 2023-10-09T05:30:48.271694+0000 mgr.y (mgr.25277) 1280 : cluster 0 pgmap v1371: 89 pgs: 89 active+clean; 734 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 61 MiB/s wr, 977 op/s 2023-10-09T05:30:49.900 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:49.924 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:49.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:49.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:50.027 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:50.058 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:50.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:50.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:50.164 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:50.190 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:50.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:50.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:50.294 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:50.318 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:50.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:50.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:50.421 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:50.454 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:50.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:50.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:50.517 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 11 16 22199 22183 126.026 62.875 0.00256401 0.0079068 2023-10-09T05:30:50.558 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:50.585 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:50.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:50.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:50.689 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:50.718 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:50.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:50.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:50.821 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:50.845 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:50.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:50.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:50.949 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:50.984 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:50.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:50.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:51.089 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:51.126 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:51.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:51.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:51.232 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:51.254 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:51.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:51.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:51.357 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:51.383 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:51.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:51.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:51.486 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:51.510 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:51.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:51.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:51.517 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 12 16 23225 23209 120.866 64.125 0.00185345 0.00823544 2023-10-09T05:30:51.613 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:51.643 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:51.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:51.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:51.750 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:51.777 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:51.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:51.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:51.882 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:51.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:51 smithi194 bash[92666]: cluster 2023-10-09T05:30:50.272593+0000 mgr.y (mgr.25277) 1281 : cluster 0 pgmap v1372: 89 pgs: 89 active+clean; 1.2 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T05:30:51.891 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:51 smithi133 bash[102858]: cluster 2023-10-09T05:30:50.272593+0000 mgr.y (mgr.25277) 1281 : cluster 0 pgmap v1372: 89 pgs: 89 active+clean; 1.2 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T05:30:51.891 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:51 smithi133 bash[105589]: cluster 2023-10-09T05:30:50.272593+0000 mgr.y (mgr.25277) 1281 : cluster 0 pgmap v1372: 89 pgs: 89 active+clean; 1.2 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T05:30:51.907 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:51.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:51.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:52.012 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:52.039 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:52.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:52.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:52.143 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:52.171 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:52.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:52.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:52.275 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:52.299 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:52.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:52.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:52.404 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:52.432 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:52.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:52.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:52.517 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 13 16 24934 24918 119.784 106.812 0.0121161 0.00834136 2023-10-09T05:30:52.535 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:52.560 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:52.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:52.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:52.664 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:52.690 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:52.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:52.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:52.793 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:52.825 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:52.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:52.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:52.929 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:52.971 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:52.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:52.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:53.075 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:53.101 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:53.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:53.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:53.204 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:53.231 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:53.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:53.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:53.335 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:53.355 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:53.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:53.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:53.457 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:53.486 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:53.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:53.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:53.517 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 14 16 26567 26551 118.518 102.062 0.00744689 0.00843083 2023-10-09T05:30:53.591 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:53.602 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:53 smithi133 bash[102858]: cluster 2023-10-09T05:30:52.273263+0000 mgr.y (mgr.25277) 1282 : cluster 0 pgmap v1373: 89 pgs: 89 active+clean; 1.3 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 110 MiB/s wr, 1.75k op/s 2023-10-09T05:30:53.603 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:53 smithi133 bash[105589]: cluster 2023-10-09T05:30:52.273263+0000 mgr.y (mgr.25277) 1282 : cluster 0 pgmap v1373: 89 pgs: 89 active+clean; 1.3 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 110 MiB/s wr, 1.75k op/s 2023-10-09T05:30:53.634 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:53.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:53.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:53.739 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:53.766 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:53.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:53.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:53.872 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:53.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:53 smithi194 bash[92666]: cluster 2023-10-09T05:30:52.273263+0000 mgr.y (mgr.25277) 1282 : cluster 0 pgmap v1373: 89 pgs: 89 active+clean; 1.3 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 110 MiB/s wr, 1.75k op/s 2023-10-09T05:30:53.896 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:53.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:53.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:54.000 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:54.033 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:54.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:54.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:54.138 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:54.166 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:54.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:54.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:54.271 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:54.309 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:54.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:54.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:54.415 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:54.433 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:54.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:54.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:54.517 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 15 16 28031 28015 116.716 91.5 0.0143515 0.00856155 2023-10-09T05:30:54.536 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:54.546 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:30:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:30:54] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.43.0" 2023-10-09T05:30:54.559 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:54.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:54.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:54.662 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:54.687 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:54.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:54.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:54.789 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:54.825 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:54.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:54.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:54.842 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:54 smithi133 bash[102858]: cluster 2023-10-09T05:30:54.273780+0000 mgr.y (mgr.25277) 1283 : cluster 0 pgmap v1374: 89 pgs: 89 active+clean; 1.4 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-10-09T05:30:54.929 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:54.957 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:54.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:54.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:55.061 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:55.072 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:54 smithi194 bash[92666]: cluster 2023-10-09T05:30:54.273780+0000 mgr.y (mgr.25277) 1283 : cluster 0 pgmap v1374: 89 pgs: 89 active+clean; 1.4 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-10-09T05:30:55.088 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:55.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:55.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:55.194 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:55.196 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:54 smithi133 bash[105589]: cluster 2023-10-09T05:30:54.273780+0000 mgr.y (mgr.25277) 1283 : cluster 0 pgmap v1374: 89 pgs: 89 active+clean; 1.4 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-10-09T05:30:55.219 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:55.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:55.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:55.323 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:55.366 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:55.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:55.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:55.470 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:55.494 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:55.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:55.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:55.517 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 16 16 29523 29507 115.249 93.25 0.00592426 0.00867097 2023-10-09T05:30:55.598 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:55.629 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:55.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:55.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:55.735 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:55.772 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:55.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:55.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:55.876 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:55.910 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:55.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:55.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:56.016 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:56.042 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:56.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:56.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:56.145 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:56.180 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:56.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:56.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:56.282 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:56.312 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:56.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:56.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:56.416 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:56.442 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:56.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:56.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:56.517 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 17 16 30727 30711 112.896 75.25 0.0023124 0.00882996 2023-10-09T05:30:56.545 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:56.575 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:56.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:56.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:56.678 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:56.713 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:56.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:56.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:56.817 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:56.837 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:56.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:56.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:56.940 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:56.969 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:56.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:56.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:57.072 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:57.101 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:57.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:57.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:57.204 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:57.235 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:57.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:57.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:57.331 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:57 smithi133 bash[102858]: cluster 2023-10-09T05:30:56.274553+0000 mgr.y (mgr.25277) 1284 : cluster 0 pgmap v1375: 89 pgs: 89 active+clean; 1.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-10-09T05:30:57.339 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:57.368 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:57.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:57.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:57.471 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:57.500 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:57.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:57.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:57.517 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 18 16 31843 31827 110.498 69.75 0.0114383 0.00904393 2023-10-09T05:30:57.605 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:57.616 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:57 smithi133 bash[105589]: cluster 2023-10-09T05:30:56.274553+0000 mgr.y (mgr.25277) 1284 : cluster 0 pgmap v1375: 89 pgs: 89 active+clean; 1.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-10-09T05:30:57.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:57 smithi194 bash[92666]: cluster 2023-10-09T05:30:56.274553+0000 mgr.y (mgr.25277) 1284 : cluster 0 pgmap v1375: 89 pgs: 89 active+clean; 1.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-10-09T05:30:57.648 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:57.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:57.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:57.752 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:57.778 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:57.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:57.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:57.882 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:57.910 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:30:57.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:57.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:30:58.013 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:58.035 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:58.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:58.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:58.137 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:58.162 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:58.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:58.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:58.266 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:58.290 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:58.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:58.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:58.393 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:58.420 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:58.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:58.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:58.517 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 19 16 32971 32955 108.393 70.5 0.0151123 0.00921977 2023-10-09T05:30:58.523 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:58.557 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:30:58.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:58.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:30:58.662 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:58.690 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:30:58.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:58.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:30:58.799 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:58.824 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:58.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:58.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:58.927 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:58.958 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:58.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:58.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:59.065 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:59.087 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:30:59.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:59.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:30:59.191 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:59.222 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:59.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:59.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:59.326 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:59.347 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:59 smithi133 bash[102858]: cluster 2023-10-09T05:30:58.274967+0000 mgr.y (mgr.25277) 1285 : cluster 0 pgmap v1376: 89 pgs: 89 active+clean; 1.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-10-09T05:30:59.351 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:59.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:59.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:59.456 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:59.467 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:59 smithi194 bash[92666]: cluster 2023-10-09T05:30:58.274967+0000 mgr.y (mgr.25277) 1285 : cluster 0 pgmap v1376: 89 pgs: 89 active+clean; 1.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-10-09T05:30:59.468 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:30:59 smithi194 bash[92666]: audit 2023-10-09T05:30:58.481599+0000 mon.a (mon.0) 1634 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:30:59.489 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:30:59.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:59.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:30:59.517 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T05:30:59.518004+0000 min lat: 0.00142329 max lat: 0.30392 avg lat: 0.00935126 2023-10-09T05:30:59.518 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:30:59.518 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 20 16 34207 34191 106.835 77.25 0.00669896 0.00935126 2023-10-09T05:30:59.592 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:59.602 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:30:59 smithi133 bash[102858]: audit 2023-10-09T05:30:58.481599+0000 mon.a (mon.0) 1634 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:30:59.603 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:59 smithi133 bash[105589]: cluster 2023-10-09T05:30:58.274967+0000 mgr.y (mgr.25277) 1285 : cluster 0 pgmap v1376: 89 pgs: 89 active+clean; 1.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-10-09T05:30:59.603 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:30:59 smithi133 bash[105589]: audit 2023-10-09T05:30:58.481599+0000 mon.a (mon.0) 1634 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:30:59.621 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:30:59.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:59.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:30:59.725 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:59.754 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:30:59.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:59.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:30:59.857 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:59.886 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:30:59.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:30:59.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:30:59.990 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:00.020 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:00.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:00.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:00.124 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:00.147 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:00.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:00.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:00.250 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:00.280 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:00.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:00.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:00.386 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:00.415 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:00.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:00.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:00.518 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 21 16 35480 35464 105.536 79.5625 0.0120416 0.00946877 2023-10-09T05:31:00.519 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:00.543 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:00.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:00.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:00.647 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:00.675 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:00.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:00.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:00.778 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:00.807 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:00.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:00.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:00.913 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:00.937 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:00.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:00.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:01.040 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:01.066 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:01.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:01.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:01.171 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:01.193 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:01.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:01.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:01.299 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:01.327 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:01.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:01.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:01.431 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:01.441 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:01 smithi133 bash[105589]: cluster 2023-10-09T05:31:00.275625+0000 mgr.y (mgr.25277) 1286 : cluster 0 pgmap v1377: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-10-09T05:31:01.442 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:01 smithi133 bash[102858]: cluster 2023-10-09T05:31:00.275625+0000 mgr.y (mgr.25277) 1286 : cluster 0 pgmap v1377: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-10-09T05:31:01.469 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:01.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:01.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:01.518 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 22 16 36924 36908 104.841 90.25 0.0165728 0.00953149 2023-10-09T05:31:01.573 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:01.608 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:01.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:01.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:01.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:01 smithi194 bash[92666]: cluster 2023-10-09T05:31:00.275625+0000 mgr.y (mgr.25277) 1286 : cluster 0 pgmap v1377: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-10-09T05:31:01.712 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:01.758 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:01.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:01.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:01.862 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:01.901 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:01.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:01.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:02.008 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:02.046 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:02.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:02.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:02.153 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:02.188 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:02.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:02.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:02.292 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:02.320 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:02.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:02.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:02.423 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:02.450 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:02.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:02.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:02.518 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 23 16 38351 38335 104.16 89.1875 0.00684635 0.00959408 2023-10-09T05:31:02.553 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:02.582 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:02.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:02.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:02.685 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:02.712 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:02.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:02.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:02.818 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:02.841 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:02.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:02.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:02.947 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:02.973 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:02.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:02.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:03.076 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:03.111 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:03.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:03.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:03.216 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:03.242 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:03.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:03.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:03.345 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:03.355 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:03 smithi133 bash[102858]: cluster 2023-10-09T05:31:02.276202+0000 mgr.y (mgr.25277) 1287 : cluster 0 pgmap v1378: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T05:31:03.355 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:03 smithi133 bash[105589]: cluster 2023-10-09T05:31:02.276202+0000 mgr.y (mgr.25277) 1287 : cluster 0 pgmap v1378: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T05:31:03.371 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:03.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:03.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:03.475 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:03.486 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:03 smithi194 bash[92666]: cluster 2023-10-09T05:31:02.276202+0000 mgr.y (mgr.25277) 1287 : cluster 0 pgmap v1378: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T05:31:03.502 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:03.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:03.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:03.518 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 24 16 39776 39760 103.531 89.0625 0.00925833 0.00965199 2023-10-09T05:31:03.610 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:03.635 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:03.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:03.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:03.739 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:03.763 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:03.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:03.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:03.866 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:03.891 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:03.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:03.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:03.993 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:04.029 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:04.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:04.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:04.132 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:04.168 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:04.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:04.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:04.274 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:04.281 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:31:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:31:04] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.43.0" 2023-10-09T05:31:04.303 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:04.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:04.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:04.408 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:04.432 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:04.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:04.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:04.518 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 25 16 40827 40811 102.017 65.6875 0.00889517 0.0097964 2023-10-09T05:31:04.535 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:04.560 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:04.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:04.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:04.664 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:04.705 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:04.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:04.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:04.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:04 smithi133 bash[105589]: cluster 2023-10-09T05:31:04.276600+0000 mgr.y (mgr.25277) 1288 : cluster 0 pgmap v1379: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T05:31:04.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:04 smithi133 bash[102858]: cluster 2023-10-09T05:31:04.276600+0000 mgr.y (mgr.25277) 1288 : cluster 0 pgmap v1379: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T05:31:04.811 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:04.841 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:04.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:04.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:04.945 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:04.956 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:04 smithi194 bash[92666]: cluster 2023-10-09T05:31:04.276600+0000 mgr.y (mgr.25277) 1288 : cluster 0 pgmap v1379: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T05:31:04.970 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:04.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:04.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:05.073 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:05.094 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:05.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:05.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:05.196 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:05.229 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:05.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:05.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:05.333 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:05.360 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:05.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:05.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:05.464 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:05.491 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:05.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:05.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:05.518 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 26 16 42000 41984 100.912 73.3125 0.00730974 0.00990405 2023-10-09T05:31:05.596 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:05.633 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:05.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:05.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:05.736 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:05.765 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:05.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:05.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:05.867 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:05.891 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:05.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:05.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:05.996 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:06.025 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:06.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:06.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:06.129 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:06.157 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:06.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:06.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:06.262 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:06.288 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:06.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:06.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:06.391 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:06.420 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:06.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:06.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:06.518 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 27 16 43086 43070 99.6887 67.875 0.00513987 0.0100261 2023-10-09T05:31:06.525 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:06.552 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:06.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:06.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:06.657 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:06.680 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:06.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:06.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:06.783 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:06.812 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:06.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:06.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:06.917 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:06.945 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:06.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:06.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:07.048 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:07.072 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:07.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:07.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:07.174 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:07.205 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:07.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:07.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:07.312 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:07.341 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:07.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:07.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:07.446 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:07.472 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:07.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:07.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:07.518 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 28 16 44005 43989 98.1795 57.4375 0.0178483 0.01018 2023-10-09T05:31:07.578 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:07.587 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:07 smithi133 bash[102858]: cluster 2023-10-09T05:31:06.277521+0000 mgr.y (mgr.25277) 1289 : cluster 0 pgmap v1380: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T05:31:07.587 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:07 smithi133 bash[105589]: cluster 2023-10-09T05:31:06.277521+0000 mgr.y (mgr.25277) 1289 : cluster 0 pgmap v1380: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T05:31:07.604 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:07.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:07.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:07.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:07 smithi194 bash[92666]: cluster 2023-10-09T05:31:06.277521+0000 mgr.y (mgr.25277) 1289 : cluster 0 pgmap v1380: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T05:31:07.709 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:07.743 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:07.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:07.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:07.846 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:07.870 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:31:07.870 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:31:07.872 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:07.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:07.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:07.976 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:08.002 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:08.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:08.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:08.105 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:08.133 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:08.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:08.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:08.237 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:08.264 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:08.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:08.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:08.368 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:08.394 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:08.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:08.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:08.498 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:08.518 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 29 16 45211 45195 97.393 75.375 0.012604 0.0102605 2023-10-09T05:31:08.533 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:08.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:08.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:08.636 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:08.657 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:08.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:08.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:08.760 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:08.789 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:08.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:08.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:08.879 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:31:08.880 INFO:teuthology.orchestra.run.smithi133.stderr:dumped all 2023-10-09T05:31:08.893 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:08.920 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:08.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:08.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:09.025 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:09.050 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:09.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:09.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:09.156 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:09.244 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:09.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:09.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:09.337 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:09 smithi194 bash[92666]: cluster 2023-10-09T05:31:08.278118+0000 mgr.y (mgr.25277) 1290 : cluster 0 pgmap v1381: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T05:31:09.349 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:09.376 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:09.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:09.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:09.480 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:09.511 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:09.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:09.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:09.518 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 30 15 46261 46246 96.3358 65.6875 0.0165829 0.0103738 2023-10-09T05:31:09.615 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:09.624 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:09 smithi133 bash[102858]: cluster 2023-10-09T05:31:08.278118+0000 mgr.y (mgr.25277) 1290 : cluster 0 pgmap v1381: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T05:31:09.624 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:09 smithi133 bash[105589]: cluster 2023-10-09T05:31:08.278118+0000 mgr.y (mgr.25277) 1290 : cluster 0 pgmap v1381: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T05:31:09.644 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:09.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:09.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:09.750 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:09.781 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:09.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:09.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:09.883 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:09.912 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:09.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:09.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:10.017 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:10.045 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:10.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:10.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:10.149 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:10.178 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:10.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:10.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:10.281 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:10.303 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:10.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:10.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:10.407 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:10.418 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:10 smithi194 bash[92666]: audit 2023-10-09T05:31:08.866190+0000 mgr.y (mgr.25277) 1291 : audit 0 from='client.55198 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:31:10.435 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:10.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:10.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:10.518 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 31 16 47306 47290 95.3329 65.25 0.0181405 0.010483 2023-10-09T05:31:10.538 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:10.569 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:10.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:10.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:10.634 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:10 smithi133 bash[102858]: audit 2023-10-09T05:31:08.866190+0000 mgr.y (mgr.25277) 1291 : audit 0 from='client.55198 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:31:10.635 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:10 smithi133 bash[105589]: audit 2023-10-09T05:31:08.866190+0000 mgr.y (mgr.25277) 1291 : audit 0 from='client.55198 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:31:10.673 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:10.700 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:10.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:10.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:10.804 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:10.835 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:10.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:10.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:10.931 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:31:10.940 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:10.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:10.974 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:10.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:10.983 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:31:11.037 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:31:11.075 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:11.078 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:31:11.099 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:11.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:11.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:11.116 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:31:11.150 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:31:11.182 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:31:11.202 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:11.216 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:31:11.227 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:31:11.227 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:31:11.228 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:31:11.228 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:31:11.228 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:31:11.228 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:31:11.228 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:31:11.228 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:31:11.229 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:11.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:11.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:11.337 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:11.371 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:11.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:11.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:11.378 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is set 2023-10-09T05:31:11.475 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:11.488 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:11 smithi133 bash[102858]: cluster 2023-10-09T05:31:10.278640+0000 mgr.y (mgr.25277) 1292 : cluster 0 pgmap v1382: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-10-09T05:31:11.488 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:11 smithi133 bash[102858]: audit 2023-10-09T05:31:10.448355+0000 mon.c (mon.2) 122 : audit 1 from='client.? 172.21.15.133:0/4067783957' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:31:11.488 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:11 smithi133 bash[102858]: audit 2023-10-09T05:31:10.449225+0000 mon.a (mon.0) 1635 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:31:11.488 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:11 smithi133 bash[102858]: audit 2023-10-09T05:31:10.886065+0000 mon.c (mon.2) 123 : audit 0 from='client.? 172.21.15.133:0/3925399780' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:31:11.488 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:11 smithi133 bash[105589]: cluster 2023-10-09T05:31:10.278640+0000 mgr.y (mgr.25277) 1292 : cluster 0 pgmap v1382: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-10-09T05:31:11.488 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:11 smithi133 bash[105589]: audit 2023-10-09T05:31:10.448355+0000 mon.c (mon.2) 122 : audit 1 from='client.? 172.21.15.133:0/4067783957' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:31:11.489 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:11 smithi133 bash[105589]: audit 2023-10-09T05:31:10.449225+0000 mon.a (mon.0) 1635 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:31:11.489 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:11 smithi133 bash[105589]: audit 2023-10-09T05:31:10.886065+0000 mon.c (mon.2) 123 : audit 0 from='client.? 172.21.15.133:0/3925399780' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:31:11.520 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 32 16 48416 48400 94.5215 69.375 0.0103718 0.0105734 2023-10-09T05:31:11.525 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:11.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:11.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:11.629 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:11.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:11 smithi194 bash[92666]: cluster 2023-10-09T05:31:10.278640+0000 mgr.y (mgr.25277) 1292 : cluster 0 pgmap v1382: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-10-09T05:31:11.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:11 smithi194 bash[92666]: audit 2023-10-09T05:31:10.448355+0000 mon.c (mon.2) 122 : audit 1 from='client.? 172.21.15.133:0/4067783957' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:31:11.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:11 smithi194 bash[92666]: audit 2023-10-09T05:31:10.449225+0000 mon.a (mon.0) 1635 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:31:11.638 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:11 smithi194 bash[92666]: audit 2023-10-09T05:31:10.886065+0000 mon.c (mon.2) 123 : audit 0 from='client.? 172.21.15.133:0/3925399780' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:31:11.670 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:11.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:11.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:11.775 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:11.823 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:11.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:11.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:11.926 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:11.969 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:11.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:11.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:12.073 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:12.098 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:12.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:12.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:12.202 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:12.239 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:12.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:12.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:12.346 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:12.374 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:12.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:12.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:12.477 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:12.487 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:12 smithi194 bash[92666]: cluster 2023-10-09T05:31:11.345755+0000 mon.a (mon.0) 1636 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:31:12.487 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:12 smithi194 bash[92666]: audit 2023-10-09T05:31:11.365622+0000 mon.a (mon.0) 1637 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:31:12.487 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:12 smithi194 bash[92666]: cluster 2023-10-09T05:31:11.365769+0000 mon.a (mon.0) 1638 : cluster 0 osdmap e294: 8 total, 8 up, 6 in 2023-10-09T05:31:12.488 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:12 smithi194 bash[92666]: cluster 2023-10-09T05:31:11.366945+0000 osd.7 (osd.7) 248 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T05:31:12.488 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:12 smithi194 bash[92666]: cluster 2023-10-09T05:31:11.367730+0000 osd.6 (osd.6) 251 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T05:31:12.488 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:12 smithi194 bash[92666]: cluster 2023-10-09T05:31:11.371472+0000 osd.7 (osd.7) 249 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T05:31:12.504 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:12.519 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 33 16 49585 49569 93.8711 73.0625 0.0144513 0.0106464 2023-10-09T05:31:12.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:12.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:12.622 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:12.638 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:12 smithi133 bash[102858]: cluster 2023-10-09T05:31:11.345755+0000 mon.a (mon.0) 1636 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:31:12.639 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:12 smithi133 bash[102858]: audit 2023-10-09T05:31:11.365622+0000 mon.a (mon.0) 1637 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:31:12.639 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:12 smithi133 bash[102858]: cluster 2023-10-09T05:31:11.365769+0000 mon.a (mon.0) 1638 : cluster 0 osdmap e294: 8 total, 8 up, 6 in 2023-10-09T05:31:12.639 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:12 smithi133 bash[102858]: cluster 2023-10-09T05:31:11.366945+0000 osd.7 (osd.7) 248 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T05:31:12.639 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:12 smithi133 bash[102858]: cluster 2023-10-09T05:31:11.367730+0000 osd.6 (osd.6) 251 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T05:31:12.639 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:12 smithi133 bash[102858]: cluster 2023-10-09T05:31:11.371472+0000 osd.7 (osd.7) 249 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T05:31:12.640 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:12 smithi133 bash[105589]: cluster 2023-10-09T05:31:11.345755+0000 mon.a (mon.0) 1636 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:31:12.640 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:12 smithi133 bash[105589]: audit 2023-10-09T05:31:11.365622+0000 mon.a (mon.0) 1637 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:31:12.640 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:12 smithi133 bash[105589]: cluster 2023-10-09T05:31:11.365769+0000 mon.a (mon.0) 1638 : cluster 0 osdmap e294: 8 total, 8 up, 6 in 2023-10-09T05:31:12.640 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:12 smithi133 bash[105589]: cluster 2023-10-09T05:31:11.366945+0000 osd.7 (osd.7) 248 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T05:31:12.640 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:12 smithi133 bash[105589]: cluster 2023-10-09T05:31:11.367730+0000 osd.6 (osd.6) 251 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T05:31:12.640 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:12 smithi133 bash[105589]: cluster 2023-10-09T05:31:11.371472+0000 osd.7 (osd.7) 249 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T05:31:12.653 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:12.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:12.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:12.761 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:12.797 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:12.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:12.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:12.902 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:12.941 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:12.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:12.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:13.045 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:13.066 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:13.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:13.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:13.168 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:13.199 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:13.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:13.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:13.303 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:13.327 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:13.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:13.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:13.432 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:13.465 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:13.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:13.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:13.519 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 34 16 50378 50362 92.5677 49.5625 0.00599606 0.0107461 2023-10-09T05:31:13.543 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:13 smithi133 bash[105589]: cluster 2023-10-09T05:31:12.279031+0000 mgr.y (mgr.25277) 1293 : cluster 0 pgmap v1384: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-10-09T05:31:13.543 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:13 smithi133 bash[105589]: cluster 2023-10-09T05:31:13.108422+0000 osd.4 (osd.4) 267 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T05:31:13.568 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:13.608 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:13.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:13.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:13.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:13 smithi194 bash[92666]: cluster 2023-10-09T05:31:12.279031+0000 mgr.y (mgr.25277) 1293 : cluster 0 pgmap v1384: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-10-09T05:31:13.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:13 smithi194 bash[92666]: cluster 2023-10-09T05:31:13.108422+0000 osd.4 (osd.4) 267 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T05:31:13.713 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:13.735 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T05:31:13.743 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:13.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:13.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:13.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:13 smithi133 bash[102858]: cluster 2023-10-09T05:31:12.279031+0000 mgr.y (mgr.25277) 1293 : cluster 0 pgmap v1384: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-10-09T05:31:13.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:13 smithi133 bash[102858]: cluster 2023-10-09T05:31:13.108422+0000 osd.4 (osd.4) 267 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T05:31:13.847 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:13.869 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:13.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:13.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:13.972 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:13.999 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:14.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:14.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:14.102 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:14.135 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:14.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:14.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:14.241 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:14.273 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:14.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:14.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:14.376 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:14.403 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:14.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:14.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:14.508 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:14.519 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 35 16 51203 51187 91.396 51.5625 0.0138443 0.0109356 2023-10-09T05:31:14.520 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:14 smithi133 bash[102858]: audit 2023-10-09T05:31:13.482292+0000 mon.a (mon.0) 1639 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:31:14.520 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:14 smithi133 bash[102858]: cluster 2023-10-09T05:31:13.542595+0000 osd.5 (osd.5) 268 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T05:31:14.520 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:31:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:31:14] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.43.0" 2023-10-09T05:31:14.521 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:14 smithi133 bash[105589]: audit 2023-10-09T05:31:13.482292+0000 mon.a (mon.0) 1639 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:31:14.521 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:14 smithi133 bash[105589]: cluster 2023-10-09T05:31:13.542595+0000 osd.5 (osd.5) 268 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T05:31:14.539 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:14.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:14.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:14.603 INFO:teuthology.orchestra.run.smithi133.stdout:{"pg_ready":true,"pg_map":{"version":1381,"stamp":"2023-10-09T05:31:08.277765+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2703890336,"num_objects":41273,"num_object_clones":0,"num_object_copies":123819,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":41273,"num_whiteouts":0,"num_read":509,"num_read_kb":934,"num_write":41549,"num_write_kb":2642381,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":69,"num_bytes_recovered":4154180,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":41471,"ondisk_log_size":41471,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":13348728,"kb_used_data":7844808,"kb_used_omap":20,"kb_used_meta":5503851,"kb_avail":549122184,"statfs":{"total":575970213888,"available":562301116416,"internally_reserved":0,"allocated":8033083392,"data_stored":8025223769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":5635944078},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":851509248,"num_objects":12993,"num_object_clones":0,"num_object_copies":38979,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12993,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12993,"num_write_kb":831552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003785"},"pg_stats":[{"pgid":"3.1a","version":"69'2","reported_seq":160,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:34.757365+0000","last_change":"2023-10-09T05:30:03.378323+0000","last_active":"2023-10-09T05:30:34.757365+0000","last_peered":"2023-10-09T05:30:34.757365+0000","last_clean":"2023-10-09T05:30:34.757365+0000","last_became_active":"2023-10-09T05:30:03.377871+0000","last_became_peered":"2023-10-09T05:30:03.377871+0000","last_unstale":"2023-10-09T05:30:34.757365+0000","last_undegraded":"2023-10-09T05:30:34.757365+0000","last_fullsized":"2023-10-09T05:30:34.757365+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:09.181202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":286,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.707941+0000","last_change":"2023-10-09T05:30:03.377792+0000","last_active":"2023-10-09T05:30:35.707941+0000","last_peered":"2023-10-09T05:30:35.707941+0000","last_clean":"2023-10-09T05:30:35.707941+0000","last_became_active":"2023-10-09T05:30:03.377456+0000","last_became_peered":"2023-10-09T05:30:03.377456+0000","last_unstale":"2023-10-09T05:30:35.707941+0000","last_undegraded":"2023-10-09T05:30:35.707941+0000","last_fullsized":"2023-10-09T05:30:35.707941+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:35:57.941046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":169,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:34.757895+0000","last_change":"2023-10-09T05:30:03.377059+0000","last_active":"2023-10-09T05:30:34.757895+0000","last_peered":"2023-10-09T05:30:34.757895+0000","last_clean":"2023-10-09T05:30:34.757895+0000","last_became_active":"2023-10-09T05:30:03.376460+0000","last_became_peered":"2023-10-09T05:30:03.376460+0000","last_unstale":"2023-10-09T05:30:34.757895+0000","last_undegraded":"2023-10-09T05:30:34.757895+0000","last_fullsized":"2023-10-09T05:30:34.757895+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:01:08.558120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":295,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T05:30:36.675839+0000","last_change":"2023-10-09T05:18:26.422923+0000","last_active":"2023-10-09T05:30:36.675839+0000","last_peered":"2023-10-09T05:30:36.675839+0000","last_clean":"2023-10-09T05:30:36.675839+0000","last_became_active":"2023-10-09T05:18:26.422508+0000","last_became_peered":"2023-10-09T05:18:26.422508+0000","last_unstale":"2023-10-09T05:30:36.675839+0000","last_undegraded":"2023-10-09T05:30:36.675839+0000","last_fullsized":"2023-10-09T05:30:36.675839+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:00.876037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":313,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.674708+0000","last_change":"2023-10-09T05:23:18.188378+0000","last_active":"2023-10-09T05:30:35.674708+0000","last_peered":"2023-10-09T05:30:35.674708+0000","last_clean":"2023-10-09T05:30:35.674708+0000","last_became_active":"2023-10-09T05:23:18.186629+0000","last_became_peered":"2023-10-09T05:23:18.186629+0000","last_unstale":"2023-10-09T05:30:35.674708+0000","last_undegraded":"2023-10-09T05:30:35.674708+0000","last_fullsized":"2023-10-09T05:30:35.674708+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:16.444225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":283,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.465919+0000","last_change":"2023-10-09T05:13:03.039637+0000","last_active":"2023-10-09T05:30:35.465919+0000","last_peered":"2023-10-09T05:30:35.465919+0000","last_clean":"2023-10-09T05:30:35.465919+0000","last_became_active":"2023-10-09T05:13:03.039268+0000","last_became_peered":"2023-10-09T05:13:03.039268+0000","last_unstale":"2023-10-09T05:30:35.465919+0000","last_undegraded":"2023-10-09T05:30:35.465919+0000","last_fullsized":"2023-10-09T05:30:35.465919+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:19.875767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":282,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T05:30:36.675794+0000","last_change":"2023-10-09T05:22:47.216769+0000","last_active":"2023-10-09T05:30:36.675794+0000","last_peered":"2023-10-09T05:30:36.675794+0000","last_clean":"2023-10-09T05:30:36.675794+0000","last_became_active":"2023-10-09T05:22:47.215847+0000","last_became_peered":"2023-10-09T05:22:47.215847+0000","last_unstale":"2023-10-09T05:30:36.675794+0000","last_undegraded":"2023-10-09T05:30:36.675794+0000","last_fullsized":"2023-10-09T05:30:36.675794+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:51.392675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":269,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.674750+0000","last_change":"2023-10-09T05:09:57.849695+0000","last_active":"2023-10-09T05:30:35.674750+0000","last_peered":"2023-10-09T05:30:35.674750+0000","last_clean":"2023-10-09T05:30:35.674750+0000","last_became_active":"2023-10-09T05:09:57.848478+0000","last_became_peered":"2023-10-09T05:09:57.848478+0000","last_unstale":"2023-10-09T05:30:35.674750+0000","last_undegraded":"2023-10-09T05:30:35.674750+0000","last_fullsized":"2023-10-09T05:30:35.674750+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:26.115312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":102,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.688748+0000","last_change":"2023-10-09T05:23:18.194144+0000","last_active":"2023-10-09T05:30:35.688748+0000","last_peered":"2023-10-09T05:30:35.688748+0000","last_clean":"2023-10-09T05:30:35.688748+0000","last_became_active":"2023-10-09T05:23:18.193390+0000","last_became_peered":"2023-10-09T05:23:18.193390+0000","last_unstale":"2023-10-09T05:30:35.688748+0000","last_undegraded":"2023-10-09T05:30:35.688748+0000","last_fullsized":"2023-10-09T05:30:35.688748+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:22.549362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":282,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.674776+0000","last_change":"2023-10-09T05:18:26.418431+0000","last_active":"2023-10-09T05:30:35.674776+0000","last_peered":"2023-10-09T05:30:35.674776+0000","last_clean":"2023-10-09T05:30:35.674776+0000","last_became_active":"2023-10-09T05:18:26.418100+0000","last_became_peered":"2023-10-09T05:18:26.418100+0000","last_unstale":"2023-10-09T05:30:35.674776+0000","last_undegraded":"2023-10-09T05:30:35.674776+0000","last_fullsized":"2023-10-09T05:30:35.674776+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:21.241286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":104,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.688085+0000","last_change":"2023-10-09T05:30:03.413915+0000","last_active":"2023-10-09T05:30:35.688085+0000","last_peered":"2023-10-09T05:30:35.688085+0000","last_clean":"2023-10-09T05:30:35.688085+0000","last_became_active":"2023-10-09T05:30:03.391102+0000","last_became_peered":"2023-10-09T05:30:03.391102+0000","last_unstale":"2023-10-09T05:30:35.688085+0000","last_undegraded":"2023-10-09T05:30:35.688085+0000","last_fullsized":"2023-10-09T05:30:35.688085+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:13.024050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":54,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:34.757176+0000","last_change":"2023-10-09T05:23:18.199181+0000","last_active":"2023-10-09T05:30:34.757176+0000","last_peered":"2023-10-09T05:30:34.757176+0000","last_clean":"2023-10-09T05:30:34.757176+0000","last_became_active":"2023-10-09T05:23:18.198401+0000","last_became_peered":"2023-10-09T05:23:18.198401+0000","last_unstale":"2023-10-09T05:30:34.757176+0000","last_undegraded":"2023-10-09T05:30:34.757176+0000","last_fullsized":"2023-10-09T05:30:34.757176+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:55:49.891420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'23","reported_seq":282,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.687938+0000","last_change":"2023-10-09T05:30:03.423577+0000","last_active":"2023-10-09T05:30:35.687938+0000","last_peered":"2023-10-09T05:30:35.687938+0000","last_clean":"2023-10-09T05:30:35.687938+0000","last_became_active":"2023-10-09T05:30:03.394939+0000","last_became_peered":"2023-10-09T05:30:03.394939+0000","last_unstale":"2023-10-09T05:30:35.687938+0000","last_undegraded":"2023-10-09T05:30:35.687938+0000","last_fullsized":"2023-10-09T05:30:35.687938+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:19:34.968600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":141,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.464266+0000","last_change":"2023-10-09T05:30:03.382454+0000","last_active":"2023-10-09T05:30:35.464266+0000","last_peered":"2023-10-09T05:30:35.464266+0000","last_clean":"2023-10-09T05:30:35.464266+0000","last_became_active":"2023-10-09T05:30:03.380820+0000","last_became_peered":"2023-10-09T05:30:03.380820+0000","last_unstale":"2023-10-09T05:30:35.464266+0000","last_undegraded":"2023-10-09T05:30:35.464266+0000","last_fullsized":"2023-10-09T05:30:35.464266+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:02.666124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":104,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.688170+0000","last_change":"2023-10-09T05:30:03.407302+0000","last_active":"2023-10-09T05:30:35.688170+0000","last_peered":"2023-10-09T05:30:35.688170+0000","last_clean":"2023-10-09T05:30:35.688170+0000","last_became_active":"2023-10-09T05:30:03.374209+0000","last_became_peered":"2023-10-09T05:30:03.374209+0000","last_unstale":"2023-10-09T05:30:35.688170+0000","last_undegraded":"2023-10-09T05:30:35.688170+0000","last_fullsized":"2023-10-09T05:30:35.688170+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:53.750483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":323,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.707518+0000","last_change":"2023-10-09T05:23:18.187593+0000","last_active":"2023-10-09T05:30:35.707518+0000","last_peered":"2023-10-09T05:30:35.707518+0000","last_clean":"2023-10-09T05:30:35.707518+0000","last_became_active":"2023-10-09T05:23:18.186445+0000","last_became_peered":"2023-10-09T05:23:18.186445+0000","last_unstale":"2023-10-09T05:30:35.707518+0000","last_undegraded":"2023-10-09T05:30:35.707518+0000","last_fullsized":"2023-10-09T05:30:35.707518+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:11.327635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":291,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:34.756938+0000","last_change":"2023-10-09T05:18:26.424399+0000","last_active":"2023-10-09T05:30:34.756938+0000","last_peered":"2023-10-09T05:30:34.756938+0000","last_clean":"2023-10-09T05:30:34.756938+0000","last_became_active":"2023-10-09T05:18:26.422122+0000","last_became_peered":"2023-10-09T05:18:26.422122+0000","last_unstale":"2023-10-09T05:30:34.756938+0000","last_undegraded":"2023-10-09T05:30:34.756938+0000","last_fullsized":"2023-10-09T05:30:34.756938+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:39.393756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":82,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.688475+0000","last_change":"2023-10-09T05:18:26.422950+0000","last_active":"2023-10-09T05:30:35.688475+0000","last_peered":"2023-10-09T05:30:35.688475+0000","last_clean":"2023-10-09T05:30:35.688475+0000","last_became_active":"2023-10-09T05:18:26.422592+0000","last_became_peered":"2023-10-09T05:18:26.422592+0000","last_unstale":"2023-10-09T05:30:35.688475+0000","last_undegraded":"2023-10-09T05:30:35.688475+0000","last_fullsized":"2023-10-09T05:30:35.688475+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:20.701107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":92,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.687286+0000","last_change":"2023-10-09T05:30:03.395964+0000","last_active":"2023-10-09T05:30:35.687286+0000","last_peered":"2023-10-09T05:30:35.687286+0000","last_clean":"2023-10-09T05:30:35.687286+0000","last_became_active":"2023-10-09T05:30:03.395634+0000","last_became_peered":"2023-10-09T05:30:03.395634+0000","last_unstale":"2023-10-09T05:30:35.687286+0000","last_undegraded":"2023-10-09T05:30:35.687286+0000","last_fullsized":"2023-10-09T05:30:35.687286+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:31.601758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":292,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T05:30:36.675945+0000","last_change":"2023-10-09T05:18:26.421678+0000","last_active":"2023-10-09T05:30:36.675945+0000","last_peered":"2023-10-09T05:30:36.675945+0000","last_clean":"2023-10-09T05:30:36.675945+0000","last_became_active":"2023-10-09T05:18:26.419673+0000","last_became_peered":"2023-10-09T05:18:26.419673+0000","last_unstale":"2023-10-09T05:30:36.675945+0000","last_undegraded":"2023-10-09T05:30:36.675945+0000","last_fullsized":"2023-10-09T05:30:36.675945+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:38.876021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":286,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.464756+0000","last_change":"2023-10-09T05:13:03.047339+0000","last_active":"2023-10-09T05:30:35.464756+0000","last_peered":"2023-10-09T05:30:35.464756+0000","last_clean":"2023-10-09T05:30:35.464756+0000","last_became_active":"2023-10-09T05:13:03.040038+0000","last_became_peered":"2023-10-09T05:13:03.040038+0000","last_unstale":"2023-10-09T05:30:35.464756+0000","last_undegraded":"2023-10-09T05:30:35.464756+0000","last_fullsized":"2023-10-09T05:30:35.464756+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:56.726249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":304,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.708226+0000","last_change":"2023-10-09T05:18:26.415173+0000","last_active":"2023-10-09T05:30:35.708226+0000","last_peered":"2023-10-09T05:30:35.708226+0000","last_clean":"2023-10-09T05:30:35.708226+0000","last_became_active":"2023-10-09T05:18:26.414452+0000","last_became_peered":"2023-10-09T05:18:26.414452+0000","last_unstale":"2023-10-09T05:30:35.708226+0000","last_undegraded":"2023-10-09T05:30:35.708226+0000","last_fullsized":"2023-10-09T05:30:35.708226+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:34.379767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"293'2573","reported_seq":2585,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T05:31:04.523436+0000","last_change":"2023-10-09T05:30:14.775023+0000","last_active":"2023-10-09T05:31:04.523436+0000","last_peered":"2023-10-09T05:31:04.523436+0000","last_clean":"2023-10-09T05:31:04.523436+0000","last_became_active":"2023-10-09T05:30:14.774270+0000","last_became_peered":"2023-10-09T05:30:14.774270+0000","last_unstale":"2023-10-09T05:31:04.523436+0000","last_undegraded":"2023-10-09T05:31:04.523436+0000","last_fullsized":"2023-10-09T05:31:04.523436+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_clean_scrub_stamp":"2023-10-09T05:30:13.731262+0000","objects_scrubbed":0,"log_size":2573,"log_dups_size":0,"ondisk_log_size":2573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:18:31.034636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168624128,"num_objects":2573,"num_object_clones":0,"num_object_copies":7719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2573,"num_write_kb":164672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":100,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.687423+0000","last_change":"2023-10-09T05:30:03.409982+0000","last_active":"2023-10-09T05:30:35.687423+0000","last_peered":"2023-10-09T05:30:35.687423+0000","last_clean":"2023-10-09T05:30:35.687423+0000","last_became_active":"2023-10-09T05:30:03.386761+0000","last_became_peered":"2023-10-09T05:30:03.386761+0000","last_unstale":"2023-10-09T05:30:35.687423+0000","last_undegraded":"2023-10-09T05:30:35.687423+0000","last_fullsized":"2023-10-09T05:30:35.687423+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:17.372036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":92,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.688403+0000","last_change":"2023-10-09T05:30:03.379437+0000","last_active":"2023-10-09T05:30:35.688403+0000","last_peered":"2023-10-09T05:30:35.688403+0000","last_clean":"2023-10-09T05:30:35.688403+0000","last_became_active":"2023-10-09T05:30:03.379060+0000","last_became_peered":"2023-10-09T05:30:03.379060+0000","last_unstale":"2023-10-09T05:30:35.688403+0000","last_undegraded":"2023-10-09T05:30:35.688403+0000","last_fullsized":"2023-10-09T05:30:35.688403+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:45.459782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.0","version":"293'2492","reported_seq":2504,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T05:31:04.571536+0000","last_change":"2023-10-09T05:30:14.774171+0000","last_active":"2023-10-09T05:31:04.571536+0000","last_peered":"2023-10-09T05:31:04.571536+0000","last_clean":"2023-10-09T05:31:04.571536+0000","last_became_active":"2023-10-09T05:30:14.773741+0000","last_became_peered":"2023-10-09T05:30:14.773741+0000","last_unstale":"2023-10-09T05:31:04.571536+0000","last_undegraded":"2023-10-09T05:31:04.571536+0000","last_fullsized":"2023-10-09T05:31:04.571536+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_clean_scrub_stamp":"2023-10-09T05:30:13.731262+0000","objects_scrubbed":0,"log_size":2492,"log_dups_size":0,"ondisk_log_size":2492,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:41.765420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163315712,"num_objects":2492,"num_object_clones":0,"num_object_copies":7476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2492,"num_write_kb":159488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":100,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.687524+0000","last_change":"2023-10-09T05:30:03.416268+0000","last_active":"2023-10-09T05:30:35.687524+0000","last_peered":"2023-10-09T05:30:35.687524+0000","last_clean":"2023-10-09T05:30:35.687524+0000","last_became_active":"2023-10-09T05:30:03.392236+0000","last_became_peered":"2023-10-09T05:30:03.392236+0000","last_unstale":"2023-10-09T05:30:35.687524+0000","last_undegraded":"2023-10-09T05:30:35.687524+0000","last_fullsized":"2023-10-09T05:30:35.687524+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:18:34.616445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":157,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:34.757772+0000","last_change":"2023-10-09T05:30:03.376778+0000","last_active":"2023-10-09T05:30:34.757772+0000","last_peered":"2023-10-09T05:30:34.757772+0000","last_clean":"2023-10-09T05:30:34.757772+0000","last_became_active":"2023-10-09T05:30:03.376056+0000","last_became_peered":"2023-10-09T05:30:03.376056+0000","last_unstale":"2023-10-09T05:30:34.757772+0000","last_undegraded":"2023-10-09T05:30:34.757772+0000","last_fullsized":"2023-10-09T05:30:34.757772+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:06.957639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.3","version":"293'2438","reported_seq":2450,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T05:31:04.466789+0000","last_change":"2023-10-09T05:30:14.775377+0000","last_active":"2023-10-09T05:31:04.466789+0000","last_peered":"2023-10-09T05:31:04.466789+0000","last_clean":"2023-10-09T05:31:04.466789+0000","last_became_active":"2023-10-09T05:30:14.774935+0000","last_became_peered":"2023-10-09T05:30:14.774935+0000","last_unstale":"2023-10-09T05:31:04.466789+0000","last_undegraded":"2023-10-09T05:31:04.466789+0000","last_fullsized":"2023-10-09T05:31:04.466789+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_clean_scrub_stamp":"2023-10-09T05:30:13.731262+0000","objects_scrubbed":0,"log_size":2438,"log_dups_size":0,"ondisk_log_size":2438,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:55.757367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159776768,"num_objects":2438,"num_object_clones":0,"num_object_copies":7314,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2438,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2438,"num_write_kb":156032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":86,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.688550+0000","last_change":"2023-10-09T05:18:26.437953+0000","last_active":"2023-10-09T05:30:35.688550+0000","last_peered":"2023-10-09T05:30:35.688550+0000","last_clean":"2023-10-09T05:30:35.688550+0000","last_became_active":"2023-10-09T05:18:26.422703+0000","last_became_peered":"2023-10-09T05:18:26.422703+0000","last_unstale":"2023-10-09T05:30:35.688550+0000","last_undegraded":"2023-10-09T05:30:35.688550+0000","last_fullsized":"2023-10-09T05:30:35.688550+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:13.926811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":323,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.465474+0000","last_change":"2023-10-09T05:18:26.423526+0000","last_active":"2023-10-09T05:30:35.465474+0000","last_peered":"2023-10-09T05:30:35.465474+0000","last_clean":"2023-10-09T05:30:35.465474+0000","last_became_active":"2023-10-09T05:18:26.423092+0000","last_became_peered":"2023-10-09T05:18:26.423092+0000","last_unstale":"2023-10-09T05:30:35.465474+0000","last_undegraded":"2023-10-09T05:30:35.465474+0000","last_fullsized":"2023-10-09T05:30:35.465474+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:54.877664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"293'2540","reported_seq":2553,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T05:31:05.293360+0000","last_change":"2023-10-09T05:30:14.771449+0000","last_active":"2023-10-09T05:31:05.293360+0000","last_peered":"2023-10-09T05:31:05.293360+0000","last_clean":"2023-10-09T05:31:05.293360+0000","last_became_active":"2023-10-09T05:30:14.771165+0000","last_became_peered":"2023-10-09T05:30:14.771165+0000","last_unstale":"2023-10-09T05:31:05.293360+0000","last_undegraded":"2023-10-09T05:31:05.293360+0000","last_fullsized":"2023-10-09T05:31:05.293360+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_clean_scrub_stamp":"2023-10-09T05:30:13.731262+0000","objects_scrubbed":0,"log_size":2540,"log_dups_size":0,"ondisk_log_size":2540,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:23:19.444872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166461440,"num_objects":2540,"num_object_clones":0,"num_object_copies":7620,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2540,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2540,"num_write_kb":162560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":216,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.759217+0000","last_change":"2023-10-09T05:13:03.037926+0000","last_active":"2023-10-09T05:30:35.759217+0000","last_peered":"2023-10-09T05:30:35.759217+0000","last_clean":"2023-10-09T05:30:35.759217+0000","last_became_active":"2023-10-09T05:13:03.037639+0000","last_became_peered":"2023-10-09T05:13:03.037639+0000","last_unstale":"2023-10-09T05:30:35.759217+0000","last_undegraded":"2023-10-09T05:30:35.759217+0000","last_fullsized":"2023-10-09T05:30:35.759217+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:52.977143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":284,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T05:30:36.675252+0000","last_change":"2023-10-09T05:30:03.387239+0000","last_active":"2023-10-09T05:30:36.675252+0000","last_peered":"2023-10-09T05:30:36.675252+0000","last_clean":"2023-10-09T05:30:36.675252+0000","last_became_active":"2023-10-09T05:30:03.386633+0000","last_became_peered":"2023-10-09T05:30:03.386633+0000","last_unstale":"2023-10-09T05:30:36.675252+0000","last_undegraded":"2023-10-09T05:30:36.675252+0000","last_fullsized":"2023-10-09T05:30:36.675252+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:51.537133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"293'2465","reported_seq":2478,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T05:31:04.473563+0000","last_change":"2023-10-09T05:30:14.771941+0000","last_active":"2023-10-09T05:31:04.473563+0000","last_peered":"2023-10-09T05:31:04.473563+0000","last_clean":"2023-10-09T05:31:04.473563+0000","last_became_active":"2023-10-09T05:30:14.771603+0000","last_became_peered":"2023-10-09T05:30:14.771603+0000","last_unstale":"2023-10-09T05:31:04.473563+0000","last_undegraded":"2023-10-09T05:31:04.473563+0000","last_fullsized":"2023-10-09T05:31:04.473563+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_clean_scrub_stamp":"2023-10-09T05:30:13.731262+0000","objects_scrubbed":0,"log_size":2465,"log_dups_size":0,"ondisk_log_size":2465,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:58.671686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161546240,"num_objects":2465,"num_object_clones":0,"num_object_copies":7395,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2465,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2465,"num_write_kb":157760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'3","reported_seq":261,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.673993+0000","last_change":"2023-10-09T05:30:03.399748+0000","last_active":"2023-10-09T05:30:35.673993+0000","last_peered":"2023-10-09T05:30:35.673993+0000","last_clean":"2023-10-09T05:30:35.673993+0000","last_became_active":"2023-10-09T05:30:03.384427+0000","last_became_peered":"2023-10-09T05:30:03.384427+0000","last_unstale":"2023-10-09T05:30:35.673993+0000","last_undegraded":"2023-10-09T05:30:35.673993+0000","last_fullsized":"2023-10-09T05:30:35.673993+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:01.128892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":92,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.673912+0000","last_change":"2023-10-09T05:30:03.378783+0000","last_active":"2023-10-09T05:30:35.673912+0000","last_peered":"2023-10-09T05:30:35.673912+0000","last_clean":"2023-10-09T05:30:35.673912+0000","last_became_active":"2023-10-09T05:30:03.378436+0000","last_became_peered":"2023-10-09T05:30:03.378436+0000","last_unstale":"2023-10-09T05:30:35.673912+0000","last_undegraded":"2023-10-09T05:30:35.673912+0000","last_fullsized":"2023-10-09T05:30:35.673912+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:54.102857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"293'2568","reported_seq":2580,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T05:31:04.542047+0000","last_change":"2023-10-09T05:30:14.773124+0000","last_active":"2023-10-09T05:31:04.542047+0000","last_peered":"2023-10-09T05:31:04.542047+0000","last_clean":"2023-10-09T05:31:04.542047+0000","last_became_active":"2023-10-09T05:30:14.772510+0000","last_became_peered":"2023-10-09T05:30:14.772510+0000","last_unstale":"2023-10-09T05:31:04.542047+0000","last_undegraded":"2023-10-09T05:31:04.542047+0000","last_fullsized":"2023-10-09T05:31:04.542047+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_clean_scrub_stamp":"2023-10-09T05:30:13.731262+0000","objects_scrubbed":0,"log_size":2568,"log_dups_size":0,"ondisk_log_size":2568,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:32:10.557323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168296448,"num_objects":2568,"num_object_clones":0,"num_object_copies":7704,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2568,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2568,"num_write_kb":164352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":304,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.464277+0000","last_change":"2023-10-09T05:30:03.381940+0000","last_active":"2023-10-09T05:30:35.464277+0000","last_peered":"2023-10-09T05:30:35.464277+0000","last_clean":"2023-10-09T05:30:35.464277+0000","last_became_active":"2023-10-09T05:30:03.381526+0000","last_became_peered":"2023-10-09T05:30:03.381526+0000","last_unstale":"2023-10-09T05:30:35.464277+0000","last_undegraded":"2023-10-09T05:30:35.464277+0000","last_fullsized":"2023-10-09T05:30:35.464277+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:06.078801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":260,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T05:30:15.778311+0000","last_change":"2023-10-09T05:30:03.387924+0000","last_active":"2023-10-09T05:30:15.778311+0000","last_peered":"2023-10-09T05:30:15.778311+0000","last_clean":"2023-10-09T05:30:15.778311+0000","last_became_active":"2023-10-09T05:30:03.387493+0000","last_became_peered":"2023-10-09T05:30:03.387493+0000","last_unstale":"2023-10-09T05:30:15.778311+0000","last_undegraded":"2023-10-09T05:30:15.778311+0000","last_fullsized":"2023-10-09T05:30:15.778311+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:16.554514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"293'2581","reported_seq":2593,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T05:31:04.594810+0000","last_change":"2023-10-09T05:30:14.780783+0000","last_active":"2023-10-09T05:31:04.594810+0000","last_peered":"2023-10-09T05:31:04.594810+0000","last_clean":"2023-10-09T05:31:04.594810+0000","last_became_active":"2023-10-09T05:30:14.780530+0000","last_became_peered":"2023-10-09T05:30:14.780530+0000","last_unstale":"2023-10-09T05:31:04.594810+0000","last_undegraded":"2023-10-09T05:31:04.594810+0000","last_fullsized":"2023-10-09T05:31:04.594810+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_clean_scrub_stamp":"2023-10-09T05:30:13.731262+0000","objects_scrubbed":0,"log_size":2581,"log_dups_size":0,"ondisk_log_size":2581,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:46.869943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169148416,"num_objects":2581,"num_object_clones":0,"num_object_copies":7743,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2581,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2581,"num_write_kb":165184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":229,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T05:30:15.779352+0000","last_change":"2023-10-09T05:30:03.372784+0000","last_active":"2023-10-09T05:30:15.779352+0000","last_peered":"2023-10-09T05:30:15.779352+0000","last_clean":"2023-10-09T05:30:15.779352+0000","last_became_active":"2023-10-09T05:30:03.372458+0000","last_became_peered":"2023-10-09T05:30:03.372458+0000","last_unstale":"2023-10-09T05:30:15.779352+0000","last_undegraded":"2023-10-09T05:30:15.779352+0000","last_fullsized":"2023-10-09T05:30:15.779352+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:36:24.881545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":262,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.464636+0000","last_change":"2023-10-09T05:30:03.382201+0000","last_active":"2023-10-09T05:30:35.464636+0000","last_peered":"2023-10-09T05:30:35.464636+0000","last_clean":"2023-10-09T05:30:35.464636+0000","last_became_active":"2023-10-09T05:30:03.381727+0000","last_became_peered":"2023-10-09T05:30:03.381727+0000","last_unstale":"2023-10-09T05:30:35.464636+0000","last_undegraded":"2023-10-09T05:30:35.464636+0000","last_fullsized":"2023-10-09T05:30:35.464636+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:14.559280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.6","version":"293'2652","reported_seq":2665,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T05:31:05.306756+0000","last_change":"2023-10-09T05:30:14.772276+0000","last_active":"2023-10-09T05:31:05.306756+0000","last_peered":"2023-10-09T05:31:05.306756+0000","last_clean":"2023-10-09T05:31:05.306756+0000","last_became_active":"2023-10-09T05:30:14.771924+0000","last_became_peered":"2023-10-09T05:30:14.771924+0000","last_unstale":"2023-10-09T05:31:05.306756+0000","last_undegraded":"2023-10-09T05:31:05.306756+0000","last_fullsized":"2023-10-09T05:31:05.306756+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_clean_scrub_stamp":"2023-10-09T05:30:13.731262+0000","objects_scrubbed":0,"log_size":2652,"log_dups_size":0,"ondisk_log_size":2652,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:46.303554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173801472,"num_objects":2652,"num_object_clones":0,"num_object_copies":7956,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2652,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2652,"num_write_kb":169728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":307,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.673846+0000","last_change":"2023-10-09T05:18:26.413106+0000","last_active":"2023-10-09T05:30:35.673846+0000","last_peered":"2023-10-09T05:30:35.673846+0000","last_clean":"2023-10-09T05:30:35.673846+0000","last_became_active":"2023-10-09T05:18:26.412334+0000","last_became_peered":"2023-10-09T05:18:26.412334+0000","last_unstale":"2023-10-09T05:30:35.673846+0000","last_undegraded":"2023-10-09T05:30:35.673846+0000","last_fullsized":"2023-10-09T05:30:35.673846+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:18.058016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":170,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:34.789120+0000","last_change":"2023-10-09T05:30:03.383456+0000","last_active":"2023-10-09T05:30:34.789120+0000","last_peered":"2023-10-09T05:30:34.789120+0000","last_clean":"2023-10-09T05:30:34.789120+0000","last_became_active":"2023-10-09T05:30:03.382656+0000","last_became_peered":"2023-10-09T05:30:03.382656+0000","last_unstale":"2023-10-09T05:30:34.789120+0000","last_undegraded":"2023-10-09T05:30:34.789120+0000","last_fullsized":"2023-10-09T05:30:34.789120+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:47:23.383409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.b","version":"293'2591","reported_seq":2604,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T05:31:05.301882+0000","last_change":"2023-10-09T05:30:14.772394+0000","last_active":"2023-10-09T05:31:05.301882+0000","last_peered":"2023-10-09T05:31:05.301882+0000","last_clean":"2023-10-09T05:31:05.301882+0000","last_became_active":"2023-10-09T05:30:14.772080+0000","last_became_peered":"2023-10-09T05:30:14.772080+0000","last_unstale":"2023-10-09T05:31:05.301882+0000","last_undegraded":"2023-10-09T05:31:05.301882+0000","last_fullsized":"2023-10-09T05:31:05.301882+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_clean_scrub_stamp":"2023-10-09T05:30:13.731262+0000","objects_scrubbed":0,"log_size":2591,"log_dups_size":0,"ondisk_log_size":2591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:30.397080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169803776,"num_objects":2591,"num_object_clones":0,"num_object_copies":7773,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2591,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2591,"num_write_kb":165824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":198,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.762640+0000","last_change":"2023-10-09T05:18:26.422150+0000","last_active":"2023-10-09T05:30:35.762640+0000","last_peered":"2023-10-09T05:30:35.762640+0000","last_clean":"2023-10-09T05:30:35.762640+0000","last_became_active":"2023-10-09T05:18:26.421853+0000","last_became_peered":"2023-10-09T05:18:26.421853+0000","last_unstale":"2023-10-09T05:30:35.762640+0000","last_undegraded":"2023-10-09T05:30:35.762640+0000","last_fullsized":"2023-10-09T05:30:35.762640+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:37:47.000556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":233,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.708021+0000","last_change":"2023-10-09T05:13:03.031243+0000","last_active":"2023-10-09T05:30:35.708021+0000","last_peered":"2023-10-09T05:30:35.708021+0000","last_clean":"2023-10-09T05:30:35.708021+0000","last_became_active":"2023-10-09T05:13:03.028769+0000","last_became_peered":"2023-10-09T05:13:03.028769+0000","last_unstale":"2023-10-09T05:30:35.708021+0000","last_undegraded":"2023-10-09T05:30:35.708021+0000","last_fullsized":"2023-10-09T05:30:35.708021+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:25:25.239799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":63,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:34.757100+0000","last_change":"2023-10-09T05:30:03.378519+0000","last_active":"2023-10-09T05:30:34.757100+0000","last_peered":"2023-10-09T05:30:34.757100+0000","last_clean":"2023-10-09T05:30:34.757100+0000","last_became_active":"2023-10-09T05:30:03.377444+0000","last_became_peered":"2023-10-09T05:30:03.377444+0000","last_unstale":"2023-10-09T05:30:34.757100+0000","last_undegraded":"2023-10-09T05:30:34.757100+0000","last_fullsized":"2023-10-09T05:30:34.757100+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:09:40.464032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.d","version":"293'2622","reported_seq":2635,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T05:31:05.178761+0000","last_change":"2023-10-09T05:30:14.776562+0000","last_active":"2023-10-09T05:31:05.178761+0000","last_peered":"2023-10-09T05:31:05.178761+0000","last_clean":"2023-10-09T05:31:05.178761+0000","last_became_active":"2023-10-09T05:30:14.776219+0000","last_became_peered":"2023-10-09T05:30:14.776219+0000","last_unstale":"2023-10-09T05:31:05.178761+0000","last_undegraded":"2023-10-09T05:31:05.178761+0000","last_fullsized":"2023-10-09T05:31:05.178761+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_clean_scrub_stamp":"2023-10-09T05:30:13.731262+0000","objects_scrubbed":0,"log_size":2622,"log_dups_size":0,"ondisk_log_size":2622,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:20.179665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171835392,"num_objects":2622,"num_object_clones":0,"num_object_copies":7866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2622,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2622,"num_write_kb":167808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":267,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:34.757247+0000","last_change":"2023-10-09T05:13:03.031386+0000","last_active":"2023-10-09T05:30:34.757247+0000","last_peered":"2023-10-09T05:30:34.757247+0000","last_clean":"2023-10-09T05:30:34.757247+0000","last_became_active":"2023-10-09T05:13:03.030816+0000","last_became_peered":"2023-10-09T05:13:03.030816+0000","last_unstale":"2023-10-09T05:30:34.757247+0000","last_undegraded":"2023-10-09T05:30:34.757247+0000","last_fullsized":"2023-10-09T05:30:34.757247+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:28.088661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":58,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:34.789866+0000","last_change":"2023-10-09T05:30:03.381036+0000","last_active":"2023-10-09T05:30:34.789866+0000","last_peered":"2023-10-09T05:30:34.789866+0000","last_clean":"2023-10-09T05:30:34.789866+0000","last_became_active":"2023-10-09T05:30:03.380698+0000","last_became_peered":"2023-10-09T05:30:03.380698+0000","last_unstale":"2023-10-09T05:30:34.789866+0000","last_undegraded":"2023-10-09T05:30:34.789866+0000","last_fullsized":"2023-10-09T05:30:34.789866+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:16.464471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":54,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:34.789435+0000","last_change":"2023-10-09T05:23:18.196260+0000","last_active":"2023-10-09T05:30:34.789435+0000","last_peered":"2023-10-09T05:30:34.789435+0000","last_clean":"2023-10-09T05:30:34.789435+0000","last_became_active":"2023-10-09T05:23:18.195907+0000","last_became_peered":"2023-10-09T05:23:18.195907+0000","last_unstale":"2023-10-09T05:30:34.789435+0000","last_undegraded":"2023-10-09T05:30:34.789435+0000","last_fullsized":"2023-10-09T05:30:34.789435+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:36.243432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.e","version":"293'2594","reported_seq":2606,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T05:31:04.598884+0000","last_change":"2023-10-09T05:30:14.779322+0000","last_active":"2023-10-09T05:31:04.598884+0000","last_peered":"2023-10-09T05:31:04.598884+0000","last_clean":"2023-10-09T05:31:04.598884+0000","last_became_active":"2023-10-09T05:30:14.778984+0000","last_became_peered":"2023-10-09T05:30:14.778984+0000","last_unstale":"2023-10-09T05:31:04.598884+0000","last_undegraded":"2023-10-09T05:31:04.598884+0000","last_fullsized":"2023-10-09T05:31:04.598884+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_clean_scrub_stamp":"2023-10-09T05:30:13.731262+0000","objects_scrubbed":0,"log_size":2594,"log_dups_size":0,"ondisk_log_size":2594,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:02.049382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170000384,"num_objects":2594,"num_object_clones":0,"num_object_copies":7782,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2594,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2594,"num_write_kb":166016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":72,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T05:30:36.293689+0000","last_change":"2023-10-09T05:20:18.123706+0000","last_active":"2023-10-09T05:30:36.293689+0000","last_peered":"2023-10-09T05:30:36.293689+0000","last_clean":"2023-10-09T05:30:36.293689+0000","last_became_active":"2023-10-09T05:20:18.122859+0000","last_became_peered":"2023-10-09T05:20:18.122859+0000","last_unstale":"2023-10-09T05:30:36.293689+0000","last_undegraded":"2023-10-09T05:30:36.293689+0000","last_fullsized":"2023-10-09T05:30:36.293689+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.906228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":162,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:34.757449+0000","last_change":"2023-10-09T05:09:57.854064+0000","last_active":"2023-10-09T05:30:34.757449+0000","last_peered":"2023-10-09T05:30:34.757449+0000","last_clean":"2023-10-09T05:30:34.757449+0000","last_became_active":"2023-10-09T05:09:57.847953+0000","last_became_peered":"2023-10-09T05:09:57.847953+0000","last_unstale":"2023-10-09T05:30:34.757449+0000","last_undegraded":"2023-10-09T05:30:34.757449+0000","last_fullsized":"2023-10-09T05:30:34.757449+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:45.914976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":36,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T05:30:15.778215+0000","last_change":"2023-10-09T05:25:15.659153+0000","last_active":"2023-10-09T05:30:15.778215+0000","last_peered":"2023-10-09T05:30:15.778215+0000","last_clean":"2023-10-09T05:30:15.778215+0000","last_became_active":"2023-10-09T05:25:15.658750+0000","last_became_peered":"2023-10-09T05:25:15.658750+0000","last_unstale":"2023-10-09T05:30:15.778215+0000","last_undegraded":"2023-10-09T05:30:15.778215+0000","last_fullsized":"2023-10-09T05:30:15.778215+0000","mapping_epoch":268,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":269,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:10.370264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.c","version":"293'2560","reported_seq":2573,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T05:31:05.173761+0000","last_change":"2023-10-09T05:30:14.774991+0000","last_active":"2023-10-09T05:31:05.173761+0000","last_peered":"2023-10-09T05:31:05.173761+0000","last_clean":"2023-10-09T05:31:05.173761+0000","last_became_active":"2023-10-09T05:30:14.773586+0000","last_became_peered":"2023-10-09T05:30:14.773586+0000","last_unstale":"2023-10-09T05:31:05.173761+0000","last_undegraded":"2023-10-09T05:31:05.173761+0000","last_fullsized":"2023-10-09T05:31:05.173761+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_clean_scrub_stamp":"2023-10-09T05:30:13.731262+0000","objects_scrubbed":0,"log_size":2560,"log_dups_size":0,"ondisk_log_size":2560,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:48:58.560129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167772160,"num_objects":2560,"num_object_clones":0,"num_object_copies":7680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2560,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2560,"num_write_kb":163840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":238,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T05:30:15.778854+0000","last_change":"2023-10-09T05:18:26.416511+0000","last_active":"2023-10-09T05:30:15.778854+0000","last_peered":"2023-10-09T05:30:15.778854+0000","last_clean":"2023-10-09T05:30:15.778854+0000","last_became_active":"2023-10-09T05:18:26.416101+0000","last_became_peered":"2023-10-09T05:18:26.416101+0000","last_unstale":"2023-10-09T05:30:15.778854+0000","last_undegraded":"2023-10-09T05:30:15.778854+0000","last_fullsized":"2023-10-09T05:30:15.778854+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:10.243472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":578,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.688427+0000","last_change":"2023-10-09T05:18:29.419527+0000","last_active":"2023-10-09T05:30:35.688427+0000","last_peered":"2023-10-09T05:30:35.688427+0000","last_clean":"2023-10-09T05:30:35.688427+0000","last_became_active":"2023-10-09T05:18:29.418852+0000","last_became_peered":"2023-10-09T05:18:29.418852+0000","last_unstale":"2023-10-09T05:30:35.688427+0000","last_undegraded":"2023-10-09T05:30:35.688427+0000","last_fullsized":"2023-10-09T05:30:35.688427+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_clean_scrub_stamp":"2023-10-09T04:36:44.261013+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:02.899170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":265,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.674416+0000","last_change":"2023-10-09T05:30:03.378832+0000","last_active":"2023-10-09T05:30:35.674416+0000","last_peered":"2023-10-09T05:30:35.674416+0000","last_clean":"2023-10-09T05:30:35.674416+0000","last_became_active":"2023-10-09T05:30:03.378442+0000","last_became_peered":"2023-10-09T05:30:03.378442+0000","last_unstale":"2023-10-09T05:30:35.674416+0000","last_undegraded":"2023-10-09T05:30:35.674416+0000","last_fullsized":"2023-10-09T05:30:35.674416+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:43:52.998584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":286,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.464662+0000","last_change":"2023-10-09T05:30:03.386395+0000","last_active":"2023-10-09T05:30:35.464662+0000","last_peered":"2023-10-09T05:30:35.464662+0000","last_clean":"2023-10-09T05:30:35.464662+0000","last_became_active":"2023-10-09T05:30:03.386024+0000","last_became_peered":"2023-10-09T05:30:03.386024+0000","last_unstale":"2023-10-09T05:30:35.464662+0000","last_undegraded":"2023-10-09T05:30:35.464662+0000","last_fullsized":"2023-10-09T05:30:35.464662+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:13.133829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"293'2571","reported_seq":2583,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T05:31:04.552368+0000","last_change":"2023-10-09T05:30:14.775740+0000","last_active":"2023-10-09T05:31:04.552368+0000","last_peered":"2023-10-09T05:31:04.552368+0000","last_clean":"2023-10-09T05:31:04.552368+0000","last_became_active":"2023-10-09T05:30:14.775386+0000","last_became_peered":"2023-10-09T05:30:14.775386+0000","last_unstale":"2023-10-09T05:31:04.552368+0000","last_undegraded":"2023-10-09T05:31:04.552368+0000","last_fullsized":"2023-10-09T05:31:04.552368+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_clean_scrub_stamp":"2023-10-09T05:30:13.731262+0000","objects_scrubbed":0,"log_size":2571,"log_dups_size":0,"ondisk_log_size":2571,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:39.263920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168493056,"num_objects":2571,"num_object_clones":0,"num_object_copies":7713,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2571,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2571,"num_write_kb":164544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":102,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.688588+0000","last_change":"2023-10-09T05:23:18.193825+0000","last_active":"2023-10-09T05:30:35.688588+0000","last_peered":"2023-10-09T05:30:35.688588+0000","last_clean":"2023-10-09T05:30:35.688588+0000","last_became_active":"2023-10-09T05:23:18.193394+0000","last_became_peered":"2023-10-09T05:23:18.193394+0000","last_unstale":"2023-10-09T05:30:35.688588+0000","last_undegraded":"2023-10-09T05:30:35.688588+0000","last_fullsized":"2023-10-09T05:30:35.688588+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:40.594469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":266,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:34.756722+0000","last_change":"2023-10-09T05:30:03.386555+0000","last_active":"2023-10-09T05:30:34.756722+0000","last_peered":"2023-10-09T05:30:34.756722+0000","last_clean":"2023-10-09T05:30:34.756722+0000","last_became_active":"2023-10-09T05:30:03.385961+0000","last_became_peered":"2023-10-09T05:30:03.385961+0000","last_unstale":"2023-10-09T05:30:34.756722+0000","last_undegraded":"2023-10-09T05:30:34.756722+0000","last_fullsized":"2023-10-09T05:30:34.756722+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:30.356878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":335,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:34.757374+0000","last_change":"2023-10-09T05:20:18.132750+0000","last_active":"2023-10-09T05:30:34.757374+0000","last_peered":"2023-10-09T05:30:34.757374+0000","last_clean":"2023-10-09T05:30:34.757374+0000","last_became_active":"2023-10-09T05:20:18.132396+0000","last_became_peered":"2023-10-09T05:20:18.132396+0000","last_unstale":"2023-10-09T05:30:34.757374+0000","last_undegraded":"2023-10-09T05:30:34.757374+0000","last_fullsized":"2023-10-09T05:30:34.757374+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:00:55.731045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"293'2755","reported_seq":2767,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T05:31:05.644052+0000","last_change":"2023-10-09T05:30:14.773249+0000","last_active":"2023-10-09T05:31:05.644052+0000","last_peered":"2023-10-09T05:31:05.644052+0000","last_clean":"2023-10-09T05:31:05.644052+0000","last_became_active":"2023-10-09T05:30:14.772928+0000","last_became_peered":"2023-10-09T05:30:14.772928+0000","last_unstale":"2023-10-09T05:31:05.644052+0000","last_undegraded":"2023-10-09T05:31:05.644052+0000","last_fullsized":"2023-10-09T05:31:05.644052+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_clean_scrub_stamp":"2023-10-09T05:30:13.731262+0000","objects_scrubbed":0,"log_size":2755,"log_dups_size":0,"ondisk_log_size":2755,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:31.972613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180551680,"num_objects":2755,"num_object_clones":0,"num_object_copies":8265,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2755,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2755,"num_write_kb":176320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":92,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.687812+0000","last_change":"2023-10-09T05:30:03.391750+0000","last_active":"2023-10-09T05:30:35.687812+0000","last_peered":"2023-10-09T05:30:35.687812+0000","last_clean":"2023-10-09T05:30:35.687812+0000","last_became_active":"2023-10-09T05:30:03.391393+0000","last_became_peered":"2023-10-09T05:30:03.391393+0000","last_unstale":"2023-10-09T05:30:35.687812+0000","last_undegraded":"2023-10-09T05:30:35.687812+0000","last_fullsized":"2023-10-09T05:30:35.687812+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:07.263598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":189,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.762771+0000","last_change":"2023-10-09T05:20:18.126997+0000","last_active":"2023-10-09T05:30:35.762771+0000","last_peered":"2023-10-09T05:30:35.762771+0000","last_clean":"2023-10-09T05:30:35.762771+0000","last_became_active":"2023-10-09T05:20:18.126641+0000","last_became_peered":"2023-10-09T05:20:18.126641+0000","last_unstale":"2023-10-09T05:30:35.762771+0000","last_undegraded":"2023-10-09T05:30:35.762771+0000","last_fullsized":"2023-10-09T05:30:35.762771+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:42.112245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":297,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:34.789778+0000","last_change":"2023-10-09T05:18:26.418524+0000","last_active":"2023-10-09T05:30:34.789778+0000","last_peered":"2023-10-09T05:30:34.789778+0000","last_clean":"2023-10-09T05:30:34.789778+0000","last_became_active":"2023-10-09T05:18:26.418058+0000","last_became_peered":"2023-10-09T05:18:26.418058+0000","last_unstale":"2023-10-09T05:30:34.789778+0000","last_undegraded":"2023-10-09T05:30:34.789778+0000","last_fullsized":"2023-10-09T05:30:34.789778+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:40.594449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.9","version":"293'2596","reported_seq":2608,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T05:31:04.444123+0000","last_change":"2023-10-09T05:30:14.770802+0000","last_active":"2023-10-09T05:31:04.444123+0000","last_peered":"2023-10-09T05:31:04.444123+0000","last_clean":"2023-10-09T05:31:04.444123+0000","last_became_active":"2023-10-09T05:30:14.770464+0000","last_became_peered":"2023-10-09T05:30:14.770464+0000","last_unstale":"2023-10-09T05:31:04.444123+0000","last_undegraded":"2023-10-09T05:31:04.444123+0000","last_fullsized":"2023-10-09T05:31:04.444123+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_clean_scrub_stamp":"2023-10-09T05:30:13.731262+0000","objects_scrubbed":0,"log_size":2596,"log_dups_size":0,"ondisk_log_size":2596,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:09.722956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170131456,"num_objects":2596,"num_object_clones":0,"num_object_copies":7788,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2596,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2596,"num_write_kb":166144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":300,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.674293+0000","last_change":"2023-10-09T05:18:26.431563+0000","last_active":"2023-10-09T05:30:35.674293+0000","last_peered":"2023-10-09T05:30:35.674293+0000","last_clean":"2023-10-09T05:30:35.674293+0000","last_became_active":"2023-10-09T05:18:26.421025+0000","last_became_peered":"2023-10-09T05:18:26.421025+0000","last_unstale":"2023-10-09T05:30:35.674293+0000","last_undegraded":"2023-10-09T05:30:35.674293+0000","last_fullsized":"2023-10-09T05:30:35.674293+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:30.333585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":63,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:34.789523+0000","last_change":"2023-10-09T05:30:03.375221+0000","last_active":"2023-10-09T05:30:34.789523+0000","last_peered":"2023-10-09T05:30:34.789523+0000","last_clean":"2023-10-09T05:30:34.789523+0000","last_became_active":"2023-10-09T05:30:03.374863+0000","last_became_peered":"2023-10-09T05:30:03.374863+0000","last_unstale":"2023-10-09T05:30:34.789523+0000","last_undegraded":"2023-10-09T05:30:34.789523+0000","last_fullsized":"2023-10-09T05:30:34.789523+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:16.058179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":299,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.674648+0000","last_change":"2023-10-09T05:21:08.179657+0000","last_active":"2023-10-09T05:30:35.674648+0000","last_peered":"2023-10-09T05:30:35.674648+0000","last_clean":"2023-10-09T05:30:35.674648+0000","last_became_active":"2023-10-09T05:21:08.178958+0000","last_became_peered":"2023-10-09T05:21:08.178958+0000","last_unstale":"2023-10-09T05:30:35.674648+0000","last_undegraded":"2023-10-09T05:30:35.674648+0000","last_fullsized":"2023-10-09T05:30:35.674648+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:20.815986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.8","version":"293'2651","reported_seq":2663,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T05:31:05.629930+0000","last_change":"2023-10-09T05:30:14.772263+0000","last_active":"2023-10-09T05:31:05.629930+0000","last_peered":"2023-10-09T05:31:05.629930+0000","last_clean":"2023-10-09T05:31:05.629930+0000","last_became_active":"2023-10-09T05:30:14.771941+0000","last_became_peered":"2023-10-09T05:30:14.771941+0000","last_unstale":"2023-10-09T05:31:05.629930+0000","last_undegraded":"2023-10-09T05:31:05.629930+0000","last_fullsized":"2023-10-09T05:31:05.629930+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:30:13.731262+0000","last_clean_scrub_stamp":"2023-10-09T05:30:13.731262+0000","objects_scrubbed":0,"log_size":2651,"log_dups_size":0,"ondisk_log_size":2651,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:50:24.406676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173735936,"num_objects":2651,"num_object_clones":0,"num_object_copies":7953,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2651,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2651,"num_write_kb":169664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":298,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.465844+0000","last_change":"2023-10-09T05:18:26.424429+0000","last_active":"2023-10-09T05:30:35.465844+0000","last_peered":"2023-10-09T05:30:35.465844+0000","last_clean":"2023-10-09T05:30:35.465844+0000","last_became_active":"2023-10-09T05:18:26.423034+0000","last_became_peered":"2023-10-09T05:18:26.423034+0000","last_unstale":"2023-10-09T05:30:35.465844+0000","last_undegraded":"2023-10-09T05:30:35.465844+0000","last_fullsized":"2023-10-09T05:30:35.465844+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:16.251181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":300,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:34.757472+0000","last_change":"2023-10-09T05:30:03.397632+0000","last_active":"2023-10-09T05:30:34.757472+0000","last_peered":"2023-10-09T05:30:34.757472+0000","last_clean":"2023-10-09T05:30:34.757472+0000","last_became_active":"2023-10-09T05:30:03.385921+0000","last_became_peered":"2023-10-09T05:30:03.385921+0000","last_unstale":"2023-10-09T05:30:34.757472+0000","last_undegraded":"2023-10-09T05:30:34.757472+0000","last_fullsized":"2023-10-09T05:30:34.757472+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:15.452117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":135,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.464300+0000","last_change":"2023-10-09T05:30:03.383694+0000","last_active":"2023-10-09T05:30:35.464300+0000","last_peered":"2023-10-09T05:30:35.464300+0000","last_clean":"2023-10-09T05:30:35.464300+0000","last_became_active":"2023-10-09T05:30:03.383157+0000","last_became_peered":"2023-10-09T05:30:03.383157+0000","last_unstale":"2023-10-09T05:30:35.464300+0000","last_undegraded":"2023-10-09T05:30:35.464300+0000","last_fullsized":"2023-10-09T05:30:35.464300+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:34:02.469475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"69'2","reported_seq":73,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:34.788710+0000","last_change":"2023-10-09T05:30:03.386435+0000","last_active":"2023-10-09T05:30:34.788710+0000","last_peered":"2023-10-09T05:30:34.788710+0000","last_clean":"2023-10-09T05:30:34.788710+0000","last_became_active":"2023-10-09T05:30:03.385418+0000","last_became_peered":"2023-10-09T05:30:03.385418+0000","last_unstale":"2023-10-09T05:30:34.788710+0000","last_undegraded":"2023-10-09T05:30:34.788710+0000","last_fullsized":"2023-10-09T05:30:34.788710+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:34.685368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":71,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T05:30:23.928606+0000","last_change":"2023-10-09T05:20:18.125172+0000","last_active":"2023-10-09T05:30:23.928606+0000","last_peered":"2023-10-09T05:30:23.928606+0000","last_clean":"2023-10-09T05:30:23.928606+0000","last_became_active":"2023-10-09T05:20:18.123624+0000","last_became_peered":"2023-10-09T05:20:18.123624+0000","last_unstale":"2023-10-09T05:30:23.928606+0000","last_undegraded":"2023-10-09T05:30:23.928606+0000","last_fullsized":"2023-10-09T05:30:23.928606+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:11.106483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":152,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:34.789556+0000","last_change":"2023-10-09T05:30:03.409245+0000","last_active":"2023-10-09T05:30:34.789556+0000","last_peered":"2023-10-09T05:30:34.789556+0000","last_clean":"2023-10-09T05:30:34.789556+0000","last_became_active":"2023-10-09T05:30:03.383531+0000","last_became_peered":"2023-10-09T05:30:03.383531+0000","last_unstale":"2023-10-09T05:30:34.789556+0000","last_undegraded":"2023-10-09T05:30:34.789556+0000","last_fullsized":"2023-10-09T05:30:34.789556+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:52.871360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":278,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.674210+0000","last_change":"2023-10-09T05:30:03.380938+0000","last_active":"2023-10-09T05:30:35.674210+0000","last_peered":"2023-10-09T05:30:35.674210+0000","last_clean":"2023-10-09T05:30:35.674210+0000","last_became_active":"2023-10-09T05:30:03.380395+0000","last_became_peered":"2023-10-09T05:30:03.380395+0000","last_unstale":"2023-10-09T05:30:35.674210+0000","last_undegraded":"2023-10-09T05:30:35.674210+0000","last_fullsized":"2023-10-09T05:30:35.674210+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:23.087303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":33,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.464399+0000","last_change":"2023-10-09T05:30:03.402738+0000","last_active":"2023-10-09T05:30:35.464399+0000","last_peered":"2023-10-09T05:30:35.464399+0000","last_clean":"2023-10-09T05:30:35.464399+0000","last_became_active":"2023-10-09T05:30:03.384635+0000","last_became_peered":"2023-10-09T05:30:03.384635+0000","last_unstale":"2023-10-09T05:30:35.464399+0000","last_undegraded":"2023-10-09T05:30:35.464399+0000","last_fullsized":"2023-10-09T05:30:35.464399+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:50.874547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":109,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.674060+0000","last_change":"2023-10-09T05:30:03.376908+0000","last_active":"2023-10-09T05:30:35.674060+0000","last_peered":"2023-10-09T05:30:35.674060+0000","last_clean":"2023-10-09T05:30:35.674060+0000","last_became_active":"2023-10-09T05:30:03.376456+0000","last_became_peered":"2023-10-09T05:30:03.376456+0000","last_unstale":"2023-10-09T05:30:35.674060+0000","last_undegraded":"2023-10-09T05:30:35.674060+0000","last_fullsized":"2023-10-09T05:30:35.674060+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:23.173899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'2","reported_seq":293,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.708101+0000","last_change":"2023-10-09T05:22:47.197002+0000","last_active":"2023-10-09T05:30:35.708101+0000","last_peered":"2023-10-09T05:30:35.708101+0000","last_clean":"2023-10-09T05:30:35.708101+0000","last_became_active":"2023-10-09T05:22:47.196685+0000","last_became_peered":"2023-10-09T05:22:47.196685+0000","last_unstale":"2023-10-09T05:30:35.708101+0000","last_undegraded":"2023-10-09T05:30:35.708101+0000","last_fullsized":"2023-10-09T05:30:35.708101+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:47:32.594447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":277,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.674750+0000","last_change":"2023-10-09T05:02:09.649674+0000","last_active":"2023-10-09T05:30:35.674750+0000","last_peered":"2023-10-09T05:30:35.674750+0000","last_clean":"2023-10-09T05:30:35.674750+0000","last_became_active":"2023-10-09T05:02:09.649041+0000","last_became_peered":"2023-10-09T05:02:09.649041+0000","last_unstale":"2023-10-09T05:30:35.674750+0000","last_undegraded":"2023-10-09T05:30:35.674750+0000","last_fullsized":"2023-10-09T05:30:35.674750+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:18.524071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'5","reported_seq":71,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:34.756686+0000","last_change":"2023-10-09T05:21:08.181594+0000","last_active":"2023-10-09T05:30:34.756686+0000","last_peered":"2023-10-09T05:30:34.756686+0000","last_clean":"2023-10-09T05:30:34.756686+0000","last_became_active":"2023-10-09T05:21:08.176895+0000","last_became_peered":"2023-10-09T05:21:08.176895+0000","last_unstale":"2023-10-09T05:30:34.756686+0000","last_undegraded":"2023-10-09T05:30:34.756686+0000","last_fullsized":"2023-10-09T05:30:34.756686+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:48:45.569544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":268,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T05:30:35.465236+0000","last_change":"2023-10-09T05:30:03.379517+0000","last_active":"2023-10-09T05:30:35.465236+0000","last_peered":"2023-10-09T05:30:35.465236+0000","last_clean":"2023-10-09T05:30:35.465236+0000","last_became_active":"2023-10-09T05:30:03.379048+0000","last_became_peered":"2023-10-09T05:30:03.379048+0000","last_unstale":"2023-10-09T05:30:35.465236+0000","last_undegraded":"2023-10-09T05:30:35.465236+0000","last_fullsized":"2023-10-09T05:30:35.465236+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:04.333337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":2703294464,"num_objects":41249,"num_object_clones":0,"num_object_copies":123747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":41249,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":41249,"num_write_kb":2639936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8126791680,"data_stored":8126791680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":41249,"ondisk_log_size":41249,"up":48,"acting":48,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":23,"num_read_kb":22,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":21604,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234586,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2349344,"kb_used_data":1442200,"kb_used_omap":0,"kb_used_meta":907136,"kb_avail":91395808,"statfs":{"total":95995035648,"available":93589307392,"internally_reserved":0,"allocated":1476812800,"data_stored":1475919513,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928907264},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:31:03 2023","interfaces":[{"interface":"back","average":{"1min":6.8470000000000004,"5min":4.3129999999999997,"15min":2.4590000000000001},"min":{"1min":0.28999999999999998,"5min":0.26500000000000001,"15min":0.248},"max":{"1min":114.259,"5min":122.248,"15min":122.248},"last":1.268},{"interface":"front","average":{"1min":6.8769999999999998,"5min":4.2800000000000002,"15min":2.5150000000000001},"min":{"1min":0.315,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":114.286,"5min":122.22199999999999,"15min":122.22199999999999},"last":0.41599999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:30:40 2023","interfaces":[{"interface":"back","average":{"1min":0.79300000000000004,"5min":1.3740000000000001,"15min":1.425},"min":{"1min":0.30299999999999999,"5min":0.29199999999999998,"15min":0.28000000000000003},"max":{"1min":1.1459999999999999,"5min":6.1109999999999998,"15min":23.530999999999999},"last":1.3280000000000001},{"interface":"front","average":{"1min":0.93600000000000005,"5min":1.3169999999999999,"15min":1.4079999999999999},"min":{"1min":0.496,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":1.3240000000000001,"5min":6.8710000000000004,"15min":23.443000000000001},"last":1.532}]},{"osd":2,"last update":"Mon Oct 9 05:31:03 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":3.0259999999999998,"15min":2.0920000000000001},"min":{"1min":0.371,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.4489999999999998,"5min":122.065,"15min":122.065},"last":1.026},{"interface":"front","average":{"1min":0.96299999999999997,"5min":3.5960000000000001,"15min":2.21},"min":{"1min":0.42099999999999999,"5min":0.32700000000000001,"15min":0.30399999999999999},"max":{"1min":2.097,"5min":122.026,"15min":122.026},"last":0.66800000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:31:03 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":3.2879999999999998,"15min":2.4500000000000002},"min":{"1min":0.39300000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.702,"5min":138.667,"15min":138.667},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.92400000000000004,"5min":3.4609999999999999,"15min":2.5880000000000001},"min":{"1min":0.42999999999999999,"5min":0.34699999999999998,"15min":0.34300000000000003},"max":{"1min":1.1879999999999999,"5min":138.54400000000001,"15min":138.54400000000001},"last":0.96999999999999997}]},{"osd":4,"last update":"Mon Oct 9 05:30:40 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":1.032,"15min":1.012},"min":{"1min":0.44700000000000001,"5min":0.311,"15min":0.311},"max":{"1min":1.8420000000000001,"5min":6.0309999999999997,"15min":9.7850000000000001},"last":1.393},{"interface":"front","average":{"1min":1.095,"5min":1.0800000000000001,"15min":1.0469999999999999},"min":{"1min":0.503,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":2.008,"5min":6.8879999999999999,"15min":6.8879999999999999},"last":0.92300000000000004}]},{"osd":5,"last update":"Mon Oct 9 05:30:40 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":1.036,"15min":0.98699999999999999},"min":{"1min":0.59999999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.272,"5min":6.2839999999999998,"15min":6.2839999999999998},"last":0.77100000000000002},{"interface":"front","average":{"1min":1.026,"5min":1.0449999999999999,"15min":0.95999999999999996},"min":{"1min":0.44900000000000001,"5min":0.373,"15min":0.373},"max":{"1min":1.4770000000000001,"5min":5.9779999999999998,"15min":5.9779999999999998},"last":1.444}]},{"osd":6,"last update":"Mon Oct 9 05:30:40 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":1.3120000000000001,"15min":1.1830000000000001},"min":{"1min":0.40000000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":1.522,"5min":8.5609999999999999,"15min":8.5609999999999999},"last":1.4990000000000001},{"interface":"front","average":{"1min":1.1890000000000001,"5min":1.083,"15min":1.1499999999999999},"min":{"1min":0.48599999999999999,"5min":0.377,"15min":0.377},"max":{"1min":1.7949999999999999,"5min":5.0030000000000001,"15min":6.7910000000000004},"last":1.1899999999999999}]}]},{"osd":6,"up_from":46,"seq":197568496228,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2213536,"kb_used_data":1309720,"kb_used_omap":0,"kb_used_meta":903808,"kb_avail":91531616,"statfs":{"total":95995035648,"available":93728374784,"internally_reserved":0,"allocated":1341153280,"data_stored":1340260758,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925499392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:31:03 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":3.1389999999999998,"15min":3.048},"min":{"1min":0.48499999999999999,"5min":0.26100000000000001,"15min":0.23599999999999999},"max":{"1min":3.1059999999999999,"5min":154.29300000000001,"15min":154.29300000000001},"last":4.2270000000000003},{"interface":"front","average":{"1min":1.2490000000000001,"5min":3.2530000000000001,"15min":3.169},"min":{"1min":0.51400000000000001,"5min":0.33100000000000002,"15min":0.309},"max":{"1min":3.1400000000000001,"5min":154.35300000000001,"15min":154.35300000000001},"last":4.3579999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:31:03 2023","interfaces":[{"interface":"back","average":{"1min":1.266,"5min":1.6060000000000001,"15min":2.2669999999999999},"min":{"1min":0.77600000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":2.8370000000000002,"5min":6.5789999999999997,"15min":120.873},"last":5.7069999999999999},{"interface":"front","average":{"1min":1.149,"5min":1.274,"15min":2.0670000000000002},"min":{"1min":0.34499999999999997,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.1349999999999998,"5min":5.7350000000000003,"15min":119.248},"last":4.1429999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:31:03 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":4.2229999999999999,"15min":2.6819999999999999},"min":{"1min":0.42699999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":9.4039999999999999,"5min":112.649,"15min":112.649},"last":4.2709999999999999},{"interface":"front","average":{"1min":1.502,"5min":4.0830000000000002,"15min":2.8490000000000002},"min":{"1min":0.45900000000000002,"5min":0.31,"15min":0.31},"max":{"1min":9.782,"5min":112.834,"15min":112.834},"last":3.8540000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:31:03 2023","interfaces":[{"interface":"back","average":{"1min":1.2909999999999999,"5min":3.8679999999999999,"15min":4.8819999999999997},"min":{"1min":0.45700000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":4.3890000000000002,"5min":169.01599999999999,"15min":169.01599999999999},"last":5.7530000000000001},{"interface":"front","average":{"1min":1.383,"5min":3.754,"15min":4.7439999999999998},"min":{"1min":0.59099999999999997,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":3.0600000000000001,"5min":169.40799999999999,"15min":169.40799999999999},"last":4.3170000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:31:03 2023","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":0.998,"15min":1.0429999999999999},"min":{"1min":0.47699999999999998,"5min":0.315,"15min":0.27000000000000002},"max":{"1min":1.24,"5min":4.5119999999999996,"15min":7.7000000000000002},"last":5.8239999999999998},{"interface":"front","average":{"1min":1.095,"5min":0.90900000000000003,"15min":0.98299999999999998},"min":{"1min":0.438,"5min":0.38100000000000001,"15min":0.34100000000000003},"max":{"1min":2.6400000000000001,"5min":4.069,"15min":4.7279999999999998},"last":4.431}]},{"osd":5,"last update":"Mon Oct 9 05:31:03 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.96899999999999997,"15min":0.98899999999999999},"min":{"1min":0.501,"5min":0.35999999999999999,"15min":0.33300000000000002},"max":{"1min":1.4119999999999999,"5min":4.5410000000000004,"15min":7.5899999999999999},"last":5.7910000000000004},{"interface":"front","average":{"1min":1.1359999999999999,"5min":1.002,"15min":1.0369999999999999},"min":{"1min":0.52600000000000002,"5min":0.40400000000000003,"15min":0.38},"max":{"1min":2.5449999999999999,"5min":4.3230000000000004,"15min":6.6459999999999999},"last":4.3959999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:30:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":1.4219999999999999,"15min":1.302},"min":{"1min":0.71299999999999997,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.3680000000000001,"5min":8.8780000000000001,"15min":8.8780000000000001},"last":3.9769999999999999},{"interface":"front","average":{"1min":1.147,"5min":1.5009999999999999,"15min":1.429},"min":{"1min":0.81399999999999995,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.462,"5min":8.923,"15min":8.923},"last":4.0220000000000002}]}]},{"osd":1,"up_from":96,"seq":412316860819,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2199532,"kb_used_data":1172196,"kb_used_omap":4,"kb_used_meta":1027323,"kb_avail":91545620,"statfs":{"total":95995035648,"available":93742714880,"internally_reserved":0,"allocated":1200328704,"data_stored":1198724784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1051979662},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:30:54 2023","interfaces":[{"interface":"back","average":{"1min":4.0659999999999998,"5min":3.242,"15min":2.617},"min":{"1min":0.42999999999999999,"5min":0.217,"15min":0.19},"max":{"1min":60.976999999999997,"5min":76.007999999999996,"15min":83.194999999999993},"last":0.22900000000000001},{"interface":"front","average":{"1min":4.0279999999999996,"5min":3.532,"15min":2.9409999999999998},"min":{"1min":0.28499999999999998,"5min":0.246,"15min":0.186},"max":{"1min":60.777000000000001,"5min":75.168999999999997,"15min":83.972999999999999},"last":0.245}]},{"osd":2,"last update":"Mon Oct 9 05:30:57 2023","interfaces":[{"interface":"back","average":{"1min":3.8780000000000001,"5min":2.496,"15min":2.3639999999999999},"min":{"1min":0.26500000000000001,"5min":0.24099999999999999,"15min":0.20999999999999999},"max":{"1min":54.429000000000002,"5min":54.429000000000002,"15min":83.337000000000003},"last":13.943},{"interface":"front","average":{"1min":4.1280000000000001,"5min":2.4710000000000001,"15min":2.4180000000000001},"min":{"1min":0.40300000000000002,"5min":0.254,"15min":0.23599999999999999},"max":{"1min":54.837000000000003,"5min":54.837000000000003,"15min":83.143000000000001},"last":12.981}]},{"osd":3,"last update":"Mon Oct 9 05:30:57 2023","interfaces":[{"interface":"back","average":{"1min":0.78800000000000003,"5min":2.3490000000000002,"15min":2.407},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":2.6139999999999999,"5min":51.780999999999999,"15min":84.105999999999995},"last":8.5299999999999994},{"interface":"front","average":{"1min":0.68700000000000006,"5min":2.6789999999999998,"15min":2.677},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":2.5649999999999999,"5min":51.813000000000002,"15min":83.566000000000003},"last":9.2579999999999991}]},{"osd":4,"last update":"Mon Oct 9 05:30:54 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":1.5529999999999999,"15min":1.986},"min":{"1min":0.39200000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.381,"5min":9.8200000000000003,"15min":83.453999999999994},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.871,"5min":1.389,"15min":1.6879999999999999},"min":{"1min":0.47799999999999998,"5min":0.34599999999999997,"15min":0.28100000000000003},"max":{"1min":1.3600000000000001,"5min":9.7850000000000001,"15min":83.375},"last":0.48499999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:30:54 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":1.3799999999999999,"15min":1.452},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.3979999999999999,"5min":9.7520000000000007,"15min":9.7520000000000007},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.84999999999999998,"5min":1.629,"15min":1.931},"min":{"1min":0.45700000000000002,"5min":0.42399999999999999,"15min":0.39900000000000002},"max":{"1min":1.2170000000000001,"5min":14.263999999999999,"15min":83.268000000000001},"last":0.434}]},{"osd":6,"last update":"Mon Oct 9 05:30:54 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":1.6000000000000001,"15min":1.655},"min":{"1min":0.45300000000000001,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.262,"5min":9.8520000000000003,"15min":9.8520000000000003},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.85999999999999999,"5min":1.671,"15min":2.0089999999999999},"min":{"1min":0.42099999999999999,"5min":0.40100000000000002,"15min":0.376},"max":{"1min":1.278,"5min":14.231,"15min":83.400000000000006},"last":0.47299999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:30:54 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":1.7150000000000001,"15min":2.0049999999999999},"min":{"1min":0.45900000000000002,"5min":0.376,"15min":0.376},"max":{"1min":1.3700000000000001,"5min":14.166,"15min":83.510000000000005},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.86599999999999999,"5min":1.4590000000000001,"15min":1.7050000000000001},"min":{"1min":0.45300000000000001,"5min":0.377,"15min":0.36899999999999999},"max":{"1min":1.3939999999999999,"5min":9.8819999999999997,"15min":83.480999999999995},"last":0.53900000000000003}]}]},{"osd":0,"up_from":88,"seq":377957122460,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2213608,"kb_used_data":1279776,"kb_used_omap":5,"kb_used_meta":933818,"kb_avail":91531544,"statfs":{"total":95995035648,"available":93728301056,"internally_reserved":0,"allocated":1310490624,"data_stored":1308948637,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":956230266},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:30:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":1.827,"15min":2.177},"min":{"1min":0.42999999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":5.774,"5min":8.1910000000000007,"15min":80.843999999999994},"last":2.1859999999999999},{"interface":"front","average":{"1min":0.78600000000000003,"5min":1.2569999999999999,"15min":2.3100000000000001},"min":{"1min":0.28000000000000003,"5min":0.221,"15min":0.21299999999999999},"max":{"1min":2.492,"5min":8.9849999999999994,"15min":81.159999999999997},"last":0.80600000000000005}]},{"osd":2,"last update":"Mon Oct 9 05:30:55 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":2.8839999999999999,"15min":2.597},"min":{"1min":0.25900000000000001,"5min":0.247,"15min":0.247},"max":{"1min":1.657,"5min":92.745000000000005,"15min":92.745000000000005},"last":0.94399999999999995},{"interface":"front","average":{"1min":1.3260000000000001,"5min":3.2989999999999999,"15min":2.6139999999999999},"min":{"1min":0.40899999999999997,"5min":0.20399999999999999,"15min":0.19700000000000001},"max":{"1min":7.3310000000000004,"5min":97.816000000000003,"15min":97.816000000000003},"last":4.827}]},{"osd":3,"last update":"Mon Oct 9 05:30:55 2023","interfaces":[{"interface":"back","average":{"1min":2.4049999999999998,"5min":2.6080000000000001,"15min":2.4729999999999999},"min":{"1min":0.48699999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":23.858000000000001,"5min":37.883000000000003,"15min":82.286000000000001},"last":4.3179999999999996},{"interface":"front","average":{"1min":2.157,"5min":2.649,"15min":2.8290000000000002},"min":{"1min":0.35799999999999998,"5min":0.313,"15min":0.28399999999999997},"max":{"1min":23.539999999999999,"5min":45.279000000000003,"15min":75.969999999999999},"last":0.871}]},{"osd":4,"last update":"Mon Oct 9 05:30:55 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":1.234,"15min":1.5840000000000001},"min":{"1min":0.436,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":2.355,"5min":8.4629999999999992,"15min":12.255000000000001},"last":0.91800000000000004},{"interface":"front","average":{"1min":1.3200000000000001,"5min":1.7849999999999999,"15min":1.6040000000000001},"min":{"1min":0.36199999999999999,"5min":0.33400000000000002,"15min":0.28199999999999997},"max":{"1min":7.3019999999999996,"5min":9.8320000000000007,"15min":9.8320000000000007},"last":2.2269999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:30:55 2023","interfaces":[{"interface":"back","average":{"1min":1.25,"5min":1.4390000000000001,"15min":1.409},"min":{"1min":0.58499999999999996,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":4.516,"5min":9.5869999999999997,"15min":11.916},"last":4.4859999999999998},{"interface":"front","average":{"1min":1.038,"5min":1.2549999999999999,"15min":1.649},"min":{"1min":0.46300000000000002,"5min":0.40000000000000002,"15min":0.375},"max":{"1min":2.3959999999999999,"5min":8.5299999999999994,"15min":12.143000000000001},"last":0.89700000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:30:55 2023","interfaces":[{"interface":"back","average":{"1min":1.4139999999999999,"5min":2.1030000000000002,"15min":1.806},"min":{"1min":0.39900000000000002,"5min":0.374,"15min":0.374},"max":{"1min":7.2569999999999997,"5min":12.077999999999999,"15min":12.125},"last":2.2610000000000001},{"interface":"front","average":{"1min":1.25,"5min":1.6020000000000001,"15min":1.573},"min":{"1min":0.67400000000000004,"5min":0.35999999999999999,"15min":0.35899999999999999},"max":{"1min":4.4909999999999997,"5min":9.6180000000000003,"15min":12.007999999999999},"last":4.5149999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:30:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3899999999999999,"5min":1.8500000000000001,"15min":1.6319999999999999},"min":{"1min":0.76700000000000002,"5min":0.308,"15min":0.308},"max":{"1min":4.46,"5min":9.5530000000000008,"15min":11.977},"last":4.4569999999999999},{"interface":"front","average":{"1min":1.0680000000000001,"5min":1.5449999999999999,"15min":1.8280000000000001},"min":{"1min":0.63,"5min":0.41599999999999998,"15min":0.40100000000000002},"max":{"1min":2.4319999999999999,"5min":10.795,"15min":12.1},"last":0.84199999999999997}]}]},{"osd":2,"up_from":106,"seq":455266533739,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2459584,"kb_used_data":1655864,"kb_used_omap":7,"kb_used_meta":803704,"kb_avail":91285568,"statfs":{"total":95995035648,"available":93476421632,"internally_reserved":0,"allocated":1695604736,"data_stored":1694101165,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":822992977},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:30:50 2023","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":2.5310000000000001,"15min":2.3119999999999998},"min":{"1min":0.22800000000000001,"5min":0.19,"15min":0.19},"max":{"1min":3.4359999999999999,"5min":27.010999999999999,"15min":40.204999999999998},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.80800000000000005,"5min":2.2490000000000001,"15min":2.1669999999999998},"min":{"1min":0.28399999999999997,"5min":0.23699999999999999,"15min":0.21199999999999999},"max":{"1min":4.2629999999999999,"5min":18.922000000000001,"15min":40.170000000000002},"last":1.1779999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:30:50 2023","interfaces":[{"interface":"back","average":{"1min":1.083,"5min":1.9910000000000001,"15min":1.9670000000000001},"min":{"1min":0.40899999999999997,"5min":0.22700000000000001,"15min":0.224},"max":{"1min":5.2869999999999999,"5min":15.419,"15min":23.251000000000001},"last":1.0049999999999999},{"interface":"front","average":{"1min":0.88800000000000001,"5min":2.391,"15min":2.113},"min":{"1min":0.39100000000000001,"5min":0.22600000000000001,"15min":0.20399999999999999},"max":{"1min":3.4279999999999999,"5min":14.034000000000001,"15min":17.632999999999999},"last":0.51000000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:31:05 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":2.6760000000000002,"15min":2.4630000000000001},"min":{"1min":0.40500000000000003,"5min":0.307,"15min":0.23599999999999999},"max":{"1min":1.6479999999999999,"5min":15.334,"15min":15.334},"last":1.198},{"interface":"front","average":{"1min":1.234,"5min":2.8399999999999999,"15min":2.3580000000000001},"min":{"1min":0.309,"5min":0.27000000000000002,"15min":0.23899999999999999},"max":{"1min":4.6349999999999998,"5min":18.515000000000001,"15min":18.515000000000001},"last":2.5939999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:30:50 2023","interfaces":[{"interface":"back","average":{"1min":1.1639999999999999,"5min":1.96,"15min":1.748},"min":{"1min":0.63800000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":5.2380000000000004,"5min":10.597,"15min":11.157999999999999},"last":1.3300000000000001},{"interface":"front","average":{"1min":1.0660000000000001,"5min":1.994,"15min":1.744},"min":{"1min":0.65500000000000003,"5min":0.40300000000000002,"15min":0.29699999999999999},"max":{"1min":3.4910000000000001,"5min":13.097,"15min":13.097},"last":0.53900000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:30:50 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":1.6639999999999999,"15min":1.6100000000000001},"min":{"1min":0.40400000000000003,"5min":0.35599999999999998,"15min":0.33000000000000002},"max":{"1min":1.9199999999999999,"5min":9.1319999999999997,"15min":9.9190000000000005},"last":0.627},{"interface":"front","average":{"1min":1.028,"5min":1.75,"15min":1.744},"min":{"1min":0.52300000000000002,"5min":0.39700000000000002,"15min":0.34699999999999998},"max":{"1min":1.8129999999999999,"5min":10.686,"15min":11.205},"last":1.3440000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:30:50 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":2.044,"15min":1.9099999999999999},"min":{"1min":0.60999999999999999,"5min":0.42999999999999999,"15min":0.32500000000000001},"max":{"1min":3.4620000000000002,"5min":13.209,"15min":16.337},"last":0.46700000000000003},{"interface":"front","average":{"1min":1.034,"5min":1.829,"15min":1.8169999999999999},"min":{"1min":0.53000000000000003,"5min":0.38,"15min":0.35199999999999998},"max":{"1min":1.95,"5min":9.2289999999999992,"15min":16.399999999999999},"last":0.65300000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:30:50 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":2.1880000000000002,"15min":1.8320000000000001},"min":{"1min":0.46300000000000002,"5min":0.436,"15min":0.38300000000000001},"max":{"1min":1.867,"5min":13.571999999999999,"15min":13.571999999999999},"last":0.56100000000000005},{"interface":"front","average":{"1min":1.177,"5min":2.383,"15min":1.9770000000000001},"min":{"1min":0.752,"5min":0.41499999999999998,"15min":0.379},"max":{"1min":2.0179999999999998,"5min":10.659000000000001,"15min":11.236000000000001},"last":1.3089999999999999}]}]},{"osd":3,"up_from":123,"seq":528280977757,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1913124,"kb_used_data":985052,"kb_used_omap":2,"kb_used_meta":928061,"kb_avail":91832028,"statfs":{"total":95995035648,"available":94035996672,"internally_reserved":0,"allocated":1008693248,"data_stored":1007268912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":950334517},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:30:03 2023","interfaces":[{"interface":"back","average":{"1min":1.6000000000000001,"5min":2.4910000000000001,"15min":3.302},"min":{"1min":0.26200000000000001,"5min":0.20499999999999999,"15min":0.19500000000000001},"max":{"1min":7.6769999999999996,"5min":46.765999999999998,"15min":199.84999999999999},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.9359999999999999,"5min":2.7320000000000002,"15min":3.0579999999999998},"min":{"1min":0.40500000000000003,"5min":0.224,"15min":0.20200000000000001},"max":{"1min":9.3119999999999994,"5min":47.069000000000003,"15min":83.234999999999999},"last":0.42399999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:30:03 2023","interfaces":[{"interface":"back","average":{"1min":1.246,"5min":2.339,"15min":2.2570000000000001},"min":{"1min":0.29999999999999999,"5min":0.22,"15min":0.22},"max":{"1min":8.3209999999999997,"5min":26.372,"15min":26.372},"last":2.2290000000000001},{"interface":"front","average":{"1min":1.446,"5min":2.4129999999999998,"15min":2.0939999999999999},"min":{"1min":0.499,"5min":0.24399999999999999,"15min":0.218},"max":{"1min":7.899,"5min":28.007999999999999,"15min":28.007999999999999},"last":1.4810000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:30:03 2023","interfaces":[{"interface":"back","average":{"1min":1.806,"5min":2.819,"15min":2.1819999999999999},"min":{"1min":0.47099999999999997,"5min":0.24299999999999999,"15min":0.22700000000000001},"max":{"1min":8.1500000000000004,"5min":29.945,"15min":29.945},"last":2.7879999999999998},{"interface":"front","average":{"1min":1.6439999999999999,"5min":2.4510000000000001,"15min":2.4409999999999998},"min":{"1min":0.45000000000000001,"5min":0.251,"15min":0.22800000000000001},"max":{"1min":8.1400000000000006,"5min":26.452999999999999,"15min":26.452999999999999},"last":2.1120000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:30:03 2023","interfaces":[{"interface":"back","average":{"1min":1.425,"5min":1.7709999999999999,"15min":1.885},"min":{"1min":0.53300000000000003,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":8.0199999999999996,"5min":8.3629999999999995,"15min":12.438000000000001},"last":1.9370000000000001},{"interface":"front","average":{"1min":1.657,"5min":1.8160000000000001,"15min":2.2669999999999999},"min":{"1min":0.55000000000000004,"5min":0.36299999999999999,"15min":0.316},"max":{"1min":8.173,"5min":13.037000000000001,"15min":199.50800000000001},"last":0.47399999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:30:03 2023","interfaces":[{"interface":"back","average":{"1min":1.6419999999999999,"5min":1.853,"15min":2.3010000000000002},"min":{"1min":0.56899999999999995,"5min":0.40400000000000003,"15min":0.33500000000000002},"max":{"1min":8.2200000000000006,"5min":13.079000000000001,"15min":199.93799999999999},"last":0.497},{"interface":"front","average":{"1min":1.6719999999999999,"5min":1.9430000000000001,"15min":1.6299999999999999},"min":{"1min":0.45900000000000002,"5min":0.38300000000000001,"15min":0.34899999999999998},"max":{"1min":9.4410000000000007,"5min":9.8109999999999999,"15min":11.135},"last":0.51300000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:30:03 2023","interfaces":[{"interface":"back","average":{"1min":1.6859999999999999,"5min":1.9430000000000001,"15min":1.708},"min":{"1min":0.502,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":9.4120000000000008,"5min":9.7789999999999999,"15min":11.173999999999999},"last":0.46100000000000002},{"interface":"front","average":{"1min":1.7090000000000001,"5min":2.0750000000000002,"15min":2.113},"min":{"1min":0.63100000000000001,"5min":0.36699999999999999,"15min":0.32800000000000001},"max":{"1min":8.3889999999999993,"5min":8.5500000000000007,"15min":12.407},"last":2.012}]},{"osd":7,"last update":"Mon Oct 9 05:30:03 2023","interfaces":[{"interface":"back","average":{"1min":1.5329999999999999,"5min":2.0760000000000001,"15min":1.9690000000000001},"min":{"1min":0.59199999999999997,"5min":0.39100000000000001,"15min":0.378},"max":{"1min":8.4169999999999998,"5min":8.6039999999999992,"15min":12.468},"last":1.9810000000000001},{"interface":"front","average":{"1min":1.7689999999999999,"5min":2.1280000000000001,"15min":2.4820000000000002},"min":{"1min":0.69099999999999995,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":8.2539999999999996,"5min":12.989000000000001,"15min":199.96799999999999},"last":0.41199999999999998}]}]},{"osd":4,"up_from":0,"seq":128849019514,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":0,"seq":163208757870,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1327759360,"data_stored":1327759360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1199833088,"data_stored":1199833088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1735917568,"data_stored":1735917568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1012203520,"data_stored":1012203520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1339621376,"data_stored":1339621376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1511456768,"data_stored":1511456768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:31:14.613 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_8 pgp_num 16 2023-10-09T05:31:14.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:14 smithi194 bash[92666]: audit 2023-10-09T05:31:13.482292+0000 mon.a (mon.0) 1639 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:31:14.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:14 smithi194 bash[92666]: cluster 2023-10-09T05:31:13.542595+0000 osd.5 (osd.5) 268 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T05:31:14.645 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:14.675 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:14.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:14.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:14.780 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:14.810 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:14.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:14.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:14.913 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:14.939 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:14.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:14.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:15.042 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:15.074 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:15.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:15.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:15.180 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:15.208 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:15.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:15.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:15.314 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:15.339 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:15.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:15.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:15.377 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:15 smithi133 bash[102858]: cluster 2023-10-09T05:31:14.279427+0000 mgr.y (mgr.25277) 1294 : cluster 0 pgmap v1385: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-10-09T05:31:15.377 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:15 smithi133 bash[102858]: audit 2023-10-09T05:31:14.649751+0000 mon.a (mon.0) 1640 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [0, 2]}]: dispatch 2023-10-09T05:31:15.377 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:15 smithi133 bash[102858]: audit 2023-10-09T05:31:14.650516+0000 mon.a (mon.0) 1641 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [6, 2]}]: dispatch 2023-10-09T05:31:15.442 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:15.469 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:15.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:15.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:15.519 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 36 16 52071 52055 90.364 54.25 0.0147137 0.0110603 2023-10-09T05:31:15.574 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:15.599 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:15.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:15.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:15.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:15 smithi194 bash[92666]: cluster 2023-10-09T05:31:14.279427+0000 mgr.y (mgr.25277) 1294 : cluster 0 pgmap v1385: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-10-09T05:31:15.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:15 smithi194 bash[92666]: audit 2023-10-09T05:31:14.649751+0000 mon.a (mon.0) 1640 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [0, 2]}]: dispatch 2023-10-09T05:31:15.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:15 smithi194 bash[92666]: audit 2023-10-09T05:31:14.650516+0000 mon.a (mon.0) 1641 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [6, 2]}]: dispatch 2023-10-09T05:31:15.703 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:15.733 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:15.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:15.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:15.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:15 smithi133 bash[105589]: cluster 2023-10-09T05:31:14.279427+0000 mgr.y (mgr.25277) 1294 : cluster 0 pgmap v1385: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-10-09T05:31:15.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:15 smithi133 bash[105589]: audit 2023-10-09T05:31:14.649751+0000 mon.a (mon.0) 1640 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [0, 2]}]: dispatch 2023-10-09T05:31:15.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:15 smithi133 bash[105589]: audit 2023-10-09T05:31:14.650516+0000 mon.a (mon.0) 1641 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [6, 2]}]: dispatch 2023-10-09T05:31:15.838 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:15.865 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:15.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:15.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:15.968 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:15.996 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:15.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:15.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:16.099 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:16.128 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:16.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:16.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:16.238 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:16.267 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:16.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:16.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:16.373 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:16.392 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:16 smithi133 bash[102858]: audit 2023-10-09T05:31:15.371682+0000 mon.a (mon.0) 1642 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [0, 2]}]': finished 2023-10-09T05:31:16.392 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:16 smithi133 bash[102858]: audit 2023-10-09T05:31:15.371921+0000 mon.a (mon.0) 1643 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [6, 2]}]': finished 2023-10-09T05:31:16.393 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:16 smithi133 bash[102858]: cluster 2023-10-09T05:31:15.372123+0000 mon.a (mon.0) 1644 : cluster 0 osdmap e295: 8 total, 8 up, 6 in 2023-10-09T05:31:16.393 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:16 smithi133 bash[102858]: cluster 2023-10-09T05:31:15.379765+0000 osd.7 (osd.7) 250 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T05:31:16.393 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:16 smithi133 bash[102858]: cluster 2023-10-09T05:31:15.387211+0000 osd.6 (osd.6) 252 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T05:31:16.393 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:16 smithi133 bash[102858]: cluster 2023-10-09T05:31:16.076906+0000 osd.5 (osd.5) 269 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T05:31:16.393 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:16 smithi133 bash[102858]: cluster 2023-10-09T05:31:16.078634+0000 osd.5 (osd.5) 270 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T05:31:16.394 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:16 smithi133 bash[102858]: cluster 2023-10-09T05:31:16.193204+0000 osd.4 (osd.4) 268 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T05:31:16.399 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:16.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:16.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:16.504 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:16.520 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 37 16 53218 53202 89.8573 71.6875 0.00973677 0.0111207 2023-10-09T05:31:16.537 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:16.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:16.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:16.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:16 smithi194 bash[92666]: audit 2023-10-09T05:31:15.371682+0000 mon.a (mon.0) 1642 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [0, 2]}]': finished 2023-10-09T05:31:16.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:16 smithi194 bash[92666]: audit 2023-10-09T05:31:15.371921+0000 mon.a (mon.0) 1643 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [6, 2]}]': finished 2023-10-09T05:31:16.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:16 smithi194 bash[92666]: cluster 2023-10-09T05:31:15.372123+0000 mon.a (mon.0) 1644 : cluster 0 osdmap e295: 8 total, 8 up, 6 in 2023-10-09T05:31:16.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:16 smithi194 bash[92666]: cluster 2023-10-09T05:31:15.379765+0000 osd.7 (osd.7) 250 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T05:31:16.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:16 smithi194 bash[92666]: cluster 2023-10-09T05:31:15.387211+0000 osd.6 (osd.6) 252 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T05:31:16.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:16 smithi194 bash[92666]: cluster 2023-10-09T05:31:16.076906+0000 osd.5 (osd.5) 269 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T05:31:16.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:16 smithi194 bash[92666]: cluster 2023-10-09T05:31:16.078634+0000 osd.5 (osd.5) 270 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T05:31:16.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:16 smithi194 bash[92666]: cluster 2023-10-09T05:31:16.193204+0000 osd.4 (osd.4) 268 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T05:31:16.643 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:16.662 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:16 smithi133 bash[105589]: audit 2023-10-09T05:31:15.371682+0000 mon.a (mon.0) 1642 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [0, 2]}]': finished 2023-10-09T05:31:16.662 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:16 smithi133 bash[105589]: audit 2023-10-09T05:31:15.371921+0000 mon.a (mon.0) 1643 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [6, 2]}]': finished 2023-10-09T05:31:16.662 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:16 smithi133 bash[105589]: cluster 2023-10-09T05:31:15.372123+0000 mon.a (mon.0) 1644 : cluster 0 osdmap e295: 8 total, 8 up, 6 in 2023-10-09T05:31:16.662 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:16 smithi133 bash[105589]: cluster 2023-10-09T05:31:15.379765+0000 osd.7 (osd.7) 250 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T05:31:16.662 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:16 smithi133 bash[105589]: cluster 2023-10-09T05:31:15.387211+0000 osd.6 (osd.6) 252 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T05:31:16.662 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:16 smithi133 bash[105589]: cluster 2023-10-09T05:31:16.076906+0000 osd.5 (osd.5) 269 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T05:31:16.663 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:16 smithi133 bash[105589]: cluster 2023-10-09T05:31:16.078634+0000 osd.5 (osd.5) 270 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T05:31:16.663 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:16 smithi133 bash[105589]: cluster 2023-10-09T05:31:16.193204+0000 osd.4 (osd.4) 268 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T05:31:16.686 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:16.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:16.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:16.795 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:16.821 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:16.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:16.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:16.925 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:16.957 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:16.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:16.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:17.061 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:17.084 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:17.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:17.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:17.190 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:17.214 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:17.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:17.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:17.320 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:17.347 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:17.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:17.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:17.453 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:17.481 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:17.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:17.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:17.520 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 38 16 54330 54314 89.3214 69.5 0.00866652 0.0111891 2023-10-09T05:31:17.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:17 smithi133 bash[102858]: cluster 2023-10-09T05:31:16.279891+0000 mgr.y (mgr.25277) 1295 : cluster 0 pgmap v1387: 89 pgs: 2 peering, 87 active+clean; 3.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T05:31:17.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:17 smithi133 bash[102858]: cluster 2023-10-09T05:31:16.375828+0000 mon.a (mon.0) 1645 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:31:17.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:17 smithi133 bash[102858]: cluster 2023-10-09T05:31:16.392434+0000 osd.5 (osd.5) 271 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T05:31:17.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:17 smithi133 bash[102858]: cluster 2023-10-09T05:31:16.396890+0000 osd.6 (osd.6) 253 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T05:31:17.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:17 smithi133 bash[102858]: cluster 2023-10-09T05:31:16.397837+0000 mon.a (mon.0) 1646 : cluster 0 osdmap e296: 8 total, 8 up, 6 in 2023-10-09T05:31:17.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:17 smithi133 bash[102858]: cluster 2023-10-09T05:31:16.398103+0000 osd.7 (osd.7) 251 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T05:31:17.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:17 smithi133 bash[102858]: cluster 2023-10-09T05:31:16.403569+0000 osd.6 (osd.6) 254 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T05:31:17.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:17 smithi133 bash[102858]: cluster 2023-10-09T05:31:16.404651+0000 osd.7 (osd.7) 252 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T05:31:17.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:17 smithi133 bash[102858]: cluster 2023-10-09T05:31:16.405102+0000 osd.6 (osd.6) 255 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T05:31:17.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:17 smithi133 bash[102858]: cluster 2023-10-09T05:31:16.447458+0000 osd.4 (osd.4) 269 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T05:31:17.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:17 smithi133 bash[105589]: cluster 2023-10-09T05:31:16.279891+0000 mgr.y (mgr.25277) 1295 : cluster 0 pgmap v1387: 89 pgs: 2 peering, 87 active+clean; 3.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T05:31:17.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:17 smithi133 bash[105589]: cluster 2023-10-09T05:31:16.375828+0000 mon.a (mon.0) 1645 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:31:17.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:17 smithi133 bash[105589]: cluster 2023-10-09T05:31:16.392434+0000 osd.5 (osd.5) 271 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T05:31:17.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:17 smithi133 bash[105589]: cluster 2023-10-09T05:31:16.396890+0000 osd.6 (osd.6) 253 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T05:31:17.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:17 smithi133 bash[105589]: cluster 2023-10-09T05:31:16.397837+0000 mon.a (mon.0) 1646 : cluster 0 osdmap e296: 8 total, 8 up, 6 in 2023-10-09T05:31:17.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:17 smithi133 bash[105589]: cluster 2023-10-09T05:31:16.398103+0000 osd.7 (osd.7) 251 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T05:31:17.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:17 smithi133 bash[105589]: cluster 2023-10-09T05:31:16.403569+0000 osd.6 (osd.6) 254 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T05:31:17.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:17 smithi133 bash[105589]: cluster 2023-10-09T05:31:16.404651+0000 osd.7 (osd.7) 252 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T05:31:17.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:17 smithi133 bash[105589]: cluster 2023-10-09T05:31:16.405102+0000 osd.6 (osd.6) 255 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T05:31:17.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:17 smithi133 bash[105589]: cluster 2023-10-09T05:31:16.447458+0000 osd.4 (osd.4) 269 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T05:31:17.585 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:17.594 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:17 smithi194 bash[92666]: cluster 2023-10-09T05:31:16.279891+0000 mgr.y (mgr.25277) 1295 : cluster 0 pgmap v1387: 89 pgs: 2 peering, 87 active+clean; 3.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T05:31:17.595 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:17 smithi194 bash[92666]: cluster 2023-10-09T05:31:16.375828+0000 mon.a (mon.0) 1645 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:31:17.595 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:17 smithi194 bash[92666]: cluster 2023-10-09T05:31:16.392434+0000 osd.5 (osd.5) 271 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T05:31:17.595 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:17 smithi194 bash[92666]: cluster 2023-10-09T05:31:16.396890+0000 osd.6 (osd.6) 253 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T05:31:17.595 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:17 smithi194 bash[92666]: cluster 2023-10-09T05:31:16.397837+0000 mon.a (mon.0) 1646 : cluster 0 osdmap e296: 8 total, 8 up, 6 in 2023-10-09T05:31:17.595 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:17 smithi194 bash[92666]: cluster 2023-10-09T05:31:16.398103+0000 osd.7 (osd.7) 251 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T05:31:17.595 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:17 smithi194 bash[92666]: cluster 2023-10-09T05:31:16.403569+0000 osd.6 (osd.6) 254 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T05:31:17.595 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:17 smithi194 bash[92666]: cluster 2023-10-09T05:31:16.404651+0000 osd.7 (osd.7) 252 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T05:31:17.595 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:17 smithi194 bash[92666]: cluster 2023-10-09T05:31:16.405102+0000 osd.6 (osd.6) 255 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T05:31:17.595 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:17 smithi194 bash[92666]: cluster 2023-10-09T05:31:16.447458+0000 osd.4 (osd.4) 269 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T05:31:17.612 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:17.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:17.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:17.715 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:17.753 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:17.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:17.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:17.857 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:17.892 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:17.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:17.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:17.965 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:31:17.980 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:31:17.999 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:18.053 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:18.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:18.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:18.157 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:18.182 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:18.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:18.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:18.286 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:18.308 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:18.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:18.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:18.413 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:18.440 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:18.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:18.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:18.520 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 39 16 55363 55347 88.6864 64.5625 0.0110702 0.011269 2023-10-09T05:31:18.544 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:18.580 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:18.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:18.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:18.670 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:31:18.670 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:31:18.686 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:18.732 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:18.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:18.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:18.842 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:18.869 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:18.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:18.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:18.974 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:19.007 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:19.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:19.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:19.114 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:19.142 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:19.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:19.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:19.245 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:19.277 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:19.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:19.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:19.381 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:19.415 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:19.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:19.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:19.520 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:19.521 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T05:31:19.520646+0000 min lat: 0.00142329 max lat: 0.308726 avg lat: 0.0113437 2023-10-09T05:31:19.521 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:31:19.521 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 40 16 56409 56393 88.1035 65.375 0.0127347 0.0113437 2023-10-09T05:31:19.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:19 smithi133 bash[105589]: cluster 2023-10-09T05:31:18.280535+0000 mgr.y (mgr.25277) 1296 : cluster 0 pgmap v1389: 89 pgs: 2 peering, 87 active+clean; 3.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 44 MiB/s wr, 701 op/s 2023-10-09T05:31:19.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:19 smithi133 bash[102858]: cluster 2023-10-09T05:31:18.280535+0000 mgr.y (mgr.25277) 1296 : cluster 0 pgmap v1389: 89 pgs: 2 peering, 87 active+clean; 3.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 44 MiB/s wr, 701 op/s 2023-10-09T05:31:19.551 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:19.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:19.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:19.655 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:19.681 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:19.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:19.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:19.786 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:19.821 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:19.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:19.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:19.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:19 smithi194 bash[92666]: cluster 2023-10-09T05:31:18.280535+0000 mgr.y (mgr.25277) 1296 : cluster 0 pgmap v1389: 89 pgs: 2 peering, 87 active+clean; 3.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 44 MiB/s wr, 701 op/s 2023-10-09T05:31:19.923 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:19.952 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:19.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:19.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:20.055 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:20.101 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:20.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:20.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:20.205 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:20.233 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:20.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:20.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:20.338 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:20.366 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:20.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:20.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:20.470 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:20.498 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:20.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:20.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:20.520 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 41 16 57501 57485 87.6191 68.25 0.0240362 0.011407 2023-10-09T05:31:20.605 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:20.637 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:20.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:20.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:20.743 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:20.797 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:20.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:20.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:20.900 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:20.932 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:20.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:20.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:21.038 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:21.064 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:21.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:21.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:21.168 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:21.207 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:21.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:21.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:21.312 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:21.334 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:21.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:21.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:21.440 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:21.469 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:21.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:21.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:21.520 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 42 16 58455 58439 86.9524 59.625 0.00528016 0.0114699 2023-10-09T05:31:21.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:21 smithi133 bash[105589]: cluster 2023-10-09T05:31:20.281378+0000 mgr.y (mgr.25277) 1297 : cluster 0 pgmap v1390: 89 pgs: 2 peering, 87 active+clean; 3.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T05:31:21.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:21 smithi133 bash[102858]: cluster 2023-10-09T05:31:20.281378+0000 mgr.y (mgr.25277) 1297 : cluster 0 pgmap v1390: 89 pgs: 2 peering, 87 active+clean; 3.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T05:31:21.572 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:21.600 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:21.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:21.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:21.703 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:21.725 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:21.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:21.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:21.830 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:21.859 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:21.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:21.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:21.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:21 smithi194 bash[92666]: cluster 2023-10-09T05:31:20.281378+0000 mgr.y (mgr.25277) 1297 : cluster 0 pgmap v1390: 89 pgs: 2 peering, 87 active+clean; 3.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T05:31:21.963 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:21.988 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:21.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:21.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:22.091 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:22.118 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:22.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:22.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:22.222 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:22.246 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:22.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:22.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:22.350 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:22.381 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:22.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:22.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:22.486 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:22.520 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 43 16 59303 59287 86.1627 53 0.0128322 0.0115994 2023-10-09T05:31:22.523 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:22.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:22.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:22.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:22 smithi133 bash[105589]: cluster 2023-10-09T05:31:22.397478+0000 mon.a (mon.0) 1647 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:31:22.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:22 smithi133 bash[102858]: cluster 2023-10-09T05:31:22.397478+0000 mon.a (mon.0) 1647 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:31:22.628 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:22.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:22 smithi194 bash[92666]: cluster 2023-10-09T05:31:22.397478+0000 mon.a (mon.0) 1647 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:31:22.665 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:22.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:22.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:22.745 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:31:22.745 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:31:22.770 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:22.796 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:22.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:22.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:22.901 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:22.935 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:22.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:22.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:23.042 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:23.078 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:23.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:23.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:23.182 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:23.212 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:23.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:23.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:23.317 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:23.344 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:23.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:23.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:23.448 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:23.456 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:23 smithi133 bash[102858]: cluster 2023-10-09T05:31:22.282002+0000 mgr.y (mgr.25277) 1298 : cluster 0 pgmap v1391: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 79 MiB/s wr, 1.27k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T05:31:23.473 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:23.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:23.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:23.520 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 44 16 60399 60383 85.7612 68.5 0.00499513 0.0116537 2023-10-09T05:31:23.576 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:23.585 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:23 smithi194 bash[92666]: cluster 2023-10-09T05:31:22.282002+0000 mgr.y (mgr.25277) 1298 : cluster 0 pgmap v1391: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 79 MiB/s wr, 1.27k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T05:31:23.600 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:23.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:23.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:23.704 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:23.732 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:23.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:23.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:23.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:23 smithi133 bash[105589]: cluster 2023-10-09T05:31:22.282002+0000 mgr.y (mgr.25277) 1298 : cluster 0 pgmap v1391: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 79 MiB/s wr, 1.27k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T05:31:23.835 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:23.866 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:23.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:23.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:23.975 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:24.002 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:24.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:24.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:24.108 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:24.145 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:24.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:24.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:24.250 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:24.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:24.284 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:24.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:24.385 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:24.411 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:24.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:24.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:24.516 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:24.521 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 45 16 61427 61411 85.283 64.25 0.00533994 0.01172 2023-10-09T05:31:24.539 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:24.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:24.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:24.547 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:31:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:31:24] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.43.0" 2023-10-09T05:31:24.642 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:24.680 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:24.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:24.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:24.783 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:24.813 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:24.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:24.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:24.916 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:24.945 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:24.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:24.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:25.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:24 smithi133 bash[105589]: cluster 2023-10-09T05:31:24.282433+0000 mgr.y (mgr.25277) 1299 : cluster 0 pgmap v1392: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 42 MiB/s wr, 664 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T05:31:25.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:24 smithi133 bash[102858]: cluster 2023-10-09T05:31:24.282433+0000 mgr.y (mgr.25277) 1299 : cluster 0 pgmap v1392: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 42 MiB/s wr, 664 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T05:31:25.052 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:25.073 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:25.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:25.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:25.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:24 smithi194 bash[92666]: cluster 2023-10-09T05:31:24.282433+0000 mgr.y (mgr.25277) 1299 : cluster 0 pgmap v1392: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 42 MiB/s wr, 664 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T05:31:25.177 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:25.209 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:25.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:25.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:25.312 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:25.342 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:25.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:25.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:25.446 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:25.474 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:25.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:25.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:25.521 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 46 16 62259 62243 84.5593 52 0.00660881 0.0118192 2023-10-09T05:31:25.577 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:25.607 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:25.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:25.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:25.711 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:25.737 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:25.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:25.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:25.842 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:25.873 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:25.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:25.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:25.980 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:26.010 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:26.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:26.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:26.115 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:26.150 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:26.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:26.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:26.252 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:26.278 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:26.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:26.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:26.384 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:26.418 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:26.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:26.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:26.521 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 47 16 63301 63285 84.1457 65.125 0.0136229 0.0118764 2023-10-09T05:31:26.521 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:26.547 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:26.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:26.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:26.650 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:26.679 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:26.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:26.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:26.782 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:26.807 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:26.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:26.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:26.912 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:26.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:26.946 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:26.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:27.047 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:27.068 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:27.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:27.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:27.170 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:27.202 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:27.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:27.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:27.306 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:27.332 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:27.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:27.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:27.437 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:27.465 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:27.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:27.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:27.521 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 48 16 64100 64084 83.433 49.9375 0.00642082 0.0119794 2023-10-09T05:31:27.571 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:27.578 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:27 smithi194 bash[92666]: cluster 2023-10-09T05:31:26.283178+0000 mgr.y (mgr.25277) 1300 : cluster 0 pgmap v1393: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T05:31:27.598 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:27.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:27.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:27.703 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:27.712 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:27 smithi133 bash[102858]: cluster 2023-10-09T05:31:26.283178+0000 mgr.y (mgr.25277) 1300 : cluster 0 pgmap v1393: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T05:31:27.712 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:27 smithi133 bash[105589]: cluster 2023-10-09T05:31:26.283178+0000 mgr.y (mgr.25277) 1300 : cluster 0 pgmap v1393: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T05:31:27.733 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:27.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:27.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:27.837 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:27.864 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:27.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:27.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:27.968 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:27.990 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:27.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:27.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:28.093 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:28.117 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:28.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:28.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:28.223 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:28.248 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:28.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:28.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:28.352 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:28.380 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:28.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:28.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:28.521 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 49 16 65028 65012 82.9138 58 0.0124056 0.0120541 2023-10-09T05:31:28.556 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:28.588 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:28.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:28.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:28.695 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:28.726 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:28.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:28.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:28.831 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:28.864 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:28.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:28.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:28.969 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:28.995 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:28.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:28.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:29.099 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:29.128 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:29.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:29.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:29.231 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:29.266 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:29.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:29.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:29.370 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:29.379 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:29 smithi194 bash[92666]: cluster 2023-10-09T05:31:28.283559+0000 mgr.y (mgr.25277) 1301 : cluster 0 pgmap v1394: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 54 MiB/s wr, 857 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T05:31:29.379 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:29 smithi194 bash[92666]: audit 2023-10-09T05:31:28.482384+0000 mon.a (mon.0) 1648 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:31:29.379 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:29 smithi194 bash[92666]: audit 2023-10-09T05:31:29.153463+0000 mon.a (mon.0) 1649 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:31:29.398 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:29.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:29.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:29.508 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:29.519 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:29 smithi133 bash[102858]: cluster 2023-10-09T05:31:28.283559+0000 mgr.y (mgr.25277) 1301 : cluster 0 pgmap v1394: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 54 MiB/s wr, 857 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T05:31:29.519 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:29 smithi133 bash[102858]: audit 2023-10-09T05:31:28.482384+0000 mon.a (mon.0) 1648 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:31:29.519 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:29 smithi133 bash[102858]: audit 2023-10-09T05:31:29.153463+0000 mon.a (mon.0) 1649 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:31:29.519 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:29 smithi133 bash[105589]: cluster 2023-10-09T05:31:28.283559+0000 mgr.y (mgr.25277) 1301 : cluster 0 pgmap v1394: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 54 MiB/s wr, 857 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T05:31:29.520 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:29 smithi133 bash[105589]: audit 2023-10-09T05:31:28.482384+0000 mon.a (mon.0) 1648 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:31:29.520 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:29 smithi133 bash[105589]: audit 2023-10-09T05:31:29.153463+0000 mon.a (mon.0) 1649 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:31:29.521 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 50 16 65860 65844 82.2947 52 0.0211013 0.0121454 2023-10-09T05:31:29.533 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:29.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:29.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:29.640 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:29.668 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:29.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:29.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:29.774 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:29.803 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:29.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:29.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:29.907 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:29.935 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:29.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:29.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:30.039 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:30.062 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:30.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:30.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:30.167 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:30.201 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:30.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:30.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:30.306 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:30.335 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:30.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:30.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:30.438 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:30.463 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:30.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:30.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:30.522 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 51 16 66790 66774 81.8207 58.125 0.0175297 0.0122108 2023-10-09T05:31:30.565 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:30.590 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:30.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:30.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:30.693 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:30.718 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:30.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:30.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:30.824 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:30.860 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:30.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:30.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:30.964 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:30.992 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:30.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:30.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:31.095 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:31.123 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:31.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:31.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:31.230 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:31.262 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:31.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:31.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:31.366 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:31.396 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:31.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:31.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:31.499 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:31 smithi133 bash[105589]: cluster 2023-10-09T05:31:30.284231+0000 mgr.y (mgr.25277) 1302 : cluster 0 pgmap v1395: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 69 MiB/s wr, 1.10k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T05:31:31.499 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:31 smithi133 bash[102858]: cluster 2023-10-09T05:31:30.284231+0000 mgr.y (mgr.25277) 1302 : cluster 0 pgmap v1395: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 69 MiB/s wr, 1.10k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T05:31:31.500 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:31.522 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 52 16 67509 67493 81.1113 44.9375 0.0130317 0.0123216 2023-10-09T05:31:31.540 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:31.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:31.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:31.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:31 smithi194 bash[92666]: cluster 2023-10-09T05:31:30.284231+0000 mgr.y (mgr.25277) 1302 : cluster 0 pgmap v1395: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 69 MiB/s wr, 1.10k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T05:31:31.643 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:31.675 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:31.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:31.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:31.780 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:31.808 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:31.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:31.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:31.912 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:31.937 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:31.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:31.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:32.044 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:32.071 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:32.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:32.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:32.174 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:32.216 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:32.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:32.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:32.320 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:32.345 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:32.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:32.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:32.451 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:32.491 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:32.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:32.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:32.522 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 53 16 68330 68314 80.549 51.3125 0.197747 0.012374 2023-10-09T05:31:32.595 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:32.624 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:32.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:32.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:32.728 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:32.771 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:32.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:32.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:32.878 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:32.910 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:32.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:32.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:33.018 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:33.049 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:33.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:33.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:33.153 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:33.172 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:33.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:33.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:33.275 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:33.305 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:33.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:33.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:33.411 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:33.445 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:33.449 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:33 smithi133 bash[105589]: cluster 2023-10-09T05:31:32.284809+0000 mgr.y (mgr.25277) 1303 : cluster 0 pgmap v1396: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 57 MiB/s wr, 919 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T05:31:33.450 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:33 smithi133 bash[102858]: cluster 2023-10-09T05:31:32.284809+0000 mgr.y (mgr.25277) 1303 : cluster 0 pgmap v1396: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 57 MiB/s wr, 919 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T05:31:33.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:33.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:33.522 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 54 16 69282 69266 80.1591 59.5 0.00586889 0.0124685 2023-10-09T05:31:33.551 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:33.585 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:33.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:33.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:33.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:33 smithi194 bash[92666]: cluster 2023-10-09T05:31:32.284809+0000 mgr.y (mgr.25277) 1303 : cluster 0 pgmap v1396: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 57 MiB/s wr, 919 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T05:31:33.689 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:33.723 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:33.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:33.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:33.826 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:33.856 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:33.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:33.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:33.962 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:33.993 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:33.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:33.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:34.099 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:34.129 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:34.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:34.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:34.237 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:34.262 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:34.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:34.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:34.365 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:34.373 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:31:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:31:34] "GET /metrics HTTP/1.1" 200 36474 "" "Prometheus/2.43.0" 2023-10-09T05:31:34.398 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:34.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:34.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:34.502 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:34.523 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 55 16 70255 70239 79.8073 60.8125 0.020652 0.0125246 2023-10-09T05:31:34.531 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:34.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:34.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:34.636 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:34.663 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:34.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:34.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:34.767 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:34.792 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:34.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:34.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:34.896 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:34.922 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:34.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:34.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:35.030 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:35.040 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:34 smithi133 bash[105589]: cluster 2023-10-09T05:31:34.285221+0000 mgr.y (mgr.25277) 1304 : cluster 0 pgmap v1397: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 49 MiB/s wr, 783 op/s 2023-10-09T05:31:35.040 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:34 smithi133 bash[102858]: cluster 2023-10-09T05:31:34.285221+0000 mgr.y (mgr.25277) 1304 : cluster 0 pgmap v1397: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 49 MiB/s wr, 783 op/s 2023-10-09T05:31:35.059 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:35.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:35.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:35.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:34 smithi194 bash[92666]: cluster 2023-10-09T05:31:34.285221+0000 mgr.y (mgr.25277) 1304 : cluster 0 pgmap v1397: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 49 MiB/s wr, 783 op/s 2023-10-09T05:31:35.165 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:35.190 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:35.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:35.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:35.296 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:35.321 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:35.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:35.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:35.425 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:35.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:35.462 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:35.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:35.522 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 56 16 71229 71213 79.4691 60.875 0.0189596 0.0125769 2023-10-09T05:31:35.563 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:35.592 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:35.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:35.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:35.695 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:35.723 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:35.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:35.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:35.829 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:35.856 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:35.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:35.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:35.961 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:35.996 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:36.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:36.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:36.102 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:36.144 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:36.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:36.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:36.247 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:36.271 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:36.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:36.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:36.375 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:36.407 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:36.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:36.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:36.511 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:36.522 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 57 16 72155 72139 79.0902 57.875 0.0167382 0.012638 2023-10-09T05:31:36.538 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:36.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:36.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:36.643 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:36.673 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:36.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:36.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:36.777 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:36.813 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:36.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:36.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:36.916 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:36.943 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:36.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:36.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:37.046 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:37.085 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:37.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:37.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:37.189 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:37.216 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:37.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:37.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:37.322 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:37.348 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:37.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:37.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:37.453 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:37.463 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:37 smithi194 bash[92666]: cluster 2023-10-09T05:31:36.288358+0000 mgr.y (mgr.25277) 1305 : cluster 0 pgmap v1398: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T05:31:37.479 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:37.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:37.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:37.522 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 58 16 72928 72912 78.5595 48.3125 0.0140984 0.0127231 2023-10-09T05:31:37.586 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:37.595 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:37 smithi133 bash[105589]: cluster 2023-10-09T05:31:36.288358+0000 mgr.y (mgr.25277) 1305 : cluster 0 pgmap v1398: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T05:31:37.596 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:37 smithi133 bash[102858]: cluster 2023-10-09T05:31:36.288358+0000 mgr.y (mgr.25277) 1305 : cluster 0 pgmap v1398: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T05:31:37.633 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:37.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:37.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:37.736 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:37.767 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:37.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:37.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:37.870 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:37.902 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:37.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:37.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:38.009 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:38.043 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:38.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:38.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:38.147 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:38.180 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:38.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:38.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:38.192 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:31:38.192 INFO:teuthology.orchestra.run.smithi133.stdout: "size": 20, 2023-10-09T05:31:38.192 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 600, 2023-10-09T05:31:38.192 INFO:teuthology.orchestra.run.smithi133.stdout: "ops": [ 2023-10-09T05:31:38.192 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.192 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35187.0:164981 11.6 11:68b38e9f:::benchmark_data_smithi133_194104_object69061:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2023-10-09T05:31:38.192 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:22:46.151468+0000", 2023-10-09T05:31:38.193 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 532.03801619700005, 2023-10-09T05:31:38.193 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.36817279600000002, 2023-10-09T05:31:38.193 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:31:38.193 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:31:38.193 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:31:38.193 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35187", 2023-10-09T05:31:38.193 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2118344251", 2023-10-09T05:31:38.193 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 164981 2023-10-09T05:31:38.193 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.194 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:31:38.194 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.194 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:31:38.194 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:22:46.151468+0000", 2023-10-09T05:31:38.194 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.194 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.194 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.194 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:31:38.194 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:22:46.151468+0000", 2023-10-09T05:31:38.194 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.194 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.195 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.195 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:31:38.195 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:22:46.151466+0000", 2023-10-09T05:31:38.195 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:31:38.195 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.195 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.195 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:31:38.195 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:22:46.151475+0000", 2023-10-09T05:31:38.195 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.174e-06 2023-10-09T05:31:38.195 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.195 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.196 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:31:38.196 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:22:46.151493+0000", 2023-10-09T05:31:38.196 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.8516999999999999e-05 2023-10-09T05:31:38.196 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.196 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.196 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:31:38.196 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:22:46.151503+0000", 2023-10-09T05:31:38.196 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.0225e-05 2023-10-09T05:31:38.196 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.196 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.196 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:31:38.196 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:22:46.151611+0000", 2023-10-09T05:31:38.197 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00010792400000000001 2023-10-09T05:31:38.197 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.197 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.197 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:31:38.197 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:22:46.151906+0000", 2023-10-09T05:31:38.197 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00029454699999999998 2023-10-09T05:31:38.197 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.197 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.197 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:31:38.197 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:22:46.152324+0000", 2023-10-09T05:31:38.197 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00041775599999999998 2023-10-09T05:31:38.198 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.198 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.198 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:31:38.198 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:22:46.178163+0000", 2023-10-09T05:31:38.198 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.025839142999999998 2023-10-09T05:31:38.198 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.198 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.198 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:31:38.199 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:22:46.204159+0000", 2023-10-09T05:31:38.199 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.025996426 2023-10-09T05:31:38.199 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.199 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.199 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:31:38.199 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:22:46.516074+0000", 2023-10-09T05:31:38.199 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.31191508699999998 2023-10-09T05:31:38.199 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.199 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.199 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:31:38.200 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:22:46.516262+0000", 2023-10-09T05:31:38.200 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000187658 2023-10-09T05:31:38.200 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.200 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.200 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:31:38.200 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:22:46.519641+0000", 2023-10-09T05:31:38.200 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0033789050000000002 2023-10-09T05:31:38.200 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.200 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:31:38.200 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.200 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.201 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.201 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45374.0:74862 12.c 12:3d72a797:::benchmark_data_smithi133_218956_object74861:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e272)", 2023-10-09T05:31:38.201 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:26:19.639206+0000", 2023-10-09T05:31:38.202 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 318.55027867000001, 2023-10-09T05:31:38.202 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.236398252, 2023-10-09T05:31:38.202 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:31:38.202 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:31:38.202 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:31:38.202 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45374", 2023-10-09T05:31:38.202 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2988063385", 2023-10-09T05:31:38.203 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 74862 2023-10-09T05:31:38.203 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.203 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:31:38.203 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.203 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:31:38.203 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.639206+0000", 2023-10-09T05:31:38.203 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.203 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.203 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.203 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:31:38.203 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.639206+0000", 2023-10-09T05:31:38.203 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.204 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.204 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.204 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:31:38.204 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.639204+0000", 2023-10-09T05:31:38.204 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999981 2023-10-09T05:31:38.204 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.204 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.204 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:31:38.204 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.639228+0000", 2023-10-09T05:31:38.204 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.4930999999999999e-05 2023-10-09T05:31:38.204 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.205 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.205 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:31:38.205 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.639241+0000", 2023-10-09T05:31:38.205 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.2924e-05 2023-10-09T05:31:38.205 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.205 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.205 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:31:38.205 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.639253+0000", 2023-10-09T05:31:38.205 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.1783999999999999e-05 2023-10-09T05:31:38.205 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.205 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.206 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:31:38.206 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.639317+0000", 2023-10-09T05:31:38.206 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.4253999999999998e-05 2023-10-09T05:31:38.206 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.206 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.206 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:31:38.206 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.639484+0000", 2023-10-09T05:31:38.206 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000166639 2023-10-09T05:31:38.206 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.206 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.206 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,7", 2023-10-09T05:31:38.207 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.639652+0000", 2023-10-09T05:31:38.207 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000167725 2023-10-09T05:31:38.207 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.207 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.207 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:31:38.207 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.642694+0000", 2023-10-09T05:31:38.207 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.003041827 2023-10-09T05:31:38.207 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.207 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.207 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:31:38.207 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.644440+0000", 2023-10-09T05:31:38.207 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.001746537 2023-10-09T05:31:38.208 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.208 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.208 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:31:38.208 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.875499+0000", 2023-10-09T05:31:38.208 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.23105906500000001 2023-10-09T05:31:38.208 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.208 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.208 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:31:38.208 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.875565+0000", 2023-10-09T05:31:38.208 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.5921999999999997e-05 2023-10-09T05:31:38.208 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.209 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.209 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:31:38.209 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.875604+0000", 2023-10-09T05:31:38.209 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.8688e-05 2023-10-09T05:31:38.209 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.209 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:31:38.209 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.209 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.209 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.209 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:63811 13.9 13:9f17c7cf:::benchmark_data_smithi133_241067_object63810:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-10-09T05:31:38.209 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:31:27.030518+0000", 2023-10-09T05:31:38.209 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 11.158966264, 2023-10-09T05:31:38.210 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.24935813600000001, 2023-10-09T05:31:38.210 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:31:38.210 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:31:38.210 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:31:38.210 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:31:38.210 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:31:38.210 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 63811 2023-10-09T05:31:38.210 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.210 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:31:38.210 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.210 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:31:38.211 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.030518+0000", 2023-10-09T05:31:38.211 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.211 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.211 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.211 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:31:38.211 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.030518+0000", 2023-10-09T05:31:38.211 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.211 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.211 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.211 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:31:38.211 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.030517+0000", 2023-10-09T05:31:38.211 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:31:38.212 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.212 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.212 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:31:38.212 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.030546+0000", 2023-10-09T05:31:38.212 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.9665000000000001e-05 2023-10-09T05:31:38.212 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.212 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.212 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:31:38.212 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.030557+0000", 2023-10-09T05:31:38.212 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.0558e-05 2023-10-09T05:31:38.212 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.212 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.213 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:31:38.213 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.030565+0000", 2023-10-09T05:31:38.213 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.7659999999999998e-06 2023-10-09T05:31:38.213 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.213 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.213 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:31:38.213 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.030614+0000", 2023-10-09T05:31:38.213 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.8863999999999998e-05 2023-10-09T05:31:38.213 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.213 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.213 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:31:38.214 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.030799+0000", 2023-10-09T05:31:38.214 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000184993 2023-10-09T05:31:38.214 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.214 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.214 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:31:38.214 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.030957+0000", 2023-10-09T05:31:38.214 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000158276 2023-10-09T05:31:38.214 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.215 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.215 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:31:38.215 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.042276+0000", 2023-10-09T05:31:38.215 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.01131908 2023-10-09T05:31:38.215 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.215 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.215 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:31:38.215 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.042315+0000", 2023-10-09T05:31:38.215 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.9026999999999998e-05 2023-10-09T05:31:38.215 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.215 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.216 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:31:38.216 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.278659+0000", 2023-10-09T05:31:38.216 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.23634405999999999 2023-10-09T05:31:38.216 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.216 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.216 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:31:38.216 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.279822+0000", 2023-10-09T05:31:38.216 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.001163063 2023-10-09T05:31:38.217 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.217 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.217 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:31:38.217 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.279876+0000", 2023-10-09T05:31:38.217 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.3912000000000002e-05 2023-10-09T05:31:38.217 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.217 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:31:38.217 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.217 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.217 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.217 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:63815 13.9 13:9184e866:::benchmark_data_smithi133_241067_object63814:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-10-09T05:31:38.218 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:31:27.031700+0000", 2023-10-09T05:31:38.218 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 11.157784061999999, 2023-10-09T05:31:38.218 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.248284432, 2023-10-09T05:31:38.218 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:31:38.218 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:31:38.218 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:31:38.218 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:31:38.219 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:31:38.219 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 63815 2023-10-09T05:31:38.219 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.219 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:31:38.219 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.219 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:31:38.219 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.031700+0000", 2023-10-09T05:31:38.219 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.219 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.219 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.219 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:31:38.220 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.031700+0000", 2023-10-09T05:31:38.220 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.220 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.220 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.220 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:31:38.220 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.031699+0000", 2023-10-09T05:31:38.220 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:31:38.220 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.221 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.221 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:31:38.221 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.031719+0000", 2023-10-09T05:31:38.221 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.0044999999999999e-05 2023-10-09T05:31:38.221 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.221 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.221 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:31:38.221 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.031729+0000", 2023-10-09T05:31:38.221 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.6660000000000003e-06 2023-10-09T05:31:38.222 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.222 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.222 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:31:38.222 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.031736+0000", 2023-10-09T05:31:38.222 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.0079999999999996e-06 2023-10-09T05:31:38.222 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.222 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.222 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:31:38.222 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.033533+0000", 2023-10-09T05:31:38.223 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00179659 2023-10-09T05:31:38.223 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.223 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.223 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:31:38.223 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.033694+0000", 2023-10-09T05:31:38.223 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00016135800000000001 2023-10-09T05:31:38.223 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.223 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.223 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:31:38.223 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.034107+0000", 2023-10-09T05:31:38.224 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00041344599999999998 2023-10-09T05:31:38.224 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.224 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.224 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:31:38.224 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.042296+0000", 2023-10-09T05:31:38.224 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0081888859999999994 2023-10-09T05:31:38.224 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.224 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.224 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:31:38.224 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.278325+0000", 2023-10-09T05:31:38.224 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.236028715 2023-10-09T05:31:38.224 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.225 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.225 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:31:38.225 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.279891+0000", 2023-10-09T05:31:38.225 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0015654239999999999 2023-10-09T05:31:38.225 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.225 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.225 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:31:38.225 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.279945+0000", 2023-10-09T05:31:38.225 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.4601999999999997e-05 2023-10-09T05:31:38.225 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.225 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.226 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:31:38.226 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.279985+0000", 2023-10-09T05:31:38.226 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.9484999999999999e-05 2023-10-09T05:31:38.226 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.226 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:31:38.226 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.226 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.226 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.226 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:63818 13.9 13:94d3e448:::benchmark_data_smithi133_241067_object63817:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-10-09T05:31:38.226 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:31:27.035673+0000", 2023-10-09T05:31:38.226 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 11.153811051, 2023-10-09T05:31:38.227 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.24440779800000001, 2023-10-09T05:31:38.227 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:31:38.227 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:31:38.227 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:31:38.227 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:31:38.227 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:31:38.227 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 63818 2023-10-09T05:31:38.227 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.228 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:31:38.228 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.228 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:31:38.228 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.035673+0000", 2023-10-09T05:31:38.228 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.228 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.228 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.228 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:31:38.229 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.035673+0000", 2023-10-09T05:31:38.229 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.229 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.229 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.229 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:31:38.229 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.035672+0000", 2023-10-09T05:31:38.229 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:31:38.229 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.230 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.230 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:31:38.230 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.035695+0000", 2023-10-09T05:31:38.230 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.2841e-05 2023-10-09T05:31:38.230 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.230 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.230 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:31:38.230 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.035707+0000", 2023-10-09T05:31:38.231 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.2442e-05 2023-10-09T05:31:38.231 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.231 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.231 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:31:38.231 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.035715+0000", 2023-10-09T05:31:38.231 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.8499999999999994e-06 2023-10-09T05:31:38.231 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.231 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.231 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:31:38.231 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.045379+0000", 2023-10-09T05:31:38.232 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0096640649999999995 2023-10-09T05:31:38.232 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.232 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.232 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:31:38.232 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.045524+0000", 2023-10-09T05:31:38.232 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000145317 2023-10-09T05:31:38.232 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.232 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.232 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:31:38.232 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.045634+0000", 2023-10-09T05:31:38.232 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00010958 2023-10-09T05:31:38.233 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.233 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.233 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:31:38.233 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.278423+0000", 2023-10-09T05:31:38.233 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.23278948999999999 2023-10-09T05:31:38.233 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.233 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.233 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:31:38.233 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.278636+0000", 2023-10-09T05:31:38.233 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00021249100000000001 2023-10-09T05:31:38.233 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.234 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.234 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:31:38.234 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.279992+0000", 2023-10-09T05:31:38.234 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0013564919999999999 2023-10-09T05:31:38.234 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.234 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.234 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:31:38.234 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.280041+0000", 2023-10-09T05:31:38.234 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.8715999999999999e-05 2023-10-09T05:31:38.234 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.234 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.235 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:31:38.235 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.280081+0000", 2023-10-09T05:31:38.235 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.9907999999999998e-05 2023-10-09T05:31:38.235 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.235 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:31:38.235 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.235 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.235 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.235 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:63826 13.5 13:a9c7d600:::benchmark_data_smithi133_241067_object63825:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-10-09T05:31:38.235 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:31:27.041516+0000", 2023-10-09T05:31:38.235 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 11.147968000000001, 2023-10-09T05:31:38.235 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.248416691, 2023-10-09T05:31:38.236 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:31:38.236 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:31:38.236 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:31:38.236 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:31:38.236 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:31:38.236 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 63826 2023-10-09T05:31:38.236 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.236 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:31:38.236 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.236 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:31:38.236 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.041516+0000", 2023-10-09T05:31:38.236 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.237 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.237 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.237 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:31:38.237 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.041516+0000", 2023-10-09T05:31:38.237 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.237 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.237 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.237 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:31:38.237 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.041515+0000", 2023-10-09T05:31:38.237 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:31:38.237 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.238 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.238 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:31:38.238 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.041539+0000", 2023-10-09T05:31:38.238 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.3842e-05 2023-10-09T05:31:38.238 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.238 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.238 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:31:38.238 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.041550+0000", 2023-10-09T05:31:38.238 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.1508999999999999e-05 2023-10-09T05:31:38.238 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.238 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.238 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:31:38.239 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.041558+0000", 2023-10-09T05:31:38.239 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.559e-06 2023-10-09T05:31:38.239 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.239 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.239 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:31:38.239 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.041618+0000", 2023-10-09T05:31:38.239 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.9933999999999999e-05 2023-10-09T05:31:38.239 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.239 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.239 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:31:38.239 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.041761+0000", 2023-10-09T05:31:38.240 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00014324300000000001 2023-10-09T05:31:38.240 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.240 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.240 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:31:38.240 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.041918+0000", 2023-10-09T05:31:38.240 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00015727300000000001 2023-10-09T05:31:38.240 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.240 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.240 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:31:38.240 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.273774+0000", 2023-10-09T05:31:38.240 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.231856064 2023-10-09T05:31:38.240 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.241 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.241 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:31:38.241 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.281981+0000", 2023-10-09T05:31:38.241 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0082065599999999999 2023-10-09T05:31:38.241 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.241 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.241 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:31:38.241 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.289741+0000", 2023-10-09T05:31:38.241 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0077605729999999998 2023-10-09T05:31:38.241 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.241 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.241 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:31:38.242 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.289890+0000", 2023-10-09T05:31:38.242 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000148355 2023-10-09T05:31:38.242 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.242 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.242 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:31:38.242 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.289933+0000", 2023-10-09T05:31:38.242 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.3312000000000001e-05 2023-10-09T05:31:38.242 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.242 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:31:38.242 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.242 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.242 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.243 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_repop_reply(client.45596.0:63815 13.9 e296/289 ondisk, result = 0)", 2023-10-09T05:31:38.243 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:31:27.045310+0000", 2023-10-09T05:31:38.243 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 11.144174045, 2023-10-09T05:31:38.243 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.233022747, 2023-10-09T05:31:38.243 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:31:38.243 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "started", 2023-10-09T05:31:38.243 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:31:38.243 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.243 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:31:38.243 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.045310+0000", 2023-10-09T05:31:38.243 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.244 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.244 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.244 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:31:38.244 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.045310+0000", 2023-10-09T05:31:38.244 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.244 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.244 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.244 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:31:38.244 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.045309+0000", 2023-10-09T05:31:38.244 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:31:38.244 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.245 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.245 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:31:38.245 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.045316+0000", 2023-10-09T05:31:38.245 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.8349999999999998e-06 2023-10-09T05:31:38.245 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.245 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.245 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:31:38.245 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.045334+0000", 2023-10-09T05:31:38.245 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.8130000000000001e-05 2023-10-09T05:31:38.245 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.245 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.245 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:31:38.246 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.045343+0000", 2023-10-09T05:31:38.246 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.5890000000000003e-06 2023-10-09T05:31:38.246 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.246 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.246 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:31:38.246 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.278285+0000", 2023-10-09T05:31:38.246 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.23294220099999999 2023-10-09T05:31:38.246 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.246 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.246 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:31:38.246 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.278307+0000", 2023-10-09T05:31:38.247 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.1713e-05 2023-10-09T05:31:38.247 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.247 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.247 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:31:38.247 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.278333+0000", 2023-10-09T05:31:38.247 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.6023000000000001e-05 2023-10-09T05:31:38.247 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.247 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:31:38.247 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.247 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.247 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.247 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_repop_reply(client.45596.0:63826 13.5 e296/289 ondisk, result = 0)", 2023-10-09T05:31:38.248 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:31:27.045482+0000", 2023-10-09T05:31:38.248 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 11.14400253, 2023-10-09T05:31:38.248 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.22831384399999999, 2023-10-09T05:31:38.248 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:31:38.248 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "started", 2023-10-09T05:31:38.248 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:31:38.248 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.248 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:31:38.248 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.045482+0000", 2023-10-09T05:31:38.248 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.248 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.248 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.249 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:31:38.249 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.045482+0000", 2023-10-09T05:31:38.249 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.249 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.249 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.249 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:31:38.249 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.045481+0000", 2023-10-09T05:31:38.249 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999995 2023-10-09T05:31:38.249 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.249 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.249 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:31:38.250 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.045486+0000", 2023-10-09T05:31:38.250 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.322e-06 2023-10-09T05:31:38.250 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.250 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.250 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:31:38.250 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.045496+0000", 2023-10-09T05:31:38.250 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.3659999999999998e-06 2023-10-09T05:31:38.250 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.250 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.250 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:31:38.250 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.045502+0000", 2023-10-09T05:31:38.251 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.7679999999999997e-06 2023-10-09T05:31:38.251 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.251 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.251 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:31:38.251 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.273731+0000", 2023-10-09T05:31:38.251 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.22822944100000001 2023-10-09T05:31:38.251 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.251 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.251 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:31:38.251 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.273756+0000", 2023-10-09T05:31:38.251 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.5343000000000001e-05 2023-10-09T05:31:38.251 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.252 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.252 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:31:38.252 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.273796+0000", 2023-10-09T05:31:38.252 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.9175999999999999e-05 2023-10-09T05:31:38.252 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.252 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:31:38.252 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.252 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.252 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.253 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:63831 13.5 13:afce8127:::benchmark_data_smithi133_241067_object63830:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-10-09T05:31:38.253 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:31:27.046128+0000", 2023-10-09T05:31:38.253 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 11.143355983999999, 2023-10-09T05:31:38.253 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.24491956100000001, 2023-10-09T05:31:38.253 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:31:38.253 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:31:38.253 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:31:38.253 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:31:38.253 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:31:38.254 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 63831 2023-10-09T05:31:38.254 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.254 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:31:38.254 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.254 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:31:38.254 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.046128+0000", 2023-10-09T05:31:38.254 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.254 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.255 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.255 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:31:38.255 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.046128+0000", 2023-10-09T05:31:38.255 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.255 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.255 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.255 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:31:38.255 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.046127+0000", 2023-10-09T05:31:38.255 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:31:38.256 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.256 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.256 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:31:38.256 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.046154+0000", 2023-10-09T05:31:38.256 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.6783000000000001e-05 2023-10-09T05:31:38.256 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.256 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.256 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:31:38.256 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.273681+0000", 2023-10-09T05:31:38.257 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.22752772199999999 2023-10-09T05:31:38.257 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.257 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.257 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:31:38.257 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.273696+0000", 2023-10-09T05:31:38.257 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.454e-05 2023-10-09T05:31:38.257 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.257 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.257 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:31:38.258 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.273876+0000", 2023-10-09T05:31:38.258 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000180514 2023-10-09T05:31:38.258 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.258 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.258 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:31:38.258 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.274253+0000", 2023-10-09T05:31:38.258 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00037650100000000001 2023-10-09T05:31:38.258 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.258 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.259 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:31:38.259 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.276586+0000", 2023-10-09T05:31:38.259 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.002332621 2023-10-09T05:31:38.259 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.259 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.259 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:31:38.259 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.286168+0000", 2023-10-09T05:31:38.259 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0095823060000000005 2023-10-09T05:31:38.260 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.260 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.260 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:31:38.260 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.289944+0000", 2023-10-09T05:31:38.260 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0037764819999999998 2023-10-09T05:31:38.260 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.260 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.260 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:31:38.261 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.290476+0000", 2023-10-09T05:31:38.261 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00053208800000000001 2023-10-09T05:31:38.261 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.261 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.261 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:31:38.261 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.290517+0000", 2023-10-09T05:31:38.261 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.0221e-05 2023-10-09T05:31:38.261 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.262 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.262 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:31:38.262 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.291048+0000", 2023-10-09T05:31:38.262 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00053120200000000004 2023-10-09T05:31:38.262 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.262 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:31:38.262 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.262 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.263 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.263 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_repop(client.45596.0:63824 13.2 e296/289 13:43aa95bc:::benchmark_data_smithi133_241067_object63823:head v 296'3962, mlcod=296'3959)", 2023-10-09T05:31:38.263 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:31:27.049105+0000", 2023-10-09T05:31:38.263 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 11.140379054, 2023-10-09T05:31:38.263 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.22934455400000001, 2023-10-09T05:31:38.263 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:31:38.263 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:31:38.263 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:31:38.264 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.264 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:31:38.264 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.049105+0000", 2023-10-09T05:31:38.264 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.264 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.264 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.264 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:31:38.264 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.049105+0000", 2023-10-09T05:31:38.265 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.265 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.265 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.265 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:31:38.265 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.049104+0000", 2023-10-09T05:31:38.265 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:31:38.265 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.265 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.266 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:31:38.266 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.049136+0000", 2023-10-09T05:31:38.266 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.2403999999999997e-05 2023-10-09T05:31:38.266 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.266 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.266 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:31:38.266 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.273689+0000", 2023-10-09T05:31:38.266 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.22455245600000001 2023-10-09T05:31:38.266 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.266 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.267 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:31:38.267 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.273707+0000", 2023-10-09T05:31:38.267 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.8359000000000002e-05 2023-10-09T05:31:38.267 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.267 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.267 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:31:38.267 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.273842+0000", 2023-10-09T05:31:38.267 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00013429100000000001 2023-10-09T05:31:38.267 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.267 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.267 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:31:38.268 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.273867+0000", 2023-10-09T05:31:38.268 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.5137999999999999e-05 2023-10-09T05:31:38.268 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.268 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.268 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:31:38.268 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.278358+0000", 2023-10-09T05:31:38.268 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00449118 2023-10-09T05:31:38.268 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.268 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.268 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:31:38.269 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.278450+0000", 2023-10-09T05:31:38.269 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.1842999999999995e-05 2023-10-09T05:31:38.269 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.269 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:31:38.269 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.269 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.269 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.269 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_repop_reply(client.45596.0:63818 13.9 e296/289 ondisk, result = 0)", 2023-10-09T05:31:38.269 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:31:27.052168+0000", 2023-10-09T05:31:38.269 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 11.137316589999999, 2023-10-09T05:31:38.270 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.22626228400000001, 2023-10-09T05:31:38.270 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:31:38.270 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "started", 2023-10-09T05:31:38.270 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:31:38.270 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.270 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:31:38.270 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.052168+0000", 2023-10-09T05:31:38.270 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.270 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.270 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.270 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:31:38.270 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.052168+0000", 2023-10-09T05:31:38.271 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.271 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.271 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.271 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:31:38.271 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.052167+0000", 2023-10-09T05:31:38.271 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:31:38.271 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.271 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.271 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:31:38.271 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.052175+0000", 2023-10-09T05:31:38.271 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.6540000000000005e-06 2023-10-09T05:31:38.272 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.272 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.272 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:31:38.272 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.273652+0000", 2023-10-09T05:31:38.272 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.221476959 2023-10-09T05:31:38.272 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.272 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.272 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:31:38.272 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.273682+0000", 2023-10-09T05:31:38.272 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.0043999999999999e-05 2023-10-09T05:31:38.272 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.272 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.273 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:31:38.273 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.278391+0000", 2023-10-09T05:31:38.273 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0047084689999999998 2023-10-09T05:31:38.273 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.273 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.273 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:31:38.273 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.278408+0000", 2023-10-09T05:31:38.273 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.7598999999999999e-05 2023-10-09T05:31:38.273 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.273 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.273 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:31:38.273 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.278430+0000", 2023-10-09T05:31:38.274 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.1690000000000001e-05 2023-10-09T05:31:38.274 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.274 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:31:38.274 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.274 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.274 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.274 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:65059 13.9 13:987ec66c:::benchmark_data_smithi133_241067_object65058:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-10-09T05:31:38.274 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:31:28.551254+0000", 2023-10-09T05:31:38.274 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 9.6382303169999997, 2023-10-09T05:31:38.274 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.26131015600000002, 2023-10-09T05:31:38.274 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:31:38.275 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:31:38.275 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:31:38.275 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:31:38.275 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:31:38.275 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 65059 2023-10-09T05:31:38.275 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.275 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:31:38.275 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.275 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:31:38.275 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.551254+0000", 2023-10-09T05:31:38.275 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.275 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.276 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.276 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:31:38.276 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.551254+0000", 2023-10-09T05:31:38.276 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.276 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.276 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.276 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:31:38.276 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.551253+0000", 2023-10-09T05:31:38.276 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:31:38.276 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.276 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.277 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:31:38.277 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.551304+0000", 2023-10-09T05:31:38.277 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.0733999999999999e-05 2023-10-09T05:31:38.277 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.277 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.277 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:31:38.277 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.551313+0000", 2023-10-09T05:31:38.277 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.9090000000000006e-06 2023-10-09T05:31:38.277 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.277 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.277 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:31:38.277 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.551320+0000", 2023-10-09T05:31:38.278 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.7850000000000003e-06 2023-10-09T05:31:38.278 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.278 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.278 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:31:38.278 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.554614+0000", 2023-10-09T05:31:38.278 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0032940859999999999 2023-10-09T05:31:38.278 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.278 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.278 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:31:38.278 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.554752+0000", 2023-10-09T05:31:38.278 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00013819400000000001 2023-10-09T05:31:38.279 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.279 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.279 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:31:38.279 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.554897+0000", 2023-10-09T05:31:38.279 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00014488500000000001 2023-10-09T05:31:38.279 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.279 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.279 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:31:38.279 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.565402+0000", 2023-10-09T05:31:38.279 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.010505216 2023-10-09T05:31:38.279 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.279 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.280 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:31:38.280 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.812439+0000", 2023-10-09T05:31:38.280 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.247037224 2023-10-09T05:31:38.280 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.280 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.280 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:31:38.280 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.812472+0000", 2023-10-09T05:31:38.280 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.2778999999999999e-05 2023-10-09T05:31:38.280 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.280 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.280 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:31:38.280 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.812523+0000", 2023-10-09T05:31:38.281 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.0522000000000002e-05 2023-10-09T05:31:38.281 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.281 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.281 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:31:38.281 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.812564+0000", 2023-10-09T05:31:38.281 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.1557000000000002e-05 2023-10-09T05:31:38.281 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.281 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:31:38.281 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.281 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.281 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.281 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:65065 13.9 13:981183f9:::benchmark_data_smithi133_241067_object65064:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-10-09T05:31:38.282 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:31:28.556065+0000", 2023-10-09T05:31:38.282 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 9.6334194340000003, 2023-10-09T05:31:38.282 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.25963333700000002, 2023-10-09T05:31:38.282 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:31:38.282 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:31:38.282 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:31:38.282 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:31:38.282 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:31:38.282 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 65065 2023-10-09T05:31:38.282 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.282 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:31:38.283 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.283 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:31:38.283 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.556065+0000", 2023-10-09T05:31:38.283 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.283 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.283 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.283 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:31:38.283 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.556065+0000", 2023-10-09T05:31:38.283 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.283 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.283 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.283 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:31:38.284 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.556064+0000", 2023-10-09T05:31:38.284 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:31:38.284 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.284 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.284 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:31:38.284 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.556109+0000", 2023-10-09T05:31:38.284 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.5240000000000001e-05 2023-10-09T05:31:38.284 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.284 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.284 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:31:38.284 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.556118+0000", 2023-10-09T05:31:38.285 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.2590000000000006e-06 2023-10-09T05:31:38.285 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.285 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.285 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:31:38.285 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.556124+0000", 2023-10-09T05:31:38.285 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.8440000000000003e-06 2023-10-09T05:31:38.285 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.285 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.285 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:31:38.286 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.565513+0000", 2023-10-09T05:31:38.286 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0093892290000000007 2023-10-09T05:31:38.286 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.286 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.286 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:31:38.286 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.565618+0000", 2023-10-09T05:31:38.286 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000104263 2023-10-09T05:31:38.286 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.287 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.287 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:31:38.287 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.565708+0000", 2023-10-09T05:31:38.287 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.0091000000000002e-05 2023-10-09T05:31:38.287 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.287 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.287 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:31:38.287 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.812603+0000", 2023-10-09T05:31:38.287 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.24689562900000001 2023-10-09T05:31:38.287 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.288 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.288 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:31:38.288 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.812637+0000", 2023-10-09T05:31:38.288 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.3318999999999997e-05 2023-10-09T05:31:38.288 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.288 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.288 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:31:38.288 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.815581+0000", 2023-10-09T05:31:38.289 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0029449070000000001 2023-10-09T05:31:38.289 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.289 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.289 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:31:38.289 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.815656+0000", 2023-10-09T05:31:38.289 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.5026999999999998e-05 2023-10-09T05:31:38.289 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.289 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.289 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:31:38.290 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.815698+0000", 2023-10-09T05:31:38.290 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.1649000000000001e-05 2023-10-09T05:31:38.290 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.290 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:31:38.290 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.290 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.290 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.290 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:65069 13.9 13:9cceab01:::benchmark_data_smithi133_241067_object65068:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-10-09T05:31:38.291 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:31:28.561659+0000", 2023-10-09T05:31:38.291 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 9.6278247439999998, 2023-10-09T05:31:38.291 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.26434785300000002, 2023-10-09T05:31:38.291 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:31:38.291 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:31:38.291 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:31:38.291 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:31:38.291 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:31:38.291 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 65069 2023-10-09T05:31:38.292 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.292 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:31:38.292 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.292 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:31:38.292 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.561659+0000", 2023-10-09T05:31:38.292 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.292 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.292 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.292 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:31:38.293 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.561659+0000", 2023-10-09T05:31:38.293 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.293 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.293 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.293 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:31:38.293 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.561658+0000", 2023-10-09T05:31:38.293 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:31:38.293 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.293 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.294 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:31:38.294 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.561722+0000", 2023-10-09T05:31:38.294 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.3973000000000006e-05 2023-10-09T05:31:38.294 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.294 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.294 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:31:38.294 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.561731+0000", 2023-10-09T05:31:38.294 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.1130000000000007e-06 2023-10-09T05:31:38.294 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.295 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.295 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:31:38.295 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.561737+0000", 2023-10-09T05:31:38.295 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.6470000000000003e-06 2023-10-09T05:31:38.295 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.295 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.295 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:31:38.295 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.568198+0000", 2023-10-09T05:31:38.296 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0064612159999999997 2023-10-09T05:31:38.296 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.296 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.296 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:31:38.296 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.568312+0000", 2023-10-09T05:31:38.296 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00011403 2023-10-09T05:31:38.296 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.296 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.296 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:31:38.296 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.809807+0000", 2023-10-09T05:31:38.297 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.24149514899999999 2023-10-09T05:31:38.297 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.297 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.297 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:31:38.297 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.815770+0000", 2023-10-09T05:31:38.297 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0059632050000000001 2023-10-09T05:31:38.297 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.297 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.297 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:31:38.297 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.818795+0000", 2023-10-09T05:31:38.297 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0030246169999999998 2023-10-09T05:31:38.298 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.298 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.298 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:31:38.298 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.825916+0000", 2023-10-09T05:31:38.298 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0071206840000000004 2023-10-09T05:31:38.298 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.298 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.298 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:31:38.298 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.825969+0000", 2023-10-09T05:31:38.298 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.3776000000000001e-05 2023-10-09T05:31:38.298 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.299 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.299 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:31:38.299 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.826007+0000", 2023-10-09T05:31:38.299 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.7892999999999998e-05 2023-10-09T05:31:38.299 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.299 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:31:38.299 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.299 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.299 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.300 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_repop_reply(client.45596.0:65059 13.9 e296/289 ondisk, result = 0)", 2023-10-09T05:31:38.300 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:31:28.566073+0000", 2023-10-09T05:31:38.300 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 9.6234111529999993, 2023-10-09T05:31:38.300 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.246369272, 2023-10-09T05:31:38.300 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:31:38.300 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "started", 2023-10-09T05:31:38.300 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:31:38.301 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.301 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:31:38.301 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.566073+0000", 2023-10-09T05:31:38.301 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.301 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.301 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.301 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:31:38.301 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.566073+0000", 2023-10-09T05:31:38.302 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.302 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.302 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.302 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:31:38.302 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.566072+0000", 2023-10-09T05:31:38.302 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:31:38.302 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.302 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.302 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:31:38.302 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.566079+0000", 2023-10-09T05:31:38.302 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.3670000000000002e-06 2023-10-09T05:31:38.302 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.303 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.303 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:31:38.303 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.566091+0000", 2023-10-09T05:31:38.303 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.2609000000000001e-05 2023-10-09T05:31:38.303 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.303 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.303 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:31:38.303 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.566099+0000", 2023-10-09T05:31:38.303 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.191e-06 2023-10-09T05:31:38.303 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.303 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.304 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:31:38.304 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.812425+0000", 2023-10-09T05:31:38.304 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.24632506300000001 2023-10-09T05:31:38.304 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.304 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.304 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:31:38.304 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.812432+0000", 2023-10-09T05:31:38.304 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.8820000000000001e-06 2023-10-09T05:31:38.304 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.304 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.305 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:31:38.305 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.812442+0000", 2023-10-09T05:31:38.305 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.9380000000000004e-06 2023-10-09T05:31:38.305 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.305 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:31:38.305 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.305 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.305 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.305 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:65073 13.9 13:96eb0669:::benchmark_data_smithi133_241067_object65072:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-10-09T05:31:38.305 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:31:28.566427+0000", 2023-10-09T05:31:38.305 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 9.6230572760000008, 2023-10-09T05:31:38.305 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.26087090499999999, 2023-10-09T05:31:38.306 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:31:38.306 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:31:38.306 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:31:38.306 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:31:38.306 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:31:38.306 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 65073 2023-10-09T05:31:38.306 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.306 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:31:38.306 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.306 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:31:38.306 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.566427+0000", 2023-10-09T05:31:38.307 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.307 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.307 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.307 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:31:38.307 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.566427+0000", 2023-10-09T05:31:38.307 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.307 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.307 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.307 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:31:38.307 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.566425+0000", 2023-10-09T05:31:38.307 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:31:38.307 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.308 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.308 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:31:38.308 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.566476+0000", 2023-10-09T05:31:38.308 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.0914000000000001e-05 2023-10-09T05:31:38.308 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.308 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.308 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:31:38.308 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.566489+0000", 2023-10-09T05:31:38.308 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.2849e-05 2023-10-09T05:31:38.308 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.308 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.308 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:31:38.309 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.566498+0000", 2023-10-09T05:31:38.309 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.9080000000000001e-06 2023-10-09T05:31:38.309 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.309 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.309 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:31:38.309 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.812698+0000", 2023-10-09T05:31:38.309 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.246199524 2023-10-09T05:31:38.309 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.309 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.309 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:31:38.309 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.812933+0000", 2023-10-09T05:31:38.309 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00023507700000000001 2023-10-09T05:31:38.310 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.310 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.310 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:31:38.310 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.813090+0000", 2023-10-09T05:31:38.310 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000157653 2023-10-09T05:31:38.310 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.310 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.310 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:31:38.310 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.815835+0000", 2023-10-09T05:31:38.310 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0027450489999999998 2023-10-09T05:31:38.310 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.310 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.311 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:31:38.311 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.818988+0000", 2023-10-09T05:31:38.311 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0031525749999999999 2023-10-09T05:31:38.311 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.311 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.311 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:31:38.311 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.826864+0000", 2023-10-09T05:31:38.311 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0078758089999999992 2023-10-09T05:31:38.311 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.311 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.311 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:31:38.312 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.827053+0000", 2023-10-09T05:31:38.312 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00018942699999999999 2023-10-09T05:31:38.312 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.312 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.312 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:31:38.312 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.827298+0000", 2023-10-09T05:31:38.312 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00024460399999999999 2023-10-09T05:31:38.312 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.312 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:31:38.312 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.312 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.313 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.313 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_repop(client.45596.0:65074 13.b e296/289 13:d738b009:::benchmark_data_smithi133_241067_object65073:head v 296'4035, mlcod=296'4033)", 2023-10-09T05:31:38.313 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:31:28.569018+0000", 2023-10-09T05:31:38.313 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 9.6204664399999995, 2023-10-09T05:31:38.313 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.25409176500000002, 2023-10-09T05:31:38.313 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:31:38.313 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:31:38.313 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:31:38.313 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.313 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:31:38.314 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.569018+0000", 2023-10-09T05:31:38.314 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.314 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.314 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.314 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:31:38.314 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.569018+0000", 2023-10-09T05:31:38.314 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.314 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.314 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.314 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:31:38.314 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.569017+0000", 2023-10-09T05:31:38.315 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:31:38.315 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.315 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.315 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:31:38.315 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.569047+0000", 2023-10-09T05:31:38.315 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.0130000000000001e-05 2023-10-09T05:31:38.315 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.315 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.315 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:31:38.315 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.809516+0000", 2023-10-09T05:31:38.315 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.240468455 2023-10-09T05:31:38.315 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.316 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.316 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:31:38.316 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.809552+0000", 2023-10-09T05:31:38.316 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.6053000000000002e-05 2023-10-09T05:31:38.316 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.316 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.316 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:31:38.316 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.809656+0000", 2023-10-09T05:31:38.316 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00010410499999999999 2023-10-09T05:31:38.316 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.317 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.317 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:31:38.317 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.809721+0000", 2023-10-09T05:31:38.317 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.5746000000000004e-05 2023-10-09T05:31:38.317 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.317 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.317 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:31:38.317 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.822926+0000", 2023-10-09T05:31:38.317 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.013204293000000001 2023-10-09T05:31:38.317 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.317 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.318 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:31:38.318 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.823110+0000", 2023-10-09T05:31:38.318 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00018383900000000001 2023-10-09T05:31:38.318 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.318 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:31:38.318 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.318 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.318 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.318 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_repop_reply(client.45596.0:65065 13.9 e296/289 ondisk, result = 0)", 2023-10-09T05:31:38.318 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:31:28.569709+0000", 2023-10-09T05:31:38.318 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 9.6197750660000008, 2023-10-09T05:31:38.319 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.242896258, 2023-10-09T05:31:38.319 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:31:38.319 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "started", 2023-10-09T05:31:38.319 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:31:38.319 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.319 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:31:38.319 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.569709+0000", 2023-10-09T05:31:38.319 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.319 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.319 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.319 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:31:38.319 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.569709+0000", 2023-10-09T05:31:38.320 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.320 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.320 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.320 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:31:38.320 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.569708+0000", 2023-10-09T05:31:38.320 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999995 2023-10-09T05:31:38.320 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.320 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.320 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:31:38.320 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.569715+0000", 2023-10-09T05:31:38.320 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.4099999999999996e-06 2023-10-09T05:31:38.321 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.321 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.321 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:31:38.321 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.809582+0000", 2023-10-09T05:31:38.321 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.239867512 2023-10-09T05:31:38.321 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.321 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.321 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:31:38.321 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.809613+0000", 2023-10-09T05:31:38.321 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.0542e-05 2023-10-09T05:31:38.321 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.321 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.322 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:31:38.322 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.812590+0000", 2023-10-09T05:31:38.322 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.002976729 2023-10-09T05:31:38.322 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.322 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.322 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:31:38.322 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.812596+0000", 2023-10-09T05:31:38.322 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.0499999999999997e-06 2023-10-09T05:31:38.323 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.323 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.323 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:31:38.323 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.812605+0000", 2023-10-09T05:31:38.323 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.7259999999999997e-06 2023-10-09T05:31:38.323 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.323 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:31:38.323 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.323 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.324 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.324 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_repop_reply(client.45596.0:65059 13.9 e296/289 ondisk, result = 0)", 2023-10-09T05:31:38.324 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:31:28.573094+0000", 2023-10-09T05:31:38.324 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 9.6163901989999996, 2023-10-09T05:31:38.324 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.239475678, 2023-10-09T05:31:38.324 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:31:38.324 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "started", 2023-10-09T05:31:38.324 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:31:38.324 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.324 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:31:38.325 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.573094+0000", 2023-10-09T05:31:38.325 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.325 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.325 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.325 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:31:38.325 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.573094+0000", 2023-10-09T05:31:38.325 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.325 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.326 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.326 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:31:38.326 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.573093+0000", 2023-10-09T05:31:38.326 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:31:38.326 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.326 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.326 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:31:38.326 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.573101+0000", 2023-10-09T05:31:38.327 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.2539999999999992e-06 2023-10-09T05:31:38.327 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.327 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.327 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:31:38.327 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.809526+0000", 2023-10-09T05:31:38.327 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.236425151 2023-10-09T05:31:38.327 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.327 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.328 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:31:38.328 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.809554+0000", 2023-10-09T05:31:38.328 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.7373e-05 2023-10-09T05:31:38.328 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.328 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.328 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:31:38.328 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.812462+0000", 2023-10-09T05:31:38.328 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0029082589999999998 2023-10-09T05:31:38.328 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.329 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.329 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:31:38.329 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.812467+0000", 2023-10-09T05:31:38.329 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.519e-06 2023-10-09T05:31:38.329 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.329 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.329 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:31:38.329 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.812570+0000", 2023-10-09T05:31:38.329 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000102256 2023-10-09T05:31:38.330 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.330 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:31:38.330 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.330 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.330 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.330 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:68312 13.3 13:c645802f:::benchmark_data_smithi133_241067_object68311:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-10-09T05:31:38.330 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:31:32.333542+0000", 2023-10-09T05:31:38.330 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 5.855942389, 2023-10-09T05:31:38.331 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.20106586800000001, 2023-10-09T05:31:38.331 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:31:38.331 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:31:38.331 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:31:38.331 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:31:38.331 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:31:38.331 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 68312 2023-10-09T05:31:38.331 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.331 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:31:38.331 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.332 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:31:38.332 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:32.333542+0000", 2023-10-09T05:31:38.332 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.332 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.332 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.332 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:31:38.332 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:32.333542+0000", 2023-10-09T05:31:38.332 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:31:38.332 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.333 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.333 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:31:38.333 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:32.333541+0000", 2023-10-09T05:31:38.333 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999995 2023-10-09T05:31:38.333 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.333 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.333 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:31:38.334 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:32.333563+0000", 2023-10-09T05:31:38.334 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.1090999999999998e-05 2023-10-09T05:31:38.334 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.334 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.334 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:31:38.334 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:32.333569+0000", 2023-10-09T05:31:38.334 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.781e-06 2023-10-09T05:31:38.334 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.334 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.335 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:31:38.335 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:32.333574+0000", 2023-10-09T05:31:38.335 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.028e-06 2023-10-09T05:31:38.335 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.335 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.335 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:31:38.335 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:32.527289+0000", 2023-10-09T05:31:38.335 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.19371456000000001 2023-10-09T05:31:38.335 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.336 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.336 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:31:38.336 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:32.527440+0000", 2023-10-09T05:31:38.336 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00015095700000000001 2023-10-09T05:31:38.336 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.336 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.336 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:31:38.336 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:32.527637+0000", 2023-10-09T05:31:38.336 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00019763199999999999 2023-10-09T05:31:38.336 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.336 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.337 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:31:38.337 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:32.530800+0000", 2023-10-09T05:31:38.337 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0031621319999999998 2023-10-09T05:31:38.337 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.337 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.337 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:31:38.337 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:32.532610+0000", 2023-10-09T05:31:38.337 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0018104519999999999 2023-10-09T05:31:38.337 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.337 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.337 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:31:38.337 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:32.534402+0000", 2023-10-09T05:31:38.338 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0017915190000000001 2023-10-09T05:31:38.338 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.338 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.338 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:31:38.338 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:32.534570+0000", 2023-10-09T05:31:38.338 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00016865700000000001 2023-10-09T05:31:38.338 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:31:38.338 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:31:38.338 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:31:38.338 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:32.534608+0000", 2023-10-09T05:31:38.338 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.7460999999999999e-05 2023-10-09T05:31:38.338 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.339 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:31:38.339 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.339 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:31:38.339 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:31:38.339 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:31:38.339 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:38.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:38.373 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:38.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:38.474 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:38.505 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:38.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:38.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:38.522 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 59 16 73703 73687 78.0489 48.4375 0.00810781 0.0128066 2023-10-09T05:31:38.615 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:38.640 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:38.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:38.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:38.743 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:38.772 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:38.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:38.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:38.880 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:38.913 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:38.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:38.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:39.018 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:39.045 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:39.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:39.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:39.149 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:39.176 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:39.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:39.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:39.280 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:39.326 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:39.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:39.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:39.369 INFO:teuthology.orchestra.run.smithi133.stderr:set pool 13 pgp_num to 16 2023-10-09T05:31:39.432 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:39.440 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:39 smithi194 bash[92666]: cluster 2023-10-09T05:31:38.288752+0000 mgr.y (mgr.25277) 1306 : cluster 0 pgmap v1399: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 46 MiB/s wr, 741 op/s 2023-10-09T05:31:39.441 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:39 smithi194 bash[92666]: audit 2023-10-09T05:31:38.727398+0000 mon.c (mon.2) 124 : audit 1 from='client.? 172.21.15.133:0/1949833559' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:31:39.441 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:39 smithi194 bash[92666]: audit 2023-10-09T05:31:38.728662+0000 mon.a (mon.0) 1650 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:31:39.454 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:39.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:39.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:39.523 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T05:31:39.523034+0000 min lat: 0.00142329 max lat: 0.347117 avg lat: 0.0128611 2023-10-09T05:31:39.523 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:31:39.523 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 60 16 74636 74620 77.7198 58.3125 0.0260778 0.0128611 2023-10-09T05:31:39.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:39 smithi133 bash[102858]: cluster 2023-10-09T05:31:38.288752+0000 mgr.y (mgr.25277) 1306 : cluster 0 pgmap v1399: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 46 MiB/s wr, 741 op/s 2023-10-09T05:31:39.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:39 smithi133 bash[102858]: audit 2023-10-09T05:31:38.727398+0000 mon.c (mon.2) 124 : audit 1 from='client.? 172.21.15.133:0/1949833559' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:31:39.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:39 smithi133 bash[102858]: audit 2023-10-09T05:31:38.728662+0000 mon.a (mon.0) 1650 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:31:39.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:39 smithi133 bash[105589]: cluster 2023-10-09T05:31:38.288752+0000 mgr.y (mgr.25277) 1306 : cluster 0 pgmap v1399: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 46 MiB/s wr, 741 op/s 2023-10-09T05:31:39.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:39 smithi133 bash[105589]: audit 2023-10-09T05:31:38.727398+0000 mon.c (mon.2) 124 : audit 1 from='client.? 172.21.15.133:0/1949833559' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:31:39.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:39 smithi133 bash[105589]: audit 2023-10-09T05:31:38.728662+0000 mon.a (mon.0) 1650 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:31:39.557 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:39.583 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:39.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:39.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:39.687 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:39.715 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:39.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:39.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:39.821 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:39.845 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:39.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:39.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:39.950 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:39.981 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:39.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:39.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:40.087 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:40.115 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:40.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:40.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:40.220 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:40.248 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:40.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:40.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:40.354 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:40.379 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:40.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:40.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:40.482 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:40.514 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:40.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:40.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:40.522 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 61 16 75443 75427 77.2724 50.4375 0.00419632 0.0129046 2023-10-09T05:31:40.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:40 smithi133 bash[102858]: cluster 2023-10-09T05:31:39.348620+0000 osd.5 (osd.5) 272 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T05:31:40.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:40 smithi133 bash[102858]: audit 2023-10-09T05:31:39.350258+0000 mon.a (mon.0) 1651 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T05:31:40.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:40 smithi133 bash[102858]: cluster 2023-10-09T05:31:39.350406+0000 mon.a (mon.0) 1652 : cluster 0 osdmap e297: 8 total, 8 up, 6 in 2023-10-09T05:31:40.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:40 smithi133 bash[102858]: cluster 2023-10-09T05:31:39.351045+0000 osd.6 (osd.6) 256 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T05:31:40.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:40 smithi133 bash[102858]: cluster 2023-10-09T05:31:39.356467+0000 osd.7 (osd.7) 253 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T05:31:40.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:40 smithi133 bash[105589]: cluster 2023-10-09T05:31:39.348620+0000 osd.5 (osd.5) 272 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T05:31:40.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:40 smithi133 bash[105589]: audit 2023-10-09T05:31:39.350258+0000 mon.a (mon.0) 1651 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T05:31:40.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:40 smithi133 bash[105589]: cluster 2023-10-09T05:31:39.350406+0000 mon.a (mon.0) 1652 : cluster 0 osdmap e297: 8 total, 8 up, 6 in 2023-10-09T05:31:40.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:40 smithi133 bash[105589]: cluster 2023-10-09T05:31:39.351045+0000 osd.6 (osd.6) 256 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T05:31:40.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:40 smithi133 bash[105589]: cluster 2023-10-09T05:31:39.356467+0000 osd.7 (osd.7) 253 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T05:31:40.618 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:40.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:40 smithi194 bash[92666]: cluster 2023-10-09T05:31:39.348620+0000 osd.5 (osd.5) 272 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T05:31:40.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:40 smithi194 bash[92666]: audit 2023-10-09T05:31:39.350258+0000 mon.a (mon.0) 1651 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T05:31:40.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:40 smithi194 bash[92666]: cluster 2023-10-09T05:31:39.350406+0000 mon.a (mon.0) 1652 : cluster 0 osdmap e297: 8 total, 8 up, 6 in 2023-10-09T05:31:40.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:40 smithi194 bash[92666]: cluster 2023-10-09T05:31:39.351045+0000 osd.6 (osd.6) 256 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T05:31:40.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:40 smithi194 bash[92666]: cluster 2023-10-09T05:31:39.356467+0000 osd.7 (osd.7) 253 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T05:31:40.642 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:40.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:40.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:40.746 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:40.775 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:40.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:40.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:40.881 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:40.909 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:40.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:40.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:41.013 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:41.042 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:41.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:41.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:41.147 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:41.179 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:41.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:41.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:41.288 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:41.309 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:41.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:41.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:41.414 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:41.438 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:41.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:41.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:41.523 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 62 16 76226 76210 76.8153 48.9375 0.0182969 0.0130102 2023-10-09T05:31:41.544 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:41.547 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:41 smithi133 bash[105589]: cluster 2023-10-09T05:31:40.289303+0000 mgr.y (mgr.25277) 1307 : cluster 0 pgmap v1401: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 55 MiB/s wr, 886 op/s 2023-10-09T05:31:41.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:41 smithi133 bash[105589]: audit 2023-10-09T05:31:41.074913+0000 mon.a (mon.0) 1653 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:31:41.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:41 smithi133 bash[102858]: cluster 2023-10-09T05:31:40.289303+0000 mgr.y (mgr.25277) 1307 : cluster 0 pgmap v1401: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 55 MiB/s wr, 886 op/s 2023-10-09T05:31:41.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:41 smithi133 bash[102858]: audit 2023-10-09T05:31:41.074913+0000 mon.a (mon.0) 1653 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:31:41.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:41.582 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:41.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:41.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:41 smithi194 bash[92666]: cluster 2023-10-09T05:31:40.289303+0000 mgr.y (mgr.25277) 1307 : cluster 0 pgmap v1401: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 55 MiB/s wr, 886 op/s 2023-10-09T05:31:41.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:41 smithi194 bash[92666]: audit 2023-10-09T05:31:41.074913+0000 mon.a (mon.0) 1653 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:31:41.682 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:41.711 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:41.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:41.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:41.815 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:41.840 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:41.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:41.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:41.855 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T05:31:41.944 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:41.971 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:41.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:41.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:42.075 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:42.096 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:42.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:42.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:42.202 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:42.231 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:42.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:42.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:42.339 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:42.362 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:42.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:42.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:42.467 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:42.498 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:42.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:42.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:42.523 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 63 16 77063 77047 76.4263 52.3125 0.0378414 0.0130784 2023-10-09T05:31:42.604 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:42.646 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:42.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:42.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:42.750 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:42.779 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:42.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:42.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:42.883 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:42.908 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:42.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:42.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:43.011 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:43.039 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:43.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:43.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:43.145 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:43.176 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:43.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:43.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:43.281 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:43.317 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:43.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:43.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:43.320 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:31:43.364 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:31:43.371 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:43 smithi133 bash[102858]: cluster 2023-10-09T05:31:42.290788+0000 mgr.y (mgr.25277) 1308 : cluster 0 pgmap v1402: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 55 MiB/s wr, 878 op/s 2023-10-09T05:31:43.371 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:43 smithi133 bash[102858]: audit 2023-10-09T05:31:43.269143+0000 mon.a (mon.0) 1654 : audit 0 from='client.? 172.21.15.133:0/604069877' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:31:43.413 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:31:43.421 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:43.430 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:43 smithi133 bash[105589]: cluster 2023-10-09T05:31:42.290788+0000 mgr.y (mgr.25277) 1308 : cluster 0 pgmap v1402: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 55 MiB/s wr, 878 op/s 2023-10-09T05:31:43.430 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:43 smithi133 bash[105589]: audit 2023-10-09T05:31:43.269143+0000 mon.a (mon.0) 1654 : audit 0 from='client.? 172.21.15.133:0/604069877' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:31:43.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:43.459 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:43.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:43.462 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:31:43.522 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:31:43.523 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 64 16 77868 77852 76.0183 50.3125 0.00727153 0.0131482 2023-10-09T05:31:43.560 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:43.561 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:31:43.586 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:43.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:43.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:43.599 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:31:43.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:43 smithi194 bash[92666]: cluster 2023-10-09T05:31:42.290788+0000 mgr.y (mgr.25277) 1308 : cluster 0 pgmap v1402: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 55 MiB/s wr, 878 op/s 2023-10-09T05:31:43.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:43 smithi194 bash[92666]: audit 2023-10-09T05:31:43.269143+0000 mon.a (mon.0) 1654 : audit 0 from='client.? 172.21.15.133:0/604069877' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:31:43.639 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:31:43.645 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:31:43.645 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:31:43.645 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:31:43.646 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:31:43.646 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:31:43.646 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:31:43.646 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:31:43.646 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:31:43.690 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:43.720 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:43.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:43.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:43.825 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:43.852 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:43.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:43.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:43.956 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:43.982 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:43.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:43.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:44.089 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:44.114 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:44.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:44.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:44.217 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:44.244 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:44.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:44.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:44.349 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:44.373 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:44.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:44.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:44.386 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:31:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:31:44] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.43.0" 2023-10-09T05:31:44.386 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is unset 2023-10-09T05:31:44.477 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:44.485 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:44 smithi194 bash[92666]: audit 2023-10-09T05:31:43.483040+0000 mon.a (mon.0) 1655 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:31:44.485 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:44 smithi194 bash[92666]: audit 2023-10-09T05:31:43.727591+0000 mon.b (mon.1) 29 : audit 1 from='client.? 172.21.15.133:0/1136814575' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:31:44.485 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:44 smithi194 bash[92666]: audit 2023-10-09T05:31:43.727892+0000 mon.a (mon.0) 1656 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:31:44.511 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:44.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:44.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:44.523 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 65 16 78728 78712 75.6756 53.75 0.019291 0.013208 2023-10-09T05:31:44.614 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:44.640 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:44.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:44.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:44.647 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:44 smithi133 bash[102858]: audit 2023-10-09T05:31:43.483040+0000 mon.a (mon.0) 1655 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:31:44.647 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:44 smithi133 bash[102858]: audit 2023-10-09T05:31:43.727591+0000 mon.b (mon.1) 29 : audit 1 from='client.? 172.21.15.133:0/1136814575' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:31:44.647 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:44 smithi133 bash[102858]: audit 2023-10-09T05:31:43.727892+0000 mon.a (mon.0) 1656 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:31:44.647 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:44 smithi133 bash[105589]: audit 2023-10-09T05:31:43.483040+0000 mon.a (mon.0) 1655 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:31:44.647 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:44 smithi133 bash[105589]: audit 2023-10-09T05:31:43.727591+0000 mon.b (mon.1) 29 : audit 1 from='client.? 172.21.15.133:0/1136814575' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:31:44.647 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:44 smithi133 bash[105589]: audit 2023-10-09T05:31:43.727892+0000 mon.a (mon.0) 1656 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:31:44.745 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:44.768 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:44.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:44.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:44.871 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:44.897 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:44.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:44.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:45.000 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:45.027 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:45.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:45.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:45.132 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:45.159 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:45.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:45.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:45.264 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:45.295 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:45.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:45.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:45.400 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:45.426 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:45.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:45.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:45.523 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 66 16 79569 79553 75.3253 52.5625 0.0347384 0.0132701 2023-10-09T05:31:45.529 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:45.540 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:45 smithi133 bash[102858]: cluster 2023-10-09T05:31:44.291146+0000 mgr.y (mgr.25277) 1309 : cluster 0 pgmap v1403: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 55 MiB/s wr, 878 op/s 2023-10-09T05:31:45.540 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:45 smithi133 bash[102858]: cluster 2023-10-09T05:31:44.370430+0000 mon.a (mon.0) 1657 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:31:45.540 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:45 smithi133 bash[102858]: cluster 2023-10-09T05:31:44.375738+0000 osd.6 (osd.6) 257 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T05:31:45.540 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:45 smithi133 bash[102858]: audit 2023-10-09T05:31:44.378766+0000 mon.a (mon.0) 1658 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:31:45.540 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:45 smithi133 bash[102858]: cluster 2023-10-09T05:31:44.378853+0000 mon.a (mon.0) 1659 : cluster 0 osdmap e298: 8 total, 8 up, 6 in 2023-10-09T05:31:45.540 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:45 smithi133 bash[102858]: cluster 2023-10-09T05:31:44.380295+0000 osd.5 (osd.5) 273 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T05:31:45.540 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:45 smithi133 bash[102858]: cluster 2023-10-09T05:31:44.385380+0000 osd.7 (osd.7) 254 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T05:31:45.540 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:45 smithi133 bash[102858]: cluster 2023-10-09T05:31:44.394265+0000 osd.7 (osd.7) 255 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T05:31:45.541 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:45 smithi133 bash[105589]: cluster 2023-10-09T05:31:44.291146+0000 mgr.y (mgr.25277) 1309 : cluster 0 pgmap v1403: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 55 MiB/s wr, 878 op/s 2023-10-09T05:31:45.541 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:45 smithi133 bash[105589]: cluster 2023-10-09T05:31:44.370430+0000 mon.a (mon.0) 1657 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:31:45.541 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:45 smithi133 bash[105589]: cluster 2023-10-09T05:31:44.375738+0000 osd.6 (osd.6) 257 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T05:31:45.541 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:45 smithi133 bash[105589]: audit 2023-10-09T05:31:44.378766+0000 mon.a (mon.0) 1658 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:31:45.541 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:45 smithi133 bash[105589]: cluster 2023-10-09T05:31:44.378853+0000 mon.a (mon.0) 1659 : cluster 0 osdmap e298: 8 total, 8 up, 6 in 2023-10-09T05:31:45.541 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:45 smithi133 bash[105589]: cluster 2023-10-09T05:31:44.380295+0000 osd.5 (osd.5) 273 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T05:31:45.542 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:45 smithi133 bash[105589]: cluster 2023-10-09T05:31:44.385380+0000 osd.7 (osd.7) 254 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T05:31:45.542 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:45 smithi133 bash[105589]: cluster 2023-10-09T05:31:44.394265+0000 osd.7 (osd.7) 255 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T05:31:45.554 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:45.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:45.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:45.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:45 smithi194 bash[92666]: cluster 2023-10-09T05:31:44.291146+0000 mgr.y (mgr.25277) 1309 : cluster 0 pgmap v1403: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 55 MiB/s wr, 878 op/s 2023-10-09T05:31:45.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:45 smithi194 bash[92666]: cluster 2023-10-09T05:31:44.370430+0000 mon.a (mon.0) 1657 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:31:45.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:45 smithi194 bash[92666]: cluster 2023-10-09T05:31:44.375738+0000 osd.6 (osd.6) 257 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T05:31:45.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:45 smithi194 bash[92666]: audit 2023-10-09T05:31:44.378766+0000 mon.a (mon.0) 1658 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:31:45.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:45 smithi194 bash[92666]: cluster 2023-10-09T05:31:44.378853+0000 mon.a (mon.0) 1659 : cluster 0 osdmap e298: 8 total, 8 up, 6 in 2023-10-09T05:31:45.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:45 smithi194 bash[92666]: cluster 2023-10-09T05:31:44.380295+0000 osd.5 (osd.5) 273 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T05:31:45.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:45 smithi194 bash[92666]: cluster 2023-10-09T05:31:44.385380+0000 osd.7 (osd.7) 254 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T05:31:45.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:45 smithi194 bash[92666]: cluster 2023-10-09T05:31:44.394265+0000 osd.7 (osd.7) 255 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T05:31:45.659 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:45.680 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:45.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:45.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:45.783 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:45.813 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:45.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:45.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:45.920 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:45.946 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:45.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:45.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:46.053 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:46.092 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:46.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:46.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:46.197 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:46.226 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:46.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:46.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:46.329 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:46.359 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:46.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:46.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:46.463 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:46.493 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:46.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:46.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:46.523 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 67 16 80451 80435 75.0238 55.125 0.0196824 0.0133223 2023-10-09T05:31:46.596 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:46.622 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:46.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:46.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:46.728 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:46.763 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:46.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:46.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:46.869 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:46.900 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:46.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:46.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:47.004 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:47.028 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:47.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:47.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:47.132 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:47.166 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:47.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:47.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:47.273 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:47.300 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:47.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:47.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:47.409 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:47.436 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:47.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:47.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:47.523 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 68 16 81285 81269 74.6869 52.125 0.0201187 0.0133825 2023-10-09T05:31:47.545 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:47.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:47 smithi133 bash[105589]: cluster 2023-10-09T05:31:46.294493+0000 mgr.y (mgr.25277) 1310 : cluster 0 pgmap v1405: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T05:31:47.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:47 smithi133 bash[102858]: cluster 2023-10-09T05:31:46.294493+0000 mgr.y (mgr.25277) 1310 : cluster 0 pgmap v1405: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T05:31:47.575 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:47.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:47.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:47.678 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:47.711 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:47.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:47.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:47.816 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:47.826 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:47 smithi194 bash[92666]: cluster 2023-10-09T05:31:46.294493+0000 mgr.y (mgr.25277) 1310 : cluster 0 pgmap v1405: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T05:31:47.843 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:47.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:47.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:47.946 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:47.975 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:47.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:47.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:48.079 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:48.080 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 6, 1, 7] out_osds: [5, 4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:31:48.080 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:31:48.080 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-10-09T05:31:48.080 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T05:31:48.080 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T05:31:48.080 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-10-09T05:31:48.107 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:48.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:48.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:48.111 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:31:48.211 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:48.239 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:48.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:48.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:48.344 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:48.410 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:48.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:48.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:48.521 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:48.525 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 69 16 82063 82047 74.3092 48.625 0.0086258 0.0134514 2023-10-09T05:31:48.545 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:48.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:48.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:48.645 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:48 smithi133 bash[102858]: audit 2023-10-09T05:31:47.645561+0000 mon.a (mon.0) 1660 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:31:48.645 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:48 smithi133 bash[102858]: audit 2023-10-09T05:31:47.661805+0000 mon.a (mon.0) 1661 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:31:48.646 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:48 smithi133 bash[102858]: cluster 2023-10-09T05:31:48.294926+0000 mgr.y (mgr.25277) 1311 : cluster 0 pgmap v1406: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 59 MiB/s wr, 941 op/s 2023-10-09T05:31:48.654 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:48.679 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:48.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:48.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:48.785 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:48.796 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:48 smithi194 bash[92666]: audit 2023-10-09T05:31:47.645561+0000 mon.a (mon.0) 1660 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:31:48.796 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:48 smithi194 bash[92666]: audit 2023-10-09T05:31:47.661805+0000 mon.a (mon.0) 1661 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:31:48.796 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:48 smithi194 bash[92666]: cluster 2023-10-09T05:31:48.294926+0000 mgr.y (mgr.25277) 1311 : cluster 0 pgmap v1406: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 59 MiB/s wr, 941 op/s 2023-10-09T05:31:48.811 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:48.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:48.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:48.914 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:48.924 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:48 smithi133 bash[105589]: audit 2023-10-09T05:31:47.645561+0000 mon.a (mon.0) 1660 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:31:48.924 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:48 smithi133 bash[105589]: audit 2023-10-09T05:31:47.661805+0000 mon.a (mon.0) 1661 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:31:48.925 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:48 smithi133 bash[105589]: cluster 2023-10-09T05:31:48.294926+0000 mgr.y (mgr.25277) 1311 : cluster 0 pgmap v1406: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 59 MiB/s wr, 941 op/s 2023-10-09T05:31:48.942 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:48.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:48.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:49.017 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:31:49.046 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:49.107 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:49.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:49.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:49.214 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:49.242 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:49.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:49.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:49.348 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:49.388 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:49.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:49.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:49.497 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:49.525 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 70 16 82896 82880 73.9912 52.0625 0.0198421 0.013509 2023-10-09T05:31:49.525 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:31:49.530 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:49.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:49.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:49.636 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:49.671 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:49.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:49.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:49.777 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:49.810 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:49.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:49.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:49.915 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:49.954 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:49.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:49.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:50.061 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:50.098 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:50.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:50.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:50.207 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:50.230 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:50.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:50.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:50.334 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:50.362 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:50.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:50.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:50.465 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:50.488 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:50.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:50.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:50.524 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 71 16 83610 83594 73.5775 44.625 0.00920337 0.0135835 2023-10-09T05:31:50.594 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:50.630 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:50.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:50.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:50.733 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:50.761 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:50.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:50.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:50.864 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:50.890 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:50.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:50.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:50.996 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:51.039 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:51.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:51.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:51.144 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:51.168 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:51.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:51.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:51.271 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:51.315 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:51.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:51.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:51.420 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:51.431 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:51 smithi194 bash[92666]: cluster 2023-10-09T05:31:50.297198+0000 mgr.y (mgr.25277) 1312 : cluster 0 pgmap v1407: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 52 MiB/s wr, 826 op/s 2023-10-09T05:31:51.431 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:51 smithi194 bash[92666]: audit 2023-10-09T05:31:51.234480+0000 mon.a (mon.0) 1662 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:31:51.431 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:51 smithi194 bash[92666]: audit 2023-10-09T05:31:51.247736+0000 mon.a (mon.0) 1663 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:31:51.448 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:51.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:51.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:51.524 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 72 16 84368 84352 73.2135 47.375 0.00956234 0.0136521 2023-10-09T05:31:51.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:51 smithi133 bash[105589]: cluster 2023-10-09T05:31:50.297198+0000 mgr.y (mgr.25277) 1312 : cluster 0 pgmap v1407: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 52 MiB/s wr, 826 op/s 2023-10-09T05:31:51.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:51 smithi133 bash[105589]: audit 2023-10-09T05:31:51.234480+0000 mon.a (mon.0) 1662 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:31:51.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:51 smithi133 bash[105589]: audit 2023-10-09T05:31:51.247736+0000 mon.a (mon.0) 1663 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:31:51.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:51 smithi133 bash[102858]: cluster 2023-10-09T05:31:50.297198+0000 mgr.y (mgr.25277) 1312 : cluster 0 pgmap v1407: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 52 MiB/s wr, 826 op/s 2023-10-09T05:31:51.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:51 smithi133 bash[102858]: audit 2023-10-09T05:31:51.234480+0000 mon.a (mon.0) 1662 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:31:51.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:51 smithi133 bash[102858]: audit 2023-10-09T05:31:51.247736+0000 mon.a (mon.0) 1663 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:31:51.551 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:51.576 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:51.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:51.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:51.681 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:51.724 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:51.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:51.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:51.832 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:51.857 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:51.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:51.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:51.960 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:51.986 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:51.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:51.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:52.091 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:52.119 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:52.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:52.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:52.230 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:52.257 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:52.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:52.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:52.363 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:52.397 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:52.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:52.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:52.502 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:52.502 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:52 smithi133 bash[105589]: audit 2023-10-09T05:31:51.871020+0000 mon.a (mon.0) 1664 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:31:52.503 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:52 smithi133 bash[105589]: audit 2023-10-09T05:31:51.873587+0000 mon.a (mon.0) 1665 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:31:52.503 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:52 smithi133 bash[105589]: audit 2023-10-09T05:31:51.886121+0000 mon.a (mon.0) 1666 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:31:52.503 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:52 smithi133 bash[102858]: audit 2023-10-09T05:31:51.871020+0000 mon.a (mon.0) 1664 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:31:52.503 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:52 smithi133 bash[102858]: audit 2023-10-09T05:31:51.873587+0000 mon.a (mon.0) 1665 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:31:52.503 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:52 smithi133 bash[102858]: audit 2023-10-09T05:31:51.886121+0000 mon.a (mon.0) 1666 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:31:52.524 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 73 16 85176 85160 72.9023 50.5 0.0127436 0.0137108 2023-10-09T05:31:52.526 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:52.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:52.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:52.581 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:31:52.581 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:31:52.629 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:52.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:52 smithi194 bash[92666]: audit 2023-10-09T05:31:51.871020+0000 mon.a (mon.0) 1664 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:31:52.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:52 smithi194 bash[92666]: audit 2023-10-09T05:31:51.873587+0000 mon.a (mon.0) 1665 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:31:52.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:52 smithi194 bash[92666]: audit 2023-10-09T05:31:51.886121+0000 mon.a (mon.0) 1666 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:31:52.657 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:52.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:52.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:52.760 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:52.779 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:52.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:52.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:52.882 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:52.914 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:52.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:52.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:53.018 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:53.042 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:53.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:53.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:53.145 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:53.170 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:53.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:53.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:53.274 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:53.303 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:53.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:53.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:53.407 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:53.417 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:53 smithi133 bash[102858]: cluster 2023-10-09T05:31:52.297760+0000 mgr.y (mgr.25277) 1313 : cluster 0 pgmap v1408: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 51 MiB/s wr, 816 op/s 2023-10-09T05:31:53.417 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:53 smithi133 bash[105589]: cluster 2023-10-09T05:31:52.297760+0000 mgr.y (mgr.25277) 1313 : cluster 0 pgmap v1408: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 51 MiB/s wr, 816 op/s 2023-10-09T05:31:53.444 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:53.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:53.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:53.524 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 74 16 85754 85738 72.4053 36.125 0.0124387 0.0138045 2023-10-09T05:31:53.550 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:53.588 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:53.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:53.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:53.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:53 smithi194 bash[92666]: cluster 2023-10-09T05:31:52.297760+0000 mgr.y (mgr.25277) 1313 : cluster 0 pgmap v1408: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 51 MiB/s wr, 816 op/s 2023-10-09T05:31:53.693 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:53.733 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:53.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:53.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:53.839 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:53.866 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:53.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:53.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:53.970 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:53.997 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:54.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:54.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:54.103 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:54.139 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:54.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:54.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:54.246 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:54.270 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:54.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:54.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:54.376 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:54.406 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:54.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:54.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:54.509 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:54.524 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 75 16 86565 86549 72.1157 50.6875 0.0207896 0.0138595 2023-10-09T05:31:54.537 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:54.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:54.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:54.552 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:31:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:31:54] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.43.0" 2023-10-09T05:31:54.640 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:54.666 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:54.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:54.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:54.772 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:54.798 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:54.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:54.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:54.902 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:54.926 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:54.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:54.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:55.031 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:55.041 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:54 smithi133 bash[102858]: cluster 2023-10-09T05:31:54.298231+0000 mgr.y (mgr.25277) 1314 : cluster 0 pgmap v1409: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 51 MiB/s wr, 816 op/s 2023-10-09T05:31:55.041 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:54 smithi133 bash[105589]: cluster 2023-10-09T05:31:54.298231+0000 mgr.y (mgr.25277) 1314 : cluster 0 pgmap v1409: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 51 MiB/s wr, 816 op/s 2023-10-09T05:31:55.060 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:55.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:55.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:55.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:54 smithi194 bash[92666]: cluster 2023-10-09T05:31:54.298231+0000 mgr.y (mgr.25277) 1314 : cluster 0 pgmap v1409: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 51 MiB/s wr, 816 op/s 2023-10-09T05:31:55.163 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:55.194 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:55.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:55.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:55.297 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:55.324 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:55.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:55.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:55.428 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:55.459 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:55.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:55.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:55.524 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 76 16 87439 87423 71.8855 54.625 0.00914393 0.0139058 2023-10-09T05:31:55.562 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:55.589 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:55.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:55.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:55.692 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:55.721 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:55.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:55.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:55.825 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:55.852 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:55.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:55.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:55.958 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:55.984 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:55.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:55.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:56.087 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:56.122 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:56.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:56.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:56.229 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:56.266 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:56.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:56.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:56.372 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:56.411 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:56.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:56.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:56.517 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:56.524 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 77 16 88278 88262 71.6328 52.4375 0.0185066 0.013954 2023-10-09T05:31:56.541 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:56.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:56.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:56.644 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:56.687 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:56.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:56.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:56.694 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:31:56.792 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:56.820 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:56.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:56.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:56.930 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:56.956 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:56.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:56.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:57.060 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:57.088 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:57.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:57.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:57.191 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:57.219 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:31:57.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:57.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:31:57.322 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:57.353 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:31:57.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:57.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:31:57.364 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:57 smithi194 bash[92666]: cluster 2023-10-09T05:31:56.299450+0000 mgr.y (mgr.25277) 1315 : cluster 0 pgmap v1410: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-10-09T05:31:57.457 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:57.489 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:57.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:57.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:57.524 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 78 16 89080 89064 71.357 50.125 0.0131331 0.0140078 2023-10-09T05:31:57.592 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:57.614 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:57.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:57.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:57.622 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:57 smithi133 bash[102858]: cluster 2023-10-09T05:31:56.299450+0000 mgr.y (mgr.25277) 1315 : cluster 0 pgmap v1410: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-10-09T05:31:57.622 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:57 smithi133 bash[105589]: cluster 2023-10-09T05:31:56.299450+0000 mgr.y (mgr.25277) 1315 : cluster 0 pgmap v1410: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-10-09T05:31:57.717 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:57.741 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:57.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:57.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:57.844 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:57.872 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:57.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:57.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:57.977 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:58.003 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:58.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:58.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:58.107 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:58.129 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:31:58.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:58.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:31:58.236 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:58.265 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:58.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:58.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:58.369 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:58.404 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:58.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:58.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:58.509 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:58.524 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 79 16 89696 89680 70.9411 38.5 0.0157503 0.01409 2023-10-09T05:31:58.540 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:58.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:58.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:58.646 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:58.674 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:58.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:58.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:58.777 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:58.805 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:58.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:58.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:58.910 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:58.945 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:58.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:58.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:59.050 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:59.082 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:31:59.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:59.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:31:59.188 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:59.217 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:59.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:59.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:59.320 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:59.347 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:31:59.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:59.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:31:59.449 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:59.459 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:59 smithi194 bash[92666]: cluster 2023-10-09T05:31:58.299818+0000 mgr.y (mgr.25277) 1316 : cluster 0 pgmap v1411: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 41 MiB/s wr, 657 op/s 2023-10-09T05:31:59.459 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:31:59 smithi194 bash[92666]: audit 2023-10-09T05:31:58.482882+0000 mon.a (mon.0) 1667 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:31:59.476 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:59.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:59.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:59.525 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T05:31:59.525156+0000 min lat: 0.00142329 max lat: 0.347117 avg lat: 0.0141502 2023-10-09T05:31:59.525 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:31:59.525 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 80 16 90441 90425 70.6363 46.5625 0.0512414 0.0141502 2023-10-09T05:31:59.582 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:59.607 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:31:59.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:59.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:31:59.712 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:59.720 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:59 smithi133 bash[105589]: cluster 2023-10-09T05:31:58.299818+0000 mgr.y (mgr.25277) 1316 : cluster 0 pgmap v1411: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 41 MiB/s wr, 657 op/s 2023-10-09T05:31:59.720 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:31:59 smithi133 bash[105589]: audit 2023-10-09T05:31:58.482882+0000 mon.a (mon.0) 1667 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:31:59.720 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:59 smithi133 bash[102858]: cluster 2023-10-09T05:31:58.299818+0000 mgr.y (mgr.25277) 1316 : cluster 0 pgmap v1411: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 41 MiB/s wr, 657 op/s 2023-10-09T05:31:59.720 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:31:59 smithi133 bash[102858]: audit 2023-10-09T05:31:58.482882+0000 mon.a (mon.0) 1667 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:31:59.741 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:31:59.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:59.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:31:59.848 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:59.872 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:31:59.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:31:59.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:31:59.980 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:00.012 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:00.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:00.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:00.116 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:00.147 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:00.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:00.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:00.252 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:00.280 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:00.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:00.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:00.386 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:00.411 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:00.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:00.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:00.517 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:00.525 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 81 16 91231 91215 70.3735 49.375 0.0295927 0.0142038 2023-10-09T05:32:00.551 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:00.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:00.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:00.655 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:00.688 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:00.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:00.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:00.792 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:00.816 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:00.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:00.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:00.921 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:00.946 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:00.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:00.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:01.051 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:01.093 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:01.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:01.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:01.196 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:01.222 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:01.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:01.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:01.326 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:01.358 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:01.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:01.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:01.463 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:01.484 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:01 smithi194 bash[92666]: cluster 2023-10-09T05:32:00.300442+0000 mgr.y (mgr.25277) 1317 : cluster 0 pgmap v1412: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 55 MiB/s wr, 876 op/s 2023-10-09T05:32:01.505 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:01.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:01.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:01.526 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 82 16 92060 92044 70.1471 51.8125 0.0201507 0.01425 2023-10-09T05:32:01.618 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:01.652 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:01.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:01.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:01.757 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:01.788 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:01.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:01.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:01.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:01 smithi133 bash[102858]: cluster 2023-10-09T05:32:00.300442+0000 mgr.y (mgr.25277) 1317 : cluster 0 pgmap v1412: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 55 MiB/s wr, 876 op/s 2023-10-09T05:32:01.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:01 smithi133 bash[105589]: cluster 2023-10-09T05:32:00.300442+0000 mgr.y (mgr.25277) 1317 : cluster 0 pgmap v1412: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 55 MiB/s wr, 876 op/s 2023-10-09T05:32:01.893 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:01.924 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:01.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:01.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:02.028 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:02.059 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:02.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:02.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:02.166 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:02.200 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:02.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:02.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:02.305 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:02.339 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:02.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:02.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:02.445 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:02.480 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:02.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:02.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:02.525 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 83 16 92730 92714 69.8065 41.875 0.0174198 0.0143185 2023-10-09T05:32:02.586 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:02.610 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:02.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:02.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:02.718 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:02.744 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:02.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:02.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:02.852 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:02.885 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:02.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:02.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:02.959 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:32:02.960 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:32:02.989 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:03.027 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:03.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:03.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:03.129 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:03.158 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:03.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:03.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:03.261 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:03.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:03.288 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:03.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:03.389 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:03.416 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:03.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:03.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:03.520 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:03.526 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 84 16 93527 93511 69.5684 49.8125 0.0168784 0.0143682 2023-10-09T05:32:03.535 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:03 smithi133 bash[105589]: cluster 2023-10-09T05:32:02.301064+0000 mgr.y (mgr.25277) 1318 : cluster 0 pgmap v1413: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 46 MiB/s wr, 736 op/s 2023-10-09T05:32:03.535 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:03 smithi133 bash[102858]: cluster 2023-10-09T05:32:02.301064+0000 mgr.y (mgr.25277) 1318 : cluster 0 pgmap v1413: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 46 MiB/s wr, 736 op/s 2023-10-09T05:32:03.553 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:03.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:03.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:03.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:03 smithi194 bash[92666]: cluster 2023-10-09T05:32:02.301064+0000 mgr.y (mgr.25277) 1318 : cluster 0 pgmap v1413: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 46 MiB/s wr, 736 op/s 2023-10-09T05:32:03.660 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:03.690 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:03.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:03.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:03.793 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:03.819 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:03.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:03.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:03.923 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:03.951 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:03.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:03.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:04.056 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:04.088 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:04.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:04.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:04.191 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:04.220 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:04.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:04.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:04.324 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:04.337 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:32:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:32:04] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.43.0" 2023-10-09T05:32:04.359 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:04.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:04.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:04.465 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:04.490 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:04.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:04.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:04.526 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 85 16 94276 94260 69.3005 46.8125 0.0249858 0.014424 2023-10-09T05:32:04.595 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:04.641 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:04.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:04.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:04.745 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:04.769 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:04.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:04.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:04.872 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:04.896 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:04.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:04.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:05.000 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:05.011 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:04 smithi194 bash[92666]: cluster 2023-10-09T05:32:04.301526+0000 mgr.y (mgr.25277) 1319 : cluster 0 pgmap v1414: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 40 MiB/s wr, 633 op/s 2023-10-09T05:32:05.029 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:05.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:05.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:05.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:04 smithi133 bash[105589]: cluster 2023-10-09T05:32:04.301526+0000 mgr.y (mgr.25277) 1319 : cluster 0 pgmap v1414: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 40 MiB/s wr, 633 op/s 2023-10-09T05:32:05.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:04 smithi133 bash[102858]: cluster 2023-10-09T05:32:04.301526+0000 mgr.y (mgr.25277) 1319 : cluster 0 pgmap v1414: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 40 MiB/s wr, 633 op/s 2023-10-09T05:32:05.131 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:05.156 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:05.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:05.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:05.260 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:05.283 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:05.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:05.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:05.386 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:05.410 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:05.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:05.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:05.513 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:05.528 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 86 16 94967 94951 68.9945 43.1875 0.018479 0.0144872 2023-10-09T05:32:05.535 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:05.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:05.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:05.638 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:05.663 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:05.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:05.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:05.766 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:05.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:05.806 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:05.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:05.907 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:05.934 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:05.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:05.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:06.038 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:06.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:06.067 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:06.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:06.168 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:06.192 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:06.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:06.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:06.296 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:06.326 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:06.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:06.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:06.429 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:06.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:06.454 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:06.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:06.529 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 87 16 95767 95751 68.7761 50 0.00791229 0.0145327 2023-10-09T05:32:06.555 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:06.579 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:06.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:06.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:06.682 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:06.711 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:06.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:06.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:06.816 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:06.845 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:06.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:06.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:06.949 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:06.980 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:06.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:06.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:07.085 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:07.119 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:07.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:07.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:07.225 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:07.257 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:07.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:07.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:07.358 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:07.362 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:07 smithi133 bash[102858]: cluster 2023-10-09T05:32:06.302267+0000 mgr.y (mgr.25277) 1320 : cluster 0 pgmap v1415: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 59 MiB/s wr, 950 op/s 2023-10-09T05:32:07.385 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:07.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:07.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:07.488 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:07.506 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:07 smithi194 bash[92666]: cluster 2023-10-09T05:32:06.302267+0000 mgr.y (mgr.25277) 1320 : cluster 0 pgmap v1415: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 59 MiB/s wr, 950 op/s 2023-10-09T05:32:07.513 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:07.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:07.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:07.528 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 88 16 96548 96532 68.5492 48.8125 0.0222007 0.0145816 2023-10-09T05:32:07.616 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:07.641 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:07.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:07.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:07.745 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:07.753 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:07 smithi133 bash[105589]: cluster 2023-10-09T05:32:06.302267+0000 mgr.y (mgr.25277) 1320 : cluster 0 pgmap v1415: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 59 MiB/s wr, 950 op/s 2023-10-09T05:32:07.772 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:07.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:07.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:07.876 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:07.899 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:07.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:07.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:08.003 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:08.034 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:08.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:08.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:08.140 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:08.169 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:08.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:08.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:08.272 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:08.310 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:08.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:08.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:08.414 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:08.443 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:08.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:08.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:08.529 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 89 16 97287 97271 68.2979 46.1875 0.0110001 0.0146353 2023-10-09T05:32:08.548 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:08.576 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:08.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:08.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:08.679 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:08.706 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:08.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:08.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:08.815 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:08.847 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:08.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:08.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:08.951 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:08.976 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:08.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:08.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:09.080 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:09.120 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:09.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:09.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:09.211 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T05:32:09.227 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:09.253 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:09.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:09.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:09.356 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:09.371 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:09 smithi133 bash[102858]: cluster 2023-10-09T05:32:08.302623+0000 mgr.y (mgr.25277) 1321 : cluster 0 pgmap v1416: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 40 MiB/s wr, 636 op/s 2023-10-09T05:32:09.383 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:09.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:09.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:09.487 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:09.498 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:09 smithi194 bash[92666]: cluster 2023-10-09T05:32:08.302623+0000 mgr.y (mgr.25277) 1321 : cluster 0 pgmap v1416: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 40 MiB/s wr, 636 op/s 2023-10-09T05:32:09.524 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:09.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:09.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:09.529 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 90 1 98037 98036 68.0703 47.8125 0.0149265 0.0146855 2023-10-09T05:32:09.627 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:09.641 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:09 smithi133 bash[105589]: cluster 2023-10-09T05:32:08.302623+0000 mgr.y (mgr.25277) 1321 : cluster 0 pgmap v1416: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 40 MiB/s wr, 636 op/s 2023-10-09T05:32:09.659 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:09.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:09.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:09.763 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:09.795 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:09.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:09.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:09.898 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:09.929 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:09.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:09.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:10.032 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:10.053 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:10.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:10.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:10.156 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:10.182 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:10.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:10.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:10.285 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:10.310 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:10.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:10.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:10.413 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:10.449 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:10.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:10.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:10.529 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Total time run: 90.0139 2023-10-09T05:32:10.529 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Total writes made: 98037 2023-10-09T05:32:10.530 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Write size: 65536 2023-10-09T05:32:10.530 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Object size: 65536 2023-10-09T05:32:10.530 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Bandwidth (MB/sec): 68.0707 2023-10-09T05:32:10.530 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Stddev Bandwidth: 27.0967 2023-10-09T05:32:10.530 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Max bandwidth (MB/sec): 153.75 2023-10-09T05:32:10.530 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Min bandwidth (MB/sec): 36.125 2023-10-09T05:32:10.530 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Average IOPS: 1089 2023-10-09T05:32:10.530 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Stddev IOPS: 433.548 2023-10-09T05:32:10.531 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Max IOPS: 2460 2023-10-09T05:32:10.531 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Min IOPS: 578 2023-10-09T05:32:10.531 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Average Latency(s): 0.0146855 2023-10-09T05:32:10.531 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Stddev Latency(s): 0.0171826 2023-10-09T05:32:10.531 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Max latency(s): 0.347117 2023-10-09T05:32:10.531 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Min latency(s): 0.00142329 2023-10-09T05:32:10.537 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T05:32:10.553 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:10.594 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:10.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:10.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:10.698 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:10.731 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:10.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:10.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:10.836 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:10.874 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:10.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:10.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:10.979 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:11.014 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:11.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:11.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:11.118 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:11.145 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:11.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:11.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:11.248 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:11.281 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:11.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:11.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:11.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:11 smithi194 bash[92666]: cluster 2023-10-09T05:32:10.303419+0000 mgr.y (mgr.25277) 1322 : cluster 0 pgmap v1417: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 53 MiB/s wr, 842 op/s 2023-10-09T05:32:11.387 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:11.416 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:11.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:11.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:11.521 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:11.532 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:11 smithi133 bash[102858]: cluster 2023-10-09T05:32:10.303419+0000 mgr.y (mgr.25277) 1322 : cluster 0 pgmap v1417: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 53 MiB/s wr, 842 op/s 2023-10-09T05:32:11.532 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:11 smithi133 bash[105589]: cluster 2023-10-09T05:32:10.303419+0000 mgr.y (mgr.25277) 1322 : cluster 0 pgmap v1417: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 53 MiB/s wr, 842 op/s 2023-10-09T05:32:11.561 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:11.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:11.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:11.664 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:11.703 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:11.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:11.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:11.806 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:11.834 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:11.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:11.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:11.937 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:11.966 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:11.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:11.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:12.072 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:12.095 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:12.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:12.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:12.198 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:12.204 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:32:12.204 INFO:teuthology.orchestra.run.smithi133.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T05:32:12.205 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:32:12.225 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:12.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:12.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:12.273 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:32:12.327 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:32:12.332 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:12.356 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:12.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:12.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:12.388 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:32:12.426 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:32:12.459 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:12.479 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:32:12.488 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:12.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:12.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:12.522 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:32:12.530 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:12 smithi133 bash[105589]: audit 2023-10-09T05:32:12.228070+0000 mon.a (mon.0) 1668 : audit 0 from='client.? 172.21.15.133:0/1285140888' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:32:12.530 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:12 smithi133 bash[102858]: audit 2023-10-09T05:32:12.228070+0000 mon.a (mon.0) 1668 : audit 0 from='client.? 172.21.15.133:0/1285140888' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:32:12.563 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:32:12.591 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:12.598 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:12 smithi194 bash[92666]: audit 2023-10-09T05:32:12.228070+0000 mon.a (mon.0) 1668 : audit 0 from='client.? 172.21.15.133:0/1285140888' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:32:12.602 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:32:12.615 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:32:12.615 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:32:12.615 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:32:12.615 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:32:12.615 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:32:12.615 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:32:12.616 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:32:12.616 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:32:12.619 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:12.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:12.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:12.722 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:12.749 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:12.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:12.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:12.852 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:12.884 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:12.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:12.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:12.987 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:13.029 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:13.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:13.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:13.133 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:13.169 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:13.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:13.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:13.276 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:13.306 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:13.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:13.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:13.410 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is unset 2023-10-09T05:32:13.410 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:13.438 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:13.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:13.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:13.545 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:13.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:13 smithi133 bash[105589]: cluster 2023-10-09T05:32:12.303987+0000 mgr.y (mgr.25277) 1323 : cluster 0 pgmap v1418: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 44 MiB/s wr, 706 op/s 2023-10-09T05:32:13.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:13 smithi133 bash[105589]: audit 2023-10-09T05:32:13.074097+0000 mon.b (mon.1) 30 : audit 1 from='client.? 172.21.15.133:0/1467625648' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:32:13.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:13 smithi133 bash[105589]: audit 2023-10-09T05:32:13.074472+0000 mon.a (mon.0) 1669 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:32:13.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:13 smithi133 bash[102858]: cluster 2023-10-09T05:32:12.303987+0000 mgr.y (mgr.25277) 1323 : cluster 0 pgmap v1418: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 44 MiB/s wr, 706 op/s 2023-10-09T05:32:13.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:13 smithi133 bash[102858]: audit 2023-10-09T05:32:13.074097+0000 mon.b (mon.1) 30 : audit 1 from='client.? 172.21.15.133:0/1467625648' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:32:13.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:13 smithi133 bash[102858]: audit 2023-10-09T05:32:13.074472+0000 mon.a (mon.0) 1669 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:32:13.564 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:13.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:13.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:13.668 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:13.704 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:13.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:13.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:13.810 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:13.820 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:13 smithi194 bash[92666]: cluster 2023-10-09T05:32:12.303987+0000 mgr.y (mgr.25277) 1323 : cluster 0 pgmap v1418: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 44 MiB/s wr, 706 op/s 2023-10-09T05:32:13.820 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:13 smithi194 bash[92666]: audit 2023-10-09T05:32:13.074097+0000 mon.b (mon.1) 30 : audit 1 from='client.? 172.21.15.133:0/1467625648' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:32:13.820 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:13 smithi194 bash[92666]: audit 2023-10-09T05:32:13.074472+0000 mon.a (mon.0) 1669 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:32:13.839 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:13.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:13.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:13.942 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:13.965 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:13.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:13.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:14.069 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:14.095 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:14.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:14.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:14.201 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:14.232 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:14.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:14.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:14.281 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:32:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:32:14] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.43.0" 2023-10-09T05:32:14.339 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:14.364 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:14.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:14.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:14.469 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:14.507 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:14.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:14.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:14.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:14 smithi133 bash[102858]: cluster 2023-10-09T05:32:13.391011+0000 mon.a (mon.0) 1670 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:32:14.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:14 smithi133 bash[102858]: cluster 2023-10-09T05:32:13.391069+0000 mon.a (mon.0) 1671 : cluster 1 Cluster is now healthy 2023-10-09T05:32:14.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:14 smithi133 bash[102858]: audit 2023-10-09T05:32:13.395920+0000 mon.a (mon.0) 1672 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:32:14.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:14 smithi133 bash[102858]: cluster 2023-10-09T05:32:13.396155+0000 mon.a (mon.0) 1673 : cluster 0 osdmap e299: 8 total, 8 up, 6 in 2023-10-09T05:32:14.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:14 smithi133 bash[102858]: cluster 2023-10-09T05:32:13.398649+0000 osd.4 (osd.4) 270 : cluster 3 failed to encode map e299 with expected crc 2023-10-09T05:32:14.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:14 smithi133 bash[102858]: cluster 2023-10-09T05:32:13.404342+0000 osd.7 (osd.7) 256 : cluster 3 failed to encode map e299 with expected crc 2023-10-09T05:32:14.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:14 smithi133 bash[102858]: audit 2023-10-09T05:32:13.487700+0000 mon.a (mon.0) 1674 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:32:14.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:14 smithi133 bash[102858]: audit 2023-10-09T05:32:13.491480+0000 mon.a (mon.0) 1675 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:32:14.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:14 smithi133 bash[105589]: cluster 2023-10-09T05:32:13.391011+0000 mon.a (mon.0) 1670 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:32:14.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:14 smithi133 bash[105589]: cluster 2023-10-09T05:32:13.391069+0000 mon.a (mon.0) 1671 : cluster 1 Cluster is now healthy 2023-10-09T05:32:14.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:14 smithi133 bash[105589]: audit 2023-10-09T05:32:13.395920+0000 mon.a (mon.0) 1672 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:32:14.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:14 smithi133 bash[105589]: cluster 2023-10-09T05:32:13.396155+0000 mon.a (mon.0) 1673 : cluster 0 osdmap e299: 8 total, 8 up, 6 in 2023-10-09T05:32:14.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:14 smithi133 bash[105589]: cluster 2023-10-09T05:32:13.398649+0000 osd.4 (osd.4) 270 : cluster 3 failed to encode map e299 with expected crc 2023-10-09T05:32:14.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:14 smithi133 bash[105589]: cluster 2023-10-09T05:32:13.404342+0000 osd.7 (osd.7) 256 : cluster 3 failed to encode map e299 with expected crc 2023-10-09T05:32:14.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:14 smithi133 bash[105589]: audit 2023-10-09T05:32:13.487700+0000 mon.a (mon.0) 1674 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:32:14.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:14 smithi133 bash[105589]: audit 2023-10-09T05:32:13.491480+0000 mon.a (mon.0) 1675 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:32:14.610 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:14.619 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:14 smithi194 bash[92666]: cluster 2023-10-09T05:32:13.391011+0000 mon.a (mon.0) 1670 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:32:14.619 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:14 smithi194 bash[92666]: cluster 2023-10-09T05:32:13.391069+0000 mon.a (mon.0) 1671 : cluster 1 Cluster is now healthy 2023-10-09T05:32:14.619 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:14 smithi194 bash[92666]: audit 2023-10-09T05:32:13.395920+0000 mon.a (mon.0) 1672 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:32:14.619 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:14 smithi194 bash[92666]: cluster 2023-10-09T05:32:13.396155+0000 mon.a (mon.0) 1673 : cluster 0 osdmap e299: 8 total, 8 up, 6 in 2023-10-09T05:32:14.619 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:14 smithi194 bash[92666]: cluster 2023-10-09T05:32:13.398649+0000 osd.4 (osd.4) 270 : cluster 3 failed to encode map e299 with expected crc 2023-10-09T05:32:14.619 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:14 smithi194 bash[92666]: cluster 2023-10-09T05:32:13.404342+0000 osd.7 (osd.7) 256 : cluster 3 failed to encode map e299 with expected crc 2023-10-09T05:32:14.619 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:14 smithi194 bash[92666]: audit 2023-10-09T05:32:13.487700+0000 mon.a (mon.0) 1674 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:32:14.619 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:14 smithi194 bash[92666]: audit 2023-10-09T05:32:13.491480+0000 mon.a (mon.0) 1675 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:32:14.634 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:14.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:14.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:14.737 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:14.764 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:14.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:14.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:14.868 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:14.895 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:14.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:14.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:15.000 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:15.037 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:15.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:15.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:15.143 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:15.169 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:15.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:15.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:15.272 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:15.297 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:15.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:15.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:15.400 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:15.416 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:15 smithi194 bash[92666]: cluster 2023-10-09T05:32:14.304394+0000 mgr.y (mgr.25277) 1324 : cluster 0 pgmap v1420: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 45 MiB/s wr, 728 op/s 2023-10-09T05:32:15.426 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:15.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:15.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:15.535 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:15.561 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:15.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:15.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:15.667 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:15.692 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:15.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:15.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:15.796 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:15.839 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:15.840 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:15 smithi133 bash[105589]: cluster 2023-10-09T05:32:14.304394+0000 mgr.y (mgr.25277) 1324 : cluster 0 pgmap v1420: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 45 MiB/s wr, 728 op/s 2023-10-09T05:32:15.840 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:15 smithi133 bash[102858]: cluster 2023-10-09T05:32:14.304394+0000 mgr.y (mgr.25277) 1324 : cluster 0 pgmap v1420: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 45 MiB/s wr, 728 op/s 2023-10-09T05:32:15.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:15.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:15.943 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:15.970 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:15.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:15.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:16.076 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:16.102 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:16.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:16.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:16.207 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:16.235 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:16.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:16.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:16.338 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:16.366 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:16.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:16.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:16.469 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:16.500 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:16.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:16.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:16.602 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:16.632 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:16.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:16.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:16.738 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:16.768 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:16.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:16.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:16.871 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:16.894 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:16.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:16.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:17.001 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:17.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:17.028 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:17.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:17.129 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:17.165 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:17.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:17.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:17.269 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:17.295 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:17.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:17.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:17.398 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:17.428 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:17.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:17.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:17.533 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:17.561 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:17.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:17.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:17.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:17 smithi194 bash[92666]: cluster 2023-10-09T05:32:16.305131+0000 mgr.y (mgr.25277) 1325 : cluster 0 pgmap v1421: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 102 B/s rd, 22 MiB/s wr, 690 op/s 2023-10-09T05:32:17.666 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:17.675 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:17 smithi133 bash[102858]: cluster 2023-10-09T05:32:16.305131+0000 mgr.y (mgr.25277) 1325 : cluster 0 pgmap v1421: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 102 B/s rd, 22 MiB/s wr, 690 op/s 2023-10-09T05:32:17.675 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:17 smithi133 bash[105589]: cluster 2023-10-09T05:32:16.305131+0000 mgr.y (mgr.25277) 1325 : cluster 0 pgmap v1421: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 102 B/s rd, 22 MiB/s wr, 690 op/s 2023-10-09T05:32:17.693 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:17.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:17.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:17.797 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:17.831 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:17.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:17.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:17.934 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:17.953 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:17.966 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:17.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:17.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:18.071 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:18.104 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:18.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:18.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:18.214 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:18.253 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:18.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:18.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:18.355 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:18.389 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:18.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:18.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:18.476 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:32:18.482 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:32:18.493 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:18.529 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:18.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:18.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:18.632 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:18.687 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:18.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:18.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:18.791 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:18.822 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:18.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:18.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:18.925 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:18.953 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:18.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:18.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:19.062 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:19.087 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:19.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:19.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:19.190 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:19.214 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:19.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:19.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:19.318 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:19.350 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:19.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:19.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:19.422 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:19 smithi133 bash[102858]: cluster 2023-10-09T05:32:18.305728+0000 mgr.y (mgr.25277) 1326 : cluster 0 pgmap v1422: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 102 B/s rd, 22 MiB/s wr, 690 op/s 2023-10-09T05:32:19.457 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:19.484 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:19.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:19.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:19.589 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:19.615 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:19.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:19.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:19.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:19 smithi194 bash[92666]: cluster 2023-10-09T05:32:18.305728+0000 mgr.y (mgr.25277) 1326 : cluster 0 pgmap v1422: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 102 B/s rd, 22 MiB/s wr, 690 op/s 2023-10-09T05:32:19.718 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:19.745 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:19.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:19.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:19.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:19 smithi133 bash[105589]: cluster 2023-10-09T05:32:18.305728+0000 mgr.y (mgr.25277) 1326 : cluster 0 pgmap v1422: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 102 B/s rd, 22 MiB/s wr, 690 op/s 2023-10-09T05:32:19.852 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:19.876 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:19.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:19.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:19.979 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:20.007 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:20.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:20.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:20.110 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:20.154 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:20.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:20.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:20.257 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:20.285 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:20.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:20.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:20.391 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:20.420 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:20.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:20.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:20.525 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:20.554 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:20.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:20.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:20.660 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:20.686 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:20.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:20.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:20.790 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:20.818 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:20.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:20.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:20.923 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:20.955 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:20.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:20.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:21.062 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:21.091 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:21.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:21.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:21.197 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 6, 1, 7] out_osds: [5, 4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:32:21.197 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:32:21.197 INFO:tasks.thrashosds.thrasher:Adding osd 5 2023-10-09T05:32:21.197 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd in 5 2023-10-09T05:32:21.198 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:21.228 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:21.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:21.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:21.332 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:21.496 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:21.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:21.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:21.600 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:21.608 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:21 smithi133 bash[102858]: cluster 2023-10-09T05:32:20.306335+0000 mgr.y (mgr.25277) 1327 : cluster 0 pgmap v1423: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 102 B/s rd, 6.1 MiB/s wr, 709 op/s 2023-10-09T05:32:21.608 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:21 smithi133 bash[105589]: cluster 2023-10-09T05:32:20.306335+0000 mgr.y (mgr.25277) 1327 : cluster 0 pgmap v1423: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 102 B/s rd, 6.1 MiB/s wr, 709 op/s 2023-10-09T05:32:21.624 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:21.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:21.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:21.730 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:21.773 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:21.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:21.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:21.882 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:21.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:21 smithi194 bash[92666]: cluster 2023-10-09T05:32:20.306335+0000 mgr.y (mgr.25277) 1327 : cluster 0 pgmap v1423: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 102 B/s rd, 6.1 MiB/s wr, 709 op/s 2023-10-09T05:32:21.906 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:21.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:21.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:22.009 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:22.031 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:22.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:22.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:22.136 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:22.162 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:22.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:22.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:22.267 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:22.298 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:22.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:22.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:22.401 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:22.425 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:22.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:22.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:22.530 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:22.562 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:22.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:22.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:22.666 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:22.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:22.710 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:22.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:22.811 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:22.841 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:22.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:22.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:22.944 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:22.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:22.986 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:22.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:23.087 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:23.115 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:23.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:23.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:23.218 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:23.253 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:23.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:23.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:23.355 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:23.378 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:23.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:23.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:23.484 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:23.493 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:23 smithi133 bash[105589]: cluster 2023-10-09T05:32:22.306880+0000 mgr.y (mgr.25277) 1328 : cluster 0 pgmap v1424: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 102 B/s rd, 102 B/s wr, 727 op/s 2023-10-09T05:32:23.493 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:23 smithi133 bash[102858]: cluster 2023-10-09T05:32:22.306880+0000 mgr.y (mgr.25277) 1328 : cluster 0 pgmap v1424: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 102 B/s rd, 102 B/s wr, 727 op/s 2023-10-09T05:32:23.510 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:23.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:23.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:23.616 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:23.624 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:23 smithi194 bash[92666]: cluster 2023-10-09T05:32:22.306880+0000 mgr.y (mgr.25277) 1328 : cluster 0 pgmap v1424: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 102 B/s rd, 102 B/s wr, 727 op/s 2023-10-09T05:32:23.643 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:23.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:23.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:23.679 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:32:23.679 INFO:teuthology.orchestra.run.smithi133.stdout: "ops": [], 2023-10-09T05:32:23.679 INFO:teuthology.orchestra.run.smithi133.stdout: "complaint_time": 30, 2023-10-09T05:32:23.679 INFO:teuthology.orchestra.run.smithi133.stdout: "num_blocked_ops": 0 2023-10-09T05:32:23.679 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:32:23.747 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:23.772 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:23.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:23.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:23.875 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:23.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:23.905 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:23.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:24.006 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:24.038 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:24.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:24.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:24.146 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:24.178 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:24.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:24.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:24.284 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:24.307 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:24.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:24.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:24.324 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:32:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:32:24] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.43.0" 2023-10-09T05:32:24.409 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:24.436 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:24.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:24.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:24.539 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:24.574 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:24.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:24.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:24.680 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:24.707 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:24.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:24.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:24.812 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:24.838 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:24.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:24.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:24.944 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:24.966 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:24.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:24.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:25.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:24 smithi133 bash[105589]: cluster 2023-10-09T05:32:24.307213+0000 mgr.y (mgr.25277) 1329 : cluster 0 pgmap v1425: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 93 B/s rd, 93 B/s wr, 666 op/s 2023-10-09T05:32:25.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:24 smithi133 bash[102858]: cluster 2023-10-09T05:32:24.307213+0000 mgr.y (mgr.25277) 1329 : cluster 0 pgmap v1425: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 93 B/s rd, 93 B/s wr, 666 op/s 2023-10-09T05:32:25.069 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:25.078 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:24 smithi194 bash[92666]: cluster 2023-10-09T05:32:24.307213+0000 mgr.y (mgr.25277) 1329 : cluster 0 pgmap v1425: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 93 B/s rd, 93 B/s wr, 666 op/s 2023-10-09T05:32:25.091 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:25.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:25.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:25.194 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:25.248 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:25.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:25.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:25.268 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T05:32:25.358 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:25.385 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:25.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:25.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:25.487 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:25.513 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:25.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:25.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:25.619 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:25.650 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:25.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:25.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:25.746 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:25.750 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:25.760 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:25.787 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:25.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:25.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:25.892 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:25.928 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:25.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:25.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:26.035 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:26.055 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:26.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:26.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:26.161 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:26.193 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:26.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:26.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:26.303 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:26.337 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:26.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:26.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:26.441 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:26.465 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:26.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:26.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:26.570 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:26.593 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:26.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:26.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:26.697 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:26.722 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:26.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:26.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:26.767 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:26.825 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:26.858 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:26.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:26.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:26.962 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:26.994 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:26.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:26.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:27.098 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:27.124 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:27.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:27.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:27.230 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:27.260 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:27.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:27.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:27.367 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:27.399 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:27.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:27.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:27.503 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:27.528 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:27.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:27.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:27.632 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:27.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:27 smithi194 bash[92666]: cluster 2023-10-09T05:32:26.307919+0000 mgr.y (mgr.25277) 1330 : cluster 0 pgmap v1426: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 85 B/s rd, 85 B/s wr, 930 op/s 2023-10-09T05:32:27.641 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:27 smithi133 bash[102858]: cluster 2023-10-09T05:32:26.307919+0000 mgr.y (mgr.25277) 1330 : cluster 0 pgmap v1426: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 85 B/s rd, 85 B/s wr, 930 op/s 2023-10-09T05:32:27.641 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:27 smithi133 bash[105589]: cluster 2023-10-09T05:32:26.307919+0000 mgr.y (mgr.25277) 1330 : cluster 0 pgmap v1426: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 85 B/s rd, 85 B/s wr, 930 op/s 2023-10-09T05:32:27.660 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:27.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:27.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:27.767 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:27.803 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:27.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:27.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:27.905 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:27.932 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:27.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:27.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:28.039 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:28.072 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:28.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:28.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:28.177 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:28.209 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:28.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:28.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:28.321 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:28.351 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:28.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:28.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:28.454 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:28.479 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:28.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:28.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:28.582 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:28.608 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:28.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:28.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:28.713 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:28.747 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:28.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:28.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:28.852 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:28.877 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:28.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:28.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:28.981 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:29.008 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:29.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:29.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:29.117 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:29.151 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:29.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:29.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:29.255 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:29.303 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:29.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:29.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:29.410 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:29.436 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:29.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:29.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:29.542 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:29.547 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:29 smithi133 bash[105589]: cluster 2023-10-09T05:32:28.308331+0000 mgr.y (mgr.25277) 1331 : cluster 0 pgmap v1427: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 645 op/s 2023-10-09T05:32:29.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:29 smithi133 bash[105589]: audit 2023-10-09T05:32:28.483784+0000 mon.a (mon.0) 1676 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:32:29.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:29 smithi133 bash[102858]: cluster 2023-10-09T05:32:28.308331+0000 mgr.y (mgr.25277) 1331 : cluster 0 pgmap v1427: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 645 op/s 2023-10-09T05:32:29.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:29 smithi133 bash[102858]: audit 2023-10-09T05:32:28.483784+0000 mon.a (mon.0) 1676 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:32:29.575 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:29.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:29.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:29.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:29 smithi194 bash[92666]: cluster 2023-10-09T05:32:28.308331+0000 mgr.y (mgr.25277) 1331 : cluster 0 pgmap v1427: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 645 op/s 2023-10-09T05:32:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:29 smithi194 bash[92666]: audit 2023-10-09T05:32:28.483784+0000 mon.a (mon.0) 1676 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:32:29.678 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:29.703 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:29.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:29.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:29.808 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:29.822 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:29.832 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:29.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:29.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:29.936 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:29.975 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:29.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:29.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:30.078 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:30.117 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:30.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:30.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:30.220 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:30.245 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:30.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:30.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:30.348 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:30.387 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:30.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:30.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:30.490 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:30.519 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:30.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:30.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:30.622 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:30.657 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:30.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:30.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:30.761 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:30.791 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:30.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:30.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:30.897 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:30.928 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:30.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:30.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:31.033 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:31.064 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:31.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:31.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:31.168 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:31.197 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:31.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:31.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:31.301 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:31.327 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:31.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:31.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:31.430 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:31.438 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:31 smithi194 bash[92666]: cluster 2023-10-09T05:32:30.308916+0000 mgr.y (mgr.25277) 1332 : cluster 0 pgmap v1428: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 830 op/s 2023-10-09T05:32:31.454 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:31.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:31.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:31.557 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:31.580 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:31.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:31.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:31.624 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:31 smithi133 bash[102858]: cluster 2023-10-09T05:32:30.308916+0000 mgr.y (mgr.25277) 1332 : cluster 0 pgmap v1428: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 830 op/s 2023-10-09T05:32:31.624 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:31 smithi133 bash[105589]: cluster 2023-10-09T05:32:30.308916+0000 mgr.y (mgr.25277) 1332 : cluster 0 pgmap v1428: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 830 op/s 2023-10-09T05:32:31.683 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:31.714 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:31.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:31.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:31.818 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:31.843 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:31.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:31.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:31.949 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:31.975 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:31.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:31.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:32.079 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:32.102 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:32.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:32.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:32.205 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:32.232 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:32.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:32.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:32.335 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:32.361 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:32.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:32.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:32.465 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:32.500 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:32.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:32.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:32.606 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:32.641 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:32.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:32.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:32.746 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:32.786 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:32.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:32.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:32.890 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:32.913 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:32.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:32.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:33.017 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:33.053 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:33.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:33.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:33.160 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:33.193 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:33.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:33.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:33.302 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:33.328 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:33.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:33.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:33.379 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:33 smithi133 bash[102858]: cluster 2023-10-09T05:32:32.309532+0000 mgr.y (mgr.25277) 1333 : cluster 0 pgmap v1429: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 748 op/s 2023-10-09T05:32:33.433 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:33.460 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:33.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:33.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:33.567 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:33.611 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:33.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:33.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:33.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:33 smithi194 bash[92666]: cluster 2023-10-09T05:32:32.309532+0000 mgr.y (mgr.25277) 1333 : cluster 0 pgmap v1429: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 748 op/s 2023-10-09T05:32:33.716 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:33.737 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:33 smithi133 bash[105589]: cluster 2023-10-09T05:32:32.309532+0000 mgr.y (mgr.25277) 1333 : cluster 0 pgmap v1429: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 748 op/s 2023-10-09T05:32:33.757 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:33.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:33.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:33.862 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:33.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:33.916 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:33.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:34.017 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:34.068 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:34.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:34.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:34.171 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:34.207 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:34.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:34.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:34.312 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:34.340 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:34.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:34.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:34.443 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:34.452 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:32:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:32:34] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.43.0" 2023-10-09T05:32:34.470 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:34.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:34.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:34.573 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:34.613 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:34.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:34.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:34.716 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:34.747 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:34.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:34.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:34.850 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:34.878 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:34.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:34.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:34.984 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:34.993 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:34 smithi133 bash[105589]: cluster 2023-10-09T05:32:34.309870+0000 mgr.y (mgr.25277) 1334 : cluster 0 pgmap v1430: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 649 op/s 2023-10-09T05:32:34.993 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:34 smithi133 bash[102858]: cluster 2023-10-09T05:32:34.309870+0000 mgr.y (mgr.25277) 1334 : cluster 0 pgmap v1430: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 649 op/s 2023-10-09T05:32:35.022 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:35.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:35.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:35.128 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:35.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:34 smithi194 bash[92666]: cluster 2023-10-09T05:32:34.309870+0000 mgr.y (mgr.25277) 1334 : cluster 0 pgmap v1430: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 649 op/s 2023-10-09T05:32:35.160 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:35.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:35.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:35.264 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:35.294 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:35.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:35.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:35.401 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:35.431 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:35.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:35.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:35.534 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:35.561 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:35.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:35.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:35.665 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:35.691 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:35.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:35.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:35.794 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:35.828 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:35.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:35.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:35.933 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:35.960 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:35.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:35.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:36.064 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:36.085 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:36.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:36.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:36.188 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:36.215 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:36.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:36.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:36.320 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:36.351 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:36.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:36.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:36.456 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:36.480 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:36.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:36.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:36.585 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:36.620 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:36.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:36.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:36.724 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:36.751 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:36.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:36.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:36.855 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:36.902 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:36.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:36.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:37.009 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:37.048 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:37.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:37.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:37.154 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:37.207 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:37.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:37.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:37.318 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:37.356 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:37.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:37.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:37.460 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:37.487 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:37.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:37.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:37.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:37 smithi133 bash[102858]: cluster 2023-10-09T05:32:36.310567+0000 mgr.y (mgr.25277) 1335 : cluster 0 pgmap v1431: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 949 op/s 2023-10-09T05:32:37.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:37 smithi133 bash[105589]: cluster 2023-10-09T05:32:36.310567+0000 mgr.y (mgr.25277) 1335 : cluster 0 pgmap v1431: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 949 op/s 2023-10-09T05:32:37.595 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:37.619 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:37.620 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:37 smithi194 bash[92666]: cluster 2023-10-09T05:32:36.310567+0000 mgr.y (mgr.25277) 1335 : cluster 0 pgmap v1431: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 949 op/s 2023-10-09T05:32:37.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:37.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:37.722 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:37.748 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:37.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:37.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:37.854 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:37.909 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:37.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:37.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:38.015 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:38.048 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:38.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:38.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:38.151 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:38.180 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:38.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:38.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:38.284 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:38.309 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:38.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:38.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:38.415 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:38.439 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:38.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:38.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:38.545 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:38.570 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:38.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:38.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:38.678 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:38.708 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:38.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:38.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:38.812 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:38.838 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:38.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:38.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:38.944 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:38.979 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:38.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:38.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:39.086 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:39.116 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:39.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:39.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:39.220 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:39.244 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:39.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:39.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:39.347 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:39.373 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:39.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:39.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:39.477 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:39.511 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:39 smithi133 bash[102858]: cluster 2023-10-09T05:32:38.310987+0000 mgr.y (mgr.25277) 1336 : cluster 0 pgmap v1432: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 625 op/s 2023-10-09T05:32:39.515 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:39.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:39.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:39.622 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:39.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:39 smithi194 bash[92666]: cluster 2023-10-09T05:32:38.310987+0000 mgr.y (mgr.25277) 1336 : cluster 0 pgmap v1432: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 625 op/s 2023-10-09T05:32:39.647 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:39.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:39.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:39.755 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:39.763 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:39 smithi133 bash[105589]: cluster 2023-10-09T05:32:38.310987+0000 mgr.y (mgr.25277) 1336 : cluster 0 pgmap v1432: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 625 op/s 2023-10-09T05:32:39.793 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:39.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:39.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:39.896 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:39.920 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:39.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:39.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:40.027 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:40.053 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:40.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:40.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:40.157 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:40.181 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:40.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:40.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:40.284 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:40.312 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:40.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:40.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:40.418 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:40.442 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:40.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:40.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:40.548 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:40.574 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:40.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:40.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:40.677 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:40.713 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:40.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:40.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:40.818 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:40.843 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:40.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:40.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:40.947 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:40.983 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:40.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:40.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:41.088 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:41.117 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:41.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:41.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:41.220 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:41.246 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:41.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:41.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:41.353 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:41.382 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:41.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:41.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:41.486 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:41.515 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:41.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:41.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:41.621 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:41.628 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:41 smithi194 bash[92666]: cluster 2023-10-09T05:32:40.311568+0000 mgr.y (mgr.25277) 1337 : cluster 0 pgmap v1433: 89 pgs: 89 active+clean; 4.7 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 802 op/s 2023-10-09T05:32:41.646 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:41.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:41.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:41.750 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:41.779 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:41.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:41.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:41.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:41 smithi133 bash[102858]: cluster 2023-10-09T05:32:40.311568+0000 mgr.y (mgr.25277) 1337 : cluster 0 pgmap v1433: 89 pgs: 89 active+clean; 4.7 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 802 op/s 2023-10-09T05:32:41.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:41 smithi133 bash[105589]: cluster 2023-10-09T05:32:40.311568+0000 mgr.y (mgr.25277) 1337 : cluster 0 pgmap v1433: 89 pgs: 89 active+clean; 4.7 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 802 op/s 2023-10-09T05:32:41.885 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:41.910 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:41.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:41.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:42.014 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:42.046 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:42.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:42.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:42.152 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:42.176 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:42.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:42.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:42.281 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:42.303 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:42.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:42.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:42.412 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:42.440 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:42.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:42.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:42.543 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:42.563 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:42.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:42.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:42.666 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:42.709 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:42.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:42.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:42.815 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:42.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:42.853 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:42.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:42.954 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:42.991 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:42.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:42.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:43.096 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:43.139 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:43.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:43.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:43.244 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:43.272 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:43.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:43.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:43.319 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:32:43.376 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:43.377 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:32:43.414 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:43.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:43.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:43.429 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:32:43.489 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:32:43.521 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:43.531 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:43 smithi194 bash[92666]: cluster 2023-10-09T05:32:42.312316+0000 mgr.y (mgr.25277) 1338 : cluster 0 pgmap v1434: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 761 op/s 2023-10-09T05:32:43.531 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:43 smithi194 bash[92666]: audit 2023-10-09T05:32:43.244454+0000 mon.a (mon.0) 1677 : audit 0 from='client.? 172.21.15.133:0/2242824392' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:32:43.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:43 smithi133 bash[102858]: cluster 2023-10-09T05:32:42.312316+0000 mgr.y (mgr.25277) 1338 : cluster 0 pgmap v1434: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 761 op/s 2023-10-09T05:32:43.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:43 smithi133 bash[102858]: audit 2023-10-09T05:32:43.244454+0000 mon.a (mon.0) 1677 : audit 0 from='client.? 172.21.15.133:0/2242824392' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:32:43.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:43 smithi133 bash[105589]: cluster 2023-10-09T05:32:42.312316+0000 mgr.y (mgr.25277) 1338 : cluster 0 pgmap v1434: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 761 op/s 2023-10-09T05:32:43.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:43 smithi133 bash[105589]: audit 2023-10-09T05:32:43.244454+0000 mon.a (mon.0) 1677 : audit 0 from='client.? 172.21.15.133:0/2242824392' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:32:43.552 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:32:43.553 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:43.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:43.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:43.619 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:32:43.657 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:43.665 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:32:43.708 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:43.709 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:32:43.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:43.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:43.716 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:32:43.716 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:32:43.716 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:32:43.716 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:32:43.716 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:32:43.717 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:32:43.717 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:32:43.717 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:32:43.812 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:43.858 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:32:43.859 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:32:43.867 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:43.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:43.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:43.973 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:44.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:44.019 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:44.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:44.120 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:44.149 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:44.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:44.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:44.254 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:44.263 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:44 smithi133 bash[102858]: debug 2023-10-09T05:32:44.236+0000 7f5eb4d1d700 -1 mon.a@0(leader).osd e299 definitely_dead 0 2023-10-09T05:32:44.290 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:44.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:44.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:44.386 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:44 smithi133 bash[102858]: audit 2023-10-09T05:32:43.484098+0000 mon.a (mon.0) 1678 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:32:44.386 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:44 smithi133 bash[102858]: audit 2023-10-09T05:32:44.019931+0000 mon.c (mon.2) 125 : audit 1 from='client.? 172.21.15.133:0/1159680049' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:32:44.386 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:44 smithi133 bash[102858]: audit 2023-10-09T05:32:44.020856+0000 mon.a (mon.0) 1679 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:32:44.386 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:44 smithi133 bash[102858]: audit 2023-10-09T05:32:44.240285+0000 mon.a (mon.0) 1680 : audit 0 from='client.? 172.21.15.133:0/1087573161' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T05:32:44.386 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:32:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:32:44] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.43.0" 2023-10-09T05:32:44.396 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:44.421 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:44.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:44.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:44.527 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:44.551 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:44.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:44.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:44.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:44 smithi194 bash[92666]: audit 2023-10-09T05:32:43.484098+0000 mon.a (mon.0) 1678 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:32:44.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:44 smithi194 bash[92666]: audit 2023-10-09T05:32:44.019931+0000 mon.c (mon.2) 125 : audit 1 from='client.? 172.21.15.133:0/1159680049' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:32:44.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:44 smithi194 bash[92666]: audit 2023-10-09T05:32:44.020856+0000 mon.a (mon.0) 1679 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:32:44.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:44 smithi194 bash[92666]: audit 2023-10-09T05:32:44.240285+0000 mon.a (mon.0) 1680 : audit 0 from='client.? 172.21.15.133:0/1087573161' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T05:32:44.654 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:44.663 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:44 smithi133 bash[105589]: audit 2023-10-09T05:32:43.484098+0000 mon.a (mon.0) 1678 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:32:44.663 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:44 smithi133 bash[105589]: audit 2023-10-09T05:32:44.019931+0000 mon.c (mon.2) 125 : audit 1 from='client.? 172.21.15.133:0/1159680049' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:32:44.663 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:44 smithi133 bash[105589]: audit 2023-10-09T05:32:44.020856+0000 mon.a (mon.0) 1679 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:32:44.663 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:44 smithi133 bash[105589]: audit 2023-10-09T05:32:44.240285+0000 mon.a (mon.0) 1680 : audit 0 from='client.? 172.21.15.133:0/1087573161' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T05:32:44.682 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:44.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:44.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:44.786 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:44.811 INFO:teuthology.orchestra.run.smithi133.stderr:marked in osd.5. 2023-10-09T05:32:44.813 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is set 2023-10-09T05:32:44.833 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:44.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:44.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:44.936 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:44.961 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:44.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:44.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:45.064 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:45.087 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:45.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:45.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:45.190 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:45.237 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:45.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:45.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:45.341 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:45.370 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:45.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:45.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:45.474 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:45.494 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:45.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:45.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:45.596 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:45.608 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:45 smithi194 bash[92666]: cluster 2023-10-09T05:32:44.312759+0000 mgr.y (mgr.25277) 1339 : cluster 0 pgmap v1435: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 620 op/s 2023-10-09T05:32:45.608 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:45 smithi194 bash[92666]: cluster 2023-10-09T05:32:44.789531+0000 mon.a (mon.0) 1681 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:32:45.608 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:45 smithi194 bash[92666]: cluster 2023-10-09T05:32:44.800166+0000 osd.4 (osd.4) 271 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T05:32:45.608 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:45 smithi194 bash[92666]: cluster 2023-10-09T05:32:44.802712+0000 osd.6 (osd.6) 258 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T05:32:45.608 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:45 smithi194 bash[92666]: cluster 2023-10-09T05:32:44.802716+0000 osd.5 (osd.5) 274 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T05:32:45.608 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:45 smithi194 bash[92666]: audit 2023-10-09T05:32:44.802759+0000 mon.a (mon.0) 1682 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:32:45.608 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:45 smithi194 bash[92666]: audit 2023-10-09T05:32:44.802907+0000 mon.a (mon.0) 1683 : audit 1 from='client.? 172.21.15.133:0/1087573161' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-09T05:32:45.608 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:45 smithi194 bash[92666]: cluster 2023-10-09T05:32:44.803098+0000 mon.a (mon.0) 1684 : cluster 0 osdmap e300: 8 total, 8 up, 7 in 2023-10-09T05:32:45.609 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:45 smithi194 bash[92666]: cluster 2023-10-09T05:32:44.804499+0000 osd.5 (osd.5) 275 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T05:32:45.609 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:45 smithi194 bash[92666]: cluster 2023-10-09T05:32:44.809265+0000 osd.7 (osd.7) 257 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T05:32:45.630 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:45.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:45.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:45.734 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:45.746 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:45 smithi133 bash[102858]: cluster 2023-10-09T05:32:44.312759+0000 mgr.y (mgr.25277) 1339 : cluster 0 pgmap v1435: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 620 op/s 2023-10-09T05:32:45.746 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:45 smithi133 bash[102858]: cluster 2023-10-09T05:32:44.789531+0000 mon.a (mon.0) 1681 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:32:45.747 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:45 smithi133 bash[102858]: cluster 2023-10-09T05:32:44.800166+0000 osd.4 (osd.4) 271 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T05:32:45.747 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:45 smithi133 bash[102858]: cluster 2023-10-09T05:32:44.802712+0000 osd.6 (osd.6) 258 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T05:32:45.747 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:45 smithi133 bash[102858]: cluster 2023-10-09T05:32:44.802716+0000 osd.5 (osd.5) 274 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T05:32:45.747 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:45 smithi133 bash[102858]: audit 2023-10-09T05:32:44.802759+0000 mon.a (mon.0) 1682 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:32:45.747 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:45 smithi133 bash[102858]: audit 2023-10-09T05:32:44.802907+0000 mon.a (mon.0) 1683 : audit 1 from='client.? 172.21.15.133:0/1087573161' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-09T05:32:45.747 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:45 smithi133 bash[102858]: cluster 2023-10-09T05:32:44.803098+0000 mon.a (mon.0) 1684 : cluster 0 osdmap e300: 8 total, 8 up, 7 in 2023-10-09T05:32:45.747 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:45 smithi133 bash[102858]: cluster 2023-10-09T05:32:44.804499+0000 osd.5 (osd.5) 275 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T05:32:45.747 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:45 smithi133 bash[102858]: cluster 2023-10-09T05:32:44.809265+0000 osd.7 (osd.7) 257 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T05:32:45.747 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:45 smithi133 bash[105589]: cluster 2023-10-09T05:32:44.312759+0000 mgr.y (mgr.25277) 1339 : cluster 0 pgmap v1435: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 620 op/s 2023-10-09T05:32:45.748 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:45 smithi133 bash[105589]: cluster 2023-10-09T05:32:44.789531+0000 mon.a (mon.0) 1681 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:32:45.748 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:45 smithi133 bash[105589]: cluster 2023-10-09T05:32:44.800166+0000 osd.4 (osd.4) 271 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T05:32:45.748 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:45 smithi133 bash[105589]: cluster 2023-10-09T05:32:44.802712+0000 osd.6 (osd.6) 258 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T05:32:45.748 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:45 smithi133 bash[105589]: cluster 2023-10-09T05:32:44.802716+0000 osd.5 (osd.5) 274 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T05:32:45.748 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:45 smithi133 bash[105589]: audit 2023-10-09T05:32:44.802759+0000 mon.a (mon.0) 1682 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:32:45.748 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:45 smithi133 bash[105589]: audit 2023-10-09T05:32:44.802907+0000 mon.a (mon.0) 1683 : audit 1 from='client.? 172.21.15.133:0/1087573161' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-09T05:32:45.748 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:45 smithi133 bash[105589]: cluster 2023-10-09T05:32:44.803098+0000 mon.a (mon.0) 1684 : cluster 0 osdmap e300: 8 total, 8 up, 7 in 2023-10-09T05:32:45.748 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:45 smithi133 bash[105589]: cluster 2023-10-09T05:32:44.804499+0000 osd.5 (osd.5) 275 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T05:32:45.748 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:45 smithi133 bash[105589]: cluster 2023-10-09T05:32:44.809265+0000 osd.7 (osd.7) 257 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T05:32:45.761 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:45.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:45.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:45.864 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:45.889 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:45.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:45.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:45.992 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:46.019 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:46.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:46.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:46.122 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:46.154 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:46.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:46.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:46.258 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:46.276 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:46.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:46.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:46.379 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:46.411 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:46.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:46.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:46.514 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:46.551 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:46.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:46.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:46.655 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:46.689 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:46.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:46.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:46.792 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:46.842 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:46.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:46.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:46.948 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:46.966 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:46.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:46.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:47.069 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:47.078 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:46 smithi194 bash[92666]: cluster 2023-10-09T05:32:45.806939+0000 osd.6 (osd.6) 259 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T05:32:47.079 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:46 smithi194 bash[92666]: cluster 2023-10-09T05:32:45.807555+0000 osd.5 (osd.5) 276 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T05:32:47.079 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:46 smithi194 bash[92666]: cluster 2023-10-09T05:32:45.808096+0000 osd.7 (osd.7) 258 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T05:32:47.079 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:46 smithi194 bash[92666]: cluster 2023-10-09T05:32:45.809708+0000 osd.5 (osd.5) 277 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T05:32:47.079 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:46 smithi194 bash[92666]: cluster 2023-10-09T05:32:45.811030+0000 osd.7 (osd.7) 259 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T05:32:47.079 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:46 smithi194 bash[92666]: cluster 2023-10-09T05:32:45.811322+0000 mon.a (mon.0) 1685 : cluster 0 osdmap e301: 8 total, 8 up, 7 in 2023-10-09T05:32:47.079 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:46 smithi194 bash[92666]: cluster 2023-10-09T05:32:45.811986+0000 osd.6 (osd.6) 260 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T05:32:47.079 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:46 smithi194 bash[92666]: cluster 2023-10-09T05:32:45.812601+0000 osd.7 (osd.7) 260 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T05:32:47.080 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:46 smithi194 bash[92666]: cluster 2023-10-09T05:32:45.814312+0000 osd.6 (osd.6) 261 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T05:32:47.080 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:46 smithi194 bash[92666]: cluster 2023-10-09T05:32:46.267180+0000 osd.4 (osd.4) 272 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T05:32:47.080 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:46 smithi194 bash[92666]: cluster 2023-10-09T05:32:46.313431+0000 mgr.y (mgr.25277) 1340 : cluster 0 pgmap v1438: 89 pgs: 1 remapped+peering, 4 peering, 84 active+clean; 4.4 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 808 op/s 2023-10-09T05:32:47.097 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:47.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:47.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:47.200 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:47.227 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:47.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:47.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:47.330 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:47.349 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:46 smithi133 bash[102858]: cluster 2023-10-09T05:32:45.806939+0000 osd.6 (osd.6) 259 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T05:32:47.349 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:46 smithi133 bash[102858]: cluster 2023-10-09T05:32:45.807555+0000 osd.5 (osd.5) 276 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T05:32:47.349 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:46 smithi133 bash[102858]: cluster 2023-10-09T05:32:45.808096+0000 osd.7 (osd.7) 258 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T05:32:47.349 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:46 smithi133 bash[102858]: cluster 2023-10-09T05:32:45.809708+0000 osd.5 (osd.5) 277 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T05:32:47.349 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:46 smithi133 bash[102858]: cluster 2023-10-09T05:32:45.811030+0000 osd.7 (osd.7) 259 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T05:32:47.349 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:46 smithi133 bash[102858]: cluster 2023-10-09T05:32:45.811322+0000 mon.a (mon.0) 1685 : cluster 0 osdmap e301: 8 total, 8 up, 7 in 2023-10-09T05:32:47.349 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:46 smithi133 bash[102858]: cluster 2023-10-09T05:32:45.811986+0000 osd.6 (osd.6) 260 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T05:32:47.349 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:46 smithi133 bash[102858]: cluster 2023-10-09T05:32:45.812601+0000 osd.7 (osd.7) 260 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T05:32:47.349 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:46 smithi133 bash[102858]: cluster 2023-10-09T05:32:45.814312+0000 osd.6 (osd.6) 261 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T05:32:47.350 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:46 smithi133 bash[102858]: cluster 2023-10-09T05:32:46.267180+0000 osd.4 (osd.4) 272 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T05:32:47.350 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:46 smithi133 bash[102858]: cluster 2023-10-09T05:32:46.313431+0000 mgr.y (mgr.25277) 1340 : cluster 0 pgmap v1438: 89 pgs: 1 remapped+peering, 4 peering, 84 active+clean; 4.4 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 808 op/s 2023-10-09T05:32:47.350 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:46 smithi133 bash[105589]: cluster 2023-10-09T05:32:45.806939+0000 osd.6 (osd.6) 259 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T05:32:47.350 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:46 smithi133 bash[105589]: cluster 2023-10-09T05:32:45.807555+0000 osd.5 (osd.5) 276 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T05:32:47.350 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:46 smithi133 bash[105589]: cluster 2023-10-09T05:32:45.808096+0000 osd.7 (osd.7) 258 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T05:32:47.350 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:46 smithi133 bash[105589]: cluster 2023-10-09T05:32:45.809708+0000 osd.5 (osd.5) 277 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T05:32:47.350 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:46 smithi133 bash[105589]: cluster 2023-10-09T05:32:45.811030+0000 osd.7 (osd.7) 259 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T05:32:47.350 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:46 smithi133 bash[105589]: cluster 2023-10-09T05:32:45.811322+0000 mon.a (mon.0) 1685 : cluster 0 osdmap e301: 8 total, 8 up, 7 in 2023-10-09T05:32:47.350 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:46 smithi133 bash[105589]: cluster 2023-10-09T05:32:45.811986+0000 osd.6 (osd.6) 260 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T05:32:47.350 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:46 smithi133 bash[105589]: cluster 2023-10-09T05:32:45.812601+0000 osd.7 (osd.7) 260 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T05:32:47.351 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:46 smithi133 bash[105589]: cluster 2023-10-09T05:32:45.814312+0000 osd.6 (osd.6) 261 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T05:32:47.351 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:46 smithi133 bash[105589]: cluster 2023-10-09T05:32:46.267180+0000 osd.4 (osd.4) 272 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T05:32:47.351 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:46 smithi133 bash[105589]: cluster 2023-10-09T05:32:46.313431+0000 mgr.y (mgr.25277) 1340 : cluster 0 pgmap v1438: 89 pgs: 1 remapped+peering, 4 peering, 84 active+clean; 4.4 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 808 op/s 2023-10-09T05:32:47.365 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:47.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:47.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:47.471 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:47.503 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:47.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:47.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:47.610 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:47.636 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:47.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:47.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:47.742 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:47.761 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:47.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:47.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:47.814 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T05:32:47.867 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:47.896 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:47.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:47.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:48.004 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:48.032 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:48.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:48.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:48.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:47 smithi133 bash[105589]: cluster 2023-10-09T05:32:46.807159+0000 osd.1 (osd.1) 1 : cluster 0 13.c starting backfill to osd.5 from (0'0,0'0] MAX to 299'7603 2023-10-09T05:32:48.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:47 smithi133 bash[105589]: cluster 2023-10-09T05:32:46.808150+0000 osd.7 (osd.7) 261 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T05:32:48.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:47 smithi133 bash[105589]: cluster 2023-10-09T05:32:46.809610+0000 osd.6 (osd.6) 262 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T05:32:48.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:47 smithi133 bash[105589]: cluster 2023-10-09T05:32:46.811832+0000 mon.a (mon.0) 1686 : cluster 0 osdmap e302: 8 total, 8 up, 7 in 2023-10-09T05:32:48.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:47 smithi133 bash[105589]: cluster 2023-10-09T05:32:46.812599+0000 osd.7 (osd.7) 262 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T05:32:48.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:47 smithi133 bash[105589]: cluster 2023-10-09T05:32:46.815304+0000 osd.0 (osd.0) 5 : cluster 0 13.9 starting backfill to osd.5 from (0'0,0'0] MAX to 299'7779 2023-10-09T05:32:48.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:47 smithi133 bash[105589]: cluster 2023-10-09T05:32:46.815349+0000 osd.0 (osd.0) 6 : cluster 0 13.b starting backfill to osd.5 from (0'0,0'0] MAX to 299'7695 2023-10-09T05:32:48.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:47 smithi133 bash[105589]: cluster 2023-10-09T05:32:46.817101+0000 osd.7 (osd.7) 263 : cluster 0 13.e starting backfill to osd.5 from (0'0,0'0] MAX to 299'7813 2023-10-09T05:32:48.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:47 smithi133 bash[105589]: cluster 2023-10-09T05:32:46.818874+0000 osd.0 (osd.0) 7 : cluster 0 13.5 starting backfill to osd.5 from (0'0,0'0] MAX to 299'7648 2023-10-09T05:32:48.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:47 smithi133 bash[105589]: cluster 2023-10-09T05:32:46.818884+0000 osd.0 (osd.0) 8 : cluster 0 13.3 starting backfill to osd.5 from (0'0,0'0] MAX to 299'7692 2023-10-09T05:32:48.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:47 smithi133 bash[105589]: cluster 2023-10-09T05:32:46.819251+0000 osd.7 (osd.7) 264 : cluster 0 13.7 starting backfill to osd.5 from (0'0,0'0] MAX to 299'7735 2023-10-09T05:32:48.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:47 smithi133 bash[105589]: cluster 2023-10-09T05:32:46.836459+0000 osd.3 (osd.3) 1 : cluster 0 13.2 starting backfill to osd.5 from (0'0,0'0] MAX to 299'7630 2023-10-09T05:32:48.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:47 smithi133 bash[105589]: cluster 2023-10-09T05:32:46.846741+0000 osd.3 (osd.3) 2 : cluster 0 13.6 starting backfill to osd.5 from (0'0,0'0] MAX to 299'7875 2023-10-09T05:32:48.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:47 smithi133 bash[105589]: cluster 2023-10-09T05:32:46.895636+0000 osd.5 (osd.5) 278 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T05:32:48.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:47 smithi133 bash[102858]: cluster 2023-10-09T05:32:46.807159+0000 osd.1 (osd.1) 1 : cluster 0 13.c starting backfill to osd.5 from (0'0,0'0] MAX to 299'7603 2023-10-09T05:32:48.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:47 smithi133 bash[102858]: cluster 2023-10-09T05:32:46.808150+0000 osd.7 (osd.7) 261 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T05:32:48.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:47 smithi133 bash[102858]: cluster 2023-10-09T05:32:46.809610+0000 osd.6 (osd.6) 262 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T05:32:48.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:47 smithi133 bash[102858]: cluster 2023-10-09T05:32:46.811832+0000 mon.a (mon.0) 1686 : cluster 0 osdmap e302: 8 total, 8 up, 7 in 2023-10-09T05:32:48.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:47 smithi133 bash[102858]: cluster 2023-10-09T05:32:46.812599+0000 osd.7 (osd.7) 262 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T05:32:48.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:47 smithi133 bash[102858]: cluster 2023-10-09T05:32:46.815304+0000 osd.0 (osd.0) 5 : cluster 0 13.9 starting backfill to osd.5 from (0'0,0'0] MAX to 299'7779 2023-10-09T05:32:48.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:47 smithi133 bash[102858]: cluster 2023-10-09T05:32:46.815349+0000 osd.0 (osd.0) 6 : cluster 0 13.b starting backfill to osd.5 from (0'0,0'0] MAX to 299'7695 2023-10-09T05:32:48.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:47 smithi133 bash[102858]: cluster 2023-10-09T05:32:46.817101+0000 osd.7 (osd.7) 263 : cluster 0 13.e starting backfill to osd.5 from (0'0,0'0] MAX to 299'7813 2023-10-09T05:32:48.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:47 smithi133 bash[102858]: cluster 2023-10-09T05:32:46.818874+0000 osd.0 (osd.0) 7 : cluster 0 13.5 starting backfill to osd.5 from (0'0,0'0] MAX to 299'7648 2023-10-09T05:32:48.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:47 smithi133 bash[102858]: cluster 2023-10-09T05:32:46.818884+0000 osd.0 (osd.0) 8 : cluster 0 13.3 starting backfill to osd.5 from (0'0,0'0] MAX to 299'7692 2023-10-09T05:32:48.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:47 smithi133 bash[102858]: cluster 2023-10-09T05:32:46.819251+0000 osd.7 (osd.7) 264 : cluster 0 13.7 starting backfill to osd.5 from (0'0,0'0] MAX to 299'7735 2023-10-09T05:32:48.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:47 smithi133 bash[102858]: cluster 2023-10-09T05:32:46.836459+0000 osd.3 (osd.3) 1 : cluster 0 13.2 starting backfill to osd.5 from (0'0,0'0] MAX to 299'7630 2023-10-09T05:32:48.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:47 smithi133 bash[102858]: cluster 2023-10-09T05:32:46.846741+0000 osd.3 (osd.3) 2 : cluster 0 13.6 starting backfill to osd.5 from (0'0,0'0] MAX to 299'7875 2023-10-09T05:32:48.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:47 smithi133 bash[102858]: cluster 2023-10-09T05:32:46.895636+0000 osd.5 (osd.5) 278 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T05:32:48.135 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:48.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:47 smithi194 bash[92666]: cluster 2023-10-09T05:32:46.807159+0000 osd.1 (osd.1) 1 : cluster 0 13.c starting backfill to osd.5 from (0'0,0'0] MAX to 299'7603 2023-10-09T05:32:48.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:47 smithi194 bash[92666]: cluster 2023-10-09T05:32:46.808150+0000 osd.7 (osd.7) 261 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T05:32:48.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:47 smithi194 bash[92666]: cluster 2023-10-09T05:32:46.809610+0000 osd.6 (osd.6) 262 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T05:32:48.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:47 smithi194 bash[92666]: cluster 2023-10-09T05:32:46.811832+0000 mon.a (mon.0) 1686 : cluster 0 osdmap e302: 8 total, 8 up, 7 in 2023-10-09T05:32:48.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:47 smithi194 bash[92666]: cluster 2023-10-09T05:32:46.812599+0000 osd.7 (osd.7) 262 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T05:32:48.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:47 smithi194 bash[92666]: cluster 2023-10-09T05:32:46.815304+0000 osd.0 (osd.0) 5 : cluster 0 13.9 starting backfill to osd.5 from (0'0,0'0] MAX to 299'7779 2023-10-09T05:32:48.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:47 smithi194 bash[92666]: cluster 2023-10-09T05:32:46.815349+0000 osd.0 (osd.0) 6 : cluster 0 13.b starting backfill to osd.5 from (0'0,0'0] MAX to 299'7695 2023-10-09T05:32:48.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:47 smithi194 bash[92666]: cluster 2023-10-09T05:32:46.817101+0000 osd.7 (osd.7) 263 : cluster 0 13.e starting backfill to osd.5 from (0'0,0'0] MAX to 299'7813 2023-10-09T05:32:48.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:47 smithi194 bash[92666]: cluster 2023-10-09T05:32:46.818874+0000 osd.0 (osd.0) 7 : cluster 0 13.5 starting backfill to osd.5 from (0'0,0'0] MAX to 299'7648 2023-10-09T05:32:48.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:47 smithi194 bash[92666]: cluster 2023-10-09T05:32:46.818884+0000 osd.0 (osd.0) 8 : cluster 0 13.3 starting backfill to osd.5 from (0'0,0'0] MAX to 299'7692 2023-10-09T05:32:48.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:47 smithi194 bash[92666]: cluster 2023-10-09T05:32:46.819251+0000 osd.7 (osd.7) 264 : cluster 0 13.7 starting backfill to osd.5 from (0'0,0'0] MAX to 299'7735 2023-10-09T05:32:48.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:47 smithi194 bash[92666]: cluster 2023-10-09T05:32:46.836459+0000 osd.3 (osd.3) 1 : cluster 0 13.2 starting backfill to osd.5 from (0'0,0'0] MAX to 299'7630 2023-10-09T05:32:48.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:47 smithi194 bash[92666]: cluster 2023-10-09T05:32:46.846741+0000 osd.3 (osd.3) 2 : cluster 0 13.6 starting backfill to osd.5 from (0'0,0'0] MAX to 299'7875 2023-10-09T05:32:48.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:47 smithi194 bash[92666]: cluster 2023-10-09T05:32:46.895636+0000 osd.5 (osd.5) 278 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T05:32:48.161 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:48.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:48.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:48.264 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:48.297 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:48.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:48.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:48.406 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:48.446 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:48.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:48.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:48.552 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:48.577 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:48.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:48.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:48.680 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:48.712 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:48.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:48.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:48.818 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:48.844 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:48.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:48.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:48.949 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:48.975 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:48.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:48.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:49.002 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:48 smithi133 bash[102858]: cluster 2023-10-09T05:32:48.316033+0000 mgr.y (mgr.25277) 1341 : cluster 0 pgmap v1440: 89 pgs: 1 remapped+peering, 4 peering, 84 active+clean; 4.4 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 436 op/s 2023-10-09T05:32:49.002 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:48 smithi133 bash[105589]: cluster 2023-10-09T05:32:48.316033+0000 mgr.y (mgr.25277) 1341 : cluster 0 pgmap v1440: 89 pgs: 1 remapped+peering, 4 peering, 84 active+clean; 4.4 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 436 op/s 2023-10-09T05:32:49.082 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:49.106 INFO:tasks.thrashosds.thrasher:Added osd 5 2023-10-09T05:32:49.109 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:49.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:49.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:49.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:48 smithi194 bash[92666]: cluster 2023-10-09T05:32:48.316033+0000 mgr.y (mgr.25277) 1341 : cluster 0 pgmap v1440: 89 pgs: 1 remapped+peering, 4 peering, 84 active+clean; 4.4 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 436 op/s 2023-10-09T05:32:49.216 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:49.249 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:49.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:49.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:49.356 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:49.390 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:49.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:49.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:49.497 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:49.522 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:49.526 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:32:49.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:49.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:49.629 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:49.668 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:49.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:49.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:49.773 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:49.812 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:49.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:49.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:49.919 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:49.955 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:49.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:49.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:50.066 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:50.094 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:50.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:50.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:50.197 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:50.235 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:50.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:50.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:50.339 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:50.360 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:50.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:50.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:50.467 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:50.498 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:50.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:50.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:50.602 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:50.630 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:50.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:50.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:50.735 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:50.777 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:50.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:50.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:50.884 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:50.917 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:50.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:50.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:51.004 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:32:51.025 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:51.075 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:51.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:51.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:51.184 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:51.212 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:51.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:51.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:51.316 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:51.347 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:51.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:51.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:51.455 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:51.501 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:51.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:51.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:51.606 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:51.615 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:51 smithi133 bash[102858]: cluster 2023-10-09T05:32:50.316692+0000 mgr.y (mgr.25277) 1342 : cluster 0 pgmap v1441: 89 pgs: 6 active+remapped+backfill_wait, 1 remapped+peering, 4 peering, 78 active+clean; 4.4 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 26417/214935 objects misplaced (12.291%); 21 KiB/s, 1 objects/s recovering 2023-10-09T05:32:51.615 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:51 smithi133 bash[105589]: cluster 2023-10-09T05:32:50.316692+0000 mgr.y (mgr.25277) 1342 : cluster 0 pgmap v1441: 89 pgs: 6 active+remapped+backfill_wait, 1 remapped+peering, 4 peering, 78 active+clean; 4.4 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 26417/214935 objects misplaced (12.291%); 21 KiB/s, 1 objects/s recovering 2023-10-09T05:32:51.633 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:51.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:51 smithi194 bash[92666]: cluster 2023-10-09T05:32:50.316692+0000 mgr.y (mgr.25277) 1342 : cluster 0 pgmap v1441: 89 pgs: 6 active+remapped+backfill_wait, 1 remapped+peering, 4 peering, 78 active+clean; 4.4 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 26417/214935 objects misplaced (12.291%); 21 KiB/s, 1 objects/s recovering 2023-10-09T05:32:51.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:51.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:51.740 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:51.773 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:51.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:51.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:51.880 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:51.905 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:51.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:51.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:52.008 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:52.034 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:52.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:52.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:52.143 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:52.165 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:52.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:52.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:52.270 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:52.299 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:52.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:52.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:52.402 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:52.443 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:52.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:52.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:52.550 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:52.575 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:52.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:52.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:52.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:52 smithi194 bash[92666]: audit 2023-10-09T05:32:51.973073+0000 mon.a (mon.0) 1687 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:32:52.637 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:52.662 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:52 smithi133 bash[105589]: audit 2023-10-09T05:32:51.973073+0000 mon.a (mon.0) 1687 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:32:52.662 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:52 smithi133 bash[102858]: audit 2023-10-09T05:32:51.973073+0000 mon.a (mon.0) 1687 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:32:52.680 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:52.709 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:52.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:52.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:52.812 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:52.845 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:52.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:52.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:52.955 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:52.976 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:52.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:52.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:53.080 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:53.113 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:53.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:53.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:53.218 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:53.261 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:53.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:53.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:53.366 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:53.395 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:53.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:53.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:53.499 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:53.508 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:53 smithi133 bash[105589]: cluster 2023-10-09T05:32:52.317389+0000 mgr.y (mgr.25277) 1343 : cluster 0 pgmap v1442: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 4.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 422 op/s; 122/211263 objects degraded (0.058%); 43809/211263 objects misplaced (20.737%); 596 KiB/s, 10 objects/s recovering 2023-10-09T05:32:53.509 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:53 smithi133 bash[105589]: cluster 2023-10-09T05:32:52.371106+0000 mon.a (mon.0) 1688 : cluster 3 Health check failed: Degraded data redundancy: 122/211263 objects degraded (0.058%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:32:53.509 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:53 smithi133 bash[102858]: cluster 2023-10-09T05:32:52.317389+0000 mgr.y (mgr.25277) 1343 : cluster 0 pgmap v1442: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 4.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 422 op/s; 122/211263 objects degraded (0.058%); 43809/211263 objects misplaced (20.737%); 596 KiB/s, 10 objects/s recovering 2023-10-09T05:32:53.509 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:53 smithi133 bash[102858]: cluster 2023-10-09T05:32:52.371106+0000 mon.a (mon.0) 1688 : cluster 3 Health check failed: Degraded data redundancy: 122/211263 objects degraded (0.058%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:32:53.532 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:53.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:53.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:53.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:53 smithi194 bash[92666]: cluster 2023-10-09T05:32:52.317389+0000 mgr.y (mgr.25277) 1343 : cluster 0 pgmap v1442: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 4.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 422 op/s; 122/211263 objects degraded (0.058%); 43809/211263 objects misplaced (20.737%); 596 KiB/s, 10 objects/s recovering 2023-10-09T05:32:53.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:53 smithi194 bash[92666]: cluster 2023-10-09T05:32:52.371106+0000 mon.a (mon.0) 1688 : cluster 3 Health check failed: Degraded data redundancy: 122/211263 objects degraded (0.058%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:32:53.640 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:53.669 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:53.676 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:53.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:53.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:53.783 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:53.822 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:53.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:53.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:53.926 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:53.952 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:53.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:53.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:54.056 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:54.085 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:54.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:54.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:54.107 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 6, 1, 7, 5] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:32:54.107 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:32:54.107 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-10-09T05:32:54.107 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 0.05 2023-10-09T05:32:54.191 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:54.241 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:54.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:54.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:54.347 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:54.388 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:54.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:54.391 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:32:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:32:54] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.43.0" 2023-10-09T05:32:54.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:54.493 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:54.528 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:54.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:54.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:54.634 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:54.689 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:54.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:54.696 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:54.796 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:55.027 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:55.030 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:54 smithi133 bash[102858]: cluster 2023-10-09T05:32:54.321684+0000 mgr.y (mgr.25277) 1344 : cluster 0 pgmap v1443: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 4.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 329 op/s; 122/211263 objects degraded (0.058%); 43809/211263 objects misplaced (20.737%); 526 KiB/s, 9 objects/s recovering 2023-10-09T05:32:55.031 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:54 smithi133 bash[105589]: cluster 2023-10-09T05:32:54.321684+0000 mgr.y (mgr.25277) 1344 : cluster 0 pgmap v1443: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 4.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 329 op/s; 122/211263 objects degraded (0.058%); 43809/211263 objects misplaced (20.737%); 526 KiB/s, 9 objects/s recovering 2023-10-09T05:32:55.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:55.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:55.133 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:55.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:54 smithi194 bash[92666]: cluster 2023-10-09T05:32:54.321684+0000 mgr.y (mgr.25277) 1344 : cluster 0 pgmap v1443: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 4.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 329 op/s; 122/211263 objects degraded (0.058%); 43809/211263 objects misplaced (20.737%); 526 KiB/s, 9 objects/s recovering 2023-10-09T05:32:55.156 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:55.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:55.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:55.260 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:55.287 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:55.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:55.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:55.395 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:55.431 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:55.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:55.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:55.537 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:55.571 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:55.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:55.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:55.676 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:55.699 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:55.716 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:55.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:55.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:55.819 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:55.843 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:55.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:55.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:55.949 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:55.978 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:55.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:55.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:56.081 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:56.153 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:56.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:56.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:56.261 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:56.294 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:56.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:56.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:56.403 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:56.542 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:56.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:56.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:56.645 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:56.675 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:56.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:56.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:56.780 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:56.809 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:56.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:56.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:56.914 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:56.945 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:56.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:56.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:57.050 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:57.083 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:57.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:57.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:57.187 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:57.217 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:57.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:57.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:57.320 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:57.343 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:57.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:57.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:57.446 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:57.479 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:57.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:57.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:57.588 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:57.596 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:57 smithi194 bash[92666]: cluster 2023-10-09T05:32:56.322518+0000 mgr.y (mgr.25277) 1345 : cluster 0 pgmap v1444: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 4.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 273/201615 objects degraded (0.135%); 41795/201615 objects misplaced (20.730%); 851 KiB/s, 14 objects/s recovering 2023-10-09T05:32:57.634 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:57.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:57.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:57.737 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:57.769 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:57.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:57.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:57.797 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:57 smithi133 bash[105589]: cluster 2023-10-09T05:32:56.322518+0000 mgr.y (mgr.25277) 1345 : cluster 0 pgmap v1444: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 4.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 273/201615 objects degraded (0.135%); 41795/201615 objects misplaced (20.730%); 851 KiB/s, 14 objects/s recovering 2023-10-09T05:32:57.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:57 smithi133 bash[102858]: cluster 2023-10-09T05:32:56.322518+0000 mgr.y (mgr.25277) 1345 : cluster 0 pgmap v1444: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 4.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 273/201615 objects degraded (0.135%); 41795/201615 objects misplaced (20.730%); 851 KiB/s, 14 objects/s recovering 2023-10-09T05:32:57.873 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:57.920 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:57.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:57.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:58.024 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:58.077 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:58.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:58.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:58.184 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:58.215 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:58.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:58.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:58.333 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:58.364 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:58.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:58.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:58.470 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:58.516 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:58.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:58.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:58.620 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:58.675 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:58.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:58.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:58.749 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:58.781 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:58.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:58.810 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:58.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:58.911 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:58.937 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:58.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:58.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:59.044 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:59.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:59.077 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:59.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:59.178 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:59.200 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:59.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:59.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:59.308 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:59.333 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:59.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:59.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:59.437 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:59.464 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:59.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:59.466 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:59 smithi133 bash[102858]: audit 2023-10-09T05:32:58.136649+0000 mon.a (mon.0) 1689 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:32:59.467 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:59 smithi133 bash[102858]: audit 2023-10-09T05:32:58.147202+0000 mon.a (mon.0) 1690 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:32:59.467 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:59 smithi133 bash[102858]: cluster 2023-10-09T05:32:58.323014+0000 mgr.y (mgr.25277) 1346 : cluster 0 pgmap v1445: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 4.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 493 op/s; 273/201615 objects degraded (0.135%); 41795/201615 objects misplaced (20.730%); 739 KiB/s, 12 objects/s recovering 2023-10-09T05:32:59.467 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:59 smithi133 bash[102858]: audit 2023-10-09T05:32:58.487956+0000 mon.a (mon.0) 1691 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:32:59.467 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:59 smithi133 bash[102858]: audit 2023-10-09T05:32:58.493440+0000 mon.a (mon.0) 1692 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:32:59.467 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:59 smithi133 bash[102858]: audit 2023-10-09T05:32:58.734911+0000 mon.a (mon.0) 1693 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:32:59.467 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:59 smithi133 bash[102858]: audit 2023-10-09T05:32:58.752365+0000 mon.a (mon.0) 1694 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:32:59.467 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:32:59 smithi133 bash[102858]: cluster 2023-10-09T05:32:58.797150+0000 mon.a (mon.0) 1695 : cluster 3 Health check update: Degraded data redundancy: 273/201615 objects degraded (0.135%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:32:59.467 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:59 smithi133 bash[105589]: audit 2023-10-09T05:32:58.136649+0000 mon.a (mon.0) 1689 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:32:59.468 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:59 smithi133 bash[105589]: audit 2023-10-09T05:32:58.147202+0000 mon.a (mon.0) 1690 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:32:59.468 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:59 smithi133 bash[105589]: cluster 2023-10-09T05:32:58.323014+0000 mgr.y (mgr.25277) 1346 : cluster 0 pgmap v1445: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 4.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 493 op/s; 273/201615 objects degraded (0.135%); 41795/201615 objects misplaced (20.730%); 739 KiB/s, 12 objects/s recovering 2023-10-09T05:32:59.468 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:59 smithi133 bash[105589]: audit 2023-10-09T05:32:58.487956+0000 mon.a (mon.0) 1691 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:32:59.468 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:59 smithi133 bash[105589]: audit 2023-10-09T05:32:58.493440+0000 mon.a (mon.0) 1692 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:32:59.468 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:59 smithi133 bash[105589]: audit 2023-10-09T05:32:58.734911+0000 mon.a (mon.0) 1693 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:32:59.468 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:59 smithi133 bash[105589]: audit 2023-10-09T05:32:58.752365+0000 mon.a (mon.0) 1694 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:32:59.468 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:32:59 smithi133 bash[105589]: cluster 2023-10-09T05:32:58.797150+0000 mon.a (mon.0) 1695 : cluster 3 Health check update: Degraded data redundancy: 273/201615 objects degraded (0.135%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:32:59.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:59.569 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:59.578 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:59 smithi194 bash[92666]: audit 2023-10-09T05:32:58.136649+0000 mon.a (mon.0) 1689 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:32:59.578 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:59 smithi194 bash[92666]: audit 2023-10-09T05:32:58.147202+0000 mon.a (mon.0) 1690 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:32:59.578 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:59 smithi194 bash[92666]: cluster 2023-10-09T05:32:58.323014+0000 mgr.y (mgr.25277) 1346 : cluster 0 pgmap v1445: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 4.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 493 op/s; 273/201615 objects degraded (0.135%); 41795/201615 objects misplaced (20.730%); 739 KiB/s, 12 objects/s recovering 2023-10-09T05:32:59.578 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:59 smithi194 bash[92666]: audit 2023-10-09T05:32:58.487956+0000 mon.a (mon.0) 1691 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:32:59.578 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:59 smithi194 bash[92666]: audit 2023-10-09T05:32:58.493440+0000 mon.a (mon.0) 1692 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:32:59.578 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:59 smithi194 bash[92666]: audit 2023-10-09T05:32:58.734911+0000 mon.a (mon.0) 1693 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:32:59.578 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:59 smithi194 bash[92666]: audit 2023-10-09T05:32:58.752365+0000 mon.a (mon.0) 1694 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:32:59.578 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:32:59 smithi194 bash[92666]: cluster 2023-10-09T05:32:58.797150+0000 mon.a (mon.0) 1695 : cluster 3 Health check update: Degraded data redundancy: 273/201615 objects degraded (0.135%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:32:59.601 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:59.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:59.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:59.705 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:59.744 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:59.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:59.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:59.848 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:59.889 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:59.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:32:59.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:59.994 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:00.024 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:00.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:00.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:00.128 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:00.150 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:00.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:00.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:00.257 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:00.267 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:00 smithi194 bash[92666]: audit 2023-10-09T05:32:59.177922+0000 mon.a (mon.0) 1696 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:33:00.267 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:00 smithi194 bash[92666]: audit 2023-10-09T05:32:59.764647+0000 mon.a (mon.0) 1697 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:33:00.267 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:00 smithi194 bash[92666]: audit 2023-10-09T05:32:59.774886+0000 mon.a (mon.0) 1698 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:33:00.291 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:00.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:00.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:00.400 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:00.424 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:00.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:00.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:00.527 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:00.539 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:00 smithi133 bash[102858]: audit 2023-10-09T05:32:59.177922+0000 mon.a (mon.0) 1696 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:33:00.539 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:00 smithi133 bash[102858]: audit 2023-10-09T05:32:59.764647+0000 mon.a (mon.0) 1697 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:33:00.539 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:00 smithi133 bash[102858]: audit 2023-10-09T05:32:59.774886+0000 mon.a (mon.0) 1698 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:33:00.540 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:00 smithi133 bash[105589]: audit 2023-10-09T05:32:59.177922+0000 mon.a (mon.0) 1696 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:33:00.540 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:00 smithi133 bash[105589]: audit 2023-10-09T05:32:59.764647+0000 mon.a (mon.0) 1697 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:33:00.540 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:00 smithi133 bash[105589]: audit 2023-10-09T05:32:59.774886+0000 mon.a (mon.0) 1698 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:33:00.564 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:00.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:00.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:00.669 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:00.706 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:00.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:00.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:00.811 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:00.836 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:00.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:00.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:00.940 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:00.964 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:00.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:00.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:01.067 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:01.093 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:01.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:01.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:01.193 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:01 smithi133 bash[102858]: cluster 2023-10-09T05:33:00.323814+0000 mgr.y (mgr.25277) 1347 : cluster 0 pgmap v1446: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 4.0 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 273/196716 objects degraded (0.139%); 40821/196716 objects misplaced (20.751%); 709 KiB/s, 11 objects/s recovering 2023-10-09T05:33:01.197 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:01.227 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:01.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:01.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:01.330 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:01.339 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:01 smithi194 bash[92666]: cluster 2023-10-09T05:33:00.323814+0000 mgr.y (mgr.25277) 1347 : cluster 0 pgmap v1446: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 4.0 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 273/196716 objects degraded (0.139%); 40821/196716 objects misplaced (20.751%); 709 KiB/s, 11 objects/s recovering 2023-10-09T05:33:01.358 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:01.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:01.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:01.473 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:01.483 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:01 smithi133 bash[105589]: cluster 2023-10-09T05:33:00.323814+0000 mgr.y (mgr.25277) 1347 : cluster 0 pgmap v1446: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 4.0 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 273/196716 objects degraded (0.139%); 40821/196716 objects misplaced (20.751%); 709 KiB/s, 11 objects/s recovering 2023-10-09T05:33:01.512 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:01.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:01.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:01.617 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:01.643 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:01.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:01.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:01.748 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:01.778 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:01.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:01.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:01.881 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:01.904 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:01.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:01.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:02.008 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:02.047 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:02.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:02.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:02.150 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:02.175 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:02.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:02.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:02.278 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:02.310 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:02.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:02.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:02.415 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:02.445 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:02.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:02.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:02.546 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:02.574 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:02.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:02.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:02.680 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:02.707 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:02.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:02.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:02.812 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:02.836 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:02.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:02.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:02.940 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:02.984 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:02.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:02.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:03.087 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:03.124 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:03.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:03.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:03.228 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:03.261 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:03.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:03.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:03.368 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:03.394 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:03.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:03.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:03.500 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:03.509 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:03 smithi133 bash[105589]: cluster 2023-10-09T05:33:02.324463+0000 mgr.y (mgr.25277) 1348 : cluster 0 pgmap v1447: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 362/193650 objects degraded (0.187%); 40156/193650 objects misplaced (20.736%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T05:33:03.509 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:03 smithi133 bash[102858]: cluster 2023-10-09T05:33:02.324463+0000 mgr.y (mgr.25277) 1348 : cluster 0 pgmap v1447: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 362/193650 objects degraded (0.187%); 40156/193650 objects misplaced (20.736%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T05:33:03.540 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:03.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:03.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:03.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:03 smithi194 bash[92666]: cluster 2023-10-09T05:33:02.324463+0000 mgr.y (mgr.25277) 1348 : cluster 0 pgmap v1447: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 362/193650 objects degraded (0.187%); 40156/193650 objects misplaced (20.736%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T05:33:03.646 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:03.678 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:03.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:03.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:03.783 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:03.815 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:03.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:03.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:03.919 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:03.943 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:03.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:03.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:04.050 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:04.071 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:04.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:04.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:04.175 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:04.208 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:04.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:04.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:04.313 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:04.342 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:04.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:04.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:04.444 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:04.474 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:04.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:04.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:04.485 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:33:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:33:04] "GET /metrics HTTP/1.1" 200 36502 "" "Prometheus/2.43.0" 2023-10-09T05:33:04.486 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:04 smithi133 bash[105589]: cluster 2023-10-09T05:33:03.798250+0000 mon.a (mon.0) 1699 : cluster 3 Health check update: Degraded data redundancy: 362/193650 objects degraded (0.187%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:33:04.486 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:04 smithi133 bash[102858]: cluster 2023-10-09T05:33:03.798250+0000 mon.a (mon.0) 1699 : cluster 3 Health check update: Degraded data redundancy: 362/193650 objects degraded (0.187%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:33:04.582 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:04.590 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:04 smithi194 bash[92666]: cluster 2023-10-09T05:33:03.798250+0000 mon.a (mon.0) 1699 : cluster 3 Health check update: Degraded data redundancy: 362/193650 objects degraded (0.187%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:33:04.613 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:04.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:04.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:04.717 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:04.749 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:04.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:04.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:04.859 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:04.886 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:04.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:04.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:04.989 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:05.017 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:05.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:05.022 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:05.122 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:05.162 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:05.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:05.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:05.266 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:05.288 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:05.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:05.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:05.391 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:05.424 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:05.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:05.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:05.531 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:05.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:05 smithi133 bash[105589]: cluster 2023-10-09T05:33:04.325029+0000 mgr.y (mgr.25277) 1349 : cluster 0 pgmap v1448: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 362/193650 objects degraded (0.187%); 40156/193650 objects misplaced (20.736%); 762 KiB/s, 11 objects/s recovering 2023-10-09T05:33:05.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:05 smithi133 bash[102858]: cluster 2023-10-09T05:33:04.325029+0000 mgr.y (mgr.25277) 1349 : cluster 0 pgmap v1448: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 362/193650 objects degraded (0.187%); 40156/193650 objects misplaced (20.736%); 762 KiB/s, 11 objects/s recovering 2023-10-09T05:33:05.559 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:05.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:05.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:05.668 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:05.676 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:05 smithi194 bash[92666]: cluster 2023-10-09T05:33:04.325029+0000 mgr.y (mgr.25277) 1349 : cluster 0 pgmap v1448: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 362/193650 objects degraded (0.187%); 40156/193650 objects misplaced (20.736%); 762 KiB/s, 11 objects/s recovering 2023-10-09T05:33:05.695 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:05.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:05.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:05.801 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:05.824 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:05.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:05.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:05.927 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:05.964 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:05.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:05.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:06.068 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:06.092 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:06.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:06.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:06.195 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:06.223 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:06.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:06.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:06.333 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:06.358 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:06.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:06.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:06.465 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:06.500 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:06.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:06.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:06.604 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:06.635 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:06.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:06.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:06.740 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:06.771 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:06.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:06.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:06.875 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:06.911 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:06.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:06.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:07.018 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:07.041 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:07.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:07.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:07.145 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:07.170 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:07.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:07.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:07.273 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:07.297 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:07.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:07.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:07.400 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:07.427 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:07.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:07.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:07.530 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:07.557 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:07.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:07.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:07.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:07 smithi194 bash[92666]: cluster 2023-10-09T05:33:06.325827+0000 mgr.y (mgr.25277) 1350 : cluster 0 pgmap v1449: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 762 op/s; 498/183798 objects degraded (0.271%); 38117/183798 objects misplaced (20.739%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T05:33:07.661 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:07.669 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:07 smithi133 bash[105589]: cluster 2023-10-09T05:33:06.325827+0000 mgr.y (mgr.25277) 1350 : cluster 0 pgmap v1449: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 762 op/s; 498/183798 objects degraded (0.271%); 38117/183798 objects misplaced (20.739%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T05:33:07.670 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:07 smithi133 bash[102858]: cluster 2023-10-09T05:33:06.325827+0000 mgr.y (mgr.25277) 1350 : cluster 0 pgmap v1449: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 762 op/s; 498/183798 objects degraded (0.271%); 38117/183798 objects misplaced (20.739%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T05:33:07.686 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:07.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:07.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:07.794 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:07.833 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:07.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:07.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:07.937 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:07.977 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:07.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:07.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:08.083 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:08.106 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:08.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:08.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:08.212 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:08.242 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:08.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:08.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:08.349 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:08.376 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:08.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:08.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:08.481 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:08.513 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:08.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:08.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:08.620 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:08.642 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:08.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:08.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:08.745 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:08.783 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:08.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:08.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:08.888 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:08.912 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:08.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:08.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:09.018 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:09.050 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:09.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:09.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:09.154 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:09.191 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:09.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:09.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:09.295 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:09.332 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:09.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:09.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:09.438 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:09.449 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:09 smithi133 bash[102858]: cluster 2023-10-09T05:33:08.327504+0000 mgr.y (mgr.25277) 1351 : cluster 0 pgmap v1450: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 498/183798 objects degraded (0.271%); 38117/183798 objects misplaced (20.739%); 826 KiB/s, 12 objects/s recovering 2023-10-09T05:33:09.449 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:09 smithi133 bash[102858]: cluster 2023-10-09T05:33:08.799446+0000 mon.a (mon.0) 1700 : cluster 3 Health check update: Degraded data redundancy: 498/183798 objects degraded (0.271%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:33:09.449 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:09 smithi133 bash[105589]: cluster 2023-10-09T05:33:08.327504+0000 mgr.y (mgr.25277) 1351 : cluster 0 pgmap v1450: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 498/183798 objects degraded (0.271%); 38117/183798 objects misplaced (20.739%); 826 KiB/s, 12 objects/s recovering 2023-10-09T05:33:09.450 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:09 smithi133 bash[105589]: cluster 2023-10-09T05:33:08.799446+0000 mon.a (mon.0) 1700 : cluster 3 Health check update: Degraded data redundancy: 498/183798 objects degraded (0.271%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:33:09.470 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:09.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:09.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:09.576 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:09.597 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:09.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:09.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:09.700 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:09.731 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:09.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:09.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:09.836 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:09.846 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:09 smithi194 bash[92666]: cluster 2023-10-09T05:33:08.327504+0000 mgr.y (mgr.25277) 1351 : cluster 0 pgmap v1450: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 498/183798 objects degraded (0.271%); 38117/183798 objects misplaced (20.739%); 826 KiB/s, 12 objects/s recovering 2023-10-09T05:33:09.846 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:09 smithi194 bash[92666]: cluster 2023-10-09T05:33:08.799446+0000 mon.a (mon.0) 1700 : cluster 3 Health check update: Degraded data redundancy: 498/183798 objects degraded (0.271%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:33:09.869 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:09.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:09.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:09.973 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:10.007 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:10.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:10.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:10.113 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:10.149 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:10.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:10.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:10.252 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:10.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:10.287 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:10.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:10.388 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:10.417 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:10.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:10.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:10.520 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:10.544 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:10.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:10.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:10.648 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:10.679 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:10.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:10.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:10.787 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:10.847 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:10.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:10.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:10.954 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:10.983 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:10.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:10.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:11.088 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:11.116 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:11.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:11.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:11.222 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:11.253 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:11.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:11.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:11.357 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:11.396 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:11.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:11.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:11.502 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:11.531 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:11.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:11.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:11.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:11 smithi133 bash[102858]: cluster 2023-10-09T05:33:10.328517+0000 mgr.y (mgr.25277) 1352 : cluster 0 pgmap v1451: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 498/178068 objects degraded (0.280%); 36998/178068 objects misplaced (20.777%); 826 KiB/s, 12 objects/s recovering 2023-10-09T05:33:11.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:11 smithi133 bash[105589]: cluster 2023-10-09T05:33:10.328517+0000 mgr.y (mgr.25277) 1352 : cluster 0 pgmap v1451: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 498/178068 objects degraded (0.280%); 36998/178068 objects misplaced (20.777%); 826 KiB/s, 12 objects/s recovering 2023-10-09T05:33:11.635 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:11.668 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:11.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:11.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:11.674 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:11 smithi194 bash[92666]: cluster 2023-10-09T05:33:10.328517+0000 mgr.y (mgr.25277) 1352 : cluster 0 pgmap v1451: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 498/178068 objects degraded (0.280%); 36998/178068 objects misplaced (20.777%); 826 KiB/s, 12 objects/s recovering 2023-10-09T05:33:11.775 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:11.800 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:11.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:11.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:11.903 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:11.936 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:11.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:11.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:12.042 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:12.078 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:12.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:12.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:12.186 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:12.216 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:12.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:12.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:12.319 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:12.344 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:12.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:12.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:12.448 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:12.481 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:12.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:12.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:12.584 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:12.614 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:12.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:12.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:12.719 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:12.768 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:12.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:12.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:12.873 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:12.903 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:12.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:12.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:13.006 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:13.030 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:13.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:13.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:13.136 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:13.168 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:13.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:13.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:13.272 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:13.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:13.304 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:13.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:13.405 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:13.426 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:13 smithi133 bash[102858]: cluster 2023-10-09T05:33:12.329195+0000 mgr.y (mgr.25277) 1353 : cluster 0 pgmap v1452: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 614/174528 objects degraded (0.352%); 36218/174528 objects misplaced (20.752%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T05:33:13.426 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:13 smithi133 bash[105589]: cluster 2023-10-09T05:33:12.329195+0000 mgr.y (mgr.25277) 1353 : cluster 0 pgmap v1452: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 614/174528 objects degraded (0.352%); 36218/174528 objects misplaced (20.752%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T05:33:13.444 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:13.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:13.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:13.551 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:13.593 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:13.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:13.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:13.697 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:13.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:13.745 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:13.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:13.846 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:13.872 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:13.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:13.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:13.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:13 smithi194 bash[92666]: cluster 2023-10-09T05:33:12.329195+0000 mgr.y (mgr.25277) 1353 : cluster 0 pgmap v1452: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 614/174528 objects degraded (0.352%); 36218/174528 objects misplaced (20.752%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T05:33:13.976 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:14.012 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:14.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:14.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:14.117 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:14.180 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:14.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:14.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:14.286 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:14.348 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:14.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:14.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:14.454 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:14.507 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:14.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:14.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:14.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:33:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:33:14] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.43.0" 2023-10-09T05:33:14.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:14 smithi133 bash[105589]: audit 2023-10-09T05:33:13.485675+0000 mon.a (mon.0) 1701 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:33:14.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:14 smithi133 bash[105589]: cluster 2023-10-09T05:33:13.800589+0000 mon.a (mon.0) 1702 : cluster 3 Health check update: Degraded data redundancy: 614/174528 objects degraded (0.352%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:33:14.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:14 smithi133 bash[102858]: audit 2023-10-09T05:33:13.485675+0000 mon.a (mon.0) 1701 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:33:14.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:14 smithi133 bash[102858]: cluster 2023-10-09T05:33:13.800589+0000 mon.a (mon.0) 1702 : cluster 3 Health check update: Degraded data redundancy: 614/174528 objects degraded (0.352%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:33:14.616 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:14.642 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:14 smithi194 bash[92666]: audit 2023-10-09T05:33:13.485675+0000 mon.a (mon.0) 1701 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:33:14.642 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:14 smithi194 bash[92666]: cluster 2023-10-09T05:33:13.800589+0000 mon.a (mon.0) 1702 : cluster 3 Health check update: Degraded data redundancy: 614/174528 objects degraded (0.352%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:33:14.655 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:14.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:14.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:14.762 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:14.792 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:14.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:14.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:14.898 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:14.936 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:14.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:14.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:15.044 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:15.073 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:15.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:15.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:15.178 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:15.225 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:15.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:15.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:15.332 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:15.376 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:15.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:15.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:15.482 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:15.520 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:15.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:15.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:15.624 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:15.633 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:15 smithi194 bash[92666]: cluster 2023-10-09T05:33:14.329678+0000 mgr.y (mgr.25277) 1354 : cluster 0 pgmap v1453: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 614/174528 objects degraded (0.352%); 36218/174528 objects misplaced (20.752%); 773 KiB/s, 12 objects/s recovering 2023-10-09T05:33:15.649 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:15.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:15.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:15.756 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:15.792 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:15.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:15.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:15.873 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:15 smithi133 bash[102858]: cluster 2023-10-09T05:33:14.329678+0000 mgr.y (mgr.25277) 1354 : cluster 0 pgmap v1453: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 614/174528 objects degraded (0.352%); 36218/174528 objects misplaced (20.752%); 773 KiB/s, 12 objects/s recovering 2023-10-09T05:33:15.873 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:15 smithi133 bash[105589]: cluster 2023-10-09T05:33:14.329678+0000 mgr.y (mgr.25277) 1354 : cluster 0 pgmap v1453: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 614/174528 objects degraded (0.352%); 36218/174528 objects misplaced (20.752%); 773 KiB/s, 12 objects/s recovering 2023-10-09T05:33:15.895 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:15.933 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:15.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:15.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:16.039 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:16.081 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:16.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:16.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:16.185 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:16.211 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:16.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:16.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:16.314 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:16.344 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:16.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:16.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:16.447 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:16.477 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:16.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:16.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:16.491 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:33:16.491 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:33:16.587 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:16.610 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:16.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:16.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:16.713 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:16.754 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:16.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:16.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:16.856 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:16.880 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:16.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:16.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:16.984 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:17.005 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:17.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:17.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:17.118 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:17.157 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:17.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:17.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:17.264 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:17.295 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:17.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:17.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:17.398 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:17.432 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:17.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:17.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:17.538 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:17.573 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:17.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:17.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:17.681 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:17.690 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:17 smithi133 bash[105589]: cluster 2023-10-09T05:33:16.330438+0000 mgr.y (mgr.25277) 1355 : cluster 0 pgmap v1454: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 793 op/s; 746/165087 objects degraded (0.452%); 34231/165087 objects misplaced (20.735%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T05:33:17.690 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:17 smithi133 bash[102858]: cluster 2023-10-09T05:33:16.330438+0000 mgr.y (mgr.25277) 1355 : cluster 0 pgmap v1454: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 793 op/s; 746/165087 objects degraded (0.452%); 34231/165087 objects misplaced (20.735%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T05:33:17.714 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:17.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:17.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:17.817 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:17.826 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:17 smithi194 bash[92666]: cluster 2023-10-09T05:33:16.330438+0000 mgr.y (mgr.25277) 1355 : cluster 0 pgmap v1454: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 793 op/s; 746/165087 objects degraded (0.452%); 34231/165087 objects misplaced (20.735%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T05:33:17.838 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:17.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:17.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:17.945 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:17.979 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:17.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:17.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:18.083 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:18.108 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:18.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:18.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:18.212 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:18.239 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:18.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:18.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:18.347 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:18.378 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:18.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:18.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:18.485 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:18.515 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:18.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:18.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:18.624 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:18.657 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:18.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:18.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:18.761 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:18.789 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:18.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:18.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:18.894 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:18.929 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:18.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:18.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:19.035 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:19.066 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:19.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:19.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:19.171 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:19.196 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:19.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:19.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:19.303 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:19.341 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:19.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:19.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:19.379 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:33:19.430 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:33:19.448 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:19.478 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:33:19.490 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:19.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:19.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:19.538 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:33:19.580 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:33:19.592 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:19.600 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:19 smithi133 bash[105589]: cluster 2023-10-09T05:33:18.330859+0000 mgr.y (mgr.25277) 1356 : cluster 0 pgmap v1455: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 519 op/s; 746/165087 objects degraded (0.452%); 34231/165087 objects misplaced (20.735%); 762 KiB/s, 11 objects/s recovering 2023-10-09T05:33:19.600 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:19 smithi133 bash[105589]: cluster 2023-10-09T05:33:18.805378+0000 mon.a (mon.0) 1703 : cluster 3 Health check update: Degraded data redundancy: 746/165087 objects degraded (0.452%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:33:19.600 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:19 smithi133 bash[105589]: audit 2023-10-09T05:33:19.300985+0000 mon.c (mon.2) 126 : audit 0 from='client.? 172.21.15.133:0/3263576553' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:33:19.601 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:19 smithi133 bash[102858]: cluster 2023-10-09T05:33:18.330859+0000 mgr.y (mgr.25277) 1356 : cluster 0 pgmap v1455: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 519 op/s; 746/165087 objects degraded (0.452%); 34231/165087 objects misplaced (20.735%); 762 KiB/s, 11 objects/s recovering 2023-10-09T05:33:19.601 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:19 smithi133 bash[102858]: cluster 2023-10-09T05:33:18.805378+0000 mon.a (mon.0) 1703 : cluster 3 Health check update: Degraded data redundancy: 746/165087 objects degraded (0.452%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:33:19.601 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:19 smithi133 bash[102858]: audit 2023-10-09T05:33:19.300985+0000 mon.c (mon.2) 126 : audit 0 from='client.? 172.21.15.133:0/3263576553' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:33:19.617 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:33:19.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:19 smithi194 bash[92666]: cluster 2023-10-09T05:33:18.330859+0000 mgr.y (mgr.25277) 1356 : cluster 0 pgmap v1455: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 519 op/s; 746/165087 objects degraded (0.452%); 34231/165087 objects misplaced (20.735%); 762 KiB/s, 11 objects/s recovering 2023-10-09T05:33:19.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:19 smithi194 bash[92666]: cluster 2023-10-09T05:33:18.805378+0000 mon.a (mon.0) 1703 : cluster 3 Health check update: Degraded data redundancy: 746/165087 objects degraded (0.452%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:33:19.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:19 smithi194 bash[92666]: audit 2023-10-09T05:33:19.300985+0000 mon.c (mon.2) 126 : audit 0 from='client.? 172.21.15.133:0/3263576553' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:33:19.639 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:19.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:19.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:19.656 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:33:19.691 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:33:19.703 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:33:19.703 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:33:19.703 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:33:19.703 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:33:19.703 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:33:19.703 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:33:19.703 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:33:19.703 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:33:19.743 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:19.772 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:19.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:19.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:19.877 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:19.909 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:19.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:19.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:20.013 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:20.046 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:20.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:20.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:20.152 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:20.186 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:20.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:20.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:20.290 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:20.342 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:20.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:20.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:20.443 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is set 2023-10-09T05:33:20.446 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:20.478 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:20.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:20.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:20.586 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:20.608 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:20.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:20.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:20.625 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:20 smithi133 bash[105589]: audit 2023-10-09T05:33:19.860889+0000 mon.c (mon.2) 127 : audit 1 from='client.? 172.21.15.133:0/3135270131' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:33:20.625 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:20 smithi133 bash[105589]: audit 2023-10-09T05:33:19.861267+0000 mon.a (mon.0) 1704 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:33:20.625 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:20 smithi133 bash[102858]: audit 2023-10-09T05:33:19.860889+0000 mon.c (mon.2) 127 : audit 1 from='client.? 172.21.15.133:0/3135270131' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:33:20.625 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:20 smithi133 bash[102858]: audit 2023-10-09T05:33:19.861267+0000 mon.a (mon.0) 1704 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:33:20.714 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:20.757 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:20.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:20.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:20.864 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:20.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:20 smithi194 bash[92666]: audit 2023-10-09T05:33:19.860889+0000 mon.c (mon.2) 127 : audit 1 from='client.? 172.21.15.133:0/3135270131' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:33:20.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:20 smithi194 bash[92666]: audit 2023-10-09T05:33:19.861267+0000 mon.a (mon.0) 1704 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:33:20.893 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:20.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:20.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:21.000 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:21.022 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:21.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:21.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:21.130 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:21.154 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:21.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:21.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:21.259 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:21.294 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:21.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:21.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:21.400 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:21.431 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:21.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:21.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:21.536 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:21.576 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:21.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:21.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:21.679 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:21.687 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:21 smithi194 bash[92666]: cluster 2023-10-09T05:33:20.331599+0000 mgr.y (mgr.25277) 1357 : cluster 0 pgmap v1456: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 673 op/s; 746/159531 objects degraded (0.468%); 33142/159531 objects misplaced (20.775%); 762 KiB/s, 11 objects/s recovering 2023-10-09T05:33:21.687 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:21 smithi194 bash[92666]: cluster 2023-10-09T05:33:20.415137+0000 mon.a (mon.0) 1705 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:33:21.687 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:21 smithi194 bash[92666]: cluster 2023-10-09T05:33:20.427838+0000 osd.7 (osd.7) 265 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T05:33:21.687 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:21 smithi194 bash[92666]: audit 2023-10-09T05:33:20.429945+0000 mon.a (mon.0) 1706 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:33:21.687 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:21 smithi194 bash[92666]: cluster 2023-10-09T05:33:20.430121+0000 mon.a (mon.0) 1707 : cluster 0 osdmap e303: 8 total, 8 up, 7 in 2023-10-09T05:33:21.687 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:21 smithi194 bash[92666]: cluster 2023-10-09T05:33:20.432137+0000 osd.5 (osd.5) 279 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T05:33:21.712 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:21.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:21.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:21.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:21 smithi133 bash[105589]: cluster 2023-10-09T05:33:20.331599+0000 mgr.y (mgr.25277) 1357 : cluster 0 pgmap v1456: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 673 op/s; 746/159531 objects degraded (0.468%); 33142/159531 objects misplaced (20.775%); 762 KiB/s, 11 objects/s recovering 2023-10-09T05:33:21.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:21 smithi133 bash[105589]: cluster 2023-10-09T05:33:20.415137+0000 mon.a (mon.0) 1705 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:33:21.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:21 smithi133 bash[105589]: cluster 2023-10-09T05:33:20.427838+0000 osd.7 (osd.7) 265 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T05:33:21.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:21 smithi133 bash[105589]: audit 2023-10-09T05:33:20.429945+0000 mon.a (mon.0) 1706 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:33:21.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:21 smithi133 bash[105589]: cluster 2023-10-09T05:33:20.430121+0000 mon.a (mon.0) 1707 : cluster 0 osdmap e303: 8 total, 8 up, 7 in 2023-10-09T05:33:21.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:21 smithi133 bash[105589]: cluster 2023-10-09T05:33:20.432137+0000 osd.5 (osd.5) 279 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T05:33:21.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:21 smithi133 bash[102858]: cluster 2023-10-09T05:33:20.331599+0000 mgr.y (mgr.25277) 1357 : cluster 0 pgmap v1456: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 673 op/s; 746/159531 objects degraded (0.468%); 33142/159531 objects misplaced (20.775%); 762 KiB/s, 11 objects/s recovering 2023-10-09T05:33:21.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:21 smithi133 bash[102858]: cluster 2023-10-09T05:33:20.415137+0000 mon.a (mon.0) 1705 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:33:21.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:21 smithi133 bash[102858]: cluster 2023-10-09T05:33:20.427838+0000 osd.7 (osd.7) 265 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T05:33:21.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:21 smithi133 bash[102858]: audit 2023-10-09T05:33:20.429945+0000 mon.a (mon.0) 1706 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:33:21.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:21 smithi133 bash[102858]: cluster 2023-10-09T05:33:20.430121+0000 mon.a (mon.0) 1707 : cluster 0 osdmap e303: 8 total, 8 up, 7 in 2023-10-09T05:33:21.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:21 smithi133 bash[102858]: cluster 2023-10-09T05:33:20.432137+0000 osd.5 (osd.5) 279 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T05:33:21.818 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:21.855 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:21.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:21.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:21.963 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:21.993 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:21.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:21.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:22.098 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:22.139 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:22.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:22.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:22.244 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:22.259 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T05:33:22.276 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:22.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:22.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:22.382 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:22.410 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:22.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:22.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:22.513 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:22.552 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:22.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:22.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:22.657 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:22.682 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:22.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:22.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:22.791 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:22.820 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:22.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:22.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:22.923 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:22.950 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:22.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:22.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:23.053 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:23.078 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:23.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:23.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:23.183 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:23.215 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:23.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:23.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:23.320 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:23.345 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:23.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:23.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:23.454 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:23.485 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:23.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:23.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:23.593 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:23.601 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:23 smithi194 bash[92666]: cluster 2023-10-09T05:33:22.332092+0000 mgr.y (mgr.25277) 1358 : cluster 0 pgmap v1458: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 608 op/s; 862/156255 objects degraded (0.552%); 32448/156255 objects misplaced (20.766%); 934 KiB/s, 14 objects/s recovering 2023-10-09T05:33:23.634 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:23.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:23.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:23.737 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:23.769 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:23.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:23.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:23.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:23 smithi133 bash[102858]: cluster 2023-10-09T05:33:22.332092+0000 mgr.y (mgr.25277) 1358 : cluster 0 pgmap v1458: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 608 op/s; 862/156255 objects degraded (0.552%); 32448/156255 objects misplaced (20.766%); 934 KiB/s, 14 objects/s recovering 2023-10-09T05:33:23.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:23 smithi133 bash[105589]: cluster 2023-10-09T05:33:22.332092+0000 mgr.y (mgr.25277) 1358 : cluster 0 pgmap v1458: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 608 op/s; 862/156255 objects degraded (0.552%); 32448/156255 objects misplaced (20.766%); 934 KiB/s, 14 objects/s recovering 2023-10-09T05:33:23.877 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:23.914 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:23.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:23.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:24.017 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:24.044 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:24.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:24.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:24.147 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:24.186 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:24.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:24.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:24.290 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:24.312 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:24.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:24.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:24.415 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:24.424 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:33:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:33:24] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.43.0" 2023-10-09T05:33:24.425 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:24 smithi133 bash[102858]: cluster 2023-10-09T05:33:23.806389+0000 mon.a (mon.0) 1708 : cluster 3 Health check update: Degraded data redundancy: 862/156255 objects degraded (0.552%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:33:24.440 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:24.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:24.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:24.549 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:24.589 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:24.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:24.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:24.694 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:24.704 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:24 smithi194 bash[92666]: cluster 2023-10-09T05:33:23.806389+0000 mon.a (mon.0) 1708 : cluster 3 Health check update: Degraded data redundancy: 862/156255 objects degraded (0.552%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:33:24.728 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:24.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:24.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:24.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:24 smithi133 bash[105589]: cluster 2023-10-09T05:33:23.806389+0000 mon.a (mon.0) 1708 : cluster 3 Health check update: Degraded data redundancy: 862/156255 objects degraded (0.552%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:33:24.833 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:24.866 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:24.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:24.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:24.969 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:25.014 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:25.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:25.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:25.120 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:25.173 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:25.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:25.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:25.282 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:25.312 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:25.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:25.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:25.417 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:25.448 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:25.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:25.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:25.544 INFO:teuthology.orchestra.run.smithi133.stdout:moved 3 / 267 (1.1236%) 2023-10-09T05:33:25.544 INFO:teuthology.orchestra.run.smithi133.stdout:avg 38.1429 2023-10-09T05:33:25.544 INFO:teuthology.orchestra.run.smithi133.stdout:stddev 4.0507 -> 4.48581 (expected baseline 5.71786) 2023-10-09T05:33:25.544 INFO:teuthology.orchestra.run.smithi133.stdout:min osd.5 with 32 -> 32 pgs (0.838951 -> 0.838951 * mean) 2023-10-09T05:33:25.544 INFO:teuthology.orchestra.run.smithi133.stdout:max osd.1 with 45 -> 45 pgs (1.17978 -> 1.17978 * mean) 2023-10-09T05:33:25.544 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:33:25.544 INFO:teuthology.orchestra.run.smithi133.stdout:oload 110 2023-10-09T05:33:25.544 INFO:teuthology.orchestra.run.smithi133.stdout:max_change 0.05 2023-10-09T05:33:25.545 INFO:teuthology.orchestra.run.smithi133.stdout:max_change_osds 4 2023-10-09T05:33:25.545 INFO:teuthology.orchestra.run.smithi133.stdout:average_utilization 0.0257 2023-10-09T05:33:25.545 INFO:teuthology.orchestra.run.smithi133.stdout:overload_utilization 0.0282 2023-10-09T05:33:25.545 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2 weight 1.0000 -> 0.9500 2023-10-09T05:33:25.545 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7 weight 1.0000 -> 0.9500 2023-10-09T05:33:25.545 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0 weight 1.0000 -> 0.9500 2023-10-09T05:33:25.551 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:25.556 INFO:teuthology.orchestra.run.smithi133.stderr:no change 2023-10-09T05:33:25.581 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:25.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:25.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:25.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:25 smithi194 bash[92666]: cluster 2023-10-09T05:33:24.332550+0000 mgr.y (mgr.25277) 1359 : cluster 0 pgmap v1459: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 608 op/s; 862/156255 objects degraded (0.552%); 32448/156255 objects misplaced (20.766%); 934 KiB/s, 14 objects/s recovering 2023-10-09T05:33:25.684 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:25.712 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:25.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:25.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:25.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:25 smithi133 bash[105589]: cluster 2023-10-09T05:33:24.332550+0000 mgr.y (mgr.25277) 1359 : cluster 0 pgmap v1459: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 608 op/s; 862/156255 objects degraded (0.552%); 32448/156255 objects misplaced (20.766%); 934 KiB/s, 14 objects/s recovering 2023-10-09T05:33:25.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:25 smithi133 bash[102858]: cluster 2023-10-09T05:33:24.332550+0000 mgr.y (mgr.25277) 1359 : cluster 0 pgmap v1459: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 608 op/s; 862/156255 objects degraded (0.552%); 32448/156255 objects misplaced (20.766%); 934 KiB/s, 14 objects/s recovering 2023-10-09T05:33:25.798 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:33:25 smithi133 bash[91128]: debug 2023-10-09T05:33:25.539+0000 7ff45d5ed700 -1 mgr.server reply reply (3) No such process no change 2023-10-09T05:33:25.817 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:25.844 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:25.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:25.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:25.947 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:25.987 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:25.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:25.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:26.091 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:26.121 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:26.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:26.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:26.226 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:26.260 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:26.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:26.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:26.367 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:26.400 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:26.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:26.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:26.434 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:26 smithi133 bash[102858]: audit 2023-10-09T05:33:25.543038+0000 mgr.y (mgr.25277) 1360 : audit 0 from='client.45662 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:33:26.504 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:26.514 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:26 smithi194 bash[92666]: audit 2023-10-09T05:33:25.543038+0000 mgr.y (mgr.25277) 1360 : audit 0 from='client.45662 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:33:26.530 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:33:26.531 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:26.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:26.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:26.577 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:33:26.635 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:26.670 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:26.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:26.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:26.775 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:26.787 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:26 smithi133 bash[105589]: audit 2023-10-09T05:33:25.543038+0000 mgr.y (mgr.25277) 1360 : audit 0 from='client.45662 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:33:26.813 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:26.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:26.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:26.916 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:26.950 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:26.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:26.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:27.057 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:27.083 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:27.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:27.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:27.190 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:27.212 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:27.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:27.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:27.322 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:27.347 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:27.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:27.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:27.452 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:27.477 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:27.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:27.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:27.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:27 smithi133 bash[105589]: cluster 2023-10-09T05:33:26.333364+0000 mgr.y (mgr.25277) 1361 : cluster 0 pgmap v1460: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 971/147105 objects degraded (0.660%); 30567/147105 objects misplaced (20.779%); 966 KiB/s, 15 objects/s recovering 2023-10-09T05:33:27.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:27 smithi133 bash[102858]: cluster 2023-10-09T05:33:26.333364+0000 mgr.y (mgr.25277) 1361 : cluster 0 pgmap v1460: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 971/147105 objects degraded (0.660%); 30567/147105 objects misplaced (20.779%); 966 KiB/s, 15 objects/s recovering 2023-10-09T05:33:27.587 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:27.615 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:27.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:27.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:27.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:27 smithi194 bash[92666]: cluster 2023-10-09T05:33:26.333364+0000 mgr.y (mgr.25277) 1361 : cluster 0 pgmap v1460: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 971/147105 objects degraded (0.660%); 30567/147105 objects misplaced (20.779%); 966 KiB/s, 15 objects/s recovering 2023-10-09T05:33:27.720 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:27.754 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:27.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:27.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:27.859 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:27.884 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:27.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:27.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:27.989 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:28.023 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:28.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:28.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:28.078 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-10-09T05:33:28.078 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-10-09T05:33:28.128 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:28.161 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:28.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:28.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:28.266 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:28.302 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:28.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:28.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:28.406 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:28.448 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:28.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:28.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:28.545 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:33:28.550 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:28.587 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:28.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:28.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:28.693 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:28.731 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:28.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:28.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:28.835 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:28.867 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:28.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:28.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:28.970 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:28.994 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:28.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:28.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:29.100 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:29.127 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:29.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:29.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:29.238 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:29.265 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:29.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:29.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:29.374 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:29.408 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:29.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:29.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:29.513 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:29.522 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:29 smithi194 bash[92666]: cluster 2023-10-09T05:33:28.333758+0000 mgr.y (mgr.25277) 1362 : cluster 0 pgmap v1461: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 971/147105 objects degraded (0.660%); 30567/147105 objects misplaced (20.779%); 966 KiB/s, 15 objects/s recovering 2023-10-09T05:33:29.522 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:29 smithi194 bash[92666]: audit 2023-10-09T05:33:28.485165+0000 mon.a (mon.0) 1709 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:33:29.522 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:29 smithi194 bash[92666]: cluster 2023-10-09T05:33:28.811213+0000 mon.a (mon.0) 1710 : cluster 3 Health check update: Degraded data redundancy: 971/147105 objects degraded (0.660%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:33:29.549 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:29.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:29.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:29.653 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:29.683 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:29.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:29.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:29.758 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:29 smithi133 bash[102858]: cluster 2023-10-09T05:33:28.333758+0000 mgr.y (mgr.25277) 1362 : cluster 0 pgmap v1461: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 971/147105 objects degraded (0.660%); 30567/147105 objects misplaced (20.779%); 966 KiB/s, 15 objects/s recovering 2023-10-09T05:33:29.758 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:29 smithi133 bash[102858]: audit 2023-10-09T05:33:28.485165+0000 mon.a (mon.0) 1709 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:33:29.758 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:29 smithi133 bash[102858]: cluster 2023-10-09T05:33:28.811213+0000 mon.a (mon.0) 1710 : cluster 3 Health check update: Degraded data redundancy: 971/147105 objects degraded (0.660%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:33:29.787 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:29.807 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:29.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:29.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:29.910 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:29.942 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:29.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:29.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:30.046 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:30.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:29 smithi133 bash[105589]: cluster 2023-10-09T05:33:28.333758+0000 mgr.y (mgr.25277) 1362 : cluster 0 pgmap v1461: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 3.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 971/147105 objects degraded (0.660%); 30567/147105 objects misplaced (20.779%); 966 KiB/s, 15 objects/s recovering 2023-10-09T05:33:30.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:29 smithi133 bash[105589]: audit 2023-10-09T05:33:28.485165+0000 mon.a (mon.0) 1709 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:33:30.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:29 smithi133 bash[105589]: cluster 2023-10-09T05:33:28.811213+0000 mon.a (mon.0) 1710 : cluster 3 Health check update: Degraded data redundancy: 971/147105 objects degraded (0.660%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:33:30.071 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:30.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:30.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:30.178 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:30.206 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:30.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:30.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:30.309 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:30.341 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:30.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:30.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:30.449 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:30.474 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:30.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:30.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:30.578 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:30.601 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:30.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:30.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:30.709 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:30.743 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:30.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:30.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:30.851 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:30.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:30.886 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:30.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:30.987 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:31.017 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:31.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:31.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:31.126 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:31.151 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:31.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:31.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:31.254 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:31.279 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:31.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:31.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:31.386 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:31.409 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:31.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:31.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:31.450 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:31 smithi194 bash[92666]: cluster 2023-10-09T05:33:30.334278+0000 mgr.y (mgr.25277) 1363 : cluster 0 pgmap v1462: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 612 op/s; 971/141162 objects degraded (0.688%); 29338/141162 objects misplaced (20.783%); 966 KiB/s, 15 objects/s recovering 2023-10-09T05:33:31.516 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:31.525 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:31 smithi133 bash[105589]: cluster 2023-10-09T05:33:30.334278+0000 mgr.y (mgr.25277) 1363 : cluster 0 pgmap v1462: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 612 op/s; 971/141162 objects degraded (0.688%); 29338/141162 objects misplaced (20.783%); 966 KiB/s, 15 objects/s recovering 2023-10-09T05:33:31.525 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:31 smithi133 bash[102858]: cluster 2023-10-09T05:33:30.334278+0000 mgr.y (mgr.25277) 1363 : cluster 0 pgmap v1462: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 612 op/s; 971/141162 objects degraded (0.688%); 29338/141162 objects misplaced (20.783%); 966 KiB/s, 15 objects/s recovering 2023-10-09T05:33:31.540 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:31.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:31.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:31.646 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:31.672 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:31.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:31.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:31.777 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:31.807 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:31.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:31.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:31.910 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:31.943 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:31.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:31.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:32.048 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:32.081 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:32.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:32.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:32.184 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:32.211 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:32.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:32.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:32.315 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:32.345 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:32.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:32.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:32.447 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:32.478 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:32.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:32.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:32.581 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:32.611 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:32.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:32.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:32.622 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:33:32.622 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:33:32.622 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:33:32.714 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:32.741 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:32.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:32.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:32.844 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:32.879 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:32.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:32.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:32.988 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:33.027 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:33.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:33.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:33.133 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:33.171 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:33.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:33.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:33.274 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:33.296 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:33.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:33.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:33.403 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:33.440 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:33.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:33.449 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:33.549 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:33.583 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:33.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:33.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:33.687 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:33.710 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:33.714 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:33 smithi133 bash[102858]: cluster 2023-10-09T05:33:32.336047+0000 mgr.y (mgr.25277) 1364 : cluster 0 pgmap v1463: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 579 op/s; 1054/137793 objects degraded (0.765%); 28594/137793 objects misplaced (20.751%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:33:33.715 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:33 smithi133 bash[105589]: cluster 2023-10-09T05:33:32.336047+0000 mgr.y (mgr.25277) 1364 : cluster 0 pgmap v1463: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 579 op/s; 1054/137793 objects degraded (0.765%); 28594/137793 objects misplaced (20.751%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:33:33.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:33.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:33.817 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:33.825 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:33 smithi194 bash[92666]: cluster 2023-10-09T05:33:32.336047+0000 mgr.y (mgr.25277) 1364 : cluster 0 pgmap v1463: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 579 op/s; 1054/137793 objects degraded (0.765%); 28594/137793 objects misplaced (20.751%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:33:33.840 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:33.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:33.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:33.945 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:33.978 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:33.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:33.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:34.083 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:34.114 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:34.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:34.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:34.218 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:34.249 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:34.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:34.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:34.353 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:34.380 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:34.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:34.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:34.483 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:34.509 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:34.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:34.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:34.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:33:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:33:34] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.43.0" 2023-10-09T05:33:34.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:34 smithi133 bash[105589]: cluster 2023-10-09T05:33:33.812092+0000 mon.a (mon.0) 1711 : cluster 3 Health check update: Degraded data redundancy: 1054/137793 objects degraded (0.765%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:33:34.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:34 smithi133 bash[102858]: cluster 2023-10-09T05:33:33.812092+0000 mon.a (mon.0) 1711 : cluster 3 Health check update: Degraded data redundancy: 1054/137793 objects degraded (0.765%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:33:34.616 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:34.659 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:34.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:34.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:34.765 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:34.775 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:34 smithi194 bash[92666]: cluster 2023-10-09T05:33:33.812092+0000 mon.a (mon.0) 1711 : cluster 3 Health check update: Degraded data redundancy: 1054/137793 objects degraded (0.765%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:33:34.799 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:34.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:34.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:34.903 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:34.935 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:34.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:34.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:35.039 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:35.071 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:35.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:35.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:35.178 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:35.201 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:35.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:35.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:35.303 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:35.333 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:35.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:35.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:35.437 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:35.462 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:35 smithi133 bash[102858]: cluster 2023-10-09T05:33:34.336543+0000 mgr.y (mgr.25277) 1365 : cluster 0 pgmap v1464: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 1054/137793 objects degraded (0.765%); 28594/137793 objects misplaced (20.751%); 906 KiB/s, 14 objects/s recovering 2023-10-09T05:33:35.479 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:35.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:35.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:35.582 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:35.592 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:35 smithi194 bash[92666]: cluster 2023-10-09T05:33:34.336543+0000 mgr.y (mgr.25277) 1365 : cluster 0 pgmap v1464: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 1054/137793 objects degraded (0.765%); 28594/137793 objects misplaced (20.751%); 906 KiB/s, 14 objects/s recovering 2023-10-09T05:33:35.613 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:35.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:35.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:35.717 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:35.749 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:35.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:35.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:35.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:35 smithi133 bash[105589]: cluster 2023-10-09T05:33:34.336543+0000 mgr.y (mgr.25277) 1365 : cluster 0 pgmap v1464: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 1054/137793 objects degraded (0.765%); 28594/137793 objects misplaced (20.751%); 906 KiB/s, 14 objects/s recovering 2023-10-09T05:33:35.852 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:35.880 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:35.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:35.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:35.983 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:36.013 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:36.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:36.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:36.119 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:36.145 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:36.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:36.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:36.249 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:36.278 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:36.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:36.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:36.384 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:36.408 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:36.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:36.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:36.514 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:36.548 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:36.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:36.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:36.653 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:36.696 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:36.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:36.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:36.800 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:36.831 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:36.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:36.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:36.938 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:36.979 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:36.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:36.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:37.085 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:37.125 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:37.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:37.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:37.231 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:37.256 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:37.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:37.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:37.363 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:37.387 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:37.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:37.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:37.490 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:37.501 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:37 smithi194 bash[92666]: cluster 2023-10-09T05:33:36.337187+0000 mgr.y (mgr.25277) 1366 : cluster 0 pgmap v1465: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 740 op/s; 1157/129591 objects degraded (0.893%); 26875/129591 objects misplaced (20.738%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T05:33:37.534 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:37.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:37.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:37.641 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:37.664 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:37.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:37.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:37.768 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:37.776 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:37 smithi133 bash[102858]: cluster 2023-10-09T05:33:36.337187+0000 mgr.y (mgr.25277) 1366 : cluster 0 pgmap v1465: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 740 op/s; 1157/129591 objects degraded (0.893%); 26875/129591 objects misplaced (20.738%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T05:33:37.776 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:37 smithi133 bash[105589]: cluster 2023-10-09T05:33:36.337187+0000 mgr.y (mgr.25277) 1366 : cluster 0 pgmap v1465: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 740 op/s; 1157/129591 objects degraded (0.893%); 26875/129591 objects misplaced (20.738%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T05:33:37.800 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:37.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:37.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:37.905 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:37.942 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:37.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:37.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:38.045 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:38.072 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:38.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:38.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:38.177 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:38.209 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:38.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:38.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:38.313 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:38.353 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:38.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:38.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:38.458 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:38.508 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:38.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:38.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:38.612 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:38.630 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:38.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:38.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:38.733 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:38.765 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:38.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:38.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:38.869 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:38.893 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:38.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:38.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:38.997 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:39.028 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:39.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:39.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:39.135 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:39.161 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:39.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:39.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:39.264 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:39.304 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:39.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:39.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:39.412 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:39.439 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:39.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:39.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:39.541 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:39.547 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:39 smithi133 bash[105589]: cluster 2023-10-09T05:33:38.337718+0000 mgr.y (mgr.25277) 1367 : cluster 0 pgmap v1466: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 486 op/s; 1157/129591 objects degraded (0.893%); 26875/129591 objects misplaced (20.738%); 864 KiB/s, 13 objects/s recovering 2023-10-09T05:33:39.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:39 smithi133 bash[105589]: cluster 2023-10-09T05:33:38.813138+0000 mon.a (mon.0) 1712 : cluster 3 Health check update: Degraded data redundancy: 1157/129591 objects degraded (0.893%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:33:39.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:39 smithi133 bash[102858]: cluster 2023-10-09T05:33:38.337718+0000 mgr.y (mgr.25277) 1367 : cluster 0 pgmap v1466: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 486 op/s; 1157/129591 objects degraded (0.893%); 26875/129591 objects misplaced (20.738%); 864 KiB/s, 13 objects/s recovering 2023-10-09T05:33:39.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:39 smithi133 bash[102858]: cluster 2023-10-09T05:33:38.813138+0000 mon.a (mon.0) 1712 : cluster 3 Health check update: Degraded data redundancy: 1157/129591 objects degraded (0.893%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:33:39.573 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:39.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:39.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:39.676 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:39.720 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:39.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:39.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:39.825 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:39.849 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:39.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:39.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:39.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:39 smithi194 bash[92666]: cluster 2023-10-09T05:33:38.337718+0000 mgr.y (mgr.25277) 1367 : cluster 0 pgmap v1466: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 486 op/s; 1157/129591 objects degraded (0.893%); 26875/129591 objects misplaced (20.738%); 864 KiB/s, 13 objects/s recovering 2023-10-09T05:33:39.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:39 smithi194 bash[92666]: cluster 2023-10-09T05:33:38.813138+0000 mon.a (mon.0) 1712 : cluster 3 Health check update: Degraded data redundancy: 1157/129591 objects degraded (0.893%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:33:39.958 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:39.990 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:39.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:39.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:40.094 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:40.123 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:40.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:40.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:40.227 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:40.256 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:40.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:40.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:40.366 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:40.399 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:40.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:40.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:40.504 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:40.536 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:40.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:40.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:40.643 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:40.673 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:40.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:40.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:40.776 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:40.802 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:40.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:40.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:40.905 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:40.930 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:40.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:40.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:41.039 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:41.077 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:41.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:41.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:41.180 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:41.207 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:41.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:41.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:41.311 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:41.333 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:41.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:41.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:41.437 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:41.465 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:41.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:41.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:41.572 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:41.579 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:41 smithi133 bash[102858]: cluster 2023-10-09T05:33:40.338436+0000 mgr.y (mgr.25277) 1368 : cluster 0 pgmap v1467: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 1157/124113 objects degraded (0.932%); 25790/124113 objects misplaced (20.779%); 864 KiB/s, 13 objects/s recovering 2023-10-09T05:33:41.582 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:41 smithi133 bash[105589]: cluster 2023-10-09T05:33:40.338436+0000 mgr.y (mgr.25277) 1368 : cluster 0 pgmap v1467: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 1157/124113 objects degraded (0.932%); 25790/124113 objects misplaced (20.779%); 864 KiB/s, 13 objects/s recovering 2023-10-09T05:33:41.605 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:41.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:41.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:41.710 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:41.719 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:41 smithi194 bash[92666]: cluster 2023-10-09T05:33:40.338436+0000 mgr.y (mgr.25277) 1368 : cluster 0 pgmap v1467: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 1157/124113 objects degraded (0.932%); 25790/124113 objects misplaced (20.779%); 864 KiB/s, 13 objects/s recovering 2023-10-09T05:33:41.738 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:41.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:41.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:41.842 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:41.873 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:41.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:41.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:41.976 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:42.003 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:42.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:42.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:42.111 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:42.141 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:42.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:42.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:42.247 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:42.278 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:42.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:42.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:42.383 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:42.422 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:42.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:42.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:42.528 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:42.553 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:42.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:42.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:42.657 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:42.690 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:42.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:42.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:42.795 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:42.831 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:42.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:42.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:42.936 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:42.973 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:42.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:42.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:43.079 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:43.105 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:43.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:43.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:43.209 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:43.246 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:43.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:43.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:43.354 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:43.372 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:43.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:43.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:43.475 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:43.492 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:43 smithi133 bash[102858]: cluster 2023-10-09T05:33:42.339182+0000 mgr.y (mgr.25277) 1369 : cluster 0 pgmap v1468: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 564 op/s; 1219/120825 objects degraded (1.009%); 25079/120825 objects misplaced (20.756%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T05:33:43.505 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:43.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:43.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:43.609 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:43.634 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:43.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:43.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:43.739 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:43.748 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:43 smithi133 bash[105589]: cluster 2023-10-09T05:33:42.339182+0000 mgr.y (mgr.25277) 1369 : cluster 0 pgmap v1468: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 564 op/s; 1219/120825 objects degraded (1.009%); 25079/120825 objects misplaced (20.756%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T05:33:43.749 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:43 smithi133 bash[105589]: audit 2023-10-09T05:33:43.485502+0000 mon.a (mon.0) 1713 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:33:43.749 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:43 smithi133 bash[102858]: audit 2023-10-09T05:33:43.485502+0000 mon.a (mon.0) 1713 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:33:43.779 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:43.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:43.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:43.884 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:43.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:43 smithi194 bash[92666]: cluster 2023-10-09T05:33:42.339182+0000 mgr.y (mgr.25277) 1369 : cluster 0 pgmap v1468: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 564 op/s; 1219/120825 objects degraded (1.009%); 25079/120825 objects misplaced (20.756%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T05:33:43.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:43 smithi194 bash[92666]: audit 2023-10-09T05:33:43.485502+0000 mon.a (mon.0) 1713 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:33:43.913 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:43.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:43.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:44.016 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:44.044 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:44.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:44.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:44.152 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:44.192 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:44.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:44.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:44.296 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:44.320 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:44.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:44.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:44.427 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:44.433 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:33:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:33:44] "GET /metrics HTTP/1.1" 200 36497 "" "Prometheus/2.43.0" 2023-10-09T05:33:44.453 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:44.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:44.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:44.557 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:44.584 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:44.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:44.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:44.695 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:44.703 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:44 smithi133 bash[105589]: audit 2023-10-09T05:33:43.636228+0000 mon.a (mon.0) 1714 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:33:44.703 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:44 smithi133 bash[105589]: audit 2023-10-09T05:33:43.649166+0000 mon.a (mon.0) 1715 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:33:44.703 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:44 smithi133 bash[105589]: audit 2023-10-09T05:33:43.651400+0000 mon.a (mon.0) 1716 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:33:44.703 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:44 smithi133 bash[105589]: audit 2023-10-09T05:33:43.652414+0000 mon.a (mon.0) 1717 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:33:44.703 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:44 smithi133 bash[105589]: audit 2023-10-09T05:33:43.664477+0000 mon.a (mon.0) 1718 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:33:44.703 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:44 smithi133 bash[105589]: cluster 2023-10-09T05:33:43.814190+0000 mon.a (mon.0) 1719 : cluster 3 Health check update: Degraded data redundancy: 1219/120825 objects degraded (1.009%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:33:44.703 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:44 smithi133 bash[105589]: cluster 2023-10-09T05:33:44.339552+0000 mgr.y (mgr.25277) 1370 : cluster 0 pgmap v1469: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 471 op/s; 1219/120825 objects degraded (1.009%); 25079/120825 objects misplaced (20.756%); 885 KiB/s, 13 objects/s recovering 2023-10-09T05:33:44.704 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:44 smithi133 bash[102858]: audit 2023-10-09T05:33:43.636228+0000 mon.a (mon.0) 1714 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:33:44.704 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:44 smithi133 bash[102858]: audit 2023-10-09T05:33:43.649166+0000 mon.a (mon.0) 1715 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:33:44.704 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:44 smithi133 bash[102858]: audit 2023-10-09T05:33:43.651400+0000 mon.a (mon.0) 1716 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:33:44.704 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:44 smithi133 bash[102858]: audit 2023-10-09T05:33:43.652414+0000 mon.a (mon.0) 1717 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:33:44.704 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:44 smithi133 bash[102858]: audit 2023-10-09T05:33:43.664477+0000 mon.a (mon.0) 1718 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:33:44.704 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:44 smithi133 bash[102858]: cluster 2023-10-09T05:33:43.814190+0000 mon.a (mon.0) 1719 : cluster 3 Health check update: Degraded data redundancy: 1219/120825 objects degraded (1.009%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:33:44.704 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:44 smithi133 bash[102858]: cluster 2023-10-09T05:33:44.339552+0000 mgr.y (mgr.25277) 1370 : cluster 0 pgmap v1469: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 471 op/s; 1219/120825 objects degraded (1.009%); 25079/120825 objects misplaced (20.756%); 885 KiB/s, 13 objects/s recovering 2023-10-09T05:33:44.729 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:44.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:44.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:44.760 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:33:44.761 INFO:teuthology.orchestra.run.smithi133.stdout: "ops": [], 2023-10-09T05:33:44.761 INFO:teuthology.orchestra.run.smithi133.stdout: "complaint_time": 30, 2023-10-09T05:33:44.761 INFO:teuthology.orchestra.run.smithi133.stdout: "num_blocked_ops": 0 2023-10-09T05:33:44.761 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:33:44.832 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:44.859 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:44.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:44.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:44.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:44 smithi194 bash[92666]: audit 2023-10-09T05:33:43.636228+0000 mon.a (mon.0) 1714 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:33:44.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:44 smithi194 bash[92666]: audit 2023-10-09T05:33:43.649166+0000 mon.a (mon.0) 1715 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:33:44.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:44 smithi194 bash[92666]: audit 2023-10-09T05:33:43.651400+0000 mon.a (mon.0) 1716 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:33:44.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:44 smithi194 bash[92666]: audit 2023-10-09T05:33:43.652414+0000 mon.a (mon.0) 1717 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:33:44.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:44 smithi194 bash[92666]: audit 2023-10-09T05:33:43.664477+0000 mon.a (mon.0) 1718 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:33:44.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:44 smithi194 bash[92666]: cluster 2023-10-09T05:33:43.814190+0000 mon.a (mon.0) 1719 : cluster 3 Health check update: Degraded data redundancy: 1219/120825 objects degraded (1.009%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:33:44.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:44 smithi194 bash[92666]: cluster 2023-10-09T05:33:44.339552+0000 mgr.y (mgr.25277) 1370 : cluster 0 pgmap v1469: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 471 op/s; 1219/120825 objects degraded (1.009%); 25079/120825 objects misplaced (20.756%); 885 KiB/s, 13 objects/s recovering 2023-10-09T05:33:44.964 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:44.999 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:45.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:45.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:45.106 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:45.140 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:45.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:45.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:45.245 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:45.275 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:45.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:45.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:45.381 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:45.399 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:45.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:45.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:45.506 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:45.543 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:45.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:45.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:45.645 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:45.673 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:45.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:45.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:45.780 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:45.813 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:45.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:45.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:45.921 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:45.945 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:45.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:45.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:46.056 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:46.086 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:46.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:46.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:46.192 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:46.216 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:46.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:46.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:46.323 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:46.361 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:46.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:46.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:46.468 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:46.504 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:46.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:46.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:46.608 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:46.636 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:46.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:46.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:46.742 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:46.774 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:46.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:46.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:46.881 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:46.917 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:46.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:46.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:47.020 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:47.045 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:47.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:47.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:47.150 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:47.186 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:47.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:47.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:47.290 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:47.314 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:47.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:47.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:47.412 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:47 smithi194 bash[92666]: cluster 2023-10-09T05:33:46.340319+0000 mgr.y (mgr.25277) 1371 : cluster 0 pgmap v1470: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 736 op/s; 1341/111273 objects degraded (1.205%); 23061/111273 objects misplaced (20.725%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:33:47.424 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:47.448 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:47.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:47.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:47.551 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:47.588 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:47.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:47.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:47.594 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:47 smithi133 bash[105589]: cluster 2023-10-09T05:33:46.340319+0000 mgr.y (mgr.25277) 1371 : cluster 0 pgmap v1470: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 736 op/s; 1341/111273 objects degraded (1.205%); 23061/111273 objects misplaced (20.725%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:33:47.594 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:47 smithi133 bash[102858]: cluster 2023-10-09T05:33:46.340319+0000 mgr.y (mgr.25277) 1371 : cluster 0 pgmap v1470: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 736 op/s; 1341/111273 objects degraded (1.205%); 23061/111273 objects misplaced (20.725%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:33:47.694 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:47.715 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:47.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:47.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:47.818 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:47.844 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:47.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:47.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:47.951 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:47.999 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:48.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:48.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:48.104 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:48.137 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:48.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:48.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:48.241 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:48.280 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:48.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:48.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:48.384 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:48.419 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:48.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:48.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:48.523 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:48.550 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:48.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:48.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:48.656 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:48.685 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:48.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:48.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:48.792 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:48.817 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:48.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:48.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:48.921 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:48.959 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:48.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:48.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:49.063 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:49.103 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:49.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:49.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:49.208 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:49.241 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:49.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:49.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:49.347 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:49.385 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:49.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:49.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:49.488 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:49.496 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:49 smithi194 bash[92666]: cluster 2023-10-09T05:33:48.340821+0000 mgr.y (mgr.25277) 1372 : cluster 0 pgmap v1471: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 1341/111273 objects degraded (1.205%); 23061/111273 objects misplaced (20.725%); 954 KiB/s, 14 objects/s recovering 2023-10-09T05:33:49.496 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:49 smithi194 bash[92666]: cluster 2023-10-09T05:33:48.816656+0000 mon.a (mon.0) 1720 : cluster 3 Health check update: Degraded data redundancy: 1341/111273 objects degraded (1.205%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:33:49.520 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:49.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:49.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:49.633 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:49.670 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:49.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:49.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:49.775 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:49.810 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:49.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:49.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:49.912 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:49.922 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:49 smithi133 bash[102858]: cluster 2023-10-09T05:33:48.340821+0000 mgr.y (mgr.25277) 1372 : cluster 0 pgmap v1471: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 1341/111273 objects degraded (1.205%); 23061/111273 objects misplaced (20.725%); 954 KiB/s, 14 objects/s recovering 2023-10-09T05:33:49.922 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:49 smithi133 bash[102858]: cluster 2023-10-09T05:33:48.816656+0000 mon.a (mon.0) 1720 : cluster 3 Health check update: Degraded data redundancy: 1341/111273 objects degraded (1.205%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:33:49.922 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:49 smithi133 bash[105589]: cluster 2023-10-09T05:33:48.340821+0000 mgr.y (mgr.25277) 1372 : cluster 0 pgmap v1471: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 1341/111273 objects degraded (1.205%); 23061/111273 objects misplaced (20.725%); 954 KiB/s, 14 objects/s recovering 2023-10-09T05:33:49.922 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:49 smithi133 bash[105589]: cluster 2023-10-09T05:33:48.816656+0000 mon.a (mon.0) 1720 : cluster 3 Health check update: Degraded data redundancy: 1341/111273 objects degraded (1.205%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:33:49.937 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:49.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:49.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:50.044 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:50.080 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:50.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:50.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:50.185 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:50.244 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:50.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:50.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:50.274 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T05:33:50.349 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:50.388 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:50.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:50.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:50.493 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:50.532 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:50.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:50.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:50.635 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:50.660 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:50.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:50.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:50.769 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:50.803 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:50.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:50.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:50.908 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:50.951 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:50.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:50.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:51.057 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:51.086 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:51.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:51.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:51.193 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:51.227 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:51.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:51.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:51.332 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:51.376 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:51.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:51.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:51.425 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:51 smithi133 bash[102858]: cluster 2023-10-09T05:33:50.341549+0000 mgr.y (mgr.25277) 1373 : cluster 0 pgmap v1472: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 674 op/s; 1341/105315 objects degraded (1.273%); 21863/105315 objects misplaced (20.760%); 954 KiB/s, 14 objects/s recovering 2023-10-09T05:33:51.479 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:51.494 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:51 smithi194 bash[92666]: cluster 2023-10-09T05:33:50.341549+0000 mgr.y (mgr.25277) 1373 : cluster 0 pgmap v1472: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 674 op/s; 1341/105315 objects degraded (1.273%); 21863/105315 objects misplaced (20.760%); 954 KiB/s, 14 objects/s recovering 2023-10-09T05:33:51.516 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:51.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:51.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:51.621 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:51.647 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:51.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:51.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:51.752 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:51.783 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:51.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:51.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:51.797 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:51 smithi133 bash[105589]: cluster 2023-10-09T05:33:50.341549+0000 mgr.y (mgr.25277) 1373 : cluster 0 pgmap v1472: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 674 op/s; 1341/105315 objects degraded (1.273%); 21863/105315 objects misplaced (20.760%); 954 KiB/s, 14 objects/s recovering 2023-10-09T05:33:51.887 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:51.944 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:51.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:51.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:52.052 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:52.088 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:52.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:52.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:52.193 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:52.227 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:52.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:52.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:52.334 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:52.360 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:52.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:52.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:52.465 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:52.513 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:52.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:52.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:52.617 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:52.651 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:52.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:52.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:52.754 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:52.789 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:52.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:52.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:52.895 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:52.919 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:52.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:52.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:52.985 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:33:53.025 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:53.052 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:33:53.067 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:53.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:53.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:53.134 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:33:53.171 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:53.183 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:33:53.219 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:53.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:53.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:53.247 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:33:53.291 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:33:53.325 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:53.349 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:33:53.354 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:53.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:53.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:53.403 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:33:53.405 INFO:teuthology.orchestra.run.smithi133.stdout:moved 20 / 267 (7.49064%) 2023-10-09T05:33:53.405 INFO:teuthology.orchestra.run.smithi133.stdout:avg 38.1429 2023-10-09T05:33:53.405 INFO:teuthology.orchestra.run.smithi133.stdout:stddev 4.0507 -> 4.94046 (expected baseline 5.71786) 2023-10-09T05:33:53.405 INFO:teuthology.orchestra.run.smithi133.stdout:min osd.5 with 32 -> 35 pgs (0.838951 -> 0.917603 * mean) 2023-10-09T05:33:53.405 INFO:teuthology.orchestra.run.smithi133.stdout:max osd.1 with 45 -> 42 pgs (1.17978 -> 1.10112 * mean) 2023-10-09T05:33:53.405 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:33:53.406 INFO:teuthology.orchestra.run.smithi133.stdout:oload 110 2023-10-09T05:33:53.406 INFO:teuthology.orchestra.run.smithi133.stdout:max_change 1 2023-10-09T05:33:53.406 INFO:teuthology.orchestra.run.smithi133.stdout:max_change_osds 4 2023-10-09T05:33:53.406 INFO:teuthology.orchestra.run.smithi133.stdout:average_utilization 0.0209 2023-10-09T05:33:53.406 INFO:teuthology.orchestra.run.smithi133.stdout:overload_utilization 0.0230 2023-10-09T05:33:53.406 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7 weight 1.0000 -> 0.8100 2023-10-09T05:33:53.406 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2 weight 1.0000 -> 0.8703 2023-10-09T05:33:53.406 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1 weight 1.0000 -> 0.8942 2023-10-09T05:33:53.412 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:33:53.412 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:33:53.412 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:33:53.412 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:33:53.412 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:33:53.412 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:33:53.413 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:33:53.413 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:33:53.414 INFO:teuthology.orchestra.run.smithi133.stderr:no change 2023-10-09T05:33:53.457 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is unset 2023-10-09T05:33:53.463 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:53.507 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:53.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:53.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:53.611 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:53.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:53 smithi194 bash[92666]: cluster 2023-10-09T05:33:52.342211+0000 mgr.y (mgr.25277) 1374 : cluster 0 pgmap v1473: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 1431/101706 objects degraded (1.407%); 21061/101706 objects misplaced (20.708%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T05:33:53.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:53 smithi194 bash[92666]: audit 2023-10-09T05:33:52.918955+0000 mon.c (mon.2) 128 : audit 0 from='client.? 172.21.15.133:0/246972634' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:33:53.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:53 smithi194 bash[92666]: audit 2023-10-09T05:33:53.328216+0000 mon.a (mon.0) 1721 : audit 1 from='client.? 172.21.15.133:0/3540691706' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:33:53.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:53.664 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:53 smithi133 bash[102858]: cluster 2023-10-09T05:33:52.342211+0000 mgr.y (mgr.25277) 1374 : cluster 0 pgmap v1473: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 1431/101706 objects degraded (1.407%); 21061/101706 objects misplaced (20.708%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T05:33:53.664 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:53 smithi133 bash[102858]: audit 2023-10-09T05:33:52.918955+0000 mon.c (mon.2) 128 : audit 0 from='client.? 172.21.15.133:0/246972634' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:33:53.664 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:53 smithi133 bash[102858]: audit 2023-10-09T05:33:53.328216+0000 mon.a (mon.0) 1721 : audit 1 from='client.? 172.21.15.133:0/3540691706' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:33:53.664 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:33:53 smithi133 bash[91128]: debug 2023-10-09T05:33:53.399+0000 7ff45d5ed700 -1 mgr.server reply reply (3) No such process no change 2023-10-09T05:33:53.665 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:53 smithi133 bash[105589]: cluster 2023-10-09T05:33:52.342211+0000 mgr.y (mgr.25277) 1374 : cluster 0 pgmap v1473: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 1431/101706 objects degraded (1.407%); 21061/101706 objects misplaced (20.708%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T05:33:53.665 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:53 smithi133 bash[105589]: audit 2023-10-09T05:33:52.918955+0000 mon.c (mon.2) 128 : audit 0 from='client.? 172.21.15.133:0/246972634' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:33:53.665 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:53 smithi133 bash[105589]: audit 2023-10-09T05:33:53.328216+0000 mon.a (mon.0) 1721 : audit 1 from='client.? 172.21.15.133:0/3540691706' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:33:53.665 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:53.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:53.766 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:53.796 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:53.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:53.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:53.904 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:53.933 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:53.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:53.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:54.035 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:54.067 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:54.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:54.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:54.171 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:54.198 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:54.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:54.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:54.308 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:54.333 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:54.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:54.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:54.432 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:54 smithi194 bash[92666]: audit 2023-10-09T05:33:53.403085+0000 mgr.y (mgr.25277) 1375 : audit 0 from='client.45677 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:33:54.432 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:54 smithi194 bash[92666]: cluster 2023-10-09T05:33:53.426453+0000 mon.a (mon.0) 1722 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:33:54.432 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:54 smithi194 bash[92666]: audit 2023-10-09T05:33:53.441910+0000 mon.a (mon.0) 1723 : audit 1 from='client.? 172.21.15.133:0/3540691706' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:33:54.432 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:54 smithi194 bash[92666]: cluster 2023-10-09T05:33:53.442125+0000 mon.a (mon.0) 1724 : cluster 0 osdmap e304: 8 total, 8 up, 7 in 2023-10-09T05:33:54.432 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:54 smithi194 bash[92666]: cluster 2023-10-09T05:33:53.450596+0000 osd.7 (osd.7) 266 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T05:33:54.432 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:54 smithi194 bash[92666]: cluster 2023-10-09T05:33:53.457117+0000 osd.6 (osd.6) 263 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T05:33:54.433 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:54 smithi194 bash[92666]: cluster 2023-10-09T05:33:53.767549+0000 osd.4 (osd.4) 273 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T05:33:54.433 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:54 smithi194 bash[92666]: cluster 2023-10-09T05:33:53.818814+0000 mon.a (mon.0) 1725 : cluster 3 Health check update: Degraded data redundancy: 1431/101706 objects degraded (1.407%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:33:54.436 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:54.461 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:54.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:54.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:54.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:33:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:33:54] "GET /metrics HTTP/1.1" 200 36497 "" "Prometheus/2.43.0" 2023-10-09T05:33:54.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:54 smithi133 bash[102858]: audit 2023-10-09T05:33:53.403085+0000 mgr.y (mgr.25277) 1375 : audit 0 from='client.45677 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:33:54.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:54 smithi133 bash[102858]: cluster 2023-10-09T05:33:53.426453+0000 mon.a (mon.0) 1722 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:33:54.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:54 smithi133 bash[102858]: audit 2023-10-09T05:33:53.441910+0000 mon.a (mon.0) 1723 : audit 1 from='client.? 172.21.15.133:0/3540691706' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:33:54.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:54 smithi133 bash[102858]: cluster 2023-10-09T05:33:53.442125+0000 mon.a (mon.0) 1724 : cluster 0 osdmap e304: 8 total, 8 up, 7 in 2023-10-09T05:33:54.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:54 smithi133 bash[102858]: cluster 2023-10-09T05:33:53.450596+0000 osd.7 (osd.7) 266 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T05:33:54.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:54 smithi133 bash[102858]: cluster 2023-10-09T05:33:53.457117+0000 osd.6 (osd.6) 263 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T05:33:54.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:54 smithi133 bash[102858]: cluster 2023-10-09T05:33:53.767549+0000 osd.4 (osd.4) 273 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T05:33:54.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:54 smithi133 bash[102858]: cluster 2023-10-09T05:33:53.818814+0000 mon.a (mon.0) 1725 : cluster 3 Health check update: Degraded data redundancy: 1431/101706 objects degraded (1.407%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:33:54.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:54 smithi133 bash[105589]: audit 2023-10-09T05:33:53.403085+0000 mgr.y (mgr.25277) 1375 : audit 0 from='client.45677 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:33:54.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:54 smithi133 bash[105589]: cluster 2023-10-09T05:33:53.426453+0000 mon.a (mon.0) 1722 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:33:54.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:54 smithi133 bash[105589]: audit 2023-10-09T05:33:53.441910+0000 mon.a (mon.0) 1723 : audit 1 from='client.? 172.21.15.133:0/3540691706' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:33:54.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:54 smithi133 bash[105589]: cluster 2023-10-09T05:33:53.442125+0000 mon.a (mon.0) 1724 : cluster 0 osdmap e304: 8 total, 8 up, 7 in 2023-10-09T05:33:54.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:54 smithi133 bash[105589]: cluster 2023-10-09T05:33:53.450596+0000 osd.7 (osd.7) 266 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T05:33:54.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:54 smithi133 bash[105589]: cluster 2023-10-09T05:33:53.457117+0000 osd.6 (osd.6) 263 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T05:33:54.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:54 smithi133 bash[105589]: cluster 2023-10-09T05:33:53.767549+0000 osd.4 (osd.4) 273 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T05:33:54.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:54 smithi133 bash[105589]: cluster 2023-10-09T05:33:53.818814+0000 mon.a (mon.0) 1725 : cluster 3 Health check update: Degraded data redundancy: 1431/101706 objects degraded (1.407%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:33:54.565 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:54.600 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:54.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:54.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:54.706 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:54.731 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:54.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:54.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:54.836 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:54.875 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:54.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:54.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:54.984 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:55.017 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:55.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:55.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:55.122 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:55.160 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:55.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:55.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:55.263 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:55.290 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:55.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:55.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:55.393 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:55.421 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:55.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:55.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:55.525 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:55.562 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:55.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:55.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:55.668 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:55.677 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:55 smithi194 bash[92666]: cluster 2023-10-09T05:33:54.342696+0000 mgr.y (mgr.25277) 1376 : cluster 0 pgmap v1475: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 637 op/s; 1431/101706 objects degraded (1.407%); 21061/101706 objects misplaced (20.708%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T05:33:55.696 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:55.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:55.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:55.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:55 smithi133 bash[102858]: cluster 2023-10-09T05:33:54.342696+0000 mgr.y (mgr.25277) 1376 : cluster 0 pgmap v1475: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 637 op/s; 1431/101706 objects degraded (1.407%); 21061/101706 objects misplaced (20.708%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T05:33:55.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:55 smithi133 bash[105589]: cluster 2023-10-09T05:33:54.342696+0000 mgr.y (mgr.25277) 1376 : cluster 0 pgmap v1475: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 637 op/s; 1431/101706 objects degraded (1.407%); 21061/101706 objects misplaced (20.708%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T05:33:55.804 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:55.844 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:55.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:55.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:55.950 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:55.975 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:55.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:55.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:56.078 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:56.107 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:56.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:56.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:56.215 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:56.240 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:56.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:56.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:56.344 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:56.373 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:56.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:56.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:56.479 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:56.510 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:56.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:56.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:56.617 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:56.641 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:56.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:56.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:56.743 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:56.776 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:56.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:56.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:56.880 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:56.903 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:56.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:56.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:57.010 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:57.040 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:57.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:57.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:57.145 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:57.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:57.174 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:57.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:57.275 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:57.312 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:57.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:57.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:57.417 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:57.422 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-09T05:33:57.422 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 0.05 2023-10-09T05:33:57.434 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:57 smithi194 bash[92666]: cluster 2023-10-09T05:33:56.343732+0000 mgr.y (mgr.25277) 1377 : cluster 0 pgmap v1476: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 624 op/s; 1516/92520 objects degraded (1.639%); 19143/92520 objects misplaced (20.691%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T05:33:57.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:57.494 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:57.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:57.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:57 smithi133 bash[102858]: cluster 2023-10-09T05:33:56.343732+0000 mgr.y (mgr.25277) 1377 : cluster 0 pgmap v1476: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 624 op/s; 1516/92520 objects degraded (1.639%); 19143/92520 objects misplaced (20.691%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T05:33:57.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:57 smithi133 bash[105589]: cluster 2023-10-09T05:33:56.343732+0000 mgr.y (mgr.25277) 1377 : cluster 0 pgmap v1476: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 624 op/s; 1516/92520 objects degraded (1.639%); 19143/92520 objects misplaced (20.691%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T05:33:57.594 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:57.628 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:57.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:57.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:57.731 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:57.767 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:57.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:57.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:57.874 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:57.909 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:57.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:57.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:58.012 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:58.036 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:58.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:58.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:58.141 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:58.164 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:58.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:58.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:58.267 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:58.293 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:58.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:58.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:58.397 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:58.445 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:58.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:58.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:58.548 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:58.576 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:58.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:58.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:58.684 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:58.709 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:58.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:58.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:58.811 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:58.844 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:58.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:58.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:58.921 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:33:58.951 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:58.989 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:58.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:58.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:59.095 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:59.130 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:59.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:59.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:59.234 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:59.269 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:59.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:59.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:59.368 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:33:59.373 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:59.400 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:33:59.453 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:59.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:59.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:59.563 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:59.603 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:59.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:59.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:59.715 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:59.726 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:59 smithi194 bash[92666]: cluster 2023-10-09T05:33:58.344223+0000 mgr.y (mgr.25277) 1378 : cluster 0 pgmap v1477: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 624 op/s; 1516/92520 objects degraded (1.639%); 19143/92520 objects misplaced (20.691%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T05:33:59.726 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:59 smithi194 bash[92666]: audit 2023-10-09T05:33:58.490567+0000 mon.a (mon.0) 1726 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:33:59.726 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:33:59 smithi194 bash[92666]: cluster 2023-10-09T05:33:58.820057+0000 mon.a (mon.0) 1727 : cluster 3 Health check update: Degraded data redundancy: 1516/92520 objects degraded (1.639%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:33:59.748 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:59.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:59.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:59.852 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:59.878 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:59.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:59.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:59.896 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:59 smithi133 bash[105589]: cluster 2023-10-09T05:33:58.344223+0000 mgr.y (mgr.25277) 1378 : cluster 0 pgmap v1477: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 624 op/s; 1516/92520 objects degraded (1.639%); 19143/92520 objects misplaced (20.691%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T05:33:59.896 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:59 smithi133 bash[105589]: audit 2023-10-09T05:33:58.490567+0000 mon.a (mon.0) 1726 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:33:59.896 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:33:59 smithi133 bash[105589]: cluster 2023-10-09T05:33:58.820057+0000 mon.a (mon.0) 1727 : cluster 3 Health check update: Degraded data redundancy: 1516/92520 objects degraded (1.639%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:33:59.896 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:59 smithi133 bash[102858]: cluster 2023-10-09T05:33:58.344223+0000 mgr.y (mgr.25277) 1378 : cluster 0 pgmap v1477: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 624 op/s; 1516/92520 objects degraded (1.639%); 19143/92520 objects misplaced (20.691%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T05:33:59.897 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:59 smithi133 bash[102858]: audit 2023-10-09T05:33:58.490567+0000 mon.a (mon.0) 1726 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:33:59.897 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:33:59 smithi133 bash[102858]: cluster 2023-10-09T05:33:58.820057+0000 mon.a (mon.0) 1727 : cluster 3 Health check update: Degraded data redundancy: 1516/92520 objects degraded (1.639%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:33:59.986 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:00.022 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:00.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:00.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:00.130 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:00.162 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:00.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:00.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:00.269 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:00.305 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:00.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:00.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:00.412 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:00.436 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:00.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:00.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:00.544 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:00.572 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:00.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:00.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:00.679 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:00.718 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:00.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:00.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:00.825 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:00.864 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:00.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:00.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:00.969 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:00.998 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:01.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:01.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:01.105 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:01.139 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:01.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:01.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:01.247 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:01.279 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:01.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:01.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:01.386 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:01.434 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:01.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:01.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:01.540 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:01.547 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:01 smithi133 bash[105589]: cluster 2023-10-09T05:34:00.344915+0000 mgr.y (mgr.25277) 1379 : cluster 0 pgmap v1478: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 1516/86778 objects degraded (1.747%); 17990/86778 objects misplaced (20.731%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T05:34:01.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:01 smithi133 bash[102858]: cluster 2023-10-09T05:34:00.344915+0000 mgr.y (mgr.25277) 1379 : cluster 0 pgmap v1478: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 1516/86778 objects degraded (1.747%); 17990/86778 objects misplaced (20.731%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T05:34:01.572 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:01.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:01.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:01.679 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:01.712 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:01.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:01.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:01.815 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:01.839 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:01.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:01.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:01.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:01 smithi194 bash[92666]: cluster 2023-10-09T05:34:00.344915+0000 mgr.y (mgr.25277) 1379 : cluster 0 pgmap v1478: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 1516/86778 objects degraded (1.747%); 17990/86778 objects misplaced (20.731%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T05:34:01.947 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:01.976 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:01.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:01.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:01.986 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:34:02.083 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:02.111 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:02.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:02.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:02.226 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:02.258 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:02.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:02.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:02.363 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:02.397 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:02.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:02.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:02.502 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:02.525 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:02.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:02.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:02.628 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:02.657 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:02.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:02.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:02.762 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:02.787 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:02.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:02.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:02.893 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:02.931 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:02.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:02.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:03.036 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:03.069 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:03.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:03.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:03.173 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:03.200 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:03.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:03.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:03.303 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:03.326 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:03.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:03.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:03.430 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:03.451 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:03.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:03.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:03.555 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:03.582 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:03.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:03.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:03.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:03 smithi194 bash[92666]: cluster 2023-10-09T05:34:02.345616+0000 mgr.y (mgr.25277) 1380 : cluster 0 pgmap v1479: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 611 op/s; 1657/83340 objects degraded (1.988%); 17218/83340 objects misplaced (20.660%); 1.1 MiB/s, 16 objects/s recovering 2023-10-09T05:34:03.691 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:03.701 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:03 smithi133 bash[102858]: cluster 2023-10-09T05:34:02.345616+0000 mgr.y (mgr.25277) 1380 : cluster 0 pgmap v1479: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 611 op/s; 1657/83340 objects degraded (1.988%); 17218/83340 objects misplaced (20.660%); 1.1 MiB/s, 16 objects/s recovering 2023-10-09T05:34:03.701 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:03 smithi133 bash[105589]: cluster 2023-10-09T05:34:02.345616+0000 mgr.y (mgr.25277) 1380 : cluster 0 pgmap v1479: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 611 op/s; 1657/83340 objects degraded (1.988%); 17218/83340 objects misplaced (20.660%); 1.1 MiB/s, 16 objects/s recovering 2023-10-09T05:34:03.721 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:03.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:03.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:03.826 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:03.854 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:03.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:03.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:03.961 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:03.987 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:03.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:03.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:04.090 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:04.136 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:04.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:04.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:04.241 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:04.263 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:04.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:04.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:04.366 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:04.396 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:04.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:04.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:04.504 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:04.513 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:04 smithi194 bash[92666]: cluster 2023-10-09T05:34:03.821050+0000 mon.a (mon.0) 1728 : cluster 3 Health check update: Degraded data redundancy: 1657/83340 objects degraded (1.988%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:04.532 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:04.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:04.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:04.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:04 smithi133 bash[102858]: cluster 2023-10-09T05:34:03.821050+0000 mon.a (mon.0) 1728 : cluster 3 Health check update: Degraded data redundancy: 1657/83340 objects degraded (1.988%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:04.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:34:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:34:04] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.43.0" 2023-10-09T05:34:04.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:04 smithi133 bash[105589]: cluster 2023-10-09T05:34:03.821050+0000 mon.a (mon.0) 1728 : cluster 3 Health check update: Degraded data redundancy: 1657/83340 objects degraded (1.988%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:04.639 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:04.678 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:04.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:04.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:04.784 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:04.807 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:04.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:04.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:04.911 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:04.938 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:34:04.938 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:04.940 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:34:04.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:04.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:05.047 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:05.080 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:05.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:05.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:05.184 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:05.206 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:05.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:05.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:05.309 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:05.335 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:05.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:05.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:05.446 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:05.482 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:05.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:05.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:05.589 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:05.618 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:05.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:05.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:05.721 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:05.745 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:05.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:05.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:05.848 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:05.858 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:05 smithi194 bash[92666]: cluster 2023-10-09T05:34:04.346091+0000 mgr.y (mgr.25277) 1381 : cluster 0 pgmap v1480: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 1657/83340 objects degraded (1.988%); 17218/83340 objects misplaced (20.660%); 997 KiB/s, 15 objects/s recovering 2023-10-09T05:34:05.877 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:05.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:05.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:05.919 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:05 smithi133 bash[105589]: cluster 2023-10-09T05:34:04.346091+0000 mgr.y (mgr.25277) 1381 : cluster 0 pgmap v1480: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 1657/83340 objects degraded (1.988%); 17218/83340 objects misplaced (20.660%); 997 KiB/s, 15 objects/s recovering 2023-10-09T05:34:05.919 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:05 smithi133 bash[102858]: cluster 2023-10-09T05:34:04.346091+0000 mgr.y (mgr.25277) 1381 : cluster 0 pgmap v1480: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 1657/83340 objects degraded (1.988%); 17218/83340 objects misplaced (20.660%); 997 KiB/s, 15 objects/s recovering 2023-10-09T05:34:05.981 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:06.020 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:06.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:06.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:06.124 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:06.150 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:06.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:06.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:06.254 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:06.295 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:06.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:06.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:06.399 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:06.433 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:06.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:06.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:06.536 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:06.575 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:06.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:06.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:06.681 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:06.716 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:06.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:06.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:06.861 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:06.891 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:06.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:06.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:06.996 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:07.026 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:07.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:07.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:07.129 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:07.154 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:07.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:07.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:07.260 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:07.304 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:07.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:07.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:07.407 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:07.444 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:07.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:07.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:07.470 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:07 smithi194 bash[92666]: cluster 2023-10-09T05:34:06.346868+0000 mgr.y (mgr.25277) 1382 : cluster 0 pgmap v1481: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 732 op/s; 1657/75330 objects degraded (2.200%); 15504/75330 objects misplaced (20.581%); 906 KiB/s, 14 objects/s recovering 2023-10-09T05:34:07.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:07 smithi133 bash[102858]: cluster 2023-10-09T05:34:06.346868+0000 mgr.y (mgr.25277) 1382 : cluster 0 pgmap v1481: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 732 op/s; 1657/75330 objects degraded (2.200%); 15504/75330 objects misplaced (20.581%); 906 KiB/s, 14 objects/s recovering 2023-10-09T05:34:07.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:07 smithi133 bash[105589]: cluster 2023-10-09T05:34:06.346868+0000 mgr.y (mgr.25277) 1382 : cluster 0 pgmap v1481: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 732 op/s; 1657/75330 objects degraded (2.200%); 15504/75330 objects misplaced (20.581%); 906 KiB/s, 14 objects/s recovering 2023-10-09T05:34:07.552 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:07.582 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:07.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:07.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:07.687 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:07.723 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:07.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:07.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:07.832 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:07.868 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:07.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:07.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:07.975 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:08.013 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:08.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:08.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:08.124 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:08.153 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:08.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:08.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:08.255 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:08.282 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:08.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:08.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:08.388 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:08.418 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:08.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:08.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:08.524 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:08.554 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:08.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:08.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:08.656 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:08.687 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:08.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:08.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:08.793 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:08.834 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:08.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:08.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:08.938 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:08.981 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:08.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:08.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:09.086 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:09.111 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:09.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:09.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:09.219 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:09.253 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:09.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:09.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:09.272 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:34:09.272 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:34:09.358 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:09.389 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:09.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:09.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:09.493 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:09.526 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:09.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:09.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:09.631 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:09 smithi133 bash[105589]: cluster 2023-10-09T05:34:08.347397+0000 mgr.y (mgr.25277) 1383 : cluster 0 pgmap v1482: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.5 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 1758/74277 objects degraded (2.367%); 15323/74277 objects misplaced (20.630%); 837 KiB/s, 13 objects/s recovering 2023-10-09T05:34:09.631 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:09 smithi133 bash[105589]: cluster 2023-10-09T05:34:08.822249+0000 mon.a (mon.0) 1729 : cluster 3 Health check update: Degraded data redundancy: 1758/74277 objects degraded (2.367%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:09.631 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:09 smithi133 bash[102858]: cluster 2023-10-09T05:34:08.347397+0000 mgr.y (mgr.25277) 1383 : cluster 0 pgmap v1482: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.5 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 1758/74277 objects degraded (2.367%); 15323/74277 objects misplaced (20.630%); 837 KiB/s, 13 objects/s recovering 2023-10-09T05:34:09.631 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:09 smithi133 bash[102858]: cluster 2023-10-09T05:34:08.822249+0000 mon.a (mon.0) 1729 : cluster 3 Health check update: Degraded data redundancy: 1758/74277 objects degraded (2.367%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:09.632 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:09.666 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:09.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:09.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:09.773 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:09.795 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:09.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:09.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:09.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:09 smithi194 bash[92666]: cluster 2023-10-09T05:34:08.347397+0000 mgr.y (mgr.25277) 1383 : cluster 0 pgmap v1482: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.5 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 1758/74277 objects degraded (2.367%); 15323/74277 objects misplaced (20.630%); 837 KiB/s, 13 objects/s recovering 2023-10-09T05:34:09.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:09 smithi194 bash[92666]: cluster 2023-10-09T05:34:08.822249+0000 mon.a (mon.0) 1729 : cluster 3 Health check update: Degraded data redundancy: 1758/74277 objects degraded (2.367%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:09.903 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:09.927 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:09.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:09.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:10.033 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:10.068 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:10.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:10.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:10.171 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:10.201 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:10.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:10.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:10.303 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:10.328 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:10.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:10.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:10.432 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:10.464 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:10.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:10.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:10.568 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:10.606 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:10.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:10.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:10.713 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:10.744 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:10.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:10.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:10.847 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:10.877 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:10.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:10.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:10.984 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:11.028 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:11.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:11.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:11.130 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:11.157 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:11.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:11.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:11.260 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:11.291 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:11.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:11.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:11.396 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:11.426 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:11.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:11.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:11.475 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:11 smithi194 bash[92666]: cluster 2023-10-09T05:34:10.348058+0000 mgr.y (mgr.25277) 1384 : cluster 0 pgmap v1483: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 1758/68799 objects degraded (2.555%); 14266/68799 objects misplaced (20.736%); 837 KiB/s, 13 objects/s recovering 2023-10-09T05:34:11.533 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:11.551 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:11.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:11.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:11.556 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:11 smithi133 bash[102858]: cluster 2023-10-09T05:34:10.348058+0000 mgr.y (mgr.25277) 1384 : cluster 0 pgmap v1483: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 1758/68799 objects degraded (2.555%); 14266/68799 objects misplaced (20.736%); 837 KiB/s, 13 objects/s recovering 2023-10-09T05:34:11.556 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:11 smithi133 bash[105589]: cluster 2023-10-09T05:34:10.348058+0000 mgr.y (mgr.25277) 1384 : cluster 0 pgmap v1483: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 1758/68799 objects degraded (2.555%); 14266/68799 objects misplaced (20.736%); 837 KiB/s, 13 objects/s recovering 2023-10-09T05:34:11.655 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:11.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:11.692 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:11.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:11.793 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:11.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:11.825 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:11.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:11.926 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:11.959 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:11.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:11.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:12.066 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:12.094 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:12.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:12.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:12.200 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:12.223 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:12.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:12.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:12.328 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:12.349 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:12.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:12.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:12.455 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:12.499 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:12.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:12.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:12.602 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:12.641 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:12.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:12.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:12.747 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:12.779 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:12.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:12.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:12.884 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:12.910 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:12.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:12.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:13.014 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:13.042 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:13.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:13.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:13.149 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:13.173 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:13.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:13.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:13.280 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:13.312 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:13.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:13.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:13.416 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:13.451 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:13.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:13.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:13.537 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T05:34:13.557 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:13.590 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:13.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:13.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:13.695 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:13.716 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:13 smithi133 bash[102858]: cluster 2023-10-09T05:34:12.348736+0000 mgr.y (mgr.25277) 1385 : cluster 0 pgmap v1484: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 1872/65241 objects degraded (2.869%); 13494/65241 objects misplaced (20.683%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:34:13.717 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:13 smithi133 bash[105589]: cluster 2023-10-09T05:34:12.348736+0000 mgr.y (mgr.25277) 1385 : cluster 0 pgmap v1484: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 1872/65241 objects degraded (2.869%); 13494/65241 objects misplaced (20.683%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:34:13.733 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:13.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:13.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:13.844 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:13.877 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:13.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:13.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:13.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:13 smithi194 bash[92666]: cluster 2023-10-09T05:34:12.348736+0000 mgr.y (mgr.25277) 1385 : cluster 0 pgmap v1484: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 1872/65241 objects degraded (2.869%); 13494/65241 objects misplaced (20.683%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:34:13.981 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:14.017 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:14.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:14.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:14.123 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:14.150 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:14.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:14.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:14.257 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:14.281 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:14.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:14.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:14.386 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:14.395 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:34:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:34:14] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.43.0" 2023-10-09T05:34:14.413 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:14.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:14.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:14.516 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:14.554 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:14.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:14.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:14.657 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:14.665 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:14 smithi133 bash[102858]: audit 2023-10-09T05:34:13.488713+0000 mon.a (mon.0) 1730 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:34:14.665 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:14 smithi133 bash[102858]: cluster 2023-10-09T05:34:13.823358+0000 mon.a (mon.0) 1731 : cluster 3 Health check update: Degraded data redundancy: 1872/65241 objects degraded (2.869%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:14.666 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:14 smithi133 bash[105589]: audit 2023-10-09T05:34:13.488713+0000 mon.a (mon.0) 1730 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:34:14.666 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:14 smithi133 bash[105589]: cluster 2023-10-09T05:34:13.823358+0000 mon.a (mon.0) 1731 : cluster 3 Health check update: Degraded data redundancy: 1872/65241 objects degraded (2.869%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:14.684 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:14.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:14.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:14.794 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:14.802 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:14 smithi194 bash[92666]: audit 2023-10-09T05:34:13.488713+0000 mon.a (mon.0) 1730 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:34:14.802 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:14 smithi194 bash[92666]: cluster 2023-10-09T05:34:13.823358+0000 mon.a (mon.0) 1731 : cluster 3 Health check update: Degraded data redundancy: 1872/65241 objects degraded (2.869%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:14.830 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:14.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:14.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:14.936 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:14.959 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:14.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:14.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:15.065 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:15.106 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:15.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:15.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:15.213 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:15.249 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:15.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:15.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:15.354 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:15.384 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:15.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:15.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:15.489 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:15 smithi133 bash[102858]: cluster 2023-10-09T05:34:14.349102+0000 mgr.y (mgr.25277) 1386 : cluster 0 pgmap v1485: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 502 op/s; 1872/65241 objects degraded (2.869%); 13494/65241 objects misplaced (20.683%); 933 KiB/s, 14 objects/s recovering 2023-10-09T05:34:15.489 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:15 smithi133 bash[105589]: cluster 2023-10-09T05:34:14.349102+0000 mgr.y (mgr.25277) 1386 : cluster 0 pgmap v1485: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 502 op/s; 1872/65241 objects degraded (2.869%); 13494/65241 objects misplaced (20.683%); 933 KiB/s, 14 objects/s recovering 2023-10-09T05:34:15.490 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:15.526 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:15.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:15.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:15.634 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:15.657 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:15.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:15.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:15.761 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:15.787 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:15.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:15.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:15.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:15 smithi194 bash[92666]: cluster 2023-10-09T05:34:14.349102+0000 mgr.y (mgr.25277) 1386 : cluster 0 pgmap v1485: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 502 op/s; 1872/65241 objects degraded (2.869%); 13494/65241 objects misplaced (20.683%); 933 KiB/s, 14 objects/s recovering 2023-10-09T05:34:15.895 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:15.939 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:15.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:15.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:16.045 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:16.077 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:16.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:16.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:16.181 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:16.206 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:16.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:16.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:16.316 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:16.344 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:16.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:16.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:16.451 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:16.482 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:16.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:16.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:16.585 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:16.615 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:16.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:16.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:16.719 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:16.760 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:16.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:16.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:16.863 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:16.888 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:16.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:16.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:16.995 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:17.040 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:17.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:17.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:17.144 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:17.182 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:17.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:17.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:17.288 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:17.318 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:17.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:17.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:17.426 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:17.459 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:17.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:17.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:17.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:17 smithi133 bash[102858]: cluster 2023-10-09T05:34:16.349727+0000 mgr.y (mgr.25277) 1387 : cluster 0 pgmap v1486: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 742 op/s; 1872/56610 objects degraded (3.307%); 11625/56610 objects misplaced (20.535%); 933 KiB/s, 14 objects/s recovering 2023-10-09T05:34:17.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:17 smithi133 bash[105589]: cluster 2023-10-09T05:34:16.349727+0000 mgr.y (mgr.25277) 1387 : cluster 0 pgmap v1486: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 742 op/s; 1872/56610 objects degraded (3.307%); 11625/56610 objects misplaced (20.535%); 933 KiB/s, 14 objects/s recovering 2023-10-09T05:34:17.567 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:17.596 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:17.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:17.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:17.703 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:17.713 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:17 smithi194 bash[92666]: cluster 2023-10-09T05:34:16.349727+0000 mgr.y (mgr.25277) 1387 : cluster 0 pgmap v1486: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 742 op/s; 1872/56610 objects degraded (3.307%); 11625/56610 objects misplaced (20.535%); 933 KiB/s, 14 objects/s recovering 2023-10-09T05:34:17.732 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:17.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:17.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:17.840 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:17.867 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:17.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:17.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:17.970 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:18.000 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:18.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:18.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:18.107 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:18.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:18.155 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:18.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:18.257 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:18.282 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:18.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:18.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:18.386 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:18.425 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:34:18.426 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:18.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:18.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:18.532 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:18.560 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:18.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:18.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:18.666 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:18.693 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:18.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:18.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:18.796 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:18.819 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:18.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:18.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:18.924 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:18.953 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:18.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:18.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:19.059 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:19.099 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:19.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:19.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:19.203 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:19.233 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:19.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:19.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:19.339 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:19.366 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:19.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:19.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:19.473 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:19.496 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:19.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:19.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:19.599 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:19.607 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:19 smithi194 bash[92666]: cluster 2023-10-09T05:34:18.350358+0000 mgr.y (mgr.25277) 1388 : cluster 0 pgmap v1487: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 1997/55377 objects degraded (3.606%); 11407/55377 objects misplaced (20.599%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T05:34:19.607 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:19 smithi194 bash[92666]: cluster 2023-10-09T05:34:18.824212+0000 mon.a (mon.0) 1732 : cluster 3 Health check update: Degraded data redundancy: 1997/55377 objects degraded (3.606%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:19.621 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:19.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:19.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:19.725 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:19.748 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:19.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:19.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:19.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:19 smithi133 bash[102858]: cluster 2023-10-09T05:34:18.350358+0000 mgr.y (mgr.25277) 1388 : cluster 0 pgmap v1487: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 1997/55377 objects degraded (3.606%); 11407/55377 objects misplaced (20.599%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T05:34:19.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:19 smithi133 bash[102858]: cluster 2023-10-09T05:34:18.824212+0000 mon.a (mon.0) 1732 : cluster 3 Health check update: Degraded data redundancy: 1997/55377 objects degraded (3.606%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:19.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:19 smithi133 bash[105589]: cluster 2023-10-09T05:34:18.350358+0000 mgr.y (mgr.25277) 1388 : cluster 0 pgmap v1487: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 1997/55377 objects degraded (3.606%); 11407/55377 objects misplaced (20.599%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T05:34:19.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:19 smithi133 bash[105589]: cluster 2023-10-09T05:34:18.824212+0000 mon.a (mon.0) 1732 : cluster 3 Health check update: Degraded data redundancy: 1997/55377 objects degraded (3.606%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:19.852 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:19.877 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:19.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:19.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:19.984 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:20.017 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:20.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:20.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:20.124 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:20.164 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:20.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:20.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:20.269 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:20.300 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:20.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:20.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:20.408 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:20.450 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:20.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:20.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:20.556 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:20.577 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:20.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:20.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:20.682 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:20.716 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:20.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:20.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:20.823 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:20.862 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:20.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:20.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:20.967 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:20.988 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:20.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:20.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:21.092 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:21.134 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:21.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:21.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:21.238 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:21.266 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:21.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:21.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:21.371 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:21.416 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:21.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:21.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:21.514 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is unset 2023-10-09T05:34:21.516 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:21 smithi133 bash[102858]: cluster 2023-10-09T05:34:20.351078+0000 mgr.y (mgr.25277) 1389 : cluster 0 pgmap v1488: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 686 op/s; 1997/49569 objects degraded (4.029%); 10245/49569 objects misplaced (20.668%); 1002 KiB/s, 15 objects/s recovering 2023-10-09T05:34:21.516 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:21 smithi133 bash[102858]: audit 2023-10-09T05:34:21.089656+0000 mon.b (mon.1) 31 : audit 1 from='client.? 172.21.15.133:0/1139589712' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:34:21.516 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:21 smithi133 bash[102858]: audit 2023-10-09T05:34:21.090047+0000 mon.a (mon.0) 1733 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:34:21.522 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:21.552 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:21.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:21.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:21.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:21 smithi194 bash[92666]: cluster 2023-10-09T05:34:20.351078+0000 mgr.y (mgr.25277) 1389 : cluster 0 pgmap v1488: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 686 op/s; 1997/49569 objects degraded (4.029%); 10245/49569 objects misplaced (20.668%); 1002 KiB/s, 15 objects/s recovering 2023-10-09T05:34:21.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:21 smithi194 bash[92666]: audit 2023-10-09T05:34:21.089656+0000 mon.b (mon.1) 31 : audit 1 from='client.? 172.21.15.133:0/1139589712' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:34:21.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:21 smithi194 bash[92666]: audit 2023-10-09T05:34:21.090047+0000 mon.a (mon.0) 1733 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:34:21.659 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:21.688 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:21.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:21.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:21.790 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:21.827 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:21.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:21.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:21.833 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:21 smithi133 bash[105589]: cluster 2023-10-09T05:34:20.351078+0000 mgr.y (mgr.25277) 1389 : cluster 0 pgmap v1488: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 1.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 686 op/s; 1997/49569 objects degraded (4.029%); 10245/49569 objects misplaced (20.668%); 1002 KiB/s, 15 objects/s recovering 2023-10-09T05:34:21.834 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:21 smithi133 bash[105589]: audit 2023-10-09T05:34:21.089656+0000 mon.b (mon.1) 31 : audit 1 from='client.? 172.21.15.133:0/1139589712' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:34:21.834 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:21 smithi133 bash[105589]: audit 2023-10-09T05:34:21.090047+0000 mon.a (mon.0) 1733 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:34:21.934 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:21.971 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:21.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:21.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:22.032 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:34:22.077 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:22.088 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:34:22.115 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:22.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:22.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:22.137 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:34:22.167 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:34:22.201 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:34:22.220 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:22.240 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:34:22.258 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:22.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:22.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:22.288 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:34:22.329 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:34:22.343 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:34:22.343 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:34:22.343 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:34:22.343 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:34:22.343 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:34:22.343 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:34:22.343 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:34:22.343 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:34:22.367 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:22.417 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:22.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:22.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:22.521 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:22.528 INFO:teuthology.orchestra.run.smithi133.stdout:moved 0 / 267 (0%) 2023-10-09T05:34:22.528 INFO:teuthology.orchestra.run.smithi133.stdout:avg 38.1429 2023-10-09T05:34:22.528 INFO:teuthology.orchestra.run.smithi133.stdout:stddev 4.0507 -> 4.0507 (expected baseline 5.71786) 2023-10-09T05:34:22.528 INFO:teuthology.orchestra.run.smithi133.stdout:min osd.5 with 32 -> 32 pgs (0.838951 -> 0.838951 * mean) 2023-10-09T05:34:22.528 INFO:teuthology.orchestra.run.smithi133.stdout:max osd.1 with 45 -> 45 pgs (1.17978 -> 1.17978 * mean) 2023-10-09T05:34:22.528 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:34:22.529 INFO:teuthology.orchestra.run.smithi133.stdout:oload 1000 2023-10-09T05:34:22.529 INFO:teuthology.orchestra.run.smithi133.stdout:max_change 0.05 2023-10-09T05:34:22.529 INFO:teuthology.orchestra.run.smithi133.stdout:max_change_osds 4 2023-10-09T05:34:22.529 INFO:teuthology.orchestra.run.smithi133.stdout:average_utilization 0.0157 2023-10-09T05:34:22.529 INFO:teuthology.orchestra.run.smithi133.stdout:overload_utilization 0.1569 2023-10-09T05:34:22.539 INFO:teuthology.orchestra.run.smithi133.stderr:no change 2023-10-09T05:34:22.558 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:22.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:22.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:22.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:22 smithi194 bash[92666]: cluster 2023-10-09T05:34:21.499528+0000 mon.a (mon.0) 1734 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:34:22.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:22 smithi194 bash[92666]: cluster 2023-10-09T05:34:21.508371+0000 osd.5 (osd.5) 280 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T05:34:22.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:22 smithi194 bash[92666]: audit 2023-10-09T05:34:21.509299+0000 mon.a (mon.0) 1735 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:34:22.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:22 smithi194 bash[92666]: cluster 2023-10-09T05:34:21.509447+0000 mon.a (mon.0) 1736 : cluster 0 osdmap e305: 8 total, 8 up, 7 in 2023-10-09T05:34:22.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:22 smithi194 bash[92666]: cluster 2023-10-09T05:34:21.510371+0000 osd.7 (osd.7) 267 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T05:34:22.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:22 smithi194 bash[92666]: cluster 2023-10-09T05:34:21.515703+0000 osd.6 (osd.6) 264 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T05:34:22.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:22 smithi194 bash[92666]: cluster 2023-10-09T05:34:21.561961+0000 osd.4 (osd.4) 274 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T05:34:22.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:22 smithi194 bash[92666]: audit 2023-10-09T05:34:21.970893+0000 mon.a (mon.0) 1737 : audit 0 from='client.? 172.21.15.133:0/296118907' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:34:22.662 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:22.679 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:22 smithi133 bash[102858]: cluster 2023-10-09T05:34:21.499528+0000 mon.a (mon.0) 1734 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:34:22.679 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:22 smithi133 bash[102858]: cluster 2023-10-09T05:34:21.508371+0000 osd.5 (osd.5) 280 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T05:34:22.679 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:22 smithi133 bash[102858]: audit 2023-10-09T05:34:21.509299+0000 mon.a (mon.0) 1735 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:34:22.679 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:22 smithi133 bash[102858]: cluster 2023-10-09T05:34:21.509447+0000 mon.a (mon.0) 1736 : cluster 0 osdmap e305: 8 total, 8 up, 7 in 2023-10-09T05:34:22.680 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:22 smithi133 bash[102858]: cluster 2023-10-09T05:34:21.510371+0000 osd.7 (osd.7) 267 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T05:34:22.680 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:22 smithi133 bash[102858]: cluster 2023-10-09T05:34:21.515703+0000 osd.6 (osd.6) 264 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T05:34:22.680 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:22 smithi133 bash[102858]: cluster 2023-10-09T05:34:21.561961+0000 osd.4 (osd.4) 274 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T05:34:22.680 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:22 smithi133 bash[102858]: audit 2023-10-09T05:34:21.970893+0000 mon.a (mon.0) 1737 : audit 0 from='client.? 172.21.15.133:0/296118907' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:34:22.680 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:22 smithi133 bash[105589]: cluster 2023-10-09T05:34:21.499528+0000 mon.a (mon.0) 1734 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:34:22.680 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:22 smithi133 bash[105589]: cluster 2023-10-09T05:34:21.508371+0000 osd.5 (osd.5) 280 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T05:34:22.680 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:22 smithi133 bash[105589]: audit 2023-10-09T05:34:21.509299+0000 mon.a (mon.0) 1735 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:34:22.680 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:22 smithi133 bash[105589]: cluster 2023-10-09T05:34:21.509447+0000 mon.a (mon.0) 1736 : cluster 0 osdmap e305: 8 total, 8 up, 7 in 2023-10-09T05:34:22.680 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:22 smithi133 bash[105589]: cluster 2023-10-09T05:34:21.510371+0000 osd.7 (osd.7) 267 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T05:34:22.681 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:22 smithi133 bash[105589]: cluster 2023-10-09T05:34:21.515703+0000 osd.6 (osd.6) 264 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T05:34:22.681 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:22 smithi133 bash[105589]: cluster 2023-10-09T05:34:21.561961+0000 osd.4 (osd.4) 274 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T05:34:22.681 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:22 smithi133 bash[105589]: audit 2023-10-09T05:34:21.970893+0000 mon.a (mon.0) 1737 : audit 0 from='client.? 172.21.15.133:0/296118907' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:34:22.696 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:22.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:22.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:22.800 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:22.835 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:22.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:22.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:22.943 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:22.989 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:22.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:22.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:23.095 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:23.142 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:23.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:23.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:23.246 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:23.280 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:23.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:23.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:23.389 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:23.417 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:23.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:23.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:23.521 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:23.625 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:23.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:23.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:23.728 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:23.752 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:23.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:23.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:23.760 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:23 smithi133 bash[102858]: cluster 2023-10-09T05:34:22.351592+0000 mgr.y (mgr.25277) 1390 : cluster 0 pgmap v1490: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 960 MiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 2078/46143 objects degraded (4.503%); 9534/46143 objects misplaced (20.662%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T05:34:23.760 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:23 smithi133 bash[102858]: audit 2023-10-09T05:34:22.524084+0000 mgr.y (mgr.25277) 1391 : audit 0 from='client.45692 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:34:23.760 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:23 smithi133 bash[105589]: cluster 2023-10-09T05:34:22.351592+0000 mgr.y (mgr.25277) 1390 : cluster 0 pgmap v1490: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 960 MiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 2078/46143 objects degraded (4.503%); 9534/46143 objects misplaced (20.662%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T05:34:23.761 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:23 smithi133 bash[105589]: audit 2023-10-09T05:34:22.524084+0000 mgr.y (mgr.25277) 1391 : audit 0 from='client.45692 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:34:23.855 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:23.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:23 smithi194 bash[92666]: cluster 2023-10-09T05:34:22.351592+0000 mgr.y (mgr.25277) 1390 : cluster 0 pgmap v1490: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 960 MiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 2078/46143 objects degraded (4.503%); 9534/46143 objects misplaced (20.662%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T05:34:23.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:23 smithi194 bash[92666]: audit 2023-10-09T05:34:22.524084+0000 mgr.y (mgr.25277) 1391 : audit 0 from='client.45692 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:34:23.893 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:23.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:23.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:23.999 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:24.028 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:24.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:24.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:24.135 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:24.173 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:24.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:24.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:24.277 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:24.289 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:34:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:34:24] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.43.0" 2023-10-09T05:34:24.301 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:24.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:24.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:24.404 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:24.452 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:24.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:24.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:24.527 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:24 smithi133 bash[102858]: cluster 2023-10-09T05:34:23.826292+0000 mon.a (mon.0) 1738 : cluster 3 Health check update: Degraded data redundancy: 2078/46143 objects degraded (4.503%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:24.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:24 smithi133 bash[105589]: cluster 2023-10-09T05:34:23.826292+0000 mon.a (mon.0) 1738 : cluster 3 Health check update: Degraded data redundancy: 2078/46143 objects degraded (4.503%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:24.560 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:24.590 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:24.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:24.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:24.699 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:24.728 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:24.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:24.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:24.831 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:24.877 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:24.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:24.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:24.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:24 smithi194 bash[92666]: cluster 2023-10-09T05:34:23.826292+0000 mon.a (mon.0) 1738 : cluster 3 Health check update: Degraded data redundancy: 2078/46143 objects degraded (4.503%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:24.984 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:25.021 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:25.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:25.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:25.129 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:25.146 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:25.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:25.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:25.249 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:25.281 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:25.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:25.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:25.384 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:25.419 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:25.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:25.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:25.526 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:25.573 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:25.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:25.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:25.680 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:25.724 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:25.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:25.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:25.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:25 smithi133 bash[105589]: cluster 2023-10-09T05:34:24.353810+0000 mgr.y (mgr.25277) 1392 : cluster 0 pgmap v1491: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 960 MiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 2078/46143 objects degraded (4.503%); 9534/46143 objects misplaced (20.662%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T05:34:25.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:25 smithi133 bash[102858]: cluster 2023-10-09T05:34:24.353810+0000 mgr.y (mgr.25277) 1392 : cluster 0 pgmap v1491: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 960 MiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 2078/46143 objects degraded (4.503%); 9534/46143 objects misplaced (20.662%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T05:34:25.828 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:25.837 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:25 smithi194 bash[92666]: cluster 2023-10-09T05:34:24.353810+0000 mgr.y (mgr.25277) 1392 : cluster 0 pgmap v1491: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 960 MiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 2078/46143 objects degraded (4.503%); 9534/46143 objects misplaced (20.662%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T05:34:25.861 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:25.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:25.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:25.964 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:25.989 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:25.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:25.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:26.094 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:26.123 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:26.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:26.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:26.227 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:26.261 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:26.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:26.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:26.368 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:26.401 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:26.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:26.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:26.505 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:26.540 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:26.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:26.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:26.644 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:26.691 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:26.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:26.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:26.776 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-10-09T05:34:26.776 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 0.05 2023-10-09T05:34:26.796 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:26.821 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:26.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:26.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:26.925 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:26.969 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:26.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:26.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:27.072 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:27.104 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:27.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:27.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:27.210 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:27.241 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:27.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:27.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:27.351 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:27.395 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:27.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:27.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:27.499 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:27.531 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:27.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:27.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:27.541 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:27 smithi133 bash[102858]: cluster 2023-10-09T05:34:26.354610+0000 mgr.y (mgr.25277) 1393 : cluster 0 pgmap v1492: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 796 MiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 611 op/s; 2078/38268 objects degraded (5.430%); 7824/38268 objects misplaced (20.445%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T05:34:27.636 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:27.669 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:27.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:27.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:27.775 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:27.783 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:27 smithi194 bash[92666]: cluster 2023-10-09T05:34:26.354610+0000 mgr.y (mgr.25277) 1393 : cluster 0 pgmap v1492: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 796 MiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 611 op/s; 2078/38268 objects degraded (5.430%); 7824/38268 objects misplaced (20.445%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T05:34:27.797 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:27 smithi133 bash[105589]: cluster 2023-10-09T05:34:26.354610+0000 mgr.y (mgr.25277) 1393 : cluster 0 pgmap v1492: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 796 MiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 611 op/s; 2078/38268 objects degraded (5.430%); 7824/38268 objects misplaced (20.445%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T05:34:27.802 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:27.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:27.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:27.904 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:27.928 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:27.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:27.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:28.031 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:28.065 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:28.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:28.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:28.173 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:28.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:28.203 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:28.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:28.304 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:28.330 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:28.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:28.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:28.438 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:28.466 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:28.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:28.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:28.572 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:28.606 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:28.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:28.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:28.711 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:28.749 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:28.752 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:34:28.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:28.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:28.762 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:34:28.858 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:28.923 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:28.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:28.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:29.033 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:29.059 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:29.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:29.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:29.163 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:29.187 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:29.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:29.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:29.272 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:34:29.272 INFO:teuthology.orchestra.run.smithi133.stdout: "ops": [ 2023-10-09T05:34:29.272 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.272 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:186277 13.2 13:48912fdf:::benchmark_data_smithi133_241067_object88237:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e305)", 2023-10-09T05:34:29.272 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:34:29.254877+0000", 2023-10-09T05:34:29.272 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.016374972000000002, 2023-10-09T05:34:29.272 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.016410194999999999, 2023-10-09T05:34:29.272 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:34:29.273 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:34:29.273 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:34:29.273 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:34:29.273 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:34:29.273 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 186277 2023-10-09T05:34:29.273 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.273 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:34:29.273 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.273 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:34:29.273 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.254877+0000", 2023-10-09T05:34:29.273 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:34:29.273 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.274 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.274 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:34:29.274 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.254877+0000", 2023-10-09T05:34:29.274 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:34:29.274 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.274 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.274 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:34:29.274 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.254875+0000", 2023-10-09T05:34:29.274 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:34:29.274 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.274 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.275 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:34:29.275 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.254883+0000", 2023-10-09T05:34:29.275 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.622e-06 2023-10-09T05:34:29.275 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.275 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.275 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:34:29.275 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.254896+0000", 2023-10-09T05:34:29.275 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.3789e-05 2023-10-09T05:34:29.275 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.275 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.275 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:34:29.275 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.254903+0000", 2023-10-09T05:34:29.276 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.6590000000000001e-06 2023-10-09T05:34:29.276 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.276 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.276 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:34:29.276 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.256692+0000", 2023-10-09T05:34:29.276 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.001789276 2023-10-09T05:34:29.276 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.276 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.276 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:34:29.276 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.259454+0000", 2023-10-09T05:34:29.276 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0027612119999999999 2023-10-09T05:34:29.276 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.277 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.277 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T05:34:29.277 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.259545+0000", 2023-10-09T05:34:29.277 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.1312999999999998e-05 2023-10-09T05:34:29.277 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.277 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.277 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:34:29.277 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.264720+0000", 2023-10-09T05:34:29.277 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0051752439999999999 2023-10-09T05:34:29.277 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:34:29.277 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:34:29.277 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:34:29.278 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.278 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.278 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:186278 13.6 13:67623971:::benchmark_data_smithi133_241067_object88238:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e305)", 2023-10-09T05:34:29.278 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:34:29.254915+0000", 2023-10-09T05:34:29.278 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.016336296, 2023-10-09T05:34:29.278 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.016470696, 2023-10-09T05:34:29.278 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:34:29.278 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:34:29.278 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:34:29.278 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:34:29.278 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:34:29.278 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 186278 2023-10-09T05:34:29.279 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.279 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:34:29.279 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.279 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:34:29.279 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.254915+0000", 2023-10-09T05:34:29.279 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:34:29.279 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.279 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.279 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:34:29.279 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.254915+0000", 2023-10-09T05:34:29.279 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:34:29.279 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.280 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.280 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:34:29.280 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.254915+0000", 2023-10-09T05:34:29.280 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999995 2023-10-09T05:34:29.280 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.280 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.280 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:34:29.281 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.254917+0000", 2023-10-09T05:34:29.281 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.142e-06 2023-10-09T05:34:29.281 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.281 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.281 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:34:29.281 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.254921+0000", 2023-10-09T05:34:29.281 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.9509999999999999e-06 2023-10-09T05:34:29.281 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.281 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.281 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:34:29.281 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.254924+0000", 2023-10-09T05:34:29.282 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.937e-06 2023-10-09T05:34:29.282 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.282 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.282 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:34:29.282 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.254974+0000", 2023-10-09T05:34:29.282 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.9654000000000003e-05 2023-10-09T05:34:29.282 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.282 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.282 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:34:29.282 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.255135+0000", 2023-10-09T05:34:29.282 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00016110000000000001 2023-10-09T05:34:29.282 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.283 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.283 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 0,2,5", 2023-10-09T05:34:29.283 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.255293+0000", 2023-10-09T05:34:29.283 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00015835400000000001 2023-10-09T05:34:29.283 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.283 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.283 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:29.283 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.263033+0000", 2023-10-09T05:34:29.283 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0077404010000000001 2023-10-09T05:34:29.283 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.283 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.283 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:34:29.284 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.263042+0000", 2023-10-09T05:34:29.284 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.3820000000000004e-06 2023-10-09T05:34:29.284 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.284 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.284 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:29.284 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.268349+0000", 2023-10-09T05:34:29.284 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0053071849999999999 2023-10-09T05:34:29.284 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:34:29.284 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:34:29.284 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:34:29.284 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.284 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.285 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:186283 13.6 13:615c6869:::benchmark_data_smithi133_241067_object88243:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e305)", 2023-10-09T05:34:29.285 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:34:29.260441+0000", 2023-10-09T05:34:29.285 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.010810181, 2023-10-09T05:34:29.285 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.01105073, 2023-10-09T05:34:29.285 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:34:29.285 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:34:29.285 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:34:29.285 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:34:29.285 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:34:29.285 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 186283 2023-10-09T05:34:29.285 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.286 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:34:29.286 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.286 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:34:29.286 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.260441+0000", 2023-10-09T05:34:29.286 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:34:29.286 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.286 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.286 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:34:29.286 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.260441+0000", 2023-10-09T05:34:29.286 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:34:29.286 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.286 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.287 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:34:29.287 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.260440+0000", 2023-10-09T05:34:29.287 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:34:29.287 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.287 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.287 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:34:29.287 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.260446+0000", 2023-10-09T05:34:29.287 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.302e-06 2023-10-09T05:34:29.287 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.287 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.287 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:34:29.287 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.260466+0000", 2023-10-09T05:34:29.288 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.9987e-05 2023-10-09T05:34:29.288 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.288 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.288 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:34:29.288 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.260475+0000", 2023-10-09T05:34:29.288 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.4479999999999994e-06 2023-10-09T05:34:29.288 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.288 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.288 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:34:29.288 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.263066+0000", 2023-10-09T05:34:29.288 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0025913300000000002 2023-10-09T05:34:29.288 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.289 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.289 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:34:29.289 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.263212+0000", 2023-10-09T05:34:29.289 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00014605799999999999 2023-10-09T05:34:29.289 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.289 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.289 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 0,2,5", 2023-10-09T05:34:29.289 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.263378+0000", 2023-10-09T05:34:29.289 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000165623 2023-10-09T05:34:29.289 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.289 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.290 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:34:29.290 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.268366+0000", 2023-10-09T05:34:29.290 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0049882080000000001 2023-10-09T05:34:29.290 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:34:29.290 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:34:29.290 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:34:29.290 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.290 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.290 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:186285 13.2 13:4f51dc9f:::benchmark_data_smithi133_241067_object88245:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e305)", 2023-10-09T05:34:29.290 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:34:29.264968+0000", 2023-10-09T05:34:29.290 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.006283711, 2023-10-09T05:34:29.290 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.006615063, 2023-10-09T05:34:29.291 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:34:29.291 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:34:29.291 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:34:29.291 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:34:29.291 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:34:29.291 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 186285 2023-10-09T05:34:29.291 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.291 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:34:29.291 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.291 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:34:29.291 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.264968+0000", 2023-10-09T05:34:29.292 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:34:29.292 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.292 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.292 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:34:29.292 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.264968+0000", 2023-10-09T05:34:29.292 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:34:29.292 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.292 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.293 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:34:29.293 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.264966+0000", 2023-10-09T05:34:29.293 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:34:29.293 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.293 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.293 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:34:29.293 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.264974+0000", 2023-10-09T05:34:29.293 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.9750000000000007e-06 2023-10-09T05:34:29.293 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.293 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.293 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:34:29.294 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.264987+0000", 2023-10-09T05:34:29.294 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.272e-05 2023-10-09T05:34:29.294 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.294 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.294 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:34:29.294 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.264995+0000", 2023-10-09T05:34:29.294 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.1049999999999995e-06 2023-10-09T05:34:29.294 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.295 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.295 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:34:29.295 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.266366+0000", 2023-10-09T05:34:29.295 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.001370436 2023-10-09T05:34:29.295 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.295 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.295 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:34:29.295 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.266839+0000", 2023-10-09T05:34:29.295 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00047381000000000002 2023-10-09T05:34:29.295 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.295 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.296 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T05:34:29.296 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.267339+0000", 2023-10-09T05:34:29.296 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000499421 2023-10-09T05:34:29.296 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:34:29.296 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:34:29.296 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:34:29.296 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.297 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.297 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_repop_reply(client.45596.0:186277 13.2 e305/301)", 2023-10-09T05:34:29.297 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:34:29.269875+0000", 2023-10-09T05:34:29.297 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.001376337, 2023-10-09T05:34:29.297 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.001784073, 2023-10-09T05:34:29.297 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:34:29.297 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "queued for pg", 2023-10-09T05:34:29.297 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:34:29.297 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.297 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:34:29.297 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.269875+0000", 2023-10-09T05:34:29.298 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:34:29.298 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.298 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.298 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:34:29.298 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.269875+0000", 2023-10-09T05:34:29.298 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:34:29.298 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.298 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.298 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:34:29.299 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.269874+0000", 2023-10-09T05:34:29.299 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:34:29.299 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.299 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.299 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:34:29.299 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.269884+0000", 2023-10-09T05:34:29.299 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.7550000000000006e-06 2023-10-09T05:34:29.299 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.299 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.299 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:34:29.300 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.269901+0000", 2023-10-09T05:34:29.300 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.7017e-05 2023-10-09T05:34:29.300 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.300 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.300 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:34:29.300 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.269909+0000", 2023-10-09T05:34:29.300 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.8019999999999992e-06 2023-10-09T05:34:29.300 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:34:29.300 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:34:29.300 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:34:29.300 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.301 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.301 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:186288 13.2 13.d5ea1a32 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e305)", 2023-10-09T05:34:29.301 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:34:29.269933+0000", 2023-10-09T05:34:29.301 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.001318128, 2023-10-09T05:34:29.301 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0017785100000000001, 2023-10-09T05:34:29.301 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:34:29.301 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "queued for pg", 2023-10-09T05:34:29.301 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:34:29.301 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:34:29.301 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:34:29.301 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 186288 2023-10-09T05:34:29.301 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.302 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:34:29.302 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.302 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:34:29.302 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.269933+0000", 2023-10-09T05:34:29.302 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:34:29.302 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.302 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.302 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:34:29.303 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.269933+0000", 2023-10-09T05:34:29.303 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:34:29.303 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.303 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.303 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:34:29.303 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.269932+0000", 2023-10-09T05:34:29.303 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:34:29.303 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.303 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.303 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:34:29.303 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.269937+0000", 2023-10-09T05:34:29.304 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.0039999999999999e-06 2023-10-09T05:34:29.304 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.304 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.304 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:34:29.304 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.269949+0000", 2023-10-09T05:34:29.304 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.2322e-05 2023-10-09T05:34:29.304 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:34:29.304 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:34:29.305 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:34:29.305 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:29.269956+0000", 2023-10-09T05:34:29.305 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.7649999999999996e-06 2023-10-09T05:34:29.305 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:34:29.305 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:34:29.305 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:34:29.305 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:34:29.305 INFO:teuthology.orchestra.run.smithi133.stdout: ], 2023-10-09T05:34:29.305 INFO:teuthology.orchestra.run.smithi133.stdout: "num_ops": 6 2023-10-09T05:34:29.306 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:34:29.306 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:29.335 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:29.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:29.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:29.442 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:29.472 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:29.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:29.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:29.575 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:29.601 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:29.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:29.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:29.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:29 smithi194 bash[92666]: cluster 2023-10-09T05:34:28.355141+0000 mgr.y (mgr.25277) 1394 : cluster 0 pgmap v1493: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 774 MiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 2143/37185 objects degraded (5.763%); 7647/37185 objects misplaced (20.565%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:34:29.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:29 smithi194 bash[92666]: audit 2023-10-09T05:34:28.491518+0000 mon.a (mon.0) 1739 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:34:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:29 smithi194 bash[92666]: audit 2023-10-09T05:34:28.492868+0000 mon.a (mon.0) 1740 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:34:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:29 smithi194 bash[92666]: cluster 2023-10-09T05:34:28.827420+0000 mon.a (mon.0) 1741 : cluster 3 Health check update: Degraded data redundancy: 2143/37185 objects degraded (5.763%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:29.710 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:29.751 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:29.758 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:29 smithi133 bash[105589]: cluster 2023-10-09T05:34:28.355141+0000 mgr.y (mgr.25277) 1394 : cluster 0 pgmap v1493: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 774 MiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 2143/37185 objects degraded (5.763%); 7647/37185 objects misplaced (20.565%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:34:29.758 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:29 smithi133 bash[105589]: audit 2023-10-09T05:34:28.491518+0000 mon.a (mon.0) 1739 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:34:29.758 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:29 smithi133 bash[105589]: audit 2023-10-09T05:34:28.492868+0000 mon.a (mon.0) 1740 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:34:29.758 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:29 smithi133 bash[105589]: cluster 2023-10-09T05:34:28.827420+0000 mon.a (mon.0) 1741 : cluster 3 Health check update: Degraded data redundancy: 2143/37185 objects degraded (5.763%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:29.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:29.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:29.759 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:29 smithi133 bash[102858]: cluster 2023-10-09T05:34:28.355141+0000 mgr.y (mgr.25277) 1394 : cluster 0 pgmap v1493: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 774 MiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 2143/37185 objects degraded (5.763%); 7647/37185 objects misplaced (20.565%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:34:29.759 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:29 smithi133 bash[102858]: audit 2023-10-09T05:34:28.491518+0000 mon.a (mon.0) 1739 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:34:29.759 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:29 smithi133 bash[102858]: audit 2023-10-09T05:34:28.492868+0000 mon.a (mon.0) 1740 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:34:29.760 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:29 smithi133 bash[102858]: cluster 2023-10-09T05:34:28.827420+0000 mon.a (mon.0) 1741 : cluster 3 Health check update: Degraded data redundancy: 2143/37185 objects degraded (5.763%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:29.859 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:29.885 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:29.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:29.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:29.989 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:30.015 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:30.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:30.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:30.123 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:30.158 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:30.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:30.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:30.266 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:30.301 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:30.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:30.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:30.404 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:30.432 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:30.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:30.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:30.536 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:30.568 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:30.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:30.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:30.672 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:30.703 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:30.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:30.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:30.808 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:30.839 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:30.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:30.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:30.945 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:30.982 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:30.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:30.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:31.089 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:31.113 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:31.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:31.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:31.128 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T05:34:31.216 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:31.244 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:31.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:31.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:31.347 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:31.372 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:31.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:31.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:31.476 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:31.512 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:31.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:31.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:31.615 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:31.640 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:31.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:31.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:31.747 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:31.756 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:31 smithi133 bash[105589]: cluster 2023-10-09T05:34:30.355848+0000 mgr.y (mgr.25277) 1395 : cluster 0 pgmap v1494: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 648 MiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 2143/31149 objects degraded (6.880%); 6442/31149 objects misplaced (20.681%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:34:31.758 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:31 smithi133 bash[102858]: cluster 2023-10-09T05:34:30.355848+0000 mgr.y (mgr.25277) 1395 : cluster 0 pgmap v1494: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 648 MiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 2143/31149 objects degraded (6.880%); 6442/31149 objects misplaced (20.681%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:34:31.785 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:31.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:31.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:31.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:31 smithi194 bash[92666]: cluster 2023-10-09T05:34:30.355848+0000 mgr.y (mgr.25277) 1395 : cluster 0 pgmap v1494: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 648 MiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 2143/31149 objects degraded (6.880%); 6442/31149 objects misplaced (20.681%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:34:31.890 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:31.916 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:31.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:31.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:32.020 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:32.059 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:32.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:32.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:32.162 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:32.190 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:32.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:32.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:32.296 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:32.321 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:32.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:32.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:32.428 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:32.465 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:32.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:32.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:32.568 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:32.593 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:32.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:32.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:32.696 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:32.737 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:32.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:32.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:32.840 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:32.870 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:32.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:32.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:32.974 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:33.006 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:33.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:33.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:33.109 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:33.150 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:33.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:33.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:33.256 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:33.290 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:33.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:33.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:33.396 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:33.436 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:33.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:33.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:33.540 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:33.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:33 smithi133 bash[102858]: cluster 2023-10-09T05:34:32.356470+0000 mgr.y (mgr.25277) 1396 : cluster 0 pgmap v1495: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 573 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 570 op/s; 2238/27567 objects degraded (8.118%); 5644/27567 objects misplaced (20.474%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T05:34:33.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:33 smithi133 bash[105589]: cluster 2023-10-09T05:34:32.356470+0000 mgr.y (mgr.25277) 1396 : cluster 0 pgmap v1495: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 573 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 570 op/s; 2238/27567 objects degraded (8.118%); 5644/27567 objects misplaced (20.474%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T05:34:33.569 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:33.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:33.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:33.658 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:34:33.660 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:34:33.664 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:34:33.673 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:33.724 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:33.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:33.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:33.835 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:33.873 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:33.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:33.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:33.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:33 smithi194 bash[92666]: cluster 2023-10-09T05:34:32.356470+0000 mgr.y (mgr.25277) 1396 : cluster 0 pgmap v1495: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 573 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 570 op/s; 2238/27567 objects degraded (8.118%); 5644/27567 objects misplaced (20.474%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T05:34:33.982 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:34.028 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:34.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:34.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:34.133 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:34.169 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:34.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:34.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:34.272 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:34.295 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:34.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:34.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:34.402 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:34.412 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:34:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:34:34] "GET /metrics HTTP/1.1" 200 36493 "" "Prometheus/2.43.0" 2023-10-09T05:34:34.435 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:34.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:34.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:34.540 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:34.548 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:34 smithi194 bash[92666]: cluster 2023-10-09T05:34:33.828585+0000 mon.a (mon.0) 1742 : cluster 3 Health check update: Degraded data redundancy: 2238/27567 objects degraded (8.118%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:34.572 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:34.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:34.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:34.677 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:34.686 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:34 smithi133 bash[102858]: cluster 2023-10-09T05:34:33.828585+0000 mon.a (mon.0) 1742 : cluster 3 Health check update: Degraded data redundancy: 2238/27567 objects degraded (8.118%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:34.687 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:34 smithi133 bash[105589]: cluster 2023-10-09T05:34:33.828585+0000 mon.a (mon.0) 1742 : cluster 3 Health check update: Degraded data redundancy: 2238/27567 objects degraded (8.118%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:34.708 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:34.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:34.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:34.813 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:34.861 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:34.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:34.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:34.967 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:35.000 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:35.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:35.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:35.104 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:35.137 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:35.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:35.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:35.245 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:35.291 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:35.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:35.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:35.397 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:35.434 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:35.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:35.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:35.538 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:35.562 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:35.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:35.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:35.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:35 smithi194 bash[92666]: cluster 2023-10-09T05:34:34.356950+0000 mgr.y (mgr.25277) 1397 : cluster 0 pgmap v1496: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 573 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 2238/27567 objects degraded (8.118%); 5644/27567 objects misplaced (20.474%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T05:34:35.668 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:35.689 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:34:35.698 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:35.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:35.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:35.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:35 smithi133 bash[105589]: cluster 2023-10-09T05:34:34.356950+0000 mgr.y (mgr.25277) 1397 : cluster 0 pgmap v1496: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 573 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 2238/27567 objects degraded (8.118%); 5644/27567 objects misplaced (20.474%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T05:34:35.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:35 smithi133 bash[102858]: cluster 2023-10-09T05:34:34.356950+0000 mgr.y (mgr.25277) 1397 : cluster 0 pgmap v1496: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 573 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 2238/27567 objects degraded (8.118%); 5644/27567 objects misplaced (20.474%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T05:34:35.804 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:35.829 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:35.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:35.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:35.933 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:35.961 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:35.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:35.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:36.065 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:36.097 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:36.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:36.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:36.203 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:36.232 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:36.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:36.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:36.336 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:36.362 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:36.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:36.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:36.466 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:36.498 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:36.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:36.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:36.605 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:36.644 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:36.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:36.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:36.748 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:36.778 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:36.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:36.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:36.881 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:36.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:36.921 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:36.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:37.023 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:37.126 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:37.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:37.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:37.235 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:37.263 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:37.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:37.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:37.366 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:37.399 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:37.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:37.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:37.503 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:37.531 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:37.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:37.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:37.635 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:37.661 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:37.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:37.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:37.765 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:37.773 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:37 smithi194 bash[92666]: cluster 2023-10-09T05:34:36.357808+0000 mgr.y (mgr.25277) 1398 : cluster 0 pgmap v1497: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 397 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 751 op/s; 2238/19098 objects degraded (11.719%); 3816/19098 objects misplaced (19.981%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T05:34:37.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:37 smithi133 bash[102858]: cluster 2023-10-09T05:34:36.357808+0000 mgr.y (mgr.25277) 1398 : cluster 0 pgmap v1497: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 397 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 751 op/s; 2238/19098 objects degraded (11.719%); 3816/19098 objects misplaced (19.981%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T05:34:37.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:37 smithi133 bash[105589]: cluster 2023-10-09T05:34:36.357808+0000 mgr.y (mgr.25277) 1398 : cluster 0 pgmap v1497: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 397 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 751 op/s; 2238/19098 objects degraded (11.719%); 3816/19098 objects misplaced (19.981%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T05:34:37.800 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:37.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:37.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:37.904 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:37.932 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:37.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:37.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:38.039 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:38.066 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:38.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:38.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:38.173 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:38.211 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:38.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:38.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:38.315 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:38.342 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:38.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:38.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:38.450 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:38.479 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:38.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:38.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:38.582 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:38.604 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:38.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:38.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:38.710 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:38.741 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:38.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:38.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:38.848 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:38.892 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:38.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:38.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:38.996 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:39.031 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:39.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:39.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:39.137 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:39.164 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:39.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:39.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:39.268 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:39.300 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:39.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:39.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:39.404 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:39.436 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:39.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:39.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:39.539 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:39.577 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:39.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:39.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:39.668 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:39 smithi133 bash[105589]: cluster 2023-10-09T05:34:38.358351+0000 mgr.y (mgr.25277) 1399 : cluster 0 pgmap v1498: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 373 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 564 op/s; 2303/17946 objects degraded (12.833%); 3635/17946 objects misplaced (20.255%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T05:34:39.668 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:39 smithi133 bash[105589]: cluster 2023-10-09T05:34:38.829846+0000 mon.a (mon.0) 1743 : cluster 3 Health check update: Degraded data redundancy: 2303/17946 objects degraded (12.833%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:39.669 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:39 smithi133 bash[102858]: cluster 2023-10-09T05:34:38.358351+0000 mgr.y (mgr.25277) 1399 : cluster 0 pgmap v1498: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 373 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 564 op/s; 2303/17946 objects degraded (12.833%); 3635/17946 objects misplaced (20.255%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T05:34:39.669 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:39 smithi133 bash[102858]: cluster 2023-10-09T05:34:38.829846+0000 mon.a (mon.0) 1743 : cluster 3 Health check update: Degraded data redundancy: 2303/17946 objects degraded (12.833%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:39.683 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:39.715 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:39.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:39.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:39.821 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:39.830 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:39 smithi194 bash[92666]: cluster 2023-10-09T05:34:38.358351+0000 mgr.y (mgr.25277) 1399 : cluster 0 pgmap v1498: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 373 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 564 op/s; 2303/17946 objects degraded (12.833%); 3635/17946 objects misplaced (20.255%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T05:34:39.830 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:39 smithi194 bash[92666]: cluster 2023-10-09T05:34:38.829846+0000 mon.a (mon.0) 1743 : cluster 3 Health check update: Degraded data redundancy: 2303/17946 objects degraded (12.833%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:39.852 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:39.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:39.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:39.960 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:40.000 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:40.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:40.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:40.105 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:40.132 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:40.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:40.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:40.237 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:40.269 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:40.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:40.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:40.377 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:40.408 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:40.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:40.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:40.514 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:40.541 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:40.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:40.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:40.644 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:40.686 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:40.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:40.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:40.790 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:40.815 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:40.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:40.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:40.919 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:40.956 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:40.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:40.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:41.065 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:41.097 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:41.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:41.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:41.202 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:41.242 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:41.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:41.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:41.346 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:41.383 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:41.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:41.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:41.486 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:41.523 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:41.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:41.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:41.627 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:41.653 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:41.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:41.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:41.680 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:41 smithi133 bash[102858]: cluster 2023-10-09T05:34:40.358923+0000 mgr.y (mgr.25277) 1400 : cluster 0 pgmap v1499: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 252 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 695 op/s; 2303/12141 objects degraded (18.969%); 2505/12141 objects misplaced (20.633%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T05:34:41.680 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:41 smithi133 bash[105589]: cluster 2023-10-09T05:34:40.358923+0000 mgr.y (mgr.25277) 1400 : cluster 0 pgmap v1499: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 252 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 695 op/s; 2303/12141 objects degraded (18.969%); 2505/12141 objects misplaced (20.633%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T05:34:41.756 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:41.789 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:41.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:41.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:41.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:41 smithi194 bash[92666]: cluster 2023-10-09T05:34:40.358923+0000 mgr.y (mgr.25277) 1400 : cluster 0 pgmap v1499: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 252 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 695 op/s; 2303/12141 objects degraded (18.969%); 2505/12141 objects misplaced (20.633%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T05:34:41.893 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:41.928 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:41.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:41.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:42.041 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:42.073 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:42.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:42.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:42.179 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:42.206 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:42.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:42.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:42.310 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:42.343 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:42.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:42.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:42.451 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:42.481 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:42.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:42.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:42.585 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:42.608 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:42.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:42.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:42.711 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:42.741 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:42.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:42.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:42.844 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:42.888 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:42.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:42.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:42.994 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:43.027 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:43.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:43.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:43.131 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:43.191 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:43.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:43.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:43.296 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:43.340 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:43.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:43.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:43.447 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:43.482 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:43.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:43.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:43.585 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:43.614 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:43.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:43.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:43.721 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:43.728 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:43 smithi194 bash[92666]: cluster 2023-10-09T05:34:42.359546+0000 mgr.y (mgr.25277) 1401 : cluster 0 pgmap v1500: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 178 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 2356/8574 objects degraded (27.478%); 1742/8574 objects misplaced (20.317%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T05:34:43.729 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:43 smithi194 bash[92666]: audit 2023-10-09T05:34:43.489424+0000 mon.a (mon.0) 1744 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:34:43.755 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:43.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:43.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:43.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:43 smithi133 bash[105589]: cluster 2023-10-09T05:34:42.359546+0000 mgr.y (mgr.25277) 1401 : cluster 0 pgmap v1500: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 178 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 2356/8574 objects degraded (27.478%); 1742/8574 objects misplaced (20.317%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T05:34:43.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:43 smithi133 bash[105589]: audit 2023-10-09T05:34:43.489424+0000 mon.a (mon.0) 1744 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:34:43.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:43 smithi133 bash[102858]: cluster 2023-10-09T05:34:42.359546+0000 mgr.y (mgr.25277) 1401 : cluster 0 pgmap v1500: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 178 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 2356/8574 objects degraded (27.478%); 1742/8574 objects misplaced (20.317%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T05:34:43.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:43 smithi133 bash[102858]: audit 2023-10-09T05:34:43.489424+0000 mon.a (mon.0) 1744 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:34:43.861 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:43.885 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:43.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:43.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:43.998 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:44.037 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:44.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:44.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:44.140 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:44.173 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:44.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:44.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:44.278 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:44.303 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:44.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:44.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:44.407 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:44.434 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:44.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:44.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:44.537 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:44.546 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:44 smithi194 bash[92666]: audit 2023-10-09T05:34:43.759794+0000 mon.a (mon.0) 1745 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:34:44.546 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:44 smithi194 bash[92666]: cluster 2023-10-09T05:34:43.833408+0000 mon.a (mon.0) 1746 : cluster 3 Health check update: Degraded data redundancy: 2356/8574 objects degraded (27.478%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:44.546 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:44 smithi194 bash[92666]: audit 2023-10-09T05:34:44.287031+0000 mon.a (mon.0) 1747 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:34:44.546 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:44 smithi194 bash[92666]: audit 2023-10-09T05:34:44.294661+0000 mon.a (mon.0) 1748 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:34:44.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:44 smithi133 bash[102858]: audit 2023-10-09T05:34:43.759794+0000 mon.a (mon.0) 1745 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:34:44.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:44 smithi133 bash[102858]: cluster 2023-10-09T05:34:43.833408+0000 mon.a (mon.0) 1746 : cluster 3 Health check update: Degraded data redundancy: 2356/8574 objects degraded (27.478%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:44.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:44 smithi133 bash[102858]: audit 2023-10-09T05:34:44.287031+0000 mon.a (mon.0) 1747 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:34:44.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:44 smithi133 bash[102858]: audit 2023-10-09T05:34:44.294661+0000 mon.a (mon.0) 1748 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:34:44.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:34:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:34:44] "GET /metrics HTTP/1.1" 200 36490 "" "Prometheus/2.43.0" 2023-10-09T05:34:44.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:44 smithi133 bash[105589]: audit 2023-10-09T05:34:43.759794+0000 mon.a (mon.0) 1745 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:34:44.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:44 smithi133 bash[105589]: cluster 2023-10-09T05:34:43.833408+0000 mon.a (mon.0) 1746 : cluster 3 Health check update: Degraded data redundancy: 2356/8574 objects degraded (27.478%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:44.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:44 smithi133 bash[105589]: audit 2023-10-09T05:34:44.287031+0000 mon.a (mon.0) 1747 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:34:44.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:44 smithi133 bash[105589]: audit 2023-10-09T05:34:44.294661+0000 mon.a (mon.0) 1748 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:34:44.571 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:44.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:44.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:44.681 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:44.719 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:44.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:44.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:44.825 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:44.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:44.886 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:44.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:44.987 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:45.024 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:45.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:45.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:45.096 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Removed 98037 objects 2023-10-09T05:34:45.096 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Clean up completed and total clean up time :154.558 2023-10-09T05:34:45.130 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:45.147 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:45.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:45.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:45.250 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:45.273 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:45.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:45.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:45.376 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:45.397 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:45.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:45.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:45.500 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:45.529 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:45.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:45.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:45.634 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:45.645 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:45 smithi133 bash[102858]: cluster 2023-10-09T05:34:44.360000+0000 mgr.y (mgr.25277) 1402 : cluster 0 pgmap v1501: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 178 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 2356/8574 objects degraded (27.478%); 1742/8574 objects misplaced (20.317%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T05:34:45.646 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:45 smithi133 bash[105589]: cluster 2023-10-09T05:34:44.360000+0000 mgr.y (mgr.25277) 1402 : cluster 0 pgmap v1501: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 178 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 2356/8574 objects degraded (27.478%); 1742/8574 objects misplaced (20.317%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T05:34:45.659 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:45.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:45.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:45.763 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:45.788 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:45.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:45.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:45.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:45 smithi194 bash[92666]: cluster 2023-10-09T05:34:44.360000+0000 mgr.y (mgr.25277) 1402 : cluster 0 pgmap v1501: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 178 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 2356/8574 objects degraded (27.478%); 1742/8574 objects misplaced (20.317%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T05:34:45.891 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:45.917 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:45.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:45.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:46.021 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:46.049 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:46.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:46.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:46.153 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:46.187 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:46.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:46.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:46.291 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:46.318 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:46.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:46.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:46.422 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:46.446 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:46.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:46.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:46.549 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:46.582 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:46.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:46.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:46.686 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:46.712 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:46.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:46.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:46.816 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:46.849 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:46.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:46.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:46.953 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:46.982 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:46.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:46.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:47.086 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:47.116 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:47.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:47.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:47.219 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:47.254 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:47.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:47.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:47.359 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:47.386 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:47.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:47.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:47.489 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:47.526 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:47.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:47.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:47.629 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:47.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:47 smithi194 bash[92666]: cluster 2023-10-09T05:34:46.361483+0000 mgr.y (mgr.25277) 1403 : cluster 0 pgmap v1502: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 23 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 733 op/s; 2356/1152 objects degraded (204.514%); 183/1152 objects misplaced (15.885%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T05:34:47.659 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:47.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:47.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:47.767 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:47.779 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:47 smithi133 bash[102858]: cluster 2023-10-09T05:34:46.361483+0000 mgr.y (mgr.25277) 1403 : cluster 0 pgmap v1502: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 23 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 733 op/s; 2356/1152 objects degraded (204.514%); 183/1152 objects misplaced (15.885%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T05:34:47.779 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:47 smithi133 bash[105589]: cluster 2023-10-09T05:34:46.361483+0000 mgr.y (mgr.25277) 1403 : cluster 0 pgmap v1502: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 23 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 733 op/s; 2356/1152 objects degraded (204.514%); 183/1152 objects misplaced (15.885%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T05:34:47.798 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:47.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:47.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:47.901 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:47.932 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:47.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:47.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:48.037 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:48.068 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:48.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:48.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:48.172 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:48.204 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:48.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:48.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:48.309 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:48.351 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:48.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:48.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:48.455 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:48.486 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:48.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:48.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:48.589 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:48.611 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:48.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:48.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:48.714 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:48.739 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:48.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:48.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:48.843 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:48.880 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:48.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:48.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:48.984 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:49.007 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:49.010 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:34:49.010 INFO:teuthology.orchestra.run.smithi133.stdout: "ops": [], 2023-10-09T05:34:49.010 INFO:teuthology.orchestra.run.smithi133.stdout: "complaint_time": 30, 2023-10-09T05:34:49.010 INFO:teuthology.orchestra.run.smithi133.stdout: "num_blocked_ops": 0 2023-10-09T05:34:49.010 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:34:49.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:49.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:49.112 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:49.134 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:49.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:49.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:49.172 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:34:49.194 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:34:49.215 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:34:49.238 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:49.250 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:34:49.264 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:49.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:49.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:49.276 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:34:49.306 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:34:49.330 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:34:49.356 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:34:49.357 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:34:49.358 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:34:49.358 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:34:49.358 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:34:49.358 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:34:49.358 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:34:49.358 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:34:49.358 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:34:49.368 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:49.393 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:49.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:49.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:49.409 INFO:teuthology.orchestra.run.smithi133.stdout:moved 6 / 267 (2.24719%) 2023-10-09T05:34:49.410 INFO:teuthology.orchestra.run.smithi133.stdout:avg 38.1429 2023-10-09T05:34:49.410 INFO:teuthology.orchestra.run.smithi133.stdout:stddev 4.0507 -> 3.35638 (expected baseline 5.71786) 2023-10-09T05:34:49.410 INFO:teuthology.orchestra.run.smithi133.stdout:min osd.5 with 32 -> 34 pgs (0.838951 -> 0.891386 * mean) 2023-10-09T05:34:49.410 INFO:teuthology.orchestra.run.smithi133.stdout:max osd.1 with 45 -> 41 pgs (1.17978 -> 1.07491 * mean) 2023-10-09T05:34:49.410 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:34:49.410 INFO:teuthology.orchestra.run.smithi133.stdout:oload 110 2023-10-09T05:34:49.410 INFO:teuthology.orchestra.run.smithi133.stdout:max_change 0.05 2023-10-09T05:34:49.410 INFO:teuthology.orchestra.run.smithi133.stdout:max_change_osds 4 2023-10-09T05:34:49.410 INFO:teuthology.orchestra.run.smithi133.stdout:average_utilization 0.0114 2023-10-09T05:34:49.411 INFO:teuthology.orchestra.run.smithi133.stdout:overload_utilization 0.0125 2023-10-09T05:34:49.411 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1 weight 1.0000 -> 0.9500 2023-10-09T05:34:49.411 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7 weight 1.0000 -> 0.9500 2023-10-09T05:34:49.414 INFO:teuthology.orchestra.run.smithi133.stderr:no change 2023-10-09T05:34:49.496 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:49.507 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:34:49 smithi133 bash[91128]: debug 2023-10-09T05:34:49.406+0000 7ff45d5ed700 -1 mgr.server reply reply (2) No such file or directory no change 2023-10-09T05:34:49.521 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:49.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:49.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:49.543 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is set 2023-10-09T05:34:49.566 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:49 smithi133 bash[105589]: cluster 2023-10-09T05:34:48.362283+0000 mgr.y (mgr.25277) 1404 : cluster 0 pgmap v1503: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 7.1 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 519 op/s; 2384/387 objects degraded (616.021%); 57/387 objects misplaced (14.729%); 1.7 MiB/s, 38 objects/s recovering 2023-10-09T05:34:49.567 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:49 smithi133 bash[105589]: audit 2023-10-09T05:34:48.762280+0000 mon.c (mon.2) 129 : audit 1 from='client.? 172.21.15.133:0/3543103700' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:34:49.567 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:49 smithi133 bash[105589]: audit 2023-10-09T05:34:48.762726+0000 mon.a (mon.0) 1749 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:34:49.567 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:49 smithi133 bash[105589]: cluster 2023-10-09T05:34:48.834478+0000 mon.a (mon.0) 1750 : cluster 3 Health check update: Degraded data redundancy: 2384/387 objects degraded (616.021%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:49.567 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:49 smithi133 bash[105589]: audit 2023-10-09T05:34:49.147206+0000 mon.a (mon.0) 1751 : audit 0 from='client.? 172.21.15.133:0/3935452575' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:34:49.567 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:49 smithi133 bash[102858]: cluster 2023-10-09T05:34:48.362283+0000 mgr.y (mgr.25277) 1404 : cluster 0 pgmap v1503: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 7.1 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 519 op/s; 2384/387 objects degraded (616.021%); 57/387 objects misplaced (14.729%); 1.7 MiB/s, 38 objects/s recovering 2023-10-09T05:34:49.568 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:49 smithi133 bash[102858]: audit 2023-10-09T05:34:48.762280+0000 mon.c (mon.2) 129 : audit 1 from='client.? 172.21.15.133:0/3543103700' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:34:49.568 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:49 smithi133 bash[102858]: audit 2023-10-09T05:34:48.762726+0000 mon.a (mon.0) 1749 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:34:49.568 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:49 smithi133 bash[102858]: cluster 2023-10-09T05:34:48.834478+0000 mon.a (mon.0) 1750 : cluster 3 Health check update: Degraded data redundancy: 2384/387 objects degraded (616.021%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:49.568 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:49 smithi133 bash[102858]: audit 2023-10-09T05:34:49.147206+0000 mon.a (mon.0) 1751 : audit 0 from='client.? 172.21.15.133:0/3935452575' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:34:49.607 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_8 2023-10-09T05:34:49.607 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_8 unique_pool_8 --yes-i-really-really-mean-it 2023-10-09T05:34:49.625 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:49.634 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:49 smithi194 bash[92666]: cluster 2023-10-09T05:34:48.362283+0000 mgr.y (mgr.25277) 1404 : cluster 0 pgmap v1503: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 7.1 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 519 op/s; 2384/387 objects degraded (616.021%); 57/387 objects misplaced (14.729%); 1.7 MiB/s, 38 objects/s recovering 2023-10-09T05:34:49.634 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:49 smithi194 bash[92666]: audit 2023-10-09T05:34:48.762280+0000 mon.c (mon.2) 129 : audit 1 from='client.? 172.21.15.133:0/3543103700' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:34:49.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:49 smithi194 bash[92666]: audit 2023-10-09T05:34:48.762726+0000 mon.a (mon.0) 1749 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:34:49.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:49 smithi194 bash[92666]: cluster 2023-10-09T05:34:48.834478+0000 mon.a (mon.0) 1750 : cluster 3 Health check update: Degraded data redundancy: 2384/387 objects degraded (616.021%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:49.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:49 smithi194 bash[92666]: audit 2023-10-09T05:34:49.147206+0000 mon.a (mon.0) 1751 : audit 0 from='client.? 172.21.15.133:0/3935452575' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:34:49.648 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:49.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:49.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:49.752 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:49.785 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:49.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:49.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:49.888 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:49.914 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:49.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:49.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:50.018 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:50.038 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:50.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:50.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:50.141 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:50.163 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:50.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:50.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:50.267 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:50.298 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:50.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:50.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:50.402 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:50.432 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:50.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:50.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:50.535 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:50.565 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:50.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:50.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:50.668 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:50.691 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:50.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:50.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:50.734 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:50 smithi133 bash[105589]: audit 2023-10-09T05:34:49.409086+0000 mgr.y (mgr.25277) 1405 : audit 0 from='client.45710 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:34:50.735 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:50 smithi133 bash[105589]: cluster 2023-10-09T05:34:49.530704+0000 mon.a (mon.0) 1752 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:34:50.735 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:50 smithi133 bash[105589]: cluster 2023-10-09T05:34:49.533640+0000 osd.4 (osd.4) 275 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T05:34:50.735 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:50 smithi133 bash[105589]: cluster 2023-10-09T05:34:49.534911+0000 osd.5 (osd.5) 281 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T05:34:50.735 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:50 smithi133 bash[105589]: audit 2023-10-09T05:34:49.535527+0000 mon.a (mon.0) 1753 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:34:50.735 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:50 smithi133 bash[105589]: cluster 2023-10-09T05:34:49.535636+0000 mon.a (mon.0) 1754 : cluster 0 osdmap e306: 8 total, 8 up, 7 in 2023-10-09T05:34:50.735 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:50 smithi133 bash[105589]: audit 2023-10-09T05:34:50.198426+0000 mon.a (mon.0) 1755 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:34:50.735 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:50 smithi133 bash[105589]: audit 2023-10-09T05:34:50.204169+0000 mon.a (mon.0) 1756 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:34:50.735 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:50 smithi133 bash[102858]: audit 2023-10-09T05:34:49.409086+0000 mgr.y (mgr.25277) 1405 : audit 0 from='client.45710 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:34:50.736 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:50 smithi133 bash[102858]: cluster 2023-10-09T05:34:49.530704+0000 mon.a (mon.0) 1752 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:34:50.736 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:50 smithi133 bash[102858]: cluster 2023-10-09T05:34:49.533640+0000 osd.4 (osd.4) 275 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T05:34:50.736 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:50 smithi133 bash[102858]: cluster 2023-10-09T05:34:49.534911+0000 osd.5 (osd.5) 281 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T05:34:50.736 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:50 smithi133 bash[102858]: audit 2023-10-09T05:34:49.535527+0000 mon.a (mon.0) 1753 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:34:50.736 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:50 smithi133 bash[102858]: cluster 2023-10-09T05:34:49.535636+0000 mon.a (mon.0) 1754 : cluster 0 osdmap e306: 8 total, 8 up, 7 in 2023-10-09T05:34:50.736 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:50 smithi133 bash[102858]: audit 2023-10-09T05:34:50.198426+0000 mon.a (mon.0) 1755 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:34:50.736 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:50 smithi133 bash[102858]: audit 2023-10-09T05:34:50.204169+0000 mon.a (mon.0) 1756 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:34:50.794 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:50.813 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:50.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:50.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:50.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:50 smithi194 bash[92666]: audit 2023-10-09T05:34:49.409086+0000 mgr.y (mgr.25277) 1405 : audit 0 from='client.45710 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:34:50.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:50 smithi194 bash[92666]: cluster 2023-10-09T05:34:49.530704+0000 mon.a (mon.0) 1752 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:34:50.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:50 smithi194 bash[92666]: cluster 2023-10-09T05:34:49.533640+0000 osd.4 (osd.4) 275 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T05:34:50.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:50 smithi194 bash[92666]: cluster 2023-10-09T05:34:49.534911+0000 osd.5 (osd.5) 281 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T05:34:50.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:50 smithi194 bash[92666]: audit 2023-10-09T05:34:49.535527+0000 mon.a (mon.0) 1753 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:34:50.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:50 smithi194 bash[92666]: cluster 2023-10-09T05:34:49.535636+0000 mon.a (mon.0) 1754 : cluster 0 osdmap e306: 8 total, 8 up, 7 in 2023-10-09T05:34:50.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:50 smithi194 bash[92666]: audit 2023-10-09T05:34:50.198426+0000 mon.a (mon.0) 1755 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:34:50.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:50 smithi194 bash[92666]: audit 2023-10-09T05:34:50.204169+0000 mon.a (mon.0) 1756 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:34:50.916 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:50.946 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:50.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:50.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:51.049 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:51.079 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:51.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:51.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:51.183 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:51.204 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:51.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:51.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:51.308 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:51.335 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:51.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:51.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:51.439 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:51.467 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:51.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:51.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:51.571 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:51.583 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:51 smithi194 bash[92666]: cluster 2023-10-09T05:34:50.363230+0000 mgr.y (mgr.25277) 1406 : cluster 0 pgmap v1505: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 402 op/s; 2384/72 objects degraded (3311.111%); 1.4 MiB/s, 34 objects/s recovering 2023-10-09T05:34:51.604 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:51.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:51.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:51.707 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:51.729 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:51.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:51.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:51.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:51 smithi133 bash[102858]: cluster 2023-10-09T05:34:50.363230+0000 mgr.y (mgr.25277) 1406 : cluster 0 pgmap v1505: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 402 op/s; 2384/72 objects degraded (3311.111%); 1.4 MiB/s, 34 objects/s recovering 2023-10-09T05:34:51.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:51 smithi133 bash[105589]: cluster 2023-10-09T05:34:50.363230+0000 mgr.y (mgr.25277) 1406 : cluster 0 pgmap v1505: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 402 op/s; 2384/72 objects degraded (3311.111%); 1.4 MiB/s, 34 objects/s recovering 2023-10-09T05:34:51.833 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:51.855 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:51.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:51.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:51.959 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:51.991 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:51.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:51.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:52.095 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:52.127 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:52.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:52.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:52.231 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:52.259 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:52.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:52.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:52.363 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:52.395 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:52.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:52.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:52.499 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:52.528 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:52.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:52.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:52.631 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:52.660 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:52.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:52.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:52.764 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:52.796 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:52.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:52.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:52.900 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:52.933 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:52.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:52.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:53.036 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:53.067 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:53.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:53.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:53.170 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:53.203 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:53.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:53.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:53.307 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:53.334 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:53.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:53.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:53.437 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:53.471 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:53.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:53.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:53.576 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:53.606 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:53.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:53.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:53.708 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:53.734 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:53.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:53.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:53.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:53 smithi133 bash[102858]: cluster 2023-10-09T05:34:52.364314+0000 mgr.y (mgr.25277) 1407 : cluster 0 pgmap v1506: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 283 op/s; 2384/72 objects degraded (3311.111%); 627 KiB/s, 86 objects/s recovering 2023-10-09T05:34:53.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:53 smithi133 bash[105589]: cluster 2023-10-09T05:34:52.364314+0000 mgr.y (mgr.25277) 1407 : cluster 0 pgmap v1506: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 283 op/s; 2384/72 objects degraded (3311.111%); 627 KiB/s, 86 objects/s recovering 2023-10-09T05:34:53.838 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:53.849 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:53 smithi194 bash[92666]: cluster 2023-10-09T05:34:52.364314+0000 mgr.y (mgr.25277) 1407 : cluster 0 pgmap v1506: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 283 op/s; 2384/72 objects degraded (3311.111%); 627 KiB/s, 86 objects/s recovering 2023-10-09T05:34:53.866 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:53.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:53.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:53.969 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:53.997 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:54.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:54.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:54.101 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:54.127 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:54.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:54.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:54.231 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:54.253 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:54.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:54.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:54.356 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:54.364 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:34:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:34:54] "GET /metrics HTTP/1.1" 200 36490 "" "Prometheus/2.43.0" 2023-10-09T05:34:54.382 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:54.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:54.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:54.485 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:54.512 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:54.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:54.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:54.547 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T05:34:54.616 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:54.646 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:54.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:54.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:54.750 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:54.780 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:54.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:54.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:54.797 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:54 smithi133 bash[102858]: cluster 2023-10-09T05:34:53.835850+0000 mon.a (mon.0) 1757 : cluster 3 Health check update: Degraded data redundancy: 2384/72 objects degraded (3311.111%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:54.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:54 smithi133 bash[105589]: cluster 2023-10-09T05:34:53.835850+0000 mon.a (mon.0) 1757 : cluster 3 Health check update: Degraded data redundancy: 2384/72 objects degraded (3311.111%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:54.883 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:54.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:54 smithi194 bash[92666]: cluster 2023-10-09T05:34:53.835850+0000 mon.a (mon.0) 1757 : cluster 3 Health check update: Degraded data redundancy: 2384/72 objects degraded (3311.111%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:34:54.915 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:54.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:54.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:55.019 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:55.050 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:55.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:55.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:55.154 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:55.186 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:55.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:55.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:55.290 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:55.321 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:55.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:55.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:55.424 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:55.455 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:55.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:55.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:55.558 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:55.591 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:55.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:55.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:55.695 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:55.706 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:55 smithi194 bash[92666]: cluster 2023-10-09T05:34:54.365100+0000 mgr.y (mgr.25277) 1408 : cluster 0 pgmap v1507: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 283 op/s; 2384/72 objects degraded (3311.111%); 627 KiB/s, 86 objects/s recovering 2023-10-09T05:34:55.725 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:55.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:55.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:55.828 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:55.839 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:55 smithi133 bash[105589]: cluster 2023-10-09T05:34:54.365100+0000 mgr.y (mgr.25277) 1408 : cluster 0 pgmap v1507: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 283 op/s; 2384/72 objects degraded (3311.111%); 627 KiB/s, 86 objects/s recovering 2023-10-09T05:34:55.839 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:55 smithi133 bash[102858]: cluster 2023-10-09T05:34:54.365100+0000 mgr.y (mgr.25277) 1408 : cluster 0 pgmap v1507: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 283 op/s; 2384/72 objects degraded (3311.111%); 627 KiB/s, 86 objects/s recovering 2023-10-09T05:34:55.854 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:55.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:55.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:55.958 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:55.986 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:55.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:55.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:56.089 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:56.120 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:56.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:56.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:56.131 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-10-09T05:34:56.131 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 0.05 2023-10-09T05:34:56.224 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:56.251 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:56.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:56.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:56.355 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:56.386 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:56.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:56.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:56.490 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:56.521 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:56.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:56.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:56.624 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:56.654 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:56.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:56.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:56.759 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:56.786 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:56.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:56.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:56.889 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:56.917 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:56.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:56.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:57.020 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:57.048 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:57.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:57.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:57.152 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:57.176 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:57.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:57.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:57.279 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:57.310 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:57.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:57.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:57.413 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:57.444 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:57.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:57.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:57.548 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:57.579 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:57.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:57.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:57.683 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:57.696 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:57 smithi133 bash[105589]: cluster 2023-10-09T05:34:56.366445+0000 mgr.y (mgr.25277) 1409 : cluster 0 pgmap v1508: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 35 op/s; 2384/72 objects degraded (3311.111%); 627 KiB/s, 86 objects/s recovering 2023-10-09T05:34:57.696 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:57 smithi133 bash[102858]: cluster 2023-10-09T05:34:56.366445+0000 mgr.y (mgr.25277) 1409 : cluster 0 pgmap v1508: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 35 op/s; 2384/72 objects degraded (3311.111%); 627 KiB/s, 86 objects/s recovering 2023-10-09T05:34:57.713 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:57.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:57.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:57.819 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:57.831 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:57 smithi194 bash[92666]: cluster 2023-10-09T05:34:56.366445+0000 mgr.y (mgr.25277) 1409 : cluster 0 pgmap v1508: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 35 op/s; 2384/72 objects degraded (3311.111%); 627 KiB/s, 86 objects/s recovering 2023-10-09T05:34:57.851 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:57.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:57.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:57.955 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:57.987 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:57.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:57.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:58.091 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:58.118 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:34:58.121 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:58.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:58.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:58.130 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:34:58.225 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:58.255 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:58.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:58.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:58.359 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:58.385 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:58.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:58.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:58.489 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:58.524 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:58.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:58.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:58.628 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:58.657 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:58.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:58.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:58.760 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:58.773 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:58 smithi194 bash[92666]: audit 2023-10-09T05:34:57.613863+0000 mon.a (mon.0) 1758 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:34:58.774 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:58 smithi194 bash[92666]: audit 2023-10-09T05:34:57.624300+0000 mon.a (mon.0) 1759 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:34:58.774 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:58 smithi194 bash[92666]: audit 2023-10-09T05:34:58.097205+0000 mon.a (mon.0) 1760 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:34:58.774 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:58 smithi194 bash[92666]: audit 2023-10-09T05:34:58.098466+0000 mon.a (mon.0) 1761 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:34:58.774 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:58 smithi194 bash[92666]: audit 2023-10-09T05:34:58.108107+0000 mon.a (mon.0) 1762 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:34:58.774 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:58 smithi194 bash[92666]: cluster 2023-10-09T05:34:58.367181+0000 mgr.y (mgr.25277) 1410 : cluster 0 pgmap v1509: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 10 op/s; 2384/72 objects degraded (3311.111%); 0 B/s, 125 objects/s recovering 2023-10-09T05:34:58.774 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:58 smithi194 bash[92666]: audit 2023-10-09T05:34:58.494398+0000 mon.a (mon.0) 1763 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:34:58.774 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:34:58 smithi194 bash[92666]: audit 2023-10-09T05:34:58.496034+0000 mon.a (mon.0) 1764 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:34:58.794 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:58.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:58.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:58.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:58 smithi133 bash[102858]: audit 2023-10-09T05:34:57.613863+0000 mon.a (mon.0) 1758 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:34:58.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:58 smithi133 bash[102858]: audit 2023-10-09T05:34:57.624300+0000 mon.a (mon.0) 1759 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:34:58.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:58 smithi133 bash[102858]: audit 2023-10-09T05:34:58.097205+0000 mon.a (mon.0) 1760 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:34:58.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:58 smithi133 bash[102858]: audit 2023-10-09T05:34:58.098466+0000 mon.a (mon.0) 1761 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:34:58.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:58 smithi133 bash[102858]: audit 2023-10-09T05:34:58.108107+0000 mon.a (mon.0) 1762 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:34:58.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:58 smithi133 bash[102858]: cluster 2023-10-09T05:34:58.367181+0000 mgr.y (mgr.25277) 1410 : cluster 0 pgmap v1509: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 10 op/s; 2384/72 objects degraded (3311.111%); 0 B/s, 125 objects/s recovering 2023-10-09T05:34:58.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:58 smithi133 bash[102858]: audit 2023-10-09T05:34:58.494398+0000 mon.a (mon.0) 1763 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:34:58.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:34:58 smithi133 bash[102858]: audit 2023-10-09T05:34:58.496034+0000 mon.a (mon.0) 1764 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:34:58.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:58 smithi133 bash[105589]: audit 2023-10-09T05:34:57.613863+0000 mon.a (mon.0) 1758 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:34:58.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:58 smithi133 bash[105589]: audit 2023-10-09T05:34:57.624300+0000 mon.a (mon.0) 1759 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:34:58.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:58 smithi133 bash[105589]: audit 2023-10-09T05:34:58.097205+0000 mon.a (mon.0) 1760 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:34:58.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:58 smithi133 bash[105589]: audit 2023-10-09T05:34:58.098466+0000 mon.a (mon.0) 1761 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:34:58.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:58 smithi133 bash[105589]: audit 2023-10-09T05:34:58.108107+0000 mon.a (mon.0) 1762 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:34:58.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:58 smithi133 bash[105589]: cluster 2023-10-09T05:34:58.367181+0000 mgr.y (mgr.25277) 1410 : cluster 0 pgmap v1509: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 10 op/s; 2384/72 objects degraded (3311.111%); 0 B/s, 125 objects/s recovering 2023-10-09T05:34:58.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:58 smithi133 bash[105589]: audit 2023-10-09T05:34:58.494398+0000 mon.a (mon.0) 1763 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:34:58.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:34:58 smithi133 bash[105589]: audit 2023-10-09T05:34:58.496034+0000 mon.a (mon.0) 1764 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:34:58.897 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:58.931 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:58.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:58.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:59.034 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:59.062 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:59.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:59.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:59.167 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:59.198 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:59.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:59.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:59.302 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:59.333 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:59.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:59.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:59.436 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:59.470 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:59.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:59.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:59.574 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:59.607 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:59.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:59.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:59.710 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:59.743 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:59.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:59.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:59.846 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:59.880 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:59.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:59.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:59.983 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:00.021 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:00.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:00.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:00.125 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:00.151 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:00.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:00.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:00.255 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:00.289 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:00.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:00.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:00.393 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:00.427 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:00.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:00.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:00.531 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:00.562 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:00.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:00.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:00.665 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:00.665 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:00.666 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:00.667 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:00.695 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:00.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:00.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:00.801 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:00.832 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:00.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:00.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:00.937 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:00.959 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:00.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:00.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:01.063 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:01.092 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:01.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:01.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:01.196 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:01.225 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:01.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:01.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:01.328 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:01.360 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:01.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:01.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:01.464 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:01.498 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:01.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:01.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:01.603 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:01.611 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:01 smithi133 bash[105589]: cluster 2023-10-09T05:35:00.368037+0000 mgr.y (mgr.25277) 1411 : cluster 0 pgmap v1510: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 9 op/s; 2384/72 objects degraded (3311.111%); 0 B/s, 116 objects/s recovering 2023-10-09T05:35:01.612 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:01 smithi133 bash[102858]: cluster 2023-10-09T05:35:00.368037+0000 mgr.y (mgr.25277) 1411 : cluster 0 pgmap v1510: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 9 op/s; 2384/72 objects degraded (3311.111%); 0 B/s, 116 objects/s recovering 2023-10-09T05:35:01.625 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:01.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:01.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:01.729 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:01.741 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:01 smithi194 bash[92666]: cluster 2023-10-09T05:35:00.368037+0000 mgr.y (mgr.25277) 1411 : cluster 0 pgmap v1510: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 9 op/s; 2384/72 objects degraded (3311.111%); 0 B/s, 116 objects/s recovering 2023-10-09T05:35:01.760 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:01.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:01.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:01.864 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:01.889 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:01.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:01.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:01.993 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:02.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:02.025 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:02.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:02.126 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:02.153 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:02.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:02.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:02.256 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:02.286 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:02.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:02.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:02.388 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:02.417 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:02.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:02.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:02.520 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:02.550 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:02.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:02.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:02.653 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:02.665 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:02 smithi133 bash[105589]: cluster 2023-10-09T05:35:02.423187+0000 mon.a (mon.0) 1765 : cluster 3 Health check update: Degraded data redundancy: 2090/72 objects degraded (2902.778%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:35:02.665 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:02 smithi133 bash[102858]: cluster 2023-10-09T05:35:02.423187+0000 mon.a (mon.0) 1765 : cluster 3 Health check update: Degraded data redundancy: 2090/72 objects degraded (2902.778%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:35:02.681 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:02.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:02.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:02.693 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:02.693 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:02.785 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:02.816 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:02.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:02.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:02.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:02 smithi194 bash[92666]: cluster 2023-10-09T05:35:02.423187+0000 mon.a (mon.0) 1765 : cluster 3 Health check update: Degraded data redundancy: 2090/72 objects degraded (2902.778%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:35:02.919 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:02.943 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:02.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:02.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:03.047 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:03.081 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:03.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:03.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:03.185 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:03.217 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:03.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:03.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:03.321 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:03.352 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:03.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:03.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:03.456 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:03.483 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:03.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:03.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:03.587 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:03.616 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:03.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:03.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:03.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:03 smithi194 bash[92666]: cluster 2023-10-09T05:35:02.369041+0000 mgr.y (mgr.25277) 1412 : cluster 0 pgmap v1511: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 2090/72 objects degraded (2902.778%); 0 B/s, 146 objects/s recovering 2023-10-09T05:35:03.720 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:03.731 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:03 smithi133 bash[105589]: cluster 2023-10-09T05:35:02.369041+0000 mgr.y (mgr.25277) 1412 : cluster 0 pgmap v1511: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 2090/72 objects degraded (2902.778%); 0 B/s, 146 objects/s recovering 2023-10-09T05:35:03.732 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:03 smithi133 bash[102858]: cluster 2023-10-09T05:35:02.369041+0000 mgr.y (mgr.25277) 1412 : cluster 0 pgmap v1511: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 2090/72 objects degraded (2902.778%); 0 B/s, 146 objects/s recovering 2023-10-09T05:35:03.748 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:03.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:03.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:03.852 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:03.881 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:03.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:03.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:03.984 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:04.015 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:04.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:04.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:04.119 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:04.155 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:04.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:04.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:04.259 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:04.296 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:04.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:04.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:04.399 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:04.432 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:04.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:04.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:04.535 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:04.547 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:35:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:35:04] "GET /metrics HTTP/1.1" 200 36474 "" "Prometheus/2.43.0" 2023-10-09T05:35:04.566 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:04.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:04.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:04.669 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:04.707 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:04.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:04.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:04.811 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:04.834 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:04.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:04.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:04.937 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:04.966 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:04.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:04.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:05.070 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:05.082 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:04 smithi194 bash[92666]: cluster 2023-10-09T05:35:04.369734+0000 mgr.y (mgr.25277) 1413 : cluster 0 pgmap v1512: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 2090/72 objects degraded (2902.778%); 0 B/s, 92 objects/s recovering 2023-10-09T05:35:05.099 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:05.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:05.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:05.203 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:05.216 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:04 smithi133 bash[105589]: cluster 2023-10-09T05:35:04.369734+0000 mgr.y (mgr.25277) 1413 : cluster 0 pgmap v1512: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 2090/72 objects degraded (2902.778%); 0 B/s, 92 objects/s recovering 2023-10-09T05:35:05.216 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:04 smithi133 bash[102858]: cluster 2023-10-09T05:35:04.369734+0000 mgr.y (mgr.25277) 1413 : cluster 0 pgmap v1512: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 2090/72 objects degraded (2902.778%); 0 B/s, 92 objects/s recovering 2023-10-09T05:35:05.231 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:05.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:05.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:05.335 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:05.363 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:05.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:05.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:05.468 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:05.496 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:05.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:05.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:05.600 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:05.630 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:05.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:05.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:05.734 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:05.765 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:05.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:05.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:05.869 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:05.900 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:05.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:05.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:06.003 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:06.035 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:06.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:06.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:06.140 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:06.163 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:06.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:06.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:06.266 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:06.294 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:06.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:06.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:06.398 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:06.430 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:06.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:06.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:06.534 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:06.560 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:06.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:06.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:06.663 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:06.695 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:06.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:06.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:06.799 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:06.825 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:06.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:06.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:06.928 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:06.958 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:06.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:06.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:07.062 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:07.091 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:07.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:07.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:07.194 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:07.221 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:07.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:07.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:07.325 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:07.360 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:07.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:07.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:07.464 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:07.488 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:07.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:07.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:07.591 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:07.618 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:07.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:07.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:07.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:07 smithi194 bash[92666]: cluster 2023-10-09T05:35:06.370625+0000 mgr.y (mgr.25277) 1414 : cluster 0 pgmap v1513: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 2090/72 objects degraded (2902.778%); 0 B/s, 92 objects/s recovering 2023-10-09T05:35:07.722 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:07.754 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:07.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:07.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:07.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:07 smithi133 bash[105589]: cluster 2023-10-09T05:35:06.370625+0000 mgr.y (mgr.25277) 1414 : cluster 0 pgmap v1513: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 2090/72 objects degraded (2902.778%); 0 B/s, 92 objects/s recovering 2023-10-09T05:35:07.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:07 smithi133 bash[102858]: cluster 2023-10-09T05:35:06.370625+0000 mgr.y (mgr.25277) 1414 : cluster 0 pgmap v1513: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 2090/72 objects degraded (2902.778%); 0 B/s, 92 objects/s recovering 2023-10-09T05:35:07.857 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:07.887 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:07.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:07.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:07.992 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:08.021 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:08.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:08.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:08.124 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:08.150 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:08.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:08.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:08.254 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:08.278 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:08.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:08.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:08.381 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:08.406 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:08.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:08.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:08.509 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:08.539 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:08.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:08.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:08.547 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:08 smithi133 bash[105589]: cluster 2023-10-09T05:35:08.426231+0000 mon.a (mon.0) 1766 : cluster 3 Health check update: Degraded data redundancy: 1448/72 objects degraded (2011.111%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:35:08.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:08 smithi133 bash[102858]: cluster 2023-10-09T05:35:08.426231+0000 mon.a (mon.0) 1766 : cluster 3 Health check update: Degraded data redundancy: 1448/72 objects degraded (2011.111%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:35:08.643 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:08.674 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:08.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:08.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:08.778 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:08.789 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:08 smithi194 bash[92666]: cluster 2023-10-09T05:35:08.426231+0000 mon.a (mon.0) 1766 : cluster 3 Health check update: Degraded data redundancy: 1448/72 objects degraded (2011.111%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:35:08.804 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:08.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:08.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:08.908 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:09.116 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:09.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:09.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:09.220 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:09.242 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:09.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:09.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:09.345 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:09.368 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:09.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:09.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:09.471 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:09.481 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:09 smithi133 bash[102858]: cluster 2023-10-09T05:35:08.371221+0000 mgr.y (mgr.25277) 1415 : cluster 0 pgmap v1514: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 1448/72 objects degraded (2011.111%); 0 B/s, 146 objects/s recovering 2023-10-09T05:35:09.498 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:09.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:09.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:09.603 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:09.635 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:09.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:09.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:09.739 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:09.750 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:09 smithi194 bash[92666]: cluster 2023-10-09T05:35:08.371221+0000 mgr.y (mgr.25277) 1415 : cluster 0 pgmap v1514: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 1448/72 objects degraded (2011.111%); 0 B/s, 146 objects/s recovering 2023-10-09T05:35:09.768 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:09.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:09.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:09.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:09 smithi133 bash[105589]: cluster 2023-10-09T05:35:08.371221+0000 mgr.y (mgr.25277) 1415 : cluster 0 pgmap v1514: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 1448/72 objects degraded (2011.111%); 0 B/s, 146 objects/s recovering 2023-10-09T05:35:09.872 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:09.905 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:09.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:09.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:10.009 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:10.034 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:10.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:10.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:10.137 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:10.168 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:10.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:10.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:10.272 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:10.294 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:10.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:10.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:10.397 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:10.423 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:10.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:10.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:10.526 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:10.561 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:10.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:10.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:10.664 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:10.693 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:10.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:10.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:10.797 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:10.818 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:10.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:10.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:10.923 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:10.951 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:10.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:10.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:11.054 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:11.076 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:11.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:11.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:11.179 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:11.200 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:11.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:11.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:11.303 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:11.334 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:11.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:11.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:11.438 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:11.464 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:11.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:11.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:11.567 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:11.578 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:11 smithi194 bash[92666]: cluster 2023-10-09T05:35:10.372218+0000 mgr.y (mgr.25277) 1416 : cluster 0 pgmap v1515: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 1448/72 objects degraded (2011.111%); 0 B/s, 94 objects/s recovering 2023-10-09T05:35:11.597 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:11.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:11.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:11.701 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:11.730 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:11.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:11.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:11.835 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:11.850 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:11 smithi133 bash[105589]: cluster 2023-10-09T05:35:10.372218+0000 mgr.y (mgr.25277) 1416 : cluster 0 pgmap v1515: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 1448/72 objects degraded (2011.111%); 0 B/s, 94 objects/s recovering 2023-10-09T05:35:11.851 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:11 smithi133 bash[102858]: cluster 2023-10-09T05:35:10.372218+0000 mgr.y (mgr.25277) 1416 : cluster 0 pgmap v1515: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 1448/72 objects degraded (2011.111%); 0 B/s, 94 objects/s recovering 2023-10-09T05:35:11.864 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:11.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:11.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:11.968 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:11.993 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:11.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:11.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:12.096 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:12.124 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:12.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:12.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:12.228 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:12.258 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:12.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:12.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:12.363 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:12.391 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:12.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:12.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:12.494 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:12.525 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:12.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:12.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:12.629 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:12.651 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:12.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:12.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:12.754 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:12.779 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:12.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:12.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:12.882 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:12.905 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:12.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:12.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:13.008 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:13.039 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:13.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:13.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:13.142 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:13.167 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:13.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:13.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:13.270 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:13.300 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:13.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:13.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:13.404 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:13.438 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:13.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:13.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:13.487 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:13 smithi133 bash[102858]: cluster 2023-10-09T05:35:12.372879+0000 mgr.y (mgr.25277) 1417 : cluster 0 pgmap v1516: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 664/72 objects degraded (922.222%); 0 B/s, 160 objects/s recovering 2023-10-09T05:35:13.541 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:13.547 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:13 smithi133 bash[105589]: cluster 2023-10-09T05:35:12.372879+0000 mgr.y (mgr.25277) 1417 : cluster 0 pgmap v1516: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 664/72 objects degraded (922.222%); 0 B/s, 160 objects/s recovering 2023-10-09T05:35:13.562 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:13.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:13.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:13.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:13 smithi194 bash[92666]: cluster 2023-10-09T05:35:12.372879+0000 mgr.y (mgr.25277) 1417 : cluster 0 pgmap v1516: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 664/72 objects degraded (922.222%); 0 B/s, 160 objects/s recovering 2023-10-09T05:35:13.665 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:13.688 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:13.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:13.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:13.791 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:13.821 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:13.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:13.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:13.925 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:13.952 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:13.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:13.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:14.055 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:14.084 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:14.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:14.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:14.188 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:14.217 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:14.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:14.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:14.321 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:14.346 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:14.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:14.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:14.448 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:14.484 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:14.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:14.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:14.487 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:35:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:35:14] "GET /metrics HTTP/1.1" 200 36474 "" "Prometheus/2.43.0" 2023-10-09T05:35:14.589 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:14.615 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:14.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:14.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:14.718 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:14.738 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:14.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:14.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:14.841 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:14.851 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:14 smithi194 bash[92666]: audit 2023-10-09T05:35:13.490886+0000 mon.a (mon.0) 1767 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:35:14.851 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:14 smithi194 bash[92666]: cluster 2023-10-09T05:35:13.840996+0000 mon.a (mon.0) 1768 : cluster 3 Health check update: Degraded data redundancy: 664/72 objects degraded (922.222%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:35:14.866 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:14.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:14.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:14.902 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:14 smithi133 bash[102858]: audit 2023-10-09T05:35:13.490886+0000 mon.a (mon.0) 1767 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:35:14.902 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:14 smithi133 bash[102858]: cluster 2023-10-09T05:35:13.840996+0000 mon.a (mon.0) 1768 : cluster 3 Health check update: Degraded data redundancy: 664/72 objects degraded (922.222%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:35:14.902 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:14 smithi133 bash[105589]: audit 2023-10-09T05:35:13.490886+0000 mon.a (mon.0) 1767 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:35:14.903 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:14 smithi133 bash[105589]: cluster 2023-10-09T05:35:13.840996+0000 mon.a (mon.0) 1768 : cluster 3 Health check update: Degraded data redundancy: 664/72 objects degraded (922.222%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:35:14.969 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:14.988 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:14.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:14.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:15.091 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:15.116 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:15.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:15.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:15.219 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:15.252 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:15.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:15.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:15.356 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:15.383 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:15.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:15.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:15.487 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:15.522 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:15.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:15.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:15.625 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:15.638 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:15 smithi133 bash[105589]: cluster 2023-10-09T05:35:14.373294+0000 mgr.y (mgr.25277) 1418 : cluster 0 pgmap v1517: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 664/72 objects degraded (922.222%); 0 B/s, 118 objects/s recovering 2023-10-09T05:35:15.638 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:15 smithi133 bash[102858]: cluster 2023-10-09T05:35:14.373294+0000 mgr.y (mgr.25277) 1418 : cluster 0 pgmap v1517: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 664/72 objects degraded (922.222%); 0 B/s, 118 objects/s recovering 2023-10-09T05:35:15.657 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:15.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:15.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:15.761 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:15.772 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:15 smithi194 bash[92666]: cluster 2023-10-09T05:35:14.373294+0000 mgr.y (mgr.25277) 1418 : cluster 0 pgmap v1517: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 664/72 objects degraded (922.222%); 0 B/s, 118 objects/s recovering 2023-10-09T05:35:15.789 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:15.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:15.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:15.893 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:15.926 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:15.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:15.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:16.028 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:16.060 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:16.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:16.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:16.164 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:16.196 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:16.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:16.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:16.301 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:16.326 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:16.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:16.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:16.430 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:16.460 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:16.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:16.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:16.564 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:16.601 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:16.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:16.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:16.705 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:16.713 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:16 smithi133 bash[105589]: cluster 2023-10-09T05:35:15.498286+0000 osd.5 (osd.5) 282 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T05:35:16.713 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:16 smithi133 bash[105589]: cluster 2023-10-09T05:35:15.498707+0000 osd.6 (osd.6) 265 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T05:35:16.713 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:16 smithi133 bash[105589]: cluster 2023-10-09T05:35:15.501256+0000 mon.a (mon.0) 1769 : cluster 0 osdmap e307: 8 total, 8 up, 7 in 2023-10-09T05:35:16.713 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:16 smithi133 bash[105589]: cluster 2023-10-09T05:35:15.502099+0000 osd.7 (osd.7) 268 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T05:35:16.713 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:16 smithi133 bash[105589]: cluster 2023-10-09T05:35:15.505605+0000 osd.7 (osd.7) 269 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T05:35:16.713 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:16 smithi133 bash[102858]: cluster 2023-10-09T05:35:15.498286+0000 osd.5 (osd.5) 282 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T05:35:16.713 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:16 smithi133 bash[102858]: cluster 2023-10-09T05:35:15.498707+0000 osd.6 (osd.6) 265 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T05:35:16.713 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:16 smithi133 bash[102858]: cluster 2023-10-09T05:35:15.501256+0000 mon.a (mon.0) 1769 : cluster 0 osdmap e307: 8 total, 8 up, 7 in 2023-10-09T05:35:16.713 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:16 smithi133 bash[102858]: cluster 2023-10-09T05:35:15.502099+0000 osd.7 (osd.7) 268 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T05:35:16.714 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:16 smithi133 bash[102858]: cluster 2023-10-09T05:35:15.505605+0000 osd.7 (osd.7) 269 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T05:35:16.730 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:16.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:16.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:16.834 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:16.847 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:16 smithi194 bash[92666]: cluster 2023-10-09T05:35:15.498286+0000 osd.5 (osd.5) 282 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T05:35:16.847 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:16 smithi194 bash[92666]: cluster 2023-10-09T05:35:15.498707+0000 osd.6 (osd.6) 265 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T05:35:16.847 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:16 smithi194 bash[92666]: cluster 2023-10-09T05:35:15.501256+0000 mon.a (mon.0) 1769 : cluster 0 osdmap e307: 8 total, 8 up, 7 in 2023-10-09T05:35:16.847 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:16 smithi194 bash[92666]: cluster 2023-10-09T05:35:15.502099+0000 osd.7 (osd.7) 268 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T05:35:16.847 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:16 smithi194 bash[92666]: cluster 2023-10-09T05:35:15.505605+0000 osd.7 (osd.7) 269 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T05:35:16.862 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:16.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:16.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:16.965 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:16.989 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:16.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:16.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:17.092 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:17.119 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:17.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:17.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:17.222 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:17.254 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:17.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:17.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:17.359 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:17.383 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:17.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:17.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:17.487 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:17.514 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:17.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:17.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:17.617 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:17.639 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:17.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:17.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:17.742 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:17.774 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:17.776 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:17 smithi194 bash[92666]: cluster 2023-10-09T05:35:16.373897+0000 mgr.y (mgr.25277) 1419 : cluster 0 pgmap v1519: 89 pgs: 4 active+clean+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 664/72 objects degraded (922.222%); 0 B/s, 142 objects/s recovering 2023-10-09T05:35:17.777 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:17 smithi194 bash[92666]: cluster 2023-10-09T05:35:16.504907+0000 osd.4 (osd.4) 276 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T05:35:17.777 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:17 smithi194 bash[92666]: cluster 2023-10-09T05:35:16.507720+0000 osd.5 (osd.5) 283 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T05:35:17.777 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:17 smithi194 bash[92666]: cluster 2023-10-09T05:35:16.509893+0000 mon.a (mon.0) 1770 : cluster 0 osdmap e308: 8 total, 8 up, 7 in 2023-10-09T05:35:17.777 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:17 smithi194 bash[92666]: cluster 2023-10-09T05:35:16.510684+0000 osd.7 (osd.7) 270 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T05:35:17.777 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:17 smithi194 bash[92666]: cluster 2023-10-09T05:35:16.512643+0000 osd.5 (osd.5) 284 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T05:35:17.777 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:17 smithi194 bash[92666]: cluster 2023-10-09T05:35:16.513737+0000 osd.7 (osd.7) 271 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T05:35:17.777 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:17 smithi194 bash[92666]: cluster 2023-10-09T05:35:16.515880+0000 osd.5 (osd.5) 285 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T05:35:17.777 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:17 smithi194 bash[92666]: cluster 2023-10-09T05:35:16.516661+0000 osd.6 (osd.6) 266 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T05:35:17.777 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:17 smithi194 bash[92666]: cluster 2023-10-09T05:35:16.518175+0000 osd.5 (osd.5) 286 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T05:35:17.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:17.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:17.879 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:17.912 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:17.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:17.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:18.016 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:18.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:17 smithi133 bash[105589]: cluster 2023-10-09T05:35:16.373897+0000 mgr.y (mgr.25277) 1419 : cluster 0 pgmap v1519: 89 pgs: 4 active+clean+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 664/72 objects degraded (922.222%); 0 B/s, 142 objects/s recovering 2023-10-09T05:35:18.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:17 smithi133 bash[105589]: cluster 2023-10-09T05:35:16.504907+0000 osd.4 (osd.4) 276 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T05:35:18.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:17 smithi133 bash[105589]: cluster 2023-10-09T05:35:16.507720+0000 osd.5 (osd.5) 283 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T05:35:18.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:17 smithi133 bash[105589]: cluster 2023-10-09T05:35:16.509893+0000 mon.a (mon.0) 1770 : cluster 0 osdmap e308: 8 total, 8 up, 7 in 2023-10-09T05:35:18.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:17 smithi133 bash[105589]: cluster 2023-10-09T05:35:16.510684+0000 osd.7 (osd.7) 270 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T05:35:18.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:17 smithi133 bash[105589]: cluster 2023-10-09T05:35:16.512643+0000 osd.5 (osd.5) 284 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T05:35:18.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:17 smithi133 bash[105589]: cluster 2023-10-09T05:35:16.513737+0000 osd.7 (osd.7) 271 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T05:35:18.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:17 smithi133 bash[105589]: cluster 2023-10-09T05:35:16.515880+0000 osd.5 (osd.5) 285 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T05:35:18.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:17 smithi133 bash[105589]: cluster 2023-10-09T05:35:16.516661+0000 osd.6 (osd.6) 266 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T05:35:18.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:17 smithi133 bash[105589]: cluster 2023-10-09T05:35:16.518175+0000 osd.5 (osd.5) 286 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T05:35:18.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:17 smithi133 bash[102858]: cluster 2023-10-09T05:35:16.373897+0000 mgr.y (mgr.25277) 1419 : cluster 0 pgmap v1519: 89 pgs: 4 active+clean+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 79 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 664/72 objects degraded (922.222%); 0 B/s, 142 objects/s recovering 2023-10-09T05:35:18.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:17 smithi133 bash[102858]: cluster 2023-10-09T05:35:16.504907+0000 osd.4 (osd.4) 276 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T05:35:18.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:17 smithi133 bash[102858]: cluster 2023-10-09T05:35:16.507720+0000 osd.5 (osd.5) 283 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T05:35:18.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:17 smithi133 bash[102858]: cluster 2023-10-09T05:35:16.509893+0000 mon.a (mon.0) 1770 : cluster 0 osdmap e308: 8 total, 8 up, 7 in 2023-10-09T05:35:18.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:17 smithi133 bash[102858]: cluster 2023-10-09T05:35:16.510684+0000 osd.7 (osd.7) 270 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T05:35:18.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:17 smithi133 bash[102858]: cluster 2023-10-09T05:35:16.512643+0000 osd.5 (osd.5) 284 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T05:35:18.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:17 smithi133 bash[102858]: cluster 2023-10-09T05:35:16.513737+0000 osd.7 (osd.7) 271 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T05:35:18.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:17 smithi133 bash[102858]: cluster 2023-10-09T05:35:16.515880+0000 osd.5 (osd.5) 285 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T05:35:18.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:17 smithi133 bash[102858]: cluster 2023-10-09T05:35:16.516661+0000 osd.6 (osd.6) 266 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T05:35:18.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:17 smithi133 bash[102858]: cluster 2023-10-09T05:35:16.518175+0000 osd.5 (osd.5) 286 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T05:35:18.050 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:18.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:18.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:18.152 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:18.173 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:18.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:18.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:18.278 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:18.300 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:18.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:18.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:18.403 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:18.437 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:18.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:18.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:18.541 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:18.564 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:18.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:18.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:18.667 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:18.692 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:18.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:18.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:18.796 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:18.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:18 smithi133 bash[105589]: cluster 2023-10-09T05:35:18.505968+0000 mon.a (mon.0) 1771 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 664/72 objects degraded (922.222%), 1 pg degraded, 1 pg undersized) 2023-10-09T05:35:18.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:18 smithi133 bash[102858]: cluster 2023-10-09T05:35:18.505968+0000 mon.a (mon.0) 1771 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 664/72 objects degraded (922.222%), 1 pg degraded, 1 pg undersized) 2023-10-09T05:35:18.807 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:18 smithi194 bash[92666]: cluster 2023-10-09T05:35:18.505968+0000 mon.a (mon.0) 1771 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 664/72 objects degraded (922.222%), 1 pg degraded, 1 pg undersized) 2023-10-09T05:35:18.826 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:18.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:18.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:18.931 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:18.960 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:18.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:18.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:19.063 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:19.091 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:19.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:19.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:19.194 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:19.222 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:19.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:19.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:19.328 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:19.361 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:19.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:19.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:19.465 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:19.497 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:19.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:19.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:19.601 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:19.614 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:19 smithi194 bash[92666]: cluster 2023-10-09T05:35:18.374441+0000 mgr.y (mgr.25277) 1420 : cluster 0 pgmap v1521: 89 pgs: 4 active+clean+remapped, 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 80 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s, 180 objects/s recovering 2023-10-09T05:35:19.635 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:19.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:19.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:19.738 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:19.769 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:19.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:19.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:19.873 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:19.886 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:19 smithi133 bash[102858]: cluster 2023-10-09T05:35:18.374441+0000 mgr.y (mgr.25277) 1420 : cluster 0 pgmap v1521: 89 pgs: 4 active+clean+remapped, 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 80 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s, 180 objects/s recovering 2023-10-09T05:35:19.886 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:19 smithi133 bash[105589]: cluster 2023-10-09T05:35:18.374441+0000 mgr.y (mgr.25277) 1420 : cluster 0 pgmap v1521: 89 pgs: 4 active+clean+remapped, 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 80 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s, 180 objects/s recovering 2023-10-09T05:35:19.900 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:19.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:19.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:20.004 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:20.025 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:20.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:20.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:20.129 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:20.162 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:20.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:20.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:20.266 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:20.289 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:20.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:20.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:20.392 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:20.419 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:20.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:20.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:20.522 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:20.550 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:20.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:20.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:20.653 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:20.675 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:20.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:20.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:20.778 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:20.803 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:20.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:20.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:20.906 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:20.936 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:20.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:20.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:21.041 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:21.070 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:21.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:21.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:21.174 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:21.196 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:21.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:21.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:21.300 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:21.323 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:21.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:21.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:21.426 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:21.456 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:21.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:21.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:21.559 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:21.586 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:21.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:21.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:21.689 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:21.721 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:21.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:21.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:21.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:21 smithi133 bash[102858]: cluster 2023-10-09T05:35:20.375450+0000 mgr.y (mgr.25277) 1421 : cluster 0 pgmap v1522: 89 pgs: 1 active+clean+remapped, 88 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s, 82 objects/s recovering 2023-10-09T05:35:21.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:21 smithi133 bash[105589]: cluster 2023-10-09T05:35:20.375450+0000 mgr.y (mgr.25277) 1421 : cluster 0 pgmap v1522: 89 pgs: 1 active+clean+remapped, 88 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s, 82 objects/s recovering 2023-10-09T05:35:21.824 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:21.857 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:21.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:21.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:21.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:21 smithi194 bash[92666]: cluster 2023-10-09T05:35:20.375450+0000 mgr.y (mgr.25277) 1421 : cluster 0 pgmap v1522: 89 pgs: 1 active+clean+remapped, 88 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s, 82 objects/s recovering 2023-10-09T05:35:21.960 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:21.994 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:21.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:21.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:22.097 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:22.124 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:22.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:22.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:22.228 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:22.257 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:22.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:22.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:22.361 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:22.391 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:22.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:22.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:22.495 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:22.523 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:22.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:22.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:22.626 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:22.658 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:22.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:22.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:22.761 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:22.792 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:22.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:22.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:22.808 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:35:22.829 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:35:22.849 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:35:22.869 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:35:22.892 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:35:22.896 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:22.914 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:35:22.926 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:22.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:22.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:22.936 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:35:22.976 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:35:22.978 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:35:22.980 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:35:22.980 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:35:22.980 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:35:22.980 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:35:22.980 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:35:22.981 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:35:22.981 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:35:23.030 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:23.058 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:23.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:23.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:23.162 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:23.182 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:23.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:23.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:23.285 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:23.310 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:23.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:23.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:23.379 INFO:teuthology.orchestra.run.smithi133.stdout:moved 0 / 267 (0%) 2023-10-09T05:35:23.379 INFO:teuthology.orchestra.run.smithi133.stdout:avg 38.1429 2023-10-09T05:35:23.379 INFO:teuthology.orchestra.run.smithi133.stdout:stddev 4.0507 -> 4.0507 (expected baseline 5.71786) 2023-10-09T05:35:23.379 INFO:teuthology.orchestra.run.smithi133.stdout:min osd.5 with 32 -> 32 pgs (0.838951 -> 0.838951 * mean) 2023-10-09T05:35:23.379 INFO:teuthology.orchestra.run.smithi133.stdout:max osd.1 with 45 -> 45 pgs (1.17978 -> 1.17978 * mean) 2023-10-09T05:35:23.379 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:35:23.379 INFO:teuthology.orchestra.run.smithi133.stdout:oload 1000 2023-10-09T05:35:23.380 INFO:teuthology.orchestra.run.smithi133.stdout:max_change 0.05 2023-10-09T05:35:23.380 INFO:teuthology.orchestra.run.smithi133.stdout:max_change_osds 4 2023-10-09T05:35:23.380 INFO:teuthology.orchestra.run.smithi133.stdout:average_utilization 0.0113 2023-10-09T05:35:23.380 INFO:teuthology.orchestra.run.smithi133.stdout:overload_utilization 0.1127 2023-10-09T05:35:23.382 INFO:teuthology.orchestra.run.smithi133.stderr:no change 2023-10-09T05:35:23.414 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:23.438 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:23.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:23.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:23.544 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:23.564 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:23.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:23.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:23.577 INFO:teuthology.orchestra.run.smithi133.stderr:pool 'unique_pool_8' removed 2023-10-09T05:35:23.590 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:23 smithi133 bash[105589]: cluster 2023-10-09T05:35:22.376417+0000 mgr.y (mgr.25277) 1422 : cluster 0 pgmap v1523: 89 pgs: 89 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s, 82 objects/s recovering 2023-10-09T05:35:23.590 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:23 smithi133 bash[105589]: audit 2023-10-09T05:35:22.786629+0000 mon.a (mon.0) 1772 : audit 0 from='client.? 172.21.15.133:0/2053763882' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:35:23.591 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:23 smithi133 bash[102858]: cluster 2023-10-09T05:35:22.376417+0000 mgr.y (mgr.25277) 1422 : cluster 0 pgmap v1523: 89 pgs: 89 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s, 82 objects/s recovering 2023-10-09T05:35:23.591 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:23 smithi133 bash[102858]: audit 2023-10-09T05:35:22.786629+0000 mon.a (mon.0) 1772 : audit 0 from='client.? 172.21.15.133:0/2053763882' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:35:23.591 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:23 smithi133 bash[102858]: audit 2023-10-09T05:35:23.060623+0000 mon.a (mon.0) 1773 : audit 1 from='client.? 172.21.15.133:0/1144116630' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:35:23.601 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:35:23.602 INFO:teuthology.orchestra.run.smithi133.stdout: "size": 20, 2023-10-09T05:35:23.602 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 600, 2023-10-09T05:35:23.602 INFO:teuthology.orchestra.run.smithi133.stdout: "ops": [ 2023-10-09T05:35:23.602 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.602 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45374.0:74846 12.b 12:d37979af:::benchmark_data_smithi133_218956_object74845:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e272)", 2023-10-09T05:35:23.602 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:26:19.622340+0000", 2023-10-09T05:35:23.602 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 543.97557374200005, 2023-10-09T05:35:23.602 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.25094175600000002, 2023-10-09T05:35:23.602 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:35:23.603 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:23.603 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:35:23.603 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45374", 2023-10-09T05:35:23.603 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2988063385", 2023-10-09T05:35:23.603 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 74846 2023-10-09T05:35:23.603 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.603 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:35:23.603 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.603 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:35:23.603 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.622340+0000", 2023-10-09T05:35:23.603 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.603 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.603 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.604 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:35:23.604 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.622340+0000", 2023-10-09T05:35:23.604 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.604 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.604 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.604 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:35:23.604 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.622339+0000", 2023-10-09T05:35:23.604 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:35:23.604 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.604 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.604 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:35:23.605 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.622362+0000", 2023-10-09T05:35:23.605 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.3439999999999999e-05 2023-10-09T05:35:23.605 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.605 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.605 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:35:23.605 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.622376+0000", 2023-10-09T05:35:23.605 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.3997e-05 2023-10-09T05:35:23.605 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.605 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.605 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:35:23.605 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.622382+0000", 2023-10-09T05:35:23.605 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.9359999999999996e-06 2023-10-09T05:35:23.606 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.606 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.606 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:35:23.606 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.622429+0000", 2023-10-09T05:35:23.606 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.6944000000000003e-05 2023-10-09T05:35:23.606 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.606 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.606 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:35:23.606 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.622576+0000", 2023-10-09T05:35:23.606 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00014735400000000001 2023-10-09T05:35:23.606 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.606 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.607 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:35:23.607 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.622780+0000", 2023-10-09T05:35:23.607 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00020392199999999999 2023-10-09T05:35:23.607 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.607 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.607 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.607 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.631546+0000", 2023-10-09T05:35:23.607 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0087656420000000006 2023-10-09T05:35:23.607 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.607 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.607 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:35:23.607 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.639159+0000", 2023-10-09T05:35:23.608 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0076136299999999997 2023-10-09T05:35:23.608 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.608 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.608 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.608 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.873163+0000", 2023-10-09T05:35:23.608 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.23400337399999999 2023-10-09T05:35:23.608 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.608 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.608 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:35:23.608 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.873232+0000", 2023-10-09T05:35:23.608 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.8916000000000002e-05 2023-10-09T05:35:23.608 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.608 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.609 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:35:23.609 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.873281+0000", 2023-10-09T05:35:23.609 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.9546999999999998e-05 2023-10-09T05:35:23.609 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.609 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:35:23.609 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.609 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.609 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.609 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45374.0:74852 12.8 12:1e53c266:::benchmark_data_smithi133_218956_object74851:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e272)", 2023-10-09T05:35:23.609 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:26:19.623024+0000", 2023-10-09T05:35:23.609 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 543.974889512, 2023-10-09T05:35:23.609 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.248889784, 2023-10-09T05:35:23.610 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:35:23.610 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:23.610 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:35:23.610 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45374", 2023-10-09T05:35:23.610 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2988063385", 2023-10-09T05:35:23.610 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 74852 2023-10-09T05:35:23.610 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.610 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:35:23.610 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.610 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:35:23.610 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.623024+0000", 2023-10-09T05:35:23.610 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.611 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.611 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.611 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:35:23.611 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.623024+0000", 2023-10-09T05:35:23.611 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.611 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.611 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.611 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:35:23.611 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.623023+0000", 2023-10-09T05:35:23.611 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:35:23.611 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.611 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.612 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:35:23.612 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.623047+0000", 2023-10-09T05:35:23.612 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.4765000000000001e-05 2023-10-09T05:35:23.612 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.612 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.612 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:35:23.612 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.623058+0000", 2023-10-09T05:35:23.612 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.0564000000000001e-05 2023-10-09T05:35:23.612 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.612 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.612 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:35:23.612 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.623066+0000", 2023-10-09T05:35:23.613 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.6149999999999999e-06 2023-10-09T05:35:23.613 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.613 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.613 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:35:23.613 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.623191+0000", 2023-10-09T05:35:23.613 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000125764 2023-10-09T05:35:23.613 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.613 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.613 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:35:23.613 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.623313+0000", 2023-10-09T05:35:23.613 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000121314 2023-10-09T05:35:23.613 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.614 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.614 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:35:23.614 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.623435+0000", 2023-10-09T05:35:23.614 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00012258500000000001 2023-10-09T05:35:23.614 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.614 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.614 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.614 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.631490+0000", 2023-10-09T05:35:23.614 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0080541999999999992 2023-10-09T05:35:23.614 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.614 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.615 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:35:23.615 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.640248+0000", 2023-10-09T05:35:23.615 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0087584829999999992 2023-10-09T05:35:23.615 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.615 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.615 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.615 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.871814+0000", 2023-10-09T05:35:23.615 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.23156628400000001 2023-10-09T05:35:23.615 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.615 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.616 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:35:23.616 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.871867+0000", 2023-10-09T05:35:23.616 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.3029999999999999e-05 2023-10-09T05:35:23.616 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.616 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.616 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:35:23.616 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.871914+0000", 2023-10-09T05:35:23.616 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.6207e-05 2023-10-09T05:35:23.616 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.616 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:35:23.617 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.617 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.617 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.617 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45374.0:74860 12.8 12:14f0766e:::benchmark_data_smithi133_218956_object74859:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e272)", 2023-10-09T05:35:23.617 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:26:19.636074+0000", 2023-10-09T05:35:23.617 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 543.96183892299996, 2023-10-09T05:35:23.617 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.26545807700000001, 2023-10-09T05:35:23.617 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:35:23.617 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:23.617 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:35:23.618 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45374", 2023-10-09T05:35:23.618 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2988063385", 2023-10-09T05:35:23.618 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 74860 2023-10-09T05:35:23.618 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.618 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:35:23.618 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.618 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:35:23.618 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.636074+0000", 2023-10-09T05:35:23.618 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.619 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.619 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.619 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:35:23.619 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.636074+0000", 2023-10-09T05:35:23.619 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.619 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.619 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.619 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:35:23.619 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.636073+0000", 2023-10-09T05:35:23.619 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:35:23.620 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.620 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.620 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:35:23.620 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.636097+0000", 2023-10-09T05:35:23.620 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.4439000000000001e-05 2023-10-09T05:35:23.620 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.620 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.620 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:35:23.620 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.636115+0000", 2023-10-09T05:35:23.620 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.8019000000000001e-05 2023-10-09T05:35:23.620 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.621 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.621 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:35:23.621 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.636124+0000", 2023-10-09T05:35:23.621 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.473e-06 2023-10-09T05:35:23.621 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.621 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.621 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:35:23.621 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.636205+0000", 2023-10-09T05:35:23.621 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.1317999999999996e-05 2023-10-09T05:35:23.621 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.621 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.621 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:35:23.622 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.636358+0000", 2023-10-09T05:35:23.622 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00015331000000000001 2023-10-09T05:35:23.622 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.622 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.622 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:35:23.622 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.636511+0000", 2023-10-09T05:35:23.622 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00015312 2023-10-09T05:35:23.622 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.622 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.622 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.622 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.644811+0000", 2023-10-09T05:35:23.623 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0082995880000000001 2023-10-09T05:35:23.623 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.623 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.623 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:35:23.623 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.644819+0000", 2023-10-09T05:35:23.623 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.0679999999999997e-06 2023-10-09T05:35:23.623 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.623 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.623 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.623 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.901431+0000", 2023-10-09T05:35:23.623 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.25661172500000001 2023-10-09T05:35:23.623 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.624 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.624 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:35:23.624 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.901477+0000", 2023-10-09T05:35:23.624 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.5976999999999997e-05 2023-10-09T05:35:23.624 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.624 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.624 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:35:23.624 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:26:19.901532+0000", 2023-10-09T05:35:23.624 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.5606999999999998e-05 2023-10-09T05:35:23.624 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.624 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:35:23.624 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.625 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.625 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.625 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:63824 13.2 13:43aa95bc:::benchmark_data_smithi133_241067_object63823:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-10-09T05:35:23.625 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:31:27.039452+0000", 2023-10-09T05:35:23.625 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 236.55846119, 2023-10-09T05:35:23.625 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.23959925500000001, 2023-10-09T05:35:23.625 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:35:23.625 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:23.625 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:35:23.625 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:35:23.625 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:35:23.625 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 63824 2023-10-09T05:35:23.626 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.626 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:35:23.626 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.626 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:35:23.626 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.039452+0000", 2023-10-09T05:35:23.626 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.626 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.626 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.626 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:35:23.626 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.039452+0000", 2023-10-09T05:35:23.626 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.627 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.627 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.627 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:35:23.627 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.039451+0000", 2023-10-09T05:35:23.627 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:35:23.627 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.627 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.627 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:35:23.627 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.039475+0000", 2023-10-09T05:35:23.627 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.4488999999999999e-05 2023-10-09T05:35:23.627 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.627 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.628 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:35:23.628 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.039487+0000", 2023-10-09T05:35:23.628 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.2302e-05 2023-10-09T05:35:23.628 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.628 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.628 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:35:23.628 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.039496+0000", 2023-10-09T05:35:23.628 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.1410000000000005e-06 2023-10-09T05:35:23.628 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.628 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.628 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:35:23.628 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.039607+0000", 2023-10-09T05:35:23.629 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000111053 2023-10-09T05:35:23.629 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.629 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.629 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:35:23.629 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.039756+0000", 2023-10-09T05:35:23.629 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000149834 2023-10-09T05:35:23.629 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.629 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.629 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:35:23.629 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.039908+0000", 2023-10-09T05:35:23.629 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00015141899999999999 2023-10-09T05:35:23.630 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.630 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.630 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.630 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.045771+0000", 2023-10-09T05:35:23.630 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0058635709999999997 2023-10-09T05:35:23.630 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.630 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.630 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:35:23.630 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.045946+0000", 2023-10-09T05:35:23.630 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00017466500000000001 2023-10-09T05:35:23.630 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.631 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.631 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.631 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.278931+0000", 2023-10-09T05:35:23.631 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.232984732 2023-10-09T05:35:23.631 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.631 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.631 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:35:23.631 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.278982+0000", 2023-10-09T05:35:23.631 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.1527000000000003e-05 2023-10-09T05:35:23.631 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.631 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.631 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:35:23.632 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.279051+0000", 2023-10-09T05:35:23.632 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.8989e-05 2023-10-09T05:35:23.632 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.632 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:35:23.632 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.632 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.632 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.632 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:63825 13.b 13:dca69c78:::benchmark_data_smithi133_241067_object63824:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-10-09T05:35:23.632 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:31:27.039511+0000", 2023-10-09T05:35:23.632 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 236.55840212999999, 2023-10-09T05:35:23.632 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.242960491, 2023-10-09T05:35:23.633 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:35:23.633 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:23.633 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:35:23.633 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:35:23.633 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:35:23.633 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 63825 2023-10-09T05:35:23.633 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.633 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:35:23.633 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.633 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:35:23.633 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.039511+0000", 2023-10-09T05:35:23.634 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.634 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.634 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.634 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:35:23.634 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.039511+0000", 2023-10-09T05:35:23.634 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.634 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.634 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.634 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:35:23.634 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.039511+0000", 2023-10-09T05:35:23.635 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999995 2023-10-09T05:35:23.635 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.635 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.635 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:35:23.635 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.039531+0000", 2023-10-09T05:35:23.635 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.012e-05 2023-10-09T05:35:23.635 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.635 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.635 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:35:23.635 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.039538+0000", 2023-10-09T05:35:23.635 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.8510000000000001e-06 2023-10-09T05:35:23.635 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.635 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.636 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:35:23.636 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.039543+0000", 2023-10-09T05:35:23.636 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.6049999999999997e-06 2023-10-09T05:35:23.636 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.636 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.636 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:35:23.636 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.039769+0000", 2023-10-09T05:35:23.636 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000225858 2023-10-09T05:35:23.636 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.636 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.636 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:35:23.636 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.039904+0000", 2023-10-09T05:35:23.637 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000134907 2023-10-09T05:35:23.637 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.637 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.637 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:35:23.637 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.040022+0000", 2023-10-09T05:35:23.637 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000117731 2023-10-09T05:35:23.637 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.637 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.637 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:35:23.637 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.042227+0000", 2023-10-09T05:35:23.637 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0022056129999999999 2023-10-09T05:35:23.638 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.638 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.638 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.638 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.045713+0000", 2023-10-09T05:35:23.638 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0034855049999999999 2023-10-09T05:35:23.638 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.638 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.638 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.638 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.282376+0000", 2023-10-09T05:35:23.638 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.23666331600000001 2023-10-09T05:35:23.638 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.639 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.639 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:35:23.639 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.282430+0000", 2023-10-09T05:35:23.639 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.3631000000000002e-05 2023-10-09T05:35:23.639 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.639 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.639 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:35:23.639 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.282472+0000", 2023-10-09T05:35:23.639 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.1863000000000002e-05 2023-10-09T05:35:23.639 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.639 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:35:23.639 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.640 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.640 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.640 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:63828 13.6 13:62a7547a:::benchmark_data_smithi133_241067_object63827:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-10-09T05:35:23.640 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:31:27.042415+0000", 2023-10-09T05:35:23.640 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 236.55549834300001, 2023-10-09T05:35:23.640 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.23717179699999999, 2023-10-09T05:35:23.640 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:35:23.640 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:23.640 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:35:23.640 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:35:23.640 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:35:23.640 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 63828 2023-10-09T05:35:23.641 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.641 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:35:23.641 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.641 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:35:23.641 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.042415+0000", 2023-10-09T05:35:23.641 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.641 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.641 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.641 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:35:23.641 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.042415+0000", 2023-10-09T05:35:23.641 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.642 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.642 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.642 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:35:23.642 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.042413+0000", 2023-10-09T05:35:23.642 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:35:23.642 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.642 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.642 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:35:23.642 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.042438+0000", 2023-10-09T05:35:23.642 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.4428000000000001e-05 2023-10-09T05:35:23.642 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.642 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.643 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:35:23.643 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.042446+0000", 2023-10-09T05:35:23.643 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.1820000000000006e-06 2023-10-09T05:35:23.643 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.643 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.643 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:35:23.643 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.042452+0000", 2023-10-09T05:35:23.643 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.9109999999999998e-06 2023-10-09T05:35:23.643 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.643 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.643 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:35:23.643 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.042751+0000", 2023-10-09T05:35:23.644 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00029903599999999997 2023-10-09T05:35:23.644 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.644 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.644 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:35:23.644 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.042879+0000", 2023-10-09T05:35:23.644 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00012782700000000001 2023-10-09T05:35:23.644 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.644 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.644 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:35:23.644 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.043008+0000", 2023-10-09T05:35:23.644 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000129295 2023-10-09T05:35:23.644 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.645 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.645 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.645 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.046022+0000", 2023-10-09T05:35:23.645 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0030138740000000002 2023-10-09T05:35:23.645 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.645 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.645 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:35:23.645 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.046031+0000", 2023-10-09T05:35:23.645 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.7590000000000003e-06 2023-10-09T05:35:23.645 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.645 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.645 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.646 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.279477+0000", 2023-10-09T05:35:23.646 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.23344657099999999 2023-10-09T05:35:23.646 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.646 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.646 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:35:23.646 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.279544+0000", 2023-10-09T05:35:23.646 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.7218000000000006e-05 2023-10-09T05:35:23.646 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.646 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.646 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:35:23.646 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.279587+0000", 2023-10-09T05:35:23.646 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.2258000000000001e-05 2023-10-09T05:35:23.647 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.647 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:35:23.647 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.647 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.647 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.647 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:63836 13.2 13:47750eaf:::benchmark_data_smithi133_241067_object63835:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-10-09T05:35:23.647 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:31:27.050093+0000", 2023-10-09T05:35:23.647 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 236.54782034999999, 2023-10-09T05:35:23.647 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.229073795, 2023-10-09T05:35:23.647 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:35:23.647 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:23.647 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:35:23.647 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:35:23.648 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:35:23.648 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 63836 2023-10-09T05:35:23.648 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.648 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:35:23.648 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.648 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:35:23.648 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.050093+0000", 2023-10-09T05:35:23.648 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.648 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.648 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.648 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:35:23.648 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.050093+0000", 2023-10-09T05:35:23.649 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.649 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.649 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.649 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:35:23.649 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.050091+0000", 2023-10-09T05:35:23.649 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999981 2023-10-09T05:35:23.649 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.649 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.649 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:35:23.649 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.050116+0000", 2023-10-09T05:35:23.649 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.4681999999999998e-05 2023-10-09T05:35:23.650 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.650 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.650 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:35:23.650 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.050134+0000", 2023-10-09T05:35:23.650 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.8289999999999999e-05 2023-10-09T05:35:23.650 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.650 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.650 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:35:23.650 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.050144+0000", 2023-10-09T05:35:23.650 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.9180000000000006e-06 2023-10-09T05:35:23.650 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.650 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.651 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:35:23.651 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.050200+0000", 2023-10-09T05:35:23.651 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.5745000000000002e-05 2023-10-09T05:35:23.651 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.651 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.651 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:35:23.651 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.050362+0000", 2023-10-09T05:35:23.651 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00016249100000000001 2023-10-09T05:35:23.651 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.651 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.651 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:35:23.651 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.050531+0000", 2023-10-09T05:35:23.651 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000169106 2023-10-09T05:35:23.652 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.652 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.652 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.652 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.053316+0000", 2023-10-09T05:35:23.652 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0027845389999999999 2023-10-09T05:35:23.652 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.652 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.652 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:35:23.652 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.053336+0000", 2023-10-09T05:35:23.652 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.0613000000000001e-05 2023-10-09T05:35:23.652 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.653 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.653 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.653 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.279092+0000", 2023-10-09T05:35:23.653 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.225755441 2023-10-09T05:35:23.653 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.653 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.653 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:35:23.653 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.279133+0000", 2023-10-09T05:35:23.653 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.1322999999999997e-05 2023-10-09T05:35:23.653 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.653 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.653 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:35:23.653 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.279167+0000", 2023-10-09T05:35:23.654 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.3497000000000001e-05 2023-10-09T05:35:23.654 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.654 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:35:23.654 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.654 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.654 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.654 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:63842 13.b 13:dbe78422:::benchmark_data_smithi133_241067_object63841:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-10-09T05:35:23.654 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:31:27.057132+0000", 2023-10-09T05:35:23.654 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 236.54078108100001, 2023-10-09T05:35:23.654 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.23049175799999999, 2023-10-09T05:35:23.654 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:35:23.655 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:23.655 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:35:23.655 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:35:23.655 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:35:23.655 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 63842 2023-10-09T05:35:23.655 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.655 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:35:23.655 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.655 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:35:23.655 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.057132+0000", 2023-10-09T05:35:23.655 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.655 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.656 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.656 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:35:23.656 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.057132+0000", 2023-10-09T05:35:23.656 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.656 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.656 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.656 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:35:23.656 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.057130+0000", 2023-10-09T05:35:23.656 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999981 2023-10-09T05:35:23.656 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.656 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.656 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:35:23.657 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.057163+0000", 2023-10-09T05:35:23.657 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.2814000000000001e-05 2023-10-09T05:35:23.657 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.657 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.657 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:35:23.657 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.057181+0000", 2023-10-09T05:35:23.657 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.7682000000000001e-05 2023-10-09T05:35:23.657 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.657 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.657 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:35:23.657 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.057191+0000", 2023-10-09T05:35:23.657 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.0118999999999999e-05 2023-10-09T05:35:23.658 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.658 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.658 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:35:23.658 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.057237+0000", 2023-10-09T05:35:23.658 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.5896000000000002e-05 2023-10-09T05:35:23.658 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.658 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.658 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:35:23.658 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.057373+0000", 2023-10-09T05:35:23.658 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000136632 2023-10-09T05:35:23.658 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.659 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.659 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:35:23.659 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.057500+0000", 2023-10-09T05:35:23.659 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00012655500000000001 2023-10-09T05:35:23.659 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.659 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.659 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:35:23.659 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.059616+0000", 2023-10-09T05:35:23.659 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0021156769999999998 2023-10-09T05:35:23.659 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.659 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.659 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.660 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.059727+0000", 2023-10-09T05:35:23.660 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00011192399999999999 2023-10-09T05:35:23.660 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.660 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.660 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.660 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.287536+0000", 2023-10-09T05:35:23.660 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.227808333 2023-10-09T05:35:23.660 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.660 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.660 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:35:23.660 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.287582+0000", 2023-10-09T05:35:23.660 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.5986999999999999e-05 2023-10-09T05:35:23.660 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.661 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.661 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:35:23.661 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:27.287624+0000", 2023-10-09T05:35:23.661 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.2200999999999998e-05 2023-10-09T05:35:23.661 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.661 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:35:23.661 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.661 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.661 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.661 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:65074 13.b 13:d738b009:::benchmark_data_smithi133_241067_object65073:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-10-09T05:35:23.661 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:31:28.566404+0000", 2023-10-09T05:35:23.662 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 235.03150973800001, 2023-10-09T05:35:23.662 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.25848331800000002, 2023-10-09T05:35:23.662 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:35:23.662 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:23.662 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:35:23.662 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:35:23.662 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:35:23.662 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 65074 2023-10-09T05:35:23.662 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.662 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:35:23.662 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.663 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:35:23.663 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.566404+0000", 2023-10-09T05:35:23.663 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.663 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.663 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.663 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:35:23.663 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.566404+0000", 2023-10-09T05:35:23.663 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.663 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.663 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.663 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:35:23.663 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.566402+0000", 2023-10-09T05:35:23.663 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999981 2023-10-09T05:35:23.664 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.664 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.664 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:35:23.664 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.566435+0000", 2023-10-09T05:35:23.664 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.3538999999999999e-05 2023-10-09T05:35:23.664 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.664 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.664 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:35:23.664 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.566454+0000", 2023-10-09T05:35:23.664 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.8992999999999999e-05 2023-10-09T05:35:23.664 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.665 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.665 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:35:23.665 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.566463+0000", 2023-10-09T05:35:23.665 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.037e-06 2023-10-09T05:35:23.665 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.665 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.665 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:35:23.665 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.566565+0000", 2023-10-09T05:35:23.665 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00010196600000000001 2023-10-09T05:35:23.665 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.665 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.665 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:35:23.666 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.566742+0000", 2023-10-09T05:35:23.666 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00017734000000000001 2023-10-09T05:35:23.666 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.666 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.666 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:35:23.666 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.566917+0000", 2023-10-09T05:35:23.666 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00017453 2023-10-09T05:35:23.666 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.666 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.666 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.666 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.575609+0000", 2023-10-09T05:35:23.667 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0086917250000000008 2023-10-09T05:35:23.667 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.667 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.667 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:35:23.667 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.575659+0000", 2023-10-09T05:35:23.667 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.041e-05 2023-10-09T05:35:23.667 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.667 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.667 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.667 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.824623+0000", 2023-10-09T05:35:23.668 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.248963821 2023-10-09T05:35:23.668 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.668 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.668 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:35:23.668 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.824836+0000", 2023-10-09T05:35:23.668 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00021325399999999999 2023-10-09T05:35:23.668 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.668 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.668 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:35:23.668 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.824887+0000", 2023-10-09T05:35:23.668 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.0705000000000003e-05 2023-10-09T05:35:23.668 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.669 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:35:23.669 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.669 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.669 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.669 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:65075 13.b 13:dee07a12:::benchmark_data_smithi133_241067_object65074:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-10-09T05:35:23.669 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:31:28.566481+0000", 2023-10-09T05:35:23.669 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 235.031431973, 2023-10-09T05:35:23.669 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.26504462200000001, 2023-10-09T05:35:23.669 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:35:23.669 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:23.669 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:35:23.670 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:35:23.670 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:35:23.670 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 65075 2023-10-09T05:35:23.670 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.670 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:35:23.670 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.670 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:35:23.670 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.566481+0000", 2023-10-09T05:35:23.670 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.670 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.670 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.671 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:35:23.671 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.566481+0000", 2023-10-09T05:35:23.671 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.671 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.671 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.671 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:35:23.671 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.566481+0000", 2023-10-09T05:35:23.671 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999995 2023-10-09T05:35:23.671 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.671 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.671 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:35:23.672 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.566505+0000", 2023-10-09T05:35:23.672 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.4392999999999998e-05 2023-10-09T05:35:23.672 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.672 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.672 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:35:23.672 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.566514+0000", 2023-10-09T05:35:23.672 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.5779999999999999e-06 2023-10-09T05:35:23.672 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.672 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.672 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:35:23.672 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.566523+0000", 2023-10-09T05:35:23.673 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.3549999999999994e-06 2023-10-09T05:35:23.673 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.673 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.673 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:35:23.673 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.569015+0000", 2023-10-09T05:35:23.673 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0024916059999999999 2023-10-09T05:35:23.673 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.673 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.673 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:35:23.673 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.569216+0000", 2023-10-09T05:35:23.673 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00020149600000000001 2023-10-09T05:35:23.674 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.674 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.674 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:35:23.674 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.569530+0000", 2023-10-09T05:35:23.674 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000314383 2023-10-09T05:35:23.674 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.674 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.674 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.674 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.575652+0000", 2023-10-09T05:35:23.674 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.006121942 2023-10-09T05:35:23.674 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.675 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.675 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:35:23.675 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.575740+0000", 2023-10-09T05:35:23.675 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.7087000000000002e-05 2023-10-09T05:35:23.675 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.675 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.675 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.675 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.831385+0000", 2023-10-09T05:35:23.675 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.255645493 2023-10-09T05:35:23.675 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.675 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.675 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:35:23.675 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.831484+0000", 2023-10-09T05:35:23.676 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.9302999999999994e-05 2023-10-09T05:35:23.676 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.676 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.676 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:35:23.676 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.831526+0000", 2023-10-09T05:35:23.676 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.1594999999999997e-05 2023-10-09T05:35:23.676 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.676 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:35:23.676 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.676 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.676 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.677 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:65085 13.6 13:6b500555:::benchmark_data_smithi133_241067_object65084:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-10-09T05:35:23.677 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:31:28.575217+0000", 2023-10-09T05:35:23.677 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 235.022696631, 2023-10-09T05:35:23.677 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.24485874599999999, 2023-10-09T05:35:23.677 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:35:23.677 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:23.677 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:35:23.677 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:35:23.677 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:35:23.677 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 65085 2023-10-09T05:35:23.677 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.678 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:35:23.678 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.678 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:35:23.678 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.575217+0000", 2023-10-09T05:35:23.678 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.678 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.678 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.678 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:35:23.678 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.575217+0000", 2023-10-09T05:35:23.678 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.679 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.679 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.679 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:35:23.679 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.575215+0000", 2023-10-09T05:35:23.679 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:35:23.679 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.679 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.679 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:35:23.679 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.575239+0000", 2023-10-09T05:35:23.679 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.3439e-05 2023-10-09T05:35:23.679 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.680 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.680 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:35:23.680 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.575253+0000", 2023-10-09T05:35:23.680 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.4406000000000001e-05 2023-10-09T05:35:23.680 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.680 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.680 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:35:23.680 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.575261+0000", 2023-10-09T05:35:23.680 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.0190000000000006e-06 2023-10-09T05:35:23.680 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.680 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.681 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:35:23.681 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.575309+0000", 2023-10-09T05:35:23.681 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.7632000000000001e-05 2023-10-09T05:35:23.681 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.681 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.681 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:35:23.681 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.575428+0000", 2023-10-09T05:35:23.681 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000119042 2023-10-09T05:35:23.681 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.681 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.682 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:35:23.682 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.575547+0000", 2023-10-09T05:35:23.682 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000119307 2023-10-09T05:35:23.682 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.682 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.682 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:35:23.682 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.577719+0000", 2023-10-09T05:35:23.682 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0021724980000000001 2023-10-09T05:35:23.682 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.682 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.682 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.682 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.578045+0000", 2023-10-09T05:35:23.683 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00032524099999999998 2023-10-09T05:35:23.683 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.683 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.683 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.683 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.819975+0000", 2023-10-09T05:35:23.683 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.241930178 2023-10-09T05:35:23.683 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.683 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.683 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:35:23.683 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.820030+0000", 2023-10-09T05:35:23.683 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.4917000000000003e-05 2023-10-09T05:35:23.683 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.684 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.684 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:35:23.684 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.820075+0000", 2023-10-09T05:35:23.684 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.5621999999999998e-05 2023-10-09T05:35:23.684 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.684 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:35:23.684 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.684 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.684 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.684 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:65087 13.2 13:467d2ec0:::benchmark_data_smithi133_241067_object65086:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-10-09T05:35:23.684 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:31:28.577008+0000", 2023-10-09T05:35:23.684 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 235.02090496100001, 2023-10-09T05:35:23.685 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.23737668100000001, 2023-10-09T05:35:23.685 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:35:23.685 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:23.685 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:35:23.685 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:35:23.685 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:35:23.685 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 65087 2023-10-09T05:35:23.685 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.685 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:35:23.685 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.685 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:35:23.686 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.577008+0000", 2023-10-09T05:35:23.686 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.686 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.686 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.686 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:35:23.686 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.577008+0000", 2023-10-09T05:35:23.686 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.686 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.686 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.686 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:35:23.686 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.577007+0000", 2023-10-09T05:35:23.686 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:35:23.687 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.687 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.687 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:35:23.687 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.577030+0000", 2023-10-09T05:35:23.687 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.3606999999999999e-05 2023-10-09T05:35:23.687 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.687 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.687 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:35:23.687 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.577045+0000", 2023-10-09T05:35:23.687 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.4144e-05 2023-10-09T05:35:23.687 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.688 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.688 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:35:23.688 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.577053+0000", 2023-10-09T05:35:23.688 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.1310000000000006e-06 2023-10-09T05:35:23.688 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.688 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.688 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:35:23.688 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.577105+0000", 2023-10-09T05:35:23.688 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.2256999999999998e-05 2023-10-09T05:35:23.688 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.688 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.689 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:35:23.689 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.577251+0000", 2023-10-09T05:35:23.689 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000145761 2023-10-09T05:35:23.689 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.689 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.689 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:35:23.689 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.577402+0000", 2023-10-09T05:35:23.689 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00015089300000000001 2023-10-09T05:35:23.689 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.689 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.689 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:35:23.690 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.579474+0000", 2023-10-09T05:35:23.690 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0020722990000000001 2023-10-09T05:35:23.690 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.690 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.690 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.690 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.580543+0000", 2023-10-09T05:35:23.690 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.001068935 2023-10-09T05:35:23.690 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.690 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.690 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.690 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.814284+0000", 2023-10-09T05:35:23.690 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.23374089100000001 2023-10-09T05:35:23.691 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.691 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.691 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:35:23.691 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.814345+0000", 2023-10-09T05:35:23.691 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.1161999999999995e-05 2023-10-09T05:35:23.691 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.691 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.691 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:35:23.691 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.814385+0000", 2023-10-09T05:35:23.691 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.0098999999999997e-05 2023-10-09T05:35:23.691 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.691 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:35:23.692 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.692 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.692 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.692 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:65090 13.b 13:d2e75b73:::benchmark_data_smithi133_241067_object65089:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-10-09T05:35:23.692 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:31:28.580916+0000", 2023-10-09T05:35:23.692 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 235.01699693200001, 2023-10-09T05:35:23.692 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.25079236700000002, 2023-10-09T05:35:23.692 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:35:23.692 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:23.692 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:35:23.692 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:35:23.692 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:35:23.693 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 65090 2023-10-09T05:35:23.693 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.693 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:35:23.693 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.693 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:35:23.693 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.580916+0000", 2023-10-09T05:35:23.693 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.693 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.693 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.693 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:35:23.693 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.580916+0000", 2023-10-09T05:35:23.693 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.694 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.694 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.694 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:35:23.694 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.580915+0000", 2023-10-09T05:35:23.694 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:35:23.694 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.694 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.694 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:35:23.694 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.580936+0000", 2023-10-09T05:35:23.694 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.0727999999999999e-05 2023-10-09T05:35:23.694 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.695 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.695 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:35:23.695 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.580947+0000", 2023-10-09T05:35:23.695 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.0729e-05 2023-10-09T05:35:23.695 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.695 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.695 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:35:23.695 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.580952+0000", 2023-10-09T05:35:23.695 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.3990000000000003e-06 2023-10-09T05:35:23.695 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.695 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.695 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:35:23.696 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.581014+0000", 2023-10-09T05:35:23.696 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.2243999999999996e-05 2023-10-09T05:35:23.696 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.696 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.696 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:35:23.696 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.581185+0000", 2023-10-09T05:35:23.696 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00017059399999999999 2023-10-09T05:35:23.696 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.696 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.696 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:35:23.696 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.581373+0000", 2023-10-09T05:35:23.696 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000188135 2023-10-09T05:35:23.697 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.697 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.697 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:35:23.697 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.583481+0000", 2023-10-09T05:35:23.697 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0021076480000000002 2023-10-09T05:35:23.697 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.697 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.697 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.697 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.584236+0000", 2023-10-09T05:35:23.697 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00075537 2023-10-09T05:35:23.697 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.697 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.698 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.698 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.831581+0000", 2023-10-09T05:35:23.698 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.247344599 2023-10-09T05:35:23.698 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.698 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.698 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:35:23.698 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.831670+0000", 2023-10-09T05:35:23.698 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.9409999999999999e-05 2023-10-09T05:35:23.698 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.698 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.698 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:35:23.699 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:28.831709+0000", 2023-10-09T05:35:23.699 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.8776999999999998e-05 2023-10-09T05:35:23.699 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.699 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:35:23.699 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.699 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.699 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.699 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:66773 13.6 13:604ede19:::benchmark_data_smithi133_241067_object66772:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-10-09T05:35:23.699 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:31:30.485253+0000", 2023-10-09T05:35:23.699 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 233.11266015999999, 2023-10-09T05:35:23.699 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.22977592999999999, 2023-10-09T05:35:23.700 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:35:23.700 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:23.700 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:35:23.700 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:35:23.700 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:35:23.700 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 66773 2023-10-09T05:35:23.700 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.700 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:35:23.700 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.700 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:35:23.700 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:30.485253+0000", 2023-10-09T05:35:23.700 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.700 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.701 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.701 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:35:23.701 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:30.485253+0000", 2023-10-09T05:35:23.701 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.701 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.701 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.701 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:35:23.701 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:30.485253+0000", 2023-10-09T05:35:23.701 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999995 2023-10-09T05:35:23.701 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.701 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.701 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:35:23.702 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:30.485303+0000", 2023-10-09T05:35:23.702 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.0744000000000001e-05 2023-10-09T05:35:23.702 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.702 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.702 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:35:23.702 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:30.485313+0000", 2023-10-09T05:35:23.702 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.9229999999999997e-06 2023-10-09T05:35:23.702 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.702 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.702 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:35:23.702 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:30.485320+0000", 2023-10-09T05:35:23.702 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.216e-06 2023-10-09T05:35:23.703 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.703 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.703 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:35:23.703 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:30.488123+0000", 2023-10-09T05:35:23.703 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0028023589999999999 2023-10-09T05:35:23.703 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.703 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.703 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:35:23.703 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:30.488213+0000", 2023-10-09T05:35:23.703 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.0725999999999998e-05 2023-10-09T05:35:23.703 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.704 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.704 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:35:23.704 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:30.488551+0000", 2023-10-09T05:35:23.704 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00033777600000000003 2023-10-09T05:35:23.704 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.704 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.704 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:35:23.704 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:30.495195+0000", 2023-10-09T05:35:23.704 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0066437049999999997 2023-10-09T05:35:23.704 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.704 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.704 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.705 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:30.502563+0000", 2023-10-09T05:35:23.705 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0073678909999999997 2023-10-09T05:35:23.705 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.705 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.705 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.705 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:30.714934+0000", 2023-10-09T05:35:23.705 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.212371592 2023-10-09T05:35:23.705 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.705 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.705 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:35:23.705 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:30.714986+0000", 2023-10-09T05:35:23.706 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.1211000000000001e-05 2023-10-09T05:35:23.706 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.706 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.706 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:35:23.706 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:31:30.715029+0000", 2023-10-09T05:35:23.706 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.3381999999999999e-05 2023-10-09T05:35:23.706 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.706 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:35:23.706 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.706 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.707 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.707 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:150052 13.6 13:63e176b9:::benchmark_data_smithi133_241067_object52012:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2023-10-09T05:35:23.707 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:33:29.787996+0000", 2023-10-09T05:35:23.707 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 113.80991763999999, 2023-10-09T05:35:23.707 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.24123220200000001, 2023-10-09T05:35:23.707 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:35:23.707 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:23.707 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:35:23.707 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:35:23.707 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:35:23.707 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 150052 2023-10-09T05:35:23.708 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.708 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:35:23.708 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.708 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:35:23.708 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.787996+0000", 2023-10-09T05:35:23.708 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.708 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.708 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.708 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:35:23.708 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.787996+0000", 2023-10-09T05:35:23.708 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.709 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.709 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.709 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:35:23.709 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.787994+0000", 2023-10-09T05:35:23.709 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999981 2023-10-09T05:35:23.709 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.709 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.709 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:35:23.709 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.788002+0000", 2023-10-09T05:35:23.709 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.0399999999999993e-06 2023-10-09T05:35:23.709 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.710 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.710 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:35:23.710 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.788018+0000", 2023-10-09T05:35:23.710 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.6545000000000002e-05 2023-10-09T05:35:23.710 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.710 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.710 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:35:23.710 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.788027+0000", 2023-10-09T05:35:23.710 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.4530000000000002e-06 2023-10-09T05:35:23.710 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.711 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.711 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:35:23.711 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.788090+0000", 2023-10-09T05:35:23.711 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.2798000000000004e-05 2023-10-09T05:35:23.711 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.711 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.711 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:35:23.711 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.788248+0000", 2023-10-09T05:35:23.711 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000157937 2023-10-09T05:35:23.711 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.711 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.712 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 0,2,5", 2023-10-09T05:35:23.712 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.788399+0000", 2023-10-09T05:35:23.712 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00015111599999999999 2023-10-09T05:35:23.712 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.712 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.712 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:35:23.712 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.792972+0000", 2023-10-09T05:35:23.712 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0045736639999999999 2023-10-09T05:35:23.712 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.712 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.712 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.713 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.793813+0000", 2023-10-09T05:35:23.713 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00084055800000000004 2023-10-09T05:35:23.713 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.713 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.713 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.713 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.796862+0000", 2023-10-09T05:35:23.713 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0030485849999999999 2023-10-09T05:35:23.713 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.713 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.713 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.713 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:30.028742+0000", 2023-10-09T05:35:23.713 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.23188023099999999 2023-10-09T05:35:23.714 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.714 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.714 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:35:23.714 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:30.029080+0000", 2023-10-09T05:35:23.714 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00033806999999999999 2023-10-09T05:35:23.714 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.714 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.714 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:35:23.714 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:30.029228+0000", 2023-10-09T05:35:23.714 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000147954 2023-10-09T05:35:23.714 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.715 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:35:23.715 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.715 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.715 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.715 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:150054 13.2 13:47013130:::benchmark_data_smithi133_241067_object52014:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2023-10-09T05:35:23.715 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:33:29.788640+0000", 2023-10-09T05:35:23.715 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 113.809273602, 2023-10-09T05:35:23.715 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.23872981099999999, 2023-10-09T05:35:23.715 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:35:23.715 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:23.715 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:35:23.715 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:35:23.716 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:35:23.716 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 150054 2023-10-09T05:35:23.716 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.716 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:35:23.716 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.716 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:35:23.716 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.788640+0000", 2023-10-09T05:35:23.716 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.716 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.716 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.716 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:35:23.717 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.788640+0000", 2023-10-09T05:35:23.717 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.717 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.717 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.717 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:35:23.717 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.788639+0000", 2023-10-09T05:35:23.717 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:35:23.717 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.717 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.717 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:35:23.717 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.788644+0000", 2023-10-09T05:35:23.717 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.6200000000000004e-06 2023-10-09T05:35:23.718 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.718 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.718 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:35:23.718 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.788656+0000", 2023-10-09T05:35:23.718 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.1086e-05 2023-10-09T05:35:23.718 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.718 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.718 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:35:23.718 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.788663+0000", 2023-10-09T05:35:23.718 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.4789999999999999e-06 2023-10-09T05:35:23.718 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.719 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.719 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:35:23.719 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.788712+0000", 2023-10-09T05:35:23.719 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.8603999999999997e-05 2023-10-09T05:35:23.719 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.719 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.719 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:35:23.719 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.788852+0000", 2023-10-09T05:35:23.719 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00014073800000000001 2023-10-09T05:35:23.719 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.719 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.719 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T05:35:23.720 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.788998+0000", 2023-10-09T05:35:23.720 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00014577399999999999 2023-10-09T05:35:23.720 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.720 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.720 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:35:23.720 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.797060+0000", 2023-10-09T05:35:23.720 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0080623459999999997 2023-10-09T05:35:23.720 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.720 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.720 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.720 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.797997+0000", 2023-10-09T05:35:23.721 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000936476 2023-10-09T05:35:23.721 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.721 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.721 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.721 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.798682+0000", 2023-10-09T05:35:23.721 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00068456500000000004 2023-10-09T05:35:23.721 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.721 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.721 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.721 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:30.027063+0000", 2023-10-09T05:35:23.721 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.22838177200000001 2023-10-09T05:35:23.721 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.722 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.722 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:35:23.722 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:30.027244+0000", 2023-10-09T05:35:23.722 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00018113600000000001 2023-10-09T05:35:23.722 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.722 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.722 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:35:23.722 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:30.027369+0000", 2023-10-09T05:35:23.722 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00012502999999999999 2023-10-09T05:35:23.722 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.722 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:35:23.723 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.723 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.723 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.723 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:150057 13.2 13:4c88d5e8:::benchmark_data_smithi133_241067_object52017:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2023-10-09T05:35:23.723 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:33:29.792847+0000", 2023-10-09T05:35:23.723 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 113.805066615, 2023-10-09T05:35:23.723 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.23571537100000001, 2023-10-09T05:35:23.723 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:35:23.723 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:23.723 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:35:23.723 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:35:23.723 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:35:23.724 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 150057 2023-10-09T05:35:23.724 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.724 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:35:23.724 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.724 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:35:23.724 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.792847+0000", 2023-10-09T05:35:23.724 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.724 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.724 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.724 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:35:23.724 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.792847+0000", 2023-10-09T05:35:23.725 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.725 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.725 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.725 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:35:23.725 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.792845+0000", 2023-10-09T05:35:23.725 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:35:23.725 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.725 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.725 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:35:23.725 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.792852+0000", 2023-10-09T05:35:23.725 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.3309999999999998e-06 2023-10-09T05:35:23.725 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.726 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.726 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:35:23.726 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.792865+0000", 2023-10-09T05:35:23.726 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.2697000000000001e-05 2023-10-09T05:35:23.726 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.726 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.726 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:35:23.726 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.792874+0000", 2023-10-09T05:35:23.726 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.5310000000000001e-06 2023-10-09T05:35:23.726 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.726 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.726 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:35:23.727 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.793218+0000", 2023-10-09T05:35:23.727 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00034467699999999999 2023-10-09T05:35:23.727 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.727 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.727 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:35:23.727 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.793457+0000", 2023-10-09T05:35:23.727 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00023867199999999999 2023-10-09T05:35:23.727 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.727 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.727 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T05:35:23.727 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.793559+0000", 2023-10-09T05:35:23.727 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000101702 2023-10-09T05:35:23.728 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.728 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.728 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:35:23.728 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.797072+0000", 2023-10-09T05:35:23.728 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0035128730000000001 2023-10-09T05:35:23.728 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.728 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.728 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.728 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.798100+0000", 2023-10-09T05:35:23.728 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.001028485 2023-10-09T05:35:23.728 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.729 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.729 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.729 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:29.798805+0000", 2023-10-09T05:35:23.729 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00070507399999999998 2023-10-09T05:35:23.729 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.729 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.729 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.729 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:30.027409+0000", 2023-10-09T05:35:23.729 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.22860408800000001 2023-10-09T05:35:23.729 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.729 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.729 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:35:23.730 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:30.027455+0000", 2023-10-09T05:35:23.730 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.5707000000000001e-05 2023-10-09T05:35:23.730 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.730 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.730 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:35:23.730 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:30.028562+0000", 2023-10-09T05:35:23.730 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0011071340000000001 2023-10-09T05:35:23.730 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.730 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:35:23.730 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.730 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.730 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.731 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:152501 13.6 13:6e47de18:::benchmark_data_smithi133_241067_object54461:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2023-10-09T05:35:23.731 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:33:33.919410+0000", 2023-10-09T05:35:23.731 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 109.678503573, 2023-10-09T05:35:23.731 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.52472146799999997, 2023-10-09T05:35:23.731 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:35:23.731 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:23.731 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:35:23.731 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:35:23.731 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:35:23.731 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 152501 2023-10-09T05:35:23.733 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.733 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:35:23.733 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.734 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:35:23.734 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:33.919410+0000", 2023-10-09T05:35:23.734 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.734 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.734 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.734 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:35:23.735 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:33.919410+0000", 2023-10-09T05:35:23.735 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.735 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.735 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.735 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:35:23.735 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:33.919408+0000", 2023-10-09T05:35:23.735 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:35:23.735 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.735 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.735 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:35:23.735 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:33.919417+0000", 2023-10-09T05:35:23.736 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.7749999999999998e-06 2023-10-09T05:35:23.736 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.736 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.736 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:35:23.736 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:33.919434+0000", 2023-10-09T05:35:23.736 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.7354e-05 2023-10-09T05:35:23.736 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.736 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.736 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:35:23.736 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:33.919441+0000", 2023-10-09T05:35:23.736 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.1829999999999997e-06 2023-10-09T05:35:23.736 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.737 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.737 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:35:23.737 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:33.923861+0000", 2023-10-09T05:35:23.737 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0044199729999999998 2023-10-09T05:35:23.737 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.737 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.737 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:35:23.737 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:33.924024+0000", 2023-10-09T05:35:23.737 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00016288000000000001 2023-10-09T05:35:23.737 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.737 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.737 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 0,2,5", 2023-10-09T05:35:23.737 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:33.924183+0000", 2023-10-09T05:35:23.738 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000158455 2023-10-09T05:35:23.738 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.738 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.738 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.738 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:33.929005+0000", 2023-10-09T05:35:23.738 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0048225860000000002 2023-10-09T05:35:23.738 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.738 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.738 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:35:23.738 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:33.929017+0000", 2023-10-09T05:35:23.739 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.1656e-05 2023-10-09T05:35:23.739 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.739 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.739 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.739 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:33.929523+0000", 2023-10-09T05:35:23.739 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00050569099999999995 2023-10-09T05:35:23.739 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.739 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.739 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.739 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:34.443880+0000", 2023-10-09T05:35:23.739 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.51435731699999998 2023-10-09T05:35:23.739 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.740 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.740 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:35:23.740 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:34.443941+0000", 2023-10-09T05:35:23.740 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.1332000000000002e-05 2023-10-09T05:35:23.740 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.740 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.740 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:35:23.740 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:33:34.444131+0000", 2023-10-09T05:35:23.740 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00018991 2023-10-09T05:35:23.740 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.740 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:35:23.740 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.741 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.741 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.741 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:194593 13.6 13:6ebbaa2b:::benchmark_data_smithi133_241067_object96553:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e305)", 2023-10-09T05:35:23.741 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:34:42.483733+0000", 2023-10-09T05:35:23.741 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 41.114180490999999, 2023-10-09T05:35:23.741 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.264407583, 2023-10-09T05:35:23.741 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:35:23.741 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:23.741 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:35:23.741 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:35:23.741 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:35:23.741 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 194593 2023-10-09T05:35:23.742 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.742 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:35:23.742 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.742 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:35:23.742 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:42.483733+0000", 2023-10-09T05:35:23.742 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.742 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.742 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.742 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:35:23.742 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:42.483733+0000", 2023-10-09T05:35:23.742 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.742 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.743 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.743 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:35:23.743 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:42.483731+0000", 2023-10-09T05:35:23.743 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:35:23.743 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.743 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.743 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:35:23.743 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:42.483740+0000", 2023-10-09T05:35:23.743 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.1449999999999997e-06 2023-10-09T05:35:23.743 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.743 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.744 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:35:23.744 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:42.483756+0000", 2023-10-09T05:35:23.744 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.5991999999999999e-05 2023-10-09T05:35:23.744 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.744 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.744 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:35:23.744 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:42.483764+0000", 2023-10-09T05:35:23.744 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.943e-06 2023-10-09T05:35:23.744 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.744 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.744 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:35:23.744 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:42.483823+0000", 2023-10-09T05:35:23.745 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.8325000000000001e-05 2023-10-09T05:35:23.745 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.745 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.745 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:35:23.745 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:42.483991+0000", 2023-10-09T05:35:23.745 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00016791500000000001 2023-10-09T05:35:23.745 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.745 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.745 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 0,2,5", 2023-10-09T05:35:23.745 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:42.484145+0000", 2023-10-09T05:35:23.745 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00015486699999999999 2023-10-09T05:35:23.745 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.746 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.746 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:35:23.746 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:42.488488+0000", 2023-10-09T05:35:23.746 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0043430819999999998 2023-10-09T05:35:23.746 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.746 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.746 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.746 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:42.488822+0000", 2023-10-09T05:35:23.746 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00033317999999999998 2023-10-09T05:35:23.746 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.746 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.746 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.747 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:42.490735+0000", 2023-10-09T05:35:23.747 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00191347 2023-10-09T05:35:23.747 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.747 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.747 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.747 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:42.747982+0000", 2023-10-09T05:35:23.747 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.25724691500000002 2023-10-09T05:35:23.747 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.747 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.747 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:35:23.747 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:42.748029+0000", 2023-10-09T05:35:23.748 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.7159e-05 2023-10-09T05:35:23.748 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.748 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.748 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:35:23.748 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:42.748140+0000", 2023-10-09T05:35:23.748 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00011116 2023-10-09T05:35:23.748 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.748 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:35:23.748 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.748 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.748 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.748 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.45596.0:194612 13.2 13:4ef838b9:::benchmark_data_smithi133_241067_object96572:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e305)", 2023-10-09T05:35:23.748 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:34:42.508160+0000", 2023-10-09T05:35:23.749 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 41.089753041999998, 2023-10-09T05:35:23.749 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.23993339599999999, 2023-10-09T05:35:23.749 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:35:23.749 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:23.749 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:35:23.749 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.45596", 2023-10-09T05:35:23.749 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:35:23.749 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 194612 2023-10-09T05:35:23.749 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.749 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:35:23.749 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.749 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:35:23.750 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:42.508160+0000", 2023-10-09T05:35:23.750 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.750 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.750 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.750 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:35:23.750 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:42.508160+0000", 2023-10-09T05:35:23.750 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:35:23.750 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.750 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.750 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:35:23.750 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:42.508159+0000", 2023-10-09T05:35:23.751 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:35:23.751 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.751 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.751 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:35:23.751 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:42.508165+0000", 2023-10-09T05:35:23.751 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.2899999999999999e-06 2023-10-09T05:35:23.751 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.751 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.751 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:35:23.751 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:42.508178+0000", 2023-10-09T05:35:23.751 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.2377e-05 2023-10-09T05:35:23.751 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.752 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.752 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:35:23.752 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:42.508184+0000", 2023-10-09T05:35:23.752 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.426e-06 2023-10-09T05:35:23.752 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.752 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.752 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:35:23.752 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:42.508240+0000", 2023-10-09T05:35:23.752 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.6156999999999998e-05 2023-10-09T05:35:23.752 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.752 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.752 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:35:23.753 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:42.508393+0000", 2023-10-09T05:35:23.753 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000152616 2023-10-09T05:35:23.753 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.753 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.753 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T05:35:23.753 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:42.508516+0000", 2023-10-09T05:35:23.753 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00012270099999999999 2023-10-09T05:35:23.753 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.753 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.753 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.753 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:42.512024+0000", 2023-10-09T05:35:23.753 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0035083900000000001 2023-10-09T05:35:23.754 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.754 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.754 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:35:23.754 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:42.512094+0000", 2023-10-09T05:35:23.754 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.0208000000000003e-05 2023-10-09T05:35:23.754 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.754 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.754 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.754 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:42.512610+0000", 2023-10-09T05:35:23.754 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00051580599999999995 2023-10-09T05:35:23.754 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.755 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.755 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:23.755 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:42.747909+0000", 2023-10-09T05:35:23.755 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.235299008 2023-10-09T05:35:23.755 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.755 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.755 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:35:23.755 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:42.747979+0000", 2023-10-09T05:35:23.755 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.0390999999999997e-05 2023-10-09T05:35:23.755 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:35:23.755 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:35:23.755 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:35:23.756 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:34:42.748094+0000", 2023-10-09T05:35:23.756 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000114145 2023-10-09T05:35:23.756 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.756 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:35:23.756 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.756 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:35:23.756 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:35:23.756 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:35:23.757 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:23.789 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:23.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:23.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:23.797 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:23 smithi133 bash[105589]: audit 2023-10-09T05:35:23.060623+0000 mon.a (mon.0) 1773 : audit 1 from='client.? 172.21.15.133:0/1144116630' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:35:23.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:23 smithi194 bash[92666]: cluster 2023-10-09T05:35:22.376417+0000 mgr.y (mgr.25277) 1422 : cluster 0 pgmap v1523: 89 pgs: 89 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s, 82 objects/s recovering 2023-10-09T05:35:23.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:23 smithi194 bash[92666]: audit 2023-10-09T05:35:22.786629+0000 mon.a (mon.0) 1772 : audit 0 from='client.? 172.21.15.133:0/2053763882' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:35:23.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:23 smithi194 bash[92666]: audit 2023-10-09T05:35:23.060623+0000 mon.a (mon.0) 1773 : audit 1 from='client.? 172.21.15.133:0/1144116630' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:35:23.893 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:23.913 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:23.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:23.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:24.016 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:24.045 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:24.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:24.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:24.149 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:24.171 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:24.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:24.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:24.274 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:24.293 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:24.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:24.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:24.396 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:24.434 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:24.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:24.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:24.538 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:24.546 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:35:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:35:24] "GET /metrics HTTP/1.1" 200 36474 "" "Prometheus/2.43.0" 2023-10-09T05:35:24.559 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:24.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:24.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:24.662 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:24.693 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:24.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:24.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:24.796 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:24.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:24 smithi133 bash[105589]: audit 2023-10-09T05:35:23.377959+0000 mgr.y (mgr.25277) 1423 : audit 0 from='client.35724 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:35:24.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:24 smithi133 bash[105589]: cluster 2023-10-09T05:35:23.567925+0000 osd.6 (osd.6) 267 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T05:35:24.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:24 smithi133 bash[105589]: audit 2023-10-09T05:35:23.570876+0000 mon.a (mon.0) 1774 : audit 1 from='client.? 172.21.15.133:0/1144116630' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:35:24.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:24 smithi133 bash[105589]: cluster 2023-10-09T05:35:23.570958+0000 mon.a (mon.0) 1775 : cluster 0 osdmap e309: 8 total, 8 up, 7 in 2023-10-09T05:35:24.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:24 smithi133 bash[105589]: audit 2023-10-09T05:35:23.864088+0000 mon.c (mon.2) 130 : audit 1 from='client.? 172.21.15.133:0/2416280972' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:35:24.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:24 smithi133 bash[105589]: audit 2023-10-09T05:35:23.864361+0000 mon.a (mon.0) 1776 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:35:24.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:24 smithi133 bash[105589]: cluster 2023-10-09T05:35:23.950809+0000 osd.4 (osd.4) 277 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T05:35:24.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:24 smithi133 bash[105589]: cluster 2023-10-09T05:35:23.953828+0000 osd.4 (osd.4) 278 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T05:35:24.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:24 smithi133 bash[105589]: cluster 2023-10-09T05:35:24.184453+0000 osd.5 (osd.5) 287 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T05:35:24.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:24 smithi133 bash[102858]: audit 2023-10-09T05:35:23.377959+0000 mgr.y (mgr.25277) 1423 : audit 0 from='client.35724 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:35:24.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:24 smithi133 bash[102858]: cluster 2023-10-09T05:35:23.567925+0000 osd.6 (osd.6) 267 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T05:35:24.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:24 smithi133 bash[102858]: audit 2023-10-09T05:35:23.570876+0000 mon.a (mon.0) 1774 : audit 1 from='client.? 172.21.15.133:0/1144116630' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:35:24.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:24 smithi133 bash[102858]: cluster 2023-10-09T05:35:23.570958+0000 mon.a (mon.0) 1775 : cluster 0 osdmap e309: 8 total, 8 up, 7 in 2023-10-09T05:35:24.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:24 smithi133 bash[102858]: audit 2023-10-09T05:35:23.864088+0000 mon.c (mon.2) 130 : audit 1 from='client.? 172.21.15.133:0/2416280972' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:35:24.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:24 smithi133 bash[102858]: audit 2023-10-09T05:35:23.864361+0000 mon.a (mon.0) 1776 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:35:24.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:24 smithi133 bash[102858]: cluster 2023-10-09T05:35:23.950809+0000 osd.4 (osd.4) 277 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T05:35:24.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:24 smithi133 bash[102858]: cluster 2023-10-09T05:35:23.953828+0000 osd.4 (osd.4) 278 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T05:35:24.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:24 smithi133 bash[102858]: cluster 2023-10-09T05:35:24.184453+0000 osd.5 (osd.5) 287 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T05:35:24.824 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:24.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:24.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:24.857 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is set 2023-10-09T05:35:24.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:24 smithi194 bash[92666]: audit 2023-10-09T05:35:23.377959+0000 mgr.y (mgr.25277) 1423 : audit 0 from='client.35724 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:35:24.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:24 smithi194 bash[92666]: cluster 2023-10-09T05:35:23.567925+0000 osd.6 (osd.6) 267 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T05:35:24.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:24 smithi194 bash[92666]: audit 2023-10-09T05:35:23.570876+0000 mon.a (mon.0) 1774 : audit 1 from='client.? 172.21.15.133:0/1144116630' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:35:24.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:24 smithi194 bash[92666]: cluster 2023-10-09T05:35:23.570958+0000 mon.a (mon.0) 1775 : cluster 0 osdmap e309: 8 total, 8 up, 7 in 2023-10-09T05:35:24.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:24 smithi194 bash[92666]: audit 2023-10-09T05:35:23.864088+0000 mon.c (mon.2) 130 : audit 1 from='client.? 172.21.15.133:0/2416280972' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:35:24.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:24 smithi194 bash[92666]: audit 2023-10-09T05:35:23.864361+0000 mon.a (mon.0) 1776 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:35:24.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:24 smithi194 bash[92666]: cluster 2023-10-09T05:35:23.950809+0000 osd.4 (osd.4) 277 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T05:35:24.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:24 smithi194 bash[92666]: cluster 2023-10-09T05:35:23.953828+0000 osd.4 (osd.4) 278 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T05:35:24.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:24 smithi194 bash[92666]: cluster 2023-10-09T05:35:24.184453+0000 osd.5 (osd.5) 287 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T05:35:24.928 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:24.958 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:24.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:24.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:25.062 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:25.092 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:25.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:25.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:25.195 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:25.223 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:25.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:25.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:25.326 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:25.352 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:25.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:25.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:25.455 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:25.488 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:25.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:25.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:25.591 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:25.605 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:25 smithi194 bash[92666]: cluster 2023-10-09T05:35:24.184469+0000 osd.7 (osd.7) 272 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T05:35:25.605 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:25 smithi194 bash[92666]: cluster 2023-10-09T05:35:24.377058+0000 mgr.y (mgr.25277) 1424 : cluster 0 pgmap v1525: 73 pgs: 73 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:25.605 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:25 smithi194 bash[92666]: cluster 2023-10-09T05:35:24.846563+0000 mon.a (mon.0) 1777 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:35:25.606 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:25 smithi194 bash[92666]: cluster 2023-10-09T05:35:24.850176+0000 osd.5 (osd.5) 288 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T05:35:25.606 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:25 smithi194 bash[92666]: audit 2023-10-09T05:35:24.852244+0000 mon.a (mon.0) 1778 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:35:25.606 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:25 smithi194 bash[92666]: cluster 2023-10-09T05:35:24.852333+0000 mon.a (mon.0) 1779 : cluster 0 osdmap e310: 8 total, 8 up, 7 in 2023-10-09T05:35:25.606 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:25 smithi194 bash[92666]: cluster 2023-10-09T05:35:25.285196+0000 osd.4 (osd.4) 279 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T05:35:25.625 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:25.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:25.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:25.728 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:25.763 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:25.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:25.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:25.867 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:25.901 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:25.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:25.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:26.004 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:26.013 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:25 smithi133 bash[102858]: cluster 2023-10-09T05:35:24.184469+0000 osd.7 (osd.7) 272 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T05:35:26.013 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:25 smithi133 bash[102858]: cluster 2023-10-09T05:35:24.377058+0000 mgr.y (mgr.25277) 1424 : cluster 0 pgmap v1525: 73 pgs: 73 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:26.013 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:25 smithi133 bash[102858]: cluster 2023-10-09T05:35:24.846563+0000 mon.a (mon.0) 1777 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:35:26.013 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:25 smithi133 bash[102858]: cluster 2023-10-09T05:35:24.850176+0000 osd.5 (osd.5) 288 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T05:35:26.013 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:25 smithi133 bash[102858]: audit 2023-10-09T05:35:24.852244+0000 mon.a (mon.0) 1778 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:35:26.013 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:25 smithi133 bash[102858]: cluster 2023-10-09T05:35:24.852333+0000 mon.a (mon.0) 1779 : cluster 0 osdmap e310: 8 total, 8 up, 7 in 2023-10-09T05:35:26.013 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:25 smithi133 bash[102858]: cluster 2023-10-09T05:35:25.285196+0000 osd.4 (osd.4) 279 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T05:35:26.013 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:25 smithi133 bash[105589]: cluster 2023-10-09T05:35:24.184469+0000 osd.7 (osd.7) 272 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T05:35:26.014 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:25 smithi133 bash[105589]: cluster 2023-10-09T05:35:24.377058+0000 mgr.y (mgr.25277) 1424 : cluster 0 pgmap v1525: 73 pgs: 73 active+clean; 582 KiB data, 7.1 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:26.014 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:25 smithi133 bash[105589]: cluster 2023-10-09T05:35:24.846563+0000 mon.a (mon.0) 1777 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:35:26.014 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:25 smithi133 bash[105589]: cluster 2023-10-09T05:35:24.850176+0000 osd.5 (osd.5) 288 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T05:35:26.014 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:25 smithi133 bash[105589]: audit 2023-10-09T05:35:24.852244+0000 mon.a (mon.0) 1778 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:35:26.014 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:25 smithi133 bash[105589]: cluster 2023-10-09T05:35:24.852333+0000 mon.a (mon.0) 1779 : cluster 0 osdmap e310: 8 total, 8 up, 7 in 2023-10-09T05:35:26.014 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:25 smithi133 bash[105589]: cluster 2023-10-09T05:35:25.285196+0000 osd.4 (osd.4) 279 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T05:35:26.026 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:26.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:26.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:26.129 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:26.151 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:26.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:26.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:26.254 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:26.274 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:26.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:26.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:26.377 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:26.402 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:26.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:26.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:26.504 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:26.531 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:26.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:26.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:26.635 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:26.667 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:26.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:26.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:26.771 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:26.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:26 smithi133 bash[102858]: cluster 2023-10-09T05:35:25.285338+0000 osd.6 (osd.6) 268 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T05:35:26.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:26 smithi133 bash[102858]: cluster 2023-10-09T05:35:25.285627+0000 osd.7 (osd.7) 273 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T05:35:26.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:26 smithi133 bash[105589]: cluster 2023-10-09T05:35:25.285338+0000 osd.6 (osd.6) 268 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T05:35:26.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:26 smithi133 bash[105589]: cluster 2023-10-09T05:35:25.285627+0000 osd.7 (osd.7) 273 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T05:35:26.799 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:26.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:26.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:26.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:26 smithi194 bash[92666]: cluster 2023-10-09T05:35:25.285338+0000 osd.6 (osd.6) 268 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T05:35:26.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:26 smithi194 bash[92666]: cluster 2023-10-09T05:35:25.285627+0000 osd.7 (osd.7) 273 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T05:35:26.902 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:26.932 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:26.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:26.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:27.037 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:27.074 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:27.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:27.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:27.178 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:27.213 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:27.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:27.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:27.316 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:27.344 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:27.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:27.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:27.447 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:27.469 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:27.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:27.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:27.571 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:27.593 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:27.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:27.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:27.696 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:27.729 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:27.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:27.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:27.833 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:27.858 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:27.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:27.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:27.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:27 smithi194 bash[92666]: cluster 2023-10-09T05:35:26.378152+0000 mgr.y (mgr.25277) 1425 : cluster 0 pgmap v1527: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:27.961 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:27.991 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:27.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:27.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:28.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:27 smithi133 bash[102858]: cluster 2023-10-09T05:35:26.378152+0000 mgr.y (mgr.25277) 1425 : cluster 0 pgmap v1527: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:28.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:27 smithi133 bash[105589]: cluster 2023-10-09T05:35:26.378152+0000 mgr.y (mgr.25277) 1425 : cluster 0 pgmap v1527: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:28.096 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:28.129 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:28.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:28.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:28.233 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:28.265 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:28.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:28.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:28.369 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:28.400 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:28.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:28.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:28.504 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:28.538 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:28.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:28.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:28.642 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:28.673 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:28.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:28.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:28.776 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:28.806 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:28.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:28.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:28.881 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T05:35:28.910 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:28.942 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:28.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:28.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:29.011 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T05:35:29.012 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T05:35:29.012 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_9 2023-10-09T05:35:29.012 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_9 16 2023-10-09T05:35:29.047 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:29.089 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:29.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:29.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:29.147 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:35:29.192 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:29.227 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:29.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:29.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:29.331 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:29.365 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:29.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:29.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:29.469 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:29.505 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:29.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:29.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:29.609 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:29.637 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:29.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:29.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:29.741 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:29.767 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:29 smithi194 bash[92666]: cluster 2023-10-09T05:35:28.379002+0000 mgr.y (mgr.25277) 1426 : cluster 0 pgmap v1528: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:29.767 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:29 smithi194 bash[92666]: audit 2023-10-09T05:35:28.493725+0000 mon.a (mon.0) 1780 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:35:29.768 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:29 smithi194 bash[92666]: audit 2023-10-09T05:35:28.494826+0000 mon.a (mon.0) 1781 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:35:29.768 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:29 smithi194 bash[92666]: audit 2023-10-09T05:35:29.223813+0000 mon.a (mon.0) 1782 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:35:29.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:29.789 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:29.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:29.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:29 smithi133 bash[102858]: cluster 2023-10-09T05:35:28.379002+0000 mgr.y (mgr.25277) 1426 : cluster 0 pgmap v1528: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:29.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:29 smithi133 bash[102858]: audit 2023-10-09T05:35:28.493725+0000 mon.a (mon.0) 1780 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:35:29.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:29 smithi133 bash[102858]: audit 2023-10-09T05:35:28.494826+0000 mon.a (mon.0) 1781 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:35:29.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:29 smithi133 bash[102858]: audit 2023-10-09T05:35:29.223813+0000 mon.a (mon.0) 1782 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:35:29.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:29 smithi133 bash[105589]: cluster 2023-10-09T05:35:28.379002+0000 mgr.y (mgr.25277) 1426 : cluster 0 pgmap v1528: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:29.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:29 smithi133 bash[105589]: audit 2023-10-09T05:35:28.493725+0000 mon.a (mon.0) 1780 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:35:29.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:29 smithi133 bash[105589]: audit 2023-10-09T05:35:28.494826+0000 mon.a (mon.0) 1781 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:35:29.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:29 smithi133 bash[105589]: audit 2023-10-09T05:35:29.223813+0000 mon.a (mon.0) 1782 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:35:29.890 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:29.926 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:29.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:29.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:30.031 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:30.062 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:30.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:30.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:30.166 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:30.197 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:30.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:30.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:30.301 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:30.336 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:30.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:30.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:30.440 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:30.474 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:30.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:30.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:30.578 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:30.611 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:30.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:30.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:30.716 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:30.750 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:30.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:30.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:30.854 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:30.888 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:30.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:30.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:30.992 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:31.020 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:35:31.029 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:31.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:31.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:31.133 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:31.164 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:31.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:31.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:31.268 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:31.300 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:31.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:31.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:31.404 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:31.436 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:31.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:31.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:31.540 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:31.570 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:31.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:31.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:31.673 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:31.687 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:31 smithi194 bash[92666]: cluster 2023-10-09T05:35:30.379946+0000 mgr.y (mgr.25277) 1427 : cluster 0 pgmap v1529: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:31.707 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:31.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:31.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:31.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:31 smithi133 bash[105589]: cluster 2023-10-09T05:35:30.379946+0000 mgr.y (mgr.25277) 1427 : cluster 0 pgmap v1529: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:31.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:31 smithi133 bash[102858]: cluster 2023-10-09T05:35:30.379946+0000 mgr.y (mgr.25277) 1427 : cluster 0 pgmap v1529: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:31.812 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:31.843 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:31.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:31.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:31.947 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:31.978 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:31.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:31.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:32.082 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:32.118 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:32.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:32.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:32.221 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:32.254 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:32.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:32.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:32.359 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:32.389 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:32.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:32.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:32.493 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:32.527 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:32.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:32.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:32.630 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:32.661 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:32.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:32.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:32.765 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:32.797 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:32.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:32.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:32.901 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:32.932 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:32.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:32.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:33.036 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:33.068 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:33.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:33.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:33.172 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:33.201 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:33.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:33.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:33.304 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:33.339 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:33.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:33.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:33.444 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:33.476 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:33.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:33.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:33.484 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 6, 1, 7, 5] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:35:33.484 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:35:33.579 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:33.591 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:33 smithi194 bash[92666]: cluster 2023-10-09T05:35:32.380842+0000 mgr.y (mgr.25277) 1428 : cluster 0 pgmap v1530: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:33.610 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:33.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:33.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:33.658 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:35:33.714 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:33.751 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:33.752 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:33 smithi133 bash[102858]: cluster 2023-10-09T05:35:32.380842+0000 mgr.y (mgr.25277) 1428 : cluster 0 pgmap v1530: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:33.752 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:33 smithi133 bash[105589]: cluster 2023-10-09T05:35:32.380842+0000 mgr.y (mgr.25277) 1428 : cluster 0 pgmap v1530: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:33.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:33.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:33.803 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:33.804 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:33.854 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:33.882 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:33.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:33.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:33.986 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:34.012 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:34.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:34.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:34.116 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:34.146 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:34.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:34.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:34.248 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:34.273 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:34.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:34.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:34.377 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:34.400 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:34.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:34.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:34.503 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:34.530 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:34.531 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:35:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:35:34] "GET /metrics HTTP/1.1" 200 34070 "" "Prometheus/2.43.0" 2023-10-09T05:35:34.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:34.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:34.633 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:34.658 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:34.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:34.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:34.762 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:34.792 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:34.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:34.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:34.817 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:34.895 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:34.906 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:34 smithi194 bash[92666]: cluster 2023-10-09T05:35:34.381465+0000 mgr.y (mgr.25277) 1429 : cluster 0 pgmap v1531: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:34.925 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:34.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:34.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:35.030 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:35.061 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:35.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:35.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:35.164 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:35.176 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:34 smithi133 bash[105589]: cluster 2023-10-09T05:35:34.381465+0000 mgr.y (mgr.25277) 1429 : cluster 0 pgmap v1531: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:35.176 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:34 smithi133 bash[102858]: cluster 2023-10-09T05:35:34.381465+0000 mgr.y (mgr.25277) 1429 : cluster 0 pgmap v1531: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:35.190 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:35.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:35.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:35.294 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:35.322 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:35.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:35.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:35.427 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:35.456 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:35.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:35.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:35.559 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:35.584 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:35.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:35.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:35.687 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:35.714 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:35.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:35.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:35.817 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:35.844 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:35.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:35.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:35.947 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:35.980 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:35.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:35.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:36.083 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:36.107 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:36.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:36.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:36.210 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:36.239 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:36.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:36.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:36.343 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:36.371 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:36.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:36.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:36.475 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:36.506 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:36.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:36.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:36.610 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:36.633 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:36.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:36.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:36.737 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:36.765 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:36.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:36.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:36.869 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:36.899 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:36.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:36.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:37.002 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:37.032 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:37.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:37.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:37.135 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:37.163 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:37.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:37.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:37.266 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:37.300 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:37.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:37.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:37.404 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:37.436 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:37.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:37.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:37.540 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:37.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:37 smithi133 bash[105589]: cluster 2023-10-09T05:35:36.382148+0000 mgr.y (mgr.25277) 1430 : cluster 0 pgmap v1532: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:37.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:37 smithi133 bash[102858]: cluster 2023-10-09T05:35:36.382148+0000 mgr.y (mgr.25277) 1430 : cluster 0 pgmap v1532: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:37.573 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:37.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:37.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:37.612 INFO:teuthology.orchestra.run.smithi194.stdout:{ 2023-10-09T05:35:37.612 INFO:teuthology.orchestra.run.smithi194.stdout: "ops": [], 2023-10-09T05:35:37.612 INFO:teuthology.orchestra.run.smithi194.stdout: "num_ops": 0 2023-10-09T05:35:37.612 INFO:teuthology.orchestra.run.smithi194.stdout:} 2023-10-09T05:35:37.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:37 smithi194 bash[92666]: cluster 2023-10-09T05:35:36.382148+0000 mgr.y (mgr.25277) 1430 : cluster 0 pgmap v1532: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:37.677 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:37.712 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:37.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:37.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:37.814 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:37.846 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:37.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:37.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:37.949 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:37.975 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:37.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:37.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:38.079 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:38.110 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:38.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:38.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:38.213 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:38.237 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:38.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:38.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:38.340 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:38.369 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:38.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:38.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:38.473 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:38.495 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:38.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:38.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:38.569 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T05:35:38.599 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:38.624 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:38.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:38.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:38.727 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:38.759 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:38.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:38.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:38.862 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:38.891 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:38.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:38.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:38.995 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:39.021 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:39.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:39.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:39.124 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:39.153 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:39.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:39.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:39.256 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:39.281 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:39.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:39.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:39.385 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:39.412 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:39.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:39.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:39.516 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:39.542 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:39.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:39.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:39.645 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:39.676 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:39.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:39.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:39.780 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:39.791 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:39 smithi194 bash[92666]: cluster 2023-10-09T05:35:38.382820+0000 mgr.y (mgr.25277) 1431 : cluster 0 pgmap v1533: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:39.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:39 smithi133 bash[102858]: cluster 2023-10-09T05:35:38.382820+0000 mgr.y (mgr.25277) 1431 : cluster 0 pgmap v1533: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:39.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:39 smithi133 bash[105589]: cluster 2023-10-09T05:35:38.382820+0000 mgr.y (mgr.25277) 1431 : cluster 0 pgmap v1533: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:39.809 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:39.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:39.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:39.912 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:39.942 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:39.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:39.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:40.045 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:40.076 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:40.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:40.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:40.179 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:40.241 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:40.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:40.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:40.344 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:40.376 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:40.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:40.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:40.481 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:40.507 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:40.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:40.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:40.610 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:40.642 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:40.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:40.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:40.746 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:40.773 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:40.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:40.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:40.876 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:40.914 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:40.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:40.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:41.017 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:41.045 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:41.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:41.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:41.148 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:41.173 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:41.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:41.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:41.276 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:41.298 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:41.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:41.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:41.402 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:41.426 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:41.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:41.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:41.529 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:41.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:41 smithi133 bash[102858]: cluster 2023-10-09T05:35:40.383560+0000 mgr.y (mgr.25277) 1432 : cluster 0 pgmap v1534: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:41.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:41 smithi133 bash[105589]: cluster 2023-10-09T05:35:40.383560+0000 mgr.y (mgr.25277) 1432 : cluster 0 pgmap v1534: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:41.560 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:41.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:41.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:41.664 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:41.700 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:41.701 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:41 smithi194 bash[92666]: cluster 2023-10-09T05:35:40.383560+0000 mgr.y (mgr.25277) 1432 : cluster 0 pgmap v1534: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:41.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:41.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:41.803 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:41.837 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:41.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:41.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:41.941 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:41.971 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:41.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:41.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:42.074 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:42.099 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:42.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:42.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:42.203 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:42.226 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:42.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:42.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:42.330 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:42.355 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:42.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:42.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:42.458 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:42.493 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:42.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:42.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:42.597 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:42.619 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:42.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:42.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:42.723 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:42.759 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:42.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:42.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:42.863 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:42.889 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:42.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:42.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:42.992 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:43.021 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:43.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:43.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:43.126 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:43.150 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:43.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:43.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:43.254 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:43.289 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:43.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:43.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:43.347 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:35:43.394 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:43.426 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:43.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:43.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:43.529 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:43.550 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:43.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:43.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:43.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:43 smithi194 bash[92666]: cluster 2023-10-09T05:35:42.384054+0000 mgr.y (mgr.25277) 1433 : cluster 0 pgmap v1535: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:43.653 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:43.676 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:43.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:43.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:43.780 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:43.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:43 smithi133 bash[102858]: cluster 2023-10-09T05:35:42.384054+0000 mgr.y (mgr.25277) 1433 : cluster 0 pgmap v1535: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:43.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:43 smithi133 bash[105589]: cluster 2023-10-09T05:35:42.384054+0000 mgr.y (mgr.25277) 1433 : cluster 0 pgmap v1535: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:43.812 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:43.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:43.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:43.916 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:43.947 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:43.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:43.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:44.052 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:44.072 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:44.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:44.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:44.175 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:44.207 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:44.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:44.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:44.311 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:44.341 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:44.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:44.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:44.444 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:44.469 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:44.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:44.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:44.547 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:35:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:35:44] "GET /metrics HTTP/1.1" 200 34059 "" "Prometheus/2.43.0" 2023-10-09T05:35:44.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:44 smithi133 bash[105589]: audit 2023-10-09T05:35:43.488065+0000 mon.a (mon.0) 1783 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:35:44.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:44 smithi133 bash[105589]: audit 2023-10-09T05:35:44.348484+0000 mon.a (mon.0) 1784 : audit 1 from='client.? 172.21.15.133:0/691781245' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:35:44.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:44 smithi133 bash[102858]: audit 2023-10-09T05:35:43.488065+0000 mon.a (mon.0) 1783 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:35:44.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:44 smithi133 bash[102858]: audit 2023-10-09T05:35:44.348484+0000 mon.a (mon.0) 1784 : audit 1 from='client.? 172.21.15.133:0/691781245' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:35:44.563 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:35:44.574 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:44.587 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:35:44.602 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:44.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:44.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:44.616 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:35:44.645 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:35:44.672 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:35:44.697 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:35:44.706 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:44.719 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:44 smithi194 bash[92666]: audit 2023-10-09T05:35:43.488065+0000 mon.a (mon.0) 1783 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:35:44.719 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:44 smithi194 bash[92666]: audit 2023-10-09T05:35:44.348484+0000 mon.a (mon.0) 1784 : audit 1 from='client.? 172.21.15.133:0/691781245' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:35:44.722 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:35:44.740 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:44.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:44.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:44.749 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:35:44.751 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:35:44.751 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:35:44.751 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:35:44.751 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:35:44.751 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:35:44.751 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:35:44.752 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:35:44.752 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:35:44.844 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:44.869 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:44.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:44.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:44.973 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:45.005 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:45.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:45.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:45.109 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:45.133 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:45.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:45.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:45.238 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:45.263 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:45.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:45.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:45.298 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is unset 2023-10-09T05:35:45.300 INFO:teuthology.orchestra.run.smithi133.stderr:pool 'unique_pool_9' created 2023-10-09T05:35:45.367 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:45.391 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:45.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:45.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:45.494 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:45.525 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:45.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:45.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:45.629 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:45.640 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:45 smithi133 bash[102858]: cluster 2023-10-09T05:35:44.384641+0000 mgr.y (mgr.25277) 1434 : cluster 0 pgmap v1536: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:45.640 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:45 smithi133 bash[102858]: audit 2023-10-09T05:35:44.535243+0000 mon.a (mon.0) 1785 : audit 0 from='client.? 172.21.15.133:0/2418343007' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:35:45.641 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:45 smithi133 bash[102858]: audit 2023-10-09T05:35:44.830689+0000 mon.c (mon.2) 131 : audit 1 from='client.? 172.21.15.133:0/331731876' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-10-09T05:35:45.641 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:45 smithi133 bash[102858]: audit 2023-10-09T05:35:44.831353+0000 mon.a (mon.0) 1786 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-10-09T05:35:45.641 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:45 smithi133 bash[102858]: cluster 2023-10-09T05:35:45.278224+0000 mon.a (mon.0) 1787 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:35:45.641 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:45 smithi133 bash[102858]: audit 2023-10-09T05:35:45.290333+0000 mon.a (mon.0) 1788 : audit 1 from='client.? 172.21.15.133:0/691781245' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:35:45.641 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:45 smithi133 bash[102858]: audit 2023-10-09T05:35:45.290411+0000 mon.a (mon.0) 1789 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2023-10-09T05:35:45.641 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:45 smithi133 bash[102858]: cluster 2023-10-09T05:35:45.290483+0000 mon.a (mon.0) 1790 : cluster 0 osdmap e311: 8 total, 8 up, 7 in 2023-10-09T05:35:45.641 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:45 smithi133 bash[105589]: cluster 2023-10-09T05:35:44.384641+0000 mgr.y (mgr.25277) 1434 : cluster 0 pgmap v1536: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:45.641 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:45 smithi133 bash[105589]: audit 2023-10-09T05:35:44.535243+0000 mon.a (mon.0) 1785 : audit 0 from='client.? 172.21.15.133:0/2418343007' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:35:45.641 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:45 smithi133 bash[105589]: audit 2023-10-09T05:35:44.830689+0000 mon.c (mon.2) 131 : audit 1 from='client.? 172.21.15.133:0/331731876' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-10-09T05:35:45.642 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:45 smithi133 bash[105589]: audit 2023-10-09T05:35:44.831353+0000 mon.a (mon.0) 1786 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-10-09T05:35:45.642 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:45 smithi133 bash[105589]: cluster 2023-10-09T05:35:45.278224+0000 mon.a (mon.0) 1787 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:35:45.642 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:45 smithi133 bash[105589]: audit 2023-10-09T05:35:45.290333+0000 mon.a (mon.0) 1788 : audit 1 from='client.? 172.21.15.133:0/691781245' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:35:45.642 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:45 smithi133 bash[105589]: audit 2023-10-09T05:35:45.290411+0000 mon.a (mon.0) 1789 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2023-10-09T05:35:45.642 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:45 smithi133 bash[105589]: cluster 2023-10-09T05:35:45.290483+0000 mon.a (mon.0) 1790 : cluster 0 osdmap e311: 8 total, 8 up, 7 in 2023-10-09T05:35:45.658 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:45.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:45.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:45.762 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:45.789 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:45.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:45.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:45.852 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:45 smithi194 bash[92666]: cluster 2023-10-09T05:35:44.384641+0000 mgr.y (mgr.25277) 1434 : cluster 0 pgmap v1536: 73 pgs: 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:45.852 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:45 smithi194 bash[92666]: audit 2023-10-09T05:35:44.535243+0000 mon.a (mon.0) 1785 : audit 0 from='client.? 172.21.15.133:0/2418343007' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:35:45.852 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:45 smithi194 bash[92666]: audit 2023-10-09T05:35:44.830689+0000 mon.c (mon.2) 131 : audit 1 from='client.? 172.21.15.133:0/331731876' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-10-09T05:35:45.852 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:45 smithi194 bash[92666]: audit 2023-10-09T05:35:44.831353+0000 mon.a (mon.0) 1786 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-10-09T05:35:45.852 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:45 smithi194 bash[92666]: cluster 2023-10-09T05:35:45.278224+0000 mon.a (mon.0) 1787 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:35:45.852 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:45 smithi194 bash[92666]: audit 2023-10-09T05:35:45.290333+0000 mon.a (mon.0) 1788 : audit 1 from='client.? 172.21.15.133:0/691781245' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:35:45.853 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:45 smithi194 bash[92666]: audit 2023-10-09T05:35:45.290411+0000 mon.a (mon.0) 1789 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2023-10-09T05:35:45.853 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:45 smithi194 bash[92666]: cluster 2023-10-09T05:35:45.290483+0000 mon.a (mon.0) 1790 : cluster 0 osdmap e311: 8 total, 8 up, 7 in 2023-10-09T05:35:45.892 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:45.913 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:45.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:45.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:46.017 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:46.045 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:46.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:46.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:46.148 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:46.179 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:46.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:46.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:46.282 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:46.313 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:46.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:46.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:46.415 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:46.446 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:46.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:46.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:46.549 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:46.585 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:46.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:46.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:46.688 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:46.696 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:46 smithi194 bash[92666]: cluster 2023-10-09T05:35:45.288362+0000 osd.5 (osd.5) 289 : cluster 3 failed to encode map e311 with expected crc 2023-10-09T05:35:46.696 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:46 smithi194 bash[92666]: cluster 2023-10-09T05:35:45.294040+0000 osd.6 (osd.6) 269 : cluster 3 failed to encode map e311 with expected crc 2023-10-09T05:35:46.696 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:46 smithi194 bash[92666]: cluster 2023-10-09T05:35:45.297626+0000 osd.6 (osd.6) 270 : cluster 3 failed to encode map e311 with expected crc 2023-10-09T05:35:46.696 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:46 smithi194 bash[92666]: cluster 2023-10-09T05:35:45.554896+0000 osd.4 (osd.4) 280 : cluster 3 failed to encode map e311 with expected crc 2023-10-09T05:35:46.696 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:46 smithi194 bash[92666]: cluster 2023-10-09T05:35:46.295024+0000 mon.a (mon.0) 1791 : cluster 0 osdmap e312: 8 total, 8 up, 7 in 2023-10-09T05:35:46.714 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:46.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:46.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:46.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:46 smithi133 bash[102858]: cluster 2023-10-09T05:35:45.288362+0000 osd.5 (osd.5) 289 : cluster 3 failed to encode map e311 with expected crc 2023-10-09T05:35:46.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:46 smithi133 bash[102858]: cluster 2023-10-09T05:35:45.294040+0000 osd.6 (osd.6) 269 : cluster 3 failed to encode map e311 with expected crc 2023-10-09T05:35:46.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:46 smithi133 bash[102858]: cluster 2023-10-09T05:35:45.297626+0000 osd.6 (osd.6) 270 : cluster 3 failed to encode map e311 with expected crc 2023-10-09T05:35:46.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:46 smithi133 bash[102858]: cluster 2023-10-09T05:35:45.554896+0000 osd.4 (osd.4) 280 : cluster 3 failed to encode map e311 with expected crc 2023-10-09T05:35:46.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:46 smithi133 bash[102858]: cluster 2023-10-09T05:35:46.295024+0000 mon.a (mon.0) 1791 : cluster 0 osdmap e312: 8 total, 8 up, 7 in 2023-10-09T05:35:46.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:46 smithi133 bash[105589]: cluster 2023-10-09T05:35:45.288362+0000 osd.5 (osd.5) 289 : cluster 3 failed to encode map e311 with expected crc 2023-10-09T05:35:46.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:46 smithi133 bash[105589]: cluster 2023-10-09T05:35:45.294040+0000 osd.6 (osd.6) 269 : cluster 3 failed to encode map e311 with expected crc 2023-10-09T05:35:46.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:46 smithi133 bash[105589]: cluster 2023-10-09T05:35:45.297626+0000 osd.6 (osd.6) 270 : cluster 3 failed to encode map e311 with expected crc 2023-10-09T05:35:46.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:46 smithi133 bash[105589]: cluster 2023-10-09T05:35:45.554896+0000 osd.4 (osd.4) 280 : cluster 3 failed to encode map e311 with expected crc 2023-10-09T05:35:46.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:46 smithi133 bash[105589]: cluster 2023-10-09T05:35:46.295024+0000 mon.a (mon.0) 1791 : cluster 0 osdmap e312: 8 total, 8 up, 7 in 2023-10-09T05:35:46.817 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:46.846 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:46.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:46.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:46.949 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:46.977 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:46.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:46.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:47.081 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:47.115 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:47.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:47.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:47.219 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:47.244 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:35:47.244 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:35:47.248 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:47.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:47.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:47.351 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:47.375 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:47.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:47.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:47.478 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:47.504 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:47.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:47.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:47.607 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:47.620 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:47 smithi194 bash[92666]: cluster 2023-10-09T05:35:46.295567+0000 osd.7 (osd.7) 274 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T05:35:47.620 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:47 smithi194 bash[92666]: cluster 2023-10-09T05:35:46.295634+0000 osd.6 (osd.6) 271 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T05:35:47.621 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:47 smithi194 bash[92666]: cluster 2023-10-09T05:35:46.295977+0000 osd.5 (osd.5) 290 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T05:35:47.621 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:47 smithi194 bash[92666]: cluster 2023-10-09T05:35:46.385299+0000 mgr.y (mgr.25277) 1435 : cluster 0 pgmap v1539: 89 pgs: 1 creating+peering, 15 unknown, 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:47.621 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:47 smithi194 bash[92666]: cluster 2023-10-09T05:35:46.687739+0000 osd.4 (osd.4) 281 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T05:35:47.621 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:47 smithi194 bash[92666]: cluster 2023-10-09T05:35:47.300952+0000 mon.a (mon.0) 1792 : cluster 0 osdmap e313: 8 total, 8 up, 7 in 2023-10-09T05:35:47.635 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:47.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:47.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:47.738 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:47.767 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:47.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:47.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:47.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:47 smithi133 bash[102858]: cluster 2023-10-09T05:35:46.295567+0000 osd.7 (osd.7) 274 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T05:35:47.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:47 smithi133 bash[102858]: cluster 2023-10-09T05:35:46.295634+0000 osd.6 (osd.6) 271 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T05:35:47.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:47 smithi133 bash[102858]: cluster 2023-10-09T05:35:46.295977+0000 osd.5 (osd.5) 290 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T05:35:47.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:47 smithi133 bash[102858]: cluster 2023-10-09T05:35:46.385299+0000 mgr.y (mgr.25277) 1435 : cluster 0 pgmap v1539: 89 pgs: 1 creating+peering, 15 unknown, 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:47.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:47 smithi133 bash[102858]: cluster 2023-10-09T05:35:46.687739+0000 osd.4 (osd.4) 281 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T05:35:47.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:47 smithi133 bash[102858]: cluster 2023-10-09T05:35:47.300952+0000 mon.a (mon.0) 1792 : cluster 0 osdmap e313: 8 total, 8 up, 7 in 2023-10-09T05:35:47.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:47 smithi133 bash[105589]: cluster 2023-10-09T05:35:46.295567+0000 osd.7 (osd.7) 274 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T05:35:47.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:47 smithi133 bash[105589]: cluster 2023-10-09T05:35:46.295634+0000 osd.6 (osd.6) 271 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T05:35:47.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:47 smithi133 bash[105589]: cluster 2023-10-09T05:35:46.295977+0000 osd.5 (osd.5) 290 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T05:35:47.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:47 smithi133 bash[105589]: cluster 2023-10-09T05:35:46.385299+0000 mgr.y (mgr.25277) 1435 : cluster 0 pgmap v1539: 89 pgs: 1 creating+peering, 15 unknown, 73 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:47.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:47 smithi133 bash[105589]: cluster 2023-10-09T05:35:46.687739+0000 osd.4 (osd.4) 281 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T05:35:47.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:47 smithi133 bash[105589]: cluster 2023-10-09T05:35:47.300952+0000 mon.a (mon.0) 1792 : cluster 0 osdmap e313: 8 total, 8 up, 7 in 2023-10-09T05:35:47.854 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_9 rados --yes-i-really-mean-it || true 2023-10-09T05:35:47.870 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:47.928 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:47.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:47.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:47.986 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:35:48.032 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:48.061 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:48.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:48.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:48.165 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:48.191 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:48.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:48.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:48.222 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T05:35:48.294 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:48.322 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:48.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:48.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:48.426 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:48.453 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:48.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:48.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:48.557 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:48.570 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:48 smithi194 bash[92666]: cluster 2023-10-09T05:35:47.301791+0000 osd.5 (osd.5) 291 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T05:35:48.570 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:48 smithi194 bash[92666]: cluster 2023-10-09T05:35:47.466287+0000 osd.6 (osd.6) 272 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T05:35:48.570 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:48 smithi194 bash[92666]: cluster 2023-10-09T05:35:47.912416+0000 osd.7 (osd.7) 275 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T05:35:48.587 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:48.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:48.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:48.691 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:48.724 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:48.727 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:48 smithi133 bash[105589]: cluster 2023-10-09T05:35:47.301791+0000 osd.5 (osd.5) 291 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T05:35:48.727 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:48 smithi133 bash[105589]: cluster 2023-10-09T05:35:47.466287+0000 osd.6 (osd.6) 272 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T05:35:48.727 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:48 smithi133 bash[105589]: cluster 2023-10-09T05:35:47.912416+0000 osd.7 (osd.7) 275 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T05:35:48.727 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:48 smithi133 bash[102858]: cluster 2023-10-09T05:35:47.301791+0000 osd.5 (osd.5) 291 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T05:35:48.727 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:48 smithi133 bash[102858]: cluster 2023-10-09T05:35:47.466287+0000 osd.6 (osd.6) 272 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T05:35:48.727 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:48 smithi133 bash[102858]: cluster 2023-10-09T05:35:47.912416+0000 osd.7 (osd.7) 275 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T05:35:48.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:48.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:48.829 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:48.863 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:48.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:48.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:48.967 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:48.994 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:48.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:48.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:49.098 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:49.130 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:49.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:49.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:49.233 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:49.265 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:49.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:49.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:49.369 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:49.400 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:49.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:49.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:49.504 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:49.537 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:49.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:49.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:49.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:49 smithi133 bash[105589]: cluster 2023-10-09T05:35:48.385972+0000 mgr.y (mgr.25277) 1436 : cluster 0 pgmap v1541: 89 pgs: 1 creating+peering, 13 unknown, 75 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:49.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:49 smithi133 bash[105589]: cluster 2023-10-09T05:35:48.956625+0000 osd.4 (osd.4) 282 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T05:35:49.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:49 smithi133 bash[105589]: cluster 2023-10-09T05:35:48.959715+0000 osd.4 (osd.4) 283 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T05:35:49.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:49 smithi133 bash[105589]: cluster 2023-10-09T05:35:48.962526+0000 osd.4 (osd.4) 284 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T05:35:49.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:49 smithi133 bash[105589]: cluster 2023-10-09T05:35:48.964828+0000 osd.4 (osd.4) 285 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T05:35:49.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:49 smithi133 bash[102858]: cluster 2023-10-09T05:35:48.385972+0000 mgr.y (mgr.25277) 1436 : cluster 0 pgmap v1541: 89 pgs: 1 creating+peering, 13 unknown, 75 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:49.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:49 smithi133 bash[102858]: cluster 2023-10-09T05:35:48.956625+0000 osd.4 (osd.4) 282 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T05:35:49.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:49 smithi133 bash[102858]: cluster 2023-10-09T05:35:48.959715+0000 osd.4 (osd.4) 283 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T05:35:49.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:49 smithi133 bash[102858]: cluster 2023-10-09T05:35:48.962526+0000 osd.4 (osd.4) 284 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T05:35:49.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:49 smithi133 bash[102858]: cluster 2023-10-09T05:35:48.964828+0000 osd.4 (osd.4) 285 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T05:35:49.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:49 smithi194 bash[92666]: cluster 2023-10-09T05:35:48.385972+0000 mgr.y (mgr.25277) 1436 : cluster 0 pgmap v1541: 89 pgs: 1 creating+peering, 13 unknown, 75 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:49.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:49 smithi194 bash[92666]: cluster 2023-10-09T05:35:48.956625+0000 osd.4 (osd.4) 282 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T05:35:49.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:49 smithi194 bash[92666]: cluster 2023-10-09T05:35:48.959715+0000 osd.4 (osd.4) 283 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T05:35:49.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:49 smithi194 bash[92666]: cluster 2023-10-09T05:35:48.962526+0000 osd.4 (osd.4) 284 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T05:35:49.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:49 smithi194 bash[92666]: cluster 2023-10-09T05:35:48.964828+0000 osd.4 (osd.4) 285 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T05:35:49.641 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:49.672 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:49.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:49.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:49.776 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:49.807 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:49.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:49.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:49.843 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:35:49.911 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:49.942 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:49.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:49.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:50.045 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:50.078 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:50.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:50.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:50.183 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:50.216 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:50.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:50.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:50.320 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:50.354 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:50.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:50.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:50.458 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:50.490 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:50.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:50.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:50.593 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:50.624 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:50.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:50.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:50.728 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:50.763 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:50.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:50.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:50.867 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:50.902 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:50.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:50.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:51.005 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:51.034 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:51.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:51.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:51.138 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:51.169 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:51.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:51.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:51.273 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:51.308 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:51.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:51.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:51.411 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:51.447 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:51.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:51.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:51.551 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:51.582 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:51.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:51.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:51.686 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:51.715 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:51.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:51.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:51.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:51 smithi133 bash[105589]: cluster 2023-10-09T05:35:50.387005+0000 mgr.y (mgr.25277) 1437 : cluster 0 pgmap v1542: 89 pgs: 1 creating+peering, 88 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:51.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:51 smithi133 bash[102858]: cluster 2023-10-09T05:35:50.387005+0000 mgr.y (mgr.25277) 1437 : cluster 0 pgmap v1542: 89 pgs: 1 creating+peering, 88 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:51.817 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:51.831 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:51 smithi194 bash[92666]: cluster 2023-10-09T05:35:50.387005+0000 mgr.y (mgr.25277) 1437 : cluster 0 pgmap v1542: 89 pgs: 1 creating+peering, 88 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:51.849 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:51.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:51.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:51.952 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:51.985 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:51.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:51.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:52.089 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:52.118 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:52.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:52.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:52.221 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:52.253 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:52.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:52.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:52.357 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:52.392 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:52.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:52.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:52.496 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:52.531 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:52.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:52.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:52.635 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:52.666 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:52.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:52.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:52.684 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:52.684 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:52.769 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:52.797 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:52.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:52.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:52.900 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:52.931 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:52.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:52.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:52.997 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:35:53.034 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:53.063 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:53.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:53.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:53.167 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:53.189 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:53.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:53.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:53.293 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:53.321 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:53.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:53.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:53.424 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:53.452 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:53.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:53.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:53.555 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:53.583 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:53.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:53.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:53.687 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:53.697 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:53.698 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:53 smithi133 bash[105589]: cluster 2023-10-09T05:35:52.387804+0000 mgr.y (mgr.25277) 1438 : cluster 0 pgmap v1543: 89 pgs: 89 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:53.698 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:53 smithi133 bash[102858]: cluster 2023-10-09T05:35:52.387804+0000 mgr.y (mgr.25277) 1438 : cluster 0 pgmap v1543: 89 pgs: 89 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:53.715 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:53.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:53.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:53.820 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:53.849 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:53.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:53.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:53.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:53 smithi194 bash[92666]: cluster 2023-10-09T05:35:52.387804+0000 mgr.y (mgr.25277) 1438 : cluster 0 pgmap v1543: 89 pgs: 89 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:53.952 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:53.985 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:53.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:53.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:54.089 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:54.121 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:54.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:54.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:54.225 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:54.261 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:54.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:54.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:54.364 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:54.394 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:54.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:54.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:54.497 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:54.526 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:54.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:54.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:54.547 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:35:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:35:54] "GET /metrics HTTP/1.1" 200 34059 "" "Prometheus/2.43.0" 2023-10-09T05:35:54.630 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:54.652 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:54.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:54.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:54.756 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:54.785 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:54.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:54.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:54.888 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:54.902 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:54 smithi194 bash[92666]: cluster 2023-10-09T05:35:54.388209+0000 mgr.y (mgr.25277) 1439 : cluster 0 pgmap v1544: 89 pgs: 89 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:54.919 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:54.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:54.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:55.023 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:55.047 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:54 smithi133 bash[105589]: cluster 2023-10-09T05:35:54.388209+0000 mgr.y (mgr.25277) 1439 : cluster 0 pgmap v1544: 89 pgs: 89 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:55.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:54 smithi133 bash[102858]: cluster 2023-10-09T05:35:54.388209+0000 mgr.y (mgr.25277) 1439 : cluster 0 pgmap v1544: 89 pgs: 89 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:55.054 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:55.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:55.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:55.159 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:55.188 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:55.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:55.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:55.292 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:55.321 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:55.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:55.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:55.425 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:55.452 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:55.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:55.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:55.556 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:55.580 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:55.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:55.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:55.683 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:55.709 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:55.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:55.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:55.812 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:55.838 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:55.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:55.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:55.942 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:55.965 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:55.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:55.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:56.068 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:56.095 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:56.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:56.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:56.198 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:56.228 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:56.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:56.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:56.332 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:56.369 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:56.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:56.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:56.472 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:56.497 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:56.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:56.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:56.600 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:56.631 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:56.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:56.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:56.734 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:56.763 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:56.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:56.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:56.869 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:56.901 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:56.903 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:35:56.903 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:35:56.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:56.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:57.005 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:57.036 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:57.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:57.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:57.140 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:57.169 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:57.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:57.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:57.272 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:57.304 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:57.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:57.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:57.408 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:57.441 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:57.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:57.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:57.545 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:57.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:57 smithi133 bash[102858]: cluster 2023-10-09T05:35:56.389359+0000 mgr.y (mgr.25277) 1440 : cluster 0 pgmap v1545: 89 pgs: 89 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:57.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:57 smithi133 bash[105589]: cluster 2023-10-09T05:35:56.389359+0000 mgr.y (mgr.25277) 1440 : cluster 0 pgmap v1545: 89 pgs: 89 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:57.571 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:57.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:57.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:57.675 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:57.710 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:57.711 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:57 smithi194 bash[92666]: cluster 2023-10-09T05:35:56.389359+0000 mgr.y (mgr.25277) 1440 : cluster 0 pgmap v1545: 89 pgs: 89 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:57.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:57.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:57.813 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:57.841 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:57.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:57.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:57.895 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T05:35:57.945 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:57.974 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:57.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:57.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:58.078 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:58.103 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:58.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:58.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:58.205 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:58.230 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:58.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:58.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:58.334 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:58.359 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:58.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:58.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:58.462 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:58.487 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:58.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:58.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:58.590 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:58.622 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:58.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:58.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:58.726 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:58.753 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:58.754 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:58 smithi133 bash[102858]: audit 2023-10-09T05:35:58.188041+0000 mon.a (mon.0) 1793 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:35:58.754 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:58 smithi133 bash[105589]: audit 2023-10-09T05:35:58.188041+0000 mon.a (mon.0) 1793 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:35:58.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:58.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:58.857 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:58.869 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:58 smithi194 bash[92666]: audit 2023-10-09T05:35:58.188041+0000 mon.a (mon.0) 1793 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:35:58.886 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:58.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:58.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:58.991 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:59.021 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:59.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:59.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:59.124 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:59.158 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:59.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:59.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:59.262 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:59.296 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:59.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:59.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:59.400 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:59.434 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:59.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:59.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:59.537 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:59.565 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:59.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:59.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:59.668 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:59.682 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:59 smithi194 bash[92666]: cluster 2023-10-09T05:35:58.389815+0000 mgr.y (mgr.25277) 1441 : cluster 0 pgmap v1546: 89 pgs: 89 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:59.682 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:59 smithi194 bash[92666]: audit 2023-10-09T05:35:58.488432+0000 mon.a (mon.0) 1794 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:35:59.682 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:59 smithi194 bash[92666]: audit 2023-10-09T05:35:58.614468+0000 mon.a (mon.0) 1795 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:35:59.682 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:59 smithi194 bash[92666]: audit 2023-10-09T05:35:58.621913+0000 mon.a (mon.0) 1796 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:35:59.682 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:35:59 smithi194 bash[92666]: audit 2023-10-09T05:35:59.249232+0000 mon.a (mon.0) 1797 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:35:59.706 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:59.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:59.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:59.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:59 smithi133 bash[105589]: cluster 2023-10-09T05:35:58.389815+0000 mgr.y (mgr.25277) 1441 : cluster 0 pgmap v1546: 89 pgs: 89 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:59.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:59 smithi133 bash[105589]: audit 2023-10-09T05:35:58.488432+0000 mon.a (mon.0) 1794 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:35:59.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:59 smithi133 bash[105589]: audit 2023-10-09T05:35:58.614468+0000 mon.a (mon.0) 1795 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:35:59.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:59 smithi133 bash[105589]: audit 2023-10-09T05:35:58.621913+0000 mon.a (mon.0) 1796 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:35:59.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:35:59 smithi133 bash[105589]: audit 2023-10-09T05:35:59.249232+0000 mon.a (mon.0) 1797 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:35:59.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:59 smithi133 bash[102858]: cluster 2023-10-09T05:35:58.389815+0000 mgr.y (mgr.25277) 1441 : cluster 0 pgmap v1546: 89 pgs: 89 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:35:59.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:59 smithi133 bash[102858]: audit 2023-10-09T05:35:58.488432+0000 mon.a (mon.0) 1794 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:35:59.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:59 smithi133 bash[102858]: audit 2023-10-09T05:35:58.614468+0000 mon.a (mon.0) 1795 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:35:59.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:59 smithi133 bash[102858]: audit 2023-10-09T05:35:58.621913+0000 mon.a (mon.0) 1796 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:35:59.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:35:59 smithi133 bash[102858]: audit 2023-10-09T05:35:59.249232+0000 mon.a (mon.0) 1797 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:35:59.810 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:59.836 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:59.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:59.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:59.940 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:59.977 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:59.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:35:59.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:00.081 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:00.112 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:00.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:00.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:00.216 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:00.241 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:00.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:00.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:00.344 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:00.368 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:00.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:00.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:00.472 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:00.500 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:00.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:00.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:00.603 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:00.634 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:00.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:00.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:00.739 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:00.767 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:00.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:00.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:00.870 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:00.906 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:00.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:00.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:01.009 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:01.031 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:01.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:01.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:01.135 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:01.165 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:01.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:01.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:01.268 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:01.295 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:01.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:01.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:01.399 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:01.425 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:01.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:01.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:01.527 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:01.566 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:01.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:01.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:01.669 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:01.693 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:01.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:01.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:01.797 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:01.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:01 smithi133 bash[102858]: cluster 2023-10-09T05:36:00.390505+0000 mgr.y (mgr.25277) 1442 : cluster 0 pgmap v1547: 89 pgs: 89 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:36:01.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:01 smithi133 bash[105589]: cluster 2023-10-09T05:36:00.390505+0000 mgr.y (mgr.25277) 1442 : cluster 0 pgmap v1547: 89 pgs: 89 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:36:01.810 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:01 smithi194 bash[92666]: cluster 2023-10-09T05:36:00.390505+0000 mgr.y (mgr.25277) 1442 : cluster 0 pgmap v1547: 89 pgs: 89 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:36:01.828 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:01.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:01.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:01.933 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:01.969 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:01.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:01.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:02.073 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:02.102 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:02.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:02.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:02.205 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:02.237 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:02.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:02.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:02.341 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:02.371 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:02.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:02.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:02.475 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:02.509 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:02.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:02.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:02.612 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:02.641 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:02.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:02.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:02.673 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:36:02.744 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:02.768 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:02.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:02.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:02.871 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:02.891 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:02.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:02.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:02.994 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:03.018 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:03.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:03.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:03.122 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:03.144 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:03.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:03.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:03.248 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:03.275 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:03.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:03.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:03.378 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:03.401 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:03.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:03.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:03.505 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:03.527 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:03.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:03.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:03.547 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:03 smithi133 bash[102858]: cluster 2023-10-09T05:36:02.391261+0000 mgr.y (mgr.25277) 1443 : cluster 0 pgmap v1548: 89 pgs: 89 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:36:03.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:03 smithi133 bash[105589]: cluster 2023-10-09T05:36:02.391261+0000 mgr.y (mgr.25277) 1443 : cluster 0 pgmap v1548: 89 pgs: 89 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:36:03.630 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:03.665 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:03.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:03.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:03.769 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:03.781 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:03 smithi194 bash[92666]: cluster 2023-10-09T05:36:02.391261+0000 mgr.y (mgr.25277) 1443 : cluster 0 pgmap v1548: 89 pgs: 89 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:36:03.808 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:03.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:03.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:03.912 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:03.944 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:03.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:03.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:04.047 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:04.076 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:04.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:04.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:04.180 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:04.206 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:04.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:04.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:04.310 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:04.335 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:04.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:04.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:04.438 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:04.448 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:36:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:36:04] "GET /metrics HTTP/1.1" 200 36409 "" "Prometheus/2.43.0" 2023-10-09T05:36:04.466 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:04.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:04.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:04.569 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:04.595 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:04.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:04.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:04.649 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:36:04.678 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:36:04.698 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:04.705 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:36:04.729 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:04.730 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:36:04.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:04.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:04.751 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:36:04.780 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:36:04.808 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:36:04.832 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:36:04.832 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:04.833 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:36:04.834 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:36:04.834 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:36:04.834 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:36:04.834 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:36:04.834 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:36:04.834 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:36:04.834 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:36:04.859 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:04.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:04.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:04.962 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:04.989 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:04.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:04.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:05.093 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:05.120 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:05.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:05.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:05.186 INFO:teuthology.orchestra.run.smithi133.stderr:enabled application 'rados' on pool 'unique_pool_9' 2023-10-09T05:36:05.186 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is unset 2023-10-09T05:36:05.224 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:05.253 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:05.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:05.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:05.356 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:05.380 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:05.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:05.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:05.483 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:05.493 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:05 smithi133 bash[105589]: audit 2023-10-09T05:36:04.162815+0000 mon.a (mon.0) 1798 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:36:05.493 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:05 smithi133 bash[105589]: audit 2023-10-09T05:36:04.172287+0000 mon.a (mon.0) 1799 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:36:05.493 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:05 smithi133 bash[105589]: cluster 2023-10-09T05:36:04.391873+0000 mgr.y (mgr.25277) 1444 : cluster 0 pgmap v1549: 89 pgs: 89 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:36:05.493 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:05 smithi133 bash[105589]: audit 2023-10-09T05:36:04.621692+0000 mon.c (mon.2) 132 : audit 0 from='client.? 172.21.15.133:0/1266348460' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:36:05.494 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:05 smithi133 bash[105589]: audit 2023-10-09T05:36:05.011055+0000 mon.a (mon.0) 1800 : audit 1 from='client.? 172.21.15.133:0/3276721442' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:36:05.494 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:05 smithi133 bash[105589]: audit 2023-10-09T05:36:05.022979+0000 mon.c (mon.2) 133 : audit 1 from='client.? 172.21.15.133:0/3362517466' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:36:05.494 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:05 smithi133 bash[105589]: audit 2023-10-09T05:36:05.023601+0000 mon.a (mon.0) 1801 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:36:05.494 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:05 smithi133 bash[102858]: audit 2023-10-09T05:36:04.162815+0000 mon.a (mon.0) 1798 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:36:05.494 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:05 smithi133 bash[102858]: audit 2023-10-09T05:36:04.172287+0000 mon.a (mon.0) 1799 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:36:05.494 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:05 smithi133 bash[102858]: cluster 2023-10-09T05:36:04.391873+0000 mgr.y (mgr.25277) 1444 : cluster 0 pgmap v1549: 89 pgs: 89 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:36:05.494 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:05 smithi133 bash[102858]: audit 2023-10-09T05:36:04.621692+0000 mon.c (mon.2) 132 : audit 0 from='client.? 172.21.15.133:0/1266348460' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:36:05.495 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:05 smithi133 bash[102858]: audit 2023-10-09T05:36:05.011055+0000 mon.a (mon.0) 1800 : audit 1 from='client.? 172.21.15.133:0/3276721442' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:36:05.495 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:05 smithi133 bash[102858]: audit 2023-10-09T05:36:05.022979+0000 mon.c (mon.2) 133 : audit 1 from='client.? 172.21.15.133:0/3362517466' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:36:05.495 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:05 smithi133 bash[102858]: audit 2023-10-09T05:36:05.023601+0000 mon.a (mon.0) 1801 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:36:05.505 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:05 smithi194 bash[92666]: audit 2023-10-09T05:36:04.162815+0000 mon.a (mon.0) 1798 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:36:05.505 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:05 smithi194 bash[92666]: audit 2023-10-09T05:36:04.172287+0000 mon.a (mon.0) 1799 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:36:05.505 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:05 smithi194 bash[92666]: cluster 2023-10-09T05:36:04.391873+0000 mgr.y (mgr.25277) 1444 : cluster 0 pgmap v1549: 89 pgs: 89 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:36:05.505 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:05 smithi194 bash[92666]: audit 2023-10-09T05:36:04.621692+0000 mon.c (mon.2) 132 : audit 0 from='client.? 172.21.15.133:0/1266348460' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:36:05.505 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:05 smithi194 bash[92666]: audit 2023-10-09T05:36:05.011055+0000 mon.a (mon.0) 1800 : audit 1 from='client.? 172.21.15.133:0/3276721442' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:36:05.505 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:05 smithi194 bash[92666]: audit 2023-10-09T05:36:05.022979+0000 mon.c (mon.2) 133 : audit 1 from='client.? 172.21.15.133:0/3362517466' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:36:05.505 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:05 smithi194 bash[92666]: audit 2023-10-09T05:36:05.023601+0000 mon.a (mon.0) 1801 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:36:05.508 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:05.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:05.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:05.612 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:05.637 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:05.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:05.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:05.740 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:05.776 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:05.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:05.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:05.880 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:05.915 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:05.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:05.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:06.019 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:06.048 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:06.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:06.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:06.152 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:06.185 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:06.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:06.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:06.289 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:06.303 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:06 smithi194 bash[92666]: cluster 2023-10-09T05:36:05.170538+0000 mon.a (mon.0) 1802 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:36:06.303 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:06 smithi194 bash[92666]: cluster 2023-10-09T05:36:05.170575+0000 mon.a (mon.0) 1803 : cluster 1 Cluster is now healthy 2023-10-09T05:36:06.303 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:06 smithi194 bash[92666]: cluster 2023-10-09T05:36:05.177735+0000 osd.6 (osd.6) 273 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T05:36:06.303 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:06 smithi194 bash[92666]: cluster 2023-10-09T05:36:05.178000+0000 osd.5 (osd.5) 292 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T05:36:06.304 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:06 smithi194 bash[92666]: audit 2023-10-09T05:36:05.180321+0000 mon.a (mon.0) 1804 : audit 1 from='client.? 172.21.15.133:0/3276721442' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:36:06.304 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:06 smithi194 bash[92666]: audit 2023-10-09T05:36:05.180416+0000 mon.a (mon.0) 1805 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:36:06.304 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:06 smithi194 bash[92666]: cluster 2023-10-09T05:36:05.180489+0000 mon.a (mon.0) 1806 : cluster 0 osdmap e314: 8 total, 8 up, 7 in 2023-10-09T05:36:06.304 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:06 smithi194 bash[92666]: cluster 2023-10-09T05:36:05.409960+0000 osd.7 (osd.7) 276 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T05:36:06.304 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:06 smithi194 bash[92666]: audit 2023-10-09T05:36:05.790080+0000 mon.a (mon.0) 1807 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:36:06.304 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:06 smithi194 bash[92666]: audit 2023-10-09T05:36:05.795568+0000 mon.a (mon.0) 1808 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:36:06.323 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:06.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:06.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:06.427 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:06.441 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:06 smithi133 bash[105589]: cluster 2023-10-09T05:36:05.170538+0000 mon.a (mon.0) 1802 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:36:06.441 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:06 smithi133 bash[105589]: cluster 2023-10-09T05:36:05.170575+0000 mon.a (mon.0) 1803 : cluster 1 Cluster is now healthy 2023-10-09T05:36:06.441 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:06 smithi133 bash[105589]: cluster 2023-10-09T05:36:05.177735+0000 osd.6 (osd.6) 273 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T05:36:06.441 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:06 smithi133 bash[105589]: cluster 2023-10-09T05:36:05.178000+0000 osd.5 (osd.5) 292 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T05:36:06.441 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:06 smithi133 bash[105589]: audit 2023-10-09T05:36:05.180321+0000 mon.a (mon.0) 1804 : audit 1 from='client.? 172.21.15.133:0/3276721442' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:36:06.442 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:06 smithi133 bash[105589]: audit 2023-10-09T05:36:05.180416+0000 mon.a (mon.0) 1805 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:36:06.442 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:06 smithi133 bash[105589]: cluster 2023-10-09T05:36:05.180489+0000 mon.a (mon.0) 1806 : cluster 0 osdmap e314: 8 total, 8 up, 7 in 2023-10-09T05:36:06.442 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:06 smithi133 bash[105589]: cluster 2023-10-09T05:36:05.409960+0000 osd.7 (osd.7) 276 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T05:36:06.442 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:06 smithi133 bash[105589]: audit 2023-10-09T05:36:05.790080+0000 mon.a (mon.0) 1807 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:36:06.442 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:06 smithi133 bash[105589]: audit 2023-10-09T05:36:05.795568+0000 mon.a (mon.0) 1808 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:36:06.442 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:06 smithi133 bash[102858]: cluster 2023-10-09T05:36:05.170538+0000 mon.a (mon.0) 1802 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:36:06.442 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:06 smithi133 bash[102858]: cluster 2023-10-09T05:36:05.170575+0000 mon.a (mon.0) 1803 : cluster 1 Cluster is now healthy 2023-10-09T05:36:06.442 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:06 smithi133 bash[102858]: cluster 2023-10-09T05:36:05.177735+0000 osd.6 (osd.6) 273 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T05:36:06.443 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:06 smithi133 bash[102858]: cluster 2023-10-09T05:36:05.178000+0000 osd.5 (osd.5) 292 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T05:36:06.443 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:06 smithi133 bash[102858]: audit 2023-10-09T05:36:05.180321+0000 mon.a (mon.0) 1804 : audit 1 from='client.? 172.21.15.133:0/3276721442' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:36:06.443 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:06 smithi133 bash[102858]: audit 2023-10-09T05:36:05.180416+0000 mon.a (mon.0) 1805 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:36:06.443 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:06 smithi133 bash[102858]: cluster 2023-10-09T05:36:05.180489+0000 mon.a (mon.0) 1806 : cluster 0 osdmap e314: 8 total, 8 up, 7 in 2023-10-09T05:36:06.443 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:06 smithi133 bash[102858]: cluster 2023-10-09T05:36:05.409960+0000 osd.7 (osd.7) 276 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T05:36:06.443 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:06 smithi133 bash[102858]: audit 2023-10-09T05:36:05.790080+0000 mon.a (mon.0) 1807 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:36:06.443 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:06 smithi133 bash[102858]: audit 2023-10-09T05:36:05.795568+0000 mon.a (mon.0) 1808 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:36:06.458 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:06.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:06.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:06.563 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:06.595 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:06.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:06.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:06.698 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:06.732 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:06.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:06.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:06.751 INFO:teuthology.orchestra.run.smithi194.stdout:{ 2023-10-09T05:36:06.752 INFO:teuthology.orchestra.run.smithi194.stdout: "ops": [], 2023-10-09T05:36:06.752 INFO:teuthology.orchestra.run.smithi194.stdout: "num_ops": 0 2023-10-09T05:36:06.752 INFO:teuthology.orchestra.run.smithi194.stdout:} 2023-10-09T05:36:06.835 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:06.868 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:06.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:06.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:06.972 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:07.001 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:07.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:07.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:07.104 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:07.132 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:07.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:07.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:07.236 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:07.244 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:07 smithi194 bash[92666]: audit 2023-10-09T05:36:06.267511+0000 mon.a (mon.0) 1809 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:07.244 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:07 smithi194 bash[92666]: audit 2023-10-09T05:36:06.268546+0000 mon.a (mon.0) 1810 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:36:07.245 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:07 smithi194 bash[92666]: audit 2023-10-09T05:36:06.275445+0000 mon.a (mon.0) 1811 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:36:07.245 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:07 smithi194 bash[92666]: cluster 2023-10-09T05:36:06.392389+0000 mgr.y (mgr.25277) 1445 : cluster 0 pgmap v1551: 89 pgs: 89 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:36:07.262 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:07.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:07.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:07.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:07 smithi133 bash[102858]: audit 2023-10-09T05:36:06.267511+0000 mon.a (mon.0) 1809 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:07.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:07 smithi133 bash[102858]: audit 2023-10-09T05:36:06.268546+0000 mon.a (mon.0) 1810 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:36:07.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:07 smithi133 bash[102858]: audit 2023-10-09T05:36:06.275445+0000 mon.a (mon.0) 1811 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:36:07.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:07 smithi133 bash[102858]: cluster 2023-10-09T05:36:06.392389+0000 mgr.y (mgr.25277) 1445 : cluster 0 pgmap v1551: 89 pgs: 89 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:36:07.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:07 smithi133 bash[105589]: audit 2023-10-09T05:36:06.267511+0000 mon.a (mon.0) 1809 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:07.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:07 smithi133 bash[105589]: audit 2023-10-09T05:36:06.268546+0000 mon.a (mon.0) 1810 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:36:07.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:07 smithi133 bash[105589]: audit 2023-10-09T05:36:06.275445+0000 mon.a (mon.0) 1811 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:36:07.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:07 smithi133 bash[105589]: cluster 2023-10-09T05:36:06.392389+0000 mgr.y (mgr.25277) 1445 : cluster 0 pgmap v1551: 89 pgs: 89 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:36:07.366 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:07.396 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:07.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:07.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:07.499 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:07.531 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:07.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:07.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:07.636 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:07.659 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:07.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:07.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:07.674 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T05:36:07.763 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:07.797 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:07.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:07.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:07.900 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:07.928 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:07.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:07.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:08.032 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:08.060 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:08.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:08.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:08.162 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:08.198 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:08.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:08.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:08.303 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:08.334 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:08.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:08.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:08.438 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:08.471 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:08.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:08.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:08.486 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:36:08.575 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:08.607 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:08.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:08.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:08.710 DEBUG:teuthology.orchestra.run.smithi133:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_9 bench 90 write' 2023-10-09T05:36:08.710 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:08.711 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_9 2023-10-09T05:36:08.711 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:36:08.713 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T05:36:08.713 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T05:36:08.743 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:08.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:08.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:08.758 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:hints = 1 2023-10-09T05:36:08.759 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T05:36:08.759 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Object prefix: benchmark_data_smithi133_267384 2023-10-09T05:36:08.760 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:36:08.760 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 0 0 0 0 0 0 - 0 2023-10-09T05:36:08.849 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:08.875 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:08.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:08.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:08.978 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:09.016 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:09.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:09.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:09.120 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:09.144 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:09.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:09.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:09.249 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:09.276 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:09.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:09.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:09.380 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:09.409 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:09.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:09.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:09.514 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:09.524 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:09 smithi194 bash[92666]: cluster 2023-10-09T05:36:08.393184+0000 mgr.y (mgr.25277) 1446 : cluster 0 pgmap v1552: 89 pgs: 89 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:36:09.541 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:09.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:09.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:09.645 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:09.655 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:09 smithi133 bash[105589]: cluster 2023-10-09T05:36:08.393184+0000 mgr.y (mgr.25277) 1446 : cluster 0 pgmap v1552: 89 pgs: 89 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:36:09.656 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:09 smithi133 bash[102858]: cluster 2023-10-09T05:36:08.393184+0000 mgr.y (mgr.25277) 1446 : cluster 0 pgmap v1552: 89 pgs: 89 active+clean; 582 KiB data, 7.0 GiB used, 619 GiB / 626 GiB avail 2023-10-09T05:36:09.675 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:09.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:09.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:09.698 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:36:09.759 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 1 16 2546 2530 158.114 158.125 0.00316923 0.00630593 2023-10-09T05:36:09.779 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:09.810 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:09.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:09.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:09.915 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:09.941 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:09.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:09.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:10.046 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:10.086 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:10.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:10.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:10.191 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:10.218 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:10.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:10.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:10.321 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:10.351 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:10.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:10.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:10.454 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:10.482 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:10.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:10.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:10.586 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:10.614 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:10.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:10.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:10.717 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:10.744 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:10.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:10.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:10.759 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 2 16 5008 4992 155.987 153.875 0.00261571 0.00639037 2023-10-09T05:36:10.848 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:10.880 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:10.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:10.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:11.009 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:11.038 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:11.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:11.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:11.143 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:11.171 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:11.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:11.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:11.277 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:11.305 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:11.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:11.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:11.409 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:11.439 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:11.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:11.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:11.543 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:11.576 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:11.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:11.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:11.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:11 smithi194 bash[92666]: cluster 2023-10-09T05:36:10.393936+0000 mgr.y (mgr.25277) 1447 : cluster 0 pgmap v1553: 89 pgs: 89 active+clean; 144 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 14 MiB/s wr, 229 op/s 2023-10-09T05:36:11.680 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:11.691 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:11 smithi133 bash[105589]: cluster 2023-10-09T05:36:10.393936+0000 mgr.y (mgr.25277) 1447 : cluster 0 pgmap v1553: 89 pgs: 89 active+clean; 144 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 14 MiB/s wr, 229 op/s 2023-10-09T05:36:11.691 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:11 smithi133 bash[102858]: cluster 2023-10-09T05:36:10.393936+0000 mgr.y (mgr.25277) 1447 : cluster 0 pgmap v1553: 89 pgs: 89 active+clean; 144 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 14 MiB/s wr, 229 op/s 2023-10-09T05:36:11.716 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:11.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:11.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:11.759 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 3 16 7304 7288 151.82 143.5 0.00693775 0.00657032 2023-10-09T05:36:11.819 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:11.846 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:11.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:11.849 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:11.949 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:11.980 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:11.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:11.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:12.085 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:12.114 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:12.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:12.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:12.220 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:12.250 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:12.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:12.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:12.354 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:12.388 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:12.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:12.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:12.492 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:12.497 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:36:12.521 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:12.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:12.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:12.625 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:12.662 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:12.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:12.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:12.760 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 4 16 9462 9446 147.575 134.875 0.00554579 0.00677018 2023-10-09T05:36:12.767 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:12.795 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:12.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:12.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:12.899 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:12.927 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:12.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:12.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:13.030 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:13.060 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:13.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:13.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:13.164 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:13.194 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:13.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:13.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:13.282 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:13.283 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:13.301 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:13.337 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:13.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:13.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:13.443 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:13.474 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:13.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:13.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:13.578 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:13.587 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:13 smithi133 bash[102858]: cluster 2023-10-09T05:36:12.394545+0000 mgr.y (mgr.25277) 1448 : cluster 0 pgmap v1554: 89 pgs: 89 active+clean; 218 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 22 MiB/s wr, 347 op/s 2023-10-09T05:36:13.587 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:13 smithi133 bash[105589]: cluster 2023-10-09T05:36:12.394545+0000 mgr.y (mgr.25277) 1448 : cluster 0 pgmap v1554: 89 pgs: 89 active+clean; 218 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 22 MiB/s wr, 347 op/s 2023-10-09T05:36:13.607 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:13.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:13.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:13.711 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:13.719 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:13 smithi194 bash[92666]: cluster 2023-10-09T05:36:12.394545+0000 mgr.y (mgr.25277) 1448 : cluster 0 pgmap v1554: 89 pgs: 89 active+clean; 218 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 22 MiB/s wr, 347 op/s 2023-10-09T05:36:13.736 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:13.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:13.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:13.759 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 5 16 11459 11443 143.02 124.812 0.00987965 0.00698153 2023-10-09T05:36:13.840 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:13.867 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:13.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:13.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:13.973 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:13.999 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:14.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:14.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:14.103 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:14.134 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:14.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:14.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:14.239 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:14.266 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:14.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:14.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:14.304 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:14.369 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:14.378 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:36:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:36:14] "GET /metrics HTTP/1.1" 200 36461 "" "Prometheus/2.43.0" 2023-10-09T05:36:14.396 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:14.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:14.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:14.500 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:14.526 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:14.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:14.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:14.629 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:14.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:14 smithi194 bash[92666]: audit 2023-10-09T05:36:13.495912+0000 mon.a (mon.0) 1812 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:36:14.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:14 smithi194 bash[92666]: audit 2023-10-09T05:36:13.497308+0000 mon.a (mon.0) 1813 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:36:14.658 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:14.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:14.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:14.760 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 6 16 13437 13421 139.786 123.625 0.00656404 0.00714624 2023-10-09T05:36:14.761 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:14.771 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:14 smithi133 bash[102858]: audit 2023-10-09T05:36:13.495912+0000 mon.a (mon.0) 1812 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:36:14.771 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:14 smithi133 bash[102858]: audit 2023-10-09T05:36:13.497308+0000 mon.a (mon.0) 1813 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:36:14.771 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:14 smithi133 bash[105589]: audit 2023-10-09T05:36:13.495912+0000 mon.a (mon.0) 1812 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:36:14.771 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:14 smithi133 bash[105589]: audit 2023-10-09T05:36:13.497308+0000 mon.a (mon.0) 1813 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:36:14.790 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:14.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:14.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:14.894 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:14.920 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:14.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:14.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:15.023 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:15.047 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:15.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:15.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:15.152 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:15.177 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:15.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:15.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:15.281 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:15.304 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:15.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:15.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:15.407 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:15.434 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:15.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:15.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:15.538 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:15.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:15 smithi133 bash[102858]: cluster 2023-10-09T05:36:14.394932+0000 mgr.y (mgr.25277) 1449 : cluster 0 pgmap v1555: 89 pgs: 89 active+clean; 218 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 22 MiB/s wr, 347 op/s 2023-10-09T05:36:15.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:15 smithi133 bash[102858]: audit 2023-10-09T05:36:15.391734+0000 mon.a (mon.0) 1814 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 5]}]: dispatch 2023-10-09T05:36:15.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:15 smithi133 bash[102858]: audit 2023-10-09T05:36:15.392627+0000 mon.a (mon.0) 1815 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [3, 5]}]: dispatch 2023-10-09T05:36:15.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:15 smithi133 bash[105589]: cluster 2023-10-09T05:36:14.394932+0000 mgr.y (mgr.25277) 1449 : cluster 0 pgmap v1555: 89 pgs: 89 active+clean; 218 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 22 MiB/s wr, 347 op/s 2023-10-09T05:36:15.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:15 smithi133 bash[105589]: audit 2023-10-09T05:36:15.391734+0000 mon.a (mon.0) 1814 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 5]}]: dispatch 2023-10-09T05:36:15.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:15 smithi133 bash[105589]: audit 2023-10-09T05:36:15.392627+0000 mon.a (mon.0) 1815 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [3, 5]}]: dispatch 2023-10-09T05:36:15.567 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:15.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:15.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:15.671 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:15.679 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:15 smithi194 bash[92666]: cluster 2023-10-09T05:36:14.394932+0000 mgr.y (mgr.25277) 1449 : cluster 0 pgmap v1555: 89 pgs: 89 active+clean; 218 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 22 MiB/s wr, 347 op/s 2023-10-09T05:36:15.679 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:15 smithi194 bash[92666]: audit 2023-10-09T05:36:15.391734+0000 mon.a (mon.0) 1814 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 5]}]: dispatch 2023-10-09T05:36:15.679 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:15 smithi194 bash[92666]: audit 2023-10-09T05:36:15.392627+0000 mon.a (mon.0) 1815 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [3, 5]}]: dispatch 2023-10-09T05:36:15.702 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:15.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:15.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:15.760 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 7 16 15451 15435 137.797 125.875 0.00885943 0.00724891 2023-10-09T05:36:15.808 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:15.831 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:15.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:15.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:15.935 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:15.960 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:15.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:15.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:16.063 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:16.091 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:16.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:16.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:16.197 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:16.226 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:16.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:16.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:16.329 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:16.355 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:16.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:16.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:16.459 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:16.489 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:16.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:16.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:16.593 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:16.621 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:16.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:16.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:16.725 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:16.755 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:16.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:16.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:16.760 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 8 16 17402 17386 135.813 121.938 0.0101915 0.00735663 2023-10-09T05:36:16.859 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:16.898 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:16.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:16.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:17.004 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:17.036 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:17.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:17.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:17.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:16 smithi194 bash[92666]: audit 2023-10-09T05:36:15.872860+0000 mon.a (mon.0) 1816 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 5]}]': finished 2023-10-09T05:36:17.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:16 smithi194 bash[92666]: audit 2023-10-09T05:36:15.872953+0000 mon.a (mon.0) 1817 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [3, 5]}]': finished 2023-10-09T05:36:17.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:16 smithi194 bash[92666]: cluster 2023-10-09T05:36:15.873118+0000 mon.a (mon.0) 1818 : cluster 0 osdmap e315: 8 total, 8 up, 7 in 2023-10-09T05:36:17.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:16 smithi194 bash[92666]: cluster 2023-10-09T05:36:15.874057+0000 osd.5 (osd.5) 293 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T05:36:17.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:16 smithi194 bash[92666]: cluster 2023-10-09T05:36:15.877767+0000 osd.6 (osd.6) 274 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T05:36:17.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:16 smithi194 bash[92666]: cluster 2023-10-09T05:36:15.877775+0000 osd.5 (osd.5) 294 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T05:36:17.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:16 smithi194 bash[92666]: cluster 2023-10-09T05:36:15.879034+0000 osd.6 (osd.6) 275 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T05:36:17.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:16 smithi194 bash[92666]: cluster 2023-10-09T05:36:16.395357+0000 mgr.y (mgr.25277) 1450 : cluster 0 pgmap v1557: 89 pgs: 89 active+clean; 808 MiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T05:36:17.140 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:17.151 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:16 smithi133 bash[105589]: audit 2023-10-09T05:36:15.872860+0000 mon.a (mon.0) 1816 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 5]}]': finished 2023-10-09T05:36:17.152 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:16 smithi133 bash[105589]: audit 2023-10-09T05:36:15.872953+0000 mon.a (mon.0) 1817 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [3, 5]}]': finished 2023-10-09T05:36:17.152 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:16 smithi133 bash[105589]: cluster 2023-10-09T05:36:15.873118+0000 mon.a (mon.0) 1818 : cluster 0 osdmap e315: 8 total, 8 up, 7 in 2023-10-09T05:36:17.152 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:16 smithi133 bash[105589]: cluster 2023-10-09T05:36:15.874057+0000 osd.5 (osd.5) 293 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T05:36:17.152 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:16 smithi133 bash[105589]: cluster 2023-10-09T05:36:15.877767+0000 osd.6 (osd.6) 274 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T05:36:17.152 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:16 smithi133 bash[105589]: cluster 2023-10-09T05:36:15.877775+0000 osd.5 (osd.5) 294 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T05:36:17.152 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:16 smithi133 bash[105589]: cluster 2023-10-09T05:36:15.879034+0000 osd.6 (osd.6) 275 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T05:36:17.152 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:16 smithi133 bash[105589]: cluster 2023-10-09T05:36:16.395357+0000 mgr.y (mgr.25277) 1450 : cluster 0 pgmap v1557: 89 pgs: 89 active+clean; 808 MiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T05:36:17.152 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:16 smithi133 bash[102858]: audit 2023-10-09T05:36:15.872860+0000 mon.a (mon.0) 1816 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 5]}]': finished 2023-10-09T05:36:17.153 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:16 smithi133 bash[102858]: audit 2023-10-09T05:36:15.872953+0000 mon.a (mon.0) 1817 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [3, 5]}]': finished 2023-10-09T05:36:17.153 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:16 smithi133 bash[102858]: cluster 2023-10-09T05:36:15.873118+0000 mon.a (mon.0) 1818 : cluster 0 osdmap e315: 8 total, 8 up, 7 in 2023-10-09T05:36:17.153 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:16 smithi133 bash[102858]: cluster 2023-10-09T05:36:15.874057+0000 osd.5 (osd.5) 293 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T05:36:17.153 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:16 smithi133 bash[102858]: cluster 2023-10-09T05:36:15.877767+0000 osd.6 (osd.6) 274 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T05:36:17.153 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:16 smithi133 bash[102858]: cluster 2023-10-09T05:36:15.877775+0000 osd.5 (osd.5) 294 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T05:36:17.153 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:16 smithi133 bash[102858]: cluster 2023-10-09T05:36:15.879034+0000 osd.6 (osd.6) 275 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T05:36:17.153 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:16 smithi133 bash[102858]: cluster 2023-10-09T05:36:16.395357+0000 mgr.y (mgr.25277) 1450 : cluster 0 pgmap v1557: 89 pgs: 89 active+clean; 808 MiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T05:36:17.172 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:17.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:17.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:17.277 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:17.308 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:17.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:17.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:17.412 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:17.443 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:17.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:17.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:17.546 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:17.576 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:17.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:17.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:17.680 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:17.713 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:17.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:17.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:17.760 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 9 16 18946 18930 131.444 96.5 0.00912117 0.00759997 2023-10-09T05:36:17.818 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:17.844 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:17.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:17.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:17.947 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:17.957 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:17 smithi133 bash[102858]: cluster 2023-10-09T05:36:16.883506+0000 osd.6 (osd.6) 276 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T05:36:17.957 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:17 smithi133 bash[102858]: cluster 2023-10-09T05:36:16.889453+0000 osd.6 (osd.6) 277 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T05:36:17.957 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:17 smithi133 bash[102858]: cluster 2023-10-09T05:36:16.890643+0000 osd.5 (osd.5) 295 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T05:36:17.958 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:17 smithi133 bash[102858]: cluster 2023-10-09T05:36:16.891996+0000 mon.a (mon.0) 1819 : cluster 0 osdmap e316: 8 total, 8 up, 7 in 2023-10-09T05:36:17.958 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:17 smithi133 bash[102858]: cluster 2023-10-09T05:36:16.893140+0000 osd.5 (osd.5) 296 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T05:36:17.958 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:17 smithi133 bash[105589]: cluster 2023-10-09T05:36:16.883506+0000 osd.6 (osd.6) 276 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T05:36:17.958 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:17 smithi133 bash[105589]: cluster 2023-10-09T05:36:16.889453+0000 osd.6 (osd.6) 277 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T05:36:17.959 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:17 smithi133 bash[105589]: cluster 2023-10-09T05:36:16.890643+0000 osd.5 (osd.5) 295 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T05:36:17.959 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:17 smithi133 bash[105589]: cluster 2023-10-09T05:36:16.891996+0000 mon.a (mon.0) 1819 : cluster 0 osdmap e316: 8 total, 8 up, 7 in 2023-10-09T05:36:17.959 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:17 smithi133 bash[105589]: cluster 2023-10-09T05:36:16.893140+0000 osd.5 (osd.5) 296 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T05:36:17.977 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:17.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:17.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:18.083 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:18.093 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:17 smithi194 bash[92666]: cluster 2023-10-09T05:36:16.883506+0000 osd.6 (osd.6) 276 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T05:36:18.093 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:17 smithi194 bash[92666]: cluster 2023-10-09T05:36:16.889453+0000 osd.6 (osd.6) 277 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T05:36:18.093 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:17 smithi194 bash[92666]: cluster 2023-10-09T05:36:16.890643+0000 osd.5 (osd.5) 295 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T05:36:18.093 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:17 smithi194 bash[92666]: cluster 2023-10-09T05:36:16.891996+0000 mon.a (mon.0) 1819 : cluster 0 osdmap e316: 8 total, 8 up, 7 in 2023-10-09T05:36:18.093 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:17 smithi194 bash[92666]: cluster 2023-10-09T05:36:16.893140+0000 osd.5 (osd.5) 296 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T05:36:18.110 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:18.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:18.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:18.213 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:18.250 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:18.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:18.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:18.353 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:18.383 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:18.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:18.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:18.488 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:18.513 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:18.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:18.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:18.617 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:18.644 INFO:teuthology.orchestra.run.smithi194.stdout:{ 2023-10-09T05:36:18.644 INFO:teuthology.orchestra.run.smithi194.stdout: "ops": [], 2023-10-09T05:36:18.644 INFO:teuthology.orchestra.run.smithi194.stdout: "complaint_time": 30, 2023-10-09T05:36:18.644 INFO:teuthology.orchestra.run.smithi194.stdout: "num_blocked_ops": 0 2023-10-09T05:36:18.644 INFO:teuthology.orchestra.run.smithi194.stdout:} 2023-10-09T05:36:18.649 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:18.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:18.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:18.752 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:18.760 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 10 16 20617 20601 128.742 104.438 0.00391629 0.00775647 2023-10-09T05:36:18.778 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:18.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:18.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:18.883 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:18.907 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:18.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:18.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:19.011 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:19.038 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:19.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:19.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:19.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:18 smithi194 bash[92666]: cluster 2023-10-09T05:36:17.496657+0000 osd.4 (osd.4) 286 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T05:36:19.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:18 smithi194 bash[92666]: cluster 2023-10-09T05:36:18.395833+0000 mgr.y (mgr.25277) 1451 : cluster 0 pgmap v1559: 89 pgs: 1 peering, 88 active+clean; 888 MiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-10-09T05:36:19.144 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:19.172 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:19.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:19.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:19.275 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:19.287 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:18 smithi133 bash[105589]: cluster 2023-10-09T05:36:17.496657+0000 osd.4 (osd.4) 286 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T05:36:19.287 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:18 smithi133 bash[105589]: cluster 2023-10-09T05:36:18.395833+0000 mgr.y (mgr.25277) 1451 : cluster 0 pgmap v1559: 89 pgs: 1 peering, 88 active+clean; 888 MiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-10-09T05:36:19.287 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:18 smithi133 bash[102858]: cluster 2023-10-09T05:36:17.496657+0000 osd.4 (osd.4) 286 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T05:36:19.287 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:18 smithi133 bash[102858]: cluster 2023-10-09T05:36:18.395833+0000 mgr.y (mgr.25277) 1451 : cluster 0 pgmap v1559: 89 pgs: 1 peering, 88 active+clean; 888 MiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-10-09T05:36:19.303 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:19.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:19.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:19.405 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:19.428 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:19.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:19.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:19.531 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:19.552 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:19.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:19.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:19.655 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:19.675 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:19.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:19.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:19.760 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 11 16 21610 21594 122.679 62.0625 0.00216558 0.00810909 2023-10-09T05:36:19.778 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:19.808 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:19.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:19.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:19.911 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:19.938 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:19.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:19.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:19.976 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T05:36:20.041 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:20.069 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:20.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:20.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:20.173 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:20.198 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:20.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:20.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:20.302 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:20.332 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:20.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:20.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:20.435 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:20.471 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:20.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:20.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:20.576 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:20.608 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:20.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:20.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:20.711 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:20.743 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:20.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:20.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:20.760 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 12 16 22994 22978 119.664 86.5 0.0069888 0.00834826 2023-10-09T05:36:20.847 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:20.874 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:20.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:20.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:20.979 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:21.002 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:21.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:21.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:21.106 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:21.132 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:21.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:21.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:21.238 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:21.266 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:21.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:21.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:21.371 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:21.397 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:21.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:21.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:21.501 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:21.511 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:21 smithi133 bash[102858]: cluster 2023-10-09T05:36:20.396891+0000 mgr.y (mgr.25277) 1452 : cluster 0 pgmap v1560: 89 pgs: 1 peering, 88 active+clean; 1.3 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 135 MiB/s wr, 2.17k op/s 2023-10-09T05:36:21.511 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:21 smithi133 bash[105589]: cluster 2023-10-09T05:36:20.396891+0000 mgr.y (mgr.25277) 1452 : cluster 0 pgmap v1560: 89 pgs: 1 peering, 88 active+clean; 1.3 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 135 MiB/s wr, 2.17k op/s 2023-10-09T05:36:21.528 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:21.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:21.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:21.630 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:21.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:21 smithi194 bash[92666]: cluster 2023-10-09T05:36:20.396891+0000 mgr.y (mgr.25277) 1452 : cluster 0 pgmap v1560: 89 pgs: 1 peering, 88 active+clean; 1.3 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 135 MiB/s wr, 2.17k op/s 2023-10-09T05:36:21.664 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:21.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:21.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:21.761 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 13 16 24512 24496 117.756 94.875 0.00586327 0.00848525 2023-10-09T05:36:21.768 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:21.795 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:21.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:21.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:21.901 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:21.927 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:21.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:21.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:22.030 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:22.071 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:22.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:22.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:22.175 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:22.207 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:22.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:22.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:22.312 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:22.331 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:22.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:22.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:22.434 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:22.462 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:22.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:22.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:22.566 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:22.595 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:22.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:22.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:22.699 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:22.733 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:22.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:22.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:22.760 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 14 16 25824 25808 115.201 82 0.00849015 0.00867214 2023-10-09T05:36:22.841 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:22.870 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:22.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:22.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:22.973 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:23.006 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:23.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:23.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:23.113 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:23.142 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:23.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:23.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:23.247 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:23.283 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:23.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:23.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:23.392 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:23.420 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:23.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:23.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:23.523 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:23.532 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:23 smithi133 bash[102858]: cluster 2023-10-09T05:36:22.397891+0000 mgr.y (mgr.25277) 1453 : cluster 0 pgmap v1561: 89 pgs: 89 active+clean; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 146 MiB/s wr, 2.34k op/s 2023-10-09T05:36:23.532 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:23 smithi133 bash[105589]: cluster 2023-10-09T05:36:22.397891+0000 mgr.y (mgr.25277) 1453 : cluster 0 pgmap v1561: 89 pgs: 89 active+clean; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 146 MiB/s wr, 2.34k op/s 2023-10-09T05:36:23.550 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:23.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:23.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:23.655 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:23.666 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:23 smithi194 bash[92666]: cluster 2023-10-09T05:36:22.397891+0000 mgr.y (mgr.25277) 1453 : cluster 0 pgmap v1561: 89 pgs: 89 active+clean; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 146 MiB/s wr, 2.34k op/s 2023-10-09T05:36:23.684 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:23.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:23.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:23.761 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 15 16 27467 27451 114.367 102.688 0.0200826 0.00873652 2023-10-09T05:36:23.789 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:23.815 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:23.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:23.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:23.920 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:23.951 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:23.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:23.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:24.056 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:24.091 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:24.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:24.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:24.195 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:24.220 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:24.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:24.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:24.323 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:24.330 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:36:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:36:24] "GET /metrics HTTP/1.1" 200 36461 "" "Prometheus/2.43.0" 2023-10-09T05:36:24.342 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:24.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:24.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:24.446 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:24.476 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:24.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:24.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:24.579 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:24.610 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:24.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:24.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:24.717 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:24.744 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:24.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:24.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:24.761 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 16 16 28566 28550 111.511 68.6875 0.00587795 0.00896009 2023-10-09T05:36:24.840 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:36:24.850 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:24.874 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:24 smithi133 bash[102858]: cluster 2023-10-09T05:36:24.398223+0000 mgr.y (mgr.25277) 1454 : cluster 0 pgmap v1562: 89 pgs: 89 active+clean; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T05:36:24.885 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:24.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:24.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:24.988 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:25.015 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:25.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:25.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:25.118 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:25.130 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:24 smithi194 bash[92666]: cluster 2023-10-09T05:36:24.398223+0000 mgr.y (mgr.25277) 1454 : cluster 0 pgmap v1562: 89 pgs: 89 active+clean; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T05:36:25.152 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:25.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:25.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:25.259 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:25.284 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:25.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:25.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:25.297 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:24 smithi133 bash[105589]: cluster 2023-10-09T05:36:24.398223+0000 mgr.y (mgr.25277) 1454 : cluster 0 pgmap v1562: 89 pgs: 89 active+clean; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T05:36:25.387 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:25.413 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:25.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:25.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:25.515 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:25.540 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:25.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:25.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:25.643 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:25.663 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:25.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:25.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:25.761 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 17 16 29879 29863 109.779 82.0625 0.00694017 0.00910372 2023-10-09T05:36:25.766 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:25.796 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:25.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:25.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:25.903 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:25.928 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:25.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:25.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:26.032 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:26.057 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:26.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:26.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:26.160 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:26.186 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:26.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:26.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:26.289 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:26.314 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:26.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:26.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:26.416 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:26.442 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:26.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:26.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:26.546 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:26.570 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:26.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:26.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:26.678 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:26.704 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:26.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:26.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:26.761 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 18 16 31344 31328 108.766 91.5625 0.00865084 0.00918759 2023-10-09T05:36:26.808 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:26.834 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:26.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:26.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:26.937 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:26.955 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:26.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:26.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:27.058 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:27.089 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:27.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:27.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:27.193 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:27.221 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:27.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:27.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:27.327 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:27.362 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:27.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:27.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:27.457 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:27 smithi133 bash[102858]: cluster 2023-10-09T05:36:26.398924+0000 mgr.y (mgr.25277) 1455 : cluster 0 pgmap v1563: 89 pgs: 89 active+clean; 1.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-09T05:36:27.468 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:27.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:27.502 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:27.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:27.603 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:27.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:27 smithi194 bash[92666]: cluster 2023-10-09T05:36:26.398924+0000 mgr.y (mgr.25277) 1455 : cluster 0 pgmap v1563: 89 pgs: 89 active+clean; 1.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-09T05:36:27.638 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:27.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:27.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:27.741 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:27.761 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 19 16 32576 32560 107.094 77 0.00696367 0.00933111 2023-10-09T05:36:27.767 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:27.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:27.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:27.797 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:27 smithi133 bash[105589]: cluster 2023-10-09T05:36:26.398924+0000 mgr.y (mgr.25277) 1455 : cluster 0 pgmap v1563: 89 pgs: 89 active+clean; 1.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-09T05:36:27.870 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:27.892 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:27.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:27.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:27.996 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:28.025 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:28.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:28.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:28.129 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:28.156 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:28.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:28.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:28.260 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:28.290 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:28.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:28.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:28.396 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:28.427 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:28.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:28.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:28.530 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:28.555 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:28.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:28.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:28.662 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:28.685 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:28.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:28.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:28.761 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T05:36:28.761826+0000 min lat: 0.00135224 max lat: 0.333322 avg lat: 0.00942617 2023-10-09T05:36:28.762 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:36:28.762 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 20 16 33946 33930 106.02 85.625 0.00987199 0.00942617 2023-10-09T05:36:28.788 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:28.816 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:28.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:28.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:28.924 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:28.955 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:28.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:28.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:29.060 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:29.088 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:29.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:29.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:29.191 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:29.226 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:29.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:29.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:29.331 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:29.357 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:29.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:29.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:29.460 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:29.492 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:29.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:29.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:29.597 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:29.627 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:29.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:29.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:29 smithi194 bash[92666]: cluster 2023-10-09T05:36:28.400632+0000 mgr.y (mgr.25277) 1456 : cluster 0 pgmap v1564: 89 pgs: 89 active+clean; 1.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T05:36:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:29 smithi194 bash[92666]: audit 2023-10-09T05:36:28.488899+0000 mon.a (mon.0) 1820 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:36:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:29 smithi194 bash[92666]: audit 2023-10-09T05:36:29.269881+0000 mon.a (mon.0) 1821 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:36:29.733 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:29.761 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 21 16 35069 35053 104.313 70.1875 0.00900838 0.00957654 2023-10-09T05:36:29.762 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:29.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:29.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:29.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:29 smithi133 bash[102858]: cluster 2023-10-09T05:36:28.400632+0000 mgr.y (mgr.25277) 1456 : cluster 0 pgmap v1564: 89 pgs: 89 active+clean; 1.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T05:36:29.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:29 smithi133 bash[102858]: audit 2023-10-09T05:36:28.488899+0000 mon.a (mon.0) 1820 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:36:29.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:29 smithi133 bash[102858]: audit 2023-10-09T05:36:29.269881+0000 mon.a (mon.0) 1821 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:36:29.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:29 smithi133 bash[105589]: cluster 2023-10-09T05:36:28.400632+0000 mgr.y (mgr.25277) 1456 : cluster 0 pgmap v1564: 89 pgs: 89 active+clean; 1.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T05:36:29.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:29 smithi133 bash[105589]: audit 2023-10-09T05:36:28.488899+0000 mon.a (mon.0) 1820 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:36:29.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:29 smithi133 bash[105589]: audit 2023-10-09T05:36:29.269881+0000 mon.a (mon.0) 1821 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:36:29.866 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:29.900 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:29.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:29.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:29.964 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:36:29.987 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:36:30.000 INFO:teuthology.orchestra.run.smithi133.stderr:dumped all 2023-10-09T05:36:30.005 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:30.035 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:30.038 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:36:30.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:30.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:30.085 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:36:30.140 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:30.148 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:36:30.164 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:30.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:30.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:30.187 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:36:30.194 INFO:teuthology.orchestra.run.smithi194.stdout:{ 2023-10-09T05:36:30.194 INFO:teuthology.orchestra.run.smithi194.stdout: "size": 20, 2023-10-09T05:36:30.194 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 600, 2023-10-09T05:36:30.194 INFO:teuthology.orchestra.run.smithi194.stdout: "ops": [ 2023-10-09T05:36:30.194 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.194 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35775.0:21528 14.9 14:9e4f0b09:::benchmark_data_smithi133_267384_object21527:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-10-09T05:36:30.194 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:36:19.512272+0000", 2023-10-09T05:36:30.194 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 10.677679900999999, 2023-10-09T05:36:30.194 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.176870578, 2023-10-09T05:36:30.195 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:36:30.195 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:30.195 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:36:30.195 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35775", 2023-10-09T05:36:30.195 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:36:30.195 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 21528 2023-10-09T05:36:30.195 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.195 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:36:30.196 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.196 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:36:30.196 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.512272+0000", 2023-10-09T05:36:30.198 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.198 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.198 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.199 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:36:30.199 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.512272+0000", 2023-10-09T05:36:30.199 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.199 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.199 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.199 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:36:30.200 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.512270+0000", 2023-10-09T05:36:30.200 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999981 2023-10-09T05:36:30.200 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.200 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.200 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:36:30.200 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.512340+0000", 2023-10-09T05:36:30.200 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.9664000000000003e-05 2023-10-09T05:36:30.200 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.200 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.200 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:36:30.200 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.512352+0000", 2023-10-09T05:36:30.201 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.2736e-05 2023-10-09T05:36:30.201 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.201 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.201 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:36:30.201 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.512360+0000", 2023-10-09T05:36:30.201 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.6459999999999993e-06 2023-10-09T05:36:30.201 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.201 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.201 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:36:30.201 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.512430+0000", 2023-10-09T05:36:30.201 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.9832999999999994e-05 2023-10-09T05:36:30.202 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.202 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.202 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:36:30.202 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.512562+0000", 2023-10-09T05:36:30.202 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00013266600000000001 2023-10-09T05:36:30.202 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.202 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.202 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:36:30.202 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.512720+0000", 2023-10-09T05:36:30.202 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000157703 2023-10-09T05:36:30.202 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.202 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.203 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:36:30.203 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.513739+0000", 2023-10-09T05:36:30.203 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00101844 2023-10-09T05:36:30.203 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.203 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.203 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.203 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.517355+0000", 2023-10-09T05:36:30.203 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0036167579999999999 2023-10-09T05:36:30.203 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.203 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.203 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.203 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.688951+0000", 2023-10-09T05:36:30.204 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.17159576300000001 2023-10-09T05:36:30.204 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.204 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.204 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:36:30.204 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.689041+0000", 2023-10-09T05:36:30.204 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.9567999999999994e-05 2023-10-09T05:36:30.204 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.204 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.204 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:36:30.204 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.689142+0000", 2023-10-09T05:36:30.204 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00010147800000000001 2023-10-09T05:36:30.204 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.205 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:36:30.205 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.205 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.205 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.205 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35775.0:21535 14.a 14:55c3d8db:::benchmark_data_smithi133_267384_object21534:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-10-09T05:36:30.205 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:36:19.513048+0000", 2023-10-09T05:36:30.205 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 10.676903593, 2023-10-09T05:36:30.205 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.17480685400000001, 2023-10-09T05:36:30.205 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:36:30.205 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:30.205 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:36:30.206 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35775", 2023-10-09T05:36:30.206 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:36:30.206 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 21535 2023-10-09T05:36:30.206 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.206 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:36:30.206 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.206 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:36:30.206 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.513048+0000", 2023-10-09T05:36:30.206 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.206 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.206 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.206 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:36:30.207 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.513048+0000", 2023-10-09T05:36:30.207 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.207 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.207 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.207 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:36:30.207 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.513047+0000", 2023-10-09T05:36:30.207 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:36:30.207 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.207 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.207 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:36:30.207 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.513104+0000", 2023-10-09T05:36:30.207 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.6857999999999997e-05 2023-10-09T05:36:30.208 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.208 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.208 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:36:30.208 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.513119+0000", 2023-10-09T05:36:30.208 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.5274e-05 2023-10-09T05:36:30.208 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.208 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.208 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:36:30.208 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.513127+0000", 2023-10-09T05:36:30.208 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.7449999999999995e-06 2023-10-09T05:36:30.208 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.209 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.209 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:36:30.209 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.513183+0000", 2023-10-09T05:36:30.209 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.6268999999999999e-05 2023-10-09T05:36:30.209 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.209 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.209 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:36:30.209 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.513314+0000", 2023-10-09T05:36:30.209 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000131018 2023-10-09T05:36:30.209 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.209 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.209 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:36:30.210 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.513488+0000", 2023-10-09T05:36:30.210 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00017402100000000001 2023-10-09T05:36:30.210 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.210 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.210 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:36:30.210 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.515448+0000", 2023-10-09T05:36:30.210 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0019603300000000001 2023-10-09T05:36:30.210 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.210 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.210 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.210 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.517990+0000", 2023-10-09T05:36:30.210 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0025412529999999998 2023-10-09T05:36:30.211 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.211 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.211 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.211 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.687676+0000", 2023-10-09T05:36:30.211 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.16968680899999999 2023-10-09T05:36:30.211 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.211 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.211 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:36:30.211 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.687780+0000", 2023-10-09T05:36:30.211 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00010389499999999999 2023-10-09T05:36:30.211 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.211 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.212 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:36:30.212 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.687855+0000", 2023-10-09T05:36:30.212 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.4521e-05 2023-10-09T05:36:30.212 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.212 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:36:30.212 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.212 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.212 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.212 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35775.0:21539 14.a 14:58632c57:::benchmark_data_smithi133_267384_object21538:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-10-09T05:36:30.212 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:36:19.516029+0000", 2023-10-09T05:36:30.212 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 10.673922745, 2023-10-09T05:36:30.212 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.17388245499999999, 2023-10-09T05:36:30.213 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:36:30.213 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:30.213 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:36:30.213 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35775", 2023-10-09T05:36:30.213 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:36:30.213 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 21539 2023-10-09T05:36:30.213 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.213 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:36:30.213 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.213 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:36:30.213 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.516029+0000", 2023-10-09T05:36:30.214 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.214 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.214 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.214 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:36:30.214 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.516029+0000", 2023-10-09T05:36:30.214 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.214 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.214 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.214 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:36:30.214 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.516027+0000", 2023-10-09T05:36:30.214 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:36:30.215 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.215 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.215 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:36:30.215 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.516089+0000", 2023-10-09T05:36:30.215 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.1653000000000001e-05 2023-10-09T05:36:30.215 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.215 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.215 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:36:30.215 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.516104+0000", 2023-10-09T05:36:30.215 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.5179e-05 2023-10-09T05:36:30.215 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.215 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.216 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:36:30.216 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.516111+0000", 2023-10-09T05:36:30.216 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.2640000000000002e-06 2023-10-09T05:36:30.216 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.216 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.216 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:36:30.216 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.516175+0000", 2023-10-09T05:36:30.216 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.3604999999999998e-05 2023-10-09T05:36:30.216 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.216 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.216 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:36:30.216 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.516308+0000", 2023-10-09T05:36:30.217 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000133258 2023-10-09T05:36:30.217 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.217 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.217 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:36:30.217 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.516407+0000", 2023-10-09T05:36:30.217 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.8432999999999998e-05 2023-10-09T05:36:30.217 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.217 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.217 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:36:30.217 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.517541+0000", 2023-10-09T05:36:30.217 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0011339049999999999 2023-10-09T05:36:30.217 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.218 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.218 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.218 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.518287+0000", 2023-10-09T05:36:30.218 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000746198 2023-10-09T05:36:30.218 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.218 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.218 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.218 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.689615+0000", 2023-10-09T05:36:30.218 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.17132782699999999 2023-10-09T05:36:30.218 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.218 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.219 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:36:30.219 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.689762+0000", 2023-10-09T05:36:30.219 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000147758 2023-10-09T05:36:30.219 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.219 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.219 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:36:30.219 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.689911+0000", 2023-10-09T05:36:30.219 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00014888100000000001 2023-10-09T05:36:30.219 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.219 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:36:30.219 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.219 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.220 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.220 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35775.0:21547 14.a 14:52a77ca5:::benchmark_data_smithi133_267384_object21546:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-10-09T05:36:30.220 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:36:19.519514+0000", 2023-10-09T05:36:30.220 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 10.670437541, 2023-10-09T05:36:30.220 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.170835553, 2023-10-09T05:36:30.220 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:36:30.220 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:30.220 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:36:30.220 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35775", 2023-10-09T05:36:30.220 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:36:30.220 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 21547 2023-10-09T05:36:30.221 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.221 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:36:30.221 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.221 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:36:30.221 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.519514+0000", 2023-10-09T05:36:30.221 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.221 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.221 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.221 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:36:30.221 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.519514+0000", 2023-10-09T05:36:30.221 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.221 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.222 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.222 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:36:30.222 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.519513+0000", 2023-10-09T05:36:30.222 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:36:30.222 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.222 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.222 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:36:30.222 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.519574+0000", 2023-10-09T05:36:30.222 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.0767999999999998e-05 2023-10-09T05:36:30.222 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.222 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.223 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:36:30.223 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.519589+0000", 2023-10-09T05:36:30.223 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.5696999999999999e-05 2023-10-09T05:36:30.223 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.223 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.223 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:36:30.223 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.519596+0000", 2023-10-09T05:36:30.223 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.9229999999999996e-06 2023-10-09T05:36:30.223 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.223 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.223 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:36:30.223 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.519638+0000", 2023-10-09T05:36:30.224 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.1802999999999999e-05 2023-10-09T05:36:30.224 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.224 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.224 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:36:30.224 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.519794+0000", 2023-10-09T05:36:30.224 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00015599799999999999 2023-10-09T05:36:30.224 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.224 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.224 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:36:30.224 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.519953+0000", 2023-10-09T05:36:30.224 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000159455 2023-10-09T05:36:30.224 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.225 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.225 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:36:30.225 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.521945+0000", 2023-10-09T05:36:30.225 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0019911220000000001 2023-10-09T05:36:30.225 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.225 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.225 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.225 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.522350+0000", 2023-10-09T05:36:30.225 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00040566300000000001 2023-10-09T05:36:30.225 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.225 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.225 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.225 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.690088+0000", 2023-10-09T05:36:30.226 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.16773806899999999 2023-10-09T05:36:30.226 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.226 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.226 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:36:30.226 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.690231+0000", 2023-10-09T05:36:30.226 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000143098 2023-10-09T05:36:30.226 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.226 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.226 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:36:30.226 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.690350+0000", 2023-10-09T05:36:30.226 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000118212 2023-10-09T05:36:30.227 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.227 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:36:30.227 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.227 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.227 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.227 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35775.0:21549 14.a 14:52e96d62:::benchmark_data_smithi133_267384_object21548:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-10-09T05:36:30.227 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:36:19.520042+0000", 2023-10-09T05:36:30.227 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 10.669909969000001, 2023-10-09T05:36:30.227 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.170660964, 2023-10-09T05:36:30.227 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:36:30.227 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:30.227 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:36:30.228 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35775", 2023-10-09T05:36:30.228 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:36:30.228 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 21549 2023-10-09T05:36:30.228 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.228 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:36:30.228 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.228 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:36:30.228 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.520042+0000", 2023-10-09T05:36:30.228 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.228 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.228 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.228 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:36:30.229 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.520042+0000", 2023-10-09T05:36:30.229 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.229 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.229 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.229 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:36:30.229 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.520041+0000", 2023-10-09T05:36:30.229 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:36:30.229 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.229 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.229 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:36:30.229 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.520072+0000", 2023-10-09T05:36:30.229 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.0970000000000003e-05 2023-10-09T05:36:30.230 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.230 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.230 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:36:30.230 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.520082+0000", 2023-10-09T05:36:30.230 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0604e-05 2023-10-09T05:36:30.230 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.230 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.230 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:36:30.230 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.520090+0000", 2023-10-09T05:36:30.230 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.1149999999999994e-06 2023-10-09T05:36:30.230 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.231 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.231 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:36:30.231 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.520743+0000", 2023-10-09T05:36:30.231 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00065322799999999995 2023-10-09T05:36:30.231 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.231 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.231 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:36:30.231 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.520956+0000", 2023-10-09T05:36:30.231 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00021233499999999999 2023-10-09T05:36:30.231 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.231 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.231 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:36:30.232 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.521098+0000", 2023-10-09T05:36:30.232 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00014223600000000001 2023-10-09T05:36:30.232 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.232 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.232 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:36:30.232 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.522094+0000", 2023-10-09T05:36:30.232 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000995943 2023-10-09T05:36:30.232 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.232 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.232 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.232 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.522508+0000", 2023-10-09T05:36:30.233 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00041353700000000001 2023-10-09T05:36:30.233 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.233 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.233 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.233 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.690461+0000", 2023-10-09T05:36:30.233 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.167953038 2023-10-09T05:36:30.233 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.233 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.233 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:36:30.233 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.690605+0000", 2023-10-09T05:36:30.233 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00014451700000000001 2023-10-09T05:36:30.233 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.233 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.234 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:36:30.234 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.690703+0000", 2023-10-09T05:36:30.234 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.7429000000000005e-05 2023-10-09T05:36:30.234 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.234 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:36:30.234 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.234 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.234 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.234 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35775.0:21554 14.9 14:985ef642:::benchmark_data_smithi133_267384_object21553:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-10-09T05:36:30.234 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:36:19.523532+0000", 2023-10-09T05:36:30.234 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 10.666419357000001, 2023-10-09T05:36:30.235 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.16777529999999999, 2023-10-09T05:36:30.235 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:36:30.235 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:30.235 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:36:30.235 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35775", 2023-10-09T05:36:30.235 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:36:30.235 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 21554 2023-10-09T05:36:30.235 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.235 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:36:30.235 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.235 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:36:30.235 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.523532+0000", 2023-10-09T05:36:30.236 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.236 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.236 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.236 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:36:30.236 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.523532+0000", 2023-10-09T05:36:30.236 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.236 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.236 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.236 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:36:30.236 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.523531+0000", 2023-10-09T05:36:30.236 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:36:30.236 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.237 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.237 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:36:30.237 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.523576+0000", 2023-10-09T05:36:30.237 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.5473999999999999e-05 2023-10-09T05:36:30.237 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.237 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.237 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:36:30.237 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.523590+0000", 2023-10-09T05:36:30.237 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.3055e-05 2023-10-09T05:36:30.237 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.237 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.237 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:36:30.238 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.523597+0000", 2023-10-09T05:36:30.238 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.7880000000000007e-06 2023-10-09T05:36:30.238 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.238 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.238 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:36:30.238 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.523642+0000", 2023-10-09T05:36:30.238 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.4274000000000003e-05 2023-10-09T05:36:30.238 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.238 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.238 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:36:30.238 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.523744+0000", 2023-10-09T05:36:30.239 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000102391 2023-10-09T05:36:30.239 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.239 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.239 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:36:30.239 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.523850+0000", 2023-10-09T05:36:30.239 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000105814 2023-10-09T05:36:30.239 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.239 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.239 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:36:30.239 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.524858+0000", 2023-10-09T05:36:30.239 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0010079170000000001 2023-10-09T05:36:30.239 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.240 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.240 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.240 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.525103+0000", 2023-10-09T05:36:30.240 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00024476700000000001 2023-10-09T05:36:30.240 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.240 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.240 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.240 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.691242+0000", 2023-10-09T05:36:30.240 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.166139499 2023-10-09T05:36:30.240 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.240 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.240 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:36:30.241 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.691281+0000", 2023-10-09T05:36:30.241 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.8992999999999997e-05 2023-10-09T05:36:30.241 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.241 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.241 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:36:30.241 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.691307+0000", 2023-10-09T05:36:30.241 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.6478999999999998e-05 2023-10-09T05:36:30.241 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.241 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:36:30.241 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.241 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.241 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.242 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35775.0:22432 14.7 14:edec2dd0:::benchmark_data_smithi133_267384_object22431:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-10-09T05:36:30.242 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:36:20.244204+0000", 2023-10-09T05:36:30.242 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 9.9457477619999999, 2023-10-09T05:36:30.242 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.18193561699999999, 2023-10-09T05:36:30.242 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:36:30.242 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:30.242 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:36:30.242 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35775", 2023-10-09T05:36:30.242 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:36:30.242 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 22432 2023-10-09T05:36:30.242 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.242 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:36:30.243 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.243 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:36:30.243 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.244204+0000", 2023-10-09T05:36:30.243 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.243 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.243 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.243 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:36:30.243 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.244204+0000", 2023-10-09T05:36:30.243 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.243 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.243 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.243 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:36:30.244 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.244197+0000", 2023-10-09T05:36:30.244 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999933 2023-10-09T05:36:30.244 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.244 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.244 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:36:30.244 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.244254+0000", 2023-10-09T05:36:30.244 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.7303999999999999e-05 2023-10-09T05:36:30.244 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.244 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.244 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:36:30.244 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.244277+0000", 2023-10-09T05:36:30.244 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.2853999999999998e-05 2023-10-09T05:36:30.245 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.245 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.245 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:36:30.245 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.244289+0000", 2023-10-09T05:36:30.245 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.1537e-05 2023-10-09T05:36:30.245 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.245 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.245 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:36:30.245 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.244647+0000", 2023-10-09T05:36:30.245 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00035822099999999998 2023-10-09T05:36:30.246 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.246 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.246 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:36:30.246 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.244792+0000", 2023-10-09T05:36:30.246 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000144986 2023-10-09T05:36:30.246 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.246 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.246 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 0,7", 2023-10-09T05:36:30.246 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.244946+0000", 2023-10-09T05:36:30.247 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000154272 2023-10-09T05:36:30.247 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.247 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.247 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:36:30.247 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.248410+0000", 2023-10-09T05:36:30.247 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0034635960000000002 2023-10-09T05:36:30.247 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.247 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.247 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.247 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.251587+0000", 2023-10-09T05:36:30.247 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.003177643 2023-10-09T05:36:30.248 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.248 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.248 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.248 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.425945+0000", 2023-10-09T05:36:30.248 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.174357918 2023-10-09T05:36:30.248 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.248 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.248 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:36:30.248 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.426048+0000", 2023-10-09T05:36:30.248 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00010265800000000001 2023-10-09T05:36:30.248 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.248 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.249 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:36:30.249 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.426139+0000", 2023-10-09T05:36:30.249 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.1415999999999993e-05 2023-10-09T05:36:30.249 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.249 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:36:30.249 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.249 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.249 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.249 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35775.0:22433 14.7 14:e6912679:::benchmark_data_smithi133_267384_object22432:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-10-09T05:36:30.249 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:36:20.246165+0000", 2023-10-09T05:36:30.249 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 9.9437861870000006, 2023-10-09T05:36:30.250 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.18023874200000001, 2023-10-09T05:36:30.250 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:36:30.250 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:30.250 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:36:30.250 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35775", 2023-10-09T05:36:30.250 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:36:30.250 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 22433 2023-10-09T05:36:30.250 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.250 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:36:30.250 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.250 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:36:30.251 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.246165+0000", 2023-10-09T05:36:30.251 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.251 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.251 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.251 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:36:30.251 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.246165+0000", 2023-10-09T05:36:30.251 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.251 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.251 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.251 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:36:30.251 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.246164+0000", 2023-10-09T05:36:30.251 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:36:30.252 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.252 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.252 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:36:30.252 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.246210+0000", 2023-10-09T05:36:30.252 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.6286999999999999e-05 2023-10-09T05:36:30.252 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.252 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.252 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:36:30.252 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.246225+0000", 2023-10-09T05:36:30.252 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.4749999999999999e-05 2023-10-09T05:36:30.252 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.252 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.253 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:36:30.253 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.246234+0000", 2023-10-09T05:36:30.253 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.8519999999999993e-06 2023-10-09T05:36:30.253 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.253 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.253 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:36:30.253 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.246389+0000", 2023-10-09T05:36:30.253 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00015498500000000001 2023-10-09T05:36:30.253 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.253 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.253 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:36:30.253 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.246625+0000", 2023-10-09T05:36:30.254 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00023645400000000001 2023-10-09T05:36:30.254 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.254 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.254 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 0,7", 2023-10-09T05:36:30.254 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.246921+0000", 2023-10-09T05:36:30.254 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00029592600000000001 2023-10-09T05:36:30.254 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.254 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.254 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:36:30.254 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.249192+0000", 2023-10-09T05:36:30.255 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.002270822 2023-10-09T05:36:30.255 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.255 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.255 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.255 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.253840+0000", 2023-10-09T05:36:30.255 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0046483790000000002 2023-10-09T05:36:30.255 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.255 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.255 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.255 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.426236+0000", 2023-10-09T05:36:30.255 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.172395672 2023-10-09T05:36:30.255 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.256 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.256 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:36:30.256 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.426324+0000", 2023-10-09T05:36:30.256 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.7589000000000005e-05 2023-10-09T05:36:30.256 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.256 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.256 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:36:30.256 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.426404+0000", 2023-10-09T05:36:30.256 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.0326000000000003e-05 2023-10-09T05:36:30.256 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.256 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:36:30.256 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.257 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.257 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.257 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35775.0:22436 14.9 14:9e85c77d:::benchmark_data_smithi133_267384_object22435:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-10-09T05:36:30.257 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:36:20.246616+0000", 2023-10-09T05:36:30.257 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 9.9433353110000002, 2023-10-09T05:36:30.257 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.17970519099999999, 2023-10-09T05:36:30.257 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:36:30.257 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:30.257 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:36:30.257 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35775", 2023-10-09T05:36:30.257 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:36:30.258 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 22436 2023-10-09T05:36:30.258 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.258 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:36:30.258 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.258 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:36:30.258 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.246616+0000", 2023-10-09T05:36:30.258 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.258 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.258 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.258 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:36:30.258 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.246616+0000", 2023-10-09T05:36:30.259 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.259 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.259 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.259 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:36:30.259 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.246614+0000", 2023-10-09T05:36:30.259 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999981 2023-10-09T05:36:30.259 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.259 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.259 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:36:30.259 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.246767+0000", 2023-10-09T05:36:30.259 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00015254900000000001 2023-10-09T05:36:30.259 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.260 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.260 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:36:30.260 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.246794+0000", 2023-10-09T05:36:30.260 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.7237e-05 2023-10-09T05:36:30.260 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.260 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.260 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:36:30.260 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.246809+0000", 2023-10-09T05:36:30.260 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.4951e-05 2023-10-09T05:36:30.260 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.260 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.260 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:36:30.261 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.247014+0000", 2023-10-09T05:36:30.261 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00020515900000000001 2023-10-09T05:36:30.261 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.261 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.261 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:36:30.261 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.247275+0000", 2023-10-09T05:36:30.261 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00026124299999999998 2023-10-09T05:36:30.261 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.261 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.261 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:36:30.261 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.247600+0000", 2023-10-09T05:36:30.261 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00032479999999999998 2023-10-09T05:36:30.262 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.262 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.262 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:36:30.262 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.250934+0000", 2023-10-09T05:36:30.262 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0033342549999999999 2023-10-09T05:36:30.262 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.262 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.262 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.262 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.250971+0000", 2023-10-09T05:36:30.262 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.6468999999999999e-05 2023-10-09T05:36:30.263 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.263 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.263 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.263 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.426072+0000", 2023-10-09T05:36:30.263 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.17510128899999999 2023-10-09T05:36:30.263 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.263 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.263 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:36:30.263 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.426210+0000", 2023-10-09T05:36:30.263 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00013765399999999999 2023-10-09T05:36:30.263 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.263 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.264 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:36:30.264 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.426321+0000", 2023-10-09T05:36:30.264 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000111708 2023-10-09T05:36:30.264 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.264 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:36:30.264 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.264 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.264 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.264 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35775.0:22439 14.9 14:99904106:::benchmark_data_smithi133_267384_object22438:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-10-09T05:36:30.264 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:36:20.246842+0000", 2023-10-09T05:36:30.264 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 9.9431095440000004, 2023-10-09T05:36:30.264 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.17965782299999999, 2023-10-09T05:36:30.265 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:36:30.265 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:30.265 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:36:30.265 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35775", 2023-10-09T05:36:30.265 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:36:30.265 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 22439 2023-10-09T05:36:30.265 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.265 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:36:30.265 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.265 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:36:30.265 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.246842+0000", 2023-10-09T05:36:30.265 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.266 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.266 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.266 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:36:30.266 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.246842+0000", 2023-10-09T05:36:30.266 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.266 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.266 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.266 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:36:30.266 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.246841+0000", 2023-10-09T05:36:30.266 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:36:30.266 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.267 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.267 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:36:30.267 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.246888+0000", 2023-10-09T05:36:30.267 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.7456000000000001e-05 2023-10-09T05:36:30.267 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.267 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.267 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:36:30.267 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.246905+0000", 2023-10-09T05:36:30.267 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.6373999999999999e-05 2023-10-09T05:36:30.267 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.267 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.268 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:36:30.268 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.246916+0000", 2023-10-09T05:36:30.268 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0991e-05 2023-10-09T05:36:30.268 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.268 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.268 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:36:30.268 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.249392+0000", 2023-10-09T05:36:30.268 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0024765360000000001 2023-10-09T05:36:30.268 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.268 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.268 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:36:30.269 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.249778+0000", 2023-10-09T05:36:30.269 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00038608199999999999 2023-10-09T05:36:30.269 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.269 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.269 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:36:30.269 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.250071+0000", 2023-10-09T05:36:30.269 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00029302700000000001 2023-10-09T05:36:30.269 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.269 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.269 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.269 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.251341+0000", 2023-10-09T05:36:30.269 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0012697209999999999 2023-10-09T05:36:30.270 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.270 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.270 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:36:30.270 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.253599+0000", 2023-10-09T05:36:30.270 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0022580489999999998 2023-10-09T05:36:30.270 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.270 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.270 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.270 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.426402+0000", 2023-10-09T05:36:30.270 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.172803503 2023-10-09T05:36:30.270 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.270 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.271 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:36:30.271 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.426449+0000", 2023-10-09T05:36:30.271 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.7033000000000002e-05 2023-10-09T05:36:30.271 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.271 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.271 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:36:30.271 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.426500+0000", 2023-10-09T05:36:30.271 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.0352999999999997e-05 2023-10-09T05:36:30.271 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.271 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:36:30.271 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.272 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.272 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.272 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35775.0:22441 14.7 14:e702730d:::benchmark_data_smithi133_267384_object22440:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-10-09T05:36:30.272 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:36:20.251559+0000", 2023-10-09T05:36:30.272 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 9.9383927930000002, 2023-10-09T05:36:30.272 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.17902673199999999, 2023-10-09T05:36:30.272 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:36:30.272 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:30.272 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:36:30.272 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35775", 2023-10-09T05:36:30.272 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:36:30.272 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 22441 2023-10-09T05:36:30.273 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.273 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:36:30.273 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.273 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:36:30.273 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.251559+0000", 2023-10-09T05:36:30.273 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.273 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.273 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.273 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:36:30.273 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.251559+0000", 2023-10-09T05:36:30.273 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.273 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.274 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.274 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:36:30.274 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.251558+0000", 2023-10-09T05:36:30.274 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:36:30.274 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.274 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.274 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:36:30.274 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.251591+0000", 2023-10-09T05:36:30.274 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.3696999999999999e-05 2023-10-09T05:36:30.274 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.274 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.275 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:36:30.275 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.251600+0000", 2023-10-09T05:36:30.275 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.6230000000000003e-06 2023-10-09T05:36:30.275 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.275 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.275 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:36:30.275 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.251605+0000", 2023-10-09T05:36:30.275 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.7530000000000001e-06 2023-10-09T05:36:30.275 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.275 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.275 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:36:30.275 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.251638+0000", 2023-10-09T05:36:30.276 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.2823000000000001e-05 2023-10-09T05:36:30.276 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.276 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.276 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:36:30.276 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.251763+0000", 2023-10-09T05:36:30.276 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000125671 2023-10-09T05:36:30.276 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.276 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.276 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 0,7", 2023-10-09T05:36:30.276 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.251944+0000", 2023-10-09T05:36:30.276 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00018070400000000001 2023-10-09T05:36:30.277 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.277 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.277 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:36:30.277 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.253325+0000", 2023-10-09T05:36:30.277 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.001381132 2023-10-09T05:36:30.277 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.277 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.277 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.277 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.253876+0000", 2023-10-09T05:36:30.277 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000550576 2023-10-09T05:36:30.277 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.277 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.278 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.278 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.430382+0000", 2023-10-09T05:36:30.278 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.17650685299999999 2023-10-09T05:36:30.278 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.278 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.278 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:36:30.278 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.430521+0000", 2023-10-09T05:36:30.278 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00013877099999999999 2023-10-09T05:36:30.278 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.278 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.278 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:36:30.279 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.430585+0000", 2023-10-09T05:36:30.279 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.4245000000000006e-05 2023-10-09T05:36:30.279 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.279 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:36:30.279 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.279 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.279 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.279 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35775.0:22443 14.9 14:9e1315c5:::benchmark_data_smithi133_267384_object22442:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-10-09T05:36:30.279 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:36:20.252107+0000", 2023-10-09T05:36:30.279 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 9.9378443839999999, 2023-10-09T05:36:30.279 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.17841742799999999, 2023-10-09T05:36:30.279 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:36:30.280 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:30.280 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:36:30.280 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35775", 2023-10-09T05:36:30.280 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:36:30.280 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 22443 2023-10-09T05:36:30.280 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.280 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:36:30.280 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.280 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:36:30.280 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.252107+0000", 2023-10-09T05:36:30.280 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.280 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.281 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.281 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:36:30.281 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.252107+0000", 2023-10-09T05:36:30.281 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.281 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.281 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.281 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:36:30.281 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.252106+0000", 2023-10-09T05:36:30.281 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:36:30.281 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.281 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.281 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:36:30.282 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.252156+0000", 2023-10-09T05:36:30.282 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.0117999999999997e-05 2023-10-09T05:36:30.282 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.282 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.282 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:36:30.282 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.252173+0000", 2023-10-09T05:36:30.282 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.7144999999999999e-05 2023-10-09T05:36:30.282 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.282 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.282 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:36:30.282 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.252182+0000", 2023-10-09T05:36:30.283 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.5439999999999999e-06 2023-10-09T05:36:30.283 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.283 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.283 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:36:30.283 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.252239+0000", 2023-10-09T05:36:30.283 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.7024000000000002e-05 2023-10-09T05:36:30.283 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.283 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.283 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:36:30.283 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.252380+0000", 2023-10-09T05:36:30.283 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000140579 2023-10-09T05:36:30.284 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.284 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.284 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:36:30.284 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.252544+0000", 2023-10-09T05:36:30.284 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00016416400000000001 2023-10-09T05:36:30.284 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.284 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.284 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:36:30.284 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.253794+0000", 2023-10-09T05:36:30.284 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0012498629999999999 2023-10-09T05:36:30.284 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.284 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.285 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.285 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.254069+0000", 2023-10-09T05:36:30.285 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00027581200000000001 2023-10-09T05:36:30.285 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.285 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.285 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.285 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.430325+0000", 2023-10-09T05:36:30.285 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.176255357 2023-10-09T05:36:30.285 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.285 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.285 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:36:30.285 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.430427+0000", 2023-10-09T05:36:30.286 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000102205 2023-10-09T05:36:30.286 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.286 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.286 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:36:30.286 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.430525+0000", 2023-10-09T05:36:30.286 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.7656000000000001e-05 2023-10-09T05:36:30.286 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.286 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:36:30.286 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.286 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.286 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.287 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35775.0:22449 14.7 14:e673581f:::benchmark_data_smithi133_267384_object22448:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-10-09T05:36:30.287 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:36:20.255429+0000", 2023-10-09T05:36:30.287 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 9.9345228599999995, 2023-10-09T05:36:30.287 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.17552141900000001, 2023-10-09T05:36:30.287 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:36:30.287 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:30.287 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:36:30.287 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35775", 2023-10-09T05:36:30.287 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:36:30.287 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 22449 2023-10-09T05:36:30.287 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.287 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:36:30.288 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.288 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:36:30.288 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.255429+0000", 2023-10-09T05:36:30.288 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.288 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.288 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.288 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:36:30.288 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.255429+0000", 2023-10-09T05:36:30.288 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.288 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.288 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.288 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:36:30.289 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.255427+0000", 2023-10-09T05:36:30.289 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:36:30.289 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.289 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.289 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:36:30.289 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.255545+0000", 2023-10-09T05:36:30.289 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000118356 2023-10-09T05:36:30.289 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.289 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.289 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:36:30.289 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.255562+0000", 2023-10-09T05:36:30.289 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.6396999999999999e-05 2023-10-09T05:36:30.290 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.290 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.290 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:36:30.290 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.255571+0000", 2023-10-09T05:36:30.290 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.2799999999999992e-06 2023-10-09T05:36:30.290 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.290 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.290 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:36:30.290 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.255624+0000", 2023-10-09T05:36:30.290 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.2766000000000003e-05 2023-10-09T05:36:30.290 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.291 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.291 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:36:30.291 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.255756+0000", 2023-10-09T05:36:30.291 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00013177299999999999 2023-10-09T05:36:30.291 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.291 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.291 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 0,7", 2023-10-09T05:36:30.291 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.255854+0000", 2023-10-09T05:36:30.291 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.8505999999999995e-05 2023-10-09T05:36:30.291 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.291 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.291 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:36:30.292 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.256846+0000", 2023-10-09T05:36:30.292 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0009923899999999999 2023-10-09T05:36:30.292 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.292 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.292 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.292 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.257255+0000", 2023-10-09T05:36:30.292 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00040804300000000002 2023-10-09T05:36:30.292 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.292 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.292 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.292 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.430767+0000", 2023-10-09T05:36:30.292 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.17351219800000001 2023-10-09T05:36:30.293 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.293 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.293 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:36:30.293 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.430899+0000", 2023-10-09T05:36:30.293 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000132392 2023-10-09T05:36:30.293 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.293 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.293 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:36:30.293 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:20.430950+0000", 2023-10-09T05:36:30.293 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.0963e-05 2023-10-09T05:36:30.293 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.293 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:36:30.294 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.294 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.294 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.294 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35775.0:24966 14.9 14:97ce26fb:::benchmark_data_smithi133_267384_object24965:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-10-09T05:36:30.294 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:36:22.075783+0000", 2023-10-09T05:36:30.294 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 8.1141683310000001, 2023-10-09T05:36:30.294 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.11302427299999999, 2023-10-09T05:36:30.294 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:36:30.294 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:30.294 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:36:30.294 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35775", 2023-10-09T05:36:30.295 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:36:30.295 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 24966 2023-10-09T05:36:30.295 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.295 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:36:30.295 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.295 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:36:30.295 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:22.075783+0000", 2023-10-09T05:36:30.295 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.295 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.295 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.295 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:36:30.295 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:22.075783+0000", 2023-10-09T05:36:30.296 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.296 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.296 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.296 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:36:30.296 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:22.075782+0000", 2023-10-09T05:36:30.296 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:36:30.296 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.296 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.296 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:36:30.296 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:22.075835+0000", 2023-10-09T05:36:30.296 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.3634000000000002e-05 2023-10-09T05:36:30.297 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.297 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.297 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:36:30.297 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:22.075856+0000", 2023-10-09T05:36:30.297 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.086e-05 2023-10-09T05:36:30.297 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.297 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.297 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:36:30.297 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:22.075865+0000", 2023-10-09T05:36:30.297 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.2350000000000005e-06 2023-10-09T05:36:30.297 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.297 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.298 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:36:30.298 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:22.075940+0000", 2023-10-09T05:36:30.298 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.4349000000000003e-05 2023-10-09T05:36:30.298 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.298 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.298 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:36:30.298 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:22.076125+0000", 2023-10-09T05:36:30.298 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000184783 2023-10-09T05:36:30.298 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.298 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.298 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:36:30.299 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:22.076324+0000", 2023-10-09T05:36:30.299 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00019923099999999999 2023-10-09T05:36:30.299 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.299 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.299 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:36:30.299 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:22.077710+0000", 2023-10-09T05:36:30.299 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0013859199999999999 2023-10-09T05:36:30.299 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.299 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.299 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.299 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:22.078875+0000", 2023-10-09T05:36:30.300 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.001165307 2023-10-09T05:36:30.300 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.300 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.300 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.300 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:22.188665+0000", 2023-10-09T05:36:30.300 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.109790055 2023-10-09T05:36:30.300 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.300 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.300 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:36:30.300 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:22.188740+0000", 2023-10-09T05:36:30.300 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.4808000000000006e-05 2023-10-09T05:36:30.301 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.301 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.301 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:36:30.301 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:22.188807+0000", 2023-10-09T05:36:30.301 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.7619000000000004e-05 2023-10-09T05:36:30.301 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.301 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:36:30.301 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.301 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.301 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.301 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35775.0:28233 14.a 14:596beaa2:::benchmark_data_smithi133_267384_object28232:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-10-09T05:36:30.301 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:36:24.246425+0000", 2023-10-09T05:36:30.302 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 5.9435265020000001, 2023-10-09T05:36:30.302 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.33090081100000002, 2023-10-09T05:36:30.302 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:36:30.302 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:30.302 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:36:30.302 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35775", 2023-10-09T05:36:30.302 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:36:30.302 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 28233 2023-10-09T05:36:30.302 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.302 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:36:30.302 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.303 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:36:30.303 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.246425+0000", 2023-10-09T05:36:30.303 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.303 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.303 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.303 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:36:30.303 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.246425+0000", 2023-10-09T05:36:30.303 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.303 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.303 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.303 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:36:30.303 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.246419+0000", 2023-10-09T05:36:30.304 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999933 2023-10-09T05:36:30.304 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.304 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.304 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:36:30.304 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.246503+0000", 2023-10-09T05:36:30.304 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.4950000000000005e-05 2023-10-09T05:36:30.304 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.304 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.304 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:36:30.304 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.246525+0000", 2023-10-09T05:36:30.304 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.105e-05 2023-10-09T05:36:30.305 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.305 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.305 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:36:30.305 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.246540+0000", 2023-10-09T05:36:30.305 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.5591e-05 2023-10-09T05:36:30.305 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.305 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.305 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:36:30.305 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.246606+0000", 2023-10-09T05:36:30.305 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.6336999999999999e-05 2023-10-09T05:36:30.305 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.305 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.306 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:36:30.306 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.246731+0000", 2023-10-09T05:36:30.306 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000124642 2023-10-09T05:36:30.306 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.306 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.306 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:36:30.306 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.246859+0000", 2023-10-09T05:36:30.306 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000127713 2023-10-09T05:36:30.306 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.306 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.306 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:36:30.307 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.248448+0000", 2023-10-09T05:36:30.307 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0015893599999999999 2023-10-09T05:36:30.307 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.307 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.307 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.307 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.252991+0000", 2023-10-09T05:36:30.307 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0045428049999999996 2023-10-09T05:36:30.307 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.307 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.307 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.307 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.576944+0000", 2023-10-09T05:36:30.307 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.323953143 2023-10-09T05:36:30.308 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.308 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.308 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:36:30.308 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.577163+0000", 2023-10-09T05:36:30.308 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00021915300000000001 2023-10-09T05:36:30.308 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.308 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.308 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:36:30.308 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.577326+0000", 2023-10-09T05:36:30.308 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000162573 2023-10-09T05:36:30.308 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.308 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:36:30.309 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.309 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.309 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.309 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35775.0:28989 14.7 14:ea31f34a:::benchmark_data_smithi133_267384_object28988:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-10-09T05:36:30.309 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:36:25.038448+0000", 2023-10-09T05:36:30.309 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 5.1515037000000001, 2023-10-09T05:36:30.309 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.104111332, 2023-10-09T05:36:30.309 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:36:30.309 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:30.309 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:36:30.309 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35775", 2023-10-09T05:36:30.309 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:36:30.310 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 28989 2023-10-09T05:36:30.310 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.310 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:36:30.310 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.310 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:36:30.310 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.038448+0000", 2023-10-09T05:36:30.310 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.310 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.310 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.310 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:36:30.310 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.038448+0000", 2023-10-09T05:36:30.311 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.311 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.311 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.311 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:36:30.311 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.038446+0000", 2023-10-09T05:36:30.311 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:36:30.311 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.311 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.311 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:36:30.311 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.038499+0000", 2023-10-09T05:36:30.311 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.2871000000000003e-05 2023-10-09T05:36:30.311 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.312 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.312 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:36:30.312 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.038516+0000", 2023-10-09T05:36:30.312 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.7053000000000001e-05 2023-10-09T05:36:30.312 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.312 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.312 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:36:30.312 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.038523+0000", 2023-10-09T05:36:30.312 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.7499999999999997e-06 2023-10-09T05:36:30.312 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.312 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.313 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:36:30.313 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.038589+0000", 2023-10-09T05:36:30.313 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.5505000000000004e-05 2023-10-09T05:36:30.313 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.313 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.313 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:36:30.313 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.038722+0000", 2023-10-09T05:36:30.313 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000133688 2023-10-09T05:36:30.313 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.313 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.313 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 0,7", 2023-10-09T05:36:30.314 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.038870+0000", 2023-10-09T05:36:30.314 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000147517 2023-10-09T05:36:30.314 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.314 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.314 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:36:30.314 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.040093+0000", 2023-10-09T05:36:30.314 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.001223284 2023-10-09T05:36:30.314 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.314 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.314 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.314 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.041873+0000", 2023-10-09T05:36:30.315 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0017797049999999999 2023-10-09T05:36:30.315 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.315 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.315 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.315 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.142460+0000", 2023-10-09T05:36:30.315 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.100587014 2023-10-09T05:36:30.315 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.315 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.315 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:36:30.315 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.142524+0000", 2023-10-09T05:36:30.315 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.4444999999999997e-05 2023-10-09T05:36:30.315 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.316 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.316 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:36:30.316 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.142559+0000", 2023-10-09T05:36:30.316 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.4835000000000001e-05 2023-10-09T05:36:30.316 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.316 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:36:30.316 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.316 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.316 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.316 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35775.0:29795 14.a 14:5e44ffb8:::benchmark_data_smithi133_267384_object29794:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-10-09T05:36:30.316 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:36:25.621684+0000", 2023-10-09T05:36:30.316 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 4.5682679889999998, 2023-10-09T05:36:30.317 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.11089439, 2023-10-09T05:36:30.317 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:36:30.317 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:30.317 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:36:30.317 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35775", 2023-10-09T05:36:30.317 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:36:30.317 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 29795 2023-10-09T05:36:30.317 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.317 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:36:30.317 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.317 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:36:30.317 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.621684+0000", 2023-10-09T05:36:30.318 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.318 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.318 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.318 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:36:30.318 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.621684+0000", 2023-10-09T05:36:30.318 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.318 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.318 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.318 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:36:30.318 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.621683+0000", 2023-10-09T05:36:30.318 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999995 2023-10-09T05:36:30.319 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.319 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.319 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:36:30.319 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.621715+0000", 2023-10-09T05:36:30.319 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.1772e-05 2023-10-09T05:36:30.319 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.319 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.319 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:36:30.319 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.621730+0000", 2023-10-09T05:36:30.319 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.4749000000000001e-05 2023-10-09T05:36:30.319 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.319 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.320 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:36:30.320 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.621736+0000", 2023-10-09T05:36:30.320 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.5350000000000001e-06 2023-10-09T05:36:30.320 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.320 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.320 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:36:30.320 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.621792+0000", 2023-10-09T05:36:30.320 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.5414000000000002e-05 2023-10-09T05:36:30.320 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.320 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.320 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:36:30.320 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.621906+0000", 2023-10-09T05:36:30.321 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000114764 2023-10-09T05:36:30.321 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.321 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.321 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:36:30.321 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.622021+0000", 2023-10-09T05:36:30.321 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000114256 2023-10-09T05:36:30.321 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.321 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.321 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:36:30.321 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.623656+0000", 2023-10-09T05:36:30.321 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0016350850000000001 2023-10-09T05:36:30.322 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.322 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.322 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.322 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.624118+0000", 2023-10-09T05:36:30.322 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00046193100000000001 2023-10-09T05:36:30.322 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.322 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.322 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.322 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.732473+0000", 2023-10-09T05:36:30.322 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.108354991 2023-10-09T05:36:30.322 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.322 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.323 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:36:30.323 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.732532+0000", 2023-10-09T05:36:30.323 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.9805000000000001e-05 2023-10-09T05:36:30.323 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.323 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.323 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:36:30.323 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.732578+0000", 2023-10-09T05:36:30.323 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.5586e-05 2023-10-09T05:36:30.323 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.323 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:36:30.323 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.323 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.324 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.324 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35775.0:29813 14.a 14:5ef94df0:::benchmark_data_smithi133_267384_object29812:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-10-09T05:36:30.324 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:36:25.631145+0000", 2023-10-09T05:36:30.324 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 4.558806873, 2023-10-09T05:36:30.324 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.108160331, 2023-10-09T05:36:30.324 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:36:30.324 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:30.324 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:36:30.324 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35775", 2023-10-09T05:36:30.324 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:36:30.324 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 29813 2023-10-09T05:36:30.324 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.325 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:36:30.325 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.325 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:36:30.325 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.631145+0000", 2023-10-09T05:36:30.325 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.325 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.325 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.325 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:36:30.325 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.631145+0000", 2023-10-09T05:36:30.325 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.325 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.325 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.326 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:36:30.326 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.631143+0000", 2023-10-09T05:36:30.326 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:36:30.326 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.326 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.326 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:36:30.326 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.631195+0000", 2023-10-09T05:36:30.326 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.2012000000000003e-05 2023-10-09T05:36:30.326 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.326 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.326 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:36:30.326 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.631212+0000", 2023-10-09T05:36:30.327 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.7161999999999999e-05 2023-10-09T05:36:30.327 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.327 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.327 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:36:30.327 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.631221+0000", 2023-10-09T05:36:30.327 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.2970000000000004e-06 2023-10-09T05:36:30.327 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.327 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.327 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:36:30.327 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.631286+0000", 2023-10-09T05:36:30.327 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.5567000000000004e-05 2023-10-09T05:36:30.327 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.328 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.328 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:36:30.328 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.631435+0000", 2023-10-09T05:36:30.328 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000149371 2023-10-09T05:36:30.328 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.328 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.328 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:36:30.328 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.631603+0000", 2023-10-09T05:36:30.328 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00016718599999999999 2023-10-09T05:36:30.328 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.328 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.328 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:36:30.329 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.632936+0000", 2023-10-09T05:36:30.329 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0013328630000000001 2023-10-09T05:36:30.329 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.329 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.329 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.329 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.633335+0000", 2023-10-09T05:36:30.329 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00039966900000000001 2023-10-09T05:36:30.329 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.329 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.329 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.329 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.739199+0000", 2023-10-09T05:36:30.329 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.105863955 2023-10-09T05:36:30.330 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.330 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.330 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:36:30.330 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.739257+0000", 2023-10-09T05:36:30.330 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.7509000000000001e-05 2023-10-09T05:36:30.330 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.330 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.330 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:36:30.330 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:25.739305+0000", 2023-10-09T05:36:30.330 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.8350999999999998e-05 2023-10-09T05:36:30.330 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.331 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:36:30.331 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.331 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.331 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.331 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35775.0:31511 14.9 14:965bab40:::benchmark_data_smithi133_267384_object31510:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-10-09T05:36:30.331 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:36:26.886274+0000", 2023-10-09T05:36:30.331 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 3.3036775660000002, 2023-10-09T05:36:30.331 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.178689558, 2023-10-09T05:36:30.331 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:36:30.331 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:30.331 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:36:30.331 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35775", 2023-10-09T05:36:30.332 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:36:30.332 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 31511 2023-10-09T05:36:30.332 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.332 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:36:30.332 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.332 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:36:30.332 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:26.886274+0000", 2023-10-09T05:36:30.332 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.332 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.332 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.332 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:36:30.332 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:26.886274+0000", 2023-10-09T05:36:30.333 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.333 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.333 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.333 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:36:30.333 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:26.886266+0000", 2023-10-09T05:36:30.333 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999914 2023-10-09T05:36:30.333 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.333 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.333 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:36:30.333 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:26.886468+0000", 2023-10-09T05:36:30.333 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00020248299999999999 2023-10-09T05:36:30.333 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.333 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.334 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:36:30.334 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:26.887059+0000", 2023-10-09T05:36:30.334 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00059091600000000003 2023-10-09T05:36:30.334 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.334 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.334 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:36:30.334 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:26.887092+0000", 2023-10-09T05:36:30.334 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.2836999999999997e-05 2023-10-09T05:36:30.334 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.334 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.334 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:36:30.335 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:26.887172+0000", 2023-10-09T05:36:30.335 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.0190000000000003e-05 2023-10-09T05:36:30.335 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.335 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.335 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:36:30.335 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:26.887450+0000", 2023-10-09T05:36:30.335 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00027758500000000001 2023-10-09T05:36:30.335 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.335 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.335 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:36:30.335 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:26.888255+0000", 2023-10-09T05:36:30.335 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00080569599999999995 2023-10-09T05:36:30.336 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.336 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.336 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:36:30.336 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:26.891948+0000", 2023-10-09T05:36:30.336 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0036925220000000002 2023-10-09T05:36:30.336 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.336 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.336 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.336 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:26.892027+0000", 2023-10-09T05:36:30.336 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.9382000000000007e-05 2023-10-09T05:36:30.336 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.336 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.337 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.337 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:27.064856+0000", 2023-10-09T05:36:30.337 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.172828967 2023-10-09T05:36:30.337 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.337 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.337 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:36:30.337 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:27.064912+0000", 2023-10-09T05:36:30.337 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.5927999999999997e-05 2023-10-09T05:36:30.337 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.337 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.337 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:36:30.337 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:27.064964+0000", 2023-10-09T05:36:30.338 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.1418000000000001e-05 2023-10-09T05:36:30.338 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.338 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:36:30.338 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.338 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.338 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.338 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35775.0:31512 14.9 14:98eeb4d0:::benchmark_data_smithi133_267384_object31511:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-10-09T05:36:30.338 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:36:26.887119+0000", 2023-10-09T05:36:30.338 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 3.3028323099999999, 2023-10-09T05:36:30.338 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.18596834900000001, 2023-10-09T05:36:30.338 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:36:30.339 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:30.339 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:36:30.339 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35775", 2023-10-09T05:36:30.339 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:36:30.339 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 31512 2023-10-09T05:36:30.339 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.339 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:36:30.339 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.339 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:36:30.339 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:26.887119+0000", 2023-10-09T05:36:30.339 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.339 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.340 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.340 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:36:30.340 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:26.887119+0000", 2023-10-09T05:36:30.340 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:36:30.340 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.340 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.340 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:36:30.340 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:26.887117+0000", 2023-10-09T05:36:30.340 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999981 2023-10-09T05:36:30.340 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.340 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.340 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:36:30.341 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:26.887230+0000", 2023-10-09T05:36:30.341 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000112998 2023-10-09T05:36:30.341 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.341 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.341 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:36:30.341 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:26.887251+0000", 2023-10-09T05:36:30.341 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.0315000000000001e-05 2023-10-09T05:36:30.341 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.341 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.341 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:36:30.341 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:26.887261+0000", 2023-10-09T05:36:30.341 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.0386e-05 2023-10-09T05:36:30.342 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.342 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.342 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:36:30.342 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:26.890386+0000", 2023-10-09T05:36:30.342 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0031245999999999999 2023-10-09T05:36:30.342 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.342 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.342 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:36:30.342 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:26.890718+0000", 2023-10-09T05:36:30.342 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000332727 2023-10-09T05:36:30.342 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.343 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.343 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:36:30.343 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:26.890898+0000", 2023-10-09T05:36:30.343 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00017900300000000001 2023-10-09T05:36:30.343 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.343 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.343 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:36:30.343 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:26.892175+0000", 2023-10-09T05:36:30.343 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.001277189 2023-10-09T05:36:30.343 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.343 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.344 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.344 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:26.892413+0000", 2023-10-09T05:36:30.344 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00023804100000000001 2023-10-09T05:36:30.344 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.344 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.344 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:30.344 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:27.072974+0000", 2023-10-09T05:36:30.344 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.180561003 2023-10-09T05:36:30.344 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.344 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.344 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:36:30.344 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:27.073034+0000", 2023-10-09T05:36:30.345 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.0761000000000003e-05 2023-10-09T05:36:30.345 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:36:30.345 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:36:30.345 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:36:30.345 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:27.073088+0000", 2023-10-09T05:36:30.345 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.3075000000000003e-05 2023-10-09T05:36:30.345 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.345 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:36:30.345 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.345 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:36:30.345 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:36:30.345 INFO:teuthology.orchestra.run.smithi194.stdout:} 2023-10-09T05:36:30.346 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:30.347 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:36:30.347 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:36:30.348 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:36:30.348 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:36:30.348 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:36:30.348 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:36:30.348 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:36:30.348 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:36:30.348 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:36:30.348 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:36:30.348 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:36:30.372 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:30.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:30.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:30.476 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:30.493 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is set 2023-10-09T05:36:30.504 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:30.506 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:30 smithi133 bash[105589]: audit 2023-10-09T05:36:29.910713+0000 mon.c (mon.2) 134 : audit 0 from='client.? 172.21.15.133:0/1728622272' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:36:30.506 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:30 smithi133 bash[105589]: audit 2023-10-09T05:36:30.304120+0000 mon.c (mon.2) 135 : audit 1 from='client.? 172.21.15.133:0/2185526697' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:36:30.507 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:30 smithi133 bash[105589]: audit 2023-10-09T05:36:30.305487+0000 mon.a (mon.0) 1822 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:36:30.507 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:30 smithi133 bash[102858]: audit 2023-10-09T05:36:29.910713+0000 mon.c (mon.2) 134 : audit 0 from='client.? 172.21.15.133:0/1728622272' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:36:30.507 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:30 smithi133 bash[102858]: audit 2023-10-09T05:36:30.304120+0000 mon.c (mon.2) 135 : audit 1 from='client.? 172.21.15.133:0/2185526697' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:36:30.507 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:30 smithi133 bash[102858]: audit 2023-10-09T05:36:30.305487+0000 mon.a (mon.0) 1822 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:36:30.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:30.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:30.609 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:30.618 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:30 smithi194 bash[92666]: audit 2023-10-09T05:36:29.910713+0000 mon.c (mon.2) 134 : audit 0 from='client.? 172.21.15.133:0/1728622272' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:36:30.618 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:30 smithi194 bash[92666]: audit 2023-10-09T05:36:30.304120+0000 mon.c (mon.2) 135 : audit 1 from='client.? 172.21.15.133:0/2185526697' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:36:30.618 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:30 smithi194 bash[92666]: audit 2023-10-09T05:36:30.305487+0000 mon.a (mon.0) 1822 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:36:30.634 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:30.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:30.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:30.737 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:30.763 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 22 16 36187 36171 102.748 69.875 0.0258383 0.00972362 2023-10-09T05:36:30.766 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:30.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:30.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:30.871 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:30.900 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:30.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:30.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:31.005 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:31.029 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:31.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:31.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:31.133 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:31.160 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:31.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:31.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:31.264 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:31.290 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:31.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:31.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:31.394 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:31.422 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:31.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:31.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:31.458 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T05:36:31.525 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:31.554 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:31.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:31.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:31.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:31 smithi194 bash[92666]: audit 2023-10-09T05:36:29.964456+0000 mgr.y (mgr.25277) 1457 : audit 0 from='client.45782 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:36:31.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:31 smithi194 bash[92666]: cluster 2023-10-09T05:36:30.401588+0000 mgr.y (mgr.25277) 1458 : cluster 0 pgmap v1565: 89 pgs: 89 active+clean; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-09T05:36:31.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:31 smithi194 bash[92666]: cluster 2023-10-09T05:36:30.460160+0000 mon.a (mon.0) 1823 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:36:31.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:31 smithi194 bash[92666]: audit 2023-10-09T05:36:30.479937+0000 mon.a (mon.0) 1824 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:36:31.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:31 smithi194 bash[92666]: cluster 2023-10-09T05:36:30.480222+0000 mon.a (mon.0) 1825 : cluster 0 osdmap e317: 8 total, 8 up, 7 in 2023-10-09T05:36:31.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:31 smithi194 bash[92666]: cluster 2023-10-09T05:36:30.485427+0000 osd.5 (osd.5) 297 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T05:36:31.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:31 smithi194 bash[92666]: cluster 2023-10-09T05:36:30.486689+0000 osd.6 (osd.6) 278 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T05:36:31.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:31 smithi194 bash[92666]: cluster 2023-10-09T05:36:30.487036+0000 osd.5 (osd.5) 298 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T05:36:31.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:31 smithi194 bash[92666]: cluster 2023-10-09T05:36:30.488350+0000 osd.6 (osd.6) 279 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T05:36:31.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:31 smithi194 bash[92666]: cluster 2023-10-09T05:36:30.491622+0000 osd.6 (osd.6) 280 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T05:36:31.637 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:31 smithi194 bash[92666]: cluster 2023-10-09T05:36:30.491744+0000 osd.7 (osd.7) 277 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T05:36:31.661 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:31.690 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:31.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:31.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:31.761 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 23 16 37604 37588 102.13 88.5625 0.00551793 0.00978443 2023-10-09T05:36:31.796 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:31.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:31 smithi133 bash[105589]: audit 2023-10-09T05:36:29.964456+0000 mgr.y (mgr.25277) 1457 : audit 0 from='client.45782 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:36:31.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:31 smithi133 bash[105589]: cluster 2023-10-09T05:36:30.401588+0000 mgr.y (mgr.25277) 1458 : cluster 0 pgmap v1565: 89 pgs: 89 active+clean; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-09T05:36:31.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:31 smithi133 bash[105589]: cluster 2023-10-09T05:36:30.460160+0000 mon.a (mon.0) 1823 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:36:31.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:31 smithi133 bash[105589]: audit 2023-10-09T05:36:30.479937+0000 mon.a (mon.0) 1824 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:36:31.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:31 smithi133 bash[105589]: cluster 2023-10-09T05:36:30.480222+0000 mon.a (mon.0) 1825 : cluster 0 osdmap e317: 8 total, 8 up, 7 in 2023-10-09T05:36:31.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:31 smithi133 bash[105589]: cluster 2023-10-09T05:36:30.485427+0000 osd.5 (osd.5) 297 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T05:36:31.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:31 smithi133 bash[105589]: cluster 2023-10-09T05:36:30.486689+0000 osd.6 (osd.6) 278 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T05:36:31.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:31 smithi133 bash[105589]: cluster 2023-10-09T05:36:30.487036+0000 osd.5 (osd.5) 298 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T05:36:31.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:31 smithi133 bash[105589]: cluster 2023-10-09T05:36:30.488350+0000 osd.6 (osd.6) 279 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T05:36:31.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:31 smithi133 bash[105589]: cluster 2023-10-09T05:36:30.491622+0000 osd.6 (osd.6) 280 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T05:36:31.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:31 smithi133 bash[105589]: cluster 2023-10-09T05:36:30.491744+0000 osd.7 (osd.7) 277 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T05:36:31.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:31 smithi133 bash[102858]: audit 2023-10-09T05:36:29.964456+0000 mgr.y (mgr.25277) 1457 : audit 0 from='client.45782 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:36:31.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:31 smithi133 bash[102858]: cluster 2023-10-09T05:36:30.401588+0000 mgr.y (mgr.25277) 1458 : cluster 0 pgmap v1565: 89 pgs: 89 active+clean; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-09T05:36:31.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:31 smithi133 bash[102858]: cluster 2023-10-09T05:36:30.460160+0000 mon.a (mon.0) 1823 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:36:31.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:31 smithi133 bash[102858]: audit 2023-10-09T05:36:30.479937+0000 mon.a (mon.0) 1824 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:36:31.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:31 smithi133 bash[102858]: cluster 2023-10-09T05:36:30.480222+0000 mon.a (mon.0) 1825 : cluster 0 osdmap e317: 8 total, 8 up, 7 in 2023-10-09T05:36:31.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:31 smithi133 bash[102858]: cluster 2023-10-09T05:36:30.485427+0000 osd.5 (osd.5) 297 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T05:36:31.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:31 smithi133 bash[102858]: cluster 2023-10-09T05:36:30.486689+0000 osd.6 (osd.6) 278 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T05:36:31.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:31 smithi133 bash[102858]: cluster 2023-10-09T05:36:30.487036+0000 osd.5 (osd.5) 298 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T05:36:31.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:31 smithi133 bash[102858]: cluster 2023-10-09T05:36:30.488350+0000 osd.6 (osd.6) 279 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T05:36:31.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:31 smithi133 bash[102858]: cluster 2023-10-09T05:36:30.491622+0000 osd.6 (osd.6) 280 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T05:36:31.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:31 smithi133 bash[102858]: cluster 2023-10-09T05:36:30.491744+0000 osd.7 (osd.7) 277 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T05:36:31.823 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:31.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:31.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:31.927 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:31.954 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:31.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:31.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:32.057 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:32.082 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:32.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:32.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:32.187 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:32.216 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:32.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:32.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:32.323 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:32.350 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:32.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:32.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:32.454 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:32.482 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:32.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:32.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:32.586 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:32.612 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:32.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:32.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:32.717 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:32.744 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:32.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:32.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:32.762 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 24 16 38924 38908 101.312 82.5 0.00682822 0.00986502 2023-10-09T05:36:32.847 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:32.877 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:32.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:32.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:32.980 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:33.003 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:33.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:33.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:33.106 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:33.130 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:33.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:33.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:33.234 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:33.260 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:33.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:33.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:33.363 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:33.393 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:33.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:33.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:33.481 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:33 smithi133 bash[102858]: cluster 2023-10-09T05:36:32.402170+0000 mgr.y (mgr.25277) 1459 : cluster 0 pgmap v1567: 89 pgs: 89 active+clean; 2.2 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T05:36:33.498 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:33.537 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:33.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:33.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:33.557 INFO:teuthology.orchestra.run.smithi133.stdout:{"pg_ready":true,"pg_map":{"version":1564,"stamp":"2023-10-09T05:36:28.400305+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1909069728,"num_objects":29145,"num_object_clones":0,"num_object_copies":87435,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29145,"num_whiteouts":0,"num_read":509,"num_read_kb":934,"num_write":29421,"num_write_kb":1866189,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":4154804,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":29343,"ondisk_log_size":29343,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":12935772,"kb_used_data":5753188,"kb_used_omap":20,"kb_used_meta":7182507,"kb_avail":643280292,"statfs":{"total":671965249536,"available":658719019008,"internally_reserved":0,"allocated":5891264512,"data_stored":5881650202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":7354887822},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[8,4,3,7,1,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":977731584,"num_objects":14919,"num_object_clones":0,"num_object_copies":44757,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14919,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14919,"num_write_kb":954816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"11.516689"},"pg_stats":[{"pgid":"3.1a","version":"69'2","reported_seq":185,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.877052+0000","last_change":"2023-10-09T05:30:03.378323+0000","last_active":"2023-10-09T05:36:15.877052+0000","last_peered":"2023-10-09T05:36:15.877052+0000","last_clean":"2023-10-09T05:36:15.877052+0000","last_became_active":"2023-10-09T05:30:03.377871+0000","last_became_peered":"2023-10-09T05:30:03.377871+0000","last_unstale":"2023-10-09T05:36:15.877052+0000","last_undegraded":"2023-10-09T05:36:15.877052+0000","last_fullsized":"2023-10-09T05:36:15.877052+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:09.181202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":311,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.878787+0000","last_change":"2023-10-09T05:30:03.377792+0000","last_active":"2023-10-09T05:36:15.878787+0000","last_peered":"2023-10-09T05:36:15.878787+0000","last_clean":"2023-10-09T05:36:15.878787+0000","last_became_active":"2023-10-09T05:30:03.377456+0000","last_became_peered":"2023-10-09T05:30:03.377456+0000","last_unstale":"2023-10-09T05:36:15.878787+0000","last_undegraded":"2023-10-09T05:36:15.878787+0000","last_fullsized":"2023-10-09T05:36:15.878787+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:35:57.941046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":194,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.876164+0000","last_change":"2023-10-09T05:30:03.377059+0000","last_active":"2023-10-09T05:36:15.876164+0000","last_peered":"2023-10-09T05:36:15.876164+0000","last_clean":"2023-10-09T05:36:15.876164+0000","last_became_active":"2023-10-09T05:30:03.376460+0000","last_became_peered":"2023-10-09T05:30:03.376460+0000","last_unstale":"2023-10-09T05:36:15.876164+0000","last_undegraded":"2023-10-09T05:36:15.876164+0000","last_fullsized":"2023-10-09T05:36:15.876164+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:01:08.558120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":320,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.881251+0000","last_change":"2023-10-09T05:18:26.422923+0000","last_active":"2023-10-09T05:36:15.881251+0000","last_peered":"2023-10-09T05:36:15.881251+0000","last_clean":"2023-10-09T05:36:15.881251+0000","last_became_active":"2023-10-09T05:18:26.422508+0000","last_became_peered":"2023-10-09T05:18:26.422508+0000","last_unstale":"2023-10-09T05:36:15.881251+0000","last_undegraded":"2023-10-09T05:36:15.881251+0000","last_fullsized":"2023-10-09T05:36:15.881251+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:00.876037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":339,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.878332+0000","last_change":"2023-10-09T05:23:18.188378+0000","last_active":"2023-10-09T05:36:15.878332+0000","last_peered":"2023-10-09T05:36:15.878332+0000","last_clean":"2023-10-09T05:36:15.878332+0000","last_became_active":"2023-10-09T05:23:18.186629+0000","last_became_peered":"2023-10-09T05:23:18.186629+0000","last_unstale":"2023-10-09T05:36:15.878332+0000","last_undegraded":"2023-10-09T05:36:15.878332+0000","last_fullsized":"2023-10-09T05:36:15.878332+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:16.444225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":316,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.884253+0000","last_change":"2023-10-09T05:32:45.829908+0000","last_active":"2023-10-09T05:36:15.884253+0000","last_peered":"2023-10-09T05:36:15.884253+0000","last_clean":"2023-10-09T05:36:15.884253+0000","last_became_active":"2023-10-09T05:32:45.829566+0000","last_became_peered":"2023-10-09T05:32:45.829566+0000","last_unstale":"2023-10-09T05:36:15.884253+0000","last_undegraded":"2023-10-09T05:36:15.884253+0000","last_fullsized":"2023-10-09T05:36:15.884253+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:05.705818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":316,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.881969+0000","last_change":"2023-10-09T05:32:45.835245+0000","last_active":"2023-10-09T05:36:15.881969+0000","last_peered":"2023-10-09T05:36:15.881969+0000","last_clean":"2023-10-09T05:36:15.881969+0000","last_became_active":"2023-10-09T05:32:45.830180+0000","last_became_peered":"2023-10-09T05:32:45.830180+0000","last_unstale":"2023-10-09T05:36:15.881969+0000","last_undegraded":"2023-10-09T05:36:15.881969+0000","last_fullsized":"2023-10-09T05:36:15.881969+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:07:06.351159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":294,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.878415+0000","last_change":"2023-10-09T05:09:57.849695+0000","last_active":"2023-10-09T05:36:15.878415+0000","last_peered":"2023-10-09T05:36:15.878415+0000","last_clean":"2023-10-09T05:36:15.878415+0000","last_became_active":"2023-10-09T05:09:57.848478+0000","last_became_peered":"2023-10-09T05:09:57.848478+0000","last_unstale":"2023-10-09T05:36:15.878415+0000","last_undegraded":"2023-10-09T05:36:15.878415+0000","last_fullsized":"2023-10-09T05:36:15.878415+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:26.115312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":25,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.881535+0000","last_change":"2023-10-09T05:32:45.830061+0000","last_active":"2023-10-09T05:36:15.881535+0000","last_peered":"2023-10-09T05:36:15.881535+0000","last_clean":"2023-10-09T05:36:15.881535+0000","last_became_active":"2023-10-09T05:32:45.829723+0000","last_became_peered":"2023-10-09T05:32:45.829723+0000","last_unstale":"2023-10-09T05:36:15.881535+0000","last_undegraded":"2023-10-09T05:36:15.881535+0000","last_fullsized":"2023-10-09T05:36:15.881535+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:21.429740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":307,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.879202+0000","last_change":"2023-10-09T05:18:26.418431+0000","last_active":"2023-10-09T05:36:15.879202+0000","last_peered":"2023-10-09T05:36:15.879202+0000","last_clean":"2023-10-09T05:36:15.879202+0000","last_became_active":"2023-10-09T05:18:26.418100+0000","last_became_peered":"2023-10-09T05:18:26.418100+0000","last_unstale":"2023-10-09T05:36:15.879202+0000","last_undegraded":"2023-10-09T05:36:15.879202+0000","last_fullsized":"2023-10-09T05:36:15.879202+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:21.241286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":127,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.884238+0000","last_change":"2023-10-09T05:30:03.413915+0000","last_active":"2023-10-09T05:36:15.884238+0000","last_peered":"2023-10-09T05:36:15.884238+0000","last_clean":"2023-10-09T05:36:15.884238+0000","last_became_active":"2023-10-09T05:30:03.391102+0000","last_became_peered":"2023-10-09T05:30:03.391102+0000","last_unstale":"2023-10-09T05:36:15.884238+0000","last_undegraded":"2023-10-09T05:36:15.884238+0000","last_fullsized":"2023-10-09T05:36:15.884238+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:13.024050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":79,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.876331+0000","last_change":"2023-10-09T05:23:18.199181+0000","last_active":"2023-10-09T05:36:15.876331+0000","last_peered":"2023-10-09T05:36:15.876331+0000","last_clean":"2023-10-09T05:36:15.876331+0000","last_became_active":"2023-10-09T05:23:18.198401+0000","last_became_peered":"2023-10-09T05:23:18.198401+0000","last_unstale":"2023-10-09T05:36:15.876331+0000","last_undegraded":"2023-10-09T05:36:15.876331+0000","last_fullsized":"2023-10-09T05:36:15.876331+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:55:49.891420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'23","reported_seq":305,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.883458+0000","last_change":"2023-10-09T05:30:03.423577+0000","last_active":"2023-10-09T05:36:15.883458+0000","last_peered":"2023-10-09T05:36:15.883458+0000","last_clean":"2023-10-09T05:36:15.883458+0000","last_became_active":"2023-10-09T05:30:03.394939+0000","last_became_peered":"2023-10-09T05:30:03.394939+0000","last_unstale":"2023-10-09T05:36:15.883458+0000","last_undegraded":"2023-10-09T05:36:15.883458+0000","last_fullsized":"2023-10-09T05:36:15.883458+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:19:34.968600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":174,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.884066+0000","last_change":"2023-10-09T05:32:45.831400+0000","last_active":"2023-10-09T05:36:15.884066+0000","last_peered":"2023-10-09T05:36:15.884066+0000","last_clean":"2023-10-09T05:36:15.884066+0000","last_became_active":"2023-10-09T05:32:45.831007+0000","last_became_peered":"2023-10-09T05:32:45.831007+0000","last_unstale":"2023-10-09T05:36:15.884066+0000","last_undegraded":"2023-10-09T05:36:15.884066+0000","last_fullsized":"2023-10-09T05:36:15.884066+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:00:20.174754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":127,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.883877+0000","last_change":"2023-10-09T05:30:03.407302+0000","last_active":"2023-10-09T05:36:15.883877+0000","last_peered":"2023-10-09T05:36:15.883877+0000","last_clean":"2023-10-09T05:36:15.883877+0000","last_became_active":"2023-10-09T05:30:03.374209+0000","last_became_peered":"2023-10-09T05:30:03.374209+0000","last_unstale":"2023-10-09T05:36:15.883877+0000","last_undegraded":"2023-10-09T05:36:15.883877+0000","last_fullsized":"2023-10-09T05:36:15.883877+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:53.750483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":348,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.881676+0000","last_change":"2023-10-09T05:23:18.187593+0000","last_active":"2023-10-09T05:36:15.881676+0000","last_peered":"2023-10-09T05:36:15.881676+0000","last_clean":"2023-10-09T05:36:15.881676+0000","last_became_active":"2023-10-09T05:23:18.186445+0000","last_became_peered":"2023-10-09T05:23:18.186445+0000","last_unstale":"2023-10-09T05:36:15.881676+0000","last_undegraded":"2023-10-09T05:36:15.881676+0000","last_fullsized":"2023-10-09T05:36:15.881676+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:11.327635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":316,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.880866+0000","last_change":"2023-10-09T05:18:26.424399+0000","last_active":"2023-10-09T05:36:15.880866+0000","last_peered":"2023-10-09T05:36:15.880866+0000","last_clean":"2023-10-09T05:36:15.880866+0000","last_became_active":"2023-10-09T05:18:26.422122+0000","last_became_peered":"2023-10-09T05:18:26.422122+0000","last_unstale":"2023-10-09T05:36:15.880866+0000","last_undegraded":"2023-10-09T05:36:15.880866+0000","last_fullsized":"2023-10-09T05:36:15.880866+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:39.393756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":25,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.881584+0000","last_change":"2023-10-09T05:32:45.831080+0000","last_active":"2023-10-09T05:36:15.881584+0000","last_peered":"2023-10-09T05:36:15.881584+0000","last_clean":"2023-10-09T05:36:15.881584+0000","last_became_active":"2023-10-09T05:32:45.830745+0000","last_became_peered":"2023-10-09T05:32:45.830745+0000","last_unstale":"2023-10-09T05:36:15.881584+0000","last_undegraded":"2023-10-09T05:36:15.881584+0000","last_fullsized":"2023-10-09T05:36:15.881584+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:16.345644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":115,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.883391+0000","last_change":"2023-10-09T05:30:03.395964+0000","last_active":"2023-10-09T05:36:15.883391+0000","last_peered":"2023-10-09T05:36:15.883391+0000","last_clean":"2023-10-09T05:36:15.883391+0000","last_became_active":"2023-10-09T05:30:03.395634+0000","last_became_peered":"2023-10-09T05:30:03.395634+0000","last_unstale":"2023-10-09T05:36:15.883391+0000","last_undegraded":"2023-10-09T05:36:15.883391+0000","last_fullsized":"2023-10-09T05:36:15.883391+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:31.601758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":317,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.881299+0000","last_change":"2023-10-09T05:18:26.421678+0000","last_active":"2023-10-09T05:36:15.881299+0000","last_peered":"2023-10-09T05:36:15.881299+0000","last_clean":"2023-10-09T05:36:15.881299+0000","last_became_active":"2023-10-09T05:18:26.419673+0000","last_became_peered":"2023-10-09T05:18:26.419673+0000","last_unstale":"2023-10-09T05:36:15.881299+0000","last_undegraded":"2023-10-09T05:36:15.881299+0000","last_fullsized":"2023-10-09T05:36:15.881299+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:38.876021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":329,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.882705+0000","last_change":"2023-10-09T05:32:45.844532+0000","last_active":"2023-10-09T05:36:15.882705+0000","last_peered":"2023-10-09T05:36:15.882705+0000","last_clean":"2023-10-09T05:36:15.882705+0000","last_became_active":"2023-10-09T05:32:45.831101+0000","last_became_peered":"2023-10-09T05:32:45.831101+0000","last_unstale":"2023-10-09T05:36:15.882705+0000","last_undegraded":"2023-10-09T05:36:15.882705+0000","last_fullsized":"2023-10-09T05:36:15.882705+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:19:37.907188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":10,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-10-09T05:36:16.901739+0000","last_change":"2023-10-09T05:36:16.901739+0000","last_active":"2023-10-09T05:36:16.901739+0000","last_peered":"2023-10-09T05:36:16.901739+0000","last_clean":"2023-10-09T05:36:16.901739+0000","last_became_active":"2023-10-09T05:36:16.901380+0000","last_became_peered":"2023-10-09T05:36:16.901380+0000","last_unstale":"2023-10-09T05:36:16.901739+0000","last_undegraded":"2023-10-09T05:36:16.901739+0000","last_fullsized":"2023-10-09T05:36:16.901739+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:26.454798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.2","version":"316'1815","reported_seq":1828,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-10-09T05:36:24.795058+0000","last_change":"2023-10-09T05:35:46.307255+0000","last_active":"2023-10-09T05:36:24.795058+0000","last_peered":"2023-10-09T05:36:24.795058+0000","last_clean":"2023-10-09T05:36:24.795058+0000","last_became_active":"2023-10-09T05:35:46.306927+0000","last_became_peered":"2023-10-09T05:35:46.306927+0000","last_unstale":"2023-10-09T05:36:24.795058+0000","last_undegraded":"2023-10-09T05:36:24.795058+0000","last_fullsized":"2023-10-09T05:36:24.795058+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":1815,"log_dups_size":0,"ondisk_log_size":1815,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:16.113713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118947840,"num_objects":1815,"num_object_clones":0,"num_object_copies":5445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1815,"num_write_kb":116160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":123,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.884647+0000","last_change":"2023-10-09T05:30:03.409982+0000","last_active":"2023-10-09T05:36:15.884647+0000","last_peered":"2023-10-09T05:36:15.884647+0000","last_clean":"2023-10-09T05:36:15.884647+0000","last_became_active":"2023-10-09T05:30:03.386761+0000","last_became_peered":"2023-10-09T05:30:03.386761+0000","last_unstale":"2023-10-09T05:36:15.884647+0000","last_undegraded":"2023-10-09T05:36:15.884647+0000","last_fullsized":"2023-10-09T05:36:15.884647+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:17.372036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":25,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.880930+0000","last_change":"2023-10-09T05:32:45.829063+0000","last_active":"2023-10-09T05:36:15.880930+0000","last_peered":"2023-10-09T05:36:15.880930+0000","last_clean":"2023-10-09T05:36:15.880930+0000","last_became_active":"2023-10-09T05:32:45.828694+0000","last_became_peered":"2023-10-09T05:32:45.828694+0000","last_unstale":"2023-10-09T05:36:15.880930+0000","last_undegraded":"2023-10-09T05:36:15.880930+0000","last_fullsized":"2023-10-09T05:36:15.880930+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:58:51.140748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"316'1746","reported_seq":1760,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-10-09T05:36:24.927278+0000","last_change":"2023-10-09T05:35:46.303310+0000","last_active":"2023-10-09T05:36:24.927278+0000","last_peered":"2023-10-09T05:36:24.927278+0000","last_clean":"2023-10-09T05:36:24.927278+0000","last_became_active":"2023-10-09T05:35:46.302834+0000","last_became_peered":"2023-10-09T05:35:46.302834+0000","last_unstale":"2023-10-09T05:36:24.927278+0000","last_undegraded":"2023-10-09T05:36:24.927278+0000","last_fullsized":"2023-10-09T05:36:24.927278+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":1746,"log_dups_size":0,"ondisk_log_size":1746,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:44:58.231473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114425856,"num_objects":1746,"num_object_clones":0,"num_object_copies":5238,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1746,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1746,"num_write_kb":111744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":123,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.884420+0000","last_change":"2023-10-09T05:30:03.416268+0000","last_active":"2023-10-09T05:36:15.884420+0000","last_peered":"2023-10-09T05:36:15.884420+0000","last_clean":"2023-10-09T05:36:15.884420+0000","last_became_active":"2023-10-09T05:30:03.392236+0000","last_became_peered":"2023-10-09T05:30:03.392236+0000","last_unstale":"2023-10-09T05:36:15.884420+0000","last_undegraded":"2023-10-09T05:36:15.884420+0000","last_fullsized":"2023-10-09T05:36:15.884420+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:18:34.616445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":182,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.876037+0000","last_change":"2023-10-09T05:30:03.376778+0000","last_active":"2023-10-09T05:36:15.876037+0000","last_peered":"2023-10-09T05:36:15.876037+0000","last_clean":"2023-10-09T05:36:15.876037+0000","last_became_active":"2023-10-09T05:30:03.376056+0000","last_became_peered":"2023-10-09T05:30:03.376056+0000","last_unstale":"2023-10-09T05:36:15.876037+0000","last_undegraded":"2023-10-09T05:36:15.876037+0000","last_fullsized":"2023-10-09T05:36:15.876037+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:06.957639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.0","version":"316'1896","reported_seq":1910,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-10-09T05:36:26.586197+0000","last_change":"2023-10-09T05:35:46.304412+0000","last_active":"2023-10-09T05:36:26.586197+0000","last_peered":"2023-10-09T05:36:26.586197+0000","last_clean":"2023-10-09T05:36:26.586197+0000","last_became_active":"2023-10-09T05:35:46.304057+0000","last_became_peered":"2023-10-09T05:35:46.304057+0000","last_unstale":"2023-10-09T05:36:26.586197+0000","last_undegraded":"2023-10-09T05:36:26.586197+0000","last_fullsized":"2023-10-09T05:36:26.586197+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":1896,"log_dups_size":0,"ondisk_log_size":1896,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:59.756942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124256256,"num_objects":1896,"num_object_clones":0,"num_object_copies":5688,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1896,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1896,"num_write_kb":121344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":123,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.884239+0000","last_change":"2023-10-09T05:32:45.838786+0000","last_active":"2023-10-09T05:36:15.884239+0000","last_peered":"2023-10-09T05:36:15.884239+0000","last_clean":"2023-10-09T05:36:15.884239+0000","last_became_active":"2023-10-09T05:32:45.830293+0000","last_became_peered":"2023-10-09T05:32:45.830293+0000","last_unstale":"2023-10-09T05:36:15.884239+0000","last_undegraded":"2023-10-09T05:36:15.884239+0000","last_fullsized":"2023-10-09T05:36:15.884239+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:47.319929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":346,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.883204+0000","last_change":"2023-10-09T05:18:26.423526+0000","last_active":"2023-10-09T05:36:15.883204+0000","last_peered":"2023-10-09T05:36:15.883204+0000","last_clean":"2023-10-09T05:36:15.883204+0000","last_became_active":"2023-10-09T05:18:26.423092+0000","last_became_peered":"2023-10-09T05:18:26.423092+0000","last_unstale":"2023-10-09T05:36:15.883204+0000","last_undegraded":"2023-10-09T05:36:15.883204+0000","last_fullsized":"2023-10-09T05:36:15.883204+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:54.877664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"316'1877","reported_seq":1891,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-10-09T05:36:25.600746+0000","last_change":"2023-10-09T05:35:46.301799+0000","last_active":"2023-10-09T05:36:25.600746+0000","last_peered":"2023-10-09T05:36:25.600746+0000","last_clean":"2023-10-09T05:36:25.600746+0000","last_became_active":"2023-10-09T05:35:46.301216+0000","last_became_peered":"2023-10-09T05:35:46.301216+0000","last_unstale":"2023-10-09T05:36:25.600746+0000","last_undegraded":"2023-10-09T05:36:25.600746+0000","last_fullsized":"2023-10-09T05:36:25.600746+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":1877,"log_dups_size":0,"ondisk_log_size":1877,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:02:04.025011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123011072,"num_objects":1877,"num_object_clones":0,"num_object_copies":5631,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1877,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1877,"num_write_kb":120128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":25,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.881381+0000","last_change":"2023-10-09T05:32:45.829450+0000","last_active":"2023-10-09T05:36:15.881381+0000","last_peered":"2023-10-09T05:36:15.881381+0000","last_clean":"2023-10-09T05:36:15.881381+0000","last_became_active":"2023-10-09T05:32:45.829076+0000","last_became_peered":"2023-10-09T05:32:45.829076+0000","last_unstale":"2023-10-09T05:36:15.881381+0000","last_undegraded":"2023-10-09T05:36:15.881381+0000","last_fullsized":"2023-10-09T05:36:15.881381+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:51:13.538770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":309,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.880789+0000","last_change":"2023-10-09T05:30:03.387239+0000","last_active":"2023-10-09T05:36:15.880789+0000","last_peered":"2023-10-09T05:36:15.880789+0000","last_clean":"2023-10-09T05:36:15.880789+0000","last_became_active":"2023-10-09T05:30:03.386633+0000","last_became_peered":"2023-10-09T05:30:03.386633+0000","last_unstale":"2023-10-09T05:36:15.880789+0000","last_undegraded":"2023-10-09T05:36:15.880789+0000","last_fullsized":"2023-10-09T05:36:15.880789+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:51.537133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.6","version":"316'1885","reported_seq":1899,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-10-09T05:36:24.935823+0000","last_change":"2023-10-09T05:35:46.308285+0000","last_active":"2023-10-09T05:36:24.935823+0000","last_peered":"2023-10-09T05:36:24.935823+0000","last_clean":"2023-10-09T05:36:24.935823+0000","last_became_active":"2023-10-09T05:35:46.307966+0000","last_became_peered":"2023-10-09T05:35:46.307966+0000","last_unstale":"2023-10-09T05:36:24.935823+0000","last_undegraded":"2023-10-09T05:36:24.935823+0000","last_fullsized":"2023-10-09T05:36:24.935823+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":1885,"log_dups_size":0,"ondisk_log_size":1885,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:09:40.433853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123535360,"num_objects":1885,"num_object_clones":0,"num_object_copies":5655,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1885,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1885,"num_write_kb":120640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'3","reported_seq":287,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.879757+0000","last_change":"2023-10-09T05:30:03.399748+0000","last_active":"2023-10-09T05:36:15.879757+0000","last_peered":"2023-10-09T05:36:15.879757+0000","last_clean":"2023-10-09T05:36:15.879757+0000","last_became_active":"2023-10-09T05:30:03.384427+0000","last_became_peered":"2023-10-09T05:30:03.384427+0000","last_unstale":"2023-10-09T05:36:15.879757+0000","last_undegraded":"2023-10-09T05:36:15.879757+0000","last_fullsized":"2023-10-09T05:36:15.879757+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:01.128892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":117,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.881772+0000","last_change":"2023-10-09T05:30:03.378783+0000","last_active":"2023-10-09T05:36:15.881772+0000","last_peered":"2023-10-09T05:36:15.881772+0000","last_clean":"2023-10-09T05:36:15.881772+0000","last_became_active":"2023-10-09T05:30:03.378436+0000","last_became_peered":"2023-10-09T05:30:03.378436+0000","last_unstale":"2023-10-09T05:36:15.881772+0000","last_undegraded":"2023-10-09T05:36:15.881772+0000","last_fullsized":"2023-10-09T05:36:15.881772+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:54.102857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.7","version":"316'1737","reported_seq":1750,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-10-09T05:36:24.896355+0000","last_change":"2023-10-09T05:35:46.307283+0000","last_active":"2023-10-09T05:36:24.896355+0000","last_peered":"2023-10-09T05:36:24.896355+0000","last_clean":"2023-10-09T05:36:24.896355+0000","last_became_active":"2023-10-09T05:35:46.306960+0000","last_became_peered":"2023-10-09T05:35:46.306960+0000","last_unstale":"2023-10-09T05:36:24.896355+0000","last_undegraded":"2023-10-09T05:36:24.896355+0000","last_fullsized":"2023-10-09T05:36:24.896355+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":1737,"log_dups_size":0,"ondisk_log_size":1737,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:04:27.884680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113836032,"num_objects":1737,"num_object_clones":0,"num_object_copies":5211,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1737,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1737,"num_write_kb":111168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":327,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.883652+0000","last_change":"2023-10-09T05:30:03.381940+0000","last_active":"2023-10-09T05:36:15.883652+0000","last_peered":"2023-10-09T05:36:15.883652+0000","last_clean":"2023-10-09T05:36:15.883652+0000","last_became_active":"2023-10-09T05:30:03.381526+0000","last_became_peered":"2023-10-09T05:30:03.381526+0000","last_unstale":"2023-10-09T05:36:15.883652+0000","last_undegraded":"2023-10-09T05:36:15.883652+0000","last_fullsized":"2023-10-09T05:36:15.883652+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:06.078801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":285,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.874047+0000","last_change":"2023-10-09T05:30:03.387924+0000","last_active":"2023-10-09T05:36:15.874047+0000","last_peered":"2023-10-09T05:36:15.874047+0000","last_clean":"2023-10-09T05:36:15.874047+0000","last_became_active":"2023-10-09T05:30:03.387493+0000","last_became_peered":"2023-10-09T05:30:03.387493+0000","last_unstale":"2023-10-09T05:36:15.874047+0000","last_undegraded":"2023-10-09T05:36:15.874047+0000","last_fullsized":"2023-10-09T05:36:15.874047+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:16.554514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.4","version":"316'1753","reported_seq":1766,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-10-09T05:36:24.795175+0000","last_change":"2023-10-09T05:35:46.305925+0000","last_active":"2023-10-09T05:36:24.795175+0000","last_peered":"2023-10-09T05:36:24.795175+0000","last_clean":"2023-10-09T05:36:24.795175+0000","last_became_active":"2023-10-09T05:35:46.305593+0000","last_became_peered":"2023-10-09T05:35:46.305593+0000","last_unstale":"2023-10-09T05:36:24.795175+0000","last_undegraded":"2023-10-09T05:36:24.795175+0000","last_fullsized":"2023-10-09T05:36:24.795175+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":1753,"log_dups_size":0,"ondisk_log_size":1753,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:14.046917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114884608,"num_objects":1753,"num_object_clones":0,"num_object_copies":5259,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1753,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1753,"num_write_kb":112192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":254,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.874981+0000","last_change":"2023-10-09T05:30:03.372784+0000","last_active":"2023-10-09T05:36:15.874981+0000","last_peered":"2023-10-09T05:36:15.874981+0000","last_clean":"2023-10-09T05:36:15.874981+0000","last_became_active":"2023-10-09T05:30:03.372458+0000","last_became_peered":"2023-10-09T05:30:03.372458+0000","last_unstale":"2023-10-09T05:36:15.874981+0000","last_undegraded":"2023-10-09T05:36:15.874981+0000","last_fullsized":"2023-10-09T05:36:15.874981+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:36:24.881545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":285,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.884935+0000","last_change":"2023-10-09T05:30:03.382201+0000","last_active":"2023-10-09T05:36:15.884935+0000","last_peered":"2023-10-09T05:36:15.884935+0000","last_clean":"2023-10-09T05:36:15.884935+0000","last_became_active":"2023-10-09T05:30:03.381727+0000","last_became_peered":"2023-10-09T05:30:03.381727+0000","last_unstale":"2023-10-09T05:36:15.884935+0000","last_undegraded":"2023-10-09T05:36:15.884935+0000","last_fullsized":"2023-10-09T05:36:15.884935+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:14.559280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.5","version":"316'1800","reported_seq":1814,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-10-09T05:36:24.892719+0000","last_change":"2023-10-09T05:35:46.305495+0000","last_active":"2023-10-09T05:36:24.892719+0000","last_peered":"2023-10-09T05:36:24.892719+0000","last_clean":"2023-10-09T05:36:24.892719+0000","last_became_active":"2023-10-09T05:35:46.305029+0000","last_became_peered":"2023-10-09T05:35:46.305029+0000","last_unstale":"2023-10-09T05:36:24.892719+0000","last_undegraded":"2023-10-09T05:36:24.892719+0000","last_fullsized":"2023-10-09T05:36:24.892719+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":1800,"log_dups_size":0,"ondisk_log_size":1800,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:08.629352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117964800,"num_objects":1800,"num_object_clones":0,"num_object_copies":5400,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1800,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1800,"num_write_kb":115200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":332,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.878172+0000","last_change":"2023-10-09T05:18:26.413106+0000","last_active":"2023-10-09T05:36:15.878172+0000","last_peered":"2023-10-09T05:36:15.878172+0000","last_clean":"2023-10-09T05:36:15.878172+0000","last_became_active":"2023-10-09T05:18:26.412334+0000","last_became_peered":"2023-10-09T05:18:26.412334+0000","last_unstale":"2023-10-09T05:36:15.878172+0000","last_undegraded":"2023-10-09T05:36:15.878172+0000","last_fullsized":"2023-10-09T05:36:15.878172+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:18.058016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":195,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.876160+0000","last_change":"2023-10-09T05:30:03.383456+0000","last_active":"2023-10-09T05:36:15.876160+0000","last_peered":"2023-10-09T05:36:15.876160+0000","last_clean":"2023-10-09T05:36:15.876160+0000","last_became_active":"2023-10-09T05:30:03.382656+0000","last_became_peered":"2023-10-09T05:30:03.382656+0000","last_unstale":"2023-10-09T05:36:15.876160+0000","last_undegraded":"2023-10-09T05:36:15.876160+0000","last_fullsized":"2023-10-09T05:36:15.876160+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:47:23.383409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.8","version":"316'1826","reported_seq":1840,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-10-09T05:36:24.916326+0000","last_change":"2023-10-09T05:35:46.307140+0000","last_active":"2023-10-09T05:36:24.916326+0000","last_peered":"2023-10-09T05:36:24.916326+0000","last_clean":"2023-10-09T05:36:24.916326+0000","last_became_active":"2023-10-09T05:35:46.304868+0000","last_became_peered":"2023-10-09T05:35:46.304868+0000","last_unstale":"2023-10-09T05:36:24.916326+0000","last_undegraded":"2023-10-09T05:36:24.916326+0000","last_fullsized":"2023-10-09T05:36:24.916326+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":1826,"log_dups_size":0,"ondisk_log_size":1826,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:51.320374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119668736,"num_objects":1826,"num_object_clones":0,"num_object_copies":5478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1826,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1826,"num_write_kb":116864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":223,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.876709+0000","last_change":"2023-10-09T05:18:26.422150+0000","last_active":"2023-10-09T05:36:15.876709+0000","last_peered":"2023-10-09T05:36:15.876709+0000","last_clean":"2023-10-09T05:36:15.876709+0000","last_became_active":"2023-10-09T05:18:26.421853+0000","last_became_peered":"2023-10-09T05:18:26.421853+0000","last_unstale":"2023-10-09T05:36:15.876709+0000","last_undegraded":"2023-10-09T05:36:15.876709+0000","last_fullsized":"2023-10-09T05:36:15.876709+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:37:47.000556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":25,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.881627+0000","last_change":"2023-10-09T05:32:45.832271+0000","last_active":"2023-10-09T05:36:15.881627+0000","last_peered":"2023-10-09T05:36:15.881627+0000","last_clean":"2023-10-09T05:36:15.881627+0000","last_became_active":"2023-10-09T05:32:45.831997+0000","last_became_peered":"2023-10-09T05:32:45.831997+0000","last_unstale":"2023-10-09T05:36:15.881627+0000","last_undegraded":"2023-10-09T05:36:15.881627+0000","last_fullsized":"2023-10-09T05:36:15.881627+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:18.970416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":98,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.877383+0000","last_change":"2023-10-09T05:32:45.819022+0000","last_active":"2023-10-09T05:36:15.877383+0000","last_peered":"2023-10-09T05:36:15.877383+0000","last_clean":"2023-10-09T05:36:15.877383+0000","last_became_active":"2023-10-09T05:32:45.818726+0000","last_became_peered":"2023-10-09T05:32:45.818726+0000","last_unstale":"2023-10-09T05:36:15.877383+0000","last_undegraded":"2023-10-09T05:36:15.877383+0000","last_fullsized":"2023-10-09T05:36:15.877383+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:04.414263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.e","version":"316'2025","reported_seq":2039,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-10-09T05:36:26.563963+0000","last_change":"2023-10-09T05:35:46.303342+0000","last_active":"2023-10-09T05:36:26.563963+0000","last_peered":"2023-10-09T05:36:26.563963+0000","last_clean":"2023-10-09T05:36:26.563963+0000","last_became_active":"2023-10-09T05:35:46.302540+0000","last_became_peered":"2023-10-09T05:35:46.302540+0000","last_unstale":"2023-10-09T05:36:26.563963+0000","last_undegraded":"2023-10-09T05:36:26.563963+0000","last_fullsized":"2023-10-09T05:36:26.563963+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":2025,"log_dups_size":0,"ondisk_log_size":2025,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:00:27.626139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132710400,"num_objects":2025,"num_object_clones":0,"num_object_copies":6075,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2025,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2025,"num_write_kb":129600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":25,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.881084+0000","last_change":"2023-10-09T05:32:45.825342+0000","last_active":"2023-10-09T05:36:15.881084+0000","last_peered":"2023-10-09T05:36:15.881084+0000","last_clean":"2023-10-09T05:36:15.881084+0000","last_became_active":"2023-10-09T05:32:45.824981+0000","last_became_peered":"2023-10-09T05:32:45.824981+0000","last_unstale":"2023-10-09T05:36:15.881084+0000","last_undegraded":"2023-10-09T05:36:15.881084+0000","last_fullsized":"2023-10-09T05:36:15.881084+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:02.868005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":84,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-10-09T05:36:16.900362+0000","last_change":"2023-10-09T05:30:03.381036+0000","last_active":"2023-10-09T05:36:16.900362+0000","last_peered":"2023-10-09T05:36:16.900362+0000","last_clean":"2023-10-09T05:36:16.900362+0000","last_became_active":"2023-10-09T05:30:03.380698+0000","last_became_peered":"2023-10-09T05:30:03.380698+0000","last_unstale":"2023-10-09T05:36:16.900362+0000","last_undegraded":"2023-10-09T05:36:16.900362+0000","last_fullsized":"2023-10-09T05:36:16.900362+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:16.464471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":80,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.876110+0000","last_change":"2023-10-09T05:23:18.196260+0000","last_active":"2023-10-09T05:36:15.876110+0000","last_peered":"2023-10-09T05:36:15.876110+0000","last_clean":"2023-10-09T05:36:15.876110+0000","last_became_active":"2023-10-09T05:23:18.195907+0000","last_became_peered":"2023-10-09T05:23:18.195907+0000","last_unstale":"2023-10-09T05:36:15.876110+0000","last_undegraded":"2023-10-09T05:36:15.876110+0000","last_fullsized":"2023-10-09T05:36:15.876110+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:36.243432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.d","version":"316'1812","reported_seq":1825,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-10-09T05:36:24.791197+0000","last_change":"2023-10-09T05:35:46.303676+0000","last_active":"2023-10-09T05:36:24.791197+0000","last_peered":"2023-10-09T05:36:24.791197+0000","last_clean":"2023-10-09T05:36:24.791197+0000","last_became_active":"2023-10-09T05:35:46.303364+0000","last_became_peered":"2023-10-09T05:35:46.303364+0000","last_unstale":"2023-10-09T05:36:24.791197+0000","last_undegraded":"2023-10-09T05:36:24.791197+0000","last_fullsized":"2023-10-09T05:36:24.791197+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":1812,"log_dups_size":0,"ondisk_log_size":1812,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:56.816946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118751232,"num_objects":1812,"num_object_clones":0,"num_object_copies":5436,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1812,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1812,"num_write_kb":115968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":97,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.874885+0000","last_change":"2023-10-09T05:20:18.123706+0000","last_active":"2023-10-09T05:36:15.874885+0000","last_peered":"2023-10-09T05:36:15.874885+0000","last_clean":"2023-10-09T05:36:15.874885+0000","last_became_active":"2023-10-09T05:20:18.122859+0000","last_became_peered":"2023-10-09T05:20:18.122859+0000","last_unstale":"2023-10-09T05:36:15.874885+0000","last_undegraded":"2023-10-09T05:36:15.874885+0000","last_fullsized":"2023-10-09T05:36:15.874885+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.906228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":188,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.876240+0000","last_change":"2023-10-09T05:09:57.854064+0000","last_active":"2023-10-09T05:36:15.876240+0000","last_peered":"2023-10-09T05:36:15.876240+0000","last_clean":"2023-10-09T05:36:15.876240+0000","last_became_active":"2023-10-09T05:09:57.847953+0000","last_became_peered":"2023-10-09T05:09:57.847953+0000","last_unstale":"2023-10-09T05:36:15.876240+0000","last_undegraded":"2023-10-09T05:36:15.876240+0000","last_fullsized":"2023-10-09T05:36:15.876240+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:45.914976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":71,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.876796+0000","last_change":"2023-10-09T05:32:45.832106+0000","last_active":"2023-10-09T05:36:15.876796+0000","last_peered":"2023-10-09T05:36:15.876796+0000","last_clean":"2023-10-09T05:36:15.876796+0000","last_became_active":"2023-10-09T05:32:45.831763+0000","last_became_peered":"2023-10-09T05:32:45.831763+0000","last_unstale":"2023-10-09T05:36:15.876796+0000","last_undegraded":"2023-10-09T05:36:15.876796+0000","last_fullsized":"2023-10-09T05:36:15.876796+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:06:37.005113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.f","version":"316'1779","reported_seq":1792,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-10-09T05:36:24.774320+0000","last_change":"2023-10-09T05:35:46.309035+0000","last_active":"2023-10-09T05:36:24.774320+0000","last_peered":"2023-10-09T05:36:24.774320+0000","last_clean":"2023-10-09T05:36:24.774320+0000","last_became_active":"2023-10-09T05:35:46.308730+0000","last_became_peered":"2023-10-09T05:35:46.308730+0000","last_unstale":"2023-10-09T05:36:24.774320+0000","last_undegraded":"2023-10-09T05:36:24.774320+0000","last_fullsized":"2023-10-09T05:36:24.774320+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":1779,"log_dups_size":0,"ondisk_log_size":1779,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:08.994300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116588544,"num_objects":1779,"num_object_clones":0,"num_object_copies":5337,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1779,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1779,"num_write_kb":113856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":25,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.881060+0000","last_change":"2023-10-09T05:32:45.825075+0000","last_active":"2023-10-09T05:36:15.881060+0000","last_peered":"2023-10-09T05:36:15.881060+0000","last_clean":"2023-10-09T05:36:15.881060+0000","last_became_active":"2023-10-09T05:32:45.824730+0000","last_became_peered":"2023-10-09T05:32:45.824730+0000","last_unstale":"2023-10-09T05:36:15.881060+0000","last_undegraded":"2023-10-09T05:36:15.881060+0000","last_fullsized":"2023-10-09T05:36:15.881060+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:33.484326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":601,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.883439+0000","last_change":"2023-10-09T05:18:29.419527+0000","last_active":"2023-10-09T05:36:15.883439+0000","last_peered":"2023-10-09T05:36:15.883439+0000","last_clean":"2023-10-09T05:36:15.883439+0000","last_became_active":"2023-10-09T05:18:29.418852+0000","last_became_peered":"2023-10-09T05:18:29.418852+0000","last_unstale":"2023-10-09T05:36:15.883439+0000","last_undegraded":"2023-10-09T05:36:15.883439+0000","last_fullsized":"2023-10-09T05:36:15.883439+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_clean_scrub_stamp":"2023-10-09T04:36:44.261013+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:02.899170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":300,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.879780+0000","last_change":"2023-10-09T05:32:45.829840+0000","last_active":"2023-10-09T05:36:15.879780+0000","last_peered":"2023-10-09T05:36:15.879780+0000","last_clean":"2023-10-09T05:36:15.879780+0000","last_became_active":"2023-10-09T05:32:45.829084+0000","last_became_peered":"2023-10-09T05:32:45.829084+0000","last_unstale":"2023-10-09T05:36:15.879780+0000","last_undegraded":"2023-10-09T05:36:15.879780+0000","last_fullsized":"2023-10-09T05:36:15.879780+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:26:48.836506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":309,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.884460+0000","last_change":"2023-10-09T05:30:03.386395+0000","last_active":"2023-10-09T05:36:15.884460+0000","last_peered":"2023-10-09T05:36:15.884460+0000","last_clean":"2023-10-09T05:36:15.884460+0000","last_became_active":"2023-10-09T05:30:03.386024+0000","last_became_peered":"2023-10-09T05:30:03.386024+0000","last_unstale":"2023-10-09T05:36:15.884460+0000","last_undegraded":"2023-10-09T05:36:15.884460+0000","last_fullsized":"2023-10-09T05:36:15.884460+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:13.133829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"316'1795","reported_seq":1809,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-10-09T05:36:24.934490+0000","last_change":"2023-10-09T05:35:46.308831+0000","last_active":"2023-10-09T05:36:24.934490+0000","last_peered":"2023-10-09T05:36:24.934490+0000","last_clean":"2023-10-09T05:36:24.934490+0000","last_became_active":"2023-10-09T05:35:46.308458+0000","last_became_peered":"2023-10-09T05:35:46.308458+0000","last_unstale":"2023-10-09T05:36:24.934490+0000","last_undegraded":"2023-10-09T05:36:24.934490+0000","last_fullsized":"2023-10-09T05:36:24.934490+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":1795,"log_dups_size":0,"ondisk_log_size":1795,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:30.298840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117637120,"num_objects":1795,"num_object_clones":0,"num_object_copies":5385,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1795,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1795,"num_write_kb":114880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":125,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.883519+0000","last_change":"2023-10-09T05:23:18.193825+0000","last_active":"2023-10-09T05:36:15.883519+0000","last_peered":"2023-10-09T05:36:15.883519+0000","last_clean":"2023-10-09T05:36:15.883519+0000","last_became_active":"2023-10-09T05:23:18.193394+0000","last_became_peered":"2023-10-09T05:23:18.193394+0000","last_unstale":"2023-10-09T05:36:15.883519+0000","last_undegraded":"2023-10-09T05:36:15.883519+0000","last_fullsized":"2023-10-09T05:36:15.883519+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:40.594469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":300,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.880696+0000","last_change":"2023-10-09T05:32:45.824031+0000","last_active":"2023-10-09T05:36:15.880696+0000","last_peered":"2023-10-09T05:36:15.880696+0000","last_clean":"2023-10-09T05:36:15.880696+0000","last_became_active":"2023-10-09T05:32:45.823651+0000","last_became_peered":"2023-10-09T05:32:45.823651+0000","last_unstale":"2023-10-09T05:36:15.880696+0000","last_undegraded":"2023-10-09T05:36:15.880696+0000","last_fullsized":"2023-10-09T05:36:15.880696+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:14:20.745117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":361,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.881223+0000","last_change":"2023-10-09T05:20:18.132750+0000","last_active":"2023-10-09T05:36:15.881223+0000","last_peered":"2023-10-09T05:36:15.881223+0000","last_clean":"2023-10-09T05:36:15.881223+0000","last_became_active":"2023-10-09T05:20:18.132396+0000","last_became_peered":"2023-10-09T05:20:18.132396+0000","last_unstale":"2023-10-09T05:36:15.881223+0000","last_undegraded":"2023-10-09T05:36:15.881223+0000","last_fullsized":"2023-10-09T05:36:15.881223+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:00:55.731045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.9","version":"316'1794","reported_seq":1807,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-10-09T05:36:24.903406+0000","last_change":"2023-10-09T05:35:46.308006+0000","last_active":"2023-10-09T05:36:24.903406+0000","last_peered":"2023-10-09T05:36:24.903406+0000","last_clean":"2023-10-09T05:36:24.903406+0000","last_became_active":"2023-10-09T05:35:46.307713+0000","last_became_peered":"2023-10-09T05:35:46.307713+0000","last_unstale":"2023-10-09T05:36:24.903406+0000","last_undegraded":"2023-10-09T05:36:24.903406+0000","last_fullsized":"2023-10-09T05:36:24.903406+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":1794,"log_dups_size":0,"ondisk_log_size":1794,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:36:40.593672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117571584,"num_objects":1794,"num_object_clones":0,"num_object_copies":5382,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1794,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1794,"num_write_kb":114816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":115,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.884326+0000","last_change":"2023-10-09T05:30:03.391750+0000","last_active":"2023-10-09T05:36:15.884326+0000","last_peered":"2023-10-09T05:36:15.884326+0000","last_clean":"2023-10-09T05:36:15.884326+0000","last_became_active":"2023-10-09T05:30:03.391393+0000","last_became_peered":"2023-10-09T05:30:03.391393+0000","last_unstale":"2023-10-09T05:36:15.884326+0000","last_undegraded":"2023-10-09T05:36:15.884326+0000","last_fullsized":"2023-10-09T05:36:15.884326+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:07.263598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":223,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.876832+0000","last_change":"2023-10-09T05:32:45.824227+0000","last_active":"2023-10-09T05:36:15.876832+0000","last_peered":"2023-10-09T05:36:15.876832+0000","last_clean":"2023-10-09T05:36:15.876832+0000","last_became_active":"2023-10-09T05:32:45.823662+0000","last_became_peered":"2023-10-09T05:32:45.823662+0000","last_unstale":"2023-10-09T05:36:15.876832+0000","last_undegraded":"2023-10-09T05:36:15.876832+0000","last_fullsized":"2023-10-09T05:36:15.876832+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:54:56.029126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":331,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.881030+0000","last_change":"2023-10-09T05:32:45.826815+0000","last_active":"2023-10-09T05:36:15.881030+0000","last_peered":"2023-10-09T05:36:15.881030+0000","last_clean":"2023-10-09T05:36:15.881030+0000","last_became_active":"2023-10-09T05:32:45.826529+0000","last_became_peered":"2023-10-09T05:32:45.826529+0000","last_unstale":"2023-10-09T05:36:15.881030+0000","last_undegraded":"2023-10-09T05:36:15.881030+0000","last_fullsized":"2023-10-09T05:36:15.881030+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:19.868533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.a","version":"316'1777","reported_seq":1790,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-10-09T05:36:24.899006+0000","last_change":"2023-10-09T05:35:46.307929+0000","last_active":"2023-10-09T05:36:24.899006+0000","last_peered":"2023-10-09T05:36:24.899006+0000","last_clean":"2023-10-09T05:36:24.899006+0000","last_became_active":"2023-10-09T05:35:46.307610+0000","last_became_peered":"2023-10-09T05:35:46.307610+0000","last_unstale":"2023-10-09T05:36:24.899006+0000","last_undegraded":"2023-10-09T05:36:24.899006+0000","last_fullsized":"2023-10-09T05:36:24.899006+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":1777,"log_dups_size":0,"ondisk_log_size":1777,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:40.352492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116457472,"num_objects":1777,"num_object_clones":0,"num_object_copies":5331,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1777,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1777,"num_write_kb":113728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":325,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.878972+0000","last_change":"2023-10-09T05:18:26.431563+0000","last_active":"2023-10-09T05:36:15.878972+0000","last_peered":"2023-10-09T05:36:15.878972+0000","last_clean":"2023-10-09T05:36:15.878972+0000","last_became_active":"2023-10-09T05:18:26.421025+0000","last_became_peered":"2023-10-09T05:18:26.421025+0000","last_unstale":"2023-10-09T05:36:15.878972+0000","last_undegraded":"2023-10-09T05:36:15.878972+0000","last_fullsized":"2023-10-09T05:36:15.878972+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:30.333585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":89,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-10-09T05:36:16.890510+0000","last_change":"2023-10-09T05:30:03.375221+0000","last_active":"2023-10-09T05:36:16.890510+0000","last_peered":"2023-10-09T05:36:16.890510+0000","last_clean":"2023-10-09T05:36:16.890510+0000","last_became_active":"2023-10-09T05:30:03.374863+0000","last_became_peered":"2023-10-09T05:30:03.374863+0000","last_unstale":"2023-10-09T05:36:16.890510+0000","last_undegraded":"2023-10-09T05:36:16.890510+0000","last_fullsized":"2023-10-09T05:36:16.890510+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:16.058179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":325,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.878251+0000","last_change":"2023-10-09T05:21:08.179657+0000","last_active":"2023-10-09T05:36:15.878251+0000","last_peered":"2023-10-09T05:36:15.878251+0000","last_clean":"2023-10-09T05:36:15.878251+0000","last_became_active":"2023-10-09T05:21:08.178958+0000","last_became_peered":"2023-10-09T05:21:08.178958+0000","last_unstale":"2023-10-09T05:36:15.878251+0000","last_undegraded":"2023-10-09T05:36:15.878251+0000","last_fullsized":"2023-10-09T05:36:15.878251+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:20.815986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"316'1804","reported_seq":1818,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-10-09T05:36:24.932085+0000","last_change":"2023-10-09T05:35:46.308213+0000","last_active":"2023-10-09T05:36:24.932085+0000","last_peered":"2023-10-09T05:36:24.932085+0000","last_clean":"2023-10-09T05:36:24.932085+0000","last_became_active":"2023-10-09T05:35:46.307896+0000","last_became_peered":"2023-10-09T05:35:46.307896+0000","last_unstale":"2023-10-09T05:36:24.932085+0000","last_undegraded":"2023-10-09T05:36:24.932085+0000","last_fullsized":"2023-10-09T05:36:24.932085+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":1804,"log_dups_size":0,"ondisk_log_size":1804,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:11.538673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118226944,"num_objects":1804,"num_object_clones":0,"num_object_copies":5412,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1804,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1804,"num_write_kb":115456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":321,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.884206+0000","last_change":"2023-10-09T05:18:26.424429+0000","last_active":"2023-10-09T05:36:15.884206+0000","last_peered":"2023-10-09T05:36:15.884206+0000","last_clean":"2023-10-09T05:36:15.884206+0000","last_became_active":"2023-10-09T05:18:26.423034+0000","last_became_peered":"2023-10-09T05:18:26.423034+0000","last_unstale":"2023-10-09T05:36:15.884206+0000","last_undegraded":"2023-10-09T05:36:15.884206+0000","last_fullsized":"2023-10-09T05:36:15.884206+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:16.251181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":37,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.879509+0000","last_change":"2023-10-09T05:31:16.444031+0000","last_active":"2023-10-09T05:36:15.879509+0000","last_peered":"2023-10-09T05:36:15.879509+0000","last_clean":"2023-10-09T05:36:15.879509+0000","last_became_active":"2023-10-09T05:31:16.412865+0000","last_became_peered":"2023-10-09T05:31:16.412865+0000","last_unstale":"2023-10-09T05:36:15.879509+0000","last_undegraded":"2023-10-09T05:36:15.879509+0000","last_fullsized":"2023-10-09T05:36:15.879509+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:04:06.520632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":25,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.881243+0000","last_change":"2023-10-09T05:32:45.828380+0000","last_active":"2023-10-09T05:36:15.881243+0000","last_peered":"2023-10-09T05:36:15.881243+0000","last_clean":"2023-10-09T05:36:15.881243+0000","last_became_active":"2023-10-09T05:32:45.827888+0000","last_became_peered":"2023-10-09T05:32:45.827888+0000","last_unstale":"2023-10-09T05:36:15.881243+0000","last_undegraded":"2023-10-09T05:36:15.881243+0000","last_fullsized":"2023-10-09T05:36:15.881243+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:48.145813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'2","reported_seq":99,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.880948+0000","last_change":"2023-10-09T05:30:03.386435+0000","last_active":"2023-10-09T05:36:15.880948+0000","last_peered":"2023-10-09T05:36:15.880948+0000","last_clean":"2023-10-09T05:36:15.880948+0000","last_became_active":"2023-10-09T05:30:03.385418+0000","last_became_peered":"2023-10-09T05:30:03.385418+0000","last_unstale":"2023-10-09T05:36:15.880948+0000","last_undegraded":"2023-10-09T05:36:15.880948+0000","last_fullsized":"2023-10-09T05:36:15.880948+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:34.685368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":105,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-10-09T05:36:16.898207+0000","last_change":"2023-10-09T05:36:16.898207+0000","last_active":"2023-10-09T05:36:16.898207+0000","last_peered":"2023-10-09T05:36:16.898207+0000","last_clean":"2023-10-09T05:36:16.898207+0000","last_became_active":"2023-10-09T05:36:16.898009+0000","last_became_peered":"2023-10-09T05:36:16.898009+0000","last_unstale":"2023-10-09T05:36:16.898207+0000","last_undegraded":"2023-10-09T05:36:16.898207+0000","last_fullsized":"2023-10-09T05:36:16.898207+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:59.694977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":29,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.881321+0000","last_change":"2023-10-09T05:32:45.847749+0000","last_active":"2023-10-09T05:36:15.881321+0000","last_peered":"2023-10-09T05:36:15.881321+0000","last_clean":"2023-10-09T05:36:15.881321+0000","last_became_active":"2023-10-09T05:32:45.825335+0000","last_became_peered":"2023-10-09T05:32:45.825335+0000","last_unstale":"2023-10-09T05:36:15.881321+0000","last_undegraded":"2023-10-09T05:36:15.881321+0000","last_fullsized":"2023-10-09T05:36:15.881321+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:42.082798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":303,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.879133+0000","last_change":"2023-10-09T05:30:03.380938+0000","last_active":"2023-10-09T05:36:15.879133+0000","last_peered":"2023-10-09T05:36:15.879133+0000","last_clean":"2023-10-09T05:36:15.879133+0000","last_became_active":"2023-10-09T05:30:03.380395+0000","last_became_peered":"2023-10-09T05:30:03.380395+0000","last_unstale":"2023-10-09T05:36:15.879133+0000","last_undegraded":"2023-10-09T05:36:15.879133+0000","last_fullsized":"2023-10-09T05:36:15.879133+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:23.087303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":29,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.881731+0000","last_change":"2023-10-09T05:32:45.860675+0000","last_active":"2023-10-09T05:36:15.881731+0000","last_peered":"2023-10-09T05:36:15.881731+0000","last_clean":"2023-10-09T05:36:15.881731+0000","last_became_active":"2023-10-09T05:32:45.828515+0000","last_became_peered":"2023-10-09T05:32:45.828515+0000","last_unstale":"2023-10-09T05:36:15.881731+0000","last_undegraded":"2023-10-09T05:36:15.881731+0000","last_fullsized":"2023-10-09T05:36:15.881731+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:39.371415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":134,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.879700+0000","last_change":"2023-10-09T05:30:03.376908+0000","last_active":"2023-10-09T05:36:15.879700+0000","last_peered":"2023-10-09T05:36:15.879700+0000","last_clean":"2023-10-09T05:36:15.879700+0000","last_became_active":"2023-10-09T05:30:03.376456+0000","last_became_peered":"2023-10-09T05:30:03.376456+0000","last_unstale":"2023-10-09T05:36:15.879700+0000","last_undegraded":"2023-10-09T05:36:15.879700+0000","last_fullsized":"2023-10-09T05:36:15.879700+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:23.173899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'2","reported_seq":319,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.878866+0000","last_change":"2023-10-09T05:22:47.197002+0000","last_active":"2023-10-09T05:36:15.878866+0000","last_peered":"2023-10-09T05:36:15.878866+0000","last_clean":"2023-10-09T05:36:15.878866+0000","last_became_active":"2023-10-09T05:22:47.196685+0000","last_became_peered":"2023-10-09T05:22:47.196685+0000","last_unstale":"2023-10-09T05:36:15.878866+0000","last_undegraded":"2023-10-09T05:36:15.878866+0000","last_fullsized":"2023-10-09T05:36:15.878866+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:47:32.594447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":303,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.879123+0000","last_change":"2023-10-09T05:02:09.649674+0000","last_active":"2023-10-09T05:36:15.879123+0000","last_peered":"2023-10-09T05:36:15.879123+0000","last_clean":"2023-10-09T05:36:15.879123+0000","last_became_active":"2023-10-09T05:02:09.649041+0000","last_became_peered":"2023-10-09T05:02:09.649041+0000","last_unstale":"2023-10-09T05:36:15.879123+0000","last_undegraded":"2023-10-09T05:36:15.879123+0000","last_fullsized":"2023-10-09T05:36:15.879123+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:18.524071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'5","reported_seq":112,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.881755+0000","last_change":"2023-10-09T05:32:45.849292+0000","last_active":"2023-10-09T05:36:15.881755+0000","last_peered":"2023-10-09T05:36:15.881755+0000","last_clean":"2023-10-09T05:36:15.881755+0000","last_became_active":"2023-10-09T05:32:45.829351+0000","last_became_peered":"2023-10-09T05:32:45.829351+0000","last_unstale":"2023-10-09T05:36:15.881755+0000","last_undegraded":"2023-10-09T05:36:15.881755+0000","last_fullsized":"2023-10-09T05:36:15.881755+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:07:15.666617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":291,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T05:36:15.883005+0000","last_change":"2023-10-09T05:30:03.379517+0000","last_active":"2023-10-09T05:36:15.883005+0000","last_peered":"2023-10-09T05:36:15.883005+0000","last_clean":"2023-10-09T05:36:15.883005+0000","last_became_active":"2023-10-09T05:30:03.379048+0000","last_became_peered":"2023-10-09T05:30:03.379048+0000","last_unstale":"2023-10-09T05:36:15.883005+0000","last_undegraded":"2023-10-09T05:36:15.883005+0000","last_fullsized":"2023-10-09T05:36:15.883005+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:04.333337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":1908473856,"num_objects":29121,"num_object_clones":0,"num_object_copies":87363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":29121,"num_write_kb":1863744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5794824192,"data_stored":5794824192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":29121,"ondisk_log_size":29121,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":23,"num_read_kb":22,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":64,"num_bytes_recovered":22228,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234650,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1629484,"kb_used_data":438052,"kb_used_omap":0,"kb_used_meta":1191424,"kb_avail":92115668,"statfs":{"total":95995035648,"available":94326444032,"internally_reserved":0,"allocated":448565248,"data_stored":447588081,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1220018176},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:36:13 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":2.8399999999999999,"15min":2.8300000000000001},"min":{"1min":0.36899999999999999,"5min":0.32300000000000001,"15min":0.248},"max":{"1min":1.165,"5min":87.216999999999999,"15min":122.248},"last":1.71},{"interface":"front","average":{"1min":0.92200000000000004,"5min":3.0510000000000002,"15min":2.931},"min":{"1min":0.59399999999999997,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.3440000000000001,"5min":87.180999999999997,"15min":122.22199999999999},"last":1.726}]},{"osd":1,"last update":"Mon Oct 9 05:35:51 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":1.5249999999999999,"15min":1.5589999999999999},"min":{"1min":0.52000000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":1.411,"5min":7.8140000000000001,"15min":23.530999999999999},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.871,"5min":1.7130000000000001,"15min":1.595},"min":{"1min":0.52500000000000002,"5min":0.318,"15min":0.29899999999999999},"max":{"1min":1.22,"5min":12.081,"15min":23.443000000000001},"last":2.7930000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:36:13 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":3.246,"15min":2.653},"min":{"1min":0.66400000000000003,"5min":0.34499999999999997,"15min":0.29799999999999999},"max":{"1min":1.206,"5min":102.146,"15min":122.065},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.80300000000000005,"5min":2.9449999999999998,"15min":2.677},"min":{"1min":0.46700000000000003,"5min":0.35999999999999999,"15min":0.30399999999999999},"max":{"1min":1.004,"5min":102.188,"15min":122.026},"last":0.42699999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:36:13 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":1.542,"15min":2.27},"min":{"1min":0.64100000000000001,"5min":0.33500000000000002,"15min":0.33200000000000002},"max":{"1min":1.1259999999999999,"5min":7.4269999999999996,"15min":138.667},"last":1.857},{"interface":"front","average":{"1min":0.97199999999999998,"5min":1.653,"15min":2.4550000000000001},"min":{"1min":0.72999999999999998,"5min":0.41399999999999998,"15min":0.34300000000000003},"max":{"1min":1.1859999999999999,"5min":7.3529999999999998,"15min":138.54400000000001},"last":0.54600000000000004}]},{"osd":4,"last update":"Mon Oct 9 05:35:51 2023","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":1.0589999999999999,"15min":0.96799999999999997},"min":{"1min":0.73399999999999999,"5min":0.33900000000000002,"15min":0.311},"max":{"1min":1.25,"5min":7.077,"15min":7.077},"last":0.53100000000000003},{"interface":"front","average":{"1min":1.0680000000000001,"5min":1.1579999999999999,"15min":1.083},"min":{"1min":0.63100000000000001,"5min":0.36699999999999999,"15min":0.33300000000000002},"max":{"1min":1.341,"5min":7.3869999999999996,"15min":7.3869999999999996},"last":0.625}]},{"osd":5,"last update":"Mon Oct 9 05:35:51 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":1.3100000000000001,"15min":1.1220000000000001},"min":{"1min":0.58799999999999997,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.161,"5min":7.4829999999999997,"15min":7.4829999999999997},"last":0.50700000000000001},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.3540000000000001,"15min":1.117},"min":{"1min":0.75900000000000001,"5min":0.375,"15min":0.373},"max":{"1min":1.4079999999999999,"5min":7.6219999999999999,"15min":7.6219999999999999},"last":0.47899999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:35:51 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.379,"15min":1.236},"min":{"1min":0.61799999999999999,"5min":0.38700000000000001,"15min":0.32800000000000001},"max":{"1min":1.401,"5min":6.8789999999999996,"15min":8.5609999999999999},"last":0.83599999999999997},{"interface":"front","average":{"1min":1.159,"5min":1.4199999999999999,"15min":1.2170000000000001},"min":{"1min":0.84199999999999997,"5min":0.44800000000000001,"15min":0.377},"max":{"1min":1.4019999999999999,"5min":7.46,"15min":7.46},"last":0.78400000000000003}]}]},{"osd":6,"up_from":46,"seq":197568496292,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1477864,"kb_used_data":574048,"kb_used_omap":0,"kb_used_meta":903808,"kb_avail":92267288,"statfs":{"total":95995035648,"available":94481702912,"internally_reserved":0,"allocated":587825152,"data_stored":586852292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925499392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:36:09 2023","interfaces":[{"interface":"back","average":{"1min":1.0649999999999999,"5min":4.2549999999999999,"15min":2.8119999999999998},"min":{"1min":0.57399999999999995,"5min":0.28000000000000003,"15min":0.26100000000000001},"max":{"1min":3.9830000000000001,"5min":106.85599999999999,"15min":154.29300000000001},"last":5.1520000000000001},{"interface":"front","average":{"1min":1.2110000000000001,"5min":4.343,"15min":2.9239999999999999},"min":{"1min":0.70099999999999996,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":4.024,"5min":107.11,"15min":154.35300000000001},"last":5.1829999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:36:09 2023","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":1.609,"15min":2.0499999999999998},"min":{"1min":0.56499999999999995,"5min":0.28299999999999997,"15min":0.27400000000000002},"max":{"1min":1.228,"5min":10.932,"15min":94.037000000000006},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.86699999999999999,"5min":1.286,"15min":1.724},"min":{"1min":0.375,"5min":0.312,"15min":0.29999999999999999},"max":{"1min":1.2989999999999999,"5min":9.7210000000000001,"15min":87.950000000000003},"last":0.504}]},{"osd":2,"last update":"Mon Oct 9 05:36:09 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":5.5880000000000001,"15min":4.157},"min":{"1min":0.47699999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.3089999999999999,"5min":211.66300000000001,"15min":211.66300000000001},"last":0.377},{"interface":"front","average":{"1min":0.82999999999999996,"5min":6.2880000000000003,"15min":4.4249999999999998},"min":{"1min":0.60999999999999999,"5min":0.317,"15min":0.31},"max":{"1min":1.1919999999999999,"5min":211.423,"15min":211.423},"last":1.1799999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:36:09 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":1.8280000000000001,"15min":3.9289999999999998},"min":{"1min":0.56200000000000006,"5min":0.309,"15min":0.309},"max":{"1min":1.4079999999999999,"5min":29.704000000000001,"15min":169.01599999999999},"last":0.71199999999999997},{"interface":"front","average":{"1min":1.036,"5min":1.7090000000000001,"15min":3.7519999999999998},"min":{"1min":0.57699999999999996,"5min":0.34200000000000003,"15min":0.33800000000000002},"max":{"1min":1.6830000000000001,"5min":30.295999999999999,"15min":169.40799999999999},"last":0.42499999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:36:09 2023","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":1.0660000000000001,"15min":1.0309999999999999},"min":{"1min":0.47099999999999997,"5min":0.35299999999999998,"15min":0.27600000000000002},"max":{"1min":1.5389999999999999,"5min":10.976000000000001,"15min":10.976000000000001},"last":0.76000000000000001},{"interface":"front","average":{"1min":1.081,"5min":1.0529999999999999,"15min":0.97299999999999998},"min":{"1min":0.438,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":1.774,"5min":5.0190000000000001,"15min":5.0190000000000001},"last":0.46600000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:36:09 2023","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":1.278,"15min":1.1200000000000001},"min":{"1min":0.435,"5min":0.39100000000000001,"15min":0.33300000000000002},"max":{"1min":1.8080000000000001,"5min":11.275,"15min":11.275},"last":0.73799999999999999},{"interface":"front","average":{"1min":1.117,"5min":1.169,"15min":1.0649999999999999},"min":{"1min":0.497,"5min":0.38,"15min":0.38},"max":{"1min":1.8700000000000001,"5min":7.3090000000000002,"15min":7.3090000000000002},"last":0.53700000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:35:27 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":1.859,"15min":1.49},"min":{"1min":0.42799999999999999,"5min":0.36599999999999999,"15min":0.28100000000000003},"max":{"1min":6.8789999999999996,"5min":18.963000000000001,"15min":18.963000000000001},"last":0.60699999999999998},{"interface":"front","average":{"1min":1.296,"5min":1.9430000000000001,"15min":1.5800000000000001},"min":{"1min":0.51500000000000001,"5min":0.39500000000000002,"15min":0.35399999999999998},"max":{"1min":6.9260000000000002,"5min":19.010000000000002,"15min":19.010000000000002},"last":0.68400000000000005}]}]},{"osd":1,"up_from":96,"seq":412316860883,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2336596,"kb_used_data":1223116,"kb_used_omap":4,"kb_used_meta":1113467,"kb_avail":91408556,"statfs":{"total":95995035648,"available":93602361344,"internally_reserved":0,"allocated":1252470784,"data_stored":1250700662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1140191118},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":3.7669999999999999,"15min":3.1520000000000001},"min":{"1min":0.307,"5min":0.21099999999999999,"15min":0.19500000000000001},"max":{"1min":0.92000000000000004,"5min":150.75200000000001,"15min":150.75200000000001},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.625,"5min":4.0170000000000003,"15min":3.488},"min":{"1min":0.35399999999999998,"5min":0.185,"15min":0.185},"max":{"1min":1.143,"5min":150.59299999999999,"15min":150.59299999999999},"last":0.60399999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":5.4580000000000002,"15min":3.714},"min":{"1min":0.379,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":1.361,"5min":154.14599999999999,"15min":154.14599999999999},"last":4.9260000000000002},{"interface":"front","average":{"1min":0.73099999999999998,"5min":5.3890000000000002,"15min":3.6789999999999998},"min":{"1min":0.40200000000000002,"5min":0.28100000000000003,"15min":0.254},"max":{"1min":1.415,"5min":154.215,"15min":154.215},"last":0.61899999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":4.5499999999999998,"15min":3.3690000000000002},"min":{"1min":0.41599999999999998,"5min":0.22800000000000001,"15min":0.17499999999999999},"max":{"1min":1.083,"5min":152.58199999999999,"15min":152.58199999999999},"last":3.0979999999999999},{"interface":"front","average":{"1min":0.746,"5min":4.2039999999999997,"15min":3.504},"min":{"1min":0.42499999999999999,"5min":0.255,"15min":0.221},"max":{"1min":1.6930000000000001,"5min":154.178,"15min":154.178},"last":4.9939999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.371,"15min":1.9710000000000001},"min":{"1min":0.65000000000000002,"5min":0.38700000000000001,"15min":0.30099999999999999},"max":{"1min":1.958,"5min":5.3579999999999997,"15min":83.453999999999994},"last":4.835},{"interface":"front","average":{"1min":0.93999999999999995,"5min":1.3049999999999999,"15min":1.6850000000000001},"min":{"1min":0.5,"5min":0.36799999999999999,"15min":0.32400000000000001},"max":{"1min":1.379,"5min":6.0650000000000004,"15min":83.375},"last":0.67700000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":1.673,"15min":1.5229999999999999},"min":{"1min":0.65000000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.446,"5min":9.2249999999999996,"15min":9.7520000000000007},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.6080000000000001,"15min":2.0249999999999999},"min":{"1min":0.64600000000000002,"5min":0.40500000000000003,"15min":0.39900000000000002},"max":{"1min":1.8999999999999999,"5min":9.8260000000000005,"15min":83.268000000000001},"last":4.8890000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":1.6619999999999999,"15min":1.647},"min":{"1min":0.57299999999999995,"5min":0.36599999999999999,"15min":0.34000000000000002},"max":{"1min":1.5549999999999999,"5min":11.315,"15min":11.315},"last":0.56999999999999995},{"interface":"front","average":{"1min":1.0700000000000001,"5min":1.655,"15min":2.0630000000000002},"min":{"1min":0.59999999999999998,"5min":0.41499999999999998,"15min":0.376},"max":{"1min":2.0590000000000002,"5min":10.446,"15min":83.400000000000006},"last":4.9550000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:36:07 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":1.635,"15min":2.0819999999999999},"min":{"1min":0.59499999999999997,"5min":0.38800000000000001,"15min":0.376},"max":{"1min":2.008,"5min":8.6059999999999999,"15min":83.510000000000005},"last":4.9749999999999996},{"interface":"front","average":{"1min":1.0549999999999999,"5min":1.504,"15min":1.782},"min":{"1min":0.69799999999999995,"5min":0.38600000000000001,"15min":0.377},"max":{"1min":1.5029999999999999,"5min":7.101,"15min":83.480999999999995},"last":0.64400000000000002}]}]},{"osd":0,"up_from":88,"seq":377957122524,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2137604,"kb_used_data":1037244,"kb_used_omap":5,"kb_used_meta":1100346,"kb_avail":91607548,"statfs":{"total":95995035648,"available":93806129152,"internally_reserved":0,"allocated":1062137856,"data_stored":1060440448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1126754938},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:36:01 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":2.8100000000000001,"15min":2.1230000000000002},"min":{"1min":0.39300000000000002,"5min":0.17699999999999999,"15min":0.17299999999999999},"max":{"1min":1.0700000000000001,"5min":111.001,"15min":111.001},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.67100000000000004,"5min":2.9980000000000002,"15min":2.1280000000000001},"min":{"1min":0.45900000000000002,"5min":0.25700000000000001,"15min":0.214},"max":{"1min":1.1479999999999999,"5min":110.919,"15min":110.919},"last":0.24299999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.74299999999999999,"5min":2.9420000000000002,"15min":2.859},"min":{"1min":0.44600000000000001,"5min":0.27100000000000002,"15min":0.247},"max":{"1min":1.079,"5min":74.817999999999998,"15min":92.745000000000005},"last":0.315},{"interface":"front","average":{"1min":0.70299999999999996,"5min":3.2669999999999999,"15min":3.0670000000000002},"min":{"1min":0.34999999999999998,"5min":0.25900000000000001,"15min":0.19700000000000001},"max":{"1min":0.99199999999999999,"5min":74.751000000000005,"15min":97.816000000000003},"last":1.9990000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":7.3040000000000003,"15min":4.1050000000000004},"min":{"1min":0.41699999999999998,"5min":0.25600000000000001,"15min":0.21099999999999999},"max":{"1min":1.0129999999999999,"5min":290.73000000000002,"15min":290.73000000000002},"last":7.1310000000000002},{"interface":"front","average":{"1min":0.83099999999999996,"5min":6.8330000000000002,"15min":4.0700000000000003},"min":{"1min":0.39500000000000002,"5min":0.246,"15min":0.246},"max":{"1min":1.2549999999999999,"5min":289.85199999999998,"15min":289.85199999999998},"last":2.887}]},{"osd":4,"last update":"Mon Oct 9 05:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":1.4370000000000001,"15min":1.472},"min":{"1min":0.71899999999999997,"5min":0.373,"15min":0.33500000000000002},"max":{"1min":1.2170000000000001,"5min":9.0069999999999997,"15min":9.7460000000000004},"last":0.51100000000000001},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.546,"15min":1.6319999999999999},"min":{"1min":0.63600000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.5029999999999999,"5min":9.7989999999999995,"15min":9.8320000000000007},"last":0.499}]},{"osd":5,"last update":"Mon Oct 9 05:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":1.9390000000000001,"15min":1.6319999999999999},"min":{"1min":0.60199999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.274,"5min":14.510999999999999,"15min":14.510999999999999},"last":7.0419999999999998},{"interface":"front","average":{"1min":1.079,"5min":1.575,"15min":1.5720000000000001},"min":{"1min":0.75800000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.4550000000000001,"5min":9.0449999999999999,"15min":9.8010000000000002},"last":0.54300000000000004}]},{"osd":6,"last update":"Mon Oct 9 05:36:05 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.9219999999999999,"15min":1.8660000000000001},"min":{"1min":0.71299999999999997,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.601,"5min":15.544,"15min":15.544},"last":3.802},{"interface":"front","average":{"1min":1.038,"5min":2.077,"15min":1.7689999999999999},"min":{"1min":0.78200000000000003,"5min":0.318,"15min":0.318},"max":{"1min":1.431,"5min":14.459,"15min":14.459},"last":7.1070000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:36:05 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":2.2869999999999999,"15min":1.8919999999999999},"min":{"1min":0.64200000000000002,"5min":0.42099999999999999,"15min":0.308},"max":{"1min":1.5649999999999999,"5min":14.492000000000001,"15min":14.492000000000001},"last":7.0819999999999999},{"interface":"front","average":{"1min":1.145,"5min":1.726,"15min":1.7290000000000001},"min":{"1min":0.86799999999999999,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.544,"5min":8.9710000000000001,"15min":10.795},"last":0.68300000000000005}]}]},{"osd":2,"up_from":106,"seq":455266533803,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1922588,"kb_used_data":970580,"kb_used_omap":7,"kb_used_meta":951992,"kb_avail":91822564,"statfs":{"total":95995035648,"available":94026305536,"internally_reserved":0,"allocated":993873920,"data_stored":992214928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":974839889},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:35:52 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":3.25,"15min":2.6080000000000001},"min":{"1min":0.41799999999999998,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.97599999999999998,"5min":91.646000000000001,"15min":91.646000000000001},"last":46.154000000000003},{"interface":"front","average":{"1min":0.65600000000000003,"5min":3.2730000000000001,"15min":2.4950000000000001},"min":{"1min":0.38600000000000001,"5min":0.22,"15min":0.21199999999999999},"max":{"1min":1.046,"5min":92.522000000000006,"15min":92.522000000000006},"last":45.789999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:35:52 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":2.1190000000000002,"15min":2.1139999999999999},"min":{"1min":0.27600000000000002,"5min":0.22900000000000001,"15min":0.22700000000000001},"max":{"1min":1.0309999999999999,"5min":13.143000000000001,"15min":23.251000000000001},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.64900000000000002,"5min":2.1539999999999999,"15min":2.327},"min":{"1min":0.33200000000000002,"5min":0.24299999999999999,"15min":0.22600000000000001},"max":{"1min":0.93100000000000005,"5min":10.063000000000001,"15min":17.632999999999999},"last":0.65400000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:36:12 2023","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":2.6030000000000002,"15min":2.5680000000000001},"min":{"1min":0.35199999999999998,"5min":0.214,"15min":0.214},"max":{"1min":2.3140000000000001,"5min":15.576000000000001,"15min":15.576000000000001},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.79700000000000004,"5min":2.133,"15min":2.4470000000000001},"min":{"1min":0.40300000000000002,"5min":0.33600000000000002,"15min":0.27000000000000002},"max":{"1min":1.1739999999999999,"5min":11.451000000000001,"15min":18.515000000000001},"last":0.496}]},{"osd":4,"last update":"Mon Oct 9 05:35:52 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":1.8540000000000001,"15min":1.839},"min":{"1min":0.49399999999999999,"5min":0.34499999999999997,"15min":0.29599999999999999},"max":{"1min":1.4830000000000001,"5min":10.379,"15min":11.157999999999999},"last":0.878},{"interface":"front","average":{"1min":1.0860000000000001,"5min":1.8560000000000001,"15min":1.8999999999999999},"min":{"1min":0.624,"5min":0.35399999999999998,"15min":0.34200000000000003},"max":{"1min":3.234,"5min":11.173,"15min":13.097},"last":0.73299999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:35:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":1.8069999999999999,"15min":1.7390000000000001},"min":{"1min":0.60699999999999998,"5min":0.373,"15min":0.33000000000000002},"max":{"1min":1.304,"5min":9.8729999999999993,"15min":9.8729999999999993},"last":0.63800000000000001},{"interface":"front","average":{"1min":1.0269999999999999,"5min":1.8859999999999999,"15min":1.8089999999999999},"min":{"1min":0.46300000000000002,"5min":0.36299999999999999,"15min":0.34699999999999998},"max":{"1min":1.3560000000000001,"5min":10.265000000000001,"15min":11.205},"last":0.80500000000000005}]},{"osd":6,"last update":"Mon Oct 9 05:35:52 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.921,"15min":1.994},"min":{"1min":0.54000000000000004,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.4530000000000001,"5min":11.381,"15min":13.209},"last":1.014},{"interface":"front","average":{"1min":1.0489999999999999,"5min":1.8899999999999999,"15min":1.855},"min":{"1min":0.63500000000000001,"5min":0.39200000000000002,"15min":0.35199999999999998},"max":{"1min":1.4159999999999999,"5min":9.9120000000000008,"15min":11.118},"last":0.81699999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:35:52 2023","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":2.0600000000000001,"15min":2.0249999999999999},"min":{"1min":0.80100000000000005,"5min":0.45900000000000002,"15min":0.38300000000000001},"max":{"1min":1.496,"5min":9.8930000000000007,"15min":13.571999999999999},"last":0.79700000000000004},{"interface":"front","average":{"1min":1.1539999999999999,"5min":2.1219999999999999,"15min":2.137},"min":{"1min":0.74199999999999999,"5min":0.434,"15min":0.379},"max":{"1min":1.5449999999999999,"5min":12.016,"15min":12.016},"last":1.006}]}]},{"osd":3,"up_from":123,"seq":528280977821,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1975288,"kb_used_data":948464,"kb_used_omap":2,"kb_used_meta":1026813,"kb_avail":91769864,"statfs":{"total":95995035648,"available":93972340736,"internally_reserved":0,"allocated":971227136,"data_stored":969654440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":1051456565},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:36:19 2023","interfaces":[{"interface":"back","average":{"1min":1.349,"5min":3.7799999999999998,"15min":2.641},"min":{"1min":0.41299999999999998,"5min":0.224,"15min":0.19500000000000001},"max":{"1min":7.04,"5min":90.947000000000003,"15min":90.947000000000003},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.70499999999999996,"5min":3.5409999999999999,"15min":2.6190000000000002},"min":{"1min":0.246,"5min":0.184,"15min":0.184},"max":{"1min":1.853,"5min":90.662999999999997,"15min":90.662999999999997},"last":0.29299999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:36:19 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":3.5779999999999998,"15min":2.5720000000000001},"min":{"1min":0.28799999999999998,"5min":0.23000000000000001,"15min":0.22},"max":{"1min":0.80400000000000005,"5min":118.20999999999999,"15min":118.20999999999999},"last":2.2349999999999999},{"interface":"front","average":{"1min":0.89700000000000002,"5min":3.597,"15min":2.5489999999999999},"min":{"1min":0.48099999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.3050000000000002,"5min":124.083,"15min":124.083},"last":0.32200000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:36:19 2023","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":6.8799999999999999,"15min":3.7999999999999998},"min":{"1min":0.48199999999999998,"5min":0.248,"15min":0.24299999999999999},"max":{"1min":3.1840000000000002,"5min":350.53899999999999,"15min":350.53899999999999},"last":3.1480000000000001},{"interface":"front","average":{"1min":0.89000000000000001,"5min":6.8940000000000001,"15min":3.7650000000000001},"min":{"1min":0.41699999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":2.9790000000000001,"5min":350.64699999999999,"15min":350.64699999999999},"last":2.2040000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:36:19 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":1.6339999999999999,"15min":1.6910000000000001},"min":{"1min":0.42499999999999999,"5min":0.36499999999999999,"15min":0.29999999999999999},"max":{"1min":1.377,"5min":9.1539999999999999,"15min":9.1539999999999999},"last":3.2069999999999999},{"interface":"front","average":{"1min":1.1599999999999999,"5min":1.7030000000000001,"15min":1.663},"min":{"1min":0.67800000000000005,"5min":0.36599999999999999,"15min":0.33300000000000002},"max":{"1min":2.266,"5min":12.132999999999999,"15min":13.234999999999999},"last":2.4279999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:36:19 2023","interfaces":[{"interface":"back","average":{"1min":1.2270000000000001,"5min":1.7569999999999999,"15min":1.6990000000000001},"min":{"1min":0.70699999999999996,"5min":0.40500000000000003,"15min":0.35999999999999999},"max":{"1min":2.331,"5min":12.092000000000001,"15min":13.287000000000001},"last":2.484},{"interface":"front","average":{"1min":0.97699999999999998,"5min":1.6040000000000001,"15min":1.5569999999999999},"min":{"1min":0.61099999999999999,"5min":0.36299999999999999,"15min":0.34899999999999998},"max":{"1min":1.327,"5min":10.135,"15min":10.135},"last":0.63300000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:36:19 2023","interfaces":[{"interface":"back","average":{"1min":2.0609999999999999,"5min":2.1560000000000001,"15min":1.7909999999999999},"min":{"1min":0.52000000000000002,"5min":0.38700000000000001,"15min":0.35699999999999998},"max":{"1min":20.463999999999999,"5min":20.463999999999999,"15min":20.463999999999999},"last":0.58499999999999996},{"interface":"front","average":{"1min":2.0739999999999998,"5min":2,"15min":1.9319999999999999},"min":{"1min":0.48999999999999999,"5min":0.38400000000000001,"15min":0.36699999999999999},"max":{"1min":20.279,"5min":20.279,"15min":20.279},"last":3.1659999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:36:19 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":2.0649999999999999,"15min":1.913},"min":{"1min":0.46100000000000002,"5min":0.41099999999999998,"15min":0.38800000000000001},"max":{"1min":1.3200000000000001,"5min":10.885,"15min":10.885},"last":3.2410000000000001},{"interface":"front","average":{"1min":1.224,"5min":2.145,"15min":1.9910000000000001},"min":{"1min":0.68600000000000005,"5min":0.45500000000000002,"15min":0.39200000000000002},"max":{"1min":2.2269999999999999,"5min":12.113,"15min":13.262},"last":2.46}]}]},{"osd":4,"up_from":0,"seq":128849019578,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757934,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1456348,"kb_used_data":561684,"kb_used_omap":0,"kb_used_meta":894656,"kb_avail":92288804,"statfs":{"total":95995035648,"available":94503735296,"internally_reserved":0,"allocated":575164416,"data_stored":574199351,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916127744},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:36:10 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":3.9710000000000001,"15min":2.3279999999999998},"min":{"1min":0.42199999999999999,"5min":0.28499999999999998,"15min":0.26900000000000002},"max":{"1min":1.234,"5min":109.193,"15min":109.193},"last":3.9910000000000001},{"interface":"front","average":{"1min":0.94299999999999995,"5min":4.0599999999999996,"15min":2.4409999999999998},"min":{"1min":0.51200000000000001,"5min":0.36199999999999999,"15min":0.35799999999999998},"max":{"1min":1.381,"5min":109.298,"15min":109.298},"last":4.0259999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:36:10 2023","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":1.367,"15min":1.734},"min":{"1min":0.50800000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.3100000000000001,"5min":9.2430000000000003,"15min":78.117000000000004},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.90600000000000003,"5min":1.361,"15min":1.74},"min":{"1min":0.48599999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.3220000000000001,"5min":8.6799999999999997,"15min":78.879999999999995},"last":0.47099999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:36:20 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":4.5270000000000001,"15min":2.6800000000000002},"min":{"1min":0.41199999999999998,"5min":0.32300000000000001,"15min":0.27800000000000002},"max":{"1min":1.2769999999999999,"5min":250.941,"15min":250.941},"last":1.667},{"interface":"front","average":{"1min":1.016,"5min":4.625,"15min":2.6709999999999998},"min":{"1min":0.57899999999999996,"5min":0.34699999999999998,"15min":0.32900000000000001},"max":{"1min":4.1779999999999999,"5min":251.48599999999999,"15min":251.48599999999999},"last":0.81899999999999995}]},{"osd":3,"last update":"Mon Oct 9 05:36:20 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":1.5109999999999999,"15min":2.0720000000000001},"min":{"1min":0.36699999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.226,"5min":9.0169999999999995,"15min":76.512},"last":0.60399999999999998},{"interface":"front","average":{"1min":1.2370000000000001,"5min":1.863,"15min":2.1389999999999998},"min":{"1min":0.49099999999999999,"5min":0.374,"15min":0.315},"max":{"1min":8.1140000000000008,"5min":10.295,"15min":76.578000000000003},"last":2.0230000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:36:10 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.95999999999999996,"15min":0.84099999999999997},"min":{"1min":0.45200000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.4039999999999999,"5min":7.6109999999999998,"15min":7.6109999999999998},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.95799999999999996,"5min":1.1870000000000001,"15min":1.0049999999999999},"min":{"1min":0.55500000000000005,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.3169999999999999,"5min":8.3480000000000008,"15min":8.3480000000000008},"last":0.76500000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:36:10 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":1.3799999999999999,"15min":1.865},"min":{"1min":0.55700000000000005,"5min":0.376,"15min":0.34899999999999998},"max":{"1min":1.194,"5min":8.5220000000000002,"15min":190.98500000000001},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.97999999999999998,"5min":1.2929999999999999,"15min":1.8620000000000001},"min":{"1min":0.47299999999999998,"5min":0.41399999999999998,"15min":0.39600000000000002},"max":{"1min":1.2989999999999999,"5min":9.282,"15min":190.928},"last":0.38800000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:35:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":1.3400000000000001,"15min":1.103},"min":{"1min":0.499,"5min":0.372,"15min":0.372},"max":{"1min":1.2969999999999999,"5min":9.3970000000000002,"15min":9.3970000000000002},"last":0.442},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.5960000000000001,"15min":1.2},"min":{"1min":0.53100000000000003,"5min":0.41799999999999998,"15min":0.35899999999999999},"max":{"1min":1.3799999999999999,"5min":11.593,"15min":11.593},"last":0.74099999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1058275328,"data_stored":1058275328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1098579968,"data_stored":1098579968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1029373952,"data_stored":1029373952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":971767808,"data_stored":971767808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":582352896,"data_stored":582352896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":591855616,"data_stored":591855616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":462618624,"data_stored":462618624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:36:33.564 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T05:36:33.564 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_9 pg_num 26 2023-10-09T05:36:33.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:33 smithi194 bash[92666]: cluster 2023-10-09T05:36:32.402170+0000 mgr.y (mgr.25277) 1459 : cluster 0 pgmap v1567: 89 pgs: 89 active+clean; 2.2 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T05:36:33.643 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:33.668 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:33.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:33.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:33.762 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 25 16 40186 40170 100.414 78.875 0.0187209 0.0099521 2023-10-09T05:36:33.772 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:33.797 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:33 smithi133 bash[105589]: cluster 2023-10-09T05:36:32.402170+0000 mgr.y (mgr.25277) 1459 : cluster 0 pgmap v1567: 89 pgs: 89 active+clean; 2.2 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T05:36:33.798 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:33.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:33.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:33.903 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:33.942 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:33.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:33.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:34.048 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:34.078 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:34.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:34.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:34.181 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:34.209 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:34.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:34.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:34.312 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:34.340 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:34.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:34.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:34.443 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:34.470 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:34.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:34.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:34.547 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:36:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:36:34] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.43.0" 2023-10-09T05:36:34.576 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:34.604 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:34.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:34.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:34.710 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:34.738 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:34.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:34.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:34.762 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 26 16 41425 41409 99.5304 77.4375 0.0139755 0.0100408 2023-10-09T05:36:34.843 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:34.873 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:34.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:34.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:34.983 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:34.992 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:34 smithi194 bash[92666]: cluster 2023-10-09T05:36:34.402819+0000 mgr.y (mgr.25277) 1460 : cluster 0 pgmap v1568: 89 pgs: 89 active+clean; 2.2 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T05:36:35.010 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:35.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:35.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:35.114 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:35.139 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:35.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:35.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:35.242 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:35.269 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:35.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:35.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:35.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:34 smithi133 bash[102858]: cluster 2023-10-09T05:36:34.402819+0000 mgr.y (mgr.25277) 1460 : cluster 0 pgmap v1568: 89 pgs: 89 active+clean; 2.2 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T05:36:35.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:34 smithi133 bash[105589]: cluster 2023-10-09T05:36:34.402819+0000 mgr.y (mgr.25277) 1460 : cluster 0 pgmap v1568: 89 pgs: 89 active+clean; 2.2 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T05:36:35.373 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:35.400 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:35.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:35.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:35.503 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:35.532 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:35.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:35.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:35.636 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:35.661 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:35.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:35.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:35.762 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 27 16 42677 42661 98.7419 78.25 0.00509869 0.0101193 2023-10-09T05:36:35.764 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:35.797 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:35.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:35.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:35.902 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:35.925 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:35.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:35.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:36.032 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:36.062 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:36.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:36.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:36.170 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:36.194 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:36.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:36.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:36.264 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:36:36.297 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:36.322 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:36.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:36.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:36.426 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:36.453 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:36.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:36.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:36.561 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:36.585 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:36.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:36.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:36.689 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:36.724 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:36.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:36.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:36.762 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 28 16 43865 43849 97.8669 74.25 0.0157219 0.0102096 2023-10-09T05:36:36.828 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:36.855 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:36.857 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:36:36.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:36.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:36.891 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:36:36.961 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:36.987 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:36.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:36.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:37.091 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:37.180 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:37.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:37.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:37.292 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:37.315 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:37.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:37.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:37.418 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:37.445 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:37.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:37.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:37.459 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:37 smithi133 bash[102858]: cluster 2023-10-09T05:36:36.403564+0000 mgr.y (mgr.25277) 1461 : cluster 0 pgmap v1569: 89 pgs: 89 active+clean; 2.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T05:36:37.549 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:37.558 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:37 smithi194 bash[92666]: cluster 2023-10-09T05:36:36.403564+0000 mgr.y (mgr.25277) 1461 : cluster 0 pgmap v1569: 89 pgs: 89 active+clean; 2.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T05:36:37.572 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:37.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:37.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:37.676 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:37.699 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:37.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:37.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:37.762 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 29 16 44984 44968 96.9035 69.9375 0.020962 0.0103107 2023-10-09T05:36:37.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:37 smithi133 bash[105589]: cluster 2023-10-09T05:36:36.403564+0000 mgr.y (mgr.25277) 1461 : cluster 0 pgmap v1569: 89 pgs: 89 active+clean; 2.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T05:36:37.803 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:37.831 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:37.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:37.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:37.934 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:37.965 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:37.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:37.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:38.069 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:38.097 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:38.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:38.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:38.200 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:38.228 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:38.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:38.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:38.331 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:38.359 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:38.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:38.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:38.465 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:38.497 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:38.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:38.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:38.600 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:38.608 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:38.629 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:38.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:38.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:38.736 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:38.762 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 30 16 46163 46147 96.1295 73.6875 0.0136043 0.0103958 2023-10-09T05:36:38.763 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:38.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:38.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:38.870 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:38.909 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:38.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:38.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:39.013 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:39.041 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:39.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:39.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:39.146 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:39.173 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:39.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:39.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:39.278 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:39.304 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:39.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:39.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:39.409 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:39.434 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:39.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:39.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:39.537 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:39.570 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:39.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:39.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:39.674 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:39.679 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:39 smithi194 bash[92666]: cluster 2023-10-09T05:36:38.404088+0000 mgr.y (mgr.25277) 1462 : cluster 0 pgmap v1570: 89 pgs: 89 active+clean; 2.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-10-09T05:36:39.700 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:39.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:39.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:39.763 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 31 16 47306 47290 95.3322 71.4375 0.00510053 0.0104813 2023-10-09T05:36:39.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:39 smithi133 bash[105589]: cluster 2023-10-09T05:36:38.404088+0000 mgr.y (mgr.25277) 1462 : cluster 0 pgmap v1570: 89 pgs: 89 active+clean; 2.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-10-09T05:36:39.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:39 smithi133 bash[102858]: cluster 2023-10-09T05:36:38.404088+0000 mgr.y (mgr.25277) 1462 : cluster 0 pgmap v1570: 89 pgs: 89 active+clean; 2.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-10-09T05:36:39.802 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:39.827 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:39.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:39.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:39.933 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:39.958 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:39.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:39.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:40.061 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:40.092 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:40.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:40.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:40.194 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:40.222 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:40.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:40.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:40.326 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:40.351 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:40.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:40.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:40.455 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:40.479 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:40.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:40.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:40.585 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:40.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:40.618 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:40.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:40.719 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:40.747 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:40.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:40.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:40.762 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 32 16 48495 48479 94.6751 74.3125 0.00492859 0.0105549 2023-10-09T05:36:40.851 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:40.876 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:40.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:40.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:40.981 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:41.018 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:41.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:41.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:41.121 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:41.145 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:41.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:41.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:41.248 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:41.273 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:41.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:41.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:41.376 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:41.407 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:41.410 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:36:41.410 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:36:41.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:41.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:41.513 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:41.541 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:41.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:41.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:41.646 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:41.670 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:41.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:41.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:41.673 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:41.673 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:41.705 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:41 smithi133 bash[102858]: cluster 2023-10-09T05:36:40.404754+0000 mgr.y (mgr.25277) 1463 : cluster 0 pgmap v1571: 89 pgs: 89 active+clean; 2.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T05:36:41.705 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:41 smithi133 bash[105589]: cluster 2023-10-09T05:36:40.404754+0000 mgr.y (mgr.25277) 1463 : cluster 0 pgmap v1571: 89 pgs: 89 active+clean; 2.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T05:36:41.763 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 33 16 49634 49618 93.9631 71.1875 0.0122514 0.0106339 2023-10-09T05:36:41.773 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:41.783 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:41 smithi194 bash[92666]: cluster 2023-10-09T05:36:40.404754+0000 mgr.y (mgr.25277) 1463 : cluster 0 pgmap v1571: 89 pgs: 89 active+clean; 2.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T05:36:41.797 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:41.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:41.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:41.902 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:41.929 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:41.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:41.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:42.032 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:42.055 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:42.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:42.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:42.159 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:42.199 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:42.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:42.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:42.303 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:42.327 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:42.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:42.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:42.429 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:42.459 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:42.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:42.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:42.563 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:42.587 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:42.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:42.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:42.689 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:42.722 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:42.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:42.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:42.761 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T05:36:42.762 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 34 16 50592 50576 92.9604 59.875 0.0177212 0.0107501 2023-10-09T05:36:42.831 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:42.861 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:42.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:42.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:42.966 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:42.996 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:42.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:42.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:43.099 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:43.128 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:43.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:43.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:43.234 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:43.261 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:43.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:43.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:43.364 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:43.393 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:43.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:43.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:43.496 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:43.535 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:43.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:43.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:43.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:43 smithi194 bash[92666]: cluster 2023-10-09T05:36:42.405321+0000 mgr.y (mgr.25277) 1464 : cluster 0 pgmap v1572: 89 pgs: 89 active+clean; 2.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T05:36:43.639 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:43.665 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:43.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:43.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:43.727 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:43 smithi133 bash[105589]: cluster 2023-10-09T05:36:42.405321+0000 mgr.y (mgr.25277) 1464 : cluster 0 pgmap v1572: 89 pgs: 89 active+clean; 2.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T05:36:43.728 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:43 smithi133 bash[102858]: cluster 2023-10-09T05:36:42.405321+0000 mgr.y (mgr.25277) 1464 : cluster 0 pgmap v1572: 89 pgs: 89 active+clean; 2.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T05:36:43.763 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 35 16 51620 51604 92.1399 64.25 0.00794546 0.0108469 2023-10-09T05:36:43.769 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:43.797 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:43.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:43.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:43.900 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:43.928 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:43.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:43.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:44.031 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:44.057 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:44.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:44.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:44.161 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:44.185 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:44.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:44.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:44.288 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:44.297 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:36:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:36:44] "GET /metrics HTTP/1.1" 200 36476 "" "Prometheus/2.43.0" 2023-10-09T05:36:44.314 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:44.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:44.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:44.419 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:44.446 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:44.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:44.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:44.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:44 smithi133 bash[102858]: audit 2023-10-09T05:36:43.492232+0000 mon.a (mon.0) 1826 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:36:44.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:44 smithi133 bash[102858]: audit 2023-10-09T05:36:43.492991+0000 mon.a (mon.0) 1827 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:36:44.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:44 smithi133 bash[105589]: audit 2023-10-09T05:36:43.492232+0000 mon.a (mon.0) 1826 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:36:44.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:44 smithi133 bash[105589]: audit 2023-10-09T05:36:43.492991+0000 mon.a (mon.0) 1827 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:36:44.550 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:44.579 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:44.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:44.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:44.683 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:44.711 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:44.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:44.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:44.763 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 36 16 52620 52604 91.3164 62.5 0.00754667 0.010944 2023-10-09T05:36:44.813 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:44.853 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:44.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:44.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:44.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:44 smithi194 bash[92666]: audit 2023-10-09T05:36:43.492232+0000 mon.a (mon.0) 1826 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:36:44.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:44 smithi194 bash[92666]: audit 2023-10-09T05:36:43.492991+0000 mon.a (mon.0) 1827 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:36:44.957 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:44.988 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:44.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:44.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:45.091 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:45.117 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:45.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:45.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:45.222 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:45.258 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:45.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:45.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:45.364 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:45.390 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:45.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:45.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:45.494 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:45.518 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:45.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:45.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:45.620 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:45.653 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:45.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:45.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:45.759 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:45.763 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 37 16 53657 53641 90.6 64.8125 0.00982398 0.0110311 2023-10-09T05:36:45.771 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:45 smithi194 bash[92666]: cluster 2023-10-09T05:36:44.405762+0000 mgr.y (mgr.25277) 1465 : cluster 0 pgmap v1573: 89 pgs: 89 active+clean; 2.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T05:36:45.787 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:45.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:45.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:45.797 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:45 smithi133 bash[102858]: cluster 2023-10-09T05:36:44.405762+0000 mgr.y (mgr.25277) 1465 : cluster 0 pgmap v1573: 89 pgs: 89 active+clean; 2.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T05:36:45.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:45 smithi133 bash[105589]: cluster 2023-10-09T05:36:44.405762+0000 mgr.y (mgr.25277) 1465 : cluster 0 pgmap v1573: 89 pgs: 89 active+clean; 2.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T05:36:45.892 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:45.923 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:45.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:45.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:46.029 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:46.050 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:46.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:46.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:46.153 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:46.191 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:46.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:46.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:46.294 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:46.322 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:46.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:46.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:46.425 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:46.452 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:46.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:46.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:46.558 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:46.587 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:46.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:46.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:46.691 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:46.717 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:46.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:46.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:46.763 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 38 16 54555 54539 89.6926 56.125 0.0175818 0.0111435 2023-10-09T05:36:46.821 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:46.854 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:46.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:46.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:46.958 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:46.984 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:46.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:46.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:47.090 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:47.116 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:47.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:47.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:47.221 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:47.250 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:47.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:47.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:47.356 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:47.381 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:47.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:47.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:47.485 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:47.512 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:47.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:47.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:47.567 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:36:47.616 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:47.651 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:47.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:47.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:47.757 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:47.763 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 39 16 55444 55428 88.8172 55.5625 0.0116573 0.0112526 2023-10-09T05:36:47.766 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:47 smithi194 bash[92666]: cluster 2023-10-09T05:36:46.406537+0000 mgr.y (mgr.25277) 1466 : cluster 0 pgmap v1574: 89 pgs: 89 active+clean; 3.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T05:36:47.781 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:47.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:47.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:47.884 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:47.893 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:47 smithi133 bash[102858]: cluster 2023-10-09T05:36:46.406537+0000 mgr.y (mgr.25277) 1466 : cluster 0 pgmap v1574: 89 pgs: 89 active+clean; 3.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T05:36:47.893 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:47 smithi133 bash[105589]: cluster 2023-10-09T05:36:46.406537+0000 mgr.y (mgr.25277) 1466 : cluster 0 pgmap v1574: 89 pgs: 89 active+clean; 3.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T05:36:47.918 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:47.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:47.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:48.024 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:48.049 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:48.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:48.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:48.155 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:48.181 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:48.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:48.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:48.286 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:48.310 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:48.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:48.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:48.413 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:48.442 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:48.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:48.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:48.548 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:48.572 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:48.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:48.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:48.675 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:48.700 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:48.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:48.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:48.763 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T05:36:48.764024+0000 min lat: 0.00135224 max lat: 0.333322 avg lat: 0.0113319 2023-10-09T05:36:48.764 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:36:48.764 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 40 16 56465 56449 88.192 63.8125 0.0167511 0.0113319 2023-10-09T05:36:48.805 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:48.831 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:48.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:48.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:48.934 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:48.962 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:48.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:48.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:49.065 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:49.093 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:49.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:49.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:49.197 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:49.221 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:49.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:49.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:49.329 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:49.355 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:49.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:49.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:49.460 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:49.488 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:49.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:49.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:49.592 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:49.620 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:49.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:49.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:49.726 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:49.736 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:49 smithi133 bash[105589]: cluster 2023-10-09T05:36:48.407077+0000 mgr.y (mgr.25277) 1467 : cluster 0 pgmap v1575: 89 pgs: 89 active+clean; 3.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 62 MiB/s wr, 994 op/s 2023-10-09T05:36:49.736 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:49 smithi133 bash[102858]: cluster 2023-10-09T05:36:48.407077+0000 mgr.y (mgr.25277) 1467 : cluster 0 pgmap v1575: 89 pgs: 89 active+clean; 3.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 62 MiB/s wr, 994 op/s 2023-10-09T05:36:49.760 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:49.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:49.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:49.766 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 41 16 57395 57379 87.4585 58.125 0.00583205 0.0114283 2023-10-09T05:36:49.815 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:49 smithi194 bash[92666]: cluster 2023-10-09T05:36:48.407077+0000 mgr.y (mgr.25277) 1467 : cluster 0 pgmap v1575: 89 pgs: 89 active+clean; 3.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 62 MiB/s wr, 994 op/s 2023-10-09T05:36:49.863 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:49.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:49.891 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:49.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:49.992 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:50.029 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:50.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:50.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:50.134 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:50.170 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:50.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:50.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:50.273 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:50.298 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:50.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:50.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:50.404 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:50.438 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:50.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:50.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:50.538 INFO:teuthology.orchestra.run.smithi133.stderr:set pool 14 pg_num to 26 2023-10-09T05:36:50.544 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:50.581 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:50.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:50.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:50.684 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:50.694 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:50 smithi133 bash[102858]: audit 2023-10-09T05:36:50.418262+0000 mon.c (mon.2) 136 : audit 1 from='client.? 172.21.15.133:0/3431508143' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T05:36:50.694 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:50 smithi133 bash[102858]: audit 2023-10-09T05:36:50.418923+0000 mon.a (mon.0) 1828 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T05:36:50.694 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:50 smithi133 bash[105589]: audit 2023-10-09T05:36:50.418262+0000 mon.c (mon.2) 136 : audit 1 from='client.? 172.21.15.133:0/3431508143' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T05:36:50.694 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:50 smithi133 bash[105589]: audit 2023-10-09T05:36:50.418923+0000 mon.a (mon.0) 1828 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T05:36:50.710 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:50.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:50.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:50.764 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 42 16 58334 58318 86.7734 58.6875 0.0346755 0.0115176 2023-10-09T05:36:50.817 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:50.827 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:50 smithi194 bash[92666]: audit 2023-10-09T05:36:50.418262+0000 mon.c (mon.2) 136 : audit 1 from='client.? 172.21.15.133:0/3431508143' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T05:36:50.827 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:50 smithi194 bash[92666]: audit 2023-10-09T05:36:50.418923+0000 mon.a (mon.0) 1828 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T05:36:50.842 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:50.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:50.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:50.947 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:50.974 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:50.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:50.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:51.082 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:51.109 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:51.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:51.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:51.217 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:51.244 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:51.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:51.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:51.349 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:51.375 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:51.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:51.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:51.478 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:51.502 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:51.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:51.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:51.606 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:51.614 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:51 smithi194 bash[92666]: cluster 2023-10-09T05:36:50.408183+0000 mgr.y (mgr.25277) 1468 : cluster 0 pgmap v1576: 89 pgs: 89 active+clean; 3.5 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T05:36:51.614 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:51 smithi194 bash[92666]: cluster 2023-10-09T05:36:50.522217+0000 osd.6 (osd.6) 281 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T05:36:51.615 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:51 smithi194 bash[92666]: cluster 2023-10-09T05:36:50.525362+0000 osd.6 (osd.6) 282 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T05:36:51.615 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:51 smithi194 bash[92666]: audit 2023-10-09T05:36:50.526876+0000 mon.a (mon.0) 1829 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]': finished 2023-10-09T05:36:51.615 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:51 smithi194 bash[92666]: cluster 2023-10-09T05:36:50.526949+0000 osd.5 (osd.5) 299 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T05:36:51.615 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:51 smithi194 bash[92666]: cluster 2023-10-09T05:36:50.526952+0000 mon.a (mon.0) 1830 : cluster 0 osdmap e318: 8 total, 8 up, 7 in 2023-10-09T05:36:51.636 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:51.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:51.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:51.739 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:51.768 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 43 16 59365 59349 86.2446 64.4375 0.0267648 0.0115873 2023-10-09T05:36:51.775 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:51.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:51.779 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:51 smithi133 bash[102858]: cluster 2023-10-09T05:36:50.408183+0000 mgr.y (mgr.25277) 1468 : cluster 0 pgmap v1576: 89 pgs: 89 active+clean; 3.5 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T05:36:51.779 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:51 smithi133 bash[102858]: cluster 2023-10-09T05:36:50.522217+0000 osd.6 (osd.6) 281 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T05:36:51.779 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:51 smithi133 bash[102858]: cluster 2023-10-09T05:36:50.525362+0000 osd.6 (osd.6) 282 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T05:36:51.779 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:51 smithi133 bash[102858]: audit 2023-10-09T05:36:50.526876+0000 mon.a (mon.0) 1829 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]': finished 2023-10-09T05:36:51.779 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:51 smithi133 bash[102858]: cluster 2023-10-09T05:36:50.526949+0000 osd.5 (osd.5) 299 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T05:36:51.779 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:51 smithi133 bash[102858]: cluster 2023-10-09T05:36:50.526952+0000 mon.a (mon.0) 1830 : cluster 0 osdmap e318: 8 total, 8 up, 7 in 2023-10-09T05:36:51.780 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:51 smithi133 bash[105589]: cluster 2023-10-09T05:36:50.408183+0000 mgr.y (mgr.25277) 1468 : cluster 0 pgmap v1576: 89 pgs: 89 active+clean; 3.5 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T05:36:51.780 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:51 smithi133 bash[105589]: cluster 2023-10-09T05:36:50.522217+0000 osd.6 (osd.6) 281 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T05:36:51.780 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:51 smithi133 bash[105589]: cluster 2023-10-09T05:36:50.525362+0000 osd.6 (osd.6) 282 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T05:36:51.780 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:51 smithi133 bash[105589]: audit 2023-10-09T05:36:50.526876+0000 mon.a (mon.0) 1829 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]': finished 2023-10-09T05:36:51.780 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:51 smithi133 bash[105589]: cluster 2023-10-09T05:36:50.526949+0000 osd.5 (osd.5) 299 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T05:36:51.780 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:51 smithi133 bash[105589]: cluster 2023-10-09T05:36:50.526952+0000 mon.a (mon.0) 1830 : cluster 0 osdmap e318: 8 total, 8 up, 7 in 2023-10-09T05:36:51.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:51.881 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:51.907 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:51.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:51.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:52.011 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:52.041 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:52.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:52.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:52.144 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:52.169 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:52.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:52.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:52.275 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:52.301 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:52.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:52.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:52.403 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:52.431 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:52.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:52.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:52.536 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:52.620 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:52.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:52.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:52.728 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:52.768 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:52 smithi133 bash[102858]: audit 2023-10-09T05:36:52.409855+0000 mon.a (mon.0) 1831 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T05:36:52.769 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:52 smithi133 bash[105589]: audit 2023-10-09T05:36:52.409855+0000 mon.a (mon.0) 1831 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T05:36:52.771 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 44 16 60181 60165 85.4416 51 0.245947 0.0116512 2023-10-09T05:36:52.784 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:52.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:52.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:52.863 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:36:52.864 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:36:52.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:52 smithi194 bash[92666]: audit 2023-10-09T05:36:52.409855+0000 mon.a (mon.0) 1831 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T05:36:52.890 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:52.922 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:52.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:52.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:53.025 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:53.052 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:53.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:53.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:53.154 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:53.178 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:53.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:53.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:53.281 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:53.307 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:53.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:53.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:53.410 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:53.439 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:53.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:53.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:53.542 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:53.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:53 smithi133 bash[105589]: cluster 2023-10-09T05:36:52.408710+0000 mgr.y (mgr.25277) 1469 : cluster 0 pgmap v1578: 89 pgs: 89 active+clean; 3.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 62 MiB/s wr, 998 op/s 2023-10-09T05:36:53.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:53 smithi133 bash[105589]: cluster 2023-10-09T05:36:52.531521+0000 osd.5 (osd.5) 300 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T05:36:53.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:53 smithi133 bash[105589]: cluster 2023-10-09T05:36:52.534193+0000 osd.6 (osd.6) 283 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T05:36:53.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:53 smithi133 bash[105589]: cluster 2023-10-09T05:36:52.534217+0000 osd.7 (osd.7) 278 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T05:36:53.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:53 smithi133 bash[105589]: audit 2023-10-09T05:36:52.535039+0000 mon.a (mon.0) 1832 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T05:36:53.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:53 smithi133 bash[105589]: cluster 2023-10-09T05:36:52.535609+0000 mon.a (mon.0) 1833 : cluster 0 osdmap e319: 8 total, 8 up, 7 in 2023-10-09T05:36:53.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:53 smithi133 bash[105589]: cluster 2023-10-09T05:36:52.714461+0000 osd.4 (osd.4) 287 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T05:36:53.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:53 smithi133 bash[102858]: cluster 2023-10-09T05:36:52.408710+0000 mgr.y (mgr.25277) 1469 : cluster 0 pgmap v1578: 89 pgs: 89 active+clean; 3.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 62 MiB/s wr, 998 op/s 2023-10-09T05:36:53.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:53 smithi133 bash[102858]: cluster 2023-10-09T05:36:52.531521+0000 osd.5 (osd.5) 300 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T05:36:53.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:53 smithi133 bash[102858]: cluster 2023-10-09T05:36:52.534193+0000 osd.6 (osd.6) 283 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T05:36:53.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:53 smithi133 bash[102858]: cluster 2023-10-09T05:36:52.534217+0000 osd.7 (osd.7) 278 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T05:36:53.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:53 smithi133 bash[102858]: audit 2023-10-09T05:36:52.535039+0000 mon.a (mon.0) 1832 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T05:36:53.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:53 smithi133 bash[102858]: cluster 2023-10-09T05:36:52.535609+0000 mon.a (mon.0) 1833 : cluster 0 osdmap e319: 8 total, 8 up, 7 in 2023-10-09T05:36:53.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:53 smithi133 bash[102858]: cluster 2023-10-09T05:36:52.714461+0000 osd.4 (osd.4) 287 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T05:36:53.568 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:53.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:53.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:53.673 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:53.701 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:53.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:53.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:53.769 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 45 16 60382 60366 83.8223 12.5625 0.0233082 0.0119233 2023-10-09T05:36:53.806 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:53.838 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:53.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:53.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:53.873 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:53 smithi194 bash[92666]: cluster 2023-10-09T05:36:52.408710+0000 mgr.y (mgr.25277) 1469 : cluster 0 pgmap v1578: 89 pgs: 89 active+clean; 3.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 62 MiB/s wr, 998 op/s 2023-10-09T05:36:53.873 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:53 smithi194 bash[92666]: cluster 2023-10-09T05:36:52.531521+0000 osd.5 (osd.5) 300 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T05:36:53.874 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:53 smithi194 bash[92666]: cluster 2023-10-09T05:36:52.534193+0000 osd.6 (osd.6) 283 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T05:36:53.874 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:53 smithi194 bash[92666]: cluster 2023-10-09T05:36:52.534217+0000 osd.7 (osd.7) 278 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T05:36:53.874 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:53 smithi194 bash[92666]: audit 2023-10-09T05:36:52.535039+0000 mon.a (mon.0) 1832 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T05:36:53.874 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:53 smithi194 bash[92666]: cluster 2023-10-09T05:36:52.535609+0000 mon.a (mon.0) 1833 : cluster 0 osdmap e319: 8 total, 8 up, 7 in 2023-10-09T05:36:53.874 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:53 smithi194 bash[92666]: cluster 2023-10-09T05:36:52.714461+0000 osd.4 (osd.4) 287 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T05:36:53.875 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T05:36:53.945 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:53.971 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:53.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:53.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:54.075 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:54.103 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:54.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:54.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:54.207 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:54.234 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:54.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:54.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:54.281 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:36:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:36:54] "GET /metrics HTTP/1.1" 200 36476 "" "Prometheus/2.43.0" 2023-10-09T05:36:54.338 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:54.366 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:54.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:54.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:54.478 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:54.503 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:54.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:54.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:54.541 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:54 smithi133 bash[102858]: cluster 2023-10-09T05:36:53.538938+0000 osd.5 (osd.5) 301 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T05:36:54.541 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:54 smithi133 bash[102858]: cluster 2023-10-09T05:36:53.540683+0000 mon.a (mon.0) 1834 : cluster 0 osdmap e320: 8 total, 8 up, 7 in 2023-10-09T05:36:54.542 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:54 smithi133 bash[102858]: cluster 2023-10-09T05:36:53.540761+0000 osd.6 (osd.6) 284 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T05:36:54.542 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:54 smithi133 bash[102858]: cluster 2023-10-09T05:36:53.541478+0000 osd.5 (osd.5) 302 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T05:36:54.542 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:54 smithi133 bash[102858]: cluster 2023-10-09T05:36:53.542040+0000 osd.6 (osd.6) 285 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T05:36:54.542 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:54 smithi133 bash[102858]: cluster 2023-10-09T05:36:53.545018+0000 osd.6 (osd.6) 286 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T05:36:54.607 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:54.634 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:54.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:54.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:54.738 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:54.748 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:54 smithi194 bash[92666]: cluster 2023-10-09T05:36:53.538938+0000 osd.5 (osd.5) 301 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T05:36:54.748 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:54 smithi194 bash[92666]: cluster 2023-10-09T05:36:53.540683+0000 mon.a (mon.0) 1834 : cluster 0 osdmap e320: 8 total, 8 up, 7 in 2023-10-09T05:36:54.749 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:54 smithi194 bash[92666]: cluster 2023-10-09T05:36:53.540761+0000 osd.6 (osd.6) 284 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T05:36:54.749 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:54 smithi194 bash[92666]: cluster 2023-10-09T05:36:53.541478+0000 osd.5 (osd.5) 302 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T05:36:54.749 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:54 smithi194 bash[92666]: cluster 2023-10-09T05:36:53.542040+0000 osd.6 (osd.6) 285 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T05:36:54.749 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:54 smithi194 bash[92666]: cluster 2023-10-09T05:36:53.545018+0000 osd.6 (osd.6) 286 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T05:36:54.763 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:54.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:54.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:54.769 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 46 16 61670 61654 83.7499 80.5 0.00516695 0.0119351 2023-10-09T05:36:54.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:54 smithi133 bash[105589]: cluster 2023-10-09T05:36:53.538938+0000 osd.5 (osd.5) 301 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T05:36:54.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:54 smithi133 bash[105589]: cluster 2023-10-09T05:36:53.540683+0000 mon.a (mon.0) 1834 : cluster 0 osdmap e320: 8 total, 8 up, 7 in 2023-10-09T05:36:54.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:54 smithi133 bash[105589]: cluster 2023-10-09T05:36:53.540761+0000 osd.6 (osd.6) 284 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T05:36:54.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:54 smithi133 bash[105589]: cluster 2023-10-09T05:36:53.541478+0000 osd.5 (osd.5) 302 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T05:36:54.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:54 smithi133 bash[105589]: cluster 2023-10-09T05:36:53.542040+0000 osd.6 (osd.6) 285 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T05:36:54.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:54 smithi133 bash[105589]: cluster 2023-10-09T05:36:53.545018+0000 osd.6 (osd.6) 286 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T05:36:54.867 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:54.904 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:54.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:54.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:55.008 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:55.036 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:55.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:55.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:55.140 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:55.165 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:55.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:55.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:55.268 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:55.296 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:55.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:55.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:55.400 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:55.427 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:55.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:55.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:55.530 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:55.562 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:55.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:55.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:55.666 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:55.694 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:55.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:55.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:55.770 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 47 16 62988 62972 83.7205 82.375 0.00532646 0.0119389 2023-10-09T05:36:55.799 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:55.825 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:55.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:55.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:55.845 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:55 smithi133 bash[102858]: cluster 2023-10-09T05:36:54.409189+0000 mgr.y (mgr.25277) 1470 : cluster 0 pgmap v1581: 99 pgs: 10 unknown, 89 active+clean; 3.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 49 MiB/s wr, 789 op/s 2023-10-09T05:36:55.845 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:55 smithi133 bash[105589]: cluster 2023-10-09T05:36:54.409189+0000 mgr.y (mgr.25277) 1470 : cluster 0 pgmap v1581: 99 pgs: 10 unknown, 89 active+clean; 3.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 49 MiB/s wr, 789 op/s 2023-10-09T05:36:55.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:55 smithi194 bash[92666]: cluster 2023-10-09T05:36:54.409189+0000 mgr.y (mgr.25277) 1470 : cluster 0 pgmap v1581: 99 pgs: 10 unknown, 89 active+clean; 3.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 49 MiB/s wr, 789 op/s 2023-10-09T05:36:55.931 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:55.960 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:55.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:55.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:56.063 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:56.087 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:56.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:56.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:56.192 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:56.218 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:56.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:56.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:56.323 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:56.348 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:56.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:56.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:56.452 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:56.480 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:56.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:56.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:56.586 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:56.613 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:56.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:56.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:56.718 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:56.743 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:56.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:56.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:56.770 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 48 16 64289 64273 83.6702 81.3125 0.00687087 0.0119462 2023-10-09T05:36:56.848 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:56.877 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:56.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:56.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:56.986 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:57.015 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:57.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:57.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:57.120 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:57.150 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:57.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:57.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:57.254 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:57.283 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:57.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:57.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:57.388 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:57.413 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:57.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:57.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:57.517 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:57.543 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:57.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:57.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:57.647 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:57.677 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:57.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:57.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:57.770 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 49 16 65580 65564 83.6092 80.6875 0.0174818 0.0119542 2023-10-09T05:36:57.781 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:57.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:57 smithi133 bash[102858]: cluster 2023-10-09T05:36:56.409976+0000 mgr.y (mgr.25277) 1471 : cluster 0 pgmap v1582: 99 pgs: 1 unknown, 98 active+clean; 3.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 49 MiB/s wr, 790 op/s; 32 KiB/s, 0 objects/s recovering 2023-10-09T05:36:57.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:57 smithi133 bash[105589]: cluster 2023-10-09T05:36:56.409976+0000 mgr.y (mgr.25277) 1471 : cluster 0 pgmap v1582: 99 pgs: 1 unknown, 98 active+clean; 3.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 49 MiB/s wr, 790 op/s; 32 KiB/s, 0 objects/s recovering 2023-10-09T05:36:57.811 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:57.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:57.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:57.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:57 smithi194 bash[92666]: cluster 2023-10-09T05:36:56.409976+0000 mgr.y (mgr.25277) 1471 : cluster 0 pgmap v1582: 99 pgs: 1 unknown, 98 active+clean; 3.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 49 MiB/s wr, 790 op/s; 32 KiB/s, 0 objects/s recovering 2023-10-09T05:36:57.916 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:57.941 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:57.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:57.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:57.986 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:36:58.015 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:36:58.045 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:58.046 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:36:58.075 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:58.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:58.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:58.103 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:36:58.149 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:36:58.178 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:58.186 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:36:58.217 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:58.218 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:36:58.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:58.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:58.247 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:36:58.249 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:36:58.250 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:36:58.250 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:36:58.250 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:36:58.250 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:36:58.250 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:36:58.250 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:36:58.250 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:36:58.321 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:58.350 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:58.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:58.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:58.458 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:58.487 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:58.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:58.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:58.570 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is set 2023-10-09T05:36:58.592 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:58.619 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:58.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:58.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:58.680 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:36:58.722 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:58.731 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:58 smithi133 bash[105589]: audit 2023-10-09T05:36:57.953784+0000 mon.a (mon.0) 1835 : audit 0 from='client.? 172.21.15.133:0/2623530143' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:36:58.731 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:58 smithi133 bash[105589]: audit 2023-10-09T05:36:58.411269+0000 mon.a (mon.0) 1836 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T05:36:58.731 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:58 smithi133 bash[105589]: audit 2023-10-09T05:36:58.465814+0000 mon.b (mon.1) 32 : audit 1 from='client.? 172.21.15.133:0/3706310693' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:36:58.732 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:58 smithi133 bash[105589]: audit 2023-10-09T05:36:58.466542+0000 mon.a (mon.0) 1837 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:36:58.732 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:58 smithi133 bash[105589]: audit 2023-10-09T05:36:58.489653+0000 mon.a (mon.0) 1838 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:36:58.732 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:58 smithi133 bash[102858]: audit 2023-10-09T05:36:57.953784+0000 mon.a (mon.0) 1835 : audit 0 from='client.? 172.21.15.133:0/2623530143' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:36:58.732 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:58 smithi133 bash[102858]: audit 2023-10-09T05:36:58.411269+0000 mon.a (mon.0) 1836 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T05:36:58.732 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:58 smithi133 bash[102858]: audit 2023-10-09T05:36:58.465814+0000 mon.b (mon.1) 32 : audit 1 from='client.? 172.21.15.133:0/3706310693' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:36:58.732 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:58 smithi133 bash[102858]: audit 2023-10-09T05:36:58.466542+0000 mon.a (mon.0) 1837 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:36:58.732 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:58 smithi133 bash[102858]: audit 2023-10-09T05:36:58.489653+0000 mon.a (mon.0) 1838 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:36:58.749 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:58.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:58.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:58.770 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 50 16 66613 66597 83.2282 64.5625 0.00701094 0.0119999 2023-10-09T05:36:58.853 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:58.880 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:58.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:58.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:58.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:58 smithi194 bash[92666]: audit 2023-10-09T05:36:57.953784+0000 mon.a (mon.0) 1835 : audit 0 from='client.? 172.21.15.133:0/2623530143' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:36:58.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:58 smithi194 bash[92666]: audit 2023-10-09T05:36:58.411269+0000 mon.a (mon.0) 1836 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T05:36:58.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:58 smithi194 bash[92666]: audit 2023-10-09T05:36:58.465814+0000 mon.b (mon.1) 32 : audit 1 from='client.? 172.21.15.133:0/3706310693' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:36:58.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:58 smithi194 bash[92666]: audit 2023-10-09T05:36:58.466542+0000 mon.a (mon.0) 1837 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:36:58.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:58 smithi194 bash[92666]: audit 2023-10-09T05:36:58.489653+0000 mon.a (mon.0) 1838 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:36:58.985 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:59.001 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 6, 1, 7, 5] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:36:59.002 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:36:59.002 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_9 2023-10-09T05:36:59.002 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:36:59.010 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:59.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:59.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:59.114 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:59.138 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:59.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:59.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:59.241 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:59.271 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:59.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:59.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:59.375 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:59.405 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:59.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:59.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:59.509 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:59.534 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:59.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:59.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:59.558 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[105589]: cluster 2023-10-09T05:36:58.410474+0000 mgr.y (mgr.25277) 1472 : cluster 0 pgmap v1583: 99 pgs: 99 active+clean; 3.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 42 MiB/s wr, 676 op/s; 32 KiB/s, 0 objects/s recovering 2023-10-09T05:36:59.559 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[105589]: cluster 2023-10-09T05:36:58.545299+0000 mon.a (mon.0) 1839 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:36:59.559 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[105589]: cluster 2023-10-09T05:36:58.557644+0000 osd.5 (osd.5) 303 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T05:36:59.559 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[105589]: cluster 2023-10-09T05:36:58.557744+0000 osd.4 (osd.4) 288 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T05:36:59.559 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[105589]: audit 2023-10-09T05:36:58.561706+0000 mon.a (mon.0) 1840 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T05:36:59.559 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[105589]: audit 2023-10-09T05:36:58.561931+0000 mon.a (mon.0) 1841 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:36:59.559 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[105589]: cluster 2023-10-09T05:36:58.562535+0000 mon.a (mon.0) 1842 : cluster 0 osdmap e321: 8 total, 8 up, 7 in 2023-10-09T05:36:59.559 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[105589]: cluster 2023-10-09T05:36:58.567050+0000 osd.5 (osd.5) 304 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T05:36:59.559 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[105589]: cluster 2023-10-09T05:36:58.569033+0000 osd.6 (osd.6) 287 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T05:36:59.559 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[105589]: cluster 2023-10-09T05:36:58.575988+0000 osd.7 (osd.7) 279 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T05:36:59.559 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[105589]: cluster 2023-10-09T05:36:58.883051+0000 osd.4 (osd.4) 289 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T05:36:59.560 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[105589]: cluster 2023-10-09T05:36:58.886202+0000 mon.a (mon.0) 1843 : cluster 0 osdmap e322: 8 total, 8 up, 7 in 2023-10-09T05:36:59.560 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[105589]: cluster 2023-10-09T05:36:58.891298+0000 osd.6 (osd.6) 288 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T05:36:59.560 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[105589]: cluster 2023-10-09T05:36:58.894634+0000 osd.7 (osd.7) 280 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T05:36:59.560 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[105589]: cluster 2023-10-09T05:36:58.897277+0000 osd.5 (osd.5) 305 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T05:36:59.560 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[105589]: cluster 2023-10-09T05:36:58.898859+0000 osd.5 (osd.5) 306 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T05:36:59.560 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[102858]: cluster 2023-10-09T05:36:58.410474+0000 mgr.y (mgr.25277) 1472 : cluster 0 pgmap v1583: 99 pgs: 99 active+clean; 3.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 42 MiB/s wr, 676 op/s; 32 KiB/s, 0 objects/s recovering 2023-10-09T05:36:59.561 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[102858]: cluster 2023-10-09T05:36:58.545299+0000 mon.a (mon.0) 1839 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:36:59.561 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[102858]: cluster 2023-10-09T05:36:58.557644+0000 osd.5 (osd.5) 303 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T05:36:59.561 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[102858]: cluster 2023-10-09T05:36:58.557744+0000 osd.4 (osd.4) 288 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T05:36:59.561 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[102858]: audit 2023-10-09T05:36:58.561706+0000 mon.a (mon.0) 1840 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T05:36:59.561 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[102858]: audit 2023-10-09T05:36:58.561931+0000 mon.a (mon.0) 1841 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:36:59.561 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[102858]: cluster 2023-10-09T05:36:58.562535+0000 mon.a (mon.0) 1842 : cluster 0 osdmap e321: 8 total, 8 up, 7 in 2023-10-09T05:36:59.561 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[102858]: cluster 2023-10-09T05:36:58.567050+0000 osd.5 (osd.5) 304 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T05:36:59.561 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[102858]: cluster 2023-10-09T05:36:58.569033+0000 osd.6 (osd.6) 287 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T05:36:59.561 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[102858]: cluster 2023-10-09T05:36:58.575988+0000 osd.7 (osd.7) 279 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T05:36:59.561 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[102858]: cluster 2023-10-09T05:36:58.883051+0000 osd.4 (osd.4) 289 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T05:36:59.561 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[102858]: cluster 2023-10-09T05:36:58.886202+0000 mon.a (mon.0) 1843 : cluster 0 osdmap e322: 8 total, 8 up, 7 in 2023-10-09T05:36:59.562 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[102858]: cluster 2023-10-09T05:36:58.891298+0000 osd.6 (osd.6) 288 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T05:36:59.562 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[102858]: cluster 2023-10-09T05:36:58.894634+0000 osd.7 (osd.7) 280 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T05:36:59.562 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[102858]: cluster 2023-10-09T05:36:58.897277+0000 osd.5 (osd.5) 305 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T05:36:59.562 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:36:59 smithi133 bash[102858]: cluster 2023-10-09T05:36:58.898859+0000 osd.5 (osd.5) 306 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T05:36:59.638 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:59.664 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:59.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:59.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:59.766 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:59.770 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 51 16 66982 66966 82.0485 23.0625 0.00396612 0.0119648 2023-10-09T05:36:59.793 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:59.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:59.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:59.853 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:59 smithi194 bash[92666]: cluster 2023-10-09T05:36:58.410474+0000 mgr.y (mgr.25277) 1472 : cluster 0 pgmap v1583: 99 pgs: 99 active+clean; 3.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 42 MiB/s wr, 676 op/s; 32 KiB/s, 0 objects/s recovering 2023-10-09T05:36:59.853 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:59 smithi194 bash[92666]: cluster 2023-10-09T05:36:58.545299+0000 mon.a (mon.0) 1839 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:36:59.853 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:59 smithi194 bash[92666]: cluster 2023-10-09T05:36:58.557644+0000 osd.5 (osd.5) 303 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T05:36:59.853 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:59 smithi194 bash[92666]: cluster 2023-10-09T05:36:58.557744+0000 osd.4 (osd.4) 288 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T05:36:59.853 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:59 smithi194 bash[92666]: audit 2023-10-09T05:36:58.561706+0000 mon.a (mon.0) 1840 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T05:36:59.853 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:59 smithi194 bash[92666]: audit 2023-10-09T05:36:58.561931+0000 mon.a (mon.0) 1841 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:36:59.854 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:59 smithi194 bash[92666]: cluster 2023-10-09T05:36:58.562535+0000 mon.a (mon.0) 1842 : cluster 0 osdmap e321: 8 total, 8 up, 7 in 2023-10-09T05:36:59.854 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:59 smithi194 bash[92666]: cluster 2023-10-09T05:36:58.567050+0000 osd.5 (osd.5) 304 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T05:36:59.854 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:59 smithi194 bash[92666]: cluster 2023-10-09T05:36:58.569033+0000 osd.6 (osd.6) 287 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T05:36:59.854 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:59 smithi194 bash[92666]: cluster 2023-10-09T05:36:58.575988+0000 osd.7 (osd.7) 279 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T05:36:59.854 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:59 smithi194 bash[92666]: cluster 2023-10-09T05:36:58.883051+0000 osd.4 (osd.4) 289 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T05:36:59.854 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:59 smithi194 bash[92666]: cluster 2023-10-09T05:36:58.886202+0000 mon.a (mon.0) 1843 : cluster 0 osdmap e322: 8 total, 8 up, 7 in 2023-10-09T05:36:59.854 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:59 smithi194 bash[92666]: cluster 2023-10-09T05:36:58.891298+0000 osd.6 (osd.6) 288 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T05:36:59.854 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:59 smithi194 bash[92666]: cluster 2023-10-09T05:36:58.894634+0000 osd.7 (osd.7) 280 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T05:36:59.854 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:59 smithi194 bash[92666]: cluster 2023-10-09T05:36:58.897277+0000 osd.5 (osd.5) 305 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T05:36:59.855 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:36:59 smithi194 bash[92666]: cluster 2023-10-09T05:36:58.898859+0000 osd.5 (osd.5) 306 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T05:36:59.897 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:59.924 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:59.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:36:59.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:00.027 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:00.051 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:00.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:00.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:00.154 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:00.177 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:00.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:00.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:00.281 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:00.311 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:00.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:00.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:00.419 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:00.443 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:00.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:00.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:00.547 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:00.570 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:00.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:00.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:00.674 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:00.696 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:00.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:00.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:00.770 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 52 16 67696 67680 81.3288 44.625 0.0108825 0.0122895 2023-10-09T05:37:00.799 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:00.843 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:00.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:00.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:00.950 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:00.959 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:00 smithi194 bash[92666]: cluster 2023-10-09T05:36:59.894563+0000 osd.5 (osd.5) 307 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T05:37:00.959 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:00 smithi194 bash[92666]: cluster 2023-10-09T05:36:59.898369+0000 mon.a (mon.0) 1844 : cluster 0 osdmap e323: 8 total, 8 up, 7 in 2023-10-09T05:37:00.959 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:00 smithi194 bash[92666]: cluster 2023-10-09T05:37:00.064210+0000 osd.4 (osd.4) 290 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T05:37:00.960 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:00 smithi194 bash[92666]: cluster 2023-10-09T05:37:00.064226+0000 osd.7 (osd.7) 281 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T05:37:00.960 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:00 smithi194 bash[92666]: cluster 2023-10-09T05:37:00.411218+0000 mgr.y (mgr.25277) 1473 : cluster 0 pgmap v1587: 99 pgs: 1 activating+undersized+degraded+remapped, 98 active+clean; 4.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s; 2091/199317 objects degraded (1.049%); 2091/199317 objects misplaced (1.049%); 43 KiB/s, 0 objects/s recovering 2023-10-09T05:37:00.976 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:00.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:00.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:01.079 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:01.103 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:01.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:01.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:01.131 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:00 smithi133 bash[102858]: cluster 2023-10-09T05:36:59.894563+0000 osd.5 (osd.5) 307 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T05:37:01.131 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:00 smithi133 bash[102858]: cluster 2023-10-09T05:36:59.898369+0000 mon.a (mon.0) 1844 : cluster 0 osdmap e323: 8 total, 8 up, 7 in 2023-10-09T05:37:01.131 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:00 smithi133 bash[102858]: cluster 2023-10-09T05:37:00.064210+0000 osd.4 (osd.4) 290 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T05:37:01.131 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:00 smithi133 bash[102858]: cluster 2023-10-09T05:37:00.064226+0000 osd.7 (osd.7) 281 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T05:37:01.131 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:00 smithi133 bash[102858]: cluster 2023-10-09T05:37:00.411218+0000 mgr.y (mgr.25277) 1473 : cluster 0 pgmap v1587: 99 pgs: 1 activating+undersized+degraded+remapped, 98 active+clean; 4.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s; 2091/199317 objects degraded (1.049%); 2091/199317 objects misplaced (1.049%); 43 KiB/s, 0 objects/s recovering 2023-10-09T05:37:01.131 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:00 smithi133 bash[105589]: cluster 2023-10-09T05:36:59.894563+0000 osd.5 (osd.5) 307 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T05:37:01.131 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:00 smithi133 bash[105589]: cluster 2023-10-09T05:36:59.898369+0000 mon.a (mon.0) 1844 : cluster 0 osdmap e323: 8 total, 8 up, 7 in 2023-10-09T05:37:01.132 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:00 smithi133 bash[105589]: cluster 2023-10-09T05:37:00.064210+0000 osd.4 (osd.4) 290 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T05:37:01.132 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:00 smithi133 bash[105589]: cluster 2023-10-09T05:37:00.064226+0000 osd.7 (osd.7) 281 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T05:37:01.132 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:00 smithi133 bash[105589]: cluster 2023-10-09T05:37:00.411218+0000 mgr.y (mgr.25277) 1473 : cluster 0 pgmap v1587: 99 pgs: 1 activating+undersized+degraded+remapped, 98 active+clean; 4.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s; 2091/199317 objects degraded (1.049%); 2091/199317 objects misplaced (1.049%); 43 KiB/s, 0 objects/s recovering 2023-10-09T05:37:01.209 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:01.243 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:01.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:01.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:01.351 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:01.379 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:01.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:01.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:01.486 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:01.520 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:01.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:01.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:01.624 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:01.651 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:01.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:01.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:01.754 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:01.773 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 53 16 68816 68800 81.1101 70 0.00891887 0.0123224 2023-10-09T05:37:01.778 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:01.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:01.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:01.884 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:01.914 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:01.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:01.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:02.019 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:02.029 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:01 smithi133 bash[105589]: cluster 2023-10-09T05:37:00.064226+0000 osd.6 (osd.6) 289 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T05:37:02.029 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:01 smithi133 bash[105589]: cluster 2023-10-09T05:37:00.892484+0000 mon.a (mon.0) 1845 : cluster 3 Health check failed: Degraded data redundancy: 2091/199317 objects degraded (1.049%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:37:02.029 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:01 smithi133 bash[102858]: cluster 2023-10-09T05:37:00.064226+0000 osd.6 (osd.6) 289 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T05:37:02.029 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:01 smithi133 bash[102858]: cluster 2023-10-09T05:37:00.892484+0000 mon.a (mon.0) 1845 : cluster 3 Health check failed: Degraded data redundancy: 2091/199317 objects degraded (1.049%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:37:02.048 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:02.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:02.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:02.154 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:02.165 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:01 smithi194 bash[92666]: cluster 2023-10-09T05:37:00.064226+0000 osd.6 (osd.6) 289 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T05:37:02.165 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:01 smithi194 bash[92666]: cluster 2023-10-09T05:37:00.892484+0000 mon.a (mon.0) 1845 : cluster 3 Health check failed: Degraded data redundancy: 2091/199317 objects degraded (1.049%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:37:02.182 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:02.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:02.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:02.286 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:02.316 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:02.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:02.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:02.420 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:02.444 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:02.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:02.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:02.548 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:02.571 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:02.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:02.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:02.675 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:02.699 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:02.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:02.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:02.773 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 54 16 69874 69858 80.8326 66.125 0.0158741 0.0123656 2023-10-09T05:37:02.803 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:02.827 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:02.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:02.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:02.931 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:02.960 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:02.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:02.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:03.066 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:03.073 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:02 smithi194 bash[92666]: cluster 2023-10-09T05:37:02.411907+0000 mgr.y (mgr.25277) 1474 : cluster 0 pgmap v1588: 99 pgs: 1 activating+undersized+degraded+remapped, 98 active+clean; 4.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 57 MiB/s wr, 919 op/s; 2091/201531 objects degraded (1.038%); 2091/201531 objects misplaced (1.038%); 11 KiB/s, 0 objects/s recovering 2023-10-09T05:37:03.091 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:03.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:03.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:03.132 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:37:03.146 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:37:03.186 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:02 smithi133 bash[105589]: cluster 2023-10-09T05:37:02.411907+0000 mgr.y (mgr.25277) 1474 : cluster 0 pgmap v1588: 99 pgs: 1 activating+undersized+degraded+remapped, 98 active+clean; 4.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 57 MiB/s wr, 919 op/s; 2091/201531 objects degraded (1.038%); 2091/201531 objects misplaced (1.038%); 11 KiB/s, 0 objects/s recovering 2023-10-09T05:37:03.187 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:02 smithi133 bash[102858]: cluster 2023-10-09T05:37:02.411907+0000 mgr.y (mgr.25277) 1474 : cluster 0 pgmap v1588: 99 pgs: 1 activating+undersized+degraded+remapped, 98 active+clean; 4.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 57 MiB/s wr, 919 op/s; 2091/201531 objects degraded (1.038%); 2091/201531 objects misplaced (1.038%); 11 KiB/s, 0 objects/s recovering 2023-10-09T05:37:03.195 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:03.252 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:03.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:03.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:03.357 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:03.385 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:03.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:03.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:03.488 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:03.515 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:03.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:03.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:03.620 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:03.656 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:03.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:03.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:03.761 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:03.773 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 55 16 70822 70806 80.4401 59.25 0.00556095 0.0124257 2023-10-09T05:37:03.786 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:03.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:03.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:03.844 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:37:03.885 INFO:teuthology.orchestra.run.smithi194.stdout:{ 2023-10-09T05:37:03.885 INFO:teuthology.orchestra.run.smithi194.stdout: "size": 20, 2023-10-09T05:37:03.885 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 600, 2023-10-09T05:37:03.885 INFO:teuthology.orchestra.run.smithi194.stdout: "ops": [ 2023-10-09T05:37:03.885 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.885 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.45374.0:128791 12.5 12:a7df30dd:::benchmark_data_smithi133_218956_object26049:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-10-09T05:37:03.886 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:27:29.043781+0000", 2023-10-09T05:37:03.886 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 574.83790250699997, 2023-10-09T05:37:03.886 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.27514119599999998, 2023-10-09T05:37:03.886 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:37:03.886 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:03.886 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:37:03.886 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.45374", 2023-10-09T05:37:03.886 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2988063385", 2023-10-09T05:37:03.886 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 128791 2023-10-09T05:37:03.886 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.886 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:37:03.887 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.887 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:37:03.887 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.043781+0000", 2023-10-09T05:37:03.887 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.887 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.887 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.887 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:37:03.887 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.043781+0000", 2023-10-09T05:37:03.887 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.887 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.887 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.887 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:37:03.888 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.043780+0000", 2023-10-09T05:37:03.888 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999995 2023-10-09T05:37:03.888 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.888 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.888 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:37:03.888 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.043783+0000", 2023-10-09T05:37:03.888 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.3020000000000002e-06 2023-10-09T05:37:03.888 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.888 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.888 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:37:03.888 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.043788+0000", 2023-10-09T05:37:03.888 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.259e-06 2023-10-09T05:37:03.889 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.889 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.889 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:37:03.889 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.043791+0000", 2023-10-09T05:37:03.889 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.9040000000000001e-06 2023-10-09T05:37:03.889 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.889 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.889 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:37:03.889 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.043852+0000", 2023-10-09T05:37:03.889 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.1141000000000003e-05 2023-10-09T05:37:03.889 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.890 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.890 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:37:03.890 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.044013+0000", 2023-10-09T05:37:03.890 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00016104300000000001 2023-10-09T05:37:03.890 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.890 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.890 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:37:03.890 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.044159+0000", 2023-10-09T05:37:03.890 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00014595799999999999 2023-10-09T05:37:03.890 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.890 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.890 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:37:03.891 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.052090+0000", 2023-10-09T05:37:03.891 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0079310939999999996 2023-10-09T05:37:03.891 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.891 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.891 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:03.891 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.125773+0000", 2023-10-09T05:37:03.891 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.073683070000000003 2023-10-09T05:37:03.891 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.891 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.891 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:03.891 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.318761+0000", 2023-10-09T05:37:03.891 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.192988401 2023-10-09T05:37:03.892 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.892 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.892 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:37:03.892 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.318851+0000", 2023-10-09T05:37:03.892 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.9759000000000006e-05 2023-10-09T05:37:03.892 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.892 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.892 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:37:03.892 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.318922+0000", 2023-10-09T05:37:03.892 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.0636000000000006e-05 2023-10-09T05:37:03.892 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.892 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:37:03.893 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.893 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.893 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.893 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.45374.0:128794 12.5 12:a35c773e:::benchmark_data_smithi133_218956_object26052:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-10-09T05:37:03.893 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:27:29.043979+0000", 2023-10-09T05:37:03.893 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 574.83770434099995, 2023-10-09T05:37:03.893 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.29330039699999999, 2023-10-09T05:37:03.893 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:37:03.893 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:03.893 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:37:03.893 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.45374", 2023-10-09T05:37:03.893 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2988063385", 2023-10-09T05:37:03.894 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 128794 2023-10-09T05:37:03.894 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.894 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:37:03.894 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.894 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:37:03.894 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.043979+0000", 2023-10-09T05:37:03.894 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.894 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.894 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.894 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:37:03.894 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.043979+0000", 2023-10-09T05:37:03.894 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.895 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.895 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.895 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:37:03.895 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.043978+0000", 2023-10-09T05:37:03.895 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:37:03.895 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.895 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.895 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:37:03.895 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.043989+0000", 2023-10-09T05:37:03.895 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.1411e-05 2023-10-09T05:37:03.895 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.895 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.896 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:37:03.896 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.043998+0000", 2023-10-09T05:37:03.896 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.5029999999999997e-06 2023-10-09T05:37:03.896 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.896 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.896 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:37:03.896 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.044005+0000", 2023-10-09T05:37:03.896 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.1679999999999998e-06 2023-10-09T05:37:03.896 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.896 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.896 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:37:03.896 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.048052+0000", 2023-10-09T05:37:03.897 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0040472240000000003 2023-10-09T05:37:03.897 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.897 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.897 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:37:03.897 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.048202+0000", 2023-10-09T05:37:03.897 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00015012399999999999 2023-10-09T05:37:03.897 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.897 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.897 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:37:03.897 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.048346+0000", 2023-10-09T05:37:03.897 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000143378 2023-10-09T05:37:03.897 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.897 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.898 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:37:03.898 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.052368+0000", 2023-10-09T05:37:03.898 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0040220009999999999 2023-10-09T05:37:03.898 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.898 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.898 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:03.898 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.126200+0000", 2023-10-09T05:37:03.898 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.073831820000000006 2023-10-09T05:37:03.898 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.898 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.898 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:03.898 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.337177+0000", 2023-10-09T05:37:03.899 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.21097736 2023-10-09T05:37:03.899 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.899 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.899 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:37:03.899 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.337235+0000", 2023-10-09T05:37:03.899 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.8139e-05 2023-10-09T05:37:03.899 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.899 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.899 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:37:03.899 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.337279+0000", 2023-10-09T05:37:03.899 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.4132999999999999e-05 2023-10-09T05:37:03.899 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.900 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:37:03.900 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.900 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.900 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.900 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.45374.0:128797 12.4 12:281bf68d:::benchmark_data_smithi133_218956_object26055:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-10-09T05:37:03.900 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:27:29.048154+0000", 2023-10-09T05:37:03.900 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 574.833528914, 2023-10-09T05:37:03.900 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.26531340399999997, 2023-10-09T05:37:03.900 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:37:03.900 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:03.900 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:37:03.900 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.45374", 2023-10-09T05:37:03.901 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2988063385", 2023-10-09T05:37:03.901 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 128797 2023-10-09T05:37:03.901 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.901 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:37:03.901 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.901 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:37:03.901 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.048154+0000", 2023-10-09T05:37:03.901 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.901 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.901 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.901 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:37:03.901 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.048154+0000", 2023-10-09T05:37:03.902 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.902 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.902 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.902 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:37:03.902 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.048153+0000", 2023-10-09T05:37:03.902 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:37:03.902 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.902 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.902 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:37:03.902 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.048159+0000", 2023-10-09T05:37:03.902 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.2280000000000003e-06 2023-10-09T05:37:03.902 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.903 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.903 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:37:03.903 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.048171+0000", 2023-10-09T05:37:03.903 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.1803e-05 2023-10-09T05:37:03.903 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.903 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.903 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:37:03.903 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.048177+0000", 2023-10-09T05:37:03.903 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.1890000000000004e-06 2023-10-09T05:37:03.903 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.903 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.903 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:37:03.904 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.048222+0000", 2023-10-09T05:37:03.904 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.4536000000000002e-05 2023-10-09T05:37:03.904 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.904 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.904 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:37:03.904 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.048331+0000", 2023-10-09T05:37:03.904 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000108856 2023-10-09T05:37:03.904 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.904 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.904 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:37:03.904 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.048436+0000", 2023-10-09T05:37:03.904 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000105123 2023-10-09T05:37:03.905 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.905 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.905 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:03.905 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.052804+0000", 2023-10-09T05:37:03.905 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0043685779999999997 2023-10-09T05:37:03.905 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.905 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.905 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:37:03.905 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.052857+0000", 2023-10-09T05:37:03.905 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.3015e-05 2023-10-09T05:37:03.905 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.905 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.906 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:03.906 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.313343+0000", 2023-10-09T05:37:03.906 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.26048575699999998 2023-10-09T05:37:03.906 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.906 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.906 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:37:03.906 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.313412+0000", 2023-10-09T05:37:03.906 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.9036999999999996e-05 2023-10-09T05:37:03.906 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.906 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.906 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:37:03.906 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.313468+0000", 2023-10-09T05:37:03.907 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.5597000000000003e-05 2023-10-09T05:37:03.907 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.907 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:37:03.907 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.907 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.907 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.907 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.45374.0:128802 12.5 12:af1ef82a:::benchmark_data_smithi133_218956_object26060:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-10-09T05:37:03.907 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:27:29.126317+0000", 2023-10-09T05:37:03.907 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 574.75536612600001, 2023-10-09T05:37:03.907 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.21120854, 2023-10-09T05:37:03.907 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:37:03.907 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:03.908 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:37:03.908 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.45374", 2023-10-09T05:37:03.908 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2988063385", 2023-10-09T05:37:03.908 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 128802 2023-10-09T05:37:03.908 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.908 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:37:03.908 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.908 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:37:03.908 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.126317+0000", 2023-10-09T05:37:03.908 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.908 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.908 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.909 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:37:03.909 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.126317+0000", 2023-10-09T05:37:03.909 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.909 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.909 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.909 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:37:03.909 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.126316+0000", 2023-10-09T05:37:03.909 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:37:03.909 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.909 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.909 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:37:03.909 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.126320+0000", 2023-10-09T05:37:03.909 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.76e-06 2023-10-09T05:37:03.910 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.910 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.910 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:37:03.910 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.126328+0000", 2023-10-09T05:37:03.910 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.9959999999999993e-06 2023-10-09T05:37:03.910 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.910 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.910 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:37:03.910 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.126333+0000", 2023-10-09T05:37:03.910 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.3310000000000003e-06 2023-10-09T05:37:03.910 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.910 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.911 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:37:03.911 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.132512+0000", 2023-10-09T05:37:03.911 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0061789219999999999 2023-10-09T05:37:03.911 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.911 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.911 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:37:03.911 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.132677+0000", 2023-10-09T05:37:03.911 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000165069 2023-10-09T05:37:03.911 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.911 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.911 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:37:03.911 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.132854+0000", 2023-10-09T05:37:03.912 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00017674300000000001 2023-10-09T05:37:03.912 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.912 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.912 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:37:03.912 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.144998+0000", 2023-10-09T05:37:03.912 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.012144339000000001 2023-10-09T05:37:03.912 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.912 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.912 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:03.912 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.145069+0000", 2023-10-09T05:37:03.912 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.0196000000000004e-05 2023-10-09T05:37:03.912 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.913 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.913 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:03.913 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.337439+0000", 2023-10-09T05:37:03.913 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.19237079800000001 2023-10-09T05:37:03.913 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.913 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.913 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:37:03.913 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.337490+0000", 2023-10-09T05:37:03.913 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.0374999999999998e-05 2023-10-09T05:37:03.913 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.913 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.913 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:37:03.914 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:27:29.337526+0000", 2023-10-09T05:37:03.914 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.5809000000000002e-05 2023-10-09T05:37:03.914 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.914 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:37:03.914 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.914 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.914 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.914 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.45596.0:51352 13.4 13:2009eb4f:::benchmark_data_smithi133_241067_object51351:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e294)", 2023-10-09T05:37:03.914 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:31:14.655898+0000", 2023-10-09T05:37:03.914 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 349.22578480700003, 2023-10-09T05:37:03.914 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.29091270400000002, 2023-10-09T05:37:03.914 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:37:03.915 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:03.915 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:37:03.915 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.45596", 2023-10-09T05:37:03.915 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:37:03.915 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 51352 2023-10-09T05:37:03.915 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.915 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:37:03.915 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.915 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:37:03.915 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.655898+0000", 2023-10-09T05:37:03.915 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.915 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.916 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.916 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:37:03.916 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.655898+0000", 2023-10-09T05:37:03.916 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.916 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.916 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.916 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:37:03.916 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.655893+0000", 2023-10-09T05:37:03.916 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999943 2023-10-09T05:37:03.916 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.916 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.916 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:37:03.916 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.655968+0000", 2023-10-09T05:37:03.917 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.5671000000000001e-05 2023-10-09T05:37:03.917 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.917 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.917 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:37:03.917 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.655983+0000", 2023-10-09T05:37:03.917 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.4973999999999999e-05 2023-10-09T05:37:03.917 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.917 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.917 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:37:03.917 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.655990+0000", 2023-10-09T05:37:03.917 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.5459999999999997e-06 2023-10-09T05:37:03.917 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.918 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.918 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:37:03.918 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.656054+0000", 2023-10-09T05:37:03.918 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.3723e-05 2023-10-09T05:37:03.918 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.918 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.918 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:37:03.918 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.656231+0000", 2023-10-09T05:37:03.918 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00017759899999999999 2023-10-09T05:37:03.918 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.918 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.918 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:37:03.919 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.656441+0000", 2023-10-09T05:37:03.919 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00021031199999999999 2023-10-09T05:37:03.919 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.919 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.919 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:37:03.919 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.659543+0000", 2023-10-09T05:37:03.919 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0031010909999999998 2023-10-09T05:37:03.919 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.919 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.919 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:03.919 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.662562+0000", 2023-10-09T05:37:03.919 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0030199319999999999 2023-10-09T05:37:03.920 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.920 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.920 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:03.920 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.946646+0000", 2023-10-09T05:37:03.920 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.28408381900000002 2023-10-09T05:37:03.920 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.920 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.920 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:37:03.920 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.946745+0000", 2023-10-09T05:37:03.920 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.8690999999999994e-05 2023-10-09T05:37:03.920 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.920 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.920 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:37:03.921 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.946811+0000", 2023-10-09T05:37:03.921 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.6123000000000004e-05 2023-10-09T05:37:03.921 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.921 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:37:03.921 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.921 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.921 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.921 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.45596.0:51363 13.4 13:2bb20a9c:::benchmark_data_smithi133_241067_object51362:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e294)", 2023-10-09T05:37:03.921 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:31:14.666805+0000", 2023-10-09T05:37:03.921 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 349.21487825499997, 2023-10-09T05:37:03.921 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.28438781000000002, 2023-10-09T05:37:03.921 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:37:03.922 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:03.922 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:37:03.922 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.45596", 2023-10-09T05:37:03.922 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:37:03.922 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 51363 2023-10-09T05:37:03.922 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.922 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:37:03.922 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.922 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:37:03.922 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.666805+0000", 2023-10-09T05:37:03.922 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.922 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.923 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.923 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:37:03.923 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.666805+0000", 2023-10-09T05:37:03.923 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.923 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.923 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.923 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:37:03.923 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.666803+0000", 2023-10-09T05:37:03.923 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999981 2023-10-09T05:37:03.923 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.923 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.923 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:37:03.923 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.666861+0000", 2023-10-09T05:37:03.924 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.7463999999999997e-05 2023-10-09T05:37:03.924 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.924 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.924 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:37:03.924 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.666876+0000", 2023-10-09T05:37:03.924 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.5642e-05 2023-10-09T05:37:03.924 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.924 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.924 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:37:03.924 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.666884+0000", 2023-10-09T05:37:03.924 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.1159999999999999e-06 2023-10-09T05:37:03.924 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.925 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.925 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:37:03.925 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.666957+0000", 2023-10-09T05:37:03.925 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.2237000000000007e-05 2023-10-09T05:37:03.925 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.925 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.925 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:37:03.925 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.667198+0000", 2023-10-09T05:37:03.925 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00024132900000000001 2023-10-09T05:37:03.925 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.925 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.925 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:37:03.925 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.667335+0000", 2023-10-09T05:37:03.926 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000137198 2023-10-09T05:37:03.926 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.926 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.926 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:03.926 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.671865+0000", 2023-10-09T05:37:03.926 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0045301849999999999 2023-10-09T05:37:03.926 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.926 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.926 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:37:03.926 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.671870+0000", 2023-10-09T05:37:03.926 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.2649999999999998e-06 2023-10-09T05:37:03.926 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.927 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.927 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:03.927 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.951060+0000", 2023-10-09T05:37:03.927 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.27919054999999998 2023-10-09T05:37:03.927 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.927 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.927 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:37:03.927 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.951133+0000", 2023-10-09T05:37:03.927 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.3277999999999999e-05 2023-10-09T05:37:03.927 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.927 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.927 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:37:03.927 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.951193+0000", 2023-10-09T05:37:03.928 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.9311999999999997e-05 2023-10-09T05:37:03.928 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.928 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:37:03.928 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.928 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.928 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.928 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.45596.0:51364 13.4 13:20a518f3:::benchmark_data_smithi133_241067_object51363:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e294)", 2023-10-09T05:37:03.928 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:31:14.666902+0000", 2023-10-09T05:37:03.928 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 349.21478125300001, 2023-10-09T05:37:03.928 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.28445334799999999, 2023-10-09T05:37:03.928 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:37:03.928 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:03.928 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:37:03.929 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.45596", 2023-10-09T05:37:03.929 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:37:03.929 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 51364 2023-10-09T05:37:03.929 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.929 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:37:03.929 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.929 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:37:03.929 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.666902+0000", 2023-10-09T05:37:03.929 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.929 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.929 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.929 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:37:03.930 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.666902+0000", 2023-10-09T05:37:03.930 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.930 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.930 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.930 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:37:03.930 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.666901+0000", 2023-10-09T05:37:03.930 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999995 2023-10-09T05:37:03.930 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.930 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.930 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:37:03.930 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.666949+0000", 2023-10-09T05:37:03.930 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.7234000000000002e-05 2023-10-09T05:37:03.931 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.931 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.931 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:37:03.931 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.666958+0000", 2023-10-09T05:37:03.931 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.3999999999999998e-06 2023-10-09T05:37:03.931 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.931 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.931 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:37:03.931 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.666963+0000", 2023-10-09T05:37:03.931 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.2599999999999996e-06 2023-10-09T05:37:03.931 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.931 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.931 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:37:03.932 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.669664+0000", 2023-10-09T05:37:03.932 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.002700418 2023-10-09T05:37:03.932 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.932 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.932 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:37:03.932 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.669811+0000", 2023-10-09T05:37:03.932 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00014760199999999999 2023-10-09T05:37:03.932 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.932 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.932 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:37:03.932 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.669958+0000", 2023-10-09T05:37:03.933 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000146766 2023-10-09T05:37:03.933 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.933 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.933 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:03.933 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.671886+0000", 2023-10-09T05:37:03.933 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.001928319 2023-10-09T05:37:03.933 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.933 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.933 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:37:03.933 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.671962+0000", 2023-10-09T05:37:03.933 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.5507999999999996e-05 2023-10-09T05:37:03.933 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.933 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.934 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:03.934 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.951249+0000", 2023-10-09T05:37:03.934 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.27928680299999997 2023-10-09T05:37:03.934 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.934 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.934 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:37:03.934 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.951306+0000", 2023-10-09T05:37:03.934 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.7800000000000002e-05 2023-10-09T05:37:03.934 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.934 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.934 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:37:03.934 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:31:14.951355+0000", 2023-10-09T05:37:03.935 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.8874e-05 2023-10-09T05:37:03.935 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.935 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:37:03.935 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.935 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.935 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.935 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.45596.0:152496 13.1 13:8168f567:::benchmark_data_smithi133_241067_object54456:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2023-10-09T05:37:03.935 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:33:33.913479+0000", 2023-10-09T05:37:03.935 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 209.96820413399999, 2023-10-09T05:37:03.935 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.52459294400000001, 2023-10-09T05:37:03.935 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:37:03.935 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:03.936 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:37:03.936 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.45596", 2023-10-09T05:37:03.936 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:37:03.936 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 152496 2023-10-09T05:37:03.936 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.936 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:37:03.936 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.936 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:37:03.936 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.913479+0000", 2023-10-09T05:37:03.936 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.936 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.936 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.936 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:37:03.937 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.913479+0000", 2023-10-09T05:37:03.937 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.937 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.937 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.937 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:37:03.937 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.913478+0000", 2023-10-09T05:37:03.937 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:37:03.937 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.937 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.937 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:37:03.937 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.913494+0000", 2023-10-09T05:37:03.937 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.5773999999999998e-05 2023-10-09T05:37:03.938 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.938 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.938 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:37:03.938 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.913506+0000", 2023-10-09T05:37:03.938 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.2294e-05 2023-10-09T05:37:03.938 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.938 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.938 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:37:03.938 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.913524+0000", 2023-10-09T05:37:03.938 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.8278999999999999e-05 2023-10-09T05:37:03.938 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.939 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.939 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:37:03.939 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.913569+0000", 2023-10-09T05:37:03.939 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.4783999999999997e-05 2023-10-09T05:37:03.939 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.939 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.939 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:37:03.939 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.913676+0000", 2023-10-09T05:37:03.939 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00010692900000000001 2023-10-09T05:37:03.939 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.939 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.939 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 2,7", 2023-10-09T05:37:03.939 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.913755+0000", 2023-10-09T05:37:03.940 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.9091000000000006e-05 2023-10-09T05:37:03.940 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.940 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.940 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:37:03.940 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.917484+0000", 2023-10-09T05:37:03.940 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0037294540000000001 2023-10-09T05:37:03.940 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.940 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.940 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:03.940 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.922781+0000", 2023-10-09T05:37:03.940 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0052965260000000002 2023-10-09T05:37:03.940 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.941 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.941 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:03.941 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:34.437782+0000", 2023-10-09T05:37:03.941 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.51500083100000005 2023-10-09T05:37:03.941 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.941 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.941 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:37:03.941 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:34.437936+0000", 2023-10-09T05:37:03.941 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00015414399999999999 2023-10-09T05:37:03.941 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.941 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.941 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:37:03.942 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:34.438072+0000", 2023-10-09T05:37:03.942 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00013613000000000001 2023-10-09T05:37:03.942 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.942 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:37:03.942 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.942 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.942 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.942 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.45596.0:152507 13.1 13:8123a5f4:::benchmark_data_smithi133_241067_object54467:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2023-10-09T05:37:03.942 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:33:33.929221+0000", 2023-10-09T05:37:03.942 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 209.95246173000001, 2023-10-09T05:37:03.942 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.52978479499999998, 2023-10-09T05:37:03.942 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:37:03.943 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:03.943 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:37:03.943 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.45596", 2023-10-09T05:37:03.943 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:37:03.943 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 152507 2023-10-09T05:37:03.943 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.943 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:37:03.943 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.943 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:37:03.943 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.929221+0000", 2023-10-09T05:37:03.943 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.943 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.944 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.944 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:37:03.944 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.929221+0000", 2023-10-09T05:37:03.944 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.944 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.944 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.944 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:37:03.944 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.929220+0000", 2023-10-09T05:37:03.944 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999981 2023-10-09T05:37:03.944 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.944 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.944 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:37:03.945 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.929229+0000", 2023-10-09T05:37:03.945 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.3239999999999992e-06 2023-10-09T05:37:03.945 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.945 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.945 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:37:03.945 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.929246+0000", 2023-10-09T05:37:03.945 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.7388e-05 2023-10-09T05:37:03.945 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.945 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.945 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:37:03.945 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.929254+0000", 2023-10-09T05:37:03.945 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.8499999999999994e-06 2023-10-09T05:37:03.946 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.946 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.946 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:37:03.946 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.929312+0000", 2023-10-09T05:37:03.946 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.8137999999999998e-05 2023-10-09T05:37:03.946 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.946 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.946 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:37:03.946 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.929400+0000", 2023-10-09T05:37:03.946 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.7204000000000001e-05 2023-10-09T05:37:03.946 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.946 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.947 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 2,7", 2023-10-09T05:37:03.947 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.929476+0000", 2023-10-09T05:37:03.947 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.6249999999999997e-05 2023-10-09T05:37:03.947 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.947 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.947 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:37:03.947 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.933345+0000", 2023-10-09T05:37:03.947 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0038694110000000001 2023-10-09T05:37:03.947 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.947 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.947 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:03.947 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.934626+0000", 2023-10-09T05:37:03.947 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0012809 2023-10-09T05:37:03.948 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.948 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.948 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:03.948 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:34.458926+0000", 2023-10-09T05:37:03.948 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.52429960900000006 2023-10-09T05:37:03.948 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.948 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.948 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:37:03.948 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:34.458964+0000", 2023-10-09T05:37:03.948 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.7968e-05 2023-10-09T05:37:03.948 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.948 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.949 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:37:03.949 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:34.459006+0000", 2023-10-09T05:37:03.949 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.2517e-05 2023-10-09T05:37:03.949 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.949 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:37:03.949 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.949 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.949 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.949 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.45596.0:152508 13.4 13:213ff3ab:::benchmark_data_smithi133_241067_object54468:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2023-10-09T05:37:03.949 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:33:33.933188+0000", 2023-10-09T05:37:03.949 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 209.94849563599999, 2023-10-09T05:37:03.949 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.51060209899999998, 2023-10-09T05:37:03.950 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:37:03.950 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:03.950 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:37:03.950 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.45596", 2023-10-09T05:37:03.950 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/1286240652", 2023-10-09T05:37:03.950 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 152508 2023-10-09T05:37:03.950 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.950 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:37:03.950 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.950 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:37:03.950 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.933188+0000", 2023-10-09T05:37:03.950 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.951 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.951 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.951 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:37:03.951 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.933188+0000", 2023-10-09T05:37:03.951 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.951 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.951 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.951 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:37:03.951 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.933186+0000", 2023-10-09T05:37:03.951 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:37:03.951 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.951 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.951 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:37:03.952 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.933195+0000", 2023-10-09T05:37:03.952 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.636e-06 2023-10-09T05:37:03.952 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.952 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.952 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:37:03.952 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.933208+0000", 2023-10-09T05:37:03.952 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.3742e-05 2023-10-09T05:37:03.952 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.952 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.952 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:37:03.952 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.933214+0000", 2023-10-09T05:37:03.952 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.1129999999999997e-06 2023-10-09T05:37:03.953 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.953 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.953 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:37:03.953 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.933276+0000", 2023-10-09T05:37:03.953 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.1517000000000001e-05 2023-10-09T05:37:03.953 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.953 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.953 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:37:03.953 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.933394+0000", 2023-10-09T05:37:03.953 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00011832099999999999 2023-10-09T05:37:03.953 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.953 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.954 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:37:03.954 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.933496+0000", 2023-10-09T05:37:03.954 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000101494 2023-10-09T05:37:03.954 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.954 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.954 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:03.954 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.937500+0000", 2023-10-09T05:37:03.954 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.004004446 2023-10-09T05:37:03.954 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.954 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.954 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:37:03.954 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:33.937643+0000", 2023-10-09T05:37:03.954 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000143098 2023-10-09T05:37:03.955 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.955 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.955 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:03.955 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:34.443563+0000", 2023-10-09T05:37:03.955 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.50591993700000004 2023-10-09T05:37:03.955 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.955 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.955 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:37:03.955 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:34.443691+0000", 2023-10-09T05:37:03.955 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000128114 2023-10-09T05:37:03.955 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.955 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.956 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:37:03.956 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:33:34.443790+0000", 2023-10-09T05:37:03.956 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.8338000000000006e-05 2023-10-09T05:37:03.956 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.956 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:37:03.956 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.956 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.956 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.956 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35775.0:21445 14.2 14:46541f4f:::benchmark_data_smithi133_267384_object21444:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-10-09T05:37:03.956 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:36:19.219681+0000", 2023-10-09T05:37:03.956 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 44.662001771999996, 2023-10-09T05:37:03.956 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.27607060700000002, 2023-10-09T05:37:03.956 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:37:03.957 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:03.957 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:37:03.957 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35775", 2023-10-09T05:37:03.957 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:37:03.957 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 21445 2023-10-09T05:37:03.957 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.957 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:37:03.957 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.957 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:37:03.957 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.219681+0000", 2023-10-09T05:37:03.957 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.957 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.958 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.958 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:37:03.958 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.219681+0000", 2023-10-09T05:37:03.958 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.958 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.958 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.958 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:37:03.958 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.219681+0000", 2023-10-09T05:37:03.958 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:37:03.958 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.958 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.959 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:37:03.959 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.219756+0000", 2023-10-09T05:37:03.959 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.5871000000000006e-05 2023-10-09T05:37:03.959 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.959 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.959 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:37:03.959 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.219768+0000", 2023-10-09T05:37:03.959 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.1647e-05 2023-10-09T05:37:03.959 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.959 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.959 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:37:03.959 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.219774+0000", 2023-10-09T05:37:03.959 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.694e-06 2023-10-09T05:37:03.960 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.960 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.960 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:37:03.960 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.219818+0000", 2023-10-09T05:37:03.960 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.4622000000000001e-05 2023-10-09T05:37:03.960 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.960 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.960 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:37:03.960 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.219931+0000", 2023-10-09T05:37:03.960 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000112963 2023-10-09T05:37:03.960 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.960 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.961 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:37:03.961 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.220074+0000", 2023-10-09T05:37:03.961 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00014241699999999999 2023-10-09T05:37:03.961 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.961 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.961 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:37:03.961 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.221191+0000", 2023-10-09T05:37:03.961 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0011171919999999999 2023-10-09T05:37:03.961 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.961 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.961 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:03.961 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.223310+0000", 2023-10-09T05:37:03.962 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0021193539999999999 2023-10-09T05:37:03.962 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.962 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.962 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:03.962 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.495652+0000", 2023-10-09T05:37:03.962 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.27234201699999999 2023-10-09T05:37:03.962 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.962 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.962 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:37:03.962 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.495703+0000", 2023-10-09T05:37:03.962 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.1007999999999997e-05 2023-10-09T05:37:03.962 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.963 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.963 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:37:03.963 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.495752+0000", 2023-10-09T05:37:03.963 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.8702000000000002e-05 2023-10-09T05:37:03.963 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.963 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:37:03.963 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.963 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.963 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.963 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35775.0:21449 14.2 14:45da398f:::benchmark_data_smithi133_267384_object21448:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-10-09T05:37:03.963 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:36:19.223731+0000", 2023-10-09T05:37:03.963 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 44.657952463000001, 2023-10-09T05:37:03.964 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.27730549199999999, 2023-10-09T05:37:03.964 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:37:03.964 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:03.964 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:37:03.964 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35775", 2023-10-09T05:37:03.964 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:37:03.964 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 21449 2023-10-09T05:37:03.964 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.964 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:37:03.964 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.964 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:37:03.964 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.223731+0000", 2023-10-09T05:37:03.965 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.965 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.965 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.965 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:37:03.965 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.223731+0000", 2023-10-09T05:37:03.965 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.965 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.965 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.965 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:37:03.965 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.223730+0000", 2023-10-09T05:37:03.965 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:37:03.965 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.965 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.966 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:37:03.966 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.223790+0000", 2023-10-09T05:37:03.966 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.0377999999999997e-05 2023-10-09T05:37:03.966 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.966 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.966 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:37:03.966 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.223799+0000", 2023-10-09T05:37:03.966 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.7120000000000006e-06 2023-10-09T05:37:03.966 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.966 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.966 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:37:03.966 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.223804+0000", 2023-10-09T05:37:03.967 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.7299999999999996e-06 2023-10-09T05:37:03.967 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.967 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.967 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:37:03.967 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.223830+0000", 2023-10-09T05:37:03.967 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.6291e-05 2023-10-09T05:37:03.967 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.967 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.967 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:37:03.967 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.493867+0000", 2023-10-09T05:37:03.967 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.27003705099999997 2023-10-09T05:37:03.967 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.967 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.968 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:37:03.968 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.494050+0000", 2023-10-09T05:37:03.968 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000183105 2023-10-09T05:37:03.968 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.968 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.968 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:37:03.968 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.497014+0000", 2023-10-09T05:37:03.968 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0029637399999999999 2023-10-09T05:37:03.968 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.968 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.968 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:03.968 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.500810+0000", 2023-10-09T05:37:03.969 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0037956790000000002 2023-10-09T05:37:03.969 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.969 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.969 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:03.969 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.500857+0000", 2023-10-09T05:37:03.969 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.7173999999999999e-05 2023-10-09T05:37:03.969 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.969 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.969 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:37:03.969 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.500993+0000", 2023-10-09T05:37:03.969 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000135976 2023-10-09T05:37:03.969 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.969 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.970 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:37:03.970 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.501036+0000", 2023-10-09T05:37:03.970 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.3492000000000003e-05 2023-10-09T05:37:03.970 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.970 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:37:03.970 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.970 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.970 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.970 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_repop_reply(client.35775.0:21445 14.2 e316/311 ondisk, result = 0)", 2023-10-09T05:37:03.970 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:36:19.228121+0000", 2023-10-09T05:37:03.970 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 44.653561969000002, 2023-10-09T05:37:03.970 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.26764012500000001, 2023-10-09T05:37:03.971 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:37:03.971 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "started", 2023-10-09T05:37:03.971 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:37:03.971 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.971 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:37:03.971 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.228121+0000", 2023-10-09T05:37:03.971 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.971 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.971 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.971 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:37:03.971 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.228121+0000", 2023-10-09T05:37:03.971 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.972 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.972 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.972 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:37:03.972 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.228120+0000", 2023-10-09T05:37:03.972 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.999999 2023-10-09T05:37:03.972 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.972 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.972 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:37:03.972 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.228127+0000", 2023-10-09T05:37:03.972 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.1190000000000002e-06 2023-10-09T05:37:03.972 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.972 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.972 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:37:03.973 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.493724+0000", 2023-10-09T05:37:03.973 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.265597788 2023-10-09T05:37:03.973 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.973 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.973 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:37:03.973 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.493746+0000", 2023-10-09T05:37:03.973 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.1590000000000002e-05 2023-10-09T05:37:03.973 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.973 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.973 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:37:03.973 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.495637+0000", 2023-10-09T05:37:03.973 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.001891181 2023-10-09T05:37:03.974 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.974 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.974 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:37:03.974 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.495646+0000", 2023-10-09T05:37:03.974 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.9660000000000002e-06 2023-10-09T05:37:03.974 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.974 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.974 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:37:03.974 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.495761+0000", 2023-10-09T05:37:03.974 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00011532900000000001 2023-10-09T05:37:03.974 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.974 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:37:03.975 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.975 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.975 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.975 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_repop(client.35775.0:21452 14.c e316/311 14:3edf1aa7:::benchmark_data_smithi133_267384_object21451:head v 316'1328, mlcod=316'1326)", 2023-10-09T05:37:03.975 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:36:19.229441+0000", 2023-10-09T05:37:03.975 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 44.652242569000002, 2023-10-09T05:37:03.975 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.26886261099999997, 2023-10-09T05:37:03.975 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:37:03.975 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:03.975 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:37:03.975 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.975 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:37:03.976 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.229441+0000", 2023-10-09T05:37:03.976 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.976 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.976 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.976 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:37:03.976 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.229441+0000", 2023-10-09T05:37:03.976 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.976 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.976 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.976 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:37:03.976 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.229439+0000", 2023-10-09T05:37:03.976 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999981 2023-10-09T05:37:03.976 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.977 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.977 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:37:03.977 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.229513+0000", 2023-10-09T05:37:03.977 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.4589e-05 2023-10-09T05:37:03.977 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.977 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.977 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:37:03.977 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.493736+0000", 2023-10-09T05:37:03.977 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.26422267199999999 2023-10-09T05:37:03.977 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.977 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.977 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:37:03.978 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.493749+0000", 2023-10-09T05:37:03.978 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.3293e-05 2023-10-09T05:37:03.978 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.978 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.978 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:37:03.978 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.494271+0000", 2023-10-09T05:37:03.978 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00052120699999999999 2023-10-09T05:37:03.978 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.978 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.978 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:37:03.978 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.494297+0000", 2023-10-09T05:37:03.978 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.6825000000000002e-05 2023-10-09T05:37:03.979 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.979 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.979 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:37:03.979 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.498163+0000", 2023-10-09T05:37:03.979 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0038654449999999999 2023-10-09T05:37:03.979 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.979 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.979 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:37:03.979 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:19.498303+0000", 2023-10-09T05:37:03.979 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00014039 2023-10-09T05:37:03.979 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.980 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:37:03.980 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.980 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.980 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.980 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35775.0:28236 14.4 14:2637005e:::benchmark_data_smithi133_267384_object28235:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-10-09T05:37:03.980 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:36:24.247516+0000", 2023-10-09T05:37:03.980 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 39.634167064000003, 2023-10-09T05:37:03.980 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.329107973, 2023-10-09T05:37:03.980 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:37:03.980 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:03.980 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:37:03.980 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35775", 2023-10-09T05:37:03.981 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:37:03.981 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 28236 2023-10-09T05:37:03.981 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.981 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:37:03.981 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.981 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:37:03.981 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.247516+0000", 2023-10-09T05:37:03.981 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.981 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.981 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.981 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:37:03.981 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.247516+0000", 2023-10-09T05:37:03.981 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.982 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.982 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.982 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:37:03.982 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.247514+0000", 2023-10-09T05:37:03.982 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999981 2023-10-09T05:37:03.982 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.982 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.982 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:37:03.982 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.247556+0000", 2023-10-09T05:37:03.982 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.1693999999999998e-05 2023-10-09T05:37:03.982 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.983 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.983 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:37:03.983 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.247580+0000", 2023-10-09T05:37:03.983 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.3884e-05 2023-10-09T05:37:03.983 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.983 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.983 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:37:03.983 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.247589+0000", 2023-10-09T05:37:03.983 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.3090000000000001e-06 2023-10-09T05:37:03.983 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.983 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.983 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:37:03.983 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.248981+0000", 2023-10-09T05:37:03.984 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0013921809999999999 2023-10-09T05:37:03.984 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.984 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.984 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:37:03.984 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.249183+0000", 2023-10-09T05:37:03.984 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000201657 2023-10-09T05:37:03.984 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.984 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.984 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:37:03.984 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.249420+0000", 2023-10-09T05:37:03.984 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00023709100000000001 2023-10-09T05:37:03.984 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.985 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.985 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:37:03.985 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.251183+0000", 2023-10-09T05:37:03.985 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0017627610000000001 2023-10-09T05:37:03.985 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.985 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.985 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:03.985 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.251527+0000", 2023-10-09T05:37:03.985 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000344657 2023-10-09T05:37:03.985 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.985 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.985 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:03.986 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.576457+0000", 2023-10-09T05:37:03.986 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.32492934299999998 2023-10-09T05:37:03.986 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.986 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.986 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:37:03.986 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.576552+0000", 2023-10-09T05:37:03.986 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.5635000000000002e-05 2023-10-09T05:37:03.986 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.986 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.986 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:37:03.986 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.576624+0000", 2023-10-09T05:37:03.986 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.1795e-05 2023-10-09T05:37:03.987 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.987 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:37:03.987 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.987 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.987 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.987 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35775.0:28244 14.4 14:2afe0bd6:::benchmark_data_smithi133_267384_object28243:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-10-09T05:37:03.987 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:36:24.252862+0000", 2023-10-09T05:37:03.987 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 39.628821567000003, 2023-10-09T05:37:03.987 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.32416269800000003, 2023-10-09T05:37:03.987 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:37:03.987 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:03.987 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:37:03.988 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35775", 2023-10-09T05:37:03.988 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:37:03.988 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 28244 2023-10-09T05:37:03.988 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.988 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:37:03.988 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.988 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:37:03.988 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.252862+0000", 2023-10-09T05:37:03.988 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.988 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.988 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.988 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:37:03.988 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.252862+0000", 2023-10-09T05:37:03.989 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.989 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.989 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.989 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:37:03.989 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.252860+0000", 2023-10-09T05:37:03.989 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:37:03.989 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.989 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.989 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:37:03.989 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.252912+0000", 2023-10-09T05:37:03.989 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.1969999999999999e-05 2023-10-09T05:37:03.989 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.990 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.990 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:37:03.990 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.252928+0000", 2023-10-09T05:37:03.990 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.6113999999999999e-05 2023-10-09T05:37:03.990 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.990 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.990 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:37:03.990 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.252935+0000", 2023-10-09T05:37:03.990 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.2380000000000002e-06 2023-10-09T05:37:03.990 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.990 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.991 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:37:03.991 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.253046+0000", 2023-10-09T05:37:03.991 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00011102100000000001 2023-10-09T05:37:03.991 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.991 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.991 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:37:03.991 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.253270+0000", 2023-10-09T05:37:03.991 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000224763 2023-10-09T05:37:03.991 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.991 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.991 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:37:03.991 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.253427+0000", 2023-10-09T05:37:03.992 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000156565 2023-10-09T05:37:03.992 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.992 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.992 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:03.992 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.255062+0000", 2023-10-09T05:37:03.992 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0016351359999999999 2023-10-09T05:37:03.992 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.992 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.992 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:37:03.992 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.255173+0000", 2023-10-09T05:37:03.992 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000110463 2023-10-09T05:37:03.992 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.993 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.993 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:03.993 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.576754+0000", 2023-10-09T05:37:03.993 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.32158121299999998 2023-10-09T05:37:03.993 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.993 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.993 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:37:03.993 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.576913+0000", 2023-10-09T05:37:03.993 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00015897 2023-10-09T05:37:03.993 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.993 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.993 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:37:03.993 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.577024+0000", 2023-10-09T05:37:03.994 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00011154099999999999 2023-10-09T05:37:03.994 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.994 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:37:03.994 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:03.994 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.994 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.994 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35775.0:28252 14.2 14:423d3746:::benchmark_data_smithi133_267384_object28251:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-10-09T05:37:03.994 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:36:24.258964+0000", 2023-10-09T05:37:03.994 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 39.622718906999999, 2023-10-09T05:37:03.994 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.32568306499999999, 2023-10-09T05:37:03.994 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:37:03.995 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:03.995 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:37:03.995 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35775", 2023-10-09T05:37:03.995 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:37:03.995 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 28252 2023-10-09T05:37:03.995 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.995 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:37:03.995 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.995 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:37:03.995 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.258964+0000", 2023-10-09T05:37:03.995 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.995 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.996 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.996 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:37:03.996 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.258964+0000", 2023-10-09T05:37:03.996 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:03.996 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.996 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.996 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:37:03.996 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.258958+0000", 2023-10-09T05:37:03.996 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999938 2023-10-09T05:37:03.996 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.996 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.996 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:37:03.997 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.259006+0000", 2023-10-09T05:37:03.997 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.8146000000000003e-05 2023-10-09T05:37:03.997 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.997 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.997 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:37:03.997 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.259024+0000", 2023-10-09T05:37:03.997 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.7462999999999999e-05 2023-10-09T05:37:03.997 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.997 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.997 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:37:03.997 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.259032+0000", 2023-10-09T05:37:03.997 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.2879999999999993e-06 2023-10-09T05:37:03.998 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.998 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.998 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:37:03.998 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.259079+0000", 2023-10-09T05:37:03.998 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.6622000000000002e-05 2023-10-09T05:37:03.998 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.998 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.998 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:37:03.998 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.259181+0000", 2023-10-09T05:37:03.998 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000102211 2023-10-09T05:37:03.998 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.998 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.999 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:37:03.999 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.259292+0000", 2023-10-09T05:37:03.999 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000111289 2023-10-09T05:37:03.999 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.999 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.999 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:37:03.999 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.260501+0000", 2023-10-09T05:37:03.999 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.001209087 2023-10-09T05:37:03.999 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:03.999 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:03.999 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:03.999 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.261041+0000", 2023-10-09T05:37:03.999 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00053940100000000001 2023-10-09T05:37:04.000 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.000 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.000 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:04.000 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.584531+0000", 2023-10-09T05:37:04.000 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.32349022799999999 2023-10-09T05:37:04.000 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.000 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.000 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:37:04.000 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.584593+0000", 2023-10-09T05:37:04.000 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 6.2463000000000001e-05 2023-10-09T05:37:04.000 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.000 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.001 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:37:04.001 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.584647+0000", 2023-10-09T05:37:04.001 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.4044e-05 2023-10-09T05:37:04.001 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:04.001 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:37:04.001 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:04.001 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.001 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.001 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35775.0:28255 14.4 14:2eaeae78:::benchmark_data_smithi133_267384_object28254:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-10-09T05:37:04.001 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:36:24.260635+0000", 2023-10-09T05:37:04.001 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 39.621048149000003, 2023-10-09T05:37:04.001 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.32027640699999999, 2023-10-09T05:37:04.001 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:37:04.002 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:04.002 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:37:04.002 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35775", 2023-10-09T05:37:04.002 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:37:04.002 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 28255 2023-10-09T05:37:04.002 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.002 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:37:04.002 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.002 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:37:04.002 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.260635+0000", 2023-10-09T05:37:04.002 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:04.002 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.003 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.003 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:37:04.003 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.260635+0000", 2023-10-09T05:37:04.003 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:04.003 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.003 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.003 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:37:04.003 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.260634+0000", 2023-10-09T05:37:04.003 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:37:04.003 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.003 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.003 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:37:04.004 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.260683+0000", 2023-10-09T05:37:04.004 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.8921999999999997e-05 2023-10-09T05:37:04.004 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.004 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.004 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:37:04.004 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.260697+0000", 2023-10-09T05:37:04.004 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.4311000000000001e-05 2023-10-09T05:37:04.004 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.004 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.004 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:37:04.004 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.260706+0000", 2023-10-09T05:37:04.004 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.3859999999999996e-06 2023-10-09T05:37:04.005 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.005 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.005 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:37:04.005 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.260798+0000", 2023-10-09T05:37:04.005 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 9.2004999999999997e-05 2023-10-09T05:37:04.005 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.005 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.005 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:37:04.005 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.260959+0000", 2023-10-09T05:37:04.005 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00016030999999999999 2023-10-09T05:37:04.005 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.005 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.006 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:37:04.006 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.261131+0000", 2023-10-09T05:37:04.006 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000172249 2023-10-09T05:37:04.006 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.006 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.006 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:37:04.006 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.263812+0000", 2023-10-09T05:37:04.006 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0026808769999999999 2023-10-09T05:37:04.006 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.006 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.006 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:04.006 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.263871+0000", 2023-10-09T05:37:04.006 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.9682999999999998e-05 2023-10-09T05:37:04.007 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.007 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.007 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:04.007 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.580708+0000", 2023-10-09T05:37:04.007 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.316836542 2023-10-09T05:37:04.007 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.007 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.007 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:37:04.007 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.580823+0000", 2023-10-09T05:37:04.007 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000114932 2023-10-09T05:37:04.007 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.007 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.008 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:37:04.008 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.580911+0000", 2023-10-09T05:37:04.008 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.8496000000000001e-05 2023-10-09T05:37:04.008 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:04.008 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:37:04.008 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:04.008 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.008 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.008 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35775.0:28256 14.4 14:238242a5:::benchmark_data_smithi133_267384_object28255:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-10-09T05:37:04.008 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:36:24.262131+0000", 2023-10-09T05:37:04.008 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 39.619552394999999, 2023-10-09T05:37:04.008 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.31915840499999998, 2023-10-09T05:37:04.009 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:37:04.009 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:04.009 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:37:04.009 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35775", 2023-10-09T05:37:04.009 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:37:04.009 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 28256 2023-10-09T05:37:04.009 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.009 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:37:04.009 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.009 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:37:04.009 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.262131+0000", 2023-10-09T05:37:04.009 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:04.010 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.010 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.010 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:37:04.010 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.262131+0000", 2023-10-09T05:37:04.010 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:04.010 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.010 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.010 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:37:04.010 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.262129+0000", 2023-10-09T05:37:04.010 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:37:04.010 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.010 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.011 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:37:04.011 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.262201+0000", 2023-10-09T05:37:04.011 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.2002999999999995e-05 2023-10-09T05:37:04.011 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.011 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.011 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:37:04.011 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.262224+0000", 2023-10-09T05:37:04.011 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.2600999999999999e-05 2023-10-09T05:37:04.011 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.011 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.011 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:37:04.011 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.262235+0000", 2023-10-09T05:37:04.012 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.1600000000000001e-05 2023-10-09T05:37:04.012 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.012 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.012 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:37:04.012 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.262511+0000", 2023-10-09T05:37:04.012 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00027547699999999999 2023-10-09T05:37:04.012 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.012 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.012 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:37:04.012 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.262663+0000", 2023-10-09T05:37:04.012 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000152065 2023-10-09T05:37:04.012 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.013 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.013 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:37:04.013 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.262813+0000", 2023-10-09T05:37:04.013 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.000150135 2023-10-09T05:37:04.013 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.013 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.013 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:37:04.013 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.263980+0000", 2023-10-09T05:37:04.013 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0011668360000000001 2023-10-09T05:37:04.013 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.013 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.014 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:04.014 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.264655+0000", 2023-10-09T05:37:04.014 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00067549500000000002 2023-10-09T05:37:04.014 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.014 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.014 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:04.014 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.581030+0000", 2023-10-09T05:37:04.014 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.31637420700000002 2023-10-09T05:37:04.014 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.014 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.014 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:37:04.014 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.581180+0000", 2023-10-09T05:37:04.015 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00015043000000000001 2023-10-09T05:37:04.015 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.015 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.015 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:37:04.015 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.581289+0000", 2023-10-09T05:37:04.015 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00010914800000000001 2023-10-09T05:37:04.015 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:04.015 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:37:04.015 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:04.015 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.015 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.015 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35775.0:28263 14.2 14:4b70acb9:::benchmark_data_smithi133_267384_object28262:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-10-09T05:37:04.016 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:36:24.269419+0000", 2023-10-09T05:37:04.016 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 39.612263753000001, 2023-10-09T05:37:04.016 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.31536946700000001, 2023-10-09T05:37:04.016 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:37:04.016 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:04.016 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:37:04.016 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35775", 2023-10-09T05:37:04.016 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:37:04.016 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 28263 2023-10-09T05:37:04.016 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.016 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:37:04.016 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.017 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:37:04.017 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.269419+0000", 2023-10-09T05:37:04.017 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:04.017 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.017 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.017 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:37:04.017 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.269419+0000", 2023-10-09T05:37:04.017 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:04.017 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.017 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.017 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:37:04.017 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.269417+0000", 2023-10-09T05:37:04.018 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999981 2023-10-09T05:37:04.018 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.018 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.018 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:37:04.018 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.269473+0000", 2023-10-09T05:37:04.018 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.5631000000000003e-05 2023-10-09T05:37:04.018 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.018 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.018 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:37:04.018 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.269495+0000", 2023-10-09T05:37:04.018 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 2.1656000000000001e-05 2023-10-09T05:37:04.018 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.019 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.019 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:37:04.019 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.269502+0000", 2023-10-09T05:37:04.019 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 7.5820000000000004e-06 2023-10-09T05:37:04.019 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.019 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.019 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:37:04.019 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.269586+0000", 2023-10-09T05:37:04.019 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.3709999999999996e-05 2023-10-09T05:37:04.019 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.019 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.019 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:37:04.020 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.269734+0000", 2023-10-09T05:37:04.020 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00014812899999999999 2023-10-09T05:37:04.020 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.020 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.020 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:37:04.020 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.269893+0000", 2023-10-09T05:37:04.020 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00015879499999999999 2023-10-09T05:37:04.020 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.020 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.020 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:04.020 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.271405+0000", 2023-10-09T05:37:04.020 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0015123630000000001 2023-10-09T05:37:04.021 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.021 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.021 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:37:04.021 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.271589+0000", 2023-10-09T05:37:04.021 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00018381100000000001 2023-10-09T05:37:04.021 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.021 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.021 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:04.021 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.584695+0000", 2023-10-09T05:37:04.021 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.31310609700000003 2023-10-09T05:37:04.021 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.021 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.022 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "commit_sent", 2023-10-09T05:37:04.022 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.584744+0000", 2023-10-09T05:37:04.022 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.8745999999999997e-05 2023-10-09T05:37:04.022 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:04.022 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:04.022 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "done", 2023-10-09T05:37:04.022 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:36:24.584789+0000", 2023-10-09T05:37:04.022 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4.4959000000000001e-05 2023-10-09T05:37:04.022 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:04.022 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:37:04.022 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:04.022 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:04.023 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:37:04.023 INFO:teuthology.orchestra.run.smithi194.stdout:} 2023-10-09T05:37:04.023 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:04.048 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:04.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:04.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:04.151 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:04.180 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:04.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:04.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:04.284 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:04.305 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:04.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:04.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:04.410 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:04.436 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:04.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:04.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:04.541 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:04.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:37:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:37:04] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.43.0" 2023-10-09T05:37:04.565 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:04.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:04.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:04.668 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:04.691 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:04.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:04.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:04.773 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 56 16 71885 71869 80.19 66.4375 0.00633059 0.0124637 2023-10-09T05:37:04.796 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:04.825 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:04.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:04.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:04.929 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:04.931 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T05:37:04.954 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:04.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:04.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:05.059 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:05.086 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:05.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:05.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:05.192 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:05.217 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:05.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:05.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:05.322 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:05.370 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:05.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:05.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:05.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:05 smithi194 bash[92666]: audit 2023-10-09T05:37:04.290654+0000 mon.a (mon.0) 1846 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:37:05.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:05 smithi194 bash[92666]: cluster 2023-10-09T05:37:04.412561+0000 mgr.y (mgr.25277) 1475 : cluster 0 pgmap v1589: 99 pgs: 1 activating+undersized+degraded+remapped, 98 active+clean; 4.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 51 MiB/s wr, 819 op/s; 2091/201531 objects degraded (1.038%); 2091/201531 objects misplaced (1.038%) 2023-10-09T05:37:05.476 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:05.486 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:05 smithi133 bash[102858]: audit 2023-10-09T05:37:04.290654+0000 mon.a (mon.0) 1846 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:37:05.486 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:05 smithi133 bash[102858]: cluster 2023-10-09T05:37:04.412561+0000 mgr.y (mgr.25277) 1475 : cluster 0 pgmap v1589: 99 pgs: 1 activating+undersized+degraded+remapped, 98 active+clean; 4.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 51 MiB/s wr, 819 op/s; 2091/201531 objects degraded (1.038%); 2091/201531 objects misplaced (1.038%) 2023-10-09T05:37:05.486 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:05 smithi133 bash[105589]: audit 2023-10-09T05:37:04.290654+0000 mon.a (mon.0) 1846 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:37:05.486 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:05 smithi133 bash[105589]: cluster 2023-10-09T05:37:04.412561+0000 mgr.y (mgr.25277) 1475 : cluster 0 pgmap v1589: 99 pgs: 1 activating+undersized+degraded+remapped, 98 active+clean; 4.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 51 MiB/s wr, 819 op/s; 2091/201531 objects degraded (1.038%); 2091/201531 objects misplaced (1.038%) 2023-10-09T05:37:05.510 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:05.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:05.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:05.613 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:05.651 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:05.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:05.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:05.754 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:05.775 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 57 16 72937 72921 79.9351 65.75 0.0102247 0.012502 2023-10-09T05:37:05.778 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:05.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:05.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:05.885 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:05.924 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:05.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:05.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:06.028 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:06.066 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:06.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:06.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:06.170 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:06.202 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:06.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:06.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:06.306 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:06.333 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:06.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:06.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:06.414 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:06 smithi133 bash[105589]: audit 2023-10-09T05:37:06.350347+0000 mon.a (mon.0) 1847 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:37:06.415 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:06 smithi133 bash[102858]: audit 2023-10-09T05:37:06.350347+0000 mon.a (mon.0) 1847 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:37:06.437 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:06.460 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:06.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:06.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:06.564 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:06.594 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:06.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:06.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:06.702 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:06.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:06.733 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:06.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:06.776 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 58 16 73946 73930 79.6422 63.0625 0.015476 0.0125498 2023-10-09T05:37:06.834 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:06.844 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:06 smithi194 bash[92666]: audit 2023-10-09T05:37:06.350347+0000 mon.a (mon.0) 1847 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:37:06.860 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:06.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:06.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:06.964 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:06.996 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:07.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:07.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:07.104 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:07.132 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:07.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:07.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:07.235 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:07.259 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:07.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:07.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:07.363 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:07.401 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:07.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:07.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:07.505 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:07.532 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:07.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:07.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:07.547 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:07 smithi133 bash[105589]: cluster 2023-10-09T05:37:06.413349+0000 mgr.y (mgr.25277) 1476 : cluster 0 pgmap v1590: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 4.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s; 3938/215412 objects misplaced (1.828%); 994 KiB/s, 15 objects/s recovering 2023-10-09T05:37:07.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:07 smithi133 bash[105589]: audit 2023-10-09T05:37:06.416415+0000 mon.a (mon.0) 1848 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T05:37:07.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:07 smithi133 bash[102858]: cluster 2023-10-09T05:37:06.413349+0000 mgr.y (mgr.25277) 1476 : cluster 0 pgmap v1590: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 4.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s; 3938/215412 objects misplaced (1.828%); 994 KiB/s, 15 objects/s recovering 2023-10-09T05:37:07.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:07 smithi133 bash[102858]: audit 2023-10-09T05:37:06.416415+0000 mon.a (mon.0) 1848 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T05:37:07.635 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:07.661 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:07.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:07.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:07.764 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:07.775 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:07 smithi194 bash[92666]: cluster 2023-10-09T05:37:06.413349+0000 mgr.y (mgr.25277) 1476 : cluster 0 pgmap v1590: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 4.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s; 3938/215412 objects misplaced (1.828%); 994 KiB/s, 15 objects/s recovering 2023-10-09T05:37:07.775 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:07 smithi194 bash[92666]: audit 2023-10-09T05:37:06.416415+0000 mon.a (mon.0) 1848 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T05:37:07.776 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 59 16 74968 74952 79.3749 63.875 0.0069205 0.0125695 2023-10-09T05:37:07.790 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:07.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:07.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:07.894 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:07.921 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:07.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:07.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:08.025 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:08.055 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:08.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:08.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:08.158 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:08.184 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:08.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:08.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:08.288 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:08.318 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:08.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:08.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:08.421 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:08.455 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:08.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:08.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:08.562 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:08.575 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:08 smithi194 bash[92666]: cluster 2023-10-09T05:37:07.403935+0000 mon.a (mon.0) 1849 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2091/201531 objects degraded (1.038%), 1 pg degraded) 2023-10-09T05:37:08.575 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:08 smithi194 bash[92666]: cluster 2023-10-09T05:37:07.426743+0000 osd.6 (osd.6) 290 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T05:37:08.576 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:08 smithi194 bash[92666]: audit 2023-10-09T05:37:07.433727+0000 mon.a (mon.0) 1850 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T05:37:08.576 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:08 smithi194 bash[92666]: cluster 2023-10-09T05:37:07.433897+0000 mon.a (mon.0) 1851 : cluster 0 osdmap e324: 8 total, 8 up, 7 in 2023-10-09T05:37:08.576 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:08 smithi194 bash[92666]: cluster 2023-10-09T05:37:07.506804+0000 osd.4 (osd.4) 291 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T05:37:08.576 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:08 smithi194 bash[92666]: audit 2023-10-09T05:37:08.414781+0000 mon.a (mon.0) 1852 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T05:37:08.594 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:08.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:08.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:08.698 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:08.725 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:08.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:08.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:08.777 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T05:37:08.777197+0000 min lat: 0.00135224 max lat: 1.55827 avg lat: 0.0125368 2023-10-09T05:37:08.777 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:37:08.777 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 60 16 75337 75321 78.4365 23.0625 0.00284538 0.0125368 2023-10-09T05:37:08.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:08 smithi133 bash[105589]: cluster 2023-10-09T05:37:07.403935+0000 mon.a (mon.0) 1849 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2091/201531 objects degraded (1.038%), 1 pg degraded) 2023-10-09T05:37:08.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:08 smithi133 bash[105589]: cluster 2023-10-09T05:37:07.426743+0000 osd.6 (osd.6) 290 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T05:37:08.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:08 smithi133 bash[105589]: audit 2023-10-09T05:37:07.433727+0000 mon.a (mon.0) 1850 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T05:37:08.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:08 smithi133 bash[105589]: cluster 2023-10-09T05:37:07.433897+0000 mon.a (mon.0) 1851 : cluster 0 osdmap e324: 8 total, 8 up, 7 in 2023-10-09T05:37:08.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:08 smithi133 bash[105589]: cluster 2023-10-09T05:37:07.506804+0000 osd.4 (osd.4) 291 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T05:37:08.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:08 smithi133 bash[105589]: audit 2023-10-09T05:37:08.414781+0000 mon.a (mon.0) 1852 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T05:37:08.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:08 smithi133 bash[102858]: cluster 2023-10-09T05:37:07.403935+0000 mon.a (mon.0) 1849 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2091/201531 objects degraded (1.038%), 1 pg degraded) 2023-10-09T05:37:08.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:08 smithi133 bash[102858]: cluster 2023-10-09T05:37:07.426743+0000 osd.6 (osd.6) 290 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T05:37:08.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:08 smithi133 bash[102858]: audit 2023-10-09T05:37:07.433727+0000 mon.a (mon.0) 1850 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T05:37:08.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:08 smithi133 bash[102858]: cluster 2023-10-09T05:37:07.433897+0000 mon.a (mon.0) 1851 : cluster 0 osdmap e324: 8 total, 8 up, 7 in 2023-10-09T05:37:08.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:08 smithi133 bash[102858]: cluster 2023-10-09T05:37:07.506804+0000 osd.4 (osd.4) 291 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T05:37:08.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:08 smithi133 bash[102858]: audit 2023-10-09T05:37:08.414781+0000 mon.a (mon.0) 1852 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T05:37:08.828 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:08.861 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:08.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:08.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:08.965 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:08.991 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:08.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:08.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:09.094 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:09.129 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:09.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:09.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:09.232 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:09.261 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:09.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:09.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:09.364 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:09.398 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:09.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:09.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:09.503 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:09.535 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:09.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:09.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:09.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:09 smithi133 bash[102858]: cluster 2023-10-09T05:37:08.413794+0000 mgr.y (mgr.25277) 1477 : cluster 0 pgmap v1592: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 4.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 64 MiB/s wr, 1.03k op/s; 3938/216882 objects misplaced (1.816%); 916 KiB/s, 14 objects/s recovering 2023-10-09T05:37:09.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:09 smithi133 bash[102858]: cluster 2023-10-09T05:37:08.444251+0000 osd.4 (osd.4) 292 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T05:37:09.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:09 smithi133 bash[102858]: cluster 2023-10-09T05:37:08.451209+0000 osd.5 (osd.5) 308 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T05:37:09.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:09 smithi133 bash[102858]: cluster 2023-10-09T05:37:08.451374+0000 osd.6 (osd.6) 291 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T05:37:09.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:09 smithi133 bash[102858]: cluster 2023-10-09T05:37:08.453334+0000 osd.6 (osd.6) 292 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T05:37:09.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:09 smithi133 bash[102858]: audit 2023-10-09T05:37:08.456270+0000 mon.a (mon.0) 1853 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T05:37:09.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:09 smithi133 bash[102858]: cluster 2023-10-09T05:37:08.456350+0000 mon.a (mon.0) 1854 : cluster 0 osdmap e325: 8 total, 8 up, 7 in 2023-10-09T05:37:09.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:09 smithi133 bash[102858]: cluster 2023-10-09T05:37:08.457089+0000 osd.6 (osd.6) 293 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T05:37:09.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:09 smithi133 bash[102858]: cluster 2023-10-09T05:37:08.472859+0000 osd.7 (osd.7) 282 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T05:37:09.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:09 smithi133 bash[102858]: cluster 2023-10-09T05:37:08.886762+0000 osd.5 (osd.5) 309 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T05:37:09.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:09 smithi133 bash[102858]: cluster 2023-10-09T05:37:08.889095+0000 mon.a (mon.0) 1855 : cluster 0 osdmap e326: 8 total, 8 up, 7 in 2023-10-09T05:37:09.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:09 smithi133 bash[102858]: cluster 2023-10-09T05:37:08.893388+0000 osd.5 (osd.5) 310 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T05:37:09.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:09 smithi133 bash[102858]: cluster 2023-10-09T05:37:08.893414+0000 osd.6 (osd.6) 294 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T05:37:09.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:09 smithi133 bash[102858]: cluster 2023-10-09T05:37:09.207851+0000 osd.4 (osd.4) 293 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T05:37:09.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:09 smithi133 bash[105589]: cluster 2023-10-09T05:37:08.413794+0000 mgr.y (mgr.25277) 1477 : cluster 0 pgmap v1592: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 4.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 64 MiB/s wr, 1.03k op/s; 3938/216882 objects misplaced (1.816%); 916 KiB/s, 14 objects/s recovering 2023-10-09T05:37:09.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:09 smithi133 bash[105589]: cluster 2023-10-09T05:37:08.444251+0000 osd.4 (osd.4) 292 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T05:37:09.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:09 smithi133 bash[105589]: cluster 2023-10-09T05:37:08.451209+0000 osd.5 (osd.5) 308 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T05:37:09.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:09 smithi133 bash[105589]: cluster 2023-10-09T05:37:08.451374+0000 osd.6 (osd.6) 291 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T05:37:09.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:09 smithi133 bash[105589]: cluster 2023-10-09T05:37:08.453334+0000 osd.6 (osd.6) 292 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T05:37:09.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:09 smithi133 bash[105589]: audit 2023-10-09T05:37:08.456270+0000 mon.a (mon.0) 1853 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T05:37:09.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:09 smithi133 bash[105589]: cluster 2023-10-09T05:37:08.456350+0000 mon.a (mon.0) 1854 : cluster 0 osdmap e325: 8 total, 8 up, 7 in 2023-10-09T05:37:09.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:09 smithi133 bash[105589]: cluster 2023-10-09T05:37:08.457089+0000 osd.6 (osd.6) 293 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T05:37:09.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:09 smithi133 bash[105589]: cluster 2023-10-09T05:37:08.472859+0000 osd.7 (osd.7) 282 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T05:37:09.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:09 smithi133 bash[105589]: cluster 2023-10-09T05:37:08.886762+0000 osd.5 (osd.5) 309 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T05:37:09.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:09 smithi133 bash[105589]: cluster 2023-10-09T05:37:08.889095+0000 mon.a (mon.0) 1855 : cluster 0 osdmap e326: 8 total, 8 up, 7 in 2023-10-09T05:37:09.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:09 smithi133 bash[105589]: cluster 2023-10-09T05:37:08.893388+0000 osd.5 (osd.5) 310 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T05:37:09.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:09 smithi133 bash[105589]: cluster 2023-10-09T05:37:08.893414+0000 osd.6 (osd.6) 294 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T05:37:09.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:09 smithi133 bash[105589]: cluster 2023-10-09T05:37:09.207851+0000 osd.4 (osd.4) 293 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T05:37:09.639 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:09.669 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:09.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:09.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:09.761 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:37:09.773 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:09.776 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 61 16 75676 75660 77.4981 21.1875 0.00269352 0.0127787 2023-10-09T05:37:09.791 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:09 smithi194 bash[92666]: cluster 2023-10-09T05:37:08.413794+0000 mgr.y (mgr.25277) 1477 : cluster 0 pgmap v1592: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 4.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 64 MiB/s wr, 1.03k op/s; 3938/216882 objects misplaced (1.816%); 916 KiB/s, 14 objects/s recovering 2023-10-09T05:37:09.791 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:09 smithi194 bash[92666]: cluster 2023-10-09T05:37:08.444251+0000 osd.4 (osd.4) 292 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T05:37:09.791 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:09 smithi194 bash[92666]: cluster 2023-10-09T05:37:08.451209+0000 osd.5 (osd.5) 308 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T05:37:09.791 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:09 smithi194 bash[92666]: cluster 2023-10-09T05:37:08.451374+0000 osd.6 (osd.6) 291 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T05:37:09.791 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:09 smithi194 bash[92666]: cluster 2023-10-09T05:37:08.453334+0000 osd.6 (osd.6) 292 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T05:37:09.792 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:09 smithi194 bash[92666]: audit 2023-10-09T05:37:08.456270+0000 mon.a (mon.0) 1853 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T05:37:09.792 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:09 smithi194 bash[92666]: cluster 2023-10-09T05:37:08.456350+0000 mon.a (mon.0) 1854 : cluster 0 osdmap e325: 8 total, 8 up, 7 in 2023-10-09T05:37:09.792 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:09 smithi194 bash[92666]: cluster 2023-10-09T05:37:08.457089+0000 osd.6 (osd.6) 293 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T05:37:09.792 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:09 smithi194 bash[92666]: cluster 2023-10-09T05:37:08.472859+0000 osd.7 (osd.7) 282 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T05:37:09.792 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:09 smithi194 bash[92666]: cluster 2023-10-09T05:37:08.886762+0000 osd.5 (osd.5) 309 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T05:37:09.792 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:09 smithi194 bash[92666]: cluster 2023-10-09T05:37:08.889095+0000 mon.a (mon.0) 1855 : cluster 0 osdmap e326: 8 total, 8 up, 7 in 2023-10-09T05:37:09.792 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:09 smithi194 bash[92666]: cluster 2023-10-09T05:37:08.893388+0000 osd.5 (osd.5) 310 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T05:37:09.792 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:09 smithi194 bash[92666]: cluster 2023-10-09T05:37:08.893414+0000 osd.6 (osd.6) 294 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T05:37:09.792 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:09 smithi194 bash[92666]: cluster 2023-10-09T05:37:09.207851+0000 osd.4 (osd.4) 293 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T05:37:09.796 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:09.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:09.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:09.814 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:37:09.814 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:37:09.899 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:09.927 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:09.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:09.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:10.030 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:10.057 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:10.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:10.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:10.162 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:10.193 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:10.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:10.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:10.299 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:10.323 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:10.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:10.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:10.429 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:10.458 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:10.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:10.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:10.564 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:10.598 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:10.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:10.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:10.700 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:10.723 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:10.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:10.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:10.777 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 62 16 76407 76391 76.985 45.6875 0.0158671 0.0129836 2023-10-09T05:37:10.826 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:10.846 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:10.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:10.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:10.949 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:10.959 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:10 smithi194 bash[92666]: cluster 2023-10-09T05:37:09.897999+0000 osd.7 (osd.7) 283 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T05:37:10.959 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:10 smithi194 bash[92666]: cluster 2023-10-09T05:37:09.901938+0000 osd.5 (osd.5) 311 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T05:37:10.959 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:10 smithi194 bash[92666]: cluster 2023-10-09T05:37:09.902119+0000 mon.a (mon.0) 1856 : cluster 0 osdmap e327: 8 total, 8 up, 7 in 2023-10-09T05:37:10.959 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:10 smithi194 bash[92666]: cluster 2023-10-09T05:37:09.956392+0000 osd.6 (osd.6) 295 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T05:37:10.959 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:10 smithi194 bash[92666]: cluster 2023-10-09T05:37:09.965660+0000 osd.4 (osd.4) 294 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T05:37:10.959 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:10 smithi194 bash[92666]: cluster 2023-10-09T05:37:10.414513+0000 mgr.y (mgr.25277) 1478 : cluster 0 pgmap v1596: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 4.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 862 op/s; 3442/225762 objects misplaced (1.525%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T05:37:10.959 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:10 smithi194 bash[92666]: audit 2023-10-09T05:37:10.415620+0000 mon.a (mon.0) 1857 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T05:37:10.975 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:10.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:10.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:11.079 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:11.107 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:11.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:11.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:11.212 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:11.221 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:10 smithi133 bash[105589]: cluster 2023-10-09T05:37:09.897999+0000 osd.7 (osd.7) 283 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T05:37:11.221 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:10 smithi133 bash[105589]: cluster 2023-10-09T05:37:09.901938+0000 osd.5 (osd.5) 311 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T05:37:11.221 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:10 smithi133 bash[105589]: cluster 2023-10-09T05:37:09.902119+0000 mon.a (mon.0) 1856 : cluster 0 osdmap e327: 8 total, 8 up, 7 in 2023-10-09T05:37:11.221 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:10 smithi133 bash[105589]: cluster 2023-10-09T05:37:09.956392+0000 osd.6 (osd.6) 295 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T05:37:11.221 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:10 smithi133 bash[105589]: cluster 2023-10-09T05:37:09.965660+0000 osd.4 (osd.4) 294 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T05:37:11.221 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:10 smithi133 bash[105589]: cluster 2023-10-09T05:37:10.414513+0000 mgr.y (mgr.25277) 1478 : cluster 0 pgmap v1596: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 4.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 862 op/s; 3442/225762 objects misplaced (1.525%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T05:37:11.221 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:10 smithi133 bash[105589]: audit 2023-10-09T05:37:10.415620+0000 mon.a (mon.0) 1857 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T05:37:11.221 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:10 smithi133 bash[102858]: cluster 2023-10-09T05:37:09.897999+0000 osd.7 (osd.7) 283 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T05:37:11.222 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:10 smithi133 bash[102858]: cluster 2023-10-09T05:37:09.901938+0000 osd.5 (osd.5) 311 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T05:37:11.222 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:10 smithi133 bash[102858]: cluster 2023-10-09T05:37:09.902119+0000 mon.a (mon.0) 1856 : cluster 0 osdmap e327: 8 total, 8 up, 7 in 2023-10-09T05:37:11.222 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:10 smithi133 bash[102858]: cluster 2023-10-09T05:37:09.956392+0000 osd.6 (osd.6) 295 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T05:37:11.222 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:10 smithi133 bash[102858]: cluster 2023-10-09T05:37:09.965660+0000 osd.4 (osd.4) 294 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T05:37:11.222 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:10 smithi133 bash[102858]: cluster 2023-10-09T05:37:10.414513+0000 mgr.y (mgr.25277) 1478 : cluster 0 pgmap v1596: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 4.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 862 op/s; 3442/225762 objects misplaced (1.525%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T05:37:11.222 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:10 smithi133 bash[102858]: audit 2023-10-09T05:37:10.415620+0000 mon.a (mon.0) 1857 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T05:37:11.247 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:11.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:11.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:11.352 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:11.375 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:11.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:11.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:11.479 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:11.502 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:11.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:11.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:11.608 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:11.633 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:11.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:11.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:11.736 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:11.759 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:11.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:11.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:11.777 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 63 16 77055 77039 76.4059 40.5 0.00536028 0.0130386 2023-10-09T05:37:11.862 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:11.908 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:11.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:11.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:12.012 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:12.064 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:12.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:12.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:12.168 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:12.170 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:11 smithi194 bash[92666]: cluster 2023-10-09T05:37:10.905795+0000 osd.5 (osd.5) 312 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T05:37:12.170 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:11 smithi194 bash[92666]: audit 2023-10-09T05:37:10.907784+0000 mon.a (mon.0) 1858 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T05:37:12.171 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:11 smithi194 bash[92666]: cluster 2023-10-09T05:37:10.907880+0000 mon.a (mon.0) 1859 : cluster 0 osdmap e328: 8 total, 8 up, 7 in 2023-10-09T05:37:12.171 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:11 smithi194 bash[92666]: cluster 2023-10-09T05:37:10.911902+0000 osd.6 (osd.6) 296 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T05:37:12.171 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:11 smithi194 bash[92666]: cluster 2023-10-09T05:37:10.923438+0000 osd.7 (osd.7) 284 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T05:37:12.180 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:11 smithi133 bash[105589]: cluster 2023-10-09T05:37:10.905795+0000 osd.5 (osd.5) 312 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T05:37:12.180 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:11 smithi133 bash[105589]: audit 2023-10-09T05:37:10.907784+0000 mon.a (mon.0) 1858 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T05:37:12.180 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:11 smithi133 bash[105589]: cluster 2023-10-09T05:37:10.907880+0000 mon.a (mon.0) 1859 : cluster 0 osdmap e328: 8 total, 8 up, 7 in 2023-10-09T05:37:12.180 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:11 smithi133 bash[105589]: cluster 2023-10-09T05:37:10.911902+0000 osd.6 (osd.6) 296 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T05:37:12.180 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:11 smithi133 bash[105589]: cluster 2023-10-09T05:37:10.923438+0000 osd.7 (osd.7) 284 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T05:37:12.180 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:11 smithi133 bash[102858]: cluster 2023-10-09T05:37:10.905795+0000 osd.5 (osd.5) 312 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T05:37:12.180 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:11 smithi133 bash[102858]: audit 2023-10-09T05:37:10.907784+0000 mon.a (mon.0) 1858 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T05:37:12.181 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:11 smithi133 bash[102858]: cluster 2023-10-09T05:37:10.907880+0000 mon.a (mon.0) 1859 : cluster 0 osdmap e328: 8 total, 8 up, 7 in 2023-10-09T05:37:12.181 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:11 smithi133 bash[102858]: cluster 2023-10-09T05:37:10.911902+0000 osd.6 (osd.6) 296 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T05:37:12.181 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:11 smithi133 bash[102858]: cluster 2023-10-09T05:37:10.923438+0000 osd.7 (osd.7) 284 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T05:37:12.191 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:12.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:12.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:12.297 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:12.319 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:12.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:12.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:12.423 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:12.449 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:12.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:12.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:12.552 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:12.575 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:12.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:12.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:12.679 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:12.700 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:12.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:12.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:12.777 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 64 16 77282 77266 75.4339 14.1875 0.00429963 0.0130358 2023-10-09T05:37:12.804 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:12.829 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:12.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:12.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:12.933 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:12.961 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:12.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:12.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:13.065 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:13.091 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:13.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:13.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:13.194 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:13.248 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:13.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:13.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:13.352 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:13.366 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:12 smithi194 bash[92666]: cluster 2023-10-09T05:37:11.917424+0000 mon.a (mon.0) 1860 : cluster 0 osdmap e329: 8 total, 8 up, 7 in 2023-10-09T05:37:13.366 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:12 smithi194 bash[92666]: cluster 2023-10-09T05:37:11.922255+0000 osd.5 (osd.5) 313 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T05:37:13.366 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:12 smithi194 bash[92666]: cluster 2023-10-09T05:37:11.922354+0000 osd.4 (osd.4) 295 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T05:37:13.366 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:12 smithi194 bash[92666]: cluster 2023-10-09T05:37:11.922374+0000 osd.7 (osd.7) 285 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T05:37:13.366 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:12 smithi194 bash[92666]: cluster 2023-10-09T05:37:11.922393+0000 osd.6 (osd.6) 297 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T05:37:13.366 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:12 smithi194 bash[92666]: cluster 2023-10-09T05:37:12.415111+0000 mgr.y (mgr.25277) 1479 : cluster 0 pgmap v1599: 99 pgs: 1 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 95 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 20 MiB/s wr, 326 op/s; 4676/220773 objects degraded (2.118%); 5798/220773 objects misplaced (2.626%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T05:37:13.366 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:12 smithi194 bash[92666]: audit 2023-10-09T05:37:12.448296+0000 mon.a (mon.0) 1861 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:37:13.367 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:12 smithi194 bash[92666]: audit 2023-10-09T05:37:12.453825+0000 mon.a (mon.0) 1862 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:37:13.380 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:13.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:13.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:13.484 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:13.509 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:13.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:13.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:13.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:13 smithi133 bash[105589]: cluster 2023-10-09T05:37:11.917424+0000 mon.a (mon.0) 1860 : cluster 0 osdmap e329: 8 total, 8 up, 7 in 2023-10-09T05:37:13.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:13 smithi133 bash[105589]: cluster 2023-10-09T05:37:11.922255+0000 osd.5 (osd.5) 313 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T05:37:13.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:13 smithi133 bash[105589]: cluster 2023-10-09T05:37:11.922354+0000 osd.4 (osd.4) 295 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T05:37:13.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:13 smithi133 bash[105589]: cluster 2023-10-09T05:37:11.922374+0000 osd.7 (osd.7) 285 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T05:37:13.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:13 smithi133 bash[105589]: cluster 2023-10-09T05:37:11.922393+0000 osd.6 (osd.6) 297 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T05:37:13.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:13 smithi133 bash[105589]: cluster 2023-10-09T05:37:12.415111+0000 mgr.y (mgr.25277) 1479 : cluster 0 pgmap v1599: 99 pgs: 1 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 95 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 20 MiB/s wr, 326 op/s; 4676/220773 objects degraded (2.118%); 5798/220773 objects misplaced (2.626%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T05:37:13.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:13 smithi133 bash[105589]: audit 2023-10-09T05:37:12.448296+0000 mon.a (mon.0) 1861 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:37:13.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:13 smithi133 bash[105589]: audit 2023-10-09T05:37:12.453825+0000 mon.a (mon.0) 1862 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:37:13.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:13 smithi133 bash[102858]: cluster 2023-10-09T05:37:11.917424+0000 mon.a (mon.0) 1860 : cluster 0 osdmap e329: 8 total, 8 up, 7 in 2023-10-09T05:37:13.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:13 smithi133 bash[102858]: cluster 2023-10-09T05:37:11.922255+0000 osd.5 (osd.5) 313 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T05:37:13.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:13 smithi133 bash[102858]: cluster 2023-10-09T05:37:11.922354+0000 osd.4 (osd.4) 295 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T05:37:13.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:13 smithi133 bash[102858]: cluster 2023-10-09T05:37:11.922374+0000 osd.7 (osd.7) 285 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T05:37:13.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:13 smithi133 bash[102858]: cluster 2023-10-09T05:37:11.922393+0000 osd.6 (osd.6) 297 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T05:37:13.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:13 smithi133 bash[102858]: cluster 2023-10-09T05:37:12.415111+0000 mgr.y (mgr.25277) 1479 : cluster 0 pgmap v1599: 99 pgs: 1 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 95 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 20 MiB/s wr, 326 op/s; 4676/220773 objects degraded (2.118%); 5798/220773 objects misplaced (2.626%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T05:37:13.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:13 smithi133 bash[102858]: audit 2023-10-09T05:37:12.448296+0000 mon.a (mon.0) 1861 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:37:13.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:13 smithi133 bash[102858]: audit 2023-10-09T05:37:12.453825+0000 mon.a (mon.0) 1862 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:37:13.613 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:13.640 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:13.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:13.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:13.744 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:13.777 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 65 16 77472 77456 74.4562 11.875 0.0105078 0.0134234 2023-10-09T05:37:13.779 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:13.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:13.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:13.887 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:13.922 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:13.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:13.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:14.026 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:14.066 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:14.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:14.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:14.171 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:14.180 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:13 smithi194 bash[92666]: cluster 2023-10-09T05:37:12.912751+0000 mon.a (mon.0) 1863 : cluster 3 Health check failed: Degraded data redundancy: 4676/220773 objects degraded (2.118%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:37:14.180 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:13 smithi194 bash[92666]: cluster 2023-10-09T05:37:12.921028+0000 osd.4 (osd.4) 296 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T05:37:14.181 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:13 smithi194 bash[92666]: cluster 2023-10-09T05:37:12.921826+0000 osd.5 (osd.5) 314 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T05:37:14.181 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:13 smithi194 bash[92666]: cluster 2023-10-09T05:37:12.925361+0000 mon.a (mon.0) 1864 : cluster 0 osdmap e330: 8 total, 8 up, 7 in 2023-10-09T05:37:14.181 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:13 smithi194 bash[92666]: audit 2023-10-09T05:37:13.459349+0000 mon.a (mon.0) 1865 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:37:14.181 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:13 smithi194 bash[92666]: audit 2023-10-09T05:37:13.465511+0000 mon.a (mon.0) 1866 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:37:14.181 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:13 smithi194 bash[92666]: audit 2023-10-09T05:37:13.494553+0000 mon.a (mon.0) 1867 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:37:14.181 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:13 smithi194 bash[92666]: audit 2023-10-09T05:37:13.495571+0000 mon.a (mon.0) 1868 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:37:14.197 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:14.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:14.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:14.300 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:14.332 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:14.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:14.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:14.438 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:14.462 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:14.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:14.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:14.547 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:37:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:37:14] "GET /metrics HTTP/1.1" 200 36496 "" "Prometheus/2.43.0" 2023-10-09T05:37:14.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:14 smithi133 bash[105589]: cluster 2023-10-09T05:37:12.912751+0000 mon.a (mon.0) 1863 : cluster 3 Health check failed: Degraded data redundancy: 4676/220773 objects degraded (2.118%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:37:14.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:14 smithi133 bash[105589]: cluster 2023-10-09T05:37:12.921028+0000 osd.4 (osd.4) 296 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T05:37:14.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:14 smithi133 bash[105589]: cluster 2023-10-09T05:37:12.921826+0000 osd.5 (osd.5) 314 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T05:37:14.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:14 smithi133 bash[105589]: cluster 2023-10-09T05:37:12.925361+0000 mon.a (mon.0) 1864 : cluster 0 osdmap e330: 8 total, 8 up, 7 in 2023-10-09T05:37:14.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:14 smithi133 bash[105589]: audit 2023-10-09T05:37:13.459349+0000 mon.a (mon.0) 1865 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:37:14.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:14 smithi133 bash[105589]: audit 2023-10-09T05:37:13.465511+0000 mon.a (mon.0) 1866 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:37:14.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:14 smithi133 bash[105589]: audit 2023-10-09T05:37:13.494553+0000 mon.a (mon.0) 1867 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:37:14.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:14 smithi133 bash[105589]: audit 2023-10-09T05:37:13.495571+0000 mon.a (mon.0) 1868 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:37:14.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:14 smithi133 bash[102858]: cluster 2023-10-09T05:37:12.912751+0000 mon.a (mon.0) 1863 : cluster 3 Health check failed: Degraded data redundancy: 4676/220773 objects degraded (2.118%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:37:14.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:14 smithi133 bash[102858]: cluster 2023-10-09T05:37:12.921028+0000 osd.4 (osd.4) 296 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T05:37:14.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:14 smithi133 bash[102858]: cluster 2023-10-09T05:37:12.921826+0000 osd.5 (osd.5) 314 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T05:37:14.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:14 smithi133 bash[102858]: cluster 2023-10-09T05:37:12.925361+0000 mon.a (mon.0) 1864 : cluster 0 osdmap e330: 8 total, 8 up, 7 in 2023-10-09T05:37:14.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:14 smithi133 bash[102858]: audit 2023-10-09T05:37:13.459349+0000 mon.a (mon.0) 1865 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:37:14.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:14 smithi133 bash[102858]: audit 2023-10-09T05:37:13.465511+0000 mon.a (mon.0) 1866 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:37:14.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:14 smithi133 bash[102858]: audit 2023-10-09T05:37:13.494553+0000 mon.a (mon.0) 1867 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:37:14.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:14 smithi133 bash[102858]: audit 2023-10-09T05:37:13.495571+0000 mon.a (mon.0) 1868 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:37:14.568 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:14.590 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:14.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:14.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:14.693 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:14.720 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:14.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:14.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:14.777 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 66 16 78397 78381 74.204 57.8125 0.0152501 0.0134696 2023-10-09T05:37:14.826 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:14.854 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:14.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:14.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:14.881 INFO:teuthology.orchestra.run.smithi194.stdout:{ 2023-10-09T05:37:14.881 INFO:teuthology.orchestra.run.smithi194.stdout: "ops": [ 2023-10-09T05:37:14.881 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:14.881 INFO:teuthology.orchestra.run.smithi194.stdout: "description": "osd_op(client.35775.0:78485 14.f 14:fe05b0c1:::benchmark_data_smithi133_267384_object78484:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-10-09T05:37:14.881 INFO:teuthology.orchestra.run.smithi194.stdout: "initiated_at": "2023-10-09T05:37:14.867510+0000", 2023-10-09T05:37:14.881 INFO:teuthology.orchestra.run.smithi194.stdout: "age": 0.013469429, 2023-10-09T05:37:14.881 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.013499997999999999, 2023-10-09T05:37:14.881 INFO:teuthology.orchestra.run.smithi194.stdout: "type_data": { 2023-10-09T05:37:14.881 INFO:teuthology.orchestra.run.smithi194.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:37:14.881 INFO:teuthology.orchestra.run.smithi194.stdout: "client_info": { 2023-10-09T05:37:14.882 INFO:teuthology.orchestra.run.smithi194.stdout: "client": "client.35775", 2023-10-09T05:37:14.882 INFO:teuthology.orchestra.run.smithi194.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:37:14.882 INFO:teuthology.orchestra.run.smithi194.stdout: "tid": 78485 2023-10-09T05:37:14.882 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:14.882 INFO:teuthology.orchestra.run.smithi194.stdout: "events": [ 2023-10-09T05:37:14.882 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:14.882 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "initiated", 2023-10-09T05:37:14.882 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:37:14.867510+0000", 2023-10-09T05:37:14.882 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:14.882 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:14.882 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:14.883 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "throttled", 2023-10-09T05:37:14.883 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:37:14.867510+0000", 2023-10-09T05:37:14.883 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0 2023-10-09T05:37:14.883 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:14.883 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:14.883 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "header_read", 2023-10-09T05:37:14.883 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:37:14.867508+0000", 2023-10-09T05:37:14.883 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 4294967295.9999986 2023-10-09T05:37:14.883 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:14.883 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:14.883 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "all_read", 2023-10-09T05:37:14.884 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:37:14.867563+0000", 2023-10-09T05:37:14.884 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.4571000000000003e-05 2023-10-09T05:37:14.884 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:14.884 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:14.884 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "dispatched", 2023-10-09T05:37:14.884 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:37:14.867575+0000", 2023-10-09T05:37:14.884 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 1.1754e-05 2023-10-09T05:37:14.884 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:14.884 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:14.884 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "queued_for_pg", 2023-10-09T05:37:14.884 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:37:14.867580+0000", 2023-10-09T05:37:14.884 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 5.3550000000000004e-06 2023-10-09T05:37:14.885 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:14.885 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:14.885 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "reached_pg", 2023-10-09T05:37:14.885 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:37:14.867612+0000", 2023-10-09T05:37:14.885 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 3.2146e-05 2023-10-09T05:37:14.885 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:14.885 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:14.885 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "started", 2023-10-09T05:37:14.885 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:37:14.867697+0000", 2023-10-09T05:37:14.885 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 8.5209000000000004e-05 2023-10-09T05:37:14.885 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:14.885 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:14.886 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:37:14.886 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:37:14.867809+0000", 2023-10-09T05:37:14.886 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.00011165799999999999 2023-10-09T05:37:14.886 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:14.886 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:14.886 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "op_commit", 2023-10-09T05:37:14.886 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:37:14.870084+0000", 2023-10-09T05:37:14.886 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.002274772 2023-10-09T05:37:14.886 INFO:teuthology.orchestra.run.smithi194.stdout: }, 2023-10-09T05:37:14.886 INFO:teuthology.orchestra.run.smithi194.stdout: { 2023-10-09T05:37:14.886 INFO:teuthology.orchestra.run.smithi194.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:14.887 INFO:teuthology.orchestra.run.smithi194.stdout: "time": "2023-10-09T05:37:14.875228+0000", 2023-10-09T05:37:14.887 INFO:teuthology.orchestra.run.smithi194.stdout: "duration": 0.0051441760000000003 2023-10-09T05:37:14.887 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:14.887 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2023-10-09T05:37:14.887 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:14.887 INFO:teuthology.orchestra.run.smithi194.stdout: } 2023-10-09T05:37:14.887 INFO:teuthology.orchestra.run.smithi194.stdout: ], 2023-10-09T05:37:14.887 INFO:teuthology.orchestra.run.smithi194.stdout: "num_ops": 1 2023-10-09T05:37:14.887 INFO:teuthology.orchestra.run.smithi194.stdout:} 2023-10-09T05:37:14.959 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:14.980 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:14.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:14.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:15.084 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:15.112 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:15.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:15.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:15.118 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:14 smithi194 bash[92666]: cluster 2023-10-09T05:37:13.431193+0000 osd.6 (osd.6) 298 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T05:37:15.118 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:14 smithi194 bash[92666]: audit 2023-10-09T05:37:14.124588+0000 mon.a (mon.0) 1869 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:15.118 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:14 smithi194 bash[92666]: audit 2023-10-09T05:37:14.125825+0000 mon.a (mon.0) 1870 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:37:15.118 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:14 smithi194 bash[92666]: audit 2023-10-09T05:37:14.133181+0000 mon.a (mon.0) 1871 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:37:15.118 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:14 smithi194 bash[92666]: cluster 2023-10-09T05:37:14.415936+0000 mgr.y (mgr.25277) 1480 : cluster 0 pgmap v1601: 99 pgs: 1 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 95 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 18 MiB/s wr, 280 op/s; 4676/223200 objects degraded (2.095%); 5798/223200 objects misplaced (2.598%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T05:37:15.188 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:14 smithi133 bash[102858]: cluster 2023-10-09T05:37:13.431193+0000 osd.6 (osd.6) 298 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T05:37:15.217 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:15.237 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:15.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:15.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:15.341 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:15.379 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:15.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:15.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:15.484 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:15.509 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:15.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:15.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:15.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:15 smithi133 bash[102858]: audit 2023-10-09T05:37:14.124588+0000 mon.a (mon.0) 1869 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:15.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:15 smithi133 bash[102858]: audit 2023-10-09T05:37:14.125825+0000 mon.a (mon.0) 1870 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:37:15.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:15 smithi133 bash[102858]: audit 2023-10-09T05:37:14.133181+0000 mon.a (mon.0) 1871 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:37:15.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:15 smithi133 bash[102858]: cluster 2023-10-09T05:37:14.415936+0000 mgr.y (mgr.25277) 1480 : cluster 0 pgmap v1601: 99 pgs: 1 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 95 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 18 MiB/s wr, 280 op/s; 4676/223200 objects degraded (2.095%); 5798/223200 objects misplaced (2.598%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T05:37:15.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:15 smithi133 bash[105589]: cluster 2023-10-09T05:37:13.431193+0000 osd.6 (osd.6) 298 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T05:37:15.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:15 smithi133 bash[105589]: audit 2023-10-09T05:37:14.124588+0000 mon.a (mon.0) 1869 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:15.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:15 smithi133 bash[105589]: audit 2023-10-09T05:37:14.125825+0000 mon.a (mon.0) 1870 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:37:15.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:15 smithi133 bash[105589]: audit 2023-10-09T05:37:14.133181+0000 mon.a (mon.0) 1871 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:37:15.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:15 smithi133 bash[105589]: cluster 2023-10-09T05:37:14.415936+0000 mgr.y (mgr.25277) 1480 : cluster 0 pgmap v1601: 99 pgs: 1 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 95 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 18 MiB/s wr, 280 op/s; 4676/223200 objects degraded (2.095%); 5798/223200 objects misplaced (2.598%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T05:37:15.613 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:15.636 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:15.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:15.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:15.741 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:15.762 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:15.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:15.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:15.777 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 67 16 79291 79275 73.9304 55.875 0.0052845 0.0135202 2023-10-09T05:37:15.869 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:15.890 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:15.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:15.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:15.994 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:16.021 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:16.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:16.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:16.026 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T05:37:16.126 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:16.161 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:16.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:16.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:16.265 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:16.295 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:16.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:16.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:16.402 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:16.429 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:16.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:16.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:16.533 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:16.566 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:16.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:16.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:16.670 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:16.706 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:16.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:16.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:16.777 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 68 16 80216 80200 73.6934 57.8125 0.0353482 0.0135628 2023-10-09T05:37:16.810 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:16.838 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:16.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:16.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:16.941 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:16.969 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:16.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:16.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:17.045 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:37:17.059 INFO:teuthology.orchestra.run.smithi133.stderr:dumped all 2023-10-09T05:37:17.078 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:17.107 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:17.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:17.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:17.211 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:17.241 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:17.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:17.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:17.346 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:17.372 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:17.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:17.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:17.475 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:17.499 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:17.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:17.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:17.603 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:17.635 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:17.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:17.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:17.731 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:17 smithi133 bash[102858]: cluster 2023-10-09T05:37:16.416642+0000 mgr.y (mgr.25277) 1481 : cluster 0 pgmap v1602: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 95 active+clean; 4.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 32 MiB/s wr, 508 op/s; 9312/234921 objects degraded (3.964%); 7562/234921 objects misplaced (3.219%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:37:17.731 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:17 smithi133 bash[105589]: cluster 2023-10-09T05:37:16.416642+0000 mgr.y (mgr.25277) 1481 : cluster 0 pgmap v1602: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 95 active+clean; 4.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 32 MiB/s wr, 508 op/s; 9312/234921 objects degraded (3.964%); 7562/234921 objects misplaced (3.219%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:37:17.740 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:17.751 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:17 smithi194 bash[92666]: cluster 2023-10-09T05:37:16.416642+0000 mgr.y (mgr.25277) 1481 : cluster 0 pgmap v1602: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 95 active+clean; 4.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 32 MiB/s wr, 508 op/s; 9312/234921 objects degraded (3.964%); 7562/234921 objects misplaced (3.219%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:37:17.768 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:17.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:17.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:17.777 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 69 16 81146 81130 73.4677 58.125 0.00901521 0.0136048 2023-10-09T05:37:17.870 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:17.902 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:17.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:17.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:18.008 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:18.037 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:18.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:18.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:18.140 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:18.178 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:18.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:18.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:18.283 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:18.311 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:18.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:18.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:18.415 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:18.441 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:18.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:18.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:18.544 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:18.570 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:18.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:18.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:18.674 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:18.684 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:18 smithi133 bash[102858]: audit 2023-10-09T05:37:17.022653+0000 mgr.y (mgr.25277) 1482 : audit 0 from='client.45812 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:37:18.684 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:18 smithi133 bash[105589]: audit 2023-10-09T05:37:17.022653+0000 mgr.y (mgr.25277) 1482 : audit 0 from='client.45812 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:37:18.720 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:18.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:18.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:18.777 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 70 16 82122 82106 73.2895 61 0.00889537 0.013639 2023-10-09T05:37:18.827 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:18.861 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:18.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:18.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:18.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:18 smithi194 bash[92666]: audit 2023-10-09T05:37:17.022653+0000 mgr.y (mgr.25277) 1482 : audit 0 from='client.45812 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:37:18.964 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:18.994 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:18.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:18.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:19.098 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:19.129 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:19.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:19.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:19.234 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:19.264 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:19.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:19.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:19.367 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:19.401 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:19.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:19.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:19.505 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:19.514 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:19 smithi194 bash[92666]: cluster 2023-10-09T05:37:18.417203+0000 mgr.y (mgr.25277) 1483 : cluster 0 pgmap v1603: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 95 active+clean; 4.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 25 MiB/s wr, 403 op/s; 9312/235815 objects degraded (3.949%); 7562/235815 objects misplaced (3.207%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:37:19.514 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:19 smithi194 bash[92666]: cluster 2023-10-09T05:37:18.474198+0000 mon.a (mon.0) 1872 : cluster 3 Health check update: Degraded data redundancy: 9312/235815 objects degraded (3.949%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:19.533 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:19.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:19.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:19.637 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:19.663 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:19.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:19.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:19.766 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:19.777 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 71 16 83041 83025 73.0662 57.4375 0.0163045 0.0136804 2023-10-09T05:37:19.794 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:19.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:19.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:19.797 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:19 smithi133 bash[105589]: cluster 2023-10-09T05:37:18.417203+0000 mgr.y (mgr.25277) 1483 : cluster 0 pgmap v1603: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 95 active+clean; 4.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 25 MiB/s wr, 403 op/s; 9312/235815 objects degraded (3.949%); 7562/235815 objects misplaced (3.207%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:37:19.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:19 smithi133 bash[105589]: cluster 2023-10-09T05:37:18.474198+0000 mon.a (mon.0) 1872 : cluster 3 Health check update: Degraded data redundancy: 9312/235815 objects degraded (3.949%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:19.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:19 smithi133 bash[102858]: cluster 2023-10-09T05:37:18.417203+0000 mgr.y (mgr.25277) 1483 : cluster 0 pgmap v1603: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 95 active+clean; 4.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 25 MiB/s wr, 403 op/s; 9312/235815 objects degraded (3.949%); 7562/235815 objects misplaced (3.207%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:37:19.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:19 smithi133 bash[102858]: cluster 2023-10-09T05:37:18.474198+0000 mon.a (mon.0) 1872 : cluster 3 Health check update: Degraded data redundancy: 9312/235815 objects degraded (3.949%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:19.897 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:19.938 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:19.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:19.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:20.045 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:20.075 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:20.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:20.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:20.180 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:20.207 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:20.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:20.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:20.312 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:20.354 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:20.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:20.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:20.460 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:20.484 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:20.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:20.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:20.586 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:20.620 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:20.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:20.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:20.726 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:20.750 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:20.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:20.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:20.778 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 72 16 83792 83776 72.7033 46.9375 0.0112728 0.0137485 2023-10-09T05:37:20.853 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:20.876 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:20.877 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:37:20.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:20.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:20.980 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:21.011 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:21.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:21.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:21.117 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:21.142 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:21.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:21.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:21.247 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:21.272 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:21.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:21.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:21.376 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:21.416 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:21.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:21.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:21.520 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:21.530 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:21 smithi194 bash[92666]: cluster 2023-10-09T05:37:20.417757+0000 mgr.y (mgr.25277) 1484 : cluster 0 pgmap v1604: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 95 active+clean; 5.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 9312/247218 objects degraded (3.767%); 7232/247218 objects misplaced (2.925%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T05:37:21.544 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:21.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:21.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:21.599 INFO:teuthology.orchestra.run.smithi133.stdout:{"pg_ready":true,"pg_map":{"version":1602,"stamp":"2023-10-09T05:37:16.416105+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5130950560,"num_objects":78307,"num_object_clones":0,"num_object_copies":234921,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9312,"num_objects_misplaced":7562,"num_objects_unfound":0,"num_objects_dirty":78307,"num_whiteouts":0,"num_read":509,"num_read_kb":934,"num_write":78583,"num_write_kb":5012557,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":729,"num_bytes_recovered":46818740,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":116091,"ondisk_log_size":116091,"up":297,"acting":294,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":306,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":23062944,"kb_used_data":15111784,"kb_used_omap":20,"kb_used_meta":7951083,"kb_avail":633153120,"statfs":{"total":671965249536,"available":648348794880,"internally_reserved":0,"allocated":15474466816,"data_stored":15464293899,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":8141909646},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,2,5,3,2,1,1,1,5,12],"upper_bound":2048},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":200081408,"num_objects":3053,"num_object_clones":0,"num_object_copies":9159,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9312,"num_objects_misplaced":4120,"num_objects_unfound":0,"num_objects_dirty":3053,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3053,"num_write_kb":195392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":277,"num_bytes_recovered":18153472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.002073"},"pg_stats":[{"pgid":"14.17","version":"330'4298","reported_seq":4334,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T05:37:14.299068+0000","last_change":"2023-10-09T05:36:53.586212+0000","last_active":"2023-10-09T05:37:14.299068+0000","last_peered":"2023-10-09T05:37:14.299068+0000","last_clean":"2023-10-09T05:37:14.299068+0000","last_became_active":"2023-10-09T05:36:53.583019+0000","last_became_peered":"2023-10-09T05:36:53.583019+0000","last_unstale":"2023-10-09T05:37:14.299068+0000","last_undegraded":"2023-10-09T05:37:14.299068+0000","last_fullsized":"2023-10-09T05:37:14.299068+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":4298,"log_dups_size":0,"ondisk_log_size":4298,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159973376,"num_objects":2441,"num_object_clones":0,"num_object_copies":7323,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2441,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2441,"num_write_kb":156224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'2","reported_seq":199,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.919068+0000","last_change":"2023-10-09T05:30:03.378323+0000","last_active":"2023-10-09T05:37:11.919068+0000","last_peered":"2023-10-09T05:37:11.919068+0000","last_clean":"2023-10-09T05:37:11.919068+0000","last_became_active":"2023-10-09T05:30:03.377871+0000","last_became_peered":"2023-10-09T05:30:03.377871+0000","last_unstale":"2023-10-09T05:37:11.919068+0000","last_undegraded":"2023-10-09T05:37:11.919068+0000","last_fullsized":"2023-10-09T05:37:11.919068+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:09.181202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":326,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.923613+0000","last_change":"2023-10-09T05:30:03.377792+0000","last_active":"2023-10-09T05:37:11.923613+0000","last_peered":"2023-10-09T05:37:11.923613+0000","last_clean":"2023-10-09T05:37:11.923613+0000","last_became_active":"2023-10-09T05:30:03.377456+0000","last_became_peered":"2023-10-09T05:30:03.377456+0000","last_unstale":"2023-10-09T05:37:11.923613+0000","last_undegraded":"2023-10-09T05:37:11.923613+0000","last_fullsized":"2023-10-09T05:37:11.923613+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:35:57.941046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.14","version":"330'4296","reported_seq":4333,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T05:37:14.258632+0000","last_change":"2023-10-09T05:36:53.583746+0000","last_active":"2023-10-09T05:37:14.258632+0000","last_peered":"2023-10-09T05:37:14.258632+0000","last_clean":"2023-10-09T05:37:14.258632+0000","last_became_active":"2023-10-09T05:36:53.580996+0000","last_became_peered":"2023-10-09T05:36:53.580996+0000","last_unstale":"2023-10-09T05:37:14.258632+0000","last_undegraded":"2023-10-09T05:37:14.258632+0000","last_fullsized":"2023-10-09T05:37:14.258632+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":4296,"log_dups_size":0,"ondisk_log_size":4296,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160497664,"num_objects":2449,"num_object_clones":0,"num_object_copies":7347,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2449,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2449,"num_write_kb":156736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":209,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.916204+0000","last_change":"2023-10-09T05:30:03.377059+0000","last_active":"2023-10-09T05:37:11.916204+0000","last_peered":"2023-10-09T05:37:11.916204+0000","last_clean":"2023-10-09T05:37:11.916204+0000","last_became_active":"2023-10-09T05:30:03.376460+0000","last_became_peered":"2023-10-09T05:30:03.376460+0000","last_unstale":"2023-10-09T05:37:11.916204+0000","last_undegraded":"2023-10-09T05:37:11.916204+0000","last_fullsized":"2023-10-09T05:37:11.916204+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:01:08.558120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":335,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:12.044829+0000","last_change":"2023-10-09T05:18:26.422923+0000","last_active":"2023-10-09T05:37:12.044829+0000","last_peered":"2023-10-09T05:37:12.044829+0000","last_clean":"2023-10-09T05:37:12.044829+0000","last_became_active":"2023-10-09T05:18:26.422508+0000","last_became_peered":"2023-10-09T05:18:26.422508+0000","last_unstale":"2023-10-09T05:37:12.044829+0000","last_undegraded":"2023-10-09T05:37:12.044829+0000","last_fullsized":"2023-10-09T05:37:12.044829+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:00.876037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.15","version":"330'4360","reported_seq":4398,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T05:37:15.043615+0000","last_change":"2023-10-09T05:36:53.627741+0000","last_active":"2023-10-09T05:37:15.043615+0000","last_peered":"2023-10-09T05:37:15.043615+0000","last_clean":"2023-10-09T05:37:15.043615+0000","last_became_active":"2023-10-09T05:36:53.619647+0000","last_became_peered":"2023-10-09T05:36:53.619647+0000","last_unstale":"2023-10-09T05:37:15.043615+0000","last_undegraded":"2023-10-09T05:37:15.043615+0000","last_fullsized":"2023-10-09T05:37:15.043615+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":4360,"log_dups_size":0,"ondisk_log_size":4360,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162627584,"num_objects":2482,"num_object_clones":0,"num_object_copies":7446,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2482,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2482,"num_write_kb":158816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":353,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.922856+0000","last_change":"2023-10-09T05:23:18.188378+0000","last_active":"2023-10-09T05:37:11.922856+0000","last_peered":"2023-10-09T05:37:11.922856+0000","last_clean":"2023-10-09T05:37:11.922856+0000","last_became_active":"2023-10-09T05:23:18.186629+0000","last_became_peered":"2023-10-09T05:23:18.186629+0000","last_unstale":"2023-10-09T05:37:11.922856+0000","last_undegraded":"2023-10-09T05:37:11.922856+0000","last_fullsized":"2023-10-09T05:37:11.922856+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:16.444225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":330,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.929771+0000","last_change":"2023-10-09T05:32:45.829908+0000","last_active":"2023-10-09T05:37:11.929771+0000","last_peered":"2023-10-09T05:37:11.929771+0000","last_clean":"2023-10-09T05:37:11.929771+0000","last_became_active":"2023-10-09T05:32:45.829566+0000","last_became_peered":"2023-10-09T05:32:45.829566+0000","last_unstale":"2023-10-09T05:37:11.929771+0000","last_undegraded":"2023-10-09T05:37:11.929771+0000","last_fullsized":"2023-10-09T05:37:11.929771+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:05.705818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":330,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:12.044642+0000","last_change":"2023-10-09T05:32:45.835245+0000","last_active":"2023-10-09T05:37:12.044642+0000","last_peered":"2023-10-09T05:37:12.044642+0000","last_clean":"2023-10-09T05:37:12.044642+0000","last_became_active":"2023-10-09T05:32:45.830180+0000","last_became_peered":"2023-10-09T05:32:45.830180+0000","last_unstale":"2023-10-09T05:37:12.044642+0000","last_undegraded":"2023-10-09T05:37:12.044642+0000","last_fullsized":"2023-10-09T05:37:12.044642+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:07:06.351159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":309,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.922921+0000","last_change":"2023-10-09T05:09:57.849695+0000","last_active":"2023-10-09T05:37:11.922921+0000","last_peered":"2023-10-09T05:37:11.922921+0000","last_clean":"2023-10-09T05:37:11.922921+0000","last_became_active":"2023-10-09T05:09:57.848478+0000","last_became_peered":"2023-10-09T05:09:57.848478+0000","last_unstale":"2023-10-09T05:37:11.922921+0000","last_undegraded":"2023-10-09T05:37:11.922921+0000","last_fullsized":"2023-10-09T05:37:11.922921+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:26.115312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":39,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.926248+0000","last_change":"2023-10-09T05:32:45.830061+0000","last_active":"2023-10-09T05:37:11.926248+0000","last_peered":"2023-10-09T05:37:11.926248+0000","last_clean":"2023-10-09T05:37:11.926248+0000","last_became_active":"2023-10-09T05:32:45.829723+0000","last_became_peered":"2023-10-09T05:32:45.829723+0000","last_unstale":"2023-10-09T05:37:11.926248+0000","last_undegraded":"2023-10-09T05:37:11.926248+0000","last_fullsized":"2023-10-09T05:37:11.926248+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:21.429740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":322,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.923009+0000","last_change":"2023-10-09T05:18:26.418431+0000","last_active":"2023-10-09T05:37:11.923009+0000","last_peered":"2023-10-09T05:37:11.923009+0000","last_clean":"2023-10-09T05:37:11.923009+0000","last_became_active":"2023-10-09T05:18:26.418100+0000","last_became_peered":"2023-10-09T05:18:26.418100+0000","last_unstale":"2023-10-09T05:37:11.923009+0000","last_undegraded":"2023-10-09T05:37:11.923009+0000","last_fullsized":"2023-10-09T05:37:11.923009+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:21.241286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.18","version":"330'4490","reported_seq":4528,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T05:37:15.053583+0000","last_change":"2023-10-09T05:36:53.612884+0000","last_active":"2023-10-09T05:37:15.053583+0000","last_peered":"2023-10-09T05:37:15.053583+0000","last_clean":"2023-10-09T05:37:15.053583+0000","last_became_active":"2023-10-09T05:36:53.610074+0000","last_became_peered":"2023-10-09T05:36:53.610074+0000","last_unstale":"2023-10-09T05:37:15.053583+0000","last_undegraded":"2023-10-09T05:37:15.053583+0000","last_fullsized":"2023-10-09T05:37:15.053583+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":4490,"log_dups_size":0,"ondisk_log_size":4490,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165871616,"num_objects":2531,"num_object_clones":0,"num_object_copies":7593,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2531,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2531,"num_write_kb":161984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":141,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.930148+0000","last_change":"2023-10-09T05:30:03.413915+0000","last_active":"2023-10-09T05:37:11.930148+0000","last_peered":"2023-10-09T05:37:11.930148+0000","last_clean":"2023-10-09T05:37:11.930148+0000","last_became_active":"2023-10-09T05:30:03.391102+0000","last_became_peered":"2023-10-09T05:30:03.391102+0000","last_unstale":"2023-10-09T05:37:11.930148+0000","last_undegraded":"2023-10-09T05:37:11.930148+0000","last_fullsized":"2023-10-09T05:37:11.930148+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:13.024050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":93,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.918978+0000","last_change":"2023-10-09T05:23:18.199181+0000","last_active":"2023-10-09T05:37:11.918978+0000","last_peered":"2023-10-09T05:37:11.918978+0000","last_clean":"2023-10-09T05:37:11.918978+0000","last_became_active":"2023-10-09T05:23:18.198401+0000","last_became_peered":"2023-10-09T05:23:18.198401+0000","last_unstale":"2023-10-09T05:37:11.918978+0000","last_undegraded":"2023-10-09T05:37:11.918978+0000","last_fullsized":"2023-10-09T05:37:11.918978+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:55:49.891420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.19","version":"330'4255","reported_seq":4295,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T05:37:14.290680+0000","last_change":"2023-10-09T05:36:53.632724+0000","last_active":"2023-10-09T05:37:14.290680+0000","last_peered":"2023-10-09T05:37:14.290680+0000","last_clean":"2023-10-09T05:37:14.290680+0000","last_became_active":"2023-10-09T05:36:53.604546+0000","last_became_peered":"2023-10-09T05:36:53.604546+0000","last_unstale":"2023-10-09T05:37:14.290680+0000","last_undegraded":"2023-10-09T05:37:14.290680+0000","last_fullsized":"2023-10-09T05:37:14.290680+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":4255,"log_dups_size":0,"ondisk_log_size":4255,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156827648,"num_objects":2393,"num_object_clones":0,"num_object_copies":7179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2393,"num_write_kb":153152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"83'23","reported_seq":319,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.928467+0000","last_change":"2023-10-09T05:30:03.423577+0000","last_active":"2023-10-09T05:37:11.928467+0000","last_peered":"2023-10-09T05:37:11.928467+0000","last_clean":"2023-10-09T05:37:11.928467+0000","last_became_active":"2023-10-09T05:30:03.394939+0000","last_became_peered":"2023-10-09T05:30:03.394939+0000","last_unstale":"2023-10-09T05:37:11.928467+0000","last_undegraded":"2023-10-09T05:37:11.928467+0000","last_fullsized":"2023-10-09T05:37:11.928467+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:19:34.968600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":188,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.928777+0000","last_change":"2023-10-09T05:32:45.831400+0000","last_active":"2023-10-09T05:37:11.928777+0000","last_peered":"2023-10-09T05:37:11.928777+0000","last_clean":"2023-10-09T05:37:11.928777+0000","last_became_active":"2023-10-09T05:32:45.831007+0000","last_became_peered":"2023-10-09T05:32:45.831007+0000","last_unstale":"2023-10-09T05:37:11.928777+0000","last_undegraded":"2023-10-09T05:37:11.928777+0000","last_fullsized":"2023-10-09T05:37:11.928777+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:00:20.174754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":141,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.927972+0000","last_change":"2023-10-09T05:30:03.407302+0000","last_active":"2023-10-09T05:37:11.927972+0000","last_peered":"2023-10-09T05:37:11.927972+0000","last_clean":"2023-10-09T05:37:11.927972+0000","last_became_active":"2023-10-09T05:30:03.374209+0000","last_became_peered":"2023-10-09T05:30:03.374209+0000","last_unstale":"2023-10-09T05:37:11.927972+0000","last_undegraded":"2023-10-09T05:37:11.927972+0000","last_fullsized":"2023-10-09T05:37:11.927972+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:53.750483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":363,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T05:37:13.019917+0000","last_change":"2023-10-09T05:23:18.187593+0000","last_active":"2023-10-09T05:37:13.019917+0000","last_peered":"2023-10-09T05:37:13.019917+0000","last_clean":"2023-10-09T05:37:13.019917+0000","last_became_active":"2023-10-09T05:23:18.186445+0000","last_became_peered":"2023-10-09T05:23:18.186445+0000","last_unstale":"2023-10-09T05:37:13.019917+0000","last_undegraded":"2023-10-09T05:37:13.019917+0000","last_fullsized":"2023-10-09T05:37:13.019917+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:11.327635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":331,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:12.040698+0000","last_change":"2023-10-09T05:18:26.424399+0000","last_active":"2023-10-09T05:37:12.040698+0000","last_peered":"2023-10-09T05:37:12.040698+0000","last_clean":"2023-10-09T05:37:12.040698+0000","last_became_active":"2023-10-09T05:18:26.422122+0000","last_became_peered":"2023-10-09T05:18:26.422122+0000","last_unstale":"2023-10-09T05:37:12.040698+0000","last_undegraded":"2023-10-09T05:37:12.040698+0000","last_fullsized":"2023-10-09T05:37:12.040698+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:39.393756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":39,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.926992+0000","last_change":"2023-10-09T05:32:45.831080+0000","last_active":"2023-10-09T05:37:11.926992+0000","last_peered":"2023-10-09T05:37:11.926992+0000","last_clean":"2023-10-09T05:37:11.926992+0000","last_became_active":"2023-10-09T05:32:45.830745+0000","last_became_peered":"2023-10-09T05:32:45.830745+0000","last_unstale":"2023-10-09T05:37:11.926992+0000","last_undegraded":"2023-10-09T05:37:11.926992+0000","last_fullsized":"2023-10-09T05:37:11.926992+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:16.345644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":129,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.927403+0000","last_change":"2023-10-09T05:30:03.395964+0000","last_active":"2023-10-09T05:37:11.927403+0000","last_peered":"2023-10-09T05:37:11.927403+0000","last_clean":"2023-10-09T05:37:11.927403+0000","last_became_active":"2023-10-09T05:30:03.395634+0000","last_became_peered":"2023-10-09T05:30:03.395634+0000","last_unstale":"2023-10-09T05:37:11.927403+0000","last_undegraded":"2023-10-09T05:37:11.927403+0000","last_fullsized":"2023-10-09T05:37:11.927403+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:31.601758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":331,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:12.044706+0000","last_change":"2023-10-09T05:18:26.421678+0000","last_active":"2023-10-09T05:37:12.044706+0000","last_peered":"2023-10-09T05:37:12.044706+0000","last_clean":"2023-10-09T05:37:12.044706+0000","last_became_active":"2023-10-09T05:18:26.419673+0000","last_became_peered":"2023-10-09T05:18:26.419673+0000","last_unstale":"2023-10-09T05:37:12.044706+0000","last_undegraded":"2023-10-09T05:37:12.044706+0000","last_fullsized":"2023-10-09T05:37:12.044706+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:38.876021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":343,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.927285+0000","last_change":"2023-10-09T05:32:45.844532+0000","last_active":"2023-10-09T05:37:11.927285+0000","last_peered":"2023-10-09T05:37:11.927285+0000","last_clean":"2023-10-09T05:37:11.927285+0000","last_became_active":"2023-10-09T05:32:45.831101+0000","last_became_peered":"2023-10-09T05:32:45.831101+0000","last_unstale":"2023-10-09T05:37:11.927285+0000","last_undegraded":"2023-10-09T05:37:11.927285+0000","last_fullsized":"2023-10-09T05:37:11.927285+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:19:37.907188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":24,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.926239+0000","last_change":"2023-10-09T05:36:16.901739+0000","last_active":"2023-10-09T05:37:11.926239+0000","last_peered":"2023-10-09T05:37:11.926239+0000","last_clean":"2023-10-09T05:37:11.926239+0000","last_became_active":"2023-10-09T05:36:16.901380+0000","last_became_peered":"2023-10-09T05:36:16.901380+0000","last_unstale":"2023-10-09T05:37:11.926239+0000","last_undegraded":"2023-10-09T05:37:11.926239+0000","last_fullsized":"2023-10-09T05:37:11.926239+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:26.454798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.2","version":"330'4328","reported_seq":4366,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T05:37:14.097072+0000","last_change":"2023-10-09T05:36:53.565811+0000","last_active":"2023-10-09T05:37:14.097072+0000","last_peered":"2023-10-09T05:37:14.097072+0000","last_clean":"2023-10-09T05:37:14.097072+0000","last_became_active":"2023-10-09T05:36:53.563696+0000","last_became_peered":"2023-10-09T05:36:53.563696+0000","last_unstale":"2023-10-09T05:37:14.097072+0000","last_undegraded":"2023-10-09T05:37:14.097072+0000","last_fullsized":"2023-10-09T05:37:14.097072+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":4328,"log_dups_size":0,"ondisk_log_size":4328,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160137216,"num_objects":2443,"num_object_clones":0,"num_object_copies":7329,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2443,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2443,"num_write_kb":156384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":137,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.929325+0000","last_change":"2023-10-09T05:30:03.409982+0000","last_active":"2023-10-09T05:37:11.929325+0000","last_peered":"2023-10-09T05:37:11.929325+0000","last_clean":"2023-10-09T05:37:11.929325+0000","last_became_active":"2023-10-09T05:30:03.386761+0000","last_became_peered":"2023-10-09T05:30:03.386761+0000","last_unstale":"2023-10-09T05:37:11.929325+0000","last_undegraded":"2023-10-09T05:37:11.929325+0000","last_fullsized":"2023-10-09T05:37:11.929325+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:17.372036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":39,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.926140+0000","last_change":"2023-10-09T05:32:45.829063+0000","last_active":"2023-10-09T05:37:11.926140+0000","last_peered":"2023-10-09T05:37:11.926140+0000","last_clean":"2023-10-09T05:37:11.926140+0000","last_became_active":"2023-10-09T05:32:45.828694+0000","last_became_peered":"2023-10-09T05:32:45.828694+0000","last_unstale":"2023-10-09T05:37:11.926140+0000","last_undegraded":"2023-10-09T05:37:11.926140+0000","last_fullsized":"2023-10-09T05:37:11.926140+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:58:51.140748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"330'4208","reported_seq":4245,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T05:37:14.975115+0000","last_change":"2023-10-09T05:36:53.608829+0000","last_active":"2023-10-09T05:37:14.975115+0000","last_peered":"2023-10-09T05:37:14.975115+0000","last_clean":"2023-10-09T05:37:14.975115+0000","last_became_active":"2023-10-09T05:36:53.607150+0000","last_became_peered":"2023-10-09T05:36:53.607150+0000","last_unstale":"2023-10-09T05:37:14.975115+0000","last_undegraded":"2023-10-09T05:37:14.975115+0000","last_fullsized":"2023-10-09T05:37:14.975115+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":4208,"log_dups_size":0,"ondisk_log_size":4208,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156270592,"num_objects":2384,"num_object_clones":0,"num_object_copies":7152,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2384,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2384,"num_write_kb":152608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":137,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.928509+0000","last_change":"2023-10-09T05:30:03.416268+0000","last_active":"2023-10-09T05:37:11.928509+0000","last_peered":"2023-10-09T05:37:11.928509+0000","last_clean":"2023-10-09T05:37:11.928509+0000","last_became_active":"2023-10-09T05:30:03.392236+0000","last_became_peered":"2023-10-09T05:30:03.392236+0000","last_unstale":"2023-10-09T05:37:11.928509+0000","last_undegraded":"2023-10-09T05:37:11.928509+0000","last_fullsized":"2023-10-09T05:37:11.928509+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:18:34.616445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":197,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.916086+0000","last_change":"2023-10-09T05:30:03.376778+0000","last_active":"2023-10-09T05:37:11.916086+0000","last_peered":"2023-10-09T05:37:11.916086+0000","last_clean":"2023-10-09T05:37:11.916086+0000","last_became_active":"2023-10-09T05:30:03.376056+0000","last_became_peered":"2023-10-09T05:30:03.376056+0000","last_unstale":"2023-10-09T05:37:11.916086+0000","last_undegraded":"2023-10-09T05:37:11.916086+0000","last_fullsized":"2023-10-09T05:37:11.916086+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:06.957639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.0","version":"328'4249","reported_seq":4289,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.536513+0000","last_change":"2023-10-09T05:36:53.624480+0000","last_active":"2023-10-09T05:37:11.536513+0000","last_peered":"2023-10-09T05:37:11.536513+0000","last_clean":"2023-10-09T05:37:11.536513+0000","last_became_active":"2023-10-09T05:36:53.593718+0000","last_became_peered":"2023-10-09T05:36:53.593718+0000","last_unstale":"2023-10-09T05:37:11.536513+0000","last_undegraded":"2023-10-09T05:37:11.536513+0000","last_fullsized":"2023-10-09T05:37:11.536513+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":4249,"log_dups_size":0,"ondisk_log_size":4249,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155942912,"num_objects":2379,"num_object_clones":0,"num_object_copies":7137,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2379,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2379,"num_write_kb":152288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":137,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.929916+0000","last_change":"2023-10-09T05:32:45.838786+0000","last_active":"2023-10-09T05:37:11.929916+0000","last_peered":"2023-10-09T05:37:11.929916+0000","last_clean":"2023-10-09T05:37:11.929916+0000","last_became_active":"2023-10-09T05:32:45.830293+0000","last_became_peered":"2023-10-09T05:32:45.830293+0000","last_unstale":"2023-10-09T05:37:11.929916+0000","last_undegraded":"2023-10-09T05:37:11.929916+0000","last_fullsized":"2023-10-09T05:37:11.929916+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:47.319929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":360,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.928672+0000","last_change":"2023-10-09T05:18:26.423526+0000","last_active":"2023-10-09T05:37:11.928672+0000","last_peered":"2023-10-09T05:37:11.928672+0000","last_clean":"2023-10-09T05:37:11.928672+0000","last_became_active":"2023-10-09T05:18:26.423092+0000","last_became_peered":"2023-10-09T05:18:26.423092+0000","last_unstale":"2023-10-09T05:37:11.928672+0000","last_undegraded":"2023-10-09T05:37:11.928672+0000","last_fullsized":"2023-10-09T05:37:11.928672+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:54.877664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"330'4281","reported_seq":4318,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T05:37:15.674337+0000","last_change":"2023-10-09T05:36:53.569914+0000","last_active":"2023-10-09T05:37:15.674337+0000","last_peered":"2023-10-09T05:37:15.674337+0000","last_clean":"2023-10-09T05:37:15.674337+0000","last_became_active":"2023-10-09T05:36:53.559813+0000","last_became_peered":"2023-10-09T05:36:53.559813+0000","last_unstale":"2023-10-09T05:37:15.674337+0000","last_undegraded":"2023-10-09T05:37:15.674337+0000","last_fullsized":"2023-10-09T05:37:15.674337+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":4281,"log_dups_size":0,"ondisk_log_size":4281,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159842304,"num_objects":2439,"num_object_clones":0,"num_object_copies":7317,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2439,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2439,"num_write_kb":156096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":39,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.926311+0000","last_change":"2023-10-09T05:32:45.829450+0000","last_active":"2023-10-09T05:37:11.926311+0000","last_peered":"2023-10-09T05:37:11.926311+0000","last_clean":"2023-10-09T05:37:11.926311+0000","last_became_active":"2023-10-09T05:32:45.829076+0000","last_became_peered":"2023-10-09T05:32:45.829076+0000","last_unstale":"2023-10-09T05:37:11.926311+0000","last_undegraded":"2023-10-09T05:37:11.926311+0000","last_fullsized":"2023-10-09T05:37:11.926311+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:51:13.538770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":323,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:12.046191+0000","last_change":"2023-10-09T05:30:03.387239+0000","last_active":"2023-10-09T05:37:12.046191+0000","last_peered":"2023-10-09T05:37:12.046191+0000","last_clean":"2023-10-09T05:37:12.046191+0000","last_became_active":"2023-10-09T05:30:03.386633+0000","last_became_peered":"2023-10-09T05:30:03.386633+0000","last_unstale":"2023-10-09T05:37:12.046191+0000","last_undegraded":"2023-10-09T05:37:12.046191+0000","last_fullsized":"2023-10-09T05:37:12.046191+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:51.537133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.6","version":"330'4518","reported_seq":4556,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T05:37:14.947881+0000","last_change":"2023-10-09T05:36:53.611524+0000","last_active":"2023-10-09T05:37:14.947881+0000","last_peered":"2023-10-09T05:37:14.947881+0000","last_clean":"2023-10-09T05:37:14.947881+0000","last_became_active":"2023-10-09T05:36:53.582927+0000","last_became_peered":"2023-10-09T05:36:53.582927+0000","last_unstale":"2023-10-09T05:37:14.947881+0000","last_undegraded":"2023-10-09T05:37:14.947881+0000","last_fullsized":"2023-10-09T05:37:14.947881+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":4518,"log_dups_size":0,"ondisk_log_size":4518,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166985728,"num_objects":2548,"num_object_clones":0,"num_object_copies":7644,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2548,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2548,"num_write_kb":163072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'3","reported_seq":301,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.923266+0000","last_change":"2023-10-09T05:30:03.399748+0000","last_active":"2023-10-09T05:37:11.923266+0000","last_peered":"2023-10-09T05:37:11.923266+0000","last_clean":"2023-10-09T05:37:11.923266+0000","last_became_active":"2023-10-09T05:30:03.384427+0000","last_became_peered":"2023-10-09T05:30:03.384427+0000","last_unstale":"2023-10-09T05:37:11.923266+0000","last_undegraded":"2023-10-09T05:37:11.923266+0000","last_fullsized":"2023-10-09T05:37:11.923266+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:01.128892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":132,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.923359+0000","last_change":"2023-10-09T05:30:03.378783+0000","last_active":"2023-10-09T05:37:11.923359+0000","last_peered":"2023-10-09T05:37:11.923359+0000","last_clean":"2023-10-09T05:37:11.923359+0000","last_became_active":"2023-10-09T05:30:03.378436+0000","last_became_peered":"2023-10-09T05:30:03.378436+0000","last_unstale":"2023-10-09T05:37:11.923359+0000","last_undegraded":"2023-10-09T05:37:11.923359+0000","last_fullsized":"2023-10-09T05:37:11.923359+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:54.102857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.7","version":"330'4312","reported_seq":4349,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T05:37:14.294595+0000","last_change":"2023-10-09T05:36:53.563801+0000","last_active":"2023-10-09T05:37:14.294595+0000","last_peered":"2023-10-09T05:37:14.294595+0000","last_clean":"2023-10-09T05:37:14.294595+0000","last_became_active":"2023-10-09T05:36:53.559374+0000","last_became_peered":"2023-10-09T05:36:53.559374+0000","last_unstale":"2023-10-09T05:37:14.294595+0000","last_undegraded":"2023-10-09T05:37:14.294595+0000","last_fullsized":"2023-10-09T05:37:14.294595+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":4312,"log_dups_size":0,"ondisk_log_size":4312,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160890880,"num_objects":2455,"num_object_clones":0,"num_object_copies":7365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2455,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2455,"num_write_kb":157120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":341,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.928410+0000","last_change":"2023-10-09T05:30:03.381940+0000","last_active":"2023-10-09T05:37:11.928410+0000","last_peered":"2023-10-09T05:37:11.928410+0000","last_clean":"2023-10-09T05:37:11.928410+0000","last_became_active":"2023-10-09T05:30:03.381526+0000","last_became_peered":"2023-10-09T05:30:03.381526+0000","last_unstale":"2023-10-09T05:37:11.928410+0000","last_undegraded":"2023-10-09T05:37:11.928410+0000","last_fullsized":"2023-10-09T05:37:11.928410+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:06.078801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":298,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-10-09T05:37:09.935682+0000","last_change":"2023-10-09T05:30:03.387924+0000","last_active":"2023-10-09T05:37:09.935682+0000","last_peered":"2023-10-09T05:37:09.935682+0000","last_clean":"2023-10-09T05:37:09.935682+0000","last_became_active":"2023-10-09T05:30:03.387493+0000","last_became_peered":"2023-10-09T05:30:03.387493+0000","last_unstale":"2023-10-09T05:37:09.935682+0000","last_undegraded":"2023-10-09T05:37:09.935682+0000","last_fullsized":"2023-10-09T05:37:09.935682+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:16.554514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.4","version":"330'4265","reported_seq":4303,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T05:37:14.221559+0000","last_change":"2023-10-09T05:36:53.579851+0000","last_active":"2023-10-09T05:37:14.221559+0000","last_peered":"2023-10-09T05:37:14.221559+0000","last_clean":"2023-10-09T05:37:14.221559+0000","last_became_active":"2023-10-09T05:36:53.577754+0000","last_became_peered":"2023-10-09T05:36:53.577754+0000","last_unstale":"2023-10-09T05:37:14.221559+0000","last_undegraded":"2023-10-09T05:37:14.221559+0000","last_fullsized":"2023-10-09T05:37:14.221559+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":4265,"log_dups_size":0,"ondisk_log_size":4265,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158466048,"num_objects":2418,"num_object_clones":0,"num_object_copies":7254,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2418,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2418,"num_write_kb":154752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":267,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-10-09T05:37:09.935110+0000","last_change":"2023-10-09T05:30:03.372784+0000","last_active":"2023-10-09T05:37:09.935110+0000","last_peered":"2023-10-09T05:37:09.935110+0000","last_clean":"2023-10-09T05:37:09.935110+0000","last_became_active":"2023-10-09T05:30:03.372458+0000","last_became_peered":"2023-10-09T05:30:03.372458+0000","last_unstale":"2023-10-09T05:37:09.935110+0000","last_undegraded":"2023-10-09T05:37:09.935110+0000","last_fullsized":"2023-10-09T05:37:09.935110+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:36:24.881545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":299,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.929032+0000","last_change":"2023-10-09T05:30:03.382201+0000","last_active":"2023-10-09T05:37:11.929032+0000","last_peered":"2023-10-09T05:37:11.929032+0000","last_clean":"2023-10-09T05:37:11.929032+0000","last_became_active":"2023-10-09T05:30:03.381727+0000","last_became_peered":"2023-10-09T05:30:03.381727+0000","last_unstale":"2023-10-09T05:37:11.929032+0000","last_undegraded":"2023-10-09T05:37:11.929032+0000","last_fullsized":"2023-10-09T05:37:11.929032+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:14.559280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.5","version":"330'4330","reported_seq":4368,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T05:37:14.967234+0000","last_change":"2023-10-09T05:36:53.572718+0000","last_active":"2023-10-09T05:37:14.967234+0000","last_peered":"2023-10-09T05:37:14.967234+0000","last_clean":"2023-10-09T05:37:14.967234+0000","last_became_active":"2023-10-09T05:36:53.564481+0000","last_became_peered":"2023-10-09T05:36:53.564481+0000","last_unstale":"2023-10-09T05:37:14.967234+0000","last_undegraded":"2023-10-09T05:37:14.967234+0000","last_fullsized":"2023-10-09T05:37:14.967234+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":4330,"log_dups_size":0,"ondisk_log_size":4330,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160661504,"num_objects":2451,"num_object_clones":0,"num_object_copies":7353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2451,"num_write_kb":156896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":347,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.922743+0000","last_change":"2023-10-09T05:18:26.413106+0000","last_active":"2023-10-09T05:37:11.922743+0000","last_peered":"2023-10-09T05:37:11.922743+0000","last_clean":"2023-10-09T05:37:11.922743+0000","last_became_active":"2023-10-09T05:18:26.412334+0000","last_became_peered":"2023-10-09T05:18:26.412334+0000","last_unstale":"2023-10-09T05:37:11.922743+0000","last_undegraded":"2023-10-09T05:37:11.922743+0000","last_fullsized":"2023-10-09T05:37:11.922743+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:18.058016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":210,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.920184+0000","last_change":"2023-10-09T05:30:03.383456+0000","last_active":"2023-10-09T05:37:11.920184+0000","last_peered":"2023-10-09T05:37:11.920184+0000","last_clean":"2023-10-09T05:37:11.920184+0000","last_became_active":"2023-10-09T05:30:03.382656+0000","last_became_peered":"2023-10-09T05:30:03.382656+0000","last_unstale":"2023-10-09T05:37:11.920184+0000","last_undegraded":"2023-10-09T05:37:11.920184+0000","last_fullsized":"2023-10-09T05:37:11.920184+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:47:23.383409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.8","version":"330'4468","reported_seq":4506,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T05:37:15.018775+0000","last_change":"2023-10-09T05:36:53.597306+0000","last_active":"2023-10-09T05:37:15.018775+0000","last_peered":"2023-10-09T05:37:15.018775+0000","last_clean":"2023-10-09T05:37:15.018775+0000","last_became_active":"2023-10-09T05:36:53.593366+0000","last_became_peered":"2023-10-09T05:36:53.593366+0000","last_unstale":"2023-10-09T05:37:15.018775+0000","last_undegraded":"2023-10-09T05:37:15.018775+0000","last_fullsized":"2023-10-09T05:37:15.018775+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":4468,"log_dups_size":0,"ondisk_log_size":4468,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164429824,"num_objects":2509,"num_object_clones":0,"num_object_copies":7527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2509,"num_write_kb":160576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":238,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T05:37:12.929387+0000","last_change":"2023-10-09T05:18:26.422150+0000","last_active":"2023-10-09T05:37:12.929387+0000","last_peered":"2023-10-09T05:37:12.929387+0000","last_clean":"2023-10-09T05:37:12.929387+0000","last_became_active":"2023-10-09T05:18:26.421853+0000","last_became_peered":"2023-10-09T05:18:26.421853+0000","last_unstale":"2023-10-09T05:37:12.929387+0000","last_undegraded":"2023-10-09T05:37:12.929387+0000","last_fullsized":"2023-10-09T05:37:12.929387+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:37:47.000556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":39,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.927074+0000","last_change":"2023-10-09T05:32:45.832271+0000","last_active":"2023-10-09T05:37:11.927074+0000","last_peered":"2023-10-09T05:37:11.927074+0000","last_clean":"2023-10-09T05:37:11.927074+0000","last_became_active":"2023-10-09T05:32:45.831997+0000","last_became_peered":"2023-10-09T05:32:45.831997+0000","last_unstale":"2023-10-09T05:37:11.927074+0000","last_undegraded":"2023-10-09T05:37:11.927074+0000","last_fullsized":"2023-10-09T05:37:11.927074+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:18.970416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":112,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.918931+0000","last_change":"2023-10-09T05:32:45.819022+0000","last_active":"2023-10-09T05:37:11.918931+0000","last_peered":"2023-10-09T05:37:11.918931+0000","last_clean":"2023-10-09T05:37:11.918931+0000","last_became_active":"2023-10-09T05:32:45.818726+0000","last_became_peered":"2023-10-09T05:32:45.818726+0000","last_unstale":"2023-10-09T05:37:11.918931+0000","last_undegraded":"2023-10-09T05:37:11.918931+0000","last_fullsized":"2023-10-09T05:37:11.918931+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:04.414263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.e","version":"328'4834","reported_seq":4860,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.536414+0000","last_change":"2023-10-09T05:35:46.303342+0000","last_active":"2023-10-09T05:37:11.536414+0000","last_peered":"2023-10-09T05:37:11.536414+0000","last_clean":"2023-10-09T05:37:11.536414+0000","last_became_active":"2023-10-09T05:35:46.302540+0000","last_became_peered":"2023-10-09T05:35:46.302540+0000","last_unstale":"2023-10-09T05:37:11.536414+0000","last_undegraded":"2023-10-09T05:37:11.536414+0000","last_fullsized":"2023-10-09T05:37:11.536414+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":4834,"log_dups_size":0,"ondisk_log_size":4834,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:00:27.626139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316801024,"num_objects":4834,"num_object_clones":0,"num_object_copies":14502,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4834,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4834,"num_write_kb":309376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":39,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.926851+0000","last_change":"2023-10-09T05:32:45.825342+0000","last_active":"2023-10-09T05:37:11.926851+0000","last_peered":"2023-10-09T05:37:11.926851+0000","last_clean":"2023-10-09T05:37:11.926851+0000","last_became_active":"2023-10-09T05:32:45.824981+0000","last_became_peered":"2023-10-09T05:32:45.824981+0000","last_unstale":"2023-10-09T05:37:11.926851+0000","last_undegraded":"2023-10-09T05:37:11.926851+0000","last_fullsized":"2023-10-09T05:37:11.926851+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:02.868005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":98,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:12.046440+0000","last_change":"2023-10-09T05:30:03.381036+0000","last_active":"2023-10-09T05:37:12.046440+0000","last_peered":"2023-10-09T05:37:12.046440+0000","last_clean":"2023-10-09T05:37:12.046440+0000","last_became_active":"2023-10-09T05:30:03.380698+0000","last_became_peered":"2023-10-09T05:30:03.380698+0000","last_unstale":"2023-10-09T05:37:12.046440+0000","last_undegraded":"2023-10-09T05:37:12.046440+0000","last_fullsized":"2023-10-09T05:37:12.046440+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:16.464471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":94,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.916390+0000","last_change":"2023-10-09T05:23:18.196260+0000","last_active":"2023-10-09T05:37:11.916390+0000","last_peered":"2023-10-09T05:37:11.916390+0000","last_clean":"2023-10-09T05:37:11.916390+0000","last_became_active":"2023-10-09T05:23:18.195907+0000","last_became_peered":"2023-10-09T05:23:18.195907+0000","last_unstale":"2023-10-09T05:37:11.916390+0000","last_undegraded":"2023-10-09T05:37:11.916390+0000","last_fullsized":"2023-10-09T05:37:11.916390+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:36.243432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.d","version":"330'4870","reported_seq":4897,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T05:37:14.260763+0000","last_change":"2023-10-09T05:35:46.303676+0000","last_active":"2023-10-09T05:37:14.260763+0000","last_peered":"2023-10-09T05:37:14.260763+0000","last_clean":"2023-10-09T05:37:14.260763+0000","last_became_active":"2023-10-09T05:35:46.303364+0000","last_became_peered":"2023-10-09T05:35:46.303364+0000","last_unstale":"2023-10-09T05:37:14.260763+0000","last_undegraded":"2023-10-09T05:37:14.260763+0000","last_fullsized":"2023-10-09T05:37:14.260763+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":4870,"log_dups_size":0,"ondisk_log_size":4870,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:56.816946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319160320,"num_objects":4870,"num_object_clones":0,"num_object_copies":14610,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4870,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4870,"num_write_kb":311680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":110,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-10-09T05:37:09.935004+0000","last_change":"2023-10-09T05:20:18.123706+0000","last_active":"2023-10-09T05:37:09.935004+0000","last_peered":"2023-10-09T05:37:09.935004+0000","last_clean":"2023-10-09T05:37:09.935004+0000","last_became_active":"2023-10-09T05:20:18.122859+0000","last_became_peered":"2023-10-09T05:20:18.122859+0000","last_unstale":"2023-10-09T05:37:09.935004+0000","last_undegraded":"2023-10-09T05:37:09.935004+0000","last_fullsized":"2023-10-09T05:37:09.935004+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.906228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":202,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.915732+0000","last_change":"2023-10-09T05:09:57.854064+0000","last_active":"2023-10-09T05:37:11.915732+0000","last_peered":"2023-10-09T05:37:11.915732+0000","last_clean":"2023-10-09T05:37:11.915732+0000","last_became_active":"2023-10-09T05:09:57.847953+0000","last_became_peered":"2023-10-09T05:09:57.847953+0000","last_unstale":"2023-10-09T05:37:11.915732+0000","last_undegraded":"2023-10-09T05:37:11.915732+0000","last_fullsized":"2023-10-09T05:37:11.915732+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:45.914976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":83,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-10-09T05:37:09.935452+0000","last_change":"2023-10-09T05:32:45.832106+0000","last_active":"2023-10-09T05:37:09.935452+0000","last_peered":"2023-10-09T05:37:09.935452+0000","last_clean":"2023-10-09T05:37:09.935452+0000","last_became_active":"2023-10-09T05:32:45.831763+0000","last_became_peered":"2023-10-09T05:32:45.831763+0000","last_unstale":"2023-10-09T05:37:09.935452+0000","last_undegraded":"2023-10-09T05:37:09.935452+0000","last_fullsized":"2023-10-09T05:37:09.935452+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:06:37.005113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.f","version":"330'4896","reported_seq":4923,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T05:37:14.761339+0000","last_change":"2023-10-09T05:35:46.309035+0000","last_active":"2023-10-09T05:37:14.761339+0000","last_peered":"2023-10-09T05:37:14.761339+0000","last_clean":"2023-10-09T05:37:14.761339+0000","last_became_active":"2023-10-09T05:35:46.308730+0000","last_became_peered":"2023-10-09T05:35:46.308730+0000","last_unstale":"2023-10-09T05:37:14.761339+0000","last_undegraded":"2023-10-09T05:37:14.761339+0000","last_fullsized":"2023-10-09T05:37:14.761339+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":4896,"log_dups_size":0,"ondisk_log_size":4896,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:08.994300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320864256,"num_objects":4896,"num_object_clones":0,"num_object_copies":14688,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4896,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4896,"num_write_kb":313344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":39,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.926200+0000","last_change":"2023-10-09T05:32:45.825075+0000","last_active":"2023-10-09T05:37:11.926200+0000","last_peered":"2023-10-09T05:37:11.926200+0000","last_clean":"2023-10-09T05:37:11.926200+0000","last_became_active":"2023-10-09T05:32:45.824730+0000","last_became_peered":"2023-10-09T05:32:45.824730+0000","last_unstale":"2023-10-09T05:37:11.926200+0000","last_undegraded":"2023-10-09T05:37:11.926200+0000","last_fullsized":"2023-10-09T05:37:11.926200+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:33.484326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":615,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.926375+0000","last_change":"2023-10-09T05:18:29.419527+0000","last_active":"2023-10-09T05:37:11.926375+0000","last_peered":"2023-10-09T05:37:11.926375+0000","last_clean":"2023-10-09T05:37:11.926375+0000","last_became_active":"2023-10-09T05:18:29.418852+0000","last_became_peered":"2023-10-09T05:18:29.418852+0000","last_unstale":"2023-10-09T05:37:11.926375+0000","last_undegraded":"2023-10-09T05:37:11.926375+0000","last_fullsized":"2023-10-09T05:37:11.926375+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_clean_scrub_stamp":"2023-10-09T04:36:44.261013+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:02.899170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":314,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.923323+0000","last_change":"2023-10-09T05:32:45.829840+0000","last_active":"2023-10-09T05:37:11.923323+0000","last_peered":"2023-10-09T05:37:11.923323+0000","last_clean":"2023-10-09T05:37:11.923323+0000","last_became_active":"2023-10-09T05:32:45.829084+0000","last_became_peered":"2023-10-09T05:32:45.829084+0000","last_unstale":"2023-10-09T05:37:11.923323+0000","last_undegraded":"2023-10-09T05:37:11.923323+0000","last_fullsized":"2023-10-09T05:37:11.923323+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:26:48.836506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":323,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.929481+0000","last_change":"2023-10-09T05:30:03.386395+0000","last_active":"2023-10-09T05:37:11.929481+0000","last_peered":"2023-10-09T05:37:11.929481+0000","last_clean":"2023-10-09T05:37:11.929481+0000","last_became_active":"2023-10-09T05:30:03.386024+0000","last_became_peered":"2023-10-09T05:30:03.386024+0000","last_unstale":"2023-10-09T05:37:11.929481+0000","last_undegraded":"2023-10-09T05:37:11.929481+0000","last_fullsized":"2023-10-09T05:37:11.929481+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:13.133829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"330'4889","reported_seq":4915,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T05:37:15.062125+0000","last_change":"2023-10-09T05:35:46.308831+0000","last_active":"2023-10-09T05:37:15.062125+0000","last_peered":"2023-10-09T05:37:15.062125+0000","last_clean":"2023-10-09T05:37:15.062125+0000","last_became_active":"2023-10-09T05:35:46.308458+0000","last_became_peered":"2023-10-09T05:35:46.308458+0000","last_unstale":"2023-10-09T05:37:15.062125+0000","last_undegraded":"2023-10-09T05:37:15.062125+0000","last_fullsized":"2023-10-09T05:37:15.062125+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":4889,"log_dups_size":0,"ondisk_log_size":4889,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:30.298840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320405504,"num_objects":4889,"num_object_clones":0,"num_object_copies":14667,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4889,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4889,"num_write_kb":312896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":139,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.926508+0000","last_change":"2023-10-09T05:23:18.193825+0000","last_active":"2023-10-09T05:37:11.926508+0000","last_peered":"2023-10-09T05:37:11.926508+0000","last_clean":"2023-10-09T05:37:11.926508+0000","last_became_active":"2023-10-09T05:23:18.193394+0000","last_became_peered":"2023-10-09T05:23:18.193394+0000","last_unstale":"2023-10-09T05:37:11.926508+0000","last_undegraded":"2023-10-09T05:37:11.926508+0000","last_fullsized":"2023-10-09T05:37:11.926508+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:40.594469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":315,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:12.040170+0000","last_change":"2023-10-09T05:32:45.824031+0000","last_active":"2023-10-09T05:37:12.040170+0000","last_peered":"2023-10-09T05:37:12.040170+0000","last_clean":"2023-10-09T05:37:12.040170+0000","last_became_active":"2023-10-09T05:32:45.823651+0000","last_became_peered":"2023-10-09T05:32:45.823651+0000","last_unstale":"2023-10-09T05:37:12.040170+0000","last_undegraded":"2023-10-09T05:37:12.040170+0000","last_fullsized":"2023-10-09T05:37:12.040170+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:14:20.745117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":375,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:12.044576+0000","last_change":"2023-10-09T05:20:18.132750+0000","last_active":"2023-10-09T05:37:12.044576+0000","last_peered":"2023-10-09T05:37:12.044576+0000","last_clean":"2023-10-09T05:37:12.044576+0000","last_became_active":"2023-10-09T05:20:18.132396+0000","last_became_peered":"2023-10-09T05:20:18.132396+0000","last_unstale":"2023-10-09T05:37:12.044576+0000","last_undegraded":"2023-10-09T05:37:12.044576+0000","last_fullsized":"2023-10-09T05:37:12.044576+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:00:55.731045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.9","version":"330'4314","reported_seq":4351,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T05:37:14.303275+0000","last_change":"2023-10-09T05:36:53.571075+0000","last_active":"2023-10-09T05:37:14.303275+0000","last_peered":"2023-10-09T05:37:14.303275+0000","last_clean":"2023-10-09T05:37:14.303275+0000","last_became_active":"2023-10-09T05:36:53.569007+0000","last_became_peered":"2023-10-09T05:36:53.569007+0000","last_unstale":"2023-10-09T05:37:14.303275+0000","last_undegraded":"2023-10-09T05:37:14.303275+0000","last_fullsized":"2023-10-09T05:37:14.303275+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":4314,"log_dups_size":0,"ondisk_log_size":4314,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160694272,"num_objects":2452,"num_object_clones":0,"num_object_copies":7356,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2452,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2452,"num_write_kb":156928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":129,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.930029+0000","last_change":"2023-10-09T05:30:03.391750+0000","last_active":"2023-10-09T05:37:11.930029+0000","last_peered":"2023-10-09T05:37:11.930029+0000","last_clean":"2023-10-09T05:37:11.930029+0000","last_became_active":"2023-10-09T05:30:03.391393+0000","last_became_peered":"2023-10-09T05:30:03.391393+0000","last_unstale":"2023-10-09T05:37:11.930029+0000","last_undegraded":"2023-10-09T05:37:11.930029+0000","last_fullsized":"2023-10-09T05:37:11.930029+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:07.263598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":238,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T05:37:12.929319+0000","last_change":"2023-10-09T05:32:45.824227+0000","last_active":"2023-10-09T05:37:12.929319+0000","last_peered":"2023-10-09T05:37:12.929319+0000","last_clean":"2023-10-09T05:37:12.929319+0000","last_became_active":"2023-10-09T05:32:45.823662+0000","last_became_peered":"2023-10-09T05:32:45.823662+0000","last_unstale":"2023-10-09T05:37:12.929319+0000","last_undegraded":"2023-10-09T05:37:12.929319+0000","last_fullsized":"2023-10-09T05:37:12.929319+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:54:56.029126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":346,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:12.046328+0000","last_change":"2023-10-09T05:32:45.826815+0000","last_active":"2023-10-09T05:37:12.046328+0000","last_peered":"2023-10-09T05:37:12.046328+0000","last_clean":"2023-10-09T05:37:12.046328+0000","last_became_active":"2023-10-09T05:32:45.826529+0000","last_became_peered":"2023-10-09T05:32:45.826529+0000","last_unstale":"2023-10-09T05:37:12.046328+0000","last_undegraded":"2023-10-09T05:37:12.046328+0000","last_fullsized":"2023-10-09T05:37:12.046328+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:19.868533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.a","version":"330'4853","reported_seq":4880,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T05:37:14.279809+0000","last_change":"2023-10-09T05:35:46.307929+0000","last_active":"2023-10-09T05:37:14.279809+0000","last_peered":"2023-10-09T05:37:14.279809+0000","last_clean":"2023-10-09T05:37:14.279809+0000","last_became_active":"2023-10-09T05:35:46.307610+0000","last_became_peered":"2023-10-09T05:35:46.307610+0000","last_unstale":"2023-10-09T05:37:14.279809+0000","last_undegraded":"2023-10-09T05:37:14.279809+0000","last_fullsized":"2023-10-09T05:37:14.279809+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":4853,"log_dups_size":0,"ondisk_log_size":4853,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:40.352492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318046208,"num_objects":4853,"num_object_clones":0,"num_object_copies":14559,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4853,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4853,"num_write_kb":310592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":340,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.923767+0000","last_change":"2023-10-09T05:18:26.431563+0000","last_active":"2023-10-09T05:37:11.923767+0000","last_peered":"2023-10-09T05:37:11.923767+0000","last_clean":"2023-10-09T05:37:11.923767+0000","last_became_active":"2023-10-09T05:18:26.421025+0000","last_became_peered":"2023-10-09T05:18:26.421025+0000","last_unstale":"2023-10-09T05:37:11.923767+0000","last_undegraded":"2023-10-09T05:37:11.923767+0000","last_fullsized":"2023-10-09T05:37:11.923767+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:30.333585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":103,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.920189+0000","last_change":"2023-10-09T05:30:03.375221+0000","last_active":"2023-10-09T05:37:11.920189+0000","last_peered":"2023-10-09T05:37:11.920189+0000","last_clean":"2023-10-09T05:37:11.920189+0000","last_became_active":"2023-10-09T05:30:03.374863+0000","last_became_peered":"2023-10-09T05:30:03.374863+0000","last_unstale":"2023-10-09T05:37:11.920189+0000","last_undegraded":"2023-10-09T05:37:11.920189+0000","last_fullsized":"2023-10-09T05:37:11.920189+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:16.058179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":339,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.922840+0000","last_change":"2023-10-09T05:21:08.179657+0000","last_active":"2023-10-09T05:37:11.922840+0000","last_peered":"2023-10-09T05:37:11.922840+0000","last_clean":"2023-10-09T05:37:11.922840+0000","last_became_active":"2023-10-09T05:21:08.178958+0000","last_became_peered":"2023-10-09T05:21:08.178958+0000","last_unstale":"2023-10-09T05:37:11.922840+0000","last_undegraded":"2023-10-09T05:37:11.922840+0000","last_fullsized":"2023-10-09T05:37:11.922840+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:20.815986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"330'4906","reported_seq":4934,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T05:37:15.045612+0000","last_change":"2023-10-09T05:35:46.308213+0000","last_active":"2023-10-09T05:37:15.045612+0000","last_peered":"2023-10-09T05:37:15.045612+0000","last_clean":"2023-10-09T05:37:15.045612+0000","last_became_active":"2023-10-09T05:35:46.307896+0000","last_became_peered":"2023-10-09T05:35:46.307896+0000","last_unstale":"2023-10-09T05:37:15.045612+0000","last_undegraded":"2023-10-09T05:37:15.045612+0000","last_fullsized":"2023-10-09T05:37:15.045612+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":4906,"log_dups_size":0,"ondisk_log_size":4906,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:11.538673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321519616,"num_objects":4906,"num_object_clones":0,"num_object_copies":14718,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4906,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4906,"num_write_kb":313984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":335,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.929639+0000","last_change":"2023-10-09T05:18:26.424429+0000","last_active":"2023-10-09T05:37:11.929639+0000","last_peered":"2023-10-09T05:37:11.929639+0000","last_clean":"2023-10-09T05:37:11.929639+0000","last_became_active":"2023-10-09T05:18:26.423034+0000","last_became_peered":"2023-10-09T05:18:26.423034+0000","last_unstale":"2023-10-09T05:37:11.929639+0000","last_undegraded":"2023-10-09T05:37:11.929639+0000","last_fullsized":"2023-10-09T05:37:11.929639+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:16.251181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":49,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-10-09T05:37:09.935676+0000","last_change":"2023-10-09T05:31:16.444031+0000","last_active":"2023-10-09T05:37:09.935676+0000","last_peered":"2023-10-09T05:37:09.935676+0000","last_clean":"2023-10-09T05:37:09.935676+0000","last_became_active":"2023-10-09T05:31:16.412865+0000","last_became_peered":"2023-10-09T05:31:16.412865+0000","last_unstale":"2023-10-09T05:37:09.935676+0000","last_undegraded":"2023-10-09T05:37:09.935676+0000","last_fullsized":"2023-10-09T05:37:09.935676+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:04:06.520632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":39,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.926746+0000","last_change":"2023-10-09T05:32:45.828380+0000","last_active":"2023-10-09T05:37:11.926746+0000","last_peered":"2023-10-09T05:37:11.926746+0000","last_clean":"2023-10-09T05:37:11.926746+0000","last_became_active":"2023-10-09T05:32:45.827888+0000","last_became_peered":"2023-10-09T05:32:45.827888+0000","last_unstale":"2023-10-09T05:37:11.926746+0000","last_undegraded":"2023-10-09T05:37:11.926746+0000","last_fullsized":"2023-10-09T05:37:11.926746+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:48.145813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.16","version":"330'4548","reported_seq":4581,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T05:37:15.004417+0000","last_change":"2023-10-09T05:36:53.613658+0000","last_active":"2023-10-09T05:37:15.004417+0000","last_peered":"2023-10-09T05:37:15.004417+0000","last_clean":"2023-10-09T05:37:15.004417+0000","last_became_active":"2023-10-09T05:36:53.610769+0000","last_became_peered":"2023-10-09T05:36:53.610769+0000","last_unstale":"2023-10-09T05:37:15.004417+0000","last_undegraded":"2023-10-09T05:37:15.004417+0000","last_fullsized":"2023-10-09T05:37:15.004417+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":4548,"log_dups_size":0,"ondisk_log_size":4548,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168951808,"num_objects":2578,"num_object_clones":0,"num_object_copies":7734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2578,"num_write_kb":164992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'2","reported_seq":113,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:12.046081+0000","last_change":"2023-10-09T05:30:03.386435+0000","last_active":"2023-10-09T05:37:12.046081+0000","last_peered":"2023-10-09T05:37:12.046081+0000","last_clean":"2023-10-09T05:37:12.046081+0000","last_became_active":"2023-10-09T05:30:03.385418+0000","last_became_peered":"2023-10-09T05:30:03.385418+0000","last_unstale":"2023-10-09T05:37:12.046081+0000","last_undegraded":"2023-10-09T05:37:12.046081+0000","last_fullsized":"2023-10-09T05:37:12.046081+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:34.685368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":117,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-10-09T05:37:09.935188+0000","last_change":"2023-10-09T05:36:16.898207+0000","last_active":"2023-10-09T05:37:09.935188+0000","last_peered":"2023-10-09T05:37:09.935188+0000","last_clean":"2023-10-09T05:37:09.935188+0000","last_became_active":"2023-10-09T05:36:16.898009+0000","last_became_peered":"2023-10-09T05:36:16.898009+0000","last_unstale":"2023-10-09T05:37:09.935188+0000","last_undegraded":"2023-10-09T05:37:09.935188+0000","last_fullsized":"2023-10-09T05:37:09.935188+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:59.694977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.11","version":"330'4288","reported_seq":4332,"reported_epoch":330,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-09T05:37:15.520742+0000","last_change":"2023-10-09T05:37:08.979506+0000","last_active":"2023-10-09T05:37:15.520742+0000","last_peered":"2023-10-09T05:37:15.520742+0000","last_clean":"2023-10-09T05:37:07.363078+0000","last_became_active":"2023-10-09T05:37:08.972861+0000","last_became_peered":"2023-10-09T05:37:08.972861+0000","last_unstale":"2023-10-09T05:37:15.520742+0000","last_undegraded":"2023-10-09T05:37:15.520742+0000","last_fullsized":"2023-10-09T05:37:15.520742+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":4288,"log_dups_size":0,"ondisk_log_size":4288,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160301056,"num_objects":2446,"num_object_clones":0,"num_object_copies":7338,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2356,"num_objects_unfound":0,"num_objects_dirty":2446,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2446,"num_write_kb":156544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[1,2,3],"avail_no_missing":["1","2","3"],"object_location_counts":[{"shards":"1,2,3","objects":90}],"blocked_by":[],"up_primary":6,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":43,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.926175+0000","last_change":"2023-10-09T05:32:45.847749+0000","last_active":"2023-10-09T05:37:11.926175+0000","last_peered":"2023-10-09T05:37:11.926175+0000","last_clean":"2023-10-09T05:37:11.926175+0000","last_became_active":"2023-10-09T05:32:45.825335+0000","last_became_peered":"2023-10-09T05:32:45.825335+0000","last_unstale":"2023-10-09T05:37:11.926175+0000","last_undegraded":"2023-10-09T05:37:11.926175+0000","last_fullsized":"2023-10-09T05:37:11.926175+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:42.082798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":317,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.923035+0000","last_change":"2023-10-09T05:30:03.380938+0000","last_active":"2023-10-09T05:37:11.923035+0000","last_peered":"2023-10-09T05:37:11.923035+0000","last_clean":"2023-10-09T05:37:11.923035+0000","last_became_active":"2023-10-09T05:30:03.380395+0000","last_became_peered":"2023-10-09T05:30:03.380395+0000","last_unstale":"2023-10-09T05:37:11.923035+0000","last_undegraded":"2023-10-09T05:37:11.923035+0000","last_fullsized":"2023-10-09T05:37:11.923035+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:23.087303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.10","version":"330'4286","reported_seq":6270,"reported_epoch":330,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T05:37:15.044525+0000","last_change":"2023-10-09T05:37:00.160898+0000","last_active":"2023-10-09T05:37:15.044525+0000","last_peered":"2023-10-09T05:37:15.044525+0000","last_clean":"2023-10-09T05:36:58.540725+0000","last_became_active":"2023-10-09T05:37:00.128129+0000","last_became_peered":"2023-10-09T05:37:00.128129+0000","last_unstale":"2023-10-09T05:37:15.044525+0000","last_undegraded":"2023-10-09T05:37:15.044525+0000","last_fullsized":"2023-10-09T05:36:59.898109+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":4286,"log_dups_size":0,"ondisk_log_size":4286,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158367744,"num_objects":2417,"num_object_clones":0,"num_object_copies":7251,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2888,"num_objects_unfound":0,"num_objects_dirty":2417,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2417,"num_write_kb":154656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":647,"num_bytes_recovered":42401792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":973},{"shards":"0,2,5","objects":1444}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":43,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.927132+0000","last_change":"2023-10-09T05:32:45.860675+0000","last_active":"2023-10-09T05:37:11.927132+0000","last_peered":"2023-10-09T05:37:11.927132+0000","last_clean":"2023-10-09T05:37:11.927132+0000","last_became_active":"2023-10-09T05:32:45.828515+0000","last_became_peered":"2023-10-09T05:32:45.828515+0000","last_unstale":"2023-10-09T05:37:11.927132+0000","last_undegraded":"2023-10-09T05:37:11.927132+0000","last_fullsized":"2023-10-09T05:37:11.927132+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:39.371415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":148,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.923242+0000","last_change":"2023-10-09T05:30:03.376908+0000","last_active":"2023-10-09T05:37:11.923242+0000","last_peered":"2023-10-09T05:37:11.923242+0000","last_clean":"2023-10-09T05:37:11.923242+0000","last_became_active":"2023-10-09T05:30:03.376456+0000","last_became_peered":"2023-10-09T05:30:03.376456+0000","last_unstale":"2023-10-09T05:37:11.923242+0000","last_undegraded":"2023-10-09T05:37:11.923242+0000","last_fullsized":"2023-10-09T05:37:11.923242+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:23.173899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.13","version":"330'4191","reported_seq":4239,"reported_epoch":330,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:37:15.004581+0000","last_change":"2023-10-09T05:37:13.510590+0000","last_active":"2023-10-09T05:37:15.004581+0000","last_peered":"2023-10-09T05:37:15.004581+0000","last_clean":"2023-10-09T05:37:10.784717+0000","last_became_active":"2023-10-09T05:37:13.505099+0000","last_became_peered":"2023-10-09T05:37:13.505099+0000","last_unstale":"2023-10-09T05:37:15.004581+0000","last_undegraded":"2023-10-09T05:37:13.016718+0000","last_fullsized":"2023-10-09T05:37:12.926732+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":4191,"log_dups_size":0,"ondisk_log_size":4191,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155156480,"num_objects":2368,"num_object_clones":0,"num_object_copies":7104,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4636,"num_objects_misplaced":2318,"num_objects_unfound":0,"num_objects_dirty":2368,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2368,"num_write_kb":151520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[0,2],"avail_no_missing":["0","1","3"],"object_location_counts":[{"shards":"0,1,3","objects":2318},{"shards":"0,2","objects":50}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"69'2","reported_seq":333,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.923737+0000","last_change":"2023-10-09T05:22:47.197002+0000","last_active":"2023-10-09T05:37:11.923737+0000","last_peered":"2023-10-09T05:37:11.923737+0000","last_clean":"2023-10-09T05:37:11.923737+0000","last_became_active":"2023-10-09T05:22:47.196685+0000","last_became_peered":"2023-10-09T05:22:47.196685+0000","last_unstale":"2023-10-09T05:37:11.923737+0000","last_undegraded":"2023-10-09T05:37:11.923737+0000","last_fullsized":"2023-10-09T05:37:11.923737+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:47:32.594447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":317,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.923003+0000","last_change":"2023-10-09T05:02:09.649674+0000","last_active":"2023-10-09T05:37:11.923003+0000","last_peered":"2023-10-09T05:37:11.923003+0000","last_clean":"2023-10-09T05:37:11.923003+0000","last_became_active":"2023-10-09T05:02:09.649041+0000","last_became_peered":"2023-10-09T05:02:09.649041+0000","last_unstale":"2023-10-09T05:37:11.923003+0000","last_undegraded":"2023-10-09T05:37:11.923003+0000","last_fullsized":"2023-10-09T05:37:11.923003+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:18.524071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.12","version":"330'4336","reported_seq":4378,"reported_epoch":330,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:37:15.375920+0000","last_change":"2023-10-09T05:37:10.014645+0000","last_active":"2023-10-09T05:37:15.375920+0000","last_peered":"2023-10-09T05:37:15.375920+0000","last_clean":"2023-10-09T05:37:08.311384+0000","last_became_active":"2023-10-09T05:37:10.009586+0000","last_became_peered":"2023-10-09T05:37:10.009586+0000","last_unstale":"2023-10-09T05:37:15.375920+0000","last_undegraded":"2023-10-09T05:37:09.955117+0000","last_fullsized":"2023-10-09T05:37:09.899530+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":4336,"log_dups_size":0,"ondisk_log_size":4336,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160661504,"num_objects":2452,"num_object_clones":0,"num_object_copies":7356,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4676,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2452,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2452,"num_write_kb":156896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[1,2],"avail_no_missing":["1","3","6"],"object_location_counts":[{"shards":"1,2","objects":114},{"shards":"1,3,6","objects":2338}],"blocked_by":[],"up_primary":2,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'5","reported_seq":126,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:12.044122+0000","last_change":"2023-10-09T05:32:45.849292+0000","last_active":"2023-10-09T05:37:12.044122+0000","last_peered":"2023-10-09T05:37:12.044122+0000","last_clean":"2023-10-09T05:37:12.044122+0000","last_became_active":"2023-10-09T05:32:45.829351+0000","last_became_peered":"2023-10-09T05:32:45.829351+0000","last_unstale":"2023-10-09T05:37:12.044122+0000","last_undegraded":"2023-10-09T05:37:12.044122+0000","last_fullsized":"2023-10-09T05:37:12.044122+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:07:15.666617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":305,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T05:37:11.927816+0000","last_change":"2023-10-09T05:30:03.379517+0000","last_active":"2023-10-09T05:37:11.927816+0000","last_peered":"2023-10-09T05:37:11.927816+0000","last_clean":"2023-10-09T05:37:11.927816+0000","last_became_active":"2023-10-09T05:30:03.379048+0000","last_became_peered":"2023-10-09T05:30:03.379048+0000","last_unstale":"2023-10-09T05:37:11.927816+0000","last_undegraded":"2023-10-09T05:37:11.927816+0000","last_fullsized":"2023-10-09T05:37:11.927816+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:04.333337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":26,"stat_sum":{"num_bytes":5130354688,"num_objects":78283,"num_object_clones":0,"num_object_copies":234849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9312,"num_objects_misplaced":7562,"num_objects_unfound":0,"num_objects_dirty":78283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":78283,"num_write_kb":5010112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":651,"num_bytes_recovered":42663936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15519449088,"data_stored":15519449088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":115869,"ondisk_log_size":115869,"up":78,"acting":75,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":23,"num_read_kb":22,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":64,"num_bytes_recovered":22228,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234660,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2808020,"kb_used_data":1307084,"kb_used_omap":0,"kb_used_meta":1500928,"kb_avail":90937132,"statfs":{"total":95995035648,"available":93119623168,"internally_reserved":0,"allocated":1338454016,"data_stored":1337421295,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1536950272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:36:13 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":2.8399999999999999,"15min":2.8300000000000001},"min":{"1min":0.36899999999999999,"5min":0.32300000000000001,"15min":0.248},"max":{"1min":1.165,"5min":87.216999999999999,"15min":122.248},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.92200000000000004,"5min":3.0510000000000002,"15min":2.931},"min":{"1min":0.59399999999999997,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.3440000000000001,"5min":87.180999999999997,"15min":122.22199999999999},"last":0.96699999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:36:53 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":1.399,"15min":1.5049999999999999},"min":{"1min":0.33200000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.5619999999999998,"5min":7.6950000000000003,"15min":23.530999999999999},"last":0.54100000000000004},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.5780000000000001,"15min":1.504},"min":{"1min":0.38200000000000001,"5min":0.318,"15min":0.29899999999999999},"max":{"1min":2.7930000000000001,"5min":12.081,"15min":23.443000000000001},"last":0.73299999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:36:13 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":3.246,"15min":2.653},"min":{"1min":0.66400000000000003,"5min":0.34499999999999997,"15min":0.29799999999999999},"max":{"1min":1.206,"5min":102.146,"15min":122.065},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.80300000000000005,"5min":2.9449999999999998,"15min":2.677},"min":{"1min":0.46700000000000003,"5min":0.35999999999999999,"15min":0.30399999999999999},"max":{"1min":1.004,"5min":102.188,"15min":122.026},"last":0.76100000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:36:13 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":1.542,"15min":2.27},"min":{"1min":0.64100000000000001,"5min":0.33500000000000002,"15min":0.33200000000000002},"max":{"1min":1.1259999999999999,"5min":7.4269999999999996,"15min":138.667},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.97199999999999998,"5min":1.653,"15min":2.4550000000000001},"min":{"1min":0.72999999999999998,"5min":0.41399999999999998,"15min":0.34300000000000003},"max":{"1min":1.1859999999999999,"5min":7.3529999999999998,"15min":138.54400000000001},"last":0.83499999999999996}]},{"osd":4,"last update":"Mon Oct 9 05:36:53 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":1.0629999999999999,"15min":0.95799999999999996},"min":{"1min":0.35699999999999998,"5min":0.33900000000000002,"15min":0.311},"max":{"1min":1.167,"5min":7.077,"15min":7.077},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.75,"5min":1.137,"15min":1.087},"min":{"1min":0.39800000000000002,"5min":0.36699999999999999,"15min":0.33300000000000002},"max":{"1min":1.2030000000000001,"5min":7.3869999999999996,"15min":7.3869999999999996},"last":0.79500000000000004}]},{"osd":5,"last update":"Mon Oct 9 05:36:53 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":1.2749999999999999,"15min":1.1100000000000001},"min":{"1min":0.34899999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.208,"5min":7.4829999999999997,"15min":7.4829999999999997},"last":0.84999999999999998},{"interface":"front","average":{"1min":0.78100000000000003,"5min":1.3640000000000001,"15min":1.113},"min":{"1min":0.374,"5min":0.374,"15min":0.373},"max":{"1min":1.4199999999999999,"5min":7.6219999999999999,"15min":7.6219999999999999},"last":0.751}]},{"osd":6,"last update":"Mon Oct 9 05:36:53 2023","interfaces":[{"interface":"back","average":{"1min":0.76800000000000002,"5min":1.371,"15min":1.226},"min":{"1min":0.51000000000000001,"5min":0.38700000000000001,"15min":0.32800000000000001},"max":{"1min":1.2769999999999999,"5min":6.8789999999999996,"15min":8.5609999999999999},"last":0.89200000000000002},{"interface":"front","average":{"1min":0.875,"5min":1.4039999999999999,"15min":1.198},"min":{"1min":0.51700000000000002,"5min":0.44800000000000001,"15min":0.377},"max":{"1min":1.345,"5min":7.46,"15min":7.46},"last":1.0189999999999999}]}]},{"osd":6,"up_from":46,"seq":197568496302,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2734724,"kb_used_data":1550716,"kb_used_omap":0,"kb_used_meta":1184000,"kb_avail":91010428,"statfs":{"total":95995035648,"available":93194678272,"internally_reserved":0,"allocated":1587933184,"data_stored":1586908666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1212416000},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:37:10 2023","interfaces":[{"interface":"back","average":{"1min":3.5699999999999998,"5min":3.4430000000000001,"15min":2.9969999999999999},"min":{"1min":0.32100000000000001,"5min":0.28199999999999997,"15min":0.26100000000000001},"max":{"1min":32.545000000000002,"5min":106.85599999999999,"15min":154.29300000000001},"last":3.2890000000000001},{"interface":"front","average":{"1min":3.6299999999999999,"5min":3.5270000000000001,"15min":3.109},"min":{"1min":0.46100000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":32.634999999999998,"5min":107.11,"15min":154.35300000000001},"last":3.3170000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:37:10 2023","interfaces":[{"interface":"back","average":{"1min":2.9849999999999999,"5min":1.768,"15min":2.109},"min":{"1min":0.38400000000000001,"5min":0.28299999999999997,"15min":0.27400000000000002},"max":{"1min":41.408000000000001,"5min":41.408000000000001,"15min":94.037000000000006},"last":2.3029999999999999},{"interface":"front","average":{"1min":2.8780000000000001,"5min":1.554,"15min":1.8109999999999999},"min":{"1min":0.38800000000000001,"5min":0.312,"15min":0.29999999999999999},"max":{"1min":40.835999999999999,"5min":40.835999999999999,"15min":87.950000000000003},"last":0.65100000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:37:10 2023","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":5.0960000000000001,"15min":4.1539999999999999},"min":{"1min":0.29599999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":5.5780000000000003,"5min":211.66300000000001,"15min":211.66300000000001},"last":0.80000000000000004},{"interface":"front","average":{"1min":1.4259999999999999,"5min":5.8949999999999996,"15min":4.3970000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.4900000000000002,"5min":211.423,"15min":211.423},"last":0.71499999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:37:11 2023","interfaces":[{"interface":"back","average":{"1min":11.858000000000001,"5min":3.5779999999999998,"15min":4.5830000000000002},"min":{"1min":0.441,"5min":0.309,"15min":0.309},"max":{"1min":209.69,"5min":209.69,"15min":209.69},"last":0.78100000000000003},{"interface":"front","average":{"1min":11.736000000000001,"5min":3.4140000000000001,"15min":4.4589999999999996},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":209.34800000000001,"5min":209.34800000000001,"15min":209.34800000000001},"last":0.75800000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:37:10 2023","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":0.98899999999999999,"15min":1.02},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.27600000000000002},"max":{"1min":2.7530000000000001,"5min":10.976000000000001,"15min":10.976000000000001},"last":0.73599999999999999},{"interface":"front","average":{"1min":0.79000000000000004,"5min":1.03,"15min":0.98199999999999998},"min":{"1min":0.39400000000000002,"5min":0.33000000000000002,"15min":0.32800000000000001},"max":{"1min":2.7029999999999998,"5min":5.0190000000000001,"15min":5.0190000000000001},"last":0.83099999999999996}]},{"osd":5,"last update":"Mon Oct 9 05:37:10 2023","interfaces":[{"interface":"back","average":{"1min":1.1879999999999999,"5min":1.2769999999999999,"15min":1.1339999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.7750000000000004,"5min":11.275,"15min":11.275},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.89800000000000002,"5min":1.1599999999999999,"15min":1.0820000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":2.7349999999999999,"5min":7.3090000000000002,"15min":7.3090000000000002},"last":0.69499999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:36:28 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":1.7370000000000001,"15min":1.4650000000000001},"min":{"1min":0.52100000000000002,"5min":0.36599999999999999,"15min":0.28100000000000003},"max":{"1min":1.72,"5min":18.963000000000001,"15min":18.963000000000001},"last":0.92100000000000004},{"interface":"front","average":{"1min":0.97899999999999998,"5min":1.8280000000000001,"15min":1.556},"min":{"1min":0.55900000000000005,"5min":0.39500000000000002,"15min":0.35399999999999998},"max":{"1min":1.8999999999999999,"5min":19.010000000000002,"15min":19.010000000000002},"last":1.0069999999999999}]}]},{"osd":1,"up_from":96,"seq":412316860893,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4112372,"kb_used_data":2966892,"kb_used_omap":4,"kb_used_meta":1145467,"kb_avail":89632780,"statfs":{"total":95995035648,"available":91783966720,"internally_reserved":0,"allocated":3038097408,"data_stored":3036222910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1172959118},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:37:12 2023","interfaces":[{"interface":"back","average":{"1min":7.0800000000000001,"5min":3.0920000000000001,"15min":3.4710000000000001},"min":{"1min":0.28199999999999997,"5min":0.221,"15min":0.19500000000000001},"max":{"1min":116.20999999999999,"5min":116.20999999999999,"15min":150.75200000000001},"last":0.39000000000000001},{"interface":"front","average":{"1min":7.5270000000000001,"5min":3.1019999999999999,"15min":3.7959999999999998},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.185},"max":{"1min":117.182,"5min":117.182,"15min":150.59299999999999},"last":4.9850000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:37:11 2023","interfaces":[{"interface":"back","average":{"1min":2.5270000000000001,"5min":2.625,"15min":3.6259999999999999},"min":{"1min":0.34799999999999998,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":11.032,"5min":27.875,"15min":154.14599999999999},"last":5.5389999999999997},{"interface":"front","average":{"1min":2.387,"5min":2.6040000000000001,"15min":3.6349999999999998},"min":{"1min":0.40300000000000002,"5min":0.28100000000000003,"15min":0.254},"max":{"1min":7.2110000000000003,"5min":27.641999999999999,"15min":154.215},"last":1.958}]},{"osd":3,"last update":"Mon Oct 9 05:37:11 2023","interfaces":[{"interface":"back","average":{"1min":1.9670000000000001,"5min":2.097,"15min":3.2730000000000001},"min":{"1min":0.29199999999999998,"5min":0.28199999999999997,"15min":0.17499999999999999},"max":{"1min":7.5529999999999999,"5min":11.377000000000001,"15min":152.58199999999999},"last":5.032},{"interface":"front","average":{"1min":2.2879999999999998,"5min":1.8149999999999999,"15min":3.4060000000000001},"min":{"1min":0.28399999999999997,"5min":0.255,"15min":0.221},"max":{"1min":6.9989999999999997,"5min":10.206,"15min":154.178},"last":0.64800000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:37:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0329999999999999,"5min":1.542,"15min":1.9530000000000001},"min":{"1min":0.54100000000000004,"5min":0.38700000000000001,"15min":0.30099999999999999},"max":{"1min":6.306,"5min":6.306,"15min":83.453999999999994},"last":0.68300000000000005},{"interface":"front","average":{"1min":1.2170000000000001,"5min":1.377,"15min":1.6439999999999999},"min":{"1min":0.442,"5min":0.37,"15min":0.32400000000000001},"max":{"1min":3.48,"5min":6.0650000000000004,"15min":83.375},"last":0.42599999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:37:11 2023","interfaces":[{"interface":"back","average":{"1min":1.5089999999999999,"5min":1.742,"15min":1.488},"min":{"1min":0.40999999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":6.8700000000000001,"5min":9.2249999999999996,"15min":9.7520000000000007},"last":5.0869999999999997},{"interface":"front","average":{"1min":2.0510000000000002,"5min":1.786,"15min":2.0179999999999998},"min":{"1min":0.49399999999999999,"5min":0.433,"15min":0.39900000000000002},"max":{"1min":6.3490000000000002,"5min":9.8260000000000005,"15min":83.268000000000001},"last":0.94299999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:37:11 2023","interfaces":[{"interface":"back","average":{"1min":1.5640000000000001,"5min":1.708,"15min":1.6220000000000001},"min":{"1min":0.374,"5min":0.374,"15min":0.34000000000000002},"max":{"1min":6.9139999999999997,"5min":11.315,"15min":11.315},"last":5.0620000000000003},{"interface":"front","average":{"1min":2.1899999999999999,"5min":1.8620000000000001,"15min":2.0569999999999999},"min":{"1min":0.47299999999999998,"5min":0.41499999999999998,"15min":0.376},"max":{"1min":6.415,"5min":10.446,"15min":83.400000000000006},"last":1.006}]},{"osd":7,"last update":"Mon Oct 9 05:37:11 2023","interfaces":[{"interface":"back","average":{"1min":2.2250000000000001,"5min":1.8440000000000001,"15min":2.0750000000000002},"min":{"1min":0.56299999999999994,"5min":0.40899999999999997,"15min":0.376},"max":{"1min":6.3810000000000002,"5min":8.6059999999999999,"15min":83.510000000000005},"last":0.90300000000000002},{"interface":"front","average":{"1min":1.282,"5min":1.5780000000000001,"15min":1.744},"min":{"1min":0.48899999999999999,"5min":0.40400000000000003,"15min":0.377},"max":{"1min":3.4550000000000001,"5min":7.101,"15min":83.480999999999995},"last":0.50900000000000001}]}]},{"osd":0,"up_from":88,"seq":377957122534,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3972644,"kb_used_data":2805788,"kb_used_omap":5,"kb_used_meta":1166842,"kb_avail":89772508,"statfs":{"total":95995035648,"available":91927048192,"internally_reserved":0,"allocated":2873126912,"data_stored":2871325128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1194846842},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,1,3,12],"upper_bound":2048},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:37:04 2023","interfaces":[{"interface":"back","average":{"1min":2.0619999999999998,"5min":2.8700000000000001,"15min":2.0979999999999999},"min":{"1min":0.26800000000000002,"5min":0.20699999999999999,"15min":0.17299999999999999},"max":{"1min":8.9469999999999992,"5min":111.001,"15min":111.001},"last":0.627},{"interface":"front","average":{"1min":1.774,"5min":2.9119999999999999,"15min":2.0550000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.217},"max":{"1min":6.5369999999999999,"5min":110.919,"15min":110.919},"last":0.64400000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:37:09 2023","interfaces":[{"interface":"back","average":{"1min":1.8220000000000001,"5min":2.8119999999999998,"15min":2.8420000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.247},"max":{"1min":6.5810000000000004,"5min":74.817999999999998,"15min":92.745000000000005},"last":0.92800000000000005},{"interface":"front","average":{"1min":2.1309999999999998,"5min":3.2000000000000002,"15min":3.0550000000000002},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.19700000000000001},"max":{"1min":6.9619999999999997,"5min":74.751000000000005,"15min":97.816000000000003},"last":0.72599999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:37:09 2023","interfaces":[{"interface":"back","average":{"1min":1.8540000000000001,"5min":5.6600000000000001,"15min":3.8500000000000001},"min":{"1min":0.42799999999999999,"5min":0.25600000000000001,"15min":0.21099999999999999},"max":{"1min":7.1310000000000002,"5min":290.73000000000002,"15min":290.73000000000002},"last":0.67200000000000004},{"interface":"front","average":{"1min":1.7410000000000001,"5min":5.452,"15min":3.8370000000000002},"min":{"1min":0.42699999999999999,"5min":0.246,"15min":0.246},"max":{"1min":5.7130000000000001,"5min":289.85199999999998,"15min":289.85199999999998},"last":1.4650000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:37:09 2023","interfaces":[{"interface":"back","average":{"1min":1.4299999999999999,"5min":1.3680000000000001,"15min":1.4330000000000001},"min":{"1min":0.39000000000000001,"5min":0.38300000000000001,"15min":0.33500000000000002},"max":{"1min":5.6790000000000003,"5min":7.4880000000000004,"15min":9.7460000000000004},"last":1.1319999999999999},{"interface":"front","average":{"1min":1.929,"5min":1.6240000000000001,"15min":1.641},"min":{"1min":0.45200000000000001,"5min":0.38100000000000001,"15min":0.32400000000000001},"max":{"1min":5.9409999999999998,"5min":9.7989999999999995,"15min":9.8320000000000007},"last":1.008}]},{"osd":5,"last update":"Mon Oct 9 05:37:09 2023","interfaces":[{"interface":"back","average":{"1min":1.484,"5min":1.845,"15min":1.5589999999999999},"min":{"1min":0.34899999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":7.0419999999999998,"5min":14.510999999999999,"15min":14.510999999999999},"last":1.0389999999999999},{"interface":"front","average":{"1min":1.5309999999999999,"5min":1.516,"15min":1.552},"min":{"1min":0.434,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":5.6100000000000003,"5min":7.4100000000000001,"15min":9.8010000000000002},"last":1.0700000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:37:09 2023","interfaces":[{"interface":"back","average":{"1min":2.1200000000000001,"5min":1.9139999999999999,"15min":1.867},"min":{"1min":0.41599999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":5.9710000000000001,"5min":15.544,"15min":15.544},"last":1.0569999999999999},{"interface":"front","average":{"1min":1.615,"5min":1.964,"15min":1.7130000000000001},"min":{"1min":0.39300000000000002,"5min":0.318,"15min":0.318},"max":{"1min":7.1070000000000002,"5min":14.459,"15min":14.459},"last":1.1499999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:37:09 2023","interfaces":[{"interface":"back","average":{"1min":1.9159999999999999,"5min":2.2650000000000001,"15min":1.861},"min":{"1min":0.42299999999999999,"5min":0.42099999999999999,"15min":0.308},"max":{"1min":7.0819999999999999,"5min":14.492000000000001,"15min":14.492000000000001},"last":1.099},{"interface":"front","average":{"1min":1.7370000000000001,"5min":1.6739999999999999,"15min":1.7130000000000001},"min":{"1min":0.46999999999999997,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":6.9400000000000004,"5min":7.4589999999999996,"15min":10.795},"last":1.1779999999999999}]}]},{"osd":2,"up_from":106,"seq":455266533812,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3448476,"kb_used_data":2464468,"kb_used_omap":7,"kb_used_meta":983992,"kb_avail":90296676,"statfs":{"total":95995035648,"available":92463796224,"internally_reserved":0,"allocated":2523615232,"data_stored":2521866909,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1007607889},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,2,2,1,1,0,2],"upper_bound":1024},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:36:54 2023","interfaces":[{"interface":"back","average":{"1min":4.6879999999999997,"5min":2.6230000000000002,"15min":2.7650000000000001},"min":{"1min":0.32500000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":46.154000000000003,"5min":46.154000000000003,"15min":91.646000000000001},"last":1.2},{"interface":"front","average":{"1min":4.2679999999999998,"5min":2.738,"15min":2.597},"min":{"1min":0.219,"5min":0.219,"15min":0.21199999999999999},"max":{"1min":45.789999999999999,"5min":45.789999999999999,"15min":92.522000000000006},"last":1.2330000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:36:54 2023","interfaces":[{"interface":"back","average":{"1min":1.5289999999999999,"5min":2.04,"15min":1.996},"min":{"1min":0.245,"5min":0.22900000000000001,"15min":0.22700000000000001},"max":{"1min":5.9699999999999998,"5min":13.143000000000001,"15min":23.251000000000001},"last":0.85699999999999998},{"interface":"front","average":{"1min":1.538,"5min":1.9430000000000001,"15min":2.1779999999999999},"min":{"1min":0.439,"5min":0.24299999999999999,"15min":0.22600000000000001},"max":{"1min":4.0250000000000004,"5min":10.063000000000001,"15min":17.632999999999999},"last":2.3159999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:36:12 2023","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":2.6030000000000002,"15min":2.5680000000000001},"min":{"1min":0.35199999999999998,"5min":0.214,"15min":0.214},"max":{"1min":2.3140000000000001,"5min":15.576000000000001,"15min":15.576000000000001},"last":0.499},{"interface":"front","average":{"1min":0.79700000000000004,"5min":2.133,"15min":2.4470000000000001},"min":{"1min":0.40300000000000002,"5min":0.33600000000000002,"15min":0.27000000000000002},"max":{"1min":1.1739999999999999,"5min":11.451000000000001,"15min":18.515000000000001},"last":2.2890000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:36:54 2023","interfaces":[{"interface":"back","average":{"1min":1.5589999999999999,"5min":1.839,"15min":1.7569999999999999},"min":{"1min":0.38500000000000001,"5min":0.34499999999999997,"15min":0.29599999999999999},"max":{"1min":5.8639999999999999,"5min":10.379,"15min":10.597},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.97499999999999998,"5min":1.708,"15min":1.774},"min":{"1min":0.42199999999999999,"5min":0.35399999999999998,"15min":0.34200000000000003},"max":{"1min":3.2789999999999999,"5min":11.173,"15min":13.097},"last":2.2589999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:36:54 2023","interfaces":[{"interface":"back","average":{"1min":1.3240000000000001,"5min":1.673,"15min":1.718},"min":{"1min":0.42699999999999999,"5min":0.375,"15min":0.33000000000000002},"max":{"1min":6.2439999999999998,"5min":9.8729999999999993,"15min":9.8729999999999993},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.589,"5min":1.871,"15min":1.712},"min":{"1min":0.55200000000000005,"5min":0.36299999999999999,"15min":0.34699999999999998},"max":{"1min":6.0170000000000003,"5min":8.4239999999999995,"15min":10.686},"last":0.39200000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:36:54 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":1.778,"15min":1.8520000000000001},"min":{"1min":0.45600000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":3.613,"5min":11.381,"15min":13.209},"last":2.2189999999999999},{"interface":"front","average":{"1min":1.194,"5min":1.7450000000000001,"15min":1.8260000000000001},"min":{"1min":0.495,"5min":0.39200000000000002,"15min":0.35199999999999998},"max":{"1min":3.8109999999999999,"5min":9.9120000000000008,"15min":11.118},"last":1.4990000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:36:54 2023","interfaces":[{"interface":"back","average":{"1min":1.357,"5min":1.9379999999999999,"15min":2.0179999999999998},"min":{"1min":0.53600000000000003,"5min":0.47899999999999998,"15min":0.38300000000000001},"max":{"1min":6.3070000000000004,"5min":9.8930000000000007,"15min":13.571999999999999},"last":0.55400000000000005},{"interface":"front","average":{"1min":1.6339999999999999,"5min":2.1200000000000001,"15min":2.0390000000000001},"min":{"1min":0.57099999999999995,"5min":0.434,"15min":0.41499999999999998},"max":{"1min":6.0519999999999996,"5min":12.016,"15min":12.016},"last":0.47499999999999998}]}]},{"osd":3,"up_from":123,"seq":528280977831,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3548368,"kb_used_data":2485448,"kb_used_omap":2,"kb_used_meta":1062909,"kb_avail":90196784,"statfs":{"total":95995035648,"available":92361506816,"internally_reserved":0,"allocated":2545098752,"data_stored":2543428298,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":1088418869},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:36:19 2023","interfaces":[{"interface":"back","average":{"1min":1.349,"5min":3.7799999999999998,"15min":2.641},"min":{"1min":0.41299999999999998,"5min":0.224,"15min":0.19500000000000001},"max":{"1min":7.04,"5min":90.947000000000003,"15min":90.947000000000003},"last":1.6479999999999999},{"interface":"front","average":{"1min":0.70499999999999996,"5min":3.5409999999999999,"15min":2.6190000000000002},"min":{"1min":0.246,"5min":0.184,"15min":0.184},"max":{"1min":1.853,"5min":90.662999999999997,"15min":90.662999999999997},"last":3.919}]},{"osd":1,"last update":"Mon Oct 9 05:36:19 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":3.5779999999999998,"15min":2.5720000000000001},"min":{"1min":0.28799999999999998,"5min":0.23000000000000001,"15min":0.22},"max":{"1min":0.80400000000000005,"5min":118.20999999999999,"15min":118.20999999999999},"last":7.4550000000000001},{"interface":"front","average":{"1min":0.89700000000000002,"5min":3.597,"15min":2.5489999999999999},"min":{"1min":0.48099999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.3050000000000002,"5min":124.083,"15min":124.083},"last":0.72399999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:36:19 2023","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":6.8799999999999999,"15min":3.7999999999999998},"min":{"1min":0.48199999999999998,"5min":0.248,"15min":0.24299999999999999},"max":{"1min":3.1840000000000002,"5min":350.53899999999999,"15min":350.53899999999999},"last":2.714},{"interface":"front","average":{"1min":0.89000000000000001,"5min":6.8940000000000001,"15min":3.7650000000000001},"min":{"1min":0.41699999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":2.9790000000000001,"5min":350.64699999999999,"15min":350.64699999999999},"last":1.0289999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:36:19 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":1.6339999999999999,"15min":1.6910000000000001},"min":{"1min":0.42499999999999999,"5min":0.36499999999999999,"15min":0.29999999999999999},"max":{"1min":1.377,"5min":9.1539999999999999,"15min":9.1539999999999999},"last":1},{"interface":"front","average":{"1min":1.1599999999999999,"5min":1.7030000000000001,"15min":1.663},"min":{"1min":0.67800000000000005,"5min":0.36599999999999999,"15min":0.33300000000000002},"max":{"1min":2.266,"5min":12.132999999999999,"15min":13.234999999999999},"last":0.63800000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:36:19 2023","interfaces":[{"interface":"back","average":{"1min":1.2270000000000001,"5min":1.7569999999999999,"15min":1.6990000000000001},"min":{"1min":0.70699999999999996,"5min":0.40500000000000003,"15min":0.35999999999999999},"max":{"1min":2.331,"5min":12.092000000000001,"15min":13.287000000000001},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.97699999999999998,"5min":1.6040000000000001,"15min":1.5569999999999999},"min":{"1min":0.61099999999999999,"5min":0.36299999999999999,"15min":0.34899999999999998},"max":{"1min":1.327,"5min":10.135,"15min":10.135},"last":2.7709999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:36:19 2023","interfaces":[{"interface":"back","average":{"1min":2.0609999999999999,"5min":2.1560000000000001,"15min":1.7909999999999999},"min":{"1min":0.52000000000000002,"5min":0.38700000000000001,"15min":0.35699999999999998},"max":{"1min":20.463999999999999,"5min":20.463999999999999,"15min":20.463999999999999},"last":2.8130000000000002},{"interface":"front","average":{"1min":2.0739999999999998,"5min":2,"15min":1.9319999999999999},"min":{"1min":0.48999999999999999,"5min":0.38400000000000001,"15min":0.36699999999999999},"max":{"1min":20.279,"5min":20.279,"15min":20.279},"last":0.96999999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:36:19 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":2.0649999999999999,"15min":1.913},"min":{"1min":0.46100000000000002,"5min":0.41099999999999998,"15min":0.38800000000000001},"max":{"1min":1.3200000000000001,"5min":10.885,"15min":10.885},"last":0.92800000000000005},{"interface":"front","average":{"1min":1.224,"5min":2.145,"15min":1.9910000000000001},"min":{"1min":0.68600000000000005,"5min":0.45500000000000002,"15min":0.39200000000000002},"max":{"1min":2.2269999999999999,"5min":12.113,"15min":13.262},"last":0.57799999999999996}]}]},{"osd":4,"up_from":0,"seq":128849019588,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757944,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2438340,"kb_used_data":1531388,"kb_used_omap":0,"kb_used_meta":906944,"kb_avail":91306812,"statfs":{"total":95995035648,"available":93498175488,"internally_reserved":0,"allocated":1568141312,"data_stored":1567120693,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928710656},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:37:11 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":2.8199999999999998,"15min":2.2759999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":3.9910000000000001,"5min":94.989999999999995,"15min":109.193},"last":0.38400000000000001},{"interface":"front","average":{"1min":1.1870000000000001,"5min":2.9009999999999998,"15min":2.3889999999999998},"min":{"1min":0.378,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":4.0259999999999998,"5min":95.034999999999997,"15min":109.298},"last":0.498}]},{"osd":1,"last update":"Mon Oct 9 05:37:11 2023","interfaces":[{"interface":"back","average":{"1min":1.591,"5min":1.476,"15min":1.7370000000000001},"min":{"1min":0.42199999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.1929999999999996,"5min":9.2430000000000003,"15min":78.117000000000004},"last":0.54600000000000004},{"interface":"front","average":{"1min":1.2869999999999999,"5min":1.3200000000000001,"15min":1.714},"min":{"1min":0.34599999999999997,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":3.7349999999999999,"5min":8.6799999999999997,"15min":78.879999999999995},"last":0.56499999999999995}]},{"osd":2,"last update":"Mon Oct 9 05:36:20 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":4.5270000000000001,"15min":2.6800000000000002},"min":{"1min":0.41199999999999998,"5min":0.32300000000000001,"15min":0.27800000000000002},"max":{"1min":1.2769999999999999,"5min":250.941,"15min":250.941},"last":2.1850000000000001},{"interface":"front","average":{"1min":1.016,"5min":4.625,"15min":2.6709999999999998},"min":{"1min":0.57899999999999996,"5min":0.34699999999999998,"15min":0.32900000000000001},"max":{"1min":4.1779999999999999,"5min":251.48599999999999,"15min":251.48599999999999},"last":0.41699999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:36:20 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":1.5109999999999999,"15min":2.0720000000000001},"min":{"1min":0.36699999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.226,"5min":9.0169999999999995,"15min":76.512},"last":0.66500000000000004},{"interface":"front","average":{"1min":1.2370000000000001,"5min":1.863,"15min":2.1389999999999998},"min":{"1min":0.49099999999999999,"5min":0.374,"15min":0.315},"max":{"1min":8.1140000000000008,"5min":10.295,"15min":76.578000000000003},"last":0.64800000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:37:11 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":0.997,"15min":0.87},"min":{"1min":0.371,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":3.5800000000000001,"5min":7.6109999999999998,"15min":7.6109999999999998},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.72799999999999998,"5min":1.173,"15min":1.0129999999999999},"min":{"1min":0.36599999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":2.1240000000000001,"5min":8.3480000000000008,"15min":8.3480000000000008},"last":0.58199999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:37:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":1.3959999999999999,"15min":1.8859999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.9240000000000004,"5min":8.5220000000000002,"15min":190.98500000000001},"last":0.52400000000000002},{"interface":"front","average":{"1min":1.018,"5min":1.3240000000000001,"15min":1.873},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":3.9009999999999998,"5min":9.282,"15min":190.928},"last":0.623}]},{"osd":7,"last update":"Mon Oct 9 05:36:44 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":1.286,"15min":1.095},"min":{"1min":0.442,"5min":0.372,"15min":0.372},"max":{"1min":1.216,"5min":9.3970000000000002,"15min":9.3970000000000002},"last":0.67800000000000005},{"interface":"front","average":{"1min":1,"5min":1.546,"15min":1.202},"min":{"1min":0.45200000000000001,"5min":0.41799999999999998,"15min":0.35899999999999999},"max":{"1min":2.899,"5min":11.593,"15min":11.593},"last":0.747}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2905145344,"data_stored":2905145344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2984181760,"data_stored":2984181760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2537226240,"data_stored":2537226240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2565079040,"data_stored":2565079040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1576009728,"data_stored":1576009728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1599799296,"data_stored":1599799296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1352007680,"data_stored":1352007680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:37:21.607 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_9 pgp_num 26 2023-10-09T05:37:21.647 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:21.660 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:21 smithi133 bash[105589]: cluster 2023-10-09T05:37:20.417757+0000 mgr.y (mgr.25277) 1484 : cluster 0 pgmap v1604: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 95 active+clean; 5.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 9312/247218 objects degraded (3.767%); 7232/247218 objects misplaced (2.925%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T05:37:21.661 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:21 smithi133 bash[102858]: cluster 2023-10-09T05:37:20.417757+0000 mgr.y (mgr.25277) 1484 : cluster 0 pgmap v1604: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 95 active+clean; 5.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 9312/247218 objects degraded (3.767%); 7232/247218 objects misplaced (2.925%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T05:37:21.678 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:21.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:21.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:21.778 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 73 16 84659 84643 72.4497 54.1875 0.00690532 0.0137976 2023-10-09T05:37:21.781 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:21.811 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:21.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:21.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:21.918 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:21.958 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:21.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:21.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:22.064 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:22.092 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:22.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:22.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:22.196 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:22.225 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:22.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:22.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:22.329 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:22.373 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:22.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:22.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:22.481 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:22.505 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:22.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:22.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:22.608 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:22.639 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:22.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:22.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:22.742 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:22.767 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:22.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:22.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:22.778 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 74 16 85595 85579 72.2611 58.5 0.00708529 0.0138322 2023-10-09T05:37:22.870 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:22.890 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:22.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:22.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:22.993 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:23.018 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:23.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:23.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:23.121 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:23.148 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:23.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:23.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:23.251 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:23.292 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:23.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:23.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:23.398 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:23.423 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:23.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:23.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:23.499 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:23 smithi133 bash[102858]: cluster 2023-10-09T05:37:22.418368+0000 mgr.y (mgr.25277) 1485 : cluster 0 pgmap v1605: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.1 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 60 MiB/s wr, 963 op/s; 9312/249693 objects degraded (3.729%); 7232/249693 objects misplaced (2.896%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T05:37:23.527 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:23.570 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:23.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:23.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:23.673 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:23.699 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:23.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:23.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:23.778 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 75 16 86302 86286 71.8868 44.1875 0.0109527 0.0139054 2023-10-09T05:37:23.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:23 smithi133 bash[105589]: cluster 2023-10-09T05:37:22.418368+0000 mgr.y (mgr.25277) 1485 : cluster 0 pgmap v1605: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.1 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 60 MiB/s wr, 963 op/s; 9312/249693 objects degraded (3.729%); 7232/249693 objects misplaced (2.896%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T05:37:23.807 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:23.815 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:23 smithi194 bash[92666]: cluster 2023-10-09T05:37:22.418368+0000 mgr.y (mgr.25277) 1485 : cluster 0 pgmap v1605: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.1 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 60 MiB/s wr, 963 op/s; 9312/249693 objects degraded (3.729%); 7232/249693 objects misplaced (2.896%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T05:37:23.831 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:23.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:23.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:23.937 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:23.968 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:23.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:23.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:24.071 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:24.107 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:24.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:24.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:24.212 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:24.244 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:24.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:24.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:24.350 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:24.374 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:24.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:24.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:24.480 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:24.501 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:37:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:37:24] "GET /metrics HTTP/1.1" 200 36496 "" "Prometheus/2.43.0" 2023-10-09T05:37:24.503 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:24.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:24.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:24.606 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:24.630 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:24.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:24.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:24.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:24 smithi194 bash[92666]: cluster 2023-10-09T05:37:23.886562+0000 mon.a (mon.0) 1873 : cluster 3 Health check update: Degraded data redundancy: 9312/249693 objects degraded (3.729%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:24.734 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:24.760 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:24.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:24.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:24.778 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 76 16 87234 87218 71.7073 58.25 0.0129637 0.0139404 2023-10-09T05:37:24.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:24 smithi133 bash[105589]: cluster 2023-10-09T05:37:23.886562+0000 mon.a (mon.0) 1873 : cluster 3 Health check update: Degraded data redundancy: 9312/249693 objects degraded (3.729%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:24.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:24 smithi133 bash[102858]: cluster 2023-10-09T05:37:23.886562+0000 mon.a (mon.0) 1873 : cluster 3 Health check update: Degraded data redundancy: 9312/249693 objects degraded (3.729%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:24.863 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:24.887 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:24.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:24.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:24.991 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:25.014 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:25.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:25.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:25.117 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:25.139 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:25.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:25.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:25.243 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:25.270 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:25.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:25.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:25.376 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:25.403 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:25.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:25.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:25.504 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:25 smithi133 bash[105589]: cluster 2023-10-09T05:37:24.418995+0000 mgr.y (mgr.25277) 1486 : cluster 0 pgmap v1606: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 60 MiB/s wr, 961 op/s; 9312/253932 objects degraded (3.667%); 7232/253932 objects misplaced (2.848%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:37:25.504 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:25 smithi133 bash[102858]: cluster 2023-10-09T05:37:24.418995+0000 mgr.y (mgr.25277) 1486 : cluster 0 pgmap v1606: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 60 MiB/s wr, 961 op/s; 9312/253932 objects degraded (3.667%); 7232/253932 objects misplaced (2.848%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:37:25.506 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:25.536 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:25.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:25.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:25.641 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:25.650 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:25 smithi194 bash[92666]: cluster 2023-10-09T05:37:24.418995+0000 mgr.y (mgr.25277) 1486 : cluster 0 pgmap v1606: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 60 MiB/s wr, 961 op/s; 9312/253932 objects degraded (3.667%); 7232/253932 objects misplaced (2.848%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:37:25.669 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:25.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:25.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:25.772 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:25.778 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 77 16 88173 88157 71.5382 58.6875 0.0215365 0.0139721 2023-10-09T05:37:25.805 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:25.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:25.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:25.909 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:25.934 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:25.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:25.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:26.040 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:26.066 INFO:teuthology.orchestra.run.smithi194.stdout:{ 2023-10-09T05:37:26.066 INFO:teuthology.orchestra.run.smithi194.stdout: "ops": [], 2023-10-09T05:37:26.066 INFO:teuthology.orchestra.run.smithi194.stdout: "complaint_time": 30, 2023-10-09T05:37:26.066 INFO:teuthology.orchestra.run.smithi194.stdout: "num_blocked_ops": 0 2023-10-09T05:37:26.066 INFO:teuthology.orchestra.run.smithi194.stdout:} 2023-10-09T05:37:26.074 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:26.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:26.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:26.186 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:26.210 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:26.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:26.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:26.317 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:26.346 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:26.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:26.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:26.454 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:26.464 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:37:26.495 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:26.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:26.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:26.572 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:37:26.599 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:26.622 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:37:26.623 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:26.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:26.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:26.666 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:37:26.704 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:37:26.727 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:26.738 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:37:26.765 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:26.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:26.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:26.775 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:37:26.778 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 78 16 88965 88949 71.2556 49.5 0.00679585 0.0140283 2023-10-09T05:37:26.816 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:37:26.855 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:37:26.859 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:37:26.859 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:37:26.859 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:37:26.859 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:37:26.859 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:37:26.860 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:37:26.860 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:37:26.860 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:37:26.869 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:26.897 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:26.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:26.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:27.000 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:27.025 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:27.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:27.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:27.127 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:27.163 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:27.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:27.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:27.212 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T05:37:27.268 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:27.299 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:27.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:27.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:27.404 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:27.440 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:27.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:27.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:27.547 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is unset 2023-10-09T05:37:27.548 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:27.578 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:27.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:27.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:27.684 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:27.710 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:27.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:27.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:27.779 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 79 16 89807 89791 71.0196 52.625 0.00529338 0.0140617 2023-10-09T05:37:27.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:27 smithi133 bash[102858]: cluster 2023-10-09T05:37:26.419810+0000 mgr.y (mgr.25277) 1487 : cluster 0 pgmap v1607: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s; 9312/261045 objects degraded (3.567%); 6906/261045 objects misplaced (2.646%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T05:37:27.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:27 smithi133 bash[102858]: audit 2023-10-09T05:37:26.508931+0000 mon.b (mon.1) 33 : audit 0 from='client.? 172.21.15.133:0/983806103' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:37:27.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:27 smithi133 bash[102858]: audit 2023-10-09T05:37:26.861804+0000 mon.c (mon.2) 137 : audit 1 from='client.? 172.21.15.133:0/1806717290' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:37:27.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:27 smithi133 bash[102858]: audit 2023-10-09T05:37:26.862872+0000 mon.a (mon.0) 1874 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:37:27.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:27 smithi133 bash[105589]: cluster 2023-10-09T05:37:26.419810+0000 mgr.y (mgr.25277) 1487 : cluster 0 pgmap v1607: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s; 9312/261045 objects degraded (3.567%); 6906/261045 objects misplaced (2.646%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T05:37:27.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:27 smithi133 bash[105589]: audit 2023-10-09T05:37:26.508931+0000 mon.b (mon.1) 33 : audit 0 from='client.? 172.21.15.133:0/983806103' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:37:27.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:27 smithi133 bash[105589]: audit 2023-10-09T05:37:26.861804+0000 mon.c (mon.2) 137 : audit 1 from='client.? 172.21.15.133:0/1806717290' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:37:27.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:27 smithi133 bash[105589]: audit 2023-10-09T05:37:26.862872+0000 mon.a (mon.0) 1874 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:37:27.813 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:27.825 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:27 smithi194 bash[92666]: cluster 2023-10-09T05:37:26.419810+0000 mgr.y (mgr.25277) 1487 : cluster 0 pgmap v1607: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s; 9312/261045 objects degraded (3.567%); 6906/261045 objects misplaced (2.646%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T05:37:27.825 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:27 smithi194 bash[92666]: audit 2023-10-09T05:37:26.508931+0000 mon.b (mon.1) 33 : audit 0 from='client.? 172.21.15.133:0/983806103' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:37:27.826 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:27 smithi194 bash[92666]: audit 2023-10-09T05:37:26.861804+0000 mon.c (mon.2) 137 : audit 1 from='client.? 172.21.15.133:0/1806717290' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:37:27.826 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:27 smithi194 bash[92666]: audit 2023-10-09T05:37:26.862872+0000 mon.a (mon.0) 1874 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:37:27.844 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:27.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:27.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:27.948 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:27.975 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:27.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:27.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:28.078 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:28.108 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:28.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:28.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:28.214 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:28.240 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:28.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:28.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:28.349 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:28.377 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:28.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:28.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:28.483 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:28.516 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:28.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:28.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:28.622 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:28.649 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:28.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:28.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:28.753 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:28.762 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:28 smithi133 bash[105589]: cluster 2023-10-09T05:37:27.505987+0000 mon.a (mon.0) 1875 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:37:28.762 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:28 smithi133 bash[105589]: cluster 2023-10-09T05:37:27.524918+0000 osd.5 (osd.5) 315 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T05:37:28.762 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:28 smithi133 bash[105589]: cluster 2023-10-09T05:37:27.529717+0000 osd.6 (osd.6) 299 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T05:37:28.762 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:28 smithi133 bash[105589]: cluster 2023-10-09T05:37:27.532642+0000 osd.5 (osd.5) 316 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T05:37:28.762 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:28 smithi133 bash[105589]: audit 2023-10-09T05:37:27.535762+0000 mon.a (mon.0) 1876 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:37:28.762 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:28 smithi133 bash[105589]: cluster 2023-10-09T05:37:27.536001+0000 mon.a (mon.0) 1877 : cluster 0 osdmap e331: 8 total, 8 up, 7 in 2023-10-09T05:37:28.763 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:28 smithi133 bash[105589]: audit 2023-10-09T05:37:28.490528+0000 mon.a (mon.0) 1878 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:37:28.763 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:28 smithi133 bash[102858]: cluster 2023-10-09T05:37:27.505987+0000 mon.a (mon.0) 1875 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:37:28.763 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:28 smithi133 bash[102858]: cluster 2023-10-09T05:37:27.524918+0000 osd.5 (osd.5) 315 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T05:37:28.763 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:28 smithi133 bash[102858]: cluster 2023-10-09T05:37:27.529717+0000 osd.6 (osd.6) 299 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T05:37:28.763 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:28 smithi133 bash[102858]: cluster 2023-10-09T05:37:27.532642+0000 osd.5 (osd.5) 316 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T05:37:28.763 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:28 smithi133 bash[102858]: audit 2023-10-09T05:37:27.535762+0000 mon.a (mon.0) 1876 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:37:28.764 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:28 smithi133 bash[102858]: cluster 2023-10-09T05:37:27.536001+0000 mon.a (mon.0) 1877 : cluster 0 osdmap e331: 8 total, 8 up, 7 in 2023-10-09T05:37:28.764 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:28 smithi133 bash[102858]: audit 2023-10-09T05:37:28.490528+0000 mon.a (mon.0) 1878 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:37:28.779 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:2023-10-09T05:37:28.779285+0000 min lat: 0.00135224 max lat: 2.40321 avg lat: 0.0141247 2023-10-09T05:37:28.779 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:37:28.779 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 80 16 90621 90605 70.7678 50.875 0.0184603 0.0141247 2023-10-09T05:37:28.779 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:28.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:28.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:28.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:28 smithi194 bash[92666]: cluster 2023-10-09T05:37:27.505987+0000 mon.a (mon.0) 1875 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:37:28.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:28 smithi194 bash[92666]: cluster 2023-10-09T05:37:27.524918+0000 osd.5 (osd.5) 315 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T05:37:28.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:28 smithi194 bash[92666]: cluster 2023-10-09T05:37:27.529717+0000 osd.6 (osd.6) 299 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T05:37:28.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:28 smithi194 bash[92666]: cluster 2023-10-09T05:37:27.532642+0000 osd.5 (osd.5) 316 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T05:37:28.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:28 smithi194 bash[92666]: audit 2023-10-09T05:37:27.535762+0000 mon.a (mon.0) 1876 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:37:28.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:28 smithi194 bash[92666]: cluster 2023-10-09T05:37:27.536001+0000 mon.a (mon.0) 1877 : cluster 0 osdmap e331: 8 total, 8 up, 7 in 2023-10-09T05:37:28.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:28 smithi194 bash[92666]: audit 2023-10-09T05:37:28.490528+0000 mon.a (mon.0) 1878 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:37:28.886 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:28.911 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:28.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:28.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:29.016 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:29.045 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:29.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:29.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:29.149 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:29.180 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:29.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:29.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:29.285 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:29.319 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:29.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:29.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:29.427 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:29.458 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:29.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:29.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:29.526 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:29 smithi133 bash[102858]: cluster 2023-10-09T05:37:27.821491+0000 osd.3 (osd.3) 3 : cluster 0 14.1 scrub starts 2023-10-09T05:37:29.562 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:29.616 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:29.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:29.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:29.720 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:29.727 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:29 smithi194 bash[92666]: cluster 2023-10-09T05:37:27.821491+0000 osd.3 (osd.3) 3 : cluster 0 14.1 scrub starts 2023-10-09T05:37:29.727 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:29 smithi194 bash[92666]: cluster 2023-10-09T05:37:28.244362+0000 osd.6 (osd.6) 300 : cluster 0 14.2 scrub starts 2023-10-09T05:37:29.727 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:29 smithi194 bash[92666]: cluster 2023-10-09T05:37:28.420332+0000 mgr.y (mgr.25277) 1488 : cluster 0 pgmap v1609: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 55 MiB/s wr, 884 op/s; 9312/262371 objects degraded (3.549%); 6906/262371 objects misplaced (2.632%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T05:37:29.727 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:29 smithi194 bash[92666]: cluster 2023-10-09T05:37:28.876783+0000 osd.5 (osd.5) 317 : cluster 0 14.9 scrub starts 2023-10-09T05:37:29.728 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:29 smithi194 bash[92666]: cluster 2023-10-09T05:37:28.889550+0000 mon.a (mon.0) 1879 : cluster 3 Health check update: Degraded data redundancy: 9312/262371 objects degraded (3.549%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:29.742 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:29.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:29.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:29.779 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 81 16 91550 91534 70.6109 58.0625 0.00937194 0.014156 2023-10-09T05:37:29.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:29 smithi133 bash[102858]: cluster 2023-10-09T05:37:28.244362+0000 osd.6 (osd.6) 300 : cluster 0 14.2 scrub starts 2023-10-09T05:37:29.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:29 smithi133 bash[102858]: cluster 2023-10-09T05:37:28.420332+0000 mgr.y (mgr.25277) 1488 : cluster 0 pgmap v1609: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 55 MiB/s wr, 884 op/s; 9312/262371 objects degraded (3.549%); 6906/262371 objects misplaced (2.632%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T05:37:29.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:29 smithi133 bash[102858]: cluster 2023-10-09T05:37:28.876783+0000 osd.5 (osd.5) 317 : cluster 0 14.9 scrub starts 2023-10-09T05:37:29.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:29 smithi133 bash[102858]: cluster 2023-10-09T05:37:28.889550+0000 mon.a (mon.0) 1879 : cluster 3 Health check update: Degraded data redundancy: 9312/262371 objects degraded (3.549%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:29.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:29 smithi133 bash[105589]: cluster 2023-10-09T05:37:27.821491+0000 osd.3 (osd.3) 3 : cluster 0 14.1 scrub starts 2023-10-09T05:37:29.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:29 smithi133 bash[105589]: cluster 2023-10-09T05:37:28.244362+0000 osd.6 (osd.6) 300 : cluster 0 14.2 scrub starts 2023-10-09T05:37:29.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:29 smithi133 bash[105589]: cluster 2023-10-09T05:37:28.420332+0000 mgr.y (mgr.25277) 1488 : cluster 0 pgmap v1609: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 55 MiB/s wr, 884 op/s; 9312/262371 objects degraded (3.549%); 6906/262371 objects misplaced (2.632%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T05:37:29.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:29 smithi133 bash[105589]: cluster 2023-10-09T05:37:28.876783+0000 osd.5 (osd.5) 317 : cluster 0 14.9 scrub starts 2023-10-09T05:37:29.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:29 smithi133 bash[105589]: cluster 2023-10-09T05:37:28.889550+0000 mon.a (mon.0) 1879 : cluster 3 Health check update: Degraded data redundancy: 9312/262371 objects degraded (3.549%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:29.847 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:29.875 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:29.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:29.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:29.981 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:30.007 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:30.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:30.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:30.109 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:30.139 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:30.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:30.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:30.243 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:30.266 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:30.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:30.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:30.371 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:30.405 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:30.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:30.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:30.508 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:30.536 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:30.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:30.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:30.640 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:30.650 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:30 smithi133 bash[105589]: cluster 2023-10-09T05:37:28.869659+0000 osd.3 (osd.3) 4 : cluster 0 14.1 scrub starts 2023-10-09T05:37:30.651 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:30 smithi133 bash[105589]: cluster 2023-10-09T05:37:29.877197+0000 osd.5 (osd.5) 318 : cluster 0 14.7 scrub starts 2023-10-09T05:37:30.651 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:30 smithi133 bash[102858]: cluster 2023-10-09T05:37:28.869659+0000 osd.3 (osd.3) 4 : cluster 0 14.1 scrub starts 2023-10-09T05:37:30.651 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:30 smithi133 bash[102858]: cluster 2023-10-09T05:37:29.877197+0000 osd.5 (osd.5) 318 : cluster 0 14.7 scrub starts 2023-10-09T05:37:30.677 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:30.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:30.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:30.779 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 82 16 92406 92390 70.4022 53.5 0.0180252 0.0141982 2023-10-09T05:37:30.783 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:30.810 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:30.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:30.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:30.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:30 smithi194 bash[92666]: cluster 2023-10-09T05:37:28.869659+0000 osd.3 (osd.3) 4 : cluster 0 14.1 scrub starts 2023-10-09T05:37:30.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:30 smithi194 bash[92666]: cluster 2023-10-09T05:37:29.877197+0000 osd.5 (osd.5) 318 : cluster 0 14.7 scrub starts 2023-10-09T05:37:30.913 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:30.950 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:30.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:30.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:31.055 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:31.082 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:31.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:31.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:31.186 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:31.213 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:31.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:31.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:31.316 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:31.350 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:31.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:31.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:31.456 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:31.473 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:31.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:31.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:31.576 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:31.606 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:31.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:31.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:31.713 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:31.746 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:31.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:31.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:31.779 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 83 16 93280 93264 70.2121 54.625 0.0253039 0.0142368 2023-10-09T05:37:31.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:31 smithi133 bash[105589]: cluster 2023-10-09T05:37:29.881828+0000 osd.3 (osd.3) 5 : cluster 0 14.1 scrub starts 2023-10-09T05:37:31.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:31 smithi133 bash[105589]: cluster 2023-10-09T05:37:30.421060+0000 mgr.y (mgr.25277) 1489 : cluster 0 pgmap v1610: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.6 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 54 MiB/s wr, 860 op/s; 9312/273033 objects degraded (3.411%); 6548/273033 objects misplaced (2.398%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T05:37:31.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:31 smithi133 bash[105589]: cluster 2023-10-09T05:37:30.876786+0000 osd.5 (osd.5) 319 : cluster 0 14.17 scrub starts 2023-10-09T05:37:31.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:31 smithi133 bash[102858]: cluster 2023-10-09T05:37:29.881828+0000 osd.3 (osd.3) 5 : cluster 0 14.1 scrub starts 2023-10-09T05:37:31.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:31 smithi133 bash[102858]: cluster 2023-10-09T05:37:30.421060+0000 mgr.y (mgr.25277) 1489 : cluster 0 pgmap v1610: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.6 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 54 MiB/s wr, 860 op/s; 9312/273033 objects degraded (3.411%); 6548/273033 objects misplaced (2.398%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T05:37:31.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:31 smithi133 bash[102858]: cluster 2023-10-09T05:37:30.876786+0000 osd.5 (osd.5) 319 : cluster 0 14.17 scrub starts 2023-10-09T05:37:31.851 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:31.880 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:31.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:31.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:31.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:31 smithi194 bash[92666]: cluster 2023-10-09T05:37:29.881828+0000 osd.3 (osd.3) 5 : cluster 0 14.1 scrub starts 2023-10-09T05:37:31.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:31 smithi194 bash[92666]: cluster 2023-10-09T05:37:30.421060+0000 mgr.y (mgr.25277) 1489 : cluster 0 pgmap v1610: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.6 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 54 MiB/s wr, 860 op/s; 9312/273033 objects degraded (3.411%); 6548/273033 objects misplaced (2.398%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T05:37:31.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:31 smithi194 bash[92666]: cluster 2023-10-09T05:37:30.876786+0000 osd.5 (osd.5) 319 : cluster 0 14.17 scrub starts 2023-10-09T05:37:31.983 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:32.010 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:37:32.011 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:32.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:32.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:32.116 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:32.148 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:32.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:32.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:32.254 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:32.296 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:32.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:32.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:32.400 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:32.432 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:32.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:32.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:32.535 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:32.551 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:32 smithi133 bash[105589]: cluster 2023-10-09T05:37:31.202690+0000 osd.6 (osd.6) 301 : cluster 0 14.4 scrub starts 2023-10-09T05:37:32.563 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:32.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:32.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:32.665 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:32.681 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:32 smithi194 bash[92666]: cluster 2023-10-09T05:37:31.202690+0000 osd.6 (osd.6) 301 : cluster 0 14.4 scrub starts 2023-10-09T05:37:32.691 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:32.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:32.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:32.779 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 84 16 94196 94180 70.0577 57.25 0.0137268 0.0142687 2023-10-09T05:37:32.796 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:32.806 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:32 smithi133 bash[102858]: cluster 2023-10-09T05:37:31.202690+0000 osd.6 (osd.6) 301 : cluster 0 14.4 scrub starts 2023-10-09T05:37:32.829 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:32.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:32.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:32.939 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:32.965 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:32.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:32.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:33.068 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:33.099 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:33.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:33.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:33.203 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:33.236 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:33.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:33.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:33.341 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:33.368 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:33.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:33.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:33.473 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:33.496 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:33.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:33.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:33.599 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:33.631 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:33.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:33 smithi194 bash[92666]: cluster 2023-10-09T05:37:32.421754+0000 mgr.y (mgr.25277) 1490 : cluster 0 pgmap v1611: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.6 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 54 MiB/s wr, 856 op/s; 9312/275385 objects degraded (3.381%); 6548/275385 objects misplaced (2.378%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T05:37:33.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:33.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:33.739 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:33.753 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:37:33.778 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:33.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:33.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:33.781 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 85 16 95128 95112 69.9175 58.25 0.0150211 0.0142963 2023-10-09T05:37:33.816 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:37:33.833 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:33 smithi133 bash[102858]: cluster 2023-10-09T05:37:32.421754+0000 mgr.y (mgr.25277) 1490 : cluster 0 pgmap v1611: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.6 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 54 MiB/s wr, 856 op/s; 9312/275385 objects degraded (3.381%); 6548/275385 objects misplaced (2.378%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T05:37:33.834 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:33 smithi133 bash[105589]: cluster 2023-10-09T05:37:32.421754+0000 mgr.y (mgr.25277) 1490 : cluster 0 pgmap v1611: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.6 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 54 MiB/s wr, 856 op/s; 9312/275385 objects degraded (3.381%); 6548/275385 objects misplaced (2.378%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T05:37:33.882 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:33.906 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:33.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:33.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:34.009 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:34.055 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:34.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:34.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:34.161 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:34.200 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:34.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:34.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:34.283 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:37:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:37:34] "GET /metrics HTTP/1.1" 200 36496 "" "Prometheus/2.43.0" 2023-10-09T05:37:34.305 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:34.330 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:34.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:34.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:34.434 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:34.458 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:34.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:34.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:34.563 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:34.589 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:34.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:34.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:34.691 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:34.713 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:34.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:34.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:34.781 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 86 16 95955 95939 69.7056 51.6875 0.0100698 0.0143409 2023-10-09T05:37:34.817 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:34.827 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:34 smithi133 bash[102858]: cluster 2023-10-09T05:37:33.890504+0000 mon.a (mon.0) 1880 : cluster 3 Health check update: Degraded data redundancy: 9312/275385 objects degraded (3.381%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:34.827 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:34 smithi133 bash[105589]: cluster 2023-10-09T05:37:33.890504+0000 mon.a (mon.0) 1880 : cluster 3 Health check update: Degraded data redundancy: 9312/275385 objects degraded (3.381%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:34.849 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:34.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:34.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:34.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:34 smithi194 bash[92666]: cluster 2023-10-09T05:37:33.890504+0000 mon.a (mon.0) 1880 : cluster 3 Health check update: Degraded data redundancy: 9312/275385 objects degraded (3.381%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:34.953 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:34.980 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:34.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:34.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:35.084 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:35.109 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:35.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:35.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:35.213 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:35.239 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:35.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:35.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:35.344 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:35.370 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:35.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:35.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:35.474 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:35.496 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:35.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:35.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:35.599 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:35.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:35 smithi194 bash[92666]: cluster 2023-10-09T05:37:34.235402+0000 osd.6 (osd.6) 302 : cluster 0 14.14 scrub starts 2023-10-09T05:37:35.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:35 smithi194 bash[92666]: cluster 2023-10-09T05:37:34.422340+0000 mgr.y (mgr.25277) 1491 : cluster 0 pgmap v1612: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.7 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 54 MiB/s wr, 862 op/s; 9312/279819 objects degraded (3.328%); 6548/279819 objects misplaced (2.340%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T05:37:35.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:35 smithi194 bash[92666]: cluster 2023-10-09T05:37:34.920985+0000 osd.5 (osd.5) 320 : cluster 0 14.19 scrub starts 2023-10-09T05:37:35.640 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:35.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:35.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:35.745 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:35.774 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:35.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:35.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:35.781 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 87 16 96842 96826 69.5415 55.4375 0.0232035 0.0143737 2023-10-09T05:37:35.787 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:35 smithi133 bash[102858]: cluster 2023-10-09T05:37:34.235402+0000 osd.6 (osd.6) 302 : cluster 0 14.14 scrub starts 2023-10-09T05:37:35.787 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:35 smithi133 bash[102858]: cluster 2023-10-09T05:37:34.422340+0000 mgr.y (mgr.25277) 1491 : cluster 0 pgmap v1612: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.7 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 54 MiB/s wr, 862 op/s; 9312/279819 objects degraded (3.328%); 6548/279819 objects misplaced (2.340%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T05:37:35.788 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:35 smithi133 bash[102858]: cluster 2023-10-09T05:37:34.920985+0000 osd.5 (osd.5) 320 : cluster 0 14.19 scrub starts 2023-10-09T05:37:35.788 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:35 smithi133 bash[105589]: cluster 2023-10-09T05:37:34.235402+0000 osd.6 (osd.6) 302 : cluster 0 14.14 scrub starts 2023-10-09T05:37:35.788 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:35 smithi133 bash[105589]: cluster 2023-10-09T05:37:34.422340+0000 mgr.y (mgr.25277) 1491 : cluster 0 pgmap v1612: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.7 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 54 MiB/s wr, 862 op/s; 9312/279819 objects degraded (3.328%); 6548/279819 objects misplaced (2.340%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T05:37:35.788 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:35 smithi133 bash[105589]: cluster 2023-10-09T05:37:34.920985+0000 osd.5 (osd.5) 320 : cluster 0 14.19 scrub starts 2023-10-09T05:37:35.878 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:35.906 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:35.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:35.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:36.010 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:36.036 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:36.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:36.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:36.143 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:36.182 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:36.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:36.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:36.288 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:36.330 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:36.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:36.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:36.437 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:36.474 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:36.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:36.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:36.578 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:36.593 INFO:teuthology.orchestra.run.smithi133.stderr:set pool 14 pgp_num to 26 2023-10-09T05:37:36.605 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:36.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:36.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:36.710 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:36.737 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:36.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:36.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:36.781 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 88 16 97637 97621 69.3159 49.6875 0.00963003 0.0144201 2023-10-09T05:37:36.839 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:36 smithi133 bash[102858]: audit 2023-10-09T05:37:36.377311+0000 mon.a (mon.0) 1881 : audit 1 from='client.? 172.21.15.133:0/172230450' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T05:37:36.839 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:36 smithi133 bash[105589]: audit 2023-10-09T05:37:36.377311+0000 mon.a (mon.0) 1881 : audit 1 from='client.? 172.21.15.133:0/172230450' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T05:37:36.841 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:36.848 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:36 smithi194 bash[92666]: audit 2023-10-09T05:37:36.377311+0000 mon.a (mon.0) 1881 : audit 1 from='client.? 172.21.15.133:0/172230450' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T05:37:36.869 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:36.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:36.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:36.972 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:36.997 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:36.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:36.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:37.100 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:37.126 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:37:37.127 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:37:37.127 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:37.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:37.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:37.231 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:37.262 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:37.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:37.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:37.366 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:37.391 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:37.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:37.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:37.494 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:37.522 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:37.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:37.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:37.580 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:37 smithi133 bash[105589]: cluster 2023-10-09T05:37:36.423063+0000 mgr.y (mgr.25277) 1492 : cluster 0 pgmap v1613: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.8 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 55 MiB/s wr, 872 op/s; 9312/287226 objects degraded (3.242%); 6170/287226 objects misplaced (2.148%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:37:37.580 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:37 smithi133 bash[105589]: cluster 2023-10-09T05:37:36.575791+0000 osd.7 (osd.7) 286 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T05:37:37.581 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:37 smithi133 bash[105589]: cluster 2023-10-09T05:37:36.576324+0000 osd.5 (osd.5) 321 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T05:37:37.581 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:37 smithi133 bash[105589]: audit 2023-10-09T05:37:36.584025+0000 mon.a (mon.0) 1882 : audit 1 from='client.? 172.21.15.133:0/172230450' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T05:37:37.581 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:37 smithi133 bash[105589]: cluster 2023-10-09T05:37:36.584250+0000 mon.a (mon.0) 1883 : cluster 0 osdmap e332: 8 total, 8 up, 7 in 2023-10-09T05:37:37.581 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:37 smithi133 bash[105589]: cluster 2023-10-09T05:37:36.935052+0000 osd.5 (osd.5) 322 : cluster 0 14.9 scrub starts 2023-10-09T05:37:37.581 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:37 smithi133 bash[105589]: cluster 2023-10-09T05:37:37.294876+0000 osd.4 (osd.4) 297 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T05:37:37.581 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:37 smithi133 bash[102858]: cluster 2023-10-09T05:37:36.423063+0000 mgr.y (mgr.25277) 1492 : cluster 0 pgmap v1613: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.8 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 55 MiB/s wr, 872 op/s; 9312/287226 objects degraded (3.242%); 6170/287226 objects misplaced (2.148%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:37:37.581 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:37 smithi133 bash[102858]: cluster 2023-10-09T05:37:36.575791+0000 osd.7 (osd.7) 286 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T05:37:37.581 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:37 smithi133 bash[102858]: cluster 2023-10-09T05:37:36.576324+0000 osd.5 (osd.5) 321 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T05:37:37.581 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:37 smithi133 bash[102858]: audit 2023-10-09T05:37:36.584025+0000 mon.a (mon.0) 1882 : audit 1 from='client.? 172.21.15.133:0/172230450' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T05:37:37.582 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:37 smithi133 bash[102858]: cluster 2023-10-09T05:37:36.584250+0000 mon.a (mon.0) 1883 : cluster 0 osdmap e332: 8 total, 8 up, 7 in 2023-10-09T05:37:37.582 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:37 smithi133 bash[102858]: cluster 2023-10-09T05:37:36.935052+0000 osd.5 (osd.5) 322 : cluster 0 14.9 scrub starts 2023-10-09T05:37:37.582 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:37 smithi133 bash[102858]: cluster 2023-10-09T05:37:37.294876+0000 osd.4 (osd.4) 297 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T05:37:37.625 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:37.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:37 smithi194 bash[92666]: cluster 2023-10-09T05:37:36.423063+0000 mgr.y (mgr.25277) 1492 : cluster 0 pgmap v1613: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.8 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 55 MiB/s wr, 872 op/s; 9312/287226 objects degraded (3.242%); 6170/287226 objects misplaced (2.148%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:37:37.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:37 smithi194 bash[92666]: cluster 2023-10-09T05:37:36.575791+0000 osd.7 (osd.7) 286 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T05:37:37.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:37 smithi194 bash[92666]: cluster 2023-10-09T05:37:36.576324+0000 osd.5 (osd.5) 321 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T05:37:37.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:37 smithi194 bash[92666]: audit 2023-10-09T05:37:36.584025+0000 mon.a (mon.0) 1882 : audit 1 from='client.? 172.21.15.133:0/172230450' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T05:37:37.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:37 smithi194 bash[92666]: cluster 2023-10-09T05:37:36.584250+0000 mon.a (mon.0) 1883 : cluster 0 osdmap e332: 8 total, 8 up, 7 in 2023-10-09T05:37:37.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:37 smithi194 bash[92666]: cluster 2023-10-09T05:37:36.935052+0000 osd.5 (osd.5) 322 : cluster 0 14.9 scrub starts 2023-10-09T05:37:37.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:37 smithi194 bash[92666]: cluster 2023-10-09T05:37:37.294876+0000 osd.4 (osd.4) 297 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T05:37:37.675 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:37.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:37.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:37.779 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:37.781 INFO:tasks.radosbench.radosbench.0.smithi133.stdout: 89 16 98512 98496 69.1515 54.6875 0.0198399 0.014455 2023-10-09T05:37:37.817 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:37.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:37.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:37.924 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:37.949 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:37.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:37.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:38.054 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:38.078 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:38.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:38.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:38.182 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:38.205 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:38.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:38.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:38.308 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:38.343 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:38.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:38.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:38.443 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T05:37:38.448 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:38.530 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:38.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:38.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:38.637 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:38.673 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:38.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:38.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:38.775 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:38.784 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Total time run: 90.0078 2023-10-09T05:37:38.785 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Total writes made: 99326 2023-10-09T05:37:38.785 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Write size: 65536 2023-10-09T05:37:38.785 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Object size: 65536 2023-10-09T05:37:38.785 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Bandwidth (MB/sec): 68.9704 2023-10-09T05:37:38.785 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Stddev Bandwidth: 27.9024 2023-10-09T05:37:38.785 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Max bandwidth (MB/sec): 158.125 2023-10-09T05:37:38.785 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Min bandwidth (MB/sec): 11.875 2023-10-09T05:37:38.785 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Average IOPS: 1103 2023-10-09T05:37:38.786 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Stddev IOPS: 446.438 2023-10-09T05:37:38.786 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Max IOPS: 2530 2023-10-09T05:37:38.786 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Min IOPS: 190 2023-10-09T05:37:38.786 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Average Latency(s): 0.0144946 2023-10-09T05:37:38.786 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Stddev Latency(s): 0.039607 2023-10-09T05:37:38.786 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Max latency(s): 2.40321 2023-10-09T05:37:38.786 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Min latency(s): 0.00135224 2023-10-09T05:37:38.787 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:38 smithi194 bash[92666]: cluster 2023-10-09T05:37:37.004351+0000 osd.3 (osd.3) 6 : cluster 0 14.1 scrub starts 2023-10-09T05:37:38.788 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:38 smithi194 bash[92666]: cluster 2023-10-09T05:37:37.981399+0000 osd.5 (osd.5) 323 : cluster 0 14.7 scrub starts 2023-10-09T05:37:38.792 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T05:37:38.807 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:38.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:38.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:38.913 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:38.938 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:38.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:38.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:39.043 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:39.047 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:38 smithi133 bash[105589]: cluster 2023-10-09T05:37:37.004351+0000 osd.3 (osd.3) 6 : cluster 0 14.1 scrub starts 2023-10-09T05:37:39.047 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:38 smithi133 bash[105589]: cluster 2023-10-09T05:37:37.981399+0000 osd.5 (osd.5) 323 : cluster 0 14.7 scrub starts 2023-10-09T05:37:39.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:38 smithi133 bash[102858]: cluster 2023-10-09T05:37:37.004351+0000 osd.3 (osd.3) 6 : cluster 0 14.1 scrub starts 2023-10-09T05:37:39.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:38 smithi133 bash[102858]: cluster 2023-10-09T05:37:37.981399+0000 osd.5 (osd.5) 323 : cluster 0 14.7 scrub starts 2023-10-09T05:37:39.084 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:39.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:39.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:39.188 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:39.220 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:39.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:39.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:39.323 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:39.352 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:39.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:39.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:39.455 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:39.488 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:39.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:39.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:39.592 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:39.616 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:39.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:39.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:39.721 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:39.751 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:39.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:39.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:39.854 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:39.864 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:39 smithi194 bash[92666]: cluster 2023-10-09T05:37:38.424972+0000 mgr.y (mgr.25277) 1493 : cluster 0 pgmap v1615: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.9 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 54 MiB/s wr, 870 op/s; 9312/288498 objects degraded (3.228%); 6170/288498 objects misplaced (2.139%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:37:39.865 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:39 smithi194 bash[92666]: cluster 2023-10-09T05:37:38.891548+0000 mon.a (mon.0) 1884 : cluster 3 Health check update: Degraded data redundancy: 9312/288498 objects degraded (3.228%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:39.880 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:39.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:39.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:39.983 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:39.993 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:39 smithi133 bash[102858]: cluster 2023-10-09T05:37:38.424972+0000 mgr.y (mgr.25277) 1493 : cluster 0 pgmap v1615: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.9 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 54 MiB/s wr, 870 op/s; 9312/288498 objects degraded (3.228%); 6170/288498 objects misplaced (2.139%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:37:39.993 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:39 smithi133 bash[102858]: cluster 2023-10-09T05:37:38.891548+0000 mon.a (mon.0) 1884 : cluster 3 Health check update: Degraded data redundancy: 9312/288498 objects degraded (3.228%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:39.993 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:39 smithi133 bash[105589]: cluster 2023-10-09T05:37:38.424972+0000 mgr.y (mgr.25277) 1493 : cluster 0 pgmap v1615: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.9 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 54 MiB/s wr, 870 op/s; 9312/288498 objects degraded (3.228%); 6170/288498 objects misplaced (2.139%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:37:39.993 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:39 smithi133 bash[105589]: cluster 2023-10-09T05:37:38.891548+0000 mon.a (mon.0) 1884 : cluster 3 Health check update: Degraded data redundancy: 9312/288498 objects degraded (3.228%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:40.019 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:40.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:40.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:40.122 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:40.161 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:40.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:40.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:40.266 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:40.308 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:40.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:40.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:40.415 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:40.448 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:40.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:40.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:40.552 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:40.583 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:40.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:40.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:40.690 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:40.721 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:40.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:40.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:40.827 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:40.838 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:40 smithi133 bash[102858]: cluster 2023-10-09T05:37:39.958751+0000 osd.5 (osd.5) 324 : cluster 0 14.17 scrub starts 2023-10-09T05:37:40.838 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:40 smithi133 bash[105589]: cluster 2023-10-09T05:37:39.958751+0000 osd.5 (osd.5) 324 : cluster 0 14.17 scrub starts 2023-10-09T05:37:40.843 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:37:40.844 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:37:40.874 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:40.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:40.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:40.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:40 smithi194 bash[92666]: cluster 2023-10-09T05:37:39.958751+0000 osd.5 (osd.5) 324 : cluster 0 14.17 scrub starts 2023-10-09T05:37:40.982 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:41.023 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:41.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:41.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:41.133 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:41.171 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:41.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:41.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:41.276 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:41.307 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:41.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:41.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:41.411 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:41.441 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:41.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:41.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:41.550 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:41.577 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:41.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:41.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:41.680 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:41.708 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:41.710 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:41 smithi133 bash[105589]: cluster 2023-10-09T05:37:40.163318+0000 osd.6 (osd.6) 303 : cluster 0 14.2 scrub starts 2023-10-09T05:37:41.710 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:41 smithi133 bash[105589]: cluster 2023-10-09T05:37:40.425768+0000 mgr.y (mgr.25277) 1494 : cluster 0 pgmap v1616: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 49 MiB/s wr, 863 op/s; 9238/294663 objects degraded (3.135%); 5683/294663 objects misplaced (1.929%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T05:37:41.710 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:41 smithi133 bash[105589]: cluster 2023-10-09T05:37:40.932493+0000 osd.5 (osd.5) 325 : cluster 0 14.19 scrub starts 2023-10-09T05:37:41.711 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:41 smithi133 bash[102858]: cluster 2023-10-09T05:37:40.163318+0000 osd.6 (osd.6) 303 : cluster 0 14.2 scrub starts 2023-10-09T05:37:41.711 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:41 smithi133 bash[102858]: cluster 2023-10-09T05:37:40.425768+0000 mgr.y (mgr.25277) 1494 : cluster 0 pgmap v1616: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 49 MiB/s wr, 863 op/s; 9238/294663 objects degraded (3.135%); 5683/294663 objects misplaced (1.929%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T05:37:41.711 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:41 smithi133 bash[102858]: cluster 2023-10-09T05:37:40.932493+0000 osd.5 (osd.5) 325 : cluster 0 14.19 scrub starts 2023-10-09T05:37:41.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:41.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:41.813 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:41.844 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:41.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:41.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:41.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:41 smithi194 bash[92666]: cluster 2023-10-09T05:37:40.163318+0000 osd.6 (osd.6) 303 : cluster 0 14.2 scrub starts 2023-10-09T05:37:41.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:41 smithi194 bash[92666]: cluster 2023-10-09T05:37:40.425768+0000 mgr.y (mgr.25277) 1494 : cluster 0 pgmap v1616: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 49 MiB/s wr, 863 op/s; 9238/294663 objects degraded (3.135%); 5683/294663 objects misplaced (1.929%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T05:37:41.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:41 smithi194 bash[92666]: cluster 2023-10-09T05:37:40.932493+0000 osd.5 (osd.5) 325 : cluster 0 14.19 scrub starts 2023-10-09T05:37:41.949 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:41.976 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:41.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:41.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:42.081 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:42.108 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:42.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:42.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:42.215 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:42.253 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:42.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:42.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:42.359 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:42.385 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:42.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:42.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:42.491 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:42.520 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:42.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:42.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:42.624 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:42.653 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:42.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:42.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:42.756 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:42.785 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:42.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:42.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:42.874 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:37:42.891 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:42.919 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:42.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:42.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:43.024 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:43.051 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:43.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:43.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:43.155 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:43.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:43.183 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:43.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:43.284 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:43.308 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:43.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:43.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:43.346 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 6, 1, 7, 5] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:37:43.346 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:37:43.347 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_9 2023-10-09T05:37:43.347 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:37:43.413 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:43.441 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:43.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:43.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:43.543 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:43.576 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:43.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:43.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:43.604 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:43 smithi133 bash[102858]: cluster 2023-10-09T05:37:42.426475+0000 mgr.y (mgr.25277) 1495 : cluster 0 pgmap v1617: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 47 MiB/s wr, 862 op/s; 9150/294825 objects degraded (3.104%); 5683/294825 objects misplaced (1.928%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T05:37:43.604 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:43 smithi133 bash[102858]: audit 2023-10-09T05:37:43.490735+0000 mon.a (mon.0) 1885 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:37:43.679 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:43.709 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:43.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:43.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:43.814 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:43.824 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:43 smithi194 bash[92666]: cluster 2023-10-09T05:37:42.426475+0000 mgr.y (mgr.25277) 1495 : cluster 0 pgmap v1617: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 47 MiB/s wr, 862 op/s; 9150/294825 objects degraded (3.104%); 5683/294825 objects misplaced (1.928%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T05:37:43.824 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:43 smithi194 bash[92666]: audit 2023-10-09T05:37:43.490735+0000 mon.a (mon.0) 1885 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:37:43.841 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:43.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:43.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:43.949 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:43.979 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:43.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:43.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:44.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:43 smithi133 bash[105589]: cluster 2023-10-09T05:37:42.426475+0000 mgr.y (mgr.25277) 1495 : cluster 0 pgmap v1617: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 47 MiB/s wr, 862 op/s; 9150/294825 objects degraded (3.104%); 5683/294825 objects misplaced (1.928%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T05:37:44.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:43 smithi133 bash[105589]: audit 2023-10-09T05:37:43.490735+0000 mon.a (mon.0) 1885 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:37:44.082 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:44.117 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:44.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:44.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:44.228 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:44.254 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:44.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:44.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:44.358 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:44.372 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:37:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:37:44] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.43.0" 2023-10-09T05:37:44.390 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:44.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:44.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:44.496 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:44.521 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:44.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:44.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:44.624 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:44.652 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:44.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:44.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:44.758 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:44.767 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:44 smithi133 bash[102858]: cluster 2023-10-09T05:37:43.892832+0000 mon.a (mon.0) 1886 : cluster 3 Health check update: Degraded data redundancy: 9150/294825 objects degraded (3.104%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:44.767 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:44 smithi133 bash[102858]: cluster 2023-10-09T05:37:43.976002+0000 osd.5 (osd.5) 326 : cluster 0 14.9 scrub starts 2023-10-09T05:37:44.768 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:44 smithi133 bash[105589]: cluster 2023-10-09T05:37:43.892832+0000 mon.a (mon.0) 1886 : cluster 3 Health check update: Degraded data redundancy: 9150/294825 objects degraded (3.104%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:44.768 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:44 smithi133 bash[105589]: cluster 2023-10-09T05:37:43.976002+0000 osd.5 (osd.5) 326 : cluster 0 14.9 scrub starts 2023-10-09T05:37:44.790 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:44.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:44.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:44.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:44 smithi194 bash[92666]: cluster 2023-10-09T05:37:43.892832+0000 mon.a (mon.0) 1886 : cluster 3 Health check update: Degraded data redundancy: 9150/294825 objects degraded (3.104%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:44.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:44 smithi194 bash[92666]: cluster 2023-10-09T05:37:43.976002+0000 osd.5 (osd.5) 326 : cluster 0 14.9 scrub starts 2023-10-09T05:37:44.894 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:44.922 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:44.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:44.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:45.029 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:45.052 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:45.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:45.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:45.156 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:45.179 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:45.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:45.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:45.287 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:45.313 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:45.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:45.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:45.419 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:45.449 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:45.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:45.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:45.557 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:45.580 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:45.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:45.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:45.683 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:45.721 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:45.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:45.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:45.825 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:45.838 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:45 smithi194 bash[92666]: cluster 2023-10-09T05:37:44.427129+0000 mgr.y (mgr.25277) 1496 : cluster 0 pgmap v1618: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.9 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 38 MiB/s wr, 848 op/s; 9150/290814 objects degraded (3.146%); 5683/290814 objects misplaced (1.954%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T05:37:45.839 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:45 smithi194 bash[92666]: cluster 2023-10-09T05:37:44.976184+0000 osd.5 (osd.5) 327 : cluster 0 14.7 scrub starts 2023-10-09T05:37:45.859 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:45.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:45.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:45.962 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:45.971 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:45 smithi133 bash[102858]: cluster 2023-10-09T05:37:44.427129+0000 mgr.y (mgr.25277) 1496 : cluster 0 pgmap v1618: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.9 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 38 MiB/s wr, 848 op/s; 9150/290814 objects degraded (3.146%); 5683/290814 objects misplaced (1.954%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T05:37:45.971 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:45 smithi133 bash[102858]: cluster 2023-10-09T05:37:44.976184+0000 osd.5 (osd.5) 327 : cluster 0 14.7 scrub starts 2023-10-09T05:37:45.971 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:45 smithi133 bash[105589]: cluster 2023-10-09T05:37:44.427129+0000 mgr.y (mgr.25277) 1496 : cluster 0 pgmap v1618: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.9 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 38 MiB/s wr, 848 op/s; 9150/290814 objects degraded (3.146%); 5683/290814 objects misplaced (1.954%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T05:37:45.971 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:45 smithi133 bash[105589]: cluster 2023-10-09T05:37:44.976184+0000 osd.5 (osd.5) 327 : cluster 0 14.7 scrub starts 2023-10-09T05:37:45.987 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:45.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:45.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:46.091 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:46.118 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:46.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:46.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:46.225 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:46.273 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:46.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:46.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:46.379 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:46.413 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:46.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:46.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:46.517 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:46.541 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:46.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:46.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:46.644 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:46.652 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:46 smithi194 bash[92666]: cluster 2023-10-09T05:37:45.196556+0000 osd.6 (osd.6) 304 : cluster 0 14.4 scrub starts 2023-10-09T05:37:46.653 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:46 smithi194 bash[92666]: cluster 2023-10-09T05:37:45.930713+0000 osd.5 (osd.5) 328 : cluster 0 14.17 scrub starts 2023-10-09T05:37:46.668 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:46.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:46.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:46.775 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:46.804 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:46.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:46.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:46.885 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:46 smithi133 bash[105589]: cluster 2023-10-09T05:37:45.196556+0000 osd.6 (osd.6) 304 : cluster 0 14.4 scrub starts 2023-10-09T05:37:46.886 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:46 smithi133 bash[105589]: cluster 2023-10-09T05:37:45.930713+0000 osd.5 (osd.5) 328 : cluster 0 14.17 scrub starts 2023-10-09T05:37:46.886 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:46 smithi133 bash[102858]: cluster 2023-10-09T05:37:45.196556+0000 osd.6 (osd.6) 304 : cluster 0 14.4 scrub starts 2023-10-09T05:37:46.886 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:46 smithi133 bash[102858]: cluster 2023-10-09T05:37:45.930713+0000 osd.5 (osd.5) 328 : cluster 0 14.17 scrub starts 2023-10-09T05:37:46.909 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:46.943 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:46.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:46.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:47.046 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:47.077 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:47.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:47.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:47.186 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:47.220 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:47.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:47.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:47.324 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:47.350 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:47.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:47.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:47.453 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:47.487 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:47.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:47.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:47.593 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:47.619 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:47.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:47.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:47.712 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:47 smithi133 bash[102858]: cluster 2023-10-09T05:37:46.235541+0000 osd.6 (osd.6) 305 : cluster 0 14.14 scrub starts 2023-10-09T05:37:47.713 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:47 smithi133 bash[102858]: cluster 2023-10-09T05:37:46.427851+0000 mgr.y (mgr.25277) 1497 : cluster 0 pgmap v1619: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.8 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 23 MiB/s wr, 842 op/s; 8648/283599 objects degraded (3.049%); 5033/283599 objects misplaced (1.775%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T05:37:47.713 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:47 smithi133 bash[105589]: cluster 2023-10-09T05:37:46.235541+0000 osd.6 (osd.6) 305 : cluster 0 14.14 scrub starts 2023-10-09T05:37:47.713 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:47 smithi133 bash[105589]: cluster 2023-10-09T05:37:46.427851+0000 mgr.y (mgr.25277) 1497 : cluster 0 pgmap v1619: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.8 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 23 MiB/s wr, 842 op/s; 8648/283599 objects degraded (3.049%); 5033/283599 objects misplaced (1.775%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T05:37:47.725 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:47.755 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:47.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:47.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:47.862 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:47.872 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:47 smithi194 bash[92666]: cluster 2023-10-09T05:37:46.235541+0000 osd.6 (osd.6) 305 : cluster 0 14.14 scrub starts 2023-10-09T05:37:47.872 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:47 smithi194 bash[92666]: cluster 2023-10-09T05:37:46.427851+0000 mgr.y (mgr.25277) 1497 : cluster 0 pgmap v1619: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.8 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 23 MiB/s wr, 842 op/s; 8648/283599 objects degraded (3.049%); 5033/283599 objects misplaced (1.775%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T05:37:47.887 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:47.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:47.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:47.990 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:48.015 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:48.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:48.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:48.119 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:48.143 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:48.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:48.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:48.250 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:48.274 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:48.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:48.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:48.380 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:48.407 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:48.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:48.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:48.511 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:48.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:48.537 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:48.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:48.638 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:48.667 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:48.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:48.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:48.772 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:48.799 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:48.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:48.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:48.903 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:48.937 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:48.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:48.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:49.044 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:49.072 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:49.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:49.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:49.177 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:49.200 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:49.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:49.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:49.305 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:49.336 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:49.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:49.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:49.440 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:49.476 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:49.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:49.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:49.582 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:49.610 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:49.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:49.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:49.713 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:49.745 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:49.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:49.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:49.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:49 smithi133 bash[102858]: cluster 2023-10-09T05:37:48.276588+0000 osd.6 (osd.6) 306 : cluster 0 14.2 scrub starts 2023-10-09T05:37:49.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:49 smithi133 bash[102858]: cluster 2023-10-09T05:37:48.428562+0000 mgr.y (mgr.25277) 1498 : cluster 0 pgmap v1620: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.7 GiB data, 25 GiB used, 600 GiB / 626 GiB avail; 86 B/s rd, 19 MiB/s wr, 742 op/s; 8648/282483 objects degraded (3.061%); 5033/282483 objects misplaced (1.782%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T05:37:49.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:49 smithi133 bash[102858]: cluster 2023-10-09T05:37:48.894056+0000 mon.a (mon.0) 1887 : cluster 3 Health check update: Degraded data redundancy: 8648/282483 objects degraded (3.061%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:49.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:49 smithi133 bash[102858]: cluster 2023-10-09T05:37:48.998553+0000 osd.5 (osd.5) 329 : cluster 0 14.19 scrub starts 2023-10-09T05:37:49.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:49 smithi133 bash[105589]: cluster 2023-10-09T05:37:48.276588+0000 osd.6 (osd.6) 306 : cluster 0 14.2 scrub starts 2023-10-09T05:37:49.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:49 smithi133 bash[105589]: cluster 2023-10-09T05:37:48.428562+0000 mgr.y (mgr.25277) 1498 : cluster 0 pgmap v1620: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.7 GiB data, 25 GiB used, 600 GiB / 626 GiB avail; 86 B/s rd, 19 MiB/s wr, 742 op/s; 8648/282483 objects degraded (3.061%); 5033/282483 objects misplaced (1.782%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T05:37:49.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:49 smithi133 bash[105589]: cluster 2023-10-09T05:37:48.894056+0000 mon.a (mon.0) 1887 : cluster 3 Health check update: Degraded data redundancy: 8648/282483 objects degraded (3.061%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:49.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:49 smithi133 bash[105589]: cluster 2023-10-09T05:37:48.998553+0000 osd.5 (osd.5) 329 : cluster 0 14.19 scrub starts 2023-10-09T05:37:49.847 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:49.870 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:49.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:49.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:49.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:49 smithi194 bash[92666]: cluster 2023-10-09T05:37:48.276588+0000 osd.6 (osd.6) 306 : cluster 0 14.2 scrub starts 2023-10-09T05:37:49.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:49 smithi194 bash[92666]: cluster 2023-10-09T05:37:48.428562+0000 mgr.y (mgr.25277) 1498 : cluster 0 pgmap v1620: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.7 GiB data, 25 GiB used, 600 GiB / 626 GiB avail; 86 B/s rd, 19 MiB/s wr, 742 op/s; 8648/282483 objects degraded (3.061%); 5033/282483 objects misplaced (1.782%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T05:37:49.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:49 smithi194 bash[92666]: cluster 2023-10-09T05:37:48.894056+0000 mon.a (mon.0) 1887 : cluster 3 Health check update: Degraded data redundancy: 8648/282483 objects degraded (3.061%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:49.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:49 smithi194 bash[92666]: cluster 2023-10-09T05:37:48.998553+0000 osd.5 (osd.5) 329 : cluster 0 14.19 scrub starts 2023-10-09T05:37:49.974 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:49.999 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:50.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:50.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:50.105 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:50.131 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:50.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:50.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:50.234 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:50.258 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:50.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:50.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:50.363 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:50.389 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:50.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:50.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:50.492 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:50.522 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:50.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:50.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:50.625 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:50.647 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:50.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:50.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:50.751 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:50.778 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:50.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:50.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:50.881 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:50.906 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:50.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:50.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:51.012 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:51.061 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:51.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:51.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:51.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:50 smithi194 bash[92666]: cluster 2023-10-09T05:37:49.907942+0000 osd.7 (osd.7) 287 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T05:37:51.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:50 smithi194 bash[92666]: cluster 2023-10-09T05:37:49.907946+0000 osd.5 (osd.5) 330 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T05:37:51.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:50 smithi194 bash[92666]: cluster 2023-10-09T05:37:49.909783+0000 mon.a (mon.0) 1888 : cluster 0 osdmap e333: 8 total, 8 up, 7 in 2023-10-09T05:37:51.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:50 smithi194 bash[92666]: cluster 2023-10-09T05:37:49.910902+0000 osd.5 (osd.5) 331 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T05:37:51.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:50 smithi194 bash[92666]: cluster 2023-10-09T05:37:49.915781+0000 osd.5 (osd.5) 332 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T05:37:51.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:50 smithi194 bash[92666]: cluster 2023-10-09T05:37:50.429053+0000 mgr.y (mgr.25277) 1499 : cluster 0 pgmap v1622: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.6 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 2.6 MiB/s wr, 792 op/s; 8424/273372 objects degraded (3.082%); 4921/273372 objects misplaced (1.800%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T05:37:51.168 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:51.191 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:51.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:51.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:51.295 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:51.297 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:50 smithi133 bash[105589]: cluster 2023-10-09T05:37:49.907942+0000 osd.7 (osd.7) 287 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T05:37:51.297 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:50 smithi133 bash[105589]: cluster 2023-10-09T05:37:49.907946+0000 osd.5 (osd.5) 330 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T05:37:51.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:50 smithi133 bash[105589]: cluster 2023-10-09T05:37:49.909783+0000 mon.a (mon.0) 1888 : cluster 0 osdmap e333: 8 total, 8 up, 7 in 2023-10-09T05:37:51.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:50 smithi133 bash[105589]: cluster 2023-10-09T05:37:49.910902+0000 osd.5 (osd.5) 331 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T05:37:51.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:50 smithi133 bash[105589]: cluster 2023-10-09T05:37:49.915781+0000 osd.5 (osd.5) 332 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T05:37:51.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:50 smithi133 bash[105589]: cluster 2023-10-09T05:37:50.429053+0000 mgr.y (mgr.25277) 1499 : cluster 0 pgmap v1622: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.6 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 2.6 MiB/s wr, 792 op/s; 8424/273372 objects degraded (3.082%); 4921/273372 objects misplaced (1.800%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T05:37:51.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:50 smithi133 bash[102858]: cluster 2023-10-09T05:37:49.907942+0000 osd.7 (osd.7) 287 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T05:37:51.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:50 smithi133 bash[102858]: cluster 2023-10-09T05:37:49.907946+0000 osd.5 (osd.5) 330 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T05:37:51.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:50 smithi133 bash[102858]: cluster 2023-10-09T05:37:49.909783+0000 mon.a (mon.0) 1888 : cluster 0 osdmap e333: 8 total, 8 up, 7 in 2023-10-09T05:37:51.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:50 smithi133 bash[102858]: cluster 2023-10-09T05:37:49.910902+0000 osd.5 (osd.5) 331 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T05:37:51.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:50 smithi133 bash[102858]: cluster 2023-10-09T05:37:49.915781+0000 osd.5 (osd.5) 332 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T05:37:51.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:50 smithi133 bash[102858]: cluster 2023-10-09T05:37:50.429053+0000 mgr.y (mgr.25277) 1499 : cluster 0 pgmap v1622: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.6 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 2.6 MiB/s wr, 792 op/s; 8424/273372 objects degraded (3.082%); 4921/273372 objects misplaced (1.800%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T05:37:51.330 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:51.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:51.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:51.436 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:51.462 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:51.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:51.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:51.566 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:51.593 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:51.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:51.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:51.697 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:51.730 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:51.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:51.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:51.833 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:51.852 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:51.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:51.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:51.955 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:51.965 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:51 smithi133 bash[102858]: cluster 2023-10-09T05:37:50.829928+0000 osd.4 (osd.4) 298 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T05:37:51.965 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:51 smithi133 bash[102858]: cluster 2023-10-09T05:37:50.912335+0000 osd.6 (osd.6) 307 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T05:37:51.966 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:51 smithi133 bash[102858]: cluster 2023-10-09T05:37:50.912715+0000 osd.5 (osd.5) 333 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T05:37:51.966 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:51 smithi133 bash[102858]: cluster 2023-10-09T05:37:50.916661+0000 mon.a (mon.0) 1889 : cluster 0 osdmap e334: 8 total, 8 up, 7 in 2023-10-09T05:37:51.966 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:51 smithi133 bash[102858]: cluster 2023-10-09T05:37:50.917624+0000 osd.5 (osd.5) 334 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T05:37:51.966 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:51 smithi133 bash[102858]: cluster 2023-10-09T05:37:50.917774+0000 osd.7 (osd.7) 288 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T05:37:51.966 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:51 smithi133 bash[105589]: cluster 2023-10-09T05:37:50.829928+0000 osd.4 (osd.4) 298 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T05:37:51.966 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:51 smithi133 bash[105589]: cluster 2023-10-09T05:37:50.912335+0000 osd.6 (osd.6) 307 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T05:37:51.966 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:51 smithi133 bash[105589]: cluster 2023-10-09T05:37:50.912715+0000 osd.5 (osd.5) 333 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T05:37:51.966 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:51 smithi133 bash[105589]: cluster 2023-10-09T05:37:50.916661+0000 mon.a (mon.0) 1889 : cluster 0 osdmap e334: 8 total, 8 up, 7 in 2023-10-09T05:37:51.967 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:51 smithi133 bash[105589]: cluster 2023-10-09T05:37:50.917624+0000 osd.5 (osd.5) 334 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T05:37:51.967 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:51 smithi133 bash[105589]: cluster 2023-10-09T05:37:50.917774+0000 osd.7 (osd.7) 288 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T05:37:51.987 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:51.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:51.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:52.092 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:52.127 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:52.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:52.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:52.232 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:52.242 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:51 smithi194 bash[92666]: cluster 2023-10-09T05:37:50.829928+0000 osd.4 (osd.4) 298 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T05:37:52.242 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:51 smithi194 bash[92666]: cluster 2023-10-09T05:37:50.912335+0000 osd.6 (osd.6) 307 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T05:37:52.242 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:51 smithi194 bash[92666]: cluster 2023-10-09T05:37:50.912715+0000 osd.5 (osd.5) 333 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T05:37:52.242 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:51 smithi194 bash[92666]: cluster 2023-10-09T05:37:50.916661+0000 mon.a (mon.0) 1889 : cluster 0 osdmap e334: 8 total, 8 up, 7 in 2023-10-09T05:37:52.243 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:51 smithi194 bash[92666]: cluster 2023-10-09T05:37:50.917624+0000 osd.5 (osd.5) 334 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T05:37:52.243 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:51 smithi194 bash[92666]: cluster 2023-10-09T05:37:50.917774+0000 osd.7 (osd.7) 288 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T05:37:52.257 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:52.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:52.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:52.361 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:52.390 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:52.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:52.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:52.494 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:52.521 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:52.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:52.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:52.626 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:52.653 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:52.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:52.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:52.757 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:52.782 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:52.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:52.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:52.886 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:52.915 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:52.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:52.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:53.018 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:53.047 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:53.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:53.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:53.150 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:53.160 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:52 smithi194 bash[92666]: cluster 2023-10-09T05:37:51.394883+0000 osd.4 (osd.4) 299 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T05:37:53.160 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:52 smithi194 bash[92666]: cluster 2023-10-09T05:37:52.429609+0000 mgr.y (mgr.25277) 1500 : cluster 0 pgmap v1624: 99 pgs: 1 active+recovering+remapped, 1 creating+peering, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 832 op/s; 8176/270828 objects degraded (3.019%); 4319/270828 objects misplaced (1.595%); 8.6 MiB/s, 137 objects/s recovering 2023-10-09T05:37:53.176 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:53.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:53.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:53.284 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:53.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:52 smithi133 bash[102858]: cluster 2023-10-09T05:37:51.394883+0000 osd.4 (osd.4) 299 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T05:37:53.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:52 smithi133 bash[102858]: cluster 2023-10-09T05:37:52.429609+0000 mgr.y (mgr.25277) 1500 : cluster 0 pgmap v1624: 99 pgs: 1 active+recovering+remapped, 1 creating+peering, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 832 op/s; 8176/270828 objects degraded (3.019%); 4319/270828 objects misplaced (1.595%); 8.6 MiB/s, 137 objects/s recovering 2023-10-09T05:37:53.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:52 smithi133 bash[105589]: cluster 2023-10-09T05:37:51.394883+0000 osd.4 (osd.4) 299 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T05:37:53.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:52 smithi133 bash[105589]: cluster 2023-10-09T05:37:52.429609+0000 mgr.y (mgr.25277) 1500 : cluster 0 pgmap v1624: 99 pgs: 1 active+recovering+remapped, 1 creating+peering, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 832 op/s; 8176/270828 objects degraded (3.019%); 4319/270828 objects misplaced (1.595%); 8.6 MiB/s, 137 objects/s recovering 2023-10-09T05:37:53.312 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:53.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:53.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:53.417 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:53.435 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:53.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:53.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:53.538 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:53.565 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:53.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:53.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:53.668 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:53.701 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:53.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:53.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:53.808 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:53.855 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:53.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:53.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:53.932 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:53 smithi133 bash[102858]: cluster 2023-10-09T05:37:53.900011+0000 mon.a (mon.0) 1890 : cluster 3 Health check update: Degraded data redundancy: 8176/270828 objects degraded (3.019%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:53.961 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:53.991 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:53.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:53.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:54.098 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:54.121 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:54.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:54.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:54.212 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:37:54.224 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:54.235 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:53 smithi194 bash[92666]: cluster 2023-10-09T05:37:53.900011+0000 mon.a (mon.0) 1890 : cluster 3 Health check update: Degraded data redundancy: 8176/270828 objects degraded (3.019%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:54.250 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:54.250 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:53 smithi133 bash[105589]: cluster 2023-10-09T05:37:53.900011+0000 mon.a (mon.0) 1890 : cluster 3 Health check update: Degraded data redundancy: 8176/270828 objects degraded (3.019%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:54.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:54.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:54.355 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:54.398 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:54.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:54.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:54.504 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:54.536 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:54.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:54.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:54.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:37:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:37:54] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.43.0" 2023-10-09T05:37:54.641 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:54.677 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:54.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:54.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:54.784 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:54.811 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:54.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:54.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:54.917 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:54.952 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:54.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:54.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:54.999 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:54 smithi133 bash[105589]: cluster 2023-10-09T05:37:54.430269+0000 mgr.y (mgr.25277) 1501 : cluster 0 pgmap v1625: 99 pgs: 1 active+recovering+remapped, 1 creating+peering, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.4 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 679 op/s; 8176/267291 objects degraded (3.059%); 4319/267291 objects misplaced (1.616%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T05:37:54.999 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:54 smithi133 bash[102858]: cluster 2023-10-09T05:37:54.430269+0000 mgr.y (mgr.25277) 1501 : cluster 0 pgmap v1625: 99 pgs: 1 active+recovering+remapped, 1 creating+peering, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.4 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 679 op/s; 8176/267291 objects degraded (3.059%); 4319/267291 objects misplaced (1.616%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T05:37:55.055 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:55.066 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:54 smithi194 bash[92666]: cluster 2023-10-09T05:37:54.430269+0000 mgr.y (mgr.25277) 1501 : cluster 0 pgmap v1625: 99 pgs: 1 active+recovering+remapped, 1 creating+peering, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.4 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 679 op/s; 8176/267291 objects degraded (3.059%); 4319/267291 objects misplaced (1.616%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T05:37:55.084 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:55.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:55.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:55.188 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:55.218 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:55.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:55.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:55.322 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:55.352 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:55.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:55.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:55.457 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:55.485 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:55.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:55.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:55.589 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:55.609 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:55.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:55.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:55.713 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:55.734 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:55.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:55.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:55.837 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:55.860 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:55.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:55.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:55.963 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:55.990 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:55.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:55.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:56.097 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:56.123 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:56.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:56.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:56.229 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:56.238 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:55 smithi194 bash[92666]: cluster 2023-10-09T05:37:54.953208+0000 osd.5 (osd.5) 335 : cluster 0 14.9 scrub starts 2023-10-09T05:37:56.256 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:56.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:56.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:56.359 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:56.381 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:56.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:56.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:56.399 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:55 smithi133 bash[102858]: cluster 2023-10-09T05:37:54.953208+0000 osd.5 (osd.5) 335 : cluster 0 14.9 scrub starts 2023-10-09T05:37:56.399 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:55 smithi133 bash[105589]: cluster 2023-10-09T05:37:54.953208+0000 osd.5 (osd.5) 335 : cluster 0 14.9 scrub starts 2023-10-09T05:37:56.484 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:56.514 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:56.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:56.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:56.619 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:56.649 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:56.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:56.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:56.753 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:56.777 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:56.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:56.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:56.881 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:56.916 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:56.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:56.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:57.022 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:57.047 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:57.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:57.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:57.151 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:57.179 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:57.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:57.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:57.283 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:57.292 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:56 smithi133 bash[105589]: cluster 2023-10-09T05:37:56.431049+0000 mgr.y (mgr.25277) 1502 : cluster 0 pgmap v1626: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 882 op/s; 7742/261291 objects degraded (2.963%); 3942/261291 objects misplaced (1.509%); 8.4 MiB/s, 133 objects/s recovering 2023-10-09T05:37:57.292 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:56 smithi133 bash[102858]: cluster 2023-10-09T05:37:56.431049+0000 mgr.y (mgr.25277) 1502 : cluster 0 pgmap v1626: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 882 op/s; 7742/261291 objects degraded (2.963%); 3942/261291 objects misplaced (1.509%); 8.4 MiB/s, 133 objects/s recovering 2023-10-09T05:37:57.321 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:57.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:57.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:57.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:56 smithi194 bash[92666]: cluster 2023-10-09T05:37:56.431049+0000 mgr.y (mgr.25277) 1502 : cluster 0 pgmap v1626: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 882 op/s; 7742/261291 objects degraded (2.963%); 3942/261291 objects misplaced (1.509%); 8.4 MiB/s, 133 objects/s recovering 2023-10-09T05:37:57.428 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:57.455 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:57.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:57.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:57.562 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:57.586 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:57.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:57.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:57.693 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:57.719 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:57.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:57.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:57.826 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:57.856 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:57.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:57.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:57.962 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:57.991 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:57.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:57.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:58.094 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:58.130 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:58.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:58.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:58.234 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:58.276 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:58.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:58.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:58.299 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:58 smithi194 bash[92666]: cluster 2023-10-09T05:37:57.962112+0000 osd.5 (osd.5) 336 : cluster 0 14.7 scrub starts 2023-10-09T05:37:58.379 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:58.392 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:58 smithi133 bash[102858]: cluster 2023-10-09T05:37:57.962112+0000 osd.5 (osd.5) 336 : cluster 0 14.7 scrub starts 2023-10-09T05:37:58.393 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:58 smithi133 bash[105589]: cluster 2023-10-09T05:37:57.962112+0000 osd.5 (osd.5) 336 : cluster 0 14.7 scrub starts 2023-10-09T05:37:58.406 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:58.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:58.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:58.510 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:58.538 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:58.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:58.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:58.642 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:58.643 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:37:58.669 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:58.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:58.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:58.691 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:37:58.729 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:37:58.773 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:58.775 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:37:58.796 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:58.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:58.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:58.813 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:37:58.851 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:37:58.883 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:37:58.900 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:58.921 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:37:58.926 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:37:58.926 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:37:58.927 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:37:58.927 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:37:58.927 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:37:58.927 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:37:58.927 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:37:58.927 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:37:58.948 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:58.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:58.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:59.052 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:59.083 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:59.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:59.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:59.189 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:59.218 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:59.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:59.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:59.323 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:59.351 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:59.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:59.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:59.455 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:59.464 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:59 smithi133 bash[105589]: audit 2023-10-09T05:37:58.424042+0000 mon.c (mon.2) 138 : audit 0 from='client.? 172.21.15.133:0/3134572516' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:37:59.464 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:59 smithi133 bash[105589]: cluster 2023-10-09T05:37:58.431552+0000 mgr.y (mgr.25277) 1503 : cluster 0 pgmap v1627: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 681 op/s; 7742/260238 objects degraded (2.975%); 3942/260238 objects misplaced (1.515%); 7.9 MiB/s, 125 objects/s recovering 2023-10-09T05:37:59.465 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:59 smithi133 bash[105589]: audit 2023-10-09T05:37:58.490881+0000 mon.a (mon.0) 1891 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:37:59.465 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:37:59 smithi133 bash[105589]: cluster 2023-10-09T05:37:58.901543+0000 mon.a (mon.0) 1892 : cluster 3 Health check update: Degraded data redundancy: 7742/261291 objects degraded (2.963%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:59.465 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:59 smithi133 bash[102858]: audit 2023-10-09T05:37:58.424042+0000 mon.c (mon.2) 138 : audit 0 from='client.? 172.21.15.133:0/3134572516' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:37:59.465 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:59 smithi133 bash[102858]: cluster 2023-10-09T05:37:58.431552+0000 mgr.y (mgr.25277) 1503 : cluster 0 pgmap v1627: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 681 op/s; 7742/260238 objects degraded (2.975%); 3942/260238 objects misplaced (1.515%); 7.9 MiB/s, 125 objects/s recovering 2023-10-09T05:37:59.465 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:59 smithi133 bash[102858]: audit 2023-10-09T05:37:58.490881+0000 mon.a (mon.0) 1891 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:37:59.465 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:37:59 smithi133 bash[102858]: cluster 2023-10-09T05:37:58.901543+0000 mon.a (mon.0) 1892 : cluster 3 Health check update: Degraded data redundancy: 7742/261291 objects degraded (2.963%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:59.484 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:59.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:59.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:59.592 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:59.602 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:59 smithi194 bash[92666]: audit 2023-10-09T05:37:58.424042+0000 mon.c (mon.2) 138 : audit 0 from='client.? 172.21.15.133:0/3134572516' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:37:59.603 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:59 smithi194 bash[92666]: cluster 2023-10-09T05:37:58.431552+0000 mgr.y (mgr.25277) 1503 : cluster 0 pgmap v1627: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 681 op/s; 7742/260238 objects degraded (2.975%); 3942/260238 objects misplaced (1.515%); 7.9 MiB/s, 125 objects/s recovering 2023-10-09T05:37:59.603 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:59 smithi194 bash[92666]: audit 2023-10-09T05:37:58.490881+0000 mon.a (mon.0) 1891 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:37:59.603 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:37:59 smithi194 bash[92666]: cluster 2023-10-09T05:37:58.901543+0000 mon.a (mon.0) 1892 : cluster 3 Health check update: Degraded data redundancy: 7742/261291 objects degraded (2.963%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:59.617 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:59.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:59.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:59.721 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:59.758 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:59.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:59.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:59.864 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:59.881 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:59.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:37:59.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:59.984 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:00.019 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:00.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:00.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:00.122 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:00.143 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:00.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:00.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:00.247 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:00.273 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:00.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:00.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:00.376 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:00.399 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:00.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:00.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:00.504 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:00.530 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:00.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:00.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:00.635 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:00.671 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:00.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:00.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:00.774 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:00.804 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:00.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:00.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:00.909 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:00.936 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:00.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:00.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:01.043 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:01.076 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:01.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:01.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:01.181 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:01.207 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:01.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:01.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:01.313 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:01.340 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:01.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:01.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:01.443 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:01.473 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:01.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:01.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:01.577 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:01.601 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:01.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:01.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:01.708 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:01.719 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:01 smithi194 bash[92666]: cluster 2023-10-09T05:38:00.433951+0000 mgr.y (mgr.25277) 1504 : cluster 0 pgmap v1628: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.1 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 711 op/s; 7548/252027 objects degraded (2.995%); 3845/252027 objects misplaced (1.526%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T05:38:01.719 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:01 smithi194 bash[92666]: cluster 2023-10-09T05:38:00.995413+0000 osd.5 (osd.5) 337 : cluster 0 14.17 scrub starts 2023-10-09T05:38:01.735 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:01.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:01.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:01.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:01 smithi133 bash[102858]: cluster 2023-10-09T05:38:00.433951+0000 mgr.y (mgr.25277) 1504 : cluster 0 pgmap v1628: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.1 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 711 op/s; 7548/252027 objects degraded (2.995%); 3845/252027 objects misplaced (1.526%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T05:38:01.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:01 smithi133 bash[102858]: cluster 2023-10-09T05:38:00.995413+0000 osd.5 (osd.5) 337 : cluster 0 14.17 scrub starts 2023-10-09T05:38:01.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:01 smithi133 bash[105589]: cluster 2023-10-09T05:38:00.433951+0000 mgr.y (mgr.25277) 1504 : cluster 0 pgmap v1628: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.1 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 711 op/s; 7548/252027 objects degraded (2.995%); 3845/252027 objects misplaced (1.526%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T05:38:01.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:01 smithi133 bash[105589]: cluster 2023-10-09T05:38:00.995413+0000 osd.5 (osd.5) 337 : cluster 0 14.17 scrub starts 2023-10-09T05:38:01.841 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:01.874 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:01.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:01.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:01.980 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:02.009 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:02.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:02.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:02.117 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:02.143 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:02.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:02.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:02.246 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:02.274 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:02.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:02.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:02.380 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:02.398 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:02.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:02.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:02.500 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:02.533 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:02.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:02.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:02.641 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:02.673 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:02.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:02.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:02.780 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:02.810 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:02.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:02.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:02.916 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:02.942 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:02.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:02.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:03.045 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:03.073 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:03.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:03.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:03.179 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:03.215 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:03.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:03.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:03.318 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:03.344 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:03.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:03.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:03.449 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:03.478 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:03.481 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:38:03.481 INFO:teuthology.orchestra.run.smithi133.stdout: "ops": [ 2023-10-09T05:38:03.481 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.481 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35775.0:117939 14.18 14:1f4caed9:::benchmark_data_smithi133_267384_object18610:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e334)", 2023-10-09T05:38:03.482 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:38:03.462964+0000", 2023-10-09T05:38:03.482 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.013000872, 2023-10-09T05:38:03.482 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.01303608, 2023-10-09T05:38:03.482 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:38:03.482 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:38:03.483 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:38:03.483 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35775", 2023-10-09T05:38:03.483 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:38:03.483 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 117939 2023-10-09T05:38:03.483 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.484 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:38:03.484 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.484 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:38:03.484 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.462964+0000", 2023-10-09T05:38:03.484 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:38:03.484 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.485 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.485 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:38:03.485 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.462964+0000", 2023-10-09T05:38:03.485 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:38:03.485 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.486 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.486 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:38:03.486 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.462963+0000", 2023-10-09T05:38:03.486 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:38:03.486 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.486 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.486 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:38:03.487 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.462970+0000", 2023-10-09T05:38:03.487 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.7970000000000003e-06 2023-10-09T05:38:03.487 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.487 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.487 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:38:03.487 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.462979+0000", 2023-10-09T05:38:03.487 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.3940000000000002e-06 2023-10-09T05:38:03.487 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.487 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.489 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:38:03.489 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.462985+0000", 2023-10-09T05:38:03.489 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.6960000000000002e-06 2023-10-09T05:38:03.489 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.492 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.492 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:38:03.492 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.464352+0000", 2023-10-09T05:38:03.492 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.001367423 2023-10-09T05:38:03.492 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.492 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.492 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:38:03.492 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.464491+0000", 2023-10-09T05:38:03.492 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00013939599999999999 2023-10-09T05:38:03.492 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.493 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.493 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:38:03.493 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.464614+0000", 2023-10-09T05:38:03.493 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00012276100000000001 2023-10-09T05:38:03.493 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.493 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.493 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:03.493 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.467423+0000", 2023-10-09T05:38:03.493 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0028092909999999998 2023-10-09T05:38:03.493 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.493 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.494 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:38:03.494 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.468219+0000", 2023-10-09T05:38:03.494 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00079553700000000002 2023-10-09T05:38:03.494 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:38:03.494 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:38:03.494 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:38:03.494 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.494 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.494 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35775.0:117915 14.b 14:df2f708d:::benchmark_data_smithi133_267384_object18586:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e334)", 2023-10-09T05:38:03.494 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:38:03.432424+0000", 2023-10-09T05:38:03.494 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.043540662000000001, 2023-10-09T05:38:03.494 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.043685644000000003, 2023-10-09T05:38:03.495 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:38:03.495 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:38:03.495 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:38:03.495 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35775", 2023-10-09T05:38:03.495 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:38:03.496 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 117915 2023-10-09T05:38:03.496 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.496 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:38:03.496 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.497 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:38:03.497 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.432424+0000", 2023-10-09T05:38:03.497 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:38:03.497 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.497 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.497 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:38:03.497 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.432424+0000", 2023-10-09T05:38:03.497 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:38:03.497 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.497 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.497 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:38:03.497 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.432423+0000", 2023-10-09T05:38:03.498 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:38:03.498 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.498 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.498 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:38:03.498 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.432428+0000", 2023-10-09T05:38:03.498 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.8389999999999997e-06 2023-10-09T05:38:03.498 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.498 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.498 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:38:03.498 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.432438+0000", 2023-10-09T05:38:03.498 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.7720000000000006e-06 2023-10-09T05:38:03.499 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.499 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.499 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:38:03.499 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.432445+0000", 2023-10-09T05:38:03.499 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.5429999999999996e-06 2023-10-09T05:38:03.499 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.499 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.499 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:38:03.500 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.433239+0000", 2023-10-09T05:38:03.500 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00079410699999999995 2023-10-09T05:38:03.500 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.500 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.500 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:38:03.501 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.433367+0000", 2023-10-09T05:38:03.501 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00012777700000000001 2023-10-09T05:38:03.501 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.501 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.501 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:38:03.501 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.433482+0000", 2023-10-09T05:38:03.501 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000115058 2023-10-09T05:38:03.502 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.502 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.502 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:38:03.502 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.436581+0000", 2023-10-09T05:38:03.502 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0030986199999999998 2023-10-09T05:38:03.502 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.502 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.502 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:03.502 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.449420+0000", 2023-10-09T05:38:03.503 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.012839465 2023-10-09T05:38:03.503 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:38:03.503 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:38:03.503 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:38:03.503 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.504 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.504 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35775.0:117942 14.5 14:a527ccd4:::benchmark_data_smithi133_267384_object18613:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e334)", 2023-10-09T05:38:03.504 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:38:03.467906+0000", 2023-10-09T05:38:03.504 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.0080590460000000003, 2023-10-09T05:38:03.504 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0083061400000000001, 2023-10-09T05:38:03.504 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:38:03.505 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:38:03.505 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:38:03.505 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35775", 2023-10-09T05:38:03.505 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:38:03.505 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 117942 2023-10-09T05:38:03.506 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.506 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:38:03.506 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.506 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:38:03.506 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.467906+0000", 2023-10-09T05:38:03.506 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:38:03.506 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.507 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.507 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:38:03.507 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.467906+0000", 2023-10-09T05:38:03.507 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:38:03.507 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.507 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.507 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:38:03.507 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.467904+0000", 2023-10-09T05:38:03.507 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999981 2023-10-09T05:38:03.507 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.507 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.508 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:38:03.508 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.467910+0000", 2023-10-09T05:38:03.508 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.3860000000000004e-06 2023-10-09T05:38:03.508 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.508 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.508 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:38:03.508 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.467919+0000", 2023-10-09T05:38:03.508 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.055e-06 2023-10-09T05:38:03.508 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.508 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.508 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:38:03.508 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.467925+0000", 2023-10-09T05:38:03.509 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.674e-06 2023-10-09T05:38:03.509 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.509 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.509 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:38:03.509 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.467969+0000", 2023-10-09T05:38:03.509 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.4026999999999997e-05 2023-10-09T05:38:03.509 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.509 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.509 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:38:03.509 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.468054+0000", 2023-10-09T05:38:03.509 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.5125999999999998e-05 2023-10-09T05:38:03.510 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.510 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.510 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:38:03.510 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.468126+0000", 2023-10-09T05:38:03.510 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.2100000000000004e-05 2023-10-09T05:38:03.510 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.510 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.510 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:38:03.510 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.472183+0000", 2023-10-09T05:38:03.510 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0040564620000000003 2023-10-09T05:38:03.510 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.510 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.511 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:03.511 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.475993+0000", 2023-10-09T05:38:03.511 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0038096129999999999 2023-10-09T05:38:03.511 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:38:03.511 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:38:03.511 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:38:03.511 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.511 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.511 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35775.0:117943 14.5 14:a194b3a5:::benchmark_data_smithi133_267384_object18614:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e334)", 2023-10-09T05:38:03.511 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:38:03.469380+0000", 2023-10-09T05:38:03.511 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.0065846330000000003, 2023-10-09T05:38:03.511 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0069321829999999997, 2023-10-09T05:38:03.512 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:38:03.512 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:38:03.512 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:38:03.512 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35775", 2023-10-09T05:38:03.512 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:38:03.512 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 117943 2023-10-09T05:38:03.512 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.512 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:38:03.512 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.512 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:38:03.512 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.469380+0000", 2023-10-09T05:38:03.512 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:38:03.513 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.513 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.513 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:38:03.513 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.469380+0000", 2023-10-09T05:38:03.513 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:38:03.513 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.513 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.513 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:38:03.513 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.469379+0000", 2023-10-09T05:38:03.513 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:38:03.513 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.513 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.514 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:38:03.514 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.469385+0000", 2023-10-09T05:38:03.514 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.1909999999999999e-06 2023-10-09T05:38:03.514 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.514 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.514 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:38:03.514 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.469395+0000", 2023-10-09T05:38:03.514 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.0159e-05 2023-10-09T05:38:03.514 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.514 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.514 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:38:03.515 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.469402+0000", 2023-10-09T05:38:03.515 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.2880000000000004e-06 2023-10-09T05:38:03.515 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.515 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.515 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:38:03.515 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.469883+0000", 2023-10-09T05:38:03.515 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00048094000000000001 2023-10-09T05:38:03.515 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.515 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.515 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:38:03.515 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.470000+0000", 2023-10-09T05:38:03.515 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00011666 2023-10-09T05:38:03.516 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.516 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.516 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:38:03.516 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.470108+0000", 2023-10-09T05:38:03.516 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000108231 2023-10-09T05:38:03.516 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.516 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.516 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:38:03.516 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.472318+0000", 2023-10-09T05:38:03.516 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0022105430000000001 2023-10-09T05:38:03.516 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:38:03.516 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:38:03.517 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:38:03.517 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.517 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.517 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35775.0:117921 14.13 14:c9d7e757:::benchmark_data_smithi133_267384_object18592:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e334)", 2023-10-09T05:38:03.517 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:38:03.437137+0000", 2023-10-09T05:38:03.517 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.038828107000000001, 2023-10-09T05:38:03.517 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.039262964999999997, 2023-10-09T05:38:03.517 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:38:03.517 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "delayed", 2023-10-09T05:38:03.517 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:38:03.517 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35775", 2023-10-09T05:38:03.518 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:38:03.518 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 117921 2023-10-09T05:38:03.518 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.518 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:38:03.518 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.518 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:38:03.518 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.437137+0000", 2023-10-09T05:38:03.518 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:38:03.518 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.518 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.518 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:38:03.519 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.437137+0000", 2023-10-09T05:38:03.519 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:38:03.519 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.519 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.519 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:38:03.519 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.437135+0000", 2023-10-09T05:38:03.519 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:38:03.519 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.519 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.519 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:38:03.519 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.437141+0000", 2023-10-09T05:38:03.519 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.9279999999999998e-06 2023-10-09T05:38:03.520 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.520 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.520 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:38:03.520 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.437154+0000", 2023-10-09T05:38:03.520 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.2424e-05 2023-10-09T05:38:03.520 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.520 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.520 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:38:03.520 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.437161+0000", 2023-10-09T05:38:03.520 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.729e-06 2023-10-09T05:38:03.520 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.520 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.521 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:38:03.521 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.440599+0000", 2023-10-09T05:38:03.521 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0034377750000000001 2023-10-09T05:38:03.521 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.521 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.521 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for degraded object", 2023-10-09T05:38:03.521 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.446127+0000", 2023-10-09T05:38:03.521 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0055282100000000004 2023-10-09T05:38:03.521 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:38:03.521 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:38:03.521 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:38:03.521 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.522 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.522 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35775.0:117945 14.13 14:c9d6de17:::benchmark_data_smithi133_267384_object18616:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e334)", 2023-10-09T05:38:03.522 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:38:03.469698+0000", 2023-10-09T05:38:03.522 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.0062672939999999996, 2023-10-09T05:38:03.522 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0067756079999999998, 2023-10-09T05:38:03.522 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:38:03.522 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "reached pg", 2023-10-09T05:38:03.522 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:38:03.522 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35775", 2023-10-09T05:38:03.522 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:38:03.522 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 117945 2023-10-09T05:38:03.522 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.523 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:38:03.523 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.523 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:38:03.523 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.469698+0000", 2023-10-09T05:38:03.523 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:38:03.523 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.523 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.523 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:38:03.523 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.469698+0000", 2023-10-09T05:38:03.523 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:38:03.523 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.523 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.523 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:38:03.524 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.469697+0000", 2023-10-09T05:38:03.524 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999995 2023-10-09T05:38:03.524 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.524 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.524 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:38:03.524 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.469701+0000", 2023-10-09T05:38:03.524 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.2699999999999998e-06 2023-10-09T05:38:03.524 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.524 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.524 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:38:03.524 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.469711+0000", 2023-10-09T05:38:03.524 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.01e-05 2023-10-09T05:38:03.525 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.525 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.525 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:38:03.525 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.469718+0000", 2023-10-09T05:38:03.525 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.8839999999999997e-06 2023-10-09T05:38:03.525 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.525 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.525 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:38:03.525 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.472126+0000", 2023-10-09T05:38:03.525 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0024078390000000002 2023-10-09T05:38:03.525 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:38:03.525 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:38:03.526 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:38:03.526 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.526 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.526 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "MOSDPGPushReply(14.13 334/329 [PushReplyOp(14:c884d94b:::benchmark_data_smithi133_267384_object18607:head)])", 2023-10-09T05:38:03.526 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:38:03.472781+0000", 2023-10-09T05:38:03.526 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.0031838679999999999, 2023-10-09T05:38:03.526 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0037613080000000001, 2023-10-09T05:38:03.526 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:38:03.526 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "queued for pg", 2023-10-09T05:38:03.526 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:38:03.526 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.527 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:38:03.527 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.472781+0000", 2023-10-09T05:38:03.527 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:38:03.527 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.527 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.527 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:38:03.527 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.472781+0000", 2023-10-09T05:38:03.527 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:38:03.527 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.527 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.527 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:38:03.527 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.472780+0000", 2023-10-09T05:38:03.528 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:38:03.528 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.528 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.528 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:38:03.528 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.472789+0000", 2023-10-09T05:38:03.528 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.9609999999999994e-06 2023-10-09T05:38:03.528 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.528 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.528 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:38:03.528 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.472801+0000", 2023-10-09T05:38:03.528 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.1798e-05 2023-10-09T05:38:03.528 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.529 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.529 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:38:03.529 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.472808+0000", 2023-10-09T05:38:03.529 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.5189999999999998e-06 2023-10-09T05:38:03.529 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:38:03.529 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:38:03.529 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:38:03.529 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.529 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.529 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35775.0:117947 14.c 14:3f615baa:::benchmark_data_smithi133_267384_object18618:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e334)", 2023-10-09T05:38:03.529 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:38:03.475800+0000", 2023-10-09T05:38:03.529 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.00016522499999999999, 2023-10-09T05:38:03.530 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000796128, 2023-10-09T05:38:03.530 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:38:03.530 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:38:03.530 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:38:03.530 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35775", 2023-10-09T05:38:03.530 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:38:03.530 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 117947 2023-10-09T05:38:03.530 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.530 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:38:03.530 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.530 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:38:03.530 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.475800+0000", 2023-10-09T05:38:03.531 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:38:03.531 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.531 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.531 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:38:03.531 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.475800+0000", 2023-10-09T05:38:03.531 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:38:03.531 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.531 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.531 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:38:03.531 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.475798+0000", 2023-10-09T05:38:03.531 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999981 2023-10-09T05:38:03.531 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.532 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.532 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:38:03.532 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.475807+0000", 2023-10-09T05:38:03.532 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.4180000000000003e-06 2023-10-09T05:38:03.532 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.532 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.532 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:38:03.532 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.475828+0000", 2023-10-09T05:38:03.532 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.0329000000000001e-05 2023-10-09T05:38:03.532 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.532 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.532 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:38:03.533 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.475838+0000", 2023-10-09T05:38:03.533 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.0096e-05 2023-10-09T05:38:03.533 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.533 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.533 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:38:03.533 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.475895+0000", 2023-10-09T05:38:03.533 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.7433999999999999e-05 2023-10-09T05:38:03.533 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.533 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.533 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:38:03.533 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.476081+0000", 2023-10-09T05:38:03.533 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00018573299999999999 2023-10-09T05:38:03.534 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.534 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.534 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:38:03.534 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.476204+0000", 2023-10-09T05:38:03.534 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00012293699999999999 2023-10-09T05:38:03.534 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:38:03.534 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:38:03.534 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:38:03.534 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.534 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.534 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35775.0:117948 14.16 14:6dfd35b0:::benchmark_data_smithi133_267384_object18619:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e334)", 2023-10-09T05:38:03.534 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:38:03.475855+0000", 2023-10-09T05:38:03.535 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.000110159, 2023-10-09T05:38:03.535 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00081638999999999995, 2023-10-09T05:38:03.535 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:38:03.535 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:38:03.535 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:38:03.535 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35775", 2023-10-09T05:38:03.535 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:38:03.535 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 117948 2023-10-09T05:38:03.535 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.535 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:38:03.535 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.535 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:38:03.536 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.475855+0000", 2023-10-09T05:38:03.536 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:38:03.536 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.536 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.536 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:38:03.536 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.475855+0000", 2023-10-09T05:38:03.536 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:38:03.536 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.536 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.536 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:38:03.536 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.475854+0000", 2023-10-09T05:38:03.536 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999995 2023-10-09T05:38:03.536 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.537 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.537 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:38:03.537 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.475857+0000", 2023-10-09T05:38:03.537 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.9960000000000002e-06 2023-10-09T05:38:03.537 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.537 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.537 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:38:03.537 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.475864+0000", 2023-10-09T05:38:03.537 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.1650000000000003e-06 2023-10-09T05:38:03.537 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.537 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.537 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:38:03.538 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.475869+0000", 2023-10-09T05:38:03.538 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.0300000000000001e-06 2023-10-09T05:38:03.538 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.538 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.538 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:38:03.538 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.475998+0000", 2023-10-09T05:38:03.538 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00012916699999999999 2023-10-09T05:38:03.538 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.538 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.538 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:38:03.538 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.476112+0000", 2023-10-09T05:38:03.539 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000114103 2023-10-09T05:38:03.539 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.539 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.539 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:38:03.539 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.476210+0000", 2023-10-09T05:38:03.539 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.7657000000000004e-05 2023-10-09T05:38:03.539 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:38:03.539 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:38:03.539 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:38:03.539 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.539 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.539 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35775.0:117950 14.b 14.83aff7fb (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e334)", 2023-10-09T05:38:03.540 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:38:03.475880+0000", 2023-10-09T05:38:03.540 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 8.4596000000000001e-05, 2023-10-09T05:38:03.540 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00086530899999999998, 2023-10-09T05:38:03.540 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:38:03.540 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "queued for pg", 2023-10-09T05:38:03.540 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:38:03.540 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35775", 2023-10-09T05:38:03.540 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:38:03.540 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 117950 2023-10-09T05:38:03.540 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.540 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:38:03.541 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.541 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:38:03.541 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.475880+0000", 2023-10-09T05:38:03.541 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:38:03.541 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.541 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.541 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:38:03.541 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.475880+0000", 2023-10-09T05:38:03.541 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:38:03.541 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.541 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.542 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:38:03.542 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.475880+0000", 2023-10-09T05:38:03.542 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999995 2023-10-09T05:38:03.542 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.542 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.542 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:38:03.542 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.475882+0000", 2023-10-09T05:38:03.542 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.181e-06 2023-10-09T05:38:03.542 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.542 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.542 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:38:03.543 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.475888+0000", 2023-10-09T05:38:03.543 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.558e-06 2023-10-09T05:38:03.543 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.543 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.543 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:38:03.543 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.475893+0000", 2023-10-09T05:38:03.543 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.7099999999999998e-06 2023-10-09T05:38:03.543 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:38:03.543 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:38:03.543 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:38:03.543 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.543 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.544 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_repop_reply(client.35775.0:117943 14.5 e334/319)", 2023-10-09T05:38:03.544 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:38:03.475949+0000", 2023-10-09T05:38:03.544 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 1.5964000000000002e-05, 2023-10-09T05:38:03.544 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00084064299999999997, 2023-10-09T05:38:03.544 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:38:03.544 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "queued for pg", 2023-10-09T05:38:03.544 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:38:03.544 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.544 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:38:03.544 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.475949+0000", 2023-10-09T05:38:03.544 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:38:03.545 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.545 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.545 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:38:03.545 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.475949+0000", 2023-10-09T05:38:03.545 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:38:03.545 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.545 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.545 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:38:03.545 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.475949+0000", 2023-10-09T05:38:03.545 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967296 2023-10-09T05:38:03.545 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.546 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.546 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:38:03.546 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.475951+0000", 2023-10-09T05:38:03.546 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.0660000000000002e-06 2023-10-09T05:38:03.546 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.546 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.546 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:38:03.546 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.475956+0000", 2023-10-09T05:38:03.546 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.6890000000000003e-06 2023-10-09T05:38:03.546 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.546 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.546 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:38:03.547 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.475960+0000", 2023-10-09T05:38:03.547 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.1200000000000004e-06 2023-10-09T05:38:03.547 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:38:03.547 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:38:03.547 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:38:03.547 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.547 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.547 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35775.0:117936 14.13 14:c884d94b:::benchmark_data_smithi133_267384_object18607:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e334)", 2023-10-09T05:38:03.547 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:38:03.455692+0000", 2023-10-09T05:38:03.547 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 0.020273559, 2023-10-09T05:38:03.547 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.021144026999999999, 2023-10-09T05:38:03.548 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:38:03.548 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "delayed", 2023-10-09T05:38:03.548 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:38:03.548 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35775", 2023-10-09T05:38:03.548 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:38:03.548 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 117936 2023-10-09T05:38:03.548 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.548 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:38:03.548 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.548 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:38:03.548 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.455692+0000", 2023-10-09T05:38:03.548 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:38:03.549 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.549 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.549 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:38:03.549 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.455692+0000", 2023-10-09T05:38:03.549 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:38:03.549 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.549 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.549 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:38:03.549 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.455691+0000", 2023-10-09T05:38:03.549 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999995 2023-10-09T05:38:03.549 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.549 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.550 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:38:03.550 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.455693+0000", 2023-10-09T05:38:03.550 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.43e-06 2023-10-09T05:38:03.550 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.550 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.550 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:38:03.550 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.455702+0000", 2023-10-09T05:38:03.550 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.1219999999999995e-06 2023-10-09T05:38:03.550 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.550 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.550 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:38:03.550 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.455708+0000", 2023-10-09T05:38:03.551 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.1360000000000002e-06 2023-10-09T05:38:03.551 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.551 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.551 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:38:03.551 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.460276+0000", 2023-10-09T05:38:03.551 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0045684890000000002 2023-10-09T05:38:03.551 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:38:03.551 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:38:03.551 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for degraded object", 2023-10-09T05:38:03.551 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:38:03.468002+0000", 2023-10-09T05:38:03.552 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0077260139999999998 2023-10-09T05:38:03.552 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:38:03.552 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:38:03.552 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:38:03.552 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:38:03.552 INFO:teuthology.orchestra.run.smithi133.stdout: ], 2023-10-09T05:38:03.552 INFO:teuthology.orchestra.run.smithi133.stdout: "num_ops": 12 2023-10-09T05:38:03.552 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:38:03.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:03.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:03.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:03 smithi194 bash[92666]: cluster 2023-10-09T05:38:02.434540+0000 mgr.y (mgr.25277) 1505 : cluster 0 pgmap v1629: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.1 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 7354/249711 objects degraded (2.945%); 3557/249711 objects misplaced (1.424%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T05:38:03.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:03 smithi194 bash[92666]: cluster 2023-10-09T05:38:02.970348+0000 osd.5 (osd.5) 338 : cluster 0 14.19 scrub starts 2023-10-09T05:38:03.654 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:03.684 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:03.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:03.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:03.794 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:03.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:03 smithi133 bash[102858]: cluster 2023-10-09T05:38:02.434540+0000 mgr.y (mgr.25277) 1505 : cluster 0 pgmap v1629: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.1 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 7354/249711 objects degraded (2.945%); 3557/249711 objects misplaced (1.424%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T05:38:03.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:03 smithi133 bash[102858]: cluster 2023-10-09T05:38:02.970348+0000 osd.5 (osd.5) 338 : cluster 0 14.19 scrub starts 2023-10-09T05:38:03.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:03 smithi133 bash[105589]: cluster 2023-10-09T05:38:02.434540+0000 mgr.y (mgr.25277) 1505 : cluster 0 pgmap v1629: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.1 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 7354/249711 objects degraded (2.945%); 3557/249711 objects misplaced (1.424%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T05:38:03.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:03 smithi133 bash[105589]: cluster 2023-10-09T05:38:02.970348+0000 osd.5 (osd.5) 338 : cluster 0 14.19 scrub starts 2023-10-09T05:38:03.822 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:03.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:03.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:03.925 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:03.953 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:03.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:03.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:04.058 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:04.090 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:04.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:04.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:04.196 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:04.233 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:04.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:04.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:04.339 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:04.364 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:04.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:04.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:04.467 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:04.476 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:38:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:38:04] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.43.0" 2023-10-09T05:38:04.500 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:04.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:04.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:04.528 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is unset 2023-10-09T05:38:04.603 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:04.611 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:04 smithi194 bash[92666]: audit 2023-10-09T05:38:03.569669+0000 mon.c (mon.2) 139 : audit 1 from='client.? 172.21.15.133:0/586968263' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:38:04.612 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:04 smithi194 bash[92666]: audit 2023-10-09T05:38:03.570662+0000 mon.a (mon.0) 1893 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:38:04.612 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:04 smithi194 bash[92666]: cluster 2023-10-09T05:38:03.902973+0000 mon.a (mon.0) 1894 : cluster 3 Health check update: Degraded data redundancy: 7354/249711 objects degraded (2.945%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:38:04.627 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:04.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:04.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:04.736 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:04.745 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:04 smithi133 bash[102858]: audit 2023-10-09T05:38:03.569669+0000 mon.c (mon.2) 139 : audit 1 from='client.? 172.21.15.133:0/586968263' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:38:04.745 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:04 smithi133 bash[102858]: audit 2023-10-09T05:38:03.570662+0000 mon.a (mon.0) 1893 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:38:04.745 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:04 smithi133 bash[102858]: cluster 2023-10-09T05:38:03.902973+0000 mon.a (mon.0) 1894 : cluster 3 Health check update: Degraded data redundancy: 7354/249711 objects degraded (2.945%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:38:04.746 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:04 smithi133 bash[105589]: audit 2023-10-09T05:38:03.569669+0000 mon.c (mon.2) 139 : audit 1 from='client.? 172.21.15.133:0/586968263' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:38:04.746 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:04 smithi133 bash[105589]: audit 2023-10-09T05:38:03.570662+0000 mon.a (mon.0) 1893 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:38:04.746 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:04 smithi133 bash[105589]: cluster 2023-10-09T05:38:03.902973+0000 mon.a (mon.0) 1894 : cluster 3 Health check update: Degraded data redundancy: 7354/249711 objects degraded (2.945%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:38:04.761 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:04.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:04.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:04.869 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:04.892 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:04.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:04.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:04.996 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:05.017 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:38:05.018 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:05.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:05.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:05.121 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:05.147 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:05.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:05.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:05.250 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:05.296 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:05.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:05.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:05.402 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:05.431 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:05.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:05.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:05.536 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:05.571 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:05.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:05.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:05.674 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:05.691 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:05 smithi194 bash[92666]: cluster 2023-10-09T05:38:04.435088+0000 mgr.y (mgr.25277) 1506 : cluster 0 pgmap v1630: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.0 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 7354/245955 objects degraded (2.990%); 3557/245955 objects misplaced (1.446%); 5.1 MiB/s, 80 objects/s recovering 2023-10-09T05:38:05.691 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:05 smithi194 bash[92666]: cluster 2023-10-09T05:38:04.498602+0000 mon.a (mon.0) 1895 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:38:05.692 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:05 smithi194 bash[92666]: cluster 2023-10-09T05:38:04.513238+0000 osd.7 (osd.7) 289 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T05:38:05.692 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:05 smithi194 bash[92666]: audit 2023-10-09T05:38:04.514287+0000 mon.a (mon.0) 1896 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:38:05.692 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:05 smithi194 bash[92666]: cluster 2023-10-09T05:38:04.514616+0000 mon.a (mon.0) 1897 : cluster 0 osdmap e335: 8 total, 8 up, 7 in 2023-10-09T05:38:05.692 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:05 smithi194 bash[92666]: cluster 2023-10-09T05:38:04.518223+0000 osd.7 (osd.7) 290 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T05:38:05.694 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:05 smithi194 bash[92666]: cluster 2023-10-09T05:38:04.519843+0000 osd.6 (osd.6) 308 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T05:38:05.694 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:05 smithi194 bash[92666]: cluster 2023-10-09T05:38:04.521277+0000 osd.5 (osd.5) 339 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T05:38:05.694 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:05 smithi194 bash[92666]: cluster 2023-10-09T05:38:04.526991+0000 osd.6 (osd.6) 309 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T05:38:05.694 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:05 smithi194 bash[92666]: cluster 2023-10-09T05:38:04.528988+0000 osd.6 (osd.6) 310 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T05:38:05.696 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:05 smithi194 bash[92666]: cluster 2023-10-09T05:38:05.176397+0000 osd.4 (osd.4) 300 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T05:38:05.703 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:05.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:05.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:05.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:05 smithi133 bash[105589]: cluster 2023-10-09T05:38:04.435088+0000 mgr.y (mgr.25277) 1506 : cluster 0 pgmap v1630: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.0 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 7354/245955 objects degraded (2.990%); 3557/245955 objects misplaced (1.446%); 5.1 MiB/s, 80 objects/s recovering 2023-10-09T05:38:05.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:05 smithi133 bash[105589]: cluster 2023-10-09T05:38:04.498602+0000 mon.a (mon.0) 1895 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:38:05.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:05 smithi133 bash[105589]: cluster 2023-10-09T05:38:04.513238+0000 osd.7 (osd.7) 289 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T05:38:05.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:05 smithi133 bash[105589]: audit 2023-10-09T05:38:04.514287+0000 mon.a (mon.0) 1896 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:38:05.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:05 smithi133 bash[105589]: cluster 2023-10-09T05:38:04.514616+0000 mon.a (mon.0) 1897 : cluster 0 osdmap e335: 8 total, 8 up, 7 in 2023-10-09T05:38:05.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:05 smithi133 bash[105589]: cluster 2023-10-09T05:38:04.518223+0000 osd.7 (osd.7) 290 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T05:38:05.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:05 smithi133 bash[105589]: cluster 2023-10-09T05:38:04.519843+0000 osd.6 (osd.6) 308 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T05:38:05.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:05 smithi133 bash[105589]: cluster 2023-10-09T05:38:04.521277+0000 osd.5 (osd.5) 339 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T05:38:05.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:05 smithi133 bash[105589]: cluster 2023-10-09T05:38:04.526991+0000 osd.6 (osd.6) 309 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T05:38:05.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:05 smithi133 bash[105589]: cluster 2023-10-09T05:38:04.528988+0000 osd.6 (osd.6) 310 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T05:38:05.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:05 smithi133 bash[105589]: cluster 2023-10-09T05:38:05.176397+0000 osd.4 (osd.4) 300 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T05:38:05.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:05 smithi133 bash[102858]: cluster 2023-10-09T05:38:04.435088+0000 mgr.y (mgr.25277) 1506 : cluster 0 pgmap v1630: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.0 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 7354/245955 objects degraded (2.990%); 3557/245955 objects misplaced (1.446%); 5.1 MiB/s, 80 objects/s recovering 2023-10-09T05:38:05.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:05 smithi133 bash[102858]: cluster 2023-10-09T05:38:04.498602+0000 mon.a (mon.0) 1895 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:38:05.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:05 smithi133 bash[102858]: cluster 2023-10-09T05:38:04.513238+0000 osd.7 (osd.7) 289 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T05:38:05.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:05 smithi133 bash[102858]: audit 2023-10-09T05:38:04.514287+0000 mon.a (mon.0) 1896 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:38:05.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:05 smithi133 bash[102858]: cluster 2023-10-09T05:38:04.514616+0000 mon.a (mon.0) 1897 : cluster 0 osdmap e335: 8 total, 8 up, 7 in 2023-10-09T05:38:05.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:05 smithi133 bash[102858]: cluster 2023-10-09T05:38:04.518223+0000 osd.7 (osd.7) 290 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T05:38:05.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:05 smithi133 bash[102858]: cluster 2023-10-09T05:38:04.519843+0000 osd.6 (osd.6) 308 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T05:38:05.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:05 smithi133 bash[102858]: cluster 2023-10-09T05:38:04.521277+0000 osd.5 (osd.5) 339 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T05:38:05.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:05 smithi133 bash[102858]: cluster 2023-10-09T05:38:04.526991+0000 osd.6 (osd.6) 309 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T05:38:05.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:05 smithi133 bash[102858]: cluster 2023-10-09T05:38:04.528988+0000 osd.6 (osd.6) 310 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T05:38:05.802 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:05 smithi133 bash[102858]: cluster 2023-10-09T05:38:05.176397+0000 osd.4 (osd.4) 300 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T05:38:05.805 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:05.829 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:05.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:05.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:05.933 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:05.960 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:05.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:05.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:06.064 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:06.090 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:06.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:06.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:06.192 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:06.224 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:06.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:06.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:06.328 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:06.375 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:06.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:06.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:06.479 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:06.502 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:06.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:06.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:06.605 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:06.632 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:06.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:06.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:06.736 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:06.759 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:06.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:06.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:06.863 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:06.890 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:06.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:06.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:06.997 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:07.018 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:07.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:07.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:07.123 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:07.147 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:07.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:07.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:07.256 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:07.281 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:07.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:07.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:07.384 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:07.413 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:07.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:07.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:07.520 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:07.532 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:07 smithi194 bash[92666]: cluster 2023-10-09T05:38:06.435839+0000 mgr.y (mgr.25277) 1507 : cluster 0 pgmap v1632: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 718 op/s; 6848/239718 objects degraded (2.857%); 3132/239718 objects misplaced (1.307%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T05:38:07.568 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:07.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:07.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:07.675 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:07.703 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:07.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:07.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:07.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:07 smithi133 bash[105589]: cluster 2023-10-09T05:38:06.435839+0000 mgr.y (mgr.25277) 1507 : cluster 0 pgmap v1632: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 718 op/s; 6848/239718 objects degraded (2.857%); 3132/239718 objects misplaced (1.307%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T05:38:07.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:07 smithi133 bash[102858]: cluster 2023-10-09T05:38:06.435839+0000 mgr.y (mgr.25277) 1507 : cluster 0 pgmap v1632: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 718 op/s; 6848/239718 objects degraded (2.857%); 3132/239718 objects misplaced (1.307%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T05:38:07.810 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:07.834 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:07.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:07.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:07.944 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:07.988 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:07.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:07.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:08.094 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:08.121 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:08.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:08.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:08.215 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T05:38:08.225 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:08.252 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:08.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:08.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:08.356 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:08.409 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:08.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:08.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:08.517 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:08.544 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:08.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:08.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:08.649 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:08.692 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:08.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:08.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:08.797 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:08.823 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:08.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:08.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:08.926 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:08.952 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:08.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:08.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:09.056 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:09.082 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:09.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:09.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:09.186 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:09.212 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:09.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:09.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:09.317 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:09.345 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:09.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:09.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:09.454 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:09.479 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:09.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:09.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:09.533 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:09 smithi133 bash[105589]: cluster 2023-10-09T05:38:08.436392+0000 mgr.y (mgr.25277) 1508 : cluster 0 pgmap v1633: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 722 op/s; 6848/238545 objects degraded (2.871%); 3132/238545 objects misplaced (1.313%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T05:38:09.534 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:09 smithi133 bash[105589]: cluster 2023-10-09T05:38:08.904118+0000 mon.a (mon.0) 1898 : cluster 3 Health check update: Degraded data redundancy: 6848/238545 objects degraded (2.871%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:38:09.534 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:09 smithi133 bash[102858]: cluster 2023-10-09T05:38:08.436392+0000 mgr.y (mgr.25277) 1508 : cluster 0 pgmap v1633: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 722 op/s; 6848/238545 objects degraded (2.871%); 3132/238545 objects misplaced (1.313%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T05:38:09.534 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:09 smithi133 bash[102858]: cluster 2023-10-09T05:38:08.904118+0000 mon.a (mon.0) 1898 : cluster 3 Health check update: Degraded data redundancy: 6848/238545 objects degraded (2.871%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:38:09.585 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:09.609 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:09.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:09.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:09.715 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:09.750 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:09.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:09.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:09.854 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:09.864 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:09 smithi194 bash[92666]: cluster 2023-10-09T05:38:08.436392+0000 mgr.y (mgr.25277) 1508 : cluster 0 pgmap v1633: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 722 op/s; 6848/238545 objects degraded (2.871%); 3132/238545 objects misplaced (1.313%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T05:38:09.864 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:09 smithi194 bash[92666]: cluster 2023-10-09T05:38:08.904118+0000 mon.a (mon.0) 1898 : cluster 3 Health check update: Degraded data redundancy: 6848/238545 objects degraded (2.871%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:38:09.882 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:09.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:09.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:09.986 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:10.011 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:10.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:10.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:10.117 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:10.143 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:10.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:10.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:10.229 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:38:10.248 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:10.306 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:10.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:10.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:10.411 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:10.454 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:10.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:10.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:10.559 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:10.614 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:10.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:10.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:10.722 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:10.756 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:10.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:10.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:10.816 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:38:10.823 INFO:teuthology.orchestra.run.smithi133.stderr:dumped all 2023-10-09T05:38:10.861 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:10.890 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:10.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:10.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:10.996 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:11.022 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:11.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:11.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:11.127 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:11.161 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:11.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:11.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:11.267 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:11.293 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:11.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:11.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:11.400 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:11.438 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:11.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:11.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:11.541 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:11.543 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:11 smithi133 bash[105589]: cluster 2023-10-09T05:38:10.437136+0000 mgr.y (mgr.25277) 1509 : cluster 0 pgmap v1634: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 762 op/s; 6590/229149 objects degraded (2.876%); 3003/229149 objects misplaced (1.311%); 6.7 MiB/s, 106 objects/s recovering 2023-10-09T05:38:11.568 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:11.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:11.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:11.674 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:11.708 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:11.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:11.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:11.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:11 smithi133 bash[102858]: cluster 2023-10-09T05:38:10.437136+0000 mgr.y (mgr.25277) 1509 : cluster 0 pgmap v1634: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 762 op/s; 6590/229149 objects degraded (2.876%); 3003/229149 objects misplaced (1.311%); 6.7 MiB/s, 106 objects/s recovering 2023-10-09T05:38:11.812 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:11.840 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:11.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:11.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:11.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:11 smithi194 bash[92666]: cluster 2023-10-09T05:38:10.437136+0000 mgr.y (mgr.25277) 1509 : cluster 0 pgmap v1634: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 762 op/s; 6590/229149 objects degraded (2.876%); 3003/229149 objects misplaced (1.311%); 6.7 MiB/s, 106 objects/s recovering 2023-10-09T05:38:11.944 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:11.975 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:11.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:11.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:12.079 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:12.115 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:12.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:12.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:12.219 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:12.248 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:12.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:12.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:12.353 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:12.382 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:12.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:12.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:12.486 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:12.510 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:12.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:12.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:12.613 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:12.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:12 smithi194 bash[92666]: audit 2023-10-09T05:38:10.792426+0000 mgr.y (mgr.25277) 1510 : audit 0 from='client.35835 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:38:12.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:12 smithi194 bash[92666]: cluster 2023-10-09T05:38:12.038415+0000 osd.5 (osd.5) 340 : cluster 0 14.9 scrub starts 2023-10-09T05:38:12.646 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:12.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:12.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:12.751 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:12.781 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:12.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:12.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:12.812 INFO:teuthology.orchestra.run.smithi133.stdout:{"pg_ready":true,"pg_map":{"version":1634,"stamp":"2023-10-09T05:38:10.436581+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5004793784,"num_objects":76383,"num_object_clones":0,"num_object_copies":229149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6590,"num_objects_misplaced":3003,"num_objects_unfound":0,"num_objects_dirty":76383,"num_whiteouts":0,"num_read":510,"num_read_kb":935,"num_write":122595,"num_write_kb":6359310,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4486,"num_bytes_recovered":292971956,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156903,"ondisk_log_size":156903,"up":297,"acting":295,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":305,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":23746952,"kb_used_data":15577296,"kb_used_omap":20,"kb_used_meta":8169259,"kb_avail":632469112,"statfs":{"total":671965249536,"available":647648370688,"internally_reserved":0,"allocated":15951151104,"data_stored":15940741898,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":8365321870},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,5,7,3,9,5],"upper_bound":128},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-499777536,"num_objects":-7626,"num_object_clones":0,"num_object_copies":-22878,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-958,"num_objects_misplaced":-842,"num_objects_unfound":0,"num_objects_dirty":-7626,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7626,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1068,"num_bytes_recovered":69992448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003172"},"pg_stats":[{"pgid":"14.17","version":"335'5660","reported_seq":5713,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T05:38:09.311392+0000","last_change":"2023-10-09T05:38:00.995969+0000","last_active":"2023-10-09T05:38:09.311392+0000","last_peered":"2023-10-09T05:38:09.311392+0000","last_clean":"2023-10-09T05:38:09.311392+0000","last_became_active":"2023-10-09T05:36:53.583019+0000","last_became_peered":"2023-10-09T05:36:53.583019+0000","last_unstale":"2023-10-09T05:38:09.311392+0000","last_undegraded":"2023-10-09T05:38:09.311392+0000","last_fullsized":"2023-10-09T05:38:09.311392+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":5660,"log_dups_size":0,"ondisk_log_size":5660,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155648000,"num_objects":2375,"num_object_clones":0,"num_object_copies":7125,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2375,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3803,"num_write_kb":197696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'2","reported_seq":205,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.917889+0000","last_change":"2023-10-09T05:30:03.378323+0000","last_active":"2023-10-09T05:37:50.917889+0000","last_peered":"2023-10-09T05:37:50.917889+0000","last_clean":"2023-10-09T05:37:50.917889+0000","last_became_active":"2023-10-09T05:30:03.377871+0000","last_became_peered":"2023-10-09T05:30:03.377871+0000","last_unstale":"2023-10-09T05:37:50.917889+0000","last_undegraded":"2023-10-09T05:37:50.917889+0000","last_fullsized":"2023-10-09T05:37:50.917889+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:09.181202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":331,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.940519+0000","last_change":"2023-10-09T05:30:03.377792+0000","last_active":"2023-10-09T05:37:50.940519+0000","last_peered":"2023-10-09T05:37:50.940519+0000","last_clean":"2023-10-09T05:37:50.940519+0000","last_became_active":"2023-10-09T05:30:03.377456+0000","last_became_peered":"2023-10-09T05:30:03.377456+0000","last_unstale":"2023-10-09T05:37:50.940519+0000","last_undegraded":"2023-10-09T05:37:50.940519+0000","last_fullsized":"2023-10-09T05:37:50.940519+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:35:57.941046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.14","version":"335'5638","reported_seq":5686,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T05:38:09.325422+0000","last_change":"2023-10-09T05:37:46.236033+0000","last_active":"2023-10-09T05:38:09.325422+0000","last_peered":"2023-10-09T05:38:09.325422+0000","last_clean":"2023-10-09T05:38:09.325422+0000","last_became_active":"2023-10-09T05:36:53.580996+0000","last_became_peered":"2023-10-09T05:36:53.580996+0000","last_unstale":"2023-10-09T05:38:09.325422+0000","last_undegraded":"2023-10-09T05:38:09.325422+0000","last_fullsized":"2023-10-09T05:38:09.325422+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":5638,"log_dups_size":0,"ondisk_log_size":5638,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157483008,"num_objects":2403,"num_object_clones":0,"num_object_copies":7209,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2403,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3791,"num_write_kb":198208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":214,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.922289+0000","last_change":"2023-10-09T05:30:03.377059+0000","last_active":"2023-10-09T05:37:50.922289+0000","last_peered":"2023-10-09T05:37:50.922289+0000","last_clean":"2023-10-09T05:37:50.922289+0000","last_became_active":"2023-10-09T05:30:03.376460+0000","last_became_peered":"2023-10-09T05:30:03.376460+0000","last_unstale":"2023-10-09T05:37:50.922289+0000","last_undegraded":"2023-10-09T05:37:50.922289+0000","last_fullsized":"2023-10-09T05:37:50.922289+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:01:08.558120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":340,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.927075+0000","last_change":"2023-10-09T05:18:26.422923+0000","last_active":"2023-10-09T05:37:50.927075+0000","last_peered":"2023-10-09T05:37:50.927075+0000","last_clean":"2023-10-09T05:37:50.927075+0000","last_became_active":"2023-10-09T05:18:26.422508+0000","last_became_peered":"2023-10-09T05:18:26.422508+0000","last_unstale":"2023-10-09T05:37:50.927075+0000","last_undegraded":"2023-10-09T05:37:50.927075+0000","last_fullsized":"2023-10-09T05:37:50.927075+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:00.876037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.15","version":"335'5706","reported_seq":5749,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T05:38:10.034276+0000","last_change":"2023-10-09T05:36:53.627741+0000","last_active":"2023-10-09T05:38:10.034276+0000","last_peered":"2023-10-09T05:38:10.034276+0000","last_clean":"2023-10-09T05:38:10.034276+0000","last_became_active":"2023-10-09T05:36:53.619647+0000","last_became_peered":"2023-10-09T05:36:53.619647+0000","last_unstale":"2023-10-09T05:38:10.034276+0000","last_undegraded":"2023-10-09T05:38:10.034276+0000","last_fullsized":"2023-10-09T05:38:10.034276+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":5706,"log_dups_size":0,"ondisk_log_size":5706,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156336128,"num_objects":2386,"num_object_clones":0,"num_object_copies":7158,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2386,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3828,"num_write_kb":198816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":358,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.939853+0000","last_change":"2023-10-09T05:23:18.188378+0000","last_active":"2023-10-09T05:37:50.939853+0000","last_peered":"2023-10-09T05:37:50.939853+0000","last_clean":"2023-10-09T05:37:50.939853+0000","last_became_active":"2023-10-09T05:23:18.186629+0000","last_became_peered":"2023-10-09T05:23:18.186629+0000","last_unstale":"2023-10-09T05:37:50.939853+0000","last_undegraded":"2023-10-09T05:37:50.939853+0000","last_fullsized":"2023-10-09T05:37:50.939853+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:16.444225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":335,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.919514+0000","last_change":"2023-10-09T05:32:45.829908+0000","last_active":"2023-10-09T05:37:50.919514+0000","last_peered":"2023-10-09T05:37:50.919514+0000","last_clean":"2023-10-09T05:37:50.919514+0000","last_became_active":"2023-10-09T05:32:45.829566+0000","last_became_peered":"2023-10-09T05:32:45.829566+0000","last_unstale":"2023-10-09T05:37:50.919514+0000","last_undegraded":"2023-10-09T05:37:50.919514+0000","last_fullsized":"2023-10-09T05:37:50.919514+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:05.705818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":336,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:38:00.205483+0000","last_change":"2023-10-09T05:32:45.835245+0000","last_active":"2023-10-09T05:38:00.205483+0000","last_peered":"2023-10-09T05:38:00.205483+0000","last_clean":"2023-10-09T05:38:00.205483+0000","last_became_active":"2023-10-09T05:32:45.830180+0000","last_became_peered":"2023-10-09T05:32:45.830180+0000","last_unstale":"2023-10-09T05:38:00.205483+0000","last_undegraded":"2023-10-09T05:38:00.205483+0000","last_fullsized":"2023-10-09T05:38:00.205483+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:07:06.351159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":314,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.939887+0000","last_change":"2023-10-09T05:09:57.849695+0000","last_active":"2023-10-09T05:37:50.939887+0000","last_peered":"2023-10-09T05:37:50.939887+0000","last_clean":"2023-10-09T05:37:50.939887+0000","last_became_active":"2023-10-09T05:09:57.848478+0000","last_became_peered":"2023-10-09T05:09:57.848478+0000","last_unstale":"2023-10-09T05:37:50.939887+0000","last_undegraded":"2023-10-09T05:37:50.939887+0000","last_fullsized":"2023-10-09T05:37:50.939887+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:26.115312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":44,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.926411+0000","last_change":"2023-10-09T05:32:45.830061+0000","last_active":"2023-10-09T05:37:50.926411+0000","last_peered":"2023-10-09T05:37:50.926411+0000","last_clean":"2023-10-09T05:37:50.926411+0000","last_became_active":"2023-10-09T05:32:45.829723+0000","last_became_peered":"2023-10-09T05:32:45.829723+0000","last_unstale":"2023-10-09T05:37:50.926411+0000","last_undegraded":"2023-10-09T05:37:50.926411+0000","last_fullsized":"2023-10-09T05:37:50.926411+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:21.429740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":327,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.940836+0000","last_change":"2023-10-09T05:18:26.418431+0000","last_active":"2023-10-09T05:37:50.940836+0000","last_peered":"2023-10-09T05:37:50.940836+0000","last_clean":"2023-10-09T05:37:50.940836+0000","last_became_active":"2023-10-09T05:18:26.418100+0000","last_became_peered":"2023-10-09T05:18:26.418100+0000","last_unstale":"2023-10-09T05:37:50.940836+0000","last_undegraded":"2023-10-09T05:37:50.940836+0000","last_fullsized":"2023-10-09T05:37:50.940836+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:21.241286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.18","version":"335'5892","reported_seq":5935,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T05:38:10.181835+0000","last_change":"2023-10-09T05:36:53.612884+0000","last_active":"2023-10-09T05:38:10.181835+0000","last_peered":"2023-10-09T05:38:10.181835+0000","last_clean":"2023-10-09T05:38:10.181835+0000","last_became_active":"2023-10-09T05:36:53.610074+0000","last_became_peered":"2023-10-09T05:36:53.610074+0000","last_unstale":"2023-10-09T05:38:10.181835+0000","last_undegraded":"2023-10-09T05:38:10.181835+0000","last_fullsized":"2023-10-09T05:38:10.181835+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":5892,"log_dups_size":0,"ondisk_log_size":5892,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159973376,"num_objects":2441,"num_object_clones":0,"num_object_copies":7323,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2441,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3933,"num_write_kb":203968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":146,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.923502+0000","last_change":"2023-10-09T05:30:03.413915+0000","last_active":"2023-10-09T05:37:50.923502+0000","last_peered":"2023-10-09T05:37:50.923502+0000","last_clean":"2023-10-09T05:37:50.923502+0000","last_became_active":"2023-10-09T05:30:03.391102+0000","last_became_peered":"2023-10-09T05:30:03.391102+0000","last_unstale":"2023-10-09T05:37:50.923502+0000","last_undegraded":"2023-10-09T05:37:50.923502+0000","last_fullsized":"2023-10-09T05:37:50.923502+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:13.024050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":99,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.917743+0000","last_change":"2023-10-09T05:23:18.199181+0000","last_active":"2023-10-09T05:37:50.917743+0000","last_peered":"2023-10-09T05:37:50.917743+0000","last_clean":"2023-10-09T05:37:50.917743+0000","last_became_active":"2023-10-09T05:23:18.198401+0000","last_became_peered":"2023-10-09T05:23:18.198401+0000","last_unstale":"2023-10-09T05:37:50.917743+0000","last_undegraded":"2023-10-09T05:37:50.917743+0000","last_fullsized":"2023-10-09T05:37:50.917743+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:55:49.891420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.19","version":"335'5658","reported_seq":5715,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T05:38:09.258413+0000","last_change":"2023-10-09T05:38:02.970850+0000","last_active":"2023-10-09T05:38:09.258413+0000","last_peered":"2023-10-09T05:38:09.258413+0000","last_clean":"2023-10-09T05:38:09.258413+0000","last_became_active":"2023-10-09T05:36:53.604546+0000","last_became_peered":"2023-10-09T05:36:53.604546+0000","last_unstale":"2023-10-09T05:38:09.258413+0000","last_undegraded":"2023-10-09T05:38:09.258413+0000","last_fullsized":"2023-10-09T05:38:09.258413+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":5658,"log_dups_size":0,"ondisk_log_size":5658,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152043520,"num_objects":2320,"num_object_clones":0,"num_object_copies":6960,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2320,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3796,"num_write_kb":195712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"83'23","reported_seq":324,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.923141+0000","last_change":"2023-10-09T05:30:03.423577+0000","last_active":"2023-10-09T05:37:50.923141+0000","last_peered":"2023-10-09T05:37:50.923141+0000","last_clean":"2023-10-09T05:37:50.923141+0000","last_became_active":"2023-10-09T05:30:03.394939+0000","last_became_peered":"2023-10-09T05:30:03.394939+0000","last_unstale":"2023-10-09T05:37:50.923141+0000","last_undegraded":"2023-10-09T05:37:50.923141+0000","last_fullsized":"2023-10-09T05:37:50.923141+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:19:34.968600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":193,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.920221+0000","last_change":"2023-10-09T05:32:45.831400+0000","last_active":"2023-10-09T05:37:50.920221+0000","last_peered":"2023-10-09T05:37:50.920221+0000","last_clean":"2023-10-09T05:37:50.920221+0000","last_became_active":"2023-10-09T05:32:45.831007+0000","last_became_peered":"2023-10-09T05:32:45.831007+0000","last_unstale":"2023-10-09T05:37:50.920221+0000","last_undegraded":"2023-10-09T05:37:50.920221+0000","last_fullsized":"2023-10-09T05:37:50.920221+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:00:20.174754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":146,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.922845+0000","last_change":"2023-10-09T05:30:03.407302+0000","last_active":"2023-10-09T05:37:50.922845+0000","last_peered":"2023-10-09T05:37:50.922845+0000","last_clean":"2023-10-09T05:37:50.922845+0000","last_became_active":"2023-10-09T05:30:03.374209+0000","last_became_peered":"2023-10-09T05:30:03.374209+0000","last_unstale":"2023-10-09T05:37:50.922845+0000","last_undegraded":"2023-10-09T05:37:50.922845+0000","last_fullsized":"2023-10-09T05:37:50.922845+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:53.750483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":368,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.940262+0000","last_change":"2023-10-09T05:23:18.187593+0000","last_active":"2023-10-09T05:37:50.940262+0000","last_peered":"2023-10-09T05:37:50.940262+0000","last_clean":"2023-10-09T05:37:50.940262+0000","last_became_active":"2023-10-09T05:23:18.186445+0000","last_became_peered":"2023-10-09T05:23:18.186445+0000","last_unstale":"2023-10-09T05:37:50.940262+0000","last_undegraded":"2023-10-09T05:37:50.940262+0000","last_fullsized":"2023-10-09T05:37:50.940262+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:11.327635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":336,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.926788+0000","last_change":"2023-10-09T05:18:26.424399+0000","last_active":"2023-10-09T05:37:50.926788+0000","last_peered":"2023-10-09T05:37:50.926788+0000","last_clean":"2023-10-09T05:37:50.926788+0000","last_became_active":"2023-10-09T05:18:26.422122+0000","last_became_peered":"2023-10-09T05:18:26.422122+0000","last_unstale":"2023-10-09T05:37:50.926788+0000","last_undegraded":"2023-10-09T05:37:50.926788+0000","last_fullsized":"2023-10-09T05:37:50.926788+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:39.393756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":44,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.926404+0000","last_change":"2023-10-09T05:32:45.831080+0000","last_active":"2023-10-09T05:37:50.926404+0000","last_peered":"2023-10-09T05:37:50.926404+0000","last_clean":"2023-10-09T05:37:50.926404+0000","last_became_active":"2023-10-09T05:32:45.830745+0000","last_became_peered":"2023-10-09T05:32:45.830745+0000","last_unstale":"2023-10-09T05:37:50.926404+0000","last_undegraded":"2023-10-09T05:37:50.926404+0000","last_fullsized":"2023-10-09T05:37:50.926404+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:16.345644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":134,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.921161+0000","last_change":"2023-10-09T05:30:03.395964+0000","last_active":"2023-10-09T05:37:50.921161+0000","last_peered":"2023-10-09T05:37:50.921161+0000","last_clean":"2023-10-09T05:37:50.921161+0000","last_became_active":"2023-10-09T05:30:03.395634+0000","last_became_peered":"2023-10-09T05:30:03.395634+0000","last_unstale":"2023-10-09T05:37:50.921161+0000","last_undegraded":"2023-10-09T05:37:50.921161+0000","last_fullsized":"2023-10-09T05:37:50.921161+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:31.601758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":337,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:38:00.205455+0000","last_change":"2023-10-09T05:18:26.421678+0000","last_active":"2023-10-09T05:38:00.205455+0000","last_peered":"2023-10-09T05:38:00.205455+0000","last_clean":"2023-10-09T05:38:00.205455+0000","last_became_active":"2023-10-09T05:18:26.419673+0000","last_became_peered":"2023-10-09T05:18:26.419673+0000","last_unstale":"2023-10-09T05:38:00.205455+0000","last_undegraded":"2023-10-09T05:38:00.205455+0000","last_fullsized":"2023-10-09T05:38:00.205455+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:38.876021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":348,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.919097+0000","last_change":"2023-10-09T05:32:45.844532+0000","last_active":"2023-10-09T05:37:50.919097+0000","last_peered":"2023-10-09T05:37:50.919097+0000","last_clean":"2023-10-09T05:37:50.919097+0000","last_became_active":"2023-10-09T05:32:45.831101+0000","last_became_peered":"2023-10-09T05:32:45.831101+0000","last_unstale":"2023-10-09T05:37:50.919097+0000","last_undegraded":"2023-10-09T05:37:50.919097+0000","last_fullsized":"2023-10-09T05:37:50.919097+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:19:37.907188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":29,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.927725+0000","last_change":"2023-10-09T05:36:16.901739+0000","last_active":"2023-10-09T05:37:50.927725+0000","last_peered":"2023-10-09T05:37:50.927725+0000","last_clean":"2023-10-09T05:37:50.927725+0000","last_became_active":"2023-10-09T05:36:16.901380+0000","last_became_peered":"2023-10-09T05:36:16.901380+0000","last_unstale":"2023-10-09T05:37:50.927725+0000","last_undegraded":"2023-10-09T05:37:50.927725+0000","last_fullsized":"2023-10-09T05:37:50.927725+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:26.454798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.2","version":"335'5728","reported_seq":5780,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T05:38:09.090131+0000","last_change":"2023-10-09T05:37:48.286156+0000","last_active":"2023-10-09T05:38:09.090131+0000","last_peered":"2023-10-09T05:38:09.090131+0000","last_clean":"2023-10-09T05:38:09.090131+0000","last_became_active":"2023-10-09T05:36:53.563696+0000","last_became_peered":"2023-10-09T05:36:53.563696+0000","last_unstale":"2023-10-09T05:38:09.090131+0000","last_undegraded":"2023-10-09T05:38:09.090131+0000","last_fullsized":"2023-10-09T05:38:09.090131+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":5728,"log_dups_size":0,"ondisk_log_size":5728,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156205056,"num_objects":2383,"num_object_clones":0,"num_object_copies":7149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2383,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3843,"num_write_kb":199264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":142,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.923027+0000","last_change":"2023-10-09T05:30:03.409982+0000","last_active":"2023-10-09T05:37:50.923027+0000","last_peered":"2023-10-09T05:37:50.923027+0000","last_clean":"2023-10-09T05:37:50.923027+0000","last_became_active":"2023-10-09T05:30:03.386761+0000","last_became_peered":"2023-10-09T05:30:03.386761+0000","last_unstale":"2023-10-09T05:37:50.923027+0000","last_undegraded":"2023-10-09T05:37:50.923027+0000","last_fullsized":"2023-10-09T05:37:50.923027+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:17.372036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":44,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.925742+0000","last_change":"2023-10-09T05:32:45.829063+0000","last_active":"2023-10-09T05:37:50.925742+0000","last_peered":"2023-10-09T05:37:50.925742+0000","last_clean":"2023-10-09T05:37:50.925742+0000","last_became_active":"2023-10-09T05:32:45.828694+0000","last_became_peered":"2023-10-09T05:32:45.828694+0000","last_unstale":"2023-10-09T05:37:50.925742+0000","last_undegraded":"2023-10-09T05:37:50.925742+0000","last_fullsized":"2023-10-09T05:37:50.925742+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:58:51.140748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"335'5583","reported_seq":5626,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T05:38:10.143323+0000","last_change":"2023-10-09T05:36:53.608829+0000","last_active":"2023-10-09T05:38:10.143323+0000","last_peered":"2023-10-09T05:38:10.143323+0000","last_clean":"2023-10-09T05:38:10.143323+0000","last_became_active":"2023-10-09T05:36:53.607150+0000","last_became_peered":"2023-10-09T05:36:53.607150+0000","last_unstale":"2023-10-09T05:38:10.143323+0000","last_undegraded":"2023-10-09T05:38:10.143323+0000","last_fullsized":"2023-10-09T05:38:10.143323+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":5583,"log_dups_size":0,"ondisk_log_size":5583,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152141824,"num_objects":2321,"num_object_clones":0,"num_object_copies":6963,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2321,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3759,"num_write_kb":194592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":142,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.922660+0000","last_change":"2023-10-09T05:30:03.416268+0000","last_active":"2023-10-09T05:37:50.922660+0000","last_peered":"2023-10-09T05:37:50.922660+0000","last_clean":"2023-10-09T05:37:50.922660+0000","last_became_active":"2023-10-09T05:30:03.392236+0000","last_became_peered":"2023-10-09T05:30:03.392236+0000","last_unstale":"2023-10-09T05:37:50.922660+0000","last_undegraded":"2023-10-09T05:37:50.922660+0000","last_fullsized":"2023-10-09T05:37:50.922660+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:18:34.616445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":202,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.922237+0000","last_change":"2023-10-09T05:30:03.376778+0000","last_active":"2023-10-09T05:37:50.922237+0000","last_peered":"2023-10-09T05:37:50.922237+0000","last_clean":"2023-10-09T05:37:50.922237+0000","last_became_active":"2023-10-09T05:30:03.376056+0000","last_became_peered":"2023-10-09T05:30:03.376056+0000","last_unstale":"2023-10-09T05:37:50.922237+0000","last_undegraded":"2023-10-09T05:37:50.922237+0000","last_fullsized":"2023-10-09T05:37:50.922237+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:06.957639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.0","version":"335'5602","reported_seq":5650,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T05:38:06.694175+0000","last_change":"2023-10-09T05:36:53.624480+0000","last_active":"2023-10-09T05:38:06.694175+0000","last_peered":"2023-10-09T05:38:06.694175+0000","last_clean":"2023-10-09T05:38:06.694175+0000","last_became_active":"2023-10-09T05:36:53.593718+0000","last_became_peered":"2023-10-09T05:36:53.593718+0000","last_unstale":"2023-10-09T05:38:06.694175+0000","last_undegraded":"2023-10-09T05:38:06.694175+0000","last_fullsized":"2023-10-09T05:38:06.694175+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":5602,"log_dups_size":0,"ondisk_log_size":5602,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160595968,"num_objects":2450,"num_object_clones":0,"num_object_copies":7350,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2450,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3732,"num_write_kb":197856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":142,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.923572+0000","last_change":"2023-10-09T05:32:45.838786+0000","last_active":"2023-10-09T05:37:50.923572+0000","last_peered":"2023-10-09T05:37:50.923572+0000","last_clean":"2023-10-09T05:37:50.923572+0000","last_became_active":"2023-10-09T05:32:45.830293+0000","last_became_peered":"2023-10-09T05:32:45.830293+0000","last_unstale":"2023-10-09T05:37:50.923572+0000","last_undegraded":"2023-10-09T05:37:50.923572+0000","last_fullsized":"2023-10-09T05:37:50.923572+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:47.319929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":365,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.919456+0000","last_change":"2023-10-09T05:18:26.423526+0000","last_active":"2023-10-09T05:37:50.919456+0000","last_peered":"2023-10-09T05:37:50.919456+0000","last_clean":"2023-10-09T05:37:50.919456+0000","last_became_active":"2023-10-09T05:18:26.423092+0000","last_became_peered":"2023-10-09T05:18:26.423092+0000","last_unstale":"2023-10-09T05:37:50.919456+0000","last_undegraded":"2023-10-09T05:37:50.919456+0000","last_fullsized":"2023-10-09T05:37:50.919456+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:54.877664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"335'5569","reported_seq":5617,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T05:38:05.724183+0000","last_change":"2023-10-09T05:37:37.046094+0000","last_active":"2023-10-09T05:38:05.724183+0000","last_peered":"2023-10-09T05:38:05.724183+0000","last_clean":"2023-10-09T05:38:05.724183+0000","last_became_active":"2023-10-09T05:36:53.559813+0000","last_became_peered":"2023-10-09T05:36:53.559813+0000","last_unstale":"2023-10-09T05:38:05.724183+0000","last_undegraded":"2023-10-09T05:38:05.724183+0000","last_fullsized":"2023-10-09T05:38:05.724183+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":5569,"log_dups_size":0,"ondisk_log_size":5569,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161415168,"num_objects":2463,"num_object_clones":0,"num_object_copies":7389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2463,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3727,"num_write_kb":198080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":44,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.926414+0000","last_change":"2023-10-09T05:32:45.829450+0000","last_active":"2023-10-09T05:37:50.926414+0000","last_peered":"2023-10-09T05:37:50.926414+0000","last_clean":"2023-10-09T05:37:50.926414+0000","last_became_active":"2023-10-09T05:32:45.829076+0000","last_became_peered":"2023-10-09T05:32:45.829076+0000","last_unstale":"2023-10-09T05:37:50.926414+0000","last_undegraded":"2023-10-09T05:37:50.926414+0000","last_fullsized":"2023-10-09T05:37:50.926414+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:51:13.538770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":329,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.927335+0000","last_change":"2023-10-09T05:30:03.387239+0000","last_active":"2023-10-09T05:37:50.927335+0000","last_peered":"2023-10-09T05:37:50.927335+0000","last_clean":"2023-10-09T05:37:50.927335+0000","last_became_active":"2023-10-09T05:30:03.386633+0000","last_became_peered":"2023-10-09T05:30:03.386633+0000","last_unstale":"2023-10-09T05:37:50.927335+0000","last_undegraded":"2023-10-09T05:37:50.927335+0000","last_fullsized":"2023-10-09T05:37:50.927335+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:51.537133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.6","version":"335'5952","reported_seq":5996,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T05:38:10.177895+0000","last_change":"2023-10-09T05:36:53.611524+0000","last_active":"2023-10-09T05:38:10.177895+0000","last_peered":"2023-10-09T05:38:10.177895+0000","last_clean":"2023-10-09T05:38:10.177895+0000","last_became_active":"2023-10-09T05:36:53.582927+0000","last_became_peered":"2023-10-09T05:36:53.582927+0000","last_unstale":"2023-10-09T05:38:10.177895+0000","last_undegraded":"2023-10-09T05:38:10.177895+0000","last_fullsized":"2023-10-09T05:38:10.177895+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":5952,"log_dups_size":0,"ondisk_log_size":5952,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157941760,"num_objects":2410,"num_object_clones":0,"num_object_copies":7230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2410,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3982,"num_write_kb":204544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'3","reported_seq":306,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.940265+0000","last_change":"2023-10-09T05:30:03.399748+0000","last_active":"2023-10-09T05:37:50.940265+0000","last_peered":"2023-10-09T05:37:50.940265+0000","last_clean":"2023-10-09T05:37:50.940265+0000","last_became_active":"2023-10-09T05:30:03.384427+0000","last_became_peered":"2023-10-09T05:30:03.384427+0000","last_unstale":"2023-10-09T05:37:50.940265+0000","last_undegraded":"2023-10-09T05:37:50.940265+0000","last_fullsized":"2023-10-09T05:37:50.940265+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:01.128892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":137,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.940346+0000","last_change":"2023-10-09T05:30:03.378783+0000","last_active":"2023-10-09T05:37:50.940346+0000","last_peered":"2023-10-09T05:37:50.940346+0000","last_clean":"2023-10-09T05:37:50.940346+0000","last_became_active":"2023-10-09T05:30:03.378436+0000","last_became_peered":"2023-10-09T05:30:03.378436+0000","last_unstale":"2023-10-09T05:37:50.940346+0000","last_undegraded":"2023-10-09T05:37:50.940346+0000","last_fullsized":"2023-10-09T05:37:50.940346+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:54.102857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.7","version":"335'5709","reported_seq":5763,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T05:38:09.291900+0000","last_change":"2023-10-09T05:37:57.962930+0000","last_active":"2023-10-09T05:38:09.291900+0000","last_peered":"2023-10-09T05:38:09.291900+0000","last_clean":"2023-10-09T05:38:09.291900+0000","last_became_active":"2023-10-09T05:36:53.559374+0000","last_became_peered":"2023-10-09T05:36:53.559374+0000","last_unstale":"2023-10-09T05:38:09.291900+0000","last_undegraded":"2023-10-09T05:38:09.291900+0000","last_fullsized":"2023-10-09T05:38:09.291900+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":5709,"log_dups_size":0,"ondisk_log_size":5709,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160169984,"num_objects":2444,"num_object_clones":0,"num_object_copies":7332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2444,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3852,"num_write_kb":201472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":346,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.919294+0000","last_change":"2023-10-09T05:30:03.381940+0000","last_active":"2023-10-09T05:37:50.919294+0000","last_peered":"2023-10-09T05:37:50.919294+0000","last_clean":"2023-10-09T05:37:50.919294+0000","last_became_active":"2023-10-09T05:30:03.381526+0000","last_became_peered":"2023-10-09T05:30:03.381526+0000","last_unstale":"2023-10-09T05:37:50.919294+0000","last_undegraded":"2023-10-09T05:37:50.919294+0000","last_fullsized":"2023-10-09T05:37:50.919294+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:06.078801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":305,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.920403+0000","last_change":"2023-10-09T05:30:03.387924+0000","last_active":"2023-10-09T05:37:50.920403+0000","last_peered":"2023-10-09T05:37:50.920403+0000","last_clean":"2023-10-09T05:37:50.920403+0000","last_became_active":"2023-10-09T05:30:03.387493+0000","last_became_peered":"2023-10-09T05:30:03.387493+0000","last_unstale":"2023-10-09T05:37:50.920403+0000","last_undegraded":"2023-10-09T05:37:50.920403+0000","last_fullsized":"2023-10-09T05:37:50.920403+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:16.554514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.4","version":"335'5608","reported_seq":5657,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T05:38:09.280176+0000","last_change":"2023-10-09T05:37:45.197054+0000","last_active":"2023-10-09T05:38:09.280176+0000","last_peered":"2023-10-09T05:38:09.280176+0000","last_clean":"2023-10-09T05:38:09.280176+0000","last_became_active":"2023-10-09T05:36:53.577754+0000","last_became_peered":"2023-10-09T05:36:53.577754+0000","last_unstale":"2023-10-09T05:38:09.280176+0000","last_undegraded":"2023-10-09T05:38:09.280176+0000","last_fullsized":"2023-10-09T05:38:09.280176+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":5608,"log_dups_size":0,"ondisk_log_size":5608,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157614080,"num_objects":2405,"num_object_clones":0,"num_object_copies":7215,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2405,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3761,"num_write_kb":197312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":274,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.919195+0000","last_change":"2023-10-09T05:30:03.372784+0000","last_active":"2023-10-09T05:37:50.919195+0000","last_peered":"2023-10-09T05:37:50.919195+0000","last_clean":"2023-10-09T05:37:50.919195+0000","last_became_active":"2023-10-09T05:30:03.372458+0000","last_became_peered":"2023-10-09T05:30:03.372458+0000","last_unstale":"2023-10-09T05:37:50.919195+0000","last_undegraded":"2023-10-09T05:37:50.919195+0000","last_fullsized":"2023-10-09T05:37:50.919195+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:36:24.881545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":304,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.919805+0000","last_change":"2023-10-09T05:30:03.382201+0000","last_active":"2023-10-09T05:37:50.919805+0000","last_peered":"2023-10-09T05:37:50.919805+0000","last_clean":"2023-10-09T05:37:50.919805+0000","last_became_active":"2023-10-09T05:30:03.381727+0000","last_became_peered":"2023-10-09T05:30:03.381727+0000","last_unstale":"2023-10-09T05:37:50.919805+0000","last_undegraded":"2023-10-09T05:37:50.919805+0000","last_fullsized":"2023-10-09T05:37:50.919805+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:14.559280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.5","version":"335'5755","reported_seq":5799,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T05:38:10.067428+0000","last_change":"2023-10-09T05:36:53.572718+0000","last_active":"2023-10-09T05:38:10.067428+0000","last_peered":"2023-10-09T05:38:10.067428+0000","last_clean":"2023-10-09T05:38:10.067428+0000","last_became_active":"2023-10-09T05:36:53.564481+0000","last_became_peered":"2023-10-09T05:36:53.564481+0000","last_unstale":"2023-10-09T05:38:10.067428+0000","last_undegraded":"2023-10-09T05:38:10.067428+0000","last_fullsized":"2023-10-09T05:38:10.067428+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":5755,"log_dups_size":0,"ondisk_log_size":5755,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152862720,"num_objects":2332,"num_object_clones":0,"num_object_copies":6996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3876,"num_write_kb":198688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":352,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.939757+0000","last_change":"2023-10-09T05:18:26.413106+0000","last_active":"2023-10-09T05:37:50.939757+0000","last_peered":"2023-10-09T05:37:50.939757+0000","last_clean":"2023-10-09T05:37:50.939757+0000","last_became_active":"2023-10-09T05:18:26.412334+0000","last_became_peered":"2023-10-09T05:18:26.412334+0000","last_unstale":"2023-10-09T05:37:50.939757+0000","last_undegraded":"2023-10-09T05:37:50.939757+0000","last_fullsized":"2023-10-09T05:37:50.939757+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:18.058016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":215,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.918555+0000","last_change":"2023-10-09T05:30:03.383456+0000","last_active":"2023-10-09T05:37:50.918555+0000","last_peered":"2023-10-09T05:37:50.918555+0000","last_clean":"2023-10-09T05:37:50.918555+0000","last_became_active":"2023-10-09T05:30:03.382656+0000","last_became_peered":"2023-10-09T05:30:03.382656+0000","last_unstale":"2023-10-09T05:37:50.918555+0000","last_undegraded":"2023-10-09T05:37:50.918555+0000","last_fullsized":"2023-10-09T05:37:50.918555+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:47:23.383409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.8","version":"335'5898","reported_seq":5941,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T05:38:10.115014+0000","last_change":"2023-10-09T05:36:53.597306+0000","last_active":"2023-10-09T05:38:10.115014+0000","last_peered":"2023-10-09T05:38:10.115014+0000","last_clean":"2023-10-09T05:38:10.115014+0000","last_became_active":"2023-10-09T05:36:53.593366+0000","last_became_peered":"2023-10-09T05:36:53.593366+0000","last_unstale":"2023-10-09T05:38:10.115014+0000","last_undegraded":"2023-10-09T05:38:10.115014+0000","last_fullsized":"2023-10-09T05:38:10.115014+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":5898,"log_dups_size":0,"ondisk_log_size":5898,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157876224,"num_objects":2409,"num_object_clones":0,"num_object_copies":7227,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2409,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3939,"num_write_kb":203136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":243,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.916403+0000","last_change":"2023-10-09T05:18:26.422150+0000","last_active":"2023-10-09T05:37:50.916403+0000","last_peered":"2023-10-09T05:37:50.916403+0000","last_clean":"2023-10-09T05:37:50.916403+0000","last_became_active":"2023-10-09T05:18:26.421853+0000","last_became_peered":"2023-10-09T05:18:26.421853+0000","last_unstale":"2023-10-09T05:37:50.916403+0000","last_undegraded":"2023-10-09T05:37:50.916403+0000","last_fullsized":"2023-10-09T05:37:50.916403+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:37:47.000556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":44,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.926552+0000","last_change":"2023-10-09T05:32:45.832271+0000","last_active":"2023-10-09T05:37:50.926552+0000","last_peered":"2023-10-09T05:37:50.926552+0000","last_clean":"2023-10-09T05:37:50.926552+0000","last_became_active":"2023-10-09T05:32:45.831997+0000","last_became_peered":"2023-10-09T05:32:45.831997+0000","last_unstale":"2023-10-09T05:37:50.926552+0000","last_undegraded":"2023-10-09T05:37:50.926552+0000","last_fullsized":"2023-10-09T05:37:50.926552+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:18.970416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":118,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:55.849366+0000","last_change":"2023-10-09T05:32:45.819022+0000","last_active":"2023-10-09T05:37:55.849366+0000","last_peered":"2023-10-09T05:37:55.849366+0000","last_clean":"2023-10-09T05:37:55.849366+0000","last_became_active":"2023-10-09T05:32:45.818726+0000","last_became_peered":"2023-10-09T05:32:45.818726+0000","last_unstale":"2023-10-09T05:37:55.849366+0000","last_undegraded":"2023-10-09T05:37:55.849366+0000","last_fullsized":"2023-10-09T05:37:55.849366+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:04.414263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.e","version":"335'7646","reported_seq":7680,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T05:38:06.744332+0000","last_change":"2023-10-09T05:35:46.303342+0000","last_active":"2023-10-09T05:38:06.744332+0000","last_peered":"2023-10-09T05:38:06.744332+0000","last_clean":"2023-10-09T05:38:06.744332+0000","last_became_active":"2023-10-09T05:35:46.302540+0000","last_became_peered":"2023-10-09T05:35:46.302540+0000","last_unstale":"2023-10-09T05:38:06.744332+0000","last_undegraded":"2023-10-09T05:38:06.744332+0000","last_fullsized":"2023-10-09T05:38:06.744332+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":7646,"log_dups_size":0,"ondisk_log_size":7646,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:00:27.626139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316801024,"num_objects":4834,"num_object_clones":0,"num_object_copies":14502,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4834,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7646,"num_write_kb":399360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":44,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.924176+0000","last_change":"2023-10-09T05:32:45.825342+0000","last_active":"2023-10-09T05:37:50.924176+0000","last_peered":"2023-10-09T05:37:50.924176+0000","last_clean":"2023-10-09T05:37:50.924176+0000","last_became_active":"2023-10-09T05:32:45.824981+0000","last_became_peered":"2023-10-09T05:32:45.824981+0000","last_unstale":"2023-10-09T05:37:50.924176+0000","last_undegraded":"2023-10-09T05:37:50.924176+0000","last_fullsized":"2023-10-09T05:37:50.924176+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:02.868005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":103,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.927663+0000","last_change":"2023-10-09T05:30:03.381036+0000","last_active":"2023-10-09T05:37:50.927663+0000","last_peered":"2023-10-09T05:37:50.927663+0000","last_clean":"2023-10-09T05:37:50.927663+0000","last_became_active":"2023-10-09T05:30:03.380698+0000","last_became_peered":"2023-10-09T05:30:03.380698+0000","last_unstale":"2023-10-09T05:37:50.927663+0000","last_undegraded":"2023-10-09T05:37:50.927663+0000","last_fullsized":"2023-10-09T05:37:50.927663+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:16.464471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":100,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:38:00.885031+0000","last_change":"2023-10-09T05:23:18.196260+0000","last_active":"2023-10-09T05:38:00.885031+0000","last_peered":"2023-10-09T05:38:00.885031+0000","last_clean":"2023-10-09T05:38:00.885031+0000","last_became_active":"2023-10-09T05:23:18.195907+0000","last_became_peered":"2023-10-09T05:23:18.195907+0000","last_unstale":"2023-10-09T05:38:00.885031+0000","last_undegraded":"2023-10-09T05:38:00.885031+0000","last_fullsized":"2023-10-09T05:38:00.885031+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:36.243432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.d","version":"335'7680","reported_seq":7713,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T05:38:09.356599+0000","last_change":"2023-10-09T05:35:46.303676+0000","last_active":"2023-10-09T05:38:09.356599+0000","last_peered":"2023-10-09T05:38:09.356599+0000","last_clean":"2023-10-09T05:38:09.356599+0000","last_became_active":"2023-10-09T05:35:46.303364+0000","last_became_peered":"2023-10-09T05:35:46.303364+0000","last_unstale":"2023-10-09T05:38:09.356599+0000","last_undegraded":"2023-10-09T05:38:09.356599+0000","last_fullsized":"2023-10-09T05:38:09.356599+0000","mapping_epoch":311,"log_start":"314'300","ondisk_log_start":"314'300","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":7380,"log_dups_size":0,"ondisk_log_size":7380,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:56.816946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308609048,"num_objects":4710,"num_object_clones":0,"num_object_copies":14130,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4710,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7680,"num_write_kb":396417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":117,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.919122+0000","last_change":"2023-10-09T05:20:18.123706+0000","last_active":"2023-10-09T05:37:50.919122+0000","last_peered":"2023-10-09T05:37:50.919122+0000","last_clean":"2023-10-09T05:37:50.919122+0000","last_became_active":"2023-10-09T05:20:18.122859+0000","last_became_peered":"2023-10-09T05:20:18.122859+0000","last_unstale":"2023-10-09T05:37:50.919122+0000","last_undegraded":"2023-10-09T05:37:50.919122+0000","last_fullsized":"2023-10-09T05:37:50.919122+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.906228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":207,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.916044+0000","last_change":"2023-10-09T05:09:57.854064+0000","last_active":"2023-10-09T05:37:50.916044+0000","last_peered":"2023-10-09T05:37:50.916044+0000","last_clean":"2023-10-09T05:37:50.916044+0000","last_became_active":"2023-10-09T05:09:57.847953+0000","last_became_peered":"2023-10-09T05:09:57.847953+0000","last_unstale":"2023-10-09T05:37:50.916044+0000","last_undegraded":"2023-10-09T05:37:50.916044+0000","last_fullsized":"2023-10-09T05:37:50.916044+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:45.914976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":90,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.919460+0000","last_change":"2023-10-09T05:32:45.832106+0000","last_active":"2023-10-09T05:37:50.919460+0000","last_peered":"2023-10-09T05:37:50.919460+0000","last_clean":"2023-10-09T05:37:50.919460+0000","last_became_active":"2023-10-09T05:32:45.831763+0000","last_became_peered":"2023-10-09T05:32:45.831763+0000","last_unstale":"2023-10-09T05:37:50.919460+0000","last_undegraded":"2023-10-09T05:37:50.919460+0000","last_fullsized":"2023-10-09T05:37:50.919460+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:06:37.005113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.f","version":"335'7686","reported_seq":7718,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T05:38:09.708531+0000","last_change":"2023-10-09T05:35:46.309035+0000","last_active":"2023-10-09T05:38:09.708531+0000","last_peered":"2023-10-09T05:38:09.708531+0000","last_clean":"2023-10-09T05:38:09.708531+0000","last_became_active":"2023-10-09T05:35:46.308730+0000","last_became_peered":"2023-10-09T05:35:46.308730+0000","last_unstale":"2023-10-09T05:38:09.708531+0000","last_undegraded":"2023-10-09T05:38:09.708531+0000","last_fullsized":"2023-10-09T05:38:09.708531+0000","mapping_epoch":311,"log_start":"314'100","ondisk_log_start":"314'100","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":7586,"log_dups_size":0,"ondisk_log_size":7586,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:08.994300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310902784,"num_objects":4744,"num_object_clones":0,"num_object_copies":14232,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4744,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7686,"num_write_kb":397760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":44,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.926260+0000","last_change":"2023-10-09T05:32:45.825075+0000","last_active":"2023-10-09T05:37:50.926260+0000","last_peered":"2023-10-09T05:37:50.926260+0000","last_clean":"2023-10-09T05:37:50.926260+0000","last_became_active":"2023-10-09T05:32:45.824730+0000","last_became_peered":"2023-10-09T05:32:45.824730+0000","last_unstale":"2023-10-09T05:37:50.926260+0000","last_undegraded":"2023-10-09T05:37:50.926260+0000","last_fullsized":"2023-10-09T05:37:50.926260+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:33.484326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":620,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.921701+0000","last_change":"2023-10-09T05:18:29.419527+0000","last_active":"2023-10-09T05:37:50.921701+0000","last_peered":"2023-10-09T05:37:50.921701+0000","last_clean":"2023-10-09T05:37:50.921701+0000","last_became_active":"2023-10-09T05:18:29.418852+0000","last_became_peered":"2023-10-09T05:18:29.418852+0000","last_unstale":"2023-10-09T05:37:50.921701+0000","last_undegraded":"2023-10-09T05:37:50.921701+0000","last_fullsized":"2023-10-09T05:37:50.921701+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_clean_scrub_stamp":"2023-10-09T04:36:44.261013+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:02.899170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":320,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.940369+0000","last_change":"2023-10-09T05:32:45.829840+0000","last_active":"2023-10-09T05:37:50.940369+0000","last_peered":"2023-10-09T05:37:50.940369+0000","last_clean":"2023-10-09T05:37:50.940369+0000","last_became_active":"2023-10-09T05:32:45.829084+0000","last_became_peered":"2023-10-09T05:32:45.829084+0000","last_unstale":"2023-10-09T05:37:50.940369+0000","last_undegraded":"2023-10-09T05:37:50.940369+0000","last_fullsized":"2023-10-09T05:37:50.940369+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:26:48.836506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":328,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.922197+0000","last_change":"2023-10-09T05:30:03.386395+0000","last_active":"2023-10-09T05:37:50.922197+0000","last_peered":"2023-10-09T05:37:50.922197+0000","last_clean":"2023-10-09T05:37:50.922197+0000","last_became_active":"2023-10-09T05:30:03.386024+0000","last_became_peered":"2023-10-09T05:30:03.386024+0000","last_unstale":"2023-10-09T05:37:50.922197+0000","last_undegraded":"2023-10-09T05:37:50.922197+0000","last_fullsized":"2023-10-09T05:37:50.922197+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:13.133829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"335'7671","reported_seq":7705,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T05:38:10.188398+0000","last_change":"2023-10-09T05:35:46.308831+0000","last_active":"2023-10-09T05:38:10.188398+0000","last_peered":"2023-10-09T05:38:10.188398+0000","last_clean":"2023-10-09T05:38:10.188398+0000","last_became_active":"2023-10-09T05:35:46.308458+0000","last_became_peered":"2023-10-09T05:35:46.308458+0000","last_unstale":"2023-10-09T05:38:10.188398+0000","last_undegraded":"2023-10-09T05:38:10.188398+0000","last_fullsized":"2023-10-09T05:38:10.188398+0000","mapping_epoch":311,"log_start":"316'1400","ondisk_log_start":"316'1400","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":6271,"log_dups_size":0,"ondisk_log_size":6271,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:30.298840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306118656,"num_objects":4671,"num_object_clones":0,"num_object_copies":14013,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4671,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7671,"num_write_kb":394944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":144,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.921557+0000","last_change":"2023-10-09T05:23:18.193825+0000","last_active":"2023-10-09T05:37:50.921557+0000","last_peered":"2023-10-09T05:37:50.921557+0000","last_clean":"2023-10-09T05:37:50.921557+0000","last_became_active":"2023-10-09T05:23:18.193394+0000","last_became_peered":"2023-10-09T05:23:18.193394+0000","last_unstale":"2023-10-09T05:37:50.921557+0000","last_undegraded":"2023-10-09T05:37:50.921557+0000","last_fullsized":"2023-10-09T05:37:50.921557+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:40.594469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":320,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.926456+0000","last_change":"2023-10-09T05:32:45.824031+0000","last_active":"2023-10-09T05:37:50.926456+0000","last_peered":"2023-10-09T05:37:50.926456+0000","last_clean":"2023-10-09T05:37:50.926456+0000","last_became_active":"2023-10-09T05:32:45.823651+0000","last_became_peered":"2023-10-09T05:32:45.823651+0000","last_unstale":"2023-10-09T05:37:50.926456+0000","last_undegraded":"2023-10-09T05:37:50.926456+0000","last_fullsized":"2023-10-09T05:37:50.926456+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:14:20.745117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":381,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T05:38:04.522348+0000","last_change":"2023-10-09T05:20:18.132750+0000","last_active":"2023-10-09T05:38:04.522348+0000","last_peered":"2023-10-09T05:38:04.522348+0000","last_clean":"2023-10-09T05:38:04.522348+0000","last_became_active":"2023-10-09T05:20:18.132396+0000","last_became_peered":"2023-10-09T05:20:18.132396+0000","last_unstale":"2023-10-09T05:38:04.522348+0000","last_undegraded":"2023-10-09T05:38:04.522348+0000","last_fullsized":"2023-10-09T05:38:04.522348+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:00:55.731045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.9","version":"335'5686","reported_seq":5740,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T05:38:09.316616+0000","last_change":"2023-10-09T05:37:54.955854+0000","last_active":"2023-10-09T05:38:09.316616+0000","last_peered":"2023-10-09T05:38:09.316616+0000","last_clean":"2023-10-09T05:38:09.316616+0000","last_became_active":"2023-10-09T05:36:53.569007+0000","last_became_peered":"2023-10-09T05:36:53.569007+0000","last_unstale":"2023-10-09T05:38:09.316616+0000","last_undegraded":"2023-10-09T05:38:09.316616+0000","last_fullsized":"2023-10-09T05:38:09.316616+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":5686,"log_dups_size":0,"ondisk_log_size":5686,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158728192,"num_objects":2422,"num_object_clones":0,"num_object_copies":7266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2422,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3824,"num_write_kb":199872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":134,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.923634+0000","last_change":"2023-10-09T05:30:03.391750+0000","last_active":"2023-10-09T05:37:50.923634+0000","last_peered":"2023-10-09T05:37:50.923634+0000","last_clean":"2023-10-09T05:37:50.923634+0000","last_became_active":"2023-10-09T05:30:03.391393+0000","last_became_peered":"2023-10-09T05:30:03.391393+0000","last_unstale":"2023-10-09T05:37:50.923634+0000","last_undegraded":"2023-10-09T05:37:50.923634+0000","last_fullsized":"2023-10-09T05:37:50.923634+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:07.263598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":243,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.916479+0000","last_change":"2023-10-09T05:32:45.824227+0000","last_active":"2023-10-09T05:37:50.916479+0000","last_peered":"2023-10-09T05:37:50.916479+0000","last_clean":"2023-10-09T05:37:50.916479+0000","last_became_active":"2023-10-09T05:32:45.823662+0000","last_became_peered":"2023-10-09T05:32:45.823662+0000","last_unstale":"2023-10-09T05:37:50.916479+0000","last_undegraded":"2023-10-09T05:37:50.916479+0000","last_fullsized":"2023-10-09T05:37:50.916479+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:54:56.029126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":351,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.927580+0000","last_change":"2023-10-09T05:32:45.826815+0000","last_active":"2023-10-09T05:37:50.927580+0000","last_peered":"2023-10-09T05:37:50.927580+0000","last_clean":"2023-10-09T05:37:50.927580+0000","last_became_active":"2023-10-09T05:32:45.826529+0000","last_became_peered":"2023-10-09T05:32:45.826529+0000","last_unstale":"2023-10-09T05:37:50.927580+0000","last_undegraded":"2023-10-09T05:37:50.927580+0000","last_fullsized":"2023-10-09T05:37:50.927580+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:19.868533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.a","version":"335'7642","reported_seq":7674,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T05:38:09.359687+0000","last_change":"2023-10-09T05:35:46.307929+0000","last_active":"2023-10-09T05:38:09.359687+0000","last_peered":"2023-10-09T05:38:09.359687+0000","last_clean":"2023-10-09T05:38:09.359687+0000","last_became_active":"2023-10-09T05:35:46.307610+0000","last_became_peered":"2023-10-09T05:35:46.307610+0000","last_unstale":"2023-10-09T05:38:09.359687+0000","last_undegraded":"2023-10-09T05:38:09.359687+0000","last_fullsized":"2023-10-09T05:38:09.359687+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":7642,"log_dups_size":0,"ondisk_log_size":7642,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:40.352492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314572800,"num_objects":4800,"num_object_clones":0,"num_object_copies":14400,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4800,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7642,"num_write_kb":398144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":345,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.940916+0000","last_change":"2023-10-09T05:18:26.431563+0000","last_active":"2023-10-09T05:37:50.940916+0000","last_peered":"2023-10-09T05:37:50.940916+0000","last_clean":"2023-10-09T05:37:50.940916+0000","last_became_active":"2023-10-09T05:18:26.421025+0000","last_became_peered":"2023-10-09T05:18:26.421025+0000","last_unstale":"2023-10-09T05:37:50.940916+0000","last_undegraded":"2023-10-09T05:37:50.940916+0000","last_fullsized":"2023-10-09T05:37:50.940916+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:30.333585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":108,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.918623+0000","last_change":"2023-10-09T05:30:03.375221+0000","last_active":"2023-10-09T05:37:50.918623+0000","last_peered":"2023-10-09T05:37:50.918623+0000","last_clean":"2023-10-09T05:37:50.918623+0000","last_became_active":"2023-10-09T05:30:03.374863+0000","last_became_peered":"2023-10-09T05:30:03.374863+0000","last_unstale":"2023-10-09T05:37:50.918623+0000","last_undegraded":"2023-10-09T05:37:50.918623+0000","last_fullsized":"2023-10-09T05:37:50.918623+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:16.058179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":344,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.939802+0000","last_change":"2023-10-09T05:21:08.179657+0000","last_active":"2023-10-09T05:37:50.939802+0000","last_peered":"2023-10-09T05:37:50.939802+0000","last_clean":"2023-10-09T05:37:50.939802+0000","last_became_active":"2023-10-09T05:21:08.178958+0000","last_became_peered":"2023-10-09T05:21:08.178958+0000","last_unstale":"2023-10-09T05:37:50.939802+0000","last_undegraded":"2023-10-09T05:37:50.939802+0000","last_fullsized":"2023-10-09T05:37:50.939802+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:20.815986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"335'7706","reported_seq":7740,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T05:38:10.182681+0000","last_change":"2023-10-09T05:35:46.308213+0000","last_active":"2023-10-09T05:38:10.182681+0000","last_peered":"2023-10-09T05:38:10.182681+0000","last_clean":"2023-10-09T05:38:10.182681+0000","last_became_active":"2023-10-09T05:35:46.307896+0000","last_became_peered":"2023-10-09T05:35:46.307896+0000","last_unstale":"2023-10-09T05:38:10.182681+0000","last_undegraded":"2023-10-09T05:38:10.182681+0000","last_fullsized":"2023-10-09T05:38:10.182681+0000","mapping_epoch":311,"log_start":"316'1400","ondisk_log_start":"316'1400","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":6306,"log_dups_size":0,"ondisk_log_size":6306,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:11.538673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304480256,"num_objects":4646,"num_object_clones":0,"num_object_copies":13938,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4646,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7706,"num_write_kb":395264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":340,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.919402+0000","last_change":"2023-10-09T05:18:26.424429+0000","last_active":"2023-10-09T05:37:50.919402+0000","last_peered":"2023-10-09T05:37:50.919402+0000","last_clean":"2023-10-09T05:37:50.919402+0000","last_became_active":"2023-10-09T05:18:26.423034+0000","last_became_peered":"2023-10-09T05:18:26.423034+0000","last_unstale":"2023-10-09T05:37:50.919402+0000","last_undegraded":"2023-10-09T05:37:50.919402+0000","last_fullsized":"2023-10-09T05:37:50.919402+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:16.251181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":57,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.919597+0000","last_change":"2023-10-09T05:31:16.444031+0000","last_active":"2023-10-09T05:37:50.919597+0000","last_peered":"2023-10-09T05:37:50.919597+0000","last_clean":"2023-10-09T05:37:50.919597+0000","last_became_active":"2023-10-09T05:31:16.412865+0000","last_became_peered":"2023-10-09T05:31:16.412865+0000","last_unstale":"2023-10-09T05:37:50.919597+0000","last_undegraded":"2023-10-09T05:37:50.919597+0000","last_fullsized":"2023-10-09T05:37:50.919597+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:04:06.520632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":44,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.923962+0000","last_change":"2023-10-09T05:32:45.828380+0000","last_active":"2023-10-09T05:37:50.923962+0000","last_peered":"2023-10-09T05:37:50.923962+0000","last_clean":"2023-10-09T05:37:50.923962+0000","last_became_active":"2023-10-09T05:32:45.827888+0000","last_became_peered":"2023-10-09T05:32:45.827888+0000","last_unstale":"2023-10-09T05:37:50.923962+0000","last_undegraded":"2023-10-09T05:37:50.923962+0000","last_fullsized":"2023-10-09T05:37:50.923962+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:48.145813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.16","version":"335'5988","reported_seq":6027,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T05:38:10.240996+0000","last_change":"2023-10-09T05:36:53.613658+0000","last_active":"2023-10-09T05:38:10.240996+0000","last_peered":"2023-10-09T05:38:10.240996+0000","last_clean":"2023-10-09T05:38:10.240996+0000","last_became_active":"2023-10-09T05:36:53.610769+0000","last_became_peered":"2023-10-09T05:36:53.610769+0000","last_unstale":"2023-10-09T05:38:10.240996+0000","last_undegraded":"2023-10-09T05:38:10.240996+0000","last_fullsized":"2023-10-09T05:38:10.240996+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":5988,"log_dups_size":0,"ondisk_log_size":5988,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158728192,"num_objects":2422,"num_object_clones":0,"num_object_copies":7266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2422,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4018,"num_write_kb":206080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'2","reported_seq":118,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.927498+0000","last_change":"2023-10-09T05:30:03.386435+0000","last_active":"2023-10-09T05:37:50.927498+0000","last_peered":"2023-10-09T05:37:50.927498+0000","last_clean":"2023-10-09T05:37:50.927498+0000","last_became_active":"2023-10-09T05:30:03.385418+0000","last_became_peered":"2023-10-09T05:30:03.385418+0000","last_unstale":"2023-10-09T05:37:50.927498+0000","last_undegraded":"2023-10-09T05:37:50.927498+0000","last_fullsized":"2023-10-09T05:37:50.927498+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:34.685368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":125,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:56.732574+0000","last_change":"2023-10-09T05:36:16.898207+0000","last_active":"2023-10-09T05:37:56.732574+0000","last_peered":"2023-10-09T05:37:56.732574+0000","last_clean":"2023-10-09T05:37:56.732574+0000","last_became_active":"2023-10-09T05:36:16.898009+0000","last_became_peered":"2023-10-09T05:36:16.898009+0000","last_unstale":"2023-10-09T05:37:56.732574+0000","last_undegraded":"2023-10-09T05:37:56.732574+0000","last_fullsized":"2023-10-09T05:37:56.732574+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:59.694977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.11","version":"335'5574","reported_seq":7525,"reported_epoch":335,"state":"active+recovering+remapped","last_fresh":"2023-10-09T05:38:05.882159+0000","last_change":"2023-10-09T05:37:49.186026+0000","last_active":"2023-10-09T05:38:05.882159+0000","last_peered":"2023-10-09T05:38:05.882159+0000","last_clean":"2023-10-09T05:37:07.363078+0000","last_became_active":"2023-10-09T05:37:08.972861+0000","last_became_peered":"2023-10-09T05:37:08.972861+0000","last_unstale":"2023-10-09T05:38:05.882159+0000","last_undegraded":"2023-10-09T05:38:05.882159+0000","last_fullsized":"2023-10-09T05:38:05.882159+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":5574,"log_dups_size":0,"ondisk_log_size":5574,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160825344,"num_objects":2454,"num_object_clones":0,"num_object_copies":7362,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1405,"num_objects_unfound":0,"num_objects_dirty":2454,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3732,"num_write_kb":197952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":951,"num_bytes_recovered":62324736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[1,2,3],"avail_no_missing":["1","2","3"],"object_location_counts":[{"shards":"1,2,3","objects":1049}],"blocked_by":[],"up_primary":6,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":48,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.926419+0000","last_change":"2023-10-09T05:32:45.847749+0000","last_active":"2023-10-09T05:37:50.926419+0000","last_peered":"2023-10-09T05:37:50.926419+0000","last_clean":"2023-10-09T05:37:50.926419+0000","last_became_active":"2023-10-09T05:32:45.825335+0000","last_became_peered":"2023-10-09T05:32:45.825335+0000","last_unstale":"2023-10-09T05:37:50.926419+0000","last_undegraded":"2023-10-09T05:37:50.926419+0000","last_fullsized":"2023-10-09T05:37:50.926419+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:42.082798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":323,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.940131+0000","last_change":"2023-10-09T05:30:03.380938+0000","last_active":"2023-10-09T05:37:50.940131+0000","last_peered":"2023-10-09T05:37:50.940131+0000","last_clean":"2023-10-09T05:37:50.940131+0000","last_became_active":"2023-10-09T05:30:03.380395+0000","last_became_peered":"2023-10-09T05:30:03.380395+0000","last_unstale":"2023-10-09T05:37:50.940131+0000","last_undegraded":"2023-10-09T05:37:50.940131+0000","last_fullsized":"2023-10-09T05:37:50.940131+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:23.087303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.10","version":"335'5524","reported_seq":11855,"reported_epoch":335,"state":"active+recovery_wait","last_fresh":"2023-10-09T05:38:05.821482+0000","last_change":"2023-10-09T05:37:50.939730+0000","last_active":"2023-10-09T05:38:05.821482+0000","last_peered":"2023-10-09T05:38:05.821482+0000","last_clean":"2023-10-09T05:36:58.540725+0000","last_became_active":"2023-10-09T05:37:50.922100+0000","last_became_peered":"2023-10-09T05:37:50.922100+0000","last_unstale":"2023-10-09T05:38:05.821482+0000","last_undegraded":"2023-10-09T05:38:05.821482+0000","last_fullsized":"2023-10-09T05:38:05.821482+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":5524,"log_dups_size":0,"ondisk_log_size":5524,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158760960,"num_objects":2423,"num_object_clones":0,"num_object_copies":7269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3655,"num_write_kb":194464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2092,"num_bytes_recovered":137035776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":["1","0","2","5","7"],"object_location_counts":[{"shards":"1,2,7","objects":2423}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":48,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.926544+0000","last_change":"2023-10-09T05:32:45.860675+0000","last_active":"2023-10-09T05:37:50.926544+0000","last_peered":"2023-10-09T05:37:50.926544+0000","last_clean":"2023-10-09T05:37:50.926544+0000","last_became_active":"2023-10-09T05:32:45.828515+0000","last_became_peered":"2023-10-09T05:32:45.828515+0000","last_unstale":"2023-10-09T05:37:50.926544+0000","last_undegraded":"2023-10-09T05:37:50.926544+0000","last_fullsized":"2023-10-09T05:37:50.926544+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:39.371415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":154,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.940295+0000","last_change":"2023-10-09T05:30:03.376908+0000","last_active":"2023-10-09T05:37:50.940295+0000","last_peered":"2023-10-09T05:37:50.940295+0000","last_clean":"2023-10-09T05:37:50.940295+0000","last_became_active":"2023-10-09T05:30:03.376456+0000","last_became_peered":"2023-10-09T05:30:03.376456+0000","last_unstale":"2023-10-09T05:37:50.940295+0000","last_undegraded":"2023-10-09T05:37:50.940295+0000","last_fullsized":"2023-10-09T05:37:50.940295+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:23.173899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.13","version":"335'5555","reported_seq":8486,"reported_epoch":335,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:38:10.242417+0000","last_change":"2023-10-09T05:37:13.510590+0000","last_active":"2023-10-09T05:38:10.242417+0000","last_peered":"2023-10-09T05:38:10.242417+0000","last_clean":"2023-10-09T05:37:10.784717+0000","last_became_active":"2023-10-09T05:37:13.505099+0000","last_became_peered":"2023-10-09T05:37:13.505099+0000","last_unstale":"2023-10-09T05:38:10.242417+0000","last_undegraded":"2023-10-09T05:37:13.016718+0000","last_fullsized":"2023-10-09T05:37:12.926732+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":5555,"log_dups_size":0,"ondisk_log_size":5555,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150175744,"num_objects":2292,"num_object_clones":0,"num_object_copies":6876,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3196,"num_objects_misplaced":1598,"num_objects_unfound":0,"num_objects_dirty":2292,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3732,"num_write_kb":192736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":721,"num_bytes_recovered":47251456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[0,2],"avail_no_missing":["0","1","3"],"object_location_counts":[{"shards":"0,1,3","objects":1598},{"shards":"0,2","objects":694}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"69'2","reported_seq":338,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.940581+0000","last_change":"2023-10-09T05:22:47.197002+0000","last_active":"2023-10-09T05:37:50.940581+0000","last_peered":"2023-10-09T05:37:50.940581+0000","last_clean":"2023-10-09T05:37:50.940581+0000","last_became_active":"2023-10-09T05:22:47.196685+0000","last_became_peered":"2023-10-09T05:22:47.196685+0000","last_unstale":"2023-10-09T05:37:50.940581+0000","last_undegraded":"2023-10-09T05:37:50.940581+0000","last_fullsized":"2023-10-09T05:37:50.940581+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:47:32.594447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":322,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.940750+0000","last_change":"2023-10-09T05:02:09.649674+0000","last_active":"2023-10-09T05:37:50.940750+0000","last_peered":"2023-10-09T05:37:50.940750+0000","last_clean":"2023-10-09T05:37:50.940750+0000","last_became_active":"2023-10-09T05:02:09.649041+0000","last_became_peered":"2023-10-09T05:02:09.649041+0000","last_unstale":"2023-10-09T05:37:50.940750+0000","last_undegraded":"2023-10-09T05:37:50.940750+0000","last_fullsized":"2023-10-09T05:37:50.940750+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:18.524071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.12","version":"335'5565","reported_seq":7535,"reported_epoch":335,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:38:05.789234+0000","last_change":"2023-10-09T05:37:10.014645+0000","last_active":"2023-10-09T05:38:05.789234+0000","last_peered":"2023-10-09T05:38:05.789234+0000","last_clean":"2023-10-09T05:37:08.311384+0000","last_became_active":"2023-10-09T05:37:10.009586+0000","last_became_peered":"2023-10-09T05:37:10.009586+0000","last_unstale":"2023-10-09T05:38:05.789234+0000","last_undegraded":"2023-10-09T05:37:09.955117+0000","last_fullsized":"2023-10-09T05:37:09.899530+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:35:45.277887+0000","last_clean_scrub_stamp":"2023-10-09T05:35:45.277887+0000","objects_scrubbed":0,"log_size":5565,"log_dups_size":0,"ondisk_log_size":5565,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157188096,"num_objects":2399,"num_object_clones":0,"num_object_copies":7197,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3394,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2399,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3681,"num_write_kb":194528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":641,"num_bytes_recovered":42008576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[1,2],"avail_no_missing":["1","3","6"],"object_location_counts":[{"shards":"1,2","objects":702},{"shards":"1,3,6","objects":1697}],"blocked_by":[],"up_primary":2,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'5","reported_seq":132,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.926831+0000","last_change":"2023-10-09T05:32:45.849292+0000","last_active":"2023-10-09T05:37:50.926831+0000","last_peered":"2023-10-09T05:37:50.926831+0000","last_clean":"2023-10-09T05:37:50.926831+0000","last_became_active":"2023-10-09T05:32:45.829351+0000","last_became_peered":"2023-10-09T05:32:45.829351+0000","last_unstale":"2023-10-09T05:37:50.926831+0000","last_undegraded":"2023-10-09T05:37:50.926831+0000","last_fullsized":"2023-10-09T05:37:50.926831+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:07:15.666617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":310,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T05:37:50.919547+0000","last_change":"2023-10-09T05:30:03.379517+0000","last_active":"2023-10-09T05:37:50.919547+0000","last_peered":"2023-10-09T05:37:50.919547+0000","last_clean":"2023-10-09T05:37:50.919547+0000","last_became_active":"2023-10-09T05:30:03.379048+0000","last_became_peered":"2023-10-09T05:30:03.379048+0000","last_unstale":"2023-10-09T05:37:50.919547+0000","last_undegraded":"2023-10-09T05:37:50.919547+0000","last_fullsized":"2023-10-09T05:37:50.919547+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:04.333337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":26,"stat_sum":{"num_bytes":5004197912,"num_objects":76359,"num_object_clones":0,"num_object_copies":229077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6590,"num_objects_misplaced":3003,"num_objects_unfound":0,"num_objects_dirty":76359,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":122295,"num_write_kb":6356865,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4408,"num_bytes_recovered":288817152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15839080448,"data_stored":15839068232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156681,"ondisk_log_size":156681,"up":78,"acting":76,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":23,"num_read_kb":22,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":64,"num_bytes_recovered":22228,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234671,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2891600,"kb_used_data":1386568,"kb_used_omap":0,"kb_used_meta":1505024,"kb_avail":90853552,"statfs":{"total":95995035648,"available":93034037248,"internally_reserved":0,"allocated":1419845632,"data_stored":1418793344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1541144576},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:37:17 2023","interfaces":[{"interface":"back","average":{"1min":1.496,"5min":2.4199999999999999,"15min":2.8540000000000001},"min":{"1min":0.38700000000000001,"5min":0.32300000000000001,"15min":0.248},"max":{"1min":5.7789999999999999,"5min":87.216999999999999,"15min":122.248},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.579,"5min":2.6030000000000002,"15min":2.9409999999999998},"min":{"1min":0.496,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.0019999999999998,"5min":87.180999999999997,"15min":122.22199999999999},"last":1.847}]},{"osd":1,"last update":"Mon Oct 9 05:37:54 2023","interfaces":[{"interface":"back","average":{"1min":0.81499999999999995,"5min":1.3540000000000001,"15min":1.353},"min":{"1min":0.28100000000000003,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.4740000000000002,"5min":7.6950000000000003,"15min":9.6189999999999998},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.89800000000000002,"5min":1.5229999999999999,"15min":1.369},"min":{"1min":0.32900000000000001,"5min":0.318,"15min":0.29899999999999999},"max":{"1min":2.3069999999999999,"5min":12.081,"15min":12.081},"last":2.1030000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:37:17 2023","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":2.839,"15min":2.5859999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":2.637,"5min":102.146,"15min":122.065},"last":1.899},{"interface":"front","average":{"1min":1.099,"5min":2.6869999999999998,"15min":2.6589999999999998},"min":{"1min":0.375,"5min":0.35999999999999999,"15min":0.30399999999999999},"max":{"1min":3.9340000000000002,"5min":102.188,"15min":122.026},"last":3.7069999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:37:17 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":1.4810000000000001,"15min":2.194},"min":{"1min":0.40899999999999997,"5min":0.33500000000000002,"15min":0.33200000000000002},"max":{"1min":2.927,"5min":7.4269999999999996,"15min":138.667},"last":0.47099999999999997},{"interface":"front","average":{"1min":1.1319999999999999,"5min":1.5860000000000001,"15min":2.411},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.34300000000000003},"max":{"1min":3.0449999999999999,"5min":7.3529999999999998,"15min":138.54400000000001},"last":1.9370000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:37:54 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.98299999999999998,"15min":0.95099999999999996},"min":{"1min":0.41899999999999998,"5min":0.33900000000000002,"15min":0.311},"max":{"1min":0.73399999999999999,"5min":6.843,"15min":7.077},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.70999999999999996,"5min":1.1220000000000001,"15min":1.05},"min":{"1min":0.42299999999999999,"5min":0.36699999999999999,"15min":0.33300000000000002},"max":{"1min":1.9750000000000001,"5min":7.3869999999999996,"15min":7.3869999999999996},"last":1.9950000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:37:54 2023","interfaces":[{"interface":"back","average":{"1min":0.79100000000000004,"5min":1.236,"15min":1.093},"min":{"1min":0.40500000000000003,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":2.1019999999999999,"5min":7.4829999999999997,"15min":7.4829999999999997},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.74099999999999999,"5min":1.3620000000000001,"15min":1.1140000000000001},"min":{"1min":0.379,"5min":0.374,"15min":0.373},"max":{"1min":2.198,"5min":7.6219999999999999,"15min":7.6219999999999999},"last":0.86599999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:37:54 2023","interfaces":[{"interface":"back","average":{"1min":0.68300000000000005,"5min":1.2490000000000001,"15min":1.2150000000000001},"min":{"1min":0.44500000000000001,"5min":0.38700000000000001,"15min":0.32800000000000001},"max":{"1min":1.0980000000000001,"5min":6.8789999999999996,"15min":8.5609999999999999},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.86399999999999999,"5min":1.3779999999999999,"15min":1.1739999999999999},"min":{"1min":0.432,"5min":0.432,"15min":0.377},"max":{"1min":2.0249999999999999,"5min":7.46,"15min":7.46},"last":1.97}]}]},{"osd":6,"up_from":46,"seq":197568496313,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2850704,"kb_used_data":1662600,"kb_used_omap":0,"kb_used_meta":1188096,"kb_avail":90894448,"statfs":{"total":95995035648,"available":93075914752,"internally_reserved":0,"allocated":1702502400,"data_stored":1701450347,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1216610304},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:37:10 2023","interfaces":[{"interface":"back","average":{"1min":3.5699999999999998,"5min":3.4430000000000001,"15min":2.9969999999999999},"min":{"1min":0.32100000000000001,"5min":0.28199999999999997,"15min":0.26100000000000001},"max":{"1min":32.545000000000002,"5min":106.85599999999999,"15min":154.29300000000001},"last":1.591},{"interface":"front","average":{"1min":3.6299999999999999,"5min":3.5270000000000001,"15min":3.109},"min":{"1min":0.46100000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":32.634999999999998,"5min":107.11,"15min":154.35300000000001},"last":1.621}]},{"osd":1,"last update":"Mon Oct 9 05:37:10 2023","interfaces":[{"interface":"back","average":{"1min":2.9849999999999999,"5min":1.768,"15min":2.109},"min":{"1min":0.38400000000000001,"5min":0.28299999999999997,"15min":0.27400000000000002},"max":{"1min":41.408000000000001,"5min":41.408000000000001,"15min":94.037000000000006},"last":2.5489999999999999},{"interface":"front","average":{"1min":2.8780000000000001,"5min":1.554,"15min":1.8109999999999999},"min":{"1min":0.38800000000000001,"5min":0.312,"15min":0.29999999999999999},"max":{"1min":40.835999999999999,"5min":40.835999999999999,"15min":87.950000000000003},"last":0.39100000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:37:10 2023","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":5.0960000000000001,"15min":4.1539999999999999},"min":{"1min":0.29599999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":5.5780000000000003,"5min":211.66300000000001,"15min":211.66300000000001},"last":0.5},{"interface":"front","average":{"1min":1.4259999999999999,"5min":5.8949999999999996,"15min":4.3970000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.4900000000000002,"5min":211.423,"15min":211.423},"last":0.36399999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:37:11 2023","interfaces":[{"interface":"back","average":{"1min":11.858000000000001,"5min":3.5779999999999998,"15min":4.5830000000000002},"min":{"1min":0.441,"5min":0.309,"15min":0.309},"max":{"1min":209.69,"5min":209.69,"15min":209.69},"last":0.47699999999999998},{"interface":"front","average":{"1min":11.736000000000001,"5min":3.4140000000000001,"15min":4.4589999999999996},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":209.34800000000001,"5min":209.34800000000001,"15min":209.34800000000001},"last":1.4079999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:37:10 2023","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":0.98899999999999999,"15min":1.02},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.27600000000000002},"max":{"1min":2.7530000000000001,"5min":10.976000000000001,"15min":10.976000000000001},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.79000000000000004,"5min":1.03,"15min":0.98199999999999998},"min":{"1min":0.39400000000000002,"5min":0.33000000000000002,"15min":0.32800000000000001},"max":{"1min":2.7029999999999998,"5min":5.0190000000000001,"15min":5.0190000000000001},"last":0.53300000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:37:10 2023","interfaces":[{"interface":"back","average":{"1min":1.1879999999999999,"5min":1.2769999999999999,"15min":1.1339999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.7750000000000004,"5min":11.275,"15min":11.275},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.89800000000000002,"5min":1.1599999999999999,"15min":1.0820000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":2.7349999999999999,"5min":7.3090000000000002,"15min":7.3090000000000002},"last":0.45500000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:37:28 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":1.5760000000000001,"15min":1.427},"min":{"1min":0.38700000000000001,"5min":0.36599999999999999,"15min":0.28100000000000003},"max":{"1min":4.4130000000000003,"5min":18.963000000000001,"15min":18.963000000000001},"last":0.77600000000000002},{"interface":"front","average":{"1min":1.107,"5min":1.671,"15min":1.5189999999999999},"min":{"1min":0.499,"5min":0.39500000000000002,"15min":0.35399999999999998},"max":{"1min":4.4539999999999997,"5min":19.010000000000002,"15min":19.010000000000002},"last":0.88}]}]},{"osd":1,"up_from":96,"seq":412316860903,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4252804,"kb_used_data":3016828,"kb_used_omap":4,"kb_used_meta":1235707,"kb_avail":89492348,"statfs":{"total":95995035648,"available":91640164352,"internally_reserved":0,"allocated":3089231872,"data_stored":3087319939,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1265364878},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,1,2,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:37:12 2023","interfaces":[{"interface":"back","average":{"1min":7.0800000000000001,"5min":3.0920000000000001,"15min":3.4710000000000001},"min":{"1min":0.28199999999999997,"5min":0.221,"15min":0.19500000000000001},"max":{"1min":116.20999999999999,"5min":116.20999999999999,"15min":150.75200000000001},"last":2.3170000000000002},{"interface":"front","average":{"1min":7.5270000000000001,"5min":3.1019999999999999,"15min":3.7959999999999998},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.185},"max":{"1min":117.182,"5min":117.182,"15min":150.59299999999999},"last":0.68999999999999995}]},{"osd":2,"last update":"Mon Oct 9 05:37:11 2023","interfaces":[{"interface":"back","average":{"1min":2.5270000000000001,"5min":2.625,"15min":3.6259999999999999},"min":{"1min":0.34799999999999998,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":11.032,"5min":27.875,"15min":154.14599999999999},"last":0.46899999999999997},{"interface":"front","average":{"1min":2.387,"5min":2.6040000000000001,"15min":3.6349999999999998},"min":{"1min":0.40300000000000002,"5min":0.28100000000000003,"15min":0.254},"max":{"1min":7.2110000000000003,"5min":27.641999999999999,"15min":154.215},"last":0.432}]},{"osd":3,"last update":"Mon Oct 9 05:37:11 2023","interfaces":[{"interface":"back","average":{"1min":1.9670000000000001,"5min":2.097,"15min":3.2730000000000001},"min":{"1min":0.29199999999999998,"5min":0.28199999999999997,"15min":0.17499999999999999},"max":{"1min":7.5529999999999999,"5min":11.377000000000001,"15min":152.58199999999999},"last":1.0349999999999999},{"interface":"front","average":{"1min":2.2879999999999998,"5min":1.8149999999999999,"15min":3.4060000000000001},"min":{"1min":0.28399999999999997,"5min":0.255,"15min":0.221},"max":{"1min":6.9989999999999997,"5min":10.206,"15min":154.178},"last":0.66300000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:37:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0329999999999999,"5min":1.542,"15min":1.9530000000000001},"min":{"1min":0.54100000000000004,"5min":0.38700000000000001,"15min":0.30099999999999999},"max":{"1min":6.306,"5min":6.306,"15min":83.453999999999994},"last":0.78300000000000003},{"interface":"front","average":{"1min":1.2170000000000001,"5min":1.377,"15min":1.6439999999999999},"min":{"1min":0.442,"5min":0.37,"15min":0.32400000000000001},"max":{"1min":3.48,"5min":6.0650000000000004,"15min":83.375},"last":1.0860000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:37:11 2023","interfaces":[{"interface":"back","average":{"1min":1.5089999999999999,"5min":1.742,"15min":1.488},"min":{"1min":0.40999999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":6.8700000000000001,"5min":9.2249999999999996,"15min":9.7520000000000007},"last":0.747},{"interface":"front","average":{"1min":2.0510000000000002,"5min":1.786,"15min":2.0179999999999998},"min":{"1min":0.49399999999999999,"5min":0.433,"15min":0.39900000000000002},"max":{"1min":6.3490000000000002,"5min":9.8260000000000005,"15min":83.268000000000001},"last":0.502}]},{"osd":6,"last update":"Mon Oct 9 05:37:11 2023","interfaces":[{"interface":"back","average":{"1min":1.5640000000000001,"5min":1.708,"15min":1.6220000000000001},"min":{"1min":0.374,"5min":0.374,"15min":0.34000000000000002},"max":{"1min":6.9139999999999997,"5min":11.315,"15min":11.315},"last":0.72599999999999998},{"interface":"front","average":{"1min":2.1899999999999999,"5min":1.8620000000000001,"15min":2.0569999999999999},"min":{"1min":0.47299999999999998,"5min":0.41499999999999998,"15min":0.376},"max":{"1min":6.415,"5min":10.446,"15min":83.400000000000006},"last":0.52900000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:37:11 2023","interfaces":[{"interface":"back","average":{"1min":2.2250000000000001,"5min":1.8440000000000001,"15min":2.0750000000000002},"min":{"1min":0.56299999999999994,"5min":0.40899999999999997,"15min":0.376},"max":{"1min":6.3810000000000002,"5min":8.6059999999999999,"15min":83.510000000000005},"last":0.755},{"interface":"front","average":{"1min":1.282,"5min":1.5780000000000001,"15min":1.744},"min":{"1min":0.48899999999999999,"5min":0.40400000000000003,"15min":0.377},"max":{"1min":3.4550000000000001,"5min":7.101,"15min":83.480999999999995},"last":1.131}]}]},{"osd":0,"up_from":88,"seq":377957122545,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3961076,"kb_used_data":2762156,"kb_used_omap":5,"kb_used_meta":1198842,"kb_avail":89784076,"statfs":{"total":95995035648,"available":91938893824,"internally_reserved":0,"allocated":2828447744,"data_stored":2826608525,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1227614842},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,1,5],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:38:06 2023","interfaces":[{"interface":"back","average":{"1min":2.2679999999999998,"5min":2.8410000000000002,"15min":2.0579999999999998},"min":{"1min":0.24099999999999999,"5min":0.20699999999999999,"15min":0.17299999999999999},"max":{"1min":12.199999999999999,"5min":111.001,"15min":111.001},"last":2.5840000000000001},{"interface":"front","average":{"1min":2.1030000000000002,"5min":2.8069999999999999,"15min":1.948},"min":{"1min":0.254,"5min":0.24299999999999999,"15min":0.217},"max":{"1min":12.279999999999999,"5min":110.919,"15min":110.919},"last":4.3550000000000004}]},{"osd":2,"last update":"Mon Oct 9 05:38:10 2023","interfaces":[{"interface":"back","average":{"1min":2.7970000000000002,"5min":2.1419999999999999,"15min":2.6960000000000002},"min":{"1min":0.34399999999999997,"5min":0.26800000000000002,"15min":0.247},"max":{"1min":11.974,"5min":14.259,"15min":92.745000000000005},"last":7.5300000000000002},{"interface":"front","average":{"1min":3.1749999999999998,"5min":2.4500000000000002,"15min":2.9969999999999999},"min":{"1min":0.33700000000000002,"5min":0.25600000000000001,"15min":0.19700000000000001},"max":{"1min":12.074999999999999,"5min":18.724,"15min":97.816000000000003},"last":2.2879999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:38:10 2023","interfaces":[{"interface":"back","average":{"1min":3.0790000000000002,"5min":2.3370000000000002,"15min":3.871},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.21099999999999999},"max":{"1min":12.039,"5min":18.821999999999999,"15min":290.73000000000002},"last":0.441},{"interface":"front","average":{"1min":3.0830000000000002,"5min":2.2850000000000001,"15min":3.7879999999999998},"min":{"1min":0.51400000000000001,"5min":0.246,"15min":0.246},"max":{"1min":12.343,"5min":18.789000000000001,"15min":289.85199999999998},"last":2.859}]},{"osd":4,"last update":"Mon Oct 9 05:38:10 2023","interfaces":[{"interface":"back","average":{"1min":2.1200000000000001,"5min":1.504,"15min":1.357},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.33500000000000002},"max":{"1min":12.377000000000001,"5min":12.377000000000001,"15min":12.377000000000001},"last":2.802},{"interface":"front","average":{"1min":2.4430000000000001,"5min":1.748,"15min":1.6619999999999999},"min":{"1min":0.38300000000000001,"5min":0.38100000000000001,"15min":0.32400000000000001},"max":{"1min":12.477,"5min":12.477,"15min":12.477},"last":2.218}]},{"osd":5,"last update":"Mon Oct 9 05:38:10 2023","interfaces":[{"interface":"back","average":{"1min":2.613,"5min":1.9750000000000001,"15min":1.5860000000000001},"min":{"1min":0.36299999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":12.523,"5min":14.510999999999999,"15min":14.510999999999999},"last":0.61899999999999999},{"interface":"front","average":{"1min":2.137,"5min":1.5880000000000001,"15min":1.46},"min":{"1min":0.51300000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":12.403,"5min":12.403,"15min":12.403},"last":2.7690000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:38:10 2023","interfaces":[{"interface":"back","average":{"1min":2.5859999999999999,"5min":2.0270000000000001,"15min":1.891},"min":{"1min":0.34899999999999998,"5min":0.34200000000000003,"15min":0.33500000000000002},"max":{"1min":13.132,"5min":15.544,"15min":15.544},"last":2.1899999999999999},{"interface":"front","average":{"1min":2.9279999999999999,"5min":2.121,"15min":1.75},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.318},"max":{"1min":12.785,"5min":14.459,"15min":14.459},"last":0.56200000000000006}]},{"osd":7,"last update":"Mon Oct 9 05:38:10 2023","interfaces":[{"interface":"back","average":{"1min":2.5070000000000001,"5min":2.258,"15min":1.8680000000000001},"min":{"1min":0.46100000000000002,"5min":0.42299999999999999,"15min":0.308},"max":{"1min":12.586,"5min":14.492000000000001,"15min":14.492000000000001},"last":0.65100000000000002},{"interface":"front","average":{"1min":2.4340000000000002,"5min":1.796,"15min":1.633},"min":{"1min":0.54100000000000004,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":12.43,"5min":12.43,"15min":12.43},"last":2.8300000000000001}]}]},{"osd":2,"up_from":106,"seq":455266533823,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3656652,"kb_used_data":2616900,"kb_used_omap":7,"kb_used_meta":1039736,"kb_avail":90088500,"statfs":{"total":95995035648,"available":92250624000,"internally_reserved":0,"allocated":2679705600,"data_stored":2677904034,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1064689745},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3,1,1,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:37:57 2023","interfaces":[{"interface":"back","average":{"1min":2.7480000000000002,"5min":2.6179999999999999,"15min":2.7770000000000001},"min":{"1min":0.23899999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":9.5749999999999993,"5min":46.154000000000003,"15min":91.646000000000001},"last":0.22},{"interface":"front","average":{"1min":2.306,"5min":2.6339999999999999,"15min":2.6070000000000002},"min":{"1min":0.33100000000000002,"5min":0.219,"15min":0.21199999999999999},"max":{"1min":8.1769999999999996,"5min":45.789999999999999,"15min":92.522000000000006},"last":11.09}]},{"osd":1,"last update":"Mon Oct 9 05:37:57 2023","interfaces":[{"interface":"back","average":{"1min":2.081,"5min":1.8520000000000001,"15min":1.9870000000000001},"min":{"1min":0.436,"5min":0.22900000000000001,"15min":0.22700000000000001},"max":{"1min":6.5679999999999996,"5min":10.334,"15min":23.251000000000001},"last":5.8650000000000002},{"interface":"front","average":{"1min":2.496,"5min":1.895,"15min":2.1160000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.22600000000000001},"max":{"1min":8.2140000000000004,"5min":10.063000000000001,"15min":17.632999999999999},"last":4.0979999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:37:15 2023","interfaces":[{"interface":"back","average":{"1min":1.482,"5min":2.3540000000000001,"15min":2.3610000000000002},"min":{"1min":0.29099999999999998,"5min":0.214,"15min":0.214},"max":{"1min":5.5549999999999997,"5min":15.576000000000001,"15min":15.576000000000001},"last":3.7869999999999999},{"interface":"front","average":{"1min":1.843,"5min":1.9390000000000001,"15min":2.246},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.27000000000000002},"max":{"1min":6.1109999999999998,"5min":11.451000000000001,"15min":18.515000000000001},"last":4.1319999999999997}]},{"osd":4,"last update":"Mon Oct 9 05:37:57 2023","interfaces":[{"interface":"back","average":{"1min":1.736,"5min":1.724,"15min":1.74},"min":{"1min":0.36199999999999999,"5min":0.34499999999999997,"15min":0.29599999999999999},"max":{"1min":6.5970000000000004,"5min":10.379,"15min":10.597},"last":3.883},{"interface":"front","average":{"1min":2.0459999999999998,"5min":1.6890000000000001,"15min":1.7669999999999999},"min":{"1min":0.376,"5min":0.35399999999999998,"15min":0.34200000000000003},"max":{"1min":6.7279999999999998,"5min":11.173,"15min":13.097},"last":4.1619999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:37:57 2023","interfaces":[{"interface":"back","average":{"1min":1.4410000000000001,"5min":1.5489999999999999,"15min":1.675},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.34100000000000003},"max":{"1min":6.6210000000000004,"5min":9.8729999999999993,"15min":9.8729999999999993},"last":0.34000000000000002},{"interface":"front","average":{"1min":1.8120000000000001,"5min":1.7629999999999999,"15min":1.698},"min":{"1min":0.39200000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":6.6459999999999999,"5min":8.4239999999999995,"15min":10.686},"last":3.8199999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:37:57 2023","interfaces":[{"interface":"back","average":{"1min":1.962,"5min":1.7150000000000001,"15min":1.7969999999999999},"min":{"1min":0.33200000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":6.7610000000000001,"5min":11.381,"15min":13.209},"last":4.1859999999999999},{"interface":"front","average":{"1min":1.6910000000000001,"5min":1.5800000000000001,"15min":1.806},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":6.7699999999999996,"5min":9.9120000000000008,"15min":11.118},"last":0.39500000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:37:57 2023","interfaces":[{"interface":"back","average":{"1min":1.603,"5min":1.758,"15min":1.9890000000000001},"min":{"1min":0.433,"5min":0.433,"15min":0.40899999999999997},"max":{"1min":6.7350000000000003,"5min":9.8930000000000007,"15min":13.571999999999999},"last":1.581},{"interface":"front","average":{"1min":1.972,"5min":2.0259999999999998,"15min":2.0139999999999998},"min":{"1min":0.47499999999999998,"5min":0.45500000000000002,"15min":0.41499999999999998},"max":{"1min":6.6219999999999999,"5min":12.016,"15min":12.016},"last":3.8570000000000002}]}]},{"osd":3,"up_from":123,"seq":528280977841,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3622560,"kb_used_data":2527640,"kb_used_omap":2,"kb_used_meta":1094909,"kb_avail":90122592,"statfs":{"total":95995035648,"available":92285534208,"internally_reserved":0,"allocated":2588303360,"data_stored":2586595471,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":1121186869},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:37:19 2023","interfaces":[{"interface":"back","average":{"1min":2.028,"5min":2.4169999999999998,"15min":2.613},"min":{"1min":0.23699999999999999,"5min":0.224,"15min":0.20499999999999999},"max":{"1min":6.6130000000000004,"5min":42.604999999999997,"15min":90.947000000000003},"last":0.24199999999999999},{"interface":"front","average":{"1min":8.8849999999999998,"5min":3.7370000000000001,"15min":3.0409999999999999},"min":{"1min":0.27500000000000002,"5min":0.184,"15min":0.184},"max":{"1min":118.372,"5min":118.372,"15min":118.372},"last":5.1379999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:37:19 2023","interfaces":[{"interface":"back","average":{"1min":1.913,"5min":3.4079999999999999,"15min":2.5070000000000001},"min":{"1min":0.29499999999999998,"5min":0.23000000000000001,"15min":0.22},"max":{"1min":7.4550000000000001,"5min":118.20999999999999,"15min":118.20999999999999},"last":2.4180000000000001},{"interface":"front","average":{"1min":1.202,"5min":3.1280000000000001,"15min":2.4590000000000001},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":4.6520000000000001,"5min":124.083,"15min":124.083},"last":0.41399999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:37:19 2023","interfaces":[{"interface":"back","average":{"1min":1.855,"5min":6.8940000000000001,"15min":3.8319999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":7.6189999999999998,"5min":350.53899999999999,"15min":350.53899999999999},"last":1.345},{"interface":"front","average":{"1min":2.137,"5min":6.7640000000000002,"15min":3.73},"min":{"1min":0.26000000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":7.0970000000000004,"5min":350.64699999999999,"15min":350.64699999999999},"last":3.8980000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:37:19 2023","interfaces":[{"interface":"back","average":{"1min":1.4810000000000001,"5min":1.4910000000000001,"15min":1.629},"min":{"1min":0.442,"5min":0.374,"15min":0.29999999999999999},"max":{"1min":4.4240000000000004,"5min":9.1539999999999999,"15min":9.1539999999999999},"last":2.4580000000000002},{"interface":"front","average":{"1min":1.169,"5min":1.4730000000000001,"15min":1.585},"min":{"1min":0.38600000000000001,"5min":0.36599999999999999,"15min":0.36299999999999999},"max":{"1min":3.593,"5min":12.132999999999999,"15min":13.037000000000001},"last":0.52300000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:37:19 2023","interfaces":[{"interface":"back","average":{"1min":1.276,"5min":1.4950000000000001,"15min":1.627},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":3.6299999999999999,"5min":12.092000000000001,"15min":13.079000000000001},"last":0.56299999999999994},{"interface":"front","average":{"1min":1.3799999999999999,"5min":1.6319999999999999,"15min":1.583},"min":{"1min":0.53800000000000003,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":4.7119999999999997,"5min":10.135,"15min":10.135},"last":1.0840000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:37:19 2023","interfaces":[{"interface":"back","average":{"1min":1.579,"5min":2.198,"15min":1.819},"min":{"1min":0.42199999999999999,"5min":0.39900000000000002,"15min":0.35699999999999998},"max":{"1min":6.5839999999999996,"5min":20.463999999999999,"15min":20.463999999999999},"last":1.1240000000000001},{"interface":"front","average":{"1min":1.6080000000000001,"5min":1.873,"15min":1.8620000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.36699999999999999},"max":{"1min":4.5359999999999996,"5min":20.279,"15min":20.279},"last":3.9420000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:37:19 2023","interfaces":[{"interface":"back","average":{"1min":1.6140000000000001,"5min":1.9339999999999999,"15min":1.875},"min":{"1min":0.49399999999999999,"5min":0.41099999999999998,"15min":0.38800000000000001},"max":{"1min":4.4770000000000003,"5min":10.885,"15min":10.885},"last":3.9729999999999999},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.806,"15min":1.8879999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.39200000000000002},"max":{"1min":3.5550000000000002,"5min":12.113,"15min":12.989000000000001},"last":0.54600000000000004}]}]},{"osd":4,"up_from":0,"seq":128849019599,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757955,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2511556,"kb_used_data":1604604,"kb_used_omap":0,"kb_used_meta":906944,"kb_avail":91233596,"statfs":{"total":95995035648,"available":93423202304,"internally_reserved":0,"allocated":1643114496,"data_stored":1642070238,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928710656},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,2,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:37:11 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":2.8199999999999998,"15min":2.2759999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":3.9910000000000001,"5min":94.989999999999995,"15min":109.193},"last":3.6560000000000001},{"interface":"front","average":{"1min":1.1870000000000001,"5min":2.9009999999999998,"15min":2.3889999999999998},"min":{"1min":0.378,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":4.0259999999999998,"5min":95.034999999999997,"15min":109.298},"last":3.7000000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:37:11 2023","interfaces":[{"interface":"back","average":{"1min":1.591,"5min":1.476,"15min":1.7370000000000001},"min":{"1min":0.42199999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.1929999999999996,"5min":9.2430000000000003,"15min":78.117000000000004},"last":1.022},{"interface":"front","average":{"1min":1.2869999999999999,"5min":1.3200000000000001,"15min":1.714},"min":{"1min":0.34599999999999997,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":3.7349999999999999,"5min":8.6799999999999997,"15min":78.879999999999995},"last":1.153}]},{"osd":2,"last update":"Mon Oct 9 05:37:23 2023","interfaces":[{"interface":"back","average":{"1min":1.534,"5min":4.5350000000000001,"15min":2.637},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.27800000000000002},"max":{"1min":5.8879999999999999,"5min":250.941,"15min":250.941},"last":1.0549999999999999},{"interface":"front","average":{"1min":1.391,"5min":4.6589999999999998,"15min":2.6339999999999999},"min":{"1min":0.34999999999999998,"5min":0.34699999999999998,"15min":0.33300000000000002},"max":{"1min":5.4169999999999998,"5min":251.48599999999999,"15min":251.48599999999999},"last":1.236}]},{"osd":3,"last update":"Mon Oct 9 05:37:23 2023","interfaces":[{"interface":"back","average":{"1min":1.4259999999999999,"5min":1.5009999999999999,"15min":2.0089999999999999},"min":{"1min":0.41699999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":6.5259999999999998,"5min":9.0169999999999995,"15min":76.512},"last":1.1220000000000001},{"interface":"front","average":{"1min":1.423,"5min":1.6899999999999999,"15min":2.1379999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.315},"max":{"1min":8.4239999999999995,"5min":10.295,"15min":76.578000000000003},"last":2.605}]},{"osd":4,"last update":"Mon Oct 9 05:37:11 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":0.997,"15min":0.87},"min":{"1min":0.371,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":3.5800000000000001,"5min":7.6109999999999998,"15min":7.6109999999999998},"last":1.2629999999999999},{"interface":"front","average":{"1min":0.72799999999999998,"5min":1.173,"15min":1.0129999999999999},"min":{"1min":0.36599999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":2.1240000000000001,"5min":8.3480000000000008,"15min":8.3480000000000008},"last":1.5429999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:37:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":1.3959999999999999,"15min":1.8859999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.9240000000000004,"5min":8.5220000000000002,"15min":190.98500000000001},"last":1.48},{"interface":"front","average":{"1min":1.018,"5min":1.3240000000000001,"15min":1.873},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":3.9009999999999998,"5min":9.282,"15min":190.928},"last":1.292}]},{"osd":7,"last update":"Mon Oct 9 05:37:47 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":1.302,"15min":1.107},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.5430000000000001,"5min":9.3970000000000002,"15min":9.3970000000000002},"last":1.3280000000000001},{"interface":"front","average":{"1min":1.2010000000000001,"5min":1.5980000000000001,"15min":1.23},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.35899999999999999},"max":{"1min":5.2350000000000003,"5min":11.593,"15min":11.593},"last":1.3400000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2814115840,"data_stored":2814115840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3063676928,"data_stored":3063676928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2654998528,"data_stored":2654994456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2571173888,"data_stored":2571173888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1633488896,"data_stored":1633484824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1697845248,"data_stored":1697841176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1403781120,"data_stored":1403781120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:38:12.818 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T05:38:12.818 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_9 pg_num 36 2023-10-09T05:38:12.819 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:12 smithi133 bash[105589]: audit 2023-10-09T05:38:10.792426+0000 mgr.y (mgr.25277) 1510 : audit 0 from='client.35835 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:38:12.819 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:12 smithi133 bash[105589]: cluster 2023-10-09T05:38:12.038415+0000 osd.5 (osd.5) 340 : cluster 0 14.9 scrub starts 2023-10-09T05:38:12.819 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:12 smithi133 bash[102858]: audit 2023-10-09T05:38:10.792426+0000 mgr.y (mgr.25277) 1510 : audit 0 from='client.35835 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:38:12.819 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:12 smithi133 bash[102858]: cluster 2023-10-09T05:38:12.038415+0000 osd.5 (osd.5) 340 : cluster 0 14.9 scrub starts 2023-10-09T05:38:12.883 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:12.915 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:12.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:12.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:13.021 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:13.059 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:13.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:13.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:13.163 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:13.212 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:13.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:13.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:13.230 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:13.315 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:13.339 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:13.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:13.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:13.444 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:13.471 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:13.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:13.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:13.575 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:13.584 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:13 smithi194 bash[92666]: cluster 2023-10-09T05:38:12.437811+0000 mgr.y (mgr.25277) 1511 : cluster 0 pgmap v1635: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 766 op/s; 6346/226695 objects degraded (2.799%); 2688/226695 objects misplaced (1.186%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T05:38:13.584 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:13 smithi194 bash[92666]: cluster 2023-10-09T05:38:13.019058+0000 osd.5 (osd.5) 341 : cluster 0 14.7 scrub starts 2023-10-09T05:38:13.585 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:13 smithi194 bash[92666]: audit 2023-10-09T05:38:13.498212+0000 mon.a (mon.0) 1899 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:38:13.585 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:13 smithi194 bash[92666]: audit 2023-10-09T05:38:13.499663+0000 mon.a (mon.0) 1900 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:38:13.600 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:13.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:13.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:13.705 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:13.715 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:13 smithi133 bash[105589]: cluster 2023-10-09T05:38:12.437811+0000 mgr.y (mgr.25277) 1511 : cluster 0 pgmap v1635: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 766 op/s; 6346/226695 objects degraded (2.799%); 2688/226695 objects misplaced (1.186%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T05:38:13.715 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:13 smithi133 bash[105589]: cluster 2023-10-09T05:38:13.019058+0000 osd.5 (osd.5) 341 : cluster 0 14.7 scrub starts 2023-10-09T05:38:13.715 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:13 smithi133 bash[105589]: audit 2023-10-09T05:38:13.498212+0000 mon.a (mon.0) 1899 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:38:13.715 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:13 smithi133 bash[105589]: audit 2023-10-09T05:38:13.499663+0000 mon.a (mon.0) 1900 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:38:13.715 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:13 smithi133 bash[102858]: cluster 2023-10-09T05:38:12.437811+0000 mgr.y (mgr.25277) 1511 : cluster 0 pgmap v1635: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 766 op/s; 6346/226695 objects degraded (2.799%); 2688/226695 objects misplaced (1.186%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T05:38:13.715 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:13 smithi133 bash[102858]: cluster 2023-10-09T05:38:13.019058+0000 osd.5 (osd.5) 341 : cluster 0 14.7 scrub starts 2023-10-09T05:38:13.716 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:13 smithi133 bash[102858]: audit 2023-10-09T05:38:13.498212+0000 mon.a (mon.0) 1899 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:38:13.716 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:13 smithi133 bash[102858]: audit 2023-10-09T05:38:13.499663+0000 mon.a (mon.0) 1900 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:38:13.741 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:13.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:13.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:13.846 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:13.873 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:13.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:13.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:13.976 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:14.006 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:14.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:14.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:14.113 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:14.146 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:14.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:14.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:14.249 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:14.278 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:14.282 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:38:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:38:14] "GET /metrics HTTP/1.1" 200 36502 "" "Prometheus/2.43.0" 2023-10-09T05:38:14.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:14.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:14.387 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:14.408 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:14.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:14.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:14.511 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:14.542 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:14.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:14.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:14.645 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:14.669 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:14.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:14.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:14.772 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:14.780 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:14 smithi194 bash[92666]: cluster 2023-10-09T05:38:13.905145+0000 mon.a (mon.0) 1901 : cluster 3 Health check update: Degraded data redundancy: 6346/226695 objects degraded (2.799%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:38:14.780 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:14 smithi194 bash[92666]: audit 2023-10-09T05:38:14.215759+0000 mon.a (mon.0) 1902 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:38:14.796 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:14.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:14.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:14.899 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:14.942 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:14.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:14.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:15.043 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:15.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:14 smithi133 bash[102858]: cluster 2023-10-09T05:38:13.905145+0000 mon.a (mon.0) 1901 : cluster 3 Health check update: Degraded data redundancy: 6346/226695 objects degraded (2.799%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:38:15.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:14 smithi133 bash[102858]: audit 2023-10-09T05:38:14.215759+0000 mon.a (mon.0) 1902 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:38:15.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:14 smithi133 bash[105589]: cluster 2023-10-09T05:38:13.905145+0000 mon.a (mon.0) 1901 : cluster 3 Health check update: Degraded data redundancy: 6346/226695 objects degraded (2.799%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:38:15.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:14 smithi133 bash[105589]: audit 2023-10-09T05:38:14.215759+0000 mon.a (mon.0) 1902 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:38:15.074 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:15.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:15.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:15.180 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:15.223 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:15.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:15.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:15.327 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:15.334 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:15.353 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:15.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:15.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:15.461 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:15.505 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:15.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:15.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:15.610 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:15.619 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:15 smithi194 bash[92666]: cluster 2023-10-09T05:38:14.438566+0000 mgr.y (mgr.25277) 1512 : cluster 0 pgmap v1636: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 775 op/s; 6346/222672 objects degraded (2.850%); 2688/222672 objects misplaced (1.207%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T05:38:15.638 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:15.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:15.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:15.699 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:15 smithi133 bash[102858]: cluster 2023-10-09T05:38:14.438566+0000 mgr.y (mgr.25277) 1512 : cluster 0 pgmap v1636: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 775 op/s; 6346/222672 objects degraded (2.850%); 2688/222672 objects misplaced (1.207%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T05:38:15.699 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:15 smithi133 bash[105589]: cluster 2023-10-09T05:38:14.438566+0000 mgr.y (mgr.25277) 1512 : cluster 0 pgmap v1636: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 775 op/s; 6346/222672 objects degraded (2.850%); 2688/222672 objects misplaced (1.207%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T05:38:15.744 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:15.772 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:15.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:15.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:15.875 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:15.906 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:15.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:15.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:16.009 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:16.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:16.038 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:16.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:16.139 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:16.163 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:16.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:16.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:16.270 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:16.297 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:16.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:16.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:16.361 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:16.406 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:16.432 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:16.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:16.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:16.535 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:16.569 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:16.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:16.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:16.677 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:16.704 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:16.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:16.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:16.807 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:16.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:16.838 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:16.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:16.940 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:16.977 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:16.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:16.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:17.080 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:17.109 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:17.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:17.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:17.216 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:17.248 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:17.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:17.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:17.352 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:17.380 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:17.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:17.385 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:17.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:17.486 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:17.521 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:17.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:17.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:17.624 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:17.650 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:17.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:17.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:17.753 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:17.764 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:17 smithi133 bash[102858]: cluster 2023-10-09T05:38:16.439239+0000 mgr.y (mgr.25277) 1513 : cluster 0 pgmap v1637: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 822 op/s; 5834/216498 objects degraded (2.695%); 2217/216498 objects misplaced (1.024%); 9.0 MiB/s, 144 objects/s recovering 2023-10-09T05:38:17.764 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:17 smithi133 bash[105589]: cluster 2023-10-09T05:38:16.439239+0000 mgr.y (mgr.25277) 1513 : cluster 0 pgmap v1637: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 822 op/s; 5834/216498 objects degraded (2.695%); 2217/216498 objects misplaced (1.024%); 9.0 MiB/s, 144 objects/s recovering 2023-10-09T05:38:17.779 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:17.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:17.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:17.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:17 smithi194 bash[92666]: cluster 2023-10-09T05:38:16.439239+0000 mgr.y (mgr.25277) 1513 : cluster 0 pgmap v1637: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 822 op/s; 5834/216498 objects degraded (2.695%); 2217/216498 objects misplaced (1.024%); 9.0 MiB/s, 144 objects/s recovering 2023-10-09T05:38:17.888 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:17.910 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:17.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:17.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:18.013 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:18.036 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:18.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:18.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:18.140 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:18.167 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:18.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:18.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:18.270 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:18.301 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:18.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:18.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:18.405 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:18.435 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:18.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:18.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:18.538 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:18.568 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:18.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:18.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:18.673 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:18.703 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:18.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:18.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:18.808 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:18.834 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:18.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:18.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:18.937 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:18.983 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:18.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:18.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:19.088 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:19.117 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:19.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:19.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:19.223 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:19.250 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:19.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:19.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:19.354 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:19.385 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:19.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:19.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:19.489 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:19.517 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:19.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:19.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:19.625 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:19.634 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:19 smithi133 bash[102858]: cluster 2023-10-09T05:38:18.439807+0000 mgr.y (mgr.25277) 1514 : cluster 0 pgmap v1638: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 673 op/s; 5834/215448 objects degraded (2.708%); 2217/215448 objects misplaced (1.029%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T05:38:19.634 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:19 smithi133 bash[102858]: cluster 2023-10-09T05:38:18.906368+0000 mon.a (mon.0) 1903 : cluster 3 Health check update: Degraded data redundancy: 5834/215448 objects degraded (2.708%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:19.635 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:19 smithi133 bash[105589]: cluster 2023-10-09T05:38:18.439807+0000 mgr.y (mgr.25277) 1514 : cluster 0 pgmap v1638: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 673 op/s; 5834/215448 objects degraded (2.708%); 2217/215448 objects misplaced (1.029%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T05:38:19.635 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:19 smithi133 bash[105589]: cluster 2023-10-09T05:38:18.906368+0000 mon.a (mon.0) 1903 : cluster 3 Health check update: Degraded data redundancy: 5834/215448 objects degraded (2.708%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:19.659 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:19.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:19.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:19.763 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:19.811 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:19.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:19.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:19.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:19 smithi194 bash[92666]: cluster 2023-10-09T05:38:18.439807+0000 mgr.y (mgr.25277) 1514 : cluster 0 pgmap v1638: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 673 op/s; 5834/215448 objects degraded (2.708%); 2217/215448 objects misplaced (1.029%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T05:38:19.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:19 smithi194 bash[92666]: cluster 2023-10-09T05:38:18.906368+0000 mon.a (mon.0) 1903 : cluster 3 Health check update: Degraded data redundancy: 5834/215448 objects degraded (2.708%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:19.917 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:19.945 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:19.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:19.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:20.052 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:20.090 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:20.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:20.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:20.197 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:20.228 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:20.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:20.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:20.334 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:20.358 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:20.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:20.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:20.465 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:20.498 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:20.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:20.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:20.600 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:20.610 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:20 smithi133 bash[102858]: cluster 2023-10-09T05:38:20.133133+0000 osd.5 (osd.5) 342 : cluster 0 14.17 scrub starts 2023-10-09T05:38:20.611 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:20 smithi133 bash[105589]: cluster 2023-10-09T05:38:20.133133+0000 osd.5 (osd.5) 342 : cluster 0 14.17 scrub starts 2023-10-09T05:38:20.627 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:20.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:20.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:20.732 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:20.762 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:20.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:20.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:20.865 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:20.872 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:20 smithi194 bash[92666]: cluster 2023-10-09T05:38:20.133133+0000 osd.5 (osd.5) 342 : cluster 0 14.17 scrub starts 2023-10-09T05:38:20.887 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:20.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:20.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:20.991 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:21.019 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:21.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:21.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:21.123 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:21.149 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:21.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:21.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:21.253 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:21.278 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:21.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:21.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:21.384 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:21.412 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:21.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:21.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:21.519 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:21.551 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:21.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:21.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:21.665 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:21.676 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:21 smithi194 bash[92666]: cluster 2023-10-09T05:38:20.440347+0000 mgr.y (mgr.25277) 1515 : cluster 0 pgmap v1639: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.3 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 772 op/s; 5834/210738 objects degraded (2.768%); 2217/210738 objects misplaced (1.052%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T05:38:21.676 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:21 smithi194 bash[92666]: audit 2023-10-09T05:38:20.765516+0000 mon.a (mon.0) 1904 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:38:21.676 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:21 smithi194 bash[92666]: audit 2023-10-09T05:38:20.781708+0000 mon.a (mon.0) 1905 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:38:21.676 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:21 smithi194 bash[92666]: cluster 2023-10-09T05:38:21.150067+0000 osd.5 (osd.5) 343 : cluster 0 14.19 scrub starts 2023-10-09T05:38:21.700 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:21.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:21.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:21.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:21 smithi133 bash[102858]: cluster 2023-10-09T05:38:20.440347+0000 mgr.y (mgr.25277) 1515 : cluster 0 pgmap v1639: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.3 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 772 op/s; 5834/210738 objects degraded (2.768%); 2217/210738 objects misplaced (1.052%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T05:38:21.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:21 smithi133 bash[102858]: audit 2023-10-09T05:38:20.765516+0000 mon.a (mon.0) 1904 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:38:21.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:21 smithi133 bash[102858]: audit 2023-10-09T05:38:20.781708+0000 mon.a (mon.0) 1905 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:38:21.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:21 smithi133 bash[102858]: cluster 2023-10-09T05:38:21.150067+0000 osd.5 (osd.5) 343 : cluster 0 14.19 scrub starts 2023-10-09T05:38:21.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:21 smithi133 bash[105589]: cluster 2023-10-09T05:38:20.440347+0000 mgr.y (mgr.25277) 1515 : cluster 0 pgmap v1639: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.3 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 772 op/s; 5834/210738 objects degraded (2.768%); 2217/210738 objects misplaced (1.052%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T05:38:21.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:21 smithi133 bash[105589]: audit 2023-10-09T05:38:20.765516+0000 mon.a (mon.0) 1904 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:38:21.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:21 smithi133 bash[105589]: audit 2023-10-09T05:38:20.781708+0000 mon.a (mon.0) 1905 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:38:21.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:21 smithi133 bash[105589]: cluster 2023-10-09T05:38:21.150067+0000 osd.5 (osd.5) 343 : cluster 0 14.19 scrub starts 2023-10-09T05:38:21.807 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:21.833 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:21.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:21.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:21.944 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:21.968 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:21.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:21.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:22.072 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:22.103 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:22.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:22.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:22.208 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:22.241 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:22.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:22.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:22.346 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:22.390 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:22.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:22.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:22.497 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:22.550 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:22.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:22.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:22.654 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:22.680 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:22.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:22.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:22.783 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:22.810 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:22.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:22.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:22.915 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:22.955 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:22.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:22.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:23.063 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:23.097 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:23.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:23.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:23.203 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:23.224 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:23.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:23.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:23.327 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:23.334 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:23 smithi133 bash[102858]: cluster 2023-10-09T05:38:22.161122+0000 osd.5 (osd.5) 344 : cluster 0 14.9 scrub starts 2023-10-09T05:38:23.334 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:23 smithi133 bash[102858]: audit 2023-10-09T05:38:22.251403+0000 mon.a (mon.0) 1906 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:38:23.334 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:23 smithi133 bash[102858]: audit 2023-10-09T05:38:22.264435+0000 mon.a (mon.0) 1907 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:38:23.334 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:23 smithi133 bash[102858]: cluster 2023-10-09T05:38:22.443349+0000 mgr.y (mgr.25277) 1516 : cluster 0 pgmap v1640: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 696 op/s; 5400/204060 objects degraded (2.646%); 1682/204060 objects misplaced (0.824%); 8.8 MiB/s, 140 objects/s recovering 2023-10-09T05:38:23.334 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:23 smithi133 bash[102858]: audit 2023-10-09T05:38:22.898658+0000 mon.a (mon.0) 1908 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:38:23.334 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:23 smithi133 bash[102858]: audit 2023-10-09T05:38:22.902697+0000 mon.a (mon.0) 1909 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:38:23.335 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:23 smithi133 bash[102858]: audit 2023-10-09T05:38:22.916166+0000 mon.a (mon.0) 1910 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:38:23.335 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:23 smithi133 bash[105589]: cluster 2023-10-09T05:38:22.161122+0000 osd.5 (osd.5) 344 : cluster 0 14.9 scrub starts 2023-10-09T05:38:23.335 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:23 smithi133 bash[105589]: audit 2023-10-09T05:38:22.251403+0000 mon.a (mon.0) 1906 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:38:23.335 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:23 smithi133 bash[105589]: audit 2023-10-09T05:38:22.264435+0000 mon.a (mon.0) 1907 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:38:23.335 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:23 smithi133 bash[105589]: cluster 2023-10-09T05:38:22.443349+0000 mgr.y (mgr.25277) 1516 : cluster 0 pgmap v1640: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 696 op/s; 5400/204060 objects degraded (2.646%); 1682/204060 objects misplaced (0.824%); 8.8 MiB/s, 140 objects/s recovering 2023-10-09T05:38:23.335 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:23 smithi133 bash[105589]: audit 2023-10-09T05:38:22.898658+0000 mon.a (mon.0) 1908 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:38:23.335 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:23 smithi133 bash[105589]: audit 2023-10-09T05:38:22.902697+0000 mon.a (mon.0) 1909 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:38:23.335 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:23 smithi133 bash[105589]: audit 2023-10-09T05:38:22.916166+0000 mon.a (mon.0) 1910 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:38:23.349 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:23.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:23.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:23.451 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:23.484 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:23.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:23.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:23.589 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:23.598 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:23 smithi194 bash[92666]: cluster 2023-10-09T05:38:22.161122+0000 osd.5 (osd.5) 344 : cluster 0 14.9 scrub starts 2023-10-09T05:38:23.598 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:23 smithi194 bash[92666]: audit 2023-10-09T05:38:22.251403+0000 mon.a (mon.0) 1906 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:38:23.598 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:23 smithi194 bash[92666]: audit 2023-10-09T05:38:22.264435+0000 mon.a (mon.0) 1907 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:38:23.598 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:23 smithi194 bash[92666]: cluster 2023-10-09T05:38:22.443349+0000 mgr.y (mgr.25277) 1516 : cluster 0 pgmap v1640: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 696 op/s; 5400/204060 objects degraded (2.646%); 1682/204060 objects misplaced (0.824%); 8.8 MiB/s, 140 objects/s recovering 2023-10-09T05:38:23.598 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:23 smithi194 bash[92666]: audit 2023-10-09T05:38:22.898658+0000 mon.a (mon.0) 1908 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:38:23.598 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:23 smithi194 bash[92666]: audit 2023-10-09T05:38:22.902697+0000 mon.a (mon.0) 1909 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:38:23.598 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:23 smithi194 bash[92666]: audit 2023-10-09T05:38:22.916166+0000 mon.a (mon.0) 1910 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:38:23.615 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:23.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:23.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:23.718 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:23.754 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:23.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:23.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:23.862 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:23.889 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:23.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:23.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:23.991 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:24.028 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:24.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:24.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:24.134 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:24.155 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:24.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:24.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:24.258 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:24.262 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:24 smithi133 bash[102858]: cluster 2023-10-09T05:38:23.310860+0000 osd.6 (osd.6) 311 : cluster 0 14.4 scrub starts 2023-10-09T05:38:24.287 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:24.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:24.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:24.390 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:24.416 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:24.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:24.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:24.521 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:24.530 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:24 smithi194 bash[92666]: cluster 2023-10-09T05:38:23.310860+0000 osd.6 (osd.6) 311 : cluster 0 14.4 scrub starts 2023-10-09T05:38:24.530 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:24 smithi194 bash[92666]: cluster 2023-10-09T05:38:23.791257+0000 osd.6 (osd.6) 312 : cluster 0 14.4 scrub ok 2023-10-09T05:38:24.530 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:24 smithi194 bash[92666]: cluster 2023-10-09T05:38:23.909676+0000 mon.a (mon.0) 1911 : cluster 3 Health check update: Degraded data redundancy: 5400/204060 objects degraded (2.646%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:24.530 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:24 smithi194 bash[92666]: cluster 2023-10-09T05:38:23.914149+0000 osd.5 (osd.5) 345 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T05:38:24.530 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:24 smithi194 bash[92666]: cluster 2023-10-09T05:38:23.914444+0000 osd.6 (osd.6) 313 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T05:38:24.530 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:24 smithi194 bash[92666]: cluster 2023-10-09T05:38:23.916258+0000 osd.6 (osd.6) 314 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T05:38:24.530 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:24 smithi194 bash[92666]: cluster 2023-10-09T05:38:23.916885+0000 mon.a (mon.0) 1912 : cluster 0 osdmap e336: 8 total, 8 up, 7 in 2023-10-09T05:38:24.530 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:24 smithi194 bash[92666]: cluster 2023-10-09T05:38:23.921154+0000 osd.7 (osd.7) 291 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T05:38:24.545 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:24.547 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:38:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:38:24] "GET /metrics HTTP/1.1" 200 36502 "" "Prometheus/2.43.0" 2023-10-09T05:38:24.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:24 smithi133 bash[105589]: cluster 2023-10-09T05:38:23.310860+0000 osd.6 (osd.6) 311 : cluster 0 14.4 scrub starts 2023-10-09T05:38:24.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:24 smithi133 bash[105589]: cluster 2023-10-09T05:38:23.791257+0000 osd.6 (osd.6) 312 : cluster 0 14.4 scrub ok 2023-10-09T05:38:24.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:24 smithi133 bash[105589]: cluster 2023-10-09T05:38:23.909676+0000 mon.a (mon.0) 1911 : cluster 3 Health check update: Degraded data redundancy: 5400/204060 objects degraded (2.646%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:24.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:24 smithi133 bash[105589]: cluster 2023-10-09T05:38:23.914149+0000 osd.5 (osd.5) 345 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T05:38:24.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:24 smithi133 bash[105589]: cluster 2023-10-09T05:38:23.914444+0000 osd.6 (osd.6) 313 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T05:38:24.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:24 smithi133 bash[105589]: cluster 2023-10-09T05:38:23.916258+0000 osd.6 (osd.6) 314 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T05:38:24.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:24 smithi133 bash[105589]: cluster 2023-10-09T05:38:23.916885+0000 mon.a (mon.0) 1912 : cluster 0 osdmap e336: 8 total, 8 up, 7 in 2023-10-09T05:38:24.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:24 smithi133 bash[105589]: cluster 2023-10-09T05:38:23.921154+0000 osd.7 (osd.7) 291 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T05:38:24.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:24 smithi133 bash[102858]: cluster 2023-10-09T05:38:23.791257+0000 osd.6 (osd.6) 312 : cluster 0 14.4 scrub ok 2023-10-09T05:38:24.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:24 smithi133 bash[102858]: cluster 2023-10-09T05:38:23.909676+0000 mon.a (mon.0) 1911 : cluster 3 Health check update: Degraded data redundancy: 5400/204060 objects degraded (2.646%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:24.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:24 smithi133 bash[102858]: cluster 2023-10-09T05:38:23.914149+0000 osd.5 (osd.5) 345 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T05:38:24.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:24 smithi133 bash[102858]: cluster 2023-10-09T05:38:23.914444+0000 osd.6 (osd.6) 313 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T05:38:24.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:24 smithi133 bash[102858]: cluster 2023-10-09T05:38:23.916258+0000 osd.6 (osd.6) 314 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T05:38:24.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:24 smithi133 bash[102858]: cluster 2023-10-09T05:38:23.916885+0000 mon.a (mon.0) 1912 : cluster 0 osdmap e336: 8 total, 8 up, 7 in 2023-10-09T05:38:24.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:24 smithi133 bash[102858]: cluster 2023-10-09T05:38:23.921154+0000 osd.7 (osd.7) 291 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T05:38:24.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:24.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:24.652 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:24.680 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:24.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:24.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:24.786 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:24.814 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:24.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:24.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:24.918 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:24.948 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:24.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:24.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:25.051 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:25.093 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:25.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:25.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:25.200 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:25.229 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:25.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:25.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:25.336 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:25.376 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:25.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:25.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:25.383 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:25 smithi133 bash[102858]: cluster 2023-10-09T05:38:24.443907+0000 mgr.y (mgr.25277) 1517 : cluster 0 pgmap v1642: 99 pgs: 1 creating+peering, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 326 op/s; 5400/200532 objects degraded (2.693%); 1373/200532 objects misplaced (0.685%); 8.0 MiB/s, 156 objects/s recovering 2023-10-09T05:38:25.383 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:25 smithi133 bash[102858]: cluster 2023-10-09T05:38:24.915227+0000 mon.a (mon.0) 1913 : cluster 0 osdmap e337: 8 total, 8 up, 7 in 2023-10-09T05:38:25.383 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:25 smithi133 bash[102858]: cluster 2023-10-09T05:38:24.915888+0000 osd.6 (osd.6) 315 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T05:38:25.383 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:25 smithi133 bash[102858]: cluster 2023-10-09T05:38:24.918746+0000 osd.6 (osd.6) 316 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T05:38:25.383 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:25 smithi133 bash[102858]: cluster 2023-10-09T05:38:24.932557+0000 osd.5 (osd.5) 346 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T05:38:25.383 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:25 smithi133 bash[102858]: cluster 2023-10-09T05:38:24.972785+0000 osd.7 (osd.7) 292 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T05:38:25.383 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:25 smithi133 bash[105589]: cluster 2023-10-09T05:38:24.443907+0000 mgr.y (mgr.25277) 1517 : cluster 0 pgmap v1642: 99 pgs: 1 creating+peering, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 326 op/s; 5400/200532 objects degraded (2.693%); 1373/200532 objects misplaced (0.685%); 8.0 MiB/s, 156 objects/s recovering 2023-10-09T05:38:25.383 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:25 smithi133 bash[105589]: cluster 2023-10-09T05:38:24.915227+0000 mon.a (mon.0) 1913 : cluster 0 osdmap e337: 8 total, 8 up, 7 in 2023-10-09T05:38:25.384 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:25 smithi133 bash[105589]: cluster 2023-10-09T05:38:24.915888+0000 osd.6 (osd.6) 315 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T05:38:25.384 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:25 smithi133 bash[105589]: cluster 2023-10-09T05:38:24.918746+0000 osd.6 (osd.6) 316 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T05:38:25.384 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:25 smithi133 bash[105589]: cluster 2023-10-09T05:38:24.932557+0000 osd.5 (osd.5) 346 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T05:38:25.384 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:25 smithi133 bash[105589]: cluster 2023-10-09T05:38:24.972785+0000 osd.7 (osd.7) 292 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T05:38:25.483 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:25.513 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:25.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:25.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:25.617 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:25.627 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:25 smithi194 bash[92666]: cluster 2023-10-09T05:38:24.443907+0000 mgr.y (mgr.25277) 1517 : cluster 0 pgmap v1642: 99 pgs: 1 creating+peering, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 326 op/s; 5400/200532 objects degraded (2.693%); 1373/200532 objects misplaced (0.685%); 8.0 MiB/s, 156 objects/s recovering 2023-10-09T05:38:25.627 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:25 smithi194 bash[92666]: cluster 2023-10-09T05:38:24.915227+0000 mon.a (mon.0) 1913 : cluster 0 osdmap e337: 8 total, 8 up, 7 in 2023-10-09T05:38:25.627 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:25 smithi194 bash[92666]: cluster 2023-10-09T05:38:24.915888+0000 osd.6 (osd.6) 315 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T05:38:25.627 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:25 smithi194 bash[92666]: cluster 2023-10-09T05:38:24.918746+0000 osd.6 (osd.6) 316 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T05:38:25.627 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:25 smithi194 bash[92666]: cluster 2023-10-09T05:38:24.932557+0000 osd.5 (osd.5) 346 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T05:38:25.627 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:25 smithi194 bash[92666]: cluster 2023-10-09T05:38:24.972785+0000 osd.7 (osd.7) 292 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T05:38:25.643 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:25.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:25.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:25.746 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:25.774 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:25.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:25.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:25.880 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:25.910 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:25.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:25.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:26.014 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:26.053 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:26.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:26.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:26.157 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:26.181 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:26.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:26.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:26.284 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:26.293 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:26 smithi133 bash[105589]: cluster 2023-10-09T05:38:25.261551+0000 mon.a (mon.0) 1914 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:38:26.293 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:26 smithi133 bash[102858]: cluster 2023-10-09T05:38:25.261551+0000 mon.a (mon.0) 1914 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:38:26.311 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:26.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:26.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:26.415 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:26.441 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:26.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:26.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:26.545 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:26.573 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:26.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:26.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:26.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:26 smithi194 bash[92666]: cluster 2023-10-09T05:38:25.261551+0000 mon.a (mon.0) 1914 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:38:26.680 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:26.697 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:26.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:26.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:26.800 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:26.828 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:26.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:26.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:26.933 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:26.966 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:26.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:26.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:27.069 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:27.095 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:27.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:27.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:27.200 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:27.230 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:27.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:27.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:27.335 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:27.346 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:27 smithi194 bash[92666]: cluster 2023-10-09T05:38:26.444670+0000 mgr.y (mgr.25277) 1518 : cluster 0 pgmap v1644: 99 pgs: 1 active+recovering+undersized+remapped, 1 creating+peering, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 310 op/s; 2468/195657 objects degraded (1.261%); 3734/195657 objects misplaced (1.908%); 7.1 MiB/s, 147 objects/s recovering 2023-10-09T05:38:27.359 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:27.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:27.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:27.463 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:27.494 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:27.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:27.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:27.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:27 smithi133 bash[105589]: cluster 2023-10-09T05:38:26.444670+0000 mgr.y (mgr.25277) 1518 : cluster 0 pgmap v1644: 99 pgs: 1 active+recovering+undersized+remapped, 1 creating+peering, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 310 op/s; 2468/195657 objects degraded (1.261%); 3734/195657 objects misplaced (1.908%); 7.1 MiB/s, 147 objects/s recovering 2023-10-09T05:38:27.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:27 smithi133 bash[102858]: cluster 2023-10-09T05:38:26.444670+0000 mgr.y (mgr.25277) 1518 : cluster 0 pgmap v1644: 99 pgs: 1 active+recovering+undersized+remapped, 1 creating+peering, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 310 op/s; 2468/195657 objects degraded (1.261%); 3734/195657 objects misplaced (1.908%); 7.1 MiB/s, 147 objects/s recovering 2023-10-09T05:38:27.599 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:27.634 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:27.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:27.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:27.738 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:27.763 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:27.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:27.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:27.869 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:27.891 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:27.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:27.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:27.994 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:28.022 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:28.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:28.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:28.125 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:28.157 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:28.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:28.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:28.261 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:28.286 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:28.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:28.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:28.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:28 smithi194 bash[92666]: cluster 2023-10-09T05:38:27.088936+0000 osd.3 (osd.3) 7 : cluster 0 14.1 scrub starts 2023-10-09T05:38:28.389 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:28.416 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:28.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:28.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:28.520 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:28.529 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:28 smithi133 bash[102858]: cluster 2023-10-09T05:38:27.088936+0000 osd.3 (osd.3) 7 : cluster 0 14.1 scrub starts 2023-10-09T05:38:28.530 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:28 smithi133 bash[105589]: cluster 2023-10-09T05:38:27.088936+0000 osd.3 (osd.3) 7 : cluster 0 14.1 scrub starts 2023-10-09T05:38:28.554 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:28.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:28.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:28.617 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:38:28.659 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:28.675 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:38:28.690 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:28.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:28.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:28.721 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:38:28.767 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:38:28.793 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:28.805 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:38:28.822 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:28.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:28.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:28.838 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:38:28.881 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:38:28.920 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:38:28.926 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:38:28.927 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:38:28.927 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:38:28.927 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:38:28.927 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:38:28.927 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:38:28.927 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:38:28.927 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:38:28.932 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:28.959 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:28.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:28.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:29.064 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:29.090 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:29.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:29.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:29.197 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:29.222 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:29.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:29.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:29.326 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:29.363 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:29.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:29.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:29.469 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:29.506 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:29.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:29.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:29.612 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:29.635 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:29.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:29 smithi194 bash[92666]: cluster 2023-10-09T05:38:28.082329+0000 osd.3 (osd.3) 8 : cluster 0 14.1 deep-scrub starts 2023-10-09T05:38:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:29 smithi194 bash[92666]: cluster 2023-10-09T05:38:28.445231+0000 mgr.y (mgr.25277) 1519 : cluster 0 pgmap v1645: 99 pgs: 1 active+recovering+undersized+remapped, 1 creating+peering, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.0 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 153 op/s; 2468/194724 objects degraded (1.267%); 3734/194724 objects misplaced (1.918%); 7.1 MiB/s, 147 objects/s recovering 2023-10-09T05:38:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:29 smithi194 bash[92666]: audit 2023-10-09T05:38:28.499742+0000 mon.a (mon.0) 1915 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:38:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:29 smithi194 bash[92666]: audit 2023-10-09T05:38:28.501192+0000 mon.a (mon.0) 1916 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:38:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:29 smithi194 bash[92666]: audit 2023-10-09T05:38:28.569796+0000 mon.a (mon.0) 1917 : audit 0 from='client.? 172.21.15.133:0/3269592501' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:38:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:29 smithi194 bash[92666]: cluster 2023-10-09T05:38:28.910840+0000 mon.a (mon.0) 1918 : cluster 3 Health check update: Degraded data redundancy: 2468/195657 objects degraded (1.261%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:29.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:29.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:29.738 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:29.763 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:29.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:29.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:29.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:29 smithi133 bash[102858]: cluster 2023-10-09T05:38:28.082329+0000 osd.3 (osd.3) 8 : cluster 0 14.1 deep-scrub starts 2023-10-09T05:38:29.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:29 smithi133 bash[102858]: cluster 2023-10-09T05:38:28.445231+0000 mgr.y (mgr.25277) 1519 : cluster 0 pgmap v1645: 99 pgs: 1 active+recovering+undersized+remapped, 1 creating+peering, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.0 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 153 op/s; 2468/194724 objects degraded (1.267%); 3734/194724 objects misplaced (1.918%); 7.1 MiB/s, 147 objects/s recovering 2023-10-09T05:38:29.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:29 smithi133 bash[102858]: audit 2023-10-09T05:38:28.499742+0000 mon.a (mon.0) 1915 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:38:29.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:29 smithi133 bash[102858]: audit 2023-10-09T05:38:28.501192+0000 mon.a (mon.0) 1916 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:38:29.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:29 smithi133 bash[102858]: audit 2023-10-09T05:38:28.569796+0000 mon.a (mon.0) 1917 : audit 0 from='client.? 172.21.15.133:0/3269592501' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:38:29.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:29 smithi133 bash[102858]: cluster 2023-10-09T05:38:28.910840+0000 mon.a (mon.0) 1918 : cluster 3 Health check update: Degraded data redundancy: 2468/195657 objects degraded (1.261%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:29.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:29 smithi133 bash[105589]: cluster 2023-10-09T05:38:28.082329+0000 osd.3 (osd.3) 8 : cluster 0 14.1 deep-scrub starts 2023-10-09T05:38:29.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:29 smithi133 bash[105589]: cluster 2023-10-09T05:38:28.445231+0000 mgr.y (mgr.25277) 1519 : cluster 0 pgmap v1645: 99 pgs: 1 active+recovering+undersized+remapped, 1 creating+peering, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.0 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 153 op/s; 2468/194724 objects degraded (1.267%); 3734/194724 objects misplaced (1.918%); 7.1 MiB/s, 147 objects/s recovering 2023-10-09T05:38:29.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:29 smithi133 bash[105589]: audit 2023-10-09T05:38:28.499742+0000 mon.a (mon.0) 1915 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:38:29.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:29 smithi133 bash[105589]: audit 2023-10-09T05:38:28.501192+0000 mon.a (mon.0) 1916 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:38:29.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:29 smithi133 bash[105589]: audit 2023-10-09T05:38:28.569796+0000 mon.a (mon.0) 1917 : audit 0 from='client.? 172.21.15.133:0/3269592501' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:38:29.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:29 smithi133 bash[105589]: cluster 2023-10-09T05:38:28.910840+0000 mon.a (mon.0) 1918 : cluster 3 Health check update: Degraded data redundancy: 2468/195657 objects degraded (1.261%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:29.867 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:29.895 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:29.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:29.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:30.000 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:30.027 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:30.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:30.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:30.130 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:30.157 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:30.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:30.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:30.260 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:30.289 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:30.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:30.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:30.392 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:30.417 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:30.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:30.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:30.524 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:30.547 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:30 smithi133 bash[105589]: cluster 2023-10-09T05:38:29.096752+0000 osd.3 (osd.3) 9 : cluster 0 14.1 scrub starts 2023-10-09T05:38:30.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:30 smithi133 bash[102858]: cluster 2023-10-09T05:38:29.096752+0000 osd.3 (osd.3) 9 : cluster 0 14.1 scrub starts 2023-10-09T05:38:30.552 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:30.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:30.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:30.658 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:30.685 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:30.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:30.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:30.788 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:30.816 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:30.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:30.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:30.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:30 smithi194 bash[92666]: cluster 2023-10-09T05:38:29.096752+0000 osd.3 (osd.3) 9 : cluster 0 14.1 scrub starts 2023-10-09T05:38:30.920 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:30.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:30.959 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:30.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:31.061 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:31.115 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:31.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:31.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:31.220 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:31.246 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:31.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:31.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:31.352 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:31.398 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:31.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:31.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:31.500 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:31.524 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:31.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:31.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:31.628 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:31.653 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:31.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:31.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:31.756 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:31.765 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:31 smithi133 bash[102858]: cluster 2023-10-09T05:38:30.445910+0000 mgr.y (mgr.25277) 1520 : cluster 0 pgmap v1646: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 237 op/s; 2468/186021 objects degraded (1.327%); 3339/186021 objects misplaced (1.795%); 3.0 MiB/s, 82 objects/s recovering 2023-10-09T05:38:31.766 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:31 smithi133 bash[102858]: cluster 2023-10-09T05:38:30.499830+0000 mon.a (mon.0) 1919 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:38:31.766 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:31 smithi133 bash[105589]: cluster 2023-10-09T05:38:30.445910+0000 mgr.y (mgr.25277) 1520 : cluster 0 pgmap v1646: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 237 op/s; 2468/186021 objects degraded (1.327%); 3339/186021 objects misplaced (1.795%); 3.0 MiB/s, 82 objects/s recovering 2023-10-09T05:38:31.766 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:31 smithi133 bash[105589]: cluster 2023-10-09T05:38:30.499830+0000 mon.a (mon.0) 1919 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:38:31.791 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:31.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:31.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:31.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:31 smithi194 bash[92666]: cluster 2023-10-09T05:38:30.445910+0000 mgr.y (mgr.25277) 1520 : cluster 0 pgmap v1646: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 237 op/s; 2468/186021 objects degraded (1.327%); 3339/186021 objects misplaced (1.795%); 3.0 MiB/s, 82 objects/s recovering 2023-10-09T05:38:31.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:31 smithi194 bash[92666]: cluster 2023-10-09T05:38:30.499830+0000 mon.a (mon.0) 1919 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:38:31.896 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:31.919 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:31.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:31.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:32.021 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:32.048 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:32.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:32.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:32.152 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:32.187 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:32.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:32.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:32.296 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:32.326 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:32.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:32.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:32.432 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:32.464 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:32.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:32.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:32.569 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:32.599 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:32.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:32.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:32.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:32 smithi194 bash[92666]: cluster 2023-10-09T05:38:31.155327+0000 osd.3 (osd.3) 10 : cluster 0 14.1 scrub starts 2023-10-09T05:38:32.704 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:32.713 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:32 smithi133 bash[102858]: cluster 2023-10-09T05:38:31.155327+0000 osd.3 (osd.3) 10 : cluster 0 14.1 scrub starts 2023-10-09T05:38:32.714 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:32 smithi133 bash[105589]: cluster 2023-10-09T05:38:31.155327+0000 osd.3 (osd.3) 10 : cluster 0 14.1 scrub starts 2023-10-09T05:38:32.731 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:32.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:32.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:32.836 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:32.869 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:32.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:32.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:32.977 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:33.016 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:33.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:33.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:33.119 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:33.146 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:33.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:33.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:33.252 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:33.284 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:33.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:33.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:33.391 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:33.419 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:33.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:33.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:33.523 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:33.527 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:33 smithi133 bash[102858]: cluster 2023-10-09T05:38:32.160890+0000 osd.3 (osd.3) 11 : cluster 0 14.1 scrub starts 2023-10-09T05:38:33.527 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:33 smithi133 bash[102858]: cluster 2023-10-09T05:38:32.446559+0000 mgr.y (mgr.25277) 1521 : cluster 0 pgmap v1647: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 96 active+clean; 3.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 305 op/s; 2232/183903 objects degraded (1.214%); 3339/183903 objects misplaced (1.816%); 3.7 MiB/s, 91 objects/s recovering 2023-10-09T05:38:33.527 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:33 smithi133 bash[105589]: cluster 2023-10-09T05:38:32.160890+0000 osd.3 (osd.3) 11 : cluster 0 14.1 scrub starts 2023-10-09T05:38:33.527 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:33 smithi133 bash[105589]: cluster 2023-10-09T05:38:32.446559+0000 mgr.y (mgr.25277) 1521 : cluster 0 pgmap v1647: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 96 active+clean; 3.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 305 op/s; 2232/183903 objects degraded (1.214%); 3339/183903 objects misplaced (1.816%); 3.7 MiB/s, 91 objects/s recovering 2023-10-09T05:38:33.555 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:33.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:33.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:33.661 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:33.693 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:33.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:33.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:33.800 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:33.835 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:33.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:33.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:33.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:33 smithi194 bash[92666]: cluster 2023-10-09T05:38:32.160890+0000 osd.3 (osd.3) 11 : cluster 0 14.1 scrub starts 2023-10-09T05:38:33.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:33 smithi194 bash[92666]: cluster 2023-10-09T05:38:32.446559+0000 mgr.y (mgr.25277) 1521 : cluster 0 pgmap v1647: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 96 active+clean; 3.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 305 op/s; 2232/183903 objects degraded (1.214%); 3339/183903 objects misplaced (1.816%); 3.7 MiB/s, 91 objects/s recovering 2023-10-09T05:38:33.943 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:33.971 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:33.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:33.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:34.074 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:34.095 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:34.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:34.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:34.199 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:34.229 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:34.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:34.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:34.342 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:34.367 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:34.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:34.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:34.370 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:38:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:38:34] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.43.0" 2023-10-09T05:38:34.470 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:34.499 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:34.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:34.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:34.603 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:34.629 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:34.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:34.636 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:34 smithi133 bash[102858]: cluster 2023-10-09T05:38:33.911825+0000 mon.a (mon.0) 1920 : cluster 3 Health check update: Degraded data redundancy: 2232/183903 objects degraded (1.214%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:34.636 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:34 smithi133 bash[105589]: cluster 2023-10-09T05:38:33.911825+0000 mon.a (mon.0) 1920 : cluster 3 Health check update: Degraded data redundancy: 2232/183903 objects degraded (1.214%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:34.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:34.737 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:34.747 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:34 smithi194 bash[92666]: cluster 2023-10-09T05:38:33.911825+0000 mon.a (mon.0) 1920 : cluster 3 Health check update: Degraded data redundancy: 2232/183903 objects degraded (1.214%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:34.762 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:34.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:34.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:34.870 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:34.912 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:34.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:34.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:35.017 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:35.050 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:35.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:35.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:35.153 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:35.185 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:35.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:35.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:35.289 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:35.317 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:35.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:35.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:35.420 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:35.447 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:35.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:35.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:35.531 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:35 smithi133 bash[102858]: cluster 2023-10-09T05:38:34.447188+0000 mgr.y (mgr.25277) 1522 : cluster 0 pgmap v1648: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 96 active+clean; 3.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 2232/179814 objects degraded (1.241%); 3339/179814 objects misplaced (1.857%); 2.9 MiB/s, 47 objects/s recovering 2023-10-09T05:38:35.553 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:35.584 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:35.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:35.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:35.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:35 smithi194 bash[92666]: cluster 2023-10-09T05:38:34.447188+0000 mgr.y (mgr.25277) 1522 : cluster 0 pgmap v1648: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 96 active+clean; 3.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 2232/179814 objects degraded (1.241%); 3339/179814 objects misplaced (1.857%); 2.9 MiB/s, 47 objects/s recovering 2023-10-09T05:38:35.690 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:35.725 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:35.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:35.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:35.836 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:35.839 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:35 smithi133 bash[105589]: cluster 2023-10-09T05:38:34.447188+0000 mgr.y (mgr.25277) 1522 : cluster 0 pgmap v1648: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 96 active+clean; 3.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 2232/179814 objects degraded (1.241%); 3339/179814 objects misplaced (1.857%); 2.9 MiB/s, 47 objects/s recovering 2023-10-09T05:38:35.871 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:35.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:35.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:35.979 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:36.006 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:36.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:36.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:36.111 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:36.145 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:36.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:36.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:36.255 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:36.283 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:38:36.293 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:36.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:36.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:36.397 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:36.421 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:36.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:36.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:36.524 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:36.550 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:36.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:36.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:36.655 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:36.685 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:36.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:36.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:36.791 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:36.816 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:36.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:36.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:36.921 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:36.961 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:36.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:36.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:37.066 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:37.092 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:37.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:37.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:37.197 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:37.231 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:37.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:37.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:37.335 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:37.362 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:37.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:37.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:37.465 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:37.489 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:37.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:37.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:37.592 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:37.618 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:37.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:37.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:37.721 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:37.742 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:37.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:37.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:37.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:37 smithi133 bash[102858]: cluster 2023-10-09T05:38:36.448675+0000 mgr.y (mgr.25277) 1523 : cluster 0 pgmap v1649: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 96 active+clean; 3.5 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 762 op/s; 2054/173577 objects degraded (1.183%); 2943/173577 objects misplaced (1.696%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T05:38:37.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:37 smithi133 bash[105589]: cluster 2023-10-09T05:38:36.448675+0000 mgr.y (mgr.25277) 1523 : cluster 0 pgmap v1649: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 96 active+clean; 3.5 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 762 op/s; 2054/173577 objects degraded (1.183%); 2943/173577 objects misplaced (1.696%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T05:38:37.848 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:37.855 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:37 smithi194 bash[92666]: cluster 2023-10-09T05:38:36.448675+0000 mgr.y (mgr.25277) 1523 : cluster 0 pgmap v1649: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 96 active+clean; 3.5 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 762 op/s; 2054/173577 objects degraded (1.183%); 2943/173577 objects misplaced (1.696%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T05:38:37.873 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:37.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:37.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:37.976 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:37.996 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:37.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:37.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:38.099 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:38.138 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:38.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:38.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:38.243 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:38.265 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:38.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:38.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:38.369 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:38.395 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:38.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:38.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:38.498 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:38.529 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:38.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:38.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:38.633 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:38.663 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:38.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:38.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:38.767 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:38.795 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:38.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:38.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:38.901 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:38.925 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:38.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:38.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:39.017 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:38:39.017 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:38:39.027 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:39.052 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:39.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:39.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:39.157 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:39.187 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:39.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:39.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:39.290 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:39.314 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:39.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:39.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:39.417 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:39.444 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:39.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:39.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:39.546 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:39.586 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:39.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:39.592 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:39 smithi133 bash[102858]: cluster 2023-10-09T05:38:38.234569+0000 osd.3 (osd.3) 12 : cluster 0 14.1 scrub starts 2023-10-09T05:38:39.592 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:39 smithi133 bash[102858]: cluster 2023-10-09T05:38:38.449198+0000 mgr.y (mgr.25277) 1524 : cluster 0 pgmap v1650: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 96 active+clean; 3.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 2054/172503 objects degraded (1.191%); 2943/172503 objects misplaced (1.706%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T05:38:39.592 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:39 smithi133 bash[102858]: cluster 2023-10-09T05:38:38.912910+0000 mon.a (mon.0) 1921 : cluster 3 Health check update: Degraded data redundancy: 2054/172503 objects degraded (1.191%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:39.592 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:39 smithi133 bash[105589]: cluster 2023-10-09T05:38:38.234569+0000 osd.3 (osd.3) 12 : cluster 0 14.1 scrub starts 2023-10-09T05:38:39.592 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:39 smithi133 bash[105589]: cluster 2023-10-09T05:38:38.449198+0000 mgr.y (mgr.25277) 1524 : cluster 0 pgmap v1650: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 96 active+clean; 3.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 2054/172503 objects degraded (1.191%); 2943/172503 objects misplaced (1.706%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T05:38:39.593 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:39 smithi133 bash[105589]: cluster 2023-10-09T05:38:38.912910+0000 mon.a (mon.0) 1921 : cluster 3 Health check update: Degraded data redundancy: 2054/172503 objects degraded (1.191%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:39.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:39.694 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:39.724 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:39.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:39.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:39.827 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:39.834 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:39 smithi194 bash[92666]: cluster 2023-10-09T05:38:38.234569+0000 osd.3 (osd.3) 12 : cluster 0 14.1 scrub starts 2023-10-09T05:38:39.834 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:39 smithi194 bash[92666]: cluster 2023-10-09T05:38:38.449198+0000 mgr.y (mgr.25277) 1524 : cluster 0 pgmap v1650: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 96 active+clean; 3.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 2054/172503 objects degraded (1.191%); 2943/172503 objects misplaced (1.706%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T05:38:39.834 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:39 smithi194 bash[92666]: cluster 2023-10-09T05:38:38.912910+0000 mon.a (mon.0) 1921 : cluster 3 Health check update: Degraded data redundancy: 2054/172503 objects degraded (1.191%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:39.856 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:39.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:39.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:39.961 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:39.988 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:39.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:39.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:40.091 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:40.115 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:40.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:40.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:40.218 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:40.255 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:40.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:40.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:40.361 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:40.386 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:40.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:40.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:40.492 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:40.515 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:40.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:40.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:40.570 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is set 2023-10-09T05:38:40.579 INFO:teuthology.orchestra.run.smithi133.stderr:set pool 14 pg_num to 36 2023-10-09T05:38:40.618 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:40.628 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:40 smithi194 bash[92666]: audit 2023-10-09T05:38:39.884008+0000 mon.a (mon.0) 1922 : audit 1 from='client.? 172.21.15.133:0/1942555985' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:38:40.628 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:40 smithi194 bash[92666]: audit 2023-10-09T05:38:40.037657+0000 mon.c (mon.2) 140 : audit 1 from='client.? 172.21.15.133:0/3928184262' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:38:40.628 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:40 smithi194 bash[92666]: audit 2023-10-09T05:38:40.038288+0000 mon.a (mon.0) 1923 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:38:40.643 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:40.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:40.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:40.747 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:40.780 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:40.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:40.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:40.885 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:40.911 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:40.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:40.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:41.015 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:41.029 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:40 smithi133 bash[102858]: audit 2023-10-09T05:38:39.884008+0000 mon.a (mon.0) 1922 : audit 1 from='client.? 172.21.15.133:0/1942555985' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:38:41.030 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:40 smithi133 bash[102858]: audit 2023-10-09T05:38:40.037657+0000 mon.c (mon.2) 140 : audit 1 from='client.? 172.21.15.133:0/3928184262' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:38:41.030 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:40 smithi133 bash[102858]: audit 2023-10-09T05:38:40.038288+0000 mon.a (mon.0) 1923 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:38:41.030 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:40 smithi133 bash[105589]: audit 2023-10-09T05:38:39.884008+0000 mon.a (mon.0) 1922 : audit 1 from='client.? 172.21.15.133:0/1942555985' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:38:41.030 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:40 smithi133 bash[105589]: audit 2023-10-09T05:38:40.037657+0000 mon.c (mon.2) 140 : audit 1 from='client.? 172.21.15.133:0/3928184262' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:38:41.030 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:40 smithi133 bash[105589]: audit 2023-10-09T05:38:40.038288+0000 mon.a (mon.0) 1923 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:38:41.049 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:41.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:41.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:41.151 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:41.213 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T05:38:41.280 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:41.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:41.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:41.384 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:41.419 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:41.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:41.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:41.524 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:41.564 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:41.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:41.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:41.668 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:41.699 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:41.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:41.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:41.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:41 smithi133 bash[105589]: cluster 2023-10-09T05:38:40.232527+0000 osd.3 (osd.3) 13 : cluster 0 14.1 scrub starts 2023-10-09T05:38:41.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:41 smithi133 bash[105589]: cluster 2023-10-09T05:38:40.449848+0000 mgr.y (mgr.25277) 1525 : cluster 0 pgmap v1651: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 96 active+clean; 3.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 754 op/s; 2054/167553 objects degraded (1.226%); 2943/167553 objects misplaced (1.756%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T05:38:41.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:41 smithi133 bash[105589]: cluster 2023-10-09T05:38:40.537454+0000 mon.a (mon.0) 1924 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:38:41.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:41 smithi133 bash[105589]: cluster 2023-10-09T05:38:40.550799+0000 osd.5 (osd.5) 347 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T05:38:41.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:41 smithi133 bash[105589]: cluster 2023-10-09T05:38:40.558022+0000 osd.6 (osd.6) 317 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T05:38:41.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:41 smithi133 bash[105589]: cluster 2023-10-09T05:38:40.562666+0000 osd.7 (osd.7) 293 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T05:38:41.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:41 smithi133 bash[105589]: audit 2023-10-09T05:38:40.565267+0000 mon.a (mon.0) 1925 : audit 1 from='client.? 172.21.15.133:0/1942555985' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:38:41.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:41 smithi133 bash[105589]: cluster 2023-10-09T05:38:40.565316+0000 osd.6 (osd.6) 318 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T05:38:41.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:41 smithi133 bash[105589]: audit 2023-10-09T05:38:40.568902+0000 mon.a (mon.0) 1926 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "36"}]': finished 2023-10-09T05:38:41.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:41 smithi133 bash[105589]: cluster 2023-10-09T05:38:40.570098+0000 mon.a (mon.0) 1927 : cluster 0 osdmap e338: 8 total, 8 up, 7 in 2023-10-09T05:38:41.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:41 smithi133 bash[105589]: cluster 2023-10-09T05:38:40.581408+0000 osd.6 (osd.6) 319 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T05:38:41.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:41 smithi133 bash[105589]: cluster 2023-10-09T05:38:40.598277+0000 osd.4 (osd.4) 301 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T05:38:41.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:41 smithi133 bash[102858]: cluster 2023-10-09T05:38:40.232527+0000 osd.3 (osd.3) 13 : cluster 0 14.1 scrub starts 2023-10-09T05:38:41.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:41 smithi133 bash[102858]: cluster 2023-10-09T05:38:40.449848+0000 mgr.y (mgr.25277) 1525 : cluster 0 pgmap v1651: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 96 active+clean; 3.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 754 op/s; 2054/167553 objects degraded (1.226%); 2943/167553 objects misplaced (1.756%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T05:38:41.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:41 smithi133 bash[102858]: cluster 2023-10-09T05:38:40.537454+0000 mon.a (mon.0) 1924 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:38:41.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:41 smithi133 bash[102858]: cluster 2023-10-09T05:38:40.550799+0000 osd.5 (osd.5) 347 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T05:38:41.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:41 smithi133 bash[102858]: cluster 2023-10-09T05:38:40.558022+0000 osd.6 (osd.6) 317 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T05:38:41.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:41 smithi133 bash[102858]: cluster 2023-10-09T05:38:40.562666+0000 osd.7 (osd.7) 293 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T05:38:41.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:41 smithi133 bash[102858]: audit 2023-10-09T05:38:40.565267+0000 mon.a (mon.0) 1925 : audit 1 from='client.? 172.21.15.133:0/1942555985' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:38:41.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:41 smithi133 bash[102858]: cluster 2023-10-09T05:38:40.565316+0000 osd.6 (osd.6) 318 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T05:38:41.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:41 smithi133 bash[102858]: audit 2023-10-09T05:38:40.568902+0000 mon.a (mon.0) 1926 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "36"}]': finished 2023-10-09T05:38:41.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:41 smithi133 bash[102858]: cluster 2023-10-09T05:38:40.570098+0000 mon.a (mon.0) 1927 : cluster 0 osdmap e338: 8 total, 8 up, 7 in 2023-10-09T05:38:41.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:41 smithi133 bash[102858]: cluster 2023-10-09T05:38:40.581408+0000 osd.6 (osd.6) 319 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T05:38:41.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:41 smithi133 bash[102858]: cluster 2023-10-09T05:38:40.598277+0000 osd.4 (osd.4) 301 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T05:38:41.803 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:41.833 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:41.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:41.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:41.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:41 smithi194 bash[92666]: cluster 2023-10-09T05:38:40.232527+0000 osd.3 (osd.3) 13 : cluster 0 14.1 scrub starts 2023-10-09T05:38:41.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:41 smithi194 bash[92666]: cluster 2023-10-09T05:38:40.449848+0000 mgr.y (mgr.25277) 1525 : cluster 0 pgmap v1651: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 96 active+clean; 3.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 754 op/s; 2054/167553 objects degraded (1.226%); 2943/167553 objects misplaced (1.756%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T05:38:41.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:41 smithi194 bash[92666]: cluster 2023-10-09T05:38:40.537454+0000 mon.a (mon.0) 1924 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:38:41.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:41 smithi194 bash[92666]: cluster 2023-10-09T05:38:40.550799+0000 osd.5 (osd.5) 347 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T05:38:41.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:41 smithi194 bash[92666]: cluster 2023-10-09T05:38:40.558022+0000 osd.6 (osd.6) 317 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T05:38:41.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:41 smithi194 bash[92666]: cluster 2023-10-09T05:38:40.562666+0000 osd.7 (osd.7) 293 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T05:38:41.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:41 smithi194 bash[92666]: audit 2023-10-09T05:38:40.565267+0000 mon.a (mon.0) 1925 : audit 1 from='client.? 172.21.15.133:0/1942555985' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:38:41.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:41 smithi194 bash[92666]: cluster 2023-10-09T05:38:40.565316+0000 osd.6 (osd.6) 318 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T05:38:41.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:41 smithi194 bash[92666]: audit 2023-10-09T05:38:40.568902+0000 mon.a (mon.0) 1926 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "36"}]': finished 2023-10-09T05:38:41.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:41 smithi194 bash[92666]: cluster 2023-10-09T05:38:40.570098+0000 mon.a (mon.0) 1927 : cluster 0 osdmap e338: 8 total, 8 up, 7 in 2023-10-09T05:38:41.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:41 smithi194 bash[92666]: cluster 2023-10-09T05:38:40.581408+0000 osd.6 (osd.6) 319 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T05:38:41.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:41 smithi194 bash[92666]: cluster 2023-10-09T05:38:40.598277+0000 osd.4 (osd.4) 301 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T05:38:41.940 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:41.962 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:41.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:41.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:42.067 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:42.100 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:42.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:42.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:42.203 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:42.241 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:42.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:42.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:42.348 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:42.374 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:42.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:42.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:42.482 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:42.508 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:42.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:42.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:42.612 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:42.625 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:42 smithi194 bash[92666]: audit 2023-10-09T05:38:42.451376+0000 mon.a (mon.0) 1928 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T05:38:42.642 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:42.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:42.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:42.746 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:42.953 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:42 smithi133 bash[102858]: audit 2023-10-09T05:38:42.451376+0000 mon.a (mon.0) 1928 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T05:38:42.953 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:42 smithi133 bash[105589]: audit 2023-10-09T05:38:42.451376+0000 mon.a (mon.0) 1928 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T05:38:42.981 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:42.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:42.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:43.085 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:43.105 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:43.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:43.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:43.208 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:43.230 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:43.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:43.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:43.333 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:43.359 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:43.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:43.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:43.462 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:43.494 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:43.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:43.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:43.599 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:43.636 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:43.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:43.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:43.740 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:43.748 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:43 smithi133 bash[105589]: cluster 2023-10-09T05:38:42.450482+0000 mgr.y (mgr.25277) 1526 : cluster 0 pgmap v1653: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 96 active+clean; 3.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 751 op/s; 1845/161343 objects degraded (1.144%); 2496/161343 objects misplaced (1.547%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T05:38:43.748 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:43 smithi133 bash[105589]: cluster 2023-10-09T05:38:42.563907+0000 osd.5 (osd.5) 348 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T05:38:43.749 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:43 smithi133 bash[105589]: cluster 2023-10-09T05:38:42.569529+0000 osd.5 (osd.5) 349 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T05:38:43.749 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:43 smithi133 bash[105589]: audit 2023-10-09T05:38:42.574545+0000 mon.a (mon.0) 1929 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T05:38:43.749 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:43 smithi133 bash[105589]: cluster 2023-10-09T05:38:42.574862+0000 mon.a (mon.0) 1930 : cluster 0 osdmap e339: 8 total, 8 up, 7 in 2023-10-09T05:38:43.749 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:43 smithi133 bash[105589]: cluster 2023-10-09T05:38:42.654516+0000 osd.7 (osd.7) 294 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T05:38:43.749 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:43 smithi133 bash[105589]: cluster 2023-10-09T05:38:42.783198+0000 osd.4 (osd.4) 302 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T05:38:43.749 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:43 smithi133 bash[105589]: audit 2023-10-09T05:38:43.497894+0000 mon.a (mon.0) 1931 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:38:43.749 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:43 smithi133 bash[105589]: audit 2023-10-09T05:38:43.499403+0000 mon.a (mon.0) 1932 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:38:43.749 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:43 smithi133 bash[102858]: cluster 2023-10-09T05:38:42.450482+0000 mgr.y (mgr.25277) 1526 : cluster 0 pgmap v1653: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 96 active+clean; 3.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 751 op/s; 1845/161343 objects degraded (1.144%); 2496/161343 objects misplaced (1.547%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T05:38:43.749 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:43 smithi133 bash[102858]: cluster 2023-10-09T05:38:42.563907+0000 osd.5 (osd.5) 348 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T05:38:43.750 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:43 smithi133 bash[102858]: cluster 2023-10-09T05:38:42.569529+0000 osd.5 (osd.5) 349 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T05:38:43.750 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:43 smithi133 bash[102858]: audit 2023-10-09T05:38:42.574545+0000 mon.a (mon.0) 1929 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T05:38:43.750 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:43 smithi133 bash[102858]: cluster 2023-10-09T05:38:42.574862+0000 mon.a (mon.0) 1930 : cluster 0 osdmap e339: 8 total, 8 up, 7 in 2023-10-09T05:38:43.750 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:43 smithi133 bash[102858]: cluster 2023-10-09T05:38:42.654516+0000 osd.7 (osd.7) 294 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T05:38:43.750 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:43 smithi133 bash[102858]: cluster 2023-10-09T05:38:42.783198+0000 osd.4 (osd.4) 302 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T05:38:43.750 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:43 smithi133 bash[102858]: audit 2023-10-09T05:38:43.497894+0000 mon.a (mon.0) 1931 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:38:43.750 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:43 smithi133 bash[102858]: audit 2023-10-09T05:38:43.499403+0000 mon.a (mon.0) 1932 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:38:43.769 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:43.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:43.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:43.873 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:43.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:43 smithi194 bash[92666]: cluster 2023-10-09T05:38:42.450482+0000 mgr.y (mgr.25277) 1526 : cluster 0 pgmap v1653: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 96 active+clean; 3.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 751 op/s; 1845/161343 objects degraded (1.144%); 2496/161343 objects misplaced (1.547%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T05:38:43.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:43 smithi194 bash[92666]: cluster 2023-10-09T05:38:42.563907+0000 osd.5 (osd.5) 348 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T05:38:43.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:43 smithi194 bash[92666]: cluster 2023-10-09T05:38:42.569529+0000 osd.5 (osd.5) 349 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T05:38:43.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:43 smithi194 bash[92666]: audit 2023-10-09T05:38:42.574545+0000 mon.a (mon.0) 1929 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T05:38:43.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:43 smithi194 bash[92666]: cluster 2023-10-09T05:38:42.574862+0000 mon.a (mon.0) 1930 : cluster 0 osdmap e339: 8 total, 8 up, 7 in 2023-10-09T05:38:43.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:43 smithi194 bash[92666]: cluster 2023-10-09T05:38:42.654516+0000 osd.7 (osd.7) 294 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T05:38:43.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:43 smithi194 bash[92666]: cluster 2023-10-09T05:38:42.783198+0000 osd.4 (osd.4) 302 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T05:38:43.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:43 smithi194 bash[92666]: audit 2023-10-09T05:38:43.497894+0000 mon.a (mon.0) 1931 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:38:43.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:43 smithi194 bash[92666]: audit 2023-10-09T05:38:43.499403+0000 mon.a (mon.0) 1932 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:38:43.902 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:43.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:43.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:43.909 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:44.010 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:44.039 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:44.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:44.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:44.144 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:44.179 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:44.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:44.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:44.285 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:44.310 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:44.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:44.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:44.416 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:44.447 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:44.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:44.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:44.553 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:44.576 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:44.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:44.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:44.682 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:44.711 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:44.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:44.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:44.719 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:44 smithi133 bash[105589]: cluster 2023-10-09T05:38:43.577881+0000 osd.5 (osd.5) 350 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T05:38:44.719 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:44 smithi133 bash[105589]: cluster 2023-10-09T05:38:43.578072+0000 mon.a (mon.0) 1933 : cluster 0 osdmap e340: 8 total, 8 up, 7 in 2023-10-09T05:38:44.719 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:44 smithi133 bash[105589]: cluster 2023-10-09T05:38:43.578624+0000 osd.6 (osd.6) 320 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T05:38:44.719 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:44 smithi133 bash[105589]: cluster 2023-10-09T05:38:43.580694+0000 osd.5 (osd.5) 351 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T05:38:44.719 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:44 smithi133 bash[105589]: cluster 2023-10-09T05:38:43.580816+0000 osd.7 (osd.7) 295 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T05:38:44.720 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:44 smithi133 bash[105589]: cluster 2023-10-09T05:38:43.580876+0000 osd.6 (osd.6) 321 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T05:38:44.720 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:44 smithi133 bash[105589]: cluster 2023-10-09T05:38:43.583289+0000 osd.7 (osd.7) 296 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T05:38:44.720 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:44 smithi133 bash[105589]: cluster 2023-10-09T05:38:43.585423+0000 osd.5 (osd.5) 352 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T05:38:44.720 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:44 smithi133 bash[105589]: cluster 2023-10-09T05:38:43.918021+0000 mon.a (mon.0) 1934 : cluster 3 Health check update: Degraded data redundancy: 1845/161343 objects degraded (1.144%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:44.720 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:38:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:38:44] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.43.0" 2023-10-09T05:38:44.720 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:44 smithi133 bash[102858]: cluster 2023-10-09T05:38:43.577881+0000 osd.5 (osd.5) 350 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T05:38:44.720 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:44 smithi133 bash[102858]: cluster 2023-10-09T05:38:43.578072+0000 mon.a (mon.0) 1933 : cluster 0 osdmap e340: 8 total, 8 up, 7 in 2023-10-09T05:38:44.721 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:44 smithi133 bash[102858]: cluster 2023-10-09T05:38:43.578624+0000 osd.6 (osd.6) 320 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T05:38:44.721 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:44 smithi133 bash[102858]: cluster 2023-10-09T05:38:43.580694+0000 osd.5 (osd.5) 351 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T05:38:44.721 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:44 smithi133 bash[102858]: cluster 2023-10-09T05:38:43.580816+0000 osd.7 (osd.7) 295 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T05:38:44.721 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:44 smithi133 bash[102858]: cluster 2023-10-09T05:38:43.580876+0000 osd.6 (osd.6) 321 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T05:38:44.721 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:44 smithi133 bash[102858]: cluster 2023-10-09T05:38:43.583289+0000 osd.7 (osd.7) 296 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T05:38:44.721 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:44 smithi133 bash[102858]: cluster 2023-10-09T05:38:43.585423+0000 osd.5 (osd.5) 352 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T05:38:44.721 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:44 smithi133 bash[102858]: cluster 2023-10-09T05:38:43.918021+0000 mon.a (mon.0) 1934 : cluster 3 Health check update: Degraded data redundancy: 1845/161343 objects degraded (1.144%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:44.815 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:44.846 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:44.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:44.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:44.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:44 smithi194 bash[92666]: cluster 2023-10-09T05:38:43.577881+0000 osd.5 (osd.5) 350 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T05:38:44.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:44 smithi194 bash[92666]: cluster 2023-10-09T05:38:43.578072+0000 mon.a (mon.0) 1933 : cluster 0 osdmap e340: 8 total, 8 up, 7 in 2023-10-09T05:38:44.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:44 smithi194 bash[92666]: cluster 2023-10-09T05:38:43.578624+0000 osd.6 (osd.6) 320 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T05:38:44.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:44 smithi194 bash[92666]: cluster 2023-10-09T05:38:43.580694+0000 osd.5 (osd.5) 351 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T05:38:44.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:44 smithi194 bash[92666]: cluster 2023-10-09T05:38:43.580816+0000 osd.7 (osd.7) 295 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T05:38:44.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:44 smithi194 bash[92666]: cluster 2023-10-09T05:38:43.580876+0000 osd.6 (osd.6) 321 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T05:38:44.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:44 smithi194 bash[92666]: cluster 2023-10-09T05:38:43.583289+0000 osd.7 (osd.7) 296 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T05:38:44.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:44 smithi194 bash[92666]: cluster 2023-10-09T05:38:43.585423+0000 osd.5 (osd.5) 352 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T05:38:44.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:44 smithi194 bash[92666]: cluster 2023-10-09T05:38:43.918021+0000 mon.a (mon.0) 1934 : cluster 3 Health check update: Degraded data redundancy: 1845/161343 objects degraded (1.144%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:44.955 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:44.983 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:44.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:44.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:45.089 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:45.121 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:45.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:45.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:45.226 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:45.249 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:45.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:45.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:45.353 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:45.385 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:45.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:45.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:45.404 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:38:45.490 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:45.520 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:45.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:45.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:45.625 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:45.650 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:45.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:45.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:45.753 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:45.762 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:45 smithi133 bash[105589]: cluster 2023-10-09T05:38:44.451280+0000 mgr.y (mgr.25277) 1527 : cluster 0 pgmap v1656: 109 pgs: 7 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 97 active+clean; 3.2 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 802 op/s; 1845/158052 objects degraded (1.167%); 2496/158052 objects misplaced (1.579%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T05:38:45.762 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:45 smithi133 bash[102858]: cluster 2023-10-09T05:38:44.451280+0000 mgr.y (mgr.25277) 1527 : cluster 0 pgmap v1656: 109 pgs: 7 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 97 active+clean; 3.2 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 802 op/s; 1845/158052 objects degraded (1.167%); 2496/158052 objects misplaced (1.579%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T05:38:45.784 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:45.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:45.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:45.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:45 smithi194 bash[92666]: cluster 2023-10-09T05:38:44.451280+0000 mgr.y (mgr.25277) 1527 : cluster 0 pgmap v1656: 109 pgs: 7 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 97 active+clean; 3.2 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 802 op/s; 1845/158052 objects degraded (1.167%); 2496/158052 objects misplaced (1.579%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T05:38:45.887 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:45.907 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:45.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:45.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:46.013 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:46.042 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:46.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:46.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:46.148 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:46.176 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:46.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:46.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:46.281 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:46.319 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:46.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:46.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:46.424 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:46.453 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:46.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:46.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:46.556 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:46.590 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:46.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:46.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:46.693 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:46.731 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:46.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:46.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:46.837 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:46.881 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:46.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:46.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:46.984 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:47.017 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:47.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:47.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:47.122 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:47.147 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:47.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:47.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:47.251 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:47.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:47.291 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:47.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:47.392 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:47.419 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:47.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:47.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:47.521 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:47.549 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:47.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:47.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:47.654 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:47.662 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:47 smithi194 bash[92666]: cluster 2023-10-09T05:38:46.452064+0000 mgr.y (mgr.25277) 1528 : cluster 0 pgmap v1657: 109 pgs: 1 active+recovering+undersized+remapped, 2 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait, 101 active+clean; 3.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 827 op/s; 1604/152661 objects degraded (1.051%); 1908/152661 objects misplaced (1.250%); 5.9 MiB/s, 95 objects/s recovering 2023-10-09T05:38:47.686 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:47.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:47.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:47.791 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:47.822 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:47.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:47.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:47.928 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:47.965 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:47.965 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:47 smithi133 bash[105589]: cluster 2023-10-09T05:38:46.452064+0000 mgr.y (mgr.25277) 1528 : cluster 0 pgmap v1657: 109 pgs: 1 active+recovering+undersized+remapped, 2 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait, 101 active+clean; 3.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 827 op/s; 1604/152661 objects degraded (1.051%); 1908/152661 objects misplaced (1.250%); 5.9 MiB/s, 95 objects/s recovering 2023-10-09T05:38:47.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:47.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:48.068 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:48.092 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:48.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:48.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:48.195 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:48.226 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:48.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:48.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:48.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:47 smithi133 bash[102858]: cluster 2023-10-09T05:38:46.452064+0000 mgr.y (mgr.25277) 1528 : cluster 0 pgmap v1657: 109 pgs: 1 active+recovering+undersized+remapped, 2 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait, 101 active+clean; 3.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 827 op/s; 1604/152661 objects degraded (1.051%); 1908/152661 objects misplaced (1.250%); 5.9 MiB/s, 95 objects/s recovering 2023-10-09T05:38:48.329 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:48.357 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:48.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:48.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:48.428 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 6, 1, 7, 5] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:38:48.428 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:38:48.428 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-10-09T05:38:48.428 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-10-09T05:38:48.463 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:48.507 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:48.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:48.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:48.619 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:48.657 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:48.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:48.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:48.760 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:48.793 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:48.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:48.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:48.898 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:48.924 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:48.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:48.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:49.028 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:49.057 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:49.058 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:49.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:49.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:49.160 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:49.189 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:49.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:49.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:49.295 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:49.319 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:49.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:49.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:49.424 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:49.453 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:49.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:49.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:49.560 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:49.598 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:49.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:49.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:49.615 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:49 smithi133 bash[102858]: cluster 2023-10-09T05:38:48.452766+0000 mgr.y (mgr.25277) 1529 : cluster 0 pgmap v1658: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait, 103 active+clean; 3.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 1604/151713 objects degraded (1.057%); 1908/151713 objects misplaced (1.258%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T05:38:49.615 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:49 smithi133 bash[102858]: cluster 2023-10-09T05:38:48.919980+0000 mon.a (mon.0) 1935 : cluster 3 Health check update: Degraded data redundancy: 1604/151713 objects degraded (1.057%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:49.616 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:49 smithi133 bash[105589]: cluster 2023-10-09T05:38:48.452766+0000 mgr.y (mgr.25277) 1529 : cluster 0 pgmap v1658: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait, 103 active+clean; 3.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 1604/151713 objects degraded (1.057%); 1908/151713 objects misplaced (1.258%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T05:38:49.616 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:49 smithi133 bash[105589]: cluster 2023-10-09T05:38:48.919980+0000 mon.a (mon.0) 1935 : cluster 3 Health check update: Degraded data redundancy: 1604/151713 objects degraded (1.057%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:49.701 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:49.743 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:49.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:49.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:49.847 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:49.857 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:49 smithi194 bash[92666]: cluster 2023-10-09T05:38:48.452766+0000 mgr.y (mgr.25277) 1529 : cluster 0 pgmap v1658: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait, 103 active+clean; 3.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 1604/151713 objects degraded (1.057%); 1908/151713 objects misplaced (1.258%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T05:38:49.857 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:49 smithi194 bash[92666]: cluster 2023-10-09T05:38:48.919980+0000 mon.a (mon.0) 1935 : cluster 3 Health check update: Degraded data redundancy: 1604/151713 objects degraded (1.057%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:49.878 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:49.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:49.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:49.985 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:50.011 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:50.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:50.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:50.118 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:50.152 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:50.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:50.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:50.256 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:50.288 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:50.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:50.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:50.392 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:50.427 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:50.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:50.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:50.532 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:50.557 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:50.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:50.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:50.661 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:50.691 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:50.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:50.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:50.797 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:50.826 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:50.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:50.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:50.929 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:50.957 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:50.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:50.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:51.061 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:51.091 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:51.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:51.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:51.194 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:51.223 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:51.224 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:51.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:51.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:51.330 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:51.357 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:51.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:51.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:51.462 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:51.488 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:51.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:51.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:51.592 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:51.598 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:51 smithi133 bash[102858]: cluster 2023-10-09T05:38:50.453606+0000 mgr.y (mgr.25277) 1530 : cluster 0 pgmap v1659: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait, 103 active+clean; 3.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 663 op/s; 1604/145422 objects degraded (1.103%); 1908/145422 objects misplaced (1.312%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T05:38:51.599 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:51 smithi133 bash[105589]: cluster 2023-10-09T05:38:50.453606+0000 mgr.y (mgr.25277) 1530 : cluster 0 pgmap v1659: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait, 103 active+clean; 3.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 663 op/s; 1604/145422 objects degraded (1.103%); 1908/145422 objects misplaced (1.312%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T05:38:51.620 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:51.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:51.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:51.725 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:51.734 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:51 smithi194 bash[92666]: cluster 2023-10-09T05:38:50.453606+0000 mgr.y (mgr.25277) 1530 : cluster 0 pgmap v1659: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait, 103 active+clean; 3.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 663 op/s; 1604/145422 objects degraded (1.103%); 1908/145422 objects misplaced (1.312%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T05:38:51.749 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:51.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:51.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:51.854 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:51.894 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:51.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:51.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:51.998 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:52.039 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:52.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:52.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:52.144 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:52.169 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:52.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:52.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:52.273 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:52.303 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:52.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:52.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:52.411 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:52.437 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:52.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:52.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:52.542 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:52.572 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:52.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:52.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:52.677 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:52.711 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:52.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:52.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:52.815 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:52.841 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:52.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:52.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:52.946 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:52.974 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:52.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:52.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:53.077 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:53.115 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:53.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:53.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:53.223 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:53.250 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:53.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:53.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:53.266 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:53.357 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:53.382 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:53.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:53.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:53.490 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:53.526 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:53.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:53.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:53.633 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:53.660 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:53.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:53.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:53.763 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:53.789 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:53.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:53.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:53.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:53 smithi194 bash[92666]: cluster 2023-10-09T05:38:52.454367+0000 mgr.y (mgr.25277) 1531 : cluster 0 pgmap v1660: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait, 103 active+clean; 2.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 819 op/s; 1312/137046 objects degraded (0.957%); 1422/137046 objects misplaced (1.038%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T05:38:53.895 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:53.920 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:53.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:53.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:54.024 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:54.043 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:54.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:54.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:54.065 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:53 smithi133 bash[105589]: cluster 2023-10-09T05:38:52.454367+0000 mgr.y (mgr.25277) 1531 : cluster 0 pgmap v1660: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait, 103 active+clean; 2.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 819 op/s; 1312/137046 objects degraded (0.957%); 1422/137046 objects misplaced (1.038%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T05:38:54.066 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:53 smithi133 bash[102858]: cluster 2023-10-09T05:38:52.454367+0000 mgr.y (mgr.25277) 1531 : cluster 0 pgmap v1660: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait, 103 active+clean; 2.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 819 op/s; 1312/137046 objects degraded (0.957%); 1422/137046 objects misplaced (1.038%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T05:38:54.147 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:54.173 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:54.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:54.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:54.276 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:54.305 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:54.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:54.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:54.409 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:54.439 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:54.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:54.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:54.546 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:54.547 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:38:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:38:54] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.43.0" 2023-10-09T05:38:54.572 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:54.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:54.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:54.675 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:54.684 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:54 smithi194 bash[92666]: cluster 2023-10-09T05:38:53.921173+0000 mon.a (mon.0) 1936 : cluster 3 Health check update: Degraded data redundancy: 1312/137046 objects degraded (0.957%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:54.705 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:54.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:54.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:54.810 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:54.846 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:54.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:54.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:54.949 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:54.975 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:54.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:54.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:55.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:54 smithi133 bash[102858]: cluster 2023-10-09T05:38:53.921173+0000 mon.a (mon.0) 1936 : cluster 3 Health check update: Degraded data redundancy: 1312/137046 objects degraded (0.957%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:55.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:54 smithi133 bash[105589]: cluster 2023-10-09T05:38:53.921173+0000 mon.a (mon.0) 1936 : cluster 3 Health check update: Degraded data redundancy: 1312/137046 objects degraded (0.957%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:55.079 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:55.113 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:55.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:55.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:55.217 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:55.242 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:55.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:55.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:55.345 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:55.374 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:55.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:55.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:55.480 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:55.507 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:55.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:55.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:55.612 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:55.638 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:55.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:55.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:55.741 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:55.752 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:55 smithi194 bash[92666]: cluster 2023-10-09T05:38:54.455012+0000 mgr.y (mgr.25277) 1532 : cluster 0 pgmap v1661: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait, 103 active+clean; 2.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 899 op/s; 1312/131979 objects degraded (0.994%); 1422/131979 objects misplaced (1.077%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T05:38:55.768 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:55.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:55.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:55.810 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:55 smithi133 bash[102858]: cluster 2023-10-09T05:38:54.455012+0000 mgr.y (mgr.25277) 1532 : cluster 0 pgmap v1661: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait, 103 active+clean; 2.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 899 op/s; 1312/131979 objects degraded (0.994%); 1422/131979 objects misplaced (1.077%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T05:38:55.811 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:55 smithi133 bash[105589]: cluster 2023-10-09T05:38:54.455012+0000 mgr.y (mgr.25277) 1532 : cluster 0 pgmap v1661: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait, 103 active+clean; 2.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 899 op/s; 1312/131979 objects degraded (0.994%); 1422/131979 objects misplaced (1.077%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T05:38:55.871 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:55.899 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:55.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:55.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:56.002 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:56.029 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:56.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:56.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:56.135 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:56.163 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:56.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:56.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:56.267 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:56.293 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:56.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:56.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:56.396 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:56.428 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:56.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:56.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:56.532 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:56.563 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:56.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:56.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:56.667 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:56.692 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:56.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:56.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:56.716 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:38:56.760 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:38:56.795 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:56.821 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:38:56.827 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:56.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:56.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:56.853 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:38:56.896 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:38:56.932 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:38:56.943 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:56.977 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:38:56.979 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:56.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:56.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:57.005 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:38:57.010 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:38:57.010 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:38:57.010 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:38:57.010 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:38:57.010 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:38:57.011 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:38:57.011 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:38:57.011 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:38:57.084 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:57.111 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:57.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:57.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:57.216 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:57.245 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:57.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:57.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:57.349 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:57.377 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:57.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:57.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:57.488 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:57.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:57.518 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:57.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:57.619 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:57.628 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:57 smithi194 bash[92666]: cluster 2023-10-09T05:38:56.455842+0000 mgr.y (mgr.25277) 1533 : cluster 0 pgmap v1662: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait, 103 active+clean; 2.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 923 op/s; 1020/124809 objects degraded (0.817%); 753/124809 objects misplaced (0.603%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T05:38:57.628 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:57 smithi194 bash[92666]: audit 2023-10-09T05:38:56.663186+0000 mon.c (mon.2) 141 : audit 0 from='client.? 172.21.15.133:0/3610763424' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:38:57.646 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:57.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:57.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:57.751 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:57.760 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:57 smithi133 bash[102858]: cluster 2023-10-09T05:38:56.455842+0000 mgr.y (mgr.25277) 1533 : cluster 0 pgmap v1662: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait, 103 active+clean; 2.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 923 op/s; 1020/124809 objects degraded (0.817%); 753/124809 objects misplaced (0.603%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T05:38:57.760 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:57 smithi133 bash[102858]: audit 2023-10-09T05:38:56.663186+0000 mon.c (mon.2) 141 : audit 0 from='client.? 172.21.15.133:0/3610763424' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:38:57.760 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:57 smithi133 bash[105589]: cluster 2023-10-09T05:38:56.455842+0000 mgr.y (mgr.25277) 1533 : cluster 0 pgmap v1662: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait, 103 active+clean; 2.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 923 op/s; 1020/124809 objects degraded (0.817%); 753/124809 objects misplaced (0.603%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T05:38:57.760 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:57 smithi133 bash[105589]: audit 2023-10-09T05:38:56.663186+0000 mon.c (mon.2) 141 : audit 0 from='client.? 172.21.15.133:0/3610763424' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:38:57.777 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:57.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:57.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:57.881 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:57.904 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:57.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:57.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:58.009 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:58.053 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:58.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:58.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:58.157 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:58.180 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:58.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:58.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:58.283 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:58.309 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:58.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:58.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:58.413 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:58.443 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:58.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:58.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:58.547 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:58.582 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:58.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:58.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:58.686 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:58.709 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:58.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:58.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:58.815 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:58.823 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:58 smithi133 bash[105589]: audit 2023-10-09T05:38:58.491976+0000 mon.a (mon.0) 1937 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:38:58.824 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:58 smithi133 bash[102858]: audit 2023-10-09T05:38:58.491976+0000 mon.a (mon.0) 1937 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:38:58.841 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:58.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:58.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:58.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:58 smithi194 bash[92666]: audit 2023-10-09T05:38:58.491976+0000 mon.a (mon.0) 1937 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:38:58.946 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:58.971 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:58.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:58.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:59.077 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:59.106 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:59.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:59.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:59.210 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:59.244 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:59.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:59.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:59.351 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:59.380 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:59.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:59.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:59.483 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:59.506 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:59.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:59.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:59.611 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:59.622 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:59 smithi133 bash[102858]: cluster 2023-10-09T05:38:58.456317+0000 mgr.y (mgr.25277) 1534 : cluster 0 pgmap v1663: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait, 103 active+clean; 2.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 803 op/s; 1020/123732 objects degraded (0.824%); 753/123732 objects misplaced (0.609%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T05:38:59.622 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:38:59 smithi133 bash[102858]: cluster 2023-10-09T05:38:58.922365+0000 mon.a (mon.0) 1938 : cluster 3 Health check update: Degraded data redundancy: 1020/123732 objects degraded (0.824%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:59.635 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:59.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:59.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:59.739 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:59.747 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:59 smithi133 bash[105589]: cluster 2023-10-09T05:38:58.456317+0000 mgr.y (mgr.25277) 1534 : cluster 0 pgmap v1663: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait, 103 active+clean; 2.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 803 op/s; 1020/123732 objects degraded (0.824%); 753/123732 objects misplaced (0.609%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T05:38:59.747 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:38:59 smithi133 bash[105589]: cluster 2023-10-09T05:38:58.922365+0000 mon.a (mon.0) 1938 : cluster 3 Health check update: Degraded data redundancy: 1020/123732 objects degraded (0.824%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:59.760 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:59.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:59.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:59.865 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:59.874 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:59 smithi194 bash[92666]: cluster 2023-10-09T05:38:58.456317+0000 mgr.y (mgr.25277) 1534 : cluster 0 pgmap v1663: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait, 103 active+clean; 2.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 803 op/s; 1020/123732 objects degraded (0.824%); 753/123732 objects misplaced (0.609%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T05:38:59.874 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:38:59 smithi194 bash[92666]: cluster 2023-10-09T05:38:58.922365+0000 mon.a (mon.0) 1938 : cluster 3 Health check update: Degraded data redundancy: 1020/123732 objects degraded (0.824%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:59.895 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:59.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:59.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:59.998 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:00.026 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:00.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:00.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:00.132 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:00.162 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:00.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:00.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:00.267 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:00.293 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:00.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:00.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:00.399 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:00.430 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:00.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:00.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:00.533 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:00.560 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:00.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:00.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:00.663 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:00.700 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:00.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:00.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:00.808 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:00.835 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:00.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:00.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:00.939 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:00.963 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:00.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:00.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:01.066 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:01.099 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:01.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:01.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:01.207 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:01.229 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:01.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:01.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:01.332 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:01.360 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:01.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:01.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:01.466 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:01.498 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:01.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:01.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:01.603 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:01.630 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:01.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:01.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:01.734 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:01.759 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:01.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:01.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:01.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:01 smithi133 bash[102858]: cluster 2023-10-09T05:39:00.456975+0000 mgr.y (mgr.25277) 1535 : cluster 0 pgmap v1664: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait, 103 active+clean; 2.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 937 op/s; 1020/117957 objects degraded (0.865%); 753/117957 objects misplaced (0.638%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T05:39:01.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:01 smithi133 bash[105589]: cluster 2023-10-09T05:39:00.456975+0000 mgr.y (mgr.25277) 1535 : cluster 0 pgmap v1664: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait, 103 active+clean; 2.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 937 op/s; 1020/117957 objects degraded (0.865%); 753/117957 objects misplaced (0.638%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T05:39:01.863 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:01.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:01 smithi194 bash[92666]: cluster 2023-10-09T05:39:00.456975+0000 mgr.y (mgr.25277) 1535 : cluster 0 pgmap v1664: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait, 103 active+clean; 2.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 937 op/s; 1020/117957 objects degraded (0.865%); 753/117957 objects misplaced (0.638%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T05:39:01.886 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:01.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:01.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:01.993 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:02.025 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:02.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:02.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:02.129 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:02.156 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:02.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:02.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:02.260 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:02.291 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:02.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:02.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:02.396 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:02.433 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:02.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:02.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:02.542 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:02.569 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:02.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:02.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:02.674 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:02.702 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:02.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:02.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:02.808 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:02.839 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:02.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:02.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:02.942 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:02.974 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:02.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:02.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:03.080 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:03.102 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:03.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:03.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:03.206 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:03.232 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:03.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:03.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:03.335 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:03.369 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:03.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:03.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:03.473 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:03.496 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:03.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:03.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:03.600 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:03.625 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:03.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:03.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:03.728 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:03.752 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:03.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:03.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:03.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:03 smithi133 bash[105589]: cluster 2023-10-09T05:39:02.457717+0000 mgr.y (mgr.25277) 1536 : cluster 0 pgmap v1665: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait, 103 active+clean; 2.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 975 op/s; 746/110277 objects degraded (0.676%); 173/110277 objects misplaced (0.157%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T05:39:03.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:03 smithi133 bash[105589]: cluster 2023-10-09T05:39:02.652025+0000 mon.a (mon.0) 1939 : cluster 0 osdmap e341: 8 total, 8 up, 7 in 2023-10-09T05:39:03.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:03 smithi133 bash[105589]: cluster 2023-10-09T05:39:02.656968+0000 osd.5 (osd.5) 353 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T05:39:03.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:03 smithi133 bash[105589]: cluster 2023-10-09T05:39:02.657675+0000 osd.7 (osd.7) 297 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T05:39:03.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:03 smithi133 bash[105589]: cluster 2023-10-09T05:39:02.659136+0000 osd.6 (osd.6) 322 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T05:39:03.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:03 smithi133 bash[105589]: cluster 2023-10-09T05:39:02.663758+0000 osd.6 (osd.6) 323 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T05:39:03.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:03 smithi133 bash[105589]: cluster 2023-10-09T05:39:02.667259+0000 osd.6 (osd.6) 324 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T05:39:03.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:03 smithi133 bash[102858]: cluster 2023-10-09T05:39:02.457717+0000 mgr.y (mgr.25277) 1536 : cluster 0 pgmap v1665: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait, 103 active+clean; 2.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 975 op/s; 746/110277 objects degraded (0.676%); 173/110277 objects misplaced (0.157%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T05:39:03.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:03 smithi133 bash[102858]: cluster 2023-10-09T05:39:02.652025+0000 mon.a (mon.0) 1939 : cluster 0 osdmap e341: 8 total, 8 up, 7 in 2023-10-09T05:39:03.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:03 smithi133 bash[102858]: cluster 2023-10-09T05:39:02.656968+0000 osd.5 (osd.5) 353 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T05:39:03.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:03 smithi133 bash[102858]: cluster 2023-10-09T05:39:02.657675+0000 osd.7 (osd.7) 297 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T05:39:03.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:03 smithi133 bash[102858]: cluster 2023-10-09T05:39:02.659136+0000 osd.6 (osd.6) 322 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T05:39:03.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:03 smithi133 bash[102858]: cluster 2023-10-09T05:39:02.663758+0000 osd.6 (osd.6) 323 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T05:39:03.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:03 smithi133 bash[102858]: cluster 2023-10-09T05:39:02.667259+0000 osd.6 (osd.6) 324 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T05:39:03.855 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:03.889 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:03.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:03.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:03.993 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:04.027 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:04.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:04.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:04.130 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:04.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:03 smithi194 bash[92666]: cluster 2023-10-09T05:39:02.457717+0000 mgr.y (mgr.25277) 1536 : cluster 0 pgmap v1665: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait, 103 active+clean; 2.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 975 op/s; 746/110277 objects degraded (0.676%); 173/110277 objects misplaced (0.157%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T05:39:04.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:03 smithi194 bash[92666]: cluster 2023-10-09T05:39:02.652025+0000 mon.a (mon.0) 1939 : cluster 0 osdmap e341: 8 total, 8 up, 7 in 2023-10-09T05:39:04.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:03 smithi194 bash[92666]: cluster 2023-10-09T05:39:02.656968+0000 osd.5 (osd.5) 353 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T05:39:04.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:03 smithi194 bash[92666]: cluster 2023-10-09T05:39:02.657675+0000 osd.7 (osd.7) 297 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T05:39:04.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:03 smithi194 bash[92666]: cluster 2023-10-09T05:39:02.659136+0000 osd.6 (osd.6) 322 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T05:39:04.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:03 smithi194 bash[92666]: cluster 2023-10-09T05:39:02.663758+0000 osd.6 (osd.6) 323 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T05:39:04.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:03 smithi194 bash[92666]: cluster 2023-10-09T05:39:02.667259+0000 osd.6 (osd.6) 324 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T05:39:04.157 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:04.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:04.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:04.262 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:04.289 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:04.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:04.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:04.394 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:04.402 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:39:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:39:04] "GET /metrics HTTP/1.1" 200 36500 "" "Prometheus/2.43.0" 2023-10-09T05:39:04.423 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:04.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:04.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:04.528 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:04.529 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:39:04.552 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:04.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:04.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:04.656 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:04.658 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:04 smithi133 bash[102858]: cluster 2023-10-09T05:39:03.655194+0000 osd.4 (osd.4) 303 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T05:39:04.658 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:04 smithi133 bash[102858]: cluster 2023-10-09T05:39:03.656415+0000 mon.a (mon.0) 1940 : cluster 0 osdmap e342: 8 total, 8 up, 7 in 2023-10-09T05:39:04.659 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:04 smithi133 bash[102858]: cluster 2023-10-09T05:39:03.657675+0000 osd.5 (osd.5) 354 : cluster 3 failed to encode map e342 with expected crc 2023-10-09T05:39:04.659 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:04 smithi133 bash[102858]: cluster 2023-10-09T05:39:03.659775+0000 osd.5 (osd.5) 355 : cluster 3 failed to encode map e342 with expected crc 2023-10-09T05:39:04.659 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:04 smithi133 bash[102858]: cluster 2023-10-09T05:39:03.662771+0000 osd.6 (osd.6) 325 : cluster 3 failed to encode map e342 with expected crc 2023-10-09T05:39:04.659 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:04 smithi133 bash[102858]: cluster 2023-10-09T05:39:03.923970+0000 mon.a (mon.0) 1941 : cluster 3 Health check update: Degraded data redundancy: 746/110277 objects degraded (0.676%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:04.682 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:04.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:04.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:04.787 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:04.797 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:04 smithi194 bash[92666]: cluster 2023-10-09T05:39:03.655194+0000 osd.4 (osd.4) 303 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T05:39:04.798 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:04 smithi194 bash[92666]: cluster 2023-10-09T05:39:03.656415+0000 mon.a (mon.0) 1940 : cluster 0 osdmap e342: 8 total, 8 up, 7 in 2023-10-09T05:39:04.798 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:04 smithi194 bash[92666]: cluster 2023-10-09T05:39:03.657675+0000 osd.5 (osd.5) 354 : cluster 3 failed to encode map e342 with expected crc 2023-10-09T05:39:04.798 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:04 smithi194 bash[92666]: cluster 2023-10-09T05:39:03.659775+0000 osd.5 (osd.5) 355 : cluster 3 failed to encode map e342 with expected crc 2023-10-09T05:39:04.798 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:04 smithi194 bash[92666]: cluster 2023-10-09T05:39:03.662771+0000 osd.6 (osd.6) 325 : cluster 3 failed to encode map e342 with expected crc 2023-10-09T05:39:04.798 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:04 smithi194 bash[92666]: cluster 2023-10-09T05:39:03.923970+0000 mon.a (mon.0) 1941 : cluster 3 Health check update: Degraded data redundancy: 746/110277 objects degraded (0.676%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:04.813 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:04.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:04.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:04.917 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:04.925 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:04 smithi133 bash[105589]: cluster 2023-10-09T05:39:03.655194+0000 osd.4 (osd.4) 303 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T05:39:04.926 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:04 smithi133 bash[105589]: cluster 2023-10-09T05:39:03.656415+0000 mon.a (mon.0) 1940 : cluster 0 osdmap e342: 8 total, 8 up, 7 in 2023-10-09T05:39:04.926 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:04 smithi133 bash[105589]: cluster 2023-10-09T05:39:03.657675+0000 osd.5 (osd.5) 354 : cluster 3 failed to encode map e342 with expected crc 2023-10-09T05:39:04.926 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:04 smithi133 bash[105589]: cluster 2023-10-09T05:39:03.659775+0000 osd.5 (osd.5) 355 : cluster 3 failed to encode map e342 with expected crc 2023-10-09T05:39:04.926 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:04 smithi133 bash[105589]: cluster 2023-10-09T05:39:03.662771+0000 osd.6 (osd.6) 325 : cluster 3 failed to encode map e342 with expected crc 2023-10-09T05:39:04.926 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:04 smithi133 bash[105589]: cluster 2023-10-09T05:39:03.923970+0000 mon.a (mon.0) 1941 : cluster 3 Health check update: Degraded data redundancy: 746/110277 objects degraded (0.676%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:04.954 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:04.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:04.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:05.059 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:05.086 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:05.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:05.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:05.189 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:05.224 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:05.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:05.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:05.329 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:05.355 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:05.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:05.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:05.459 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:05.494 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:05.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:05.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:05.597 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:05.626 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:05.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:05.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:05.660 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:05 smithi133 bash[102858]: cluster 2023-10-09T05:39:04.458319+0000 mgr.y (mgr.25277) 1537 : cluster 0 pgmap v1668: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait, 103 active+clean; 2.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 813 op/s; 746/105276 objects degraded (0.709%); 173/105276 objects misplaced (0.164%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:39:05.729 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:05.755 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:05.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:05.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:05.860 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:05.869 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:05 smithi194 bash[92666]: cluster 2023-10-09T05:39:04.458319+0000 mgr.y (mgr.25277) 1537 : cluster 0 pgmap v1668: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait, 103 active+clean; 2.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 813 op/s; 746/105276 objects degraded (0.709%); 173/105276 objects misplaced (0.164%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:39:05.887 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:05.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:05.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:05.991 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:06.003 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:05 smithi133 bash[105589]: cluster 2023-10-09T05:39:04.458319+0000 mgr.y (mgr.25277) 1537 : cluster 0 pgmap v1668: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait, 103 active+clean; 2.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 813 op/s; 746/105276 objects degraded (0.709%); 173/105276 objects misplaced (0.164%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:39:06.032 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:06.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:06.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:06.136 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:06.173 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:06.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:06.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:06.275 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:06.299 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:06.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:06.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:06.405 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:06.434 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:06.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:06.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:06.539 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:06.566 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:06.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:06.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:06.669 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:06.697 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:06.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:06.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:06.801 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:06.830 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:06.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:06.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:06.937 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:06.947 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:06 smithi133 bash[102858]: cluster 2023-10-09T05:39:06.459170+0000 mgr.y (mgr.25277) 1538 : cluster 0 pgmap v1669: 109 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait, 103 active+clean; 2.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s; 227/98850 objects misplaced (0.230%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T05:39:06.947 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:06 smithi133 bash[102858]: audit 2023-10-09T05:39:06.460298+0000 mon.a (mon.0) 1942 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T05:39:06.948 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:06 smithi133 bash[105589]: cluster 2023-10-09T05:39:06.459170+0000 mgr.y (mgr.25277) 1538 : cluster 0 pgmap v1669: 109 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait, 103 active+clean; 2.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s; 227/98850 objects misplaced (0.230%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T05:39:06.948 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:06 smithi133 bash[105589]: audit 2023-10-09T05:39:06.460298+0000 mon.a (mon.0) 1942 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T05:39:06.964 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:06.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:06.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:07.067 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:07.088 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:07.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:07.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:07.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:06 smithi194 bash[92666]: cluster 2023-10-09T05:39:06.459170+0000 mgr.y (mgr.25277) 1538 : cluster 0 pgmap v1669: 109 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait, 103 active+clean; 2.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s; 227/98850 objects misplaced (0.230%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T05:39:07.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:06 smithi194 bash[92666]: audit 2023-10-09T05:39:06.460298+0000 mon.a (mon.0) 1942 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T05:39:07.190 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:07.218 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:07.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:07.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:07.321 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:07.354 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:07.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:07.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:07.458 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:07.486 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:07.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:07.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:07.590 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:07.620 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:07.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:07.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:07.723 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:07.753 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:07.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:07.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:07.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:07 smithi133 bash[102858]: audit 2023-10-09T05:39:06.676219+0000 mon.a (mon.0) 1943 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T05:39:07.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:07 smithi133 bash[102858]: cluster 2023-10-09T05:39:06.676389+0000 mon.a (mon.0) 1944 : cluster 0 osdmap e343: 8 total, 8 up, 7 in 2023-10-09T05:39:07.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:07 smithi133 bash[102858]: cluster 2023-10-09T05:39:06.679964+0000 osd.6 (osd.6) 326 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T05:39:07.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:07 smithi133 bash[102858]: cluster 2023-10-09T05:39:06.681103+0000 osd.6 (osd.6) 327 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T05:39:07.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:07 smithi133 bash[102858]: cluster 2023-10-09T05:39:06.685254+0000 osd.7 (osd.7) 298 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T05:39:07.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:07 smithi133 bash[102858]: cluster 2023-10-09T05:39:06.685554+0000 osd.5 (osd.5) 356 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T05:39:07.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:07 smithi133 bash[105589]: audit 2023-10-09T05:39:06.676219+0000 mon.a (mon.0) 1943 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T05:39:07.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:07 smithi133 bash[105589]: cluster 2023-10-09T05:39:06.676389+0000 mon.a (mon.0) 1944 : cluster 0 osdmap e343: 8 total, 8 up, 7 in 2023-10-09T05:39:07.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:07 smithi133 bash[105589]: cluster 2023-10-09T05:39:06.679964+0000 osd.6 (osd.6) 326 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T05:39:07.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:07 smithi133 bash[105589]: cluster 2023-10-09T05:39:06.681103+0000 osd.6 (osd.6) 327 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T05:39:07.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:07 smithi133 bash[105589]: cluster 2023-10-09T05:39:06.685254+0000 osd.7 (osd.7) 298 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T05:39:07.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:07 smithi133 bash[105589]: cluster 2023-10-09T05:39:06.685554+0000 osd.5 (osd.5) 356 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T05:39:07.857 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:07.880 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:07.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:07 smithi194 bash[92666]: audit 2023-10-09T05:39:06.676219+0000 mon.a (mon.0) 1943 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T05:39:07.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:07 smithi194 bash[92666]: cluster 2023-10-09T05:39:06.676389+0000 mon.a (mon.0) 1944 : cluster 0 osdmap e343: 8 total, 8 up, 7 in 2023-10-09T05:39:07.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:07 smithi194 bash[92666]: cluster 2023-10-09T05:39:06.679964+0000 osd.6 (osd.6) 326 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T05:39:07.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:07 smithi194 bash[92666]: cluster 2023-10-09T05:39:06.681103+0000 osd.6 (osd.6) 327 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T05:39:07.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:07 smithi194 bash[92666]: cluster 2023-10-09T05:39:06.685254+0000 osd.7 (osd.7) 298 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T05:39:07.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:07 smithi194 bash[92666]: cluster 2023-10-09T05:39:06.685554+0000 osd.5 (osd.5) 356 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T05:39:07.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:07.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:07.988 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:08.020 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:08.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:08.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:08.123 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:08.158 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:08.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:08.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:08.264 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:08.294 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:08.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:08.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:08.398 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:08.428 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:08.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:08.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:08.532 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:08.563 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:08.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:08.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:08.671 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:08.698 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:08.698 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:08 smithi133 bash[102858]: cluster 2023-10-09T05:39:07.676275+0000 mon.a (mon.0) 1945 : cluster 0 osdmap e344: 8 total, 8 up, 7 in 2023-10-09T05:39:08.699 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:08 smithi133 bash[102858]: cluster 2023-10-09T05:39:07.680092+0000 osd.5 (osd.5) 357 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T05:39:08.699 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:08 smithi133 bash[102858]: cluster 2023-10-09T05:39:07.682084+0000 osd.6 (osd.6) 328 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T05:39:08.699 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:08 smithi133 bash[102858]: cluster 2023-10-09T05:39:07.682128+0000 osd.7 (osd.7) 299 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T05:39:08.699 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:08 smithi133 bash[102858]: cluster 2023-10-09T05:39:08.459691+0000 mgr.y (mgr.25277) 1539 : cluster 0 pgmap v1672: 109 pgs: 1 peering, 4 active+recovery_wait, 104 active+clean; 2.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 755 op/s; 4.6 MiB/s, 74 objects/s recovering 2023-10-09T05:39:08.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:08.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:08.806 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:08.850 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:08.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:08.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:08.880 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:39:08.880 INFO:teuthology.orchestra.run.smithi133.stdout: "size": 20, 2023-10-09T05:39:08.880 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 600, 2023-10-09T05:39:08.880 INFO:teuthology.orchestra.run.smithi133.stdout: "ops": [ 2023-10-09T05:39:08.880 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.881 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35775.0:66510 14.10 14:09dfeace:::benchmark_data_smithi133_267384_object66509:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e322)", 2023-10-09T05:39:08.881 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:36:58.892009+0000", 2023-10-09T05:39:08.881 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 129.982506274, 2023-10-09T05:39:08.881 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.2699861109999999, 2023-10-09T05:39:08.881 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:39:08.881 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:08.881 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:39:08.881 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35775", 2023-10-09T05:39:08.881 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:39:08.881 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 66510 2023-10-09T05:39:08.882 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.882 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:39:08.882 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.882 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:39:08.882 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892009+0000", 2023-10-09T05:39:08.882 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:08.882 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.882 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.882 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:39:08.883 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892009+0000", 2023-10-09T05:39:08.883 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:08.883 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.883 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.883 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:39:08.883 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892008+0000", 2023-10-09T05:39:08.883 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999995 2023-10-09T05:39:08.883 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.883 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.883 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:39:08.883 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892030+0000", 2023-10-09T05:39:08.884 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.2277999999999999e-05 2023-10-09T05:39:08.884 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.884 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.884 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:39:08.884 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892037+0000", 2023-10-09T05:39:08.884 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.5529999999999998e-06 2023-10-09T05:39:08.884 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.884 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.884 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:39:08.884 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892042+0000", 2023-10-09T05:39:08.885 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.0370000000000002e-06 2023-10-09T05:39:08.885 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.885 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.885 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.885 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892142+0000", 2023-10-09T05:39:08.885 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.9840999999999995e-05 2023-10-09T05:39:08.885 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.885 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.885 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting_for_map not empty", 2023-10-09T05:39:08.885 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892146+0000", 2023-10-09T05:39:08.885 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.3880000000000002e-06 2023-10-09T05:39:08.885 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.885 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.886 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.886 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.894335+0000", 2023-10-09T05:39:08.886 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0021892169999999998 2023-10-09T05:39:08.886 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.886 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.886 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:39:08.886 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.894342+0000", 2023-10-09T05:39:08.886 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.6930000000000001e-06 2023-10-09T05:39:08.886 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.886 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.887 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.887 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.141623+0000", 2023-10-09T05:39:08.887 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.2472812099999999 2023-10-09T05:39:08.887 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.887 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.887 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:39:08.887 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.141786+0000", 2023-10-09T05:39:08.887 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00016233900000000001 2023-10-09T05:39:08.887 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.887 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.887 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 1,2,7", 2023-10-09T05:39:08.887 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.141985+0000", 2023-10-09T05:39:08.888 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00019933900000000001 2023-10-09T05:39:08.888 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.888 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.888 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:39:08.888 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.147371+0000", 2023-10-09T05:39:08.888 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0053859290000000002 2023-10-09T05:39:08.888 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.888 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.888 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.888 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.147535+0000", 2023-10-09T05:39:08.888 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00016411400000000001 2023-10-09T05:39:08.888 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.889 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.889 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.889 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.160575+0000", 2023-10-09T05:39:08.889 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.013040144 2023-10-09T05:39:08.889 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.889 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.889 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.889 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.160855+0000", 2023-10-09T05:39:08.889 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000280294 2023-10-09T05:39:08.889 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.889 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.889 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:39:08.890 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.160897+0000", 2023-10-09T05:39:08.890 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.1418000000000002e-05 2023-10-09T05:39:08.890 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.890 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.890 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:39:08.890 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.161995+0000", 2023-10-09T05:39:08.890 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.001097879 2023-10-09T05:39:08.890 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:08.890 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:39:08.890 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:08.890 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.890 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.891 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35775.0:66600 14.10 14:0a7576d4:::benchmark_data_smithi133_267384_object66599:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e322)", 2023-10-09T05:39:08.891 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:36:58.892061+0000", 2023-10-09T05:39:08.891 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 129.98245422400001, 2023-10-09T05:39:08.891 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.2716371099999999, 2023-10-09T05:39:08.891 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:39:08.891 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:08.891 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:39:08.891 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35775", 2023-10-09T05:39:08.891 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:39:08.891 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 66600 2023-10-09T05:39:08.891 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.891 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:39:08.892 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.892 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:39:08.892 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892061+0000", 2023-10-09T05:39:08.892 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:08.892 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.892 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.892 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:39:08.892 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892061+0000", 2023-10-09T05:39:08.892 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:08.892 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.892 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.892 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:39:08.892 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892060+0000", 2023-10-09T05:39:08.893 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999995 2023-10-09T05:39:08.893 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.893 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.893 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:39:08.893 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892084+0000", 2023-10-09T05:39:08.893 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.3387000000000002e-05 2023-10-09T05:39:08.893 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.893 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.893 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:39:08.893 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892090+0000", 2023-10-09T05:39:08.893 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.6159999999999998e-06 2023-10-09T05:39:08.893 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.894 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.894 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:39:08.894 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892096+0000", 2023-10-09T05:39:08.894 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.0249999999999999e-06 2023-10-09T05:39:08.894 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.894 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.894 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.894 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892171+0000", 2023-10-09T05:39:08.894 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.4323e-05 2023-10-09T05:39:08.894 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.894 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.894 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting_for_map not empty", 2023-10-09T05:39:08.895 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892175+0000", 2023-10-09T05:39:08.895 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.3939999999999998e-06 2023-10-09T05:39:08.895 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.895 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.895 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.895 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.894365+0000", 2023-10-09T05:39:08.895 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.002190081 2023-10-09T05:39:08.895 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.895 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.895 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:39:08.895 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.894372+0000", 2023-10-09T05:39:08.895 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.8920000000000003e-06 2023-10-09T05:39:08.895 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.896 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.896 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.896 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.143159+0000", 2023-10-09T05:39:08.896 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.2487875159999999 2023-10-09T05:39:08.896 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.896 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.896 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:39:08.896 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.143328+0000", 2023-10-09T05:39:08.899 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000168374 2023-10-09T05:39:08.899 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.899 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.899 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 1,2,7", 2023-10-09T05:39:08.899 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.143498+0000", 2023-10-09T05:39:08.899 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000169968 2023-10-09T05:39:08.899 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.899 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.899 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:39:08.899 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.147380+0000", 2023-10-09T05:39:08.900 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0038826759999999998 2023-10-09T05:39:08.900 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.900 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.900 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.900 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.150634+0000", 2023-10-09T05:39:08.900 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0032535120000000001 2023-10-09T05:39:08.900 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.900 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.900 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.900 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.160606+0000", 2023-10-09T05:39:08.900 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0099722600000000002 2023-10-09T05:39:08.900 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.900 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.901 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.901 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.162368+0000", 2023-10-09T05:39:08.901 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.001761592 2023-10-09T05:39:08.901 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.901 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.901 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:39:08.901 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.162426+0000", 2023-10-09T05:39:08.901 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.8572000000000001e-05 2023-10-09T05:39:08.901 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.901 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.901 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:39:08.901 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.163698+0000", 2023-10-09T05:39:08.902 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.001271356 2023-10-09T05:39:08.902 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:08.902 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:39:08.902 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:08.902 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.902 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.902 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35775.0:66615 14.10 14:0ce83358:::benchmark_data_smithi133_267384_object66614:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e322)", 2023-10-09T05:39:08.902 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:36:58.892116+0000", 2023-10-09T05:39:08.902 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 129.98239863200001, 2023-10-09T05:39:08.902 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.2731761150000001, 2023-10-09T05:39:08.902 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:39:08.902 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:08.903 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:39:08.903 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35775", 2023-10-09T05:39:08.903 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:39:08.903 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 66615 2023-10-09T05:39:08.903 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.903 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:39:08.903 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.903 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:39:08.903 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892116+0000", 2023-10-09T05:39:08.903 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:08.903 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.903 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.904 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:39:08.904 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892116+0000", 2023-10-09T05:39:08.904 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:08.904 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.904 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.904 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:39:08.904 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892116+0000", 2023-10-09T05:39:08.904 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999995 2023-10-09T05:39:08.904 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.904 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.904 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:39:08.904 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892138+0000", 2023-10-09T05:39:08.905 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.1837000000000001e-05 2023-10-09T05:39:08.905 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.905 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.905 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:39:08.905 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892144+0000", 2023-10-09T05:39:08.905 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.5379999999999999e-06 2023-10-09T05:39:08.905 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.905 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.905 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:39:08.905 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892150+0000", 2023-10-09T05:39:08.905 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.4709999999999998e-06 2023-10-09T05:39:08.905 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.906 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.906 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.906 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892195+0000", 2023-10-09T05:39:08.906 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.5145999999999998e-05 2023-10-09T05:39:08.906 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.906 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.906 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting_for_map not empty", 2023-10-09T05:39:08.906 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892199+0000", 2023-10-09T05:39:08.906 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.4730000000000002e-06 2023-10-09T05:39:08.906 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.906 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.906 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.907 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.894395+0000", 2023-10-09T05:39:08.907 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.002196253 2023-10-09T05:39:08.907 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.907 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.907 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:39:08.907 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.894402+0000", 2023-10-09T05:39:08.907 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.7009999999999999e-06 2023-10-09T05:39:08.907 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.907 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.908 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.908 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.144612+0000", 2023-10-09T05:39:08.908 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.250209999 2023-10-09T05:39:08.908 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.908 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.908 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:39:08.908 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.144772+0000", 2023-10-09T05:39:08.908 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000159715 2023-10-09T05:39:08.908 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.908 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.909 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 1,2,7", 2023-10-09T05:39:08.909 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.144944+0000", 2023-10-09T05:39:08.909 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00017210500000000001 2023-10-09T05:39:08.909 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.909 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.909 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:39:08.909 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.147546+0000", 2023-10-09T05:39:08.909 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0026021299999999998 2023-10-09T05:39:08.909 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.909 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.909 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.909 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.150660+0000", 2023-10-09T05:39:08.909 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0031138149999999998 2023-10-09T05:39:08.910 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.910 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.910 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.910 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.160646+0000", 2023-10-09T05:39:08.910 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0099859519999999993 2023-10-09T05:39:08.910 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.910 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.910 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.910 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.163788+0000", 2023-10-09T05:39:08.910 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0031419210000000002 2023-10-09T05:39:08.910 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.910 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.911 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:39:08.911 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.163870+0000", 2023-10-09T05:39:08.911 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.1784000000000002e-05 2023-10-09T05:39:08.911 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.911 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.911 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:39:08.911 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.165292+0000", 2023-10-09T05:39:08.911 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00142268 2023-10-09T05:39:08.911 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:08.911 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:39:08.911 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:08.911 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.912 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.912 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35775.0:66690 14.10 14:0ea502a9:::benchmark_data_smithi133_267384_object66689:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e322)", 2023-10-09T05:39:08.912 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:36:58.892169+0000", 2023-10-09T05:39:08.912 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 129.98234577700001, 2023-10-09T05:39:08.912 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.278409036, 2023-10-09T05:39:08.912 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:39:08.912 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:08.912 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:39:08.912 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35775", 2023-10-09T05:39:08.912 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:39:08.913 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 66690 2023-10-09T05:39:08.913 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.913 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:39:08.913 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.913 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:39:08.913 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892169+0000", 2023-10-09T05:39:08.913 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:08.913 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.913 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.913 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:39:08.913 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892169+0000", 2023-10-09T05:39:08.914 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:08.914 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.914 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.914 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:39:08.914 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892169+0000", 2023-10-09T05:39:08.914 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999995 2023-10-09T05:39:08.914 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.914 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.915 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:39:08.915 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892191+0000", 2023-10-09T05:39:08.915 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.2578999999999998e-05 2023-10-09T05:39:08.915 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.915 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.915 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:39:08.915 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892198+0000", 2023-10-09T05:39:08.915 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.4899999999999997e-06 2023-10-09T05:39:08.915 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.916 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.916 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:39:08.916 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892203+0000", 2023-10-09T05:39:08.917 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.1819999999999997e-06 2023-10-09T05:39:08.917 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.917 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.917 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.917 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892250+0000", 2023-10-09T05:39:08.917 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.7091e-05 2023-10-09T05:39:08.917 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.917 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.917 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting_for_map not empty", 2023-10-09T05:39:08.917 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892255+0000", 2023-10-09T05:39:08.917 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.6280000000000004e-06 2023-10-09T05:39:08.918 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.918 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.918 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.918 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.894426+0000", 2023-10-09T05:39:08.918 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0021716270000000002 2023-10-09T05:39:08.918 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.918 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.918 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:39:08.918 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.894433+0000", 2023-10-09T05:39:08.918 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.5509999999999996e-06 2023-10-09T05:39:08.918 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.919 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.919 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.919 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.146078+0000", 2023-10-09T05:39:08.919 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.2516449380000001 2023-10-09T05:39:08.919 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.919 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.919 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:39:08.919 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.146250+0000", 2023-10-09T05:39:08.919 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00017212799999999999 2023-10-09T05:39:08.919 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.919 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.919 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 1,2,7", 2023-10-09T05:39:08.919 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.146432+0000", 2023-10-09T05:39:08.920 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00018165 2023-10-09T05:39:08.920 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.920 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.920 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:39:08.920 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.150606+0000", 2023-10-09T05:39:08.920 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0041747470000000004 2023-10-09T05:39:08.920 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.920 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.920 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.920 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.150680+0000", 2023-10-09T05:39:08.920 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.3325999999999996e-05 2023-10-09T05:39:08.920 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.921 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.921 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.921 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.160701+0000", 2023-10-09T05:39:08.921 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.010021831 2023-10-09T05:39:08.921 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.921 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.921 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.921 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.168937+0000", 2023-10-09T05:39:08.921 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.008235582 2023-10-09T05:39:08.921 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.921 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.921 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:39:08.922 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.169001+0000", 2023-10-09T05:39:08.922 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.4363999999999996e-05 2023-10-09T05:39:08.922 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.922 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.922 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:39:08.922 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.170578+0000", 2023-10-09T05:39:08.922 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0015767050000000001 2023-10-09T05:39:08.922 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:08.922 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:39:08.922 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:08.922 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.922 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.923 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35775.0:66720 14.10 14:096700a5:::benchmark_data_smithi133_267384_object66719:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e322)", 2023-10-09T05:39:08.923 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:36:58.892221+0000", 2023-10-09T05:39:08.923 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 129.98229406499999, 2023-10-09T05:39:08.923 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.2796633019999999, 2023-10-09T05:39:08.923 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:39:08.923 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:08.923 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:39:08.923 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35775", 2023-10-09T05:39:08.923 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:39:08.923 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 66720 2023-10-09T05:39:08.923 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.923 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:39:08.924 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.924 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:39:08.924 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892221+0000", 2023-10-09T05:39:08.924 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:08.924 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.924 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.924 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:39:08.924 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892221+0000", 2023-10-09T05:39:08.924 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:08.924 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.924 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.924 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:39:08.924 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892220+0000", 2023-10-09T05:39:08.925 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999995 2023-10-09T05:39:08.925 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.925 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.925 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:39:08.925 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892244+0000", 2023-10-09T05:39:08.925 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.404e-05 2023-10-09T05:39:08.925 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.925 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.925 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:39:08.925 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892251+0000", 2023-10-09T05:39:08.925 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.8839999999999997e-06 2023-10-09T05:39:08.925 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.926 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.926 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:39:08.926 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892256+0000", 2023-10-09T05:39:08.926 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.1429999999999997e-06 2023-10-09T05:39:08.926 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.926 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.926 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.926 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892287+0000", 2023-10-09T05:39:08.926 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.0164000000000001e-05 2023-10-09T05:39:08.926 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.926 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.926 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting_for_map not empty", 2023-10-09T05:39:08.927 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892291+0000", 2023-10-09T05:39:08.927 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.9230000000000001e-06 2023-10-09T05:39:08.927 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.927 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.927 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.927 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.894457+0000", 2023-10-09T05:39:08.927 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0021654529999999999 2023-10-09T05:39:08.927 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.927 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.927 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:39:08.927 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.894464+0000", 2023-10-09T05:39:08.927 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.635e-06 2023-10-09T05:39:08.928 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.928 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.928 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.928 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.147606+0000", 2023-10-09T05:39:08.928 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.2531424310000001 2023-10-09T05:39:08.928 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.928 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.928 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:39:08.928 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.147804+0000", 2023-10-09T05:39:08.928 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000197751 2023-10-09T05:39:08.928 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.928 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.928 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 1,2,7", 2023-10-09T05:39:08.929 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.148177+0000", 2023-10-09T05:39:08.929 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000372927 2023-10-09T05:39:08.929 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.929 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.929 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:39:08.929 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.150639+0000", 2023-10-09T05:39:08.929 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0024625239999999998 2023-10-09T05:39:08.929 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.929 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.929 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.929 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.150774+0000", 2023-10-09T05:39:08.929 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00013438999999999999 2023-10-09T05:39:08.930 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.930 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.930 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.930 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.160748+0000", 2023-10-09T05:39:08.930 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0099741129999999997 2023-10-09T05:39:08.930 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.930 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.930 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.930 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.170639+0000", 2023-10-09T05:39:08.930 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0098917520000000002 2023-10-09T05:39:08.930 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.930 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.931 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:39:08.931 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.170694+0000", 2023-10-09T05:39:08.931 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.4731000000000002e-05 2023-10-09T05:39:08.931 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.931 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.931 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:39:08.931 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.171884+0000", 2023-10-09T05:39:08.931 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.001189863 2023-10-09T05:39:08.931 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:08.931 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:39:08.931 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:08.931 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.931 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.932 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35775.0:66743 14.10 14:08bb0852:::benchmark_data_smithi133_267384_object66742:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e322)", 2023-10-09T05:39:08.932 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:36:58.892265+0000", 2023-10-09T05:39:08.932 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 129.98224992900001, 2023-10-09T05:39:08.932 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.2972036309999999, 2023-10-09T05:39:08.932 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:39:08.932 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:08.932 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:39:08.932 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35775", 2023-10-09T05:39:08.932 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:39:08.932 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 66743 2023-10-09T05:39:08.932 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.933 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:39:08.933 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.933 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:39:08.933 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892265+0000", 2023-10-09T05:39:08.933 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:08.933 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.933 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.933 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:39:08.933 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892265+0000", 2023-10-09T05:39:08.933 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:08.933 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.933 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.934 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:39:08.934 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892264+0000", 2023-10-09T05:39:08.934 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999995 2023-10-09T05:39:08.934 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.934 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.934 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:39:08.934 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892287+0000", 2023-10-09T05:39:08.934 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.2554000000000001e-05 2023-10-09T05:39:08.934 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.934 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.934 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:39:08.934 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892293+0000", 2023-10-09T05:39:08.935 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.1870000000000002e-06 2023-10-09T05:39:08.935 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.935 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.935 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:39:08.935 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892298+0000", 2023-10-09T05:39:08.935 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.9729999999999997e-06 2023-10-09T05:39:08.935 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.935 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.935 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.935 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892326+0000", 2023-10-09T05:39:08.935 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.8059999999999999e-05 2023-10-09T05:39:08.935 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.935 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.936 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting_for_map not empty", 2023-10-09T05:39:08.936 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.892331+0000", 2023-10-09T05:39:08.936 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.7169999999999999e-06 2023-10-09T05:39:08.936 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.936 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.936 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.936 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.894498+0000", 2023-10-09T05:39:08.936 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00216709 2023-10-09T05:39:08.936 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.936 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.936 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:39:08.936 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:36:58.894505+0000", 2023-10-09T05:39:08.937 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.3270000000000003e-06 2023-10-09T05:39:08.937 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.937 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.937 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.937 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.149450+0000", 2023-10-09T05:39:08.937 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.2549450609999999 2023-10-09T05:39:08.937 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.937 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.937 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:39:08.937 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.149594+0000", 2023-10-09T05:39:08.937 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000143771 2023-10-09T05:39:08.937 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.938 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.938 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 1,2,7", 2023-10-09T05:39:08.938 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.149702+0000", 2023-10-09T05:39:08.938 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00010822100000000001 2023-10-09T05:39:08.938 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.938 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.938 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:39:08.938 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.153308+0000", 2023-10-09T05:39:08.938 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0036053700000000001 2023-10-09T05:39:08.938 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.938 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.938 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.939 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.153392+0000", 2023-10-09T05:39:08.939 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.3756000000000002e-05 2023-10-09T05:39:08.939 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.939 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.939 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.939 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.165378+0000", 2023-10-09T05:39:08.939 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.011986358000000001 2023-10-09T05:39:08.939 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.939 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.939 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.939 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.187462+0000", 2023-10-09T05:39:08.939 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.022083645999999998 2023-10-09T05:39:08.940 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.940 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.940 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:39:08.940 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.187519+0000", 2023-10-09T05:39:08.940 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.7252e-05 2023-10-09T05:39:08.940 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.940 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.940 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:39:08.940 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:00.189469+0000", 2023-10-09T05:39:08.940 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0019496750000000001 2023-10-09T05:39:08.940 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:08.940 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:39:08.940 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:08.941 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.941 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.941 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35775.0:76475 14.13 14:cf1fcee7:::benchmark_data_smithi133_267384_object76474:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T05:39:08.941 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:37:12.037694+0000", 2023-10-09T05:39:08.941 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 116.83682044699999, 2023-10-09T05:39:08.941 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.488639995, 2023-10-09T05:39:08.941 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:39:08.941 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:08.941 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:39:08.941 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35775", 2023-10-09T05:39:08.941 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:39:08.941 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 76475 2023-10-09T05:39:08.942 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.942 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:39:08.942 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.942 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:39:08.942 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.037694+0000", 2023-10-09T05:39:08.942 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:08.942 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.942 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.942 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:39:08.942 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.037694+0000", 2023-10-09T05:39:08.942 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:08.942 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.943 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.943 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:39:08.943 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.037693+0000", 2023-10-09T05:39:08.943 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:39:08.943 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.943 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.943 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:39:08.943 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.037791+0000", 2023-10-09T05:39:08.943 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.7294999999999996e-05 2023-10-09T05:39:08.943 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.943 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.943 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:39:08.943 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.037808+0000", 2023-10-09T05:39:08.944 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.7799999999999999e-05 2023-10-09T05:39:08.944 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.944 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.944 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:39:08.944 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.037822+0000", 2023-10-09T05:39:08.944 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.3465000000000001e-05 2023-10-09T05:39:08.944 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.944 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.944 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.944 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.038527+0000", 2023-10-09T05:39:08.944 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00070549899999999995 2023-10-09T05:39:08.944 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.945 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.945 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op must wait for map", 2023-10-09T05:39:08.945 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.038545+0000", 2023-10-09T05:39:08.945 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.8031000000000001e-05 2023-10-09T05:39:08.945 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.945 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.945 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.945 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.045191+0000", 2023-10-09T05:39:08.945 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0066453099999999998 2023-10-09T05:39:08.945 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.945 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.945 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:39:08.945 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.045213+0000", 2023-10-09T05:39:08.946 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.1926999999999999e-05 2023-10-09T05:39:08.946 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.946 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.946 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.946 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.508709+0000", 2023-10-09T05:39:08.946 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.463496159 2023-10-09T05:39:08.946 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.946 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.946 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:39:08.946 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.508912+0000", 2023-10-09T05:39:08.946 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000202881 2023-10-09T05:39:08.946 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.947 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.947 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T05:39:08.947 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.509102+0000", 2023-10-09T05:39:08.947 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00019044900000000001 2023-10-09T05:39:08.947 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.947 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.947 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:39:08.947 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.517594+0000", 2023-10-09T05:39:08.947 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0084915630000000006 2023-10-09T05:39:08.947 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.947 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.947 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.948 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.517655+0000", 2023-10-09T05:39:08.948 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.1241999999999994e-05 2023-10-09T05:39:08.948 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.948 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.948 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.948 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.524420+0000", 2023-10-09T05:39:08.948 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0067653699999999997 2023-10-09T05:39:08.948 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.948 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.948 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.948 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.524553+0000", 2023-10-09T05:39:08.949 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00013306700000000001 2023-10-09T05:39:08.949 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.949 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.949 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:39:08.949 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.524604+0000", 2023-10-09T05:39:08.949 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.0513000000000002e-05 2023-10-09T05:39:08.949 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.949 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.949 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:39:08.949 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.526334+0000", 2023-10-09T05:39:08.949 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.001730526 2023-10-09T05:39:08.949 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:08.949 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:39:08.950 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:08.950 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.950 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.950 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35775.0:76477 14.13 14:cb71fb63:::benchmark_data_smithi133_267384_object76476:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T05:39:08.950 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:37:12.037844+0000", 2023-10-09T05:39:08.950 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 116.836671263, 2023-10-09T05:39:08.950 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.4980485219999999, 2023-10-09T05:39:08.950 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:39:08.950 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:08.950 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:39:08.950 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35775", 2023-10-09T05:39:08.950 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:39:08.951 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 76477 2023-10-09T05:39:08.951 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.951 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:39:08.951 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.951 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:39:08.951 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.037844+0000", 2023-10-09T05:39:08.951 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:08.951 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.951 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.951 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:39:08.951 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.037844+0000", 2023-10-09T05:39:08.951 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:08.952 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.952 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.952 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:39:08.952 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.037843+0000", 2023-10-09T05:39:08.952 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:39:08.952 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.952 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.952 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:39:08.952 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.037937+0000", 2023-10-09T05:39:08.952 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.4739000000000002e-05 2023-10-09T05:39:08.952 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.952 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.953 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:39:08.953 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.037954+0000", 2023-10-09T05:39:08.953 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.6211000000000001e-05 2023-10-09T05:39:08.953 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.953 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.953 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:39:08.953 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.037966+0000", 2023-10-09T05:39:08.953 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.2543e-05 2023-10-09T05:39:08.953 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.953 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.953 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.953 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.038667+0000", 2023-10-09T05:39:08.954 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000701185 2023-10-09T05:39:08.954 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.954 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.954 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting_for_map not empty", 2023-10-09T05:39:08.954 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.038679+0000", 2023-10-09T05:39:08.954 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.1426999999999999e-05 2023-10-09T05:39:08.954 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.954 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.954 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.954 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.049748+0000", 2023-10-09T05:39:08.954 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.011069281 2023-10-09T05:39:08.954 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.955 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.955 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:39:08.955 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.049761+0000", 2023-10-09T05:39:08.955 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.3342e-05 2023-10-09T05:39:08.955 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.955 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.955 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.955 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.511860+0000", 2023-10-09T05:39:08.955 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.462098321 2023-10-09T05:39:08.955 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.955 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.955 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:39:08.955 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.512071+0000", 2023-10-09T05:39:08.956 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000210924 2023-10-09T05:39:08.956 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.956 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.956 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T05:39:08.956 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.512270+0000", 2023-10-09T05:39:08.956 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00019986599999999999 2023-10-09T05:39:08.956 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.956 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.956 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:39:08.956 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.524335+0000", 2023-10-09T05:39:08.956 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.012064470000000001 2023-10-09T05:39:08.956 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.957 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.957 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.957 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.524484+0000", 2023-10-09T05:39:08.957 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00014883299999999999 2023-10-09T05:39:08.957 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.957 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.957 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.957 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.526416+0000", 2023-10-09T05:39:08.957 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0019319750000000001 2023-10-09T05:39:08.957 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.957 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.957 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.958 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.533890+0000", 2023-10-09T05:39:08.958 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0074746860000000004 2023-10-09T05:39:08.958 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.958 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.958 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:39:08.958 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.533946+0000", 2023-10-09T05:39:08.958 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.5690999999999999e-05 2023-10-09T05:39:08.958 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.958 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.958 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:39:08.958 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.535892+0000", 2023-10-09T05:39:08.958 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0019460199999999999 2023-10-09T05:39:08.959 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:08.959 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:39:08.959 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:08.959 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.959 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.959 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35775.0:76731 14.13 14:cf55af91:::benchmark_data_smithi133_267384_object76730:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T05:39:08.959 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:37:12.037977+0000", 2023-10-09T05:39:08.959 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 116.836538171, 2023-10-09T05:39:08.959 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.4995373060000001, 2023-10-09T05:39:08.959 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:39:08.959 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:08.959 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:39:08.960 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35775", 2023-10-09T05:39:08.960 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:39:08.960 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 76731 2023-10-09T05:39:08.960 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.960 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:39:08.960 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.960 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:39:08.960 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.037977+0000", 2023-10-09T05:39:08.960 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:08.960 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.960 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.960 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:39:08.960 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.037977+0000", 2023-10-09T05:39:08.961 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:08.961 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.961 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.961 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:39:08.961 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.037976+0000", 2023-10-09T05:39:08.961 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:39:08.961 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.961 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.961 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:39:08.961 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.038070+0000", 2023-10-09T05:39:08.961 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.4067000000000006e-05 2023-10-09T05:39:08.961 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.961 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.962 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:39:08.962 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.038085+0000", 2023-10-09T05:39:08.962 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.5645e-05 2023-10-09T05:39:08.962 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.962 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.962 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:39:08.962 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.038098+0000", 2023-10-09T05:39:08.962 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.2203e-05 2023-10-09T05:39:08.962 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.962 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.962 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.963 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.038733+0000", 2023-10-09T05:39:08.963 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00063517599999999999 2023-10-09T05:39:08.963 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.963 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.963 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting_for_map not empty", 2023-10-09T05:39:08.963 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.038743+0000", 2023-10-09T05:39:08.963 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.0712e-05 2023-10-09T05:39:08.963 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.963 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.963 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.963 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.049786+0000", 2023-10-09T05:39:08.963 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.011043017 2023-10-09T05:39:08.964 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.964 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.964 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:39:08.964 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.049794+0000", 2023-10-09T05:39:08.964 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.6450000000000005e-06 2023-10-09T05:39:08.964 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.964 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.964 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.964 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.513716+0000", 2023-10-09T05:39:08.964 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.4639221199999999 2023-10-09T05:39:08.964 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.964 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.964 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:39:08.965 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.513936+0000", 2023-10-09T05:39:08.965 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000220125 2023-10-09T05:39:08.965 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.965 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.965 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T05:39:08.965 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.514158+0000", 2023-10-09T05:39:08.965 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000221905 2023-10-09T05:39:08.965 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.965 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.965 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:39:08.965 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.524346+0000", 2023-10-09T05:39:08.965 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.010187408 2023-10-09T05:39:08.966 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.966 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.966 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.966 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.530190+0000", 2023-10-09T05:39:08.966 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0058438470000000001 2023-10-09T05:39:08.966 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.966 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.966 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.966 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.530252+0000", 2023-10-09T05:39:08.966 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.2471999999999994e-05 2023-10-09T05:39:08.966 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.966 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.967 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.967 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.535963+0000", 2023-10-09T05:39:08.967 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.005711369 2023-10-09T05:39:08.967 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.967 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.967 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:39:08.967 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.536033+0000", 2023-10-09T05:39:08.967 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.9380000000000003e-05 2023-10-09T05:39:08.967 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.967 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.967 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:39:08.967 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.537514+0000", 2023-10-09T05:39:08.968 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0014812149999999999 2023-10-09T05:39:08.968 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:08.968 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:39:08.968 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:08.968 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.968 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.968 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35775.0:76834 14.13 14:cf9caae7:::benchmark_data_smithi133_267384_object76833:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T05:39:08.968 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:37:12.038108+0000", 2023-10-09T05:39:08.968 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 116.83640669899999, 2023-10-09T05:39:08.968 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.501464551, 2023-10-09T05:39:08.968 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:39:08.968 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:08.969 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:39:08.969 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35775", 2023-10-09T05:39:08.969 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:39:08.969 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 76834 2023-10-09T05:39:08.969 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.969 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:39:08.969 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.969 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:39:08.969 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.038108+0000", 2023-10-09T05:39:08.969 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:08.969 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.969 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.970 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:39:08.970 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.038108+0000", 2023-10-09T05:39:08.970 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:08.970 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.970 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.970 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:39:08.970 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.038107+0000", 2023-10-09T05:39:08.970 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:39:08.970 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.970 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.970 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:39:08.970 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.038853+0000", 2023-10-09T05:39:08.971 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000745654 2023-10-09T05:39:08.971 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.971 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.971 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:39:08.971 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.038881+0000", 2023-10-09T05:39:08.971 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.8348e-05 2023-10-09T05:39:08.971 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.971 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.971 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:39:08.971 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.038900+0000", 2023-10-09T05:39:08.971 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.8496999999999999e-05 2023-10-09T05:39:08.971 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.972 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.972 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.972 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.049817+0000", 2023-10-09T05:39:08.972 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.010917484999999999 2023-10-09T05:39:08.972 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.972 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.972 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:39:08.972 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.049824+0000", 2023-10-09T05:39:08.972 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.0040000000000002e-06 2023-10-09T05:39:08.972 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.972 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.972 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.972 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.515816+0000", 2023-10-09T05:39:08.973 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.465992159 2023-10-09T05:39:08.973 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.973 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.973 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:39:08.973 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.516025+0000", 2023-10-09T05:39:08.973 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00020837399999999999 2023-10-09T05:39:08.973 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.973 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.973 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T05:39:08.973 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.516270+0000", 2023-10-09T05:39:08.973 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000245049 2023-10-09T05:39:08.973 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.974 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.974 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:39:08.974 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.524496+0000", 2023-10-09T05:39:08.974 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0082267959999999998 2023-10-09T05:39:08.974 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.974 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.974 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.974 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.530308+0000", 2023-10-09T05:39:08.974 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0058113460000000002 2023-10-09T05:39:08.974 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.974 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.974 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.975 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.530383+0000", 2023-10-09T05:39:08.975 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.5180999999999997e-05 2023-10-09T05:39:08.975 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.975 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.975 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.975 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.537726+0000", 2023-10-09T05:39:08.975 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0073427620000000001 2023-10-09T05:39:08.975 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.975 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.975 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:39:08.975 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.537791+0000", 2023-10-09T05:39:08.975 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.5401000000000006e-05 2023-10-09T05:39:08.976 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.976 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.976 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:39:08.976 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.539573+0000", 2023-10-09T05:39:08.976 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0017815210000000001 2023-10-09T05:39:08.976 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:08.976 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:39:08.976 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:08.976 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.977 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.977 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35775.0:76841 14.13 14:ca2a818a:::benchmark_data_smithi133_267384_object76840:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T05:39:08.977 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:37:12.038930+0000", 2023-10-09T05:39:08.977 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 116.835584575, 2023-10-09T05:39:08.977 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.502841648, 2023-10-09T05:39:08.977 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:39:08.977 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:08.977 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:39:08.977 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35775", 2023-10-09T05:39:08.977 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:39:08.977 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 76841 2023-10-09T05:39:08.978 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.978 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:39:08.978 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.978 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:39:08.978 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.038930+0000", 2023-10-09T05:39:08.978 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:08.978 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.978 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.978 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:39:08.978 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.038930+0000", 2023-10-09T05:39:08.978 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:08.978 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.979 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.979 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:39:08.979 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.038929+0000", 2023-10-09T05:39:08.979 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:39:08.979 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.979 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.979 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:39:08.979 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039038+0000", 2023-10-09T05:39:08.979 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00010893 2023-10-09T05:39:08.979 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.979 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.979 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:39:08.980 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039059+0000", 2023-10-09T05:39:08.980 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.1780000000000002e-05 2023-10-09T05:39:08.980 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.980 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.980 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:39:08.980 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039076+0000", 2023-10-09T05:39:08.980 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.6545000000000002e-05 2023-10-09T05:39:08.980 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.980 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.980 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.980 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.049846+0000", 2023-10-09T05:39:08.980 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.010769562999999999 2023-10-09T05:39:08.981 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.981 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.981 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:39:08.981 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.049852+0000", 2023-10-09T05:39:08.981 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.4910000000000002e-06 2023-10-09T05:39:08.981 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.981 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.981 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.981 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.517677+0000", 2023-10-09T05:39:08.981 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.467825156 2023-10-09T05:39:08.981 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.981 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.982 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:39:08.982 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.517873+0000", 2023-10-09T05:39:08.982 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000195964 2023-10-09T05:39:08.982 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.982 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.982 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T05:39:08.982 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.518111+0000", 2023-10-09T05:39:08.982 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00023771299999999999 2023-10-09T05:39:08.982 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.982 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.982 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:39:08.982 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.524507+0000", 2023-10-09T05:39:08.983 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0063956849999999999 2023-10-09T05:39:08.983 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.983 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.983 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.983 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.537629+0000", 2023-10-09T05:39:08.983 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.013122002000000001 2023-10-09T05:39:08.983 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.983 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.983 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.983 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.537687+0000", 2023-10-09T05:39:08.983 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.8334000000000001e-05 2023-10-09T05:39:08.984 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.984 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.984 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.984 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.539698+0000", 2023-10-09T05:39:08.984 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.002010581 2023-10-09T05:39:08.984 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.984 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.984 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:39:08.984 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.539761+0000", 2023-10-09T05:39:08.984 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.3713999999999993e-05 2023-10-09T05:39:08.984 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.984 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.984 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:39:08.985 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.541772+0000", 2023-10-09T05:39:08.985 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.002010642 2023-10-09T05:39:08.985 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:08.985 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:39:08.985 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:08.985 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.985 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.985 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35775.0:76843 14.13 14:ccbd50ad:::benchmark_data_smithi133_267384_object76842:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T05:39:08.985 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:37:12.039093+0000", 2023-10-09T05:39:08.985 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 116.83542160099999, 2023-10-09T05:39:08.985 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.506720743, 2023-10-09T05:39:08.985 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:39:08.986 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:08.986 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:39:08.986 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35775", 2023-10-09T05:39:08.986 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:39:08.986 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 76843 2023-10-09T05:39:08.986 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.986 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:39:08.986 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.986 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:39:08.986 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039093+0000", 2023-10-09T05:39:08.986 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:08.987 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.987 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.987 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:39:08.987 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039093+0000", 2023-10-09T05:39:08.987 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:08.987 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.987 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.987 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:39:08.987 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039092+0000", 2023-10-09T05:39:08.987 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:39:08.987 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.987 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.987 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:39:08.988 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039198+0000", 2023-10-09T05:39:08.988 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00010567 2023-10-09T05:39:08.988 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.988 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.988 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:39:08.988 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039215+0000", 2023-10-09T05:39:08.988 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.7110999999999999e-05 2023-10-09T05:39:08.988 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.988 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.988 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:39:08.988 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039230+0000", 2023-10-09T05:39:08.988 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.4902000000000001e-05 2023-10-09T05:39:08.989 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.989 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.989 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.989 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.049874+0000", 2023-10-09T05:39:08.989 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.010644696 2023-10-09T05:39:08.989 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.989 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.989 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:39:08.989 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.049881+0000", 2023-10-09T05:39:08.989 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.4899999999999997e-06 2023-10-09T05:39:08.989 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.989 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.990 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.990 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.519399+0000", 2023-10-09T05:39:08.990 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.4695184459999999 2023-10-09T05:39:08.990 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.990 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.990 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:39:08.990 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.519575+0000", 2023-10-09T05:39:08.990 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00017567 2023-10-09T05:39:08.990 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.990 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.990 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T05:39:08.990 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.519784+0000", 2023-10-09T05:39:08.991 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00020906399999999999 2023-10-09T05:39:08.991 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.991 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.991 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:39:08.991 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.524517+0000", 2023-10-09T05:39:08.991 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0047327920000000004 2023-10-09T05:39:08.991 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.991 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.991 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.991 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.539627+0000", 2023-10-09T05:39:08.991 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.015110755999999999 2023-10-09T05:39:08.991 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.991 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.992 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.992 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.539662+0000", 2023-10-09T05:39:08.992 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.4879000000000003e-05 2023-10-09T05:39:08.992 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.992 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.992 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:08.992 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.541997+0000", 2023-10-09T05:39:08.992 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0023351750000000001 2023-10-09T05:39:08.992 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.992 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.992 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:39:08.992 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.542080+0000", 2023-10-09T05:39:08.993 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.2100000000000003e-05 2023-10-09T05:39:08.993 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.993 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.993 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:39:08.993 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.545814+0000", 2023-10-09T05:39:08.993 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0037343659999999998 2023-10-09T05:39:08.993 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:08.993 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:39:08.993 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:08.993 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.993 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.993 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35775.0:76876 14.13 14:ccea55dd:::benchmark_data_smithi133_267384_object76875:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T05:39:08.994 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:37:12.039242+0000", 2023-10-09T05:39:08.994 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 116.835273172, 2023-10-09T05:39:08.994 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.508785176, 2023-10-09T05:39:08.994 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:39:08.994 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:08.994 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:39:08.994 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35775", 2023-10-09T05:39:08.994 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:39:08.994 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 76876 2023-10-09T05:39:08.994 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.994 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:39:08.995 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.995 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:39:08.995 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039242+0000", 2023-10-09T05:39:08.995 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:08.995 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.995 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.995 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:39:08.995 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039242+0000", 2023-10-09T05:39:08.995 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:08.995 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.995 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.995 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:39:08.996 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039240+0000", 2023-10-09T05:39:08.996 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:39:08.996 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.996 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.996 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:39:08.996 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039343+0000", 2023-10-09T05:39:08.996 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000102642 2023-10-09T05:39:08.996 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.996 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.996 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:39:08.996 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039363+0000", 2023-10-09T05:39:08.996 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.0358999999999999e-05 2023-10-09T05:39:08.996 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.997 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.997 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:39:08.997 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039378+0000", 2023-10-09T05:39:08.997 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.4185999999999999e-05 2023-10-09T05:39:08.997 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.997 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.997 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.997 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.049903+0000", 2023-10-09T05:39:08.997 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.010525184999999999 2023-10-09T05:39:08.997 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.997 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.997 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:39:08.998 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.049909+0000", 2023-10-09T05:39:08.998 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.3339999999999998e-06 2023-10-09T05:39:08.998 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.998 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.998 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:08.998 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.520988+0000", 2023-10-09T05:39:08.998 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.471079289 2023-10-09T05:39:08.998 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.998 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.998 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:39:08.998 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.521147+0000", 2023-10-09T05:39:08.998 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00015849599999999999 2023-10-09T05:39:08.999 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.999 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.999 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T05:39:08.999 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.521508+0000", 2023-10-09T05:39:08.999 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00036124699999999998 2023-10-09T05:39:08.999 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.999 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:08.999 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:39:08.999 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.524526+0000", 2023-10-09T05:39:08.999 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0030177210000000001 2023-10-09T05:39:08.999 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:08.999 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.000 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:09.000 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.541827+0000", 2023-10-09T05:39:09.000 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.017300784 2023-10-09T05:39:09.000 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.000 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.000 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:09.000 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.541959+0000", 2023-10-09T05:39:09.000 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00013233599999999999 2023-10-09T05:39:09.000 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.000 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.000 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:09.000 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.545993+0000", 2023-10-09T05:39:09.000 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.004033465 2023-10-09T05:39:09.001 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.001 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.001 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:39:09.001 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.546064+0000", 2023-10-09T05:39:09.001 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.1749999999999996e-05 2023-10-09T05:39:09.001 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.001 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.001 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:39:09.001 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.548027+0000", 2023-10-09T05:39:09.001 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0019625649999999999 2023-10-09T05:39:09.001 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:09.001 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:39:09.002 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:09.002 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.002 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.002 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35775.0:76891 14.13 14:cdf2addd:::benchmark_data_smithi133_267384_object76890:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T05:39:09.002 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:37:12.039390+0000", 2023-10-09T05:39:09.002 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 116.83512529399999, 2023-10-09T05:39:09.002 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.510686201, 2023-10-09T05:39:09.002 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:39:09.002 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:09.002 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:39:09.002 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35775", 2023-10-09T05:39:09.002 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:39:09.003 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 76891 2023-10-09T05:39:09.003 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.003 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:39:09.003 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.003 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:39:09.003 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039390+0000", 2023-10-09T05:39:09.003 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:09.003 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.003 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.003 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:39:09.003 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039390+0000", 2023-10-09T05:39:09.003 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:09.004 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.004 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.004 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:39:09.004 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039388+0000", 2023-10-09T05:39:09.004 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:39:09.004 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.004 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.004 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:39:09.004 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039498+0000", 2023-10-09T05:39:09.004 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000109224 2023-10-09T05:39:09.004 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.004 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.005 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:39:09.005 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039515+0000", 2023-10-09T05:39:09.005 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.6988000000000001e-05 2023-10-09T05:39:09.005 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.005 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.005 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:39:09.005 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039529+0000", 2023-10-09T05:39:09.005 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.4826e-05 2023-10-09T05:39:09.005 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.005 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.005 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:09.005 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.050130+0000", 2023-10-09T05:39:09.006 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.010600228999999999 2023-10-09T05:39:09.006 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.006 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.006 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:39:09.006 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.050223+0000", 2023-10-09T05:39:09.006 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.3770000000000005e-05 2023-10-09T05:39:09.006 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.006 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.006 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:09.006 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.522777+0000", 2023-10-09T05:39:09.006 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.472553998 2023-10-09T05:39:09.006 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.006 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.007 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:39:09.007 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.522980+0000", 2023-10-09T05:39:09.007 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00020236700000000001 2023-10-09T05:39:09.007 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.007 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.007 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T05:39:09.007 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.523196+0000", 2023-10-09T05:39:09.007 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00021654700000000001 2023-10-09T05:39:09.007 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.007 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.007 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:39:09.007 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.526431+0000", 2023-10-09T05:39:09.008 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0032350949999999999 2023-10-09T05:39:09.008 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.008 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.008 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:09.008 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.545890+0000", 2023-10-09T05:39:09.008 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.019458915 2023-10-09T05:39:09.008 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.008 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.008 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:09.008 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.545942+0000", 2023-10-09T05:39:09.008 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.1687000000000002e-05 2023-10-09T05:39:09.008 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.009 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.009 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:09.009 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.548224+0000", 2023-10-09T05:39:09.009 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.002281749 2023-10-09T05:39:09.009 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.009 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.009 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:39:09.009 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.548282+0000", 2023-10-09T05:39:09.009 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.8492000000000002e-05 2023-10-09T05:39:09.009 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.009 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.009 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:39:09.010 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.550076+0000", 2023-10-09T05:39:09.010 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.001793383 2023-10-09T05:39:09.010 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:09.010 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:39:09.010 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:09.010 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.010 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.010 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35775.0:76994 14.13 14:c8938304:::benchmark_data_smithi133_267384_object76993:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T05:39:09.010 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:37:12.039543+0000", 2023-10-09T05:39:09.010 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 116.834972205, 2023-10-09T05:39:09.010 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.5123379099999998, 2023-10-09T05:39:09.010 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:39:09.011 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:09.011 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:39:09.011 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35775", 2023-10-09T05:39:09.011 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:39:09.011 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 76994 2023-10-09T05:39:09.011 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.011 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:39:09.011 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.011 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:39:09.011 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039543+0000", 2023-10-09T05:39:09.011 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:09.011 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.012 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.012 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:39:09.012 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039543+0000", 2023-10-09T05:39:09.012 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:09.012 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.012 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.012 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:39:09.012 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039541+0000", 2023-10-09T05:39:09.012 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999981 2023-10-09T05:39:09.012 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.012 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.012 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:39:09.013 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039648+0000", 2023-10-09T05:39:09.013 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000107379 2023-10-09T05:39:09.013 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.013 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.013 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:39:09.013 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039667+0000", 2023-10-09T05:39:09.013 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.8799000000000001e-05 2023-10-09T05:39:09.013 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.013 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.013 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:39:09.013 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039682+0000", 2023-10-09T05:39:09.013 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.5383000000000002e-05 2023-10-09T05:39:09.014 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.014 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.014 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:09.014 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.050286+0000", 2023-10-09T05:39:09.014 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.010603382999999999 2023-10-09T05:39:09.014 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.014 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.014 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:39:09.014 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.050294+0000", 2023-10-09T05:39:09.014 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.2910000000000007e-06 2023-10-09T05:39:09.014 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.014 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.015 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:09.015 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.526458+0000", 2023-10-09T05:39:09.015 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.476163855 2023-10-09T05:39:09.015 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.015 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.015 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:39:09.015 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.526627+0000", 2023-10-09T05:39:09.015 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00016936699999999999 2023-10-09T05:39:09.015 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.015 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.015 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T05:39:09.015 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.526796+0000", 2023-10-09T05:39:09.016 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00016881 2023-10-09T05:39:09.016 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.016 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.016 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:39:09.016 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.530325+0000", 2023-10-09T05:39:09.016 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0035291799999999998 2023-10-09T05:39:09.016 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.016 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.016 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:09.016 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.548161+0000", 2023-10-09T05:39:09.016 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.017836029999999999 2023-10-09T05:39:09.016 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.017 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.017 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:09.017 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.548194+0000", 2023-10-09T05:39:09.017 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.2663000000000002e-05 2023-10-09T05:39:09.017 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.017 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.017 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:09.017 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.550186+0000", 2023-10-09T05:39:09.017 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0019923940000000002 2023-10-09T05:39:09.017 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.017 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.017 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:39:09.018 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.550233+0000", 2023-10-09T05:39:09.018 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.702e-05 2023-10-09T05:39:09.018 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.018 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.018 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:39:09.018 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.551881+0000", 2023-10-09T05:39:09.018 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0016471280000000001 2023-10-09T05:39:09.018 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:09.018 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:39:09.018 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:09.018 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.018 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.019 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35775.0:77050 14.13 14:cc70d974:::benchmark_data_smithi133_267384_object77049:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T05:39:09.019 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:37:12.039695+0000", 2023-10-09T05:39:09.019 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 116.834819368, 2023-10-09T05:39:09.019 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.514257757, 2023-10-09T05:39:09.019 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:39:09.019 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:09.019 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:39:09.019 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35775", 2023-10-09T05:39:09.019 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:39:09.019 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 77050 2023-10-09T05:39:09.019 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.019 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:39:09.019 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.020 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:39:09.020 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039695+0000", 2023-10-09T05:39:09.020 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:09.020 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.020 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.020 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:39:09.020 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039695+0000", 2023-10-09T05:39:09.020 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:09.020 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.020 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.020 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:39:09.020 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039694+0000", 2023-10-09T05:39:09.021 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:39:09.021 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.021 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.021 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:39:09.021 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039802+0000", 2023-10-09T05:39:09.021 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000107316 2023-10-09T05:39:09.021 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.021 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.021 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:39:09.021 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039819+0000", 2023-10-09T05:39:09.021 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.7779e-05 2023-10-09T05:39:09.021 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.022 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.022 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:39:09.022 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039834+0000", 2023-10-09T05:39:09.022 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.4939999999999999e-05 2023-10-09T05:39:09.022 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.022 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.022 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:09.022 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.050337+0000", 2023-10-09T05:39:09.022 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.010502727 2023-10-09T05:39:09.022 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.022 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.022 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:39:09.023 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.050346+0000", 2023-10-09T05:39:09.023 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.5259999999999993e-06 2023-10-09T05:39:09.023 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.023 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.023 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:09.023 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.528530+0000", 2023-10-09T05:39:09.023 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.4781847699999999 2023-10-09T05:39:09.023 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.023 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.023 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:39:09.023 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.528692+0000", 2023-10-09T05:39:09.023 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00016168000000000001 2023-10-09T05:39:09.024 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.024 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.024 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T05:39:09.024 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.528815+0000", 2023-10-09T05:39:09.024 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000122726 2023-10-09T05:39:09.024 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.024 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.024 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:39:09.024 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.533848+0000", 2023-10-09T05:39:09.024 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0050328650000000001 2023-10-09T05:39:09.024 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.025 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.025 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:09.025 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.550123+0000", 2023-10-09T05:39:09.025 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.016275540000000002 2023-10-09T05:39:09.025 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.025 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.025 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:09.025 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.550155+0000", 2023-10-09T05:39:09.025 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.2121e-05 2023-10-09T05:39:09.025 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.025 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.025 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:09.026 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.551998+0000", 2023-10-09T05:39:09.026 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0018423809999999999 2023-10-09T05:39:09.026 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.026 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.026 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:39:09.026 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.552055+0000", 2023-10-09T05:39:09.026 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.7114000000000002e-05 2023-10-09T05:39:09.026 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.026 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.026 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:39:09.026 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.553953+0000", 2023-10-09T05:39:09.027 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.001898495 2023-10-09T05:39:09.027 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:09.027 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:39:09.027 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:09.027 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.027 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.027 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35775.0:77054 14.13 14:c8fa2879:::benchmark_data_smithi133_267384_object77053:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T05:39:09.027 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:37:12.039847+0000", 2023-10-09T05:39:09.027 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 116.834668259, 2023-10-09T05:39:09.027 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.515861406, 2023-10-09T05:39:09.028 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:39:09.028 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:09.028 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:39:09.028 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35775", 2023-10-09T05:39:09.028 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:39:09.028 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 77054 2023-10-09T05:39:09.028 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.028 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:39:09.028 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.028 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:39:09.029 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039847+0000", 2023-10-09T05:39:09.029 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:09.029 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.029 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.029 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:39:09.029 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039847+0000", 2023-10-09T05:39:09.029 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:09.029 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.029 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.029 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:39:09.029 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039845+0000", 2023-10-09T05:39:09.029 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999986 2023-10-09T05:39:09.030 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.030 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.030 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:39:09.030 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039951+0000", 2023-10-09T05:39:09.030 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000105933 2023-10-09T05:39:09.030 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.030 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.030 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:39:09.030 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039968+0000", 2023-10-09T05:39:09.030 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.7217000000000001e-05 2023-10-09T05:39:09.030 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.030 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.031 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:39:09.031 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.039984+0000", 2023-10-09T05:39:09.031 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.5577e-05 2023-10-09T05:39:09.031 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.031 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.031 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:09.031 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.050384+0000", 2023-10-09T05:39:09.031 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.010400352 2023-10-09T05:39:09.031 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.031 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.031 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:39:09.031 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.050394+0000", 2023-10-09T05:39:09.032 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.9720000000000004e-06 2023-10-09T05:39:09.032 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.032 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.032 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:09.032 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.530412+0000", 2023-10-09T05:39:09.032 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.4800173889999999 2023-10-09T05:39:09.032 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.032 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.032 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:39:09.032 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.530532+0000", 2023-10-09T05:39:09.032 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000120081 2023-10-09T05:39:09.032 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.033 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.033 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T05:39:09.033 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.530645+0000", 2023-10-09T05:39:09.033 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0001136 2023-10-09T05:39:09.033 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.033 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.033 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:39:09.033 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.535911+0000", 2023-10-09T05:39:09.033 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0052657030000000001 2023-10-09T05:39:09.033 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.033 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.033 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:09.034 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.551928+0000", 2023-10-09T05:39:09.034 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.016017291999999999 2023-10-09T05:39:09.034 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.034 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.034 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:09.034 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.551965+0000", 2023-10-09T05:39:09.034 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 3.6513000000000001e-05 2023-10-09T05:39:09.034 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.034 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.034 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:09.034 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.554099+0000", 2023-10-09T05:39:09.034 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00213423 2023-10-09T05:39:09.035 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.035 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.035 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:39:09.035 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.554158+0000", 2023-10-09T05:39:09.035 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.8488e-05 2023-10-09T05:39:09.035 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.035 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.035 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:39:09.035 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.555708+0000", 2023-10-09T05:39:09.035 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.001550282 2023-10-09T05:39:09.035 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:09.035 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:39:09.036 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:09.036 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.036 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.036 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35775.0:77152 14.13 14:cd85531c:::benchmark_data_smithi133_267384_object77151:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T05:39:09.036 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:37:12.170359+0000", 2023-10-09T05:39:09.036 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 116.704155701, 2023-10-09T05:39:09.036 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.4148245859999999, 2023-10-09T05:39:09.036 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:39:09.036 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:09.036 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:39:09.036 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35775", 2023-10-09T05:39:09.036 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:39:09.037 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 77152 2023-10-09T05:39:09.037 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.037 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:39:09.037 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.037 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:39:09.037 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.170359+0000", 2023-10-09T05:39:09.037 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:09.037 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.037 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.037 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:39:09.037 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.170359+0000", 2023-10-09T05:39:09.037 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:09.038 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.038 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.038 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:39:09.038 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.170358+0000", 2023-10-09T05:39:09.038 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:39:09.038 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.038 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.038 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:39:09.038 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.170415+0000", 2023-10-09T05:39:09.038 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.6771999999999998e-05 2023-10-09T05:39:09.038 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.038 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.039 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:39:09.039 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.170426+0000", 2023-10-09T05:39:09.039 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.1301000000000001e-05 2023-10-09T05:39:09.039 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.039 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.039 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:39:09.039 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.170436+0000", 2023-10-09T05:39:09.039 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 9.1570000000000006e-06 2023-10-09T05:39:09.039 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.039 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.039 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:09.039 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.170548+0000", 2023-10-09T05:39:09.040 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0001125 2023-10-09T05:39:09.040 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.040 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.040 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:39:09.040 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.170556+0000", 2023-10-09T05:39:09.040 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 8.4060000000000005e-06 2023-10-09T05:39:09.040 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.040 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.040 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:09.040 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.531908+0000", 2023-10-09T05:39:09.040 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.3613512169999999 2023-10-09T05:39:09.040 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.041 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.041 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:39:09.041 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.532081+0000", 2023-10-09T05:39:09.041 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00017301600000000001 2023-10-09T05:39:09.041 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.041 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.041 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T05:39:09.041 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.532269+0000", 2023-10-09T05:39:09.041 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00018811199999999999 2023-10-09T05:39:09.041 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.041 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.042 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:39:09.042 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.548115+0000", 2023-10-09T05:39:09.042 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.015846446 2023-10-09T05:39:09.042 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.042 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.042 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:09.042 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.554021+0000", 2023-10-09T05:39:09.042 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0059060739999999999 2023-10-09T05:39:09.042 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.042 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.042 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:09.042 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.554062+0000", 2023-10-09T05:39:09.043 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4.1143999999999998e-05 2023-10-09T05:39:09.043 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.043 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.043 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:09.043 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.582032+0000", 2023-10-09T05:39:09.043 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.027969344 2023-10-09T05:39:09.043 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.043 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.043 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:39:09.043 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.582092+0000", 2023-10-09T05:39:09.043 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.0253000000000001e-05 2023-10-09T05:39:09.043 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.044 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.044 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:39:09.044 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.585184+0000", 2023-10-09T05:39:09.044 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.003091669 2023-10-09T05:39:09.044 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:09.044 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:39:09.044 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:09.044 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.044 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.044 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35775.0:77155 14.13 14:cbfb989f:::benchmark_data_smithi133_267384_object77154:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T05:39:09.044 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:37:12.174986+0000", 2023-10-09T05:39:09.044 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 116.699528595, 2023-10-09T05:39:09.045 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.4301789949999999, 2023-10-09T05:39:09.045 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:39:09.045 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:09.045 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:39:09.045 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35775", 2023-10-09T05:39:09.045 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:39:09.045 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 77155 2023-10-09T05:39:09.045 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.045 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:39:09.045 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.045 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:39:09.045 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.174986+0000", 2023-10-09T05:39:09.046 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:09.046 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.046 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.046 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:39:09.046 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.174986+0000", 2023-10-09T05:39:09.046 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:09.046 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.046 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.046 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:39:09.046 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.174984+0000", 2023-10-09T05:39:09.046 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.9999981 2023-10-09T05:39:09.047 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.047 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.047 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:39:09.047 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.175041+0000", 2023-10-09T05:39:09.047 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.6953000000000002e-05 2023-10-09T05:39:09.047 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.047 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.047 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:39:09.047 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.175058+0000", 2023-10-09T05:39:09.047 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.6920999999999999e-05 2023-10-09T05:39:09.047 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.047 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.048 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:39:09.048 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.175070+0000", 2023-10-09T05:39:09.048 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.1758e-05 2023-10-09T05:39:09.048 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.048 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.048 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:09.048 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.175133+0000", 2023-10-09T05:39:09.048 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.3281999999999995e-05 2023-10-09T05:39:09.048 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.048 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.048 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:39:09.048 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.175146+0000", 2023-10-09T05:39:09.049 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.2442e-05 2023-10-09T05:39:09.049 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.049 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.049 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:09.049 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.555746+0000", 2023-10-09T05:39:09.049 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.3805996920000001 2023-10-09T05:39:09.049 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.049 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.049 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:39:09.049 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.555945+0000", 2023-10-09T05:39:09.049 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00019914499999999999 2023-10-09T05:39:09.049 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.049 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.050 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T05:39:09.050 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.556135+0000", 2023-10-09T05:39:09.050 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00018987100000000001 2023-10-09T05:39:09.050 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.050 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.050 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:39:09.050 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.566188+0000", 2023-10-09T05:39:09.050 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.010053809 2023-10-09T05:39:09.050 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.050 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.050 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:09.051 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.569836+0000", 2023-10-09T05:39:09.051 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0036472700000000002 2023-10-09T05:39:09.051 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.051 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.051 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:09.051 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.577011+0000", 2023-10-09T05:39:09.051 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0071756709999999998 2023-10-09T05:39:09.051 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.051 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.051 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:09.051 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.599452+0000", 2023-10-09T05:39:09.051 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.022440495000000001 2023-10-09T05:39:09.051 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.052 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.052 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:39:09.052 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.599514+0000", 2023-10-09T05:39:09.052 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 6.1797999999999993e-05 2023-10-09T05:39:09.052 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.052 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.052 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:39:09.052 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.605165+0000", 2023-10-09T05:39:09.052 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.005651639 2023-10-09T05:39:09.052 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:09.052 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:39:09.052 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:09.053 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.053 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.053 INFO:teuthology.orchestra.run.smithi133.stdout: "description": "osd_op(client.35775.0:77202 14.13 14:ce821ae6:::benchmark_data_smithi133_267384_object77201:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T05:39:09.053 INFO:teuthology.orchestra.run.smithi133.stdout: "initiated_at": "2023-10-09T05:37:12.246945+0000", 2023-10-09T05:39:09.053 INFO:teuthology.orchestra.run.smithi133.stdout: "age": 116.627569866, 2023-10-09T05:39:09.053 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.3599690629999999, 2023-10-09T05:39:09.053 INFO:teuthology.orchestra.run.smithi133.stdout: "type_data": { 2023-10-09T05:39:09.053 INFO:teuthology.orchestra.run.smithi133.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:09.053 INFO:teuthology.orchestra.run.smithi133.stdout: "client_info": { 2023-10-09T05:39:09.053 INFO:teuthology.orchestra.run.smithi133.stdout: "client": "client.35775", 2023-10-09T05:39:09.053 INFO:teuthology.orchestra.run.smithi133.stdout: "client_addr": "172.21.15.133:0/2229030341", 2023-10-09T05:39:09.053 INFO:teuthology.orchestra.run.smithi133.stdout: "tid": 77202 2023-10-09T05:39:09.054 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.054 INFO:teuthology.orchestra.run.smithi133.stdout: "events": [ 2023-10-09T05:39:09.054 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.054 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "initiated", 2023-10-09T05:39:09.054 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.246945+0000", 2023-10-09T05:39:09.054 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:09.054 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.054 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.054 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "throttled", 2023-10-09T05:39:09.054 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.246945+0000", 2023-10-09T05:39:09.054 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0 2023-10-09T05:39:09.054 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.055 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.055 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "header_read", 2023-10-09T05:39:09.055 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.246944+0000", 2023-10-09T05:39:09.055 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 4294967295.999999 2023-10-09T05:39:09.055 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.055 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.055 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "all_read", 2023-10-09T05:39:09.055 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.246968+0000", 2023-10-09T05:39:09.055 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 2.4403999999999999e-05 2023-10-09T05:39:09.055 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.055 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.056 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "dispatched", 2023-10-09T05:39:09.056 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.246988+0000", 2023-10-09T05:39:09.056 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.9351000000000002e-05 2023-10-09T05:39:09.056 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.056 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.056 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "queued_for_pg", 2023-10-09T05:39:09.056 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.246995+0000", 2023-10-09T05:39:09.056 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 7.0899999999999999e-06 2023-10-09T05:39:09.056 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.056 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.056 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:09.057 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.247178+0000", 2023-10-09T05:39:09.057 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.000183294 2023-10-09T05:39:09.057 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.057 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.057 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for peered", 2023-10-09T05:39:09.057 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:12.247189+0000", 2023-10-09T05:39:09.057 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.0719e-05 2023-10-09T05:39:09.057 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.057 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.057 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "reached_pg", 2023-10-09T05:39:09.057 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.557767+0000", 2023-10-09T05:39:09.057 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 1.3105783899999999 2023-10-09T05:39:09.058 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.058 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.058 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "started", 2023-10-09T05:39:09.058 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.557941+0000", 2023-10-09T05:39:09.058 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00017421400000000001 2023-10-09T05:39:09.058 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.058 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.058 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T05:39:09.058 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.558135+0000", 2023-10-09T05:39:09.058 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00019360199999999999 2023-10-09T05:39:09.058 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.059 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.059 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "op_commit", 2023-10-09T05:39:09.059 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.566195+0000", 2023-10-09T05:39:09.059 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0080602550000000005 2023-10-09T05:39:09.059 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.059 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.059 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:09.059 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.569881+0000", 2023-10-09T05:39:09.059 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0036858440000000002 2023-10-09T05:39:09.059 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.059 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.060 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:09.060 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.578735+0000", 2023-10-09T05:39:09.060 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.0088533629999999995 2023-10-09T05:39:09.060 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.060 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.060 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:09.060 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.605224+0000", 2023-10-09T05:39:09.060 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.026489902999999999 2023-10-09T05:39:09.060 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.060 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.060 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "commit_sent", 2023-10-09T05:39:09.061 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.605277+0000", 2023-10-09T05:39:09.061 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 5.2151999999999998e-05 2023-10-09T05:39:09.061 INFO:teuthology.orchestra.run.smithi133.stdout: }, 2023-10-09T05:39:09.061 INFO:teuthology.orchestra.run.smithi133.stdout: { 2023-10-09T05:39:09.061 INFO:teuthology.orchestra.run.smithi133.stdout: "event": "done", 2023-10-09T05:39:09.061 INFO:teuthology.orchestra.run.smithi133.stdout: "time": "2023-10-09T05:37:13.606914+0000", 2023-10-09T05:39:09.061 INFO:teuthology.orchestra.run.smithi133.stdout: "duration": 0.00163744 2023-10-09T05:39:09.061 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:09.061 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:39:09.061 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:09.061 INFO:teuthology.orchestra.run.smithi133.stdout: } 2023-10-09T05:39:09.061 INFO:teuthology.orchestra.run.smithi133.stdout: ] 2023-10-09T05:39:09.062 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:39:09.062 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:09.063 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:08 smithi133 bash[105589]: cluster 2023-10-09T05:39:07.676275+0000 mon.a (mon.0) 1945 : cluster 0 osdmap e344: 8 total, 8 up, 7 in 2023-10-09T05:39:09.063 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:08 smithi133 bash[105589]: cluster 2023-10-09T05:39:07.680092+0000 osd.5 (osd.5) 357 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T05:39:09.063 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:08 smithi133 bash[105589]: cluster 2023-10-09T05:39:07.682084+0000 osd.6 (osd.6) 328 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T05:39:09.063 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:08 smithi133 bash[105589]: cluster 2023-10-09T05:39:07.682128+0000 osd.7 (osd.7) 299 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T05:39:09.063 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:08 smithi133 bash[105589]: cluster 2023-10-09T05:39:08.459691+0000 mgr.y (mgr.25277) 1539 : cluster 0 pgmap v1672: 109 pgs: 1 peering, 4 active+recovery_wait, 104 active+clean; 2.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 755 op/s; 4.6 MiB/s, 74 objects/s recovering 2023-10-09T05:39:09.073 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:08 smithi194 bash[92666]: cluster 2023-10-09T05:39:07.676275+0000 mon.a (mon.0) 1945 : cluster 0 osdmap e344: 8 total, 8 up, 7 in 2023-10-09T05:39:09.073 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:08 smithi194 bash[92666]: cluster 2023-10-09T05:39:07.680092+0000 osd.5 (osd.5) 357 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T05:39:09.073 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:08 smithi194 bash[92666]: cluster 2023-10-09T05:39:07.682084+0000 osd.6 (osd.6) 328 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T05:39:09.073 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:08 smithi194 bash[92666]: cluster 2023-10-09T05:39:07.682128+0000 osd.7 (osd.7) 299 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T05:39:09.073 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:08 smithi194 bash[92666]: cluster 2023-10-09T05:39:08.459691+0000 mgr.y (mgr.25277) 1539 : cluster 0 pgmap v1672: 109 pgs: 1 peering, 4 active+recovery_wait, 104 active+clean; 2.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 755 op/s; 4.6 MiB/s, 74 objects/s recovering 2023-10-09T05:39:09.091 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:09.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:09.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:09.194 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:09.224 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:09.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:09.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:09.327 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:09.356 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:09.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:09.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:09.461 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:09.488 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:09.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:09.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:09.591 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:09.619 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:09.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:09.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:09.722 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:09.740 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:09.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:09.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:09.843 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:09.874 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:09.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:09.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:09.944 INFO:teuthology.orchestra.run.smithi133.stdout:moved 10 / 327 (3.0581%) 2023-10-09T05:39:09.945 INFO:teuthology.orchestra.run.smithi133.stdout:avg 46.7143 2023-10-09T05:39:09.945 INFO:teuthology.orchestra.run.smithi133.stdout:stddev 7.20544 -> 4.74234 (expected baseline 6.32778) 2023-10-09T05:39:09.945 INFO:teuthology.orchestra.run.smithi133.stdout:min osd.5 with 36 -> 38 pgs (0.770642 -> 0.813456 * mean) 2023-10-09T05:39:09.945 INFO:teuthology.orchestra.run.smithi133.stdout:max osd.1 with 60 -> 51 pgs (1.2844 -> 1.09174 * mean) 2023-10-09T05:39:09.945 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:39:09.945 INFO:teuthology.orchestra.run.smithi133.stdout:oload 110 2023-10-09T05:39:09.945 INFO:teuthology.orchestra.run.smithi133.stdout:max_change 3 2023-10-09T05:39:09.945 INFO:teuthology.orchestra.run.smithi133.stdout:max_change_osds 4 2023-10-09T05:39:09.945 INFO:teuthology.orchestra.run.smithi133.stdout:average_utilization 0.0244 2023-10-09T05:39:09.945 INFO:teuthology.orchestra.run.smithi133.stdout:overload_utilization 0.0269 2023-10-09T05:39:09.946 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1 weight 1.0000 -> 0.8631 2023-10-09T05:39:09.951 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is set 2023-10-09T05:39:09.977 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:09.986 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:09 smithi133 bash[105589]: cluster 2023-10-09T05:39:08.682791+0000 mon.a (mon.0) 1946 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2 pgs undersized) 2023-10-09T05:39:09.986 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:09 smithi133 bash[105589]: audit 2023-10-09T05:39:09.045516+0000 mgr.y (mgr.25277) 1540 : audit 0 from='client.55393 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:39:09.987 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:09 smithi133 bash[105589]: audit 2023-10-09T05:39:09.046750+0000 mon.a (mon.0) 1947 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'56565'}"}]: dispatch 2023-10-09T05:39:09.987 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:09 smithi133 bash[105589]: audit 2023-10-09T05:39:09.290863+0000 mon.a (mon.0) 1948 : audit 1 from='client.? 172.21.15.133:0/135655062' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:39:09.987 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:09 smithi133 bash[102858]: cluster 2023-10-09T05:39:08.682791+0000 mon.a (mon.0) 1946 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2 pgs undersized) 2023-10-09T05:39:09.987 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:09 smithi133 bash[102858]: audit 2023-10-09T05:39:09.045516+0000 mgr.y (mgr.25277) 1540 : audit 0 from='client.55393 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:39:09.987 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:09 smithi133 bash[102858]: audit 2023-10-09T05:39:09.046750+0000 mon.a (mon.0) 1947 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'56565'}"}]: dispatch 2023-10-09T05:39:09.987 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:09 smithi133 bash[102858]: audit 2023-10-09T05:39:09.290863+0000 mon.a (mon.0) 1948 : audit 1 from='client.? 172.21.15.133:0/135655062' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:39:10.018 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:10.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:10.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:10.124 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:10.133 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:09 smithi194 bash[92666]: cluster 2023-10-09T05:39:08.682791+0000 mon.a (mon.0) 1946 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2 pgs undersized) 2023-10-09T05:39:10.134 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:09 smithi194 bash[92666]: audit 2023-10-09T05:39:09.045516+0000 mgr.y (mgr.25277) 1540 : audit 0 from='client.55393 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:39:10.134 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:09 smithi194 bash[92666]: audit 2023-10-09T05:39:09.046750+0000 mon.a (mon.0) 1947 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'56565'}"}]: dispatch 2023-10-09T05:39:10.134 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:09 smithi194 bash[92666]: audit 2023-10-09T05:39:09.290863+0000 mon.a (mon.0) 1948 : audit 1 from='client.? 172.21.15.133:0/135655062' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:39:10.152 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:10.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:10.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:10.256 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:10.280 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:10.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:10.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:10.385 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:10.408 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:10.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:10.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:10.511 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:10.542 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:10.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:10.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:10.646 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:10.668 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:10.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:10.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:10.772 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:10.784 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:10 smithi194 bash[92666]: cluster 2023-10-09T05:39:09.937310+0000 mon.a (mon.0) 1949 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:39:10.784 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:10 smithi194 bash[92666]: cluster 2023-10-09T05:39:09.941707+0000 osd.5 (osd.5) 358 : cluster 3 failed to encode map e345 with expected crc 2023-10-09T05:39:10.784 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:10 smithi194 bash[92666]: audit 2023-10-09T05:39:09.944172+0000 mon.a (mon.0) 1950 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'56565'}"}]': finished 2023-10-09T05:39:10.784 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:10 smithi194 bash[92666]: audit 2023-10-09T05:39:09.944244+0000 mon.a (mon.0) 1951 : audit 1 from='client.? 172.21.15.133:0/135655062' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:39:10.784 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:10 smithi194 bash[92666]: cluster 2023-10-09T05:39:09.944287+0000 mon.a (mon.0) 1952 : cluster 0 osdmap e345: 8 total, 8 up, 7 in 2023-10-09T05:39:10.785 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:10 smithi194 bash[92666]: cluster 2023-10-09T05:39:09.944539+0000 osd.6 (osd.6) 329 : cluster 3 failed to encode map e345 with expected crc 2023-10-09T05:39:10.785 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:10 smithi194 bash[92666]: cluster 2023-10-09T05:39:09.944636+0000 osd.7 (osd.7) 300 : cluster 3 failed to encode map e345 with expected crc 2023-10-09T05:39:10.785 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:10 smithi194 bash[92666]: cluster 2023-10-09T05:39:10.460132+0000 mgr.y (mgr.25277) 1541 : cluster 0 pgmap v1674: 109 pgs: 1 peering, 2 active+recovery_wait, 106 active+clean; 1.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 720 op/s; 4.5 MiB/s, 71 objects/s recovering 2023-10-09T05:39:10.802 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:10.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:10.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:10.907 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:10.938 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:10.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:10.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:11.042 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:11.047 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:10 smithi133 bash[102858]: cluster 2023-10-09T05:39:09.937310+0000 mon.a (mon.0) 1949 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:39:11.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:10 smithi133 bash[102858]: cluster 2023-10-09T05:39:09.941707+0000 osd.5 (osd.5) 358 : cluster 3 failed to encode map e345 with expected crc 2023-10-09T05:39:11.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:10 smithi133 bash[102858]: audit 2023-10-09T05:39:09.944172+0000 mon.a (mon.0) 1950 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'56565'}"}]': finished 2023-10-09T05:39:11.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:10 smithi133 bash[102858]: audit 2023-10-09T05:39:09.944244+0000 mon.a (mon.0) 1951 : audit 1 from='client.? 172.21.15.133:0/135655062' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:39:11.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:10 smithi133 bash[102858]: cluster 2023-10-09T05:39:09.944287+0000 mon.a (mon.0) 1952 : cluster 0 osdmap e345: 8 total, 8 up, 7 in 2023-10-09T05:39:11.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:10 smithi133 bash[102858]: cluster 2023-10-09T05:39:09.944539+0000 osd.6 (osd.6) 329 : cluster 3 failed to encode map e345 with expected crc 2023-10-09T05:39:11.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:10 smithi133 bash[102858]: cluster 2023-10-09T05:39:09.944636+0000 osd.7 (osd.7) 300 : cluster 3 failed to encode map e345 with expected crc 2023-10-09T05:39:11.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:10 smithi133 bash[102858]: cluster 2023-10-09T05:39:10.460132+0000 mgr.y (mgr.25277) 1541 : cluster 0 pgmap v1674: 109 pgs: 1 peering, 2 active+recovery_wait, 106 active+clean; 1.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 720 op/s; 4.5 MiB/s, 71 objects/s recovering 2023-10-09T05:39:11.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:10 smithi133 bash[105589]: cluster 2023-10-09T05:39:09.937310+0000 mon.a (mon.0) 1949 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:39:11.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:10 smithi133 bash[105589]: cluster 2023-10-09T05:39:09.941707+0000 osd.5 (osd.5) 358 : cluster 3 failed to encode map e345 with expected crc 2023-10-09T05:39:11.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:10 smithi133 bash[105589]: audit 2023-10-09T05:39:09.944172+0000 mon.a (mon.0) 1950 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'56565'}"}]': finished 2023-10-09T05:39:11.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:10 smithi133 bash[105589]: audit 2023-10-09T05:39:09.944244+0000 mon.a (mon.0) 1951 : audit 1 from='client.? 172.21.15.133:0/135655062' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:39:11.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:10 smithi133 bash[105589]: cluster 2023-10-09T05:39:09.944287+0000 mon.a (mon.0) 1952 : cluster 0 osdmap e345: 8 total, 8 up, 7 in 2023-10-09T05:39:11.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:10 smithi133 bash[105589]: cluster 2023-10-09T05:39:09.944539+0000 osd.6 (osd.6) 329 : cluster 3 failed to encode map e345 with expected crc 2023-10-09T05:39:11.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:10 smithi133 bash[105589]: cluster 2023-10-09T05:39:09.944636+0000 osd.7 (osd.7) 300 : cluster 3 failed to encode map e345 with expected crc 2023-10-09T05:39:11.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:10 smithi133 bash[105589]: cluster 2023-10-09T05:39:10.460132+0000 mgr.y (mgr.25277) 1541 : cluster 0 pgmap v1674: 109 pgs: 1 peering, 2 active+recovery_wait, 106 active+clean; 1.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 720 op/s; 4.5 MiB/s, 71 objects/s recovering 2023-10-09T05:39:11.070 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:11.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:11.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:11.075 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T05:39:11.173 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:11.189 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:11.203 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:11.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:11.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:11.306 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:11.337 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:11.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:11.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:11.440 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:11.463 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:11.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:11.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:11.566 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:11.591 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:11.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:11.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:11.693 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:11.715 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:11.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:11.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:11.819 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:11.855 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:11.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:11.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:11.962 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:11.983 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:11.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:11.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:12.086 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:12.098 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:11 smithi194 bash[92666]: cluster 2023-10-09T05:39:10.952280+0000 osd.4 (osd.4) 304 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T05:39:12.098 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:11 smithi194 bash[92666]: cluster 2023-10-09T05:39:10.953487+0000 osd.7 (osd.7) 301 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T05:39:12.098 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:11 smithi194 bash[92666]: cluster 2023-10-09T05:39:10.953990+0000 osd.5 (osd.5) 359 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T05:39:12.098 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:11 smithi194 bash[92666]: cluster 2023-10-09T05:39:10.954349+0000 mon.a (mon.0) 1953 : cluster 0 osdmap e346: 8 total, 8 up, 7 in 2023-10-09T05:39:12.098 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:11 smithi194 bash[92666]: cluster 2023-10-09T05:39:10.955147+0000 osd.6 (osd.6) 330 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T05:39:12.098 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:11 smithi194 bash[92666]: cluster 2023-10-09T05:39:10.956931+0000 osd.6 (osd.6) 331 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T05:39:12.099 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:11 smithi194 bash[92666]: cluster 2023-10-09T05:39:10.957301+0000 osd.7 (osd.7) 302 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T05:39:12.099 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:11 smithi194 bash[92666]: cluster 2023-10-09T05:39:10.959997+0000 osd.5 (osd.5) 360 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T05:39:12.099 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:11 smithi194 bash[92666]: cluster 2023-10-09T05:39:10.964306+0000 osd.5 (osd.5) 361 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T05:39:12.099 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:11 smithi194 bash[92666]: cluster 2023-10-09T05:39:10.965977+0000 osd.5 (osd.5) 362 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T05:39:12.119 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:12.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:12.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:12.222 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:12.251 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:12.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:12.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:12.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[105589]: cluster 2023-10-09T05:39:10.952280+0000 osd.4 (osd.4) 304 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T05:39:12.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[105589]: cluster 2023-10-09T05:39:10.953487+0000 osd.7 (osd.7) 301 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T05:39:12.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[105589]: cluster 2023-10-09T05:39:10.953990+0000 osd.5 (osd.5) 359 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T05:39:12.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[105589]: cluster 2023-10-09T05:39:10.954349+0000 mon.a (mon.0) 1953 : cluster 0 osdmap e346: 8 total, 8 up, 7 in 2023-10-09T05:39:12.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[105589]: cluster 2023-10-09T05:39:10.955147+0000 osd.6 (osd.6) 330 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T05:39:12.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[105589]: cluster 2023-10-09T05:39:10.956931+0000 osd.6 (osd.6) 331 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T05:39:12.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[105589]: cluster 2023-10-09T05:39:10.957301+0000 osd.7 (osd.7) 302 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T05:39:12.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[105589]: cluster 2023-10-09T05:39:10.959997+0000 osd.5 (osd.5) 360 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T05:39:12.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[105589]: cluster 2023-10-09T05:39:10.964306+0000 osd.5 (osd.5) 361 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T05:39:12.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[105589]: cluster 2023-10-09T05:39:10.965977+0000 osd.5 (osd.5) 362 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T05:39:12.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[102858]: cluster 2023-10-09T05:39:10.952280+0000 osd.4 (osd.4) 304 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T05:39:12.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[102858]: cluster 2023-10-09T05:39:10.953487+0000 osd.7 (osd.7) 301 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T05:39:12.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[102858]: cluster 2023-10-09T05:39:10.953990+0000 osd.5 (osd.5) 359 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T05:39:12.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[102858]: cluster 2023-10-09T05:39:10.954349+0000 mon.a (mon.0) 1953 : cluster 0 osdmap e346: 8 total, 8 up, 7 in 2023-10-09T05:39:12.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[102858]: cluster 2023-10-09T05:39:10.955147+0000 osd.6 (osd.6) 330 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T05:39:12.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[102858]: cluster 2023-10-09T05:39:10.956931+0000 osd.6 (osd.6) 331 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T05:39:12.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[102858]: cluster 2023-10-09T05:39:10.957301+0000 osd.7 (osd.7) 302 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T05:39:12.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[102858]: cluster 2023-10-09T05:39:10.959997+0000 osd.5 (osd.5) 360 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T05:39:12.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[102858]: cluster 2023-10-09T05:39:10.964306+0000 osd.5 (osd.5) 361 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T05:39:12.300 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[102858]: cluster 2023-10-09T05:39:10.965977+0000 osd.5 (osd.5) 362 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T05:39:12.353 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:12.387 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:12.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:12.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:12.658 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:12.682 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:12.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:12.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:12.788 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:12.813 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:12.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:12.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:12.917 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:12.946 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:12.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:12.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:13.052 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:13.080 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:13.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:13.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:13.183 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:13.205 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:13.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:13.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:13.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[102858]: cluster 2023-10-09T05:39:11.954408+0000 osd.5 (osd.5) 363 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T05:39:13.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[102858]: cluster 2023-10-09T05:39:11.956605+0000 mon.a (mon.0) 1954 : cluster 0 osdmap e347: 8 total, 8 up, 7 in 2023-10-09T05:39:13.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[102858]: cluster 2023-10-09T05:39:11.957155+0000 osd.7 (osd.7) 303 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T05:39:13.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[102858]: cluster 2023-10-09T05:39:11.960085+0000 osd.7 (osd.7) 304 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T05:39:13.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[102858]: cluster 2023-10-09T05:39:11.964738+0000 osd.7 (osd.7) 305 : cluster 0 14.10 starting backfill to osd.6 from (0'0,0'0] MAX to 344'7115 2023-10-09T05:39:13.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[102858]: cluster 2023-10-09T05:39:11.990460+0000 osd.4 (osd.4) 305 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T05:39:13.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[102858]: cluster 2023-10-09T05:39:11.993103+0000 osd.4 (osd.4) 306 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T05:39:13.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[102858]: cluster 2023-10-09T05:39:12.460583+0000 mgr.y (mgr.25277) 1542 : cluster 0 pgmap v1677: 109 pgs: 4 peering, 1 active+recovery_wait, 104 active+clean; 1.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 720 op/s 2023-10-09T05:39:13.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[105589]: cluster 2023-10-09T05:39:11.954408+0000 osd.5 (osd.5) 363 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T05:39:13.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[105589]: cluster 2023-10-09T05:39:11.956605+0000 mon.a (mon.0) 1954 : cluster 0 osdmap e347: 8 total, 8 up, 7 in 2023-10-09T05:39:13.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[105589]: cluster 2023-10-09T05:39:11.957155+0000 osd.7 (osd.7) 303 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T05:39:13.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[105589]: cluster 2023-10-09T05:39:11.960085+0000 osd.7 (osd.7) 304 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T05:39:13.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[105589]: cluster 2023-10-09T05:39:11.964738+0000 osd.7 (osd.7) 305 : cluster 0 14.10 starting backfill to osd.6 from (0'0,0'0] MAX to 344'7115 2023-10-09T05:39:13.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[105589]: cluster 2023-10-09T05:39:11.990460+0000 osd.4 (osd.4) 305 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T05:39:13.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[105589]: cluster 2023-10-09T05:39:11.993103+0000 osd.4 (osd.4) 306 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T05:39:13.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:12 smithi133 bash[105589]: cluster 2023-10-09T05:39:12.460583+0000 mgr.y (mgr.25277) 1542 : cluster 0 pgmap v1677: 109 pgs: 4 peering, 1 active+recovery_wait, 104 active+clean; 1.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 720 op/s 2023-10-09T05:39:13.310 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:13.320 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:12 smithi194 bash[92666]: cluster 2023-10-09T05:39:11.954408+0000 osd.5 (osd.5) 363 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T05:39:13.320 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:12 smithi194 bash[92666]: cluster 2023-10-09T05:39:11.956605+0000 mon.a (mon.0) 1954 : cluster 0 osdmap e347: 8 total, 8 up, 7 in 2023-10-09T05:39:13.320 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:12 smithi194 bash[92666]: cluster 2023-10-09T05:39:11.957155+0000 osd.7 (osd.7) 303 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T05:39:13.320 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:12 smithi194 bash[92666]: cluster 2023-10-09T05:39:11.960085+0000 osd.7 (osd.7) 304 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T05:39:13.320 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:12 smithi194 bash[92666]: cluster 2023-10-09T05:39:11.964738+0000 osd.7 (osd.7) 305 : cluster 0 14.10 starting backfill to osd.6 from (0'0,0'0] MAX to 344'7115 2023-10-09T05:39:13.320 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:12 smithi194 bash[92666]: cluster 2023-10-09T05:39:11.990460+0000 osd.4 (osd.4) 305 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T05:39:13.320 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:12 smithi194 bash[92666]: cluster 2023-10-09T05:39:11.993103+0000 osd.4 (osd.4) 306 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T05:39:13.320 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:12 smithi194 bash[92666]: cluster 2023-10-09T05:39:12.460583+0000 mgr.y (mgr.25277) 1542 : cluster 0 pgmap v1677: 109 pgs: 4 peering, 1 active+recovery_wait, 104 active+clean; 1.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 720 op/s 2023-10-09T05:39:13.337 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:13.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:13.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:13.443 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:13.507 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:13.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:13.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:13.613 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:13.643 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:13.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:13.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:13.746 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:13.779 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:13.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:13.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:13.883 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:13.912 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:13.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:13.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:14.016 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:14.035 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:14.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:14.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:14.138 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:14.157 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:14.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:14.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:14.265 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:14.281 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:39:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:39:14] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.43.0" 2023-10-09T05:39:14.290 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:14.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:14.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:14.296 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-10-09T05:39:14.296 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 1.0 2023-10-09T05:39:14.394 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:14.421 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:14.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:14.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:14.525 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:14.533 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:14 smithi194 bash[92666]: audit 2023-10-09T05:39:13.498432+0000 mon.a (mon.0) 1955 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:39:14.533 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:14 smithi194 bash[92666]: audit 2023-10-09T05:39:13.499753+0000 mon.a (mon.0) 1956 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:14.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:14 smithi133 bash[102858]: audit 2023-10-09T05:39:13.498432+0000 mon.a (mon.0) 1955 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:39:14.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:14 smithi133 bash[102858]: audit 2023-10-09T05:39:13.499753+0000 mon.a (mon.0) 1956 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:14.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:14 smithi133 bash[105589]: audit 2023-10-09T05:39:13.498432+0000 mon.a (mon.0) 1955 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:39:14.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:14 smithi133 bash[105589]: audit 2023-10-09T05:39:13.499753+0000 mon.a (mon.0) 1956 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:14.551 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:14.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:14.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:14.655 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:14.691 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:14.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:14.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:14.797 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:14.833 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:14.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:14.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:14.940 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:14.980 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:14.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:14.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:15.084 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:15.108 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:15.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:15.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:15.212 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:15.248 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:15.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:15.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:15.353 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:15.382 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:15.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:15.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:15.485 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:15.499 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:15 smithi133 bash[102858]: cluster 2023-10-09T05:39:14.461592+0000 mgr.y (mgr.25277) 1543 : cluster 0 pgmap v1678: 109 pgs: 4 peering, 1 active+recovery_wait, 104 active+clean; 1.7 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 723 op/s 2023-10-09T05:39:15.511 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:15.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:15.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:15.616 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:15.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:15 smithi194 bash[92666]: cluster 2023-10-09T05:39:14.461592+0000 mgr.y (mgr.25277) 1543 : cluster 0 pgmap v1678: 109 pgs: 4 peering, 1 active+recovery_wait, 104 active+clean; 1.7 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 723 op/s 2023-10-09T05:39:15.641 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:15.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:15.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:15.747 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:15.777 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:15.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:15.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:15.884 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:15.915 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:15.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:15.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:15.948 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:15 smithi133 bash[105589]: cluster 2023-10-09T05:39:14.461592+0000 mgr.y (mgr.25277) 1543 : cluster 0 pgmap v1678: 109 pgs: 4 peering, 1 active+recovery_wait, 104 active+clean; 1.7 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 723 op/s 2023-10-09T05:39:16.018 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:16.050 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:16.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:16.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:16.161 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:16.189 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:16.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:16.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:16.291 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:39:16.297 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:16.380 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:16.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:16.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:16.485 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:16.513 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:16.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:16.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:16.618 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:16.652 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:16.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:16.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:16.755 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:16.792 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:16.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:16.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:16.897 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:16.924 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:16.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:16.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:17.028 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:17.054 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:17.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:17.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:17.138 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:17.159 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:17.161 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:16 smithi133 bash[105589]: cluster 2023-10-09T05:39:15.962664+0000 osd.5 (osd.5) 364 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T05:39:17.161 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:16 smithi133 bash[105589]: cluster 2023-10-09T05:39:15.964572+0000 osd.5 (osd.5) 365 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T05:39:17.161 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:16 smithi133 bash[105589]: cluster 2023-10-09T05:39:15.964738+0000 osd.7 (osd.7) 306 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T05:39:17.161 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:16 smithi133 bash[105589]: cluster 2023-10-09T05:39:15.964738+0000 mon.a (mon.0) 1957 : cluster 0 osdmap e348: 8 total, 8 up, 7 in 2023-10-09T05:39:17.161 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:16 smithi133 bash[105589]: cluster 2023-10-09T05:39:15.966261+0000 osd.6 (osd.6) 332 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T05:39:17.161 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:16 smithi133 bash[105589]: cluster 2023-10-09T05:39:15.967480+0000 osd.7 (osd.7) 307 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T05:39:17.161 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:16 smithi133 bash[105589]: cluster 2023-10-09T05:39:15.969843+0000 osd.6 (osd.6) 333 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T05:39:17.161 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:16 smithi133 bash[105589]: cluster 2023-10-09T05:39:15.970275+0000 osd.7 (osd.7) 308 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T05:39:17.162 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:16 smithi133 bash[105589]: cluster 2023-10-09T05:39:16.177706+0000 osd.4 (osd.4) 307 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T05:39:17.162 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:16 smithi133 bash[105589]: cluster 2023-10-09T05:39:16.462115+0000 mgr.y (mgr.25277) 1544 : cluster 0 pgmap v1680: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 1.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 718 op/s; 12/79356 objects misplaced (0.015%); 8.0 MiB/s, 128 objects/s recovering 2023-10-09T05:39:17.162 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:16 smithi133 bash[105589]: audit 2023-10-09T05:39:16.462919+0000 mon.a (mon.0) 1958 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-10-09T05:39:17.162 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:16 smithi133 bash[102858]: cluster 2023-10-09T05:39:15.962664+0000 osd.5 (osd.5) 364 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T05:39:17.162 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:16 smithi133 bash[102858]: cluster 2023-10-09T05:39:15.964572+0000 osd.5 (osd.5) 365 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T05:39:17.163 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:16 smithi133 bash[102858]: cluster 2023-10-09T05:39:15.964738+0000 osd.7 (osd.7) 306 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T05:39:17.163 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:16 smithi133 bash[102858]: cluster 2023-10-09T05:39:15.964738+0000 mon.a (mon.0) 1957 : cluster 0 osdmap e348: 8 total, 8 up, 7 in 2023-10-09T05:39:17.163 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:16 smithi133 bash[102858]: cluster 2023-10-09T05:39:15.966261+0000 osd.6 (osd.6) 332 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T05:39:17.163 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:16 smithi133 bash[102858]: cluster 2023-10-09T05:39:15.967480+0000 osd.7 (osd.7) 307 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T05:39:17.163 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:16 smithi133 bash[102858]: cluster 2023-10-09T05:39:15.969843+0000 osd.6 (osd.6) 333 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T05:39:17.163 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:16 smithi133 bash[102858]: cluster 2023-10-09T05:39:15.970275+0000 osd.7 (osd.7) 308 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T05:39:17.163 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:16 smithi133 bash[102858]: cluster 2023-10-09T05:39:16.177706+0000 osd.4 (osd.4) 307 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T05:39:17.163 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:16 smithi133 bash[102858]: cluster 2023-10-09T05:39:16.462115+0000 mgr.y (mgr.25277) 1544 : cluster 0 pgmap v1680: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 1.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 718 op/s; 12/79356 objects misplaced (0.015%); 8.0 MiB/s, 128 objects/s recovering 2023-10-09T05:39:17.163 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:16 smithi133 bash[102858]: audit 2023-10-09T05:39:16.462919+0000 mon.a (mon.0) 1958 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-10-09T05:39:17.193 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:17.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:17.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:17.301 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:17.310 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:16 smithi194 bash[92666]: cluster 2023-10-09T05:39:15.962664+0000 osd.5 (osd.5) 364 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T05:39:17.310 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:16 smithi194 bash[92666]: cluster 2023-10-09T05:39:15.964572+0000 osd.5 (osd.5) 365 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T05:39:17.310 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:16 smithi194 bash[92666]: cluster 2023-10-09T05:39:15.964738+0000 osd.7 (osd.7) 306 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T05:39:17.310 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:16 smithi194 bash[92666]: cluster 2023-10-09T05:39:15.964738+0000 mon.a (mon.0) 1957 : cluster 0 osdmap e348: 8 total, 8 up, 7 in 2023-10-09T05:39:17.310 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:16 smithi194 bash[92666]: cluster 2023-10-09T05:39:15.966261+0000 osd.6 (osd.6) 332 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T05:39:17.310 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:16 smithi194 bash[92666]: cluster 2023-10-09T05:39:15.967480+0000 osd.7 (osd.7) 307 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T05:39:17.311 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:16 smithi194 bash[92666]: cluster 2023-10-09T05:39:15.969843+0000 osd.6 (osd.6) 333 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T05:39:17.311 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:16 smithi194 bash[92666]: cluster 2023-10-09T05:39:15.970275+0000 osd.7 (osd.7) 308 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T05:39:17.311 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:16 smithi194 bash[92666]: cluster 2023-10-09T05:39:16.177706+0000 osd.4 (osd.4) 307 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T05:39:17.311 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:16 smithi194 bash[92666]: cluster 2023-10-09T05:39:16.462115+0000 mgr.y (mgr.25277) 1544 : cluster 0 pgmap v1680: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 1.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 718 op/s; 12/79356 objects misplaced (0.015%); 8.0 MiB/s, 128 objects/s recovering 2023-10-09T05:39:17.311 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:16 smithi194 bash[92666]: audit 2023-10-09T05:39:16.462919+0000 mon.a (mon.0) 1958 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-10-09T05:39:17.328 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:17.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:17.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:17.431 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:17.461 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:17.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:17.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:17.565 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:17.586 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:17.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:17.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:17.688 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:17.711 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:17.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:17.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:17.814 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:17.847 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:17.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:17.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:17.951 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:17.983 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:17.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:17.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:18.087 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:18.098 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:17 smithi194 bash[92666]: cluster 2023-10-09T05:39:16.968350+0000 osd.7 (osd.7) 309 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T05:39:18.098 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:17 smithi194 bash[92666]: audit 2023-10-09T05:39:16.969531+0000 mon.a (mon.0) 1959 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "22"}]': finished 2023-10-09T05:39:18.099 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:17 smithi194 bash[92666]: cluster 2023-10-09T05:39:16.969631+0000 mon.a (mon.0) 1960 : cluster 0 osdmap e349: 8 total, 8 up, 7 in 2023-10-09T05:39:18.099 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:17 smithi194 bash[92666]: cluster 2023-10-09T05:39:16.971476+0000 osd.7 (osd.7) 310 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T05:39:18.099 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:17 smithi194 bash[92666]: cluster 2023-10-09T05:39:16.974248+0000 osd.7 (osd.7) 311 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T05:39:18.099 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:17 smithi194 bash[92666]: cluster 2023-10-09T05:39:16.975951+0000 osd.7 (osd.7) 312 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T05:39:18.124 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:18.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:18.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:18.228 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:18.266 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:18.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:18.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:18.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:18 smithi133 bash[105589]: cluster 2023-10-09T05:39:16.968350+0000 osd.7 (osd.7) 309 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T05:39:18.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:18 smithi133 bash[105589]: audit 2023-10-09T05:39:16.969531+0000 mon.a (mon.0) 1959 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "22"}]': finished 2023-10-09T05:39:18.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:18 smithi133 bash[105589]: cluster 2023-10-09T05:39:16.969631+0000 mon.a (mon.0) 1960 : cluster 0 osdmap e349: 8 total, 8 up, 7 in 2023-10-09T05:39:18.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:18 smithi133 bash[105589]: cluster 2023-10-09T05:39:16.971476+0000 osd.7 (osd.7) 310 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T05:39:18.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:18 smithi133 bash[105589]: cluster 2023-10-09T05:39:16.974248+0000 osd.7 (osd.7) 311 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T05:39:18.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:18 smithi133 bash[105589]: cluster 2023-10-09T05:39:16.975951+0000 osd.7 (osd.7) 312 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T05:39:18.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:18 smithi133 bash[102858]: cluster 2023-10-09T05:39:16.968350+0000 osd.7 (osd.7) 309 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T05:39:18.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:18 smithi133 bash[102858]: audit 2023-10-09T05:39:16.969531+0000 mon.a (mon.0) 1959 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "22"}]': finished 2023-10-09T05:39:18.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:18 smithi133 bash[102858]: cluster 2023-10-09T05:39:16.969631+0000 mon.a (mon.0) 1960 : cluster 0 osdmap e349: 8 total, 8 up, 7 in 2023-10-09T05:39:18.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:18 smithi133 bash[102858]: cluster 2023-10-09T05:39:16.971476+0000 osd.7 (osd.7) 310 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T05:39:18.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:18 smithi133 bash[102858]: cluster 2023-10-09T05:39:16.974248+0000 osd.7 (osd.7) 311 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T05:39:18.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:18 smithi133 bash[102858]: cluster 2023-10-09T05:39:16.975951+0000 osd.7 (osd.7) 312 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T05:39:18.369 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:18.402 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:18.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:18.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:18.506 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:18.537 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:18.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:18.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:18.644 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:18.672 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:18.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:18.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:18.775 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:18.806 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:18.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:18.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:18.910 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:18.938 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:18.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:18.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:19.042 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:19.050 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:18 smithi194 bash[92666]: cluster 2023-10-09T05:39:17.967774+0000 osd.4 (osd.4) 308 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T05:39:19.050 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:18 smithi194 bash[92666]: cluster 2023-10-09T05:39:17.972060+0000 mon.a (mon.0) 1961 : cluster 0 osdmap e350: 8 total, 8 up, 7 in 2023-10-09T05:39:19.050 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:18 smithi194 bash[92666]: cluster 2023-10-09T05:39:17.972778+0000 osd.7 (osd.7) 313 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T05:39:19.050 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:18 smithi194 bash[92666]: cluster 2023-10-09T05:39:18.462916+0000 mgr.y (mgr.25277) 1545 : cluster 0 pgmap v1683: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 1.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 12/77622 objects misplaced (0.015%); 8.0 MiB/s, 128 objects/s recovering 2023-10-09T05:39:19.051 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:18 smithi194 bash[92666]: audit 2023-10-09T05:39:18.464558+0000 mon.a (mon.0) 1962 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-10-09T05:39:19.069 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:19.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:19.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:19.169 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:19.172 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:19.192 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:18 smithi133 bash[105589]: cluster 2023-10-09T05:39:17.967774+0000 osd.4 (osd.4) 308 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T05:39:19.192 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:18 smithi133 bash[105589]: cluster 2023-10-09T05:39:17.972060+0000 mon.a (mon.0) 1961 : cluster 0 osdmap e350: 8 total, 8 up, 7 in 2023-10-09T05:39:19.192 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:18 smithi133 bash[105589]: cluster 2023-10-09T05:39:17.972778+0000 osd.7 (osd.7) 313 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T05:39:19.193 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:18 smithi133 bash[105589]: cluster 2023-10-09T05:39:18.462916+0000 mgr.y (mgr.25277) 1545 : cluster 0 pgmap v1683: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 1.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 12/77622 objects misplaced (0.015%); 8.0 MiB/s, 128 objects/s recovering 2023-10-09T05:39:19.193 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:18 smithi133 bash[105589]: audit 2023-10-09T05:39:18.464558+0000 mon.a (mon.0) 1962 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-10-09T05:39:19.193 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:18 smithi133 bash[102858]: cluster 2023-10-09T05:39:17.967774+0000 osd.4 (osd.4) 308 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T05:39:19.193 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:18 smithi133 bash[102858]: cluster 2023-10-09T05:39:17.972060+0000 mon.a (mon.0) 1961 : cluster 0 osdmap e350: 8 total, 8 up, 7 in 2023-10-09T05:39:19.194 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:18 smithi133 bash[102858]: cluster 2023-10-09T05:39:17.972778+0000 osd.7 (osd.7) 313 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T05:39:19.194 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:18 smithi133 bash[102858]: cluster 2023-10-09T05:39:18.462916+0000 mgr.y (mgr.25277) 1545 : cluster 0 pgmap v1683: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 1.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 12/77622 objects misplaced (0.015%); 8.0 MiB/s, 128 objects/s recovering 2023-10-09T05:39:19.194 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:18 smithi133 bash[102858]: audit 2023-10-09T05:39:18.464558+0000 mon.a (mon.0) 1962 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-10-09T05:39:19.200 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:19.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:19.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:19.305 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:19.330 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:19.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:19.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:19.438 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:19.456 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:19.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:19.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:19.560 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:19.595 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:19.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:19.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:19.700 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:19.730 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:19.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:19.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:19.835 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:19.856 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:19.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:19.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:19.959 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:19.988 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:19.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:19.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:19.992 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:19 smithi133 bash[102858]: cluster 2023-10-09T05:39:18.122798+0000 osd.6 (osd.6) 334 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T05:39:19.992 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:19 smithi133 bash[102858]: cluster 2023-10-09T05:39:18.984457+0000 osd.6 (osd.6) 335 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T05:39:19.992 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:19 smithi133 bash[102858]: cluster 2023-10-09T05:39:18.984738+0000 osd.5 (osd.5) 366 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T05:39:19.992 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:19 smithi133 bash[102858]: audit 2023-10-09T05:39:18.986513+0000 mon.a (mon.0) 1963 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "23"}]': finished 2023-10-09T05:39:19.993 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:19 smithi133 bash[102858]: cluster 2023-10-09T05:39:18.986586+0000 mon.a (mon.0) 1964 : cluster 0 osdmap e351: 8 total, 8 up, 7 in 2023-10-09T05:39:19.993 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:19 smithi133 bash[102858]: cluster 2023-10-09T05:39:18.987783+0000 osd.7 (osd.7) 314 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T05:39:19.993 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:19 smithi133 bash[102858]: cluster 2023-10-09T05:39:18.991878+0000 osd.4 (osd.4) 309 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T05:39:19.993 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:19 smithi133 bash[102858]: cluster 2023-10-09T05:39:18.991918+0000 osd.7 (osd.7) 315 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T05:39:19.993 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:19 smithi133 bash[102858]: cluster 2023-10-09T05:39:18.994257+0000 osd.7 (osd.7) 316 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T05:39:19.993 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:19 smithi133 bash[102858]: cluster 2023-10-09T05:39:18.994370+0000 osd.4 (osd.4) 310 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T05:39:20.092 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:20.120 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:20.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:20.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:20.186 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:20.224 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:20.233 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:19 smithi194 bash[92666]: cluster 2023-10-09T05:39:18.122798+0000 osd.6 (osd.6) 334 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T05:39:20.233 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:19 smithi194 bash[92666]: cluster 2023-10-09T05:39:18.984457+0000 osd.6 (osd.6) 335 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T05:39:20.233 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:19 smithi194 bash[92666]: cluster 2023-10-09T05:39:18.984738+0000 osd.5 (osd.5) 366 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T05:39:20.233 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:19 smithi194 bash[92666]: audit 2023-10-09T05:39:18.986513+0000 mon.a (mon.0) 1963 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "23"}]': finished 2023-10-09T05:39:20.233 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:19 smithi194 bash[92666]: cluster 2023-10-09T05:39:18.986586+0000 mon.a (mon.0) 1964 : cluster 0 osdmap e351: 8 total, 8 up, 7 in 2023-10-09T05:39:20.233 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:19 smithi194 bash[92666]: cluster 2023-10-09T05:39:18.987783+0000 osd.7 (osd.7) 314 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T05:39:20.234 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:19 smithi194 bash[92666]: cluster 2023-10-09T05:39:18.991878+0000 osd.4 (osd.4) 309 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T05:39:20.234 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:19 smithi194 bash[92666]: cluster 2023-10-09T05:39:18.991918+0000 osd.7 (osd.7) 315 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T05:39:20.234 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:19 smithi194 bash[92666]: cluster 2023-10-09T05:39:18.994257+0000 osd.7 (osd.7) 316 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T05:39:20.234 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:19 smithi194 bash[92666]: cluster 2023-10-09T05:39:18.994370+0000 osd.4 (osd.4) 310 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T05:39:20.249 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:20.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:20.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:20.348 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:19 smithi133 bash[105589]: cluster 2023-10-09T05:39:18.122798+0000 osd.6 (osd.6) 334 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T05:39:20.348 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:19 smithi133 bash[105589]: cluster 2023-10-09T05:39:18.984457+0000 osd.6 (osd.6) 335 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T05:39:20.348 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:19 smithi133 bash[105589]: cluster 2023-10-09T05:39:18.984738+0000 osd.5 (osd.5) 366 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T05:39:20.349 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:19 smithi133 bash[105589]: audit 2023-10-09T05:39:18.986513+0000 mon.a (mon.0) 1963 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "23"}]': finished 2023-10-09T05:39:20.349 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:19 smithi133 bash[105589]: cluster 2023-10-09T05:39:18.986586+0000 mon.a (mon.0) 1964 : cluster 0 osdmap e351: 8 total, 8 up, 7 in 2023-10-09T05:39:20.349 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:19 smithi133 bash[105589]: cluster 2023-10-09T05:39:18.987783+0000 osd.7 (osd.7) 314 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T05:39:20.349 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:19 smithi133 bash[105589]: cluster 2023-10-09T05:39:18.991878+0000 osd.4 (osd.4) 309 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T05:39:20.349 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:19 smithi133 bash[105589]: cluster 2023-10-09T05:39:18.991918+0000 osd.7 (osd.7) 315 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T05:39:20.349 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:19 smithi133 bash[105589]: cluster 2023-10-09T05:39:18.994257+0000 osd.7 (osd.7) 316 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T05:39:20.349 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:19 smithi133 bash[105589]: cluster 2023-10-09T05:39:18.994370+0000 osd.4 (osd.4) 310 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T05:39:20.351 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:20.379 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:20.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:20.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:20.483 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:20.508 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:20.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:20.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:20.611 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:20.631 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:20.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:20.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:20.734 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:20.759 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:20.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:20.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:20.862 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:20.884 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:20.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:20.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:20.987 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:21.016 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:21.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:21.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:21.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:21 smithi133 bash[105589]: cluster 2023-10-09T05:39:19.985021+0000 osd.6 (osd.6) 336 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T05:39:21.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:21 smithi133 bash[105589]: cluster 2023-10-09T05:39:19.986984+0000 mon.a (mon.0) 1965 : cluster 0 osdmap e352: 8 total, 8 up, 7 in 2023-10-09T05:39:21.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:21 smithi133 bash[105589]: cluster 2023-10-09T05:39:19.993787+0000 osd.5 (osd.5) 367 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T05:39:21.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:21 smithi133 bash[105589]: cluster 2023-10-09T05:39:20.463867+0000 mgr.y (mgr.25277) 1546 : cluster 0 pgmap v1686: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 1.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 464 op/s; 17/73779 objects degraded (0.023%); 703/73779 objects misplaced (0.953%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T05:39:21.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:21 smithi133 bash[102858]: cluster 2023-10-09T05:39:19.985021+0000 osd.6 (osd.6) 336 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T05:39:21.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:21 smithi133 bash[102858]: cluster 2023-10-09T05:39:19.986984+0000 mon.a (mon.0) 1965 : cluster 0 osdmap e352: 8 total, 8 up, 7 in 2023-10-09T05:39:21.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:21 smithi133 bash[102858]: cluster 2023-10-09T05:39:19.993787+0000 osd.5 (osd.5) 367 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T05:39:21.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:21 smithi133 bash[102858]: cluster 2023-10-09T05:39:20.463867+0000 mgr.y (mgr.25277) 1546 : cluster 0 pgmap v1686: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 1.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 464 op/s; 17/73779 objects degraded (0.023%); 703/73779 objects misplaced (0.953%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T05:39:21.119 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:21.150 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:21.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:21.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:21.253 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:21.264 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:20 smithi194 bash[92666]: cluster 2023-10-09T05:39:19.985021+0000 osd.6 (osd.6) 336 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T05:39:21.264 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:20 smithi194 bash[92666]: cluster 2023-10-09T05:39:19.986984+0000 mon.a (mon.0) 1965 : cluster 0 osdmap e352: 8 total, 8 up, 7 in 2023-10-09T05:39:21.264 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:20 smithi194 bash[92666]: cluster 2023-10-09T05:39:19.993787+0000 osd.5 (osd.5) 367 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T05:39:21.264 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:20 smithi194 bash[92666]: cluster 2023-10-09T05:39:20.463867+0000 mgr.y (mgr.25277) 1546 : cluster 0 pgmap v1686: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 1.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 464 op/s; 17/73779 objects degraded (0.023%); 703/73779 objects misplaced (0.953%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T05:39:21.283 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:21.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:21.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:21.387 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:21.414 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:21.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:21.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:21.517 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:21.539 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:21.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:21.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:21.641 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:21.670 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:21.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:21.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:21.751 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:39:21.776 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:21.803 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:21.804 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:39:21.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:21.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:21.845 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:39:21.889 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:39:21.910 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:21.922 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:39:21.940 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:21.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:21.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:21.961 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:39:22.000 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:39:22.037 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:39:22.040 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:39:22.041 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:39:22.041 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:39:22.041 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:39:22.041 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:39:22.041 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:39:22.041 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:39:22.041 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:39:22.044 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:22.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:21 smithi133 bash[102858]: cluster 2023-10-09T05:39:20.990845+0000 mon.a (mon.0) 1966 : cluster 3 Health check failed: Degraded data redundancy: 17/73779 objects degraded (0.023%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:39:22.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:21 smithi133 bash[102858]: cluster 2023-10-09T05:39:21.010399+0000 mon.a (mon.0) 1967 : cluster 0 osdmap e353: 8 total, 8 up, 7 in 2023-10-09T05:39:22.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:21 smithi133 bash[102858]: cluster 2023-10-09T05:39:21.010654+0000 osd.0 (osd.0) 9 : cluster 0 14.16 starting backfill to osd.3 from (0'0,0'0] MAX to 349'7670 2023-10-09T05:39:22.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:22 smithi133 bash[102858]: cluster 2023-10-09T05:39:21.023745+0000 osd.0 (osd.0) 10 : cluster 0 14.16 starting backfill to osd.5 from (0'0,0'0] MAX to 349'7670 2023-10-09T05:39:22.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:22 smithi133 bash[102858]: audit 2023-10-09T05:39:21.692580+0000 mon.c (mon.2) 142 : audit 0 from='client.? 172.21.15.133:0/2757216218' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:39:22.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:22 smithi133 bash[105589]: cluster 2023-10-09T05:39:20.990845+0000 mon.a (mon.0) 1966 : cluster 3 Health check failed: Degraded data redundancy: 17/73779 objects degraded (0.023%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:39:22.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:22 smithi133 bash[105589]: cluster 2023-10-09T05:39:21.010399+0000 mon.a (mon.0) 1967 : cluster 0 osdmap e353: 8 total, 8 up, 7 in 2023-10-09T05:39:22.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:22 smithi133 bash[105589]: cluster 2023-10-09T05:39:21.010654+0000 osd.0 (osd.0) 9 : cluster 0 14.16 starting backfill to osd.3 from (0'0,0'0] MAX to 349'7670 2023-10-09T05:39:22.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:22 smithi133 bash[105589]: cluster 2023-10-09T05:39:21.023745+0000 osd.0 (osd.0) 10 : cluster 0 14.16 starting backfill to osd.5 from (0'0,0'0] MAX to 349'7670 2023-10-09T05:39:22.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:22 smithi133 bash[105589]: audit 2023-10-09T05:39:21.692580+0000 mon.c (mon.2) 142 : audit 0 from='client.? 172.21.15.133:0/2757216218' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:39:22.076 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:22.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:22.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:22.183 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:22.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:22.223 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:22.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:22.327 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:22.373 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:22.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:22.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:22.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:22 smithi194 bash[92666]: cluster 2023-10-09T05:39:20.990845+0000 mon.a (mon.0) 1966 : cluster 3 Health check failed: Degraded data redundancy: 17/73779 objects degraded (0.023%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:39:22.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:22 smithi194 bash[92666]: cluster 2023-10-09T05:39:21.010399+0000 mon.a (mon.0) 1967 : cluster 0 osdmap e353: 8 total, 8 up, 7 in 2023-10-09T05:39:22.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:22 smithi194 bash[92666]: cluster 2023-10-09T05:39:21.010654+0000 osd.0 (osd.0) 9 : cluster 0 14.16 starting backfill to osd.3 from (0'0,0'0] MAX to 349'7670 2023-10-09T05:39:22.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:22 smithi194 bash[92666]: cluster 2023-10-09T05:39:21.023745+0000 osd.0 (osd.0) 10 : cluster 0 14.16 starting backfill to osd.5 from (0'0,0'0] MAX to 349'7670 2023-10-09T05:39:22.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:22 smithi194 bash[92666]: audit 2023-10-09T05:39:21.692580+0000 mon.c (mon.2) 142 : audit 0 from='client.? 172.21.15.133:0/2757216218' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:39:22.479 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:22.507 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:22.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:22.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:22.611 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:22.635 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:22.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:22.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:22.738 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:22.771 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:22.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:22.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:22.877 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:22.899 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:22.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:22.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:23.003 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:23.043 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:23.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:23.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:23.109 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:23 smithi133 bash[102858]: cluster 2023-10-09T05:39:21.283619+0000 osd.5 (osd.5) 368 : cluster 3 failed to encode map e353 with expected crc 2023-10-09T05:39:23.148 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:23.179 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:23.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:23.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:23.283 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:23.295 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:23 smithi133 bash[102858]: cluster 2023-10-09T05:39:21.399078+0000 osd.6 (osd.6) 337 : cluster 3 failed to encode map e353 with expected crc 2023-10-09T05:39:23.296 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:23 smithi133 bash[102858]: cluster 2023-10-09T05:39:22.464408+0000 mgr.y (mgr.25277) 1547 : cluster 0 pgmap v1688: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 487 op/s; 17/71046 objects degraded (0.024%); 703/71046 objects misplaced (0.989%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T05:39:23.296 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:23 smithi133 bash[105589]: cluster 2023-10-09T05:39:21.283619+0000 osd.5 (osd.5) 368 : cluster 3 failed to encode map e353 with expected crc 2023-10-09T05:39:23.296 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:23 smithi133 bash[105589]: cluster 2023-10-09T05:39:21.399078+0000 osd.6 (osd.6) 337 : cluster 3 failed to encode map e353 with expected crc 2023-10-09T05:39:23.296 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:23 smithi133 bash[105589]: cluster 2023-10-09T05:39:22.464408+0000 mgr.y (mgr.25277) 1547 : cluster 0 pgmap v1688: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 487 op/s; 17/71046 objects degraded (0.024%); 703/71046 objects misplaced (0.989%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T05:39:23.312 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:23.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:23.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:23.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:23 smithi194 bash[92666]: cluster 2023-10-09T05:39:21.283619+0000 osd.5 (osd.5) 368 : cluster 3 failed to encode map e353 with expected crc 2023-10-09T05:39:23.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:23 smithi194 bash[92666]: cluster 2023-10-09T05:39:21.399078+0000 osd.6 (osd.6) 337 : cluster 3 failed to encode map e353 with expected crc 2023-10-09T05:39:23.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:23 smithi194 bash[92666]: cluster 2023-10-09T05:39:22.464408+0000 mgr.y (mgr.25277) 1547 : cluster 0 pgmap v1688: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 487 op/s; 17/71046 objects degraded (0.024%); 703/71046 objects misplaced (0.989%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T05:39:23.417 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:23.448 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:23.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:23.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:23.553 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:23.582 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:23.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:23.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:23.689 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:23.724 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:23.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:23.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:23.829 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:23.858 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:23.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:23.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:23.961 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:23.988 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:23.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:23.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:24.093 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:24.110 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:24.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:24.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:24.213 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:24.242 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:24.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:24.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:24.336 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:24 smithi133 bash[105589]: audit 2023-10-09T05:39:23.003397+0000 mon.a (mon.0) 1968 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:39:24.336 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:24 smithi133 bash[102858]: audit 2023-10-09T05:39:23.003397+0000 mon.a (mon.0) 1968 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:39:24.337 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:39:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:39:24] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.43.0" 2023-10-09T05:39:24.346 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:24.376 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:24.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:24.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:24.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:24 smithi194 bash[92666]: audit 2023-10-09T05:39:23.003397+0000 mon.a (mon.0) 1968 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:39:24.478 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:24.512 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:24.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:24.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:24.617 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:24.640 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:24.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:24.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:24.744 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:24.780 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:24.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:24.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:24.884 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:24.918 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:24.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:24.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:25.021 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:25.042 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:25.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:25.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:25.145 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:25.155 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:25 smithi194 bash[92666]: cluster 2023-10-09T05:39:24.465062+0000 mgr.y (mgr.25277) 1548 : cluster 0 pgmap v1689: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.4 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 17/67944 objects degraded (0.025%); 703/67944 objects misplaced (1.035%); 1.1 MiB/s, 16 objects/s recovering 2023-10-09T05:39:25.175 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:25.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:25.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:25.279 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:25.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:25 smithi133 bash[102858]: cluster 2023-10-09T05:39:24.465062+0000 mgr.y (mgr.25277) 1548 : cluster 0 pgmap v1689: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.4 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 17/67944 objects degraded (0.025%); 703/67944 objects misplaced (1.035%); 1.1 MiB/s, 16 objects/s recovering 2023-10-09T05:39:25.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:25 smithi133 bash[105589]: cluster 2023-10-09T05:39:24.465062+0000 mgr.y (mgr.25277) 1548 : cluster 0 pgmap v1689: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.4 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 17/67944 objects degraded (0.025%); 703/67944 objects misplaced (1.035%); 1.1 MiB/s, 16 objects/s recovering 2023-10-09T05:39:25.310 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:25.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:25.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:25.416 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:25.449 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:25.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:25.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:25.552 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:25.582 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:25.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:25.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:25.688 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:25.729 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:25.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:25.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:25.836 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:25.860 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:25.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:25.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:25.965 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:25.982 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:25.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:25.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:26.084 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:26.110 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:26.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:26.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:26.216 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:26.244 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:26.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:26.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:26.352 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:26.378 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:26.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:26.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:26.481 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:26.521 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:26.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:26.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:26.624 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:26.649 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:26.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:26.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:26.693 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:26 smithi133 bash[102858]: cluster 2023-10-09T05:39:26.517002+0000 mon.a (mon.0) 1969 : cluster 3 Health check update: Degraded data redundancy: 17/62445 objects degraded (0.027%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:39:26.753 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:26.763 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:26 smithi194 bash[92666]: cluster 2023-10-09T05:39:26.517002+0000 mon.a (mon.0) 1969 : cluster 3 Health check update: Degraded data redundancy: 17/62445 objects degraded (0.027%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:39:26.784 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:26.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:26.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:26.887 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:26.921 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:26.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:26.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:27.026 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:27.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:26 smithi133 bash[105589]: cluster 2023-10-09T05:39:26.517002+0000 mon.a (mon.0) 1969 : cluster 3 Health check update: Degraded data redundancy: 17/62445 objects degraded (0.027%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:39:27.052 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:27.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:27.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:27.158 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:27.188 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:27.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:27.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:27.291 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:27.327 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:27.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:27.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:27.430 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:27.459 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:27.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:27.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:27.562 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:27.598 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:27.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:27.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:27.689 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:27 smithi133 bash[102858]: cluster 2023-10-09T05:39:26.465941+0000 mgr.y (mgr.25277) 1549 : cluster 0 pgmap v1690: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 676 op/s; 17/62445 objects degraded (0.027%); 1415/62445 objects misplaced (2.266%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T05:39:27.689 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:27 smithi133 bash[105589]: cluster 2023-10-09T05:39:26.465941+0000 mgr.y (mgr.25277) 1549 : cluster 0 pgmap v1690: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 676 op/s; 17/62445 objects degraded (0.027%); 1415/62445 objects misplaced (2.266%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T05:39:27.702 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:27.728 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:27.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:27.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:27.831 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:27.862 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:27.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:27.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:27.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:27 smithi194 bash[92666]: cluster 2023-10-09T05:39:26.465941+0000 mgr.y (mgr.25277) 1549 : cluster 0 pgmap v1690: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 676 op/s; 17/62445 objects degraded (0.027%); 1415/62445 objects misplaced (2.266%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T05:39:27.967 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:27.998 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:28.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:28.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:28.103 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:28.134 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:28.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:28.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:28.242 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:28.269 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:28.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:28.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:28.376 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:28.408 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:28.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:28.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:28.515 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:28.545 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:28.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:28.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:28.649 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:28.680 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:28.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:28.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:28.786 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:28.816 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:28.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:28.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:28.920 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:28.944 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:28.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:28.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:29.049 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:29.118 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:29.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:29.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:29.223 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:29.248 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:29.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:29.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:29.354 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:29.384 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:29.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:29.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:29.489 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:29.519 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:29.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:29.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:29.549 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:39:29.625 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:29.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:29 smithi194 bash[92666]: cluster 2023-10-09T05:39:28.466478+0000 mgr.y (mgr.25277) 1550 : cluster 0 pgmap v1691: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 17/60366 objects degraded (0.028%); 1415/60366 objects misplaced (2.344%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T05:39:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:29 smithi194 bash[92666]: audit 2023-10-09T05:39:28.499326+0000 mon.a (mon.0) 1970 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:39:29.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:29 smithi194 bash[92666]: audit 2023-10-09T05:39:28.501247+0000 mon.a (mon.0) 1971 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:29.656 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:29.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:29.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:29.760 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:29.769 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:29 smithi133 bash[102858]: cluster 2023-10-09T05:39:28.466478+0000 mgr.y (mgr.25277) 1550 : cluster 0 pgmap v1691: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 17/60366 objects degraded (0.028%); 1415/60366 objects misplaced (2.344%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T05:39:29.769 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:29 smithi133 bash[102858]: audit 2023-10-09T05:39:28.499326+0000 mon.a (mon.0) 1970 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:39:29.769 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:29 smithi133 bash[102858]: audit 2023-10-09T05:39:28.501247+0000 mon.a (mon.0) 1971 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:29.770 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:29 smithi133 bash[105589]: cluster 2023-10-09T05:39:28.466478+0000 mgr.y (mgr.25277) 1550 : cluster 0 pgmap v1691: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 17/60366 objects degraded (0.028%); 1415/60366 objects misplaced (2.344%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T05:39:29.770 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:29 smithi133 bash[105589]: audit 2023-10-09T05:39:28.499326+0000 mon.a (mon.0) 1970 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:39:29.770 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:29 smithi133 bash[105589]: audit 2023-10-09T05:39:28.501247+0000 mon.a (mon.0) 1971 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:29.793 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:29.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:29.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:29.897 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:29.928 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:29.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:29.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:30.032 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:30.057 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:30.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:30.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:30.163 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:30.200 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:30.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:30.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:30.303 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:30.333 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:30.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:30.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:30.440 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:30.479 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:30.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:30.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:30.584 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:30.612 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:30.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:30.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:30.716 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:30.728 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:30 smithi133 bash[102858]: audit 2023-10-09T05:39:29.730545+0000 mon.a (mon.0) 1972 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:39:30.728 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:30 smithi133 bash[102858]: audit 2023-10-09T05:39:29.738732+0000 mon.a (mon.0) 1973 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:39:30.728 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:30 smithi133 bash[102858]: cluster 2023-10-09T05:39:30.467170+0000 mgr.y (mgr.25277) 1551 : cluster 0 pgmap v1692: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 107 active+clean; 1.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 674 op/s; 17/53544 objects degraded (0.032%); 1234/53544 objects misplaced (2.305%); 4.4 MiB/s, 90 objects/s recovering 2023-10-09T05:39:30.756 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:30.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:30.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:30.862 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:30.871 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:30 smithi194 bash[92666]: audit 2023-10-09T05:39:29.730545+0000 mon.a (mon.0) 1972 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:39:30.871 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:30 smithi194 bash[92666]: audit 2023-10-09T05:39:29.738732+0000 mon.a (mon.0) 1973 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:39:30.871 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:30 smithi194 bash[92666]: cluster 2023-10-09T05:39:30.467170+0000 mgr.y (mgr.25277) 1551 : cluster 0 pgmap v1692: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 107 active+clean; 1.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 674 op/s; 17/53544 objects degraded (0.032%); 1234/53544 objects misplaced (2.305%); 4.4 MiB/s, 90 objects/s recovering 2023-10-09T05:39:30.894 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:30.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:30.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:30.997 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:30.998 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:30 smithi133 bash[105589]: audit 2023-10-09T05:39:29.730545+0000 mon.a (mon.0) 1972 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:39:30.998 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:30 smithi133 bash[105589]: audit 2023-10-09T05:39:29.738732+0000 mon.a (mon.0) 1973 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:39:30.998 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:30 smithi133 bash[105589]: cluster 2023-10-09T05:39:30.467170+0000 mgr.y (mgr.25277) 1551 : cluster 0 pgmap v1692: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 107 active+clean; 1.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 674 op/s; 17/53544 objects degraded (0.032%); 1234/53544 objects misplaced (2.305%); 4.4 MiB/s, 90 objects/s recovering 2023-10-09T05:39:31.025 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:31.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:31.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:31.128 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:31.158 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:31.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:31.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:31.263 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:31.289 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:31.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:31.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:31.392 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:31.424 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:31.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:31.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:31.529 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:31.554 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:31.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:31.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:31.659 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:31.682 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:31.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:31.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:31.785 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:31.812 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:31.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:31.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:31.917 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:31.940 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:31.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:31.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:32.043 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:32.082 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:32.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:32.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:32.191 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:32.226 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:32.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:32.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:32.330 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:32.356 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:32.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:32.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:32.432 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:39:32.432 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:39:32.460 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:32.485 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:32.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:32.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:32.588 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:32.611 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:32.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:32.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:32.714 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:32.738 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:32.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:32.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:32.845 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:32.855 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:32 smithi194 bash[92666]: cluster 2023-10-09T05:39:32.519022+0000 mon.a (mon.0) 1974 : cluster 3 Health check update: Degraded data redundancy: 17/45855 objects degraded (0.037%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:39:32.874 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:32.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:32.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:32.976 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:33.005 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:33.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:33.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:33.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:32 smithi133 bash[102858]: cluster 2023-10-09T05:39:32.519022+0000 mon.a (mon.0) 1974 : cluster 3 Health check update: Degraded data redundancy: 17/45855 objects degraded (0.037%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:39:33.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:32 smithi133 bash[105589]: cluster 2023-10-09T05:39:32.519022+0000 mon.a (mon.0) 1974 : cluster 3 Health check update: Degraded data redundancy: 17/45855 objects degraded (0.037%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:39:33.108 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:33.137 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:33.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:33.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:33.245 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:33.272 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:33.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:33.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:33.376 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:33.403 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:33.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:33.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:33.507 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:33.529 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:33.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:33.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:33.634 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:33.647 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:33 smithi133 bash[102858]: cluster 2023-10-09T05:39:32.467877+0000 mgr.y (mgr.25277) 1552 : cluster 0 pgmap v1693: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 954 MiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 752 op/s; 17/45855 objects degraded (0.037%); 928/45855 objects misplaced (2.024%); 3.8 MiB/s, 78 objects/s recovering 2023-10-09T05:39:33.664 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:33.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:33.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:33.772 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:33.816 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:33.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:33.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:33.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:33 smithi194 bash[92666]: cluster 2023-10-09T05:39:32.467877+0000 mgr.y (mgr.25277) 1552 : cluster 0 pgmap v1693: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 954 MiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 752 op/s; 17/45855 objects degraded (0.037%); 928/45855 objects misplaced (2.024%); 3.8 MiB/s, 78 objects/s recovering 2023-10-09T05:39:33.926 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:33.934 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:33 smithi133 bash[105589]: cluster 2023-10-09T05:39:32.467877+0000 mgr.y (mgr.25277) 1552 : cluster 0 pgmap v1693: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 954 MiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 752 op/s; 17/45855 objects degraded (0.037%); 928/45855 objects misplaced (2.024%); 3.8 MiB/s, 78 objects/s recovering 2023-10-09T05:39:33.953 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:33.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:33.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:34.056 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:34.081 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:34.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:34.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:34.187 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:34.206 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:34.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:34.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:34.309 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:34.317 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:39:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:39:34] "GET /metrics HTTP/1.1" 200 36512 "" "Prometheus/2.43.0" 2023-10-09T05:39:34.334 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:34.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:34.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:34.440 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:34.475 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:34.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:34.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:34.580 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:34.607 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:34.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:34.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:34.712 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:34.753 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:34.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:34.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:34.857 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:34.884 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:34.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:34.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:34.991 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:35.000 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:34 smithi194 bash[92666]: cluster 2023-10-09T05:39:34.468506+0000 mgr.y (mgr.25277) 1553 : cluster 0 pgmap v1694: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 864 MiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 819 op/s; 17/41541 objects degraded (0.041%); 928/41541 objects misplaced (2.234%); 3.7 MiB/s, 75 objects/s recovering 2023-10-09T05:39:35.018 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:35.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:35.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:35.123 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:35.153 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:35.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:35.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:35.240 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:34 smithi133 bash[102858]: cluster 2023-10-09T05:39:34.468506+0000 mgr.y (mgr.25277) 1553 : cluster 0 pgmap v1694: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 864 MiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 819 op/s; 17/41541 objects degraded (0.041%); 928/41541 objects misplaced (2.234%); 3.7 MiB/s, 75 objects/s recovering 2023-10-09T05:39:35.241 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:34 smithi133 bash[105589]: cluster 2023-10-09T05:39:34.468506+0000 mgr.y (mgr.25277) 1553 : cluster 0 pgmap v1694: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 864 MiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 819 op/s; 17/41541 objects degraded (0.041%); 928/41541 objects misplaced (2.234%); 3.7 MiB/s, 75 objects/s recovering 2023-10-09T05:39:35.258 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:35.290 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:35.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:35.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:35.396 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:35.425 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:35.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:35.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:35.532 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:35.555 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:35.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:35.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:35.657 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:35.678 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:35.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:35.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:35.783 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:35.814 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:35.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:35.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:35.919 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:35.947 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:35.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:35.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:36.051 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:36.082 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:36.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:36.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:36.188 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:36.221 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:36.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:36.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:36.253 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T05:39:36.325 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:36.377 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:36.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:36.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:36.491 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:36.515 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:36.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:36.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:36.618 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:36.680 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:36.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:36.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:36.716 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:36.785 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:36.831 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:36.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:36.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:36.939 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:36.964 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:36.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:36.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:37.058 INFO:teuthology.orchestra.run.smithi133.stdout:moved 0 / 327 (0%) 2023-10-09T05:39:37.059 INFO:teuthology.orchestra.run.smithi133.stdout:avg 46.7143 2023-10-09T05:39:37.059 INFO:teuthology.orchestra.run.smithi133.stdout:stddev 4.19913 -> 4.19913 (expected baseline 6.32778) 2023-10-09T05:39:37.059 INFO:teuthology.orchestra.run.smithi133.stdout:min osd.5 with 40 -> 40 pgs (0.856269 -> 0.856269 * mean) 2023-10-09T05:39:37.059 INFO:teuthology.orchestra.run.smithi133.stdout:max osd.3 with 53 -> 53 pgs (1.13456 -> 1.13456 * mean) 2023-10-09T05:39:37.059 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:39:37.059 INFO:teuthology.orchestra.run.smithi133.stdout:oload 1000 2023-10-09T05:39:37.059 INFO:teuthology.orchestra.run.smithi133.stdout:max_change 1 2023-10-09T05:39:37.060 INFO:teuthology.orchestra.run.smithi133.stdout:max_change_osds 4 2023-10-09T05:39:37.060 INFO:teuthology.orchestra.run.smithi133.stdout:average_utilization 0.0179 2023-10-09T05:39:37.060 INFO:teuthology.orchestra.run.smithi133.stdout:overload_utilization 0.1789 2023-10-09T05:39:37.062 INFO:teuthology.orchestra.run.smithi133.stderr:no change 2023-10-09T05:39:37.072 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:37.100 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:37.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:37.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:37.205 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:37.235 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:37.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:37.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:37.338 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:37.384 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:37.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:37.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:37.491 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:37.523 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:37.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:37.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:37.536 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is unset 2023-10-09T05:39:37.627 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:37.645 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:37.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:37.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:37.746 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:37 smithi133 bash[102858]: cluster 2023-10-09T05:39:36.469258+0000 mgr.y (mgr.25277) 1554 : cluster 0 pgmap v1695: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 733 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 908 op/s; 17/35211 objects degraded (0.048%); 682/35211 objects misplaced (1.937%); 3.7 MiB/s, 117 objects/s recovering 2023-10-09T05:39:37.747 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:37.756 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:37 smithi194 bash[92666]: cluster 2023-10-09T05:39:36.469258+0000 mgr.y (mgr.25277) 1554 : cluster 0 pgmap v1695: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 733 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 908 op/s; 17/35211 objects degraded (0.048%); 682/35211 objects misplaced (1.937%); 3.7 MiB/s, 117 objects/s recovering 2023-10-09T05:39:37.756 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:37 smithi194 bash[92666]: audit 2023-10-09T05:39:37.212127+0000 mon.c (mon.2) 143 : audit 1 from='client.? 172.21.15.133:0/1168813077' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:39:37.756 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:37 smithi194 bash[92666]: audit 2023-10-09T05:39:37.213300+0000 mon.a (mon.0) 1975 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:39:37.777 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:37.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:37.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:37.881 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:37.907 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:37.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:37.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:38.011 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:38.020 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:37 smithi133 bash[102858]: audit 2023-10-09T05:39:37.212127+0000 mon.c (mon.2) 143 : audit 1 from='client.? 172.21.15.133:0/1168813077' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:39:38.021 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:37 smithi133 bash[102858]: audit 2023-10-09T05:39:37.213300+0000 mon.a (mon.0) 1975 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:39:38.024 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:37 smithi133 bash[105589]: cluster 2023-10-09T05:39:36.469258+0000 mgr.y (mgr.25277) 1554 : cluster 0 pgmap v1695: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 733 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 908 op/s; 17/35211 objects degraded (0.048%); 682/35211 objects misplaced (1.937%); 3.7 MiB/s, 117 objects/s recovering 2023-10-09T05:39:38.024 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:37 smithi133 bash[105589]: audit 2023-10-09T05:39:37.212127+0000 mon.c (mon.2) 143 : audit 1 from='client.? 172.21.15.133:0/1168813077' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:39:38.024 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:37 smithi133 bash[105589]: audit 2023-10-09T05:39:37.213300+0000 mon.a (mon.0) 1975 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:39:38.065 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:38.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:38.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:38.170 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:38.193 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:38.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:38.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:38.297 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:38.324 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:38.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:38.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:38.430 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:38.456 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:38.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:38.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:38.564 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:38.598 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:38.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:38.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:38.703 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:38.737 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:38.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:38.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:38.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:38 smithi133 bash[102858]: audit 2023-10-09T05:39:37.058019+0000 mgr.y (mgr.25277) 1555 : audit 0 from='client.35874 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:39:38.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:38 smithi133 bash[102858]: cluster 2023-10-09T05:39:37.521851+0000 mon.a (mon.0) 1976 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:39:38.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:38 smithi133 bash[102858]: audit 2023-10-09T05:39:37.528741+0000 mon.a (mon.0) 1977 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:39:38.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:38 smithi133 bash[102858]: cluster 2023-10-09T05:39:37.528813+0000 mon.a (mon.0) 1978 : cluster 0 osdmap e354: 8 total, 8 up, 7 in 2023-10-09T05:39:38.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:38 smithi133 bash[102858]: cluster 2023-10-09T05:39:37.533204+0000 osd.7 (osd.7) 317 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T05:39:38.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:38 smithi133 bash[102858]: cluster 2023-10-09T05:39:37.536119+0000 osd.6 (osd.6) 338 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T05:39:38.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:38 smithi133 bash[102858]: audit 2023-10-09T05:39:38.433485+0000 mon.a (mon.0) 1979 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:39:38.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:38 smithi133 bash[102858]: audit 2023-10-09T05:39:38.438604+0000 mon.a (mon.0) 1980 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:39:38.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:38 smithi133 bash[105589]: audit 2023-10-09T05:39:37.058019+0000 mgr.y (mgr.25277) 1555 : audit 0 from='client.35874 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:39:38.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:38 smithi133 bash[105589]: cluster 2023-10-09T05:39:37.521851+0000 mon.a (mon.0) 1976 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:39:38.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:38 smithi133 bash[105589]: audit 2023-10-09T05:39:37.528741+0000 mon.a (mon.0) 1977 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:39:38.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:38 smithi133 bash[105589]: cluster 2023-10-09T05:39:37.528813+0000 mon.a (mon.0) 1978 : cluster 0 osdmap e354: 8 total, 8 up, 7 in 2023-10-09T05:39:38.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:38 smithi133 bash[105589]: cluster 2023-10-09T05:39:37.533204+0000 osd.7 (osd.7) 317 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T05:39:38.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:38 smithi133 bash[105589]: cluster 2023-10-09T05:39:37.536119+0000 osd.6 (osd.6) 338 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T05:39:38.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:38 smithi133 bash[105589]: audit 2023-10-09T05:39:38.433485+0000 mon.a (mon.0) 1979 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:39:38.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:38 smithi133 bash[105589]: audit 2023-10-09T05:39:38.438604+0000 mon.a (mon.0) 1980 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:39:38.844 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:38.855 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:38 smithi194 bash[92666]: audit 2023-10-09T05:39:37.058019+0000 mgr.y (mgr.25277) 1555 : audit 0 from='client.35874 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:39:38.855 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:38 smithi194 bash[92666]: cluster 2023-10-09T05:39:37.521851+0000 mon.a (mon.0) 1976 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:39:38.855 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:38 smithi194 bash[92666]: audit 2023-10-09T05:39:37.528741+0000 mon.a (mon.0) 1977 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:39:38.855 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:38 smithi194 bash[92666]: cluster 2023-10-09T05:39:37.528813+0000 mon.a (mon.0) 1978 : cluster 0 osdmap e354: 8 total, 8 up, 7 in 2023-10-09T05:39:38.855 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:38 smithi194 bash[92666]: cluster 2023-10-09T05:39:37.533204+0000 osd.7 (osd.7) 317 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T05:39:38.855 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:38 smithi194 bash[92666]: cluster 2023-10-09T05:39:37.536119+0000 osd.6 (osd.6) 338 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T05:39:38.855 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:38 smithi194 bash[92666]: audit 2023-10-09T05:39:38.433485+0000 mon.a (mon.0) 1979 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:39:38.855 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:38 smithi194 bash[92666]: audit 2023-10-09T05:39:38.438604+0000 mon.a (mon.0) 1980 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:39:38.880 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:38.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:38.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:38.985 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:39.013 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:39.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:39.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:39.115 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:39.145 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:39.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:39.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:39.249 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:39.275 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:39.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:39.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:39.378 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:39.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:39.411 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:39.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:39.512 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:39.538 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:39.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:39.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:39.641 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:39.665 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:39.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:39.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:39.768 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:39.778 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:39 smithi194 bash[92666]: cluster 2023-10-09T05:39:38.470315+0000 mgr.y (mgr.25277) 1556 : cluster 0 pgmap v1697: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 693 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 901 op/s; 17/33312 objects degraded (0.051%); 682/33312 objects misplaced (2.047%); 1.1 MiB/s, 88 objects/s recovering 2023-10-09T05:39:39.778 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:39 smithi194 bash[92666]: cluster 2023-10-09T05:39:38.941388+0000 mon.a (mon.0) 1981 : cluster 3 Health check update: Degraded data redundancy: 17/35211 objects degraded (0.048%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:39:39.778 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:39 smithi194 bash[92666]: audit 2023-10-09T05:39:39.021095+0000 mon.a (mon.0) 1982 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:39:39.778 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:39 smithi194 bash[92666]: audit 2023-10-09T05:39:39.021859+0000 mon.a (mon.0) 1983 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:39:39.778 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:39 smithi194 bash[92666]: audit 2023-10-09T05:39:39.026222+0000 mon.a (mon.0) 1984 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:39:39.796 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:39.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:39 smithi133 bash[105589]: cluster 2023-10-09T05:39:38.470315+0000 mgr.y (mgr.25277) 1556 : cluster 0 pgmap v1697: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 693 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 901 op/s; 17/33312 objects degraded (0.051%); 682/33312 objects misplaced (2.047%); 1.1 MiB/s, 88 objects/s recovering 2023-10-09T05:39:39.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:39 smithi133 bash[105589]: cluster 2023-10-09T05:39:38.941388+0000 mon.a (mon.0) 1981 : cluster 3 Health check update: Degraded data redundancy: 17/35211 objects degraded (0.048%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:39:39.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:39 smithi133 bash[105589]: audit 2023-10-09T05:39:39.021095+0000 mon.a (mon.0) 1982 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:39:39.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:39 smithi133 bash[105589]: audit 2023-10-09T05:39:39.021859+0000 mon.a (mon.0) 1983 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:39:39.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:39 smithi133 bash[105589]: audit 2023-10-09T05:39:39.026222+0000 mon.a (mon.0) 1984 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:39:39.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:39 smithi133 bash[102858]: cluster 2023-10-09T05:39:38.470315+0000 mgr.y (mgr.25277) 1556 : cluster 0 pgmap v1697: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 693 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 901 op/s; 17/33312 objects degraded (0.051%); 682/33312 objects misplaced (2.047%); 1.1 MiB/s, 88 objects/s recovering 2023-10-09T05:39:39.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:39 smithi133 bash[102858]: cluster 2023-10-09T05:39:38.941388+0000 mon.a (mon.0) 1981 : cluster 3 Health check update: Degraded data redundancy: 17/35211 objects degraded (0.048%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:39:39.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:39 smithi133 bash[102858]: audit 2023-10-09T05:39:39.021095+0000 mon.a (mon.0) 1982 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:39:39.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:39 smithi133 bash[102858]: audit 2023-10-09T05:39:39.021859+0000 mon.a (mon.0) 1983 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:39:39.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:39 smithi133 bash[102858]: audit 2023-10-09T05:39:39.026222+0000 mon.a (mon.0) 1984 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:39:39.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:39.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:39.902 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:39.929 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:39.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:39.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:40.033 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:40.061 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:40.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:40.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:40.166 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:40.190 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:40.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:40.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:40.294 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:40.331 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:40.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:40.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:40.436 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:40.453 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:40.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:40.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:40.555 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:40.592 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:40.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:40.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:40.695 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:40.718 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:40.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:40.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:40.823 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:40.841 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:40.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:40.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:40.943 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:40.961 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:40.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:40.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:41.067 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:41.095 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:41.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:41.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:41.200 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:41.230 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:41.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:41.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:41.337 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:41.365 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:41.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:41.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:41.470 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:41.500 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:41.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:41.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:41.603 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:41.625 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-10-09T05:39:41.625 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 110 0.05 2023-10-09T05:39:41.627 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:41.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:41.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:41.730 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:41.762 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:41.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:41.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:41.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:41 smithi133 bash[102858]: cluster 2023-10-09T05:39:40.470973+0000 mgr.y (mgr.25277) 1557 : cluster 0 pgmap v1698: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 107 active+clean; 577 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 859 op/s; 17/27756 objects degraded (0.061%); 682/27756 objects misplaced (2.457%); 0 B/s, 123 objects/s recovering 2023-10-09T05:39:41.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:41 smithi133 bash[105589]: cluster 2023-10-09T05:39:40.470973+0000 mgr.y (mgr.25277) 1557 : cluster 0 pgmap v1698: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 107 active+clean; 577 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 859 op/s; 17/27756 objects degraded (0.061%); 682/27756 objects misplaced (2.457%); 0 B/s, 123 objects/s recovering 2023-10-09T05:39:41.869 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:41.879 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:41 smithi194 bash[92666]: cluster 2023-10-09T05:39:40.470973+0000 mgr.y (mgr.25277) 1557 : cluster 0 pgmap v1698: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 107 active+clean; 577 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 859 op/s; 17/27756 objects degraded (0.061%); 682/27756 objects misplaced (2.457%); 0 B/s, 123 objects/s recovering 2023-10-09T05:39:41.895 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:41.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:41.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:41.999 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:42.034 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:42.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:42.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:42.138 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:42.164 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:42.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:42.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:42.268 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:42.288 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:42.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:42.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:42.393 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:42.425 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:42.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:42.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:42.530 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:42.566 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:42.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:42.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:42.669 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:42.676 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:42 smithi133 bash[105589]: cluster 2023-10-09T05:39:41.272630+0000 osd.2 (osd.2) 3 : cluster 0 14.0 scrub starts 2023-10-09T05:39:42.677 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:42 smithi133 bash[102858]: cluster 2023-10-09T05:39:41.272630+0000 osd.2 (osd.2) 3 : cluster 0 14.0 scrub starts 2023-10-09T05:39:42.696 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:42.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:42.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:42.801 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:42.810 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:42 smithi194 bash[92666]: cluster 2023-10-09T05:39:41.272630+0000 osd.2 (osd.2) 3 : cluster 0 14.0 scrub starts 2023-10-09T05:39:42.834 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:42.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:42.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:42.937 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:42.959 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:42.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:42.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:43.062 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:43.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:43.095 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:43.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:43.196 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:43.241 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:43.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:43.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:43.349 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:43.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:43.388 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:43.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:43.489 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:43.519 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:43.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:43.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:43.622 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:43.632 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:43 smithi194 bash[92666]: cluster 2023-10-09T05:39:42.253659+0000 osd.2 (osd.2) 4 : cluster 0 14.e scrub starts 2023-10-09T05:39:43.633 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:43 smithi194 bash[92666]: cluster 2023-10-09T05:39:42.457225+0000 osd.2 (osd.2) 5 : cluster 0 14.e scrub ok 2023-10-09T05:39:43.633 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:43 smithi194 bash[92666]: cluster 2023-10-09T05:39:42.471756+0000 mgr.y (mgr.25277) 1558 : cluster 0 pgmap v1699: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 448 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 809 op/s; 17/21555 objects degraded (0.079%); 428/21555 objects misplaced (1.986%); 0 B/s, 123 objects/s recovering 2023-10-09T05:39:43.633 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:43 smithi194 bash[92666]: audit 2023-10-09T05:39:43.495182+0000 mon.a (mon.0) 1985 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:43.645 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:39:43.649 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:43.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:43.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:43.755 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:43.781 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:43.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:43.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:43.885 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:43.895 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:43 smithi133 bash[102858]: cluster 2023-10-09T05:39:42.253659+0000 osd.2 (osd.2) 4 : cluster 0 14.e scrub starts 2023-10-09T05:39:43.895 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:43 smithi133 bash[102858]: cluster 2023-10-09T05:39:42.457225+0000 osd.2 (osd.2) 5 : cluster 0 14.e scrub ok 2023-10-09T05:39:43.895 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:43 smithi133 bash[102858]: cluster 2023-10-09T05:39:42.471756+0000 mgr.y (mgr.25277) 1558 : cluster 0 pgmap v1699: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 448 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 809 op/s; 17/21555 objects degraded (0.079%); 428/21555 objects misplaced (1.986%); 0 B/s, 123 objects/s recovering 2023-10-09T05:39:43.895 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:43 smithi133 bash[102858]: audit 2023-10-09T05:39:43.495182+0000 mon.a (mon.0) 1985 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:43.896 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:43 smithi133 bash[105589]: cluster 2023-10-09T05:39:42.253659+0000 osd.2 (osd.2) 4 : cluster 0 14.e scrub starts 2023-10-09T05:39:43.896 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:43 smithi133 bash[105589]: cluster 2023-10-09T05:39:42.457225+0000 osd.2 (osd.2) 5 : cluster 0 14.e scrub ok 2023-10-09T05:39:43.896 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:43 smithi133 bash[105589]: cluster 2023-10-09T05:39:42.471756+0000 mgr.y (mgr.25277) 1558 : cluster 0 pgmap v1699: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 448 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 809 op/s; 17/21555 objects degraded (0.079%); 428/21555 objects misplaced (1.986%); 0 B/s, 123 objects/s recovering 2023-10-09T05:39:43.896 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:43 smithi133 bash[105589]: audit 2023-10-09T05:39:43.495182+0000 mon.a (mon.0) 1985 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:43.927 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:43.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:43.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:44.033 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:44.064 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:44.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:44.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:44.167 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:44.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:44.201 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:44.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:44.303 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:44.341 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:44.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:44.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:44.447 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:44.478 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:44.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:44.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:44.547 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:39:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:39:44] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.43.0" 2023-10-09T05:39:44.582 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:44.617 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:44.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:44.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:44.724 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:44.733 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:44 smithi194 bash[92666]: cluster 2023-10-09T05:39:43.302659+0000 osd.6 (osd.6) 339 : cluster 0 14.11 scrub starts 2023-10-09T05:39:44.733 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:44 smithi194 bash[92666]: cluster 2023-10-09T05:39:43.942560+0000 mon.a (mon.0) 1986 : cluster 3 Health check update: Degraded data redundancy: 17/21555 objects degraded (0.079%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:39:44.752 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:44.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:44.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:44.857 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:44.865 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:44 smithi133 bash[102858]: cluster 2023-10-09T05:39:43.302659+0000 osd.6 (osd.6) 339 : cluster 0 14.11 scrub starts 2023-10-09T05:39:44.865 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:44 smithi133 bash[102858]: cluster 2023-10-09T05:39:43.942560+0000 mon.a (mon.0) 1986 : cluster 3 Health check update: Degraded data redundancy: 17/21555 objects degraded (0.079%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:39:44.865 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:44 smithi133 bash[105589]: cluster 2023-10-09T05:39:43.302659+0000 osd.6 (osd.6) 339 : cluster 0 14.11 scrub starts 2023-10-09T05:39:44.865 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:44 smithi133 bash[105589]: cluster 2023-10-09T05:39:43.942560+0000 mon.a (mon.0) 1986 : cluster 3 Health check update: Degraded data redundancy: 17/21555 objects degraded (0.079%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:39:44.894 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:44.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:44.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:44.999 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:45.024 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:45.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:45.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:45.129 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:45.156 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:45.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:45.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:45.262 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:45.301 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:39:45.310 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:45.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:45.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:45.337 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:39:45.394 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:39:45.414 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:45.439 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:45.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:45.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:45.450 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:39:45.492 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:39:45.532 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:39:45.544 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:45.569 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:39:45.573 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:45.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:45.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:45.609 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:39:45.620 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:39:45.620 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:39:45.620 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:39:45.620 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:39:45.620 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:39:45.620 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:39:45.620 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:39:45.620 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:39:45.681 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:45.709 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:45.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:45.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:45.816 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:45.825 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:45 smithi133 bash[102858]: cluster 2023-10-09T05:39:44.472405+0000 mgr.y (mgr.25277) 1559 : cluster 0 pgmap v1700: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 355 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 814 op/s; 17/17103 objects degraded (0.099%); 428/17103 objects misplaced (2.502%); 0 B/s, 123 objects/s recovering 2023-10-09T05:39:45.825 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:45 smithi133 bash[102858]: audit 2023-10-09T05:39:45.256402+0000 mon.b (mon.1) 34 : audit 0 from='client.? 172.21.15.133:0/3817317757' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:39:45.825 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:45 smithi133 bash[105589]: cluster 2023-10-09T05:39:44.472405+0000 mgr.y (mgr.25277) 1559 : cluster 0 pgmap v1700: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 355 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 814 op/s; 17/17103 objects degraded (0.099%); 428/17103 objects misplaced (2.502%); 0 B/s, 123 objects/s recovering 2023-10-09T05:39:45.825 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:45 smithi133 bash[105589]: audit 2023-10-09T05:39:45.256402+0000 mon.b (mon.1) 34 : audit 0 from='client.? 172.21.15.133:0/3817317757' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:39:45.846 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:45.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:45.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:45.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:45 smithi194 bash[92666]: cluster 2023-10-09T05:39:44.472405+0000 mgr.y (mgr.25277) 1559 : cluster 0 pgmap v1700: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 355 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 814 op/s; 17/17103 objects degraded (0.099%); 428/17103 objects misplaced (2.502%); 0 B/s, 123 objects/s recovering 2023-10-09T05:39:45.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:45 smithi194 bash[92666]: audit 2023-10-09T05:39:45.256402+0000 mon.b (mon.1) 34 : audit 0 from='client.? 172.21.15.133:0/3817317757' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:39:45.949 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:45.977 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:45.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:45.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:46.081 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:46.107 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:46.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:46.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:46.213 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:46.257 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:46.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:46.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:46.364 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:46.403 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:46.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:46.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:46.506 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:46.542 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:46.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:46.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:46.646 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:46.668 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:46.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:46.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:46.771 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:46.802 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:46.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:46.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:46.905 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:46.930 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:46.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:46.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:47.036 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:47.065 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:47.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:47.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:47.168 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:47.193 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:47.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:47.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:47.299 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:47.329 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:47.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:47.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:47.432 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:47.457 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:47.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:47.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:47.560 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:47.593 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:47.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:47.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:47.697 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:47.721 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:47.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:47.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:47.797 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:47 smithi133 bash[105589]: cluster 2023-10-09T05:39:46.473963+0000 mgr.y (mgr.25277) 1560 : cluster 0 pgmap v1701: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 220 MiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 819 op/s; 17/10608 objects degraded (0.160%); 146/10608 objects misplaced (1.376%); 0 B/s, 142 objects/s recovering 2023-10-09T05:39:47.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:47 smithi133 bash[105589]: cluster 2023-10-09T05:39:46.568307+0000 osd.4 (osd.4) 311 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T05:39:47.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:47 smithi133 bash[105589]: cluster 2023-10-09T05:39:46.576417+0000 mon.a (mon.0) 1987 : cluster 0 osdmap e355: 8 total, 8 up, 7 in 2023-10-09T05:39:47.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:47 smithi133 bash[105589]: cluster 2023-10-09T05:39:46.576976+0000 osd.7 (osd.7) 318 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T05:39:47.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:47 smithi133 bash[102858]: cluster 2023-10-09T05:39:46.473963+0000 mgr.y (mgr.25277) 1560 : cluster 0 pgmap v1701: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 220 MiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 819 op/s; 17/10608 objects degraded (0.160%); 146/10608 objects misplaced (1.376%); 0 B/s, 142 objects/s recovering 2023-10-09T05:39:47.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:47 smithi133 bash[102858]: cluster 2023-10-09T05:39:46.568307+0000 osd.4 (osd.4) 311 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T05:39:47.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:47 smithi133 bash[102858]: cluster 2023-10-09T05:39:46.576417+0000 mon.a (mon.0) 1987 : cluster 0 osdmap e355: 8 total, 8 up, 7 in 2023-10-09T05:39:47.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:47 smithi133 bash[102858]: cluster 2023-10-09T05:39:46.576976+0000 osd.7 (osd.7) 318 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T05:39:47.824 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:47.832 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:47 smithi194 bash[92666]: cluster 2023-10-09T05:39:46.473963+0000 mgr.y (mgr.25277) 1560 : cluster 0 pgmap v1701: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 220 MiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 819 op/s; 17/10608 objects degraded (0.160%); 146/10608 objects misplaced (1.376%); 0 B/s, 142 objects/s recovering 2023-10-09T05:39:47.832 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:47 smithi194 bash[92666]: cluster 2023-10-09T05:39:46.568307+0000 osd.4 (osd.4) 311 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T05:39:47.832 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:47 smithi194 bash[92666]: cluster 2023-10-09T05:39:46.576417+0000 mon.a (mon.0) 1987 : cluster 0 osdmap e355: 8 total, 8 up, 7 in 2023-10-09T05:39:47.832 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:47 smithi194 bash[92666]: cluster 2023-10-09T05:39:46.576976+0000 osd.7 (osd.7) 318 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T05:39:47.848 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:47.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:47.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:47.864 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:47.867 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:47.953 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:47.980 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:47.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:47.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:48.084 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:48.111 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:48.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:48.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:48.213 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:48.250 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:48.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:48.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:48.354 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:48.388 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:48.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:48.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:48.492 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:48.526 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:48.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:48.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:48.630 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:48.662 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:48.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:48.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:48.765 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:48.775 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:48 smithi194 bash[92666]: cluster 2023-10-09T05:39:47.164671+0000 osd.7 (osd.7) 319 : cluster 0 14.f scrub starts 2023-10-09T05:39:48.775 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:48 smithi194 bash[92666]: cluster 2023-10-09T05:39:47.176643+0000 osd.7 (osd.7) 320 : cluster 0 14.f scrub ok 2023-10-09T05:39:48.775 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:48 smithi194 bash[92666]: cluster 2023-10-09T05:39:47.576918+0000 osd.4 (osd.4) 312 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T05:39:48.775 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:48 smithi194 bash[92666]: cluster 2023-10-09T05:39:47.579832+0000 mon.a (mon.0) 1988 : cluster 0 osdmap e356: 8 total, 8 up, 7 in 2023-10-09T05:39:48.775 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:48 smithi194 bash[92666]: cluster 2023-10-09T05:39:47.580415+0000 osd.7 (osd.7) 321 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T05:39:48.775 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:48 smithi194 bash[92666]: cluster 2023-10-09T05:39:47.583285+0000 osd.5 (osd.5) 369 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T05:39:48.775 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:48 smithi194 bash[92666]: cluster 2023-10-09T05:39:47.590780+0000 osd.6 (osd.6) 340 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T05:39:48.795 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:48.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:48.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:48.899 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:48.909 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:48 smithi133 bash[105589]: cluster 2023-10-09T05:39:47.164671+0000 osd.7 (osd.7) 319 : cluster 0 14.f scrub starts 2023-10-09T05:39:48.910 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:48 smithi133 bash[105589]: cluster 2023-10-09T05:39:47.176643+0000 osd.7 (osd.7) 320 : cluster 0 14.f scrub ok 2023-10-09T05:39:48.910 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:48 smithi133 bash[105589]: cluster 2023-10-09T05:39:47.576918+0000 osd.4 (osd.4) 312 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T05:39:48.910 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:48 smithi133 bash[105589]: cluster 2023-10-09T05:39:47.579832+0000 mon.a (mon.0) 1988 : cluster 0 osdmap e356: 8 total, 8 up, 7 in 2023-10-09T05:39:48.910 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:48 smithi133 bash[105589]: cluster 2023-10-09T05:39:47.580415+0000 osd.7 (osd.7) 321 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T05:39:48.911 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:48 smithi133 bash[105589]: cluster 2023-10-09T05:39:47.583285+0000 osd.5 (osd.5) 369 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T05:39:48.911 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:48 smithi133 bash[105589]: cluster 2023-10-09T05:39:47.590780+0000 osd.6 (osd.6) 340 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T05:39:48.911 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:48 smithi133 bash[102858]: cluster 2023-10-09T05:39:47.164671+0000 osd.7 (osd.7) 319 : cluster 0 14.f scrub starts 2023-10-09T05:39:48.911 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:48 smithi133 bash[102858]: cluster 2023-10-09T05:39:47.176643+0000 osd.7 (osd.7) 320 : cluster 0 14.f scrub ok 2023-10-09T05:39:48.911 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:48 smithi133 bash[102858]: cluster 2023-10-09T05:39:47.576918+0000 osd.4 (osd.4) 312 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T05:39:48.912 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:48 smithi133 bash[102858]: cluster 2023-10-09T05:39:47.579832+0000 mon.a (mon.0) 1988 : cluster 0 osdmap e356: 8 total, 8 up, 7 in 2023-10-09T05:39:48.912 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:48 smithi133 bash[102858]: cluster 2023-10-09T05:39:47.580415+0000 osd.7 (osd.7) 321 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T05:39:48.912 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:48 smithi133 bash[102858]: cluster 2023-10-09T05:39:47.583285+0000 osd.5 (osd.5) 369 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T05:39:48.912 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:48 smithi133 bash[102858]: cluster 2023-10-09T05:39:47.590780+0000 osd.6 (osd.6) 340 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T05:39:48.928 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:48.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:48.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:49.032 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:49.069 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:49.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:49.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:49.175 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:49.202 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:49.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:49.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:49.307 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:49.338 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:49.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:49.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:49.381 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:39:49.443 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:49.467 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:49.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:49.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:49.572 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:49.601 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:49.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:49.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:49.636 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Removed 99326 objects 2023-10-09T05:39:49.636 INFO:tasks.radosbench.radosbench.0.smithi133.stdout:Clean up completed and total clean up time :130.844 2023-10-09T05:39:49.707 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:49.718 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:49 smithi194 bash[92666]: cluster 2023-10-09T05:39:48.140004+0000 osd.7 (osd.7) 322 : cluster 0 14.1f scrub starts 2023-10-09T05:39:49.719 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:49 smithi194 bash[92666]: cluster 2023-10-09T05:39:48.151768+0000 osd.7 (osd.7) 323 : cluster 0 14.1f scrub ok 2023-10-09T05:39:49.719 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:49 smithi194 bash[92666]: cluster 2023-10-09T05:39:48.474660+0000 mgr.y (mgr.25277) 1561 : cluster 0 pgmap v1704: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 178 MiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 43 op/s; 17/8580 objects degraded (0.198%); 146/8580 objects misplaced (1.702%); 0 B/s, 87 objects/s recovering 2023-10-09T05:39:49.719 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:49 smithi194 bash[92666]: cluster 2023-10-09T05:39:48.590953+0000 osd.5 (osd.5) 370 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T05:39:49.719 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:49 smithi194 bash[92666]: cluster 2023-10-09T05:39:48.595361+0000 osd.5 (osd.5) 371 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T05:39:49.719 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:49 smithi194 bash[92666]: cluster 2023-10-09T05:39:48.597108+0000 mon.a (mon.0) 1989 : cluster 0 osdmap e357: 8 total, 8 up, 7 in 2023-10-09T05:39:49.720 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:49 smithi194 bash[92666]: cluster 2023-10-09T05:39:48.646363+0000 osd.6 (osd.6) 341 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T05:39:49.720 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:49 smithi194 bash[92666]: cluster 2023-10-09T05:39:48.667215+0000 osd.7 (osd.7) 324 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T05:39:49.720 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:49 smithi194 bash[92666]: cluster 2023-10-09T05:39:48.785685+0000 osd.4 (osd.4) 313 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T05:39:49.720 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:49 smithi194 bash[92666]: cluster 2023-10-09T05:39:48.943750+0000 mon.a (mon.0) 1990 : cluster 3 Health check update: Degraded data redundancy: 17/8580 objects degraded (0.198%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:39:49.737 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:49.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:49.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:49.841 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:49.874 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:49.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:49.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:49.899 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:49.979 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:50.011 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:50.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:50.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:50.114 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:50.151 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:50.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:50.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:50.254 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:50.286 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:50.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:50.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:50.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:49 smithi133 bash[102858]: cluster 2023-10-09T05:39:48.140004+0000 osd.7 (osd.7) 322 : cluster 0 14.1f scrub starts 2023-10-09T05:39:50.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:49 smithi133 bash[102858]: cluster 2023-10-09T05:39:48.151768+0000 osd.7 (osd.7) 323 : cluster 0 14.1f scrub ok 2023-10-09T05:39:50.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:49 smithi133 bash[102858]: cluster 2023-10-09T05:39:48.474660+0000 mgr.y (mgr.25277) 1561 : cluster 0 pgmap v1704: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 178 MiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 43 op/s; 17/8580 objects degraded (0.198%); 146/8580 objects misplaced (1.702%); 0 B/s, 87 objects/s recovering 2023-10-09T05:39:50.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:49 smithi133 bash[102858]: cluster 2023-10-09T05:39:48.590953+0000 osd.5 (osd.5) 370 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T05:39:50.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:49 smithi133 bash[102858]: cluster 2023-10-09T05:39:48.595361+0000 osd.5 (osd.5) 371 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T05:39:50.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:49 smithi133 bash[102858]: cluster 2023-10-09T05:39:48.597108+0000 mon.a (mon.0) 1989 : cluster 0 osdmap e357: 8 total, 8 up, 7 in 2023-10-09T05:39:50.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:49 smithi133 bash[102858]: cluster 2023-10-09T05:39:48.646363+0000 osd.6 (osd.6) 341 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T05:39:50.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:49 smithi133 bash[102858]: cluster 2023-10-09T05:39:48.667215+0000 osd.7 (osd.7) 324 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T05:39:50.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:49 smithi133 bash[102858]: cluster 2023-10-09T05:39:48.785685+0000 osd.4 (osd.4) 313 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T05:39:50.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:49 smithi133 bash[102858]: cluster 2023-10-09T05:39:48.943750+0000 mon.a (mon.0) 1990 : cluster 3 Health check update: Degraded data redundancy: 17/8580 objects degraded (0.198%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:39:50.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:49 smithi133 bash[105589]: cluster 2023-10-09T05:39:48.140004+0000 osd.7 (osd.7) 322 : cluster 0 14.1f scrub starts 2023-10-09T05:39:50.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:49 smithi133 bash[105589]: cluster 2023-10-09T05:39:48.151768+0000 osd.7 (osd.7) 323 : cluster 0 14.1f scrub ok 2023-10-09T05:39:50.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:49 smithi133 bash[105589]: cluster 2023-10-09T05:39:48.474660+0000 mgr.y (mgr.25277) 1561 : cluster 0 pgmap v1704: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 178 MiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 43 op/s; 17/8580 objects degraded (0.198%); 146/8580 objects misplaced (1.702%); 0 B/s, 87 objects/s recovering 2023-10-09T05:39:50.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:49 smithi133 bash[105589]: cluster 2023-10-09T05:39:48.590953+0000 osd.5 (osd.5) 370 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T05:39:50.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:49 smithi133 bash[105589]: cluster 2023-10-09T05:39:48.595361+0000 osd.5 (osd.5) 371 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T05:39:50.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:49 smithi133 bash[105589]: cluster 2023-10-09T05:39:48.597108+0000 mon.a (mon.0) 1989 : cluster 0 osdmap e357: 8 total, 8 up, 7 in 2023-10-09T05:39:50.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:49 smithi133 bash[105589]: cluster 2023-10-09T05:39:48.646363+0000 osd.6 (osd.6) 341 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T05:39:50.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:49 smithi133 bash[105589]: cluster 2023-10-09T05:39:48.667215+0000 osd.7 (osd.7) 324 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T05:39:50.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:49 smithi133 bash[105589]: cluster 2023-10-09T05:39:48.785685+0000 osd.4 (osd.4) 313 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T05:39:50.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:49 smithi133 bash[105589]: cluster 2023-10-09T05:39:48.943750+0000 mon.a (mon.0) 1990 : cluster 3 Health check update: Degraded data redundancy: 17/8580 objects degraded (0.198%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:39:50.389 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:50.415 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:50.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:50.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:50.518 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:50.548 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:50.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:50.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:50.652 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:50.663 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:50 smithi133 bash[102858]: cluster 2023-10-09T05:39:49.178256+0000 osd.3 (osd.3) 14 : cluster 0 14.1 scrub starts 2023-10-09T05:39:50.663 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:50 smithi133 bash[102858]: cluster 2023-10-09T05:39:49.184965+0000 osd.3 (osd.3) 15 : cluster 0 14.1 scrub ok 2023-10-09T05:39:50.663 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:50 smithi133 bash[102858]: cluster 2023-10-09T05:39:50.079976+0000 osd.7 (osd.7) 325 : cluster 0 14.10 scrub starts 2023-10-09T05:39:50.663 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:50 smithi133 bash[102858]: cluster 2023-10-09T05:39:50.095443+0000 osd.7 (osd.7) 326 : cluster 0 14.10 scrub ok 2023-10-09T05:39:50.663 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:50 smithi133 bash[102858]: audit 2023-10-09T05:39:50.476717+0000 mon.a (mon.0) 1991 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-10-09T05:39:50.663 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:50 smithi133 bash[105589]: cluster 2023-10-09T05:39:49.178256+0000 osd.3 (osd.3) 14 : cluster 0 14.1 scrub starts 2023-10-09T05:39:50.663 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:50 smithi133 bash[105589]: cluster 2023-10-09T05:39:49.184965+0000 osd.3 (osd.3) 15 : cluster 0 14.1 scrub ok 2023-10-09T05:39:50.663 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:50 smithi133 bash[105589]: cluster 2023-10-09T05:39:50.079976+0000 osd.7 (osd.7) 325 : cluster 0 14.10 scrub starts 2023-10-09T05:39:50.664 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:50 smithi133 bash[105589]: cluster 2023-10-09T05:39:50.095443+0000 osd.7 (osd.7) 326 : cluster 0 14.10 scrub ok 2023-10-09T05:39:50.664 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:50 smithi133 bash[105589]: audit 2023-10-09T05:39:50.476717+0000 mon.a (mon.0) 1991 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-10-09T05:39:50.678 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:50.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:50.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:50.781 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:50.794 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:50 smithi194 bash[92666]: cluster 2023-10-09T05:39:49.178256+0000 osd.3 (osd.3) 14 : cluster 0 14.1 scrub starts 2023-10-09T05:39:50.794 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:50 smithi194 bash[92666]: cluster 2023-10-09T05:39:49.184965+0000 osd.3 (osd.3) 15 : cluster 0 14.1 scrub ok 2023-10-09T05:39:50.795 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:50 smithi194 bash[92666]: cluster 2023-10-09T05:39:50.079976+0000 osd.7 (osd.7) 325 : cluster 0 14.10 scrub starts 2023-10-09T05:39:50.795 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:50 smithi194 bash[92666]: cluster 2023-10-09T05:39:50.095443+0000 osd.7 (osd.7) 326 : cluster 0 14.10 scrub ok 2023-10-09T05:39:50.795 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:50 smithi194 bash[92666]: audit 2023-10-09T05:39:50.476717+0000 mon.a (mon.0) 1991 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-10-09T05:39:50.814 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:50.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:50.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:50.917 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:50.949 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:50.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:50.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:51.053 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:51.082 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:51.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:51.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:51.185 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:51.214 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:51.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:51.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:51.317 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:51.337 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:51.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:51.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:51.440 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:51.466 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:51.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:51.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:51.569 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:51.589 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:51.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:51.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:51.692 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:51.704 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:51 smithi133 bash[105589]: cluster 2023-10-09T05:39:50.376488+0000 osd.2 (osd.2) 6 : cluster 0 14.1e scrub starts 2023-10-09T05:39:51.704 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:51 smithi133 bash[105589]: cluster 2023-10-09T05:39:50.388950+0000 osd.2 (osd.2) 7 : cluster 0 14.1e scrub ok 2023-10-09T05:39:51.704 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:51 smithi133 bash[105589]: cluster 2023-10-09T05:39:50.475709+0000 mgr.y (mgr.25277) 1562 : cluster 0 pgmap v1706: 109 pgs: 1 active+remapped+backfill_wait, 108 active+clean; 93 MiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 146/4488 objects misplaced (3.253%); 0 B/s, 156 objects/s recovering 2023-10-09T05:39:51.704 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:51 smithi133 bash[105589]: cluster 2023-10-09T05:39:50.586387+0000 mon.a (mon.0) 1992 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17/8580 objects degraded (0.198%), 1 pg degraded) 2023-10-09T05:39:51.704 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:51 smithi133 bash[105589]: cluster 2023-10-09T05:39:50.600974+0000 osd.5 (osd.5) 372 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T05:39:51.704 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:51 smithi133 bash[105589]: audit 2023-10-09T05:39:50.603739+0000 mon.a (mon.0) 1993 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "24"}]': finished 2023-10-09T05:39:51.704 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:51 smithi133 bash[105589]: cluster 2023-10-09T05:39:50.603904+0000 mon.a (mon.0) 1994 : cluster 0 osdmap e358: 8 total, 8 up, 7 in 2023-10-09T05:39:51.704 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:51 smithi133 bash[105589]: cluster 2023-10-09T05:39:50.614024+0000 osd.7 (osd.7) 327 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T05:39:51.704 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:51 smithi133 bash[105589]: cluster 2023-10-09T05:39:51.036088+0000 osd.7 (osd.7) 328 : cluster 0 14.15 scrub starts 2023-10-09T05:39:51.705 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:51 smithi133 bash[105589]: cluster 2023-10-09T05:39:51.050461+0000 osd.7 (osd.7) 329 : cluster 0 14.15 scrub ok 2023-10-09T05:39:51.705 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:51 smithi133 bash[102858]: cluster 2023-10-09T05:39:50.376488+0000 osd.2 (osd.2) 6 : cluster 0 14.1e scrub starts 2023-10-09T05:39:51.705 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:51 smithi133 bash[102858]: cluster 2023-10-09T05:39:50.388950+0000 osd.2 (osd.2) 7 : cluster 0 14.1e scrub ok 2023-10-09T05:39:51.705 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:51 smithi133 bash[102858]: cluster 2023-10-09T05:39:50.475709+0000 mgr.y (mgr.25277) 1562 : cluster 0 pgmap v1706: 109 pgs: 1 active+remapped+backfill_wait, 108 active+clean; 93 MiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 146/4488 objects misplaced (3.253%); 0 B/s, 156 objects/s recovering 2023-10-09T05:39:51.705 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:51 smithi133 bash[102858]: cluster 2023-10-09T05:39:50.586387+0000 mon.a (mon.0) 1992 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17/8580 objects degraded (0.198%), 1 pg degraded) 2023-10-09T05:39:51.705 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:51 smithi133 bash[102858]: cluster 2023-10-09T05:39:50.600974+0000 osd.5 (osd.5) 372 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T05:39:51.705 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:51 smithi133 bash[102858]: audit 2023-10-09T05:39:50.603739+0000 mon.a (mon.0) 1993 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "24"}]': finished 2023-10-09T05:39:51.706 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:51 smithi133 bash[102858]: cluster 2023-10-09T05:39:50.603904+0000 mon.a (mon.0) 1994 : cluster 0 osdmap e358: 8 total, 8 up, 7 in 2023-10-09T05:39:51.706 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:51 smithi133 bash[102858]: cluster 2023-10-09T05:39:50.614024+0000 osd.7 (osd.7) 327 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T05:39:51.706 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:51 smithi133 bash[102858]: cluster 2023-10-09T05:39:51.036088+0000 osd.7 (osd.7) 328 : cluster 0 14.15 scrub starts 2023-10-09T05:39:51.706 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:51 smithi133 bash[102858]: cluster 2023-10-09T05:39:51.050461+0000 osd.7 (osd.7) 329 : cluster 0 14.15 scrub ok 2023-10-09T05:39:51.721 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:51.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:51.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:51.824 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:51.852 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:51.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:51.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:51.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:51 smithi194 bash[92666]: cluster 2023-10-09T05:39:50.376488+0000 osd.2 (osd.2) 6 : cluster 0 14.1e scrub starts 2023-10-09T05:39:51.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:51 smithi194 bash[92666]: cluster 2023-10-09T05:39:50.388950+0000 osd.2 (osd.2) 7 : cluster 0 14.1e scrub ok 2023-10-09T05:39:51.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:51 smithi194 bash[92666]: cluster 2023-10-09T05:39:50.475709+0000 mgr.y (mgr.25277) 1562 : cluster 0 pgmap v1706: 109 pgs: 1 active+remapped+backfill_wait, 108 active+clean; 93 MiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 146/4488 objects misplaced (3.253%); 0 B/s, 156 objects/s recovering 2023-10-09T05:39:51.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:51 smithi194 bash[92666]: cluster 2023-10-09T05:39:50.586387+0000 mon.a (mon.0) 1992 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17/8580 objects degraded (0.198%), 1 pg degraded) 2023-10-09T05:39:51.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:51 smithi194 bash[92666]: cluster 2023-10-09T05:39:50.600974+0000 osd.5 (osd.5) 372 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T05:39:51.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:51 smithi194 bash[92666]: audit 2023-10-09T05:39:50.603739+0000 mon.a (mon.0) 1993 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "24"}]': finished 2023-10-09T05:39:51.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:51 smithi194 bash[92666]: cluster 2023-10-09T05:39:50.603904+0000 mon.a (mon.0) 1994 : cluster 0 osdmap e358: 8 total, 8 up, 7 in 2023-10-09T05:39:51.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:51 smithi194 bash[92666]: cluster 2023-10-09T05:39:50.614024+0000 osd.7 (osd.7) 327 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T05:39:51.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:51 smithi194 bash[92666]: cluster 2023-10-09T05:39:51.036088+0000 osd.7 (osd.7) 328 : cluster 0 14.15 scrub starts 2023-10-09T05:39:51.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:51 smithi194 bash[92666]: cluster 2023-10-09T05:39:51.050461+0000 osd.7 (osd.7) 329 : cluster 0 14.15 scrub ok 2023-10-09T05:39:51.955 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:51.986 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:51.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:51.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:52.090 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:52.122 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:52.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:52.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:52.226 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:52.258 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:52.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:52.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:52.362 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:52.386 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:52.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:52.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:52.490 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:52.520 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:52.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:52.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:52.624 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:52.655 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:52.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:52.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:52.674 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:52 smithi133 bash[102858]: cluster 2023-10-09T05:39:51.600324+0000 osd.6 (osd.6) 342 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T05:39:52.674 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:52 smithi133 bash[102858]: cluster 2023-10-09T05:39:51.601705+0000 mon.a (mon.0) 1995 : cluster 0 osdmap e359: 8 total, 8 up, 7 in 2023-10-09T05:39:52.674 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:52 smithi133 bash[102858]: cluster 2023-10-09T05:39:51.605791+0000 osd.5 (osd.5) 373 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T05:39:52.674 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:52 smithi133 bash[102858]: cluster 2023-10-09T05:39:51.609651+0000 osd.5 (osd.5) 374 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T05:39:52.674 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:52 smithi133 bash[102858]: cluster 2023-10-09T05:39:51.609695+0000 osd.7 (osd.7) 330 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T05:39:52.760 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:52.783 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:52.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:52.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:52.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:52 smithi194 bash[92666]: cluster 2023-10-09T05:39:51.600324+0000 osd.6 (osd.6) 342 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T05:39:52.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:52 smithi194 bash[92666]: cluster 2023-10-09T05:39:51.601705+0000 mon.a (mon.0) 1995 : cluster 0 osdmap e359: 8 total, 8 up, 7 in 2023-10-09T05:39:52.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:52 smithi194 bash[92666]: cluster 2023-10-09T05:39:51.605791+0000 osd.5 (osd.5) 373 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T05:39:52.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:52 smithi194 bash[92666]: cluster 2023-10-09T05:39:51.609651+0000 osd.5 (osd.5) 374 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T05:39:52.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:52 smithi194 bash[92666]: cluster 2023-10-09T05:39:51.609695+0000 osd.7 (osd.7) 330 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T05:39:52.887 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:52.919 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:52.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:52.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:53.025 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:53.038 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:52 smithi133 bash[105589]: cluster 2023-10-09T05:39:51.600324+0000 osd.6 (osd.6) 342 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T05:39:53.038 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:52 smithi133 bash[105589]: cluster 2023-10-09T05:39:51.601705+0000 mon.a (mon.0) 1995 : cluster 0 osdmap e359: 8 total, 8 up, 7 in 2023-10-09T05:39:53.038 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:52 smithi133 bash[105589]: cluster 2023-10-09T05:39:51.605791+0000 osd.5 (osd.5) 373 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T05:39:53.038 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:52 smithi133 bash[105589]: cluster 2023-10-09T05:39:51.609651+0000 osd.5 (osd.5) 374 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T05:39:53.038 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:52 smithi133 bash[105589]: cluster 2023-10-09T05:39:51.609695+0000 osd.7 (osd.7) 330 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T05:39:53.053 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:53.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:53.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:53.157 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:53.188 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:53.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:53.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:53.292 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:53.318 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:53.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:53.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:53.420 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:53.456 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:53.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:53.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:53.560 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:53.581 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:53.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:53.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:53.684 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:53.696 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:53 smithi194 bash[92666]: cluster 2023-10-09T05:39:52.182642+0000 osd.0 (osd.0) 11 : cluster 0 14.8 scrub starts 2023-10-09T05:39:53.697 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:53 smithi194 bash[92666]: cluster 2023-10-09T05:39:52.196288+0000 osd.0 (osd.0) 12 : cluster 0 14.8 scrub ok 2023-10-09T05:39:53.697 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:53 smithi194 bash[92666]: cluster 2023-10-09T05:39:52.476334+0000 mgr.y (mgr.25277) 1563 : cluster 0 pgmap v1709: 109 pgs: 1 unknown, 108 active+clean; 19 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 666 KiB/s, 58 objects/s recovering 2023-10-09T05:39:53.697 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:53 smithi194 bash[92666]: cluster 2023-10-09T05:39:52.621397+0000 osd.4 (osd.4) 314 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T05:39:53.697 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:53 smithi194 bash[92666]: cluster 2023-10-09T05:39:52.626682+0000 mon.a (mon.0) 1996 : cluster 0 osdmap e360: 8 total, 8 up, 7 in 2023-10-09T05:39:53.697 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:53 smithi194 bash[92666]: cluster 2023-10-09T05:39:52.697882+0000 osd.7 (osd.7) 331 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T05:39:53.697 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:53 smithi194 bash[92666]: cluster 2023-10-09T05:39:52.697961+0000 osd.5 (osd.5) 375 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T05:39:53.716 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:53.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:53.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:53.820 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:53.829 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:53 smithi133 bash[102858]: cluster 2023-10-09T05:39:52.182642+0000 osd.0 (osd.0) 11 : cluster 0 14.8 scrub starts 2023-10-09T05:39:53.830 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:53 smithi133 bash[102858]: cluster 2023-10-09T05:39:52.196288+0000 osd.0 (osd.0) 12 : cluster 0 14.8 scrub ok 2023-10-09T05:39:53.830 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:53 smithi133 bash[102858]: cluster 2023-10-09T05:39:52.476334+0000 mgr.y (mgr.25277) 1563 : cluster 0 pgmap v1709: 109 pgs: 1 unknown, 108 active+clean; 19 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 666 KiB/s, 58 objects/s recovering 2023-10-09T05:39:53.830 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:53 smithi133 bash[102858]: cluster 2023-10-09T05:39:52.621397+0000 osd.4 (osd.4) 314 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T05:39:53.830 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:53 smithi133 bash[102858]: cluster 2023-10-09T05:39:52.626682+0000 mon.a (mon.0) 1996 : cluster 0 osdmap e360: 8 total, 8 up, 7 in 2023-10-09T05:39:53.830 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:53 smithi133 bash[102858]: cluster 2023-10-09T05:39:52.697882+0000 osd.7 (osd.7) 331 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T05:39:53.830 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:53 smithi133 bash[102858]: cluster 2023-10-09T05:39:52.697961+0000 osd.5 (osd.5) 375 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T05:39:53.830 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:53 smithi133 bash[105589]: cluster 2023-10-09T05:39:52.182642+0000 osd.0 (osd.0) 11 : cluster 0 14.8 scrub starts 2023-10-09T05:39:53.830 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:53 smithi133 bash[105589]: cluster 2023-10-09T05:39:52.196288+0000 osd.0 (osd.0) 12 : cluster 0 14.8 scrub ok 2023-10-09T05:39:53.830 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:53 smithi133 bash[105589]: cluster 2023-10-09T05:39:52.476334+0000 mgr.y (mgr.25277) 1563 : cluster 0 pgmap v1709: 109 pgs: 1 unknown, 108 active+clean; 19 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 666 KiB/s, 58 objects/s recovering 2023-10-09T05:39:53.831 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:53 smithi133 bash[105589]: cluster 2023-10-09T05:39:52.621397+0000 osd.4 (osd.4) 314 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T05:39:53.831 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:53 smithi133 bash[105589]: cluster 2023-10-09T05:39:52.626682+0000 mon.a (mon.0) 1996 : cluster 0 osdmap e360: 8 total, 8 up, 7 in 2023-10-09T05:39:53.831 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:53 smithi133 bash[105589]: cluster 2023-10-09T05:39:52.697882+0000 osd.7 (osd.7) 331 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T05:39:53.831 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:53 smithi133 bash[105589]: cluster 2023-10-09T05:39:52.697961+0000 osd.5 (osd.5) 375 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T05:39:53.844 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:53.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:53.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:53.947 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:53.979 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:53.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:53.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:54.083 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:54.115 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:54.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:54.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:54.219 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:54.250 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:54.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:54.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:54.354 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:54.378 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:54.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:54.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:54.460 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_9 2023-10-09T05:39:54.461 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_9 unique_pool_9 --yes-i-really-really-mean-it 2023-10-09T05:39:54.481 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:54.513 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:54.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:54.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:54.534 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:39:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:39:54] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.43.0" 2023-10-09T05:39:54.617 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:54.647 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:54.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:54.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:54.751 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:54.774 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:54.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:54.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:54.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:54 smithi133 bash[102858]: cluster 2023-10-09T05:39:53.177773+0000 osd.3 (osd.3) 16 : cluster 0 14.21 scrub starts 2023-10-09T05:39:54.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:54 smithi133 bash[105589]: cluster 2023-10-09T05:39:53.177773+0000 osd.3 (osd.3) 16 : cluster 0 14.21 scrub starts 2023-10-09T05:39:54.878 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:54.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:54 smithi194 bash[92666]: cluster 2023-10-09T05:39:53.177773+0000 osd.3 (osd.3) 16 : cluster 0 14.21 scrub starts 2023-10-09T05:39:54.908 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:54.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:54.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:55.011 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:55.044 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:55.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:55.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:55.147 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:55.174 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:55.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:55.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:55.278 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:55.306 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:55.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:55.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:55.410 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:55.435 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:55.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:55.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:55.538 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:55.569 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:55.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:55.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:55.673 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:55.685 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:55 smithi133 bash[105589]: cluster 2023-10-09T05:39:54.209168+0000 osd.3 (osd.3) 17 : cluster 0 14.14 scrub starts 2023-10-09T05:39:55.685 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:55 smithi133 bash[105589]: cluster 2023-10-09T05:39:54.220328+0000 osd.3 (osd.3) 18 : cluster 0 14.14 scrub ok 2023-10-09T05:39:55.685 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:55 smithi133 bash[105589]: cluster 2023-10-09T05:39:54.472359+0000 osd.6 (osd.6) 343 : cluster 0 14.2 scrub starts 2023-10-09T05:39:55.685 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:55 smithi133 bash[105589]: cluster 2023-10-09T05:39:54.476850+0000 mgr.y (mgr.25277) 1564 : cluster 0 pgmap v1711: 109 pgs: 1 unknown, 108 active+clean; 17 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 555 KiB/s, 48 objects/s recovering 2023-10-09T05:39:55.685 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:55 smithi133 bash[102858]: cluster 2023-10-09T05:39:54.209168+0000 osd.3 (osd.3) 17 : cluster 0 14.14 scrub starts 2023-10-09T05:39:55.686 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:55 smithi133 bash[102858]: cluster 2023-10-09T05:39:54.220328+0000 osd.3 (osd.3) 18 : cluster 0 14.14 scrub ok 2023-10-09T05:39:55.686 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:55 smithi133 bash[102858]: cluster 2023-10-09T05:39:54.472359+0000 osd.6 (osd.6) 343 : cluster 0 14.2 scrub starts 2023-10-09T05:39:55.686 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:55 smithi133 bash[102858]: cluster 2023-10-09T05:39:54.476850+0000 mgr.y (mgr.25277) 1564 : cluster 0 pgmap v1711: 109 pgs: 1 unknown, 108 active+clean; 17 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 555 KiB/s, 48 objects/s recovering 2023-10-09T05:39:55.698 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:55.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:55.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:55.801 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:55.834 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:55.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:55.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:55.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:55 smithi194 bash[92666]: cluster 2023-10-09T05:39:54.209168+0000 osd.3 (osd.3) 17 : cluster 0 14.14 scrub starts 2023-10-09T05:39:55.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:55 smithi194 bash[92666]: cluster 2023-10-09T05:39:54.220328+0000 osd.3 (osd.3) 18 : cluster 0 14.14 scrub ok 2023-10-09T05:39:55.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:55 smithi194 bash[92666]: cluster 2023-10-09T05:39:54.472359+0000 osd.6 (osd.6) 343 : cluster 0 14.2 scrub starts 2023-10-09T05:39:55.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:55 smithi194 bash[92666]: cluster 2023-10-09T05:39:54.476850+0000 mgr.y (mgr.25277) 1564 : cluster 0 pgmap v1711: 109 pgs: 1 unknown, 108 active+clean; 17 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 555 KiB/s, 48 objects/s recovering 2023-10-09T05:39:55.937 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:55.965 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:55.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:55.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:56.069 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:56.100 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:56.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:56.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:56.204 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:56.226 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:56.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:56.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:56.330 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:56.354 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:56.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:56.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:56.456 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:56.478 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:56.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:56.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:56.581 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:56.603 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:56.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:56.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:56.706 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:56.729 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:56.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:56.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:56.825 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:56 smithi133 bash[105589]: cluster 2023-10-09T05:39:55.445290+0000 osd.2 (osd.2) 8 : cluster 0 14.20 scrub starts 2023-10-09T05:39:56.825 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:56 smithi133 bash[105589]: cluster 2023-10-09T05:39:55.475416+0000 osd.6 (osd.6) 344 : cluster 0 14.d scrub starts 2023-10-09T05:39:56.825 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:56 smithi133 bash[105589]: audit 2023-10-09T05:39:56.478337+0000 mon.a (mon.0) 1997 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-10-09T05:39:56.825 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:56 smithi133 bash[102858]: cluster 2023-10-09T05:39:55.445290+0000 osd.2 (osd.2) 8 : cluster 0 14.20 scrub starts 2023-10-09T05:39:56.825 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:56 smithi133 bash[102858]: cluster 2023-10-09T05:39:55.475416+0000 osd.6 (osd.6) 344 : cluster 0 14.d scrub starts 2023-10-09T05:39:56.826 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:56 smithi133 bash[102858]: audit 2023-10-09T05:39:56.478337+0000 mon.a (mon.0) 1997 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-10-09T05:39:56.832 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:56.854 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:56.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:56.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:56.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:56 smithi194 bash[92666]: cluster 2023-10-09T05:39:55.445290+0000 osd.2 (osd.2) 8 : cluster 0 14.20 scrub starts 2023-10-09T05:39:56.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:56 smithi194 bash[92666]: cluster 2023-10-09T05:39:55.475416+0000 osd.6 (osd.6) 344 : cluster 0 14.d scrub starts 2023-10-09T05:39:56.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:56 smithi194 bash[92666]: audit 2023-10-09T05:39:56.478337+0000 mon.a (mon.0) 1997 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-10-09T05:39:56.957 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:56.991 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:56.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:56.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:57.095 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:57.126 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:57.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:57.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:57.230 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:57.261 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:57.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:57.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:57.365 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:57.401 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:57.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:57.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:57.505 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:57.536 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:57.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:57.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:57.640 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:57.663 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:57.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:57.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:57.766 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:57.787 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:57.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:57.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:57.885 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:57 smithi133 bash[102858]: cluster 2023-10-09T05:39:56.477519+0000 mgr.y (mgr.25277) 1565 : cluster 0 pgmap v1712: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 16 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:39:57.885 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:57 smithi133 bash[102858]: cluster 2023-10-09T05:39:56.632059+0000 osd.7 (osd.7) 332 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T05:39:57.885 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:57 smithi133 bash[102858]: cluster 2023-10-09T05:39:56.633257+0000 osd.5 (osd.5) 376 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T05:39:57.886 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:57 smithi133 bash[102858]: audit 2023-10-09T05:39:56.635343+0000 mon.a (mon.0) 1998 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "25"}]': finished 2023-10-09T05:39:57.886 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:57 smithi133 bash[102858]: cluster 2023-10-09T05:39:56.635511+0000 mon.a (mon.0) 1999 : cluster 0 osdmap e361: 8 total, 8 up, 7 in 2023-10-09T05:39:57.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:57 smithi194 bash[92666]: cluster 2023-10-09T05:39:56.477519+0000 mgr.y (mgr.25277) 1565 : cluster 0 pgmap v1712: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 16 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:39:57.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:57 smithi194 bash[92666]: cluster 2023-10-09T05:39:56.632059+0000 osd.7 (osd.7) 332 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T05:39:57.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:57 smithi194 bash[92666]: cluster 2023-10-09T05:39:56.633257+0000 osd.5 (osd.5) 376 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T05:39:57.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:57 smithi194 bash[92666]: audit 2023-10-09T05:39:56.635343+0000 mon.a (mon.0) 1998 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "25"}]': finished 2023-10-09T05:39:57.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:57 smithi194 bash[92666]: cluster 2023-10-09T05:39:56.635511+0000 mon.a (mon.0) 1999 : cluster 0 osdmap e361: 8 total, 8 up, 7 in 2023-10-09T05:39:57.891 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:57.921 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:57.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:57.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:58.026 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:58.057 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:58.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:58.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:58.162 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:58.195 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:58.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:58.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:58.297 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:57 smithi133 bash[105589]: cluster 2023-10-09T05:39:56.477519+0000 mgr.y (mgr.25277) 1565 : cluster 0 pgmap v1712: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 16 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:39:58.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:57 smithi133 bash[105589]: cluster 2023-10-09T05:39:56.632059+0000 osd.7 (osd.7) 332 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T05:39:58.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:57 smithi133 bash[105589]: cluster 2023-10-09T05:39:56.633257+0000 osd.5 (osd.5) 376 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T05:39:58.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:57 smithi133 bash[105589]: audit 2023-10-09T05:39:56.635343+0000 mon.a (mon.0) 1998 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "25"}]': finished 2023-10-09T05:39:58.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:57 smithi133 bash[105589]: cluster 2023-10-09T05:39:56.635511+0000 mon.a (mon.0) 1999 : cluster 0 osdmap e361: 8 total, 8 up, 7 in 2023-10-09T05:39:58.298 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:58.318 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:58.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:58.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:58.421 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:58.452 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:58.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:58.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:58.556 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:58.592 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:58.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:58.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:58.651 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:58 smithi133 bash[102858]: cluster 2023-10-09T05:39:57.643812+0000 osd.6 (osd.6) 345 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T05:39:58.651 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:58 smithi133 bash[102858]: cluster 2023-10-09T05:39:57.643973+0000 osd.5 (osd.5) 377 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T05:39:58.652 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:58 smithi133 bash[102858]: cluster 2023-10-09T05:39:57.645825+0000 mon.a (mon.0) 2000 : cluster 0 osdmap e362: 8 total, 8 up, 7 in 2023-10-09T05:39:58.652 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:58 smithi133 bash[102858]: cluster 2023-10-09T05:39:57.817245+0000 osd.7 (osd.7) 333 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T05:39:58.652 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:58 smithi133 bash[102858]: cluster 2023-10-09T05:39:57.817494+0000 osd.4 (osd.4) 315 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T05:39:58.652 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:58 smithi133 bash[102858]: audit 2023-10-09T05:39:58.479461+0000 mon.a (mon.0) 2001 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-10-09T05:39:58.652 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:58 smithi133 bash[102858]: audit 2023-10-09T05:39:58.500238+0000 mon.a (mon.0) 2002 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:39:58.652 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:58 smithi133 bash[102858]: audit 2023-10-09T05:39:58.501358+0000 mon.a (mon.0) 2003 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:58.695 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:58.719 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:58.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:58.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:58.822 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:58.834 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:58 smithi194 bash[92666]: cluster 2023-10-09T05:39:57.643812+0000 osd.6 (osd.6) 345 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T05:39:58.834 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:58 smithi194 bash[92666]: cluster 2023-10-09T05:39:57.643973+0000 osd.5 (osd.5) 377 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T05:39:58.834 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:58 smithi194 bash[92666]: cluster 2023-10-09T05:39:57.645825+0000 mon.a (mon.0) 2000 : cluster 0 osdmap e362: 8 total, 8 up, 7 in 2023-10-09T05:39:58.834 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:58 smithi194 bash[92666]: cluster 2023-10-09T05:39:57.817245+0000 osd.7 (osd.7) 333 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T05:39:58.834 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:58 smithi194 bash[92666]: cluster 2023-10-09T05:39:57.817494+0000 osd.4 (osd.4) 315 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T05:39:58.834 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:58 smithi194 bash[92666]: audit 2023-10-09T05:39:58.479461+0000 mon.a (mon.0) 2001 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-10-09T05:39:58.834 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:58 smithi194 bash[92666]: audit 2023-10-09T05:39:58.500238+0000 mon.a (mon.0) 2002 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:39:58.834 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:58 smithi194 bash[92666]: audit 2023-10-09T05:39:58.501358+0000 mon.a (mon.0) 2003 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:58.855 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:58.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:58.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:58.959 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:58.987 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:58.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:58.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:59.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:58 smithi133 bash[105589]: cluster 2023-10-09T05:39:57.643812+0000 osd.6 (osd.6) 345 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T05:39:59.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:58 smithi133 bash[105589]: cluster 2023-10-09T05:39:57.643973+0000 osd.5 (osd.5) 377 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T05:39:59.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:58 smithi133 bash[105589]: cluster 2023-10-09T05:39:57.645825+0000 mon.a (mon.0) 2000 : cluster 0 osdmap e362: 8 total, 8 up, 7 in 2023-10-09T05:39:59.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:58 smithi133 bash[105589]: cluster 2023-10-09T05:39:57.817245+0000 osd.7 (osd.7) 333 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T05:39:59.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:58 smithi133 bash[105589]: cluster 2023-10-09T05:39:57.817494+0000 osd.4 (osd.4) 315 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T05:39:59.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:58 smithi133 bash[105589]: audit 2023-10-09T05:39:58.479461+0000 mon.a (mon.0) 2001 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-10-09T05:39:59.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:58 smithi133 bash[105589]: audit 2023-10-09T05:39:58.500238+0000 mon.a (mon.0) 2002 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:39:59.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:58 smithi133 bash[105589]: audit 2023-10-09T05:39:58.501358+0000 mon.a (mon.0) 2003 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:59.091 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:59.122 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:59.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:59.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:59.225 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:59.248 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:59.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:59.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:59.351 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:59.385 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:59.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:59.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:59.488 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:59.520 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:59.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:59.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:59.623 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:59.647 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:59.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:59.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:59.750 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:59.771 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:59.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:59.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:59.874 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:59.894 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:59 smithi133 bash[102858]: cluster 2023-10-09T05:39:58.140304+0000 osd.3 (osd.3) 19 : cluster 0 14.16 scrub starts 2023-10-09T05:39:59.894 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:59 smithi133 bash[102858]: cluster 2023-10-09T05:39:58.420422+0000 osd.2 (osd.2) 9 : cluster 0 14.13 scrub starts 2023-10-09T05:39:59.894 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:59 smithi133 bash[102858]: cluster 2023-10-09T05:39:58.477992+0000 mgr.y (mgr.25277) 1566 : cluster 0 pgmap v1715: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 16 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:39:59.894 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:59 smithi133 bash[102858]: cluster 2023-10-09T05:39:58.652048+0000 osd.5 (osd.5) 378 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T05:39:59.894 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:59 smithi133 bash[102858]: cluster 2023-10-09T05:39:58.654313+0000 osd.6 (osd.6) 346 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T05:39:59.894 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:59 smithi133 bash[102858]: audit 2023-10-09T05:39:58.655023+0000 mon.a (mon.0) 2004 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "26"}]': finished 2023-10-09T05:39:59.894 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:59 smithi133 bash[102858]: cluster 2023-10-09T05:39:58.655153+0000 mon.a (mon.0) 2005 : cluster 0 osdmap e363: 8 total, 8 up, 7 in 2023-10-09T05:39:59.894 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:59 smithi133 bash[102858]: cluster 2023-10-09T05:39:58.658619+0000 osd.1 (osd.1) 2 : cluster 0 14.18 starting backfill to osd.3 from (0'0,0'0] MAX to 357'8348 2023-10-09T05:39:59.894 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:59 smithi133 bash[102858]: cluster 2023-10-09T05:39:58.952568+0000 osd.5 (osd.5) 379 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T05:39:59.894 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:59 smithi133 bash[102858]: cluster 2023-10-09T05:39:58.953728+0000 mon.a (mon.0) 2006 : cluster 0 osdmap e364: 8 total, 8 up, 7 in 2023-10-09T05:39:59.895 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:59 smithi133 bash[102858]: cluster 2023-10-09T05:39:58.954600+0000 osd.6 (osd.6) 347 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T05:39:59.895 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:59 smithi133 bash[102858]: cluster 2023-10-09T05:39:58.969698+0000 osd.7 (osd.7) 334 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T05:39:59.895 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:39:59 smithi133 bash[102858]: cluster 2023-10-09T05:39:59.031629+0000 osd.4 (osd.4) 316 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T05:39:59.895 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:59 smithi133 bash[105589]: cluster 2023-10-09T05:39:58.140304+0000 osd.3 (osd.3) 19 : cluster 0 14.16 scrub starts 2023-10-09T05:39:59.895 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:59 smithi133 bash[105589]: cluster 2023-10-09T05:39:58.420422+0000 osd.2 (osd.2) 9 : cluster 0 14.13 scrub starts 2023-10-09T05:39:59.895 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:59 smithi133 bash[105589]: cluster 2023-10-09T05:39:58.477992+0000 mgr.y (mgr.25277) 1566 : cluster 0 pgmap v1715: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 16 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:39:59.895 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:59 smithi133 bash[105589]: cluster 2023-10-09T05:39:58.652048+0000 osd.5 (osd.5) 378 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T05:39:59.895 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:59 smithi133 bash[105589]: cluster 2023-10-09T05:39:58.654313+0000 osd.6 (osd.6) 346 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T05:39:59.895 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:59 smithi133 bash[105589]: audit 2023-10-09T05:39:58.655023+0000 mon.a (mon.0) 2004 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "26"}]': finished 2023-10-09T05:39:59.895 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:59 smithi133 bash[105589]: cluster 2023-10-09T05:39:58.655153+0000 mon.a (mon.0) 2005 : cluster 0 osdmap e363: 8 total, 8 up, 7 in 2023-10-09T05:39:59.896 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:59 smithi133 bash[105589]: cluster 2023-10-09T05:39:58.658619+0000 osd.1 (osd.1) 2 : cluster 0 14.18 starting backfill to osd.3 from (0'0,0'0] MAX to 357'8348 2023-10-09T05:39:59.896 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:59 smithi133 bash[105589]: cluster 2023-10-09T05:39:58.952568+0000 osd.5 (osd.5) 379 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T05:39:59.896 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:59 smithi133 bash[105589]: cluster 2023-10-09T05:39:58.953728+0000 mon.a (mon.0) 2006 : cluster 0 osdmap e364: 8 total, 8 up, 7 in 2023-10-09T05:39:59.896 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:59 smithi133 bash[105589]: cluster 2023-10-09T05:39:58.954600+0000 osd.6 (osd.6) 347 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T05:39:59.896 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:59 smithi133 bash[105589]: cluster 2023-10-09T05:39:58.969698+0000 osd.7 (osd.7) 334 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T05:39:59.896 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:39:59 smithi133 bash[105589]: cluster 2023-10-09T05:39:59.031629+0000 osd.4 (osd.4) 316 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T05:39:59.902 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:59.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:39:59.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:00.005 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:00.018 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:59 smithi194 bash[92666]: cluster 2023-10-09T05:39:58.140304+0000 osd.3 (osd.3) 19 : cluster 0 14.16 scrub starts 2023-10-09T05:40:00.018 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:59 smithi194 bash[92666]: cluster 2023-10-09T05:39:58.420422+0000 osd.2 (osd.2) 9 : cluster 0 14.13 scrub starts 2023-10-09T05:40:00.018 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:59 smithi194 bash[92666]: cluster 2023-10-09T05:39:58.477992+0000 mgr.y (mgr.25277) 1566 : cluster 0 pgmap v1715: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 16 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:00.018 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:59 smithi194 bash[92666]: cluster 2023-10-09T05:39:58.652048+0000 osd.5 (osd.5) 378 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T05:40:00.019 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:59 smithi194 bash[92666]: cluster 2023-10-09T05:39:58.654313+0000 osd.6 (osd.6) 346 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T05:40:00.019 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:59 smithi194 bash[92666]: audit 2023-10-09T05:39:58.655023+0000 mon.a (mon.0) 2004 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "26"}]': finished 2023-10-09T05:40:00.019 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:59 smithi194 bash[92666]: cluster 2023-10-09T05:39:58.655153+0000 mon.a (mon.0) 2005 : cluster 0 osdmap e363: 8 total, 8 up, 7 in 2023-10-09T05:40:00.019 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:59 smithi194 bash[92666]: cluster 2023-10-09T05:39:58.658619+0000 osd.1 (osd.1) 2 : cluster 0 14.18 starting backfill to osd.3 from (0'0,0'0] MAX to 357'8348 2023-10-09T05:40:00.019 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:59 smithi194 bash[92666]: cluster 2023-10-09T05:39:58.952568+0000 osd.5 (osd.5) 379 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T05:40:00.019 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:59 smithi194 bash[92666]: cluster 2023-10-09T05:39:58.953728+0000 mon.a (mon.0) 2006 : cluster 0 osdmap e364: 8 total, 8 up, 7 in 2023-10-09T05:40:00.019 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:59 smithi194 bash[92666]: cluster 2023-10-09T05:39:58.954600+0000 osd.6 (osd.6) 347 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T05:40:00.019 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:59 smithi194 bash[92666]: cluster 2023-10-09T05:39:58.969698+0000 osd.7 (osd.7) 334 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T05:40:00.019 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:39:59 smithi194 bash[92666]: cluster 2023-10-09T05:39:59.031629+0000 osd.4 (osd.4) 316 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T05:40:00.036 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:00.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:00.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:00.139 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:00.167 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:00.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:00.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:00.271 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:00.295 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:00.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:00.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:00.398 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:00.427 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:00.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:00.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:00.530 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:00.555 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:00.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:00.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:00.658 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:00.678 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:00.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:00.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:00.767 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:00.781 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:00.793 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:00 smithi194 bash[92666]: cluster 2023-10-09T05:39:59.395721+0000 osd.2 (osd.2) 10 : cluster 0 14.12 scrub starts 2023-10-09T05:40:00.793 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:00 smithi194 bash[92666]: cluster 2023-10-09T05:39:59.960998+0000 osd.5 (osd.5) 380 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T05:40:00.793 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:00 smithi194 bash[92666]: cluster 2023-10-09T05:39:59.962515+0000 mon.a (mon.0) 2007 : cluster 0 osdmap e365: 8 total, 8 up, 7 in 2023-10-09T05:40:00.793 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:00 smithi194 bash[92666]: cluster 2023-10-09T05:40:00.000117+0000 mon.a (mon.0) 2008 : cluster 3 overall HEALTH_WARN nodeep-scrub flag(s) set 2023-10-09T05:40:00.793 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:00 smithi194 bash[92666]: audit 2023-10-09T05:40:00.480141+0000 mon.a (mon.0) 2009 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "27"}]: dispatch 2023-10-09T05:40:00.793 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:00 smithi133 bash[105589]: cluster 2023-10-09T05:39:59.395721+0000 osd.2 (osd.2) 10 : cluster 0 14.12 scrub starts 2023-10-09T05:40:00.794 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:00 smithi133 bash[105589]: cluster 2023-10-09T05:39:59.960998+0000 osd.5 (osd.5) 380 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T05:40:00.794 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:00 smithi133 bash[105589]: cluster 2023-10-09T05:39:59.962515+0000 mon.a (mon.0) 2007 : cluster 0 osdmap e365: 8 total, 8 up, 7 in 2023-10-09T05:40:00.794 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:00 smithi133 bash[105589]: cluster 2023-10-09T05:40:00.000117+0000 mon.a (mon.0) 2008 : cluster 3 overall HEALTH_WARN nodeep-scrub flag(s) set 2023-10-09T05:40:00.794 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:00 smithi133 bash[105589]: audit 2023-10-09T05:40:00.480141+0000 mon.a (mon.0) 2009 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "27"}]: dispatch 2023-10-09T05:40:00.794 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:00 smithi133 bash[102858]: cluster 2023-10-09T05:39:59.395721+0000 osd.2 (osd.2) 10 : cluster 0 14.12 scrub starts 2023-10-09T05:40:00.794 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:00 smithi133 bash[102858]: cluster 2023-10-09T05:39:59.960998+0000 osd.5 (osd.5) 380 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T05:40:00.794 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:00 smithi133 bash[102858]: cluster 2023-10-09T05:39:59.962515+0000 mon.a (mon.0) 2007 : cluster 0 osdmap e365: 8 total, 8 up, 7 in 2023-10-09T05:40:00.794 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:00 smithi133 bash[102858]: cluster 2023-10-09T05:40:00.000117+0000 mon.a (mon.0) 2008 : cluster 3 overall HEALTH_WARN nodeep-scrub flag(s) set 2023-10-09T05:40:00.794 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:00 smithi133 bash[102858]: audit 2023-10-09T05:40:00.480141+0000 mon.a (mon.0) 2009 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "27"}]: dispatch 2023-10-09T05:40:00.810 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:00.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:00.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:00.913 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:00.937 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:00.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:00.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:01.040 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:01.065 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:01.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:01.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:01.169 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:01.203 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:01.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:01.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:01.307 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:01.338 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:01.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:01.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:01.442 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:01.475 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:01.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:01.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:01.578 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:01.614 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:01.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:01.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:01.718 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:01.732 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:01 smithi194 bash[92666]: cluster 2023-10-09T05:40:00.478570+0000 mgr.y (mgr.25277) 1567 : cluster 0 pgmap v1719: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 16 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:01.732 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:01 smithi194 bash[92666]: cluster 2023-10-09T05:40:00.965965+0000 osd.6 (osd.6) 348 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T05:40:01.732 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:01 smithi194 bash[92666]: audit 2023-10-09T05:40:00.968414+0000 mon.a (mon.0) 2010 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "27"}]': finished 2023-10-09T05:40:01.732 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:01 smithi194 bash[92666]: cluster 2023-10-09T05:40:00.968509+0000 mon.a (mon.0) 2011 : cluster 0 osdmap e366: 8 total, 8 up, 7 in 2023-10-09T05:40:01.732 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:01 smithi194 bash[92666]: cluster 2023-10-09T05:40:00.973512+0000 osd.5 (osd.5) 381 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T05:40:01.749 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:01.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:01.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:01.853 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:01.875 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:01.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:01.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:01.978 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:02.006 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:02.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:02.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:02.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:01 smithi133 bash[102858]: cluster 2023-10-09T05:40:00.478570+0000 mgr.y (mgr.25277) 1567 : cluster 0 pgmap v1719: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 16 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:02.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:01 smithi133 bash[102858]: cluster 2023-10-09T05:40:00.965965+0000 osd.6 (osd.6) 348 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T05:40:02.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:01 smithi133 bash[102858]: audit 2023-10-09T05:40:00.968414+0000 mon.a (mon.0) 2010 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "27"}]': finished 2023-10-09T05:40:02.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:01 smithi133 bash[102858]: cluster 2023-10-09T05:40:00.968509+0000 mon.a (mon.0) 2011 : cluster 0 osdmap e366: 8 total, 8 up, 7 in 2023-10-09T05:40:02.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:01 smithi133 bash[102858]: cluster 2023-10-09T05:40:00.973512+0000 osd.5 (osd.5) 381 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T05:40:02.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:01 smithi133 bash[105589]: cluster 2023-10-09T05:40:00.478570+0000 mgr.y (mgr.25277) 1567 : cluster 0 pgmap v1719: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 16 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:02.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:01 smithi133 bash[105589]: cluster 2023-10-09T05:40:00.965965+0000 osd.6 (osd.6) 348 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T05:40:02.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:01 smithi133 bash[105589]: audit 2023-10-09T05:40:00.968414+0000 mon.a (mon.0) 2010 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "27"}]': finished 2023-10-09T05:40:02.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:01 smithi133 bash[105589]: cluster 2023-10-09T05:40:00.968509+0000 mon.a (mon.0) 2011 : cluster 0 osdmap e366: 8 total, 8 up, 7 in 2023-10-09T05:40:02.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:01 smithi133 bash[105589]: cluster 2023-10-09T05:40:00.973512+0000 osd.5 (osd.5) 381 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T05:40:02.109 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:02.143 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:02.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:02.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:02.247 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:02.282 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:02.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:02.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:02.386 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:02.414 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:02.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:02.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:02.518 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:02.546 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:02.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:02.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:02.650 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:02.675 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:02.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:02.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:02.778 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:02.802 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:02.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:02.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:02.905 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:02.930 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:02.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:02.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:03.033 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:03.054 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:03.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:03.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:03.159 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:03.166 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:02 smithi133 bash[102858]: cluster 2023-10-09T05:40:01.478304+0000 osd.6 (osd.6) 349 : cluster 0 14.22 scrub starts 2023-10-09T05:40:03.166 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:02 smithi133 bash[102858]: cluster 2023-10-09T05:40:01.973379+0000 osd.6 (osd.6) 350 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T05:40:03.166 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:02 smithi133 bash[102858]: cluster 2023-10-09T05:40:01.975212+0000 mon.a (mon.0) 2012 : cluster 0 osdmap e367: 8 total, 8 up, 7 in 2023-10-09T05:40:03.166 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:02 smithi133 bash[102858]: cluster 2023-10-09T05:40:02.288379+0000 osd.4 (osd.4) 317 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T05:40:03.167 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:02 smithi133 bash[102858]: cluster 2023-10-09T05:40:02.479263+0000 mgr.y (mgr.25277) 1568 : cluster 0 pgmap v1722: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 16 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 6200/801 objects degraded (774.032%); 0 B/s, 388 objects/s recovering 2023-10-09T05:40:03.167 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:02 smithi133 bash[105589]: cluster 2023-10-09T05:40:01.478304+0000 osd.6 (osd.6) 349 : cluster 0 14.22 scrub starts 2023-10-09T05:40:03.167 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:02 smithi133 bash[105589]: cluster 2023-10-09T05:40:01.973379+0000 osd.6 (osd.6) 350 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T05:40:03.167 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:02 smithi133 bash[105589]: cluster 2023-10-09T05:40:01.975212+0000 mon.a (mon.0) 2012 : cluster 0 osdmap e367: 8 total, 8 up, 7 in 2023-10-09T05:40:03.168 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:02 smithi133 bash[105589]: cluster 2023-10-09T05:40:02.288379+0000 osd.4 (osd.4) 317 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T05:40:03.168 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:02 smithi133 bash[105589]: cluster 2023-10-09T05:40:02.479263+0000 mgr.y (mgr.25277) 1568 : cluster 0 pgmap v1722: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 16 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 6200/801 objects degraded (774.032%); 0 B/s, 388 objects/s recovering 2023-10-09T05:40:03.182 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:03.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:03.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:03.285 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:03.306 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:03.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:03.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:03.326 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:03.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:02 smithi194 bash[92666]: cluster 2023-10-09T05:40:01.478304+0000 osd.6 (osd.6) 349 : cluster 0 14.22 scrub starts 2023-10-09T05:40:03.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:02 smithi194 bash[92666]: cluster 2023-10-09T05:40:01.973379+0000 osd.6 (osd.6) 350 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T05:40:03.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:02 smithi194 bash[92666]: cluster 2023-10-09T05:40:01.975212+0000 mon.a (mon.0) 2012 : cluster 0 osdmap e367: 8 total, 8 up, 7 in 2023-10-09T05:40:03.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:02 smithi194 bash[92666]: cluster 2023-10-09T05:40:02.288379+0000 osd.4 (osd.4) 317 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T05:40:03.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:02 smithi194 bash[92666]: cluster 2023-10-09T05:40:02.479263+0000 mgr.y (mgr.25277) 1568 : cluster 0 pgmap v1722: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 16 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 6200/801 objects degraded (774.032%); 0 B/s, 388 objects/s recovering 2023-10-09T05:40:03.409 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:03.440 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:03.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:03.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:03.544 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:03.574 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:03.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:03.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:03.678 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:03.711 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:03.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:03.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:03.815 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:03.847 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:03.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:03.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:03.951 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:03.983 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:03.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:03.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:04.089 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:04.113 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:04.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:04.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:04.217 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:04.226 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:03 smithi133 bash[102858]: cluster 2023-10-09T05:40:02.288147+0000 osd.5 (osd.5) 382 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T05:40:04.226 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:03 smithi133 bash[102858]: cluster 2023-10-09T05:40:02.288352+0000 osd.7 (osd.7) 335 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T05:40:04.226 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:03 smithi133 bash[102858]: cluster 2023-10-09T05:40:02.969694+0000 mon.a (mon.0) 2013 : cluster 3 Health check failed: Degraded data redundancy: 6200/801 objects degraded (774.032%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:40:04.226 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:03 smithi133 bash[102858]: cluster 2023-10-09T05:40:02.980336+0000 osd.4 (osd.4) 318 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T05:40:04.226 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:03 smithi133 bash[102858]: cluster 2023-10-09T05:40:02.980698+0000 osd.5 (osd.5) 383 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T05:40:04.226 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:03 smithi133 bash[102858]: cluster 2023-10-09T05:40:02.984678+0000 mon.a (mon.0) 2014 : cluster 0 osdmap e368: 8 total, 8 up, 7 in 2023-10-09T05:40:04.226 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:03 smithi133 bash[102858]: cluster 2023-10-09T05:40:02.986271+0000 osd.1 (osd.1) 3 : cluster 0 14.1a starting backfill to osd.0 from (0'0,0'0] MAX to 357'10764 2023-10-09T05:40:04.226 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:03 smithi133 bash[102858]: cluster 2023-10-09T05:40:03.097616+0000 osd.7 (osd.7) 336 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T05:40:04.226 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:03 smithi133 bash[102858]: cluster 2023-10-09T05:40:03.427174+0000 osd.2 (osd.2) 11 : cluster 0 14.0 scrub starts 2023-10-09T05:40:04.226 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:03 smithi133 bash[102858]: cluster 2023-10-09T05:40:03.958636+0000 mon.a (mon.0) 2015 : cluster 0 osdmap e369: 8 total, 8 up, 7 in 2023-10-09T05:40:04.227 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:03 smithi133 bash[105589]: cluster 2023-10-09T05:40:02.288147+0000 osd.5 (osd.5) 382 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T05:40:04.227 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:03 smithi133 bash[105589]: cluster 2023-10-09T05:40:02.288352+0000 osd.7 (osd.7) 335 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T05:40:04.227 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:03 smithi133 bash[105589]: cluster 2023-10-09T05:40:02.969694+0000 mon.a (mon.0) 2013 : cluster 3 Health check failed: Degraded data redundancy: 6200/801 objects degraded (774.032%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:40:04.227 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:03 smithi133 bash[105589]: cluster 2023-10-09T05:40:02.980336+0000 osd.4 (osd.4) 318 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T05:40:04.227 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:03 smithi133 bash[105589]: cluster 2023-10-09T05:40:02.980698+0000 osd.5 (osd.5) 383 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T05:40:04.227 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:03 smithi133 bash[105589]: cluster 2023-10-09T05:40:02.984678+0000 mon.a (mon.0) 2014 : cluster 0 osdmap e368: 8 total, 8 up, 7 in 2023-10-09T05:40:04.227 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:03 smithi133 bash[105589]: cluster 2023-10-09T05:40:02.986271+0000 osd.1 (osd.1) 3 : cluster 0 14.1a starting backfill to osd.0 from (0'0,0'0] MAX to 357'10764 2023-10-09T05:40:04.227 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:03 smithi133 bash[105589]: cluster 2023-10-09T05:40:03.097616+0000 osd.7 (osd.7) 336 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T05:40:04.227 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:03 smithi133 bash[105589]: cluster 2023-10-09T05:40:03.427174+0000 osd.2 (osd.2) 11 : cluster 0 14.0 scrub starts 2023-10-09T05:40:04.227 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:03 smithi133 bash[105589]: cluster 2023-10-09T05:40:03.958636+0000 mon.a (mon.0) 2015 : cluster 0 osdmap e369: 8 total, 8 up, 7 in 2023-10-09T05:40:04.241 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:04.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:04.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:04.344 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:04.366 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:04.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:04.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:04.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:03 smithi194 bash[92666]: cluster 2023-10-09T05:40:02.288147+0000 osd.5 (osd.5) 382 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T05:40:04.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:03 smithi194 bash[92666]: cluster 2023-10-09T05:40:02.288352+0000 osd.7 (osd.7) 335 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T05:40:04.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:03 smithi194 bash[92666]: cluster 2023-10-09T05:40:02.969694+0000 mon.a (mon.0) 2013 : cluster 3 Health check failed: Degraded data redundancy: 6200/801 objects degraded (774.032%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:40:04.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:03 smithi194 bash[92666]: cluster 2023-10-09T05:40:02.980336+0000 osd.4 (osd.4) 318 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T05:40:04.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:03 smithi194 bash[92666]: cluster 2023-10-09T05:40:02.980698+0000 osd.5 (osd.5) 383 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T05:40:04.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:03 smithi194 bash[92666]: cluster 2023-10-09T05:40:02.984678+0000 mon.a (mon.0) 2014 : cluster 0 osdmap e368: 8 total, 8 up, 7 in 2023-10-09T05:40:04.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:03 smithi194 bash[92666]: cluster 2023-10-09T05:40:02.986271+0000 osd.1 (osd.1) 3 : cluster 0 14.1a starting backfill to osd.0 from (0'0,0'0] MAX to 357'10764 2023-10-09T05:40:04.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:03 smithi194 bash[92666]: cluster 2023-10-09T05:40:03.097616+0000 osd.7 (osd.7) 336 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T05:40:04.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:03 smithi194 bash[92666]: cluster 2023-10-09T05:40:03.427174+0000 osd.2 (osd.2) 11 : cluster 0 14.0 scrub starts 2023-10-09T05:40:04.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:03 smithi194 bash[92666]: cluster 2023-10-09T05:40:03.958636+0000 mon.a (mon.0) 2015 : cluster 0 osdmap e369: 8 total, 8 up, 7 in 2023-10-09T05:40:04.470 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:04.483 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:40:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:40:04] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.43.0" 2023-10-09T05:40:04.498 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:04.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:04.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:04.601 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:04.630 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:04.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:04.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:04.734 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:04.765 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:04.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:04.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:04.870 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:04.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:04.903 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:04.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:05.004 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:05.037 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:05.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:05.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:05.140 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:05.171 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:05.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:05.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:05.275 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:05.295 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:04 smithi133 bash[105589]: cluster 2023-10-09T05:40:03.117544+0000 osd.6 (osd.6) 351 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T05:40:05.295 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:04 smithi133 bash[105589]: cluster 2023-10-09T05:40:03.953045+0000 osd.5 (osd.5) 384 : cluster 3 failed to encode map e369 with expected crc 2023-10-09T05:40:05.295 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:04 smithi133 bash[105589]: cluster 2023-10-09T05:40:04.138717+0000 osd.3 (osd.3) 20 : cluster 0 14.21 scrub starts 2023-10-09T05:40:05.295 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:04 smithi133 bash[105589]: cluster 2023-10-09T05:40:04.479961+0000 mgr.y (mgr.25277) 1569 : cluster 0 pgmap v1725: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 16 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 6200/801 objects degraded (774.032%); 0 B/s, 371 objects/s recovering 2023-10-09T05:40:05.295 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:04 smithi133 bash[105589]: cluster 2023-10-09T05:40:04.970775+0000 mon.a (mon.0) 2016 : cluster 0 osdmap e370: 8 total, 8 up, 7 in 2023-10-09T05:40:05.295 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:04 smithi133 bash[102858]: cluster 2023-10-09T05:40:03.117544+0000 osd.6 (osd.6) 351 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T05:40:05.295 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:04 smithi133 bash[102858]: cluster 2023-10-09T05:40:03.953045+0000 osd.5 (osd.5) 384 : cluster 3 failed to encode map e369 with expected crc 2023-10-09T05:40:05.295 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:04 smithi133 bash[102858]: cluster 2023-10-09T05:40:04.138717+0000 osd.3 (osd.3) 20 : cluster 0 14.21 scrub starts 2023-10-09T05:40:05.296 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:04 smithi133 bash[102858]: cluster 2023-10-09T05:40:04.479961+0000 mgr.y (mgr.25277) 1569 : cluster 0 pgmap v1725: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 16 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 6200/801 objects degraded (774.032%); 0 B/s, 371 objects/s recovering 2023-10-09T05:40:05.296 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:04 smithi133 bash[102858]: cluster 2023-10-09T05:40:04.970775+0000 mon.a (mon.0) 2016 : cluster 0 osdmap e370: 8 total, 8 up, 7 in 2023-10-09T05:40:05.302 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:05.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:05.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:05.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:04 smithi194 bash[92666]: cluster 2023-10-09T05:40:03.117544+0000 osd.6 (osd.6) 351 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T05:40:05.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:04 smithi194 bash[92666]: cluster 2023-10-09T05:40:03.953045+0000 osd.5 (osd.5) 384 : cluster 3 failed to encode map e369 with expected crc 2023-10-09T05:40:05.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:04 smithi194 bash[92666]: cluster 2023-10-09T05:40:04.138717+0000 osd.3 (osd.3) 20 : cluster 0 14.21 scrub starts 2023-10-09T05:40:05.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:04 smithi194 bash[92666]: cluster 2023-10-09T05:40:04.479961+0000 mgr.y (mgr.25277) 1569 : cluster 0 pgmap v1725: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 16 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 6200/801 objects degraded (774.032%); 0 B/s, 371 objects/s recovering 2023-10-09T05:40:05.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:04 smithi194 bash[92666]: cluster 2023-10-09T05:40:04.970775+0000 mon.a (mon.0) 2016 : cluster 0 osdmap e370: 8 total, 8 up, 7 in 2023-10-09T05:40:05.405 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:05.440 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:05.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:05.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:05.544 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:05.568 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:05.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:05.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:05.671 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:05.703 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:05.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:05.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:05.807 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:05.833 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:05.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:05.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:05.936 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:05.967 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:05.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:05.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:06.073 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:06.104 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:06.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:06.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:06.207 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:06.216 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:05 smithi194 bash[92666]: cluster 2023-10-09T05:40:04.967262+0000 osd.7 (osd.7) 337 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T05:40:06.216 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:05 smithi194 bash[92666]: cluster 2023-10-09T05:40:04.972625+0000 osd.5 (osd.5) 385 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T05:40:06.217 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:05 smithi194 bash[92666]: cluster 2023-10-09T05:40:04.972680+0000 osd.7 (osd.7) 338 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T05:40:06.217 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:05 smithi194 bash[92666]: cluster 2023-10-09T05:40:05.969387+0000 mon.a (mon.0) 2017 : cluster 0 osdmap e371: 8 total, 8 up, 7 in 2023-10-09T05:40:06.232 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:06.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:06.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:06.306 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:05 smithi133 bash[102858]: cluster 2023-10-09T05:40:04.967262+0000 osd.7 (osd.7) 337 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T05:40:06.307 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:05 smithi133 bash[102858]: cluster 2023-10-09T05:40:04.972625+0000 osd.5 (osd.5) 385 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T05:40:06.307 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:05 smithi133 bash[102858]: cluster 2023-10-09T05:40:04.972680+0000 osd.7 (osd.7) 338 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T05:40:06.307 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:05 smithi133 bash[102858]: cluster 2023-10-09T05:40:05.969387+0000 mon.a (mon.0) 2017 : cluster 0 osdmap e371: 8 total, 8 up, 7 in 2023-10-09T05:40:06.307 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:05 smithi133 bash[105589]: cluster 2023-10-09T05:40:04.967262+0000 osd.7 (osd.7) 337 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T05:40:06.307 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:05 smithi133 bash[105589]: cluster 2023-10-09T05:40:04.972625+0000 osd.5 (osd.5) 385 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T05:40:06.307 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:05 smithi133 bash[105589]: cluster 2023-10-09T05:40:04.972680+0000 osd.7 (osd.7) 338 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T05:40:06.307 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:05 smithi133 bash[105589]: cluster 2023-10-09T05:40:05.969387+0000 mon.a (mon.0) 2017 : cluster 0 osdmap e371: 8 total, 8 up, 7 in 2023-10-09T05:40:06.335 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:06.366 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:06.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:06.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:06.469 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:06.492 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:06.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:06.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:06.595 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:06.620 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:06.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:06.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:06.723 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:06.746 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:06.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:06.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:06.850 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:06.883 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:06.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:06.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:06.987 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:07.016 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:07.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:07.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:07.120 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:07.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:06 smithi194 bash[92666]: cluster 2023-10-09T05:40:05.470340+0000 osd.6 (osd.6) 352 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T05:40:07.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:06 smithi194 bash[92666]: cluster 2023-10-09T05:40:05.967899+0000 osd.6 (osd.6) 353 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T05:40:07.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:06 smithi194 bash[92666]: cluster 2023-10-09T05:40:05.970946+0000 osd.5 (osd.5) 386 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T05:40:07.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:06 smithi194 bash[92666]: cluster 2023-10-09T05:40:05.973898+0000 osd.5 (osd.5) 387 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T05:40:07.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:06 smithi194 bash[92666]: cluster 2023-10-09T05:40:06.189220+0000 osd.4 (osd.4) 319 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T05:40:07.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:06 smithi194 bash[92666]: cluster 2023-10-09T05:40:06.480425+0000 mgr.y (mgr.25277) 1570 : cluster 0 pgmap v1728: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 16 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s, 334 objects/s recovering 2023-10-09T05:40:07.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:06 smithi194 bash[92666]: audit 2023-10-09T05:40:06.481220+0000 mon.a (mon.0) 2018 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "28"}]: dispatch 2023-10-09T05:40:07.142 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:07.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:07.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:07.246 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:07.273 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:07.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:07.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:07.372 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:06 smithi133 bash[105589]: cluster 2023-10-09T05:40:05.470340+0000 osd.6 (osd.6) 352 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T05:40:07.372 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:06 smithi133 bash[105589]: cluster 2023-10-09T05:40:05.967899+0000 osd.6 (osd.6) 353 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T05:40:07.372 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:06 smithi133 bash[105589]: cluster 2023-10-09T05:40:05.970946+0000 osd.5 (osd.5) 386 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T05:40:07.372 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:06 smithi133 bash[105589]: cluster 2023-10-09T05:40:05.973898+0000 osd.5 (osd.5) 387 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T05:40:07.372 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:06 smithi133 bash[105589]: cluster 2023-10-09T05:40:06.189220+0000 osd.4 (osd.4) 319 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T05:40:07.372 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:06 smithi133 bash[105589]: cluster 2023-10-09T05:40:06.480425+0000 mgr.y (mgr.25277) 1570 : cluster 0 pgmap v1728: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 16 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s, 334 objects/s recovering 2023-10-09T05:40:07.372 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:06 smithi133 bash[105589]: audit 2023-10-09T05:40:06.481220+0000 mon.a (mon.0) 2018 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "28"}]: dispatch 2023-10-09T05:40:07.373 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:06 smithi133 bash[102858]: cluster 2023-10-09T05:40:05.470340+0000 osd.6 (osd.6) 352 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T05:40:07.373 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:06 smithi133 bash[102858]: cluster 2023-10-09T05:40:05.967899+0000 osd.6 (osd.6) 353 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T05:40:07.373 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:06 smithi133 bash[102858]: cluster 2023-10-09T05:40:05.970946+0000 osd.5 (osd.5) 386 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T05:40:07.373 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:06 smithi133 bash[102858]: cluster 2023-10-09T05:40:05.973898+0000 osd.5 (osd.5) 387 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T05:40:07.373 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:06 smithi133 bash[102858]: cluster 2023-10-09T05:40:06.189220+0000 osd.4 (osd.4) 319 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T05:40:07.373 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:06 smithi133 bash[102858]: cluster 2023-10-09T05:40:06.480425+0000 mgr.y (mgr.25277) 1570 : cluster 0 pgmap v1728: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 16 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s, 334 objects/s recovering 2023-10-09T05:40:07.373 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:06 smithi133 bash[102858]: audit 2023-10-09T05:40:06.481220+0000 mon.a (mon.0) 2018 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "28"}]: dispatch 2023-10-09T05:40:07.376 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:07.398 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:07.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:07.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:07.501 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:07.525 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:07.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:07.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:07.628 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:07.654 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:07.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:07.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:07.758 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:07.781 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:07.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:07.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:07.884 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:07.917 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:07.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:07.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:07.997 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:07 smithi133 bash[102858]: cluster 2023-10-09T05:40:06.189322+0000 osd.7 (osd.7) 339 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T05:40:07.997 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:07 smithi133 bash[102858]: cluster 2023-10-09T05:40:06.986643+0000 mon.a (mon.0) 2019 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6200/801 objects degraded (774.032%), 1 pg degraded) 2023-10-09T05:40:07.997 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:07 smithi133 bash[102858]: cluster 2023-10-09T05:40:06.996400+0000 osd.5 (osd.5) 388 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T05:40:07.997 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:07 smithi133 bash[102858]: audit 2023-10-09T05:40:07.000597+0000 mon.a (mon.0) 2020 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "28"}]': finished 2023-10-09T05:40:07.997 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:07 smithi133 bash[102858]: cluster 2023-10-09T05:40:07.000688+0000 mon.a (mon.0) 2021 : cluster 0 osdmap e372: 8 total, 8 up, 7 in 2023-10-09T05:40:08.021 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:08.050 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:08.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:08.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:08.153 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:08.175 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:08.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:08.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:08.278 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:08.288 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:07 smithi133 bash[102858]: cluster 2023-10-09T05:40:07.004607+0000 osd.6 (osd.6) 354 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T05:40:08.288 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:07 smithi133 bash[102858]: cluster 2023-10-09T05:40:07.006532+0000 osd.6 (osd.6) 355 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T05:40:08.288 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:07 smithi133 bash[102858]: cluster 2023-10-09T05:40:07.132791+0000 osd.4 (osd.4) 320 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T05:40:08.288 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:07 smithi133 bash[102858]: cluster 2023-10-09T05:40:07.261659+0000 osd.5 (osd.5) 389 : cluster 0 14.9 scrub starts 2023-10-09T05:40:08.288 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:07 smithi133 bash[102858]: cluster 2023-10-09T05:40:07.436887+0000 osd.1 (osd.1) 4 : cluster 0 14.18 scrub starts 2023-10-09T05:40:08.289 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:07 smithi133 bash[105589]: cluster 2023-10-09T05:40:06.189322+0000 osd.7 (osd.7) 339 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T05:40:08.289 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:07 smithi133 bash[105589]: cluster 2023-10-09T05:40:06.986643+0000 mon.a (mon.0) 2019 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6200/801 objects degraded (774.032%), 1 pg degraded) 2023-10-09T05:40:08.289 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:07 smithi133 bash[105589]: cluster 2023-10-09T05:40:06.996400+0000 osd.5 (osd.5) 388 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T05:40:08.289 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:07 smithi133 bash[105589]: audit 2023-10-09T05:40:07.000597+0000 mon.a (mon.0) 2020 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "28"}]': finished 2023-10-09T05:40:08.289 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:07 smithi133 bash[105589]: cluster 2023-10-09T05:40:07.000688+0000 mon.a (mon.0) 2021 : cluster 0 osdmap e372: 8 total, 8 up, 7 in 2023-10-09T05:40:08.289 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:07 smithi133 bash[105589]: cluster 2023-10-09T05:40:07.004607+0000 osd.6 (osd.6) 354 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T05:40:08.289 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:07 smithi133 bash[105589]: cluster 2023-10-09T05:40:07.006532+0000 osd.6 (osd.6) 355 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T05:40:08.289 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:07 smithi133 bash[105589]: cluster 2023-10-09T05:40:07.132791+0000 osd.4 (osd.4) 320 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T05:40:08.289 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:07 smithi133 bash[105589]: cluster 2023-10-09T05:40:07.261659+0000 osd.5 (osd.5) 389 : cluster 0 14.9 scrub starts 2023-10-09T05:40:08.289 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:07 smithi133 bash[105589]: cluster 2023-10-09T05:40:07.436887+0000 osd.1 (osd.1) 4 : cluster 0 14.18 scrub starts 2023-10-09T05:40:08.307 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:08.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:08.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:08.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:07 smithi194 bash[92666]: cluster 2023-10-09T05:40:06.189322+0000 osd.7 (osd.7) 339 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T05:40:08.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:07 smithi194 bash[92666]: cluster 2023-10-09T05:40:06.986643+0000 mon.a (mon.0) 2019 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6200/801 objects degraded (774.032%), 1 pg degraded) 2023-10-09T05:40:08.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:07 smithi194 bash[92666]: cluster 2023-10-09T05:40:06.996400+0000 osd.5 (osd.5) 388 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T05:40:08.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:07 smithi194 bash[92666]: audit 2023-10-09T05:40:07.000597+0000 mon.a (mon.0) 2020 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "28"}]': finished 2023-10-09T05:40:08.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:07 smithi194 bash[92666]: cluster 2023-10-09T05:40:07.000688+0000 mon.a (mon.0) 2021 : cluster 0 osdmap e372: 8 total, 8 up, 7 in 2023-10-09T05:40:08.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:07 smithi194 bash[92666]: cluster 2023-10-09T05:40:07.004607+0000 osd.6 (osd.6) 354 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T05:40:08.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:07 smithi194 bash[92666]: cluster 2023-10-09T05:40:07.006532+0000 osd.6 (osd.6) 355 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T05:40:08.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:07 smithi194 bash[92666]: cluster 2023-10-09T05:40:07.132791+0000 osd.4 (osd.4) 320 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T05:40:08.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:07 smithi194 bash[92666]: cluster 2023-10-09T05:40:07.261659+0000 osd.5 (osd.5) 389 : cluster 0 14.9 scrub starts 2023-10-09T05:40:08.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:07 smithi194 bash[92666]: cluster 2023-10-09T05:40:07.436887+0000 osd.1 (osd.1) 4 : cluster 0 14.18 scrub starts 2023-10-09T05:40:08.409 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:08.438 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:08.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:08.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:08.543 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:08.572 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:08.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:08.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:08.675 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:08.700 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:08.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:08.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:08.803 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:08.835 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:08.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:08.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:08.939 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:08.970 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:08.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:08.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:09.011 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:09 smithi133 bash[102858]: cluster 2023-10-09T05:40:08.003664+0000 osd.4 (osd.4) 321 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T05:40:09.011 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:09 smithi133 bash[102858]: cluster 2023-10-09T05:40:08.004458+0000 osd.5 (osd.5) 390 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T05:40:09.011 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:09 smithi133 bash[102858]: cluster 2023-10-09T05:40:08.005231+0000 mon.a (mon.0) 2022 : cluster 0 osdmap e373: 8 total, 8 up, 7 in 2023-10-09T05:40:09.011 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:09 smithi133 bash[102858]: cluster 2023-10-09T05:40:08.005894+0000 osd.6 (osd.6) 356 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T05:40:09.011 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:09 smithi133 bash[102858]: cluster 2023-10-09T05:40:08.014084+0000 osd.7 (osd.7) 340 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T05:40:09.011 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:09 smithi133 bash[102858]: cluster 2023-10-09T05:40:08.145049+0000 osd.3 (osd.3) 21 : cluster 0 14.16 scrub starts 2023-10-09T05:40:09.011 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:09 smithi133 bash[102858]: cluster 2023-10-09T05:40:08.481143+0000 mgr.y (mgr.25277) 1571 : cluster 0 pgmap v1731: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 16 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s, 334 objects/s recovering 2023-10-09T05:40:09.012 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:09 smithi133 bash[102858]: audit 2023-10-09T05:40:08.482327+0000 mon.a (mon.0) 2023 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "29"}]: dispatch 2023-10-09T05:40:09.074 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:09.086 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:09 smithi194 bash[92666]: cluster 2023-10-09T05:40:08.003664+0000 osd.4 (osd.4) 321 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T05:40:09.086 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:09 smithi194 bash[92666]: cluster 2023-10-09T05:40:08.004458+0000 osd.5 (osd.5) 390 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T05:40:09.086 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:09 smithi194 bash[92666]: cluster 2023-10-09T05:40:08.005231+0000 mon.a (mon.0) 2022 : cluster 0 osdmap e373: 8 total, 8 up, 7 in 2023-10-09T05:40:09.086 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:09 smithi194 bash[92666]: cluster 2023-10-09T05:40:08.005894+0000 osd.6 (osd.6) 356 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T05:40:09.086 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:09 smithi194 bash[92666]: cluster 2023-10-09T05:40:08.014084+0000 osd.7 (osd.7) 340 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T05:40:09.086 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:09 smithi194 bash[92666]: cluster 2023-10-09T05:40:08.145049+0000 osd.3 (osd.3) 21 : cluster 0 14.16 scrub starts 2023-10-09T05:40:09.087 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:09 smithi194 bash[92666]: cluster 2023-10-09T05:40:08.481143+0000 mgr.y (mgr.25277) 1571 : cluster 0 pgmap v1731: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 16 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s, 334 objects/s recovering 2023-10-09T05:40:09.087 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:09 smithi194 bash[92666]: audit 2023-10-09T05:40:08.482327+0000 mon.a (mon.0) 2023 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "29"}]: dispatch 2023-10-09T05:40:09.104 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:09.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:09.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:09.207 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:09.242 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:09.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:09.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:09.310 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:09 smithi133 bash[105589]: cluster 2023-10-09T05:40:08.003664+0000 osd.4 (osd.4) 321 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T05:40:09.311 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:09 smithi133 bash[105589]: cluster 2023-10-09T05:40:08.004458+0000 osd.5 (osd.5) 390 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T05:40:09.311 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:09 smithi133 bash[105589]: cluster 2023-10-09T05:40:08.005231+0000 mon.a (mon.0) 2022 : cluster 0 osdmap e373: 8 total, 8 up, 7 in 2023-10-09T05:40:09.311 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:09 smithi133 bash[105589]: cluster 2023-10-09T05:40:08.005894+0000 osd.6 (osd.6) 356 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T05:40:09.311 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:09 smithi133 bash[105589]: cluster 2023-10-09T05:40:08.014084+0000 osd.7 (osd.7) 340 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T05:40:09.311 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:09 smithi133 bash[105589]: cluster 2023-10-09T05:40:08.145049+0000 osd.3 (osd.3) 21 : cluster 0 14.16 scrub starts 2023-10-09T05:40:09.311 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:09 smithi133 bash[105589]: cluster 2023-10-09T05:40:08.481143+0000 mgr.y (mgr.25277) 1571 : cluster 0 pgmap v1731: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 16 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s, 334 objects/s recovering 2023-10-09T05:40:09.311 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:09 smithi133 bash[105589]: audit 2023-10-09T05:40:08.482327+0000 mon.a (mon.0) 2023 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "29"}]: dispatch 2023-10-09T05:40:09.345 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:09.370 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:09.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:09.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:09.476 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:09.500 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:09.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:09.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:09.603 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:09.637 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:09.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:09.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:09.741 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:09.769 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:09.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:09.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:09.873 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:09.897 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:09.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:09.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:10.000 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:10.027 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:10.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:10.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:10.132 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:10.168 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:10.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:10.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:10.271 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:10.284 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:10 smithi194 bash[92666]: cluster 2023-10-09T05:40:09.010894+0000 osd.7 (osd.7) 341 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T05:40:10.284 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:10 smithi194 bash[92666]: cluster 2023-10-09T05:40:09.013397+0000 osd.5 (osd.5) 391 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T05:40:10.284 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:10 smithi194 bash[92666]: audit 2023-10-09T05:40:09.015869+0000 mon.a (mon.0) 2024 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "29"}]': finished 2023-10-09T05:40:10.284 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:10 smithi194 bash[92666]: cluster 2023-10-09T05:40:09.015965+0000 mon.a (mon.0) 2025 : cluster 0 osdmap e374: 8 total, 8 up, 7 in 2023-10-09T05:40:10.284 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:10 smithi194 bash[92666]: cluster 2023-10-09T05:40:09.017635+0000 osd.0 (osd.0) 13 : cluster 0 14.1b starting backfill to osd.5 from (0'0,0'0] MAX to 357'10744 2023-10-09T05:40:10.285 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:10 smithi194 bash[92666]: cluster 2023-10-09T05:40:09.021603+0000 osd.6 (osd.6) 357 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T05:40:10.285 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:10 smithi194 bash[92666]: cluster 2023-10-09T05:40:09.032278+0000 osd.0 (osd.0) 14 : cluster 0 14.1b starting backfill to osd.6 from (0'0,0'0] MAX to 357'10744 2023-10-09T05:40:10.285 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:10 smithi194 bash[92666]: cluster 2023-10-09T05:40:09.039701+0000 osd.0 (osd.0) 15 : cluster 0 14.1b starting backfill to osd.7 from (0'0,0'0] MAX to 357'10744 2023-10-09T05:40:10.285 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:10 smithi194 bash[92666]: cluster 2023-10-09T05:40:09.089626+0000 osd.4 (osd.4) 322 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T05:40:10.285 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:10 smithi194 bash[92666]: cluster 2023-10-09T05:40:09.449140+0000 osd.1 (osd.1) 5 : cluster 0 14.17 scrub starts 2023-10-09T05:40:10.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:10 smithi133 bash[105589]: cluster 2023-10-09T05:40:09.010894+0000 osd.7 (osd.7) 341 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T05:40:10.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:10 smithi133 bash[105589]: cluster 2023-10-09T05:40:09.013397+0000 osd.5 (osd.5) 391 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T05:40:10.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:10 smithi133 bash[105589]: audit 2023-10-09T05:40:09.015869+0000 mon.a (mon.0) 2024 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "29"}]': finished 2023-10-09T05:40:10.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:10 smithi133 bash[105589]: cluster 2023-10-09T05:40:09.015965+0000 mon.a (mon.0) 2025 : cluster 0 osdmap e374: 8 total, 8 up, 7 in 2023-10-09T05:40:10.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:10 smithi133 bash[105589]: cluster 2023-10-09T05:40:09.017635+0000 osd.0 (osd.0) 13 : cluster 0 14.1b starting backfill to osd.5 from (0'0,0'0] MAX to 357'10744 2023-10-09T05:40:10.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:10 smithi133 bash[105589]: cluster 2023-10-09T05:40:09.021603+0000 osd.6 (osd.6) 357 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T05:40:10.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:10 smithi133 bash[105589]: cluster 2023-10-09T05:40:09.032278+0000 osd.0 (osd.0) 14 : cluster 0 14.1b starting backfill to osd.6 from (0'0,0'0] MAX to 357'10744 2023-10-09T05:40:10.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:10 smithi133 bash[105589]: cluster 2023-10-09T05:40:09.039701+0000 osd.0 (osd.0) 15 : cluster 0 14.1b starting backfill to osd.7 from (0'0,0'0] MAX to 357'10744 2023-10-09T05:40:10.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:10 smithi133 bash[105589]: cluster 2023-10-09T05:40:09.089626+0000 osd.4 (osd.4) 322 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T05:40:10.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:10 smithi133 bash[105589]: cluster 2023-10-09T05:40:09.449140+0000 osd.1 (osd.1) 5 : cluster 0 14.17 scrub starts 2023-10-09T05:40:10.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:10 smithi133 bash[102858]: cluster 2023-10-09T05:40:09.010894+0000 osd.7 (osd.7) 341 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T05:40:10.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:10 smithi133 bash[102858]: cluster 2023-10-09T05:40:09.013397+0000 osd.5 (osd.5) 391 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T05:40:10.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:10 smithi133 bash[102858]: audit 2023-10-09T05:40:09.015869+0000 mon.a (mon.0) 2024 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "29"}]': finished 2023-10-09T05:40:10.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:10 smithi133 bash[102858]: cluster 2023-10-09T05:40:09.015965+0000 mon.a (mon.0) 2025 : cluster 0 osdmap e374: 8 total, 8 up, 7 in 2023-10-09T05:40:10.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:10 smithi133 bash[102858]: cluster 2023-10-09T05:40:09.017635+0000 osd.0 (osd.0) 13 : cluster 0 14.1b starting backfill to osd.5 from (0'0,0'0] MAX to 357'10744 2023-10-09T05:40:10.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:10 smithi133 bash[102858]: cluster 2023-10-09T05:40:09.021603+0000 osd.6 (osd.6) 357 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T05:40:10.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:10 smithi133 bash[102858]: cluster 2023-10-09T05:40:09.032278+0000 osd.0 (osd.0) 14 : cluster 0 14.1b starting backfill to osd.6 from (0'0,0'0] MAX to 357'10744 2023-10-09T05:40:10.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:10 smithi133 bash[102858]: cluster 2023-10-09T05:40:09.039701+0000 osd.0 (osd.0) 15 : cluster 0 14.1b starting backfill to osd.7 from (0'0,0'0] MAX to 357'10744 2023-10-09T05:40:10.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:10 smithi133 bash[102858]: cluster 2023-10-09T05:40:09.089626+0000 osd.4 (osd.4) 322 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T05:40:10.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:10 smithi133 bash[102858]: cluster 2023-10-09T05:40:09.449140+0000 osd.1 (osd.1) 5 : cluster 0 14.17 scrub starts 2023-10-09T05:40:10.301 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:10.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:10.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:10.405 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:10.435 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:10.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:10.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:10.539 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:10.568 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:10.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:10.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:10.671 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:10.691 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:10.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:10.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:10.794 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:10.818 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:10.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:10.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:10.923 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:10.945 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:10.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:10.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:11.050 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:11.074 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:11.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:11.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:11.178 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:11.209 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:11.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:11.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:11.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:11 smithi133 bash[102858]: cluster 2023-10-09T05:40:09.296362+0000 osd.5 (osd.5) 392 : cluster 0 14.7 scrub starts 2023-10-09T05:40:11.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:11 smithi133 bash[102858]: cluster 2023-10-09T05:40:10.028045+0000 osd.6 (osd.6) 358 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T05:40:11.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:11 smithi133 bash[102858]: cluster 2023-10-09T05:40:10.028571+0000 osd.5 (osd.5) 393 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T05:40:11.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:11 smithi133 bash[102858]: cluster 2023-10-09T05:40:10.030035+0000 mon.a (mon.0) 2026 : cluster 0 osdmap e375: 8 total, 8 up, 7 in 2023-10-09T05:40:11.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:11 smithi133 bash[102858]: cluster 2023-10-09T05:40:10.036494+0000 osd.7 (osd.7) 342 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T05:40:11.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:11 smithi133 bash[102858]: cluster 2023-10-09T05:40:10.164107+0000 osd.3 (osd.3) 22 : cluster 0 14.21 scrub starts 2023-10-09T05:40:11.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:11 smithi133 bash[102858]: cluster 2023-10-09T05:40:10.481767+0000 mgr.y (mgr.25277) 1572 : cluster 0 pgmap v1734: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 16 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:11.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:11 smithi133 bash[102858]: audit 2023-10-09T05:40:10.482557+0000 mon.a (mon.0) 2027 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "30"}]: dispatch 2023-10-09T05:40:11.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:11 smithi133 bash[105589]: cluster 2023-10-09T05:40:09.296362+0000 osd.5 (osd.5) 392 : cluster 0 14.7 scrub starts 2023-10-09T05:40:11.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:11 smithi133 bash[105589]: cluster 2023-10-09T05:40:10.028045+0000 osd.6 (osd.6) 358 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T05:40:11.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:11 smithi133 bash[105589]: cluster 2023-10-09T05:40:10.028571+0000 osd.5 (osd.5) 393 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T05:40:11.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:11 smithi133 bash[105589]: cluster 2023-10-09T05:40:10.030035+0000 mon.a (mon.0) 2026 : cluster 0 osdmap e375: 8 total, 8 up, 7 in 2023-10-09T05:40:11.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:11 smithi133 bash[105589]: cluster 2023-10-09T05:40:10.036494+0000 osd.7 (osd.7) 342 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T05:40:11.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:11 smithi133 bash[105589]: cluster 2023-10-09T05:40:10.164107+0000 osd.3 (osd.3) 22 : cluster 0 14.21 scrub starts 2023-10-09T05:40:11.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:11 smithi133 bash[105589]: cluster 2023-10-09T05:40:10.481767+0000 mgr.y (mgr.25277) 1572 : cluster 0 pgmap v1734: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 16 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:11.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:11 smithi133 bash[105589]: audit 2023-10-09T05:40:10.482557+0000 mon.a (mon.0) 2027 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "30"}]: dispatch 2023-10-09T05:40:11.313 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:11.343 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:11.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:11.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:11.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:11 smithi194 bash[92666]: cluster 2023-10-09T05:40:09.296362+0000 osd.5 (osd.5) 392 : cluster 0 14.7 scrub starts 2023-10-09T05:40:11.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:11 smithi194 bash[92666]: cluster 2023-10-09T05:40:10.028045+0000 osd.6 (osd.6) 358 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T05:40:11.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:11 smithi194 bash[92666]: cluster 2023-10-09T05:40:10.028571+0000 osd.5 (osd.5) 393 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T05:40:11.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:11 smithi194 bash[92666]: cluster 2023-10-09T05:40:10.030035+0000 mon.a (mon.0) 2026 : cluster 0 osdmap e375: 8 total, 8 up, 7 in 2023-10-09T05:40:11.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:11 smithi194 bash[92666]: cluster 2023-10-09T05:40:10.036494+0000 osd.7 (osd.7) 342 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T05:40:11.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:11 smithi194 bash[92666]: cluster 2023-10-09T05:40:10.164107+0000 osd.3 (osd.3) 22 : cluster 0 14.21 scrub starts 2023-10-09T05:40:11.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:11 smithi194 bash[92666]: cluster 2023-10-09T05:40:10.481767+0000 mgr.y (mgr.25277) 1572 : cluster 0 pgmap v1734: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 16 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:11.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:11 smithi194 bash[92666]: audit 2023-10-09T05:40:10.482557+0000 mon.a (mon.0) 2027 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "30"}]: dispatch 2023-10-09T05:40:11.447 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:11.474 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:11.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:11.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:11.577 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:11.597 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:11.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:11.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:11.700 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:11.723 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:11.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:11.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:11.828 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:11.849 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:11.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:11.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:11.952 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:11.987 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:11.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:11.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:12.091 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:12.101 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:12 smithi133 bash[102858]: cluster 2023-10-09T05:40:10.341837+0000 osd.5 (osd.5) 394 : cluster 0 14.a scrub starts 2023-10-09T05:40:12.101 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:12 smithi133 bash[102858]: cluster 2023-10-09T05:40:10.359237+0000 osd.5 (osd.5) 395 : cluster 0 14.a scrub ok 2023-10-09T05:40:12.101 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:12 smithi133 bash[102858]: cluster 2023-10-09T05:40:11.030673+0000 osd.6 (osd.6) 359 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T05:40:12.101 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:12 smithi133 bash[102858]: audit 2023-10-09T05:40:11.033652+0000 mon.a (mon.0) 2028 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "30"}]': finished 2023-10-09T05:40:12.101 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:12 smithi133 bash[102858]: cluster 2023-10-09T05:40:11.033709+0000 mon.a (mon.0) 2029 : cluster 0 osdmap e376: 8 total, 8 up, 7 in 2023-10-09T05:40:12.101 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:12 smithi133 bash[102858]: cluster 2023-10-09T05:40:11.037352+0000 osd.0 (osd.0) 16 : cluster 0 14.1c starting backfill to osd.3 from (0'0,0'0] MAX to 357'10732 2023-10-09T05:40:12.101 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:12 smithi133 bash[102858]: cluster 2023-10-09T05:40:11.041335+0000 osd.5 (osd.5) 396 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T05:40:12.101 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:12 smithi133 bash[102858]: cluster 2023-10-09T05:40:11.049723+0000 osd.0 (osd.0) 17 : cluster 0 14.1c starting backfill to osd.7 from (0'0,0'0] MAX to 357'10732 2023-10-09T05:40:12.101 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:12 smithi133 bash[102858]: cluster 2023-10-09T05:40:11.128094+0000 osd.3 (osd.3) 23 : cluster 0 14.16 scrub starts 2023-10-09T05:40:12.101 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:12 smithi133 bash[102858]: cluster 2023-10-09T05:40:11.202839+0000 osd.4 (osd.4) 323 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T05:40:12.102 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:12 smithi133 bash[102858]: cluster 2023-10-09T05:40:11.204717+0000 osd.4 (osd.4) 324 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T05:40:12.102 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:12 smithi133 bash[105589]: cluster 2023-10-09T05:40:10.341837+0000 osd.5 (osd.5) 394 : cluster 0 14.a scrub starts 2023-10-09T05:40:12.102 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:12 smithi133 bash[105589]: cluster 2023-10-09T05:40:10.359237+0000 osd.5 (osd.5) 395 : cluster 0 14.a scrub ok 2023-10-09T05:40:12.102 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:12 smithi133 bash[105589]: cluster 2023-10-09T05:40:11.030673+0000 osd.6 (osd.6) 359 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T05:40:12.102 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:12 smithi133 bash[105589]: audit 2023-10-09T05:40:11.033652+0000 mon.a (mon.0) 2028 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "30"}]': finished 2023-10-09T05:40:12.102 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:12 smithi133 bash[105589]: cluster 2023-10-09T05:40:11.033709+0000 mon.a (mon.0) 2029 : cluster 0 osdmap e376: 8 total, 8 up, 7 in 2023-10-09T05:40:12.102 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:12 smithi133 bash[105589]: cluster 2023-10-09T05:40:11.037352+0000 osd.0 (osd.0) 16 : cluster 0 14.1c starting backfill to osd.3 from (0'0,0'0] MAX to 357'10732 2023-10-09T05:40:12.103 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:12 smithi133 bash[105589]: cluster 2023-10-09T05:40:11.041335+0000 osd.5 (osd.5) 396 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T05:40:12.103 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:12 smithi133 bash[105589]: cluster 2023-10-09T05:40:11.049723+0000 osd.0 (osd.0) 17 : cluster 0 14.1c starting backfill to osd.7 from (0'0,0'0] MAX to 357'10732 2023-10-09T05:40:12.103 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:12 smithi133 bash[105589]: cluster 2023-10-09T05:40:11.128094+0000 osd.3 (osd.3) 23 : cluster 0 14.16 scrub starts 2023-10-09T05:40:12.103 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:12 smithi133 bash[105589]: cluster 2023-10-09T05:40:11.202839+0000 osd.4 (osd.4) 323 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T05:40:12.103 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:12 smithi133 bash[105589]: cluster 2023-10-09T05:40:11.204717+0000 osd.4 (osd.4) 324 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T05:40:12.119 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:12.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:12.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:12.223 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:12.247 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:12.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:12.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:12.350 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:12.374 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:12.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:12.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:12.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:12 smithi194 bash[92666]: cluster 2023-10-09T05:40:10.341837+0000 osd.5 (osd.5) 394 : cluster 0 14.a scrub starts 2023-10-09T05:40:12.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:12 smithi194 bash[92666]: cluster 2023-10-09T05:40:10.359237+0000 osd.5 (osd.5) 395 : cluster 0 14.a scrub ok 2023-10-09T05:40:12.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:12 smithi194 bash[92666]: cluster 2023-10-09T05:40:11.030673+0000 osd.6 (osd.6) 359 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T05:40:12.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:12 smithi194 bash[92666]: audit 2023-10-09T05:40:11.033652+0000 mon.a (mon.0) 2028 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "30"}]': finished 2023-10-09T05:40:12.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:12 smithi194 bash[92666]: cluster 2023-10-09T05:40:11.033709+0000 mon.a (mon.0) 2029 : cluster 0 osdmap e376: 8 total, 8 up, 7 in 2023-10-09T05:40:12.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:12 smithi194 bash[92666]: cluster 2023-10-09T05:40:11.037352+0000 osd.0 (osd.0) 16 : cluster 0 14.1c starting backfill to osd.3 from (0'0,0'0] MAX to 357'10732 2023-10-09T05:40:12.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:12 smithi194 bash[92666]: cluster 2023-10-09T05:40:11.041335+0000 osd.5 (osd.5) 396 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T05:40:12.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:12 smithi194 bash[92666]: cluster 2023-10-09T05:40:11.049723+0000 osd.0 (osd.0) 17 : cluster 0 14.1c starting backfill to osd.7 from (0'0,0'0] MAX to 357'10732 2023-10-09T05:40:12.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:12 smithi194 bash[92666]: cluster 2023-10-09T05:40:11.128094+0000 osd.3 (osd.3) 23 : cluster 0 14.16 scrub starts 2023-10-09T05:40:12.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:12 smithi194 bash[92666]: cluster 2023-10-09T05:40:11.202839+0000 osd.4 (osd.4) 323 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T05:40:12.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:12 smithi194 bash[92666]: cluster 2023-10-09T05:40:11.204717+0000 osd.4 (osd.4) 324 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T05:40:12.477 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:12.506 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:12.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:12.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:12.609 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:12.641 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:12.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:12.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:12.745 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:12.777 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:12.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:12.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:12.881 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:12.901 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:12.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:12.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:13.005 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:13.036 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:13.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:13.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:13.140 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:13.170 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:13.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:13.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:13.273 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:13.280 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:13 smithi133 bash[105589]: cluster 2023-10-09T05:40:11.152220+0000 osd.7 (osd.7) 343 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T05:40:13.280 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:13 smithi133 bash[105589]: cluster 2023-10-09T05:40:11.452542+0000 osd.1 (osd.1) 6 : cluster 0 14.18 scrub starts 2023-10-09T05:40:13.280 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:13 smithi133 bash[105589]: cluster 2023-10-09T05:40:12.033338+0000 osd.4 (osd.4) 325 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T05:40:13.280 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:13 smithi133 bash[105589]: cluster 2023-10-09T05:40:12.035476+0000 mon.a (mon.0) 2030 : cluster 0 osdmap e377: 8 total, 8 up, 7 in 2023-10-09T05:40:13.280 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:13 smithi133 bash[105589]: cluster 2023-10-09T05:40:12.039075+0000 osd.5 (osd.5) 397 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T05:40:13.280 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:13 smithi133 bash[105589]: cluster 2023-10-09T05:40:12.113636+0000 osd.3 (osd.3) 24 : cluster 0 14.21 scrub starts 2023-10-09T05:40:13.280 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:13 smithi133 bash[105589]: cluster 2023-10-09T05:40:12.431403+0000 osd.1 (osd.1) 7 : cluster 0 14.17 scrub starts 2023-10-09T05:40:13.281 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:13 smithi133 bash[105589]: cluster 2023-10-09T05:40:12.482296+0000 mgr.y (mgr.25277) 1573 : cluster 0 pgmap v1737: 109 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 16 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 15/801 objects misplaced (1.873%); 0 B/s, 171 objects/s recovering 2023-10-09T05:40:13.281 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:13 smithi133 bash[102858]: cluster 2023-10-09T05:40:11.152220+0000 osd.7 (osd.7) 343 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T05:40:13.281 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:13 smithi133 bash[102858]: cluster 2023-10-09T05:40:11.452542+0000 osd.1 (osd.1) 6 : cluster 0 14.18 scrub starts 2023-10-09T05:40:13.281 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:13 smithi133 bash[102858]: cluster 2023-10-09T05:40:12.033338+0000 osd.4 (osd.4) 325 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T05:40:13.281 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:13 smithi133 bash[102858]: cluster 2023-10-09T05:40:12.035476+0000 mon.a (mon.0) 2030 : cluster 0 osdmap e377: 8 total, 8 up, 7 in 2023-10-09T05:40:13.281 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:13 smithi133 bash[102858]: cluster 2023-10-09T05:40:12.039075+0000 osd.5 (osd.5) 397 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T05:40:13.281 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:13 smithi133 bash[102858]: cluster 2023-10-09T05:40:12.113636+0000 osd.3 (osd.3) 24 : cluster 0 14.21 scrub starts 2023-10-09T05:40:13.281 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:13 smithi133 bash[102858]: cluster 2023-10-09T05:40:12.431403+0000 osd.1 (osd.1) 7 : cluster 0 14.17 scrub starts 2023-10-09T05:40:13.282 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:13 smithi133 bash[102858]: cluster 2023-10-09T05:40:12.482296+0000 mgr.y (mgr.25277) 1573 : cluster 0 pgmap v1737: 109 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 16 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 15/801 objects misplaced (1.873%); 0 B/s, 171 objects/s recovering 2023-10-09T05:40:13.293 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:13.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:13.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:13.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:13 smithi194 bash[92666]: cluster 2023-10-09T05:40:11.152220+0000 osd.7 (osd.7) 343 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T05:40:13.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:13 smithi194 bash[92666]: cluster 2023-10-09T05:40:11.452542+0000 osd.1 (osd.1) 6 : cluster 0 14.18 scrub starts 2023-10-09T05:40:13.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:13 smithi194 bash[92666]: cluster 2023-10-09T05:40:12.033338+0000 osd.4 (osd.4) 325 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T05:40:13.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:13 smithi194 bash[92666]: cluster 2023-10-09T05:40:12.035476+0000 mon.a (mon.0) 2030 : cluster 0 osdmap e377: 8 total, 8 up, 7 in 2023-10-09T05:40:13.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:13 smithi194 bash[92666]: cluster 2023-10-09T05:40:12.039075+0000 osd.5 (osd.5) 397 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T05:40:13.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:13 smithi194 bash[92666]: cluster 2023-10-09T05:40:12.113636+0000 osd.3 (osd.3) 24 : cluster 0 14.21 scrub starts 2023-10-09T05:40:13.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:13 smithi194 bash[92666]: cluster 2023-10-09T05:40:12.431403+0000 osd.1 (osd.1) 7 : cluster 0 14.17 scrub starts 2023-10-09T05:40:13.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:13 smithi194 bash[92666]: cluster 2023-10-09T05:40:12.482296+0000 mgr.y (mgr.25277) 1573 : cluster 0 pgmap v1737: 109 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 16 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 15/801 objects misplaced (1.873%); 0 B/s, 171 objects/s recovering 2023-10-09T05:40:13.396 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:13.425 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:13.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:13.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:13.529 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:13.557 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:13.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:13.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:13.660 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:13.685 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:13.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:13.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:13.787 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:13.808 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:13.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:13.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:13.912 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:13.941 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:13.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:13.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:14.045 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:14.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:14 smithi133 bash[102858]: cluster 2023-10-09T05:40:12.046261+0000 osd.6 (osd.6) 360 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T05:40:14.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:14 smithi133 bash[102858]: cluster 2023-10-09T05:40:12.372480+0000 osd.5 (osd.5) 398 : cluster 0 14.9 scrub starts 2023-10-09T05:40:14.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:14 smithi133 bash[102858]: cluster 2023-10-09T05:40:12.434893+0000 osd.7 (osd.7) 344 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T05:40:14.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:14 smithi133 bash[102858]: cluster 2023-10-09T05:40:13.038996+0000 mon.a (mon.0) 2031 : cluster 3 Health check failed: Degraded data redundancy: 1 pg degraded (PG_DEGRADED) 2023-10-09T05:40:14.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:14 smithi133 bash[102858]: cluster 2023-10-09T05:40:13.047271+0000 osd.6 (osd.6) 361 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T05:40:14.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:14 smithi133 bash[102858]: cluster 2023-10-09T05:40:13.049632+0000 osd.2 (osd.2) 12 : cluster 0 14.1d starting backfill to osd.0 from (0'0,0'0] MAX to 357'10792 2023-10-09T05:40:14.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:14 smithi133 bash[102858]: cluster 2023-10-09T05:40:13.053133+0000 mon.a (mon.0) 2032 : cluster 0 osdmap e378: 8 total, 8 up, 7 in 2023-10-09T05:40:14.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:14 smithi133 bash[102858]: cluster 2023-10-09T05:40:13.063604+0000 osd.2 (osd.2) 13 : cluster 0 14.1d starting backfill to osd.3 from (0'0,0'0] MAX to 357'10792 2023-10-09T05:40:14.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:14 smithi133 bash[102858]: cluster 2023-10-09T05:40:13.078240+0000 osd.5 (osd.5) 399 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T05:40:14.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:14 smithi133 bash[102858]: audit 2023-10-09T05:40:13.498970+0000 mon.a (mon.0) 2033 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:40:14.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:14 smithi133 bash[102858]: audit 2023-10-09T05:40:13.499950+0000 mon.a (mon.0) 2034 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:14.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:14 smithi133 bash[102858]: cluster 2023-10-09T05:40:13.962224+0000 mon.a (mon.0) 2035 : cluster 0 osdmap e379: 8 total, 8 up, 7 in 2023-10-09T05:40:14.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:14 smithi133 bash[102858]: cluster 2023-10-09T05:40:13.972475+0000 osd.6 (osd.6) 362 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T05:40:14.074 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:14.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:14.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:14.178 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:14.205 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:14.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:14.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:14.297 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:40:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:40:14] "GET /metrics HTTP/1.1" 200 36488 "" "Prometheus/2.43.0" 2023-10-09T05:40:14.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:14 smithi133 bash[105589]: cluster 2023-10-09T05:40:12.046261+0000 osd.6 (osd.6) 360 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T05:40:14.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:14 smithi133 bash[105589]: cluster 2023-10-09T05:40:12.372480+0000 osd.5 (osd.5) 398 : cluster 0 14.9 scrub starts 2023-10-09T05:40:14.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:14 smithi133 bash[105589]: cluster 2023-10-09T05:40:12.434893+0000 osd.7 (osd.7) 344 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T05:40:14.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:14 smithi133 bash[105589]: cluster 2023-10-09T05:40:13.038996+0000 mon.a (mon.0) 2031 : cluster 3 Health check failed: Degraded data redundancy: 1 pg degraded (PG_DEGRADED) 2023-10-09T05:40:14.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:14 smithi133 bash[105589]: cluster 2023-10-09T05:40:13.047271+0000 osd.6 (osd.6) 361 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T05:40:14.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:14 smithi133 bash[105589]: cluster 2023-10-09T05:40:13.049632+0000 osd.2 (osd.2) 12 : cluster 0 14.1d starting backfill to osd.0 from (0'0,0'0] MAX to 357'10792 2023-10-09T05:40:14.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:14 smithi133 bash[105589]: cluster 2023-10-09T05:40:13.053133+0000 mon.a (mon.0) 2032 : cluster 0 osdmap e378: 8 total, 8 up, 7 in 2023-10-09T05:40:14.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:14 smithi133 bash[105589]: cluster 2023-10-09T05:40:13.063604+0000 osd.2 (osd.2) 13 : cluster 0 14.1d starting backfill to osd.3 from (0'0,0'0] MAX to 357'10792 2023-10-09T05:40:14.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:14 smithi133 bash[105589]: cluster 2023-10-09T05:40:13.078240+0000 osd.5 (osd.5) 399 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T05:40:14.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:14 smithi133 bash[105589]: audit 2023-10-09T05:40:13.498970+0000 mon.a (mon.0) 2033 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:40:14.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:14 smithi133 bash[105589]: audit 2023-10-09T05:40:13.499950+0000 mon.a (mon.0) 2034 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:14.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:14 smithi133 bash[105589]: cluster 2023-10-09T05:40:13.962224+0000 mon.a (mon.0) 2035 : cluster 0 osdmap e379: 8 total, 8 up, 7 in 2023-10-09T05:40:14.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:14 smithi133 bash[105589]: cluster 2023-10-09T05:40:13.972475+0000 osd.6 (osd.6) 362 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T05:40:14.308 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:14.330 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:14.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:14.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:14.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:14 smithi194 bash[92666]: cluster 2023-10-09T05:40:12.046261+0000 osd.6 (osd.6) 360 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T05:40:14.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:14 smithi194 bash[92666]: cluster 2023-10-09T05:40:12.372480+0000 osd.5 (osd.5) 398 : cluster 0 14.9 scrub starts 2023-10-09T05:40:14.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:14 smithi194 bash[92666]: cluster 2023-10-09T05:40:12.434893+0000 osd.7 (osd.7) 344 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T05:40:14.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:14 smithi194 bash[92666]: cluster 2023-10-09T05:40:13.038996+0000 mon.a (mon.0) 2031 : cluster 3 Health check failed: Degraded data redundancy: 1 pg degraded (PG_DEGRADED) 2023-10-09T05:40:14.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:14 smithi194 bash[92666]: cluster 2023-10-09T05:40:13.047271+0000 osd.6 (osd.6) 361 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T05:40:14.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:14 smithi194 bash[92666]: cluster 2023-10-09T05:40:13.049632+0000 osd.2 (osd.2) 12 : cluster 0 14.1d starting backfill to osd.0 from (0'0,0'0] MAX to 357'10792 2023-10-09T05:40:14.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:14 smithi194 bash[92666]: cluster 2023-10-09T05:40:13.053133+0000 mon.a (mon.0) 2032 : cluster 0 osdmap e378: 8 total, 8 up, 7 in 2023-10-09T05:40:14.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:14 smithi194 bash[92666]: cluster 2023-10-09T05:40:13.063604+0000 osd.2 (osd.2) 13 : cluster 0 14.1d starting backfill to osd.3 from (0'0,0'0] MAX to 357'10792 2023-10-09T05:40:14.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:14 smithi194 bash[92666]: cluster 2023-10-09T05:40:13.078240+0000 osd.5 (osd.5) 399 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T05:40:14.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:14 smithi194 bash[92666]: audit 2023-10-09T05:40:13.498970+0000 mon.a (mon.0) 2033 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:40:14.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:14 smithi194 bash[92666]: audit 2023-10-09T05:40:13.499950+0000 mon.a (mon.0) 2034 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:14.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:14 smithi194 bash[92666]: cluster 2023-10-09T05:40:13.962224+0000 mon.a (mon.0) 2035 : cluster 0 osdmap e379: 8 total, 8 up, 7 in 2023-10-09T05:40:14.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:14 smithi194 bash[92666]: cluster 2023-10-09T05:40:13.972475+0000 osd.6 (osd.6) 362 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T05:40:14.433 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:14.455 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:14.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:14.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:14.558 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:14.591 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:14.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:14.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:14.694 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:14.727 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:14.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:14.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:14.831 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:14.866 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:14.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:14.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:14.970 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:14.971 INFO:teuthology.orchestra.run.smithi133.stdout:moved 1 / 327 (0.30581%) 2023-10-09T05:40:14.972 INFO:teuthology.orchestra.run.smithi133.stdout:avg 46.7143 2023-10-09T05:40:14.972 INFO:teuthology.orchestra.run.smithi133.stdout:stddev 5.06287 -> 5.39085 (expected baseline 6.32778) 2023-10-09T05:40:14.972 INFO:teuthology.orchestra.run.smithi133.stdout:min osd.5 with 38 -> 38 pgs (0.813456 -> 0.813456 * mean) 2023-10-09T05:40:14.972 INFO:teuthology.orchestra.run.smithi133.stdout:max osd.3 with 55 -> 56 pgs (1.17737 -> 1.19878 * mean) 2023-10-09T05:40:14.972 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:40:14.972 INFO:teuthology.orchestra.run.smithi133.stdout:oload 110 2023-10-09T05:40:14.972 INFO:teuthology.orchestra.run.smithi133.stdout:max_change 0.05 2023-10-09T05:40:14.972 INFO:teuthology.orchestra.run.smithi133.stdout:max_change_osds 4 2023-10-09T05:40:14.972 INFO:teuthology.orchestra.run.smithi133.stdout:average_utilization 0.0136 2023-10-09T05:40:14.972 INFO:teuthology.orchestra.run.smithi133.stdout:overload_utilization 0.0150 2023-10-09T05:40:14.973 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6 weight 1.0000 -> 0.9500 2023-10-09T05:40:14.973 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7 weight 1.0000 -> 0.9500 2023-10-09T05:40:14.998 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:15.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:15.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:15.103 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:15.115 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:15 smithi194 bash[92666]: cluster 2023-10-09T05:40:13.434619+0000 osd.4 (osd.4) 326 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T05:40:15.116 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:15 smithi194 bash[92666]: cluster 2023-10-09T05:40:13.434658+0000 osd.7 (osd.7) 345 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T05:40:15.116 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:15 smithi194 bash[92666]: cluster 2023-10-09T05:40:13.959625+0000 osd.7 (osd.7) 346 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T05:40:15.116 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:15 smithi194 bash[92666]: cluster 2023-10-09T05:40:13.969375+0000 osd.5 (osd.5) 400 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T05:40:15.116 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:15 smithi194 bash[92666]: cluster 2023-10-09T05:40:14.118512+0000 osd.3 (osd.3) 25 : cluster 0 14.16 scrub starts 2023-10-09T05:40:15.116 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:15 smithi194 bash[92666]: audit 2023-10-09T05:40:14.299996+0000 mgr.y (mgr.25277) 1574 : audit 0 from='client.55417 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:40:15.116 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:15 smithi194 bash[92666]: audit 2023-10-09T05:40:14.300993+0000 mon.a (mon.0) 2036 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'62260','7':'62260'}"}]: dispatch 2023-10-09T05:40:15.116 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:15 smithi194 bash[92666]: cluster 2023-10-09T05:40:14.482835+0000 mgr.y (mgr.25277) 1575 : cluster 0 pgmap v1740: 109 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 16 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 15/801 objects misplaced (1.873%); 0 B/s, 171 objects/s recovering 2023-10-09T05:40:15.116 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:15 smithi194 bash[92666]: cluster 2023-10-09T05:40:14.967496+0000 osd.6 (osd.6) 363 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T05:40:15.117 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:15 smithi194 bash[92666]: audit 2023-10-09T05:40:14.969866+0000 mon.a (mon.0) 2037 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'6':'62260','7':'62260'}"}]': finished 2023-10-09T05:40:15.117 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:15 smithi194 bash[92666]: cluster 2023-10-09T05:40:14.970029+0000 mon.a (mon.0) 2038 : cluster 0 osdmap e380: 8 total, 8 up, 7 in 2023-10-09T05:40:15.135 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:15.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:15.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:15.240 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:15.271 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:15.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:15.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:15.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:15 smithi133 bash[102858]: cluster 2023-10-09T05:40:13.434619+0000 osd.4 (osd.4) 326 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T05:40:15.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:15 smithi133 bash[102858]: cluster 2023-10-09T05:40:13.434658+0000 osd.7 (osd.7) 345 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T05:40:15.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:15 smithi133 bash[102858]: cluster 2023-10-09T05:40:13.959625+0000 osd.7 (osd.7) 346 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T05:40:15.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:15 smithi133 bash[102858]: cluster 2023-10-09T05:40:13.969375+0000 osd.5 (osd.5) 400 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T05:40:15.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:15 smithi133 bash[102858]: cluster 2023-10-09T05:40:14.118512+0000 osd.3 (osd.3) 25 : cluster 0 14.16 scrub starts 2023-10-09T05:40:15.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:15 smithi133 bash[102858]: audit 2023-10-09T05:40:14.299996+0000 mgr.y (mgr.25277) 1574 : audit 0 from='client.55417 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:40:15.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:15 smithi133 bash[102858]: audit 2023-10-09T05:40:14.300993+0000 mon.a (mon.0) 2036 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'62260','7':'62260'}"}]: dispatch 2023-10-09T05:40:15.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:15 smithi133 bash[102858]: cluster 2023-10-09T05:40:14.482835+0000 mgr.y (mgr.25277) 1575 : cluster 0 pgmap v1740: 109 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 16 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 15/801 objects misplaced (1.873%); 0 B/s, 171 objects/s recovering 2023-10-09T05:40:15.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:15 smithi133 bash[102858]: cluster 2023-10-09T05:40:14.967496+0000 osd.6 (osd.6) 363 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T05:40:15.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:15 smithi133 bash[102858]: audit 2023-10-09T05:40:14.969866+0000 mon.a (mon.0) 2037 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'6':'62260','7':'62260'}"}]': finished 2023-10-09T05:40:15.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:15 smithi133 bash[102858]: cluster 2023-10-09T05:40:14.970029+0000 mon.a (mon.0) 2038 : cluster 0 osdmap e380: 8 total, 8 up, 7 in 2023-10-09T05:40:15.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:15 smithi133 bash[105589]: cluster 2023-10-09T05:40:13.434619+0000 osd.4 (osd.4) 326 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T05:40:15.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:15 smithi133 bash[105589]: cluster 2023-10-09T05:40:13.434658+0000 osd.7 (osd.7) 345 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T05:40:15.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:15 smithi133 bash[105589]: cluster 2023-10-09T05:40:13.959625+0000 osd.7 (osd.7) 346 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T05:40:15.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:15 smithi133 bash[105589]: cluster 2023-10-09T05:40:13.969375+0000 osd.5 (osd.5) 400 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T05:40:15.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:15 smithi133 bash[105589]: cluster 2023-10-09T05:40:14.118512+0000 osd.3 (osd.3) 25 : cluster 0 14.16 scrub starts 2023-10-09T05:40:15.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:15 smithi133 bash[105589]: audit 2023-10-09T05:40:14.299996+0000 mgr.y (mgr.25277) 1574 : audit 0 from='client.55417 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:40:15.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:15 smithi133 bash[105589]: audit 2023-10-09T05:40:14.300993+0000 mon.a (mon.0) 2036 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'62260','7':'62260'}"}]: dispatch 2023-10-09T05:40:15.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:15 smithi133 bash[105589]: cluster 2023-10-09T05:40:14.482835+0000 mgr.y (mgr.25277) 1575 : cluster 0 pgmap v1740: 109 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 16 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 15/801 objects misplaced (1.873%); 0 B/s, 171 objects/s recovering 2023-10-09T05:40:15.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:15 smithi133 bash[105589]: cluster 2023-10-09T05:40:14.967496+0000 osd.6 (osd.6) 363 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T05:40:15.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:15 smithi133 bash[105589]: audit 2023-10-09T05:40:14.969866+0000 mon.a (mon.0) 2037 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'6':'62260','7':'62260'}"}]': finished 2023-10-09T05:40:15.300 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:15 smithi133 bash[105589]: cluster 2023-10-09T05:40:14.970029+0000 mon.a (mon.0) 2038 : cluster 0 osdmap e380: 8 total, 8 up, 7 in 2023-10-09T05:40:15.374 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:15.453 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:15.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:15.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:15.556 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:15.591 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:15.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:15.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:15.695 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:15.718 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:15.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:15.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:15.824 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:15.858 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:15.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:15.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:15.962 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:15.994 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:15.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:15.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:16.098 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:16.106 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:16 smithi194 bash[92666]: cluster 2023-10-09T05:40:15.096198+0000 osd.3 (osd.3) 26 : cluster 0 14.21 scrub starts 2023-10-09T05:40:16.106 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:16 smithi194 bash[92666]: cluster 2023-10-09T05:40:15.972567+0000 mon.a (mon.0) 2039 : cluster 0 osdmap e381: 8 total, 8 up, 7 in 2023-10-09T05:40:16.106 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:16 smithi194 bash[92666]: cluster 2023-10-09T05:40:15.975304+0000 osd.6 (osd.6) 364 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T05:40:16.124 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:16.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:16.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:16.227 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:16.255 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:16.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:16.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:16.358 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:16.365 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:16 smithi133 bash[105589]: cluster 2023-10-09T05:40:15.096198+0000 osd.3 (osd.3) 26 : cluster 0 14.21 scrub starts 2023-10-09T05:40:16.365 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:16 smithi133 bash[105589]: cluster 2023-10-09T05:40:15.972567+0000 mon.a (mon.0) 2039 : cluster 0 osdmap e381: 8 total, 8 up, 7 in 2023-10-09T05:40:16.365 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:16 smithi133 bash[105589]: cluster 2023-10-09T05:40:15.975304+0000 osd.6 (osd.6) 364 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T05:40:16.366 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:16 smithi133 bash[102858]: cluster 2023-10-09T05:40:15.096198+0000 osd.3 (osd.3) 26 : cluster 0 14.21 scrub starts 2023-10-09T05:40:16.366 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:16 smithi133 bash[102858]: cluster 2023-10-09T05:40:15.972567+0000 mon.a (mon.0) 2039 : cluster 0 osdmap e381: 8 total, 8 up, 7 in 2023-10-09T05:40:16.366 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:16 smithi133 bash[102858]: cluster 2023-10-09T05:40:15.975304+0000 osd.6 (osd.6) 364 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T05:40:16.379 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:16.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:16.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:16.482 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:16.505 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:16.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:16.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:16.609 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:16.638 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:16.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:16.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:16.742 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:16.772 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:16.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:16.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:16.875 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:16.904 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:16.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:16.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:17.008 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:17.034 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:17.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:17.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:17.137 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:17.159 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:17.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:17.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:17.263 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:17.274 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:17 smithi194 bash[92666]: cluster 2023-10-09T05:40:15.365365+0000 osd.5 (osd.5) 401 : cluster 0 14.7 scrub starts 2023-10-09T05:40:17.274 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:17 smithi194 bash[92666]: cluster 2023-10-09T05:40:16.064831+0000 osd.3 (osd.3) 27 : cluster 0 14.16 scrub starts 2023-10-09T05:40:17.274 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:17 smithi194 bash[92666]: cluster 2023-10-09T05:40:16.065611+0000 osd.5 (osd.5) 402 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T05:40:17.275 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:17 smithi194 bash[92666]: cluster 2023-10-09T05:40:16.067683+0000 osd.5 (osd.5) 403 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T05:40:17.275 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:17 smithi194 bash[92666]: cluster 2023-10-09T05:40:16.090627+0000 osd.4 (osd.4) 327 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T05:40:17.275 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:17 smithi194 bash[92666]: cluster 2023-10-09T05:40:16.090735+0000 osd.7 (osd.7) 347 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T05:40:17.275 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:17 smithi194 bash[92666]: cluster 2023-10-09T05:40:16.414899+0000 osd.1 (osd.1) 8 : cluster 0 14.18 scrub starts 2023-10-09T05:40:17.275 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:17 smithi194 bash[92666]: cluster 2023-10-09T05:40:16.483481+0000 mgr.y (mgr.25277) 1576 : cluster 0 pgmap v1743: 109 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 16 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 15/801 objects misplaced (1.873%); 0 B/s, 208 objects/s recovering 2023-10-09T05:40:17.275 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:17 smithi194 bash[92666]: audit 2023-10-09T05:40:16.484941+0000 mon.a (mon.0) 2040 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-10-09T05:40:17.290 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:17.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:17.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:17.297 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:17 smithi133 bash[102858]: cluster 2023-10-09T05:40:15.365365+0000 osd.5 (osd.5) 401 : cluster 0 14.7 scrub starts 2023-10-09T05:40:17.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:17 smithi133 bash[102858]: cluster 2023-10-09T05:40:16.064831+0000 osd.3 (osd.3) 27 : cluster 0 14.16 scrub starts 2023-10-09T05:40:17.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:17 smithi133 bash[102858]: cluster 2023-10-09T05:40:16.065611+0000 osd.5 (osd.5) 402 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T05:40:17.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:17 smithi133 bash[102858]: cluster 2023-10-09T05:40:16.067683+0000 osd.5 (osd.5) 403 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T05:40:17.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:17 smithi133 bash[102858]: cluster 2023-10-09T05:40:16.090627+0000 osd.4 (osd.4) 327 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T05:40:17.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:17 smithi133 bash[102858]: cluster 2023-10-09T05:40:16.090735+0000 osd.7 (osd.7) 347 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T05:40:17.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:17 smithi133 bash[102858]: cluster 2023-10-09T05:40:16.414899+0000 osd.1 (osd.1) 8 : cluster 0 14.18 scrub starts 2023-10-09T05:40:17.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:17 smithi133 bash[102858]: cluster 2023-10-09T05:40:16.483481+0000 mgr.y (mgr.25277) 1576 : cluster 0 pgmap v1743: 109 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 16 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 15/801 objects misplaced (1.873%); 0 B/s, 208 objects/s recovering 2023-10-09T05:40:17.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:17 smithi133 bash[102858]: audit 2023-10-09T05:40:16.484941+0000 mon.a (mon.0) 2040 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-10-09T05:40:17.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:17 smithi133 bash[105589]: cluster 2023-10-09T05:40:15.365365+0000 osd.5 (osd.5) 401 : cluster 0 14.7 scrub starts 2023-10-09T05:40:17.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:17 smithi133 bash[105589]: cluster 2023-10-09T05:40:16.064831+0000 osd.3 (osd.3) 27 : cluster 0 14.16 scrub starts 2023-10-09T05:40:17.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:17 smithi133 bash[105589]: cluster 2023-10-09T05:40:16.065611+0000 osd.5 (osd.5) 402 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T05:40:17.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:17 smithi133 bash[105589]: cluster 2023-10-09T05:40:16.067683+0000 osd.5 (osd.5) 403 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T05:40:17.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:17 smithi133 bash[105589]: cluster 2023-10-09T05:40:16.090627+0000 osd.4 (osd.4) 327 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T05:40:17.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:17 smithi133 bash[105589]: cluster 2023-10-09T05:40:16.090735+0000 osd.7 (osd.7) 347 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T05:40:17.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:17 smithi133 bash[105589]: cluster 2023-10-09T05:40:16.414899+0000 osd.1 (osd.1) 8 : cluster 0 14.18 scrub starts 2023-10-09T05:40:17.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:17 smithi133 bash[105589]: cluster 2023-10-09T05:40:16.483481+0000 mgr.y (mgr.25277) 1576 : cluster 0 pgmap v1743: 109 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 16 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 15/801 objects misplaced (1.873%); 0 B/s, 208 objects/s recovering 2023-10-09T05:40:17.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:17 smithi133 bash[105589]: audit 2023-10-09T05:40:16.484941+0000 mon.a (mon.0) 2040 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-10-09T05:40:17.393 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:17.423 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:17.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:17.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:17.526 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:17.553 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:17.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:17.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:17.657 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:17.678 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:17.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:17.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:17.780 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:17.801 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:17.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:17.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:17.905 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:17.925 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:17.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:17.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:18.028 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:18.049 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:18.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:18.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:18.073 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:18 smithi133 bash[105589]: cluster 2023-10-09T05:40:16.334567+0000 osd.5 (osd.5) 404 : cluster 0 14.9 scrub starts 2023-10-09T05:40:18.073 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:18 smithi133 bash[105589]: cluster 2023-10-09T05:40:17.048557+0000 osd.3 (osd.3) 28 : cluster 0 14.21 scrub starts 2023-10-09T05:40:18.073 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:18 smithi133 bash[105589]: cluster 2023-10-09T05:40:17.064453+0000 osd.4 (osd.4) 328 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T05:40:18.073 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:18 smithi133 bash[105589]: audit 2023-10-09T05:40:17.069359+0000 mon.a (mon.0) 2041 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "31"}]': finished 2023-10-09T05:40:18.073 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:18 smithi133 bash[105589]: cluster 2023-10-09T05:40:17.069363+0000 osd.7 (osd.7) 348 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T05:40:18.073 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:18 smithi133 bash[105589]: cluster 2023-10-09T05:40:17.069610+0000 mon.a (mon.0) 2042 : cluster 0 osdmap e382: 8 total, 8 up, 7 in 2023-10-09T05:40:18.074 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:18 smithi133 bash[102858]: cluster 2023-10-09T05:40:16.334567+0000 osd.5 (osd.5) 404 : cluster 0 14.9 scrub starts 2023-10-09T05:40:18.074 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:18 smithi133 bash[102858]: cluster 2023-10-09T05:40:17.048557+0000 osd.3 (osd.3) 28 : cluster 0 14.21 scrub starts 2023-10-09T05:40:18.074 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:18 smithi133 bash[102858]: cluster 2023-10-09T05:40:17.064453+0000 osd.4 (osd.4) 328 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T05:40:18.074 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:18 smithi133 bash[102858]: audit 2023-10-09T05:40:17.069359+0000 mon.a (mon.0) 2041 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "31"}]': finished 2023-10-09T05:40:18.074 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:18 smithi133 bash[102858]: cluster 2023-10-09T05:40:17.069363+0000 osd.7 (osd.7) 348 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T05:40:18.074 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:18 smithi133 bash[102858]: cluster 2023-10-09T05:40:17.069610+0000 mon.a (mon.0) 2042 : cluster 0 osdmap e382: 8 total, 8 up, 7 in 2023-10-09T05:40:18.152 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:18.173 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:18.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:18.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:18.276 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:18.307 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:18.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:18.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:18.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:18 smithi194 bash[92666]: cluster 2023-10-09T05:40:16.334567+0000 osd.5 (osd.5) 404 : cluster 0 14.9 scrub starts 2023-10-09T05:40:18.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:18 smithi194 bash[92666]: cluster 2023-10-09T05:40:17.048557+0000 osd.3 (osd.3) 28 : cluster 0 14.21 scrub starts 2023-10-09T05:40:18.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:18 smithi194 bash[92666]: cluster 2023-10-09T05:40:17.064453+0000 osd.4 (osd.4) 328 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T05:40:18.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:18 smithi194 bash[92666]: audit 2023-10-09T05:40:17.069359+0000 mon.a (mon.0) 2041 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "31"}]': finished 2023-10-09T05:40:18.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:18 smithi194 bash[92666]: cluster 2023-10-09T05:40:17.069363+0000 osd.7 (osd.7) 348 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T05:40:18.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:18 smithi194 bash[92666]: cluster 2023-10-09T05:40:17.069610+0000 mon.a (mon.0) 2042 : cluster 0 osdmap e382: 8 total, 8 up, 7 in 2023-10-09T05:40:18.411 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:18.431 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:18.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:18.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:18.536 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:18.558 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:18.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:18.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:18.660 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:18.668 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:40:18.668 INFO:teuthology.orchestra.run.smithi133.stdout: "ops": [], 2023-10-09T05:40:18.668 INFO:teuthology.orchestra.run.smithi133.stdout: "complaint_time": 30, 2023-10-09T05:40:18.668 INFO:teuthology.orchestra.run.smithi133.stdout: "num_blocked_ops": 0 2023-10-09T05:40:18.669 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:40:18.690 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:18.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:18.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:18.794 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:18.822 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:18.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:18.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:18.926 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:18.954 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:18.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:18.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:19.058 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:19.089 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:19.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:19.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:19.192 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:19.217 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:19.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:19.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:19.319 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:19.342 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:19.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:19.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:19.365 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:19 smithi133 bash[105589]: cluster 2023-10-09T05:40:17.535278+0000 osd.5 (osd.5) 405 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T05:40:19.366 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:19 smithi133 bash[105589]: cluster 2023-10-09T05:40:18.066083+0000 osd.4 (osd.4) 329 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T05:40:19.366 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:19 smithi133 bash[105589]: cluster 2023-10-09T05:40:18.068166+0000 mon.a (mon.0) 2043 : cluster 0 osdmap e383: 8 total, 8 up, 7 in 2023-10-09T05:40:19.366 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:19 smithi133 bash[105589]: cluster 2023-10-09T05:40:18.068857+0000 osd.7 (osd.7) 349 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T05:40:19.366 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:19 smithi133 bash[105589]: cluster 2023-10-09T05:40:18.072641+0000 osd.7 (osd.7) 350 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T05:40:19.366 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:19 smithi133 bash[105589]: cluster 2023-10-09T05:40:18.072720+0000 osd.6 (osd.6) 365 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T05:40:19.366 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:19 smithi133 bash[105589]: cluster 2023-10-09T05:40:18.484072+0000 mgr.y (mgr.25277) 1577 : cluster 0 pgmap v1746: 109 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 16 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 15/801 objects misplaced (1.873%); 0 B/s, 208 objects/s recovering 2023-10-09T05:40:19.366 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:19 smithi133 bash[105589]: audit 2023-10-09T05:40:18.484877+0000 mon.a (mon.0) 2044 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "32"}]: dispatch 2023-10-09T05:40:19.366 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:19 smithi133 bash[102858]: cluster 2023-10-09T05:40:17.535278+0000 osd.5 (osd.5) 405 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T05:40:19.367 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:19 smithi133 bash[102858]: cluster 2023-10-09T05:40:18.066083+0000 osd.4 (osd.4) 329 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T05:40:19.367 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:19 smithi133 bash[102858]: cluster 2023-10-09T05:40:18.068166+0000 mon.a (mon.0) 2043 : cluster 0 osdmap e383: 8 total, 8 up, 7 in 2023-10-09T05:40:19.367 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:19 smithi133 bash[102858]: cluster 2023-10-09T05:40:18.068857+0000 osd.7 (osd.7) 349 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T05:40:19.367 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:19 smithi133 bash[102858]: cluster 2023-10-09T05:40:18.072641+0000 osd.7 (osd.7) 350 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T05:40:19.367 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:19 smithi133 bash[102858]: cluster 2023-10-09T05:40:18.072720+0000 osd.6 (osd.6) 365 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T05:40:19.367 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:19 smithi133 bash[102858]: cluster 2023-10-09T05:40:18.484072+0000 mgr.y (mgr.25277) 1577 : cluster 0 pgmap v1746: 109 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 16 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 15/801 objects misplaced (1.873%); 0 B/s, 208 objects/s recovering 2023-10-09T05:40:19.367 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:19 smithi133 bash[102858]: audit 2023-10-09T05:40:18.484877+0000 mon.a (mon.0) 2044 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "32"}]: dispatch 2023-10-09T05:40:19.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:19 smithi194 bash[92666]: cluster 2023-10-09T05:40:17.535278+0000 osd.5 (osd.5) 405 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T05:40:19.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:19 smithi194 bash[92666]: cluster 2023-10-09T05:40:18.066083+0000 osd.4 (osd.4) 329 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T05:40:19.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:19 smithi194 bash[92666]: cluster 2023-10-09T05:40:18.068166+0000 mon.a (mon.0) 2043 : cluster 0 osdmap e383: 8 total, 8 up, 7 in 2023-10-09T05:40:19.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:19 smithi194 bash[92666]: cluster 2023-10-09T05:40:18.068857+0000 osd.7 (osd.7) 349 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T05:40:19.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:19 smithi194 bash[92666]: cluster 2023-10-09T05:40:18.072641+0000 osd.7 (osd.7) 350 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T05:40:19.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:19 smithi194 bash[92666]: cluster 2023-10-09T05:40:18.072720+0000 osd.6 (osd.6) 365 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T05:40:19.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:19 smithi194 bash[92666]: cluster 2023-10-09T05:40:18.484072+0000 mgr.y (mgr.25277) 1577 : cluster 0 pgmap v1746: 109 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 16 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 15/801 objects misplaced (1.873%); 0 B/s, 208 objects/s recovering 2023-10-09T05:40:19.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:19 smithi194 bash[92666]: audit 2023-10-09T05:40:18.484877+0000 mon.a (mon.0) 2044 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "32"}]: dispatch 2023-10-09T05:40:19.447 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:19.476 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:19.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:19.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:19.580 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:19.612 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:19.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:19.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:19.716 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:19.746 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:19.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:19.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:19.849 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:19.872 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:19.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:19.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:19.975 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:20.010 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:20.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:20.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:20.114 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:20.142 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:20.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:20.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:20.246 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:20.267 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:20 smithi133 bash[102858]: cluster 2023-10-09T05:40:18.299395+0000 osd.5 (osd.5) 406 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T05:40:20.268 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:20 smithi133 bash[102858]: cluster 2023-10-09T05:40:18.301579+0000 osd.5 (osd.5) 407 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T05:40:20.268 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:20 smithi133 bash[102858]: cluster 2023-10-09T05:40:18.394887+0000 osd.5 (osd.5) 408 : cluster 0 14.7 scrub starts 2023-10-09T05:40:20.268 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:20 smithi133 bash[102858]: cluster 2023-10-09T05:40:18.464622+0000 osd.1 (osd.1) 9 : cluster 0 14.17 scrub starts 2023-10-09T05:40:20.268 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:20 smithi133 bash[102858]: cluster 2023-10-09T05:40:19.084549+0000 osd.5 (osd.5) 409 : cluster 3 failed to encode map e384 with expected crc 2023-10-09T05:40:20.268 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:20 smithi133 bash[102858]: cluster 2023-10-09T05:40:19.087790+0000 osd.1 (osd.1) 10 : cluster 0 14.1e starting backfill to osd.7 from (0'0,0'0] MAX to 357'10833 2023-10-09T05:40:20.268 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:20 smithi133 bash[102858]: audit 2023-10-09T05:40:19.088688+0000 mon.a (mon.0) 2045 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "32"}]': finished 2023-10-09T05:40:20.268 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:20 smithi133 bash[102858]: cluster 2023-10-09T05:40:19.088756+0000 mon.a (mon.0) 2046 : cluster 0 osdmap e384: 8 total, 8 up, 7 in 2023-10-09T05:40:20.268 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:20 smithi133 bash[102858]: cluster 2023-10-09T05:40:19.094232+0000 osd.6 (osd.6) 366 : cluster 3 failed to encode map e384 with expected crc 2023-10-09T05:40:20.268 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:20 smithi133 bash[102858]: cluster 2023-10-09T05:40:19.094269+0000 osd.7 (osd.7) 351 : cluster 3 failed to encode map e384 with expected crc 2023-10-09T05:40:20.268 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:20 smithi133 bash[102858]: cluster 2023-10-09T05:40:19.238971+0000 osd.4 (osd.4) 330 : cluster 3 failed to encode map e384 with expected crc 2023-10-09T05:40:20.268 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:20 smithi133 bash[102858]: cluster 2023-10-09T05:40:19.451150+0000 osd.1 (osd.1) 11 : cluster 0 14.1e scrub starts 2023-10-09T05:40:20.269 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:20 smithi133 bash[105589]: cluster 2023-10-09T05:40:18.299395+0000 osd.5 (osd.5) 406 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T05:40:20.269 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:20 smithi133 bash[105589]: cluster 2023-10-09T05:40:18.301579+0000 osd.5 (osd.5) 407 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T05:40:20.269 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:20 smithi133 bash[105589]: cluster 2023-10-09T05:40:18.394887+0000 osd.5 (osd.5) 408 : cluster 0 14.7 scrub starts 2023-10-09T05:40:20.269 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:20 smithi133 bash[105589]: cluster 2023-10-09T05:40:18.464622+0000 osd.1 (osd.1) 9 : cluster 0 14.17 scrub starts 2023-10-09T05:40:20.269 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:20 smithi133 bash[105589]: cluster 2023-10-09T05:40:19.084549+0000 osd.5 (osd.5) 409 : cluster 3 failed to encode map e384 with expected crc 2023-10-09T05:40:20.269 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:20 smithi133 bash[105589]: cluster 2023-10-09T05:40:19.087790+0000 osd.1 (osd.1) 10 : cluster 0 14.1e starting backfill to osd.7 from (0'0,0'0] MAX to 357'10833 2023-10-09T05:40:20.269 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:20 smithi133 bash[105589]: audit 2023-10-09T05:40:19.088688+0000 mon.a (mon.0) 2045 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "32"}]': finished 2023-10-09T05:40:20.269 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:20 smithi133 bash[105589]: cluster 2023-10-09T05:40:19.088756+0000 mon.a (mon.0) 2046 : cluster 0 osdmap e384: 8 total, 8 up, 7 in 2023-10-09T05:40:20.269 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:20 smithi133 bash[105589]: cluster 2023-10-09T05:40:19.094232+0000 osd.6 (osd.6) 366 : cluster 3 failed to encode map e384 with expected crc 2023-10-09T05:40:20.270 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:20 smithi133 bash[105589]: cluster 2023-10-09T05:40:19.094269+0000 osd.7 (osd.7) 351 : cluster 3 failed to encode map e384 with expected crc 2023-10-09T05:40:20.270 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:20 smithi133 bash[105589]: cluster 2023-10-09T05:40:19.238971+0000 osd.4 (osd.4) 330 : cluster 3 failed to encode map e384 with expected crc 2023-10-09T05:40:20.270 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:20 smithi133 bash[105589]: cluster 2023-10-09T05:40:19.451150+0000 osd.1 (osd.1) 11 : cluster 0 14.1e scrub starts 2023-10-09T05:40:20.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:20.271 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:20.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:20.372 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:20.384 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:20 smithi194 bash[92666]: cluster 2023-10-09T05:40:18.299395+0000 osd.5 (osd.5) 406 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T05:40:20.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:20 smithi194 bash[92666]: cluster 2023-10-09T05:40:18.301579+0000 osd.5 (osd.5) 407 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T05:40:20.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:20 smithi194 bash[92666]: cluster 2023-10-09T05:40:18.394887+0000 osd.5 (osd.5) 408 : cluster 0 14.7 scrub starts 2023-10-09T05:40:20.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:20 smithi194 bash[92666]: cluster 2023-10-09T05:40:18.464622+0000 osd.1 (osd.1) 9 : cluster 0 14.17 scrub starts 2023-10-09T05:40:20.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:20 smithi194 bash[92666]: cluster 2023-10-09T05:40:19.084549+0000 osd.5 (osd.5) 409 : cluster 3 failed to encode map e384 with expected crc 2023-10-09T05:40:20.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:20 smithi194 bash[92666]: cluster 2023-10-09T05:40:19.087790+0000 osd.1 (osd.1) 10 : cluster 0 14.1e starting backfill to osd.7 from (0'0,0'0] MAX to 357'10833 2023-10-09T05:40:20.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:20 smithi194 bash[92666]: audit 2023-10-09T05:40:19.088688+0000 mon.a (mon.0) 2045 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "32"}]': finished 2023-10-09T05:40:20.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:20 smithi194 bash[92666]: cluster 2023-10-09T05:40:19.088756+0000 mon.a (mon.0) 2046 : cluster 0 osdmap e384: 8 total, 8 up, 7 in 2023-10-09T05:40:20.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:20 smithi194 bash[92666]: cluster 2023-10-09T05:40:19.094232+0000 osd.6 (osd.6) 366 : cluster 3 failed to encode map e384 with expected crc 2023-10-09T05:40:20.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:20 smithi194 bash[92666]: cluster 2023-10-09T05:40:19.094269+0000 osd.7 (osd.7) 351 : cluster 3 failed to encode map e384 with expected crc 2023-10-09T05:40:20.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:20 smithi194 bash[92666]: cluster 2023-10-09T05:40:19.238971+0000 osd.4 (osd.4) 330 : cluster 3 failed to encode map e384 with expected crc 2023-10-09T05:40:20.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:20 smithi194 bash[92666]: cluster 2023-10-09T05:40:19.451150+0000 osd.1 (osd.1) 11 : cluster 0 14.1e scrub starts 2023-10-09T05:40:20.404 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:20.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:20.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:20.508 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:20.530 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:20.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:20.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:20.633 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:20.661 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:20.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:20.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:20.764 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:20.793 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:20.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:20.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:20.897 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:20.921 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:20.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:20.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:21.026 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:21.058 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:21.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:21.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:21.116 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is unset 2023-10-09T05:40:21.162 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:21 smithi133 bash[105589]: cluster 2023-10-09T05:40:20.092811+0000 osd.4 (osd.4) 331 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T05:40:21.163 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:21 smithi133 bash[105589]: cluster 2023-10-09T05:40:20.095027+0000 mon.a (mon.0) 2047 : cluster 0 osdmap e385: 8 total, 8 up, 7 in 2023-10-09T05:40:21.163 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:21 smithi133 bash[105589]: cluster 2023-10-09T05:40:20.096642+0000 osd.6 (osd.6) 367 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T05:40:21.163 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:21 smithi133 bash[105589]: cluster 2023-10-09T05:40:20.097675+0000 osd.7 (osd.7) 352 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T05:40:21.163 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:21 smithi133 bash[105589]: cluster 2023-10-09T05:40:20.098712+0000 osd.5 (osd.5) 410 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T05:40:21.163 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:21 smithi133 bash[105589]: cluster 2023-10-09T05:40:20.100008+0000 osd.6 (osd.6) 368 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T05:40:21.163 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:21 smithi133 bash[105589]: cluster 2023-10-09T05:40:20.103487+0000 osd.6 (osd.6) 369 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T05:40:21.164 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:21 smithi133 bash[105589]: cluster 2023-10-09T05:40:20.406962+0000 osd.6 (osd.6) 370 : cluster 0 14.11 scrub starts 2023-10-09T05:40:21.164 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:21 smithi133 bash[105589]: cluster 2023-10-09T05:40:20.421168+0000 osd.6 (osd.6) 371 : cluster 0 14.11 scrub ok 2023-10-09T05:40:21.164 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:21 smithi133 bash[105589]: cluster 2023-10-09T05:40:20.484613+0000 mgr.y (mgr.25277) 1578 : cluster 0 pgmap v1749: 109 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 16 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 15/801 objects misplaced (1.873%) 2023-10-09T05:40:21.164 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:21 smithi133 bash[105589]: audit 2023-10-09T05:40:20.485763+0000 mon.a (mon.0) 2048 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "33"}]: dispatch 2023-10-09T05:40:21.164 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:21 smithi133 bash[105589]: audit 2023-10-09T05:40:20.927477+0000 mon.a (mon.0) 2049 : audit 1 from='client.? 172.21.15.133:0/3108925922' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:40:21.164 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:21 smithi133 bash[102858]: cluster 2023-10-09T05:40:20.092811+0000 osd.4 (osd.4) 331 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T05:40:21.164 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:21 smithi133 bash[102858]: cluster 2023-10-09T05:40:20.095027+0000 mon.a (mon.0) 2047 : cluster 0 osdmap e385: 8 total, 8 up, 7 in 2023-10-09T05:40:21.164 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:21 smithi133 bash[102858]: cluster 2023-10-09T05:40:20.096642+0000 osd.6 (osd.6) 367 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T05:40:21.164 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:21 smithi133 bash[102858]: cluster 2023-10-09T05:40:20.097675+0000 osd.7 (osd.7) 352 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T05:40:21.164 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:21 smithi133 bash[102858]: cluster 2023-10-09T05:40:20.098712+0000 osd.5 (osd.5) 410 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T05:40:21.165 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:21 smithi133 bash[102858]: cluster 2023-10-09T05:40:20.100008+0000 osd.6 (osd.6) 368 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T05:40:21.165 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:21 smithi133 bash[102858]: cluster 2023-10-09T05:40:20.103487+0000 osd.6 (osd.6) 369 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T05:40:21.165 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:21 smithi133 bash[102858]: cluster 2023-10-09T05:40:20.406962+0000 osd.6 (osd.6) 370 : cluster 0 14.11 scrub starts 2023-10-09T05:40:21.165 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:21 smithi133 bash[102858]: cluster 2023-10-09T05:40:20.421168+0000 osd.6 (osd.6) 371 : cluster 0 14.11 scrub ok 2023-10-09T05:40:21.165 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:21 smithi133 bash[102858]: cluster 2023-10-09T05:40:20.484613+0000 mgr.y (mgr.25277) 1578 : cluster 0 pgmap v1749: 109 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 16 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 15/801 objects misplaced (1.873%) 2023-10-09T05:40:21.165 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:21 smithi133 bash[102858]: audit 2023-10-09T05:40:20.485763+0000 mon.a (mon.0) 2048 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "33"}]: dispatch 2023-10-09T05:40:21.165 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:21 smithi133 bash[102858]: audit 2023-10-09T05:40:20.927477+0000 mon.a (mon.0) 2049 : audit 1 from='client.? 172.21.15.133:0/3108925922' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:40:21.165 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:21.190 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:21.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:21.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:21.294 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:21.308 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:21 smithi194 bash[92666]: cluster 2023-10-09T05:40:20.092811+0000 osd.4 (osd.4) 331 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T05:40:21.309 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:21 smithi194 bash[92666]: cluster 2023-10-09T05:40:20.095027+0000 mon.a (mon.0) 2047 : cluster 0 osdmap e385: 8 total, 8 up, 7 in 2023-10-09T05:40:21.309 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:21 smithi194 bash[92666]: cluster 2023-10-09T05:40:20.096642+0000 osd.6 (osd.6) 367 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T05:40:21.309 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:21 smithi194 bash[92666]: cluster 2023-10-09T05:40:20.097675+0000 osd.7 (osd.7) 352 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T05:40:21.309 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:21 smithi194 bash[92666]: cluster 2023-10-09T05:40:20.098712+0000 osd.5 (osd.5) 410 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T05:40:21.309 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:21 smithi194 bash[92666]: cluster 2023-10-09T05:40:20.100008+0000 osd.6 (osd.6) 368 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T05:40:21.309 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:21 smithi194 bash[92666]: cluster 2023-10-09T05:40:20.103487+0000 osd.6 (osd.6) 369 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T05:40:21.309 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:21 smithi194 bash[92666]: cluster 2023-10-09T05:40:20.406962+0000 osd.6 (osd.6) 370 : cluster 0 14.11 scrub starts 2023-10-09T05:40:21.309 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:21 smithi194 bash[92666]: cluster 2023-10-09T05:40:20.421168+0000 osd.6 (osd.6) 371 : cluster 0 14.11 scrub ok 2023-10-09T05:40:21.309 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:21 smithi194 bash[92666]: cluster 2023-10-09T05:40:20.484613+0000 mgr.y (mgr.25277) 1578 : cluster 0 pgmap v1749: 109 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 16 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 15/801 objects misplaced (1.873%) 2023-10-09T05:40:21.310 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:21 smithi194 bash[92666]: audit 2023-10-09T05:40:20.485763+0000 mon.a (mon.0) 2048 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "33"}]: dispatch 2023-10-09T05:40:21.310 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:21 smithi194 bash[92666]: audit 2023-10-09T05:40:20.927477+0000 mon.a (mon.0) 2049 : audit 1 from='client.? 172.21.15.133:0/3108925922' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:40:21.327 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:21.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:21.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:21.430 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:21.466 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:21.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:21.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:21.569 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:21.603 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:21.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:21.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:21.706 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:21.731 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:21.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:21.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:21.834 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:21.864 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:21.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:21.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:21.968 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:21.998 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:22.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:22.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:22.102 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:22.128 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:22.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:22.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:22.231 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:22.254 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:22.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:22.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:22.357 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:22.365 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[105589]: cluster 2023-10-09T05:40:20.481556+0000 osd.1 (osd.1) 12 : cluster 0 14.18 scrub starts 2023-10-09T05:40:22.365 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[105589]: cluster 2023-10-09T05:40:20.497803+0000 osd.1 (osd.1) 13 : cluster 0 14.18 scrub ok 2023-10-09T05:40:22.365 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[105589]: cluster 2023-10-09T05:40:21.092363+0000 mon.a (mon.0) 2050 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:40:22.365 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[105589]: cluster 2023-10-09T05:40:21.105259+0000 osd.6 (osd.6) 372 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T05:40:22.366 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[105589]: audit 2023-10-09T05:40:21.105387+0000 mon.a (mon.0) 2051 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "33"}]': finished 2023-10-09T05:40:22.366 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[105589]: audit 2023-10-09T05:40:21.105500+0000 mon.a (mon.0) 2052 : audit 1 from='client.? 172.21.15.133:0/3108925922' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:40:22.366 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[105589]: cluster 2023-10-09T05:40:21.105557+0000 mon.a (mon.0) 2053 : cluster 0 osdmap e386: 8 total, 8 up, 7 in 2023-10-09T05:40:22.366 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[105589]: cluster 2023-10-09T05:40:21.105930+0000 osd.7 (osd.7) 353 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T05:40:22.366 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[105589]: cluster 2023-10-09T05:40:21.108167+0000 osd.6 (osd.6) 373 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T05:40:22.366 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[105589]: cluster 2023-10-09T05:40:21.108694+0000 osd.7 (osd.7) 354 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T05:40:22.366 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[105589]: cluster 2023-10-09T05:40:21.109084+0000 osd.1 (osd.1) 14 : cluster 0 14.1f starting backfill to osd.0 from (0'0,0'0] MAX to 357'10780 2023-10-09T05:40:22.366 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[105589]: cluster 2023-10-09T05:40:21.110218+0000 osd.5 (osd.5) 411 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T05:40:22.366 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[105589]: cluster 2023-10-09T05:40:21.110289+0000 osd.6 (osd.6) 374 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T05:40:22.366 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[105589]: cluster 2023-10-09T05:40:21.112509+0000 osd.6 (osd.6) 375 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T05:40:22.367 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[105589]: cluster 2023-10-09T05:40:21.126705+0000 osd.1 (osd.1) 15 : cluster 0 14.1f starting backfill to osd.5 from (0'0,0'0] MAX to 357'10780 2023-10-09T05:40:22.367 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[105589]: cluster 2023-10-09T05:40:21.292408+0000 osd.2 (osd.2) 14 : cluster 0 14.13 scrub starts 2023-10-09T05:40:22.367 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[105589]: cluster 2023-10-09T05:40:21.304053+0000 osd.2 (osd.2) 15 : cluster 0 14.13 scrub ok 2023-10-09T05:40:22.367 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[105589]: cluster 2023-10-09T05:40:21.375757+0000 osd.6 (osd.6) 376 : cluster 0 14.2 scrub starts 2023-10-09T05:40:22.367 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[105589]: cluster 2023-10-09T05:40:21.387104+0000 osd.6 (osd.6) 377 : cluster 0 14.2 scrub ok 2023-10-09T05:40:22.367 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[102858]: cluster 2023-10-09T05:40:20.481556+0000 osd.1 (osd.1) 12 : cluster 0 14.18 scrub starts 2023-10-09T05:40:22.367 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[102858]: cluster 2023-10-09T05:40:20.497803+0000 osd.1 (osd.1) 13 : cluster 0 14.18 scrub ok 2023-10-09T05:40:22.367 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[102858]: cluster 2023-10-09T05:40:21.092363+0000 mon.a (mon.0) 2050 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:40:22.367 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[102858]: cluster 2023-10-09T05:40:21.105259+0000 osd.6 (osd.6) 372 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T05:40:22.367 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[102858]: audit 2023-10-09T05:40:21.105387+0000 mon.a (mon.0) 2051 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "33"}]': finished 2023-10-09T05:40:22.368 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[102858]: audit 2023-10-09T05:40:21.105500+0000 mon.a (mon.0) 2052 : audit 1 from='client.? 172.21.15.133:0/3108925922' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:40:22.368 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[102858]: cluster 2023-10-09T05:40:21.105557+0000 mon.a (mon.0) 2053 : cluster 0 osdmap e386: 8 total, 8 up, 7 in 2023-10-09T05:40:22.368 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[102858]: cluster 2023-10-09T05:40:21.105930+0000 osd.7 (osd.7) 353 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T05:40:22.368 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[102858]: cluster 2023-10-09T05:40:21.108167+0000 osd.6 (osd.6) 373 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T05:40:22.368 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[102858]: cluster 2023-10-09T05:40:21.108694+0000 osd.7 (osd.7) 354 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T05:40:22.368 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[102858]: cluster 2023-10-09T05:40:21.109084+0000 osd.1 (osd.1) 14 : cluster 0 14.1f starting backfill to osd.0 from (0'0,0'0] MAX to 357'10780 2023-10-09T05:40:22.368 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[102858]: cluster 2023-10-09T05:40:21.110218+0000 osd.5 (osd.5) 411 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T05:40:22.368 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[102858]: cluster 2023-10-09T05:40:21.110289+0000 osd.6 (osd.6) 374 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T05:40:22.368 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[102858]: cluster 2023-10-09T05:40:21.112509+0000 osd.6 (osd.6) 375 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T05:40:22.368 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[102858]: cluster 2023-10-09T05:40:21.126705+0000 osd.1 (osd.1) 15 : cluster 0 14.1f starting backfill to osd.5 from (0'0,0'0] MAX to 357'10780 2023-10-09T05:40:22.368 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[102858]: cluster 2023-10-09T05:40:21.292408+0000 osd.2 (osd.2) 14 : cluster 0 14.13 scrub starts 2023-10-09T05:40:22.368 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[102858]: cluster 2023-10-09T05:40:21.304053+0000 osd.2 (osd.2) 15 : cluster 0 14.13 scrub ok 2023-10-09T05:40:22.368 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[102858]: cluster 2023-10-09T05:40:21.375757+0000 osd.6 (osd.6) 376 : cluster 0 14.2 scrub starts 2023-10-09T05:40:22.369 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:22 smithi133 bash[102858]: cluster 2023-10-09T05:40:21.387104+0000 osd.6 (osd.6) 377 : cluster 0 14.2 scrub ok 2023-10-09T05:40:22.379 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:22.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:22.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:22.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:22 smithi194 bash[92666]: cluster 2023-10-09T05:40:20.481556+0000 osd.1 (osd.1) 12 : cluster 0 14.18 scrub starts 2023-10-09T05:40:22.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:22 smithi194 bash[92666]: cluster 2023-10-09T05:40:20.497803+0000 osd.1 (osd.1) 13 : cluster 0 14.18 scrub ok 2023-10-09T05:40:22.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:22 smithi194 bash[92666]: cluster 2023-10-09T05:40:21.092363+0000 mon.a (mon.0) 2050 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:40:22.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:22 smithi194 bash[92666]: cluster 2023-10-09T05:40:21.105259+0000 osd.6 (osd.6) 372 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T05:40:22.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:22 smithi194 bash[92666]: audit 2023-10-09T05:40:21.105387+0000 mon.a (mon.0) 2051 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "33"}]': finished 2023-10-09T05:40:22.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:22 smithi194 bash[92666]: audit 2023-10-09T05:40:21.105500+0000 mon.a (mon.0) 2052 : audit 1 from='client.? 172.21.15.133:0/3108925922' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:40:22.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:22 smithi194 bash[92666]: cluster 2023-10-09T05:40:21.105557+0000 mon.a (mon.0) 2053 : cluster 0 osdmap e386: 8 total, 8 up, 7 in 2023-10-09T05:40:22.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:22 smithi194 bash[92666]: cluster 2023-10-09T05:40:21.105930+0000 osd.7 (osd.7) 353 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T05:40:22.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:22 smithi194 bash[92666]: cluster 2023-10-09T05:40:21.108167+0000 osd.6 (osd.6) 373 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T05:40:22.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:22 smithi194 bash[92666]: cluster 2023-10-09T05:40:21.108694+0000 osd.7 (osd.7) 354 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T05:40:22.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:22 smithi194 bash[92666]: cluster 2023-10-09T05:40:21.109084+0000 osd.1 (osd.1) 14 : cluster 0 14.1f starting backfill to osd.0 from (0'0,0'0] MAX to 357'10780 2023-10-09T05:40:22.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:22 smithi194 bash[92666]: cluster 2023-10-09T05:40:21.110218+0000 osd.5 (osd.5) 411 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T05:40:22.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:22 smithi194 bash[92666]: cluster 2023-10-09T05:40:21.110289+0000 osd.6 (osd.6) 374 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T05:40:22.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:22 smithi194 bash[92666]: cluster 2023-10-09T05:40:21.112509+0000 osd.6 (osd.6) 375 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T05:40:22.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:22 smithi194 bash[92666]: cluster 2023-10-09T05:40:21.126705+0000 osd.1 (osd.1) 15 : cluster 0 14.1f starting backfill to osd.5 from (0'0,0'0] MAX to 357'10780 2023-10-09T05:40:22.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:22 smithi194 bash[92666]: cluster 2023-10-09T05:40:21.292408+0000 osd.2 (osd.2) 14 : cluster 0 14.13 scrub starts 2023-10-09T05:40:22.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:22 smithi194 bash[92666]: cluster 2023-10-09T05:40:21.304053+0000 osd.2 (osd.2) 15 : cluster 0 14.13 scrub ok 2023-10-09T05:40:22.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:22 smithi194 bash[92666]: cluster 2023-10-09T05:40:21.375757+0000 osd.6 (osd.6) 376 : cluster 0 14.2 scrub starts 2023-10-09T05:40:22.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:22 smithi194 bash[92666]: cluster 2023-10-09T05:40:21.387104+0000 osd.6 (osd.6) 377 : cluster 0 14.2 scrub ok 2023-10-09T05:40:22.482 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:22.503 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:22.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:22.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:22.606 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:22.631 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:22.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:22.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:22.734 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:22.766 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:22.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:22.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:22.870 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:22.901 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:22.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:22.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:23.005 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:23.031 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:23.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:23.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:23.134 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:23.159 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:23.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:23.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:23.263 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:23.289 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:23.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:23.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:23.334 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:23 smithi133 bash[102858]: cluster 2023-10-09T05:40:21.981654+0000 osd.3 (osd.3) 29 : cluster 0 14.16 deep-scrub starts 2023-10-09T05:40:23.334 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:23 smithi133 bash[102858]: cluster 2023-10-09T05:40:21.995662+0000 osd.3 (osd.3) 30 : cluster 0 14.16 deep-scrub ok 2023-10-09T05:40:23.334 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:23 smithi133 bash[102858]: cluster 2023-10-09T05:40:22.077102+0000 osd.7 (osd.7) 355 : cluster 0 14.1e scrub starts 2023-10-09T05:40:23.334 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:23 smithi133 bash[102858]: cluster 2023-10-09T05:40:22.090471+0000 osd.7 (osd.7) 356 : cluster 0 14.1e scrub ok 2023-10-09T05:40:23.334 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:23 smithi133 bash[102858]: cluster 2023-10-09T05:40:22.110792+0000 osd.6 (osd.6) 378 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T05:40:23.334 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:23 smithi133 bash[102858]: cluster 2023-10-09T05:40:22.112961+0000 mon.a (mon.0) 2054 : cluster 0 osdmap e387: 8 total, 8 up, 7 in 2023-10-09T05:40:23.334 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:23 smithi133 bash[102858]: cluster 2023-10-09T05:40:22.118954+0000 osd.5 (osd.5) 412 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T05:40:23.334 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:23 smithi133 bash[102858]: cluster 2023-10-09T05:40:22.118993+0000 osd.7 (osd.7) 357 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T05:40:23.334 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:23 smithi133 bash[102858]: cluster 2023-10-09T05:40:22.485148+0000 mgr.y (mgr.25277) 1579 : cluster 0 pgmap v1752: 109 pgs: 1 remapped+peering, 1 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 16 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 15/801 objects misplaced (1.873%) 2023-10-09T05:40:23.334 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:23 smithi133 bash[102858]: cluster 2023-10-09T05:40:22.549953+0000 osd.0 (osd.0) 18 : cluster 0 14.5 scrub starts 2023-10-09T05:40:23.335 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:23 smithi133 bash[102858]: cluster 2023-10-09T05:40:22.563926+0000 osd.0 (osd.0) 19 : cluster 0 14.5 scrub ok 2023-10-09T05:40:23.335 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:23 smithi133 bash[105589]: cluster 2023-10-09T05:40:21.981654+0000 osd.3 (osd.3) 29 : cluster 0 14.16 deep-scrub starts 2023-10-09T05:40:23.335 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:23 smithi133 bash[105589]: cluster 2023-10-09T05:40:21.995662+0000 osd.3 (osd.3) 30 : cluster 0 14.16 deep-scrub ok 2023-10-09T05:40:23.335 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:23 smithi133 bash[105589]: cluster 2023-10-09T05:40:22.077102+0000 osd.7 (osd.7) 355 : cluster 0 14.1e scrub starts 2023-10-09T05:40:23.335 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:23 smithi133 bash[105589]: cluster 2023-10-09T05:40:22.090471+0000 osd.7 (osd.7) 356 : cluster 0 14.1e scrub ok 2023-10-09T05:40:23.335 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:23 smithi133 bash[105589]: cluster 2023-10-09T05:40:22.110792+0000 osd.6 (osd.6) 378 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T05:40:23.335 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:23 smithi133 bash[105589]: cluster 2023-10-09T05:40:22.112961+0000 mon.a (mon.0) 2054 : cluster 0 osdmap e387: 8 total, 8 up, 7 in 2023-10-09T05:40:23.335 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:23 smithi133 bash[105589]: cluster 2023-10-09T05:40:22.118954+0000 osd.5 (osd.5) 412 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T05:40:23.335 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:23 smithi133 bash[105589]: cluster 2023-10-09T05:40:22.118993+0000 osd.7 (osd.7) 357 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T05:40:23.336 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:23 smithi133 bash[105589]: cluster 2023-10-09T05:40:22.485148+0000 mgr.y (mgr.25277) 1579 : cluster 0 pgmap v1752: 109 pgs: 1 remapped+peering, 1 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 16 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 15/801 objects misplaced (1.873%) 2023-10-09T05:40:23.336 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:23 smithi133 bash[105589]: cluster 2023-10-09T05:40:22.549953+0000 osd.0 (osd.0) 18 : cluster 0 14.5 scrub starts 2023-10-09T05:40:23.336 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:23 smithi133 bash[105589]: cluster 2023-10-09T05:40:22.563926+0000 osd.0 (osd.0) 19 : cluster 0 14.5 scrub ok 2023-10-09T05:40:23.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:23 smithi194 bash[92666]: cluster 2023-10-09T05:40:21.981654+0000 osd.3 (osd.3) 29 : cluster 0 14.16 deep-scrub starts 2023-10-09T05:40:23.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:23 smithi194 bash[92666]: cluster 2023-10-09T05:40:21.995662+0000 osd.3 (osd.3) 30 : cluster 0 14.16 deep-scrub ok 2023-10-09T05:40:23.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:23 smithi194 bash[92666]: cluster 2023-10-09T05:40:22.077102+0000 osd.7 (osd.7) 355 : cluster 0 14.1e scrub starts 2023-10-09T05:40:23.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:23 smithi194 bash[92666]: cluster 2023-10-09T05:40:22.090471+0000 osd.7 (osd.7) 356 : cluster 0 14.1e scrub ok 2023-10-09T05:40:23.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:23 smithi194 bash[92666]: cluster 2023-10-09T05:40:22.110792+0000 osd.6 (osd.6) 378 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T05:40:23.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:23 smithi194 bash[92666]: cluster 2023-10-09T05:40:22.112961+0000 mon.a (mon.0) 2054 : cluster 0 osdmap e387: 8 total, 8 up, 7 in 2023-10-09T05:40:23.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:23 smithi194 bash[92666]: cluster 2023-10-09T05:40:22.118954+0000 osd.5 (osd.5) 412 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T05:40:23.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:23 smithi194 bash[92666]: cluster 2023-10-09T05:40:22.118993+0000 osd.7 (osd.7) 357 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T05:40:23.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:23 smithi194 bash[92666]: cluster 2023-10-09T05:40:22.485148+0000 mgr.y (mgr.25277) 1579 : cluster 0 pgmap v1752: 109 pgs: 1 remapped+peering, 1 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 16 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 15/801 objects misplaced (1.873%) 2023-10-09T05:40:23.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:23 smithi194 bash[92666]: cluster 2023-10-09T05:40:22.549953+0000 osd.0 (osd.0) 18 : cluster 0 14.5 scrub starts 2023-10-09T05:40:23.388 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:23 smithi194 bash[92666]: cluster 2023-10-09T05:40:22.563926+0000 osd.0 (osd.0) 19 : cluster 0 14.5 scrub ok 2023-10-09T05:40:23.392 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:23.416 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:23.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:23.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:23.519 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:23.549 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:23.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:23.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:23.651 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:23.676 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:23.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:23.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:23.779 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:23.807 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:23.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:23.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:23.911 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:23.945 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:23.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:23.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:24.049 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:24.079 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:24.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:24.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:24.183 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:24.207 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:24.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:24.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:24.310 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:24.322 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:24 smithi194 bash[92666]: cluster 2023-10-09T05:40:22.436179+0000 osd.4 (osd.4) 332 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T05:40:24.322 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:24 smithi194 bash[92666]: cluster 2023-10-09T05:40:22.454432+0000 osd.1 (osd.1) 16 : cluster 0 14.17 scrub starts 2023-10-09T05:40:24.322 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:24 smithi194 bash[92666]: cluster 2023-10-09T05:40:22.461314+0000 osd.1 (osd.1) 17 : cluster 0 14.17 scrub ok 2023-10-09T05:40:24.322 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:24 smithi194 bash[92666]: cluster 2023-10-09T05:40:23.126426+0000 mon.a (mon.0) 2055 : cluster 0 osdmap e388: 8 total, 8 up, 7 in 2023-10-09T05:40:24.322 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:24 smithi194 bash[92666]: cluster 2023-10-09T05:40:23.137759+0000 osd.5 (osd.5) 413 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T05:40:24.322 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:24 smithi194 bash[92666]: cluster 2023-10-09T05:40:23.420980+0000 osd.6 (osd.6) 379 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T05:40:24.323 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:24 smithi194 bash[92666]: cluster 2023-10-09T05:40:23.423817+0000 osd.6 (osd.6) 380 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T05:40:24.338 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:24.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:24.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:24.442 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:24.452 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:40:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:40:24] "GET /metrics HTTP/1.1" 200 36488 "" "Prometheus/2.43.0" 2023-10-09T05:40:24.452 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:24 smithi133 bash[105589]: cluster 2023-10-09T05:40:22.436179+0000 osd.4 (osd.4) 332 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T05:40:24.452 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:24 smithi133 bash[105589]: cluster 2023-10-09T05:40:22.454432+0000 osd.1 (osd.1) 16 : cluster 0 14.17 scrub starts 2023-10-09T05:40:24.452 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:24 smithi133 bash[105589]: cluster 2023-10-09T05:40:22.461314+0000 osd.1 (osd.1) 17 : cluster 0 14.17 scrub ok 2023-10-09T05:40:24.453 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:24 smithi133 bash[105589]: cluster 2023-10-09T05:40:23.126426+0000 mon.a (mon.0) 2055 : cluster 0 osdmap e388: 8 total, 8 up, 7 in 2023-10-09T05:40:24.453 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:24 smithi133 bash[105589]: cluster 2023-10-09T05:40:23.137759+0000 osd.5 (osd.5) 413 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T05:40:24.453 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:24 smithi133 bash[105589]: cluster 2023-10-09T05:40:23.420980+0000 osd.6 (osd.6) 379 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T05:40:24.453 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:24 smithi133 bash[105589]: cluster 2023-10-09T05:40:23.423817+0000 osd.6 (osd.6) 380 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T05:40:24.453 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:24 smithi133 bash[102858]: cluster 2023-10-09T05:40:22.436179+0000 osd.4 (osd.4) 332 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T05:40:24.453 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:24 smithi133 bash[102858]: cluster 2023-10-09T05:40:22.454432+0000 osd.1 (osd.1) 16 : cluster 0 14.17 scrub starts 2023-10-09T05:40:24.453 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:24 smithi133 bash[102858]: cluster 2023-10-09T05:40:22.461314+0000 osd.1 (osd.1) 17 : cluster 0 14.17 scrub ok 2023-10-09T05:40:24.453 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:24 smithi133 bash[102858]: cluster 2023-10-09T05:40:23.126426+0000 mon.a (mon.0) 2055 : cluster 0 osdmap e388: 8 total, 8 up, 7 in 2023-10-09T05:40:24.453 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:24 smithi133 bash[102858]: cluster 2023-10-09T05:40:23.137759+0000 osd.5 (osd.5) 413 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T05:40:24.454 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:24 smithi133 bash[102858]: cluster 2023-10-09T05:40:23.420980+0000 osd.6 (osd.6) 379 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T05:40:24.454 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:24 smithi133 bash[102858]: cluster 2023-10-09T05:40:23.423817+0000 osd.6 (osd.6) 380 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T05:40:24.467 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:24.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:24.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:24.571 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:24.593 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:24.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:24.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:24.696 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:24.722 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:24.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:24.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:24.825 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:24.845 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:24.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:24.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:24.948 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:24.970 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:24.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:24.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:25.073 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:25.101 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:25.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:25.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:25.205 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:25.235 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:25.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:25.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:25.340 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:25.361 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:25.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:25.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:25.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:25 smithi194 bash[92666]: cluster 2023-10-09T05:40:23.156006+0000 osd.7 (osd.7) 358 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T05:40:25.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:25 smithi194 bash[92666]: cluster 2023-10-09T05:40:23.436735+0000 osd.4 (osd.4) 333 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T05:40:25.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:25 smithi194 bash[92666]: cluster 2023-10-09T05:40:24.404920+0000 osd.6 (osd.6) 381 : cluster 0 14.d scrub starts 2023-10-09T05:40:25.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:25 smithi194 bash[92666]: cluster 2023-10-09T05:40:24.485765+0000 mgr.y (mgr.25277) 1580 : cluster 0 pgmap v1754: 109 pgs: 1 remapped+peering, 1 peering, 107 active+clean; 16 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:25.457 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:25 smithi133 bash[102858]: cluster 2023-10-09T05:40:23.156006+0000 osd.7 (osd.7) 358 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T05:40:25.457 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:25 smithi133 bash[102858]: cluster 2023-10-09T05:40:23.436735+0000 osd.4 (osd.4) 333 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T05:40:25.457 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:25 smithi133 bash[102858]: cluster 2023-10-09T05:40:24.404920+0000 osd.6 (osd.6) 381 : cluster 0 14.d scrub starts 2023-10-09T05:40:25.457 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:25 smithi133 bash[102858]: cluster 2023-10-09T05:40:24.485765+0000 mgr.y (mgr.25277) 1580 : cluster 0 pgmap v1754: 109 pgs: 1 remapped+peering, 1 peering, 107 active+clean; 16 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:25.457 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:25 smithi133 bash[105589]: cluster 2023-10-09T05:40:23.156006+0000 osd.7 (osd.7) 358 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T05:40:25.458 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:25 smithi133 bash[105589]: cluster 2023-10-09T05:40:23.436735+0000 osd.4 (osd.4) 333 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T05:40:25.458 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:25 smithi133 bash[105589]: cluster 2023-10-09T05:40:24.404920+0000 osd.6 (osd.6) 381 : cluster 0 14.d scrub starts 2023-10-09T05:40:25.458 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:25 smithi133 bash[105589]: cluster 2023-10-09T05:40:24.485765+0000 mgr.y (mgr.25277) 1580 : cluster 0 pgmap v1754: 109 pgs: 1 remapped+peering, 1 peering, 107 active+clean; 16 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:25.464 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:25.489 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:25.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:25.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:25.595 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:25.619 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:25.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:25.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:25.722 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:25.758 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:25.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:25.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:25.812 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-09T05:40:25.812 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 1.0 2023-10-09T05:40:25.863 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:25.888 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:25.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:25.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:25.991 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:26.018 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:26.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:26.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:26.122 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:26.145 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:26.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:26.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:26.248 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:26.281 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:26.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:26.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:26.385 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:26.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:26 smithi194 bash[92666]: cluster 2023-10-09T05:40:25.121452+0000 mon.a (mon.0) 2056 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg degraded) 2023-10-09T05:40:26.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:26 smithi194 bash[92666]: cluster 2023-10-09T05:40:25.121482+0000 mon.a (mon.0) 2057 : cluster 1 Cluster is now healthy 2023-10-09T05:40:26.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:26 smithi194 bash[92666]: cluster 2023-10-09T05:40:25.403344+0000 osd.6 (osd.6) 382 : cluster 0 14.22 deep-scrub starts 2023-10-09T05:40:26.418 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:26.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:26.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:26.522 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:26.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:26 smithi133 bash[102858]: cluster 2023-10-09T05:40:25.121452+0000 mon.a (mon.0) 2056 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg degraded) 2023-10-09T05:40:26.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:26 smithi133 bash[102858]: cluster 2023-10-09T05:40:25.121482+0000 mon.a (mon.0) 2057 : cluster 1 Cluster is now healthy 2023-10-09T05:40:26.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:26 smithi133 bash[102858]: cluster 2023-10-09T05:40:25.403344+0000 osd.6 (osd.6) 382 : cluster 0 14.22 deep-scrub starts 2023-10-09T05:40:26.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:26 smithi133 bash[105589]: cluster 2023-10-09T05:40:25.121452+0000 mon.a (mon.0) 2056 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg degraded) 2023-10-09T05:40:26.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:26 smithi133 bash[105589]: cluster 2023-10-09T05:40:25.121482+0000 mon.a (mon.0) 2057 : cluster 1 Cluster is now healthy 2023-10-09T05:40:26.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:26 smithi133 bash[105589]: cluster 2023-10-09T05:40:25.403344+0000 osd.6 (osd.6) 382 : cluster 0 14.22 deep-scrub starts 2023-10-09T05:40:26.557 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:26.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:26.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:26.662 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:26.685 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:26.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:26.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:26.789 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:26.823 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:26.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:26.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:26.926 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:26.951 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:26.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:26.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:27.055 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:27.087 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:27.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:27.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:27.191 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:27.198 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:27 smithi133 bash[102858]: cluster 2023-10-09T05:40:26.486394+0000 mgr.y (mgr.25277) 1581 : cluster 0 pgmap v1755: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 15 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:27.199 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:27 smithi133 bash[102858]: audit 2023-10-09T05:40:26.487180+0000 mon.a (mon.0) 2058 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "34"}]: dispatch 2023-10-09T05:40:27.199 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:27 smithi133 bash[105589]: cluster 2023-10-09T05:40:26.486394+0000 mgr.y (mgr.25277) 1581 : cluster 0 pgmap v1755: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 15 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:27.199 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:27 smithi133 bash[105589]: audit 2023-10-09T05:40:26.487180+0000 mon.a (mon.0) 2058 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "34"}]: dispatch 2023-10-09T05:40:27.214 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:27.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:27.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:27.318 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:27.332 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:27 smithi194 bash[92666]: cluster 2023-10-09T05:40:26.486394+0000 mgr.y (mgr.25277) 1581 : cluster 0 pgmap v1755: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 15 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:27.332 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:27 smithi194 bash[92666]: audit 2023-10-09T05:40:26.487180+0000 mon.a (mon.0) 2058 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "34"}]: dispatch 2023-10-09T05:40:27.347 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:27.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:27.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:27.451 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:27.479 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:27.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:27.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:27.583 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:27.615 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:27.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:27.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:27.720 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:27.739 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:27.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:27.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:27.842 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:27.876 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:27.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:27.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:27.934 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T05:40:27.980 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:28.012 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:28.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:28.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:28.117 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:28.131 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 05:40:27 smithi194 bash[80638]: logger=cleanup t=2023-10-09T05:40:27.974514591Z level=info msg="Completed cleanup jobs" duration=68.301986ms 2023-10-09T05:40:28.152 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:28.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:28.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:28.256 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:28.288 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:28.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:28.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:28.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:28 smithi194 bash[92666]: cluster 2023-10-09T05:40:27.143767+0000 osd.5 (osd.5) 414 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T05:40:28.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:28 smithi194 bash[92666]: audit 2023-10-09T05:40:27.145946+0000 mon.a (mon.0) 2059 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "34"}]': finished 2023-10-09T05:40:28.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:28 smithi194 bash[92666]: cluster 2023-10-09T05:40:27.146795+0000 mon.a (mon.0) 2060 : cluster 0 osdmap e389: 8 total, 8 up, 7 in 2023-10-09T05:40:28.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:28 smithi194 bash[92666]: cluster 2023-10-09T05:40:27.149034+0000 osd.5 (osd.5) 415 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T05:40:28.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:28 smithi194 bash[92666]: cluster 2023-10-09T05:40:27.323456+0000 osd.6 (osd.6) 383 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T05:40:28.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:28 smithi194 bash[92666]: cluster 2023-10-09T05:40:27.326685+0000 osd.6 (osd.6) 384 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T05:40:28.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:28 smithi194 bash[92666]: cluster 2023-10-09T05:40:27.362037+0000 osd.2 (osd.2) 16 : cluster 0 14.12 scrub starts 2023-10-09T05:40:28.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:28 smithi194 bash[92666]: cluster 2023-10-09T05:40:27.940426+0000 osd.7 (osd.7) 359 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T05:40:28.393 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:28.403 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:28 smithi133 bash[102858]: cluster 2023-10-09T05:40:27.143767+0000 osd.5 (osd.5) 414 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T05:40:28.403 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:28 smithi133 bash[102858]: audit 2023-10-09T05:40:27.145946+0000 mon.a (mon.0) 2059 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "34"}]': finished 2023-10-09T05:40:28.403 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:28 smithi133 bash[102858]: cluster 2023-10-09T05:40:27.146795+0000 mon.a (mon.0) 2060 : cluster 0 osdmap e389: 8 total, 8 up, 7 in 2023-10-09T05:40:28.403 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:28 smithi133 bash[102858]: cluster 2023-10-09T05:40:27.149034+0000 osd.5 (osd.5) 415 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T05:40:28.404 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:28 smithi133 bash[102858]: cluster 2023-10-09T05:40:27.323456+0000 osd.6 (osd.6) 383 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T05:40:28.404 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:28 smithi133 bash[102858]: cluster 2023-10-09T05:40:27.326685+0000 osd.6 (osd.6) 384 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T05:40:28.404 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:28 smithi133 bash[102858]: cluster 2023-10-09T05:40:27.362037+0000 osd.2 (osd.2) 16 : cluster 0 14.12 scrub starts 2023-10-09T05:40:28.404 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:28 smithi133 bash[102858]: cluster 2023-10-09T05:40:27.940426+0000 osd.7 (osd.7) 359 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T05:40:28.404 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:28 smithi133 bash[105589]: cluster 2023-10-09T05:40:27.143767+0000 osd.5 (osd.5) 414 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T05:40:28.404 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:28 smithi133 bash[105589]: audit 2023-10-09T05:40:27.145946+0000 mon.a (mon.0) 2059 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "34"}]': finished 2023-10-09T05:40:28.404 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:28 smithi133 bash[105589]: cluster 2023-10-09T05:40:27.146795+0000 mon.a (mon.0) 2060 : cluster 0 osdmap e389: 8 total, 8 up, 7 in 2023-10-09T05:40:28.404 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:28 smithi133 bash[105589]: cluster 2023-10-09T05:40:27.149034+0000 osd.5 (osd.5) 415 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T05:40:28.404 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:28 smithi133 bash[105589]: cluster 2023-10-09T05:40:27.323456+0000 osd.6 (osd.6) 383 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T05:40:28.404 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:28 smithi133 bash[105589]: cluster 2023-10-09T05:40:27.326685+0000 osd.6 (osd.6) 384 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T05:40:28.405 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:28 smithi133 bash[105589]: cluster 2023-10-09T05:40:27.362037+0000 osd.2 (osd.2) 16 : cluster 0 14.12 scrub starts 2023-10-09T05:40:28.405 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:28 smithi133 bash[105589]: cluster 2023-10-09T05:40:27.940426+0000 osd.7 (osd.7) 359 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T05:40:28.416 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:28.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:28.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:28.522 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:28.554 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:28.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:28.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:28.657 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:28.685 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:28.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:28.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:28.789 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:28.824 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:28.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:28.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:28.927 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:28.950 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:28.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:28.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:29.054 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:29.083 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:29.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:29.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:29.187 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:29.197 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:29 smithi133 bash[105589]: cluster 2023-10-09T05:40:27.940219+0000 osd.4 (osd.4) 334 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T05:40:29.197 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:29 smithi133 bash[105589]: cluster 2023-10-09T05:40:28.150748+0000 osd.6 (osd.6) 385 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T05:40:29.197 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:29 smithi133 bash[105589]: cluster 2023-10-09T05:40:28.151902+0000 mon.a (mon.0) 2061 : cluster 0 osdmap e390: 8 total, 8 up, 7 in 2023-10-09T05:40:29.197 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:29 smithi133 bash[105589]: cluster 2023-10-09T05:40:28.153559+0000 osd.5 (osd.5) 416 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T05:40:29.197 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:29 smithi133 bash[105589]: cluster 2023-10-09T05:40:28.156580+0000 osd.5 (osd.5) 417 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T05:40:29.197 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:29 smithi133 bash[105589]: cluster 2023-10-09T05:40:28.157812+0000 osd.5 (osd.5) 418 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T05:40:29.197 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:29 smithi133 bash[105589]: cluster 2023-10-09T05:40:28.487031+0000 mgr.y (mgr.25277) 1582 : cluster 0 pgmap v1758: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 15 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:29.197 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:29 smithi133 bash[105589]: audit 2023-10-09T05:40:28.488788+0000 mon.a (mon.0) 2062 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "35"}]: dispatch 2023-10-09T05:40:29.197 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:29 smithi133 bash[105589]: audit 2023-10-09T05:40:28.503628+0000 mon.a (mon.0) 2063 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:40:29.198 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:29 smithi133 bash[105589]: audit 2023-10-09T05:40:28.504888+0000 mon.a (mon.0) 2064 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:29.198 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:29 smithi133 bash[105589]: cluster 2023-10-09T05:40:29.037588+0000 osd.7 (osd.7) 360 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T05:40:29.198 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:29 smithi133 bash[102858]: cluster 2023-10-09T05:40:27.940219+0000 osd.4 (osd.4) 334 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T05:40:29.198 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:29 smithi133 bash[102858]: cluster 2023-10-09T05:40:28.150748+0000 osd.6 (osd.6) 385 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T05:40:29.198 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:29 smithi133 bash[102858]: cluster 2023-10-09T05:40:28.151902+0000 mon.a (mon.0) 2061 : cluster 0 osdmap e390: 8 total, 8 up, 7 in 2023-10-09T05:40:29.198 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:29 smithi133 bash[102858]: cluster 2023-10-09T05:40:28.153559+0000 osd.5 (osd.5) 416 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T05:40:29.198 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:29 smithi133 bash[102858]: cluster 2023-10-09T05:40:28.156580+0000 osd.5 (osd.5) 417 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T05:40:29.198 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:29 smithi133 bash[102858]: cluster 2023-10-09T05:40:28.157812+0000 osd.5 (osd.5) 418 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T05:40:29.198 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:29 smithi133 bash[102858]: cluster 2023-10-09T05:40:28.487031+0000 mgr.y (mgr.25277) 1582 : cluster 0 pgmap v1758: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 15 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:29.198 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:29 smithi133 bash[102858]: audit 2023-10-09T05:40:28.488788+0000 mon.a (mon.0) 2062 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "35"}]: dispatch 2023-10-09T05:40:29.199 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:29 smithi133 bash[102858]: audit 2023-10-09T05:40:28.503628+0000 mon.a (mon.0) 2063 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:40:29.199 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:29 smithi133 bash[102858]: audit 2023-10-09T05:40:28.504888+0000 mon.a (mon.0) 2064 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:29.199 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:29 smithi133 bash[102858]: cluster 2023-10-09T05:40:29.037588+0000 osd.7 (osd.7) 360 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T05:40:29.211 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:29.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:29.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:29.319 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:29.346 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:29.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:29.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:29.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:29 smithi194 bash[92666]: cluster 2023-10-09T05:40:27.940219+0000 osd.4 (osd.4) 334 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T05:40:29.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:29 smithi194 bash[92666]: cluster 2023-10-09T05:40:28.150748+0000 osd.6 (osd.6) 385 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T05:40:29.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:29 smithi194 bash[92666]: cluster 2023-10-09T05:40:28.151902+0000 mon.a (mon.0) 2061 : cluster 0 osdmap e390: 8 total, 8 up, 7 in 2023-10-09T05:40:29.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:29 smithi194 bash[92666]: cluster 2023-10-09T05:40:28.153559+0000 osd.5 (osd.5) 416 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T05:40:29.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:29 smithi194 bash[92666]: cluster 2023-10-09T05:40:28.156580+0000 osd.5 (osd.5) 417 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T05:40:29.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:29 smithi194 bash[92666]: cluster 2023-10-09T05:40:28.157812+0000 osd.5 (osd.5) 418 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T05:40:29.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:29 smithi194 bash[92666]: cluster 2023-10-09T05:40:28.487031+0000 mgr.y (mgr.25277) 1582 : cluster 0 pgmap v1758: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 15 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:29.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:29 smithi194 bash[92666]: audit 2023-10-09T05:40:28.488788+0000 mon.a (mon.0) 2062 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "35"}]: dispatch 2023-10-09T05:40:29.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:29 smithi194 bash[92666]: audit 2023-10-09T05:40:28.503628+0000 mon.a (mon.0) 2063 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:40:29.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:29 smithi194 bash[92666]: audit 2023-10-09T05:40:28.504888+0000 mon.a (mon.0) 2064 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:29.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:29 smithi194 bash[92666]: cluster 2023-10-09T05:40:29.037588+0000 osd.7 (osd.7) 360 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T05:40:29.449 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:29.474 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:29.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:29.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:29.576 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:29.604 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:29.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:29.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:29.708 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:29.730 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:29.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:29.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:29.833 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:29.862 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:29.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:29.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:29.966 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:29.989 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:29.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:29.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:30.092 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:30.127 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:30.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:30.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:30.231 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:30.262 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:30.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:30.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:30.366 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:30.376 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:30 smithi194 bash[92666]: cluster 2023-10-09T05:40:28.306872+0000 osd.5 (osd.5) 419 : cluster 0 14.9 scrub starts 2023-10-09T05:40:30.376 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:30 smithi194 bash[92666]: cluster 2023-10-09T05:40:28.322898+0000 osd.5 (osd.5) 420 : cluster 0 14.9 scrub ok 2023-10-09T05:40:30.376 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:30 smithi194 bash[92666]: cluster 2023-10-09T05:40:29.159705+0000 osd.4 (osd.4) 335 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T05:40:30.376 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:30 smithi194 bash[92666]: audit 2023-10-09T05:40:29.160981+0000 mon.a (mon.0) 2065 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "35"}]': finished 2023-10-09T05:40:30.377 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:30 smithi194 bash[92666]: cluster 2023-10-09T05:40:29.161041+0000 mon.a (mon.0) 2066 : cluster 0 osdmap e391: 8 total, 8 up, 7 in 2023-10-09T05:40:30.377 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:30 smithi194 bash[92666]: cluster 2023-10-09T05:40:29.165129+0000 osd.5 (osd.5) 421 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T05:40:30.377 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:30 smithi194 bash[92666]: cluster 2023-10-09T05:40:29.168029+0000 osd.5 (osd.5) 422 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T05:40:30.377 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:30 smithi194 bash[92666]: cluster 2023-10-09T05:40:29.170643+0000 osd.5 (osd.5) 423 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T05:40:30.377 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:30 smithi194 bash[92666]: cluster 2023-10-09T05:40:29.327625+0000 osd.2 (osd.2) 17 : cluster 0 14.0 scrub starts 2023-10-09T05:40:30.397 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:30.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:30.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:30.502 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:30.526 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:30.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:30.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:30.630 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:30.641 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:30 smithi133 bash[105589]: cluster 2023-10-09T05:40:28.306872+0000 osd.5 (osd.5) 419 : cluster 0 14.9 scrub starts 2023-10-09T05:40:30.641 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:30 smithi133 bash[105589]: cluster 2023-10-09T05:40:28.322898+0000 osd.5 (osd.5) 420 : cluster 0 14.9 scrub ok 2023-10-09T05:40:30.641 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:30 smithi133 bash[105589]: cluster 2023-10-09T05:40:29.159705+0000 osd.4 (osd.4) 335 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T05:40:30.641 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:30 smithi133 bash[105589]: audit 2023-10-09T05:40:29.160981+0000 mon.a (mon.0) 2065 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "35"}]': finished 2023-10-09T05:40:30.642 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:30 smithi133 bash[105589]: cluster 2023-10-09T05:40:29.161041+0000 mon.a (mon.0) 2066 : cluster 0 osdmap e391: 8 total, 8 up, 7 in 2023-10-09T05:40:30.642 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:30 smithi133 bash[105589]: cluster 2023-10-09T05:40:29.165129+0000 osd.5 (osd.5) 421 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T05:40:30.642 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:30 smithi133 bash[105589]: cluster 2023-10-09T05:40:29.168029+0000 osd.5 (osd.5) 422 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T05:40:30.642 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:30 smithi133 bash[105589]: cluster 2023-10-09T05:40:29.170643+0000 osd.5 (osd.5) 423 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T05:40:30.642 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:30 smithi133 bash[105589]: cluster 2023-10-09T05:40:29.327625+0000 osd.2 (osd.2) 17 : cluster 0 14.0 scrub starts 2023-10-09T05:40:30.642 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:30 smithi133 bash[102858]: cluster 2023-10-09T05:40:28.306872+0000 osd.5 (osd.5) 419 : cluster 0 14.9 scrub starts 2023-10-09T05:40:30.642 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:30 smithi133 bash[102858]: cluster 2023-10-09T05:40:28.322898+0000 osd.5 (osd.5) 420 : cluster 0 14.9 scrub ok 2023-10-09T05:40:30.642 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:30 smithi133 bash[102858]: cluster 2023-10-09T05:40:29.159705+0000 osd.4 (osd.4) 335 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T05:40:30.642 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:30 smithi133 bash[102858]: audit 2023-10-09T05:40:29.160981+0000 mon.a (mon.0) 2065 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "35"}]': finished 2023-10-09T05:40:30.643 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:30 smithi133 bash[102858]: cluster 2023-10-09T05:40:29.161041+0000 mon.a (mon.0) 2066 : cluster 0 osdmap e391: 8 total, 8 up, 7 in 2023-10-09T05:40:30.643 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:30 smithi133 bash[102858]: cluster 2023-10-09T05:40:29.165129+0000 osd.5 (osd.5) 421 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T05:40:30.643 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:30 smithi133 bash[102858]: cluster 2023-10-09T05:40:29.168029+0000 osd.5 (osd.5) 422 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T05:40:30.643 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:30 smithi133 bash[102858]: cluster 2023-10-09T05:40:29.170643+0000 osd.5 (osd.5) 423 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T05:40:30.643 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:30 smithi133 bash[102858]: cluster 2023-10-09T05:40:29.327625+0000 osd.2 (osd.2) 17 : cluster 0 14.0 scrub starts 2023-10-09T05:40:30.658 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:30.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:30.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:30.762 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:30.793 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:30.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:30.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:30.896 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:30.928 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:30.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:30.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:31.032 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:31.053 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:31.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:31.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:31.156 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:31.188 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:31.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:31.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:31.290 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:40:31.292 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:31.314 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:31.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:31.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:31.417 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:31.443 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:31.445 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:31 smithi133 bash[102858]: cluster 2023-10-09T05:40:30.171519+0000 mon.a (mon.0) 2067 : cluster 0 osdmap e392: 8 total, 8 up, 7 in 2023-10-09T05:40:31.445 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:31 smithi133 bash[102858]: cluster 2023-10-09T05:40:30.172758+0000 osd.5 (osd.5) 424 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T05:40:31.445 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:31 smithi133 bash[102858]: cluster 2023-10-09T05:40:30.174706+0000 osd.6 (osd.6) 386 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T05:40:31.445 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:31 smithi133 bash[102858]: cluster 2023-10-09T05:40:30.174936+0000 osd.7 (osd.7) 361 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T05:40:31.445 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:31 smithi133 bash[102858]: cluster 2023-10-09T05:40:30.175652+0000 osd.5 (osd.5) 425 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T05:40:31.445 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:31 smithi133 bash[102858]: cluster 2023-10-09T05:40:30.360080+0000 osd.6 (osd.6) 387 : cluster 0 14.1b scrub starts 2023-10-09T05:40:31.445 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:31 smithi133 bash[102858]: cluster 2023-10-09T05:40:30.487656+0000 mgr.y (mgr.25277) 1583 : cluster 0 pgmap v1761: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 12 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:31.445 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:31 smithi133 bash[102858]: audit 2023-10-09T05:40:30.489123+0000 mon.a (mon.0) 2068 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "36"}]: dispatch 2023-10-09T05:40:31.446 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:31 smithi133 bash[105589]: cluster 2023-10-09T05:40:30.171519+0000 mon.a (mon.0) 2067 : cluster 0 osdmap e392: 8 total, 8 up, 7 in 2023-10-09T05:40:31.446 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:31 smithi133 bash[105589]: cluster 2023-10-09T05:40:30.172758+0000 osd.5 (osd.5) 424 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T05:40:31.446 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:31 smithi133 bash[105589]: cluster 2023-10-09T05:40:30.174706+0000 osd.6 (osd.6) 386 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T05:40:31.446 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:31 smithi133 bash[105589]: cluster 2023-10-09T05:40:30.174936+0000 osd.7 (osd.7) 361 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T05:40:31.446 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:31 smithi133 bash[105589]: cluster 2023-10-09T05:40:30.175652+0000 osd.5 (osd.5) 425 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T05:40:31.446 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:31 smithi133 bash[105589]: cluster 2023-10-09T05:40:30.360080+0000 osd.6 (osd.6) 387 : cluster 0 14.1b scrub starts 2023-10-09T05:40:31.446 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:31 smithi133 bash[105589]: cluster 2023-10-09T05:40:30.487656+0000 mgr.y (mgr.25277) 1583 : cluster 0 pgmap v1761: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 12 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:31.446 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:31 smithi133 bash[105589]: audit 2023-10-09T05:40:30.489123+0000 mon.a (mon.0) 2068 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "36"}]: dispatch 2023-10-09T05:40:31.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:31.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:31.548 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:31.575 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:31.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:31.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:31.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:31 smithi194 bash[92666]: cluster 2023-10-09T05:40:30.171519+0000 mon.a (mon.0) 2067 : cluster 0 osdmap e392: 8 total, 8 up, 7 in 2023-10-09T05:40:31.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:31 smithi194 bash[92666]: cluster 2023-10-09T05:40:30.172758+0000 osd.5 (osd.5) 424 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T05:40:31.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:31 smithi194 bash[92666]: cluster 2023-10-09T05:40:30.174706+0000 osd.6 (osd.6) 386 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T05:40:31.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:31 smithi194 bash[92666]: cluster 2023-10-09T05:40:30.174936+0000 osd.7 (osd.7) 361 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T05:40:31.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:31 smithi194 bash[92666]: cluster 2023-10-09T05:40:30.175652+0000 osd.5 (osd.5) 425 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T05:40:31.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:31 smithi194 bash[92666]: cluster 2023-10-09T05:40:30.360080+0000 osd.6 (osd.6) 387 : cluster 0 14.1b scrub starts 2023-10-09T05:40:31.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:31 smithi194 bash[92666]: cluster 2023-10-09T05:40:30.487656+0000 mgr.y (mgr.25277) 1583 : cluster 0 pgmap v1761: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 12 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:31.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:31 smithi194 bash[92666]: audit 2023-10-09T05:40:30.489123+0000 mon.a (mon.0) 2068 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "36"}]: dispatch 2023-10-09T05:40:31.678 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:31.687 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:31.687 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:31.704 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:31.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:31.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:31.815 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:31.836 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:31.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:31.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:31.940 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:31.972 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:31.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:31.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:32.076 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:32.102 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:32.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:32.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:32.205 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:32.215 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:32 smithi194 bash[92666]: cluster 2023-10-09T05:40:30.693250+0000 osd.4 (osd.4) 336 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T05:40:32.215 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:32 smithi194 bash[92666]: audit 2023-10-09T05:40:31.175020+0000 mon.a (mon.0) 2069 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "36"}]': finished 2023-10-09T05:40:32.215 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:32 smithi194 bash[92666]: cluster 2023-10-09T05:40:31.175270+0000 mon.a (mon.0) 2070 : cluster 0 osdmap e393: 8 total, 8 up, 7 in 2023-10-09T05:40:32.215 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:32 smithi194 bash[92666]: cluster 2023-10-09T05:40:31.180170+0000 osd.7 (osd.7) 362 : cluster 3 failed to encode map e393 with expected crc 2023-10-09T05:40:32.215 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:32 smithi194 bash[92666]: cluster 2023-10-09T05:40:31.194732+0000 osd.5 (osd.5) 426 : cluster 3 failed to encode map e393 with expected crc 2023-10-09T05:40:32.215 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:32 smithi194 bash[92666]: cluster 2023-10-09T05:40:31.242159+0000 osd.6 (osd.6) 388 : cluster 3 failed to encode map e393 with expected crc 2023-10-09T05:40:32.232 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:32.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:32.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:32.336 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:32.367 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:32.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:32.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:32.471 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:32.502 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:32.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:32.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:32.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:32 smithi133 bash[105589]: cluster 2023-10-09T05:40:30.693250+0000 osd.4 (osd.4) 336 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T05:40:32.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:32 smithi133 bash[105589]: audit 2023-10-09T05:40:31.175020+0000 mon.a (mon.0) 2069 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "36"}]': finished 2023-10-09T05:40:32.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:32 smithi133 bash[105589]: cluster 2023-10-09T05:40:31.175270+0000 mon.a (mon.0) 2070 : cluster 0 osdmap e393: 8 total, 8 up, 7 in 2023-10-09T05:40:32.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:32 smithi133 bash[105589]: cluster 2023-10-09T05:40:31.180170+0000 osd.7 (osd.7) 362 : cluster 3 failed to encode map e393 with expected crc 2023-10-09T05:40:32.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:32 smithi133 bash[105589]: cluster 2023-10-09T05:40:31.194732+0000 osd.5 (osd.5) 426 : cluster 3 failed to encode map e393 with expected crc 2023-10-09T05:40:32.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:32 smithi133 bash[105589]: cluster 2023-10-09T05:40:31.242159+0000 osd.6 (osd.6) 388 : cluster 3 failed to encode map e393 with expected crc 2023-10-09T05:40:32.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:32 smithi133 bash[102858]: cluster 2023-10-09T05:40:30.693250+0000 osd.4 (osd.4) 336 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T05:40:32.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:32 smithi133 bash[102858]: audit 2023-10-09T05:40:31.175020+0000 mon.a (mon.0) 2069 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "36"}]': finished 2023-10-09T05:40:32.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:32 smithi133 bash[102858]: cluster 2023-10-09T05:40:31.175270+0000 mon.a (mon.0) 2070 : cluster 0 osdmap e393: 8 total, 8 up, 7 in 2023-10-09T05:40:32.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:32 smithi133 bash[102858]: cluster 2023-10-09T05:40:31.180170+0000 osd.7 (osd.7) 362 : cluster 3 failed to encode map e393 with expected crc 2023-10-09T05:40:32.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:32 smithi133 bash[102858]: cluster 2023-10-09T05:40:31.194732+0000 osd.5 (osd.5) 426 : cluster 3 failed to encode map e393 with expected crc 2023-10-09T05:40:32.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:32 smithi133 bash[102858]: cluster 2023-10-09T05:40:31.242159+0000 osd.6 (osd.6) 388 : cluster 3 failed to encode map e393 with expected crc 2023-10-09T05:40:32.607 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:32.635 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:32.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:32.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:32.739 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:32.767 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:32.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:32.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:32.871 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:32.898 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:32.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:32.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:33.002 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:33.032 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:33.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:33.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:33.136 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:33.166 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:33.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:33.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:33.270 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:33.300 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:33.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:33.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:33.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:33 smithi194 bash[92666]: cluster 2023-10-09T05:40:32.193845+0000 osd.5 (osd.5) 427 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T05:40:33.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:33 smithi194 bash[92666]: cluster 2023-10-09T05:40:32.194246+0000 osd.7 (osd.7) 363 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T05:40:33.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:33 smithi194 bash[92666]: cluster 2023-10-09T05:40:32.196383+0000 mon.a (mon.0) 2071 : cluster 0 osdmap e394: 8 total, 8 up, 7 in 2023-10-09T05:40:33.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:33 smithi194 bash[92666]: cluster 2023-10-09T05:40:32.198136+0000 osd.7 (osd.7) 364 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T05:40:33.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:33 smithi194 bash[92666]: cluster 2023-10-09T05:40:32.325916+0000 osd.6 (osd.6) 389 : cluster 0 14.19 scrub starts 2023-10-09T05:40:33.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:33 smithi194 bash[92666]: cluster 2023-10-09T05:40:32.326706+0000 osd.6 (osd.6) 390 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T05:40:33.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:33 smithi194 bash[92666]: cluster 2023-10-09T05:40:32.328307+0000 osd.6 (osd.6) 391 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T05:40:33.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:33 smithi194 bash[92666]: cluster 2023-10-09T05:40:32.391957+0000 osd.2 (osd.2) 18 : cluster 0 14.12 scrub starts 2023-10-09T05:40:33.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:33 smithi194 bash[92666]: cluster 2023-10-09T05:40:32.488180+0000 mgr.y (mgr.25277) 1584 : cluster 0 pgmap v1764: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 107 active+clean; 12 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s, 319 objects/s recovering 2023-10-09T05:40:33.403 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:33.427 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:33.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:33.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:33.530 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:33.540 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:33 smithi133 bash[102858]: cluster 2023-10-09T05:40:32.193845+0000 osd.5 (osd.5) 427 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T05:40:33.541 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:33 smithi133 bash[102858]: cluster 2023-10-09T05:40:32.194246+0000 osd.7 (osd.7) 363 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T05:40:33.541 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:33 smithi133 bash[102858]: cluster 2023-10-09T05:40:32.196383+0000 mon.a (mon.0) 2071 : cluster 0 osdmap e394: 8 total, 8 up, 7 in 2023-10-09T05:40:33.541 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:33 smithi133 bash[102858]: cluster 2023-10-09T05:40:32.198136+0000 osd.7 (osd.7) 364 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T05:40:33.541 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:33 smithi133 bash[102858]: cluster 2023-10-09T05:40:32.325916+0000 osd.6 (osd.6) 389 : cluster 0 14.19 scrub starts 2023-10-09T05:40:33.541 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:33 smithi133 bash[102858]: cluster 2023-10-09T05:40:32.326706+0000 osd.6 (osd.6) 390 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T05:40:33.541 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:33 smithi133 bash[102858]: cluster 2023-10-09T05:40:32.328307+0000 osd.6 (osd.6) 391 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T05:40:33.541 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:33 smithi133 bash[102858]: cluster 2023-10-09T05:40:32.391957+0000 osd.2 (osd.2) 18 : cluster 0 14.12 scrub starts 2023-10-09T05:40:33.541 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:33 smithi133 bash[102858]: cluster 2023-10-09T05:40:32.488180+0000 mgr.y (mgr.25277) 1584 : cluster 0 pgmap v1764: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 107 active+clean; 12 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s, 319 objects/s recovering 2023-10-09T05:40:33.541 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:33 smithi133 bash[105589]: cluster 2023-10-09T05:40:32.193845+0000 osd.5 (osd.5) 427 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T05:40:33.542 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:33 smithi133 bash[105589]: cluster 2023-10-09T05:40:32.194246+0000 osd.7 (osd.7) 363 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T05:40:33.542 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:33 smithi133 bash[105589]: cluster 2023-10-09T05:40:32.196383+0000 mon.a (mon.0) 2071 : cluster 0 osdmap e394: 8 total, 8 up, 7 in 2023-10-09T05:40:33.542 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:33 smithi133 bash[105589]: cluster 2023-10-09T05:40:32.198136+0000 osd.7 (osd.7) 364 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T05:40:33.542 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:33 smithi133 bash[105589]: cluster 2023-10-09T05:40:32.325916+0000 osd.6 (osd.6) 389 : cluster 0 14.19 scrub starts 2023-10-09T05:40:33.542 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:33 smithi133 bash[105589]: cluster 2023-10-09T05:40:32.326706+0000 osd.6 (osd.6) 390 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T05:40:33.542 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:33 smithi133 bash[105589]: cluster 2023-10-09T05:40:32.328307+0000 osd.6 (osd.6) 391 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T05:40:33.542 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:33 smithi133 bash[105589]: cluster 2023-10-09T05:40:32.391957+0000 osd.2 (osd.2) 18 : cluster 0 14.12 scrub starts 2023-10-09T05:40:33.542 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:33 smithi133 bash[105589]: cluster 2023-10-09T05:40:32.488180+0000 mgr.y (mgr.25277) 1584 : cluster 0 pgmap v1764: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 107 active+clean; 12 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s, 319 objects/s recovering 2023-10-09T05:40:33.557 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:33.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:33.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:33.660 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:33.692 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:33.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:33.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:33.795 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:33.821 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:33.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:33.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:33.925 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:33.952 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:33.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:33.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:34.057 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:34.084 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:34.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:34.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:34.187 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:34.212 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:34.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:34.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:34.316 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:34.348 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:34.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:34.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:34.452 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:34.464 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:34 smithi194 bash[92666]: cluster 2023-10-09T05:40:33.109129+0000 osd.4 (osd.4) 337 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T05:40:34.464 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:34 smithi194 bash[92666]: cluster 2023-10-09T05:40:33.111919+0000 osd.4 (osd.4) 338 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T05:40:34.464 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:34 smithi194 bash[92666]: cluster 2023-10-09T05:40:33.204035+0000 osd.4 (osd.4) 339 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T05:40:34.464 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:34 smithi194 bash[92666]: cluster 2023-10-09T05:40:33.210620+0000 mon.a (mon.0) 2072 : cluster 0 osdmap e395: 8 total, 8 up, 7 in 2023-10-09T05:40:34.464 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:34 smithi194 bash[92666]: cluster 2023-10-09T05:40:33.213040+0000 osd.5 (osd.5) 428 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T05:40:34.464 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:34 smithi194 bash[92666]: cluster 2023-10-09T05:40:33.223576+0000 osd.6 (osd.6) 392 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T05:40:34.464 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:34 smithi194 bash[92666]: cluster 2023-10-09T05:40:33.225772+0000 osd.6 (osd.6) 393 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T05:40:34.464 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:34 smithi194 bash[92666]: cluster 2023-10-09T05:40:33.228036+0000 osd.6 (osd.6) 394 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T05:40:34.465 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:34 smithi194 bash[92666]: cluster 2023-10-09T05:40:33.723232+0000 osd.7 (osd.7) 365 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T05:40:34.483 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:34.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:34.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:34.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:40:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:40:34] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.43.0" 2023-10-09T05:40:34.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:34 smithi133 bash[105589]: cluster 2023-10-09T05:40:33.109129+0000 osd.4 (osd.4) 337 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T05:40:34.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:34 smithi133 bash[105589]: cluster 2023-10-09T05:40:33.111919+0000 osd.4 (osd.4) 338 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T05:40:34.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:34 smithi133 bash[105589]: cluster 2023-10-09T05:40:33.204035+0000 osd.4 (osd.4) 339 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T05:40:34.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:34 smithi133 bash[105589]: cluster 2023-10-09T05:40:33.210620+0000 mon.a (mon.0) 2072 : cluster 0 osdmap e395: 8 total, 8 up, 7 in 2023-10-09T05:40:34.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:34 smithi133 bash[105589]: cluster 2023-10-09T05:40:33.213040+0000 osd.5 (osd.5) 428 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T05:40:34.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:34 smithi133 bash[105589]: cluster 2023-10-09T05:40:33.223576+0000 osd.6 (osd.6) 392 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T05:40:34.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:34 smithi133 bash[105589]: cluster 2023-10-09T05:40:33.225772+0000 osd.6 (osd.6) 393 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T05:40:34.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:34 smithi133 bash[105589]: cluster 2023-10-09T05:40:33.228036+0000 osd.6 (osd.6) 394 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T05:40:34.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:34 smithi133 bash[105589]: cluster 2023-10-09T05:40:33.723232+0000 osd.7 (osd.7) 365 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T05:40:34.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:34 smithi133 bash[102858]: cluster 2023-10-09T05:40:33.109129+0000 osd.4 (osd.4) 337 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T05:40:34.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:34 smithi133 bash[102858]: cluster 2023-10-09T05:40:33.111919+0000 osd.4 (osd.4) 338 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T05:40:34.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:34 smithi133 bash[102858]: cluster 2023-10-09T05:40:33.204035+0000 osd.4 (osd.4) 339 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T05:40:34.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:34 smithi133 bash[102858]: cluster 2023-10-09T05:40:33.210620+0000 mon.a (mon.0) 2072 : cluster 0 osdmap e395: 8 total, 8 up, 7 in 2023-10-09T05:40:34.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:34 smithi133 bash[102858]: cluster 2023-10-09T05:40:33.213040+0000 osd.5 (osd.5) 428 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T05:40:34.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:34 smithi133 bash[102858]: cluster 2023-10-09T05:40:33.223576+0000 osd.6 (osd.6) 392 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T05:40:34.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:34 smithi133 bash[102858]: cluster 2023-10-09T05:40:33.225772+0000 osd.6 (osd.6) 393 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T05:40:34.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:34 smithi133 bash[102858]: cluster 2023-10-09T05:40:33.228036+0000 osd.6 (osd.6) 394 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T05:40:34.550 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:34 smithi133 bash[102858]: cluster 2023-10-09T05:40:33.723232+0000 osd.7 (osd.7) 365 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T05:40:34.586 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:34.616 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:34.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:34.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:34.721 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:34.749 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:34.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:34.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:34.852 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:34.880 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:34.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:34.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:34.983 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:35.006 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:35.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:35.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:35.108 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:35.136 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:35.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:35.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:35.239 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:35.260 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:35.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:35.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:35.344 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:35 smithi133 bash[102858]: cluster 2023-10-09T05:40:34.488679+0000 mgr.y (mgr.25277) 1585 : cluster 0 pgmap v1766: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 107 active+clean; 12 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s, 296 objects/s recovering 2023-10-09T05:40:35.363 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:35.395 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:35.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:35.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:35.499 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:35.511 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:35 smithi194 bash[92666]: cluster 2023-10-09T05:40:34.488679+0000 mgr.y (mgr.25277) 1585 : cluster 0 pgmap v1766: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 107 active+clean; 12 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s, 296 objects/s recovering 2023-10-09T05:40:35.532 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:35.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:35.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:35.636 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:35.668 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:35.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:35.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:35.745 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:35.771 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:35.801 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:35.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:35.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:35.805 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:35 smithi133 bash[105589]: cluster 2023-10-09T05:40:34.488679+0000 mgr.y (mgr.25277) 1585 : cluster 0 pgmap v1766: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 107 active+clean; 12 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s, 296 objects/s recovering 2023-10-09T05:40:35.905 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:35.933 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:35.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:35.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:36.036 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:36.067 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:36.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:36.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:36.173 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:36.202 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:36.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:36.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:36.305 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:36.317 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:36 smithi194 bash[92666]: cluster 2023-10-09T05:40:35.219120+0000 osd.4 (osd.4) 340 : cluster 3 failed to encode map e396 with expected crc 2023-10-09T05:40:36.317 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:36 smithi194 bash[92666]: cluster 2023-10-09T05:40:35.219273+0000 osd.5 (osd.5) 429 : cluster 3 failed to encode map e396 with expected crc 2023-10-09T05:40:36.317 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:36 smithi194 bash[92666]: cluster 2023-10-09T05:40:35.222222+0000 mon.a (mon.0) 2073 : cluster 0 osdmap e396: 8 total, 8 up, 7 in 2023-10-09T05:40:36.317 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:36 smithi194 bash[92666]: cluster 2023-10-09T05:40:35.406159+0000 osd.6 (osd.6) 395 : cluster 0 14.d scrub starts 2023-10-09T05:40:36.338 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:36.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:36.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:36.441 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:36.449 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:36 smithi133 bash[102858]: cluster 2023-10-09T05:40:35.219120+0000 osd.4 (osd.4) 340 : cluster 3 failed to encode map e396 with expected crc 2023-10-09T05:40:36.449 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:36 smithi133 bash[102858]: cluster 2023-10-09T05:40:35.219273+0000 osd.5 (osd.5) 429 : cluster 3 failed to encode map e396 with expected crc 2023-10-09T05:40:36.449 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:36 smithi133 bash[102858]: cluster 2023-10-09T05:40:35.222222+0000 mon.a (mon.0) 2073 : cluster 0 osdmap e396: 8 total, 8 up, 7 in 2023-10-09T05:40:36.449 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:36 smithi133 bash[102858]: cluster 2023-10-09T05:40:35.406159+0000 osd.6 (osd.6) 395 : cluster 0 14.d scrub starts 2023-10-09T05:40:36.449 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:36 smithi133 bash[105589]: cluster 2023-10-09T05:40:35.219120+0000 osd.4 (osd.4) 340 : cluster 3 failed to encode map e396 with expected crc 2023-10-09T05:40:36.449 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:36 smithi133 bash[105589]: cluster 2023-10-09T05:40:35.219273+0000 osd.5 (osd.5) 429 : cluster 3 failed to encode map e396 with expected crc 2023-10-09T05:40:36.449 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:36 smithi133 bash[105589]: cluster 2023-10-09T05:40:35.222222+0000 mon.a (mon.0) 2073 : cluster 0 osdmap e396: 8 total, 8 up, 7 in 2023-10-09T05:40:36.450 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:36 smithi133 bash[105589]: cluster 2023-10-09T05:40:35.406159+0000 osd.6 (osd.6) 395 : cluster 0 14.d scrub starts 2023-10-09T05:40:36.465 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:36.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:36.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:36.568 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:36.592 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:36.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:36.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:36.696 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:36.724 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:36.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:36.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:36.828 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:36.858 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:36.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:36.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:36.962 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:36.990 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:36.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:36.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:37.093 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:37.118 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:37.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:37.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:37.221 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:37.249 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:37.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:37.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:37.352 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:37.379 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:37.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:37.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:37.483 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:37.491 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:37 smithi133 bash[105589]: cluster 2023-10-09T05:40:36.216270+0000 osd.4 (osd.4) 341 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T05:40:37.491 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:37 smithi133 bash[105589]: cluster 2023-10-09T05:40:36.217739+0000 osd.5 (osd.5) 430 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T05:40:37.491 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:37 smithi133 bash[105589]: cluster 2023-10-09T05:40:36.218712+0000 mon.a (mon.0) 2074 : cluster 0 osdmap e397: 8 total, 8 up, 7 in 2023-10-09T05:40:37.491 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:37 smithi133 bash[105589]: cluster 2023-10-09T05:40:36.222812+0000 osd.5 (osd.5) 431 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T05:40:37.491 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:37 smithi133 bash[105589]: cluster 2023-10-09T05:40:36.225284+0000 osd.5 (osd.5) 432 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T05:40:37.491 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:37 smithi133 bash[105589]: cluster 2023-10-09T05:40:36.489135+0000 mgr.y (mgr.25277) 1586 : cluster 0 pgmap v1769: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 106 active+clean; 12 MiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 1586/624 objects degraded (254.167%); 1586/624 objects misplaced (254.167%); 0 B/s, 67 objects/s recovering 2023-10-09T05:40:37.491 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:37 smithi133 bash[105589]: cluster 2023-10-09T05:40:36.593731+0000 osd.6 (osd.6) 396 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T05:40:37.491 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:37 smithi133 bash[102858]: cluster 2023-10-09T05:40:36.216270+0000 osd.4 (osd.4) 341 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T05:40:37.492 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:37 smithi133 bash[102858]: cluster 2023-10-09T05:40:36.217739+0000 osd.5 (osd.5) 430 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T05:40:37.492 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:37 smithi133 bash[102858]: cluster 2023-10-09T05:40:36.218712+0000 mon.a (mon.0) 2074 : cluster 0 osdmap e397: 8 total, 8 up, 7 in 2023-10-09T05:40:37.492 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:37 smithi133 bash[102858]: cluster 2023-10-09T05:40:36.222812+0000 osd.5 (osd.5) 431 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T05:40:37.492 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:37 smithi133 bash[102858]: cluster 2023-10-09T05:40:36.225284+0000 osd.5 (osd.5) 432 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T05:40:37.492 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:37 smithi133 bash[102858]: cluster 2023-10-09T05:40:36.489135+0000 mgr.y (mgr.25277) 1586 : cluster 0 pgmap v1769: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 106 active+clean; 12 MiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 1586/624 objects degraded (254.167%); 1586/624 objects misplaced (254.167%); 0 B/s, 67 objects/s recovering 2023-10-09T05:40:37.492 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:37 smithi133 bash[102858]: cluster 2023-10-09T05:40:36.593731+0000 osd.6 (osd.6) 396 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T05:40:37.508 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:37.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:37.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:37.611 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:37.624 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:37 smithi194 bash[92666]: cluster 2023-10-09T05:40:36.216270+0000 osd.4 (osd.4) 341 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T05:40:37.624 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:37 smithi194 bash[92666]: cluster 2023-10-09T05:40:36.217739+0000 osd.5 (osd.5) 430 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T05:40:37.624 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:37 smithi194 bash[92666]: cluster 2023-10-09T05:40:36.218712+0000 mon.a (mon.0) 2074 : cluster 0 osdmap e397: 8 total, 8 up, 7 in 2023-10-09T05:40:37.624 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:37 smithi194 bash[92666]: cluster 2023-10-09T05:40:36.222812+0000 osd.5 (osd.5) 431 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T05:40:37.625 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:37 smithi194 bash[92666]: cluster 2023-10-09T05:40:36.225284+0000 osd.5 (osd.5) 432 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T05:40:37.625 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:37 smithi194 bash[92666]: cluster 2023-10-09T05:40:36.489135+0000 mgr.y (mgr.25277) 1586 : cluster 0 pgmap v1769: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 106 active+clean; 12 MiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 1586/624 objects degraded (254.167%); 1586/624 objects misplaced (254.167%); 0 B/s, 67 objects/s recovering 2023-10-09T05:40:37.625 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:37 smithi194 bash[92666]: cluster 2023-10-09T05:40:36.593731+0000 osd.6 (osd.6) 396 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T05:40:37.639 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:37.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:37.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:37.743 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:37.775 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:37.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:37.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:37.879 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:37.906 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:37.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:37.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:38.009 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:38.043 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:38.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:38.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:38.146 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:38.171 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:38.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:38.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:38.275 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:38.297 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:38.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:38.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:38.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:38 smithi194 bash[92666]: cluster 2023-10-09T05:40:37.223187+0000 mon.a (mon.0) 2075 : cluster 3 Health check failed: Degraded data redundancy: 1586/624 objects degraded (254.167%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:40:38.401 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:38.408 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:38 smithi133 bash[105589]: cluster 2023-10-09T05:40:37.223187+0000 mon.a (mon.0) 2075 : cluster 3 Health check failed: Degraded data redundancy: 1586/624 objects degraded (254.167%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:40:38.408 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:38 smithi133 bash[102858]: cluster 2023-10-09T05:40:37.223187+0000 mon.a (mon.0) 2075 : cluster 3 Health check failed: Degraded data redundancy: 1586/624 objects degraded (254.167%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:40:38.422 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:38.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:38.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:38.525 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:38.554 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:38.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:38.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:38.658 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:38.679 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:38.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:38.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:38.783 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:38.806 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:38.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:38.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:38.910 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:38.933 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:38.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:38.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:38.971 INFO:teuthology.orchestra.run.smithi133.stderr:pool 'unique_pool_9' removed 2023-10-09T05:40:39.036 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:39.063 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:39.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:39.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:39.167 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:39.205 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:39.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:39.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:39.313 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:39.341 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:39.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:39.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:39.447 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:39.457 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:39 smithi133 bash[102858]: cluster 2023-10-09T05:40:38.489694+0000 mgr.y (mgr.25277) 1587 : cluster 0 pgmap v1770: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 106 active+clean; 12 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 1586/624 objects degraded (254.167%); 1586/624 objects misplaced (254.167%); 0 B/s, 48 objects/s recovering 2023-10-09T05:40:39.457 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:39 smithi133 bash[102858]: audit 2023-10-09T05:40:38.928509+0000 mon.c (mon.2) 144 : audit 1 from='client.? 172.21.15.133:0/2898147833' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:40:39.458 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:39 smithi133 bash[102858]: audit 2023-10-09T05:40:38.928957+0000 mon.a (mon.0) 2076 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:40:39.458 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:39 smithi133 bash[102858]: audit 2023-10-09T05:40:38.963783+0000 mon.a (mon.0) 2077 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:40:39.458 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:39 smithi133 bash[102858]: cluster 2023-10-09T05:40:38.963869+0000 mon.a (mon.0) 2078 : cluster 0 osdmap e398: 8 total, 8 up, 7 in 2023-10-09T05:40:39.458 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:39 smithi133 bash[102858]: audit 2023-10-09T05:40:39.093528+0000 mon.a (mon.0) 2079 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:40:39.458 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:39 smithi133 bash[105589]: cluster 2023-10-09T05:40:38.489694+0000 mgr.y (mgr.25277) 1587 : cluster 0 pgmap v1770: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 106 active+clean; 12 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 1586/624 objects degraded (254.167%); 1586/624 objects misplaced (254.167%); 0 B/s, 48 objects/s recovering 2023-10-09T05:40:39.458 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:39 smithi133 bash[105589]: audit 2023-10-09T05:40:38.928509+0000 mon.c (mon.2) 144 : audit 1 from='client.? 172.21.15.133:0/2898147833' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:40:39.458 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:39 smithi133 bash[105589]: audit 2023-10-09T05:40:38.928957+0000 mon.a (mon.0) 2076 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:40:39.458 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:39 smithi133 bash[105589]: audit 2023-10-09T05:40:38.963783+0000 mon.a (mon.0) 2077 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:40:39.458 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:39 smithi133 bash[105589]: cluster 2023-10-09T05:40:38.963869+0000 mon.a (mon.0) 2078 : cluster 0 osdmap e398: 8 total, 8 up, 7 in 2023-10-09T05:40:39.459 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:39 smithi133 bash[105589]: audit 2023-10-09T05:40:39.093528+0000 mon.a (mon.0) 2079 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:40:39.485 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:39.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:39.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:39.588 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:39.608 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:39.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:39.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:39.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:39 smithi194 bash[92666]: cluster 2023-10-09T05:40:38.489694+0000 mgr.y (mgr.25277) 1587 : cluster 0 pgmap v1770: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 106 active+clean; 12 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 1586/624 objects degraded (254.167%); 1586/624 objects misplaced (254.167%); 0 B/s, 48 objects/s recovering 2023-10-09T05:40:39.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:39 smithi194 bash[92666]: audit 2023-10-09T05:40:38.928509+0000 mon.c (mon.2) 144 : audit 1 from='client.? 172.21.15.133:0/2898147833' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:40:39.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:39 smithi194 bash[92666]: audit 2023-10-09T05:40:38.928957+0000 mon.a (mon.0) 2076 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:40:39.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:39 smithi194 bash[92666]: audit 2023-10-09T05:40:38.963783+0000 mon.a (mon.0) 2077 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:40:39.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:39 smithi194 bash[92666]: cluster 2023-10-09T05:40:38.963869+0000 mon.a (mon.0) 2078 : cluster 0 osdmap e398: 8 total, 8 up, 7 in 2023-10-09T05:40:39.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:39 smithi194 bash[92666]: audit 2023-10-09T05:40:39.093528+0000 mon.a (mon.0) 2079 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:40:39.711 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:39.744 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:39.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:39.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:39.848 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:39.881 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:39.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:39.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:39.985 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:40.006 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:40.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:40.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:40.110 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:40.138 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:40.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:40.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:40.242 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:40.275 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:40.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:40.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:40.379 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:40.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:40 smithi194 bash[92666]: cluster 2023-10-09T05:40:38.962616+0000 osd.5 (osd.5) 433 : cluster 3 failed to encode map e398 with expected crc 2023-10-09T05:40:40.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:40 smithi194 bash[92666]: cluster 2023-10-09T05:40:39.334346+0000 osd.7 (osd.7) 366 : cluster 3 failed to encode map e398 with expected crc 2023-10-09T05:40:40.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:40 smithi194 bash[92666]: cluster 2023-10-09T05:40:39.387556+0000 osd.6 (osd.6) 397 : cluster 0 14.1b scrub starts 2023-10-09T05:40:40.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:40 smithi194 bash[92666]: cluster 2023-10-09T05:40:39.387973+0000 osd.6 (osd.6) 398 : cluster 3 failed to encode map e398 with expected crc 2023-10-09T05:40:40.409 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:40.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:40.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:40.513 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:40.527 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:40 smithi133 bash[102858]: cluster 2023-10-09T05:40:38.962616+0000 osd.5 (osd.5) 433 : cluster 3 failed to encode map e398 with expected crc 2023-10-09T05:40:40.527 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:40 smithi133 bash[102858]: cluster 2023-10-09T05:40:39.334346+0000 osd.7 (osd.7) 366 : cluster 3 failed to encode map e398 with expected crc 2023-10-09T05:40:40.527 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:40 smithi133 bash[102858]: cluster 2023-10-09T05:40:39.387556+0000 osd.6 (osd.6) 397 : cluster 0 14.1b scrub starts 2023-10-09T05:40:40.527 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:40 smithi133 bash[102858]: cluster 2023-10-09T05:40:39.387973+0000 osd.6 (osd.6) 398 : cluster 3 failed to encode map e398 with expected crc 2023-10-09T05:40:40.528 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:40 smithi133 bash[105589]: cluster 2023-10-09T05:40:38.962616+0000 osd.5 (osd.5) 433 : cluster 3 failed to encode map e398 with expected crc 2023-10-09T05:40:40.528 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:40 smithi133 bash[105589]: cluster 2023-10-09T05:40:39.334346+0000 osd.7 (osd.7) 366 : cluster 3 failed to encode map e398 with expected crc 2023-10-09T05:40:40.528 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:40 smithi133 bash[105589]: cluster 2023-10-09T05:40:39.387556+0000 osd.6 (osd.6) 397 : cluster 0 14.1b scrub starts 2023-10-09T05:40:40.528 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:40 smithi133 bash[105589]: cluster 2023-10-09T05:40:39.387973+0000 osd.6 (osd.6) 398 : cluster 3 failed to encode map e398 with expected crc 2023-10-09T05:40:40.545 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:40.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:40.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:40.649 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:40.673 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:40.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:40.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:40.777 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:40.804 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:40.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:40.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:40.907 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:40.932 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:40.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:40.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:41.035 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:41.071 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:41.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:41.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:41.175 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:41.196 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:41.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:41.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:41.299 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:41.329 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:41.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:41.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:41.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:41 smithi194 bash[92666]: cluster 2023-10-09T05:40:40.124306+0000 osd.4 (osd.4) 342 : cluster 3 failed to encode map e398 with expected crc 2023-10-09T05:40:41.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:41 smithi194 bash[92666]: cluster 2023-10-09T05:40:40.490575+0000 mgr.y (mgr.25277) 1588 : cluster 0 pgmap v1772: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:41.432 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:41.450 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:41 smithi133 bash[105589]: cluster 2023-10-09T05:40:40.124306+0000 osd.4 (osd.4) 342 : cluster 3 failed to encode map e398 with expected crc 2023-10-09T05:40:41.450 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:41 smithi133 bash[105589]: cluster 2023-10-09T05:40:40.490575+0000 mgr.y (mgr.25277) 1588 : cluster 0 pgmap v1772: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:41.451 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:41 smithi133 bash[102858]: cluster 2023-10-09T05:40:40.124306+0000 osd.4 (osd.4) 342 : cluster 3 failed to encode map e398 with expected crc 2023-10-09T05:40:41.451 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:41 smithi133 bash[102858]: cluster 2023-10-09T05:40:40.490575+0000 mgr.y (mgr.25277) 1588 : cluster 0 pgmap v1772: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:41.453 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:41.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:41.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:41.557 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:41.586 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:41.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:41.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:41.689 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:41.724 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:41.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:41.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:41.828 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:41.858 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:41.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:41.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:41.962 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:41.987 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:41.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:41.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:42.091 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:42.113 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:42.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:42.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:42.217 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:42.248 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:42.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:42.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:42.352 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:42.378 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:42.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:42.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:42.481 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:42.491 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:42 smithi194 bash[92666]: cluster 2023-10-09T05:40:41.243765+0000 mon.a (mon.0) 2080 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1586/624 objects degraded (254.167%), 2 pgs degraded) 2023-10-09T05:40:42.491 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:42 smithi194 bash[92666]: cluster 2023-10-09T05:40:41.243827+0000 mon.a (mon.0) 2081 : cluster 1 Cluster is now healthy 2023-10-09T05:40:42.513 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:42.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:42.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:42.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:42 smithi133 bash[105589]: cluster 2023-10-09T05:40:41.243765+0000 mon.a (mon.0) 2080 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1586/624 objects degraded (254.167%), 2 pgs degraded) 2023-10-09T05:40:42.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:42 smithi133 bash[105589]: cluster 2023-10-09T05:40:41.243827+0000 mon.a (mon.0) 2081 : cluster 1 Cluster is now healthy 2023-10-09T05:40:42.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:42 smithi133 bash[102858]: cluster 2023-10-09T05:40:41.243765+0000 mon.a (mon.0) 2080 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1586/624 objects degraded (254.167%), 2 pgs degraded) 2023-10-09T05:40:42.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:42 smithi133 bash[102858]: cluster 2023-10-09T05:40:41.243827+0000 mon.a (mon.0) 2081 : cluster 1 Cluster is now healthy 2023-10-09T05:40:42.617 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:42.647 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:42.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:42.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:42.751 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:42.783 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:42.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:42.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:42.887 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:42.915 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:42.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:42.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:43.018 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:43.049 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:43.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:43.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:43.152 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:43.184 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:43.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:43.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:43.200 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:40:43.222 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:40:43.245 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:40:43.261 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:43 smithi133 bash[102858]: cluster 2023-10-09T05:40:42.491539+0000 mgr.y (mgr.25277) 1589 : cluster 0 pgmap v1773: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:43.266 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:40:43.288 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:43.292 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:40:43.319 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:40:43.319 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:43.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:43.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:43.347 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:40:43.369 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:40:43.370 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:40:43.370 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:40:43.371 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:40:43.371 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:40:43.371 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:40:43.371 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:40:43.371 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:40:43.371 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:40:43.423 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:43.451 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:43.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:43.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:43.530 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:43 smithi133 bash[102858]: audit 2023-10-09T05:40:43.177809+0000 mon.a (mon.0) 2082 : audit 0 from='client.? 172.21.15.133:0/2312991609' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:40:43.530 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:43 smithi133 bash[105589]: cluster 2023-10-09T05:40:42.491539+0000 mgr.y (mgr.25277) 1589 : cluster 0 pgmap v1773: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:43.530 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:43 smithi133 bash[105589]: audit 2023-10-09T05:40:43.177809+0000 mon.a (mon.0) 2082 : audit 0 from='client.? 172.21.15.133:0/2312991609' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:40:43.554 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:43.573 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:43.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:43.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:43.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:43 smithi194 bash[92666]: cluster 2023-10-09T05:40:42.491539+0000 mgr.y (mgr.25277) 1589 : cluster 0 pgmap v1773: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:43.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:43 smithi194 bash[92666]: audit 2023-10-09T05:40:43.177809+0000 mon.a (mon.0) 2082 : audit 0 from='client.? 172.21.15.133:0/2312991609' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:40:43.677 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:43.703 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:43.705 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T05:40:43.706 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-10-09T05:40:43.706 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T05:40:43.710 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-10-09T05:40:43.710 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T05:40:43.710 DEBUG:teuthology.orchestra.run.smithi133:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T05:40:43.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:43.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:43.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:43.811 INFO:teuthology.orchestra.run.smithi133.stderr:stat: cannot stat '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T05:40:43.811 DEBUG:teuthology.orchestra.run.smithi133:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T05:40:43.812 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:43.840 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:43.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:43.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:43.878 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T05:40:43.878 DEBUG:teuthology.orchestra.run.smithi133:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T05:40:43.926 INFO:tasks.workunit:timeout=3h 2023-10-09T05:40:43.927 INFO:tasks.workunit:cleanup=True 2023-10-09T05:40:43.927 DEBUG:teuthology.orchestra.run.smithi133:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T05:40:43.943 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:43.978 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:43.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:43.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:44.082 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:44.109 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:44.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:44.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:44.212 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:44.237 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:44.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:44.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:44.340 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:44.341 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:44 smithi133 bash[102858]: audit 2023-10-09T05:40:43.495883+0000 mon.a (mon.0) 2083 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:44.341 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:40:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:40:44] "GET /metrics HTTP/1.1" 200 34110 "" "Prometheus/2.43.0" 2023-10-09T05:40:44.342 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:44 smithi133 bash[105589]: audit 2023-10-09T05:40:43.495883+0000 mon.a (mon.0) 2083 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:44.364 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:44.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:44.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:44.467 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:44.473 INFO:tasks.workunit.client.0.smithi133.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T05:40:44.489 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:44.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:44.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:44.593 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:44.619 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:44.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:44.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:44.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:44 smithi194 bash[92666]: audit 2023-10-09T05:40:43.495883+0000 mon.a (mon.0) 2083 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:44.723 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:44.751 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:44.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:44.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:44.854 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:44.884 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:44.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:44.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:44.988 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:45.010 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:45.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:45.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:45.114 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:45.135 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:45.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:45.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:45.238 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:45.260 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:45.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:45.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:45.364 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:45.392 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:45.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:45.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:45.497 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:45.500 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:45 smithi133 bash[105589]: audit 2023-10-09T05:40:44.347145+0000 mon.a (mon.0) 2084 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:40:45.500 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:45 smithi133 bash[105589]: cluster 2023-10-09T05:40:44.492044+0000 mgr.y (mgr.25277) 1590 : cluster 0 pgmap v1774: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:45.500 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:45 smithi133 bash[102858]: audit 2023-10-09T05:40:44.347145+0000 mon.a (mon.0) 2084 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:40:45.500 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:45 smithi133 bash[102858]: cluster 2023-10-09T05:40:44.492044+0000 mgr.y (mgr.25277) 1590 : cluster 0 pgmap v1774: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:45.522 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:45.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:45.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:45.626 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:45.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:45 smithi194 bash[92666]: audit 2023-10-09T05:40:44.347145+0000 mon.a (mon.0) 2084 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:40:45.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:45 smithi194 bash[92666]: cluster 2023-10-09T05:40:44.492044+0000 mgr.y (mgr.25277) 1590 : cluster 0 pgmap v1774: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:45.654 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:45.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:45.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:45.758 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:45.788 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:45.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:45.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:45.891 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:45.912 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:45.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:45.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:46.016 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:46.035 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:46.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:46.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:46.138 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:47.182 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:46 smithi133 bash[102858]: audit 2023-10-09T05:40:45.538699+0000 mon.a (mon.0) 2085 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:40:47.183 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:46 smithi133 bash[102858]: audit 2023-10-09T05:40:45.545987+0000 mon.a (mon.0) 2086 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:40:47.183 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:46 smithi133 bash[105589]: audit 2023-10-09T05:40:45.538699+0000 mon.a (mon.0) 2085 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:40:47.183 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:46 smithi133 bash[105589]: audit 2023-10-09T05:40:45.545987+0000 mon.a (mon.0) 2086 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:40:47.184 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:46 smithi194 bash[92666]: audit 2023-10-09T05:40:45.538699+0000 mon.a (mon.0) 2085 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:40:47.184 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:46 smithi194 bash[92666]: audit 2023-10-09T05:40:45.545987+0000 mon.a (mon.0) 2086 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:40:47.213 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:47.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:47.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:47.317 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:47.361 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:47.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:47.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:47.464 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:47.503 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:47.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:47.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:47.607 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:47.623 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:47 smithi194 bash[92666]: cluster 2023-10-09T05:40:46.493028+0000 mgr.y (mgr.25277) 1591 : cluster 0 pgmap v1775: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:47.643 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:47.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:47.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:47.747 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:47.774 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:47.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:47.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:47.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:47 smithi133 bash[105589]: cluster 2023-10-09T05:40:46.493028+0000 mgr.y (mgr.25277) 1591 : cluster 0 pgmap v1775: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:47.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:47 smithi133 bash[102858]: cluster 2023-10-09T05:40:46.493028+0000 mgr.y (mgr.25277) 1591 : cluster 0 pgmap v1775: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:47.878 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:47.915 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:47.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:47.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:48.019 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:48.046 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:48.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:48.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:48.150 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:48.173 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:48.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:48.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:48.276 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:48.311 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:48.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:48.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:48.415 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:48.447 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:48.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:48.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:48.551 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:48.574 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:48.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:48.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:48.677 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:48.709 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:48.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:48.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:48.812 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:48.844 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:48.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:48.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:48.948 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:48.983 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:48.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:48.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:49.087 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:49.118 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:49.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:49.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:49.222 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:49.256 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:49.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:49.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:49.359 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:49.384 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:49.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:49.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:49.488 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:49.519 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:49.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:49.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:49.622 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:49.630 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:49 smithi133 bash[102858]: cluster 2023-10-09T05:40:48.493648+0000 mgr.y (mgr.25277) 1592 : cluster 0 pgmap v1776: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:49.630 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:49 smithi133 bash[105589]: cluster 2023-10-09T05:40:48.493648+0000 mgr.y (mgr.25277) 1592 : cluster 0 pgmap v1776: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:49.645 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:49.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:49.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:49.748 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:49.775 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:49.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:49.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:49.878 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:49.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:49 smithi194 bash[92666]: cluster 2023-10-09T05:40:48.493648+0000 mgr.y (mgr.25277) 1592 : cluster 0 pgmap v1776: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:49.887 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:40:49.906 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:49.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:49.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:50.010 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:50.036 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:50.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:50.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:50.139 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:50.160 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:50.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:50.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:50.263 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:50.287 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:50.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:50.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:50.390 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:50.426 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:50.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:50.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:50.530 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:50.561 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:50.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:50.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:50.665 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:50.689 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:50.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:50.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:50.795 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:50.807 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:40:50.807 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:40:50.826 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:50.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:50.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:50.931 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:50.961 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:50.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:50.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:51.010 INFO:teuthology.orchestra.run.smithi133.stdout:moved 0 / 219 (0%) 2023-10-09T05:40:51.010 INFO:teuthology.orchestra.run.smithi133.stdout:avg 31.2857 2023-10-09T05:40:51.010 INFO:teuthology.orchestra.run.smithi133.stdout:stddev 3.28261 -> 3.28261 (expected baseline 5.17845) 2023-10-09T05:40:51.011 INFO:teuthology.orchestra.run.smithi133.stdout:min osd.5 with 26 -> 26 pgs (0.83105 -> 0.83105 * mean) 2023-10-09T05:40:51.011 INFO:teuthology.orchestra.run.smithi133.stdout:max osd.3 with 36 -> 36 pgs (1.15068 -> 1.15068 * mean) 2023-10-09T05:40:51.011 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:40:51.011 INFO:teuthology.orchestra.run.smithi133.stdout:oload 1000 2023-10-09T05:40:51.011 INFO:teuthology.orchestra.run.smithi133.stdout:max_change 1 2023-10-09T05:40:51.011 INFO:teuthology.orchestra.run.smithi133.stdout:max_change_osds 4 2023-10-09T05:40:51.011 INFO:teuthology.orchestra.run.smithi133.stdout:average_utilization 0.0134 2023-10-09T05:40:51.011 INFO:teuthology.orchestra.run.smithi133.stdout:overload_utilization 0.1339 2023-10-09T05:40:51.016 INFO:teuthology.orchestra.run.smithi133.stderr:no change 2023-10-09T05:40:51.066 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:51.102 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:51.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:51.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:51.208 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:51.230 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:51.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:51.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:51.334 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:51.357 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:51.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:51.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:51.460 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:51.481 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:51.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:51.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:51.569 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is set 2023-10-09T05:40:51.585 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:51.618 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:51.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:51.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:51.721 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:51.727 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:51 smithi133 bash[105589]: cluster 2023-10-09T05:40:50.494192+0000 mgr.y (mgr.25277) 1593 : cluster 0 pgmap v1777: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:51.728 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:51 smithi133 bash[105589]: audit 2023-10-09T05:40:50.948308+0000 mon.c (mon.2) 145 : audit 1 from='client.? 172.21.15.133:0/2719989694' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:40:51.728 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:51 smithi133 bash[105589]: audit 2023-10-09T05:40:50.948634+0000 mon.a (mon.0) 2087 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:40:51.728 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:51 smithi133 bash[102858]: cluster 2023-10-09T05:40:50.494192+0000 mgr.y (mgr.25277) 1593 : cluster 0 pgmap v1777: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:51.728 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:51 smithi133 bash[102858]: audit 2023-10-09T05:40:50.948308+0000 mon.c (mon.2) 145 : audit 1 from='client.? 172.21.15.133:0/2719989694' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:40:51.728 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:51 smithi133 bash[102858]: audit 2023-10-09T05:40:50.948634+0000 mon.a (mon.0) 2087 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:40:51.739 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:51.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:51.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:51.842 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:51.861 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:51.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:51.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:51.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:51 smithi194 bash[92666]: cluster 2023-10-09T05:40:50.494192+0000 mgr.y (mgr.25277) 1593 : cluster 0 pgmap v1777: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:51.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:51 smithi194 bash[92666]: audit 2023-10-09T05:40:50.948308+0000 mon.c (mon.2) 145 : audit 1 from='client.? 172.21.15.133:0/2719989694' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:40:51.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:51 smithi194 bash[92666]: audit 2023-10-09T05:40:50.948634+0000 mon.a (mon.0) 2087 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:40:51.963 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:51.996 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:51.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:52.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:52.100 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:52.125 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:52.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:52.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:52.229 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:52.260 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:52.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:52.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:52.364 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:52.388 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:52.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:52.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:52.494 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:52.523 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:52.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:52.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:52.626 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:52.647 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:52.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:52.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:52.750 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:52.757 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:52 smithi133 bash[105589]: audit 2023-10-09T05:40:51.009976+0000 mgr.y (mgr.25277) 1594 : audit 0 from='client.35916 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:40:52.757 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:52 smithi133 bash[105589]: cluster 2023-10-09T05:40:51.551472+0000 mon.a (mon.0) 2088 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:40:52.758 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:52 smithi133 bash[105589]: cluster 2023-10-09T05:40:51.555950+0000 osd.5 (osd.5) 434 : cluster 3 failed to encode map e399 with expected crc 2023-10-09T05:40:52.758 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:52 smithi133 bash[105589]: audit 2023-10-09T05:40:51.557689+0000 mon.a (mon.0) 2089 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:40:52.758 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:52 smithi133 bash[105589]: cluster 2023-10-09T05:40:51.557821+0000 mon.a (mon.0) 2090 : cluster 0 osdmap e399: 8 total, 8 up, 7 in 2023-10-09T05:40:52.758 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:52 smithi133 bash[105589]: cluster 2023-10-09T05:40:51.714024+0000 osd.4 (osd.4) 343 : cluster 3 failed to encode map e399 with expected crc 2023-10-09T05:40:52.758 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:52 smithi133 bash[105589]: cluster 2023-10-09T05:40:51.716986+0000 osd.4 (osd.4) 344 : cluster 3 failed to encode map e399 with expected crc 2023-10-09T05:40:52.758 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:52 smithi133 bash[105589]: cluster 2023-10-09T05:40:51.762469+0000 osd.7 (osd.7) 367 : cluster 3 failed to encode map e399 with expected crc 2023-10-09T05:40:52.758 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:52 smithi133 bash[105589]: cluster 2023-10-09T05:40:51.764937+0000 osd.7 (osd.7) 368 : cluster 3 failed to encode map e399 with expected crc 2023-10-09T05:40:52.758 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:52 smithi133 bash[105589]: audit 2023-10-09T05:40:51.908777+0000 mon.a (mon.0) 2091 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:40:52.759 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:52 smithi133 bash[105589]: audit 2023-10-09T05:40:51.915036+0000 mon.a (mon.0) 2092 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:40:52.759 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:52 smithi133 bash[105589]: audit 2023-10-09T05:40:52.336470+0000 mon.a (mon.0) 2093 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:40:52.759 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:52 smithi133 bash[105589]: audit 2023-10-09T05:40:52.337831+0000 mon.a (mon.0) 2094 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:40:52.759 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:52 smithi133 bash[105589]: audit 2023-10-09T05:40:52.344041+0000 mon.a (mon.0) 2095 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:40:52.759 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:52 smithi133 bash[102858]: audit 2023-10-09T05:40:51.009976+0000 mgr.y (mgr.25277) 1594 : audit 0 from='client.35916 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:40:52.759 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:52 smithi133 bash[102858]: cluster 2023-10-09T05:40:51.551472+0000 mon.a (mon.0) 2088 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:40:52.759 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:52 smithi133 bash[102858]: cluster 2023-10-09T05:40:51.555950+0000 osd.5 (osd.5) 434 : cluster 3 failed to encode map e399 with expected crc 2023-10-09T05:40:52.759 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:52 smithi133 bash[102858]: audit 2023-10-09T05:40:51.557689+0000 mon.a (mon.0) 2089 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:40:52.759 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:52 smithi133 bash[102858]: cluster 2023-10-09T05:40:51.557821+0000 mon.a (mon.0) 2090 : cluster 0 osdmap e399: 8 total, 8 up, 7 in 2023-10-09T05:40:52.760 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:52 smithi133 bash[102858]: cluster 2023-10-09T05:40:51.714024+0000 osd.4 (osd.4) 343 : cluster 3 failed to encode map e399 with expected crc 2023-10-09T05:40:52.760 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:52 smithi133 bash[102858]: cluster 2023-10-09T05:40:51.716986+0000 osd.4 (osd.4) 344 : cluster 3 failed to encode map e399 with expected crc 2023-10-09T05:40:52.760 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:52 smithi133 bash[102858]: cluster 2023-10-09T05:40:51.762469+0000 osd.7 (osd.7) 367 : cluster 3 failed to encode map e399 with expected crc 2023-10-09T05:40:52.760 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:52 smithi133 bash[102858]: cluster 2023-10-09T05:40:51.764937+0000 osd.7 (osd.7) 368 : cluster 3 failed to encode map e399 with expected crc 2023-10-09T05:40:52.760 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:52 smithi133 bash[102858]: audit 2023-10-09T05:40:51.908777+0000 mon.a (mon.0) 2091 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:40:52.760 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:52 smithi133 bash[102858]: audit 2023-10-09T05:40:51.915036+0000 mon.a (mon.0) 2092 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:40:52.760 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:52 smithi133 bash[102858]: audit 2023-10-09T05:40:52.336470+0000 mon.a (mon.0) 2093 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:40:52.760 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:52 smithi133 bash[102858]: audit 2023-10-09T05:40:52.337831+0000 mon.a (mon.0) 2094 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:40:52.760 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:52 smithi133 bash[102858]: audit 2023-10-09T05:40:52.344041+0000 mon.a (mon.0) 2095 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:40:52.773 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:52.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:52.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:52.877 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:52.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:52 smithi194 bash[92666]: audit 2023-10-09T05:40:51.009976+0000 mgr.y (mgr.25277) 1594 : audit 0 from='client.35916 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:40:52.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:52 smithi194 bash[92666]: cluster 2023-10-09T05:40:51.551472+0000 mon.a (mon.0) 2088 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:40:52.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:52 smithi194 bash[92666]: cluster 2023-10-09T05:40:51.555950+0000 osd.5 (osd.5) 434 : cluster 3 failed to encode map e399 with expected crc 2023-10-09T05:40:52.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:52 smithi194 bash[92666]: audit 2023-10-09T05:40:51.557689+0000 mon.a (mon.0) 2089 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:40:52.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:52 smithi194 bash[92666]: cluster 2023-10-09T05:40:51.557821+0000 mon.a (mon.0) 2090 : cluster 0 osdmap e399: 8 total, 8 up, 7 in 2023-10-09T05:40:52.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:52 smithi194 bash[92666]: cluster 2023-10-09T05:40:51.714024+0000 osd.4 (osd.4) 343 : cluster 3 failed to encode map e399 with expected crc 2023-10-09T05:40:52.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:52 smithi194 bash[92666]: cluster 2023-10-09T05:40:51.716986+0000 osd.4 (osd.4) 344 : cluster 3 failed to encode map e399 with expected crc 2023-10-09T05:40:52.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:52 smithi194 bash[92666]: cluster 2023-10-09T05:40:51.762469+0000 osd.7 (osd.7) 367 : cluster 3 failed to encode map e399 with expected crc 2023-10-09T05:40:52.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:52 smithi194 bash[92666]: cluster 2023-10-09T05:40:51.764937+0000 osd.7 (osd.7) 368 : cluster 3 failed to encode map e399 with expected crc 2023-10-09T05:40:52.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:52 smithi194 bash[92666]: audit 2023-10-09T05:40:51.908777+0000 mon.a (mon.0) 2091 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:40:52.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:52 smithi194 bash[92666]: audit 2023-10-09T05:40:51.915036+0000 mon.a (mon.0) 2092 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:40:52.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:52 smithi194 bash[92666]: audit 2023-10-09T05:40:52.336470+0000 mon.a (mon.0) 2093 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:40:52.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:52 smithi194 bash[92666]: audit 2023-10-09T05:40:52.337831+0000 mon.a (mon.0) 2094 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:40:52.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:52 smithi194 bash[92666]: audit 2023-10-09T05:40:52.344041+0000 mon.a (mon.0) 2095 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:40:52.908 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:52.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:52.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:53.012 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:53.043 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:53.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:53.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:53.146 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:53.166 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:53.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:53.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:53.269 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:53.294 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:53.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:53.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:53.398 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:53.421 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:53.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:53.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:53.496 DEBUG:teuthology.orchestra.run.smithi133:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T05:40:53.524 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:53.555 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:53.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:53.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:53.658 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:53.689 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:53.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:53.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:53.792 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:53.797 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:53 smithi133 bash[105589]: cluster 2023-10-09T05:40:52.326335+0000 osd.6 (osd.6) 399 : cluster 3 failed to encode map e399 with expected crc 2023-10-09T05:40:53.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:53 smithi133 bash[105589]: cluster 2023-10-09T05:40:52.494845+0000 mgr.y (mgr.25277) 1595 : cluster 0 pgmap v1779: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:53.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:53 smithi133 bash[102858]: cluster 2023-10-09T05:40:52.326335+0000 osd.6 (osd.6) 399 : cluster 3 failed to encode map e399 with expected crc 2023-10-09T05:40:53.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:53 smithi133 bash[102858]: cluster 2023-10-09T05:40:52.494845+0000 mgr.y (mgr.25277) 1595 : cluster 0 pgmap v1779: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:53.818 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:53.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:53.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:53.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:53 smithi194 bash[92666]: cluster 2023-10-09T05:40:52.326335+0000 osd.6 (osd.6) 399 : cluster 3 failed to encode map e399 with expected crc 2023-10-09T05:40:53.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:53 smithi194 bash[92666]: cluster 2023-10-09T05:40:52.494845+0000 mgr.y (mgr.25277) 1595 : cluster 0 pgmap v1779: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:53.922 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:53.953 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:53.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:53.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:54.014 INFO:tasks.workunit.client.0.smithi133.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T05:40:54.057 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:54.087 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:54.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:54.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:54.192 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:54.216 INFO:tasks.workunit.client.0.smithi133.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T05:40:54.216 INFO:tasks.workunit.client.0.smithi133.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T05:40:54.225 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:54.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:54.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:54.292 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T05:40:54.329 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:54.364 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:54.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:54.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:54.467 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:54.502 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:54.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:54.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:54.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:40:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:40:54] "GET /metrics HTTP/1.1" 200 34110 "" "Prometheus/2.43.0" 2023-10-09T05:40:54.606 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:54.626 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:54.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:54.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:54.728 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:54.761 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:54.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:54.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:54.864 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:54.893 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:54.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:54.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:54.996 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:55.009 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:54 smithi194 bash[92666]: cluster 2023-10-09T05:40:54.495469+0000 mgr.y (mgr.25277) 1596 : cluster 0 pgmap v1780: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:55.027 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:55.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:55.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:55.132 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:55.140 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:54 smithi133 bash[105589]: cluster 2023-10-09T05:40:54.495469+0000 mgr.y (mgr.25277) 1596 : cluster 0 pgmap v1780: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:55.140 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:54 smithi133 bash[102858]: cluster 2023-10-09T05:40:54.495469+0000 mgr.y (mgr.25277) 1596 : cluster 0 pgmap v1780: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:55.155 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:55.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:55.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:55.259 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:55.287 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:55.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:55.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:55.391 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:55.420 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:55.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:55.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:55.523 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:55.561 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:55.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:55.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:55.665 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:55.690 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:55.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:55.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:55.793 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:55.827 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:55.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:55.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:55.932 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:55.968 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:55.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:55.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:56.072 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:56.101 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:56.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:56.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:56.204 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:56.233 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:56.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:56.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:56.337 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:56.367 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:56.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:56.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:56.471 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:56.505 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:56.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:56.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:56.610 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:56.640 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:56.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:56.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:56.743 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:56.771 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:56.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:56.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:56.874 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:56.907 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:56.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:56.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:57.010 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:57.036 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:57.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:57.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:57.139 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:57.175 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:57.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:57.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:57.279 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:57.307 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:57.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:57.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:57.338 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-10-09T05:40:57.339 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 0.05 2023-10-09T05:40:57.413 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:57.442 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:57.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:57.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:57.545 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:57.572 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:57.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:57.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:57.675 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:57.707 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:57.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:57.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:57.790 INFO:tasks.workunit.client.0.smithi133.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T05:40:57.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:57 smithi133 bash[102858]: cluster 2023-10-09T05:40:56.496478+0000 mgr.y (mgr.25277) 1597 : cluster 0 pgmap v1781: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:57.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:57 smithi133 bash[105589]: cluster 2023-10-09T05:40:56.496478+0000 mgr.y (mgr.25277) 1597 : cluster 0 pgmap v1781: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:57.810 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:57.826 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:57 smithi194 bash[92666]: cluster 2023-10-09T05:40:56.496478+0000 mgr.y (mgr.25277) 1597 : cluster 0 pgmap v1781: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:57.845 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:57.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:57.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:57.872 INFO:tasks.workunit.client.0.smithi133.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T05:40:57.929 INFO:tasks.workunit.client.0.smithi133.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T05:40:57.932 INFO:tasks.workunit.client.0.smithi133.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T05:40:57.932 INFO:tasks.workunit.client.0.smithi133.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T05:40:57.949 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:57.979 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:57.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:57.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:57.996 INFO:tasks.workunit.client.0.smithi133.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T05:40:58.006 DEBUG:teuthology.orchestra.run.smithi133:> set -ex 2023-10-09T05:40:58.006 DEBUG:teuthology.orchestra.run.smithi133:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T05:40:58.023 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-10-09T05:40:58.025 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-10-09T05:40:58.025 DEBUG:teuthology.orchestra.run.smithi133:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-10-09T05:40:58.083 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:58.146 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:58.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:58.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:58.158 INFO:tasks.workunit.client.0.smithi133.stdout:Running main() from gmock_main.cc 2023-10-09T05:40:58.159 INFO:tasks.workunit.client.0.smithi133.stdout:[==========] Running 70 tests from 1 test suite. 2023-10-09T05:40:58.159 INFO:tasks.workunit.client.0.smithi133.stdout:[----------] Global test environment set-up. 2023-10-09T05:40:58.159 INFO:tasks.workunit.client.0.smithi133.stdout:[----------] 70 tests from TestClsRbd 2023-10-09T05:40:58.250 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:58.282 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:58.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:58.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:58.386 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:58.414 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:58.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:58.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:58.518 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:58.550 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:58.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:58.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:58.654 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:58.696 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:58.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:58.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:58.799 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:58.831 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:58.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:58.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:58.934 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:58.967 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:58.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:58.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:59.072 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:59.104 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:59.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:59.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:59.208 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:59.239 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:59.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:59.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:59.344 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:59.370 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:40:59.375 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:59.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:59.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:59.478 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:59.506 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:59.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:59.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:59.610 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:59.640 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:59.642 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:59 smithi133 bash[105589]: cluster 2023-10-09T05:40:58.497053+0000 mgr.y (mgr.25277) 1598 : cluster 0 pgmap v1782: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:59.642 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:59 smithi133 bash[105589]: audit 2023-10-09T05:40:58.634625+0000 mon.a (mon.0) 2096 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:40:59.642 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:59 smithi133 bash[105589]: audit 2023-10-09T05:40:58.635865+0000 mon.a (mon.0) 2097 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:59.643 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:59 smithi133 bash[105589]: cluster 2023-10-09T05:40:58.685561+0000 mon.a (mon.0) 2098 : cluster 0 osdmap e400: 8 total, 8 up, 7 in 2023-10-09T05:40:59.643 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:59 smithi133 bash[105589]: cluster 2023-10-09T05:40:58.686940+0000 osd.6 (osd.6) 400 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T05:40:59.643 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:59 smithi133 bash[105589]: cluster 2023-10-09T05:40:58.688129+0000 osd.7 (osd.7) 369 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T05:40:59.643 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:59 smithi133 bash[105589]: cluster 2023-10-09T05:40:58.689853+0000 osd.5 (osd.5) 435 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T05:40:59.643 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:59 smithi133 bash[105589]: cluster 2023-10-09T05:40:58.690247+0000 osd.6 (osd.6) 401 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T05:40:59.643 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:59 smithi133 bash[105589]: cluster 2023-10-09T05:40:58.691336+0000 osd.7 (osd.7) 370 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T05:40:59.643 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:59 smithi133 bash[105589]: audit 2023-10-09T05:40:58.695530+0000 mon.a (mon.0) 2099 : audit 1 from='client.? 172.21.15.133:0/1018273991' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi133-286028-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:40:59.643 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:40:59 smithi133 bash[105589]: cluster 2023-10-09T05:40:58.696723+0000 osd.5 (osd.5) 436 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T05:40:59.643 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:59 smithi133 bash[102858]: cluster 2023-10-09T05:40:58.497053+0000 mgr.y (mgr.25277) 1598 : cluster 0 pgmap v1782: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:59.644 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:59 smithi133 bash[102858]: audit 2023-10-09T05:40:58.634625+0000 mon.a (mon.0) 2096 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:40:59.644 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:59 smithi133 bash[102858]: audit 2023-10-09T05:40:58.635865+0000 mon.a (mon.0) 2097 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:59.644 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:59 smithi133 bash[102858]: cluster 2023-10-09T05:40:58.685561+0000 mon.a (mon.0) 2098 : cluster 0 osdmap e400: 8 total, 8 up, 7 in 2023-10-09T05:40:59.644 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:59 smithi133 bash[102858]: cluster 2023-10-09T05:40:58.686940+0000 osd.6 (osd.6) 400 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T05:40:59.644 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:59 smithi133 bash[102858]: cluster 2023-10-09T05:40:58.688129+0000 osd.7 (osd.7) 369 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T05:40:59.644 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:59 smithi133 bash[102858]: cluster 2023-10-09T05:40:58.689853+0000 osd.5 (osd.5) 435 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T05:40:59.644 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:59 smithi133 bash[102858]: cluster 2023-10-09T05:40:58.690247+0000 osd.6 (osd.6) 401 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T05:40:59.644 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:59 smithi133 bash[102858]: cluster 2023-10-09T05:40:58.691336+0000 osd.7 (osd.7) 370 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T05:40:59.644 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:59 smithi133 bash[102858]: audit 2023-10-09T05:40:58.695530+0000 mon.a (mon.0) 2099 : audit 1 from='client.? 172.21.15.133:0/1018273991' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi133-286028-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:40:59.644 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:40:59 smithi133 bash[102858]: cluster 2023-10-09T05:40:58.696723+0000 osd.5 (osd.5) 436 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T05:40:59.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:59.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:59.693 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.get_all_features 2023-10-09T05:40:59.714 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.get_all_features (21 ms) 2023-10-09T05:40:59.714 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.copyup 2023-10-09T05:40:59.746 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:59.778 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:59.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:59.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:59.855 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:59.881 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:59 smithi194 bash[92666]: cluster 2023-10-09T05:40:58.497053+0000 mgr.y (mgr.25277) 1598 : cluster 0 pgmap v1782: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:40:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:59 smithi194 bash[92666]: audit 2023-10-09T05:40:58.634625+0000 mon.a (mon.0) 2096 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:40:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:59 smithi194 bash[92666]: audit 2023-10-09T05:40:58.635865+0000 mon.a (mon.0) 2097 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:59 smithi194 bash[92666]: cluster 2023-10-09T05:40:58.685561+0000 mon.a (mon.0) 2098 : cluster 0 osdmap e400: 8 total, 8 up, 7 in 2023-10-09T05:40:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:59 smithi194 bash[92666]: cluster 2023-10-09T05:40:58.686940+0000 osd.6 (osd.6) 400 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T05:40:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:59 smithi194 bash[92666]: cluster 2023-10-09T05:40:58.688129+0000 osd.7 (osd.7) 369 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T05:40:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:59 smithi194 bash[92666]: cluster 2023-10-09T05:40:58.689853+0000 osd.5 (osd.5) 435 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T05:40:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:59 smithi194 bash[92666]: cluster 2023-10-09T05:40:58.690247+0000 osd.6 (osd.6) 401 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T05:40:59.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:59 smithi194 bash[92666]: cluster 2023-10-09T05:40:58.691336+0000 osd.7 (osd.7) 370 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T05:40:59.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:59 smithi194 bash[92666]: audit 2023-10-09T05:40:58.695530+0000 mon.a (mon.0) 2099 : audit 1 from='client.? 172.21.15.133:0/1018273991' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi133-286028-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:40:59.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:40:59 smithi194 bash[92666]: cluster 2023-10-09T05:40:58.696723+0000 osd.5 (osd.5) 436 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T05:40:59.905 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:59.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:40:59.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:59.924 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.copyup (210 ms) 2023-10-09T05:40:59.924 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-10-09T05:41:00.009 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:00.037 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:00.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:00.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:00.141 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:00.170 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:00.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:00.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:00.274 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:00.306 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:00.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:00.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:00.350 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.sparse_copyup (426 ms) 2023-10-09T05:41:00.350 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-10-09T05:41:00.410 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:00.439 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.get_and_set_id (89 ms) 2023-10-09T05:41:00.439 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-10-09T05:41:00.442 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:00.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:00.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:00.546 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:00.554 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.add_remove_child (115 ms) 2023-10-09T05:41:00.554 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.directory_methods 2023-10-09T05:41:00.573 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:00.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:00.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:00.677 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:00.712 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:00.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:00.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:00.815 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:00.828 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:00 smithi194 bash[92666]: audit 2023-10-09T05:40:59.689113+0000 mon.a (mon.0) 2100 : audit 1 from='client.? 172.21.15.133:0/1018273991' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi133-286028-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T05:41:00.828 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:00 smithi194 bash[92666]: cluster 2023-10-09T05:40:59.689272+0000 mon.a (mon.0) 2101 : cluster 0 osdmap e401: 8 total, 8 up, 7 in 2023-10-09T05:41:00.828 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:00 smithi194 bash[92666]: cluster 2023-10-09T05:40:59.690028+0000 osd.7 (osd.7) 371 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T05:41:00.828 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:00 smithi194 bash[92666]: cluster 2023-10-09T05:40:59.691297+0000 osd.6 (osd.6) 402 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T05:41:00.828 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:00 smithi194 bash[92666]: cluster 2023-10-09T05:40:59.695200+0000 osd.5 (osd.5) 437 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T05:41:00.828 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:00 smithi194 bash[92666]: cluster 2023-10-09T05:40:59.716251+0000 osd.4 (osd.4) 345 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T05:41:00.829 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:00 smithi194 bash[92666]: cluster 2023-10-09T05:40:59.717759+0000 osd.4 (osd.4) 346 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T05:41:00.829 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:00 smithi194 bash[92666]: cluster 2023-10-09T05:41:00.497809+0000 mgr.y (mgr.25277) 1599 : cluster 0 pgmap v1785: 105 pgs: 12 creating+peering, 18 unknown, 75 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:41:00.850 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:00.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:00.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:00.955 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:00.985 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:00.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:00.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:01.009 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.directory_methods (455 ms) 2023-10-09T05:41:01.009 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.create 2023-10-09T05:41:01.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:00 smithi133 bash[102858]: audit 2023-10-09T05:40:59.689113+0000 mon.a (mon.0) 2100 : audit 1 from='client.? 172.21.15.133:0/1018273991' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi133-286028-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T05:41:01.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:00 smithi133 bash[102858]: cluster 2023-10-09T05:40:59.689272+0000 mon.a (mon.0) 2101 : cluster 0 osdmap e401: 8 total, 8 up, 7 in 2023-10-09T05:41:01.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:00 smithi133 bash[102858]: cluster 2023-10-09T05:40:59.690028+0000 osd.7 (osd.7) 371 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T05:41:01.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:00 smithi133 bash[102858]: cluster 2023-10-09T05:40:59.691297+0000 osd.6 (osd.6) 402 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T05:41:01.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:00 smithi133 bash[102858]: cluster 2023-10-09T05:40:59.695200+0000 osd.5 (osd.5) 437 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T05:41:01.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:00 smithi133 bash[102858]: cluster 2023-10-09T05:40:59.716251+0000 osd.4 (osd.4) 345 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T05:41:01.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:00 smithi133 bash[102858]: cluster 2023-10-09T05:40:59.717759+0000 osd.4 (osd.4) 346 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T05:41:01.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:00 smithi133 bash[102858]: cluster 2023-10-09T05:41:00.497809+0000 mgr.y (mgr.25277) 1599 : cluster 0 pgmap v1785: 105 pgs: 12 creating+peering, 18 unknown, 75 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:41:01.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:00 smithi133 bash[105589]: audit 2023-10-09T05:40:59.689113+0000 mon.a (mon.0) 2100 : audit 1 from='client.? 172.21.15.133:0/1018273991' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi133-286028-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T05:41:01.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:00 smithi133 bash[105589]: cluster 2023-10-09T05:40:59.689272+0000 mon.a (mon.0) 2101 : cluster 0 osdmap e401: 8 total, 8 up, 7 in 2023-10-09T05:41:01.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:00 smithi133 bash[105589]: cluster 2023-10-09T05:40:59.690028+0000 osd.7 (osd.7) 371 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T05:41:01.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:00 smithi133 bash[105589]: cluster 2023-10-09T05:40:59.691297+0000 osd.6 (osd.6) 402 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T05:41:01.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:00 smithi133 bash[105589]: cluster 2023-10-09T05:40:59.695200+0000 osd.5 (osd.5) 437 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T05:41:01.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:00 smithi133 bash[105589]: cluster 2023-10-09T05:40:59.716251+0000 osd.4 (osd.4) 345 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T05:41:01.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:00 smithi133 bash[105589]: cluster 2023-10-09T05:40:59.717759+0000 osd.4 (osd.4) 346 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T05:41:01.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:00 smithi133 bash[105589]: cluster 2023-10-09T05:41:00.497809+0000 mgr.y (mgr.25277) 1599 : cluster 0 pgmap v1785: 105 pgs: 12 creating+peering, 18 unknown, 75 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:41:01.060 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.create (51 ms) 2023-10-09T05:41:01.060 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.get_features 2023-10-09T05:41:01.064 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.get_features (4 ms) 2023-10-09T05:41:01.064 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-10-09T05:41:01.067 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.get_object_prefix (3 ms) 2023-10-09T05:41:01.067 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-10-09T05:41:01.089 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:01.119 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:01.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:01.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:01.223 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:01.252 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.get_create_timestamp (185 ms) 2023-10-09T05:41:01.252 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-10-09T05:41:01.254 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:01.255 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.get_access_timestamp (4 ms) 2023-10-09T05:41:01.255 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-10-09T05:41:01.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:01.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:01.258 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.get_modify_timestamp (2 ms) 2023-10-09T05:41:01.258 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-10-09T05:41:01.263 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.get_data_pool (6 ms) 2023-10-09T05:41:01.263 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.get_size 2023-10-09T05:41:01.270 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.get_size (6 ms) 2023-10-09T05:41:01.270 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.set_size 2023-10-09T05:41:01.279 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.set_size (9 ms) 2023-10-09T05:41:01.279 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.protection_status 2023-10-09T05:41:01.325 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.protection_status (46 ms) 2023-10-09T05:41:01.325 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-10-09T05:41:01.339 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.snapshot_limits (15 ms) 2023-10-09T05:41:01.340 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.parents_v1 2023-10-09T05:41:01.358 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:01.388 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:01.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:01.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:01.415 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.parents_v1 (75 ms) 2023-10-09T05:41:01.415 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.parents_v2 2023-10-09T05:41:01.460 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.parents_v2 (45 ms) 2023-10-09T05:41:01.460 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.snapshots 2023-10-09T05:41:01.492 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:01.504 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.snapshots (44 ms) 2023-10-09T05:41:01.504 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.snapid_race 2023-10-09T05:41:01.513 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.snapid_race (9 ms) 2023-10-09T05:41:01.513 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.stripingv2 2023-10-09T05:41:01.519 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:01.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:01.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:01.529 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.stripingv2 (16 ms) 2023-10-09T05:41:01.529 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.object_map_save 2023-10-09T05:41:01.535 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.object_map_save (7 ms) 2023-10-09T05:41:01.535 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-10-09T05:41:01.564 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.object_map_resize (28 ms) 2023-10-09T05:41:01.564 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.object_map_update 2023-10-09T05:41:01.572 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.object_map_update (8 ms) 2023-10-09T05:41:01.572 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-10-09T05:41:01.572 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2023-10-09T05:41:01.572 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-10-09T05:41:01.581 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.object_map_snap_add (10 ms) 2023-10-09T05:41:01.581 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-10-09T05:41:01.592 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.object_map_snap_remove (10 ms) 2023-10-09T05:41:01.592 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.flags 2023-10-09T05:41:01.601 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.flags (9 ms) 2023-10-09T05:41:01.601 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.metadata 2023-10-09T05:41:01.623 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:01.638 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.metadata (37 ms) 2023-10-09T05:41:01.638 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.set_features 2023-10-09T05:41:01.651 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:01.652 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.set_features (15 ms) 2023-10-09T05:41:01.652 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.mirror 2023-10-09T05:41:01.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:01.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:01.718 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.mirror (64 ms) 2023-10-09T05:41:01.718 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.mirror_image 2023-10-09T05:41:01.746 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.mirror_image (29 ms) 2023-10-09T05:41:01.746 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-10-09T05:41:01.754 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:01.777 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:01.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:01.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:01.880 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:01.890 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:01.891 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:01.916 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:01.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:01.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:01.921 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:01 smithi133 bash[105589]: cluster 2023-10-09T05:41:00.691973+0000 osd.6 (osd.6) 403 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T05:41:01.921 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:01 smithi133 bash[105589]: cluster 2023-10-09T05:41:00.697516+0000 mon.a (mon.0) 2102 : cluster 0 osdmap e402: 8 total, 8 up, 7 in 2023-10-09T05:41:01.922 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:01 smithi133 bash[105589]: cluster 2023-10-09T05:41:00.743237+0000 osd.5 (osd.5) 438 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T05:41:01.922 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:01 smithi133 bash[105589]: cluster 2023-10-09T05:41:00.743237+0000 osd.4 (osd.4) 347 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T05:41:01.922 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:01 smithi133 bash[105589]: cluster 2023-10-09T05:41:00.743239+0000 osd.7 (osd.7) 372 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T05:41:01.922 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:01 smithi133 bash[102858]: cluster 2023-10-09T05:41:00.691973+0000 osd.6 (osd.6) 403 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T05:41:01.922 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:01 smithi133 bash[102858]: cluster 2023-10-09T05:41:00.697516+0000 mon.a (mon.0) 2102 : cluster 0 osdmap e402: 8 total, 8 up, 7 in 2023-10-09T05:41:01.922 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:01 smithi133 bash[102858]: cluster 2023-10-09T05:41:00.743237+0000 osd.5 (osd.5) 438 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T05:41:01.923 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:01 smithi133 bash[102858]: cluster 2023-10-09T05:41:00.743237+0000 osd.4 (osd.4) 347 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T05:41:01.923 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:01 smithi133 bash[102858]: cluster 2023-10-09T05:41:00.743239+0000 osd.7 (osd.7) 372 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T05:41:02.022 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:02.050 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:02.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:02.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:02.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:01 smithi194 bash[92666]: cluster 2023-10-09T05:41:00.691973+0000 osd.6 (osd.6) 403 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T05:41:02.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:01 smithi194 bash[92666]: cluster 2023-10-09T05:41:00.697516+0000 mon.a (mon.0) 2102 : cluster 0 osdmap e402: 8 total, 8 up, 7 in 2023-10-09T05:41:02.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:01 smithi194 bash[92666]: cluster 2023-10-09T05:41:00.743237+0000 osd.5 (osd.5) 438 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T05:41:02.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:01 smithi194 bash[92666]: cluster 2023-10-09T05:41:00.743237+0000 osd.4 (osd.4) 347 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T05:41:02.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:01 smithi194 bash[92666]: cluster 2023-10-09T05:41:00.743239+0000 osd.7 (osd.7) 372 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T05:41:02.153 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:02.180 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:02.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:02.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:02.283 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:02.313 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:02.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:02.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:02.417 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:02.449 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:02.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:02.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:02.552 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:02.585 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:02.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:02.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:02.688 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:02.722 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:02.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:02.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:02.826 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:02.839 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:02 smithi194 bash[92666]: cluster 2023-10-09T05:41:02.498506+0000 mgr.y (mgr.25277) 1600 : cluster 0 pgmap v1787: 105 pgs: 12 creating+peering, 93 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 59 KiB/s rd, 19 KiB/s wr, 78 op/s 2023-10-09T05:41:02.859 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:02.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:02.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:02.962 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:02.995 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:02.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:02.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:03.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:02 smithi133 bash[102858]: cluster 2023-10-09T05:41:02.498506+0000 mgr.y (mgr.25277) 1600 : cluster 0 pgmap v1787: 105 pgs: 12 creating+peering, 93 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 59 KiB/s rd, 19 KiB/s wr, 78 op/s 2023-10-09T05:41:03.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:02 smithi133 bash[105589]: cluster 2023-10-09T05:41:02.498506+0000 mgr.y (mgr.25277) 1600 : cluster 0 pgmap v1787: 105 pgs: 12 creating+peering, 93 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 59 KiB/s rd, 19 KiB/s wr, 78 op/s 2023-10-09T05:41:03.098 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:03.124 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:03.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:03.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:03.231 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:03.264 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:03.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:03.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:03.368 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:03.397 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:03.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:03.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:03.501 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:03.530 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:03.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:03.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:03.634 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:03.661 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:03.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:03.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:03.764 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:03.789 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:03.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:03.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:03.893 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:03.918 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:03.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:03.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:04.022 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:04.044 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:04.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:04.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:04.147 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:04.171 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:04.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:04.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:04.275 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:04.297 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:04.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:04.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:04.401 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:04.423 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:04.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:04.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:04.527 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:04.547 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:41:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:41:04] "GET /metrics HTTP/1.1" 200 34101 "" "Prometheus/2.43.0" 2023-10-09T05:41:04.556 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:04.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:04.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:04.660 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:04.687 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:04.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:04.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:04.790 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:04.814 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:04.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:04.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:04.917 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:04.948 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:04.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:04.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:04.978 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:04 smithi133 bash[105589]: cluster 2023-10-09T05:41:04.499141+0000 mgr.y (mgr.25277) 1601 : cluster 0 pgmap v1788: 105 pgs: 6 creating+peering, 99 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 62 KiB/s rd, 20 KiB/s wr, 82 op/s 2023-10-09T05:41:04.979 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:04 smithi133 bash[102858]: cluster 2023-10-09T05:41:04.499141+0000 mgr.y (mgr.25277) 1601 : cluster 0 pgmap v1788: 105 pgs: 6 creating+peering, 99 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 62 KiB/s rd, 20 KiB/s wr, 82 op/s 2023-10-09T05:41:05.051 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:05.085 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:05.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:05.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:05.189 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:05.200 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:04 smithi194 bash[92666]: cluster 2023-10-09T05:41:04.499141+0000 mgr.y (mgr.25277) 1601 : cluster 0 pgmap v1788: 105 pgs: 6 creating+peering, 99 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 62 KiB/s rd, 20 KiB/s wr, 82 op/s 2023-10-09T05:41:05.216 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:05.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:05.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:05.319 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:05.344 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:05.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:05.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:05.403 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:05.447 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:05.480 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:05.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:05.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:05.584 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:05.611 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:05.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:05.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:05.714 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:05.737 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:05.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:05.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:05.840 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:05.863 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:05.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:05.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:05.966 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:05.991 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:05.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:05.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:06.095 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:06.128 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:06.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:06.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:06.231 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:06.263 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:06.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:06.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:06.317 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.mirror_image_status (4571 ms) 2023-10-09T05:41:06.317 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-10-09T05:41:06.367 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:06.386 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:06.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:06.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:06.488 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.mirror_image_map (171 ms) 2023-10-09T05:41:06.488 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-10-09T05:41:06.489 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:06.501 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.mirror_instances (14 ms) 2023-10-09T05:41:06.501 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-10-09T05:41:06.512 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:06.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:06.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:06.516 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.mirror_snapshot (14 ms) 2023-10-09T05:41:06.516 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-10-09T05:41:06.521 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.group_dir_list (6 ms) 2023-10-09T05:41:06.522 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-10-09T05:41:06.526 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.group_dir_add (4 ms) 2023-10-09T05:41:06.526 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-10-09T05:41:06.531 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.dir_add_already_existing (5 ms) 2023-10-09T05:41:06.531 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-10-09T05:41:06.540 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.group_dir_rename (9 ms) 2023-10-09T05:41:06.540 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-10-09T05:41:06.545 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.group_dir_remove (5 ms) 2023-10-09T05:41:06.545 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-10-09T05:41:06.551 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (6 ms) 2023-10-09T05:41:06.551 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.group_image_add 2023-10-09T05:41:06.556 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.group_image_add (5 ms) 2023-10-09T05:41:06.556 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-10-09T05:41:06.561 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.group_image_remove (5 ms) 2023-10-09T05:41:06.561 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.group_image_list 2023-10-09T05:41:06.564 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.group_image_list (4 ms) 2023-10-09T05:41:06.565 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-10-09T05:41:06.571 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.group_image_clean (6 ms) 2023-10-09T05:41:06.571 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.image_group_add 2023-10-09T05:41:06.575 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.image_group_add (5 ms) 2023-10-09T05:41:06.576 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-10-09T05:41:06.581 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.image_group_remove (5 ms) 2023-10-09T05:41:06.581 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.image_group_get 2023-10-09T05:41:06.585 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.image_group_get (4 ms) 2023-10-09T05:41:06.585 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-10-09T05:41:06.588 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (3 ms) 2023-10-09T05:41:06.588 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-10-09T05:41:06.591 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (3 ms) 2023-10-09T05:41:06.591 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-10-09T05:41:06.594 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (4 ms) 2023-10-09T05:41:06.594 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-10-09T05:41:06.600 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (5 ms) 2023-10-09T05:41:06.600 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-10-09T05:41:06.603 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.group_snap_set (4 ms) 2023-10-09T05:41:06.603 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-10-09T05:41:06.610 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.group_snap_list (6 ms) 2023-10-09T05:41:06.610 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-10-09T05:41:06.615 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:06.631 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (21 ms) 2023-10-09T05:41:06.631 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-10-09T05:41:06.647 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:06.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:06.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:06.751 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:06.779 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:06.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:06.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:06.882 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:06.902 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:06.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:06.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:07.005 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:07.034 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:07.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:07.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:07.071 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (440 ms) 2023-10-09T05:41:07.071 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-10-09T05:41:07.076 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.group_snap_remove (5 ms) 2023-10-09T05:41:07.076 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-10-09T05:41:07.079 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (3 ms) 2023-10-09T05:41:07.079 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.trash_methods 2023-10-09T05:41:07.091 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.trash_methods (12 ms) 2023-10-09T05:41:07.091 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.op_features 2023-10-09T05:41:07.101 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.op_features (10 ms) 2023-10-09T05:41:07.101 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.clone_parent 2023-10-09T05:41:07.125 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.clone_parent (24 ms) 2023-10-09T05:41:07.125 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-10-09T05:41:07.138 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:07.140 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.clone_parent_ns (16 ms) 2023-10-09T05:41:07.141 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.clone_child 2023-10-09T05:41:07.159 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.clone_child (18 ms) 2023-10-09T05:41:07.159 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-10-09T05:41:07.166 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:07.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:07.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:07.171 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.namespace_methods (12 ms) 2023-10-09T05:41:07.171 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.migration 2023-10-09T05:41:07.182 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.migration (12 ms) 2023-10-09T05:41:07.183 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.migration_v1 2023-10-09T05:41:07.191 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.migration_v1 (8 ms) 2023-10-09T05:41:07.191 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-10-09T05:41:07.269 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:07.299 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:07.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:07.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:07.403 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:07.428 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:07.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:07.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:07.535 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:07.569 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:07.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:07.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:07.672 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:07.686 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:07 smithi133 bash[102858]: cluster 2023-10-09T05:41:06.500111+0000 mgr.y (mgr.25277) 1602 : cluster 0 pgmap v1789: 105 pgs: 105 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 1.6 MiB/s rd, 909 KiB/s wr, 988 op/s 2023-10-09T05:41:07.687 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:07 smithi133 bash[105589]: cluster 2023-10-09T05:41:06.500111+0000 mgr.y (mgr.25277) 1602 : cluster 0 pgmap v1789: 105 pgs: 105 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 1.6 MiB/s rd, 909 KiB/s wr, 988 op/s 2023-10-09T05:41:07.704 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:07.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:07.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:07.807 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:07.830 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:07.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:07.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:07.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:07 smithi194 bash[92666]: cluster 2023-10-09T05:41:06.500111+0000 mgr.y (mgr.25277) 1602 : cluster 0 pgmap v1789: 105 pgs: 105 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 1.6 MiB/s rd, 909 KiB/s wr, 988 op/s 2023-10-09T05:41:07.934 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:07.969 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:07.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:07.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:08.073 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:08.101 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:08.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:08.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:08.204 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:08.236 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:08.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:08.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:08.340 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:08.365 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:08.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:08.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:08.467 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:08.499 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:08.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:08.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:08.579 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1388 ms) 2023-10-09T05:41:08.579 INFO:tasks.workunit.client.0.smithi133.stdout:[ RUN ] TestClsRbd.sparsify 2023-10-09T05:41:08.603 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:08.629 INFO:tasks.workunit.client.0.smithi133.stdout:[ OK ] TestClsRbd.sparsify (50 ms) 2023-10-09T05:41:08.632 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:08.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:08.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:08.736 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:08.746 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:08 smithi133 bash[102858]: cluster 2023-10-09T05:41:07.557507+0000 osd.5 (osd.5) 439 : cluster 3 failed to encode map e403 with expected crc 2023-10-09T05:41:08.747 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:08 smithi133 bash[102858]: cluster 2023-10-09T05:41:07.560305+0000 mon.a (mon.0) 2103 : cluster 0 osdmap e403: 8 total, 8 up, 7 in 2023-10-09T05:41:08.747 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:08 smithi133 bash[105589]: cluster 2023-10-09T05:41:07.557507+0000 osd.5 (osd.5) 439 : cluster 3 failed to encode map e403 with expected crc 2023-10-09T05:41:08.747 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:08 smithi133 bash[105589]: cluster 2023-10-09T05:41:07.560305+0000 mon.a (mon.0) 2103 : cluster 0 osdmap e403: 8 total, 8 up, 7 in 2023-10-09T05:41:08.762 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:08.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:08.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:08.865 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:08.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:08 smithi194 bash[92666]: cluster 2023-10-09T05:41:07.557507+0000 osd.5 (osd.5) 439 : cluster 3 failed to encode map e403 with expected crc 2023-10-09T05:41:08.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:08 smithi194 bash[92666]: cluster 2023-10-09T05:41:07.560305+0000 mon.a (mon.0) 2103 : cluster 0 osdmap e403: 8 total, 8 up, 7 in 2023-10-09T05:41:08.887 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:08.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:08.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:08.990 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:09.021 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:09.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:09.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:09.125 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:09.159 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:09.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:09.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:09.263 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:09.293 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:09.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:09.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:09.398 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:09.425 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:09.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:09.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:09.529 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:09.559 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:09.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:09.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:09.592 INFO:tasks.workunit.client.0.smithi133.stdout:[----------] 70 tests from TestClsRbd (8936 ms total) 2023-10-09T05:41:09.592 INFO:tasks.workunit.client.0.smithi133.stdout: 2023-10-09T05:41:09.592 INFO:tasks.workunit.client.0.smithi133.stdout:[----------] Global test environment tear-down 2023-10-09T05:41:09.593 INFO:tasks.workunit.client.0.smithi133.stdout:[==========] 70 tests from 1 test suite ran. (11430 ms total) 2023-10-09T05:41:09.593 INFO:tasks.workunit.client.0.smithi133.stdout:[ PASSED ] 70 tests. 2023-10-09T05:41:09.593 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T05:41:09.593 DEBUG:teuthology.orchestra.run.smithi133:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T05:41:09.604 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-10-09T05:41:09.604 DEBUG:teuthology.orchestra.run.smithi133:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T05:41:09.664 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:09.694 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:09.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:09.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:09.797 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:09.817 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:09.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:09.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:09.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:09 smithi194 bash[92666]: cluster 2023-10-09T05:41:08.500796+0000 mgr.y (mgr.25277) 1603 : cluster 0 pgmap v1791: 105 pgs: 105 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 1.6 MiB/s rd, 889 KiB/s wr, 968 op/s 2023-10-09T05:41:09.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:09 smithi194 bash[92666]: cluster 2023-10-09T05:41:08.572186+0000 osd.6 (osd.6) 404 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T05:41:09.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:09 smithi194 bash[92666]: cluster 2023-10-09T05:41:08.577889+0000 mon.a (mon.0) 2104 : cluster 0 osdmap e404: 8 total, 8 up, 7 in 2023-10-09T05:41:09.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:09 smithi194 bash[92666]: cluster 2023-10-09T05:41:08.649364+0000 osd.4 (osd.4) 348 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T05:41:09.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:09 smithi194 bash[92666]: cluster 2023-10-09T05:41:08.649364+0000 osd.5 (osd.5) 440 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T05:41:09.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:09 smithi194 bash[92666]: cluster 2023-10-09T05:41:08.649489+0000 osd.7 (osd.7) 373 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T05:41:09.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:09 smithi194 bash[92666]: audit 2023-10-09T05:41:09.361598+0000 mon.a (mon.0) 2105 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:41:09.887 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:09 smithi133 bash[105589]: cluster 2023-10-09T05:41:08.500796+0000 mgr.y (mgr.25277) 1603 : cluster 0 pgmap v1791: 105 pgs: 105 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 1.6 MiB/s rd, 889 KiB/s wr, 968 op/s 2023-10-09T05:41:09.887 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:09 smithi133 bash[105589]: cluster 2023-10-09T05:41:08.572186+0000 osd.6 (osd.6) 404 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T05:41:09.887 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:09 smithi133 bash[105589]: cluster 2023-10-09T05:41:08.577889+0000 mon.a (mon.0) 2104 : cluster 0 osdmap e404: 8 total, 8 up, 7 in 2023-10-09T05:41:09.887 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:09 smithi133 bash[105589]: cluster 2023-10-09T05:41:08.649364+0000 osd.4 (osd.4) 348 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T05:41:09.887 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:09 smithi133 bash[105589]: cluster 2023-10-09T05:41:08.649364+0000 osd.5 (osd.5) 440 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T05:41:09.887 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:09 smithi133 bash[105589]: cluster 2023-10-09T05:41:08.649489+0000 osd.7 (osd.7) 373 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T05:41:09.888 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:09 smithi133 bash[105589]: audit 2023-10-09T05:41:09.361598+0000 mon.a (mon.0) 2105 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:41:09.888 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:09 smithi133 bash[102858]: cluster 2023-10-09T05:41:08.500796+0000 mgr.y (mgr.25277) 1603 : cluster 0 pgmap v1791: 105 pgs: 105 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 1.6 MiB/s rd, 889 KiB/s wr, 968 op/s 2023-10-09T05:41:09.888 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:09 smithi133 bash[102858]: cluster 2023-10-09T05:41:08.572186+0000 osd.6 (osd.6) 404 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T05:41:09.888 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:09 smithi133 bash[102858]: cluster 2023-10-09T05:41:08.577889+0000 mon.a (mon.0) 2104 : cluster 0 osdmap e404: 8 total, 8 up, 7 in 2023-10-09T05:41:09.888 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:09 smithi133 bash[102858]: cluster 2023-10-09T05:41:08.649364+0000 osd.4 (osd.4) 348 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T05:41:09.888 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:09 smithi133 bash[102858]: cluster 2023-10-09T05:41:08.649364+0000 osd.5 (osd.5) 440 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T05:41:09.888 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:09 smithi133 bash[102858]: cluster 2023-10-09T05:41:08.649489+0000 osd.7 (osd.7) 373 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T05:41:09.888 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:09 smithi133 bash[102858]: audit 2023-10-09T05:41:09.361598+0000 mon.a (mon.0) 2105 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:41:09.920 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:09.942 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:09.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:09.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:09.950 DEBUG:teuthology.parallel:result is None 2023-10-09T05:41:09.951 DEBUG:teuthology.orchestra.run.smithi133:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T05:41:10.001 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T05:41:10.001 DEBUG:teuthology.orchestra.run.smithi133:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T05:41:10.046 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:10.079 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:10.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:10.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:10.143 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T05:41:10.143 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T05:41:10.143 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-10-09T05:41:10.144 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T05:41:10.144 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-10-09T05:41:10.144 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T05:41:10.144 DEBUG:teuthology.orchestra.run.smithi133:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T05:41:10.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:10.152 INFO:teuthology.orchestra.run.smithi133.stderr:stat: cannot stat '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T05:41:10.152 DEBUG:teuthology.orchestra.run.smithi133:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T05:41:10.184 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:10.202 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T05:41:10.202 DEBUG:teuthology.orchestra.run.smithi133:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T05:41:10.217 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:10.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:10.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:10.222 INFO:tasks.workunit:timeout=3h 2023-10-09T05:41:10.222 INFO:tasks.workunit:cleanup=True 2023-10-09T05:41:10.223 DEBUG:teuthology.orchestra.run.smithi133:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T05:41:10.321 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:10.327 INFO:tasks.workunit.client.0.smithi133.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T05:41:10.342 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:10.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:10.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:10.445 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:10.465 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:10.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:10.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:10.559 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:41:10.569 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:10.586 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:41:10.596 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:10.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:10.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:10.611 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:41:10.637 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:41:10.668 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:41:10.692 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:41:10.700 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:10.713 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:10 smithi133 bash[102858]: cluster 2023-10-09T05:41:09.583135+0000 osd.6 (osd.6) 405 : cluster 3 failed to encode map e405 with expected crc 2023-10-09T05:41:10.713 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:10 smithi133 bash[102858]: cluster 2023-10-09T05:41:09.583210+0000 osd.5 (osd.5) 441 : cluster 3 failed to encode map e405 with expected crc 2023-10-09T05:41:10.713 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:10 smithi133 bash[102858]: cluster 2023-10-09T05:41:09.585610+0000 mon.a (mon.0) 2106 : cluster 0 osdmap e405: 8 total, 8 up, 7 in 2023-10-09T05:41:10.713 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:10 smithi133 bash[102858]: cluster 2023-10-09T05:41:09.866714+0000 osd.7 (osd.7) 374 : cluster 3 failed to encode map e405 with expected crc 2023-10-09T05:41:10.713 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:10 smithi133 bash[102858]: audit 2023-10-09T05:41:10.530033+0000 mon.c (mon.2) 146 : audit 0 from='client.? 172.21.15.133:0/931602622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:41:10.713 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:10 smithi133 bash[105589]: cluster 2023-10-09T05:41:09.583135+0000 osd.6 (osd.6) 405 : cluster 3 failed to encode map e405 with expected crc 2023-10-09T05:41:10.714 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:10 smithi133 bash[105589]: cluster 2023-10-09T05:41:09.583210+0000 osd.5 (osd.5) 441 : cluster 3 failed to encode map e405 with expected crc 2023-10-09T05:41:10.714 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:10 smithi133 bash[105589]: cluster 2023-10-09T05:41:09.585610+0000 mon.a (mon.0) 2106 : cluster 0 osdmap e405: 8 total, 8 up, 7 in 2023-10-09T05:41:10.714 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:10 smithi133 bash[105589]: cluster 2023-10-09T05:41:09.866714+0000 osd.7 (osd.7) 374 : cluster 3 failed to encode map e405 with expected crc 2023-10-09T05:41:10.714 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:10 smithi133 bash[105589]: audit 2023-10-09T05:41:10.530033+0000 mon.c (mon.2) 146 : audit 0 from='client.? 172.21.15.133:0/931602622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:41:10.715 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:41:10.726 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:10.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:10.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:10.739 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:41:10.741 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:41:10.741 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:41:10.741 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:41:10.741 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:41:10.741 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:41:10.742 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:41:10.742 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:41:10.742 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:41:10.829 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:10.843 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:10 smithi194 bash[92666]: cluster 2023-10-09T05:41:09.583135+0000 osd.6 (osd.6) 405 : cluster 3 failed to encode map e405 with expected crc 2023-10-09T05:41:10.843 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:10 smithi194 bash[92666]: cluster 2023-10-09T05:41:09.583210+0000 osd.5 (osd.5) 441 : cluster 3 failed to encode map e405 with expected crc 2023-10-09T05:41:10.843 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:10 smithi194 bash[92666]: cluster 2023-10-09T05:41:09.585610+0000 mon.a (mon.0) 2106 : cluster 0 osdmap e405: 8 total, 8 up, 7 in 2023-10-09T05:41:10.843 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:10 smithi194 bash[92666]: cluster 2023-10-09T05:41:09.866714+0000 osd.7 (osd.7) 374 : cluster 3 failed to encode map e405 with expected crc 2023-10-09T05:41:10.843 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:10 smithi194 bash[92666]: audit 2023-10-09T05:41:10.530033+0000 mon.c (mon.2) 146 : audit 0 from='client.? 172.21.15.133:0/931602622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:41:10.862 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:10.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:10.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:10.966 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:10.990 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:10.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:10.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:11.095 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:11.116 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:11.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:11.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:11.219 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:11.250 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:11.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:11.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:11.354 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:11.387 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:11.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:11.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:11.491 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:11.522 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:11.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:11.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:11.626 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:11.655 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:11.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:11.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:11.760 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:11.796 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:11.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:11 smithi133 bash[102858]: cluster 2023-10-09T05:41:10.501280+0000 mgr.y (mgr.25277) 1604 : cluster 0 pgmap v1794: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:41:11.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:11 smithi133 bash[102858]: cluster 2023-10-09T05:41:11.000151+0000 osd.4 (osd.4) 349 : cluster 3 failed to encode map e405 with expected crc 2023-10-09T05:41:11.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:11 smithi133 bash[105589]: cluster 2023-10-09T05:41:10.501280+0000 mgr.y (mgr.25277) 1604 : cluster 0 pgmap v1794: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:41:11.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:11 smithi133 bash[105589]: cluster 2023-10-09T05:41:11.000151+0000 osd.4 (osd.4) 349 : cluster 3 failed to encode map e405 with expected crc 2023-10-09T05:41:11.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:11.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:11.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:11 smithi194 bash[92666]: cluster 2023-10-09T05:41:10.501280+0000 mgr.y (mgr.25277) 1604 : cluster 0 pgmap v1794: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:41:11.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:11 smithi194 bash[92666]: cluster 2023-10-09T05:41:11.000151+0000 osd.4 (osd.4) 349 : cluster 3 failed to encode map e405 with expected crc 2023-10-09T05:41:11.901 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:11.929 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:11.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:11.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:12.032 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:12.057 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:12.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:12.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:12.160 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:12.186 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:12.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:12.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:12.289 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:12.311 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:12.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:12.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:12.415 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:12.445 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:12.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:12.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:12.550 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:12.575 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:12.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:12.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:12.678 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:12.707 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:12.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:12.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:12.811 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:12.844 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:12.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:12.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:12.948 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:12.979 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:12.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:12.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:13.083 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:13.118 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:13.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:13.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:13.222 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:13.248 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:13.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:13.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:13.351 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:13.376 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:13.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:13.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:13.479 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:13.504 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:13.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:13.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:13.607 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:13.632 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:13.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:13.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:13.736 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:13.742 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:13 smithi133 bash[105589]: cluster 2023-10-09T05:41:12.501901+0000 mgr.y (mgr.25277) 1605 : cluster 0 pgmap v1795: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:41:13.743 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:13 smithi133 bash[105589]: audit 2023-10-09T05:41:13.496882+0000 mon.a (mon.0) 2107 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:13.743 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:13 smithi133 bash[102858]: cluster 2023-10-09T05:41:12.501901+0000 mgr.y (mgr.25277) 1605 : cluster 0 pgmap v1795: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:41:13.743 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:13 smithi133 bash[102858]: audit 2023-10-09T05:41:13.496882+0000 mon.a (mon.0) 2107 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:13.756 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:13.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:13.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:13.860 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:13.878 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:13.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:13.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:13.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:13 smithi194 bash[92666]: cluster 2023-10-09T05:41:12.501901+0000 mgr.y (mgr.25277) 1605 : cluster 0 pgmap v1795: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:41:13.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:13 smithi194 bash[92666]: audit 2023-10-09T05:41:13.496882+0000 mon.a (mon.0) 2107 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:13.981 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:14.013 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:14.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:14.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:14.117 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:14.149 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:14.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:14.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:14.253 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:14.275 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:14.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:14.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:14.379 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:14.412 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:14.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:14.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:14.515 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:14.539 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:41:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:41:14] "GET /metrics HTTP/1.1" 200 34101 "" "Prometheus/2.43.0" 2023-10-09T05:41:14.546 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:14.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:14.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:14.649 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:14.685 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:14.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:14.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:14.789 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:14.820 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:14.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:14.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:14.924 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:14.946 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:14.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:14.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:15.050 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:15.076 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:15.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:15.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:15.180 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:15.206 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:15.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:15.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:15.288 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:14 smithi133 bash[105589]: cluster 2023-10-09T05:41:14.502599+0000 mgr.y (mgr.25277) 1606 : cluster 0 pgmap v1796: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:41:15.288 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:14 smithi133 bash[102858]: cluster 2023-10-09T05:41:14.502599+0000 mgr.y (mgr.25277) 1606 : cluster 0 pgmap v1796: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:41:15.310 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:15.322 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:14 smithi194 bash[92666]: cluster 2023-10-09T05:41:14.502599+0000 mgr.y (mgr.25277) 1606 : cluster 0 pgmap v1796: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:41:15.348 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:15.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:15.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:15.451 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:15.474 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:15.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:15.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:15.576 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:15.596 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:15.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:15.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:15.700 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:15.720 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:15.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:15.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:15.823 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:15.854 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:15.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:15.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:15.894 DEBUG:teuthology.orchestra.run.smithi133:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T05:41:15.946 INFO:tasks.workunit.client.0.smithi133.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T05:41:15.949 INFO:tasks.workunit.client.0.smithi133.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T05:41:15.949 INFO:tasks.workunit.client.0.smithi133.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T05:41:15.957 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:15.982 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:15.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:15.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:16.086 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:16.118 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:16.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:16.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:16.221 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:16.244 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:16.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:16.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:16.348 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:16.381 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:16.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:16.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:16.435 INFO:tasks.workunit.client.0.smithi133.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T05:41:16.484 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:16.508 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:16.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:16.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:16.512 INFO:tasks.workunit.client.0.smithi133.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T05:41:16.570 INFO:tasks.workunit.client.0.smithi133.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T05:41:16.573 INFO:tasks.workunit.client.0.smithi133.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T05:41:16.573 INFO:tasks.workunit.client.0.smithi133.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T05:41:16.612 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:16.639 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:16.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:16.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:16.650 INFO:tasks.workunit.client.0.smithi133.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T05:41:16.656 DEBUG:teuthology.orchestra.run.smithi133:> set -ex 2023-10-09T05:41:16.656 DEBUG:teuthology.orchestra.run.smithi133:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T05:41:16.703 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-10-09T05:41:16.704 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-10-09T05:41:16.704 DEBUG:teuthology.orchestra.run.smithi133:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-10-09T05:41:16.742 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:16.808 INFO:tasks.workunit.client.0.smithi133.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-10-09T05:41:16.808 INFO:tasks.workunit.client.0.smithi133.stderr:+ TMPDIR=/tmp/rbd_import_export_287447 2023-10-09T05:41:16.809 INFO:tasks.workunit.client.0.smithi133.stderr:+ rm -rf /tmp/rbd_import_export_287447 2023-10-09T05:41:16.809 INFO:tasks.workunit.client.0.smithi133.stderr:+ mkdir /tmp/rbd_import_export_287447 2023-10-09T05:41:16.810 INFO:tasks.workunit.client.0.smithi133.stderr:+ trap rm -rf /tmp/rbd_import_export_287447 INT TERM EXIT 2023-10-09T05:41:16.810 INFO:tasks.workunit.client.0.smithi133.stderr:+ mkdir foo.287447 2023-10-09T05:41:16.811 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd import foo.287447 foo.dir 2023-10-09T05:41:16.825 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:16.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:16.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:16.929 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:16.955 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:16.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:16.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:17.059 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:17.090 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:17.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:17.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:17.193 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:17.226 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:17.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:17.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:17.330 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:17.359 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:17.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:17.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:17.386 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:41:17.464 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:17.496 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:17.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:17.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:17.600 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:17.619 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:17 smithi133 bash[102858]: cluster 2023-10-09T05:41:16.503232+0000 mgr.y (mgr.25277) 1607 : cluster 0 pgmap v1797: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:41:17.619 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:17 smithi133 bash[105589]: cluster 2023-10-09T05:41:16.503232+0000 mgr.y (mgr.25277) 1607 : cluster 0 pgmap v1797: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:41:17.639 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:17.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:17.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:17.742 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:17.754 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:17 smithi194 bash[92666]: cluster 2023-10-09T05:41:16.503232+0000 mgr.y (mgr.25277) 1607 : cluster 0 pgmap v1797: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:41:17.774 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:17.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:17.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:17.879 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:17.912 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:17.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:17.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:18.016 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:18.049 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:18.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:18.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:18.152 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:18.184 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:18.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:18.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:18.288 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:18.316 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:18.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:18.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:18.419 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:18.440 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:18.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:18.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:18.544 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:18.573 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:18.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:18.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:18.677 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:18.707 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:18.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:18.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:18.811 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:18.844 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:18.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:18.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:18.947 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:18.973 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:18.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:18.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:19.080 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:19.102 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:19.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:19.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:19.206 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:19.237 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:19.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:19.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:19.341 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:19.363 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:19.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:19.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:19.467 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:19.491 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:19.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:19.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:19.595 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:19.626 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:19.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:19.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:19.696 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:19 smithi133 bash[102858]: cluster 2023-10-09T05:41:18.503947+0000 mgr.y (mgr.25277) 1608 : cluster 0 pgmap v1798: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:41:19.730 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:19.751 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:19.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:19.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:19.854 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:19.876 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:19.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:19.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:19.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:19 smithi194 bash[92666]: cluster 2023-10-09T05:41:18.503947+0000 mgr.y (mgr.25277) 1608 : cluster 0 pgmap v1798: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:41:19.980 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:19.993 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:19 smithi133 bash[105589]: cluster 2023-10-09T05:41:18.503947+0000 mgr.y (mgr.25277) 1608 : cluster 0 pgmap v1798: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:41:20.010 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:20.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:20.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:20.113 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:20.142 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:20.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:20.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:20.246 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:20.268 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:20.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:20.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:20.371 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:20.395 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:20.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:20.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:20.498 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:20.533 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:20.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:20.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:20.638 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:20.670 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:20.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:20.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:20.734 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:41:20.734 INFO:teuthology.orchestra.run.smithi133.stdout: "ops": [], 2023-10-09T05:41:20.734 INFO:teuthology.orchestra.run.smithi133.stdout: "num_ops": 0 2023-10-09T05:41:20.734 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:41:20.775 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:20.808 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:20.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:20.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:20.912 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:20.934 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:20.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:20.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:20.988 INFO:tasks.workunit.client.0.smithi133.stderr:rbd: cannot import a directory 2023-10-09T05:41:20.989 INFO:tasks.workunit.client.0.smithi133.stderr: Importing image: 0% complete...failed. 2023-10-09T05:41:20.989 INFO:tasks.workunit.client.0.smithi133.stderr:rbd: import failed: (21) Is a directory 2023-10-09T05:41:21.037 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:21.048 INFO:tasks.workunit.client.0.smithi133.stderr:+ true 2023-10-09T05:41:21.048 INFO:tasks.workunit.client.0.smithi133.stderr:+ rmdir foo.287447 2023-10-09T05:41:21.049 INFO:tasks.workunit.client.0.smithi133.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_287447/img bs=1k count=1 seek=10 2023-10-09T05:41:21.051 INFO:tasks.workunit.client.0.smithi133.stderr:1+0 records in 2023-10-09T05:41:21.051 INFO:tasks.workunit.client.0.smithi133.stderr:1+0 records out 2023-10-09T05:41:21.051 INFO:tasks.workunit.client.0.smithi133.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 0.000192164 s, 5.3 MB/s 2023-10-09T05:41:21.051 INFO:tasks.workunit.client.0.smithi133.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_287447/img bs=1k count=10 seek=100 2023-10-09T05:41:21.052 INFO:tasks.workunit.client.0.smithi133.stderr:10+0 records in 2023-10-09T05:41:21.052 INFO:tasks.workunit.client.0.smithi133.stderr:10+0 records out 2023-10-09T05:41:21.052 INFO:tasks.workunit.client.0.smithi133.stderr:10240 bytes (10 kB, 10 KiB) copied, 0.000223897 s, 45.7 MB/s 2023-10-09T05:41:21.052 INFO:tasks.workunit.client.0.smithi133.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_287447/img bs=1k count=100 seek=1000 2023-10-09T05:41:21.053 INFO:tasks.workunit.client.0.smithi133.stderr:70+1 records in 2023-10-09T05:41:21.053 INFO:tasks.workunit.client.0.smithi133.stderr:70+1 records out 2023-10-09T05:41:21.054 INFO:tasks.workunit.client.0.smithi133.stderr:72056 bytes (72 kB, 70 KiB) copied, 0.000256215 s, 281 MB/s 2023-10-09T05:41:21.054 INFO:tasks.workunit.client.0.smithi133.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_287447/img bs=1k seek=10000 2023-10-09T05:41:21.056 INFO:tasks.workunit.client.0.smithi133.stderr:138+1 records in 2023-10-09T05:41:21.056 INFO:tasks.workunit.client.0.smithi133.stderr:138+1 records out 2023-10-09T05:41:21.056 INFO:tasks.workunit.client.0.smithi133.stderr:142144 bytes (142 kB, 139 KiB) copied, 0.00107544 s, 132 MB/s 2023-10-09T05:41:21.056 INFO:tasks.workunit.client.0.smithi133.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_287447/img bs=1k seek=100000 2023-10-09T05:41:21.072 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:21.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:21.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:21.090 INFO:tasks.workunit.client.0.smithi133.stderr:74+1 records in 2023-10-09T05:41:21.090 INFO:tasks.workunit.client.0.smithi133.stderr:74+1 records out 2023-10-09T05:41:21.091 INFO:tasks.workunit.client.0.smithi133.stderr:76160 bytes (76 kB, 74 KiB) copied, 0.0333297 s, 2.3 MB/s 2023-10-09T05:41:21.091 INFO:tasks.workunit.client.0.smithi133.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_287447/img bs=1k seek=1000000 2023-10-09T05:41:21.141 INFO:tasks.workunit.client.0.smithi133.stderr:194+1 records in 2023-10-09T05:41:21.141 INFO:tasks.workunit.client.0.smithi133.stderr:194+1 records out 2023-10-09T05:41:21.141 INFO:tasks.workunit.client.0.smithi133.stderr:199136 bytes (199 kB, 194 KiB) copied, 0.049228 s, 4.0 MB/s 2023-10-09T05:41:21.141 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd rm testimg 2023-10-09T05:41:21.176 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:21.210 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:21.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:21.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:21.294 INFO:tasks.workunit.client.0.smithi133.stderr: Removing image: 0% complete...failed. 2023-10-09T05:41:21.294 INFO:tasks.workunit.client.0.smithi133.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T05:41:21.314 INFO:tasks.workunit.client.0.smithi133.stderr:+ true 2023-10-09T05:41:21.314 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd import --new-format /tmp/rbd_import_export_287447/img testimg 2023-10-09T05:41:21.315 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:21.338 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:21.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:21.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:21.442 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:21.474 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:21.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:21.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:21.578 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:21.597 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:21.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:21.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:21.630 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:21 smithi133 bash[102858]: cluster 2023-10-09T05:41:20.504541+0000 mgr.y (mgr.25277) 1609 : cluster 0 pgmap v1799: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:41:21.700 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:21.733 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:21.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:21.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:21.838 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:21.857 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:21.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:21.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:21.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:21 smithi194 bash[92666]: cluster 2023-10-09T05:41:20.504541+0000 mgr.y (mgr.25277) 1609 : cluster 0 pgmap v1799: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:41:21.900 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:21 smithi133 bash[105589]: cluster 2023-10-09T05:41:20.504541+0000 mgr.y (mgr.25277) 1609 : cluster 0 pgmap v1799: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:41:21.959 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:21.985 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:21.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:21.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:22.088 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:22.110 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:22.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:22.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:22.214 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:22.245 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:22.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:22.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:22.348 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:22.370 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:22.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:22.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:22.473 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:22.498 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:22.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:22.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:22.593 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is set 2023-10-09T05:41:22.601 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:22.621 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:22.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:22.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:22.624 INFO:teuthology.orchestra.run.smithi133.stdout:moved 2 / 219 (0.913242%) 2023-10-09T05:41:22.624 INFO:teuthology.orchestra.run.smithi133.stdout:avg 31.2857 2023-10-09T05:41:22.624 INFO:teuthology.orchestra.run.smithi133.stdout:stddev 3.28261 -> 3.05728 (expected baseline 5.17845) 2023-10-09T05:41:22.624 INFO:teuthology.orchestra.run.smithi133.stdout:min osd.5 with 26 -> 27 pgs (0.83105 -> 0.863014 * mean) 2023-10-09T05:41:22.624 INFO:teuthology.orchestra.run.smithi133.stdout:max osd.3 with 36 -> 36 pgs (1.15068 -> 1.15068 * mean) 2023-10-09T05:41:22.624 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:41:22.625 INFO:teuthology.orchestra.run.smithi133.stdout:oload 110 2023-10-09T05:41:22.625 INFO:teuthology.orchestra.run.smithi133.stdout:max_change 0.05 2023-10-09T05:41:22.625 INFO:teuthology.orchestra.run.smithi133.stdout:max_change_osds 4 2023-10-09T05:41:22.625 INFO:teuthology.orchestra.run.smithi133.stdout:average_utilization 0.0134 2023-10-09T05:41:22.625 INFO:teuthology.orchestra.run.smithi133.stdout:overload_utilization 0.0148 2023-10-09T05:41:22.625 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6 weight 0.9500 -> 0.9000 2023-10-09T05:41:22.625 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7 weight 0.9500 -> 0.9000 2023-10-09T05:41:22.627 INFO:teuthology.orchestra.run.smithi133.stderr:no change 2023-10-09T05:41:22.725 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:22.764 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:22.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:22.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:22.797 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:41:22 smithi133 bash[91128]: debug 2023-10-09T05:41:22.619+0000 7ff45d5ed700 -1 mgr.server reply reply (2) No such file or directory no change 2023-10-09T05:41:22.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:22 smithi133 bash[105589]: cluster 2023-10-09T05:41:21.568427+0000 mon.a (mon.0) 2108 : cluster 0 osdmap e406: 8 total, 8 up, 7 in 2023-10-09T05:41:22.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:22 smithi133 bash[105589]: cluster 2023-10-09T05:41:21.571027+0000 osd.5 (osd.5) 442 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T05:41:22.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:22 smithi133 bash[105589]: audit 2023-10-09T05:41:21.886034+0000 mon.c (mon.2) 147 : audit 1 from='client.? 172.21.15.133:0/2779894080' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:41:22.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:22 smithi133 bash[105589]: audit 2023-10-09T05:41:21.886404+0000 mon.a (mon.0) 2109 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:41:22.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:22 smithi133 bash[105589]: cluster 2023-10-09T05:41:22.070954+0000 osd.7 (osd.7) 375 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T05:41:22.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:22 smithi133 bash[105589]: cluster 2023-10-09T05:41:22.074004+0000 osd.7 (osd.7) 376 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T05:41:22.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:22 smithi133 bash[105589]: cluster 2023-10-09T05:41:22.102267+0000 osd.4 (osd.4) 350 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T05:41:22.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:22 smithi133 bash[102858]: cluster 2023-10-09T05:41:21.568427+0000 mon.a (mon.0) 2108 : cluster 0 osdmap e406: 8 total, 8 up, 7 in 2023-10-09T05:41:22.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:22 smithi133 bash[102858]: cluster 2023-10-09T05:41:21.571027+0000 osd.5 (osd.5) 442 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T05:41:22.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:22 smithi133 bash[102858]: audit 2023-10-09T05:41:21.886034+0000 mon.c (mon.2) 147 : audit 1 from='client.? 172.21.15.133:0/2779894080' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:41:22.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:22 smithi133 bash[102858]: audit 2023-10-09T05:41:21.886404+0000 mon.a (mon.0) 2109 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:41:22.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:22 smithi133 bash[102858]: cluster 2023-10-09T05:41:22.070954+0000 osd.7 (osd.7) 375 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T05:41:22.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:22 smithi133 bash[102858]: cluster 2023-10-09T05:41:22.074004+0000 osd.7 (osd.7) 376 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T05:41:22.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:22 smithi133 bash[102858]: cluster 2023-10-09T05:41:22.102267+0000 osd.4 (osd.4) 350 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T05:41:22.868 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:22.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:22 smithi194 bash[92666]: cluster 2023-10-09T05:41:21.568427+0000 mon.a (mon.0) 2108 : cluster 0 osdmap e406: 8 total, 8 up, 7 in 2023-10-09T05:41:22.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:22 smithi194 bash[92666]: cluster 2023-10-09T05:41:21.571027+0000 osd.5 (osd.5) 442 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T05:41:22.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:22 smithi194 bash[92666]: audit 2023-10-09T05:41:21.886034+0000 mon.c (mon.2) 147 : audit 1 from='client.? 172.21.15.133:0/2779894080' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:41:22.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:22 smithi194 bash[92666]: audit 2023-10-09T05:41:21.886404+0000 mon.a (mon.0) 2109 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:41:22.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:22 smithi194 bash[92666]: cluster 2023-10-09T05:41:22.070954+0000 osd.7 (osd.7) 375 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T05:41:22.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:22 smithi194 bash[92666]: cluster 2023-10-09T05:41:22.074004+0000 osd.7 (osd.7) 376 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T05:41:22.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:22 smithi194 bash[92666]: cluster 2023-10-09T05:41:22.102267+0000 osd.4 (osd.4) 350 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T05:41:22.895 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:22.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:22.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:22.999 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:23.037 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:23.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:23.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:23.047 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T05:41:23.142 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:23.172 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:23.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:23.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:23.276 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:23.301 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:23.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:23.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:23.344 INFO:tasks.workunit.client.0.smithi133.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-10-09T05:41:23.345 INFO:tasks.workunit.client.0.smithi133.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T05:41:23.348 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd export testimg /tmp/rbd_import_export_287447/img2 2023-10-09T05:41:23.405 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:23.439 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:23.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:23.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:23.543 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:23.573 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:23.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:23.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:23.676 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:23.690 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:23 smithi194 bash[92666]: cluster 2023-10-09T05:41:22.504924+0000 mgr.y (mgr.25277) 1610 : cluster 0 pgmap v1801: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:41:23.690 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:23 smithi194 bash[92666]: cluster 2023-10-09T05:41:22.573083+0000 mon.a (mon.0) 2110 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:41:23.690 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:23 smithi194 bash[92666]: cluster 2023-10-09T05:41:22.582172+0000 osd.4 (osd.4) 351 : cluster 3 failed to encode map e407 with expected crc 2023-10-09T05:41:23.690 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:23 smithi194 bash[92666]: audit 2023-10-09T05:41:22.589752+0000 mon.a (mon.0) 2111 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:41:23.690 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:23 smithi194 bash[92666]: cluster 2023-10-09T05:41:22.589830+0000 mon.a (mon.0) 2112 : cluster 0 osdmap e407: 8 total, 8 up, 7 in 2023-10-09T05:41:23.690 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:23 smithi194 bash[92666]: cluster 2023-10-09T05:41:22.591406+0000 osd.5 (osd.5) 443 : cluster 3 failed to encode map e407 with expected crc 2023-10-09T05:41:23.690 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:23 smithi194 bash[92666]: cluster 2023-10-09T05:41:22.599582+0000 osd.7 (osd.7) 377 : cluster 3 failed to encode map e407 with expected crc 2023-10-09T05:41:23.690 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:23 smithi194 bash[92666]: audit 2023-10-09T05:41:22.623540+0000 mgr.y (mgr.25277) 1611 : audit 0 from='client.35958 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:41:23.708 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:23.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:23.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:23.761 INFO:tasks.workunit.client.0.smithi133.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T05:41:23.769 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd export testimg - 2023-10-09T05:41:23.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:23 smithi133 bash[102858]: cluster 2023-10-09T05:41:22.504924+0000 mgr.y (mgr.25277) 1610 : cluster 0 pgmap v1801: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:41:23.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:23 smithi133 bash[102858]: cluster 2023-10-09T05:41:22.573083+0000 mon.a (mon.0) 2110 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:41:23.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:23 smithi133 bash[102858]: cluster 2023-10-09T05:41:22.582172+0000 osd.4 (osd.4) 351 : cluster 3 failed to encode map e407 with expected crc 2023-10-09T05:41:23.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:23 smithi133 bash[102858]: audit 2023-10-09T05:41:22.589752+0000 mon.a (mon.0) 2111 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:41:23.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:23 smithi133 bash[102858]: cluster 2023-10-09T05:41:22.589830+0000 mon.a (mon.0) 2112 : cluster 0 osdmap e407: 8 total, 8 up, 7 in 2023-10-09T05:41:23.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:23 smithi133 bash[102858]: cluster 2023-10-09T05:41:22.591406+0000 osd.5 (osd.5) 443 : cluster 3 failed to encode map e407 with expected crc 2023-10-09T05:41:23.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:23 smithi133 bash[102858]: cluster 2023-10-09T05:41:22.599582+0000 osd.7 (osd.7) 377 : cluster 3 failed to encode map e407 with expected crc 2023-10-09T05:41:23.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:23 smithi133 bash[102858]: audit 2023-10-09T05:41:22.623540+0000 mgr.y (mgr.25277) 1611 : audit 0 from='client.35958 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:41:23.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:23 smithi133 bash[105589]: cluster 2023-10-09T05:41:22.504924+0000 mgr.y (mgr.25277) 1610 : cluster 0 pgmap v1801: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:41:23.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:23 smithi133 bash[105589]: cluster 2023-10-09T05:41:22.573083+0000 mon.a (mon.0) 2110 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:41:23.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:23 smithi133 bash[105589]: cluster 2023-10-09T05:41:22.582172+0000 osd.4 (osd.4) 351 : cluster 3 failed to encode map e407 with expected crc 2023-10-09T05:41:23.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:23 smithi133 bash[105589]: audit 2023-10-09T05:41:22.589752+0000 mon.a (mon.0) 2111 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:41:23.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:23 smithi133 bash[105589]: cluster 2023-10-09T05:41:22.589830+0000 mon.a (mon.0) 2112 : cluster 0 osdmap e407: 8 total, 8 up, 7 in 2023-10-09T05:41:23.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:23 smithi133 bash[105589]: cluster 2023-10-09T05:41:22.591406+0000 osd.5 (osd.5) 443 : cluster 3 failed to encode map e407 with expected crc 2023-10-09T05:41:23.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:23 smithi133 bash[105589]: cluster 2023-10-09T05:41:22.599582+0000 osd.7 (osd.7) 377 : cluster 3 failed to encode map e407 with expected crc 2023-10-09T05:41:23.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:23 smithi133 bash[105589]: audit 2023-10-09T05:41:22.623540+0000 mgr.y (mgr.25277) 1611 : audit 0 from='client.35958 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:41:23.812 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:23.843 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:23.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:23.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:23.947 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:23.977 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:23.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:23.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:24.080 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:24.107 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:24.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:24.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:24.211 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:24.244 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:24.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:24.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:24.348 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:24.376 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:24.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:24.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:24.480 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:24.502 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:24.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:24.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:24.506 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:41:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:41:24] "GET /metrics HTTP/1.1" 200 34101 "" "Prometheus/2.43.0" 2023-10-09T05:41:24.605 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:24.635 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:24.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:24.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:24.739 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:24.771 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:24.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:24.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:24.875 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:24.879 INFO:tasks.workunit.client.0.smithi133.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T05:41:24.883 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd rm testimg 2023-10-09T05:41:24.896 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:24.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:24.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:24.999 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:25.022 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:25.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:25.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:25.124 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:25.153 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:25.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:25.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:25.256 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:25.264 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:24 smithi133 bash[105589]: cluster 2023-10-09T05:41:24.505559+0000 mgr.y (mgr.25277) 1612 : cluster 0 pgmap v1803: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 383 B/s rd, 255 B/s wr, 0 op/s 2023-10-09T05:41:25.264 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:24 smithi133 bash[102858]: cluster 2023-10-09T05:41:24.505559+0000 mgr.y (mgr.25277) 1612 : cluster 0 pgmap v1803: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 383 B/s rd, 255 B/s wr, 0 op/s 2023-10-09T05:41:25.274 INFO:tasks.workunit.client.0.smithi133.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-09T05:41:25.278 INFO:tasks.workunit.client.0.smithi133.stderr:+ cmp /tmp/rbd_import_export_287447/img /tmp/rbd_import_export_287447/img2 2023-10-09T05:41:25.282 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:25.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:25.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:25.385 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:25.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:24 smithi194 bash[92666]: cluster 2023-10-09T05:41:24.505559+0000 mgr.y (mgr.25277) 1612 : cluster 0 pgmap v1803: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 383 B/s rd, 255 B/s wr, 0 op/s 2023-10-09T05:41:25.415 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:25.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:25.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:25.519 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:25.551 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:25.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:25.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:25.655 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:25.680 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:25.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:25.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:25.784 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:25.809 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:25.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:25.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:25.912 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:25.945 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:25.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:25.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:26.049 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:26.080 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:26.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:26.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:26.168 INFO:tasks.workunit.client.0.smithi133.stderr:+ cmp /tmp/rbd_import_export_287447/img /tmp/rbd_import_export_287447/img3 2023-10-09T05:41:26.185 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:26.213 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:26.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:26.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:26.316 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:26.344 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:26.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:26.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:26.447 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:26.469 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:26.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:26.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:26.572 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:26.611 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:26.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:26.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:26.700 INFO:tasks.workunit.client.0.smithi133.stderr:+ rm /tmp/rbd_import_export_287447/img2 /tmp/rbd_import_export_287447/img3 2023-10-09T05:41:26.715 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:26.739 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:26.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:26.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:26.841 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:26.874 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:26.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:26.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:26.977 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:27.001 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:27.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:27.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:27.105 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:27.141 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:27.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:27.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:27.164 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd import --new-format - testimg 2023-10-09T05:41:27.246 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:27.271 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:27.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:27.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:27.375 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:27.407 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:27.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:27.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:27.511 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:27.539 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:27.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:27.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:27.636 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:27.644 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:27.669 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:27.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:27.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:27.755 INFO:tasks.workunit.client.0.smithi133.stderr: Importing image: 100% complete...done. 2023-10-09T05:41:27.756 INFO:tasks.workunit.client.0.smithi133.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T05:41:27.759 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd export testimg /tmp/rbd_import_export_287447/img2 2023-10-09T05:41:27.772 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:27.797 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:27 smithi133 bash[105589]: cluster 2023-10-09T05:41:26.506322+0000 mgr.y (mgr.25277) 1613 : cluster 0 pgmap v1804: 73 pgs: 73 active+clean; 5.2 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 137 KiB/s rd, 64 KiB/s wr, 18 op/s 2023-10-09T05:41:27.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:27 smithi133 bash[102858]: cluster 2023-10-09T05:41:26.506322+0000 mgr.y (mgr.25277) 1613 : cluster 0 pgmap v1804: 73 pgs: 73 active+clean; 5.2 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 137 KiB/s rd, 64 KiB/s wr, 18 op/s 2023-10-09T05:41:27.805 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:27.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:27.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:27.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:27 smithi194 bash[92666]: cluster 2023-10-09T05:41:26.506322+0000 mgr.y (mgr.25277) 1613 : cluster 0 pgmap v1804: 73 pgs: 73 active+clean; 5.2 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 137 KiB/s rd, 64 KiB/s wr, 18 op/s 2023-10-09T05:41:27.909 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:27.940 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:27.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:27.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:28.042 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:41:28.043 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:28.064 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:28.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:28.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:28.129 INFO:tasks.workunit.client.0.smithi133.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T05:41:28.133 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd export testimg - 2023-10-09T05:41:28.167 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:28.193 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:28.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:28.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:28.296 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:28.324 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:28.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:28.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:28.427 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:28.466 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:28.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:28.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:28.570 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:28.602 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:28.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:28.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:28.706 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:28.719 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:28 smithi194 bash[92666]: audit 2023-10-09T05:41:28.497396+0000 mon.a (mon.0) 2113 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:28.737 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:28.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:28.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:28.840 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:28.849 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:28 smithi133 bash[102858]: audit 2023-10-09T05:41:28.497396+0000 mon.a (mon.0) 2113 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:28.850 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:28 smithi133 bash[105589]: audit 2023-10-09T05:41:28.497396+0000 mon.a (mon.0) 2113 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:28.863 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:28.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:28.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:28.967 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:28.998 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:29.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:29.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:29.088 INFO:tasks.workunit.client.0.smithi133.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T05:41:29.092 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd rm testimg 2023-10-09T05:41:29.103 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:29.133 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:29.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:29.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:29.227 INFO:tasks.workunit.client.0.smithi133.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-09T05:41:29.233 INFO:tasks.workunit.client.0.smithi133.stderr:+ cmp /tmp/rbd_import_export_287447/img /tmp/rbd_import_export_287447/img2 2023-10-09T05:41:29.238 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:29.274 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:29.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:29.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:29.377 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:29.408 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:29.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:29.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:29.512 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:29.541 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:29.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:29.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:29.643 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:29.677 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:29.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:29.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:29.782 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:29.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:29 smithi133 bash[102858]: cluster 2023-10-09T05:41:28.506925+0000 mgr.y (mgr.25277) 1614 : cluster 0 pgmap v1805: 73 pgs: 73 active+clean; 5.2 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 137 KiB/s rd, 64 KiB/s wr, 20 op/s 2023-10-09T05:41:29.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:29 smithi133 bash[102858]: cluster 2023-10-09T05:41:28.981439+0000 mon.a (mon.0) 2114 : cluster 0 osdmap e408: 8 total, 8 up, 7 in 2023-10-09T05:41:29.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:29 smithi133 bash[102858]: cluster 2023-10-09T05:41:29.072041+0000 osd.7 (osd.7) 378 : cluster 3 failed to encode map e408 with expected crc 2023-10-09T05:41:29.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:29 smithi133 bash[102858]: cluster 2023-10-09T05:41:29.074908+0000 osd.7 (osd.7) 379 : cluster 3 failed to encode map e408 with expected crc 2023-10-09T05:41:29.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:29 smithi133 bash[102858]: cluster 2023-10-09T05:41:29.126530+0000 osd.5 (osd.5) 444 : cluster 3 failed to encode map e408 with expected crc 2023-10-09T05:41:29.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:29 smithi133 bash[105589]: cluster 2023-10-09T05:41:28.506925+0000 mgr.y (mgr.25277) 1614 : cluster 0 pgmap v1805: 73 pgs: 73 active+clean; 5.2 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 137 KiB/s rd, 64 KiB/s wr, 20 op/s 2023-10-09T05:41:29.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:29 smithi133 bash[105589]: cluster 2023-10-09T05:41:28.981439+0000 mon.a (mon.0) 2114 : cluster 0 osdmap e408: 8 total, 8 up, 7 in 2023-10-09T05:41:29.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:29 smithi133 bash[105589]: cluster 2023-10-09T05:41:29.072041+0000 osd.7 (osd.7) 378 : cluster 3 failed to encode map e408 with expected crc 2023-10-09T05:41:29.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:29 smithi133 bash[105589]: cluster 2023-10-09T05:41:29.074908+0000 osd.7 (osd.7) 379 : cluster 3 failed to encode map e408 with expected crc 2023-10-09T05:41:29.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:29 smithi133 bash[105589]: cluster 2023-10-09T05:41:29.126530+0000 osd.5 (osd.5) 444 : cluster 3 failed to encode map e408 with expected crc 2023-10-09T05:41:29.815 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:29.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:29.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:29.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:29 smithi194 bash[92666]: cluster 2023-10-09T05:41:28.506925+0000 mgr.y (mgr.25277) 1614 : cluster 0 pgmap v1805: 73 pgs: 73 active+clean; 5.2 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 137 KiB/s rd, 64 KiB/s wr, 20 op/s 2023-10-09T05:41:29.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:29 smithi194 bash[92666]: cluster 2023-10-09T05:41:28.981439+0000 mon.a (mon.0) 2114 : cluster 0 osdmap e408: 8 total, 8 up, 7 in 2023-10-09T05:41:29.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:29 smithi194 bash[92666]: cluster 2023-10-09T05:41:29.072041+0000 osd.7 (osd.7) 378 : cluster 3 failed to encode map e408 with expected crc 2023-10-09T05:41:29.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:29 smithi194 bash[92666]: cluster 2023-10-09T05:41:29.074908+0000 osd.7 (osd.7) 379 : cluster 3 failed to encode map e408 with expected crc 2023-10-09T05:41:29.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:29 smithi194 bash[92666]: cluster 2023-10-09T05:41:29.126530+0000 osd.5 (osd.5) 444 : cluster 3 failed to encode map e408 with expected crc 2023-10-09T05:41:29.921 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:29.953 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:29.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:29.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:30.010 INFO:tasks.workunit.client.0.smithi133.stderr:+ cmp /tmp/rbd_import_export_287447/img /tmp/rbd_import_export_287447/img3 2023-10-09T05:41:30.057 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:30.091 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:30.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:30.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:30.195 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:30.222 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:30.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:30.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:30.325 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:30.355 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:30.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:30.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:30.458 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:30.485 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:30.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:30.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:30.570 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:30.572 INFO:tasks.workunit.client.0.smithi133.stderr:+ rm /tmp/rbd_import_export_287447/img /tmp/rbd_import_export_287447/img2 /tmp/rbd_import_export_287447/img3 2023-10-09T05:41:30.589 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:30.624 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:30.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:30.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:30.727 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:30.759 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:30.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:30.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:30.797 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:30 smithi133 bash[105589]: cluster 2023-10-09T05:41:29.135398+0000 osd.6 (osd.6) 406 : cluster 3 failed to encode map e408 with expected crc 2023-10-09T05:41:30.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:30 smithi133 bash[105589]: cluster 2023-10-09T05:41:29.136973+0000 osd.6 (osd.6) 407 : cluster 3 failed to encode map e408 with expected crc 2023-10-09T05:41:30.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:30 smithi133 bash[102858]: cluster 2023-10-09T05:41:29.135398+0000 osd.6 (osd.6) 406 : cluster 3 failed to encode map e408 with expected crc 2023-10-09T05:41:30.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:30 smithi133 bash[102858]: cluster 2023-10-09T05:41:29.136973+0000 osd.6 (osd.6) 407 : cluster 3 failed to encode map e408 with expected crc 2023-10-09T05:41:30.863 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:30.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:30 smithi194 bash[92666]: cluster 2023-10-09T05:41:29.135398+0000 osd.6 (osd.6) 406 : cluster 3 failed to encode map e408 with expected crc 2023-10-09T05:41:30.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:30 smithi194 bash[92666]: cluster 2023-10-09T05:41:29.136973+0000 osd.6 (osd.6) 407 : cluster 3 failed to encode map e408 with expected crc 2023-10-09T05:41:30.890 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd help export 2023-10-09T05:41:30.890 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep -q export-format 2023-10-09T05:41:30.891 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:30.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:30.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:30.993 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:31.026 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:31.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:31.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:31.036 INFO:tasks.workunit.client.0.smithi133.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_287447/img bs=1k count=10 seek=100 2023-10-09T05:41:31.057 INFO:tasks.workunit.client.0.smithi133.stderr:10+0 records in 2023-10-09T05:41:31.058 INFO:tasks.workunit.client.0.smithi133.stderr:10+0 records out 2023-10-09T05:41:31.058 INFO:tasks.workunit.client.0.smithi133.stderr:10240 bytes (10 kB, 10 KiB) copied, 8.5862e-05 s, 119 MB/s 2023-10-09T05:41:31.058 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd import --new-format /tmp/rbd_import_export_287447/img testimg 2023-10-09T05:41:31.130 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:31.133 INFO:tasks.workunit.client.0.smithi133.stderr: Importing image: 100% complete...done. 2023-10-09T05:41:31.134 INFO:tasks.workunit.client.0.smithi133.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T05:41:31.137 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd snap create testimg@snap 2023-10-09T05:41:31.154 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:31.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:31.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:31.259 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:31.285 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:31.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:31.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:31.388 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:31.423 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:31.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:31.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:31.526 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:31.557 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:31.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:31.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:31.608 INFO:tasks.workunit.client.0.smithi133.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T05:41:31.617 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd image-meta set testimg key1 value1 2023-10-09T05:41:31.662 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:31.684 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd image-meta list testimg 2023-10-09T05:41:31.684 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:31.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:31.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:31.727 INFO:tasks.workunit.client.0.smithi133.stderr:+ IMAGEMETA_BEFORE=There is 1 metadatum on this image: 2023-10-09T05:41:31.728 INFO:tasks.workunit.client.0.smithi133.stderr: 2023-10-09T05:41:31.728 INFO:tasks.workunit.client.0.smithi133.stderr:Key Value 2023-10-09T05:41:31.728 INFO:tasks.workunit.client.0.smithi133.stderr:key1 value1 2023-10-09T05:41:31.728 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_287447/img_v2 2023-10-09T05:41:31.769 INFO:tasks.workunit.client.0.smithi133.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T05:41:31.773 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_287447/img_v2 testimg_import 2023-10-09T05:41:31.787 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:31.809 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:31.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:31.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:31.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:31 smithi194 bash[92666]: cluster 2023-10-09T05:41:30.507771+0000 mgr.y (mgr.25277) 1615 : cluster 0 pgmap v1807: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 245 KiB/s rd, 110 KiB/s wr, 56 op/s 2023-10-09T05:41:31.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:31 smithi194 bash[92666]: cluster 2023-10-09T05:41:30.548705+0000 osd.4 (osd.4) 352 : cluster 3 failed to encode map e408 with expected crc 2023-10-09T05:41:31.912 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:31.943 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:31.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:31.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:32.047 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:32.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:31 smithi133 bash[102858]: cluster 2023-10-09T05:41:30.507771+0000 mgr.y (mgr.25277) 1615 : cluster 0 pgmap v1807: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 245 KiB/s rd, 110 KiB/s wr, 56 op/s 2023-10-09T05:41:32.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:31 smithi133 bash[102858]: cluster 2023-10-09T05:41:30.548705+0000 osd.4 (osd.4) 352 : cluster 3 failed to encode map e408 with expected crc 2023-10-09T05:41:32.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:31 smithi133 bash[105589]: cluster 2023-10-09T05:41:30.507771+0000 mgr.y (mgr.25277) 1615 : cluster 0 pgmap v1807: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 245 KiB/s rd, 110 KiB/s wr, 56 op/s 2023-10-09T05:41:32.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:31 smithi133 bash[105589]: cluster 2023-10-09T05:41:30.548705+0000 osd.4 (osd.4) 352 : cluster 3 failed to encode map e408 with expected crc 2023-10-09T05:41:32.071 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:32.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:32.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:32.162 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 6, 1, 7, 5] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:41:32.163 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:41:32.163 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-10-09T05:41:32.163 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T05:41:32.163 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T05:41:32.163 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-10-09T05:41:32.174 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:32.203 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:32.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:32.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:32.307 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:32.340 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:32.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:32.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:32.444 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:32.477 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:32.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:32.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:32.580 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:32.610 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:32.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:32.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:32.627 INFO:tasks.workunit.client.0.smithi133.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T05:41:32.632 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd info testimg_import 2023-10-09T05:41:32.676 INFO:tasks.workunit.client.0.smithi133.stdout:rbd image 'testimg_import': 2023-10-09T05:41:32.676 INFO:tasks.workunit.client.0.smithi133.stdout: size 110 KiB in 1 objects 2023-10-09T05:41:32.676 INFO:tasks.workunit.client.0.smithi133.stdout: order 22 (4 MiB objects) 2023-10-09T05:41:32.676 INFO:tasks.workunit.client.0.smithi133.stdout: snapshot_count: 1 2023-10-09T05:41:32.677 INFO:tasks.workunit.client.0.smithi133.stdout: id: 8cc4cd5fbc26 2023-10-09T05:41:32.677 INFO:tasks.workunit.client.0.smithi133.stdout: block_name_prefix: rbd_data.8cc4cd5fbc26 2023-10-09T05:41:32.677 INFO:tasks.workunit.client.0.smithi133.stdout: format: 2 2023-10-09T05:41:32.677 INFO:tasks.workunit.client.0.smithi133.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-09T05:41:32.677 INFO:tasks.workunit.client.0.smithi133.stdout: op_features: 2023-10-09T05:41:32.677 INFO:tasks.workunit.client.0.smithi133.stdout: flags: 2023-10-09T05:41:32.677 INFO:tasks.workunit.client.0.smithi133.stdout: create_timestamp: Mon Oct 9 05:41:31 2023 2023-10-09T05:41:32.677 INFO:tasks.workunit.client.0.smithi133.stdout: access_timestamp: Mon Oct 9 05:41:31 2023 2023-10-09T05:41:32.677 INFO:tasks.workunit.client.0.smithi133.stdout: modify_timestamp: Mon Oct 9 05:41:31 2023 2023-10-09T05:41:32.682 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd info testimg_import@snap 2023-10-09T05:41:32.714 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:32.723 INFO:tasks.workunit.client.0.smithi133.stdout:rbd image 'testimg_import': 2023-10-09T05:41:32.723 INFO:tasks.workunit.client.0.smithi133.stdout: size 110 KiB in 1 objects 2023-10-09T05:41:32.724 INFO:tasks.workunit.client.0.smithi133.stdout: order 22 (4 MiB objects) 2023-10-09T05:41:32.724 INFO:tasks.workunit.client.0.smithi133.stdout: snapshot_count: 1 2023-10-09T05:41:32.724 INFO:tasks.workunit.client.0.smithi133.stdout: id: 8cc4cd5fbc26 2023-10-09T05:41:32.724 INFO:tasks.workunit.client.0.smithi133.stdout: block_name_prefix: rbd_data.8cc4cd5fbc26 2023-10-09T05:41:32.724 INFO:tasks.workunit.client.0.smithi133.stdout: format: 2 2023-10-09T05:41:32.724 INFO:tasks.workunit.client.0.smithi133.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-09T05:41:32.724 INFO:tasks.workunit.client.0.smithi133.stdout: op_features: 2023-10-09T05:41:32.724 INFO:tasks.workunit.client.0.smithi133.stdout: flags: 2023-10-09T05:41:32.724 INFO:tasks.workunit.client.0.smithi133.stdout: create_timestamp: Mon Oct 9 05:41:31 2023 2023-10-09T05:41:32.724 INFO:tasks.workunit.client.0.smithi133.stdout: access_timestamp: Mon Oct 9 05:41:31 2023 2023-10-09T05:41:32.724 INFO:tasks.workunit.client.0.smithi133.stdout: modify_timestamp: Mon Oct 9 05:41:31 2023 2023-10-09T05:41:32.725 INFO:tasks.workunit.client.0.smithi133.stdout: protected: False 2023-10-09T05:41:32.727 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd image-meta list testimg_import 2023-10-09T05:41:32.748 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:32.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:32.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:32.777 INFO:tasks.workunit.client.0.smithi133.stderr:+ IMAGEMETA_AFTER=There is 1 metadatum on this image: 2023-10-09T05:41:32.777 INFO:tasks.workunit.client.0.smithi133.stderr: 2023-10-09T05:41:32.777 INFO:tasks.workunit.client.0.smithi133.stderr:Key Value 2023-10-09T05:41:32.777 INFO:tasks.workunit.client.0.smithi133.stderr:key1 value1 2023-10-09T05:41:32.777 INFO:tasks.workunit.client.0.smithi133.stderr:+ [ There is 1 metadatum on this image: 2023-10-09T05:41:32.777 INFO:tasks.workunit.client.0.smithi133.stderr: 2023-10-09T05:41:32.777 INFO:tasks.workunit.client.0.smithi133.stderr:Key Value 2023-10-09T05:41:32.778 INFO:tasks.workunit.client.0.smithi133.stderr:key1 value1 = There is 1 metadatum on this image: 2023-10-09T05:41:32.778 INFO:tasks.workunit.client.0.smithi133.stderr: 2023-10-09T05:41:32.778 INFO:tasks.workunit.client.0.smithi133.stderr:Key Value 2023-10-09T05:41:32.778 INFO:tasks.workunit.client.0.smithi133.stderr:key1 value1 ] 2023-10-09T05:41:32.778 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd export testimg_import /tmp/rbd_import_export_287447/img_import 2023-10-09T05:41:32.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:32 smithi133 bash[102858]: cluster 2023-10-09T05:41:31.592770+0000 osd.7 (osd.7) 380 : cluster 3 failed to encode map e409 with expected crc 2023-10-09T05:41:32.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:32 smithi133 bash[102858]: cluster 2023-10-09T05:41:31.594920+0000 mon.a (mon.0) 2115 : cluster 0 osdmap e409: 8 total, 8 up, 7 in 2023-10-09T05:41:32.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:32 smithi133 bash[102858]: cluster 2023-10-09T05:41:31.598692+0000 osd.6 (osd.6) 408 : cluster 3 failed to encode map e409 with expected crc 2023-10-09T05:41:32.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:32 smithi133 bash[102858]: cluster 2023-10-09T05:41:31.600563+0000 osd.6 (osd.6) 409 : cluster 3 failed to encode map e409 with expected crc 2023-10-09T05:41:32.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:32 smithi133 bash[102858]: cluster 2023-10-09T05:41:31.655913+0000 osd.5 (osd.5) 445 : cluster 3 failed to encode map e409 with expected crc 2023-10-09T05:41:32.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:32 smithi133 bash[105589]: cluster 2023-10-09T05:41:31.592770+0000 osd.7 (osd.7) 380 : cluster 3 failed to encode map e409 with expected crc 2023-10-09T05:41:32.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:32 smithi133 bash[105589]: cluster 2023-10-09T05:41:31.594920+0000 mon.a (mon.0) 2115 : cluster 0 osdmap e409: 8 total, 8 up, 7 in 2023-10-09T05:41:32.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:32 smithi133 bash[105589]: cluster 2023-10-09T05:41:31.598692+0000 osd.6 (osd.6) 408 : cluster 3 failed to encode map e409 with expected crc 2023-10-09T05:41:32.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:32 smithi133 bash[105589]: cluster 2023-10-09T05:41:31.600563+0000 osd.6 (osd.6) 409 : cluster 3 failed to encode map e409 with expected crc 2023-10-09T05:41:32.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:32 smithi133 bash[105589]: cluster 2023-10-09T05:41:31.655913+0000 osd.5 (osd.5) 445 : cluster 3 failed to encode map e409 with expected crc 2023-10-09T05:41:32.821 INFO:tasks.workunit.client.0.smithi133.stderr: Exporting image: 100% complete...done. 2023-10-09T05:41:32.824 INFO:tasks.workunit.client.0.smithi133.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_287447/img /tmp/rbd_import_export_287447/img_import 2023-10-09T05:41:32.824 INFO:tasks.workunit.client.0.smithi133.stderr:+ cmp -l /tmp/rbd_import_export_287447/img /tmp/rbd_import_export_287447/img_import 2023-10-09T05:41:32.825 INFO:tasks.workunit.client.0.smithi133.stderr:+ stat /tmp/rbd_import_export_287447/img --format %b 2023-10-09T05:41:32.852 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:32.881 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:32.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:32.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:32.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:32 smithi194 bash[92666]: cluster 2023-10-09T05:41:31.592770+0000 osd.7 (osd.7) 380 : cluster 3 failed to encode map e409 with expected crc 2023-10-09T05:41:32.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:32 smithi194 bash[92666]: cluster 2023-10-09T05:41:31.594920+0000 mon.a (mon.0) 2115 : cluster 0 osdmap e409: 8 total, 8 up, 7 in 2023-10-09T05:41:32.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:32 smithi194 bash[92666]: cluster 2023-10-09T05:41:31.598692+0000 osd.6 (osd.6) 408 : cluster 3 failed to encode map e409 with expected crc 2023-10-09T05:41:32.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:32 smithi194 bash[92666]: cluster 2023-10-09T05:41:31.600563+0000 osd.6 (osd.6) 409 : cluster 3 failed to encode map e409 with expected crc 2023-10-09T05:41:32.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:32 smithi194 bash[92666]: cluster 2023-10-09T05:41:31.655913+0000 osd.5 (osd.5) 445 : cluster 3 failed to encode map e409 with expected crc 2023-10-09T05:41:32.964 INFO:tasks.workunit.client.0.smithi133.stderr:+ origsize=24 2023-10-09T05:41:32.964 INFO:tasks.workunit.client.0.smithi133.stderr:+ stat /tmp/rbd_import_export_287447/img_import --format %b 2023-10-09T05:41:32.965 INFO:tasks.workunit.client.0.smithi133.stderr:+ exportsize=224 2023-10-09T05:41:32.966 INFO:tasks.workunit.client.0.smithi133.stderr:+ difference=200 2023-10-09T05:41:32.966 INFO:tasks.workunit.client.0.smithi133.stderr:+ difference=200 2023-10-09T05:41:32.966 INFO:tasks.workunit.client.0.smithi133.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-10-09T05:41:32.966 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_287447/img_snap 2023-10-09T05:41:32.984 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:33.017 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:33.018 INFO:tasks.workunit.client.0.smithi133.stderr: Exporting image: 100% complete...done. 2023-10-09T05:41:33.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:33.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:33.023 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_287447/img_snap_import 2023-10-09T05:41:33.068 INFO:tasks.workunit.client.0.smithi133.stderr: Exporting image: 100% complete...done. 2023-10-09T05:41:33.072 INFO:tasks.workunit.client.0.smithi133.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_287447/img_snap /tmp/rbd_import_export_287447/img_snap_import 2023-10-09T05:41:33.072 INFO:tasks.workunit.client.0.smithi133.stderr:+ cmp -l /tmp/rbd_import_export_287447/img_snap /tmp/rbd_import_export_287447/img_snap_import 2023-10-09T05:41:33.073 INFO:tasks.workunit.client.0.smithi133.stderr:+ stat /tmp/rbd_import_export_287447/img_snap --format %b 2023-10-09T05:41:33.075 INFO:tasks.workunit.client.0.smithi133.stderr:+ origsize=224 2023-10-09T05:41:33.075 INFO:tasks.workunit.client.0.smithi133.stderr:+ stat /tmp/rbd_import_export_287447/img_snap_import --format %b 2023-10-09T05:41:33.076 INFO:tasks.workunit.client.0.smithi133.stderr:+ exportsize=224 2023-10-09T05:41:33.076 INFO:tasks.workunit.client.0.smithi133.stderr:+ difference=0 2023-10-09T05:41:33.076 INFO:tasks.workunit.client.0.smithi133.stderr:+ difference=0 2023-10-09T05:41:33.076 INFO:tasks.workunit.client.0.smithi133.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T05:41:33.076 INFO:tasks.workunit.client.0.smithi133.stderr:+ rm /tmp/rbd_import_export_287447/img_v2 2023-10-09T05:41:33.077 INFO:tasks.workunit.client.0.smithi133.stderr:+ rm /tmp/rbd_import_export_287447/img_import 2023-10-09T05:41:33.078 INFO:tasks.workunit.client.0.smithi133.stderr:+ rm /tmp/rbd_import_export_287447/img_snap 2023-10-09T05:41:33.078 INFO:tasks.workunit.client.0.smithi133.stderr:+ rm /tmp/rbd_import_export_287447/img_snap_import 2023-10-09T05:41:33.079 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd snap rm testimg_import@snap 2023-10-09T05:41:33.120 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:33.149 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:33.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:33.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:33.252 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:33.283 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:33.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:33.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:33.387 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:33.419 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:33.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:33.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:33.523 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:33.553 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:33.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:33.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:33.611 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:33.621 INFO:tasks.workunit.client.0.smithi133.stderr: Removing snap: 100% complete...done. 2023-10-09T05:41:33.633 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd remove testimg_import 2023-10-09T05:41:33.656 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:33.688 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:33.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:33.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:33.759 INFO:tasks.workunit.client.0.smithi133.stderr: Removing image: 100% complete...done. 2023-10-09T05:41:33.763 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd snap rm testimg@snap 2023-10-09T05:41:33.792 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:33.813 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:33.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:33.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:33.831 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:33 smithi133 bash[105589]: cluster 2023-10-09T05:41:32.508469+0000 mgr.y (mgr.25277) 1616 : cluster 0 pgmap v1809: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 320 KiB/s rd, 132 KiB/s wr, 102 op/s 2023-10-09T05:41:33.832 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:33 smithi133 bash[105589]: cluster 2023-10-09T05:41:32.602011+0000 osd.5 (osd.5) 446 : cluster 3 failed to encode map e410 with expected crc 2023-10-09T05:41:33.832 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:33 smithi133 bash[105589]: cluster 2023-10-09T05:41:32.604507+0000 mon.a (mon.0) 2116 : cluster 0 osdmap e410: 8 total, 8 up, 7 in 2023-10-09T05:41:33.832 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:33 smithi133 bash[105589]: cluster 2023-10-09T05:41:32.610735+0000 osd.7 (osd.7) 381 : cluster 3 failed to encode map e410 with expected crc 2023-10-09T05:41:33.832 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:33 smithi133 bash[105589]: cluster 2023-10-09T05:41:33.011584+0000 osd.6 (osd.6) 410 : cluster 3 failed to encode map e410 with expected crc 2023-10-09T05:41:33.833 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:33 smithi133 bash[102858]: cluster 2023-10-09T05:41:32.508469+0000 mgr.y (mgr.25277) 1616 : cluster 0 pgmap v1809: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 320 KiB/s rd, 132 KiB/s wr, 102 op/s 2023-10-09T05:41:33.833 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:33 smithi133 bash[102858]: cluster 2023-10-09T05:41:32.602011+0000 osd.5 (osd.5) 446 : cluster 3 failed to encode map e410 with expected crc 2023-10-09T05:41:33.833 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:33 smithi133 bash[102858]: cluster 2023-10-09T05:41:32.604507+0000 mon.a (mon.0) 2116 : cluster 0 osdmap e410: 8 total, 8 up, 7 in 2023-10-09T05:41:33.833 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:33 smithi133 bash[102858]: cluster 2023-10-09T05:41:32.610735+0000 osd.7 (osd.7) 381 : cluster 3 failed to encode map e410 with expected crc 2023-10-09T05:41:33.834 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:33 smithi133 bash[102858]: cluster 2023-10-09T05:41:33.011584+0000 osd.6 (osd.6) 410 : cluster 3 failed to encode map e410 with expected crc 2023-10-09T05:41:33.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:33 smithi194 bash[92666]: cluster 2023-10-09T05:41:32.508469+0000 mgr.y (mgr.25277) 1616 : cluster 0 pgmap v1809: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 320 KiB/s rd, 132 KiB/s wr, 102 op/s 2023-10-09T05:41:33.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:33 smithi194 bash[92666]: cluster 2023-10-09T05:41:32.602011+0000 osd.5 (osd.5) 446 : cluster 3 failed to encode map e410 with expected crc 2023-10-09T05:41:33.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:33 smithi194 bash[92666]: cluster 2023-10-09T05:41:32.604507+0000 mon.a (mon.0) 2116 : cluster 0 osdmap e410: 8 total, 8 up, 7 in 2023-10-09T05:41:33.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:33 smithi194 bash[92666]: cluster 2023-10-09T05:41:32.610735+0000 osd.7 (osd.7) 381 : cluster 3 failed to encode map e410 with expected crc 2023-10-09T05:41:33.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:33 smithi194 bash[92666]: cluster 2023-10-09T05:41:33.011584+0000 osd.6 (osd.6) 410 : cluster 3 failed to encode map e410 with expected crc 2023-10-09T05:41:33.916 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:33.947 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:33.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:33.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:34.051 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:34.077 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:34.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:34.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:34.181 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:34.204 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:34.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:34.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:34.308 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:34.342 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:34.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:34.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:34.446 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:34.483 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:34.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:34.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:34.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:41:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:41:34] "GET /metrics HTTP/1.1" 200 34148 "" "Prometheus/2.43.0" 2023-10-09T05:41:34.587 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:34.616 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:34.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:34.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:34.631 INFO:tasks.workunit.client.0.smithi133.stderr: Removing snap: 100% complete...done. 2023-10-09T05:41:34.645 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd rm testimg 2023-10-09T05:41:34.720 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:34.742 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:34.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:34.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:34.771 INFO:tasks.workunit.client.0.smithi133.stderr: Removing image: 100% complete...done. 2023-10-09T05:41:34.775 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd import --order 20 /tmp/rbd_import_export_287447/img testimg 2023-10-09T05:41:34.845 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:34.849 INFO:tasks.workunit.client.0.smithi133.stderr: Importing image: 100% complete...done. 2023-10-09T05:41:34.850 INFO:tasks.workunit.client.0.smithi133.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T05:41:34.853 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:34 smithi133 bash[105589]: cluster 2023-10-09T05:41:33.611330+0000 osd.7 (osd.7) 382 : cluster 3 failed to encode map e411 with expected crc 2023-10-09T05:41:34.853 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:34 smithi133 bash[105589]: cluster 2023-10-09T05:41:33.613730+0000 mon.a (mon.0) 2117 : cluster 0 osdmap e411: 8 total, 8 up, 7 in 2023-10-09T05:41:34.853 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:34 smithi133 bash[105589]: cluster 2023-10-09T05:41:33.667046+0000 osd.5 (osd.5) 447 : cluster 3 failed to encode map e411 with expected crc 2023-10-09T05:41:34.853 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:34 smithi133 bash[105589]: cluster 2023-10-09T05:41:33.717726+0000 osd.6 (osd.6) 411 : cluster 3 failed to encode map e411 with expected crc 2023-10-09T05:41:34.854 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:34 smithi133 bash[102858]: cluster 2023-10-09T05:41:33.611330+0000 osd.7 (osd.7) 382 : cluster 3 failed to encode map e411 with expected crc 2023-10-09T05:41:34.854 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:34 smithi133 bash[102858]: cluster 2023-10-09T05:41:33.613730+0000 mon.a (mon.0) 2117 : cluster 0 osdmap e411: 8 total, 8 up, 7 in 2023-10-09T05:41:34.854 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:34 smithi133 bash[102858]: cluster 2023-10-09T05:41:33.667046+0000 osd.5 (osd.5) 447 : cluster 3 failed to encode map e411 with expected crc 2023-10-09T05:41:34.854 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:34 smithi133 bash[102858]: cluster 2023-10-09T05:41:33.717726+0000 osd.6 (osd.6) 411 : cluster 3 failed to encode map e411 with expected crc 2023-10-09T05:41:34.854 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_287447/img_v2 2023-10-09T05:41:34.867 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:34.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:34.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:34.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:34 smithi194 bash[92666]: cluster 2023-10-09T05:41:33.611330+0000 osd.7 (osd.7) 382 : cluster 3 failed to encode map e411 with expected crc 2023-10-09T05:41:34.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:34 smithi194 bash[92666]: cluster 2023-10-09T05:41:33.613730+0000 mon.a (mon.0) 2117 : cluster 0 osdmap e411: 8 total, 8 up, 7 in 2023-10-09T05:41:34.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:34 smithi194 bash[92666]: cluster 2023-10-09T05:41:33.667046+0000 osd.5 (osd.5) 447 : cluster 3 failed to encode map e411 with expected crc 2023-10-09T05:41:34.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:34 smithi194 bash[92666]: cluster 2023-10-09T05:41:33.717726+0000 osd.6 (osd.6) 411 : cluster 3 failed to encode map e411 with expected crc 2023-10-09T05:41:34.900 INFO:tasks.workunit.client.0.smithi133.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T05:41:34.904 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_287447/img_v2 testimg_import 2023-10-09T05:41:34.971 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:34.989 INFO:tasks.workunit.client.0.smithi133.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T05:41:34.994 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:34.995 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd info testimg_import 2023-10-09T05:41:34.995 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep order 2023-10-09T05:41:34.995 INFO:tasks.workunit.client.0.smithi133.stderr:+ awk {print $2} 2023-10-09T05:41:34.995 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep 20 2023-10-09T05:41:34.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:34.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:35.098 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:35.130 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:35.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:35.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:35.234 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:35.255 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:35.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:35.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:35.358 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:35.382 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:35.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:35.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:35.453 INFO:tasks.workunit.client.0.smithi133.stdout:20 2023-10-09T05:41:35.454 INFO:tasks.workunit.client.0.smithi133.stderr:+ rm /tmp/rbd_import_export_287447/img_v2 2023-10-09T05:41:35.485 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:35.506 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:35.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:35.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:35.517 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd remove testimg_import 2023-10-09T05:41:35.610 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:35.634 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:35.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:35.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:35.647 INFO:tasks.workunit.client.0.smithi133.stderr: Removing image: 100% complete...done. 2023-10-09T05:41:35.651 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd remove testimg 2023-10-09T05:41:35.738 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:35.765 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:35.767 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:35 smithi133 bash[105589]: cluster 2023-10-09T05:41:34.509247+0000 mgr.y (mgr.25277) 1617 : cluster 0 pgmap v1812: 73 pgs: 73 active+clean; 692 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 278 KiB/s rd, 103 KiB/s wr, 127 op/s 2023-10-09T05:41:35.767 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:35 smithi133 bash[105589]: cluster 2023-10-09T05:41:34.623489+0000 osd.5 (osd.5) 448 : cluster 3 failed to encode map e412 with expected crc 2023-10-09T05:41:35.767 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:35 smithi133 bash[105589]: cluster 2023-10-09T05:41:34.626125+0000 mon.a (mon.0) 2118 : cluster 0 osdmap e412: 8 total, 8 up, 7 in 2023-10-09T05:41:35.767 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:35 smithi133 bash[105589]: cluster 2023-10-09T05:41:34.629921+0000 osd.6 (osd.6) 412 : cluster 3 failed to encode map e412 with expected crc 2023-10-09T05:41:35.767 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:35 smithi133 bash[102858]: cluster 2023-10-09T05:41:34.509247+0000 mgr.y (mgr.25277) 1617 : cluster 0 pgmap v1812: 73 pgs: 73 active+clean; 692 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 278 KiB/s rd, 103 KiB/s wr, 127 op/s 2023-10-09T05:41:35.767 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:35 smithi133 bash[102858]: cluster 2023-10-09T05:41:34.623489+0000 osd.5 (osd.5) 448 : cluster 3 failed to encode map e412 with expected crc 2023-10-09T05:41:35.768 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:35 smithi133 bash[102858]: cluster 2023-10-09T05:41:34.626125+0000 mon.a (mon.0) 2118 : cluster 0 osdmap e412: 8 total, 8 up, 7 in 2023-10-09T05:41:35.768 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:35 smithi133 bash[102858]: cluster 2023-10-09T05:41:34.629921+0000 osd.6 (osd.6) 412 : cluster 3 failed to encode map e412 with expected crc 2023-10-09T05:41:35.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:35.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:35.776 INFO:tasks.workunit.client.0.smithi133.stderr: Removing image: 100% complete...done. 2023-10-09T05:41:35.779 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_287447/img testimg 2023-10-09T05:41:35.843 INFO:tasks.workunit.client.0.smithi133.stderr: Importing image: 100% complete...done. 2023-10-09T05:41:35.849 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd info testimg 2023-10-09T05:41:35.850 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep features 2023-10-09T05:41:35.869 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:35.883 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:35 smithi194 bash[92666]: cluster 2023-10-09T05:41:34.509247+0000 mgr.y (mgr.25277) 1617 : cluster 0 pgmap v1812: 73 pgs: 73 active+clean; 692 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 278 KiB/s rd, 103 KiB/s wr, 127 op/s 2023-10-09T05:41:35.883 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:35 smithi194 bash[92666]: cluster 2023-10-09T05:41:34.623489+0000 osd.5 (osd.5) 448 : cluster 3 failed to encode map e412 with expected crc 2023-10-09T05:41:35.884 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:35 smithi194 bash[92666]: cluster 2023-10-09T05:41:34.626125+0000 mon.a (mon.0) 2118 : cluster 0 osdmap e412: 8 total, 8 up, 7 in 2023-10-09T05:41:35.884 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:35 smithi194 bash[92666]: cluster 2023-10-09T05:41:34.629921+0000 osd.6 (osd.6) 412 : cluster 3 failed to encode map e412 with expected crc 2023-10-09T05:41:35.898 INFO:tasks.workunit.client.0.smithi133.stderr:+ FEATURES_BEFORE= features: layering 2023-10-09T05:41:35.898 INFO:tasks.workunit.client.0.smithi133.stderr: op_features: 2023-10-09T05:41:35.898 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_287447/img_v2 2023-10-09T05:41:35.900 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:35.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:35.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:36.004 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:36.039 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:36.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:36.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:36.047 INFO:tasks.workunit.client.0.smithi133.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T05:41:36.052 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_287447/img_v2 testimg_import 2023-10-09T05:41:36.120 INFO:tasks.workunit.client.0.smithi133.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T05:41:36.124 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd info testimg_import 2023-10-09T05:41:36.125 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep features 2023-10-09T05:41:36.143 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:36.171 INFO:tasks.workunit.client.0.smithi133.stderr:+ FEATURES_AFTER= features: layering 2023-10-09T05:41:36.171 INFO:tasks.workunit.client.0.smithi133.stderr: op_features: 2023-10-09T05:41:36.172 INFO:tasks.workunit.client.0.smithi133.stderr:+ [ features: layering 2023-10-09T05:41:36.172 INFO:tasks.workunit.client.0.smithi133.stderr: op_features: != features: layering 2023-10-09T05:41:36.172 INFO:tasks.workunit.client.0.smithi133.stderr: op_features: ] 2023-10-09T05:41:36.172 INFO:tasks.workunit.client.0.smithi133.stderr:+ rm /tmp/rbd_import_export_287447/img_v2 2023-10-09T05:41:36.172 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd remove testimg_import 2023-10-09T05:41:36.174 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:36.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:36.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:36.269 INFO:tasks.workunit.client.0.smithi133.stderr: Removing image: 100% complete...done. 2023-10-09T05:41:36.274 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd remove testimg 2023-10-09T05:41:36.278 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:36.305 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:36.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:36.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:36.380 INFO:tasks.workunit.client.0.smithi133.stderr: Removing image: 100% complete...done. 2023-10-09T05:41:36.384 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_287447/img testimg 2023-10-09T05:41:36.409 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:36.443 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:36.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:36.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:36.466 INFO:tasks.workunit.client.0.smithi133.stderr: Importing image: 100% complete...done. 2023-10-09T05:41:36.469 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_287447/img_v2 2023-10-09T05:41:36.547 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:36.570 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:36.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:36.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:36.613 INFO:tasks.workunit.client.0.smithi133.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T05:41:36.617 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_287447/img_v2 testimg_import 2023-10-09T05:41:36.674 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:36.695 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:36.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:36.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:36.709 INFO:tasks.workunit.client.0.smithi133.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T05:41:36.714 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd info testimg_import 2023-10-09T05:41:36.714 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep stripe unit 2023-10-09T05:41:36.714 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep -Ei (4 KiB|4096) 2023-10-09T05:41:36.759 INFO:tasks.workunit.client.0.smithi133.stdout: stripe unit: 4 KiB 2023-10-09T05:41:36.760 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd info testimg_import 2023-10-09T05:41:36.760 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep stripe count 2023-10-09T05:41:36.760 INFO:tasks.workunit.client.0.smithi133.stderr:+ awk {print $3} 2023-10-09T05:41:36.760 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep 1000 2023-10-09T05:41:36.800 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:36.806 INFO:tasks.workunit.client.0.smithi133.stdout:1000 2023-10-09T05:41:36.807 INFO:tasks.workunit.client.0.smithi133.stderr:+ rm /tmp/rbd_import_export_287447/img_v2 2023-10-09T05:41:36.832 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:36.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:36.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:36.840 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd remove testimg_import 2023-10-09T05:41:36.936 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:36.960 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:36.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:36.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:36.975 INFO:tasks.workunit.client.0.smithi133.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-09T05:41:36.979 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd remove testimg 2023-10-09T05:41:37.064 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:37.092 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:37.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:37.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:37.105 INFO:tasks.workunit.client.0.smithi133.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-09T05:41:37.110 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_287447/img testimg 2023-10-09T05:41:37.188 INFO:tasks.workunit.client.0.smithi133.stderr: Importing image: 100% complete...done. 2023-10-09T05:41:37.191 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd snap create testimg@snap1 2023-10-09T05:41:37.196 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:37.228 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:37.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:37.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:37.332 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:37.366 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:37.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:37.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:37.469 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:37.495 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:37.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:37.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:37.598 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:37.624 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:37.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:37.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:37.649 INFO:tasks.workunit.client.0.smithi133.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T05:41:37.659 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd snap create testimg@snap2 2023-10-09T05:41:37.728 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:37.759 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:37.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:37.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:37.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:37 smithi133 bash[102858]: cluster 2023-10-09T05:41:36.509844+0000 mgr.y (mgr.25277) 1618 : cluster 0 pgmap v1814: 73 pgs: 73 active+clean; 692 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 195 KiB/s rd, 35 KiB/s wr, 214 op/s 2023-10-09T05:41:37.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:37 smithi133 bash[105589]: cluster 2023-10-09T05:41:36.509844+0000 mgr.y (mgr.25277) 1618 : cluster 0 pgmap v1814: 73 pgs: 73 active+clean; 692 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 195 KiB/s rd, 35 KiB/s wr, 214 op/s 2023-10-09T05:41:37.862 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:37.884 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:37.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:37 smithi194 bash[92666]: cluster 2023-10-09T05:41:36.509844+0000 mgr.y (mgr.25277) 1618 : cluster 0 pgmap v1814: 73 pgs: 73 active+clean; 692 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 195 KiB/s rd, 35 KiB/s wr, 214 op/s 2023-10-09T05:41:37.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:37.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:37.988 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:38.021 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:38.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:38.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:38.124 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:38.159 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:38.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:38.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:38.263 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:38.293 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:38.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:38.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:38.397 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:38.427 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:38.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:38.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:38.530 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:38.561 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:38.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:38.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:38.665 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:38.667 INFO:tasks.workunit.client.0.smithi133.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T05:41:38.674 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:38 smithi194 bash[92666]: cluster 2023-10-09T05:41:37.631438+0000 mon.a (mon.0) 2119 : cluster 0 osdmap e413: 8 total, 8 up, 7 in 2023-10-09T05:41:38.675 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:38 smithi194 bash[92666]: cluster 2023-10-09T05:41:37.636866+0000 osd.5 (osd.5) 449 : cluster 3 failed to encode map e413 with expected crc 2023-10-09T05:41:38.677 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd snap protect testimg@snap2 2023-10-09T05:41:38.690 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:38.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:38.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:38.732 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_287447/snap_protect 2023-10-09T05:41:38.791 INFO:tasks.workunit.client.0.smithi133.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T05:41:38.794 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:38.795 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_287447/snap_protect testimg_import 2023-10-09T05:41:38.819 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:38.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:38.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:38.923 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:38.931 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:38 smithi133 bash[102858]: cluster 2023-10-09T05:41:37.631438+0000 mon.a (mon.0) 2119 : cluster 0 osdmap e413: 8 total, 8 up, 7 in 2023-10-09T05:41:38.932 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:38 smithi133 bash[102858]: cluster 2023-10-09T05:41:37.636866+0000 osd.5 (osd.5) 449 : cluster 3 failed to encode map e413 with expected crc 2023-10-09T05:41:38.933 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:38 smithi133 bash[105589]: cluster 2023-10-09T05:41:37.631438+0000 mon.a (mon.0) 2119 : cluster 0 osdmap e413: 8 total, 8 up, 7 in 2023-10-09T05:41:38.933 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:38 smithi133 bash[105589]: cluster 2023-10-09T05:41:37.636866+0000 osd.5 (osd.5) 449 : cluster 3 failed to encode map e413 with expected crc 2023-10-09T05:41:38.945 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:38.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:38.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:39.049 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:39.080 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:39.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:39.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:39.183 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:39.215 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:39.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:39.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:39.319 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:39.354 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:39.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:39.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:39.458 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:39.489 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:39.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:39.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:39.593 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:39.621 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:39.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:39.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:39.724 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:39.749 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:39.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:39.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:39.853 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:39.884 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:39.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:39.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:39.987 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:40.000 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:39 smithi194 bash[92666]: cluster 2023-10-09T05:41:38.510434+0000 mgr.y (mgr.25277) 1619 : cluster 0 pgmap v1816: 73 pgs: 73 active+clean; 702 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 156 KiB/s rd, 18 KiB/s wr, 205 op/s 2023-10-09T05:41:40.001 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:39 smithi194 bash[92666]: cluster 2023-10-09T05:41:38.645263+0000 mon.a (mon.0) 2120 : cluster 0 osdmap e414: 8 total, 8 up, 7 in 2023-10-09T05:41:40.001 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:39 smithi194 bash[92666]: cluster 2023-10-09T05:41:38.648543+0000 osd.5 (osd.5) 450 : cluster 3 failed to encode map e414 with expected crc 2023-10-09T05:41:40.001 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:39 smithi194 bash[92666]: cluster 2023-10-09T05:41:38.658426+0000 osd.7 (osd.7) 383 : cluster 3 failed to encode map e414 with expected crc 2023-10-09T05:41:40.001 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:39 smithi194 bash[92666]: cluster 2023-10-09T05:41:38.983365+0000 mon.a (mon.0) 2121 : cluster 0 osdmap e415: 8 total, 8 up, 7 in 2023-10-09T05:41:40.001 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:39 smithi194 bash[92666]: cluster 2023-10-09T05:41:38.986019+0000 osd.5 (osd.5) 451 : cluster 3 failed to encode map e415 with expected crc 2023-10-09T05:41:40.001 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:39 smithi194 bash[92666]: cluster 2023-10-09T05:41:38.996007+0000 osd.6 (osd.6) 413 : cluster 3 failed to encode map e415 with expected crc 2023-10-09T05:41:40.018 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:40.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:40.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:40.031 INFO:tasks.workunit.client.0.smithi133.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T05:41:40.035 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep protected: False 2023-10-09T05:41:40.035 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd info testimg_import@snap1 2023-10-09T05:41:40.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:39 smithi133 bash[102858]: cluster 2023-10-09T05:41:38.510434+0000 mgr.y (mgr.25277) 1619 : cluster 0 pgmap v1816: 73 pgs: 73 active+clean; 702 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 156 KiB/s rd, 18 KiB/s wr, 205 op/s 2023-10-09T05:41:40.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:39 smithi133 bash[102858]: cluster 2023-10-09T05:41:38.645263+0000 mon.a (mon.0) 2120 : cluster 0 osdmap e414: 8 total, 8 up, 7 in 2023-10-09T05:41:40.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:39 smithi133 bash[102858]: cluster 2023-10-09T05:41:38.648543+0000 osd.5 (osd.5) 450 : cluster 3 failed to encode map e414 with expected crc 2023-10-09T05:41:40.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:39 smithi133 bash[102858]: cluster 2023-10-09T05:41:38.658426+0000 osd.7 (osd.7) 383 : cluster 3 failed to encode map e414 with expected crc 2023-10-09T05:41:40.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:39 smithi133 bash[102858]: cluster 2023-10-09T05:41:38.983365+0000 mon.a (mon.0) 2121 : cluster 0 osdmap e415: 8 total, 8 up, 7 in 2023-10-09T05:41:40.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:39 smithi133 bash[102858]: cluster 2023-10-09T05:41:38.986019+0000 osd.5 (osd.5) 451 : cluster 3 failed to encode map e415 with expected crc 2023-10-09T05:41:40.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:39 smithi133 bash[102858]: cluster 2023-10-09T05:41:38.996007+0000 osd.6 (osd.6) 413 : cluster 3 failed to encode map e415 with expected crc 2023-10-09T05:41:40.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:39 smithi133 bash[105589]: cluster 2023-10-09T05:41:38.510434+0000 mgr.y (mgr.25277) 1619 : cluster 0 pgmap v1816: 73 pgs: 73 active+clean; 702 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 156 KiB/s rd, 18 KiB/s wr, 205 op/s 2023-10-09T05:41:40.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:39 smithi133 bash[105589]: cluster 2023-10-09T05:41:38.645263+0000 mon.a (mon.0) 2120 : cluster 0 osdmap e414: 8 total, 8 up, 7 in 2023-10-09T05:41:40.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:39 smithi133 bash[105589]: cluster 2023-10-09T05:41:38.648543+0000 osd.5 (osd.5) 450 : cluster 3 failed to encode map e414 with expected crc 2023-10-09T05:41:40.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:39 smithi133 bash[105589]: cluster 2023-10-09T05:41:38.658426+0000 osd.7 (osd.7) 383 : cluster 3 failed to encode map e414 with expected crc 2023-10-09T05:41:40.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:39 smithi133 bash[105589]: cluster 2023-10-09T05:41:38.983365+0000 mon.a (mon.0) 2121 : cluster 0 osdmap e415: 8 total, 8 up, 7 in 2023-10-09T05:41:40.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:39 smithi133 bash[105589]: cluster 2023-10-09T05:41:38.986019+0000 osd.5 (osd.5) 451 : cluster 3 failed to encode map e415 with expected crc 2023-10-09T05:41:40.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:39 smithi133 bash[105589]: cluster 2023-10-09T05:41:38.996007+0000 osd.6 (osd.6) 413 : cluster 3 failed to encode map e415 with expected crc 2023-10-09T05:41:40.082 INFO:tasks.workunit.client.0.smithi133.stdout: protected: False 2023-10-09T05:41:40.083 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep protected: True 2023-10-09T05:41:40.083 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd info testimg_import@snap2 2023-10-09T05:41:40.108 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:40.121 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:40.136 INFO:tasks.workunit.client.0.smithi133.stdout: protected: True 2023-10-09T05:41:40.136 INFO:tasks.workunit.client.0.smithi133.stderr:+ rm /tmp/rbd_import_export_287447/snap_protect 2023-10-09T05:41:40.137 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd snap unprotect testimg@snap2 2023-10-09T05:41:40.149 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:40.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:40.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:40.196 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd snap unprotect testimg_import@snap2 2023-10-09T05:41:40.252 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:40.258 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd snap purge testimg 2023-10-09T05:41:40.283 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:40.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:40.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:40.387 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:40.414 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:40.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:40.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:40.518 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:40.539 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:40.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:40.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:40.643 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:40.673 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:40.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:40.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:40.778 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:40.810 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:40.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:40.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:40.913 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:40.949 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:40.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:40.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:41.054 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:41.084 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:41.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:41.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:41.187 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:41.214 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:41.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:41.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:41.317 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:41.343 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:41.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:41.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:41.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:41 smithi194 bash[92666]: cluster 2023-10-09T05:41:39.850774+0000 osd.4 (osd.4) 353 : cluster 3 failed to encode map e414 with expected crc 2023-10-09T05:41:41.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:41 smithi194 bash[92666]: cluster 2023-10-09T05:41:39.995157+0000 osd.4 (osd.4) 354 : cluster 3 failed to encode map e416 with expected crc 2023-10-09T05:41:41.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:41 smithi194 bash[92666]: cluster 2023-10-09T05:41:39.997387+0000 mon.a (mon.0) 2122 : cluster 0 osdmap e416: 8 total, 8 up, 7 in 2023-10-09T05:41:41.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:41 smithi194 bash[92666]: cluster 2023-10-09T05:41:39.999718+0000 osd.5 (osd.5) 452 : cluster 3 failed to encode map e416 with expected crc 2023-10-09T05:41:41.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:41 smithi194 bash[92666]: cluster 2023-10-09T05:41:40.007089+0000 osd.6 (osd.6) 414 : cluster 3 failed to encode map e416 with expected crc 2023-10-09T05:41:41.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:41 smithi194 bash[92666]: cluster 2023-10-09T05:41:40.510972+0000 mgr.y (mgr.25277) 1620 : cluster 0 pgmap v1820: 73 pgs: 73 active+clean; 702 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 354 KiB/s rd, 37 KiB/s wr, 451 op/s 2023-10-09T05:41:41.447 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:41.480 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:41.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:41.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:41.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:41 smithi133 bash[102858]: cluster 2023-10-09T05:41:39.850774+0000 osd.4 (osd.4) 353 : cluster 3 failed to encode map e414 with expected crc 2023-10-09T05:41:41.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:41 smithi133 bash[102858]: cluster 2023-10-09T05:41:39.995157+0000 osd.4 (osd.4) 354 : cluster 3 failed to encode map e416 with expected crc 2023-10-09T05:41:41.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:41 smithi133 bash[102858]: cluster 2023-10-09T05:41:39.997387+0000 mon.a (mon.0) 2122 : cluster 0 osdmap e416: 8 total, 8 up, 7 in 2023-10-09T05:41:41.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:41 smithi133 bash[102858]: cluster 2023-10-09T05:41:39.999718+0000 osd.5 (osd.5) 452 : cluster 3 failed to encode map e416 with expected crc 2023-10-09T05:41:41.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:41 smithi133 bash[102858]: cluster 2023-10-09T05:41:40.007089+0000 osd.6 (osd.6) 414 : cluster 3 failed to encode map e416 with expected crc 2023-10-09T05:41:41.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:41 smithi133 bash[102858]: cluster 2023-10-09T05:41:40.510972+0000 mgr.y (mgr.25277) 1620 : cluster 0 pgmap v1820: 73 pgs: 73 active+clean; 702 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 354 KiB/s rd, 37 KiB/s wr, 451 op/s 2023-10-09T05:41:41.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:41 smithi133 bash[105589]: cluster 2023-10-09T05:41:39.850774+0000 osd.4 (osd.4) 353 : cluster 3 failed to encode map e414 with expected crc 2023-10-09T05:41:41.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:41 smithi133 bash[105589]: cluster 2023-10-09T05:41:39.995157+0000 osd.4 (osd.4) 354 : cluster 3 failed to encode map e416 with expected crc 2023-10-09T05:41:41.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:41 smithi133 bash[105589]: cluster 2023-10-09T05:41:39.997387+0000 mon.a (mon.0) 2122 : cluster 0 osdmap e416: 8 total, 8 up, 7 in 2023-10-09T05:41:41.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:41 smithi133 bash[105589]: cluster 2023-10-09T05:41:39.999718+0000 osd.5 (osd.5) 452 : cluster 3 failed to encode map e416 with expected crc 2023-10-09T05:41:41.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:41 smithi133 bash[105589]: cluster 2023-10-09T05:41:40.007089+0000 osd.6 (osd.6) 414 : cluster 3 failed to encode map e416 with expected crc 2023-10-09T05:41:41.549 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:41 smithi133 bash[105589]: cluster 2023-10-09T05:41:40.510972+0000 mgr.y (mgr.25277) 1620 : cluster 0 pgmap v1820: 73 pgs: 73 active+clean; 702 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 354 KiB/s rd, 37 KiB/s wr, 451 op/s 2023-10-09T05:41:41.583 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:41.620 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:41.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:41.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:41.723 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:41.745 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:41.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:41.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:41.848 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:41.883 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:41.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:41.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:41.987 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:42.019 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:42.020 INFO:tasks.workunit.client.0.smithi133.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-09T05:41:42.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:42.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:42.028 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd snap purge testimg_import 2023-10-09T05:41:42.122 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:42.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:42 smithi194 bash[92666]: cluster 2023-10-09T05:41:41.005008+0000 osd.5 (osd.5) 453 : cluster 3 failed to encode map e417 with expected crc 2023-10-09T05:41:42.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:42 smithi194 bash[92666]: cluster 2023-10-09T05:41:41.012611+0000 mon.a (mon.0) 2123 : cluster 0 osdmap e417: 8 total, 8 up, 7 in 2023-10-09T05:41:42.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:42 smithi194 bash[92666]: cluster 2023-10-09T05:41:41.024909+0000 osd.7 (osd.7) 384 : cluster 3 failed to encode map e417 with expected crc 2023-10-09T05:41:42.146 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:42.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:42.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:42.249 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:42.260 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:42 smithi133 bash[105589]: cluster 2023-10-09T05:41:41.005008+0000 osd.5 (osd.5) 453 : cluster 3 failed to encode map e417 with expected crc 2023-10-09T05:41:42.260 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:42 smithi133 bash[105589]: cluster 2023-10-09T05:41:41.012611+0000 mon.a (mon.0) 2123 : cluster 0 osdmap e417: 8 total, 8 up, 7 in 2023-10-09T05:41:42.260 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:42 smithi133 bash[105589]: cluster 2023-10-09T05:41:41.024909+0000 osd.7 (osd.7) 384 : cluster 3 failed to encode map e417 with expected crc 2023-10-09T05:41:42.261 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:42 smithi133 bash[102858]: cluster 2023-10-09T05:41:41.005008+0000 osd.5 (osd.5) 453 : cluster 3 failed to encode map e417 with expected crc 2023-10-09T05:41:42.261 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:42 smithi133 bash[102858]: cluster 2023-10-09T05:41:41.012611+0000 mon.a (mon.0) 2123 : cluster 0 osdmap e417: 8 total, 8 up, 7 in 2023-10-09T05:41:42.261 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:42 smithi133 bash[102858]: cluster 2023-10-09T05:41:41.024909+0000 osd.7 (osd.7) 384 : cluster 3 failed to encode map e417 with expected crc 2023-10-09T05:41:42.274 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:42.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:42.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:42.378 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:42.404 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:42.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:42.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:42.508 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:42.534 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:42.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:42.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:42.637 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:42.664 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:42.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:42.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:42.768 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:42.802 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:42.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:42.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:42.905 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:42.932 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:42.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:42.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:43.035 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:43.044 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:43 smithi194 bash[92666]: cluster 2023-10-09T05:41:41.434906+0000 osd.4 (osd.4) 355 : cluster 3 failed to encode map e417 with expected crc 2023-10-09T05:41:43.045 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:43 smithi194 bash[92666]: cluster 2023-10-09T05:41:42.011720+0000 osd.5 (osd.5) 454 : cluster 3 failed to encode map e418 with expected crc 2023-10-09T05:41:43.045 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:43 smithi194 bash[92666]: cluster 2023-10-09T05:41:42.014116+0000 mon.a (mon.0) 2124 : cluster 0 osdmap e418: 8 total, 8 up, 7 in 2023-10-09T05:41:43.045 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:43 smithi194 bash[92666]: cluster 2023-10-09T05:41:42.511344+0000 mgr.y (mgr.25277) 1621 : cluster 0 pgmap v1823: 73 pgs: 73 active+clean; 802 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 339 KiB/s rd, 39 KiB/s wr, 446 op/s 2023-10-09T05:41:43.060 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:43.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:43.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:43.163 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:43.175 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:43 smithi133 bash[102858]: cluster 2023-10-09T05:41:41.434906+0000 osd.4 (osd.4) 355 : cluster 3 failed to encode map e417 with expected crc 2023-10-09T05:41:43.176 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:43 smithi133 bash[102858]: cluster 2023-10-09T05:41:42.011720+0000 osd.5 (osd.5) 454 : cluster 3 failed to encode map e418 with expected crc 2023-10-09T05:41:43.176 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:43 smithi133 bash[102858]: cluster 2023-10-09T05:41:42.014116+0000 mon.a (mon.0) 2124 : cluster 0 osdmap e418: 8 total, 8 up, 7 in 2023-10-09T05:41:43.176 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:43 smithi133 bash[102858]: cluster 2023-10-09T05:41:42.511344+0000 mgr.y (mgr.25277) 1621 : cluster 0 pgmap v1823: 73 pgs: 73 active+clean; 802 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 339 KiB/s rd, 39 KiB/s wr, 446 op/s 2023-10-09T05:41:43.176 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:43 smithi133 bash[105589]: cluster 2023-10-09T05:41:41.434906+0000 osd.4 (osd.4) 355 : cluster 3 failed to encode map e417 with expected crc 2023-10-09T05:41:43.176 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:43 smithi133 bash[105589]: cluster 2023-10-09T05:41:42.011720+0000 osd.5 (osd.5) 454 : cluster 3 failed to encode map e418 with expected crc 2023-10-09T05:41:43.176 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:43 smithi133 bash[105589]: cluster 2023-10-09T05:41:42.014116+0000 mon.a (mon.0) 2124 : cluster 0 osdmap e418: 8 total, 8 up, 7 in 2023-10-09T05:41:43.176 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:43 smithi133 bash[105589]: cluster 2023-10-09T05:41:42.511344+0000 mgr.y (mgr.25277) 1621 : cluster 0 pgmap v1823: 73 pgs: 73 active+clean; 802 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 339 KiB/s rd, 39 KiB/s wr, 446 op/s 2023-10-09T05:41:43.194 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:43.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:43.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:43.297 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:43.327 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:43.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:43.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:43.436 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:43.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:43.475 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:43.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:43.578 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:43.614 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:43.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:43.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:43.720 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:43.750 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:43.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:43.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:43.854 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:43.880 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:43.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:43.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:43.986 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:44.002 INFO:tasks.workunit.client.0.smithi133.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-09T05:41:44.018 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd remove testimg 2023-10-09T05:41:44.023 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:44.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:44.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:44.127 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:44.140 INFO:tasks.workunit.client.0.smithi133.stderr: Removing image: 100% complete...done. 2023-10-09T05:41:44.143 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd remove testimg_import 2023-10-09T05:41:44.157 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:44.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:44.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:44.262 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:44.273 INFO:tasks.workunit.client.0.smithi133.stderr: Removing image: 100% complete...done. 2023-10-09T05:41:44.274 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:44 smithi133 bash[105589]: cluster 2023-10-09T05:41:43.021016+0000 osd.4 (osd.4) 356 : cluster 3 failed to encode map e419 with expected crc 2023-10-09T05:41:44.274 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:44 smithi133 bash[105589]: cluster 2023-10-09T05:41:43.026144+0000 mon.a (mon.0) 2125 : cluster 0 osdmap e419: 8 total, 8 up, 7 in 2023-10-09T05:41:44.274 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:44 smithi133 bash[105589]: cluster 2023-10-09T05:41:43.030870+0000 osd.5 (osd.5) 455 : cluster 3 failed to encode map e419 with expected crc 2023-10-09T05:41:44.274 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:44 smithi133 bash[105589]: audit 2023-10-09T05:41:43.497437+0000 mon.a (mon.0) 2126 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:44.275 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:44 smithi133 bash[105589]: cluster 2023-10-09T05:41:43.986105+0000 mon.a (mon.0) 2127 : cluster 0 osdmap e420: 8 total, 8 up, 7 in 2023-10-09T05:41:44.275 INFO:tasks.workunit.client.0.smithi133.stderr:+ tiered=0 2023-10-09T05:41:44.275 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:44 smithi133 bash[102858]: cluster 2023-10-09T05:41:43.021016+0000 osd.4 (osd.4) 356 : cluster 3 failed to encode map e419 with expected crc 2023-10-09T05:41:44.275 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:44 smithi133 bash[102858]: cluster 2023-10-09T05:41:43.026144+0000 mon.a (mon.0) 2125 : cluster 0 osdmap e419: 8 total, 8 up, 7 in 2023-10-09T05:41:44.275 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:44 smithi133 bash[102858]: cluster 2023-10-09T05:41:43.030870+0000 osd.5 (osd.5) 455 : cluster 3 failed to encode map e419 with expected crc 2023-10-09T05:41:44.275 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:44 smithi133 bash[102858]: audit 2023-10-09T05:41:43.497437+0000 mon.a (mon.0) 2126 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:44.276 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:44 smithi133 bash[102858]: cluster 2023-10-09T05:41:43.986105+0000 mon.a (mon.0) 2127 : cluster 0 osdmap e420: 8 total, 8 up, 7 in 2023-10-09T05:41:44.276 INFO:tasks.workunit.client.0.smithi133.stderr:+ ceph osd dump 2023-10-09T05:41:44.276 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep ^pool 2023-10-09T05:41:44.276 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep 'rbd' 2023-10-09T05:41:44.277 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep tier 2023-10-09T05:41:44.286 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:44.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:44.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:44.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:44 smithi194 bash[92666]: cluster 2023-10-09T05:41:43.021016+0000 osd.4 (osd.4) 356 : cluster 3 failed to encode map e419 with expected crc 2023-10-09T05:41:44.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:44 smithi194 bash[92666]: cluster 2023-10-09T05:41:43.026144+0000 mon.a (mon.0) 2125 : cluster 0 osdmap e419: 8 total, 8 up, 7 in 2023-10-09T05:41:44.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:44 smithi194 bash[92666]: cluster 2023-10-09T05:41:43.030870+0000 osd.5 (osd.5) 455 : cluster 3 failed to encode map e419 with expected crc 2023-10-09T05:41:44.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:44 smithi194 bash[92666]: audit 2023-10-09T05:41:43.497437+0000 mon.a (mon.0) 2126 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:44.387 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:44 smithi194 bash[92666]: cluster 2023-10-09T05:41:43.986105+0000 mon.a (mon.0) 2127 : cluster 0 osdmap e420: 8 total, 8 up, 7 in 2023-10-09T05:41:44.388 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:44.422 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:44.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:44.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:44.526 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:44.559 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:44.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:44.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:44.563 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:41:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:41:44] "GET /metrics HTTP/1.1" 200 34149 "" "Prometheus/2.43.0" 2023-10-09T05:41:44.663 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:44.688 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:44.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:44.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:44.791 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:44.809 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:44.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:44.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:44.913 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:44.935 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:44.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:44.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:45.039 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:45.057 INFO:teuthology.orchestra.run.smithi133.stdout:osd.0: {} 2023-10-09T05:41:45.060 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:45.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:45.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:45.077 INFO:teuthology.orchestra.run.smithi133.stdout:osd.1: {} 2023-10-09T05:41:45.096 INFO:teuthology.orchestra.run.smithi133.stdout:osd.2: {} 2023-10-09T05:41:45.115 INFO:teuthology.orchestra.run.smithi133.stdout:osd.3: {} 2023-10-09T05:41:45.144 INFO:teuthology.orchestra.run.smithi133.stdout:osd.4: {} 2023-10-09T05:41:45.164 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:45.165 INFO:teuthology.orchestra.run.smithi133.stdout:osd.5: {} 2023-10-09T05:41:45.187 INFO:teuthology.orchestra.run.smithi133.stdout:osd.6: {} 2023-10-09T05:41:45.195 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:45.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:45.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:45.209 INFO:teuthology.orchestra.run.smithi133.stdout:osd.7: {} 2023-10-09T05:41:45.210 INFO:teuthology.orchestra.run.smithi133.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:41:45.210 INFO:teuthology.orchestra.run.smithi133.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:41:45.210 INFO:teuthology.orchestra.run.smithi133.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:41:45.210 INFO:teuthology.orchestra.run.smithi133.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:41:45.210 INFO:teuthology.orchestra.run.smithi133.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:41:45.211 INFO:teuthology.orchestra.run.smithi133.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:41:45.211 INFO:teuthology.orchestra.run.smithi133.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:41:45.211 INFO:teuthology.orchestra.run.smithi133.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:41:45.266 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:45 smithi133 bash[105589]: cluster 2023-10-09T05:41:43.082121+0000 osd.6 (osd.6) 415 : cluster 3 failed to encode map e419 with expected crc 2023-10-09T05:41:45.266 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:45 smithi133 bash[105589]: cluster 2023-10-09T05:41:43.984452+0000 osd.7 (osd.7) 385 : cluster 3 failed to encode map e420 with expected crc 2023-10-09T05:41:45.266 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:45 smithi133 bash[105589]: cluster 2023-10-09T05:41:43.987359+0000 osd.5 (osd.5) 456 : cluster 3 failed to encode map e420 with expected crc 2023-10-09T05:41:45.266 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:45 smithi133 bash[105589]: cluster 2023-10-09T05:41:43.991523+0000 osd.5 (osd.5) 457 : cluster 3 failed to encode map e420 with expected crc 2023-10-09T05:41:45.267 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:45 smithi133 bash[105589]: cluster 2023-10-09T05:41:44.003526+0000 osd.6 (osd.6) 416 : cluster 3 failed to encode map e420 with expected crc 2023-10-09T05:41:45.267 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:45 smithi133 bash[105589]: cluster 2023-10-09T05:41:44.511733+0000 mgr.y (mgr.25277) 1622 : cluster 0 pgmap v1826: 73 pgs: 73 active+clean; 802 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 213 KiB/s rd, 17 KiB/s wr, 286 op/s 2023-10-09T05:41:45.267 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:45 smithi133 bash[105589]: audit 2023-10-09T05:41:45.033632+0000 mon.a (mon.0) 2128 : audit 0 from='client.? 172.21.15.133:0/3362123186' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:41:45.268 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:45 smithi133 bash[102858]: cluster 2023-10-09T05:41:43.082121+0000 osd.6 (osd.6) 415 : cluster 3 failed to encode map e419 with expected crc 2023-10-09T05:41:45.268 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:45 smithi133 bash[102858]: cluster 2023-10-09T05:41:43.984452+0000 osd.7 (osd.7) 385 : cluster 3 failed to encode map e420 with expected crc 2023-10-09T05:41:45.268 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:45 smithi133 bash[102858]: cluster 2023-10-09T05:41:43.987359+0000 osd.5 (osd.5) 456 : cluster 3 failed to encode map e420 with expected crc 2023-10-09T05:41:45.268 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:45 smithi133 bash[102858]: cluster 2023-10-09T05:41:43.991523+0000 osd.5 (osd.5) 457 : cluster 3 failed to encode map e420 with expected crc 2023-10-09T05:41:45.269 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:45 smithi133 bash[102858]: cluster 2023-10-09T05:41:44.003526+0000 osd.6 (osd.6) 416 : cluster 3 failed to encode map e420 with expected crc 2023-10-09T05:41:45.269 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:45 smithi133 bash[102858]: cluster 2023-10-09T05:41:44.511733+0000 mgr.y (mgr.25277) 1622 : cluster 0 pgmap v1826: 73 pgs: 73 active+clean; 802 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 213 KiB/s rd, 17 KiB/s wr, 286 op/s 2023-10-09T05:41:45.269 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:45 smithi133 bash[102858]: audit 2023-10-09T05:41:45.033632+0000 mon.a (mon.0) 2128 : audit 0 from='client.? 172.21.15.133:0/3362123186' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:41:45.298 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:45.309 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:45 smithi194 bash[92666]: cluster 2023-10-09T05:41:43.082121+0000 osd.6 (osd.6) 415 : cluster 3 failed to encode map e419 with expected crc 2023-10-09T05:41:45.310 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:45 smithi194 bash[92666]: cluster 2023-10-09T05:41:43.984452+0000 osd.7 (osd.7) 385 : cluster 3 failed to encode map e420 with expected crc 2023-10-09T05:41:45.310 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:45 smithi194 bash[92666]: cluster 2023-10-09T05:41:43.987359+0000 osd.5 (osd.5) 456 : cluster 3 failed to encode map e420 with expected crc 2023-10-09T05:41:45.310 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:45 smithi194 bash[92666]: cluster 2023-10-09T05:41:43.991523+0000 osd.5 (osd.5) 457 : cluster 3 failed to encode map e420 with expected crc 2023-10-09T05:41:45.310 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:45 smithi194 bash[92666]: cluster 2023-10-09T05:41:44.003526+0000 osd.6 (osd.6) 416 : cluster 3 failed to encode map e420 with expected crc 2023-10-09T05:41:45.310 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:45 smithi194 bash[92666]: cluster 2023-10-09T05:41:44.511733+0000 mgr.y (mgr.25277) 1622 : cluster 0 pgmap v1826: 73 pgs: 73 active+clean; 802 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 213 KiB/s rd, 17 KiB/s wr, 286 op/s 2023-10-09T05:41:45.310 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:45 smithi194 bash[92666]: audit 2023-10-09T05:41:45.033632+0000 mon.a (mon.0) 2128 : audit 0 from='client.? 172.21.15.133:0/3362123186' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:41:45.331 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:45.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:45.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:45.433 INFO:tasks.workunit.client.0.smithi133.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_287447/sparse1 2023-10-09T05:41:45.435 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:45.437 INFO:tasks.workunit.client.0.smithi133.stderr:1+0 records in 2023-10-09T05:41:45.437 INFO:tasks.workunit.client.0.smithi133.stderr:1+0 records out 2023-10-09T05:41:45.437 INFO:tasks.workunit.client.0.smithi133.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00348995 s, 300 MB/s 2023-10-09T05:41:45.437 INFO:tasks.workunit.client.0.smithi133.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_287447/sparse2 2023-10-09T05:41:45.441 INFO:tasks.workunit.client.0.smithi133.stderr:1+0 records in 2023-10-09T05:41:45.441 INFO:tasks.workunit.client.0.smithi133.stderr:1+0 records out 2023-10-09T05:41:45.441 INFO:tasks.workunit.client.0.smithi133.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00341742 s, 307 MB/s 2023-10-09T05:41:45.441 INFO:tasks.workunit.client.0.smithi133.stderr:+ truncate /tmp/rbd_import_export_287447/sparse2 -s 2M 2023-10-09T05:41:45.466 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:45.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:45.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:45.561 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd rm sparse1 2023-10-09T05:41:45.570 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:45.594 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:45.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:45.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:45.613 INFO:tasks.workunit.client.0.smithi133.stderr: Removing image: 0% complete...failed. 2023-10-09T05:41:45.613 INFO:tasks.workunit.client.0.smithi133.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T05:41:45.618 INFO:tasks.workunit.client.0.smithi133.stderr:+ true 2023-10-09T05:41:45.618 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_287447/sparse1 2023-10-09T05:41:45.698 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:45.705 INFO:tasks.workunit.client.0.smithi133.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-09T05:41:45.706 INFO:tasks.workunit.client.0.smithi133.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T05:41:45.706 INFO:tasks.workunit.client.0.smithi133.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T05:41:45.709 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd ls -l 2023-10-09T05:41:45.710 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep sparse1 2023-10-09T05:41:45.710 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep -Ei (2 MiB|2048k) 2023-10-09T05:41:45.728 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:45.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:45.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:45.784 INFO:tasks.workunit.client.0.smithi133.stdout:sparse1 2 MiB 2 2023-10-09T05:41:45.785 INFO:tasks.workunit.client.0.smithi133.stderr:+ objects sparse1 2023-10-09T05:41:45.785 INFO:tasks.workunit.client.0.smithi133.stderr:+ image=sparse1 2023-10-09T05:41:45.785 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd info sparse1 2023-10-09T05:41:45.785 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep+ block_name_prefix 2023-10-09T05:41:45.786 INFO:tasks.workunit.client.0.smithi133.stderr:awk {print $NF;} 2023-10-09T05:41:45.830 INFO:tasks.workunit.client.0.smithi133.stderr:+ prefix=rbd_data.8de4d5038d3 2023-10-09T05:41:45.831 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep rbd_data.8de4d5038d3 2023-10-09T05:41:45.831 INFO:tasks.workunit.client.0.smithi133.stderr:+ + sed -e s/rbd_data.8de4d5038d3\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T05:41:45.831 INFO:tasks.workunit.client.0.smithi133.stderr:get_image_data_pool sparse1 2023-10-09T05:41:45.831 INFO:tasks.workunit.client.0.smithi133.stderr:+ image=sparse1 2023-10-09T05:41:45.831 INFO:tasks.workunit.client.0.smithi133.stderr:+ sort -u 2023-10-09T05:41:45.831 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd info sparse1 2023-10-09T05:41:45.831 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep data_pool: 2023-10-09T05:41:45.832 INFO:tasks.workunit.client.0.smithi133.stderr:+ awk -F: { print $NF } 2023-10-09T05:41:45.832 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:45.855 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:45.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:45.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:45.871 INFO:tasks.workunit.client.0.smithi133.stderr:+ data_pool= 2023-10-09T05:41:45.872 INFO:tasks.workunit.client.0.smithi133.stderr:+ [ -z ] 2023-10-09T05:41:45.872 INFO:tasks.workunit.client.0.smithi133.stderr:+ data_pool=rbd 2023-10-09T05:41:45.872 INFO:tasks.workunit.client.0.smithi133.stderr:+ echo rbd 2023-10-09T05:41:45.872 INFO:tasks.workunit.client.0.smithi133.stderr:+ rados ls -p rbd 2023-10-09T05:41:45.959 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:45.991 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:45.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:45.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:46.077 INFO:tasks.workunit.client.0.smithi133.stderr:+ objects=1 2023-10-09T05:41:46.077 INFO:tasks.workunit.client.0.smithi133.stderr:+ echo 1 2023-10-09T05:41:46.078 INFO:tasks.workunit.client.0.smithi133.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-10-09T05:41:46.078 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd export sparse1 /tmp/rbd_import_export_287447/sparse1.out 2023-10-09T05:41:46.094 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:46.101 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:46 smithi133 bash[105589]: audit 2023-10-09T05:41:45.311895+0000 mon.a (mon.0) 2129 : audit 0 from='client.? 172.21.15.133:0/2457134380' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T05:41:46.101 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:46 smithi133 bash[105589]: audit 2023-10-09T05:41:45.416118+0000 mon.a (mon.0) 2130 : audit 0 from='client.? 172.21.15.133:0/2457134380' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T05:41:46.102 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:46 smithi133 bash[102858]: audit 2023-10-09T05:41:45.311895+0000 mon.a (mon.0) 2129 : audit 0 from='client.? 172.21.15.133:0/2457134380' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T05:41:46.102 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:46 smithi133 bash[102858]: audit 2023-10-09T05:41:45.416118+0000 mon.a (mon.0) 2130 : audit 0 from='client.? 172.21.15.133:0/2457134380' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T05:41:46.119 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:46.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:46.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:46.124 INFO:tasks.workunit.client.0.smithi133.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-09T05:41:46.128 INFO:tasks.workunit.client.0.smithi133.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_287447/sparse1 /tmp/rbd_import_export_287447/sparse1.out 2023-10-09T05:41:46.129 INFO:tasks.workunit.client.0.smithi133.stderr:+ cmp -l /tmp/rbd_import_export_287447/sparse1 /tmp/rbd_import_export_287447/sparse1.out 2023-10-09T05:41:46.132 INFO:tasks.workunit.client.0.smithi133.stderr:+ stat /tmp/rbd_import_export_287447/sparse1 --format %b 2023-10-09T05:41:46.133 INFO:tasks.workunit.client.0.smithi133.stderr:+ origsize=2048 2023-10-09T05:41:46.133 INFO:tasks.workunit.client.0.smithi133.stderr:+ stat /tmp/rbd_import_export_287447/sparse1.out --format %b 2023-10-09T05:41:46.134 INFO:tasks.workunit.client.0.smithi133.stderr:+ exportsize=2048 2023-10-09T05:41:46.134 INFO:tasks.workunit.client.0.smithi133.stderr:+ difference=0 2023-10-09T05:41:46.134 INFO:tasks.workunit.client.0.smithi133.stderr:+ difference=0 2023-10-09T05:41:46.134 INFO:tasks.workunit.client.0.smithi133.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T05:41:46.134 INFO:tasks.workunit.client.0.smithi133.stderr:+ rm /tmp/rbd_import_export_287447/sparse1.out 2023-10-09T05:41:46.135 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd rm sparse1 2023-10-09T05:41:46.223 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:46.241 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:46.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:46.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:46.324 INFO:tasks.workunit.client.0.smithi133.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-09T05:41:46.327 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd rm sparse2 2023-10-09T05:41:46.344 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:46.357 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:46 smithi194 bash[92666]: audit 2023-10-09T05:41:45.311895+0000 mon.a (mon.0) 2129 : audit 0 from='client.? 172.21.15.133:0/2457134380' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T05:41:46.357 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:46 smithi194 bash[92666]: audit 2023-10-09T05:41:45.416118+0000 mon.a (mon.0) 2130 : audit 0 from='client.? 172.21.15.133:0/2457134380' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T05:41:46.370 INFO:tasks.workunit.client.0.smithi133.stderr: Removing image: 0% complete...failed. 2023-10-09T05:41:46.371 INFO:tasks.workunit.client.0.smithi133.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T05:41:46.374 INFO:tasks.workunit.client.0.smithi133.stderr:+ true 2023-10-09T05:41:46.374 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_287447/sparse2 2023-10-09T05:41:46.374 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:46.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:46.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:46.455 INFO:tasks.workunit.client.0.smithi133.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-09T05:41:46.456 INFO:tasks.workunit.client.0.smithi133.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T05:41:46.456 INFO:tasks.workunit.client.0.smithi133.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T05:41:46.461 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd ls -l 2023-10-09T05:41:46.461 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep sparse2 2023-10-09T05:41:46.461 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep -Ei (2 MiB|2048k) 2023-10-09T05:41:46.480 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:46.506 INFO:tasks.workunit.client.0.smithi133.stdout:sparse2 2 MiB 2 2023-10-09T05:41:46.507 INFO:tasks.workunit.client.0.smithi133.stderr:+ objects sparse2 2023-10-09T05:41:46.508 INFO:tasks.workunit.client.0.smithi133.stderr:+ image=sparse2 2023-10-09T05:41:46.508 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd+ info sparse2 2023-10-09T05:41:46.508 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep block_name_prefix 2023-10-09T05:41:46.508 INFO:tasks.workunit.client.0.smithi133.stderr:awk {print $NF;} 2023-10-09T05:41:46.513 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:46.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:46.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:46.554 INFO:tasks.workunit.client.0.smithi133.stderr:+ prefix=rbd_data.b53aca3ebd1d 2023-10-09T05:41:46.556 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep rbd_data.b53aca3ebd1d 2023-10-09T05:41:46.556 INFO:tasks.workunit.client.0.smithi133.stderr:+ sed -e s/rbd_data.b53aca3ebd1d\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T05:41:46.556 INFO:tasks.workunit.client.0.smithi133.stderr:+ get_image_data_pool sparse2 2023-10-09T05:41:46.556 INFO:tasks.workunit.client.0.smithi133.stderr:+ image=sparse2 2023-10-09T05:41:46.557 INFO:tasks.workunit.client.0.smithi133.stderr:+ sort -u 2023-10-09T05:41:46.557 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd info sparse2 2023-10-09T05:41:46.557 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep data_pool: 2023-10-09T05:41:46.557 INFO:tasks.workunit.client.0.smithi133.stderr:+ awk -F: { print $NF } 2023-10-09T05:41:46.605 INFO:tasks.workunit.client.0.smithi133.stderr:+ data_pool= 2023-10-09T05:41:46.605 INFO:tasks.workunit.client.0.smithi133.stderr:+ [ -z ] 2023-10-09T05:41:46.606 INFO:tasks.workunit.client.0.smithi133.stderr:+ data_pool=rbd 2023-10-09T05:41:46.606 INFO:tasks.workunit.client.0.smithi133.stderr:+ echo rbd 2023-10-09T05:41:46.606 INFO:tasks.workunit.client.0.smithi133.stderr:+ rados ls -p rbd 2023-10-09T05:41:46.618 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:46.644 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:46.646 INFO:tasks.workunit.client.0.smithi133.stderr:+ objects=0 2023-10-09T05:41:46.647 INFO:tasks.workunit.client.0.smithi133.stderr:+ echo 0 2023-10-09T05:41:46.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:46.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:46.648 INFO:tasks.workunit.client.0.smithi133.stderr:+ [ 0 -eq 1 -o 0 = 0 ] 2023-10-09T05:41:46.648 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd export sparse2 /tmp/rbd_import_export_287447/sparse2.out 2023-10-09T05:41:46.692 INFO:tasks.workunit.client.0.smithi133.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-09T05:41:46.698 INFO:tasks.workunit.client.0.smithi133.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_287447/sparse2 /tmp/rbd_import_export_287447/sparse2.out 2023-10-09T05:41:46.698 INFO:tasks.workunit.client.0.smithi133.stderr:+ cmp -l /tmp/rbd_import_export_287447/sparse2 /tmp/rbd_import_export_287447/sparse2.out 2023-10-09T05:41:46.700 INFO:tasks.workunit.client.0.smithi133.stderr:+ stat /tmp/rbd_import_export_287447/sparse2 --format %b 2023-10-09T05:41:46.701 INFO:tasks.workunit.client.0.smithi133.stderr:+ origsize=2048 2023-10-09T05:41:46.701 INFO:tasks.workunit.client.0.smithi133.stderr:+ stat /tmp/rbd_import_export_287447/sparse2.out --format %b 2023-10-09T05:41:46.702 INFO:tasks.workunit.client.0.smithi133.stderr:+ exportsize=2048 2023-10-09T05:41:46.702 INFO:tasks.workunit.client.0.smithi133.stderr:+ difference=0 2023-10-09T05:41:46.702 INFO:tasks.workunit.client.0.smithi133.stderr:+ difference=0 2023-10-09T05:41:46.702 INFO:tasks.workunit.client.0.smithi133.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T05:41:46.703 INFO:tasks.workunit.client.0.smithi133.stderr:+ rm /tmp/rbd_import_export_287447/sparse2.out 2023-10-09T05:41:46.703 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd rm sparse2 2023-10-09T05:41:46.748 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:46.781 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:46.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:46.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:46.867 INFO:tasks.workunit.client.0.smithi133.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-09T05:41:46.872 INFO:tasks.workunit.client.0.smithi133.stderr:+ truncate /tmp/rbd_import_export_287447/sparse1 -s 10M 2023-10-09T05:41:46.873 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd import --new-format --order 20 - sparse1 2023-10-09T05:41:46.885 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:46.921 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:46.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:46.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:47.009 INFO:tasks.workunit.client.0.smithi133.stderr: Importing image: 100% complete...done. 2023-10-09T05:41:47.010 INFO:tasks.workunit.client.0.smithi133.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T05:41:47.010 INFO:tasks.workunit.client.0.smithi133.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T05:41:47.014 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd ls -l 2023-10-09T05:41:47.014 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep sparse1 2023-10-09T05:41:47.014 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep -Ei (10 MiB|10240k) 2023-10-09T05:41:47.025 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:47.056 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:47.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:47.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:47.064 INFO:tasks.workunit.client.0.smithi133.stdout:sparse1 10 MiB 2 2023-10-09T05:41:47.065 INFO:tasks.workunit.client.0.smithi133.stderr:+ objects sparse1 2023-10-09T05:41:47.065 INFO:tasks.workunit.client.0.smithi133.stderr:+ image=sparse1 2023-10-09T05:41:47.066 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd info sparse1 2023-10-09T05:41:47.066 INFO:tasks.workunit.client.0.smithi133.stderr:+ + grep block_name_prefix 2023-10-09T05:41:47.066 INFO:tasks.workunit.client.0.smithi133.stderr:awk {print $NF;} 2023-10-09T05:41:47.104 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:47 smithi133 bash[102858]: cluster 2023-10-09T05:41:46.512234+0000 mgr.y (mgr.25277) 1623 : cluster 0 pgmap v1827: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 180 KiB/s rd, 14 KiB/s wr, 244 op/s 2023-10-09T05:41:47.112 INFO:tasks.workunit.client.0.smithi133.stderr:+ prefix=rbd_data.8e4415fa6b90 2023-10-09T05:41:47.113 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep rbd_data.8e4415fa6b90 2023-10-09T05:41:47.113 INFO:tasks.workunit.client.0.smithi133.stderr:+ get_image_data_pool sparse1 2023-10-09T05:41:47.113 INFO:tasks.workunit.client.0.smithi133.stderr:+ image=sparse1 2023-10-09T05:41:47.113 INFO:tasks.workunit.client.0.smithi133.stderr:+ sed -e s/rbd_data.8e4415fa6b90\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T05:41:47.113 INFO:tasks.workunit.client.0.smithi133.stderr:+ sort -u 2023-10-09T05:41:47.113 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd info sparse1 2023-10-09T05:41:47.114 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep data_pool: 2023-10-09T05:41:47.114 INFO:tasks.workunit.client.0.smithi133.stderr:+ awk -F: { print $NF } 2023-10-09T05:41:47.159 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:47.160 INFO:tasks.workunit.client.0.smithi133.stderr:+ data_pool= 2023-10-09T05:41:47.160 INFO:tasks.workunit.client.0.smithi133.stderr:+ [ -z ] 2023-10-09T05:41:47.160 INFO:tasks.workunit.client.0.smithi133.stderr:+ data_pool=rbd 2023-10-09T05:41:47.160 INFO:tasks.workunit.client.0.smithi133.stderr:+ echo rbd 2023-10-09T05:41:47.161 INFO:tasks.workunit.client.0.smithi133.stderr:+ rados ls -p rbd 2023-10-09T05:41:47.180 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:47.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:47.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:47.197 INFO:tasks.workunit.client.0.smithi133.stderr:+ objects=1 2023-10-09T05:41:47.198 INFO:tasks.workunit.client.0.smithi133.stderr:+ echo 1 2023-10-09T05:41:47.198 INFO:tasks.workunit.client.0.smithi133.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-10-09T05:41:47.198 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd export sparse1 /tmp/rbd_import_export_287447/sparse1.out 2023-10-09T05:41:47.284 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:47.308 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:47.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:47.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:47.331 INFO:tasks.workunit.client.0.smithi133.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-10-09T05:41:47.336 INFO:tasks.workunit.client.0.smithi133.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_287447/sparse1 /tmp/rbd_import_export_287447/sparse1.out 2023-10-09T05:41:47.336 INFO:tasks.workunit.client.0.smithi133.stderr:+ cmp -l /tmp/rbd_import_export_287447/sparse1 /tmp/rbd_import_export_287447/sparse1.out 2023-10-09T05:41:47.347 INFO:tasks.workunit.client.0.smithi133.stderr:+ stat /tmp/rbd_import_export_287447/sparse1 --format %b 2023-10-09T05:41:47.349 INFO:tasks.workunit.client.0.smithi133.stderr:+ origsize=2048 2023-10-09T05:41:47.350 INFO:tasks.workunit.client.0.smithi133.stderr:+ stat /tmp/rbd_import_export_287447/sparse1.out --format %b 2023-10-09T05:41:47.351 INFO:tasks.workunit.client.0.smithi133.stderr:+ exportsize=2048 2023-10-09T05:41:47.351 INFO:tasks.workunit.client.0.smithi133.stderr:+ difference=0 2023-10-09T05:41:47.351 INFO:tasks.workunit.client.0.smithi133.stderr:+ difference=0 2023-10-09T05:41:47.351 INFO:tasks.workunit.client.0.smithi133.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T05:41:47.351 INFO:tasks.workunit.client.0.smithi133.stderr:+ rm /tmp/rbd_import_export_287447/sparse1.out 2023-10-09T05:41:47.354 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd rm sparse1 2023-10-09T05:41:47.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:47 smithi194 bash[92666]: cluster 2023-10-09T05:41:46.512234+0000 mgr.y (mgr.25277) 1623 : cluster 0 pgmap v1827: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 180 KiB/s rd, 14 KiB/s wr, 244 op/s 2023-10-09T05:41:47.412 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:47.443 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:47.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:47.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:47.498 INFO:tasks.workunit.client.0.smithi133.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-09T05:41:47.503 INFO:tasks.workunit.client.0.smithi133.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_287447/sparse2 oflag=append conv=notrunc 2023-10-09T05:41:47.511 INFO:tasks.workunit.client.0.smithi133.stderr:1+0 records in 2023-10-09T05:41:47.511 INFO:tasks.workunit.client.0.smithi133.stderr:1+0 records out 2023-10-09T05:41:47.511 INFO:tasks.workunit.client.0.smithi133.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.0076237 s, 275 MB/s 2023-10-09T05:41:47.512 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd import --new-format --order 20 - sparse2 2023-10-09T05:41:47.547 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:47.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:47 smithi133 bash[105589]: cluster 2023-10-09T05:41:46.512234+0000 mgr.y (mgr.25277) 1623 : cluster 0 pgmap v1827: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 180 KiB/s rd, 14 KiB/s wr, 244 op/s 2023-10-09T05:41:47.574 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:47.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:47.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:47.610 INFO:tasks.workunit.client.0.smithi133.stderr: Importing image: 100% complete...done. 2023-10-09T05:41:47.611 INFO:tasks.workunit.client.0.smithi133.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T05:41:47.611 INFO:tasks.workunit.client.0.smithi133.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T05:41:47.615 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd ls -l 2023-10-09T05:41:47.615 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep sparse2 2023-10-09T05:41:47.615 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep -Ei (4 MiB|4096k) 2023-10-09T05:41:47.661 INFO:tasks.workunit.client.0.smithi133.stdout:sparse2 4 MiB 2 2023-10-09T05:41:47.662 INFO:tasks.workunit.client.0.smithi133.stderr:+ objects sparse2 2023-10-09T05:41:47.662 INFO:tasks.workunit.client.0.smithi133.stderr:+ image=sparse2 2023-10-09T05:41:47.663 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd info sparse2 2023-10-09T05:41:47.663 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep block_name_prefix 2023-10-09T05:41:47.663 INFO:tasks.workunit.client.0.smithi133.stderr:+ awk {print $NF;} 2023-10-09T05:41:47.678 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:47.699 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:47.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:47.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:47.706 INFO:tasks.workunit.client.0.smithi133.stderr:+ prefix=rbd_data.8e71178558a6 2023-10-09T05:41:47.708 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep rbd_data.8e71178558a6 2023-10-09T05:41:47.708 INFO:tasks.workunit.client.0.smithi133.stderr:+ sed -e s/rbd_data.8e71178558a6\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T05:41:47.708 INFO:tasks.workunit.client.0.smithi133.stderr:+ get_image_data_pool sparse2 2023-10-09T05:41:47.708 INFO:tasks.workunit.client.0.smithi133.stderr:+ image=sparse2 2023-10-09T05:41:47.708 INFO:tasks.workunit.client.0.smithi133.stderr:+ sort -u 2023-10-09T05:41:47.708 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd info sparse2 2023-10-09T05:41:47.709 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep data_pool: 2023-10-09T05:41:47.709 INFO:tasks.workunit.client.0.smithi133.stderr:+ awk -F: { print $NF } 2023-10-09T05:41:47.753 INFO:tasks.workunit.client.0.smithi133.stderr:+ data_pool= 2023-10-09T05:41:47.753 INFO:tasks.workunit.client.0.smithi133.stderr:+ [ -z ] 2023-10-09T05:41:47.753 INFO:tasks.workunit.client.0.smithi133.stderr:+ data_pool=rbd 2023-10-09T05:41:47.753 INFO:tasks.workunit.client.0.smithi133.stderr:+ echo rbd 2023-10-09T05:41:47.753 INFO:tasks.workunit.client.0.smithi133.stderr:+ rados ls -p rbd 2023-10-09T05:41:47.796 INFO:tasks.workunit.client.0.smithi133.stderr:+ objects=0 2023-10-09T05:41:47.796 INFO:tasks.workunit.client.0.smithi133.stderr:2 2023-10-09T05:41:47.796 INFO:tasks.workunit.client.0.smithi133.stderr:3 2023-10-09T05:41:47.796 INFO:tasks.workunit.client.0.smithi133.stderr:+ echo 0 2 3 2023-10-09T05:41:47.796 INFO:tasks.workunit.client.0.smithi133.stderr:+ [ 0 -eq 1 -o 0 2 3 = 0 2 3 ] 2023-10-09T05:41:47.796 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd export sparse2 /tmp/rbd_import_export_287447/sparse2.out 2023-10-09T05:41:47.803 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:47.830 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:47.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:47.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:47.840 INFO:tasks.workunit.client.0.smithi133.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-10-09T05:41:47.845 INFO:tasks.workunit.client.0.smithi133.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_287447/sparse2 /tmp/rbd_import_export_287447/sparse2.out 2023-10-09T05:41:47.845 INFO:tasks.workunit.client.0.smithi133.stderr:+ cmp -l /tmp/rbd_import_export_287447/sparse2 /tmp/rbd_import_export_287447/sparse2.out 2023-10-09T05:41:47.850 INFO:tasks.workunit.client.0.smithi133.stderr:+ stat /tmp/rbd_import_export_287447/sparse2 --format %b 2023-10-09T05:41:47.852 INFO:tasks.workunit.client.0.smithi133.stderr:+ origsize=6144 2023-10-09T05:41:47.852 INFO:tasks.workunit.client.0.smithi133.stderr:+ stat /tmp/rbd_import_export_287447/sparse2.out --format %b 2023-10-09T05:41:47.853 INFO:tasks.workunit.client.0.smithi133.stderr:+ exportsize=6144 2023-10-09T05:41:47.853 INFO:tasks.workunit.client.0.smithi133.stderr:+ difference=0 2023-10-09T05:41:47.853 INFO:tasks.workunit.client.0.smithi133.stderr:+ difference=0 2023-10-09T05:41:47.853 INFO:tasks.workunit.client.0.smithi133.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T05:41:47.854 INFO:tasks.workunit.client.0.smithi133.stderr:+ rm /tmp/rbd_import_export_287447/sparse2.out 2023-10-09T05:41:47.855 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd rm sparse2 2023-10-09T05:41:47.933 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:47.967 INFO:tasks.workunit.client.0.smithi133.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-10-09T05:41:47.967 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:47.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:47.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:47.971 INFO:tasks.workunit.client.0.smithi133.stderr:+ echo partially-sparse file imports to partially-sparse image 2023-10-09T05:41:47.971 INFO:tasks.workunit.client.0.smithi133.stdout:partially-sparse file imports to partially-sparse image 2023-10-09T05:41:47.971 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_287447/sparse1 sparse 2023-10-09T05:41:48.056 INFO:tasks.workunit.client.0.smithi133.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-10-09T05:41:48.057 INFO:tasks.workunit.client.0.smithi133.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T05:41:48.057 INFO:tasks.workunit.client.0.smithi133.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T05:41:48.062 INFO:tasks.workunit.client.0.smithi133.stderr:+ objects sparse 2023-10-09T05:41:48.063 INFO:tasks.workunit.client.0.smithi133.stderr:+ image=sparse 2023-10-09T05:41:48.063 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd info sparse 2023-10-09T05:41:48.063 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep block_name_prefix 2023-10-09T05:41:48.063 INFO:tasks.workunit.client.0.smithi133.stderr:+ awk {print $NF;} 2023-10-09T05:41:48.071 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:48.102 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:48.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:48.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:48.109 INFO:tasks.workunit.client.0.smithi133.stderr:+ prefix=rbd_data.b5c152d5b213 2023-10-09T05:41:48.110 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep rbd_data.b5c152d5b213 2023-10-09T05:41:48.111 INFO:tasks.workunit.client.0.smithi133.stderr:+ get_image_data_pool sparse 2023-10-09T05:41:48.111 INFO:tasks.workunit.client.0.smithi133.stderr:+ + image=sparse 2023-10-09T05:41:48.111 INFO:tasks.workunit.client.0.smithi133.stderr:sed -e s/rbd_data.b5c152d5b213\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T05:41:48.111 INFO:tasks.workunit.client.0.smithi133.stderr:+ sort -u 2023-10-09T05:41:48.111 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd info sparse 2023-10-09T05:41:48.111 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep data_pool: 2023-10-09T05:41:48.112 INFO:tasks.workunit.client.0.smithi133.stderr:+ awk -F: { print $NF } 2023-10-09T05:41:48.165 INFO:tasks.workunit.client.0.smithi133.stderr:+ data_pool= 2023-10-09T05:41:48.165 INFO:tasks.workunit.client.0.smithi133.stderr:+ [ -z ] 2023-10-09T05:41:48.165 INFO:tasks.workunit.client.0.smithi133.stderr:+ data_pool=rbd 2023-10-09T05:41:48.165 INFO:tasks.workunit.client.0.smithi133.stderr:+ echo rbd 2023-10-09T05:41:48.166 INFO:tasks.workunit.client.0.smithi133.stderr:+ rados ls -p rbd 2023-10-09T05:41:48.206 INFO:tasks.workunit.client.0.smithi133.stderr:+ objects=1 2023-10-09T05:41:48.206 INFO:tasks.workunit.client.0.smithi133.stderr:+ echo 1 2023-10-09T05:41:48.206 INFO:tasks.workunit.client.0.smithi133.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-10-09T05:41:48.206 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd rm sparse 2023-10-09T05:41:48.206 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:48.235 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:48.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:48.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:48.338 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:48.339 INFO:tasks.workunit.client.0.smithi133.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-09T05:41:48.342 INFO:tasks.workunit.client.0.smithi133.stderr:+ echo zeros import through stdin to sparse image 2023-10-09T05:41:48.343 INFO:tasks.workunit.client.0.smithi133.stdout:zeros import through stdin to sparse image 2023-10-09T05:41:48.343 INFO:tasks.workunit.client.0.smithi133.stderr:+ dd if=/dev/zero bs=1M count=4 2023-10-09T05:41:48.343 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd import --new-format - sparse 2023-10-09T05:41:48.364 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:48.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:48.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:48.408 INFO:tasks.workunit.client.0.smithi133.stderr:4+0 records in 2023-10-09T05:41:48.408 INFO:tasks.workunit.client.0.smithi133.stderr:4+0 records out 2023-10-09T05:41:48.409 INFO:tasks.workunit.client.0.smithi133.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0640015 s, 65.5 MB/s 2023-10-09T05:41:48.425 INFO:tasks.workunit.client.0.smithi133.stderr: Importing image: 100% complete...done. 2023-10-09T05:41:48.426 INFO:tasks.workunit.client.0.smithi133.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T05:41:48.432 INFO:tasks.workunit.client.0.smithi133.stderr:+ objects sparse 2023-10-09T05:41:48.432 INFO:tasks.workunit.client.0.smithi133.stderr:+ image=sparse 2023-10-09T05:41:48.432 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd info sparse 2023-10-09T05:41:48.432 INFO:tasks.workunit.client.0.smithi133.stderr:+ + awk {print $NF;} 2023-10-09T05:41:48.432 INFO:tasks.workunit.client.0.smithi133.stderr:grep block_name_prefix 2023-10-09T05:41:48.466 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:48.478 INFO:tasks.workunit.client.0.smithi133.stderr:+ prefix=rbd_data.8ebf74f7ad24 2023-10-09T05:41:48.478 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep rbd_data.8ebf74f7ad24 2023-10-09T05:41:48.478 INFO:tasks.workunit.client.0.smithi133.stderr:+ sed -e s/rbd_data.8ebf74f7ad24\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T05:41:48.479 INFO:tasks.workunit.client.0.smithi133.stderr:+ + get_image_data_pool sparse 2023-10-09T05:41:48.479 INFO:tasks.workunit.client.0.smithi133.stderr:sort+ -u 2023-10-09T05:41:48.479 INFO:tasks.workunit.client.0.smithi133.stderr:image=sparse 2023-10-09T05:41:48.479 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd info sparse 2023-10-09T05:41:48.479 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep data_pool: 2023-10-09T05:41:48.479 INFO:tasks.workunit.client.0.smithi133.stderr:+ awk -F: { print $NF } 2023-10-09T05:41:48.487 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:48.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:48.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:48.534 INFO:tasks.workunit.client.0.smithi133.stderr:+ data_pool= 2023-10-09T05:41:48.534 INFO:tasks.workunit.client.0.smithi133.stderr:+ [ -z ] 2023-10-09T05:41:48.534 INFO:tasks.workunit.client.0.smithi133.stderr:+ data_pool=rbd 2023-10-09T05:41:48.534 INFO:tasks.workunit.client.0.smithi133.stderr:+ echo rbd 2023-10-09T05:41:48.534 INFO:tasks.workunit.client.0.smithi133.stderr:+ rados ls -p rbd 2023-10-09T05:41:48.577 INFO:tasks.workunit.client.0.smithi133.stderr:+ objects= 2023-10-09T05:41:48.577 INFO:tasks.workunit.client.0.smithi133.stderr:+ echo 2023-10-09T05:41:48.578 INFO:tasks.workunit.client.0.smithi133.stderr:+ [ 0 -eq 1 -o = ] 2023-10-09T05:41:48.578 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd rm sparse 2023-10-09T05:41:48.590 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:48.621 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:48.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:48.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:48.708 INFO:tasks.workunit.client.0.smithi133.stderr: Removing image: 100% complete...done. 2023-10-09T05:41:48.714 INFO:tasks.workunit.client.0.smithi133.stdout:zeros export to sparse file 2023-10-09T05:41:48.715 INFO:tasks.workunit.client.0.smithi133.stderr:+ echo zeros export to sparse file 2023-10-09T05:41:48.726 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:48.758 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:48.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:48.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:48.763 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd create --new-format sparse --size 4 2023-10-09T05:41:48.811 INFO:tasks.workunit.client.0.smithi133.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T05:41:48.844 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd info sparse 2023-10-09T05:41:48.844 INFO:tasks.workunit.client.0.smithi133.stderr:+ awk {print $NF;} 2023-10-09T05:41:48.844 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep block_name_prefix 2023-10-09T05:41:48.863 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:48.867 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:41:48.867 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:41:48.868 INFO:tasks.workunit.client.0.smithi133.stderr:+ prefix=rbd_data.b603447a12e1 2023-10-09T05:41:48.868 INFO:tasks.workunit.client.0.smithi133.stderr:+ dd if=/dev/zero bs=4M count=1 2023-10-09T05:41:48.868 INFO:tasks.workunit.client.0.smithi133.stderr:+ get_image_data_pool sparse 2023-10-09T05:41:48.868 INFO:tasks.workunit.client.0.smithi133.stderr:+ image=sparse 2023-10-09T05:41:48.869 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd info sparse 2023-10-09T05:41:48.869 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep data_pool: 2023-10-09T05:41:48.869 INFO:tasks.workunit.client.0.smithi133.stderr:+ awk -F: { print $NF } 2023-10-09T05:41:48.888 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:48.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:48.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:48.919 INFO:tasks.workunit.client.0.smithi133.stderr:+ data_pool= 2023-10-09T05:41:48.919 INFO:tasks.workunit.client.0.smithi133.stderr:+ [ -z ] 2023-10-09T05:41:48.919 INFO:tasks.workunit.client.0.smithi133.stderr:+ data_pool=rbd 2023-10-09T05:41:48.920 INFO:tasks.workunit.client.0.smithi133.stderr:+ echo rbd 2023-10-09T05:41:48.920 INFO:tasks.workunit.client.0.smithi133.stderr:+ rados -p rbd put rbd_data.b603447a12e1.000000000000 - 2023-10-09T05:41:48.954 INFO:tasks.workunit.client.0.smithi133.stderr:1+0 records in 2023-10-09T05:41:48.954 INFO:tasks.workunit.client.0.smithi133.stderr:1+0 records out 2023-10-09T05:41:48.954 INFO:tasks.workunit.client.0.smithi133.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.085167 s, 49.2 MB/s 2023-10-09T05:41:48.982 INFO:tasks.workunit.client.0.smithi133.stderr:+ objects sparse 2023-10-09T05:41:48.983 INFO:tasks.workunit.client.0.smithi133.stderr:+ image=sparse 2023-10-09T05:41:48.983 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd info sparse 2023-10-09T05:41:48.983 INFO:tasks.workunit.client.0.smithi133.stderr:+ + awk {print $NF;} 2023-10-09T05:41:48.983 INFO:tasks.workunit.client.0.smithi133.stderr:grep block_name_prefix 2023-10-09T05:41:48.992 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:49.021 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:49.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:49.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:49.037 INFO:tasks.workunit.client.0.smithi133.stderr:+ prefix=rbd_data.b603447a12e1 2023-10-09T05:41:49.039 INFO:tasks.workunit.client.0.smithi133.stderr:+ grep rbd_data.b603447a12e1 2023-10-09T05:41:49.039 INFO:tasks.workunit.client.0.smithi133.stderr:+ sed -e s/rbd_data.b603447a12e1\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T05:41:49.039 INFO:tasks.workunit.client.0.smithi133.stderr:+ get_image_data_pool sparse 2023-10-09T05:41:49.039 INFO:tasks.workunit.client.0.smithi133.stderr:+ + image=sparse 2023-10-09T05:41:49.039 INFO:tasks.workunit.client.0.smithi133.stderr:sort -u 2023-10-09T05:41:49.039 INFO:tasks.workunit.client.0.smithi133.stderr:+ + grep data_pool: 2023-10-09T05:41:49.039 INFO:tasks.workunit.client.0.smithi133.stderr:rbd info sparse 2023-10-09T05:41:49.039 INFO:tasks.workunit.client.0.smithi133.stderr:+ awk -F: { print $NF } 2023-10-09T05:41:49.083 INFO:tasks.workunit.client.0.smithi133.stderr:+ data_pool= 2023-10-09T05:41:49.083 INFO:tasks.workunit.client.0.smithi133.stderr:+ [ -z ] 2023-10-09T05:41:49.083 INFO:tasks.workunit.client.0.smithi133.stderr:+ data_pool=rbd 2023-10-09T05:41:49.083 INFO:tasks.workunit.client.0.smithi133.stderr:+ echo rbd 2023-10-09T05:41:49.083 INFO:tasks.workunit.client.0.smithi133.stderr:+ rados ls -p rbd 2023-10-09T05:41:49.125 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:49.131 INFO:tasks.workunit.client.0.smithi133.stderr:+ objects=0 2023-10-09T05:41:49.131 INFO:tasks.workunit.client.0.smithi133.stderr:+ echo 0 2023-10-09T05:41:49.131 INFO:tasks.workunit.client.0.smithi133.stderr:+ [ 0 -eq 1 -o 0 = 0 ] 2023-10-09T05:41:49.131 INFO:tasks.workunit.client.0.smithi133.stderr:+ rm /tmp/rbd_import_export_287447/sparse 2023-10-09T05:41:49.132 INFO:tasks.workunit.client.0.smithi133.stderr:rm: cannot remove '/tmp/rbd_import_export_287447/sparse': No such file or directory 2023-10-09T05:41:49.132 INFO:tasks.workunit.client.0.smithi133.stderr:+ true 2023-10-09T05:41:49.132 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd export sparse /tmp/rbd_import_export_287447/sparse 2023-10-09T05:41:49.149 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:49.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:49.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:49.182 INFO:tasks.workunit.client.0.smithi133.stderr: Exporting image: 100% complete...done. 2023-10-09T05:41:49.186 INFO:tasks.workunit.client.0.smithi133.stderr:+ stat /tmp/rbd_import_export_287447/sparse --format=%b 2023-10-09T05:41:49.187 INFO:tasks.workunit.client.0.smithi133.stderr:+ [ 0 = 0 ] 2023-10-09T05:41:49.187 INFO:tasks.workunit.client.0.smithi133.stderr:+ rbd rm sparse 2023-10-09T05:41:49.253 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:49.278 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:49.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:49.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:49.318 INFO:tasks.workunit.client.0.smithi133.stderr: Removing image: 100% complete...done. 2023-10-09T05:41:49.321 INFO:tasks.workunit.client.0.smithi133.stderr:+ rm /tmp/rbd_import_export_287447/sparse /tmp/rbd_import_export_287447/sparse1 /tmp/rbd_import_export_287447/sparse2 /tmp/rbd_import_export_287447/sparse3 2023-10-09T05:41:49.325 INFO:tasks.workunit.client.0.smithi133.stderr:rm: cannot remove '/tmp/rbd_import_export_287447/sparse3': No such file or directory 2023-10-09T05:41:49.325 INFO:tasks.workunit.client.0.smithi133.stderr:+ true 2023-10-09T05:41:49.325 INFO:tasks.workunit.client.0.smithi133.stderr:+ echo OK 2023-10-09T05:41:49.326 INFO:tasks.workunit.client.0.smithi133.stdout:OK 2023-10-09T05:41:49.327 INFO:tasks.workunit.client.0.smithi133.stderr:+ rm -rf /tmp/rbd_import_export_287447 2023-10-09T05:41:49.327 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T05:41:49.327 DEBUG:teuthology.orchestra.run.smithi133:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T05:41:49.382 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-10-09T05:41:49.382 DEBUG:teuthology.orchestra.run.smithi133:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T05:41:49.383 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:49.450 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:49.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:49.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:49.554 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:49.585 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:49.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:49.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:49.690 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:49.724 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:49.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:49.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:49.828 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:49.842 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:49 smithi194 bash[92666]: cluster 2023-10-09T05:41:48.512600+0000 mgr.y (mgr.25277) 1624 : cluster 0 pgmap v1828: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 136 KiB/s rd, 6.9 KiB/s wr, 175 op/s 2023-10-09T05:41:49.842 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:49 smithi194 bash[92666]: cluster 2023-10-09T05:41:48.989169+0000 osd.6 (osd.6) 417 : cluster 3 failed to encode map e421 with expected crc 2023-10-09T05:41:49.842 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:49 smithi194 bash[92666]: cluster 2023-10-09T05:41:48.992810+0000 mon.a (mon.0) 2131 : cluster 0 osdmap e421: 8 total, 8 up, 7 in 2023-10-09T05:41:49.842 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:49 smithi194 bash[92666]: cluster 2023-10-09T05:41:49.023658+0000 osd.5 (osd.5) 458 : cluster 3 failed to encode map e421 with expected crc 2023-10-09T05:41:49.842 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:49 smithi194 bash[92666]: cluster 2023-10-09T05:41:49.115176+0000 osd.7 (osd.7) 386 : cluster 3 failed to encode map e421 with expected crc 2023-10-09T05:41:49.842 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:49 smithi194 bash[92666]: audit 2023-10-09T05:41:49.380802+0000 mon.a (mon.0) 2132 : audit 1 from='client.? 172.21.15.133:0/2767057728' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:41:49.858 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:49.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:49.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:49.953 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:49 smithi133 bash[105589]: cluster 2023-10-09T05:41:48.512600+0000 mgr.y (mgr.25277) 1624 : cluster 0 pgmap v1828: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 136 KiB/s rd, 6.9 KiB/s wr, 175 op/s 2023-10-09T05:41:49.953 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:49 smithi133 bash[105589]: cluster 2023-10-09T05:41:48.989169+0000 osd.6 (osd.6) 417 : cluster 3 failed to encode map e421 with expected crc 2023-10-09T05:41:49.953 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:49 smithi133 bash[105589]: cluster 2023-10-09T05:41:48.992810+0000 mon.a (mon.0) 2131 : cluster 0 osdmap e421: 8 total, 8 up, 7 in 2023-10-09T05:41:49.953 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:49 smithi133 bash[105589]: cluster 2023-10-09T05:41:49.023658+0000 osd.5 (osd.5) 458 : cluster 3 failed to encode map e421 with expected crc 2023-10-09T05:41:49.954 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:49 smithi133 bash[105589]: cluster 2023-10-09T05:41:49.115176+0000 osd.7 (osd.7) 386 : cluster 3 failed to encode map e421 with expected crc 2023-10-09T05:41:49.954 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:49 smithi133 bash[105589]: audit 2023-10-09T05:41:49.380802+0000 mon.a (mon.0) 2132 : audit 1 from='client.? 172.21.15.133:0/2767057728' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:41:49.954 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:49 smithi133 bash[102858]: cluster 2023-10-09T05:41:48.512600+0000 mgr.y (mgr.25277) 1624 : cluster 0 pgmap v1828: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 136 KiB/s rd, 6.9 KiB/s wr, 175 op/s 2023-10-09T05:41:49.954 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:49 smithi133 bash[102858]: cluster 2023-10-09T05:41:48.989169+0000 osd.6 (osd.6) 417 : cluster 3 failed to encode map e421 with expected crc 2023-10-09T05:41:49.954 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:49 smithi133 bash[102858]: cluster 2023-10-09T05:41:48.992810+0000 mon.a (mon.0) 2131 : cluster 0 osdmap e421: 8 total, 8 up, 7 in 2023-10-09T05:41:49.954 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:49 smithi133 bash[102858]: cluster 2023-10-09T05:41:49.023658+0000 osd.5 (osd.5) 458 : cluster 3 failed to encode map e421 with expected crc 2023-10-09T05:41:49.954 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:49 smithi133 bash[102858]: cluster 2023-10-09T05:41:49.115176+0000 osd.7 (osd.7) 386 : cluster 3 failed to encode map e421 with expected crc 2023-10-09T05:41:49.954 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:49 smithi133 bash[102858]: audit 2023-10-09T05:41:49.380802+0000 mon.a (mon.0) 2132 : audit 1 from='client.? 172.21.15.133:0/2767057728' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:41:49.955 DEBUG:teuthology.parallel:result is None 2023-10-09T05:41:49.956 DEBUG:teuthology.orchestra.run.smithi133:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T05:41:49.963 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:49.968 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T05:41:49.968 DEBUG:teuthology.orchestra.run.smithi133:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T05:41:49.993 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:49.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:49.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:50.013 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is unset 2023-10-09T05:41:50.099 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:50.131 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:50.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:50.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:50.234 DEBUG:teuthology.orchestra.run.smithi194:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:50.264 INFO:teuthology.orchestra.run.smithi194.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:50.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:50.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi194 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:50.284 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T05:41:50.284 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T05:41:50.284 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-10-09T05:41:50.285 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T05:41:50.285 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-10-09T05:41:50.285 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T05:41:50.286 DEBUG:teuthology.orchestra.run.smithi133:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T05:41:50.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:50.294 INFO:teuthology.orchestra.run.smithi133.stderr:stat: cannot stat '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T05:41:50.294 DEBUG:teuthology.orchestra.run.smithi133:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T05:41:50.363 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T05:41:50.364 DEBUG:teuthology.orchestra.run.smithi133:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T05:41:50.368 DEBUG:teuthology.orchestra.run.smithi133:> sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:50.413 INFO:tasks.thrashosds:joining thrashosds 2023-10-09T05:41:50.436 INFO:teuthology.orchestra.run.smithi133.stderr:Error response from daemon: Cannot kill container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:50.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:50.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi133 with status 1: 'sudo docker kill -s 1 ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:50.821 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:50 smithi133 bash[105589]: cluster 2023-10-09T05:41:49.998125+0000 mon.a (mon.0) 2133 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:41:50.821 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:50 smithi133 bash[105589]: audit 2023-10-09T05:41:50.008146+0000 mon.a (mon.0) 2134 : audit 1 from='client.? 172.21.15.133:0/2767057728' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:41:50.821 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:50 smithi133 bash[105589]: cluster 2023-10-09T05:41:50.008207+0000 mon.a (mon.0) 2135 : cluster 0 osdmap e422: 8 total, 8 up, 7 in 2023-10-09T05:41:50.821 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:50 smithi133 bash[105589]: cluster 2023-10-09T05:41:50.129603+0000 osd.5 (osd.5) 459 : cluster 3 failed to encode map e422 with expected crc 2023-10-09T05:41:50.821 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:50 smithi133 bash[105589]: cluster 2023-10-09T05:41:50.235846+0000 osd.4 (osd.4) 357 : cluster 3 failed to encode map e422 with expected crc 2023-10-09T05:41:50.821 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:50 smithi133 bash[102858]: cluster 2023-10-09T05:41:49.998125+0000 mon.a (mon.0) 2133 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:41:50.821 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:50 smithi133 bash[102858]: audit 2023-10-09T05:41:50.008146+0000 mon.a (mon.0) 2134 : audit 1 from='client.? 172.21.15.133:0/2767057728' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:41:50.822 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:50 smithi133 bash[102858]: cluster 2023-10-09T05:41:50.008207+0000 mon.a (mon.0) 2135 : cluster 0 osdmap e422: 8 total, 8 up, 7 in 2023-10-09T05:41:50.822 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:50 smithi133 bash[102858]: cluster 2023-10-09T05:41:50.129603+0000 osd.5 (osd.5) 459 : cluster 3 failed to encode map e422 with expected crc 2023-10-09T05:41:50.822 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:50 smithi133 bash[102858]: cluster 2023-10-09T05:41:50.235846+0000 osd.4 (osd.4) 357 : cluster 3 failed to encode map e422 with expected crc 2023-10-09T05:41:50.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:50 smithi194 bash[92666]: cluster 2023-10-09T05:41:49.998125+0000 mon.a (mon.0) 2133 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:41:50.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:50 smithi194 bash[92666]: audit 2023-10-09T05:41:50.008146+0000 mon.a (mon.0) 2134 : audit 1 from='client.? 172.21.15.133:0/2767057728' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:41:50.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:50 smithi194 bash[92666]: cluster 2023-10-09T05:41:50.008207+0000 mon.a (mon.0) 2135 : cluster 0 osdmap e422: 8 total, 8 up, 7 in 2023-10-09T05:41:50.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:50 smithi194 bash[92666]: cluster 2023-10-09T05:41:50.129603+0000 osd.5 (osd.5) 459 : cluster 3 failed to encode map e422 with expected crc 2023-10-09T05:41:50.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:50 smithi194 bash[92666]: cluster 2023-10-09T05:41:50.235846+0000 osd.4 (osd.4) 357 : cluster 3 failed to encode map e422 with expected crc 2023-10-09T05:41:51.130 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T05:41:51.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:51 smithi194 bash[92666]: cluster 2023-10-09T05:41:50.235850+0000 osd.6 (osd.6) 418 : cluster 3 failed to encode map e422 with expected crc 2023-10-09T05:41:51.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:51 smithi194 bash[92666]: cluster 2023-10-09T05:41:50.235905+0000 osd.7 (osd.7) 387 : cluster 3 failed to encode map e422 with expected crc 2023-10-09T05:41:51.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:51 smithi194 bash[92666]: cluster 2023-10-09T05:41:50.513217+0000 mgr.y (mgr.25277) 1625 : cluster 0 pgmap v1831: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 906 KiB/s rd, 1.4 MiB/s wr, 361 op/s 2023-10-09T05:41:52.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:51 smithi133 bash[102858]: cluster 2023-10-09T05:41:50.235850+0000 osd.6 (osd.6) 418 : cluster 3 failed to encode map e422 with expected crc 2023-10-09T05:41:52.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:51 smithi133 bash[102858]: cluster 2023-10-09T05:41:50.235905+0000 osd.7 (osd.7) 387 : cluster 3 failed to encode map e422 with expected crc 2023-10-09T05:41:52.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:51 smithi133 bash[102858]: cluster 2023-10-09T05:41:50.513217+0000 mgr.y (mgr.25277) 1625 : cluster 0 pgmap v1831: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 906 KiB/s rd, 1.4 MiB/s wr, 361 op/s 2023-10-09T05:41:52.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:51 smithi133 bash[105589]: cluster 2023-10-09T05:41:50.235850+0000 osd.6 (osd.6) 418 : cluster 3 failed to encode map e422 with expected crc 2023-10-09T05:41:52.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:51 smithi133 bash[105589]: cluster 2023-10-09T05:41:50.235905+0000 osd.7 (osd.7) 387 : cluster 3 failed to encode map e422 with expected crc 2023-10-09T05:41:52.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:51 smithi133 bash[105589]: cluster 2023-10-09T05:41:50.513217+0000 mgr.y (mgr.25277) 1625 : cluster 0 pgmap v1831: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 906 KiB/s rd, 1.4 MiB/s wr, 361 op/s 2023-10-09T05:41:52.847 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:52 smithi133 bash[105589]: audit 2023-10-09T05:41:52.430245+0000 mon.a (mon.0) 2136 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:41:52.848 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:52 smithi133 bash[102858]: audit 2023-10-09T05:41:52.430245+0000 mon.a (mon.0) 2136 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:41:52.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:52 smithi194 bash[92666]: audit 2023-10-09T05:41:52.430245+0000 mon.a (mon.0) 2136 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:41:53.101 INFO:teuthology.orchestra.run.smithi133.stdout:{ 2023-10-09T05:41:53.101 INFO:teuthology.orchestra.run.smithi133.stdout: "success": "" 2023-10-09T05:41:53.101 INFO:teuthology.orchestra.run.smithi133.stdout:} 2023-10-09T05:41:53.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:53 smithi194 bash[92666]: cluster 2023-10-09T05:41:52.513805+0000 mgr.y (mgr.25277) 1626 : cluster 0 pgmap v1832: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 987 KiB/s rd, 1.4 MiB/s wr, 296 op/s 2023-10-09T05:41:54.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:53 smithi133 bash[105589]: cluster 2023-10-09T05:41:52.513805+0000 mgr.y (mgr.25277) 1626 : cluster 0 pgmap v1832: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 987 KiB/s rd, 1.4 MiB/s wr, 296 op/s 2023-10-09T05:41:54.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:53 smithi133 bash[102858]: cluster 2023-10-09T05:41:52.513805+0000 mgr.y (mgr.25277) 1626 : cluster 0 pgmap v1832: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 987 KiB/s rd, 1.4 MiB/s wr, 296 op/s 2023-10-09T05:41:54.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:41:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:41:54] "GET /metrics HTTP/1.1" 200 34149 "" "Prometheus/2.43.0" 2023-10-09T05:41:54.576 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:41:55.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:54 smithi133 bash[105589]: cluster 2023-10-09T05:41:54.514594+0000 mgr.y (mgr.25277) 1627 : cluster 0 pgmap v1833: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 970 KiB/s rd, 1.4 MiB/s wr, 271 op/s 2023-10-09T05:41:55.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:54 smithi133 bash[102858]: cluster 2023-10-09T05:41:54.514594+0000 mgr.y (mgr.25277) 1627 : cluster 0 pgmap v1833: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 970 KiB/s rd, 1.4 MiB/s wr, 271 op/s 2023-10-09T05:41:55.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:54 smithi194 bash[92666]: cluster 2023-10-09T05:41:54.514594+0000 mgr.y (mgr.25277) 1627 : cluster 0 pgmap v1833: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 970 KiB/s rd, 1.4 MiB/s wr, 271 op/s 2023-10-09T05:41:56.694 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:57.825 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:57 smithi133 bash[105589]: cluster 2023-10-09T05:41:56.515580+0000 mgr.y (mgr.25277) 1628 : cluster 0 pgmap v1834: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 946 KiB/s rd, 1.4 MiB/s wr, 241 op/s 2023-10-09T05:41:57.825 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:57 smithi133 bash[102858]: cluster 2023-10-09T05:41:56.515580+0000 mgr.y (mgr.25277) 1628 : cluster 0 pgmap v1834: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 946 KiB/s rd, 1.4 MiB/s wr, 241 op/s 2023-10-09T05:41:57.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:57 smithi194 bash[92666]: cluster 2023-10-09T05:41:56.515580+0000 mgr.y (mgr.25277) 1628 : cluster 0 pgmap v1834: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 946 KiB/s rd, 1.4 MiB/s wr, 241 op/s 2023-10-09T05:41:58.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:58 smithi194 bash[92666]: audit 2023-10-09T05:41:58.498772+0000 mon.a (mon.0) 2137 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:58.986 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:58 smithi133 bash[105589]: audit 2023-10-09T05:41:58.498772+0000 mon.a (mon.0) 2137 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:58.987 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:58 smithi133 bash[102858]: audit 2023-10-09T05:41:58.498772+0000 mon.a (mon.0) 2137 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:59.191 INFO:tasks.thrashosds.thrasher:inning osd 2023-10-09T05:41:59.191 INFO:tasks.thrashosds.thrasher:Adding osd 4 2023-10-09T05:41:59.191 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd in 4 2023-10-09T05:41:59.389 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:59.804 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:59 smithi133 bash[105589]: cluster 2023-10-09T05:41:58.516117+0000 mgr.y (mgr.25277) 1629 : cluster 0 pgmap v1835: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 794 KiB/s rd, 1.2 MiB/s wr, 202 op/s 2023-10-09T05:41:59.835 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:59 smithi133 bash[105589]: audit 2023-10-09T05:41:58.882041+0000 mon.a (mon.0) 2138 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:41:59.835 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:59 smithi133 bash[105589]: audit 2023-10-09T05:41:58.891443+0000 mon.a (mon.0) 2139 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:41:59.835 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:59 smithi133 bash[105589]: audit 2023-10-09T05:41:58.926964+0000 mon.a (mon.0) 2140 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:41:59.835 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:59 smithi133 bash[105589]: audit 2023-10-09T05:41:58.937045+0000 mon.a (mon.0) 2141 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:41:59.835 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:59 smithi133 bash[105589]: audit 2023-10-09T05:41:59.469403+0000 mon.a (mon.0) 2142 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:41:59.835 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:59 smithi133 bash[105589]: audit 2023-10-09T05:41:59.470627+0000 mon.a (mon.0) 2143 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:41:59.835 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:41:59 smithi133 bash[105589]: audit 2023-10-09T05:41:59.479990+0000 mon.a (mon.0) 2144 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:41:59.836 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:59 smithi133 bash[102858]: cluster 2023-10-09T05:41:58.516117+0000 mgr.y (mgr.25277) 1629 : cluster 0 pgmap v1835: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 794 KiB/s rd, 1.2 MiB/s wr, 202 op/s 2023-10-09T05:41:59.836 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:59 smithi133 bash[102858]: audit 2023-10-09T05:41:58.882041+0000 mon.a (mon.0) 2138 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:41:59.836 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:59 smithi133 bash[102858]: audit 2023-10-09T05:41:58.891443+0000 mon.a (mon.0) 2139 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:41:59.836 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:59 smithi133 bash[102858]: audit 2023-10-09T05:41:58.926964+0000 mon.a (mon.0) 2140 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:41:59.836 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:59 smithi133 bash[102858]: audit 2023-10-09T05:41:58.937045+0000 mon.a (mon.0) 2141 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:41:59.836 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:59 smithi133 bash[102858]: audit 2023-10-09T05:41:59.469403+0000 mon.a (mon.0) 2142 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:41:59.836 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:59 smithi133 bash[102858]: audit 2023-10-09T05:41:59.470627+0000 mon.a (mon.0) 2143 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:41:59.836 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:41:59 smithi133 bash[102858]: audit 2023-10-09T05:41:59.479990+0000 mon.a (mon.0) 2144 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:41:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:59 smithi194 bash[92666]: cluster 2023-10-09T05:41:58.516117+0000 mgr.y (mgr.25277) 1629 : cluster 0 pgmap v1835: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 794 KiB/s rd, 1.2 MiB/s wr, 202 op/s 2023-10-09T05:41:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:59 smithi194 bash[92666]: audit 2023-10-09T05:41:58.882041+0000 mon.a (mon.0) 2138 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:41:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:59 smithi194 bash[92666]: audit 2023-10-09T05:41:58.891443+0000 mon.a (mon.0) 2139 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:41:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:59 smithi194 bash[92666]: audit 2023-10-09T05:41:58.926964+0000 mon.a (mon.0) 2140 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:41:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:59 smithi194 bash[92666]: audit 2023-10-09T05:41:58.937045+0000 mon.a (mon.0) 2141 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:41:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:59 smithi194 bash[92666]: audit 2023-10-09T05:41:59.469403+0000 mon.a (mon.0) 2142 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:41:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:59 smithi194 bash[92666]: audit 2023-10-09T05:41:59.470627+0000 mon.a (mon.0) 2143 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:41:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:41:59 smithi194 bash[92666]: audit 2023-10-09T05:41:59.479990+0000 mon.a (mon.0) 2144 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:42:01.242 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:42:01.243 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:42:01.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:01 smithi133 bash[102858]: cluster 2023-10-09T05:42:00.517065+0000 mgr.y (mgr.25277) 1630 : cluster 0 pgmap v1836: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 241 KiB/s rd, 199 KiB/s wr, 67 op/s 2023-10-09T05:42:01.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:01 smithi133 bash[105589]: cluster 2023-10-09T05:42:00.517065+0000 mgr.y (mgr.25277) 1630 : cluster 0 pgmap v1836: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 241 KiB/s rd, 199 KiB/s wr, 67 op/s 2023-10-09T05:42:01.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:01 smithi194 bash[92666]: cluster 2023-10-09T05:42:00.517065+0000 mgr.y (mgr.25277) 1630 : cluster 0 pgmap v1836: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 241 KiB/s rd, 199 KiB/s wr, 67 op/s 2023-10-09T05:42:02.843 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T05:42:03.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:03 smithi194 bash[92666]: cluster 2023-10-09T05:42:02.517992+0000 mgr.y (mgr.25277) 1631 : cluster 0 pgmap v1837: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 211 KiB/s rd, 174 KiB/s wr, 59 op/s 2023-10-09T05:42:03.937 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:03 smithi133 bash[105589]: cluster 2023-10-09T05:42:02.517992+0000 mgr.y (mgr.25277) 1631 : cluster 0 pgmap v1837: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 211 KiB/s rd, 174 KiB/s wr, 59 op/s 2023-10-09T05:42:03.937 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:03 smithi133 bash[102858]: cluster 2023-10-09T05:42:02.517992+0000 mgr.y (mgr.25277) 1631 : cluster 0 pgmap v1837: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 211 KiB/s rd, 174 KiB/s wr, 59 op/s 2023-10-09T05:42:04.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:42:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:42:04] "GET /metrics HTTP/1.1" 200 34159 "" "Prometheus/2.43.0" 2023-10-09T05:42:05.018 INFO:teuthology.orchestra.run.smithi133.stderr:noscrub is unset 2023-10-09T05:42:05.250 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:04 smithi133 bash[105589]: audit 2023-10-09T05:42:04.409056+0000 mon.c (mon.2) 148 : audit 1 from='client.? 172.21.15.133:0/4251949056' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:42:05.251 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:04 smithi133 bash[105589]: audit 2023-10-09T05:42:04.409565+0000 mon.a (mon.0) 2145 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:42:05.252 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:04 smithi133 bash[105589]: cluster 2023-10-09T05:42:04.518755+0000 mgr.y (mgr.25277) 1632 : cluster 0 pgmap v1838: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:42:05.252 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:04 smithi133 bash[102858]: audit 2023-10-09T05:42:04.409056+0000 mon.c (mon.2) 148 : audit 1 from='client.? 172.21.15.133:0/4251949056' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:42:05.252 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:04 smithi133 bash[102858]: audit 2023-10-09T05:42:04.409565+0000 mon.a (mon.0) 2145 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:42:05.252 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:04 smithi133 bash[102858]: cluster 2023-10-09T05:42:04.518755+0000 mgr.y (mgr.25277) 1632 : cluster 0 pgmap v1838: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:42:05.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:04 smithi194 bash[92666]: audit 2023-10-09T05:42:04.409056+0000 mon.c (mon.2) 148 : audit 1 from='client.? 172.21.15.133:0/4251949056' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:42:05.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:04 smithi194 bash[92666]: audit 2023-10-09T05:42:04.409565+0000 mon.a (mon.0) 2145 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:42:05.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:04 smithi194 bash[92666]: cluster 2023-10-09T05:42:04.518755+0000 mgr.y (mgr.25277) 1632 : cluster 0 pgmap v1838: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:42:06.009 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:42:06.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:06 smithi133 bash[102858]: cluster 2023-10-09T05:42:05.009394+0000 osd.5 (osd.5) 460 : cluster 3 failed to encode map e423 with expected crc 2023-10-09T05:42:06.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:06 smithi133 bash[102858]: audit 2023-10-09T05:42:05.012118+0000 mon.a (mon.0) 2146 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:42:06.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:06 smithi133 bash[102858]: cluster 2023-10-09T05:42:05.012225+0000 mon.a (mon.0) 2147 : cluster 0 osdmap e423: 8 total, 8 up, 7 in 2023-10-09T05:42:06.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:06 smithi133 bash[105589]: cluster 2023-10-09T05:42:05.009394+0000 osd.5 (osd.5) 460 : cluster 3 failed to encode map e423 with expected crc 2023-10-09T05:42:06.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:06 smithi133 bash[105589]: audit 2023-10-09T05:42:05.012118+0000 mon.a (mon.0) 2146 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:42:06.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:06 smithi133 bash[105589]: cluster 2023-10-09T05:42:05.012225+0000 mon.a (mon.0) 2147 : cluster 0 osdmap e423: 8 total, 8 up, 7 in 2023-10-09T05:42:06.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:06 smithi194 bash[92666]: cluster 2023-10-09T05:42:05.009394+0000 osd.5 (osd.5) 460 : cluster 3 failed to encode map e423 with expected crc 2023-10-09T05:42:06.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:06 smithi194 bash[92666]: audit 2023-10-09T05:42:05.012118+0000 mon.a (mon.0) 2146 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:42:06.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:06 smithi194 bash[92666]: cluster 2023-10-09T05:42:05.012225+0000 mon.a (mon.0) 2147 : cluster 0 osdmap e423: 8 total, 8 up, 7 in 2023-10-09T05:42:06.428 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:42:07.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:07 smithi133 bash[105589]: cluster 2023-10-09T05:42:05.439177+0000 osd.4 (osd.4) 358 : cluster 3 failed to encode map e423 with expected crc 2023-10-09T05:42:07.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:07 smithi133 bash[105589]: cluster 2023-10-09T05:42:05.439235+0000 osd.6 (osd.6) 419 : cluster 3 failed to encode map e423 with expected crc 2023-10-09T05:42:07.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:07 smithi133 bash[105589]: cluster 2023-10-09T05:42:05.439291+0000 osd.7 (osd.7) 388 : cluster 3 failed to encode map e423 with expected crc 2023-10-09T05:42:07.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:07 smithi133 bash[105589]: cluster 2023-10-09T05:42:06.519641+0000 mgr.y (mgr.25277) 1633 : cluster 0 pgmap v1840: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:42:07.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:07 smithi133 bash[102858]: cluster 2023-10-09T05:42:05.439177+0000 osd.4 (osd.4) 358 : cluster 3 failed to encode map e423 with expected crc 2023-10-09T05:42:07.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:07 smithi133 bash[102858]: cluster 2023-10-09T05:42:05.439235+0000 osd.6 (osd.6) 419 : cluster 3 failed to encode map e423 with expected crc 2023-10-09T05:42:07.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:07 smithi133 bash[102858]: cluster 2023-10-09T05:42:05.439291+0000 osd.7 (osd.7) 388 : cluster 3 failed to encode map e423 with expected crc 2023-10-09T05:42:07.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:07 smithi133 bash[102858]: cluster 2023-10-09T05:42:06.519641+0000 mgr.y (mgr.25277) 1633 : cluster 0 pgmap v1840: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:42:07.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:07 smithi194 bash[92666]: cluster 2023-10-09T05:42:05.439177+0000 osd.4 (osd.4) 358 : cluster 3 failed to encode map e423 with expected crc 2023-10-09T05:42:07.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:07 smithi194 bash[92666]: cluster 2023-10-09T05:42:05.439235+0000 osd.6 (osd.6) 419 : cluster 3 failed to encode map e423 with expected crc 2023-10-09T05:42:07.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:07 smithi194 bash[92666]: cluster 2023-10-09T05:42:05.439291+0000 osd.7 (osd.7) 388 : cluster 3 failed to encode map e423 with expected crc 2023-10-09T05:42:07.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:07 smithi194 bash[92666]: cluster 2023-10-09T05:42:06.519641+0000 mgr.y (mgr.25277) 1633 : cluster 0 pgmap v1840: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:42:09.211 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:42:09.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:09 smithi194 bash[92666]: cluster 2023-10-09T05:42:08.520365+0000 mgr.y (mgr.25277) 1634 : cluster 0 pgmap v1841: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:42:10.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:09 smithi133 bash[102858]: cluster 2023-10-09T05:42:08.520365+0000 mgr.y (mgr.25277) 1634 : cluster 0 pgmap v1841: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:42:10.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:09 smithi133 bash[105589]: cluster 2023-10-09T05:42:08.520365+0000 mgr.y (mgr.25277) 1634 : cluster 0 pgmap v1841: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:42:11.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:11 smithi133 bash[102858]: debug 2023-10-09T05:42:11.214+0000 7f5eb4d1d700 -1 mon.a@0(leader).osd e423 definitely_dead 0 2023-10-09T05:42:11.598 INFO:teuthology.orchestra.run.smithi133.stderr:marked in osd.4. 2023-10-09T05:42:11.625 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:11 smithi133 bash[105589]: cluster 2023-10-09T05:42:10.521265+0000 mgr.y (mgr.25277) 1635 : cluster 0 pgmap v1842: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:42:11.625 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:11 smithi133 bash[105589]: audit 2023-10-09T05:42:11.219040+0000 mon.a (mon.0) 2148 : audit 0 from='client.? 172.21.15.133:0/3544822612' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T05:42:11.625 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:11 smithi133 bash[102858]: cluster 2023-10-09T05:42:10.521265+0000 mgr.y (mgr.25277) 1635 : cluster 0 pgmap v1842: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:42:11.625 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:11 smithi133 bash[102858]: audit 2023-10-09T05:42:11.219040+0000 mon.a (mon.0) 2148 : audit 0 from='client.? 172.21.15.133:0/3544822612' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T05:42:11.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:11 smithi194 bash[92666]: cluster 2023-10-09T05:42:10.521265+0000 mgr.y (mgr.25277) 1635 : cluster 0 pgmap v1842: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:42:11.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:11 smithi194 bash[92666]: audit 2023-10-09T05:42:11.219040+0000 mon.a (mon.0) 2148 : audit 0 from='client.? 172.21.15.133:0/3544822612' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T05:42:12.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:12 smithi194 bash[92666]: cluster 2023-10-09T05:42:11.586265+0000 osd.7 (osd.7) 389 : cluster 3 failed to encode map e424 with expected crc 2023-10-09T05:42:12.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:12 smithi194 bash[92666]: cluster 2023-10-09T05:42:11.586392+0000 osd.4 (osd.4) 359 : cluster 3 failed to encode map e424 with expected crc 2023-10-09T05:42:12.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:12 smithi194 bash[92666]: cluster 2023-10-09T05:42:11.587457+0000 osd.5 (osd.5) 461 : cluster 3 failed to encode map e424 with expected crc 2023-10-09T05:42:12.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:12 smithi194 bash[92666]: audit 2023-10-09T05:42:11.587670+0000 mon.a (mon.0) 2149 : audit 1 from='client.? 172.21.15.133:0/3544822612' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-10-09T05:42:12.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:12 smithi194 bash[92666]: cluster 2023-10-09T05:42:11.587757+0000 mon.a (mon.0) 2150 : cluster 0 osdmap e424: 8 total, 8 up, 8 in 2023-10-09T05:42:12.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:12 smithi194 bash[92666]: cluster 2023-10-09T05:42:11.588577+0000 osd.4 (osd.4) 360 : cluster 3 failed to encode map e424 with expected crc 2023-10-09T05:42:12.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:12 smithi194 bash[92666]: cluster 2023-10-09T05:42:11.589155+0000 osd.7 (osd.7) 390 : cluster 3 failed to encode map e424 with expected crc 2023-10-09T05:42:12.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:12 smithi194 bash[92666]: cluster 2023-10-09T05:42:11.590333+0000 osd.6 (osd.6) 420 : cluster 3 failed to encode map e424 with expected crc 2023-10-09T05:42:12.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:12 smithi194 bash[92666]: cluster 2023-10-09T05:42:11.593141+0000 osd.4 (osd.4) 361 : cluster 3 failed to encode map e424 with expected crc 2023-10-09T05:42:12.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:12 smithi194 bash[92666]: cluster 2023-10-09T05:42:11.595382+0000 osd.5 (osd.5) 462 : cluster 3 failed to encode map e424 with expected crc 2023-10-09T05:42:12.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:12 smithi194 bash[92666]: cluster 2023-10-09T05:42:11.597697+0000 osd.5 (osd.5) 463 : cluster 3 failed to encode map e424 with expected crc 2023-10-09T05:42:13.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:12 smithi133 bash[105589]: cluster 2023-10-09T05:42:11.586265+0000 osd.7 (osd.7) 389 : cluster 3 failed to encode map e424 with expected crc 2023-10-09T05:42:13.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:12 smithi133 bash[105589]: cluster 2023-10-09T05:42:11.586392+0000 osd.4 (osd.4) 359 : cluster 3 failed to encode map e424 with expected crc 2023-10-09T05:42:13.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:12 smithi133 bash[105589]: cluster 2023-10-09T05:42:11.587457+0000 osd.5 (osd.5) 461 : cluster 3 failed to encode map e424 with expected crc 2023-10-09T05:42:13.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:12 smithi133 bash[105589]: audit 2023-10-09T05:42:11.587670+0000 mon.a (mon.0) 2149 : audit 1 from='client.? 172.21.15.133:0/3544822612' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-10-09T05:42:13.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:12 smithi133 bash[105589]: cluster 2023-10-09T05:42:11.587757+0000 mon.a (mon.0) 2150 : cluster 0 osdmap e424: 8 total, 8 up, 8 in 2023-10-09T05:42:13.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:12 smithi133 bash[105589]: cluster 2023-10-09T05:42:11.588577+0000 osd.4 (osd.4) 360 : cluster 3 failed to encode map e424 with expected crc 2023-10-09T05:42:13.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:12 smithi133 bash[105589]: cluster 2023-10-09T05:42:11.589155+0000 osd.7 (osd.7) 390 : cluster 3 failed to encode map e424 with expected crc 2023-10-09T05:42:13.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:12 smithi133 bash[105589]: cluster 2023-10-09T05:42:11.590333+0000 osd.6 (osd.6) 420 : cluster 3 failed to encode map e424 with expected crc 2023-10-09T05:42:13.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:12 smithi133 bash[105589]: cluster 2023-10-09T05:42:11.593141+0000 osd.4 (osd.4) 361 : cluster 3 failed to encode map e424 with expected crc 2023-10-09T05:42:13.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:12 smithi133 bash[105589]: cluster 2023-10-09T05:42:11.595382+0000 osd.5 (osd.5) 462 : cluster 3 failed to encode map e424 with expected crc 2023-10-09T05:42:13.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:12 smithi133 bash[105589]: cluster 2023-10-09T05:42:11.597697+0000 osd.5 (osd.5) 463 : cluster 3 failed to encode map e424 with expected crc 2023-10-09T05:42:13.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:12 smithi133 bash[102858]: cluster 2023-10-09T05:42:11.586265+0000 osd.7 (osd.7) 389 : cluster 3 failed to encode map e424 with expected crc 2023-10-09T05:42:13.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:12 smithi133 bash[102858]: cluster 2023-10-09T05:42:11.586392+0000 osd.4 (osd.4) 359 : cluster 3 failed to encode map e424 with expected crc 2023-10-09T05:42:13.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:12 smithi133 bash[102858]: cluster 2023-10-09T05:42:11.587457+0000 osd.5 (osd.5) 461 : cluster 3 failed to encode map e424 with expected crc 2023-10-09T05:42:13.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:12 smithi133 bash[102858]: audit 2023-10-09T05:42:11.587670+0000 mon.a (mon.0) 2149 : audit 1 from='client.? 172.21.15.133:0/3544822612' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-10-09T05:42:13.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:12 smithi133 bash[102858]: cluster 2023-10-09T05:42:11.587757+0000 mon.a (mon.0) 2150 : cluster 0 osdmap e424: 8 total, 8 up, 8 in 2023-10-09T05:42:13.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:12 smithi133 bash[102858]: cluster 2023-10-09T05:42:11.588577+0000 osd.4 (osd.4) 360 : cluster 3 failed to encode map e424 with expected crc 2023-10-09T05:42:13.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:12 smithi133 bash[102858]: cluster 2023-10-09T05:42:11.589155+0000 osd.7 (osd.7) 390 : cluster 3 failed to encode map e424 with expected crc 2023-10-09T05:42:13.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:12 smithi133 bash[102858]: cluster 2023-10-09T05:42:11.590333+0000 osd.6 (osd.6) 420 : cluster 3 failed to encode map e424 with expected crc 2023-10-09T05:42:13.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:12 smithi133 bash[102858]: cluster 2023-10-09T05:42:11.593141+0000 osd.4 (osd.4) 361 : cluster 3 failed to encode map e424 with expected crc 2023-10-09T05:42:13.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:12 smithi133 bash[102858]: cluster 2023-10-09T05:42:11.595382+0000 osd.5 (osd.5) 462 : cluster 3 failed to encode map e424 with expected crc 2023-10-09T05:42:13.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:12 smithi133 bash[102858]: cluster 2023-10-09T05:42:11.597697+0000 osd.5 (osd.5) 463 : cluster 3 failed to encode map e424 with expected crc 2023-10-09T05:42:13.352 INFO:tasks.thrashosds.thrasher:Added osd 4 2023-10-09T05:42:13.352 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2023-10-09T05:42:13.451 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:42:13.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:13 smithi133 bash[105589]: cluster 2023-10-09T05:42:12.521895+0000 mgr.y (mgr.25277) 1636 : cluster 0 pgmap v1844: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:42:13.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:13 smithi133 bash[105589]: cluster 2023-10-09T05:42:12.602590+0000 osd.5 (osd.5) 464 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:13 smithi133 bash[105589]: cluster 2023-10-09T05:42:12.604613+0000 mon.a (mon.0) 2151 : cluster 0 osdmap e425: 8 total, 8 up, 8 in 2023-10-09T05:42:13.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:13 smithi133 bash[105589]: cluster 2023-10-09T05:42:12.604831+0000 osd.7 (osd.7) 391 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:13 smithi133 bash[105589]: cluster 2023-10-09T05:42:12.604897+0000 osd.4 (osd.4) 362 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:13 smithi133 bash[105589]: cluster 2023-10-09T05:42:12.604935+0000 osd.5 (osd.5) 465 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:13 smithi133 bash[105589]: cluster 2023-10-09T05:42:12.605049+0000 osd.6 (osd.6) 421 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:13 smithi133 bash[105589]: cluster 2023-10-09T05:42:12.606735+0000 osd.6 (osd.6) 422 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:13 smithi133 bash[105589]: cluster 2023-10-09T05:42:12.607589+0000 osd.5 (osd.5) 466 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:13 smithi133 bash[105589]: cluster 2023-10-09T05:42:12.608163+0000 osd.4 (osd.4) 363 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:13 smithi133 bash[105589]: cluster 2023-10-09T05:42:12.608594+0000 osd.7 (osd.7) 392 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:13 smithi133 bash[105589]: cluster 2023-10-09T05:42:12.609831+0000 osd.6 (osd.6) 423 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:13 smithi133 bash[105589]: cluster 2023-10-09T05:42:12.610860+0000 osd.7 (osd.7) 393 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:13 smithi133 bash[105589]: cluster 2023-10-09T05:42:12.611134+0000 osd.5 (osd.5) 467 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:13 smithi133 bash[105589]: audit 2023-10-09T05:42:13.497986+0000 mon.a (mon.0) 2152 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:13.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:13 smithi133 bash[102858]: cluster 2023-10-09T05:42:12.521895+0000 mgr.y (mgr.25277) 1636 : cluster 0 pgmap v1844: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:42:13.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:13 smithi133 bash[102858]: cluster 2023-10-09T05:42:12.602590+0000 osd.5 (osd.5) 464 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:13 smithi133 bash[102858]: cluster 2023-10-09T05:42:12.604613+0000 mon.a (mon.0) 2151 : cluster 0 osdmap e425: 8 total, 8 up, 8 in 2023-10-09T05:42:13.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:13 smithi133 bash[102858]: cluster 2023-10-09T05:42:12.604831+0000 osd.7 (osd.7) 391 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:13 smithi133 bash[102858]: cluster 2023-10-09T05:42:12.604897+0000 osd.4 (osd.4) 362 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:13 smithi133 bash[102858]: cluster 2023-10-09T05:42:12.604935+0000 osd.5 (osd.5) 465 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:13 smithi133 bash[102858]: cluster 2023-10-09T05:42:12.605049+0000 osd.6 (osd.6) 421 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:13 smithi133 bash[102858]: cluster 2023-10-09T05:42:12.606735+0000 osd.6 (osd.6) 422 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:13 smithi133 bash[102858]: cluster 2023-10-09T05:42:12.607589+0000 osd.5 (osd.5) 466 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:13 smithi133 bash[102858]: cluster 2023-10-09T05:42:12.608163+0000 osd.4 (osd.4) 363 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:13 smithi133 bash[102858]: cluster 2023-10-09T05:42:12.608594+0000 osd.7 (osd.7) 392 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:13 smithi133 bash[102858]: cluster 2023-10-09T05:42:12.609831+0000 osd.6 (osd.6) 423 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.800 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:13 smithi133 bash[102858]: cluster 2023-10-09T05:42:12.610860+0000 osd.7 (osd.7) 393 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:13 smithi133 bash[102858]: cluster 2023-10-09T05:42:12.611134+0000 osd.5 (osd.5) 467 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.801 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:13 smithi133 bash[102858]: audit 2023-10-09T05:42:13.497986+0000 mon.a (mon.0) 2152 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:13.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:13 smithi194 bash[92666]: cluster 2023-10-09T05:42:12.521895+0000 mgr.y (mgr.25277) 1636 : cluster 0 pgmap v1844: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T05:42:13.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:13 smithi194 bash[92666]: cluster 2023-10-09T05:42:12.602590+0000 osd.5 (osd.5) 464 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:13 smithi194 bash[92666]: cluster 2023-10-09T05:42:12.604613+0000 mon.a (mon.0) 2151 : cluster 0 osdmap e425: 8 total, 8 up, 8 in 2023-10-09T05:42:13.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:13 smithi194 bash[92666]: cluster 2023-10-09T05:42:12.604831+0000 osd.7 (osd.7) 391 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:13 smithi194 bash[92666]: cluster 2023-10-09T05:42:12.604897+0000 osd.4 (osd.4) 362 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:13 smithi194 bash[92666]: cluster 2023-10-09T05:42:12.604935+0000 osd.5 (osd.5) 465 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:13 smithi194 bash[92666]: cluster 2023-10-09T05:42:12.605049+0000 osd.6 (osd.6) 421 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:13 smithi194 bash[92666]: cluster 2023-10-09T05:42:12.606735+0000 osd.6 (osd.6) 422 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:13 smithi194 bash[92666]: cluster 2023-10-09T05:42:12.607589+0000 osd.5 (osd.5) 466 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:13 smithi194 bash[92666]: cluster 2023-10-09T05:42:12.608163+0000 osd.4 (osd.4) 363 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:13 smithi194 bash[92666]: cluster 2023-10-09T05:42:12.608594+0000 osd.7 (osd.7) 392 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:13 smithi194 bash[92666]: cluster 2023-10-09T05:42:12.609831+0000 osd.6 (osd.6) 423 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:13 smithi194 bash[92666]: cluster 2023-10-09T05:42:12.610860+0000 osd.7 (osd.7) 393 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:13 smithi194 bash[92666]: cluster 2023-10-09T05:42:12.611134+0000 osd.5 (osd.5) 467 : cluster 3 failed to encode map e425 with expected crc 2023-10-09T05:42:13.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:13 smithi194 bash[92666]: audit 2023-10-09T05:42:13.497986+0000 mon.a (mon.0) 2152 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:14.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:42:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:42:14] "GET /metrics HTTP/1.1" 200 34157 "" "Prometheus/2.43.0" 2023-10-09T05:42:14.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:14 smithi194 bash[92666]: cluster 2023-10-09T05:42:13.604732+0000 osd.6 (osd.6) 424 : cluster 3 failed to encode map e426 with expected crc 2023-10-09T05:42:14.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:14 smithi194 bash[92666]: cluster 2023-10-09T05:42:13.605442+0000 osd.5 (osd.5) 468 : cluster 3 failed to encode map e426 with expected crc 2023-10-09T05:42:14.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:14 smithi194 bash[92666]: cluster 2023-10-09T05:42:13.606380+0000 mon.a (mon.0) 2153 : cluster 0 osdmap e426: 8 total, 8 up, 8 in 2023-10-09T05:42:14.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:14 smithi194 bash[92666]: cluster 2023-10-09T05:42:13.832833+0000 osd.4 (osd.4) 364 : cluster 3 failed to encode map e426 with expected crc 2023-10-09T05:42:14.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:14 smithi194 bash[92666]: cluster 2023-10-09T05:42:13.835805+0000 osd.4 (osd.4) 365 : cluster 3 failed to encode map e426 with expected crc 2023-10-09T05:42:14.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:14 smithi194 bash[92666]: cluster 2023-10-09T05:42:13.838717+0000 osd.4 (osd.4) 366 : cluster 3 failed to encode map e426 with expected crc 2023-10-09T05:42:14.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:14 smithi194 bash[92666]: cluster 2023-10-09T05:42:13.996298+0000 osd.6 (osd.6) 425 : cluster 3 failed to encode map e427 with expected crc 2023-10-09T05:42:14.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:14 smithi194 bash[92666]: cluster 2023-10-09T05:42:13.996791+0000 osd.5 (osd.5) 469 : cluster 3 failed to encode map e427 with expected crc 2023-10-09T05:42:14.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:14 smithi194 bash[92666]: cluster 2023-10-09T05:42:14.000066+0000 mon.a (mon.0) 2154 : cluster 0 osdmap e427: 8 total, 8 up, 8 in 2023-10-09T05:42:14.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:14 smithi194 bash[92666]: cluster 2023-10-09T05:42:14.002196+0000 osd.4 (osd.4) 367 : cluster 3 failed to encode map e427 with expected crc 2023-10-09T05:42:14.968 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:14 smithi133 bash[102858]: cluster 2023-10-09T05:42:13.604732+0000 osd.6 (osd.6) 424 : cluster 3 failed to encode map e426 with expected crc 2023-10-09T05:42:14.969 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:14 smithi133 bash[102858]: cluster 2023-10-09T05:42:13.605442+0000 osd.5 (osd.5) 468 : cluster 3 failed to encode map e426 with expected crc 2023-10-09T05:42:14.969 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:14 smithi133 bash[102858]: cluster 2023-10-09T05:42:13.606380+0000 mon.a (mon.0) 2153 : cluster 0 osdmap e426: 8 total, 8 up, 8 in 2023-10-09T05:42:14.969 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:14 smithi133 bash[102858]: cluster 2023-10-09T05:42:13.832833+0000 osd.4 (osd.4) 364 : cluster 3 failed to encode map e426 with expected crc 2023-10-09T05:42:14.969 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:14 smithi133 bash[102858]: cluster 2023-10-09T05:42:13.835805+0000 osd.4 (osd.4) 365 : cluster 3 failed to encode map e426 with expected crc 2023-10-09T05:42:14.969 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:14 smithi133 bash[102858]: cluster 2023-10-09T05:42:13.838717+0000 osd.4 (osd.4) 366 : cluster 3 failed to encode map e426 with expected crc 2023-10-09T05:42:14.969 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:14 smithi133 bash[102858]: cluster 2023-10-09T05:42:13.996298+0000 osd.6 (osd.6) 425 : cluster 3 failed to encode map e427 with expected crc 2023-10-09T05:42:14.969 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:14 smithi133 bash[102858]: cluster 2023-10-09T05:42:13.996791+0000 osd.5 (osd.5) 469 : cluster 3 failed to encode map e427 with expected crc 2023-10-09T05:42:14.969 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:14 smithi133 bash[102858]: cluster 2023-10-09T05:42:14.000066+0000 mon.a (mon.0) 2154 : cluster 0 osdmap e427: 8 total, 8 up, 8 in 2023-10-09T05:42:14.969 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:14 smithi133 bash[102858]: cluster 2023-10-09T05:42:14.002196+0000 osd.4 (osd.4) 367 : cluster 3 failed to encode map e427 with expected crc 2023-10-09T05:42:14.970 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:14 smithi133 bash[105589]: cluster 2023-10-09T05:42:13.604732+0000 osd.6 (osd.6) 424 : cluster 3 failed to encode map e426 with expected crc 2023-10-09T05:42:14.970 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:14 smithi133 bash[105589]: cluster 2023-10-09T05:42:13.605442+0000 osd.5 (osd.5) 468 : cluster 3 failed to encode map e426 with expected crc 2023-10-09T05:42:14.970 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:14 smithi133 bash[105589]: cluster 2023-10-09T05:42:13.606380+0000 mon.a (mon.0) 2153 : cluster 0 osdmap e426: 8 total, 8 up, 8 in 2023-10-09T05:42:14.970 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:14 smithi133 bash[105589]: cluster 2023-10-09T05:42:13.832833+0000 osd.4 (osd.4) 364 : cluster 3 failed to encode map e426 with expected crc 2023-10-09T05:42:14.970 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:14 smithi133 bash[105589]: cluster 2023-10-09T05:42:13.835805+0000 osd.4 (osd.4) 365 : cluster 3 failed to encode map e426 with expected crc 2023-10-09T05:42:14.970 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:14 smithi133 bash[105589]: cluster 2023-10-09T05:42:13.838717+0000 osd.4 (osd.4) 366 : cluster 3 failed to encode map e426 with expected crc 2023-10-09T05:42:14.970 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:14 smithi133 bash[105589]: cluster 2023-10-09T05:42:13.996298+0000 osd.6 (osd.6) 425 : cluster 3 failed to encode map e427 with expected crc 2023-10-09T05:42:14.970 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:14 smithi133 bash[105589]: cluster 2023-10-09T05:42:13.996791+0000 osd.5 (osd.5) 469 : cluster 3 failed to encode map e427 with expected crc 2023-10-09T05:42:14.971 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:14 smithi133 bash[105589]: cluster 2023-10-09T05:42:14.000066+0000 mon.a (mon.0) 2154 : cluster 0 osdmap e427: 8 total, 8 up, 8 in 2023-10-09T05:42:14.971 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:14 smithi133 bash[105589]: cluster 2023-10-09T05:42:14.002196+0000 osd.4 (osd.4) 367 : cluster 3 failed to encode map e427 with expected crc 2023-10-09T05:42:15.219 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:42:15.220 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:42:15.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:15 smithi133 bash[102858]: cluster 2023-10-09T05:42:14.522796+0000 mgr.y (mgr.25277) 1637 : cluster 0 pgmap v1848: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T05:42:15.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:15 smithi133 bash[102858]: cluster 2023-10-09T05:42:14.759218+0000 osd.7 (osd.7) 394 : cluster 3 failed to encode map e426 with expected crc 2023-10-09T05:42:15.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:15 smithi133 bash[102858]: cluster 2023-10-09T05:42:15.013456+0000 osd.4 (osd.4) 368 : cluster 3 failed to encode map e428 with expected crc 2023-10-09T05:42:15.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:15 smithi133 bash[102858]: cluster 2023-10-09T05:42:15.015019+0000 mon.a (mon.0) 2155 : cluster 0 osdmap e428: 8 total, 8 up, 8 in 2023-10-09T05:42:15.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:15 smithi133 bash[102858]: cluster 2023-10-09T05:42:15.017584+0000 osd.6 (osd.6) 426 : cluster 3 failed to encode map e428 with expected crc 2023-10-09T05:42:15.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:15 smithi133 bash[105589]: cluster 2023-10-09T05:42:14.522796+0000 mgr.y (mgr.25277) 1637 : cluster 0 pgmap v1848: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T05:42:15.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:15 smithi133 bash[105589]: cluster 2023-10-09T05:42:14.759218+0000 osd.7 (osd.7) 394 : cluster 3 failed to encode map e426 with expected crc 2023-10-09T05:42:15.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:15 smithi133 bash[105589]: cluster 2023-10-09T05:42:15.013456+0000 osd.4 (osd.4) 368 : cluster 3 failed to encode map e428 with expected crc 2023-10-09T05:42:15.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:15 smithi133 bash[105589]: cluster 2023-10-09T05:42:15.015019+0000 mon.a (mon.0) 2155 : cluster 0 osdmap e428: 8 total, 8 up, 8 in 2023-10-09T05:42:15.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:15 smithi133 bash[105589]: cluster 2023-10-09T05:42:15.017584+0000 osd.6 (osd.6) 426 : cluster 3 failed to encode map e428 with expected crc 2023-10-09T05:42:15.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:15 smithi194 bash[92666]: cluster 2023-10-09T05:42:14.522796+0000 mgr.y (mgr.25277) 1637 : cluster 0 pgmap v1848: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T05:42:15.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:15 smithi194 bash[92666]: cluster 2023-10-09T05:42:14.759218+0000 osd.7 (osd.7) 394 : cluster 3 failed to encode map e426 with expected crc 2023-10-09T05:42:15.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:15 smithi194 bash[92666]: cluster 2023-10-09T05:42:15.013456+0000 osd.4 (osd.4) 368 : cluster 3 failed to encode map e428 with expected crc 2023-10-09T05:42:15.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:15 smithi194 bash[92666]: cluster 2023-10-09T05:42:15.015019+0000 mon.a (mon.0) 2155 : cluster 0 osdmap e428: 8 total, 8 up, 8 in 2023-10-09T05:42:15.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:15 smithi194 bash[92666]: cluster 2023-10-09T05:42:15.017584+0000 osd.6 (osd.6) 426 : cluster 3 failed to encode map e428 with expected crc 2023-10-09T05:42:16.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:16 smithi194 bash[92666]: cluster 2023-10-09T05:42:15.414943+0000 osd.5 (osd.5) 470 : cluster 3 failed to encode map e428 with expected crc 2023-10-09T05:42:16.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:16 smithi194 bash[92666]: cluster 2023-10-09T05:42:15.415099+0000 osd.7 (osd.7) 395 : cluster 3 failed to encode map e428 with expected crc 2023-10-09T05:42:16.969 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:16 smithi133 bash[105589]: cluster 2023-10-09T05:42:15.414943+0000 osd.5 (osd.5) 470 : cluster 3 failed to encode map e428 with expected crc 2023-10-09T05:42:16.969 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:16 smithi133 bash[105589]: cluster 2023-10-09T05:42:15.415099+0000 osd.7 (osd.7) 395 : cluster 3 failed to encode map e428 with expected crc 2023-10-09T05:42:16.969 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:16 smithi133 bash[102858]: cluster 2023-10-09T05:42:15.414943+0000 osd.5 (osd.5) 470 : cluster 3 failed to encode map e428 with expected crc 2023-10-09T05:42:16.970 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:16 smithi133 bash[102858]: cluster 2023-10-09T05:42:15.415099+0000 osd.7 (osd.7) 395 : cluster 3 failed to encode map e428 with expected crc 2023-10-09T05:42:16.970 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T05:42:17.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:17 smithi194 bash[92666]: cluster 2023-10-09T05:42:16.523373+0000 mgr.y (mgr.25277) 1638 : cluster 0 pgmap v1850: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 36 B/s, 2 keys/s, 6 objects/s recovering 2023-10-09T05:42:18.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:17 smithi133 bash[105589]: cluster 2023-10-09T05:42:16.523373+0000 mgr.y (mgr.25277) 1638 : cluster 0 pgmap v1850: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 36 B/s, 2 keys/s, 6 objects/s recovering 2023-10-09T05:42:18.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:17 smithi133 bash[102858]: cluster 2023-10-09T05:42:16.523373+0000 mgr.y (mgr.25277) 1638 : cluster 0 pgmap v1850: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 36 B/s, 2 keys/s, 6 objects/s recovering 2023-10-09T05:42:19.642 INFO:teuthology.orchestra.run.smithi133.stderr:nodeep-scrub is unset 2023-10-09T05:42:19.889 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:19 smithi133 bash[102858]: cluster 2023-10-09T05:42:18.523999+0000 mgr.y (mgr.25277) 1639 : cluster 0 pgmap v1851: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 24 B/s, 1 keys/s, 4 objects/s recovering 2023-10-09T05:42:19.890 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:19 smithi133 bash[102858]: audit 2023-10-09T05:42:18.730611+0000 mon.a (mon.0) 2156 : audit 1 from='client.? 172.21.15.133:0/2127118854' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:42:19.890 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:19 smithi133 bash[105589]: cluster 2023-10-09T05:42:18.523999+0000 mgr.y (mgr.25277) 1639 : cluster 0 pgmap v1851: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 24 B/s, 1 keys/s, 4 objects/s recovering 2023-10-09T05:42:19.890 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:19 smithi133 bash[105589]: audit 2023-10-09T05:42:18.730611+0000 mon.a (mon.0) 2156 : audit 1 from='client.? 172.21.15.133:0/2127118854' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:42:20.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:19 smithi194 bash[92666]: cluster 2023-10-09T05:42:18.523999+0000 mgr.y (mgr.25277) 1639 : cluster 0 pgmap v1851: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 24 B/s, 1 keys/s, 4 objects/s recovering 2023-10-09T05:42:20.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:19 smithi194 bash[92666]: audit 2023-10-09T05:42:18.730611+0000 mon.a (mon.0) 2156 : audit 1 from='client.? 172.21.15.133:0/2127118854' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:42:21.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:20 smithi133 bash[102858]: cluster 2023-10-09T05:42:19.628001+0000 mon.a (mon.0) 2157 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:42:21.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:20 smithi133 bash[102858]: cluster 2023-10-09T05:42:19.628054+0000 mon.a (mon.0) 2158 : cluster 1 Cluster is now healthy 2023-10-09T05:42:21.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:20 smithi133 bash[102858]: audit 2023-10-09T05:42:19.637006+0000 mon.a (mon.0) 2159 : audit 1 from='client.? 172.21.15.133:0/2127118854' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:42:21.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:20 smithi133 bash[102858]: cluster 2023-10-09T05:42:19.637105+0000 mon.a (mon.0) 2160 : cluster 0 osdmap e429: 8 total, 8 up, 8 in 2023-10-09T05:42:21.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:20 smithi133 bash[102858]: cluster 2023-10-09T05:42:19.733862+0000 osd.4 (osd.4) 369 : cluster 3 failed to encode map e429 with expected crc 2023-10-09T05:42:21.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:20 smithi133 bash[102858]: cluster 2023-10-09T05:42:19.737163+0000 osd.4 (osd.4) 370 : cluster 3 failed to encode map e429 with expected crc 2023-10-09T05:42:21.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:20 smithi133 bash[102858]: cluster 2023-10-09T05:42:19.740152+0000 osd.4 (osd.4) 371 : cluster 3 failed to encode map e429 with expected crc 2023-10-09T05:42:21.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:20 smithi133 bash[102858]: cluster 2023-10-09T05:42:20.016224+0000 osd.6 (osd.6) 427 : cluster 3 failed to encode map e429 with expected crc 2023-10-09T05:42:21.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:20 smithi133 bash[102858]: cluster 2023-10-09T05:42:20.016227+0000 osd.7 (osd.7) 396 : cluster 3 failed to encode map e429 with expected crc 2023-10-09T05:42:21.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:20 smithi133 bash[102858]: cluster 2023-10-09T05:42:20.016294+0000 osd.5 (osd.5) 471 : cluster 3 failed to encode map e429 with expected crc 2023-10-09T05:42:21.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:20 smithi133 bash[105589]: cluster 2023-10-09T05:42:19.628001+0000 mon.a (mon.0) 2157 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:42:21.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:20 smithi133 bash[105589]: cluster 2023-10-09T05:42:19.628054+0000 mon.a (mon.0) 2158 : cluster 1 Cluster is now healthy 2023-10-09T05:42:21.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:20 smithi133 bash[105589]: audit 2023-10-09T05:42:19.637006+0000 mon.a (mon.0) 2159 : audit 1 from='client.? 172.21.15.133:0/2127118854' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:42:21.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:20 smithi133 bash[105589]: cluster 2023-10-09T05:42:19.637105+0000 mon.a (mon.0) 2160 : cluster 0 osdmap e429: 8 total, 8 up, 8 in 2023-10-09T05:42:21.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:20 smithi133 bash[105589]: cluster 2023-10-09T05:42:19.733862+0000 osd.4 (osd.4) 369 : cluster 3 failed to encode map e429 with expected crc 2023-10-09T05:42:21.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:20 smithi133 bash[105589]: cluster 2023-10-09T05:42:19.737163+0000 osd.4 (osd.4) 370 : cluster 3 failed to encode map e429 with expected crc 2023-10-09T05:42:21.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:20 smithi133 bash[105589]: cluster 2023-10-09T05:42:19.740152+0000 osd.4 (osd.4) 371 : cluster 3 failed to encode map e429 with expected crc 2023-10-09T05:42:21.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:20 smithi133 bash[105589]: cluster 2023-10-09T05:42:20.016224+0000 osd.6 (osd.6) 427 : cluster 3 failed to encode map e429 with expected crc 2023-10-09T05:42:21.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:20 smithi133 bash[105589]: cluster 2023-10-09T05:42:20.016227+0000 osd.7 (osd.7) 396 : cluster 3 failed to encode map e429 with expected crc 2023-10-09T05:42:21.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:20 smithi133 bash[105589]: cluster 2023-10-09T05:42:20.016294+0000 osd.5 (osd.5) 471 : cluster 3 failed to encode map e429 with expected crc 2023-10-09T05:42:21.056 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:42:21.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:20 smithi194 bash[92666]: cluster 2023-10-09T05:42:19.628001+0000 mon.a (mon.0) 2157 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:42:21.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:20 smithi194 bash[92666]: cluster 2023-10-09T05:42:19.628054+0000 mon.a (mon.0) 2158 : cluster 1 Cluster is now healthy 2023-10-09T05:42:21.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:20 smithi194 bash[92666]: audit 2023-10-09T05:42:19.637006+0000 mon.a (mon.0) 2159 : audit 1 from='client.? 172.21.15.133:0/2127118854' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:42:21.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:20 smithi194 bash[92666]: cluster 2023-10-09T05:42:19.637105+0000 mon.a (mon.0) 2160 : cluster 0 osdmap e429: 8 total, 8 up, 8 in 2023-10-09T05:42:21.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:20 smithi194 bash[92666]: cluster 2023-10-09T05:42:19.733862+0000 osd.4 (osd.4) 369 : cluster 3 failed to encode map e429 with expected crc 2023-10-09T05:42:21.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:20 smithi194 bash[92666]: cluster 2023-10-09T05:42:19.737163+0000 osd.4 (osd.4) 370 : cluster 3 failed to encode map e429 with expected crc 2023-10-09T05:42:21.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:20 smithi194 bash[92666]: cluster 2023-10-09T05:42:19.740152+0000 osd.4 (osd.4) 371 : cluster 3 failed to encode map e429 with expected crc 2023-10-09T05:42:21.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:20 smithi194 bash[92666]: cluster 2023-10-09T05:42:20.016224+0000 osd.6 (osd.6) 427 : cluster 3 failed to encode map e429 with expected crc 2023-10-09T05:42:21.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:20 smithi194 bash[92666]: cluster 2023-10-09T05:42:20.016227+0000 osd.7 (osd.7) 396 : cluster 3 failed to encode map e429 with expected crc 2023-10-09T05:42:21.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:20 smithi194 bash[92666]: cluster 2023-10-09T05:42:20.016294+0000 osd.5 (osd.5) 471 : cluster 3 failed to encode map e429 with expected crc 2023-10-09T05:42:22.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:21 smithi133 bash[105589]: cluster 2023-10-09T05:42:20.524774+0000 mgr.y (mgr.25277) 1640 : cluster 0 pgmap v1853: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 22 B/s, 1 keys/s, 4 objects/s recovering 2023-10-09T05:42:22.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:21 smithi133 bash[102858]: cluster 2023-10-09T05:42:20.524774+0000 mgr.y (mgr.25277) 1640 : cluster 0 pgmap v1853: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 22 B/s, 1 keys/s, 4 objects/s recovering 2023-10-09T05:42:22.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:21 smithi194 bash[92666]: cluster 2023-10-09T05:42:20.524774+0000 mgr.y (mgr.25277) 1640 : cluster 0 pgmap v1853: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 22 B/s, 1 keys/s, 4 objects/s recovering 2023-10-09T05:42:23.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:23 smithi133 bash[102858]: cluster 2023-10-09T05:42:22.525682+0000 mgr.y (mgr.25277) 1641 : cluster 0 pgmap v1854: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 18 B/s, 1 keys/s, 3 objects/s recovering 2023-10-09T05:42:23.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:23 smithi133 bash[105589]: cluster 2023-10-09T05:42:22.525682+0000 mgr.y (mgr.25277) 1641 : cluster 0 pgmap v1854: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 18 B/s, 1 keys/s, 3 objects/s recovering 2023-10-09T05:42:23.856 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:42:24.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:23 smithi194 bash[92666]: cluster 2023-10-09T05:42:22.525682+0000 mgr.y (mgr.25277) 1641 : cluster 0 pgmap v1854: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 18 B/s, 1 keys/s, 3 objects/s recovering 2023-10-09T05:42:24.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:42:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:42:24] "GET /metrics HTTP/1.1" 200 34157 "" "Prometheus/2.43.0" 2023-10-09T05:42:25.708 INFO:teuthology.orchestra.run.smithi133.stdout:mon.a: {} 2023-10-09T05:42:25.729 INFO:teuthology.orchestra.run.smithi133.stdout:mon.b: {} 2023-10-09T05:42:25.740 INFO:teuthology.orchestra.run.smithi133.stdout:mon.c: {} 2023-10-09T05:42:25.742 INFO:teuthology.orchestra.run.smithi133.stderr:mon.a: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T05:42:25.742 INFO:teuthology.orchestra.run.smithi133.stderr:mon.b: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T05:42:25.742 INFO:teuthology.orchestra.run.smithi133.stderr:mon.c: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T05:42:25.765 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:25 smithi133 bash[105589]: audit 2023-10-09T05:42:24.390541+0000 mon.a (mon.0) 2161 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:42:25.765 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:25 smithi133 bash[105589]: cluster 2023-10-09T05:42:24.526514+0000 mgr.y (mgr.25277) 1642 : cluster 0 pgmap v1855: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T05:42:25.765 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:25 smithi133 bash[102858]: audit 2023-10-09T05:42:24.390541+0000 mon.a (mon.0) 2161 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:42:25.765 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:25 smithi133 bash[102858]: cluster 2023-10-09T05:42:24.526514+0000 mgr.y (mgr.25277) 1642 : cluster 0 pgmap v1855: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T05:42:25.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:25 smithi194 bash[92666]: audit 2023-10-09T05:42:24.390541+0000 mon.a (mon.0) 2161 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:42:25.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:25 smithi194 bash[92666]: cluster 2023-10-09T05:42:24.526514+0000 mgr.y (mgr.25277) 1642 : cluster 0 pgmap v1855: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T05:42:26.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:26 smithi133 bash[102858]: audit 2023-10-09T05:42:25.689588+0000 mon.a (mon.0) 2162 : audit 0 from='client.? 172.21.15.133:0/1287115529' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:42:26.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:26 smithi133 bash[105589]: audit 2023-10-09T05:42:25.689588+0000 mon.a (mon.0) 2162 : audit 0 from='client.? 172.21.15.133:0/1287115529' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:42:26.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:26 smithi194 bash[92666]: audit 2023-10-09T05:42:25.689588+0000 mon.a (mon.0) 2162 : audit 0 from='client.? 172.21.15.133:0/1287115529' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:42:27.316 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T05:42:27.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:27 smithi133 bash[105589]: cluster 2023-10-09T05:42:26.527409+0000 mgr.y (mgr.25277) 1643 : cluster 0 pgmap v1856: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T05:42:27.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:27 smithi133 bash[102858]: cluster 2023-10-09T05:42:26.527409+0000 mgr.y (mgr.25277) 1643 : cluster 0 pgmap v1856: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T05:42:27.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:27 smithi194 bash[92666]: cluster 2023-10-09T05:42:26.527409+0000 mgr.y (mgr.25277) 1643 : cluster 0 pgmap v1856: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T05:42:28.885 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:42:28.885 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:42:29.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:29 smithi133 bash[102858]: audit 2023-10-09T05:42:28.504916+0000 mon.a (mon.0) 2163 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:42:29.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:29 smithi133 bash[102858]: audit 2023-10-09T05:42:28.506399+0000 mon.a (mon.0) 2164 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:29.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:29 smithi133 bash[102858]: cluster 2023-10-09T05:42:28.527832+0000 mgr.y (mgr.25277) 1644 : cluster 0 pgmap v1857: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:29.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:29 smithi133 bash[105589]: audit 2023-10-09T05:42:28.504916+0000 mon.a (mon.0) 2163 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:42:29.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:29 smithi133 bash[105589]: audit 2023-10-09T05:42:28.506399+0000 mon.a (mon.0) 2164 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:29.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:29 smithi133 bash[105589]: cluster 2023-10-09T05:42:28.527832+0000 mgr.y (mgr.25277) 1644 : cluster 0 pgmap v1857: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:29.864 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T05:42:29.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:29 smithi194 bash[92666]: audit 2023-10-09T05:42:28.504916+0000 mon.a (mon.0) 2163 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:42:29.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:29 smithi194 bash[92666]: audit 2023-10-09T05:42:28.506399+0000 mon.a (mon.0) 2164 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:29.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:29 smithi194 bash[92666]: cluster 2023-10-09T05:42:28.527832+0000 mgr.y (mgr.25277) 1644 : cluster 0 pgmap v1857: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:31.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:31 smithi194 bash[92666]: cluster 2023-10-09T05:42:30.528799+0000 mgr.y (mgr.25277) 1645 : cluster 0 pgmap v1858: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:32.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:31 smithi133 bash[102858]: cluster 2023-10-09T05:42:30.528799+0000 mgr.y (mgr.25277) 1645 : cluster 0 pgmap v1858: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:32.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:31 smithi133 bash[105589]: cluster 2023-10-09T05:42:30.528799+0000 mgr.y (mgr.25277) 1645 : cluster 0 pgmap v1858: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:32.384 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:42:33.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:33 smithi133 bash[105589]: cluster 2023-10-09T05:42:32.529720+0000 mgr.y (mgr.25277) 1646 : cluster 0 pgmap v1859: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:33.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:33 smithi133 bash[102858]: cluster 2023-10-09T05:42:32.529720+0000 mgr.y (mgr.25277) 1646 : cluster 0 pgmap v1859: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:33.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:33 smithi194 bash[92666]: cluster 2023-10-09T05:42:32.529720+0000 mgr.y (mgr.25277) 1646 : cluster 0 pgmap v1859: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:34.547 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:42:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:42:34] "GET /metrics HTTP/1.1" 200 34159 "" "Prometheus/2.43.0" 2023-10-09T05:42:35.093 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:42:35.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:35 smithi133 bash[102858]: cluster 2023-10-09T05:42:34.530525+0000 mgr.y (mgr.25277) 1647 : cluster 0 pgmap v1860: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:35.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:35 smithi133 bash[105589]: cluster 2023-10-09T05:42:34.530525+0000 mgr.y (mgr.25277) 1647 : cluster 0 pgmap v1860: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:35.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:35 smithi194 bash[92666]: cluster 2023-10-09T05:42:34.530525+0000 mgr.y (mgr.25277) 1647 : cluster 0 pgmap v1860: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:37.601 INFO:teuthology.orchestra.run.smithi133.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T05:42:37.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:37 smithi194 bash[92666]: cluster 2023-10-09T05:42:36.531220+0000 mgr.y (mgr.25277) 1648 : cluster 0 pgmap v1861: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:37.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:37 smithi194 bash[92666]: audit 2023-10-09T05:42:37.052832+0000 mon.c (mon.2) 149 : audit 1 from='client.? 172.21.15.133:0/780745714' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:42:37.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:37 smithi194 bash[92666]: audit 2023-10-09T05:42:37.053432+0000 mon.a (mon.0) 2165 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:42:37.911 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:37 smithi133 bash[105589]: cluster 2023-10-09T05:42:36.531220+0000 mgr.y (mgr.25277) 1648 : cluster 0 pgmap v1861: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:37.911 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:37 smithi133 bash[105589]: audit 2023-10-09T05:42:37.052832+0000 mon.c (mon.2) 149 : audit 1 from='client.? 172.21.15.133:0/780745714' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:42:37.911 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:37 smithi133 bash[105589]: audit 2023-10-09T05:42:37.053432+0000 mon.a (mon.0) 2165 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:42:37.911 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:37 smithi133 bash[102858]: cluster 2023-10-09T05:42:36.531220+0000 mgr.y (mgr.25277) 1648 : cluster 0 pgmap v1861: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:37.911 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:37 smithi133 bash[102858]: audit 2023-10-09T05:42:37.052832+0000 mon.c (mon.2) 149 : audit 1 from='client.? 172.21.15.133:0/780745714' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:42:37.912 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:37 smithi133 bash[102858]: audit 2023-10-09T05:42:37.053432+0000 mon.a (mon.0) 2165 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:42:38.777 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 0 1 2023-10-09T05:42:38.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:38 smithi194 bash[92666]: cluster 2023-10-09T05:42:37.588863+0000 osd.6 (osd.6) 428 : cluster 3 failed to encode map e430 with expected crc 2023-10-09T05:42:38.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:38 smithi194 bash[92666]: audit 2023-10-09T05:42:37.593926+0000 mon.a (mon.0) 2166 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:42:38.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:38 smithi194 bash[92666]: cluster 2023-10-09T05:42:37.594002+0000 mon.a (mon.0) 2167 : cluster 0 osdmap e430: 8 total, 8 up, 8 in 2023-10-09T05:42:38.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:38 smithi194 bash[92666]: cluster 2023-10-09T05:42:37.785949+0000 osd.7 (osd.7) 397 : cluster 3 failed to encode map e430 with expected crc 2023-10-09T05:42:38.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:38 smithi194 bash[92666]: cluster 2023-10-09T05:42:37.788919+0000 osd.7 (osd.7) 398 : cluster 3 failed to encode map e430 with expected crc 2023-10-09T05:42:38.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:38 smithi194 bash[92666]: cluster 2023-10-09T05:42:37.791579+0000 osd.7 (osd.7) 399 : cluster 3 failed to encode map e430 with expected crc 2023-10-09T05:42:39.047 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:38 smithi133 bash[105589]: cluster 2023-10-09T05:42:37.588863+0000 osd.6 (osd.6) 428 : cluster 3 failed to encode map e430 with expected crc 2023-10-09T05:42:39.047 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:38 smithi133 bash[105589]: audit 2023-10-09T05:42:37.593926+0000 mon.a (mon.0) 2166 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:42:39.047 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:38 smithi133 bash[105589]: cluster 2023-10-09T05:42:37.594002+0000 mon.a (mon.0) 2167 : cluster 0 osdmap e430: 8 total, 8 up, 8 in 2023-10-09T05:42:39.047 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:38 smithi133 bash[105589]: cluster 2023-10-09T05:42:37.785949+0000 osd.7 (osd.7) 397 : cluster 3 failed to encode map e430 with expected crc 2023-10-09T05:42:39.047 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:38 smithi133 bash[105589]: cluster 2023-10-09T05:42:37.788919+0000 osd.7 (osd.7) 398 : cluster 3 failed to encode map e430 with expected crc 2023-10-09T05:42:39.047 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:38 smithi133 bash[105589]: cluster 2023-10-09T05:42:37.791579+0000 osd.7 (osd.7) 399 : cluster 3 failed to encode map e430 with expected crc 2023-10-09T05:42:39.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:38 smithi133 bash[102858]: cluster 2023-10-09T05:42:37.588863+0000 osd.6 (osd.6) 428 : cluster 3 failed to encode map e430 with expected crc 2023-10-09T05:42:39.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:38 smithi133 bash[102858]: audit 2023-10-09T05:42:37.593926+0000 mon.a (mon.0) 2166 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:42:39.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:38 smithi133 bash[102858]: cluster 2023-10-09T05:42:37.594002+0000 mon.a (mon.0) 2167 : cluster 0 osdmap e430: 8 total, 8 up, 8 in 2023-10-09T05:42:39.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:38 smithi133 bash[102858]: cluster 2023-10-09T05:42:37.785949+0000 osd.7 (osd.7) 397 : cluster 3 failed to encode map e430 with expected crc 2023-10-09T05:42:39.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:38 smithi133 bash[102858]: cluster 2023-10-09T05:42:37.788919+0000 osd.7 (osd.7) 398 : cluster 3 failed to encode map e430 with expected crc 2023-10-09T05:42:39.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:38 smithi133 bash[102858]: cluster 2023-10-09T05:42:37.791579+0000 osd.7 (osd.7) 399 : cluster 3 failed to encode map e430 with expected crc 2023-10-09T05:42:39.305 INFO:teuthology.orchestra.run.smithi133.stdout:ERROR: (22) Invalid argument 2023-10-09T05:42:39.306 INFO:teuthology.orchestra.run.smithi133.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:42:39.922 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:39 smithi194 bash[92666]: cluster 2023-10-09T05:42:38.329353+0000 osd.5 (osd.5) 472 : cluster 3 failed to encode map e430 with expected crc 2023-10-09T05:42:39.922 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:39 smithi194 bash[92666]: cluster 2023-10-09T05:42:38.332529+0000 osd.5 (osd.5) 473 : cluster 3 failed to encode map e430 with expected crc 2023-10-09T05:42:39.922 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:39 smithi194 bash[92666]: cluster 2023-10-09T05:42:38.423113+0000 osd.4 (osd.4) 372 : cluster 3 failed to encode map e430 with expected crc 2023-10-09T05:42:39.922 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:39 smithi194 bash[92666]: cluster 2023-10-09T05:42:38.531678+0000 mgr.y (mgr.25277) 1649 : cluster 0 pgmap v1863: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:40.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:39 smithi133 bash[105589]: cluster 2023-10-09T05:42:38.329353+0000 osd.5 (osd.5) 472 : cluster 3 failed to encode map e430 with expected crc 2023-10-09T05:42:40.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:39 smithi133 bash[105589]: cluster 2023-10-09T05:42:38.332529+0000 osd.5 (osd.5) 473 : cluster 3 failed to encode map e430 with expected crc 2023-10-09T05:42:40.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:39 smithi133 bash[105589]: cluster 2023-10-09T05:42:38.423113+0000 osd.4 (osd.4) 372 : cluster 3 failed to encode map e430 with expected crc 2023-10-09T05:42:40.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:39 smithi133 bash[105589]: cluster 2023-10-09T05:42:38.531678+0000 mgr.y (mgr.25277) 1649 : cluster 0 pgmap v1863: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:40.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:39 smithi133 bash[102858]: cluster 2023-10-09T05:42:38.329353+0000 osd.5 (osd.5) 472 : cluster 3 failed to encode map e430 with expected crc 2023-10-09T05:42:40.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:39 smithi133 bash[102858]: cluster 2023-10-09T05:42:38.332529+0000 osd.5 (osd.5) 473 : cluster 3 failed to encode map e430 with expected crc 2023-10-09T05:42:40.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:39 smithi133 bash[102858]: cluster 2023-10-09T05:42:38.423113+0000 osd.4 (osd.4) 372 : cluster 3 failed to encode map e430 with expected crc 2023-10-09T05:42:40.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:39 smithi133 bash[102858]: cluster 2023-10-09T05:42:38.531678+0000 mgr.y (mgr.25277) 1649 : cluster 0 pgmap v1863: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:40.518 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T05:42:41.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:41 smithi194 bash[92666]: cluster 2023-10-09T05:42:40.532644+0000 mgr.y (mgr.25277) 1650 : cluster 0 pgmap v1864: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:42.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:41 smithi133 bash[102858]: cluster 2023-10-09T05:42:40.532644+0000 mgr.y (mgr.25277) 1650 : cluster 0 pgmap v1864: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:42.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:41 smithi133 bash[105589]: cluster 2023-10-09T05:42:40.532644+0000 mgr.y (mgr.25277) 1650 : cluster 0 pgmap v1864: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:43.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:43 smithi194 bash[92666]: cluster 2023-10-09T05:42:42.533207+0000 mgr.y (mgr.25277) 1651 : cluster 0 pgmap v1865: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:43.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:43 smithi194 bash[92666]: audit 2023-10-09T05:42:43.499626+0000 mon.a (mon.0) 2168 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:44.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:43 smithi133 bash[102858]: cluster 2023-10-09T05:42:42.533207+0000 mgr.y (mgr.25277) 1651 : cluster 0 pgmap v1865: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:44.052 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:43 smithi133 bash[102858]: audit 2023-10-09T05:42:43.499626+0000 mon.a (mon.0) 2168 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:44.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:43 smithi133 bash[105589]: cluster 2023-10-09T05:42:42.533207+0000 mgr.y (mgr.25277) 1651 : cluster 0 pgmap v1865: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:44.053 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:43 smithi133 bash[105589]: audit 2023-10-09T05:42:43.499626+0000 mon.a (mon.0) 2168 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:44.053 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:42:44.549 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:42:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:42:44] "GET /metrics HTTP/1.1" 200 34155 "" "Prometheus/2.43.0" 2023-10-09T05:42:45.295 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:42:45.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:45 smithi133 bash[102858]: cluster 2023-10-09T05:42:44.534070+0000 mgr.y (mgr.25277) 1652 : cluster 0 pgmap v1866: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:45.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:45 smithi133 bash[105589]: cluster 2023-10-09T05:42:44.534070+0000 mgr.y (mgr.25277) 1652 : cluster 0 pgmap v1866: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:45.327 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:45 smithi194 bash[92666]: cluster 2023-10-09T05:42:44.534070+0000 mgr.y (mgr.25277) 1652 : cluster 0 pgmap v1866: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:47.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:47 smithi133 bash[102858]: cluster 2023-10-09T05:42:46.534888+0000 mgr.y (mgr.25277) 1653 : cluster 0 pgmap v1867: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:47.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:47 smithi133 bash[105589]: cluster 2023-10-09T05:42:46.534888+0000 mgr.y (mgr.25277) 1653 : cluster 0 pgmap v1867: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:47.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:47 smithi194 bash[92666]: cluster 2023-10-09T05:42:46.534888+0000 mgr.y (mgr.25277) 1653 : cluster 0 pgmap v1867: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:48.611 INFO:teuthology.orchestra.run.smithi133.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-10-09T05:42:48.740 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:42:48.740 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:42:48.785 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:48 smithi194 bash[92666]: audit 2023-10-09T05:42:47.965834+0000 mon.a (mon.0) 2169 : audit 1 from='client.? 172.21.15.133:0/3130302282' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:42:48.843 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:48 smithi133 bash[105589]: audit 2023-10-09T05:42:47.965834+0000 mon.a (mon.0) 2169 : audit 1 from='client.? 172.21.15.133:0/3130302282' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:42:48.843 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:48 smithi133 bash[102858]: audit 2023-10-09T05:42:47.965834+0000 mon.a (mon.0) 2169 : audit 1 from='client.? 172.21.15.133:0/3130302282' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:42:49.607 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T05:42:49.609 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T05:42:49.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:49 smithi194 bash[92666]: cluster 2023-10-09T05:42:48.535592+0000 mgr.y (mgr.25277) 1654 : cluster 0 pgmap v1868: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:49.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:49 smithi194 bash[92666]: cluster 2023-10-09T05:42:48.603564+0000 osd.4 (osd.4) 373 : cluster 3 failed to encode map e431 with expected crc 2023-10-09T05:42:49.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:49 smithi194 bash[92666]: audit 2023-10-09T05:42:48.605077+0000 mon.a (mon.0) 2170 : audit 1 from='client.? 172.21.15.133:0/3130302282' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:42:49.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:49 smithi194 bash[92666]: cluster 2023-10-09T05:42:48.605172+0000 mon.a (mon.0) 2171 : cluster 0 osdmap e431: 8 total, 8 up, 8 in 2023-10-09T05:42:49.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:49 smithi194 bash[92666]: cluster 2023-10-09T05:42:48.924802+0000 osd.6 (osd.6) 429 : cluster 3 failed to encode map e431 with expected crc 2023-10-09T05:42:49.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:49 smithi194 bash[92666]: cluster 2023-10-09T05:42:48.925029+0000 osd.7 (osd.7) 400 : cluster 3 failed to encode map e431 with expected crc 2023-10-09T05:42:49.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:49 smithi194 bash[92666]: cluster 2023-10-09T05:42:48.925046+0000 osd.5 (osd.5) 474 : cluster 3 failed to encode map e431 with expected crc 2023-10-09T05:42:50.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:49 smithi133 bash[102858]: cluster 2023-10-09T05:42:48.535592+0000 mgr.y (mgr.25277) 1654 : cluster 0 pgmap v1868: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:50.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:49 smithi133 bash[102858]: cluster 2023-10-09T05:42:48.603564+0000 osd.4 (osd.4) 373 : cluster 3 failed to encode map e431 with expected crc 2023-10-09T05:42:50.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:49 smithi133 bash[102858]: audit 2023-10-09T05:42:48.605077+0000 mon.a (mon.0) 2170 : audit 1 from='client.? 172.21.15.133:0/3130302282' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:42:50.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:49 smithi133 bash[102858]: cluster 2023-10-09T05:42:48.605172+0000 mon.a (mon.0) 2171 : cluster 0 osdmap e431: 8 total, 8 up, 8 in 2023-10-09T05:42:50.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:49 smithi133 bash[102858]: cluster 2023-10-09T05:42:48.924802+0000 osd.6 (osd.6) 429 : cluster 3 failed to encode map e431 with expected crc 2023-10-09T05:42:50.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:49 smithi133 bash[102858]: cluster 2023-10-09T05:42:48.925029+0000 osd.7 (osd.7) 400 : cluster 3 failed to encode map e431 with expected crc 2023-10-09T05:42:50.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:49 smithi133 bash[102858]: cluster 2023-10-09T05:42:48.925046+0000 osd.5 (osd.5) 474 : cluster 3 failed to encode map e431 with expected crc 2023-10-09T05:42:50.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:49 smithi133 bash[105589]: cluster 2023-10-09T05:42:48.535592+0000 mgr.y (mgr.25277) 1654 : cluster 0 pgmap v1868: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:50.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:49 smithi133 bash[105589]: cluster 2023-10-09T05:42:48.603564+0000 osd.4 (osd.4) 373 : cluster 3 failed to encode map e431 with expected crc 2023-10-09T05:42:50.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:49 smithi133 bash[105589]: audit 2023-10-09T05:42:48.605077+0000 mon.a (mon.0) 2170 : audit 1 from='client.? 172.21.15.133:0/3130302282' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:42:50.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:49 smithi133 bash[105589]: cluster 2023-10-09T05:42:48.605172+0000 mon.a (mon.0) 2171 : cluster 0 osdmap e431: 8 total, 8 up, 8 in 2023-10-09T05:42:50.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:49 smithi133 bash[105589]: cluster 2023-10-09T05:42:48.924802+0000 osd.6 (osd.6) 429 : cluster 3 failed to encode map e431 with expected crc 2023-10-09T05:42:50.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:49 smithi133 bash[105589]: cluster 2023-10-09T05:42:48.925029+0000 osd.7 (osd.7) 400 : cluster 3 failed to encode map e431 with expected crc 2023-10-09T05:42:50.052 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:49 smithi133 bash[105589]: cluster 2023-10-09T05:42:48.925046+0000 osd.5 (osd.5) 474 : cluster 3 failed to encode map e431 with expected crc 2023-10-09T05:42:51.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:51 smithi194 bash[92666]: cluster 2023-10-09T05:42:50.536611+0000 mgr.y (mgr.25277) 1655 : cluster 0 pgmap v1870: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:52.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:51 smithi133 bash[102858]: cluster 2023-10-09T05:42:50.536611+0000 mgr.y (mgr.25277) 1655 : cluster 0 pgmap v1870: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:52.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:51 smithi133 bash[105589]: cluster 2023-10-09T05:42:50.536611+0000 mgr.y (mgr.25277) 1655 : cluster 0 pgmap v1870: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:53.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:53 smithi194 bash[92666]: cluster 2023-10-09T05:42:52.537427+0000 mgr.y (mgr.25277) 1656 : cluster 0 pgmap v1871: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:54.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:53 smithi133 bash[105589]: cluster 2023-10-09T05:42:52.537427+0000 mgr.y (mgr.25277) 1656 : cluster 0 pgmap v1871: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:54.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:53 smithi133 bash[102858]: cluster 2023-10-09T05:42:52.537427+0000 mgr.y (mgr.25277) 1656 : cluster 0 pgmap v1871: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:54.382 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:42:54.387 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:42:54.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:42:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:42:54] "GET /metrics HTTP/1.1" 200 34155 "" "Prometheus/2.43.0" 2023-10-09T05:42:55.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:55 smithi133 bash[102858]: cluster 2023-10-09T05:42:54.538278+0000 mgr.y (mgr.25277) 1657 : cluster 0 pgmap v1872: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:55.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:55 smithi133 bash[105589]: cluster 2023-10-09T05:42:54.538278+0000 mgr.y (mgr.25277) 1657 : cluster 0 pgmap v1872: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:55.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:55 smithi194 bash[92666]: cluster 2023-10-09T05:42:54.538278+0000 mgr.y (mgr.25277) 1657 : cluster 0 pgmap v1872: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:57.726 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:42:57.726 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:42:57.752 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:57 smithi194 bash[92666]: cluster 2023-10-09T05:42:56.539177+0000 mgr.y (mgr.25277) 1658 : cluster 0 pgmap v1873: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:58.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:57 smithi133 bash[105589]: cluster 2023-10-09T05:42:56.539177+0000 mgr.y (mgr.25277) 1658 : cluster 0 pgmap v1873: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:58.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:57 smithi133 bash[102858]: cluster 2023-10-09T05:42:56.539177+0000 mgr.y (mgr.25277) 1658 : cluster 0 pgmap v1873: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:58.522 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T05:42:58.623 INFO:teuthology.orchestra.run.smithi133.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T05:42:58.866 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:58 smithi133 bash[102858]: audit 2023-10-09T05:42:58.212254+0000 mon.a (mon.0) 2172 : audit 1 from='client.? 172.21.15.133:0/2032643360' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:42:58.866 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:58 smithi133 bash[102858]: audit 2023-10-09T05:42:58.499986+0000 mon.a (mon.0) 2173 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:58.866 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:58 smithi133 bash[105589]: audit 2023-10-09T05:42:58.212254+0000 mon.a (mon.0) 2172 : audit 1 from='client.? 172.21.15.133:0/2032643360' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:42:58.866 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:58 smithi133 bash[105589]: audit 2023-10-09T05:42:58.499986+0000 mon.a (mon.0) 2173 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:58.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:58 smithi194 bash[92666]: audit 2023-10-09T05:42:58.212254+0000 mon.a (mon.0) 2172 : audit 1 from='client.? 172.21.15.133:0/2032643360' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:42:58.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:58 smithi194 bash[92666]: audit 2023-10-09T05:42:58.499986+0000 mon.a (mon.0) 2173 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:59.576 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-10-09T05:42:59.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:59 smithi194 bash[92666]: cluster 2023-10-09T05:42:58.539834+0000 mgr.y (mgr.25277) 1659 : cluster 0 pgmap v1874: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:42:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:59 smithi194 bash[92666]: cluster 2023-10-09T05:42:58.610945+0000 osd.4 (osd.4) 374 : cluster 3 failed to encode map e432 with expected crc 2023-10-09T05:42:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:59 smithi194 bash[92666]: audit 2023-10-09T05:42:58.612969+0000 mon.a (mon.0) 2174 : audit 1 from='client.? 172.21.15.133:0/2032643360' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:42:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:59 smithi194 bash[92666]: cluster 2023-10-09T05:42:58.613042+0000 mon.a (mon.0) 2175 : cluster 0 osdmap e432: 8 total, 8 up, 8 in 2023-10-09T05:42:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:59 smithi194 bash[92666]: cluster 2023-10-09T05:42:58.613991+0000 osd.5 (osd.5) 475 : cluster 3 failed to encode map e432 with expected crc 2023-10-09T05:42:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:59 smithi194 bash[92666]: cluster 2023-10-09T05:42:58.623243+0000 osd.7 (osd.7) 401 : cluster 3 failed to encode map e432 with expected crc 2023-10-09T05:42:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:42:59 smithi194 bash[92666]: audit 2023-10-09T05:42:59.553771+0000 mon.a (mon.0) 2176 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:00.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:59 smithi133 bash[105589]: cluster 2023-10-09T05:42:58.539834+0000 mgr.y (mgr.25277) 1659 : cluster 0 pgmap v1874: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:00.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:59 smithi133 bash[105589]: cluster 2023-10-09T05:42:58.610945+0000 osd.4 (osd.4) 374 : cluster 3 failed to encode map e432 with expected crc 2023-10-09T05:43:00.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:59 smithi133 bash[105589]: audit 2023-10-09T05:42:58.612969+0000 mon.a (mon.0) 2174 : audit 1 from='client.? 172.21.15.133:0/2032643360' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:43:00.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:59 smithi133 bash[105589]: cluster 2023-10-09T05:42:58.613042+0000 mon.a (mon.0) 2175 : cluster 0 osdmap e432: 8 total, 8 up, 8 in 2023-10-09T05:43:00.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:59 smithi133 bash[105589]: cluster 2023-10-09T05:42:58.613991+0000 osd.5 (osd.5) 475 : cluster 3 failed to encode map e432 with expected crc 2023-10-09T05:43:00.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:59 smithi133 bash[105589]: cluster 2023-10-09T05:42:58.623243+0000 osd.7 (osd.7) 401 : cluster 3 failed to encode map e432 with expected crc 2023-10-09T05:43:00.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:42:59 smithi133 bash[105589]: audit 2023-10-09T05:42:59.553771+0000 mon.a (mon.0) 2176 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:00.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:59 smithi133 bash[102858]: cluster 2023-10-09T05:42:58.539834+0000 mgr.y (mgr.25277) 1659 : cluster 0 pgmap v1874: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:00.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:59 smithi133 bash[102858]: cluster 2023-10-09T05:42:58.610945+0000 osd.4 (osd.4) 374 : cluster 3 failed to encode map e432 with expected crc 2023-10-09T05:43:00.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:59 smithi133 bash[102858]: audit 2023-10-09T05:42:58.612969+0000 mon.a (mon.0) 2174 : audit 1 from='client.? 172.21.15.133:0/2032643360' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:43:00.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:59 smithi133 bash[102858]: cluster 2023-10-09T05:42:58.613042+0000 mon.a (mon.0) 2175 : cluster 0 osdmap e432: 8 total, 8 up, 8 in 2023-10-09T05:43:00.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:59 smithi133 bash[102858]: cluster 2023-10-09T05:42:58.613991+0000 osd.5 (osd.5) 475 : cluster 3 failed to encode map e432 with expected crc 2023-10-09T05:43:00.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:59 smithi133 bash[102858]: cluster 2023-10-09T05:42:58.623243+0000 osd.7 (osd.7) 401 : cluster 3 failed to encode map e432 with expected crc 2023-10-09T05:43:00.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:42:59 smithi133 bash[102858]: audit 2023-10-09T05:42:59.553771+0000 mon.a (mon.0) 2176 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:00.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:00 smithi194 bash[92666]: cluster 2023-10-09T05:42:59.621119+0000 osd.4 (osd.4) 375 : cluster 3 failed to encode map e433 with expected crc 2023-10-09T05:43:00.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:00 smithi194 bash[92666]: cluster 2023-10-09T05:42:59.622715+0000 mon.a (mon.0) 2177 : cluster 0 osdmap e433: 8 total, 8 up, 8 in 2023-10-09T05:43:00.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:00 smithi194 bash[92666]: cluster 2023-10-09T05:42:59.623445+0000 osd.6 (osd.6) 430 : cluster 3 failed to encode map e433 with expected crc 2023-10-09T05:43:00.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:00 smithi194 bash[92666]: cluster 2023-10-09T05:42:59.623508+0000 osd.4 (osd.4) 376 : cluster 3 failed to encode map e433 with expected crc 2023-10-09T05:43:00.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:00 smithi194 bash[92666]: cluster 2023-10-09T05:42:59.628006+0000 osd.5 (osd.5) 476 : cluster 3 failed to encode map e433 with expected crc 2023-10-09T05:43:01.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:00 smithi133 bash[102858]: cluster 2023-10-09T05:42:59.621119+0000 osd.4 (osd.4) 375 : cluster 3 failed to encode map e433 with expected crc 2023-10-09T05:43:01.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:00 smithi133 bash[102858]: cluster 2023-10-09T05:42:59.622715+0000 mon.a (mon.0) 2177 : cluster 0 osdmap e433: 8 total, 8 up, 8 in 2023-10-09T05:43:01.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:00 smithi133 bash[102858]: cluster 2023-10-09T05:42:59.623445+0000 osd.6 (osd.6) 430 : cluster 3 failed to encode map e433 with expected crc 2023-10-09T05:43:01.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:00 smithi133 bash[102858]: cluster 2023-10-09T05:42:59.623508+0000 osd.4 (osd.4) 376 : cluster 3 failed to encode map e433 with expected crc 2023-10-09T05:43:01.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:00 smithi133 bash[102858]: cluster 2023-10-09T05:42:59.628006+0000 osd.5 (osd.5) 476 : cluster 3 failed to encode map e433 with expected crc 2023-10-09T05:43:01.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:00 smithi133 bash[105589]: cluster 2023-10-09T05:42:59.621119+0000 osd.4 (osd.4) 375 : cluster 3 failed to encode map e433 with expected crc 2023-10-09T05:43:01.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:00 smithi133 bash[105589]: cluster 2023-10-09T05:42:59.622715+0000 mon.a (mon.0) 2177 : cluster 0 osdmap e433: 8 total, 8 up, 8 in 2023-10-09T05:43:01.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:00 smithi133 bash[105589]: cluster 2023-10-09T05:42:59.623445+0000 osd.6 (osd.6) 430 : cluster 3 failed to encode map e433 with expected crc 2023-10-09T05:43:01.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:00 smithi133 bash[105589]: cluster 2023-10-09T05:42:59.623508+0000 osd.4 (osd.4) 376 : cluster 3 failed to encode map e433 with expected crc 2023-10-09T05:43:01.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:00 smithi133 bash[105589]: cluster 2023-10-09T05:42:59.628006+0000 osd.5 (osd.5) 476 : cluster 3 failed to encode map e433 with expected crc 2023-10-09T05:43:01.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:01 smithi194 bash[92666]: cluster 2023-10-09T05:43:00.540615+0000 mgr.y (mgr.25277) 1660 : cluster 0 pgmap v1877: 73 pgs: 3 peering, 70 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:02.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:01 smithi133 bash[102858]: cluster 2023-10-09T05:43:00.540615+0000 mgr.y (mgr.25277) 1660 : cluster 0 pgmap v1877: 73 pgs: 3 peering, 70 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:02.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:01 smithi133 bash[105589]: cluster 2023-10-09T05:43:00.540615+0000 mgr.y (mgr.25277) 1660 : cluster 0 pgmap v1877: 73 pgs: 3 peering, 70 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:03.305 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:43:04.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:03 smithi133 bash[105589]: cluster 2023-10-09T05:43:02.541308+0000 mgr.y (mgr.25277) 1661 : cluster 0 pgmap v1878: 73 pgs: 3 peering, 70 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:04.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:03 smithi133 bash[102858]: cluster 2023-10-09T05:43:02.541308+0000 mgr.y (mgr.25277) 1661 : cluster 0 pgmap v1878: 73 pgs: 3 peering, 70 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:04.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:03 smithi194 bash[92666]: cluster 2023-10-09T05:43:02.541308+0000 mgr.y (mgr.25277) 1661 : cluster 0 pgmap v1878: 73 pgs: 3 peering, 70 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:04.397 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:43:04.547 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:43:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:43:04] "GET /metrics HTTP/1.1" 200 34158 "" "Prometheus/2.43.0" 2023-10-09T05:43:05.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:05 smithi194 bash[92666]: cluster 2023-10-09T05:43:04.541998+0000 mgr.y (mgr.25277) 1662 : cluster 0 pgmap v1879: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:05.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:05 smithi133 bash[102858]: cluster 2023-10-09T05:43:04.541998+0000 mgr.y (mgr.25277) 1662 : cluster 0 pgmap v1879: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:05.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:05 smithi133 bash[105589]: cluster 2023-10-09T05:43:04.541998+0000 mgr.y (mgr.25277) 1662 : cluster 0 pgmap v1879: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:06.728 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:43:06.728 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:43:06.751 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:06 smithi194 bash[92666]: audit 2023-10-09T05:43:05.496133+0000 mon.a (mon.0) 2178 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:43:06.751 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:06 smithi194 bash[92666]: audit 2023-10-09T05:43:05.504124+0000 mon.a (mon.0) 2179 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:43:06.751 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:06 smithi194 bash[92666]: audit 2023-10-09T05:43:05.749697+0000 mon.a (mon.0) 2180 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:43:06.751 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:06 smithi194 bash[92666]: audit 2023-10-09T05:43:05.758024+0000 mon.a (mon.0) 2181 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:43:06.752 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:06 smithi194 bash[92666]: audit 2023-10-09T05:43:06.234392+0000 mon.a (mon.0) 2182 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:43:06.752 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:06 smithi194 bash[92666]: audit 2023-10-09T05:43:06.235503+0000 mon.a (mon.0) 2183 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:43:06.752 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:06 smithi194 bash[92666]: audit 2023-10-09T05:43:06.243434+0000 mon.a (mon.0) 2184 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:43:06.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:06 smithi133 bash[105589]: audit 2023-10-09T05:43:05.496133+0000 mon.a (mon.0) 2178 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:43:06.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:06 smithi133 bash[105589]: audit 2023-10-09T05:43:05.504124+0000 mon.a (mon.0) 2179 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:43:06.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:06 smithi133 bash[105589]: audit 2023-10-09T05:43:05.749697+0000 mon.a (mon.0) 2180 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:43:06.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:06 smithi133 bash[105589]: audit 2023-10-09T05:43:05.758024+0000 mon.a (mon.0) 2181 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:43:06.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:06 smithi133 bash[105589]: audit 2023-10-09T05:43:06.234392+0000 mon.a (mon.0) 2182 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:43:06.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:06 smithi133 bash[105589]: audit 2023-10-09T05:43:06.235503+0000 mon.a (mon.0) 2183 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:43:06.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:06 smithi133 bash[105589]: audit 2023-10-09T05:43:06.243434+0000 mon.a (mon.0) 2184 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:43:06.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:06 smithi133 bash[102858]: audit 2023-10-09T05:43:05.496133+0000 mon.a (mon.0) 2178 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:43:06.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:06 smithi133 bash[102858]: audit 2023-10-09T05:43:05.504124+0000 mon.a (mon.0) 2179 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:43:06.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:06 smithi133 bash[102858]: audit 2023-10-09T05:43:05.749697+0000 mon.a (mon.0) 2180 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:43:06.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:06 smithi133 bash[102858]: audit 2023-10-09T05:43:05.758024+0000 mon.a (mon.0) 2181 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:43:06.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:06 smithi133 bash[102858]: audit 2023-10-09T05:43:06.234392+0000 mon.a (mon.0) 2182 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:43:06.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:06 smithi133 bash[102858]: audit 2023-10-09T05:43:06.235503+0000 mon.a (mon.0) 2183 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:43:06.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:06 smithi133 bash[102858]: audit 2023-10-09T05:43:06.243434+0000 mon.a (mon.0) 2184 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:43:07.629 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T05:43:07.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:07 smithi133 bash[102858]: cluster 2023-10-09T05:43:06.542900+0000 mgr.y (mgr.25277) 1663 : cluster 0 pgmap v1880: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:07.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:07 smithi133 bash[105589]: cluster 2023-10-09T05:43:06.542900+0000 mgr.y (mgr.25277) 1663 : cluster 0 pgmap v1880: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:07.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:07 smithi194 bash[92666]: cluster 2023-10-09T05:43:06.542900+0000 mgr.y (mgr.25277) 1663 : cluster 0 pgmap v1880: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:08.523 INFO:teuthology.orchestra.run.smithi133.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-10-09T05:43:08.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:08 smithi133 bash[105589]: audit 2023-10-09T05:43:08.499580+0000 mon.a (mon.0) 2185 : audit 1 from='client.? 172.21.15.133:0/2424342537' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:43:08.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:08 smithi133 bash[102858]: audit 2023-10-09T05:43:08.499580+0000 mon.a (mon.0) 2185 : audit 1 from='client.? 172.21.15.133:0/2424342537' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:43:08.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:08 smithi194 bash[92666]: audit 2023-10-09T05:43:08.499580+0000 mon.a (mon.0) 2185 : audit 1 from='client.? 172.21.15.133:0/2424342537' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:43:09.518 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T05:43:09.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:09 smithi133 bash[105589]: audit 2023-10-09T05:43:08.518507+0000 mon.a (mon.0) 2186 : audit 1 from='client.? 172.21.15.133:0/2424342537' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:43:09.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:09 smithi133 bash[105589]: cluster 2023-10-09T05:43:08.518576+0000 mon.a (mon.0) 2187 : cluster 0 osdmap e434: 8 total, 8 up, 8 in 2023-10-09T05:43:09.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:09 smithi133 bash[105589]: cluster 2023-10-09T05:43:08.543303+0000 mgr.y (mgr.25277) 1664 : cluster 0 pgmap v1882: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:09.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:09 smithi133 bash[105589]: cluster 2023-10-09T05:43:08.639015+0000 osd.7 (osd.7) 402 : cluster 3 failed to encode map e434 with expected crc 2023-10-09T05:43:09.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:09 smithi133 bash[105589]: cluster 2023-10-09T05:43:08.729328+0000 osd.4 (osd.4) 377 : cluster 3 failed to encode map e434 with expected crc 2023-10-09T05:43:09.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:09 smithi133 bash[105589]: cluster 2023-10-09T05:43:08.729395+0000 osd.6 (osd.6) 431 : cluster 3 failed to encode map e434 with expected crc 2023-10-09T05:43:09.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:09 smithi133 bash[105589]: cluster 2023-10-09T05:43:08.729593+0000 osd.5 (osd.5) 477 : cluster 3 failed to encode map e434 with expected crc 2023-10-09T05:43:09.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:09 smithi133 bash[102858]: audit 2023-10-09T05:43:08.518507+0000 mon.a (mon.0) 2186 : audit 1 from='client.? 172.21.15.133:0/2424342537' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:43:09.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:09 smithi133 bash[102858]: cluster 2023-10-09T05:43:08.518576+0000 mon.a (mon.0) 2187 : cluster 0 osdmap e434: 8 total, 8 up, 8 in 2023-10-09T05:43:09.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:09 smithi133 bash[102858]: cluster 2023-10-09T05:43:08.543303+0000 mgr.y (mgr.25277) 1664 : cluster 0 pgmap v1882: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:09.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:09 smithi133 bash[102858]: cluster 2023-10-09T05:43:08.639015+0000 osd.7 (osd.7) 402 : cluster 3 failed to encode map e434 with expected crc 2023-10-09T05:43:09.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:09 smithi133 bash[102858]: cluster 2023-10-09T05:43:08.729328+0000 osd.4 (osd.4) 377 : cluster 3 failed to encode map e434 with expected crc 2023-10-09T05:43:09.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:09 smithi133 bash[102858]: cluster 2023-10-09T05:43:08.729395+0000 osd.6 (osd.6) 431 : cluster 3 failed to encode map e434 with expected crc 2023-10-09T05:43:09.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:09 smithi133 bash[102858]: cluster 2023-10-09T05:43:08.729593+0000 osd.5 (osd.5) 477 : cluster 3 failed to encode map e434 with expected crc 2023-10-09T05:43:09.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:09 smithi194 bash[92666]: audit 2023-10-09T05:43:08.518507+0000 mon.a (mon.0) 2186 : audit 1 from='client.? 172.21.15.133:0/2424342537' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:43:09.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:09 smithi194 bash[92666]: cluster 2023-10-09T05:43:08.518576+0000 mon.a (mon.0) 2187 : cluster 0 osdmap e434: 8 total, 8 up, 8 in 2023-10-09T05:43:09.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:09 smithi194 bash[92666]: cluster 2023-10-09T05:43:08.543303+0000 mgr.y (mgr.25277) 1664 : cluster 0 pgmap v1882: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:09.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:09 smithi194 bash[92666]: cluster 2023-10-09T05:43:08.639015+0000 osd.7 (osd.7) 402 : cluster 3 failed to encode map e434 with expected crc 2023-10-09T05:43:09.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:09 smithi194 bash[92666]: cluster 2023-10-09T05:43:08.729328+0000 osd.4 (osd.4) 377 : cluster 3 failed to encode map e434 with expected crc 2023-10-09T05:43:09.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:09 smithi194 bash[92666]: cluster 2023-10-09T05:43:08.729395+0000 osd.6 (osd.6) 431 : cluster 3 failed to encode map e434 with expected crc 2023-10-09T05:43:09.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:09 smithi194 bash[92666]: cluster 2023-10-09T05:43:08.729593+0000 osd.5 (osd.5) 477 : cluster 3 failed to encode map e434 with expected crc 2023-10-09T05:43:11.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:11 smithi194 bash[92666]: cluster 2023-10-09T05:43:10.544331+0000 mgr.y (mgr.25277) 1665 : cluster 0 pgmap v1883: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:12.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:11 smithi133 bash[105589]: cluster 2023-10-09T05:43:10.544331+0000 mgr.y (mgr.25277) 1665 : cluster 0 pgmap v1883: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:12.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:11 smithi133 bash[102858]: cluster 2023-10-09T05:43:10.544331+0000 mgr.y (mgr.25277) 1665 : cluster 0 pgmap v1883: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:12.407 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:43:13.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:13 smithi194 bash[92666]: cluster 2023-10-09T05:43:12.545105+0000 mgr.y (mgr.25277) 1666 : cluster 0 pgmap v1884: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:13.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:13 smithi194 bash[92666]: audit 2023-10-09T05:43:13.500453+0000 mon.a (mon.0) 2188 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:14.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:13 smithi133 bash[102858]: cluster 2023-10-09T05:43:12.545105+0000 mgr.y (mgr.25277) 1666 : cluster 0 pgmap v1884: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:14.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:13 smithi133 bash[102858]: audit 2023-10-09T05:43:13.500453+0000 mon.a (mon.0) 2188 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:14.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:13 smithi133 bash[105589]: cluster 2023-10-09T05:43:12.545105+0000 mgr.y (mgr.25277) 1666 : cluster 0 pgmap v1884: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:14.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:13 smithi133 bash[105589]: audit 2023-10-09T05:43:13.500453+0000 mon.a (mon.0) 2188 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:14.288 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:43:14.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:43:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:43:14] "GET /metrics HTTP/1.1" 200 34143 "" "Prometheus/2.43.0" 2023-10-09T05:43:15.738 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:15 smithi133 bash[105589]: audit 2023-10-09T05:43:14.417196+0000 mon.a (mon.0) 2189 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:43:15.738 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:15 smithi133 bash[105589]: cluster 2023-10-09T05:43:14.546025+0000 mgr.y (mgr.25277) 1667 : cluster 0 pgmap v1885: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:15.738 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:15 smithi133 bash[102858]: audit 2023-10-09T05:43:14.417196+0000 mon.a (mon.0) 2189 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:43:15.738 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:15 smithi133 bash[102858]: cluster 2023-10-09T05:43:14.546025+0000 mgr.y (mgr.25277) 1667 : cluster 0 pgmap v1885: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:15.820 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:43:15.821 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:43:15.865 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:15 smithi194 bash[92666]: audit 2023-10-09T05:43:14.417196+0000 mon.a (mon.0) 2189 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:43:15.866 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:15 smithi194 bash[92666]: cluster 2023-10-09T05:43:14.546025+0000 mgr.y (mgr.25277) 1667 : cluster 0 pgmap v1885: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:16.662 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T05:43:16.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:16 smithi133 bash[102858]: audit 2023-10-09T05:43:15.900369+0000 mon.a (mon.0) 2190 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2023-10-09T05:43:16.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:16 smithi133 bash[105589]: audit 2023-10-09T05:43:15.900369+0000 mon.a (mon.0) 2190 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2023-10-09T05:43:16.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:16 smithi194 bash[92666]: audit 2023-10-09T05:43:15.900369+0000 mon.a (mon.0) 2190 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2023-10-09T05:43:17.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:17 smithi133 bash[105589]: audit 2023-10-09T05:43:16.439018+0000 mon.a (mon.0) 2191 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2023-10-09T05:43:17.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:17 smithi133 bash[105589]: cluster 2023-10-09T05:43:16.439116+0000 mon.a (mon.0) 2192 : cluster 0 osdmap e435: 8 total, 8 up, 8 in 2023-10-09T05:43:17.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:17 smithi133 bash[105589]: cluster 2023-10-09T05:43:16.442686+0000 osd.4 (osd.4) 378 : cluster 3 failed to encode map e435 with expected crc 2023-10-09T05:43:17.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:17 smithi133 bash[105589]: cluster 2023-10-09T05:43:16.546689+0000 mgr.y (mgr.25277) 1668 : cluster 0 pgmap v1887: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:17.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:17 smithi133 bash[105589]: cluster 2023-10-09T05:43:16.674862+0000 osd.7 (osd.7) 403 : cluster 3 failed to encode map e435 with expected crc 2023-10-09T05:43:17.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:17 smithi133 bash[102858]: audit 2023-10-09T05:43:16.439018+0000 mon.a (mon.0) 2191 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2023-10-09T05:43:17.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:17 smithi133 bash[102858]: cluster 2023-10-09T05:43:16.439116+0000 mon.a (mon.0) 2192 : cluster 0 osdmap e435: 8 total, 8 up, 8 in 2023-10-09T05:43:17.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:17 smithi133 bash[102858]: cluster 2023-10-09T05:43:16.442686+0000 osd.4 (osd.4) 378 : cluster 3 failed to encode map e435 with expected crc 2023-10-09T05:43:17.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:17 smithi133 bash[102858]: cluster 2023-10-09T05:43:16.546689+0000 mgr.y (mgr.25277) 1668 : cluster 0 pgmap v1887: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:17.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:17 smithi133 bash[102858]: cluster 2023-10-09T05:43:16.674862+0000 osd.7 (osd.7) 403 : cluster 3 failed to encode map e435 with expected crc 2023-10-09T05:43:17.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:17 smithi194 bash[92666]: audit 2023-10-09T05:43:16.439018+0000 mon.a (mon.0) 2191 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2023-10-09T05:43:17.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:17 smithi194 bash[92666]: cluster 2023-10-09T05:43:16.439116+0000 mon.a (mon.0) 2192 : cluster 0 osdmap e435: 8 total, 8 up, 8 in 2023-10-09T05:43:17.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:17 smithi194 bash[92666]: cluster 2023-10-09T05:43:16.442686+0000 osd.4 (osd.4) 378 : cluster 3 failed to encode map e435 with expected crc 2023-10-09T05:43:17.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:17 smithi194 bash[92666]: cluster 2023-10-09T05:43:16.546689+0000 mgr.y (mgr.25277) 1668 : cluster 0 pgmap v1887: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:17.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:17 smithi194 bash[92666]: cluster 2023-10-09T05:43:16.674862+0000 osd.7 (osd.7) 403 : cluster 3 failed to encode map e435 with expected crc 2023-10-09T05:43:18.457 INFO:teuthology.orchestra.run.smithi133.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T05:43:18.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:18 smithi133 bash[102858]: cluster 2023-10-09T05:43:17.448150+0000 mon.a (mon.0) 2193 : cluster 0 osdmap e436: 8 total, 8 up, 8 in 2023-10-09T05:43:18.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:18 smithi133 bash[102858]: cluster 2023-10-09T05:43:17.449000+0000 osd.4 (osd.4) 379 : cluster 3 failed to encode map e436 with expected crc 2023-10-09T05:43:18.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:18 smithi133 bash[102858]: cluster 2023-10-09T05:43:17.451778+0000 osd.4 (osd.4) 380 : cluster 3 failed to encode map e436 with expected crc 2023-10-09T05:43:18.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:18 smithi133 bash[102858]: cluster 2023-10-09T05:43:17.478163+0000 osd.5 (osd.5) 478 : cluster 3 failed to encode map e435 with expected crc 2023-10-09T05:43:18.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:18 smithi133 bash[102858]: cluster 2023-10-09T05:43:17.483397+0000 osd.5 (osd.5) 479 : cluster 3 failed to encode map e436 with expected crc 2023-10-09T05:43:18.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:18 smithi133 bash[102858]: cluster 2023-10-09T05:43:17.657259+0000 osd.7 (osd.7) 404 : cluster 3 failed to encode map e436 with expected crc 2023-10-09T05:43:18.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:18 smithi133 bash[102858]: cluster 2023-10-09T05:43:17.860705+0000 osd.6 (osd.6) 432 : cluster 3 failed to encode map e436 with expected crc 2023-10-09T05:43:18.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:18 smithi133 bash[102858]: cluster 2023-10-09T05:43:17.863459+0000 osd.6 (osd.6) 433 : cluster 3 failed to encode map e436 with expected crc 2023-10-09T05:43:18.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:18 smithi133 bash[102858]: audit 2023-10-09T05:43:18.334471+0000 mon.c (mon.2) 150 : audit 1 from='client.? 172.21.15.133:0/3514482262' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:43:18.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:18 smithi133 bash[102858]: audit 2023-10-09T05:43:18.335141+0000 mon.a (mon.0) 2194 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:43:18.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:18 smithi133 bash[105589]: cluster 2023-10-09T05:43:17.448150+0000 mon.a (mon.0) 2193 : cluster 0 osdmap e436: 8 total, 8 up, 8 in 2023-10-09T05:43:18.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:18 smithi133 bash[105589]: cluster 2023-10-09T05:43:17.449000+0000 osd.4 (osd.4) 379 : cluster 3 failed to encode map e436 with expected crc 2023-10-09T05:43:18.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:18 smithi133 bash[105589]: cluster 2023-10-09T05:43:17.451778+0000 osd.4 (osd.4) 380 : cluster 3 failed to encode map e436 with expected crc 2023-10-09T05:43:18.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:18 smithi133 bash[105589]: cluster 2023-10-09T05:43:17.478163+0000 osd.5 (osd.5) 478 : cluster 3 failed to encode map e435 with expected crc 2023-10-09T05:43:18.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:18 smithi133 bash[105589]: cluster 2023-10-09T05:43:17.483397+0000 osd.5 (osd.5) 479 : cluster 3 failed to encode map e436 with expected crc 2023-10-09T05:43:18.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:18 smithi133 bash[105589]: cluster 2023-10-09T05:43:17.657259+0000 osd.7 (osd.7) 404 : cluster 3 failed to encode map e436 with expected crc 2023-10-09T05:43:18.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:18 smithi133 bash[105589]: cluster 2023-10-09T05:43:17.860705+0000 osd.6 (osd.6) 432 : cluster 3 failed to encode map e436 with expected crc 2023-10-09T05:43:18.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:18 smithi133 bash[105589]: cluster 2023-10-09T05:43:17.863459+0000 osd.6 (osd.6) 433 : cluster 3 failed to encode map e436 with expected crc 2023-10-09T05:43:18.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:18 smithi133 bash[105589]: audit 2023-10-09T05:43:18.334471+0000 mon.c (mon.2) 150 : audit 1 from='client.? 172.21.15.133:0/3514482262' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:43:18.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:18 smithi133 bash[105589]: audit 2023-10-09T05:43:18.335141+0000 mon.a (mon.0) 2194 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:43:18.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:18 smithi194 bash[92666]: cluster 2023-10-09T05:43:17.448150+0000 mon.a (mon.0) 2193 : cluster 0 osdmap e436: 8 total, 8 up, 8 in 2023-10-09T05:43:18.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:18 smithi194 bash[92666]: cluster 2023-10-09T05:43:17.449000+0000 osd.4 (osd.4) 379 : cluster 3 failed to encode map e436 with expected crc 2023-10-09T05:43:18.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:18 smithi194 bash[92666]: cluster 2023-10-09T05:43:17.451778+0000 osd.4 (osd.4) 380 : cluster 3 failed to encode map e436 with expected crc 2023-10-09T05:43:18.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:18 smithi194 bash[92666]: cluster 2023-10-09T05:43:17.478163+0000 osd.5 (osd.5) 478 : cluster 3 failed to encode map e435 with expected crc 2023-10-09T05:43:18.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:18 smithi194 bash[92666]: cluster 2023-10-09T05:43:17.483397+0000 osd.5 (osd.5) 479 : cluster 3 failed to encode map e436 with expected crc 2023-10-09T05:43:18.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:18 smithi194 bash[92666]: cluster 2023-10-09T05:43:17.657259+0000 osd.7 (osd.7) 404 : cluster 3 failed to encode map e436 with expected crc 2023-10-09T05:43:18.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:18 smithi194 bash[92666]: cluster 2023-10-09T05:43:17.860705+0000 osd.6 (osd.6) 432 : cluster 3 failed to encode map e436 with expected crc 2023-10-09T05:43:18.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:18 smithi194 bash[92666]: cluster 2023-10-09T05:43:17.863459+0000 osd.6 (osd.6) 433 : cluster 3 failed to encode map e436 with expected crc 2023-10-09T05:43:18.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:18 smithi194 bash[92666]: audit 2023-10-09T05:43:18.334471+0000 mon.c (mon.2) 150 : audit 1 from='client.? 172.21.15.133:0/3514482262' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:43:18.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:18 smithi194 bash[92666]: audit 2023-10-09T05:43:18.335141+0000 mon.a (mon.0) 2194 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:43:19.461 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-10-09T05:43:19.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:19 smithi133 bash[102858]: cluster 2023-10-09T05:43:18.447894+0000 osd.5 (osd.5) 480 : cluster 3 failed to encode map e437 with expected crc 2023-10-09T05:43:19.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:19 smithi133 bash[102858]: audit 2023-10-09T05:43:18.451582+0000 mon.a (mon.0) 2195 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:43:19.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:19 smithi133 bash[102858]: cluster 2023-10-09T05:43:18.451764+0000 mon.a (mon.0) 2196 : cluster 0 osdmap e437: 8 total, 8 up, 8 in 2023-10-09T05:43:19.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:19 smithi133 bash[102858]: cluster 2023-10-09T05:43:18.471177+0000 osd.7 (osd.7) 405 : cluster 3 failed to encode map e437 with expected crc 2023-10-09T05:43:19.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:19 smithi133 bash[102858]: cluster 2023-10-09T05:43:18.471282+0000 osd.4 (osd.4) 381 : cluster 3 failed to encode map e437 with expected crc 2023-10-09T05:43:19.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:19 smithi133 bash[102858]: cluster 2023-10-09T05:43:18.471414+0000 osd.6 (osd.6) 434 : cluster 3 failed to encode map e437 with expected crc 2023-10-09T05:43:19.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:19 smithi133 bash[102858]: cluster 2023-10-09T05:43:18.547325+0000 mgr.y (mgr.25277) 1669 : cluster 0 pgmap v1890: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:19.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:19 smithi133 bash[105589]: cluster 2023-10-09T05:43:18.447894+0000 osd.5 (osd.5) 480 : cluster 3 failed to encode map e437 with expected crc 2023-10-09T05:43:19.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:19 smithi133 bash[105589]: audit 2023-10-09T05:43:18.451582+0000 mon.a (mon.0) 2195 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:43:19.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:19 smithi133 bash[105589]: cluster 2023-10-09T05:43:18.451764+0000 mon.a (mon.0) 2196 : cluster 0 osdmap e437: 8 total, 8 up, 8 in 2023-10-09T05:43:19.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:19 smithi133 bash[105589]: cluster 2023-10-09T05:43:18.471177+0000 osd.7 (osd.7) 405 : cluster 3 failed to encode map e437 with expected crc 2023-10-09T05:43:19.800 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:19 smithi133 bash[105589]: cluster 2023-10-09T05:43:18.471282+0000 osd.4 (osd.4) 381 : cluster 3 failed to encode map e437 with expected crc 2023-10-09T05:43:19.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:19 smithi133 bash[105589]: cluster 2023-10-09T05:43:18.471414+0000 osd.6 (osd.6) 434 : cluster 3 failed to encode map e437 with expected crc 2023-10-09T05:43:19.801 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:19 smithi133 bash[105589]: cluster 2023-10-09T05:43:18.547325+0000 mgr.y (mgr.25277) 1669 : cluster 0 pgmap v1890: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:19.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:19 smithi194 bash[92666]: cluster 2023-10-09T05:43:18.447894+0000 osd.5 (osd.5) 480 : cluster 3 failed to encode map e437 with expected crc 2023-10-09T05:43:19.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:19 smithi194 bash[92666]: audit 2023-10-09T05:43:18.451582+0000 mon.a (mon.0) 2195 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:43:19.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:19 smithi194 bash[92666]: cluster 2023-10-09T05:43:18.451764+0000 mon.a (mon.0) 2196 : cluster 0 osdmap e437: 8 total, 8 up, 8 in 2023-10-09T05:43:19.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:19 smithi194 bash[92666]: cluster 2023-10-09T05:43:18.471177+0000 osd.7 (osd.7) 405 : cluster 3 failed to encode map e437 with expected crc 2023-10-09T05:43:19.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:19 smithi194 bash[92666]: cluster 2023-10-09T05:43:18.471282+0000 osd.4 (osd.4) 381 : cluster 3 failed to encode map e437 with expected crc 2023-10-09T05:43:19.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:19 smithi194 bash[92666]: cluster 2023-10-09T05:43:18.471414+0000 osd.6 (osd.6) 434 : cluster 3 failed to encode map e437 with expected crc 2023-10-09T05:43:19.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:19 smithi194 bash[92666]: cluster 2023-10-09T05:43:18.547325+0000 mgr.y (mgr.25277) 1669 : cluster 0 pgmap v1890: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:21.443 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:43:21.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:21 smithi194 bash[92666]: cluster 2023-10-09T05:43:20.548380+0000 mgr.y (mgr.25277) 1670 : cluster 0 pgmap v1891: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:22.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:21 smithi133 bash[102858]: cluster 2023-10-09T05:43:20.548380+0000 mgr.y (mgr.25277) 1670 : cluster 0 pgmap v1891: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:22.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:21 smithi133 bash[105589]: cluster 2023-10-09T05:43:20.548380+0000 mgr.y (mgr.25277) 1670 : cluster 0 pgmap v1891: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:23.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:23 smithi194 bash[92666]: cluster 2023-10-09T05:43:22.549129+0000 mgr.y (mgr.25277) 1671 : cluster 0 pgmap v1892: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:24.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:23 smithi133 bash[105589]: cluster 2023-10-09T05:43:22.549129+0000 mgr.y (mgr.25277) 1671 : cluster 0 pgmap v1892: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:24.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:23 smithi133 bash[102858]: cluster 2023-10-09T05:43:22.549129+0000 mgr.y (mgr.25277) 1671 : cluster 0 pgmap v1892: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:24.236 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:43:24.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:43:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:43:24] "GET /metrics HTTP/1.1" 200 34143 "" "Prometheus/2.43.0" 2023-10-09T05:43:24.825 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:43:24.826 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:43:25.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:25 smithi194 bash[92666]: cluster 2023-10-09T05:43:24.550011+0000 mgr.y (mgr.25277) 1672 : cluster 0 pgmap v1893: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:25.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:25 smithi133 bash[105589]: cluster 2023-10-09T05:43:24.550011+0000 mgr.y (mgr.25277) 1672 : cluster 0 pgmap v1893: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:25.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:25 smithi133 bash[102858]: cluster 2023-10-09T05:43:24.550011+0000 mgr.y (mgr.25277) 1672 : cluster 0 pgmap v1893: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:25.648 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T05:43:27.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:27 smithi194 bash[92666]: cluster 2023-10-09T05:43:26.550895+0000 mgr.y (mgr.25277) 1673 : cluster 0 pgmap v1894: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:28.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:27 smithi133 bash[105589]: cluster 2023-10-09T05:43:26.550895+0000 mgr.y (mgr.25277) 1673 : cluster 0 pgmap v1894: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:28.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:27 smithi133 bash[102858]: cluster 2023-10-09T05:43:26.550895+0000 mgr.y (mgr.25277) 1673 : cluster 0 pgmap v1894: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:28.630 INFO:teuthology.orchestra.run.smithi133.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-10-09T05:43:28.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:28 smithi194 bash[92666]: audit 2023-10-09T05:43:28.120979+0000 mon.c (mon.2) 151 : audit 1 from='client.? 172.21.15.133:0/3610177940' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:43:28.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:28 smithi194 bash[92666]: audit 2023-10-09T05:43:28.121691+0000 mon.a (mon.0) 2197 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:43:28.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:28 smithi194 bash[92666]: audit 2023-10-09T05:43:28.501069+0000 mon.a (mon.0) 2198 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:29.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:28 smithi133 bash[102858]: audit 2023-10-09T05:43:28.120979+0000 mon.c (mon.2) 151 : audit 1 from='client.? 172.21.15.133:0/3610177940' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:43:29.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:28 smithi133 bash[102858]: audit 2023-10-09T05:43:28.121691+0000 mon.a (mon.0) 2197 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:43:29.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:28 smithi133 bash[102858]: audit 2023-10-09T05:43:28.501069+0000 mon.a (mon.0) 2198 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:29.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:28 smithi133 bash[105589]: audit 2023-10-09T05:43:28.120979+0000 mon.c (mon.2) 151 : audit 1 from='client.? 172.21.15.133:0/3610177940' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:43:29.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:28 smithi133 bash[105589]: audit 2023-10-09T05:43:28.121691+0000 mon.a (mon.0) 2197 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:43:29.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:28 smithi133 bash[105589]: audit 2023-10-09T05:43:28.501069+0000 mon.a (mon.0) 2198 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:29.606 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T05:43:29.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:29 smithi194 bash[92666]: cluster 2023-10-09T05:43:28.551488+0000 mgr.y (mgr.25277) 1674 : cluster 0 pgmap v1895: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:29.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:29 smithi194 bash[92666]: cluster 2023-10-09T05:43:28.622118+0000 osd.6 (osd.6) 435 : cluster 3 failed to encode map e438 with expected crc 2023-10-09T05:43:29.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:29 smithi194 bash[92666]: audit 2023-10-09T05:43:28.625507+0000 mon.a (mon.0) 2199 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:43:29.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:29 smithi194 bash[92666]: cluster 2023-10-09T05:43:28.625649+0000 mon.a (mon.0) 2200 : cluster 0 osdmap e438: 8 total, 8 up, 8 in 2023-10-09T05:43:29.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:29 smithi194 bash[92666]: cluster 2023-10-09T05:43:28.750957+0000 osd.4 (osd.4) 382 : cluster 3 failed to encode map e438 with expected crc 2023-10-09T05:43:29.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:29 smithi194 bash[92666]: cluster 2023-10-09T05:43:28.753927+0000 osd.4 (osd.4) 383 : cluster 3 failed to encode map e438 with expected crc 2023-10-09T05:43:30.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:29 smithi133 bash[105589]: cluster 2023-10-09T05:43:28.551488+0000 mgr.y (mgr.25277) 1674 : cluster 0 pgmap v1895: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:30.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:29 smithi133 bash[105589]: cluster 2023-10-09T05:43:28.622118+0000 osd.6 (osd.6) 435 : cluster 3 failed to encode map e438 with expected crc 2023-10-09T05:43:30.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:29 smithi133 bash[105589]: audit 2023-10-09T05:43:28.625507+0000 mon.a (mon.0) 2199 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:43:30.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:29 smithi133 bash[105589]: cluster 2023-10-09T05:43:28.625649+0000 mon.a (mon.0) 2200 : cluster 0 osdmap e438: 8 total, 8 up, 8 in 2023-10-09T05:43:30.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:29 smithi133 bash[105589]: cluster 2023-10-09T05:43:28.750957+0000 osd.4 (osd.4) 382 : cluster 3 failed to encode map e438 with expected crc 2023-10-09T05:43:30.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:29 smithi133 bash[105589]: cluster 2023-10-09T05:43:28.753927+0000 osd.4 (osd.4) 383 : cluster 3 failed to encode map e438 with expected crc 2023-10-09T05:43:30.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:29 smithi133 bash[102858]: cluster 2023-10-09T05:43:28.551488+0000 mgr.y (mgr.25277) 1674 : cluster 0 pgmap v1895: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:30.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:29 smithi133 bash[102858]: cluster 2023-10-09T05:43:28.622118+0000 osd.6 (osd.6) 435 : cluster 3 failed to encode map e438 with expected crc 2023-10-09T05:43:30.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:29 smithi133 bash[102858]: audit 2023-10-09T05:43:28.625507+0000 mon.a (mon.0) 2199 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:43:30.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:29 smithi133 bash[102858]: cluster 2023-10-09T05:43:28.625649+0000 mon.a (mon.0) 2200 : cluster 0 osdmap e438: 8 total, 8 up, 8 in 2023-10-09T05:43:30.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:29 smithi133 bash[102858]: cluster 2023-10-09T05:43:28.750957+0000 osd.4 (osd.4) 382 : cluster 3 failed to encode map e438 with expected crc 2023-10-09T05:43:30.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:29 smithi133 bash[102858]: cluster 2023-10-09T05:43:28.753927+0000 osd.4 (osd.4) 383 : cluster 3 failed to encode map e438 with expected crc 2023-10-09T05:43:30.428 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:43:30.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:30 smithi194 bash[92666]: cluster 2023-10-09T05:43:29.443496+0000 osd.5 (osd.5) 481 : cluster 3 failed to encode map e438 with expected crc 2023-10-09T05:43:30.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:30 smithi194 bash[92666]: cluster 2023-10-09T05:43:29.445802+0000 osd.5 (osd.5) 482 : cluster 3 failed to encode map e438 with expected crc 2023-10-09T05:43:31.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:30 smithi133 bash[105589]: cluster 2023-10-09T05:43:29.443496+0000 osd.5 (osd.5) 481 : cluster 3 failed to encode map e438 with expected crc 2023-10-09T05:43:31.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:30 smithi133 bash[105589]: cluster 2023-10-09T05:43:29.445802+0000 osd.5 (osd.5) 482 : cluster 3 failed to encode map e438 with expected crc 2023-10-09T05:43:31.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:30 smithi133 bash[102858]: cluster 2023-10-09T05:43:29.443496+0000 osd.5 (osd.5) 481 : cluster 3 failed to encode map e438 with expected crc 2023-10-09T05:43:31.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:30 smithi133 bash[102858]: cluster 2023-10-09T05:43:29.445802+0000 osd.5 (osd.5) 482 : cluster 3 failed to encode map e438 with expected crc 2023-10-09T05:43:31.814 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:31 smithi194 bash[92666]: cluster 2023-10-09T05:43:30.552597+0000 mgr.y (mgr.25277) 1675 : cluster 0 pgmap v1897: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:32.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:31 smithi133 bash[102858]: cluster 2023-10-09T05:43:30.552597+0000 mgr.y (mgr.25277) 1675 : cluster 0 pgmap v1897: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:32.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:31 smithi133 bash[105589]: cluster 2023-10-09T05:43:30.552597+0000 mgr.y (mgr.25277) 1675 : cluster 0 pgmap v1897: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:33.435 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:43:33.435 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:43:33.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:33 smithi194 bash[92666]: cluster 2023-10-09T05:43:32.553391+0000 mgr.y (mgr.25277) 1676 : cluster 0 pgmap v1898: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:34.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:33 smithi133 bash[102858]: cluster 2023-10-09T05:43:32.553391+0000 mgr.y (mgr.25277) 1676 : cluster 0 pgmap v1898: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:34.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:33 smithi133 bash[105589]: cluster 2023-10-09T05:43:32.553391+0000 mgr.y (mgr.25277) 1676 : cluster 0 pgmap v1898: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:34.343 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T05:43:34.382 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:43:34.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:43:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:43:34] "GET /metrics HTTP/1.1" 200 34140 "" "Prometheus/2.43.0" 2023-10-09T05:43:35.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:35 smithi133 bash[102858]: cluster 2023-10-09T05:43:34.554211+0000 mgr.y (mgr.25277) 1677 : cluster 0 pgmap v1899: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:35.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:35 smithi133 bash[105589]: cluster 2023-10-09T05:43:34.554211+0000 mgr.y (mgr.25277) 1677 : cluster 0 pgmap v1899: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:35.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:35 smithi194 bash[92666]: cluster 2023-10-09T05:43:34.554211+0000 mgr.y (mgr.25277) 1677 : cluster 0 pgmap v1899: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:37.866 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:37 smithi133 bash[102858]: cluster 2023-10-09T05:43:36.555074+0000 mgr.y (mgr.25277) 1678 : cluster 0 pgmap v1900: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:37.866 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:37 smithi133 bash[105589]: cluster 2023-10-09T05:43:36.555074+0000 mgr.y (mgr.25277) 1678 : cluster 0 pgmap v1900: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:37.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:37 smithi194 bash[92666]: cluster 2023-10-09T05:43:36.555074+0000 mgr.y (mgr.25277) 1678 : cluster 0 pgmap v1900: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:38.630 INFO:teuthology.orchestra.run.smithi133.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T05:43:38.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:38 smithi194 bash[92666]: audit 2023-10-09T05:43:38.274249+0000 mon.c (mon.2) 152 : audit 1 from='client.? 172.21.15.133:0/1751851645' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:43:38.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:38 smithi194 bash[92666]: audit 2023-10-09T05:43:38.274796+0000 mon.a (mon.0) 2201 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:43:39.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:38 smithi133 bash[102858]: audit 2023-10-09T05:43:38.274249+0000 mon.c (mon.2) 152 : audit 1 from='client.? 172.21.15.133:0/1751851645' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:43:39.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:38 smithi133 bash[102858]: audit 2023-10-09T05:43:38.274796+0000 mon.a (mon.0) 2201 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:43:39.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:38 smithi133 bash[105589]: audit 2023-10-09T05:43:38.274249+0000 mon.c (mon.2) 152 : audit 1 from='client.? 172.21.15.133:0/1751851645' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:43:39.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:38 smithi133 bash[105589]: audit 2023-10-09T05:43:38.274796+0000 mon.a (mon.0) 2201 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:43:39.121 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:43:39.751 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-10-09T05:43:39.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:39 smithi194 bash[92666]: cluster 2023-10-09T05:43:38.555737+0000 mgr.y (mgr.25277) 1679 : cluster 0 pgmap v1901: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:39.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:39 smithi194 bash[92666]: audit 2023-10-09T05:43:38.625186+0000 mon.a (mon.0) 2202 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:43:39.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:39 smithi194 bash[92666]: cluster 2023-10-09T05:43:38.625310+0000 mon.a (mon.0) 2203 : cluster 0 osdmap e439: 8 total, 8 up, 8 in 2023-10-09T05:43:39.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:39 smithi194 bash[92666]: cluster 2023-10-09T05:43:39.074110+0000 osd.6 (osd.6) 436 : cluster 3 failed to encode map e439 with expected crc 2023-10-09T05:43:39.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:39 smithi194 bash[92666]: cluster 2023-10-09T05:43:39.135492+0000 osd.5 (osd.5) 483 : cluster 3 failed to encode map e439 with expected crc 2023-10-09T05:43:39.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:39 smithi194 bash[92666]: cluster 2023-10-09T05:43:39.135540+0000 osd.4 (osd.4) 384 : cluster 3 failed to encode map e439 with expected crc 2023-10-09T05:43:39.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:39 smithi194 bash[92666]: cluster 2023-10-09T05:43:39.135685+0000 osd.7 (osd.7) 406 : cluster 3 failed to encode map e439 with expected crc 2023-10-09T05:43:40.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:39 smithi133 bash[102858]: cluster 2023-10-09T05:43:38.555737+0000 mgr.y (mgr.25277) 1679 : cluster 0 pgmap v1901: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:40.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:39 smithi133 bash[102858]: audit 2023-10-09T05:43:38.625186+0000 mon.a (mon.0) 2202 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:43:40.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:39 smithi133 bash[102858]: cluster 2023-10-09T05:43:38.625310+0000 mon.a (mon.0) 2203 : cluster 0 osdmap e439: 8 total, 8 up, 8 in 2023-10-09T05:43:40.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:39 smithi133 bash[102858]: cluster 2023-10-09T05:43:39.074110+0000 osd.6 (osd.6) 436 : cluster 3 failed to encode map e439 with expected crc 2023-10-09T05:43:40.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:39 smithi133 bash[102858]: cluster 2023-10-09T05:43:39.135492+0000 osd.5 (osd.5) 483 : cluster 3 failed to encode map e439 with expected crc 2023-10-09T05:43:40.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:39 smithi133 bash[102858]: cluster 2023-10-09T05:43:39.135540+0000 osd.4 (osd.4) 384 : cluster 3 failed to encode map e439 with expected crc 2023-10-09T05:43:40.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:39 smithi133 bash[102858]: cluster 2023-10-09T05:43:39.135685+0000 osd.7 (osd.7) 406 : cluster 3 failed to encode map e439 with expected crc 2023-10-09T05:43:40.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:39 smithi133 bash[105589]: cluster 2023-10-09T05:43:38.555737+0000 mgr.y (mgr.25277) 1679 : cluster 0 pgmap v1901: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:40.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:39 smithi133 bash[105589]: audit 2023-10-09T05:43:38.625186+0000 mon.a (mon.0) 2202 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:43:40.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:39 smithi133 bash[105589]: cluster 2023-10-09T05:43:38.625310+0000 mon.a (mon.0) 2203 : cluster 0 osdmap e439: 8 total, 8 up, 8 in 2023-10-09T05:43:40.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:39 smithi133 bash[105589]: cluster 2023-10-09T05:43:39.074110+0000 osd.6 (osd.6) 436 : cluster 3 failed to encode map e439 with expected crc 2023-10-09T05:43:40.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:39 smithi133 bash[105589]: cluster 2023-10-09T05:43:39.135492+0000 osd.5 (osd.5) 483 : cluster 3 failed to encode map e439 with expected crc 2023-10-09T05:43:40.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:39 smithi133 bash[105589]: cluster 2023-10-09T05:43:39.135540+0000 osd.4 (osd.4) 384 : cluster 3 failed to encode map e439 with expected crc 2023-10-09T05:43:40.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:39 smithi133 bash[105589]: cluster 2023-10-09T05:43:39.135685+0000 osd.7 (osd.7) 406 : cluster 3 failed to encode map e439 with expected crc 2023-10-09T05:43:41.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:41 smithi194 bash[92666]: cluster 2023-10-09T05:43:40.556867+0000 mgr.y (mgr.25277) 1680 : cluster 0 pgmap v1903: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:42.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:41 smithi133 bash[105589]: cluster 2023-10-09T05:43:40.556867+0000 mgr.y (mgr.25277) 1680 : cluster 0 pgmap v1903: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:42.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:41 smithi133 bash[102858]: cluster 2023-10-09T05:43:40.556867+0000 mgr.y (mgr.25277) 1680 : cluster 0 pgmap v1903: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:42.829 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:43:42.829 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:43:43.669 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T05:43:43.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:43 smithi194 bash[92666]: cluster 2023-10-09T05:43:42.557797+0000 mgr.y (mgr.25277) 1681 : cluster 0 pgmap v1904: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:43.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:43 smithi194 bash[92666]: audit 2023-10-09T05:43:43.501407+0000 mon.a (mon.0) 2204 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:44.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:43 smithi133 bash[102858]: cluster 2023-10-09T05:43:42.557797+0000 mgr.y (mgr.25277) 1681 : cluster 0 pgmap v1904: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:44.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:43 smithi133 bash[102858]: audit 2023-10-09T05:43:43.501407+0000 mon.a (mon.0) 2204 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:44.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:43 smithi133 bash[105589]: cluster 2023-10-09T05:43:42.557797+0000 mgr.y (mgr.25277) 1681 : cluster 0 pgmap v1904: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:44.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:43 smithi133 bash[105589]: audit 2023-10-09T05:43:43.501407+0000 mon.a (mon.0) 2204 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:44.538 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:43:44.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:43:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:43:44] "GET /metrics HTTP/1.1" 200 34143 "" "Prometheus/2.43.0" 2023-10-09T05:43:45.035 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:45 smithi133 bash[105589]: cluster 2023-10-09T05:43:44.558408+0000 mgr.y (mgr.25277) 1682 : cluster 0 pgmap v1905: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:45.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:45 smithi133 bash[102858]: cluster 2023-10-09T05:43:44.558408+0000 mgr.y (mgr.25277) 1682 : cluster 0 pgmap v1905: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:45.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:45 smithi194 bash[92666]: cluster 2023-10-09T05:43:44.558408+0000 mgr.y (mgr.25277) 1682 : cluster 0 pgmap v1905: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:47.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:47 smithi194 bash[92666]: cluster 2023-10-09T05:43:46.559277+0000 mgr.y (mgr.25277) 1683 : cluster 0 pgmap v1906: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:48.037 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:47 smithi133 bash[105589]: cluster 2023-10-09T05:43:46.559277+0000 mgr.y (mgr.25277) 1683 : cluster 0 pgmap v1906: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:48.037 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:47 smithi133 bash[102858]: cluster 2023-10-09T05:43:46.559277+0000 mgr.y (mgr.25277) 1683 : cluster 0 pgmap v1906: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:48.447 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:43:48.636 INFO:teuthology.orchestra.run.smithi133.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-10-09T05:43:48.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:48 smithi194 bash[92666]: audit 2023-10-09T05:43:48.512337+0000 mon.c (mon.2) 153 : audit 1 from='client.? 172.21.15.133:0/3990414854' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:43:48.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:48 smithi194 bash[92666]: audit 2023-10-09T05:43:48.513164+0000 mon.a (mon.0) 2205 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:43:49.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:48 smithi133 bash[105589]: audit 2023-10-09T05:43:48.512337+0000 mon.c (mon.2) 153 : audit 1 from='client.? 172.21.15.133:0/3990414854' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:43:49.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:48 smithi133 bash[105589]: audit 2023-10-09T05:43:48.513164+0000 mon.a (mon.0) 2205 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:43:49.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:48 smithi133 bash[102858]: audit 2023-10-09T05:43:48.512337+0000 mon.c (mon.2) 153 : audit 1 from='client.? 172.21.15.133:0/3990414854' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:43:49.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:48 smithi133 bash[102858]: audit 2023-10-09T05:43:48.513164+0000 mon.a (mon.0) 2205 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:43:49.657 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T05:43:49.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:49 smithi194 bash[92666]: cluster 2023-10-09T05:43:48.560004+0000 mgr.y (mgr.25277) 1684 : cluster 0 pgmap v1907: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:49.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:49 smithi194 bash[92666]: cluster 2023-10-09T05:43:48.627312+0000 osd.5 (osd.5) 484 : cluster 3 failed to encode map e440 with expected crc 2023-10-09T05:43:49.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:49 smithi194 bash[92666]: audit 2023-10-09T05:43:48.629080+0000 mon.a (mon.0) 2206 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:43:49.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:49 smithi194 bash[92666]: cluster 2023-10-09T05:43:48.629288+0000 mon.a (mon.0) 2207 : cluster 0 osdmap e440: 8 total, 8 up, 8 in 2023-10-09T05:43:49.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:49 smithi194 bash[92666]: cluster 2023-10-09T05:43:48.786067+0000 osd.7 (osd.7) 407 : cluster 3 failed to encode map e440 with expected crc 2023-10-09T05:43:50.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:49 smithi133 bash[105589]: cluster 2023-10-09T05:43:48.560004+0000 mgr.y (mgr.25277) 1684 : cluster 0 pgmap v1907: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:50.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:49 smithi133 bash[105589]: cluster 2023-10-09T05:43:48.627312+0000 osd.5 (osd.5) 484 : cluster 3 failed to encode map e440 with expected crc 2023-10-09T05:43:50.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:49 smithi133 bash[105589]: audit 2023-10-09T05:43:48.629080+0000 mon.a (mon.0) 2206 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:43:50.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:49 smithi133 bash[105589]: cluster 2023-10-09T05:43:48.629288+0000 mon.a (mon.0) 2207 : cluster 0 osdmap e440: 8 total, 8 up, 8 in 2023-10-09T05:43:50.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:49 smithi133 bash[105589]: cluster 2023-10-09T05:43:48.786067+0000 osd.7 (osd.7) 407 : cluster 3 failed to encode map e440 with expected crc 2023-10-09T05:43:50.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:49 smithi133 bash[102858]: cluster 2023-10-09T05:43:48.560004+0000 mgr.y (mgr.25277) 1684 : cluster 0 pgmap v1907: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:50.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:49 smithi133 bash[102858]: cluster 2023-10-09T05:43:48.627312+0000 osd.5 (osd.5) 484 : cluster 3 failed to encode map e440 with expected crc 2023-10-09T05:43:50.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:49 smithi133 bash[102858]: audit 2023-10-09T05:43:48.629080+0000 mon.a (mon.0) 2206 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:43:50.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:49 smithi133 bash[102858]: cluster 2023-10-09T05:43:48.629288+0000 mon.a (mon.0) 2207 : cluster 0 osdmap e440: 8 total, 8 up, 8 in 2023-10-09T05:43:50.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:49 smithi133 bash[102858]: cluster 2023-10-09T05:43:48.786067+0000 osd.7 (osd.7) 407 : cluster 3 failed to encode map e440 with expected crc 2023-10-09T05:43:51.778 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:43:51.779 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:43:51.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:51 smithi194 bash[92666]: cluster 2023-10-09T05:43:50.561006+0000 mgr.y (mgr.25277) 1685 : cluster 0 pgmap v1909: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:52.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:51 smithi133 bash[105589]: cluster 2023-10-09T05:43:50.561006+0000 mgr.y (mgr.25277) 1685 : cluster 0 pgmap v1909: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:52.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:51 smithi133 bash[102858]: cluster 2023-10-09T05:43:50.561006+0000 mgr.y (mgr.25277) 1685 : cluster 0 pgmap v1909: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:52.577 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T05:43:54.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:53 smithi133 bash[105589]: cluster 2023-10-09T05:43:52.561769+0000 mgr.y (mgr.25277) 1686 : cluster 0 pgmap v1910: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:54.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:53 smithi133 bash[102858]: cluster 2023-10-09T05:43:52.561769+0000 mgr.y (mgr.25277) 1686 : cluster 0 pgmap v1910: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:54.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:53 smithi194 bash[92666]: cluster 2023-10-09T05:43:52.561769+0000 mgr.y (mgr.25277) 1686 : cluster 0 pgmap v1910: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:54.433 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:43:54.547 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:43:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:43:54] "GET /metrics HTTP/1.1" 200 34143 "" "Prometheus/2.43.0" 2023-10-09T05:43:55.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:55 smithi133 bash[105589]: cluster 2023-10-09T05:43:54.562542+0000 mgr.y (mgr.25277) 1687 : cluster 0 pgmap v1911: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:55.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:55 smithi133 bash[102858]: cluster 2023-10-09T05:43:54.562542+0000 mgr.y (mgr.25277) 1687 : cluster 0 pgmap v1911: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:55.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:55 smithi194 bash[92666]: cluster 2023-10-09T05:43:54.562542+0000 mgr.y (mgr.25277) 1687 : cluster 0 pgmap v1911: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:57.353 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:43:57.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:57 smithi194 bash[92666]: cluster 2023-10-09T05:43:56.563397+0000 mgr.y (mgr.25277) 1688 : cluster 0 pgmap v1912: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:57.914 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:57 smithi133 bash[102858]: cluster 2023-10-09T05:43:56.563397+0000 mgr.y (mgr.25277) 1688 : cluster 0 pgmap v1912: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:57.914 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:57 smithi133 bash[105589]: cluster 2023-10-09T05:43:56.563397+0000 mgr.y (mgr.25277) 1688 : cluster 0 pgmap v1912: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:58.632 INFO:teuthology.orchestra.run.smithi133.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T05:43:58.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:58 smithi194 bash[92666]: audit 2023-10-09T05:43:58.310116+0000 mon.c (mon.2) 154 : audit 1 from='client.? 172.21.15.133:0/1937686276' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:43:58.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:58 smithi194 bash[92666]: audit 2023-10-09T05:43:58.310673+0000 mon.a (mon.0) 2208 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:43:58.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:58 smithi194 bash[92666]: audit 2023-10-09T05:43:58.501678+0000 mon.a (mon.0) 2209 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:59.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:58 smithi133 bash[105589]: audit 2023-10-09T05:43:58.310116+0000 mon.c (mon.2) 154 : audit 1 from='client.? 172.21.15.133:0/1937686276' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:43:59.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:58 smithi133 bash[105589]: audit 2023-10-09T05:43:58.310673+0000 mon.a (mon.0) 2208 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:43:59.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:58 smithi133 bash[105589]: audit 2023-10-09T05:43:58.501678+0000 mon.a (mon.0) 2209 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:59.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:58 smithi133 bash[102858]: audit 2023-10-09T05:43:58.310116+0000 mon.c (mon.2) 154 : audit 1 from='client.? 172.21.15.133:0/1937686276' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:43:59.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:58 smithi133 bash[102858]: audit 2023-10-09T05:43:58.310673+0000 mon.a (mon.0) 2208 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:43:59.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:58 smithi133 bash[102858]: audit 2023-10-09T05:43:58.501678+0000 mon.a (mon.0) 2209 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:59.587 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 4 1 2023-10-09T05:43:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:59 smithi194 bash[92666]: cluster 2023-10-09T05:43:58.563982+0000 mgr.y (mgr.25277) 1689 : cluster 0 pgmap v1913: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:43:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:59 smithi194 bash[92666]: audit 2023-10-09T05:43:58.626836+0000 mon.a (mon.0) 2210 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:43:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:59 smithi194 bash[92666]: cluster 2023-10-09T05:43:58.626997+0000 mon.a (mon.0) 2211 : cluster 0 osdmap e441: 8 total, 8 up, 8 in 2023-10-09T05:43:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:59 smithi194 bash[92666]: cluster 2023-10-09T05:43:58.649468+0000 osd.5 (osd.5) 485 : cluster 3 failed to encode map e441 with expected crc 2023-10-09T05:43:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:59 smithi194 bash[92666]: cluster 2023-10-09T05:43:58.652118+0000 osd.5 (osd.5) 486 : cluster 3 failed to encode map e441 with expected crc 2023-10-09T05:43:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:59 smithi194 bash[92666]: cluster 2023-10-09T05:43:58.705142+0000 osd.7 (osd.7) 408 : cluster 3 failed to encode map e441 with expected crc 2023-10-09T05:43:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:59 smithi194 bash[92666]: cluster 2023-10-09T05:43:58.707545+0000 osd.7 (osd.7) 409 : cluster 3 failed to encode map e441 with expected crc 2023-10-09T05:43:59.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:59 smithi194 bash[92666]: cluster 2023-10-09T05:43:58.779516+0000 osd.4 (osd.4) 385 : cluster 3 failed to encode map e441 with expected crc 2023-10-09T05:43:59.887 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:43:59 smithi194 bash[92666]: cluster 2023-10-09T05:43:58.779644+0000 osd.6 (osd.6) 437 : cluster 3 failed to encode map e441 with expected crc 2023-10-09T05:44:00.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:59 smithi133 bash[105589]: cluster 2023-10-09T05:43:58.563982+0000 mgr.y (mgr.25277) 1689 : cluster 0 pgmap v1913: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:00.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:59 smithi133 bash[105589]: audit 2023-10-09T05:43:58.626836+0000 mon.a (mon.0) 2210 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:44:00.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:59 smithi133 bash[105589]: cluster 2023-10-09T05:43:58.626997+0000 mon.a (mon.0) 2211 : cluster 0 osdmap e441: 8 total, 8 up, 8 in 2023-10-09T05:44:00.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:59 smithi133 bash[105589]: cluster 2023-10-09T05:43:58.649468+0000 osd.5 (osd.5) 485 : cluster 3 failed to encode map e441 with expected crc 2023-10-09T05:44:00.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:59 smithi133 bash[105589]: cluster 2023-10-09T05:43:58.652118+0000 osd.5 (osd.5) 486 : cluster 3 failed to encode map e441 with expected crc 2023-10-09T05:44:00.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:59 smithi133 bash[105589]: cluster 2023-10-09T05:43:58.705142+0000 osd.7 (osd.7) 408 : cluster 3 failed to encode map e441 with expected crc 2023-10-09T05:44:00.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:59 smithi133 bash[105589]: cluster 2023-10-09T05:43:58.707545+0000 osd.7 (osd.7) 409 : cluster 3 failed to encode map e441 with expected crc 2023-10-09T05:44:00.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:59 smithi133 bash[105589]: cluster 2023-10-09T05:43:58.779516+0000 osd.4 (osd.4) 385 : cluster 3 failed to encode map e441 with expected crc 2023-10-09T05:44:00.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:43:59 smithi133 bash[105589]: cluster 2023-10-09T05:43:58.779644+0000 osd.6 (osd.6) 437 : cluster 3 failed to encode map e441 with expected crc 2023-10-09T05:44:00.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:59 smithi133 bash[102858]: cluster 2023-10-09T05:43:58.563982+0000 mgr.y (mgr.25277) 1689 : cluster 0 pgmap v1913: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:00.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:59 smithi133 bash[102858]: audit 2023-10-09T05:43:58.626836+0000 mon.a (mon.0) 2210 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:44:00.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:59 smithi133 bash[102858]: cluster 2023-10-09T05:43:58.626997+0000 mon.a (mon.0) 2211 : cluster 0 osdmap e441: 8 total, 8 up, 8 in 2023-10-09T05:44:00.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:59 smithi133 bash[102858]: cluster 2023-10-09T05:43:58.649468+0000 osd.5 (osd.5) 485 : cluster 3 failed to encode map e441 with expected crc 2023-10-09T05:44:00.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:59 smithi133 bash[102858]: cluster 2023-10-09T05:43:58.652118+0000 osd.5 (osd.5) 486 : cluster 3 failed to encode map e441 with expected crc 2023-10-09T05:44:00.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:59 smithi133 bash[102858]: cluster 2023-10-09T05:43:58.705142+0000 osd.7 (osd.7) 408 : cluster 3 failed to encode map e441 with expected crc 2023-10-09T05:44:00.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:59 smithi133 bash[102858]: cluster 2023-10-09T05:43:58.707545+0000 osd.7 (osd.7) 409 : cluster 3 failed to encode map e441 with expected crc 2023-10-09T05:44:00.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:59 smithi133 bash[102858]: cluster 2023-10-09T05:43:58.779516+0000 osd.4 (osd.4) 385 : cluster 3 failed to encode map e441 with expected crc 2023-10-09T05:44:00.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:43:59 smithi133 bash[102858]: cluster 2023-10-09T05:43:58.779644+0000 osd.6 (osd.6) 437 : cluster 3 failed to encode map e441 with expected crc 2023-10-09T05:44:00.917 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:44:00.918 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:44:01.864 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T05:44:02.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:01 smithi133 bash[105589]: cluster 2023-10-09T05:44:00.565094+0000 mgr.y (mgr.25277) 1690 : cluster 0 pgmap v1915: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:02.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:01 smithi133 bash[102858]: cluster 2023-10-09T05:44:00.565094+0000 mgr.y (mgr.25277) 1690 : cluster 0 pgmap v1915: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:02.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:01 smithi194 bash[92666]: cluster 2023-10-09T05:44:00.565094+0000 mgr.y (mgr.25277) 1690 : cluster 0 pgmap v1915: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:04.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:03 smithi133 bash[102858]: cluster 2023-10-09T05:44:02.565922+0000 mgr.y (mgr.25277) 1691 : cluster 0 pgmap v1916: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:04.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:03 smithi133 bash[105589]: cluster 2023-10-09T05:44:02.565922+0000 mgr.y (mgr.25277) 1691 : cluster 0 pgmap v1916: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:04.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:03 smithi194 bash[92666]: cluster 2023-10-09T05:44:02.565922+0000 mgr.y (mgr.25277) 1691 : cluster 0 pgmap v1916: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:04.371 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:04.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:44:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:44:04] "GET /metrics HTTP/1.1" 200 34140 "" "Prometheus/2.43.0" 2023-10-09T05:44:05.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:05 smithi133 bash[105589]: cluster 2023-10-09T05:44:04.566632+0000 mgr.y (mgr.25277) 1692 : cluster 0 pgmap v1917: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:05.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:05 smithi133 bash[102858]: cluster 2023-10-09T05:44:04.566632+0000 mgr.y (mgr.25277) 1692 : cluster 0 pgmap v1917: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:05.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:05 smithi194 bash[92666]: cluster 2023-10-09T05:44:04.566632+0000 mgr.y (mgr.25277) 1692 : cluster 0 pgmap v1917: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:06.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:06 smithi194 bash[92666]: audit 2023-10-09T05:44:06.322125+0000 mon.a (mon.0) 2212 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:44:06.642 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:44:06.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:06 smithi133 bash[105589]: audit 2023-10-09T05:44:06.322125+0000 mon.a (mon.0) 2212 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:44:06.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:06 smithi133 bash[102858]: audit 2023-10-09T05:44:06.322125+0000 mon.a (mon.0) 2212 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:44:07.740 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:07 smithi194 bash[92666]: cluster 2023-10-09T05:44:06.567382+0000 mgr.y (mgr.25277) 1693 : cluster 0 pgmap v1918: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:07.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:07 smithi133 bash[102858]: cluster 2023-10-09T05:44:06.567382+0000 mgr.y (mgr.25277) 1693 : cluster 0 pgmap v1918: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:07.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:07 smithi133 bash[105589]: cluster 2023-10-09T05:44:06.567382+0000 mgr.y (mgr.25277) 1693 : cluster 0 pgmap v1918: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:08.487 INFO:teuthology.orchestra.run.smithi133.stderr:set osd.4 primary-affinity to 1 (8655362) 2023-10-09T05:44:08.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:08 smithi133 bash[105589]: audit 2023-10-09T05:44:08.411210+0000 mon.b (mon.1) 35 : audit 1 from='client.? 172.21.15.133:0/3669430470' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:44:08.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:08 smithi133 bash[105589]: audit 2023-10-09T05:44:08.411930+0000 mon.a (mon.0) 2213 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:44:08.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:08 smithi133 bash[102858]: audit 2023-10-09T05:44:08.411210+0000 mon.b (mon.1) 35 : audit 1 from='client.? 172.21.15.133:0/3669430470' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:44:08.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:08 smithi133 bash[102858]: audit 2023-10-09T05:44:08.411930+0000 mon.a (mon.0) 2213 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:44:08.844 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:08 smithi194 bash[92666]: audit 2023-10-09T05:44:08.411210+0000 mon.b (mon.1) 35 : audit 1 from='client.? 172.21.15.133:0/3669430470' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:44:08.844 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:08 smithi194 bash[92666]: audit 2023-10-09T05:44:08.411930+0000 mon.a (mon.0) 2213 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:44:09.503 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T05:44:09.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:09 smithi133 bash[105589]: audit 2023-10-09T05:44:08.481767+0000 mon.a (mon.0) 2214 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:44:09.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:09 smithi133 bash[105589]: cluster 2023-10-09T05:44:08.481869+0000 mon.a (mon.0) 2215 : cluster 0 osdmap e442: 8 total, 8 up, 8 in 2023-10-09T05:44:09.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:09 smithi133 bash[105589]: cluster 2023-10-09T05:44:08.568056+0000 mgr.y (mgr.25277) 1694 : cluster 0 pgmap v1920: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:09.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:09 smithi133 bash[105589]: cluster 2023-10-09T05:44:09.052648+0000 osd.5 (osd.5) 487 : cluster 3 failed to encode map e442 with expected crc 2023-10-09T05:44:09.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:09 smithi133 bash[105589]: cluster 2023-10-09T05:44:09.055635+0000 osd.5 (osd.5) 488 : cluster 3 failed to encode map e442 with expected crc 2023-10-09T05:44:09.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:09 smithi133 bash[102858]: audit 2023-10-09T05:44:08.481767+0000 mon.a (mon.0) 2214 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:44:09.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:09 smithi133 bash[102858]: cluster 2023-10-09T05:44:08.481869+0000 mon.a (mon.0) 2215 : cluster 0 osdmap e442: 8 total, 8 up, 8 in 2023-10-09T05:44:09.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:09 smithi133 bash[102858]: cluster 2023-10-09T05:44:08.568056+0000 mgr.y (mgr.25277) 1694 : cluster 0 pgmap v1920: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:09.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:09 smithi133 bash[102858]: cluster 2023-10-09T05:44:09.052648+0000 osd.5 (osd.5) 487 : cluster 3 failed to encode map e442 with expected crc 2023-10-09T05:44:09.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:09 smithi133 bash[102858]: cluster 2023-10-09T05:44:09.055635+0000 osd.5 (osd.5) 488 : cluster 3 failed to encode map e442 with expected crc 2023-10-09T05:44:09.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:09 smithi194 bash[92666]: audit 2023-10-09T05:44:08.481767+0000 mon.a (mon.0) 2214 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:44:09.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:09 smithi194 bash[92666]: cluster 2023-10-09T05:44:08.481869+0000 mon.a (mon.0) 2215 : cluster 0 osdmap e442: 8 total, 8 up, 8 in 2023-10-09T05:44:09.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:09 smithi194 bash[92666]: cluster 2023-10-09T05:44:08.568056+0000 mgr.y (mgr.25277) 1694 : cluster 0 pgmap v1920: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:09.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:09 smithi194 bash[92666]: cluster 2023-10-09T05:44:09.052648+0000 osd.5 (osd.5) 487 : cluster 3 failed to encode map e442 with expected crc 2023-10-09T05:44:09.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:09 smithi194 bash[92666]: cluster 2023-10-09T05:44:09.055635+0000 osd.5 (osd.5) 488 : cluster 3 failed to encode map e442 with expected crc 2023-10-09T05:44:10.165 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:44:10.165 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:44:10.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:10 smithi133 bash[102858]: cluster 2023-10-09T05:44:09.372327+0000 osd.6 (osd.6) 438 : cluster 3 failed to encode map e442 with expected crc 2023-10-09T05:44:10.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:10 smithi133 bash[102858]: cluster 2023-10-09T05:44:09.373387+0000 osd.6 (osd.6) 439 : cluster 3 failed to encode map e442 with expected crc 2023-10-09T05:44:10.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:10 smithi133 bash[105589]: cluster 2023-10-09T05:44:09.372327+0000 osd.6 (osd.6) 438 : cluster 3 failed to encode map e442 with expected crc 2023-10-09T05:44:10.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:10 smithi133 bash[105589]: cluster 2023-10-09T05:44:09.373387+0000 osd.6 (osd.6) 439 : cluster 3 failed to encode map e442 with expected crc 2023-10-09T05:44:10.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:10 smithi194 bash[92666]: cluster 2023-10-09T05:44:09.372327+0000 osd.6 (osd.6) 438 : cluster 3 failed to encode map e442 with expected crc 2023-10-09T05:44:10.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:10 smithi194 bash[92666]: cluster 2023-10-09T05:44:09.373387+0000 osd.6 (osd.6) 439 : cluster 3 failed to encode map e442 with expected crc 2023-10-09T05:44:11.127 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T05:44:11.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:11 smithi133 bash[105589]: cluster 2023-10-09T05:44:10.569098+0000 mgr.y (mgr.25277) 1695 : cluster 0 pgmap v1921: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:11.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:11 smithi133 bash[105589]: cluster 2023-10-09T05:44:10.782153+0000 osd.7 (osd.7) 410 : cluster 3 failed to encode map e442 with expected crc 2023-10-09T05:44:11.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:11 smithi133 bash[102858]: cluster 2023-10-09T05:44:10.569098+0000 mgr.y (mgr.25277) 1695 : cluster 0 pgmap v1921: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:11.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:11 smithi133 bash[102858]: cluster 2023-10-09T05:44:10.782153+0000 osd.7 (osd.7) 410 : cluster 3 failed to encode map e442 with expected crc 2023-10-09T05:44:11.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:11 smithi194 bash[92666]: cluster 2023-10-09T05:44:10.569098+0000 mgr.y (mgr.25277) 1695 : cluster 0 pgmap v1921: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:11.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:11 smithi194 bash[92666]: cluster 2023-10-09T05:44:10.782153+0000 osd.7 (osd.7) 410 : cluster 3 failed to encode map e442 with expected crc 2023-10-09T05:44:13.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:13 smithi194 bash[92666]: cluster 2023-10-09T05:44:12.569842+0000 mgr.y (mgr.25277) 1696 : cluster 0 pgmap v1922: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:13.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:13 smithi194 bash[92666]: audit 2023-10-09T05:44:13.217415+0000 mon.a (mon.0) 2216 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:44:13.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:13 smithi194 bash[92666]: audit 2023-10-09T05:44:13.223837+0000 mon.a (mon.0) 2217 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:44:13.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:13 smithi194 bash[92666]: audit 2023-10-09T05:44:13.502074+0000 mon.a (mon.0) 2218 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:14.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:13 smithi133 bash[102858]: cluster 2023-10-09T05:44:12.569842+0000 mgr.y (mgr.25277) 1696 : cluster 0 pgmap v1922: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:14.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:13 smithi133 bash[102858]: audit 2023-10-09T05:44:13.217415+0000 mon.a (mon.0) 2216 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:44:14.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:13 smithi133 bash[102858]: audit 2023-10-09T05:44:13.223837+0000 mon.a (mon.0) 2217 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:44:14.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:13 smithi133 bash[102858]: audit 2023-10-09T05:44:13.502074+0000 mon.a (mon.0) 2218 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:14.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:13 smithi133 bash[105589]: cluster 2023-10-09T05:44:12.569842+0000 mgr.y (mgr.25277) 1696 : cluster 0 pgmap v1922: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:14.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:13 smithi133 bash[105589]: audit 2023-10-09T05:44:13.217415+0000 mon.a (mon.0) 2216 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:44:14.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:13 smithi133 bash[105589]: audit 2023-10-09T05:44:13.223837+0000 mon.a (mon.0) 2217 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:44:14.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:13 smithi133 bash[105589]: audit 2023-10-09T05:44:13.502074+0000 mon.a (mon.0) 2218 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:14.057 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:14.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:44:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:44:14] "GET /metrics HTTP/1.1" 200 34139 "" "Prometheus/2.43.0" 2023-10-09T05:44:15.478 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:15 smithi133 bash[105589]: audit 2023-10-09T05:44:14.170098+0000 mon.a (mon.0) 2219 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:44:15.478 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:15 smithi133 bash[105589]: audit 2023-10-09T05:44:14.177826+0000 mon.a (mon.0) 2220 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:44:15.479 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:15 smithi133 bash[105589]: cluster 2023-10-09T05:44:14.570468+0000 mgr.y (mgr.25277) 1697 : cluster 0 pgmap v1923: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:15.479 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:15 smithi133 bash[105589]: audit 2023-10-09T05:44:14.646391+0000 mon.a (mon.0) 2221 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:44:15.479 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:15 smithi133 bash[105589]: audit 2023-10-09T05:44:14.648084+0000 mon.a (mon.0) 2222 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:44:15.479 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:15 smithi133 bash[105589]: audit 2023-10-09T05:44:14.655978+0000 mon.a (mon.0) 2223 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:44:15.479 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:15 smithi133 bash[102858]: audit 2023-10-09T05:44:14.170098+0000 mon.a (mon.0) 2219 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:44:15.479 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:15 smithi133 bash[102858]: audit 2023-10-09T05:44:14.177826+0000 mon.a (mon.0) 2220 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:44:15.479 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:15 smithi133 bash[102858]: cluster 2023-10-09T05:44:14.570468+0000 mgr.y (mgr.25277) 1697 : cluster 0 pgmap v1923: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:15.479 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:15 smithi133 bash[102858]: audit 2023-10-09T05:44:14.646391+0000 mon.a (mon.0) 2221 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:44:15.479 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:15 smithi133 bash[102858]: audit 2023-10-09T05:44:14.648084+0000 mon.a (mon.0) 2222 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:44:15.480 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:15 smithi133 bash[102858]: audit 2023-10-09T05:44:14.655978+0000 mon.a (mon.0) 2223 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:44:15.635 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:15 smithi194 bash[92666]: audit 2023-10-09T05:44:14.170098+0000 mon.a (mon.0) 2219 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:44:15.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:15 smithi194 bash[92666]: audit 2023-10-09T05:44:14.177826+0000 mon.a (mon.0) 2220 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:44:15.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:15 smithi194 bash[92666]: cluster 2023-10-09T05:44:14.570468+0000 mgr.y (mgr.25277) 1697 : cluster 0 pgmap v1923: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:15.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:15 smithi194 bash[92666]: audit 2023-10-09T05:44:14.646391+0000 mon.a (mon.0) 2221 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:44:15.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:15 smithi194 bash[92666]: audit 2023-10-09T05:44:14.648084+0000 mon.a (mon.0) 2222 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:44:15.636 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:15 smithi194 bash[92666]: audit 2023-10-09T05:44:14.655978+0000 mon.a (mon.0) 2223 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:44:15.677 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:44:17.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:17 smithi194 bash[92666]: cluster 2023-10-09T05:44:16.570950+0000 mgr.y (mgr.25277) 1698 : cluster 0 pgmap v1924: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:18.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:17 smithi133 bash[102858]: cluster 2023-10-09T05:44:16.570950+0000 mgr.y (mgr.25277) 1698 : cluster 0 pgmap v1924: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:18.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:17 smithi133 bash[105589]: cluster 2023-10-09T05:44:16.570950+0000 mgr.y (mgr.25277) 1698 : cluster 0 pgmap v1924: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:18.644 INFO:teuthology.orchestra.run.smithi133.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T05:44:18.906 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:18 smithi194 bash[92666]: audit 2023-10-09T05:44:18.046439+0000 mon.a (mon.0) 2224 : audit 1 from='client.? 172.21.15.133:0/2450280789' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:44:19.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:18 smithi133 bash[105589]: audit 2023-10-09T05:44:18.046439+0000 mon.a (mon.0) 2224 : audit 1 from='client.? 172.21.15.133:0/2450280789' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:44:19.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:18 smithi133 bash[102858]: audit 2023-10-09T05:44:18.046439+0000 mon.a (mon.0) 2224 : audit 1 from='client.? 172.21.15.133:0/2450280789' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:44:19.189 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:44:19.189 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:44:19.620 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 5 1 2023-10-09T05:44:19.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:19 smithi194 bash[92666]: cluster 2023-10-09T05:44:18.571626+0000 mgr.y (mgr.25277) 1699 : cluster 0 pgmap v1925: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:19.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:19 smithi194 bash[92666]: cluster 2023-10-09T05:44:18.636746+0000 osd.5 (osd.5) 489 : cluster 3 failed to encode map e443 with expected crc 2023-10-09T05:44:19.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:19 smithi194 bash[92666]: audit 2023-10-09T05:44:18.639695+0000 mon.a (mon.0) 2225 : audit 1 from='client.? 172.21.15.133:0/2450280789' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:44:19.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:19 smithi194 bash[92666]: cluster 2023-10-09T05:44:18.639794+0000 mon.a (mon.0) 2226 : cluster 0 osdmap e443: 8 total, 8 up, 8 in 2023-10-09T05:44:19.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:19 smithi194 bash[92666]: cluster 2023-10-09T05:44:18.683465+0000 osd.4 (osd.4) 386 : cluster 3 failed to encode map e443 with expected crc 2023-10-09T05:44:19.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:19 smithi194 bash[92666]: cluster 2023-10-09T05:44:18.683775+0000 osd.7 (osd.7) 411 : cluster 3 failed to encode map e443 with expected crc 2023-10-09T05:44:19.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:19 smithi194 bash[92666]: cluster 2023-10-09T05:44:18.683843+0000 osd.6 (osd.6) 440 : cluster 3 failed to encode map e443 with expected crc 2023-10-09T05:44:20.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:19 smithi133 bash[102858]: cluster 2023-10-09T05:44:18.571626+0000 mgr.y (mgr.25277) 1699 : cluster 0 pgmap v1925: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:20.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:19 smithi133 bash[102858]: cluster 2023-10-09T05:44:18.636746+0000 osd.5 (osd.5) 489 : cluster 3 failed to encode map e443 with expected crc 2023-10-09T05:44:20.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:19 smithi133 bash[102858]: audit 2023-10-09T05:44:18.639695+0000 mon.a (mon.0) 2225 : audit 1 from='client.? 172.21.15.133:0/2450280789' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:44:20.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:19 smithi133 bash[102858]: cluster 2023-10-09T05:44:18.639794+0000 mon.a (mon.0) 2226 : cluster 0 osdmap e443: 8 total, 8 up, 8 in 2023-10-09T05:44:20.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:19 smithi133 bash[102858]: cluster 2023-10-09T05:44:18.683465+0000 osd.4 (osd.4) 386 : cluster 3 failed to encode map e443 with expected crc 2023-10-09T05:44:20.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:19 smithi133 bash[102858]: cluster 2023-10-09T05:44:18.683775+0000 osd.7 (osd.7) 411 : cluster 3 failed to encode map e443 with expected crc 2023-10-09T05:44:20.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:19 smithi133 bash[102858]: cluster 2023-10-09T05:44:18.683843+0000 osd.6 (osd.6) 440 : cluster 3 failed to encode map e443 with expected crc 2023-10-09T05:44:20.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:19 smithi133 bash[105589]: cluster 2023-10-09T05:44:18.571626+0000 mgr.y (mgr.25277) 1699 : cluster 0 pgmap v1925: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:20.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:19 smithi133 bash[105589]: cluster 2023-10-09T05:44:18.636746+0000 osd.5 (osd.5) 489 : cluster 3 failed to encode map e443 with expected crc 2023-10-09T05:44:20.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:19 smithi133 bash[105589]: audit 2023-10-09T05:44:18.639695+0000 mon.a (mon.0) 2225 : audit 1 from='client.? 172.21.15.133:0/2450280789' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:44:20.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:19 smithi133 bash[105589]: cluster 2023-10-09T05:44:18.639794+0000 mon.a (mon.0) 2226 : cluster 0 osdmap e443: 8 total, 8 up, 8 in 2023-10-09T05:44:20.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:19 smithi133 bash[105589]: cluster 2023-10-09T05:44:18.683465+0000 osd.4 (osd.4) 386 : cluster 3 failed to encode map e443 with expected crc 2023-10-09T05:44:20.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:19 smithi133 bash[105589]: cluster 2023-10-09T05:44:18.683775+0000 osd.7 (osd.7) 411 : cluster 3 failed to encode map e443 with expected crc 2023-10-09T05:44:20.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:19 smithi133 bash[105589]: cluster 2023-10-09T05:44:18.683843+0000 osd.6 (osd.6) 440 : cluster 3 failed to encode map e443 with expected crc 2023-10-09T05:44:20.108 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T05:44:22.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:21 smithi133 bash[102858]: cluster 2023-10-09T05:44:20.572671+0000 mgr.y (mgr.25277) 1700 : cluster 0 pgmap v1927: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:22.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:21 smithi133 bash[105589]: cluster 2023-10-09T05:44:20.572671+0000 mgr.y (mgr.25277) 1700 : cluster 0 pgmap v1927: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:22.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:21 smithi194 bash[92666]: cluster 2023-10-09T05:44:20.572671+0000 mgr.y (mgr.25277) 1700 : cluster 0 pgmap v1927: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:24.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:23 smithi133 bash[102858]: cluster 2023-10-09T05:44:22.573456+0000 mgr.y (mgr.25277) 1701 : cluster 0 pgmap v1928: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:24.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:23 smithi133 bash[105589]: cluster 2023-10-09T05:44:22.573456+0000 mgr.y (mgr.25277) 1701 : cluster 0 pgmap v1928: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:24.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:23 smithi194 bash[92666]: cluster 2023-10-09T05:44:22.573456+0000 mgr.y (mgr.25277) 1701 : cluster 0 pgmap v1928: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:24.399 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:24.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:44:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:44:24] "GET /metrics HTTP/1.1" 200 34139 "" "Prometheus/2.43.0" 2023-10-09T05:44:24.886 INFO:teuthology.orchestra.run.smithi194.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:44:25.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:25 smithi194 bash[92666]: cluster 2023-10-09T05:44:24.574299+0000 mgr.y (mgr.25277) 1702 : cluster 0 pgmap v1929: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:25.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:25 smithi133 bash[102858]: cluster 2023-10-09T05:44:24.574299+0000 mgr.y (mgr.25277) 1702 : cluster 0 pgmap v1929: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:25.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:25 smithi133 bash[105589]: cluster 2023-10-09T05:44:24.574299+0000 mgr.y (mgr.25277) 1702 : cluster 0 pgmap v1929: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:27.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:27 smithi194 bash[92666]: cluster 2023-10-09T05:44:26.575200+0000 mgr.y (mgr.25277) 1703 : cluster 0 pgmap v1930: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:28.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:27 smithi133 bash[102858]: cluster 2023-10-09T05:44:26.575200+0000 mgr.y (mgr.25277) 1703 : cluster 0 pgmap v1930: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:28.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:27 smithi133 bash[105589]: cluster 2023-10-09T05:44:26.575200+0000 mgr.y (mgr.25277) 1703 : cluster 0 pgmap v1930: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:28.197 INFO:teuthology.orchestra.run.smithi194.stdout:ERROR: (22) Invalid argument 2023-10-09T05:44:28.197 INFO:teuthology.orchestra.run.smithi194.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:44:28.652 INFO:teuthology.orchestra.run.smithi133.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-10-09T05:44:28.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:28 smithi194 bash[92666]: audit 2023-10-09T05:44:28.288555+0000 mon.c (mon.2) 155 : audit 1 from='client.? 172.21.15.133:0/1348400862' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:44:28.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:28 smithi194 bash[92666]: audit 2023-10-09T05:44:28.289365+0000 mon.a (mon.0) 2227 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:44:28.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:28 smithi194 bash[92666]: audit 2023-10-09T05:44:28.502484+0000 mon.a (mon.0) 2228 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:29.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:28 smithi133 bash[102858]: audit 2023-10-09T05:44:28.288555+0000 mon.c (mon.2) 155 : audit 1 from='client.? 172.21.15.133:0/1348400862' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:44:29.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:28 smithi133 bash[102858]: audit 2023-10-09T05:44:28.289365+0000 mon.a (mon.0) 2227 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:44:29.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:28 smithi133 bash[102858]: audit 2023-10-09T05:44:28.502484+0000 mon.a (mon.0) 2228 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:29.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:28 smithi133 bash[105589]: audit 2023-10-09T05:44:28.288555+0000 mon.c (mon.2) 155 : audit 1 from='client.? 172.21.15.133:0/1348400862' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:44:29.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:28 smithi133 bash[105589]: audit 2023-10-09T05:44:28.289365+0000 mon.a (mon.0) 2227 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:44:29.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:28 smithi133 bash[105589]: audit 2023-10-09T05:44:28.502484+0000 mon.a (mon.0) 2228 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:29.650 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T05:44:30.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:29 smithi133 bash[102858]: cluster 2023-10-09T05:44:28.575947+0000 mgr.y (mgr.25277) 1704 : cluster 0 pgmap v1931: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:30.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:29 smithi133 bash[102858]: audit 2023-10-09T05:44:28.647232+0000 mon.a (mon.0) 2229 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:44:30.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:29 smithi133 bash[102858]: cluster 2023-10-09T05:44:28.647336+0000 mon.a (mon.0) 2230 : cluster 0 osdmap e444: 8 total, 8 up, 8 in 2023-10-09T05:44:30.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:29 smithi133 bash[105589]: cluster 2023-10-09T05:44:28.575947+0000 mgr.y (mgr.25277) 1704 : cluster 0 pgmap v1931: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:30.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:29 smithi133 bash[105589]: audit 2023-10-09T05:44:28.647232+0000 mon.a (mon.0) 2229 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:44:30.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:29 smithi133 bash[105589]: cluster 2023-10-09T05:44:28.647336+0000 mon.a (mon.0) 2230 : cluster 0 osdmap e444: 8 total, 8 up, 8 in 2023-10-09T05:44:30.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:29 smithi194 bash[92666]: cluster 2023-10-09T05:44:28.575947+0000 mgr.y (mgr.25277) 1704 : cluster 0 pgmap v1931: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:30.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:29 smithi194 bash[92666]: audit 2023-10-09T05:44:28.647232+0000 mon.a (mon.0) 2229 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:44:30.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:29 smithi194 bash[92666]: cluster 2023-10-09T05:44:28.647336+0000 mon.a (mon.0) 2230 : cluster 0 osdmap e444: 8 total, 8 up, 8 in 2023-10-09T05:44:31.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:30 smithi133 bash[105589]: cluster 2023-10-09T05:44:29.464025+0000 osd.4 (osd.4) 387 : cluster 3 failed to encode map e444 with expected crc 2023-10-09T05:44:31.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:30 smithi133 bash[105589]: cluster 2023-10-09T05:44:29.466625+0000 osd.4 (osd.4) 388 : cluster 3 failed to encode map e444 with expected crc 2023-10-09T05:44:31.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:30 smithi133 bash[105589]: cluster 2023-10-09T05:44:29.469301+0000 osd.4 (osd.4) 389 : cluster 3 failed to encode map e444 with expected crc 2023-10-09T05:44:31.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:30 smithi133 bash[105589]: cluster 2023-10-09T05:44:29.471928+0000 osd.4 (osd.4) 390 : cluster 3 failed to encode map e444 with expected crc 2023-10-09T05:44:31.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:30 smithi133 bash[105589]: cluster 2023-10-09T05:44:29.791386+0000 osd.7 (osd.7) 412 : cluster 3 failed to encode map e444 with expected crc 2023-10-09T05:44:31.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:30 smithi133 bash[105589]: cluster 2023-10-09T05:44:29.958093+0000 osd.5 (osd.5) 490 : cluster 3 failed to encode map e444 with expected crc 2023-10-09T05:44:31.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:30 smithi133 bash[105589]: cluster 2023-10-09T05:44:29.961097+0000 osd.5 (osd.5) 491 : cluster 3 failed to encode map e444 with expected crc 2023-10-09T05:44:31.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:30 smithi133 bash[105589]: cluster 2023-10-09T05:44:30.050614+0000 osd.6 (osd.6) 441 : cluster 3 failed to encode map e444 with expected crc 2023-10-09T05:44:31.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:30 smithi133 bash[102858]: cluster 2023-10-09T05:44:29.464025+0000 osd.4 (osd.4) 387 : cluster 3 failed to encode map e444 with expected crc 2023-10-09T05:44:31.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:30 smithi133 bash[102858]: cluster 2023-10-09T05:44:29.466625+0000 osd.4 (osd.4) 388 : cluster 3 failed to encode map e444 with expected crc 2023-10-09T05:44:31.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:30 smithi133 bash[102858]: cluster 2023-10-09T05:44:29.469301+0000 osd.4 (osd.4) 389 : cluster 3 failed to encode map e444 with expected crc 2023-10-09T05:44:31.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:30 smithi133 bash[102858]: cluster 2023-10-09T05:44:29.471928+0000 osd.4 (osd.4) 390 : cluster 3 failed to encode map e444 with expected crc 2023-10-09T05:44:31.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:30 smithi133 bash[102858]: cluster 2023-10-09T05:44:29.791386+0000 osd.7 (osd.7) 412 : cluster 3 failed to encode map e444 with expected crc 2023-10-09T05:44:31.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:30 smithi133 bash[102858]: cluster 2023-10-09T05:44:29.958093+0000 osd.5 (osd.5) 490 : cluster 3 failed to encode map e444 with expected crc 2023-10-09T05:44:31.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:30 smithi133 bash[102858]: cluster 2023-10-09T05:44:29.961097+0000 osd.5 (osd.5) 491 : cluster 3 failed to encode map e444 with expected crc 2023-10-09T05:44:31.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:30 smithi133 bash[102858]: cluster 2023-10-09T05:44:30.050614+0000 osd.6 (osd.6) 441 : cluster 3 failed to encode map e444 with expected crc 2023-10-09T05:44:31.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:30 smithi194 bash[92666]: cluster 2023-10-09T05:44:29.464025+0000 osd.4 (osd.4) 387 : cluster 3 failed to encode map e444 with expected crc 2023-10-09T05:44:31.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:30 smithi194 bash[92666]: cluster 2023-10-09T05:44:29.466625+0000 osd.4 (osd.4) 388 : cluster 3 failed to encode map e444 with expected crc 2023-10-09T05:44:31.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:30 smithi194 bash[92666]: cluster 2023-10-09T05:44:29.469301+0000 osd.4 (osd.4) 389 : cluster 3 failed to encode map e444 with expected crc 2023-10-09T05:44:31.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:30 smithi194 bash[92666]: cluster 2023-10-09T05:44:29.471928+0000 osd.4 (osd.4) 390 : cluster 3 failed to encode map e444 with expected crc 2023-10-09T05:44:31.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:30 smithi194 bash[92666]: cluster 2023-10-09T05:44:29.791386+0000 osd.7 (osd.7) 412 : cluster 3 failed to encode map e444 with expected crc 2023-10-09T05:44:31.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:30 smithi194 bash[92666]: cluster 2023-10-09T05:44:29.958093+0000 osd.5 (osd.5) 490 : cluster 3 failed to encode map e444 with expected crc 2023-10-09T05:44:31.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:30 smithi194 bash[92666]: cluster 2023-10-09T05:44:29.961097+0000 osd.5 (osd.5) 491 : cluster 3 failed to encode map e444 with expected crc 2023-10-09T05:44:31.137 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:30 smithi194 bash[92666]: cluster 2023-10-09T05:44:30.050614+0000 osd.6 (osd.6) 441 : cluster 3 failed to encode map e444 with expected crc 2023-10-09T05:44:32.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:31 smithi133 bash[105589]: cluster 2023-10-09T05:44:30.576973+0000 mgr.y (mgr.25277) 1705 : cluster 0 pgmap v1933: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:32.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:31 smithi133 bash[102858]: cluster 2023-10-09T05:44:30.576973+0000 mgr.y (mgr.25277) 1705 : cluster 0 pgmap v1933: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:32.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:31 smithi194 bash[92666]: cluster 2023-10-09T05:44:30.576973+0000 mgr.y (mgr.25277) 1705 : cluster 0 pgmap v1933: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:34.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:33 smithi133 bash[105589]: cluster 2023-10-09T05:44:32.577812+0000 mgr.y (mgr.25277) 1706 : cluster 0 pgmap v1934: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:34.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:33 smithi133 bash[102858]: cluster 2023-10-09T05:44:32.577812+0000 mgr.y (mgr.25277) 1706 : cluster 0 pgmap v1934: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:34.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:33 smithi194 bash[92666]: cluster 2023-10-09T05:44:32.577812+0000 mgr.y (mgr.25277) 1706 : cluster 0 pgmap v1934: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:34.425 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:34.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:44:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:44:34] "GET /metrics HTTP/1.1" 200 34142 "" "Prometheus/2.43.0" 2023-10-09T05:44:35.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:35 smithi194 bash[92666]: cluster 2023-10-09T05:44:34.578676+0000 mgr.y (mgr.25277) 1707 : cluster 0 pgmap v1935: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:35.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:35 smithi133 bash[105589]: cluster 2023-10-09T05:44:34.578676+0000 mgr.y (mgr.25277) 1707 : cluster 0 pgmap v1935: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:35.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:35 smithi133 bash[102858]: cluster 2023-10-09T05:44:34.578676+0000 mgr.y (mgr.25277) 1707 : cluster 0 pgmap v1935: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:37.917 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:37 smithi133 bash[105589]: cluster 2023-10-09T05:44:36.579587+0000 mgr.y (mgr.25277) 1708 : cluster 0 pgmap v1936: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:37.917 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:37 smithi133 bash[102858]: cluster 2023-10-09T05:44:36.579587+0000 mgr.y (mgr.25277) 1708 : cluster 0 pgmap v1936: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:38.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:37 smithi194 bash[92666]: cluster 2023-10-09T05:44:36.579587+0000 mgr.y (mgr.25277) 1708 : cluster 0 pgmap v1936: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:38.639 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:38 smithi133 bash[102858]: audit 2023-10-09T05:44:38.302276+0000 mon.c (mon.2) 156 : audit 1 from='client.? 172.21.15.133:0/572512730' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:44:38.639 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:38 smithi133 bash[102858]: audit 2023-10-09T05:44:38.302996+0000 mon.a (mon.0) 2231 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:44:38.655 INFO:teuthology.orchestra.run.smithi133.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T05:44:38.890 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:38 smithi133 bash[105589]: audit 2023-10-09T05:44:38.302276+0000 mon.c (mon.2) 156 : audit 1 from='client.? 172.21.15.133:0/572512730' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:44:38.890 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:38 smithi133 bash[105589]: audit 2023-10-09T05:44:38.302996+0000 mon.a (mon.0) 2231 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:44:39.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:38 smithi194 bash[92666]: audit 2023-10-09T05:44:38.302276+0000 mon.c (mon.2) 156 : audit 1 from='client.? 172.21.15.133:0/572512730' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:44:39.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:38 smithi194 bash[92666]: audit 2023-10-09T05:44:38.302996+0000 mon.a (mon.0) 2231 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:44:39.664 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 6 1 2023-10-09T05:44:40.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:39 smithi133 bash[102858]: cluster 2023-10-09T05:44:38.580320+0000 mgr.y (mgr.25277) 1709 : cluster 0 pgmap v1937: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:40.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:39 smithi133 bash[102858]: cluster 2023-10-09T05:44:38.646185+0000 osd.6 (osd.6) 442 : cluster 3 failed to encode map e445 with expected crc 2023-10-09T05:44:40.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:39 smithi133 bash[102858]: audit 2023-10-09T05:44:38.648039+0000 mon.a (mon.0) 2232 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:44:40.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:39 smithi133 bash[102858]: cluster 2023-10-09T05:44:38.648136+0000 mon.a (mon.0) 2233 : cluster 0 osdmap e445: 8 total, 8 up, 8 in 2023-10-09T05:44:40.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:39 smithi133 bash[105589]: cluster 2023-10-09T05:44:38.580320+0000 mgr.y (mgr.25277) 1709 : cluster 0 pgmap v1937: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:40.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:39 smithi133 bash[105589]: cluster 2023-10-09T05:44:38.646185+0000 osd.6 (osd.6) 442 : cluster 3 failed to encode map e445 with expected crc 2023-10-09T05:44:40.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:39 smithi133 bash[105589]: audit 2023-10-09T05:44:38.648039+0000 mon.a (mon.0) 2232 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:44:40.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:39 smithi133 bash[105589]: cluster 2023-10-09T05:44:38.648136+0000 mon.a (mon.0) 2233 : cluster 0 osdmap e445: 8 total, 8 up, 8 in 2023-10-09T05:44:40.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:39 smithi194 bash[92666]: cluster 2023-10-09T05:44:38.580320+0000 mgr.y (mgr.25277) 1709 : cluster 0 pgmap v1937: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:40.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:39 smithi194 bash[92666]: cluster 2023-10-09T05:44:38.646185+0000 osd.6 (osd.6) 442 : cluster 3 failed to encode map e445 with expected crc 2023-10-09T05:44:40.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:39 smithi194 bash[92666]: audit 2023-10-09T05:44:38.648039+0000 mon.a (mon.0) 2232 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:44:40.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:39 smithi194 bash[92666]: cluster 2023-10-09T05:44:38.648136+0000 mon.a (mon.0) 2233 : cluster 0 osdmap e445: 8 total, 8 up, 8 in 2023-10-09T05:44:41.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:40 smithi133 bash[102858]: cluster 2023-10-09T05:44:39.867426+0000 osd.4 (osd.4) 391 : cluster 3 failed to encode map e445 with expected crc 2023-10-09T05:44:41.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:40 smithi133 bash[102858]: cluster 2023-10-09T05:44:39.870515+0000 osd.4 (osd.4) 392 : cluster 3 failed to encode map e445 with expected crc 2023-10-09T05:44:41.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:40 smithi133 bash[102858]: cluster 2023-10-09T05:44:40.077256+0000 osd.5 (osd.5) 492 : cluster 3 failed to encode map e445 with expected crc 2023-10-09T05:44:41.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:40 smithi133 bash[102858]: cluster 2023-10-09T05:44:40.077477+0000 osd.7 (osd.7) 413 : cluster 3 failed to encode map e445 with expected crc 2023-10-09T05:44:41.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:40 smithi133 bash[105589]: cluster 2023-10-09T05:44:39.867426+0000 osd.4 (osd.4) 391 : cluster 3 failed to encode map e445 with expected crc 2023-10-09T05:44:41.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:40 smithi133 bash[105589]: cluster 2023-10-09T05:44:39.870515+0000 osd.4 (osd.4) 392 : cluster 3 failed to encode map e445 with expected crc 2023-10-09T05:44:41.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:40 smithi133 bash[105589]: cluster 2023-10-09T05:44:40.077256+0000 osd.5 (osd.5) 492 : cluster 3 failed to encode map e445 with expected crc 2023-10-09T05:44:41.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:40 smithi133 bash[105589]: cluster 2023-10-09T05:44:40.077477+0000 osd.7 (osd.7) 413 : cluster 3 failed to encode map e445 with expected crc 2023-10-09T05:44:41.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:40 smithi194 bash[92666]: cluster 2023-10-09T05:44:39.867426+0000 osd.4 (osd.4) 391 : cluster 3 failed to encode map e445 with expected crc 2023-10-09T05:44:41.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:40 smithi194 bash[92666]: cluster 2023-10-09T05:44:39.870515+0000 osd.4 (osd.4) 392 : cluster 3 failed to encode map e445 with expected crc 2023-10-09T05:44:41.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:40 smithi194 bash[92666]: cluster 2023-10-09T05:44:40.077256+0000 osd.5 (osd.5) 492 : cluster 3 failed to encode map e445 with expected crc 2023-10-09T05:44:41.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:40 smithi194 bash[92666]: cluster 2023-10-09T05:44:40.077477+0000 osd.7 (osd.7) 413 : cluster 3 failed to encode map e445 with expected crc 2023-10-09T05:44:42.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:41 smithi133 bash[102858]: cluster 2023-10-09T05:44:40.581444+0000 mgr.y (mgr.25277) 1710 : cluster 0 pgmap v1939: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:42.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:41 smithi133 bash[105589]: cluster 2023-10-09T05:44:40.581444+0000 mgr.y (mgr.25277) 1710 : cluster 0 pgmap v1939: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:42.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:41 smithi194 bash[92666]: cluster 2023-10-09T05:44:40.581444+0000 mgr.y (mgr.25277) 1710 : cluster 0 pgmap v1939: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:44.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:43 smithi133 bash[105589]: cluster 2023-10-09T05:44:42.582191+0000 mgr.y (mgr.25277) 1711 : cluster 0 pgmap v1940: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:44.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:43 smithi133 bash[105589]: audit 2023-10-09T05:44:43.502822+0000 mon.a (mon.0) 2234 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:44.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:43 smithi133 bash[102858]: cluster 2023-10-09T05:44:42.582191+0000 mgr.y (mgr.25277) 1711 : cluster 0 pgmap v1940: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:44.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:43 smithi133 bash[102858]: audit 2023-10-09T05:44:43.502822+0000 mon.a (mon.0) 2234 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:44.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:43 smithi194 bash[92666]: cluster 2023-10-09T05:44:42.582191+0000 mgr.y (mgr.25277) 1711 : cluster 0 pgmap v1940: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:44.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:43 smithi194 bash[92666]: audit 2023-10-09T05:44:43.502822+0000 mon.a (mon.0) 2234 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:44.431 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:44.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:44:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:44:44] "GET /metrics HTTP/1.1" 200 34128 "" "Prometheus/2.43.0" 2023-10-09T05:44:45.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:45 smithi194 bash[92666]: cluster 2023-10-09T05:44:44.583147+0000 mgr.y (mgr.25277) 1712 : cluster 0 pgmap v1941: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:45.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:45 smithi133 bash[102858]: cluster 2023-10-09T05:44:44.583147+0000 mgr.y (mgr.25277) 1712 : cluster 0 pgmap v1941: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:45.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:45 smithi133 bash[105589]: cluster 2023-10-09T05:44:44.583147+0000 mgr.y (mgr.25277) 1712 : cluster 0 pgmap v1941: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:48.009 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:47 smithi133 bash[102858]: cluster 2023-10-09T05:44:46.584029+0000 mgr.y (mgr.25277) 1713 : cluster 0 pgmap v1942: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:48.010 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:47 smithi133 bash[105589]: cluster 2023-10-09T05:44:46.584029+0000 mgr.y (mgr.25277) 1713 : cluster 0 pgmap v1942: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:48.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:47 smithi194 bash[92666]: cluster 2023-10-09T05:44:46.584029+0000 mgr.y (mgr.25277) 1713 : cluster 0 pgmap v1942: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:48.653 INFO:teuthology.orchestra.run.smithi133.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-10-09T05:44:49.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:48 smithi133 bash[105589]: audit 2023-10-09T05:44:48.483060+0000 mon.a (mon.0) 2235 : audit 1 from='client.? 172.21.15.133:0/3311059137' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:44:49.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:48 smithi133 bash[102858]: audit 2023-10-09T05:44:48.483060+0000 mon.a (mon.0) 2235 : audit 1 from='client.? 172.21.15.133:0/3311059137' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:44:49.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:48 smithi194 bash[92666]: audit 2023-10-09T05:44:48.483060+0000 mon.a (mon.0) 2235 : audit 1 from='client.? 172.21.15.133:0/3311059137' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:44:49.630 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T05:44:50.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:49 smithi133 bash[105589]: cluster 2023-10-09T05:44:48.584745+0000 mgr.y (mgr.25277) 1714 : cluster 0 pgmap v1943: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:50.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:49 smithi133 bash[105589]: cluster 2023-10-09T05:44:48.644172+0000 osd.4 (osd.4) 393 : cluster 3 failed to encode map e446 with expected crc 2023-10-09T05:44:50.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:49 smithi133 bash[105589]: audit 2023-10-09T05:44:48.647895+0000 mon.a (mon.0) 2236 : audit 1 from='client.? 172.21.15.133:0/3311059137' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:44:50.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:49 smithi133 bash[105589]: cluster 2023-10-09T05:44:48.647971+0000 mon.a (mon.0) 2237 : cluster 0 osdmap e446: 8 total, 8 up, 8 in 2023-10-09T05:44:50.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:49 smithi133 bash[105589]: cluster 2023-10-09T05:44:49.162203+0000 osd.5 (osd.5) 493 : cluster 3 failed to encode map e446 with expected crc 2023-10-09T05:44:50.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:49 smithi133 bash[105589]: cluster 2023-10-09T05:44:49.165176+0000 osd.5 (osd.5) 494 : cluster 3 failed to encode map e446 with expected crc 2023-10-09T05:44:50.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:49 smithi133 bash[102858]: cluster 2023-10-09T05:44:48.584745+0000 mgr.y (mgr.25277) 1714 : cluster 0 pgmap v1943: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:50.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:49 smithi133 bash[102858]: cluster 2023-10-09T05:44:48.644172+0000 osd.4 (osd.4) 393 : cluster 3 failed to encode map e446 with expected crc 2023-10-09T05:44:50.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:49 smithi133 bash[102858]: audit 2023-10-09T05:44:48.647895+0000 mon.a (mon.0) 2236 : audit 1 from='client.? 172.21.15.133:0/3311059137' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:44:50.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:49 smithi133 bash[102858]: cluster 2023-10-09T05:44:48.647971+0000 mon.a (mon.0) 2237 : cluster 0 osdmap e446: 8 total, 8 up, 8 in 2023-10-09T05:44:50.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:49 smithi133 bash[102858]: cluster 2023-10-09T05:44:49.162203+0000 osd.5 (osd.5) 493 : cluster 3 failed to encode map e446 with expected crc 2023-10-09T05:44:50.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:49 smithi133 bash[102858]: cluster 2023-10-09T05:44:49.165176+0000 osd.5 (osd.5) 494 : cluster 3 failed to encode map e446 with expected crc 2023-10-09T05:44:50.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:49 smithi194 bash[92666]: cluster 2023-10-09T05:44:48.584745+0000 mgr.y (mgr.25277) 1714 : cluster 0 pgmap v1943: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:50.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:49 smithi194 bash[92666]: cluster 2023-10-09T05:44:48.644172+0000 osd.4 (osd.4) 393 : cluster 3 failed to encode map e446 with expected crc 2023-10-09T05:44:50.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:49 smithi194 bash[92666]: audit 2023-10-09T05:44:48.647895+0000 mon.a (mon.0) 2236 : audit 1 from='client.? 172.21.15.133:0/3311059137' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:44:50.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:49 smithi194 bash[92666]: cluster 2023-10-09T05:44:48.647971+0000 mon.a (mon.0) 2237 : cluster 0 osdmap e446: 8 total, 8 up, 8 in 2023-10-09T05:44:50.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:49 smithi194 bash[92666]: cluster 2023-10-09T05:44:49.162203+0000 osd.5 (osd.5) 493 : cluster 3 failed to encode map e446 with expected crc 2023-10-09T05:44:50.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:49 smithi194 bash[92666]: cluster 2023-10-09T05:44:49.165176+0000 osd.5 (osd.5) 494 : cluster 3 failed to encode map e446 with expected crc 2023-10-09T05:44:51.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:50 smithi133 bash[102858]: cluster 2023-10-09T05:44:49.954687+0000 osd.6 (osd.6) 443 : cluster 3 failed to encode map e446 with expected crc 2023-10-09T05:44:51.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:50 smithi133 bash[102858]: cluster 2023-10-09T05:44:49.954877+0000 osd.7 (osd.7) 414 : cluster 3 failed to encode map e446 with expected crc 2023-10-09T05:44:51.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:50 smithi133 bash[105589]: cluster 2023-10-09T05:44:49.954687+0000 osd.6 (osd.6) 443 : cluster 3 failed to encode map e446 with expected crc 2023-10-09T05:44:51.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:50 smithi133 bash[105589]: cluster 2023-10-09T05:44:49.954877+0000 osd.7 (osd.7) 414 : cluster 3 failed to encode map e446 with expected crc 2023-10-09T05:44:51.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:50 smithi194 bash[92666]: cluster 2023-10-09T05:44:49.954687+0000 osd.6 (osd.6) 443 : cluster 3 failed to encode map e446 with expected crc 2023-10-09T05:44:51.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:50 smithi194 bash[92666]: cluster 2023-10-09T05:44:49.954877+0000 osd.7 (osd.7) 414 : cluster 3 failed to encode map e446 with expected crc 2023-10-09T05:44:52.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:51 smithi133 bash[102858]: cluster 2023-10-09T05:44:50.585828+0000 mgr.y (mgr.25277) 1715 : cluster 0 pgmap v1945: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:52.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:51 smithi133 bash[105589]: cluster 2023-10-09T05:44:50.585828+0000 mgr.y (mgr.25277) 1715 : cluster 0 pgmap v1945: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:52.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:51 smithi194 bash[92666]: cluster 2023-10-09T05:44:50.585828+0000 mgr.y (mgr.25277) 1715 : cluster 0 pgmap v1945: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:54.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:53 smithi133 bash[105589]: cluster 2023-10-09T05:44:52.586573+0000 mgr.y (mgr.25277) 1716 : cluster 0 pgmap v1946: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:54.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:53 smithi133 bash[102858]: cluster 2023-10-09T05:44:52.586573+0000 mgr.y (mgr.25277) 1716 : cluster 0 pgmap v1946: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:54.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:53 smithi194 bash[92666]: cluster 2023-10-09T05:44:52.586573+0000 mgr.y (mgr.25277) 1716 : cluster 0 pgmap v1946: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:54.405 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:54.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:44:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:44:54] "GET /metrics HTTP/1.1" 200 34128 "" "Prometheus/2.43.0" 2023-10-09T05:44:55.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:55 smithi194 bash[92666]: cluster 2023-10-09T05:44:54.587473+0000 mgr.y (mgr.25277) 1717 : cluster 0 pgmap v1947: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:55.549 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:55 smithi133 bash[102858]: cluster 2023-10-09T05:44:54.587473+0000 mgr.y (mgr.25277) 1717 : cluster 0 pgmap v1947: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:55.550 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:55 smithi133 bash[105589]: cluster 2023-10-09T05:44:54.587473+0000 mgr.y (mgr.25277) 1717 : cluster 0 pgmap v1947: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:58.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:57 smithi133 bash[105589]: cluster 2023-10-09T05:44:56.588374+0000 mgr.y (mgr.25277) 1718 : cluster 0 pgmap v1948: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:58.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:57 smithi133 bash[102858]: cluster 2023-10-09T05:44:56.588374+0000 mgr.y (mgr.25277) 1718 : cluster 0 pgmap v1948: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:58.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:57 smithi194 bash[92666]: cluster 2023-10-09T05:44:56.588374+0000 mgr.y (mgr.25277) 1718 : cluster 0 pgmap v1948: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:44:58.663 INFO:teuthology.orchestra.run.smithi133.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T05:44:58.903 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:58 smithi133 bash[105589]: audit 2023-10-09T05:44:58.290408+0000 mon.a (mon.0) 2238 : audit 1 from='client.? 172.21.15.133:0/971040073' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:44:58.903 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:58 smithi133 bash[105589]: audit 2023-10-09T05:44:58.503546+0000 mon.a (mon.0) 2239 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:58.903 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:58 smithi133 bash[102858]: audit 2023-10-09T05:44:58.290408+0000 mon.a (mon.0) 2238 : audit 1 from='client.? 172.21.15.133:0/971040073' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:44:58.903 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:58 smithi133 bash[102858]: audit 2023-10-09T05:44:58.503546+0000 mon.a (mon.0) 2239 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:59.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:58 smithi194 bash[92666]: audit 2023-10-09T05:44:58.290408+0000 mon.a (mon.0) 2238 : audit 1 from='client.? 172.21.15.133:0/971040073' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:44:59.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:58 smithi194 bash[92666]: audit 2023-10-09T05:44:58.503546+0000 mon.a (mon.0) 2239 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:59.637 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 7 1 2023-10-09T05:45:00.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:59 smithi133 bash[102858]: cluster 2023-10-09T05:44:58.589046+0000 mgr.y (mgr.25277) 1719 : cluster 0 pgmap v1949: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:00.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:59 smithi133 bash[102858]: cluster 2023-10-09T05:44:58.656663+0000 osd.6 (osd.6) 444 : cluster 3 failed to encode map e447 with expected crc 2023-10-09T05:45:00.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:59 smithi133 bash[102858]: audit 2023-10-09T05:44:58.657650+0000 mon.a (mon.0) 2240 : audit 1 from='client.? 172.21.15.133:0/971040073' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:45:00.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:44:59 smithi133 bash[102858]: cluster 2023-10-09T05:44:58.657715+0000 mon.a (mon.0) 2241 : cluster 0 osdmap e447: 8 total, 8 up, 8 in 2023-10-09T05:45:00.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:59 smithi133 bash[105589]: cluster 2023-10-09T05:44:58.589046+0000 mgr.y (mgr.25277) 1719 : cluster 0 pgmap v1949: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:00.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:59 smithi133 bash[105589]: cluster 2023-10-09T05:44:58.656663+0000 osd.6 (osd.6) 444 : cluster 3 failed to encode map e447 with expected crc 2023-10-09T05:45:00.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:59 smithi133 bash[105589]: audit 2023-10-09T05:44:58.657650+0000 mon.a (mon.0) 2240 : audit 1 from='client.? 172.21.15.133:0/971040073' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:45:00.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:44:59 smithi133 bash[105589]: cluster 2023-10-09T05:44:58.657715+0000 mon.a (mon.0) 2241 : cluster 0 osdmap e447: 8 total, 8 up, 8 in 2023-10-09T05:45:00.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:59 smithi194 bash[92666]: cluster 2023-10-09T05:44:58.589046+0000 mgr.y (mgr.25277) 1719 : cluster 0 pgmap v1949: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:00.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:59 smithi194 bash[92666]: cluster 2023-10-09T05:44:58.656663+0000 osd.6 (osd.6) 444 : cluster 3 failed to encode map e447 with expected crc 2023-10-09T05:45:00.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:59 smithi194 bash[92666]: audit 2023-10-09T05:44:58.657650+0000 mon.a (mon.0) 2240 : audit 1 from='client.? 172.21.15.133:0/971040073' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:45:00.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:44:59 smithi194 bash[92666]: cluster 2023-10-09T05:44:58.657715+0000 mon.a (mon.0) 2241 : cluster 0 osdmap e447: 8 total, 8 up, 8 in 2023-10-09T05:45:01.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:00 smithi133 bash[105589]: cluster 2023-10-09T05:44:59.669937+0000 osd.5 (osd.5) 495 : cluster 3 failed to encode map e448 with expected crc 2023-10-09T05:45:01.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:00 smithi133 bash[105589]: cluster 2023-10-09T05:44:59.672338+0000 osd.6 (osd.6) 445 : cluster 3 failed to encode map e448 with expected crc 2023-10-09T05:45:01.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:00 smithi133 bash[105589]: cluster 2023-10-09T05:44:59.672711+0000 mon.a (mon.0) 2242 : cluster 0 osdmap e448: 8 total, 8 up, 8 in 2023-10-09T05:45:01.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:00 smithi133 bash[105589]: cluster 2023-10-09T05:44:59.675411+0000 osd.6 (osd.6) 446 : cluster 3 failed to encode map e448 with expected crc 2023-10-09T05:45:01.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:00 smithi133 bash[105589]: cluster 2023-10-09T05:44:59.958553+0000 osd.7 (osd.7) 415 : cluster 3 failed to encode map e448 with expected crc 2023-10-09T05:45:01.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:00 smithi133 bash[105589]: cluster 2023-10-09T05:45:00.080014+0000 osd.4 (osd.4) 394 : cluster 3 failed to encode map e448 with expected crc 2023-10-09T05:45:01.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:00 smithi133 bash[102858]: cluster 2023-10-09T05:44:59.669937+0000 osd.5 (osd.5) 495 : cluster 3 failed to encode map e448 with expected crc 2023-10-09T05:45:01.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:00 smithi133 bash[102858]: cluster 2023-10-09T05:44:59.672338+0000 osd.6 (osd.6) 445 : cluster 3 failed to encode map e448 with expected crc 2023-10-09T05:45:01.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:00 smithi133 bash[102858]: cluster 2023-10-09T05:44:59.672711+0000 mon.a (mon.0) 2242 : cluster 0 osdmap e448: 8 total, 8 up, 8 in 2023-10-09T05:45:01.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:00 smithi133 bash[102858]: cluster 2023-10-09T05:44:59.675411+0000 osd.6 (osd.6) 446 : cluster 3 failed to encode map e448 with expected crc 2023-10-09T05:45:01.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:00 smithi133 bash[102858]: cluster 2023-10-09T05:44:59.958553+0000 osd.7 (osd.7) 415 : cluster 3 failed to encode map e448 with expected crc 2023-10-09T05:45:01.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:00 smithi133 bash[102858]: cluster 2023-10-09T05:45:00.080014+0000 osd.4 (osd.4) 394 : cluster 3 failed to encode map e448 with expected crc 2023-10-09T05:45:01.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:00 smithi194 bash[92666]: cluster 2023-10-09T05:44:59.669937+0000 osd.5 (osd.5) 495 : cluster 3 failed to encode map e448 with expected crc 2023-10-09T05:45:01.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:00 smithi194 bash[92666]: cluster 2023-10-09T05:44:59.672338+0000 osd.6 (osd.6) 445 : cluster 3 failed to encode map e448 with expected crc 2023-10-09T05:45:01.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:00 smithi194 bash[92666]: cluster 2023-10-09T05:44:59.672711+0000 mon.a (mon.0) 2242 : cluster 0 osdmap e448: 8 total, 8 up, 8 in 2023-10-09T05:45:01.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:00 smithi194 bash[92666]: cluster 2023-10-09T05:44:59.675411+0000 osd.6 (osd.6) 446 : cluster 3 failed to encode map e448 with expected crc 2023-10-09T05:45:01.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:00 smithi194 bash[92666]: cluster 2023-10-09T05:44:59.958553+0000 osd.7 (osd.7) 415 : cluster 3 failed to encode map e448 with expected crc 2023-10-09T05:45:01.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:00 smithi194 bash[92666]: cluster 2023-10-09T05:45:00.080014+0000 osd.4 (osd.4) 394 : cluster 3 failed to encode map e448 with expected crc 2023-10-09T05:45:01.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:01 smithi194 bash[92666]: cluster 2023-10-09T05:45:00.589971+0000 mgr.y (mgr.25277) 1720 : cluster 0 pgmap v1952: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:02.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:01 smithi133 bash[105589]: cluster 2023-10-09T05:45:00.589971+0000 mgr.y (mgr.25277) 1720 : cluster 0 pgmap v1952: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:02.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:01 smithi133 bash[102858]: cluster 2023-10-09T05:45:00.589971+0000 mgr.y (mgr.25277) 1720 : cluster 0 pgmap v1952: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:04.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:03 smithi133 bash[105589]: cluster 2023-10-09T05:45:02.590818+0000 mgr.y (mgr.25277) 1721 : cluster 0 pgmap v1953: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:04.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:03 smithi133 bash[102858]: cluster 2023-10-09T05:45:02.590818+0000 mgr.y (mgr.25277) 1721 : cluster 0 pgmap v1953: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:04.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:03 smithi194 bash[92666]: cluster 2023-10-09T05:45:02.590818+0000 mgr.y (mgr.25277) 1721 : cluster 0 pgmap v1953: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:04.411 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:04.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:45:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:45:04] "GET /metrics HTTP/1.1" 200 34130 "" "Prometheus/2.43.0" 2023-10-09T05:45:05.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:05 smithi194 bash[92666]: cluster 2023-10-09T05:45:04.591728+0000 mgr.y (mgr.25277) 1722 : cluster 0 pgmap v1954: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T05:45:05.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:05 smithi133 bash[102858]: cluster 2023-10-09T05:45:04.591728+0000 mgr.y (mgr.25277) 1722 : cluster 0 pgmap v1954: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T05:45:05.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:05 smithi133 bash[105589]: cluster 2023-10-09T05:45:04.591728+0000 mgr.y (mgr.25277) 1722 : cluster 0 pgmap v1954: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T05:45:07.947 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:07 smithi133 bash[105589]: cluster 2023-10-09T05:45:06.592681+0000 mgr.y (mgr.25277) 1723 : cluster 0 pgmap v1955: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T05:45:07.947 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:07 smithi133 bash[102858]: cluster 2023-10-09T05:45:06.592681+0000 mgr.y (mgr.25277) 1723 : cluster 0 pgmap v1955: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T05:45:08.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:07 smithi194 bash[92666]: cluster 2023-10-09T05:45:06.592681+0000 mgr.y (mgr.25277) 1723 : cluster 0 pgmap v1955: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T05:45:08.661 INFO:teuthology.orchestra.run.smithi133.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-10-09T05:45:09.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:08 smithi133 bash[102858]: audit 2023-10-09T05:45:08.347424+0000 mon.c (mon.2) 157 : audit 1 from='client.? 172.21.15.133:0/1407039496' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:45:09.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:08 smithi133 bash[102858]: audit 2023-10-09T05:45:08.348162+0000 mon.a (mon.0) 2243 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:45:09.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:08 smithi133 bash[105589]: audit 2023-10-09T05:45:08.347424+0000 mon.c (mon.2) 157 : audit 1 from='client.? 172.21.15.133:0/1407039496' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:45:09.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:08 smithi133 bash[105589]: audit 2023-10-09T05:45:08.348162+0000 mon.a (mon.0) 2243 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:45:09.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:08 smithi194 bash[92666]: audit 2023-10-09T05:45:08.347424+0000 mon.c (mon.2) 157 : audit 1 from='client.? 172.21.15.133:0/1407039496' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:45:09.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:08 smithi194 bash[92666]: audit 2023-10-09T05:45:08.348162+0000 mon.a (mon.0) 2243 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:45:09.585 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2023-10-09T05:45:09.585 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2023-10-09T05:45:09.585 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2023-10-09T05:45:09.585 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2023-10-09T05:45:09.585 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T05:45:09.585 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T05:45:10.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:09 smithi133 bash[102858]: cluster 2023-10-09T05:45:08.593383+0000 mgr.y (mgr.25277) 1724 : cluster 0 pgmap v1956: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T05:45:10.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:09 smithi133 bash[102858]: cluster 2023-10-09T05:45:08.650524+0000 osd.6 (osd.6) 447 : cluster 3 failed to encode map e449 with expected crc 2023-10-09T05:45:10.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:09 smithi133 bash[102858]: cluster 2023-10-09T05:45:08.651086+0000 osd.5 (osd.5) 496 : cluster 3 failed to encode map e449 with expected crc 2023-10-09T05:45:10.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:09 smithi133 bash[102858]: audit 2023-10-09T05:45:08.654957+0000 mon.a (mon.0) 2244 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:45:10.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:09 smithi133 bash[102858]: cluster 2023-10-09T05:45:08.655124+0000 mon.a (mon.0) 2245 : cluster 0 osdmap e449: 8 total, 8 up, 8 in 2023-10-09T05:45:10.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:09 smithi133 bash[105589]: cluster 2023-10-09T05:45:08.593383+0000 mgr.y (mgr.25277) 1724 : cluster 0 pgmap v1956: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T05:45:10.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:09 smithi133 bash[105589]: cluster 2023-10-09T05:45:08.650524+0000 osd.6 (osd.6) 447 : cluster 3 failed to encode map e449 with expected crc 2023-10-09T05:45:10.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:09 smithi133 bash[105589]: cluster 2023-10-09T05:45:08.651086+0000 osd.5 (osd.5) 496 : cluster 3 failed to encode map e449 with expected crc 2023-10-09T05:45:10.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:09 smithi133 bash[105589]: audit 2023-10-09T05:45:08.654957+0000 mon.a (mon.0) 2244 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:45:10.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:09 smithi133 bash[105589]: cluster 2023-10-09T05:45:08.655124+0000 mon.a (mon.0) 2245 : cluster 0 osdmap e449: 8 total, 8 up, 8 in 2023-10-09T05:45:10.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:09 smithi194 bash[92666]: cluster 2023-10-09T05:45:08.593383+0000 mgr.y (mgr.25277) 1724 : cluster 0 pgmap v1956: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T05:45:10.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:09 smithi194 bash[92666]: cluster 2023-10-09T05:45:08.650524+0000 osd.6 (osd.6) 447 : cluster 3 failed to encode map e449 with expected crc 2023-10-09T05:45:10.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:09 smithi194 bash[92666]: cluster 2023-10-09T05:45:08.651086+0000 osd.5 (osd.5) 496 : cluster 3 failed to encode map e449 with expected crc 2023-10-09T05:45:10.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:09 smithi194 bash[92666]: audit 2023-10-09T05:45:08.654957+0000 mon.a (mon.0) 2244 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:45:10.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:09 smithi194 bash[92666]: cluster 2023-10-09T05:45:08.655124+0000 mon.a (mon.0) 2245 : cluster 0 osdmap e449: 8 total, 8 up, 8 in 2023-10-09T05:45:12.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:11 smithi133 bash[105589]: cluster 2023-10-09T05:45:10.594391+0000 mgr.y (mgr.25277) 1725 : cluster 0 pgmap v1958: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T05:45:12.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:11 smithi133 bash[102858]: cluster 2023-10-09T05:45:10.594391+0000 mgr.y (mgr.25277) 1725 : cluster 0 pgmap v1958: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T05:45:12.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:11 smithi194 bash[92666]: cluster 2023-10-09T05:45:10.594391+0000 mgr.y (mgr.25277) 1725 : cluster 0 pgmap v1958: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T05:45:14.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:13 smithi133 bash[105589]: cluster 2023-10-09T05:45:12.595214+0000 mgr.y (mgr.25277) 1726 : cluster 0 pgmap v1959: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T05:45:14.050 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:13 smithi133 bash[105589]: audit 2023-10-09T05:45:13.503668+0000 mon.a (mon.0) 2246 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:14.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:13 smithi133 bash[102858]: cluster 2023-10-09T05:45:12.595214+0000 mgr.y (mgr.25277) 1726 : cluster 0 pgmap v1959: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T05:45:14.050 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:13 smithi133 bash[102858]: audit 2023-10-09T05:45:13.503668+0000 mon.a (mon.0) 2246 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:14.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:13 smithi194 bash[92666]: cluster 2023-10-09T05:45:12.595214+0000 mgr.y (mgr.25277) 1726 : cluster 0 pgmap v1959: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T05:45:14.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:13 smithi194 bash[92666]: audit 2023-10-09T05:45:13.503668+0000 mon.a (mon.0) 2246 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:14.366 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:14.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:45:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:45:14] "GET /metrics HTTP/1.1" 200 34116 "" "Prometheus/2.43.0" 2023-10-09T05:45:15.709 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:15 smithi133 bash[105589]: audit 2023-10-09T05:45:14.457938+0000 mon.a (mon.0) 2247 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:45:15.709 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:15 smithi133 bash[105589]: cluster 2023-10-09T05:45:14.596107+0000 mgr.y (mgr.25277) 1727 : cluster 0 pgmap v1960: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:15.710 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:15 smithi133 bash[105589]: audit 2023-10-09T05:45:14.733929+0000 mon.a (mon.0) 2248 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:45:15.710 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:15 smithi133 bash[105589]: audit 2023-10-09T05:45:15.110927+0000 mon.a (mon.0) 2249 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:45:15.710 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:15 smithi133 bash[105589]: audit 2023-10-09T05:45:15.119721+0000 mon.a (mon.0) 2250 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:45:15.710 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:15 smithi133 bash[102858]: audit 2023-10-09T05:45:14.457938+0000 mon.a (mon.0) 2247 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:45:15.710 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:15 smithi133 bash[102858]: cluster 2023-10-09T05:45:14.596107+0000 mgr.y (mgr.25277) 1727 : cluster 0 pgmap v1960: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:15.710 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:15 smithi133 bash[102858]: audit 2023-10-09T05:45:14.733929+0000 mon.a (mon.0) 2248 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:45:15.710 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:15 smithi133 bash[102858]: audit 2023-10-09T05:45:15.110927+0000 mon.a (mon.0) 2249 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:45:15.711 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:15 smithi133 bash[102858]: audit 2023-10-09T05:45:15.119721+0000 mon.a (mon.0) 2250 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:45:15.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:15 smithi194 bash[92666]: audit 2023-10-09T05:45:14.457938+0000 mon.a (mon.0) 2247 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:45:15.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:15 smithi194 bash[92666]: cluster 2023-10-09T05:45:14.596107+0000 mgr.y (mgr.25277) 1727 : cluster 0 pgmap v1960: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:15.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:15 smithi194 bash[92666]: audit 2023-10-09T05:45:14.733929+0000 mon.a (mon.0) 2248 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:45:15.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:15 smithi194 bash[92666]: audit 2023-10-09T05:45:15.110927+0000 mon.a (mon.0) 2249 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:45:15.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:15 smithi194 bash[92666]: audit 2023-10-09T05:45:15.119721+0000 mon.a (mon.0) 2250 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:45:18.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:17 smithi133 bash[102858]: cluster 2023-10-09T05:45:16.596919+0000 mgr.y (mgr.25277) 1728 : cluster 0 pgmap v1961: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:18.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:17 smithi133 bash[105589]: cluster 2023-10-09T05:45:16.596919+0000 mgr.y (mgr.25277) 1728 : cluster 0 pgmap v1961: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:18.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:17 smithi194 bash[92666]: cluster 2023-10-09T05:45:16.596919+0000 mgr.y (mgr.25277) 1728 : cluster 0 pgmap v1961: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:18.199 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:45:18.199 INFO:teuthology.orchestra.run.smithi133.stdout:{"epoch":449,"fsid":"855b5206-665c-11ee-8db5-212e2dc638e7","created":"2023-10-09T04:31:08.633675+0000","modified":"2023-10-09T05:45:08.647138+0000","last_up_change":"2023-10-09T05:02:08.616931+0000","last_in_change":"2023-10-09T05:42:11.577502+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T04:36:44.260202+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"211","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T04:42:47.518485+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"420","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":420,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T04:44:51.685755+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T04:44:52.162649+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"32633ec2-9b8f-42d8-bdfb-5c9668a26071","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":83,"up_from":88,"up_thru":424,"down_at":84,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6802","nonce":2098404708},{"type":"v1","addr":"172.21.15.133:6803","nonce":2098404708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6804","nonce":2098404708},{"type":"v1","addr":"172.21.15.133:6805","nonce":2098404708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6808","nonce":2098404708},{"type":"v1","addr":"172.21.15.133:6809","nonce":2098404708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6806","nonce":2098404708},{"type":"v1","addr":"172.21.15.133:6807","nonce":2098404708}]},"public_addr":"172.21.15.133:6803/2098404708","cluster_addr":"172.21.15.133:6805/2098404708","heartbeat_back_addr":"172.21.15.133:6809/2098404708","heartbeat_front_addr":"172.21.15.133:6807/2098404708","state":["exists","up"]},{"osd":1,"uuid":"7f2a961d-a790-4337-91c7-1e45f900bbd4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":92,"up_from":96,"up_thru":432,"down_at":93,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6810","nonce":2668659948},{"type":"v1","addr":"172.21.15.133:6811","nonce":2668659948}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6812","nonce":2668659948},{"type":"v1","addr":"172.21.15.133:6813","nonce":2668659948}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6816","nonce":2668659948},{"type":"v1","addr":"172.21.15.133:6817","nonce":2668659948}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6814","nonce":2668659948},{"type":"v1","addr":"172.21.15.133:6815","nonce":2668659948}]},"public_addr":"172.21.15.133:6811/2668659948","cluster_addr":"172.21.15.133:6813/2668659948","heartbeat_back_addr":"172.21.15.133:6817/2668659948","heartbeat_front_addr":"172.21.15.133:6815/2668659948","state":["exists","up"]},{"osd":2,"uuid":"08751330-a327-4bf8-9c4d-9ff986b1c67f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":102,"up_from":106,"up_thru":424,"down_at":103,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6818","nonce":3595178639},{"type":"v1","addr":"172.21.15.133:6819","nonce":3595178639}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6820","nonce":3595178639},{"type":"v1","addr":"172.21.15.133:6821","nonce":3595178639}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6824","nonce":3595178639},{"type":"v1","addr":"172.21.15.133:6825","nonce":3595178639}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6822","nonce":3595178639},{"type":"v1","addr":"172.21.15.133:6823","nonce":3595178639}]},"public_addr":"172.21.15.133:6819/3595178639","cluster_addr":"172.21.15.133:6821/3595178639","heartbeat_back_addr":"172.21.15.133:6825/3595178639","heartbeat_front_addr":"172.21.15.133:6823/3595178639","state":["exists","up"]},{"osd":3,"uuid":"56bbf807-c8f7-4226-b46e-802b539274ca","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":117,"up_from":123,"up_thru":432,"down_at":118,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6826","nonce":1082233372},{"type":"v1","addr":"172.21.15.133:6827","nonce":1082233372}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6828","nonce":1082233372},{"type":"v1","addr":"172.21.15.133:6829","nonce":1082233372}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6832","nonce":1082233372},{"type":"v1","addr":"172.21.15.133:6833","nonce":1082233372}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6830","nonce":1082233372},{"type":"v1","addr":"172.21.15.133:6831","nonce":1082233372}]},"public_addr":"172.21.15.133:6827/1082233372","cluster_addr":"172.21.15.133:6829/1082233372","heartbeat_back_addr":"172.21.15.133:6833/1082233372","heartbeat_front_addr":"172.21.15.133:6831/1082233372","state":["exists","up"]},{"osd":4,"uuid":"509ec8cc-5042-428e-9a69-8e602b5afce8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":435,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6801","nonce":1874338403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6803","nonce":1874338403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6807","nonce":1874338403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6805","nonce":1874338403}]},"public_addr":"172.21.15.194:6801/1874338403","cluster_addr":"172.21.15.194:6803/1874338403","heartbeat_back_addr":"172.21.15.194:6807/1874338403","heartbeat_front_addr":"172.21.15.194:6805/1874338403","state":["exists","up"]},{"osd":5,"uuid":"bbdc1e67-fd5b-49d0-9e42-cf151d167943","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":424,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6809","nonce":3989986662}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6811","nonce":3989986662}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6815","nonce":3989986662}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6813","nonce":3989986662}]},"public_addr":"172.21.15.194:6809/3989986662","cluster_addr":"172.21.15.194:6811/3989986662","heartbeat_back_addr":"172.21.15.194:6815/3989986662","heartbeat_front_addr":"172.21.15.194:6813/3989986662","state":["exists","up"]},{"osd":6,"uuid":"7b69d014-f4c0-4e73-a14e-7d51b7ee3ea2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":447,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6817","nonce":2607555245}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6819","nonce":2607555245}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6823","nonce":2607555245}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6820","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6821","nonce":2607555245}]},"public_addr":"172.21.15.194:6817/2607555245","cluster_addr":"172.21.15.194:6819/2607555245","heartbeat_back_addr":"172.21.15.194:6823/2607555245","heartbeat_front_addr":"172.21.15.194:6821/2607555245","state":["exists","up"]},{"osd":7,"uuid":"02506268-102c-4654-9b5c-68d36c28f7f7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":424,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6824","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6825","nonce":1051622545}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6827","nonce":1051622545}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6831","nonce":1051622545}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6828","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6829","nonce":1051622545}]},"public_addr":"172.21.15.194:6825/1051622545","cluster_addr":"172.21.15.194:6827/1051622545","heartbeat_back_addr":"172.21.15.194:6831/1051622545","heartbeat_front_addr":"172.21.15.194:6829/1051622545","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:34:53.028753+0000","dead_epoch":83},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:35:47.001796+0000","dead_epoch":92},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:36:41.220545+0000","dead_epoch":102},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:37:36.237434+0000","dead_epoch":117},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:38:28.301119+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:39:20.099747+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:40:12.106681+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:41:04.212546+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.6","mappings":[{"from":0,"to":2}]},{"pgid":"3.1d","mappings":[{"from":6,"to":2}]},{"pgid":"4.e","mappings":[{"from":1,"to":2}]},{"pgid":"4.17","mappings":[{"from":3,"to":5}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]},{"pgid":"4.1c","mappings":[{"from":3,"to":5}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.133:6801/3027596458":"2023-10-10T04:52:13.301898+0000","172.21.15.133:6800/3027596458":"2023-10-10T04:52:13.301898+0000","172.21.15.133:0/499357090":"2023-10-10T04:52:13.301898+0000","172.21.15.133:6800/788799879":"2023-10-10T04:32:17.611609+0000","172.21.15.133:6801/788799879":"2023-10-10T04:32:17.611609+0000","172.21.15.194:0/430115207":"2023-10-10T04:50:55.051958+0000","172.21.15.194:0/3848858405":"2023-10-10T04:50:55.051958+0000","172.21.15.133:6834/4227843513":"2023-10-10T04:45:00.021661+0000","172.21.15.194:0/2434253169":"2023-10-10T04:50:55.051958+0000","172.21.15.133:6800/768614786":"2023-10-10T04:42:05.063462+0000","172.21.15.133:6801/768614786":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/1575170036":"2023-10-10T04:49:30.896539+0000","172.21.15.133:0/320185713":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/1787141186":"2023-10-10T04:49:30.896539+0000","172.21.15.133:0/3141871970":"2023-10-10T04:52:13.301898+0000","172.21.15.133:0/3636510019":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/2419271305":"2023-10-10T04:49:30.896539+0000","172.21.15.133:0/740467257":"2023-10-10T04:31:42.639571+0000","172.21.15.133:6800/275314455":"2023-10-10T04:49:30.896539+0000","172.21.15.133:6835/4227843513":"2023-10-10T04:45:00.021661+0000","172.21.15.194:0/2197565959":"2023-10-10T04:50:55.051958+0000","172.21.15.133:0/55818837":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/1793292086":"2023-10-10T04:32:17.611609+0000","172.21.15.133:6801/275314455":"2023-10-10T04:49:30.896539+0000","172.21.15.133:0/2159982120":"2023-10-10T04:49:30.896539+0000","172.21.15.194:0/1505438352":"2023-10-10T04:50:55.051958+0000","172.21.15.133:0/1309073801":"2023-10-10T04:42:05.063462+0000","172.21.15.194:0/152828820":"2023-10-10T04:50:55.051958+0000","172.21.15.133:0/4290202542":"2023-10-10T04:32:17.611609+0000","172.21.15.194:6834/2184279986":"2023-10-10T04:50:55.051958+0000","172.21.15.194:6835/2184279986":"2023-10-10T04:50:55.051958+0000","172.21.15.133:0/1411744140":"2023-10-10T04:31:42.639571+0000","172.21.15.133:0/3619387632":"2023-10-10T04:32:17.611609+0000","172.21.15.133:6800/1761810618":"2023-10-10T04:31:42.639571+0000","172.21.15.133:6801/1761810618":"2023-10-10T04:31:42.639571+0000","172.21.15.133:0/4173745360":"2023-10-10T04:31:42.639571+0000","172.21.15.133:0/70449665":"2023-10-10T04:52:13.301898+0000","172.21.15.133:0/3777348175":"2023-10-10T04:52:13.301898+0000","172.21.15.133:0/2479656939":"2023-10-10T04:52:13.301898+0000","172.21.15.133:0/4053521787":"2023-10-10T04:52:13.301898+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":15,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:45:18.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:18 smithi133 bash[102858]: audit 2023-10-09T05:45:18.195549+0000 mon.a (mon.0) 2251 : audit 0 from='client.? 172.21.15.133:0/3191570114' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:45:18.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:18 smithi133 bash[105589]: audit 2023-10-09T05:45:18.195549+0000 mon.a (mon.0) 2251 : audit 0 from='client.? 172.21.15.133:0/3191570114' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:45:19.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:18 smithi194 bash[92666]: audit 2023-10-09T05:45:18.195549+0000 mon.a (mon.0) 2251 : audit 0 from='client.? 172.21.15.133:0/3191570114' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:45:19.157 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T05:45:19.157 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T05:45:20.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:19 smithi133 bash[102858]: cluster 2023-10-09T05:45:18.597566+0000 mgr.y (mgr.25277) 1729 : cluster 0 pgmap v1962: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:20.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:19 smithi133 bash[105589]: cluster 2023-10-09T05:45:18.597566+0000 mgr.y (mgr.25277) 1729 : cluster 0 pgmap v1962: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:20.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:19 smithi194 bash[92666]: cluster 2023-10-09T05:45:18.597566+0000 mgr.y (mgr.25277) 1729 : cluster 0 pgmap v1962: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:22.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:21 smithi133 bash[105589]: cluster 2023-10-09T05:45:20.598557+0000 mgr.y (mgr.25277) 1730 : cluster 0 pgmap v1963: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:22.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:21 smithi133 bash[105589]: audit 2023-10-09T05:45:21.181604+0000 mon.a (mon.0) 2252 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:45:22.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:21 smithi133 bash[105589]: audit 2023-10-09T05:45:21.192083+0000 mon.a (mon.0) 2253 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:45:22.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:21 smithi133 bash[102858]: cluster 2023-10-09T05:45:20.598557+0000 mgr.y (mgr.25277) 1730 : cluster 0 pgmap v1963: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:22.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:21 smithi133 bash[102858]: audit 2023-10-09T05:45:21.181604+0000 mon.a (mon.0) 2252 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:45:22.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:21 smithi133 bash[102858]: audit 2023-10-09T05:45:21.192083+0000 mon.a (mon.0) 2253 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:45:22.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:21 smithi194 bash[92666]: cluster 2023-10-09T05:45:20.598557+0000 mgr.y (mgr.25277) 1730 : cluster 0 pgmap v1963: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:22.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:21 smithi194 bash[92666]: audit 2023-10-09T05:45:21.181604+0000 mon.a (mon.0) 2252 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:45:22.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:21 smithi194 bash[92666]: audit 2023-10-09T05:45:21.192083+0000 mon.a (mon.0) 2253 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:45:23.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:22 smithi194 bash[92666]: audit 2023-10-09T05:45:21.860762+0000 mon.a (mon.0) 2254 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:45:23.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:22 smithi194 bash[92666]: audit 2023-10-09T05:45:21.868301+0000 mon.a (mon.0) 2255 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:45:23.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:22 smithi194 bash[92666]: audit 2023-10-09T05:45:22.337091+0000 mon.a (mon.0) 2256 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:23.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:22 smithi194 bash[92666]: audit 2023-10-09T05:45:22.338735+0000 mon.a (mon.0) 2257 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:45:23.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:22 smithi194 bash[92666]: audit 2023-10-09T05:45:22.347253+0000 mon.a (mon.0) 2258 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:45:23.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:22 smithi194 bash[92666]: cluster 2023-10-09T05:45:22.599329+0000 mgr.y (mgr.25277) 1731 : cluster 0 pgmap v1964: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:23.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:22 smithi133 bash[102858]: audit 2023-10-09T05:45:21.860762+0000 mon.a (mon.0) 2254 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:45:23.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:22 smithi133 bash[102858]: audit 2023-10-09T05:45:21.868301+0000 mon.a (mon.0) 2255 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:45:23.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:22 smithi133 bash[102858]: audit 2023-10-09T05:45:22.337091+0000 mon.a (mon.0) 2256 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:23.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:22 smithi133 bash[102858]: audit 2023-10-09T05:45:22.338735+0000 mon.a (mon.0) 2257 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:45:23.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:22 smithi133 bash[102858]: audit 2023-10-09T05:45:22.347253+0000 mon.a (mon.0) 2258 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:45:23.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:22 smithi133 bash[102858]: cluster 2023-10-09T05:45:22.599329+0000 mgr.y (mgr.25277) 1731 : cluster 0 pgmap v1964: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:23.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:22 smithi133 bash[105589]: audit 2023-10-09T05:45:21.860762+0000 mon.a (mon.0) 2254 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:45:23.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:22 smithi133 bash[105589]: audit 2023-10-09T05:45:21.868301+0000 mon.a (mon.0) 2255 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:45:23.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:22 smithi133 bash[105589]: audit 2023-10-09T05:45:22.337091+0000 mon.a (mon.0) 2256 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:23.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:22 smithi133 bash[105589]: audit 2023-10-09T05:45:22.338735+0000 mon.a (mon.0) 2257 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:45:23.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:22 smithi133 bash[105589]: audit 2023-10-09T05:45:22.347253+0000 mon.a (mon.0) 2258 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:45:23.299 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:22 smithi133 bash[105589]: cluster 2023-10-09T05:45:22.599329+0000 mgr.y (mgr.25277) 1731 : cluster 0 pgmap v1964: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:23.700 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:24.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:45:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:45:24] "GET /metrics HTTP/1.1" 200 34116 "" "Prometheus/2.43.0" 2023-10-09T05:45:25.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:25 smithi194 bash[92666]: cluster 2023-10-09T05:45:24.600274+0000 mgr.y (mgr.25277) 1732 : cluster 0 pgmap v1965: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:25.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:25 smithi133 bash[102858]: cluster 2023-10-09T05:45:24.600274+0000 mgr.y (mgr.25277) 1732 : cluster 0 pgmap v1965: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:25.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:25 smithi133 bash[105589]: cluster 2023-10-09T05:45:24.600274+0000 mgr.y (mgr.25277) 1732 : cluster 0 pgmap v1965: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:27.396 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:45:27.396 INFO:teuthology.orchestra.run.smithi133.stdout:{"epoch":449,"fsid":"855b5206-665c-11ee-8db5-212e2dc638e7","created":"2023-10-09T04:31:08.633675+0000","modified":"2023-10-09T05:45:08.647138+0000","last_up_change":"2023-10-09T05:02:08.616931+0000","last_in_change":"2023-10-09T05:42:11.577502+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T04:36:44.260202+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"211","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T04:42:47.518485+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"420","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":420,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T04:44:51.685755+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T04:44:52.162649+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"32633ec2-9b8f-42d8-bdfb-5c9668a26071","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":83,"up_from":88,"up_thru":424,"down_at":84,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6802","nonce":2098404708},{"type":"v1","addr":"172.21.15.133:6803","nonce":2098404708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6804","nonce":2098404708},{"type":"v1","addr":"172.21.15.133:6805","nonce":2098404708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6808","nonce":2098404708},{"type":"v1","addr":"172.21.15.133:6809","nonce":2098404708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6806","nonce":2098404708},{"type":"v1","addr":"172.21.15.133:6807","nonce":2098404708}]},"public_addr":"172.21.15.133:6803/2098404708","cluster_addr":"172.21.15.133:6805/2098404708","heartbeat_back_addr":"172.21.15.133:6809/2098404708","heartbeat_front_addr":"172.21.15.133:6807/2098404708","state":["exists","up"]},{"osd":1,"uuid":"7f2a961d-a790-4337-91c7-1e45f900bbd4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":92,"up_from":96,"up_thru":432,"down_at":93,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6810","nonce":2668659948},{"type":"v1","addr":"172.21.15.133:6811","nonce":2668659948}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6812","nonce":2668659948},{"type":"v1","addr":"172.21.15.133:6813","nonce":2668659948}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6816","nonce":2668659948},{"type":"v1","addr":"172.21.15.133:6817","nonce":2668659948}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6814","nonce":2668659948},{"type":"v1","addr":"172.21.15.133:6815","nonce":2668659948}]},"public_addr":"172.21.15.133:6811/2668659948","cluster_addr":"172.21.15.133:6813/2668659948","heartbeat_back_addr":"172.21.15.133:6817/2668659948","heartbeat_front_addr":"172.21.15.133:6815/2668659948","state":["exists","up"]},{"osd":2,"uuid":"08751330-a327-4bf8-9c4d-9ff986b1c67f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":102,"up_from":106,"up_thru":424,"down_at":103,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6818","nonce":3595178639},{"type":"v1","addr":"172.21.15.133:6819","nonce":3595178639}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6820","nonce":3595178639},{"type":"v1","addr":"172.21.15.133:6821","nonce":3595178639}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6824","nonce":3595178639},{"type":"v1","addr":"172.21.15.133:6825","nonce":3595178639}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6822","nonce":3595178639},{"type":"v1","addr":"172.21.15.133:6823","nonce":3595178639}]},"public_addr":"172.21.15.133:6819/3595178639","cluster_addr":"172.21.15.133:6821/3595178639","heartbeat_back_addr":"172.21.15.133:6825/3595178639","heartbeat_front_addr":"172.21.15.133:6823/3595178639","state":["exists","up"]},{"osd":3,"uuid":"56bbf807-c8f7-4226-b46e-802b539274ca","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":117,"up_from":123,"up_thru":432,"down_at":118,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6826","nonce":1082233372},{"type":"v1","addr":"172.21.15.133:6827","nonce":1082233372}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6828","nonce":1082233372},{"type":"v1","addr":"172.21.15.133:6829","nonce":1082233372}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6832","nonce":1082233372},{"type":"v1","addr":"172.21.15.133:6833","nonce":1082233372}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.133:6830","nonce":1082233372},{"type":"v1","addr":"172.21.15.133:6831","nonce":1082233372}]},"public_addr":"172.21.15.133:6827/1082233372","cluster_addr":"172.21.15.133:6829/1082233372","heartbeat_back_addr":"172.21.15.133:6833/1082233372","heartbeat_front_addr":"172.21.15.133:6831/1082233372","state":["exists","up"]},{"osd":4,"uuid":"509ec8cc-5042-428e-9a69-8e602b5afce8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":435,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6801","nonce":1874338403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6803","nonce":1874338403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6807","nonce":1874338403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":1874338403},{"type":"v1","addr":"172.21.15.194:6805","nonce":1874338403}]},"public_addr":"172.21.15.194:6801/1874338403","cluster_addr":"172.21.15.194:6803/1874338403","heartbeat_back_addr":"172.21.15.194:6807/1874338403","heartbeat_front_addr":"172.21.15.194:6805/1874338403","state":["exists","up"]},{"osd":5,"uuid":"bbdc1e67-fd5b-49d0-9e42-cf151d167943","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":424,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6809","nonce":3989986662}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6811","nonce":3989986662}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6815","nonce":3989986662}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":3989986662},{"type":"v1","addr":"172.21.15.194:6813","nonce":3989986662}]},"public_addr":"172.21.15.194:6809/3989986662","cluster_addr":"172.21.15.194:6811/3989986662","heartbeat_back_addr":"172.21.15.194:6815/3989986662","heartbeat_front_addr":"172.21.15.194:6813/3989986662","state":["exists","up"]},{"osd":6,"uuid":"7b69d014-f4c0-4e73-a14e-7d51b7ee3ea2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":447,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6817","nonce":2607555245}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6819","nonce":2607555245}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6823","nonce":2607555245}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6820","nonce":2607555245},{"type":"v1","addr":"172.21.15.194:6821","nonce":2607555245}]},"public_addr":"172.21.15.194:6817/2607555245","cluster_addr":"172.21.15.194:6819/2607555245","heartbeat_back_addr":"172.21.15.194:6823/2607555245","heartbeat_front_addr":"172.21.15.194:6821/2607555245","state":["exists","up"]},{"osd":7,"uuid":"02506268-102c-4654-9b5c-68d36c28f7f7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":424,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6824","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6825","nonce":1051622545}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6827","nonce":1051622545}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6831","nonce":1051622545}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6828","nonce":1051622545},{"type":"v1","addr":"172.21.15.194:6829","nonce":1051622545}]},"public_addr":"172.21.15.194:6825/1051622545","cluster_addr":"172.21.15.194:6827/1051622545","heartbeat_back_addr":"172.21.15.194:6831/1051622545","heartbeat_front_addr":"172.21.15.194:6829/1051622545","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:34:53.028753+0000","dead_epoch":83},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:35:47.001796+0000","dead_epoch":92},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:36:41.220545+0000","dead_epoch":102},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:37:36.237434+0000","dead_epoch":117},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:38:28.301119+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:39:20.099747+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:40:12.106681+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:41:04.212546+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.6","mappings":[{"from":0,"to":2}]},{"pgid":"3.1d","mappings":[{"from":6,"to":2}]},{"pgid":"4.e","mappings":[{"from":1,"to":2}]},{"pgid":"4.17","mappings":[{"from":3,"to":5}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]},{"pgid":"4.1c","mappings":[{"from":3,"to":5}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.133:6801/3027596458":"2023-10-10T04:52:13.301898+0000","172.21.15.133:6800/3027596458":"2023-10-10T04:52:13.301898+0000","172.21.15.133:0/499357090":"2023-10-10T04:52:13.301898+0000","172.21.15.133:6800/788799879":"2023-10-10T04:32:17.611609+0000","172.21.15.133:6801/788799879":"2023-10-10T04:32:17.611609+0000","172.21.15.194:0/430115207":"2023-10-10T04:50:55.051958+0000","172.21.15.194:0/3848858405":"2023-10-10T04:50:55.051958+0000","172.21.15.133:6834/4227843513":"2023-10-10T04:45:00.021661+0000","172.21.15.194:0/2434253169":"2023-10-10T04:50:55.051958+0000","172.21.15.133:6800/768614786":"2023-10-10T04:42:05.063462+0000","172.21.15.133:6801/768614786":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/1575170036":"2023-10-10T04:49:30.896539+0000","172.21.15.133:0/320185713":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/1787141186":"2023-10-10T04:49:30.896539+0000","172.21.15.133:0/3141871970":"2023-10-10T04:52:13.301898+0000","172.21.15.133:0/3636510019":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/2419271305":"2023-10-10T04:49:30.896539+0000","172.21.15.133:0/740467257":"2023-10-10T04:31:42.639571+0000","172.21.15.133:6800/275314455":"2023-10-10T04:49:30.896539+0000","172.21.15.133:6835/4227843513":"2023-10-10T04:45:00.021661+0000","172.21.15.194:0/2197565959":"2023-10-10T04:50:55.051958+0000","172.21.15.133:0/55818837":"2023-10-10T04:42:05.063462+0000","172.21.15.133:0/1793292086":"2023-10-10T04:32:17.611609+0000","172.21.15.133:6801/275314455":"2023-10-10T04:49:30.896539+0000","172.21.15.133:0/2159982120":"2023-10-10T04:49:30.896539+0000","172.21.15.194:0/1505438352":"2023-10-10T04:50:55.051958+0000","172.21.15.133:0/1309073801":"2023-10-10T04:42:05.063462+0000","172.21.15.194:0/152828820":"2023-10-10T04:50:55.051958+0000","172.21.15.133:0/4290202542":"2023-10-10T04:32:17.611609+0000","172.21.15.194:6834/2184279986":"2023-10-10T04:50:55.051958+0000","172.21.15.194:6835/2184279986":"2023-10-10T04:50:55.051958+0000","172.21.15.133:0/1411744140":"2023-10-10T04:31:42.639571+0000","172.21.15.133:0/3619387632":"2023-10-10T04:32:17.611609+0000","172.21.15.133:6800/1761810618":"2023-10-10T04:31:42.639571+0000","172.21.15.133:6801/1761810618":"2023-10-10T04:31:42.639571+0000","172.21.15.133:0/4173745360":"2023-10-10T04:31:42.639571+0000","172.21.15.133:0/70449665":"2023-10-10T04:52:13.301898+0000","172.21.15.133:0/3777348175":"2023-10-10T04:52:13.301898+0000","172.21.15.133:0/2479656939":"2023-10-10T04:52:13.301898+0000","172.21.15.133:0/4053521787":"2023-10-10T04:52:13.301898+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":15,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:45:27.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:27 smithi133 bash[105589]: cluster 2023-10-09T05:45:26.600989+0000 mgr.y (mgr.25277) 1733 : cluster 0 pgmap v1966: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:27.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:27 smithi133 bash[105589]: audit 2023-10-09T05:45:27.392702+0000 mon.a (mon.0) 2259 : audit 0 from='client.? 172.21.15.133:0/2993991703' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:45:27.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:27 smithi133 bash[102858]: cluster 2023-10-09T05:45:26.600989+0000 mgr.y (mgr.25277) 1733 : cluster 0 pgmap v1966: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:27.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:27 smithi133 bash[102858]: audit 2023-10-09T05:45:27.392702+0000 mon.a (mon.0) 2259 : audit 0 from='client.? 172.21.15.133:0/2993991703' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:45:28.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:27 smithi194 bash[92666]: cluster 2023-10-09T05:45:26.600989+0000 mgr.y (mgr.25277) 1733 : cluster 0 pgmap v1966: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:28.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:27 smithi194 bash[92666]: audit 2023-10-09T05:45:27.392702+0000 mon.a (mon.0) 2259 : audit 0 from='client.? 172.21.15.133:0/2993991703' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:45:28.333 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T05:45:28.333 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T05:45:28.334 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T05:45:28.334 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T05:45:28.334 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T05:45:28.335 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T05:45:28.335 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T05:45:28.336 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T05:45:29.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:28 smithi133 bash[102858]: audit 2023-10-09T05:45:28.503404+0000 mon.a (mon.0) 2260 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:29.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:28 smithi133 bash[105589]: audit 2023-10-09T05:45:28.503404+0000 mon.a (mon.0) 2260 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:29.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:28 smithi194 bash[92666]: audit 2023-10-09T05:45:28.503404+0000 mon.a (mon.0) 2260 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:30.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:29 smithi133 bash[105589]: cluster 2023-10-09T05:45:28.601457+0000 mgr.y (mgr.25277) 1734 : cluster 0 pgmap v1967: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:30.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:29 smithi133 bash[102858]: cluster 2023-10-09T05:45:28.601457+0000 mgr.y (mgr.25277) 1734 : cluster 0 pgmap v1967: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:30.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:29 smithi194 bash[92666]: cluster 2023-10-09T05:45:28.601457+0000 mgr.y (mgr.25277) 1734 : cluster 0 pgmap v1967: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:32.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:31 smithi133 bash[105589]: cluster 2023-10-09T05:45:30.603500+0000 mgr.y (mgr.25277) 1735 : cluster 0 pgmap v1968: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:32.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:31 smithi133 bash[102858]: cluster 2023-10-09T05:45:30.603500+0000 mgr.y (mgr.25277) 1735 : cluster 0 pgmap v1968: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:32.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:31 smithi194 bash[92666]: cluster 2023-10-09T05:45:30.603500+0000 mgr.y (mgr.25277) 1735 : cluster 0 pgmap v1968: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:33.232 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:33.235 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:33.236 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:33.236 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:33.237 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:33.237 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:33.240 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:33.241 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:34.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:33 smithi133 bash[102858]: cluster 2023-10-09T05:45:32.604318+0000 mgr.y (mgr.25277) 1736 : cluster 0 pgmap v1969: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:34.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:33 smithi133 bash[105589]: cluster 2023-10-09T05:45:32.604318+0000 mgr.y (mgr.25277) 1736 : cluster 0 pgmap v1969: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:34.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:33 smithi194 bash[92666]: cluster 2023-10-09T05:45:32.604318+0000 mgr.y (mgr.25277) 1736 : cluster 0 pgmap v1969: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:34.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:45:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:45:34] "GET /metrics HTTP/1.1" 200 34113 "" "Prometheus/2.43.0" 2023-10-09T05:45:35.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:35 smithi194 bash[92666]: cluster 2023-10-09T05:45:34.605269+0000 mgr.y (mgr.25277) 1737 : cluster 0 pgmap v1970: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:35.399 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:35 smithi133 bash[105589]: cluster 2023-10-09T05:45:34.605269+0000 mgr.y (mgr.25277) 1737 : cluster 0 pgmap v1970: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:35.399 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:35 smithi133 bash[102858]: cluster 2023-10-09T05:45:34.605269+0000 mgr.y (mgr.25277) 1737 : cluster 0 pgmap v1970: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:37.698 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:37 smithi133 bash[102858]: cluster 2023-10-09T05:45:36.605864+0000 mgr.y (mgr.25277) 1738 : cluster 0 pgmap v1971: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:38.049 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:37 smithi133 bash[105589]: cluster 2023-10-09T05:45:36.605864+0000 mgr.y (mgr.25277) 1738 : cluster 0 pgmap v1971: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:38.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:37 smithi194 bash[92666]: cluster 2023-10-09T05:45:36.605864+0000 mgr.y (mgr.25277) 1738 : cluster 0 pgmap v1971: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:40.047 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:39 smithi133 bash[105589]: cluster 2023-10-09T05:45:38.606263+0000 mgr.y (mgr.25277) 1739 : cluster 0 pgmap v1972: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:40.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:39 smithi133 bash[102858]: cluster 2023-10-09T05:45:38.606263+0000 mgr.y (mgr.25277) 1739 : cluster 0 pgmap v1972: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:40.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:39 smithi194 bash[92666]: cluster 2023-10-09T05:45:38.606263+0000 mgr.y (mgr.25277) 1739 : cluster 0 pgmap v1972: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:42.059 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:41 smithi133 bash[102858]: cluster 2023-10-09T05:45:40.606994+0000 mgr.y (mgr.25277) 1740 : cluster 0 pgmap v1973: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:42.059 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:41 smithi133 bash[105589]: cluster 2023-10-09T05:45:40.606994+0000 mgr.y (mgr.25277) 1740 : cluster 0 pgmap v1973: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:42.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:41 smithi194 bash[92666]: cluster 2023-10-09T05:45:40.606994+0000 mgr.y (mgr.25277) 1740 : cluster 0 pgmap v1973: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:43.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:43 smithi133 bash[105589]: cluster 2023-10-09T05:45:42.607452+0000 mgr.y (mgr.25277) 1741 : cluster 0 pgmap v1974: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:43.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:43 smithi133 bash[105589]: audit 2023-10-09T05:45:43.503879+0000 mon.a (mon.0) 2261 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:43.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:43 smithi133 bash[102858]: cluster 2023-10-09T05:45:42.607452+0000 mgr.y (mgr.25277) 1741 : cluster 0 pgmap v1974: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:43.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:43 smithi133 bash[102858]: audit 2023-10-09T05:45:43.503879+0000 mon.a (mon.0) 2261 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:44.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:43 smithi194 bash[92666]: cluster 2023-10-09T05:45:42.607452+0000 mgr.y (mgr.25277) 1741 : cluster 0 pgmap v1974: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:44.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:43 smithi194 bash[92666]: audit 2023-10-09T05:45:43.503879+0000 mon.a (mon.0) 2261 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:44.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:45:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:45:44] "GET /metrics HTTP/1.1" 200 34114 "" "Prometheus/2.43.0" 2023-10-09T05:45:45.260 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:45 smithi133 bash[102858]: cluster 2023-10-09T05:45:44.608347+0000 mgr.y (mgr.25277) 1742 : cluster 0 pgmap v1975: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:45.260 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:45 smithi133 bash[105589]: cluster 2023-10-09T05:45:44.608347+0000 mgr.y (mgr.25277) 1742 : cluster 0 pgmap v1975: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:45.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:45 smithi194 bash[92666]: cluster 2023-10-09T05:45:44.608347+0000 mgr.y (mgr.25277) 1742 : cluster 0 pgmap v1975: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:48.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:47 smithi133 bash[105589]: cluster 2023-10-09T05:45:46.609309+0000 mgr.y (mgr.25277) 1743 : cluster 0 pgmap v1976: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:48.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:47 smithi133 bash[102858]: cluster 2023-10-09T05:45:46.609309+0000 mgr.y (mgr.25277) 1743 : cluster 0 pgmap v1976: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:48.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:47 smithi194 bash[92666]: cluster 2023-10-09T05:45:46.609309+0000 mgr.y (mgr.25277) 1743 : cluster 0 pgmap v1976: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:49.895 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:49 smithi133 bash[102858]: cluster 2023-10-09T05:45:48.609753+0000 mgr.y (mgr.25277) 1744 : cluster 0 pgmap v1977: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:50.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:49 smithi194 bash[92666]: cluster 2023-10-09T05:45:48.609753+0000 mgr.y (mgr.25277) 1744 : cluster 0 pgmap v1977: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:50.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:49 smithi133 bash[105589]: cluster 2023-10-09T05:45:48.609753+0000 mgr.y (mgr.25277) 1744 : cluster 0 pgmap v1977: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:51.327 INFO:teuthology.orchestra.run.smithi133.stdout:128849019691 2023-10-09T05:45:51.327 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T05:45:51.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:51 smithi133 bash[105589]: cluster 2023-10-09T05:45:50.610748+0000 mgr.y (mgr.25277) 1745 : cluster 0 pgmap v1978: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:51.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:51 smithi133 bash[102858]: cluster 2023-10-09T05:45:50.610748+0000 mgr.y (mgr.25277) 1745 : cluster 0 pgmap v1978: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:52.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:51 smithi194 bash[92666]: cluster 2023-10-09T05:45:50.610748+0000 mgr.y (mgr.25277) 1745 : cluster 0 pgmap v1978: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:53.170 INFO:teuthology.orchestra.run.smithi133.stdout:163208758047 2023-10-09T05:45:53.171 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T05:45:53.688 INFO:teuthology.orchestra.run.smithi133.stdout:455266533916 2023-10-09T05:45:53.689 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T05:45:53.934 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:53 smithi133 bash[105589]: cluster 2023-10-09T05:45:52.611231+0000 mgr.y (mgr.25277) 1746 : cluster 0 pgmap v1979: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:53.934 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:53 smithi133 bash[102858]: cluster 2023-10-09T05:45:52.611231+0000 mgr.y (mgr.25277) 1746 : cluster 0 pgmap v1979: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:54.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:53 smithi194 bash[92666]: cluster 2023-10-09T05:45:52.611231+0000 mgr.y (mgr.25277) 1746 : cluster 0 pgmap v1979: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:54.547 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:45:54 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:45:54] "GET /metrics HTTP/1.1" 200 34114 "" "Prometheus/2.43.0" 2023-10-09T05:45:55.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:55 smithi194 bash[92666]: cluster 2023-10-09T05:45:54.612232+0000 mgr.y (mgr.25277) 1747 : cluster 0 pgmap v1980: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:55.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:55 smithi133 bash[102858]: cluster 2023-10-09T05:45:54.612232+0000 mgr.y (mgr.25277) 1747 : cluster 0 pgmap v1980: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:55.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:55 smithi133 bash[105589]: cluster 2023-10-09T05:45:54.612232+0000 mgr.y (mgr.25277) 1747 : cluster 0 pgmap v1980: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:57.566 INFO:teuthology.orchestra.run.smithi133.stdout:412316860997 2023-10-09T05:45:57.566 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T05:45:57.569 INFO:teuthology.orchestra.run.smithi133.stdout:197568496406 2023-10-09T05:45:57.569 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T05:45:57.643 INFO:teuthology.orchestra.run.smithi133.stdout:231928234764 2023-10-09T05:45:57.644 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T05:45:57.695 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:57 smithi133 bash[105589]: cluster 2023-10-09T05:45:56.613234+0000 mgr.y (mgr.25277) 1748 : cluster 0 pgmap v1981: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:57.695 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:57 smithi133 bash[102858]: cluster 2023-10-09T05:45:56.613234+0000 mgr.y (mgr.25277) 1748 : cluster 0 pgmap v1981: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:57.920 INFO:teuthology.orchestra.run.smithi133.stdout:528280977935 2023-10-09T05:45:57.921 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T05:45:57.934 INFO:teuthology.orchestra.run.smithi133.stdout:377957122639 2023-10-09T05:45:57.934 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T05:45:58.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:57 smithi194 bash[92666]: cluster 2023-10-09T05:45:56.613234+0000 mgr.y (mgr.25277) 1748 : cluster 0 pgmap v1981: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:58.412 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:58.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:58 smithi133 bash[105589]: audit 2023-10-09T05:45:58.504008+0000 mon.a (mon.0) 2262 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:58.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:58 smithi133 bash[102858]: audit 2023-10-09T05:45:58.504008+0000 mon.a (mon.0) 2262 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:59.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:58 smithi194 bash[92666]: audit 2023-10-09T05:45:58.504008+0000 mon.a (mon.0) 2262 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:59.952 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:45:59 smithi133 bash[105589]: cluster 2023-10-09T05:45:58.613934+0000 mgr.y (mgr.25277) 1749 : cluster 0 pgmap v1982: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:45:59.953 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:45:59 smithi133 bash[102858]: cluster 2023-10-09T05:45:58.613934+0000 mgr.y (mgr.25277) 1749 : cluster 0 pgmap v1982: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:00.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:45:59 smithi194 bash[92666]: cluster 2023-10-09T05:45:58.613934+0000 mgr.y (mgr.25277) 1749 : cluster 0 pgmap v1982: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:00.309 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:00.311 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:01.761 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:01 smithi133 bash[102858]: cluster 2023-10-09T05:46:00.614841+0000 mgr.y (mgr.25277) 1750 : cluster 0 pgmap v1983: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:02.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:01 smithi133 bash[105589]: cluster 2023-10-09T05:46:00.614841+0000 mgr.y (mgr.25277) 1750 : cluster 0 pgmap v1983: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:02.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:01 smithi194 bash[92666]: cluster 2023-10-09T05:46:00.614841+0000 mgr.y (mgr.25277) 1750 : cluster 0 pgmap v1983: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:02.338 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:02.339 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:02.339 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:02.341 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:02.342 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:04.051 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:03 smithi133 bash[102858]: cluster 2023-10-09T05:46:02.615509+0000 mgr.y (mgr.25277) 1751 : cluster 0 pgmap v1984: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:04.051 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:03 smithi133 bash[105589]: cluster 2023-10-09T05:46:02.615509+0000 mgr.y (mgr.25277) 1751 : cluster 0 pgmap v1984: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:04.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:03 smithi194 bash[92666]: cluster 2023-10-09T05:46:02.615509+0000 mgr.y (mgr.25277) 1751 : cluster 0 pgmap v1984: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:04.493 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:46:04 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:46:04] "GET /metrics HTTP/1.1" 200 34116 "" "Prometheus/2.43.0" 2023-10-09T05:46:04.926 INFO:teuthology.orchestra.run.smithi133.stdout:128849019694 2023-10-09T05:46:05.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:05 smithi133 bash[105589]: cluster 2023-10-09T05:46:04.616125+0000 mgr.y (mgr.25277) 1752 : cluster 0 pgmap v1985: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:05.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:05 smithi133 bash[105589]: audit 2023-10-09T05:46:04.926306+0000 mon.a (mon.0) 2263 : audit 0 from='client.? 172.21.15.133:0/1926971707' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:46:05.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:05 smithi133 bash[102858]: cluster 2023-10-09T05:46:04.616125+0000 mgr.y (mgr.25277) 1752 : cluster 0 pgmap v1985: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:05.299 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:05 smithi133 bash[102858]: audit 2023-10-09T05:46:04.926306+0000 mon.a (mon.0) 2263 : audit 0 from='client.? 172.21.15.133:0/1926971707' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:46:05.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:05 smithi194 bash[92666]: cluster 2023-10-09T05:46:04.616125+0000 mgr.y (mgr.25277) 1752 : cluster 0 pgmap v1985: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:05.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:05 smithi194 bash[92666]: audit 2023-10-09T05:46:04.926306+0000 mon.a (mon.0) 2263 : audit 0 from='client.? 172.21.15.133:0/1926971707' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:46:07.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:07 smithi133 bash[102858]: cluster 2023-10-09T05:46:06.616811+0000 mgr.y (mgr.25277) 1753 : cluster 0 pgmap v1986: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:07.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:07 smithi133 bash[105589]: cluster 2023-10-09T05:46:06.616811+0000 mgr.y (mgr.25277) 1753 : cluster 0 pgmap v1986: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:08.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:07 smithi194 bash[92666]: cluster 2023-10-09T05:46:06.616811+0000 mgr.y (mgr.25277) 1753 : cluster 0 pgmap v1986: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:09.262 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019691 got 128849019694 for osd.4 2023-10-09T05:46:09.262 DEBUG:teuthology.parallel:result is None 2023-10-09T05:46:10.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:09 smithi133 bash[102858]: cluster 2023-10-09T05:46:08.617378+0000 mgr.y (mgr.25277) 1754 : cluster 0 pgmap v1987: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:10.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:09 smithi133 bash[105589]: cluster 2023-10-09T05:46:08.617378+0000 mgr.y (mgr.25277) 1754 : cluster 0 pgmap v1987: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:10.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:09 smithi194 bash[92666]: cluster 2023-10-09T05:46:08.617378+0000 mgr.y (mgr.25277) 1754 : cluster 0 pgmap v1987: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:12.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:11 smithi133 bash[105589]: cluster 2023-10-09T05:46:10.617990+0000 mgr.y (mgr.25277) 1755 : cluster 0 pgmap v1988: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:12.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:11 smithi133 bash[102858]: cluster 2023-10-09T05:46:10.617990+0000 mgr.y (mgr.25277) 1755 : cluster 0 pgmap v1988: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:12.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:11 smithi194 bash[92666]: cluster 2023-10-09T05:46:10.617990+0000 mgr.y (mgr.25277) 1755 : cluster 0 pgmap v1988: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:14.047 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:13 smithi133 bash[102858]: cluster 2023-10-09T05:46:12.618516+0000 mgr.y (mgr.25277) 1756 : cluster 0 pgmap v1989: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:14.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:13 smithi133 bash[102858]: audit 2023-10-09T05:46:13.504264+0000 mon.a (mon.0) 2264 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:46:14.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:13 smithi133 bash[105589]: cluster 2023-10-09T05:46:12.618516+0000 mgr.y (mgr.25277) 1756 : cluster 0 pgmap v1989: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:14.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:13 smithi133 bash[105589]: audit 2023-10-09T05:46:13.504264+0000 mon.a (mon.0) 2264 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:46:14.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:13 smithi194 bash[92666]: cluster 2023-10-09T05:46:12.618516+0000 mgr.y (mgr.25277) 1756 : cluster 0 pgmap v1989: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:14.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:13 smithi194 bash[92666]: audit 2023-10-09T05:46:13.504264+0000 mon.a (mon.0) 2264 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:46:14.776 INFO:teuthology.orchestra.run.smithi133.stdout:163208758052 2023-10-09T05:46:14.797 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:46:14 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:46:14] "GET /metrics HTTP/1.1" 200 34113 "" "Prometheus/2.43.0" 2023-10-09T05:46:15.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:15 smithi133 bash[105589]: cluster 2023-10-09T05:46:14.618971+0000 mgr.y (mgr.25277) 1757 : cluster 0 pgmap v1990: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:15.298 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:15 smithi133 bash[105589]: audit 2023-10-09T05:46:14.776227+0000 mon.a (mon.0) 2265 : audit 0 from='client.? 172.21.15.133:0/802158205' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:46:15.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:15 smithi133 bash[102858]: cluster 2023-10-09T05:46:14.618971+0000 mgr.y (mgr.25277) 1757 : cluster 0 pgmap v1990: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:15.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:15 smithi133 bash[102858]: audit 2023-10-09T05:46:14.776227+0000 mon.a (mon.0) 2265 : audit 0 from='client.? 172.21.15.133:0/802158205' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:46:15.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:15 smithi194 bash[92666]: cluster 2023-10-09T05:46:14.618971+0000 mgr.y (mgr.25277) 1757 : cluster 0 pgmap v1990: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:15.386 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:15 smithi194 bash[92666]: audit 2023-10-09T05:46:14.776227+0000 mon.a (mon.0) 2265 : audit 0 from='client.? 172.21.15.133:0/802158205' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:46:15.941 INFO:teuthology.orchestra.run.smithi133.stdout:455266533921 2023-10-09T05:46:16.214 INFO:teuthology.orchestra.run.smithi133.stdout:412316861001 2023-10-09T05:46:16.340 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:16 smithi133 bash[105589]: audit 2023-10-09T05:46:15.940922+0000 mon.a (mon.0) 2266 : audit 0 from='client.? 172.21.15.133:0/862798541' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:46:16.341 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:16 smithi133 bash[102858]: audit 2023-10-09T05:46:15.940922+0000 mon.a (mon.0) 2266 : audit 0 from='client.? 172.21.15.133:0/862798541' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:46:16.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:16 smithi194 bash[92666]: audit 2023-10-09T05:46:15.940922+0000 mon.a (mon.0) 2266 : audit 0 from='client.? 172.21.15.133:0/862798541' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:46:17.300 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208758047 got 163208758052 for osd.5 2023-10-09T05:46:17.300 DEBUG:teuthology.parallel:result is None 2023-10-09T05:46:17.349 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:17 smithi133 bash[102858]: audit 2023-10-09T05:46:16.214493+0000 mon.a (mon.0) 2267 : audit 0 from='client.? 172.21.15.133:0/3800131353' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:46:17.349 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:17 smithi133 bash[102858]: cluster 2023-10-09T05:46:16.619775+0000 mgr.y (mgr.25277) 1758 : cluster 0 pgmap v1991: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:17.350 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:17 smithi133 bash[105589]: audit 2023-10-09T05:46:16.214493+0000 mon.a (mon.0) 2267 : audit 0 from='client.? 172.21.15.133:0/3800131353' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:46:17.350 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:17 smithi133 bash[105589]: cluster 2023-10-09T05:46:16.619775+0000 mgr.y (mgr.25277) 1758 : cluster 0 pgmap v1991: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:17.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:17 smithi194 bash[92666]: audit 2023-10-09T05:46:16.214493+0000 mon.a (mon.0) 2267 : audit 0 from='client.? 172.21.15.133:0/3800131353' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:46:17.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:17 smithi194 bash[92666]: cluster 2023-10-09T05:46:16.619775+0000 mgr.y (mgr.25277) 1758 : cluster 0 pgmap v1991: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:17.796 INFO:teuthology.orchestra.run.smithi133.stdout:528280977940 2023-10-09T05:46:18.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:18 smithi194 bash[92666]: audit 2023-10-09T05:46:17.796044+0000 mon.a (mon.0) 2268 : audit 0 from='client.? 172.21.15.133:0/820989373' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:46:18.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:18 smithi133 bash[102858]: audit 2023-10-09T05:46:17.796044+0000 mon.a (mon.0) 2268 : audit 0 from='client.? 172.21.15.133:0/820989373' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:46:18.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:18 smithi133 bash[105589]: audit 2023-10-09T05:46:17.796044+0000 mon.a (mon.0) 2268 : audit 0 from='client.? 172.21.15.133:0/820989373' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:46:19.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:19 smithi194 bash[92666]: cluster 2023-10-09T05:46:18.620290+0000 mgr.y (mgr.25277) 1759 : cluster 0 pgmap v1992: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:19.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:19 smithi133 bash[105589]: cluster 2023-10-09T05:46:18.620290+0000 mgr.y (mgr.25277) 1759 : cluster 0 pgmap v1992: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:19.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:19 smithi133 bash[102858]: cluster 2023-10-09T05:46:18.620290+0000 mgr.y (mgr.25277) 1759 : cluster 0 pgmap v1992: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:20.294 INFO:tasks.cephadm.ceph_manager.ceph:need seq 412316860997 got 412316861001 for osd.1 2023-10-09T05:46:20.294 DEBUG:teuthology.parallel:result is None 2023-10-09T05:46:20.432 INFO:tasks.cephadm.ceph_manager.ceph:need seq 455266533916 got 455266533921 for osd.2 2023-10-09T05:46:20.432 DEBUG:teuthology.parallel:result is None 2023-10-09T05:46:20.636 INFO:tasks.cephadm.ceph_manager.ceph:need seq 528280977935 got 528280977940 for osd.3 2023-10-09T05:46:20.636 DEBUG:teuthology.parallel:result is None 2023-10-09T05:46:20.695 INFO:teuthology.orchestra.run.smithi133.stdout:231928234770 2023-10-09T05:46:20.851 INFO:teuthology.orchestra.run.smithi133.stdout:197568496412 2023-10-09T05:46:20.882 INFO:teuthology.orchestra.run.smithi133.stdout:377957122644 2023-10-09T05:46:22.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:21 smithi133 bash[105589]: cluster 2023-10-09T05:46:20.620908+0000 mgr.y (mgr.25277) 1760 : cluster 0 pgmap v1993: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:22.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:21 smithi133 bash[105589]: audit 2023-10-09T05:46:20.695625+0000 mon.a (mon.0) 2269 : audit 0 from='client.? 172.21.15.133:0/498997207' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:46:22.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:21 smithi133 bash[105589]: audit 2023-10-09T05:46:20.851279+0000 mon.a (mon.0) 2270 : audit 0 from='client.? 172.21.15.133:0/2338155575' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:46:22.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:21 smithi133 bash[105589]: audit 2023-10-09T05:46:20.882253+0000 mon.c (mon.2) 158 : audit 0 from='client.? 172.21.15.133:0/3174447474' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:46:22.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:21 smithi133 bash[102858]: cluster 2023-10-09T05:46:20.620908+0000 mgr.y (mgr.25277) 1760 : cluster 0 pgmap v1993: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:22.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:21 smithi133 bash[102858]: audit 2023-10-09T05:46:20.695625+0000 mon.a (mon.0) 2269 : audit 0 from='client.? 172.21.15.133:0/498997207' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:46:22.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:21 smithi133 bash[102858]: audit 2023-10-09T05:46:20.851279+0000 mon.a (mon.0) 2270 : audit 0 from='client.? 172.21.15.133:0/2338155575' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:46:22.049 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:21 smithi133 bash[102858]: audit 2023-10-09T05:46:20.882253+0000 mon.c (mon.2) 158 : audit 0 from='client.? 172.21.15.133:0/3174447474' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:46:22.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:21 smithi194 bash[92666]: cluster 2023-10-09T05:46:20.620908+0000 mgr.y (mgr.25277) 1760 : cluster 0 pgmap v1993: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:22.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:21 smithi194 bash[92666]: audit 2023-10-09T05:46:20.695625+0000 mon.a (mon.0) 2269 : audit 0 from='client.? 172.21.15.133:0/498997207' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:46:22.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:21 smithi194 bash[92666]: audit 2023-10-09T05:46:20.851279+0000 mon.a (mon.0) 2270 : audit 0 from='client.? 172.21.15.133:0/2338155575' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:46:22.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:21 smithi194 bash[92666]: audit 2023-10-09T05:46:20.882253+0000 mon.c (mon.2) 158 : audit 0 from='client.? 172.21.15.133:0/3174447474' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:46:22.875 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234764 got 231928234770 for osd.7 2023-10-09T05:46:22.875 DEBUG:teuthology.parallel:result is None 2023-10-09T05:46:23.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:22 smithi133 bash[105589]: audit 2023-10-09T05:46:22.428931+0000 mon.a (mon.0) 2271 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:46:23.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:22 smithi133 bash[102858]: audit 2023-10-09T05:46:22.428931+0000 mon.a (mon.0) 2271 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:46:23.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:22 smithi194 bash[92666]: audit 2023-10-09T05:46:22.428931+0000 mon.a (mon.0) 2271 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:46:23.469 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568496406 got 197568496412 for osd.6 2023-10-09T05:46:23.470 DEBUG:teuthology.parallel:result is None 2023-10-09T05:46:23.513 INFO:tasks.cephadm.ceph_manager.ceph:need seq 377957122639 got 377957122644 for osd.0 2023-10-09T05:46:23.513 DEBUG:teuthology.parallel:result is None 2023-10-09T05:46:23.513 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T05:46:23.513 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:46:23.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:23 smithi133 bash[105589]: cluster 2023-10-09T05:46:22.621399+0000 mgr.y (mgr.25277) 1761 : cluster 0 pgmap v1994: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:23.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:23 smithi133 bash[105589]: audit 2023-10-09T05:46:22.806451+0000 mon.a (mon.0) 2272 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:46:23.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:23 smithi133 bash[105589]: audit 2023-10-09T05:46:22.814484+0000 mon.a (mon.0) 2273 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:46:23.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:23 smithi133 bash[102858]: cluster 2023-10-09T05:46:22.621399+0000 mgr.y (mgr.25277) 1761 : cluster 0 pgmap v1994: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:23.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:23 smithi133 bash[102858]: audit 2023-10-09T05:46:22.806451+0000 mon.a (mon.0) 2272 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:46:23.799 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:23 smithi133 bash[102858]: audit 2023-10-09T05:46:22.814484+0000 mon.a (mon.0) 2273 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:46:24.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:23 smithi194 bash[92666]: cluster 2023-10-09T05:46:22.621399+0000 mgr.y (mgr.25277) 1761 : cluster 0 pgmap v1994: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:24.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:23 smithi194 bash[92666]: audit 2023-10-09T05:46:22.806451+0000 mon.a (mon.0) 2272 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:46:24.136 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:23 smithi194 bash[92666]: audit 2023-10-09T05:46:22.814484+0000 mon.a (mon.0) 2273 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:46:24.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:46:24 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:46:24] "GET /metrics HTTP/1.1" 200 34113 "" "Prometheus/2.43.0" 2023-10-09T05:46:25.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:25 smithi194 bash[92666]: cluster 2023-10-09T05:46:24.622147+0000 mgr.y (mgr.25277) 1762 : cluster 0 pgmap v1995: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:25.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:25 smithi133 bash[102858]: cluster 2023-10-09T05:46:24.622147+0000 mgr.y (mgr.25277) 1762 : cluster 0 pgmap v1995: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:25.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:25 smithi133 bash[105589]: cluster 2023-10-09T05:46:24.622147+0000 mgr.y (mgr.25277) 1762 : cluster 0 pgmap v1995: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:27.285 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:28.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:27 smithi133 bash[102858]: cluster 2023-10-09T05:46:26.623023+0000 mgr.y (mgr.25277) 1763 : cluster 0 pgmap v1996: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:28.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:27 smithi133 bash[105589]: cluster 2023-10-09T05:46:26.623023+0000 mgr.y (mgr.25277) 1763 : cluster 0 pgmap v1996: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:28.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:27 smithi194 bash[92666]: cluster 2023-10-09T05:46:26.623023+0000 mgr.y (mgr.25277) 1763 : cluster 0 pgmap v1996: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:29.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:29 smithi133 bash[102858]: audit 2023-10-09T05:46:28.473865+0000 mon.a (mon.0) 2274 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:46:29.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:29 smithi133 bash[102858]: audit 2023-10-09T05:46:28.480882+0000 mon.a (mon.0) 2275 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:46:29.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:29 smithi133 bash[102858]: audit 2023-10-09T05:46:28.504854+0000 mon.a (mon.0) 2276 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:46:29.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:29 smithi133 bash[102858]: cluster 2023-10-09T05:46:28.623619+0000 mgr.y (mgr.25277) 1764 : cluster 0 pgmap v1997: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:29.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:29 smithi133 bash[102858]: audit 2023-10-09T05:46:29.305956+0000 mon.a (mon.0) 2277 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:46:29.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:29 smithi133 bash[102858]: audit 2023-10-09T05:46:29.314479+0000 mon.a (mon.0) 2278 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:46:29.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:29 smithi133 bash[105589]: audit 2023-10-09T05:46:28.473865+0000 mon.a (mon.0) 2274 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:46:29.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:29 smithi133 bash[105589]: audit 2023-10-09T05:46:28.480882+0000 mon.a (mon.0) 2275 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:46:29.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:29 smithi133 bash[105589]: audit 2023-10-09T05:46:28.504854+0000 mon.a (mon.0) 2276 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:46:29.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:29 smithi133 bash[105589]: cluster 2023-10-09T05:46:28.623619+0000 mgr.y (mgr.25277) 1764 : cluster 0 pgmap v1997: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:29.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:29 smithi133 bash[105589]: audit 2023-10-09T05:46:29.305956+0000 mon.a (mon.0) 2277 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:46:29.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:29 smithi133 bash[105589]: audit 2023-10-09T05:46:29.314479+0000 mon.a (mon.0) 2278 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:46:29.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:29 smithi194 bash[92666]: audit 2023-10-09T05:46:28.473865+0000 mon.a (mon.0) 2274 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:46:29.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:29 smithi194 bash[92666]: audit 2023-10-09T05:46:28.480882+0000 mon.a (mon.0) 2275 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:46:29.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:29 smithi194 bash[92666]: audit 2023-10-09T05:46:28.504854+0000 mon.a (mon.0) 2276 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:46:29.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:29 smithi194 bash[92666]: cluster 2023-10-09T05:46:28.623619+0000 mgr.y (mgr.25277) 1764 : cluster 0 pgmap v1997: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:29.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:29 smithi194 bash[92666]: audit 2023-10-09T05:46:29.305956+0000 mon.a (mon.0) 2277 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:46:29.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:29 smithi194 bash[92666]: audit 2023-10-09T05:46:29.314479+0000 mon.a (mon.0) 2278 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:46:30.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:30 smithi133 bash[102858]: audit 2023-10-09T05:46:29.783078+0000 mon.a (mon.0) 2279 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:46:30.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:30 smithi133 bash[102858]: audit 2023-10-09T05:46:29.784389+0000 mon.a (mon.0) 2280 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:46:30.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:30 smithi133 bash[102858]: audit 2023-10-09T05:46:29.794658+0000 mon.a (mon.0) 2281 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:46:30.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:30 smithi133 bash[105589]: audit 2023-10-09T05:46:29.783078+0000 mon.a (mon.0) 2279 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:46:30.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:30 smithi133 bash[105589]: audit 2023-10-09T05:46:29.784389+0000 mon.a (mon.0) 2280 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:46:30.799 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:30 smithi133 bash[105589]: audit 2023-10-09T05:46:29.794658+0000 mon.a (mon.0) 2281 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:46:30.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:30 smithi194 bash[92666]: audit 2023-10-09T05:46:29.783078+0000 mon.a (mon.0) 2279 : audit 0 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:46:30.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:30 smithi194 bash[92666]: audit 2023-10-09T05:46:29.784389+0000 mon.a (mon.0) 2280 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:46:30.886 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:30 smithi194 bash[92666]: audit 2023-10-09T05:46:29.794658+0000 mon.a (mon.0) 2281 : audit 1 from='mgr.25277 172.21.15.133:0/3725309899' entity='mgr.y' 2023-10-09T05:46:31.335 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:46:31.337 INFO:teuthology.orchestra.run.smithi133.stderr:dumped all 2023-10-09T05:46:31.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:31 smithi133 bash[105589]: cluster 2023-10-09T05:46:30.624650+0000 mgr.y (mgr.25277) 1765 : cluster 0 pgmap v1998: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:31.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:31 smithi133 bash[102858]: cluster 2023-10-09T05:46:30.624650+0000 mgr.y (mgr.25277) 1765 : cluster 0 pgmap v1998: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:31.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:31 smithi194 bash[92666]: cluster 2023-10-09T05:46:30.624650+0000 mgr.y (mgr.25277) 1765 : cluster 0 pgmap v1998: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:32.260 INFO:teuthology.orchestra.run.smithi133.stdout:{"pg_ready":true,"pg_map":{"version":1998,"stamp":"2023-10-09T05:46:30.623904+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6504,"num_read_kb":14182,"num_write":1146,"num_write_kb":15159,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":115,"num_bytes_recovered":4154950,"num_keys_recovered":30,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1198,"ondisk_log_size":1198,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9713848,"kb_used_data":56184,"kb_used_omap":20,"kb_used_meta":9657579,"kb_avail":740247368,"statfs":{"total":767960285184,"available":758013304832,"internally_reserved":0,"allocated":57532416,"data_stored":42363632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":9889361550},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003836"},"pg_stats":[{"pgid":"3.1a","version":"69'2","reported_seq":34,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.084877+0000","last_change":"2023-10-09T05:42:12.627925+0000","last_active":"2023-10-09T05:45:00.084877+0000","last_peered":"2023-10-09T05:45:00.084877+0000","last_clean":"2023-10-09T05:45:00.084877+0000","last_became_active":"2023-10-09T05:42:12.625644+0000","last_became_peered":"2023-10-09T05:42:12.625644+0000","last_unstale":"2023-10-09T05:45:00.084877+0000","last_undegraded":"2023-10-09T05:45:00.084877+0000","last_fullsized":"2023-10-09T05:45:00.084877+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:08:11.129668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":457,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.671010+0000","last_change":"2023-10-09T05:42:12.631985+0000","last_active":"2023-10-09T05:44:59.671010+0000","last_peered":"2023-10-09T05:44:59.671010+0000","last_clean":"2023-10-09T05:44:59.671010+0000","last_became_active":"2023-10-09T05:42:12.631320+0000","last_became_peered":"2023-10-09T05:42:12.631320+0000","last_unstale":"2023-10-09T05:44:59.671010+0000","last_undegraded":"2023-10-09T05:44:59.671010+0000","last_fullsized":"2023-10-09T05:44:59.671010+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:08.504947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":341,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.652400+0000","last_change":"2023-10-09T05:42:12.632126+0000","last_active":"2023-10-09T05:45:08.652400+0000","last_peered":"2023-10-09T05:45:08.652400+0000","last_clean":"2023-10-09T05:45:08.652400+0000","last_became_active":"2023-10-09T05:42:12.631615+0000","last_became_peered":"2023-10-09T05:42:12.631615+0000","last_unstale":"2023-10-09T05:45:08.652400+0000","last_undegraded":"2023-10-09T05:45:08.652400+0000","last_fullsized":"2023-10-09T05:45:08.652400+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:52.067376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":457,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.968679+0000","last_change":"2023-10-09T05:18:26.422923+0000","last_active":"2023-10-09T05:45:08.968679+0000","last_peered":"2023-10-09T05:45:08.968679+0000","last_clean":"2023-10-09T05:45:08.968679+0000","last_became_active":"2023-10-09T05:18:26.422508+0000","last_became_peered":"2023-10-09T05:18:26.422508+0000","last_unstale":"2023-10-09T05:45:08.968679+0000","last_undegraded":"2023-10-09T05:45:08.968679+0000","last_fullsized":"2023-10-09T05:45:08.968679+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:00.876037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":494,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.968165+0000","last_change":"2023-10-09T05:42:59.843555+0000","last_active":"2023-10-09T05:45:08.968165+0000","last_peered":"2023-10-09T05:45:08.968165+0000","last_clean":"2023-10-09T05:45:08.968165+0000","last_became_active":"2023-10-09T05:42:59.842903+0000","last_became_peered":"2023-10-09T05:42:59.842903+0000","last_unstale":"2023-10-09T05:45:08.968165+0000","last_undegraded":"2023-10-09T05:45:08.968165+0000","last_fullsized":"2023-10-09T05:45:08.968165+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:04:42.450057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":449,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.680210+0000","last_change":"2023-10-09T05:32:45.829908+0000","last_active":"2023-10-09T05:44:59.680210+0000","last_peered":"2023-10-09T05:44:59.680210+0000","last_clean":"2023-10-09T05:44:59.680210+0000","last_became_active":"2023-10-09T05:32:45.829566+0000","last_became_peered":"2023-10-09T05:32:45.829566+0000","last_unstale":"2023-10-09T05:44:59.680210+0000","last_undegraded":"2023-10-09T05:44:59.680210+0000","last_fullsized":"2023-10-09T05:44:59.680210+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:05.705818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":452,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.969138+0000","last_change":"2023-10-09T05:32:45.835245+0000","last_active":"2023-10-09T05:45:08.969138+0000","last_peered":"2023-10-09T05:45:08.969138+0000","last_clean":"2023-10-09T05:45:08.969138+0000","last_became_active":"2023-10-09T05:32:45.830180+0000","last_became_peered":"2023-10-09T05:32:45.830180+0000","last_unstale":"2023-10-09T05:45:08.969138+0000","last_undegraded":"2023-10-09T05:45:08.969138+0000","last_fullsized":"2023-10-09T05:45:08.969138+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:07:06.351159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":432,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.968151+0000","last_change":"2023-10-09T05:09:57.849695+0000","last_active":"2023-10-09T05:45:08.968151+0000","last_peered":"2023-10-09T05:45:08.968151+0000","last_clean":"2023-10-09T05:45:08.968151+0000","last_became_active":"2023-10-09T05:09:57.848478+0000","last_became_peered":"2023-10-09T05:09:57.848478+0000","last_unstale":"2023-10-09T05:45:08.968151+0000","last_undegraded":"2023-10-09T05:45:08.968151+0000","last_fullsized":"2023-10-09T05:45:08.968151+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:26.115312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":158,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.675112+0000","last_change":"2023-10-09T05:32:45.830061+0000","last_active":"2023-10-09T05:44:59.675112+0000","last_peered":"2023-10-09T05:44:59.675112+0000","last_clean":"2023-10-09T05:44:59.675112+0000","last_became_active":"2023-10-09T05:32:45.829723+0000","last_became_peered":"2023-10-09T05:32:45.829723+0000","last_unstale":"2023-10-09T05:44:59.675112+0000","last_undegraded":"2023-10-09T05:44:59.675112+0000","last_fullsized":"2023-10-09T05:44:59.675112+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:21.429740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":444,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.670736+0000","last_change":"2023-10-09T05:18:26.418431+0000","last_active":"2023-10-09T05:44:59.670736+0000","last_peered":"2023-10-09T05:44:59.670736+0000","last_clean":"2023-10-09T05:44:59.670736+0000","last_became_active":"2023-10-09T05:18:26.418100+0000","last_became_peered":"2023-10-09T05:18:26.418100+0000","last_unstale":"2023-10-09T05:44:59.670736+0000","last_undegraded":"2023-10-09T05:44:59.670736+0000","last_fullsized":"2023-10-09T05:44:59.670736+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:21.241286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":273,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.965184+0000","last_change":"2023-10-09T05:42:12.652600+0000","last_active":"2023-10-09T05:44:59.965184+0000","last_peered":"2023-10-09T05:44:59.965184+0000","last_clean":"2023-10-09T05:44:59.965184+0000","last_became_active":"2023-10-09T05:42:12.631619+0000","last_became_peered":"2023-10-09T05:42:12.631619+0000","last_unstale":"2023-10-09T05:44:59.965184+0000","last_undegraded":"2023-10-09T05:44:59.965184+0000","last_fullsized":"2023-10-09T05:44:59.965184+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:43:00.862938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":28,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.653090+0000","last_change":"2023-10-09T05:42:59.633392+0000","last_active":"2023-10-09T05:45:08.653090+0000","last_peered":"2023-10-09T05:45:08.653090+0000","last_clean":"2023-10-09T05:45:08.653090+0000","last_became_active":"2023-10-09T05:42:59.632990+0000","last_became_peered":"2023-10-09T05:42:59.632990+0000","last_unstale":"2023-10-09T05:45:08.653090+0000","last_undegraded":"2023-10-09T05:45:08.653090+0000","last_fullsized":"2023-10-09T05:45:08.653090+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:38.874996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'23","reported_seq":44,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.085242+0000","last_change":"2023-10-09T05:42:12.644859+0000","last_active":"2023-10-09T05:45:00.085242+0000","last_peered":"2023-10-09T05:45:00.085242+0000","last_clean":"2023-10-09T05:45:00.085242+0000","last_became_active":"2023-10-09T05:42:12.626225+0000","last_became_peered":"2023-10-09T05:42:12.626225+0000","last_unstale":"2023-10-09T05:45:00.085242+0000","last_undegraded":"2023-10-09T05:45:00.085242+0000","last_fullsized":"2023-10-09T05:45:00.085242+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:04.905187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":34,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.085048+0000","last_change":"2023-10-09T05:42:12.638386+0000","last_active":"2023-10-09T05:45:00.085048+0000","last_peered":"2023-10-09T05:45:00.085048+0000","last_clean":"2023-10-09T05:45:00.085048+0000","last_became_active":"2023-10-09T05:42:12.637267+0000","last_became_peered":"2023-10-09T05:42:12.637267+0000","last_unstale":"2023-10-09T05:45:00.085048+0000","last_undegraded":"2023-10-09T05:45:00.085048+0000","last_fullsized":"2023-10-09T05:45:00.085048+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:15.501636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":273,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.964773+0000","last_change":"2023-10-09T05:42:12.656052+0000","last_active":"2023-10-09T05:44:59.964773+0000","last_peered":"2023-10-09T05:44:59.964773+0000","last_clean":"2023-10-09T05:44:59.964773+0000","last_became_active":"2023-10-09T05:42:12.631633+0000","last_became_peered":"2023-10-09T05:42:12.631633+0000","last_unstale":"2023-10-09T05:44:59.964773+0000","last_undegraded":"2023-10-09T05:44:59.964773+0000","last_fullsized":"2023-10-09T05:44:59.964773+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:57:42.256197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":503,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.670697+0000","last_change":"2023-10-09T05:43:00.348706+0000","last_active":"2023-10-09T05:44:59.670697+0000","last_peered":"2023-10-09T05:44:59.670697+0000","last_clean":"2023-10-09T05:44:59.670697+0000","last_became_active":"2023-10-09T05:43:00.347927+0000","last_became_peered":"2023-10-09T05:43:00.347927+0000","last_unstale":"2023-10-09T05:44:59.670697+0000","last_undegraded":"2023-10-09T05:44:59.670697+0000","last_fullsized":"2023-10-09T05:44:59.670697+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:40:48.770688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":453,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.969210+0000","last_change":"2023-10-09T05:18:26.424399+0000","last_active":"2023-10-09T05:45:08.969210+0000","last_peered":"2023-10-09T05:45:08.969210+0000","last_clean":"2023-10-09T05:45:08.969210+0000","last_became_active":"2023-10-09T05:18:26.422122+0000","last_became_peered":"2023-10-09T05:18:26.422122+0000","last_unstale":"2023-10-09T05:45:08.969210+0000","last_undegraded":"2023-10-09T05:45:08.969210+0000","last_fullsized":"2023-10-09T05:45:08.969210+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:39.393756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":158,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.675096+0000","last_change":"2023-10-09T05:32:45.831080+0000","last_active":"2023-10-09T05:44:59.675096+0000","last_peered":"2023-10-09T05:44:59.675096+0000","last_clean":"2023-10-09T05:44:59.675096+0000","last_became_active":"2023-10-09T05:32:45.830745+0000","last_became_peered":"2023-10-09T05:32:45.830745+0000","last_unstale":"2023-10-09T05:44:59.675096+0000","last_undegraded":"2023-10-09T05:44:59.675096+0000","last_fullsized":"2023-10-09T05:44:59.675096+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:16.345644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":255,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.964209+0000","last_change":"2023-10-09T05:42:12.629712+0000","last_active":"2023-10-09T05:44:59.964209+0000","last_peered":"2023-10-09T05:44:59.964209+0000","last_clean":"2023-10-09T05:44:59.964209+0000","last_became_active":"2023-10-09T05:42:12.629312+0000","last_became_peered":"2023-10-09T05:42:12.629312+0000","last_unstale":"2023-10-09T05:44:59.964209+0000","last_undegraded":"2023-10-09T05:44:59.964209+0000","last_fullsized":"2023-10-09T05:44:59.964209+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:28.590232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":453,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.968728+0000","last_change":"2023-10-09T05:18:26.421678+0000","last_active":"2023-10-09T05:45:08.968728+0000","last_peered":"2023-10-09T05:45:08.968728+0000","last_clean":"2023-10-09T05:45:08.968728+0000","last_became_active":"2023-10-09T05:18:26.419673+0000","last_became_peered":"2023-10-09T05:18:26.419673+0000","last_unstale":"2023-10-09T05:45:08.968728+0000","last_undegraded":"2023-10-09T05:45:08.968728+0000","last_fullsized":"2023-10-09T05:45:08.968728+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:38.876021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":462,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.679072+0000","last_change":"2023-10-09T05:32:45.844532+0000","last_active":"2023-10-09T05:44:59.679072+0000","last_peered":"2023-10-09T05:44:59.679072+0000","last_clean":"2023-10-09T05:44:59.679072+0000","last_became_active":"2023-10-09T05:32:45.831101+0000","last_became_peered":"2023-10-09T05:32:45.831101+0000","last_unstale":"2023-10-09T05:44:59.679072+0000","last_undegraded":"2023-10-09T05:44:59.679072+0000","last_fullsized":"2023-10-09T05:44:59.679072+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:19:37.907188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":143,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.675105+0000","last_change":"2023-10-09T05:36:16.901739+0000","last_active":"2023-10-09T05:44:59.675105+0000","last_peered":"2023-10-09T05:44:59.675105+0000","last_clean":"2023-10-09T05:44:59.675105+0000","last_became_active":"2023-10-09T05:36:16.901380+0000","last_became_peered":"2023-10-09T05:36:16.901380+0000","last_unstale":"2023-10-09T05:44:59.675105+0000","last_undegraded":"2023-10-09T05:44:59.675105+0000","last_fullsized":"2023-10-09T05:44:59.675105+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:26.454798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":267,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.967869+0000","last_change":"2023-10-09T05:42:12.649212+0000","last_active":"2023-10-09T05:44:59.967869+0000","last_peered":"2023-10-09T05:44:59.967869+0000","last_clean":"2023-10-09T05:44:59.967869+0000","last_became_active":"2023-10-09T05:42:12.630943+0000","last_became_peered":"2023-10-09T05:42:12.630943+0000","last_unstale":"2023-10-09T05:44:59.967869+0000","last_undegraded":"2023-10-09T05:44:59.967869+0000","last_fullsized":"2023-10-09T05:44:59.967869+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:25.895604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":158,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.674736+0000","last_change":"2023-10-09T05:32:45.829063+0000","last_active":"2023-10-09T05:44:59.674736+0000","last_peered":"2023-10-09T05:44:59.674736+0000","last_clean":"2023-10-09T05:44:59.674736+0000","last_became_active":"2023-10-09T05:32:45.828694+0000","last_became_peered":"2023-10-09T05:32:45.828694+0000","last_unstale":"2023-10-09T05:44:59.674736+0000","last_undegraded":"2023-10-09T05:44:59.674736+0000","last_fullsized":"2023-10-09T05:44:59.674736+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:58:51.140748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":267,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.965297+0000","last_change":"2023-10-09T05:42:12.646999+0000","last_active":"2023-10-09T05:44:59.965297+0000","last_peered":"2023-10-09T05:44:59.965297+0000","last_clean":"2023-10-09T05:44:59.965297+0000","last_became_active":"2023-10-09T05:42:12.629954+0000","last_became_peered":"2023-10-09T05:42:12.629954+0000","last_unstale":"2023-10-09T05:44:59.965297+0000","last_undegraded":"2023-10-09T05:44:59.965297+0000","last_fullsized":"2023-10-09T05:44:59.965297+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:33.042876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":34,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.084585+0000","last_change":"2023-10-09T05:42:12.620059+0000","last_active":"2023-10-09T05:45:00.084585+0000","last_peered":"2023-10-09T05:45:00.084585+0000","last_clean":"2023-10-09T05:45:00.084585+0000","last_became_active":"2023-10-09T05:42:12.619703+0000","last_became_peered":"2023-10-09T05:42:12.619703+0000","last_unstale":"2023-10-09T05:45:00.084585+0000","last_undegraded":"2023-10-09T05:45:00.084585+0000","last_fullsized":"2023-10-09T05:45:00.084585+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:02.983347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":253,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.965184+0000","last_change":"2023-10-09T05:32:45.838786+0000","last_active":"2023-10-09T05:44:59.965184+0000","last_peered":"2023-10-09T05:44:59.965184+0000","last_clean":"2023-10-09T05:44:59.965184+0000","last_became_active":"2023-10-09T05:32:45.830293+0000","last_became_peered":"2023-10-09T05:32:45.830293+0000","last_unstale":"2023-10-09T05:44:59.965184+0000","last_undegraded":"2023-10-09T05:44:59.965184+0000","last_fullsized":"2023-10-09T05:44:59.965184+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:47.319929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":479,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.679359+0000","last_change":"2023-10-09T05:18:26.423526+0000","last_active":"2023-10-09T05:44:59.679359+0000","last_peered":"2023-10-09T05:44:59.679359+0000","last_clean":"2023-10-09T05:44:59.679359+0000","last_became_active":"2023-10-09T05:18:26.423092+0000","last_became_peered":"2023-10-09T05:18:26.423092+0000","last_unstale":"2023-10-09T05:44:59.679359+0000","last_undegraded":"2023-10-09T05:44:59.679359+0000","last_fullsized":"2023-10-09T05:44:59.679359+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:54.877664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":158,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.675228+0000","last_change":"2023-10-09T05:32:45.829450+0000","last_active":"2023-10-09T05:44:59.675228+0000","last_peered":"2023-10-09T05:44:59.675228+0000","last_clean":"2023-10-09T05:44:59.675228+0000","last_became_active":"2023-10-09T05:32:45.829076+0000","last_became_peered":"2023-10-09T05:32:45.829076+0000","last_unstale":"2023-10-09T05:44:59.675228+0000","last_undegraded":"2023-10-09T05:44:59.675228+0000","last_fullsized":"2023-10-09T05:44:59.675228+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:51:13.538770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":455,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.969457+0000","last_change":"2023-10-09T05:42:12.629497+0000","last_active":"2023-10-09T05:45:08.969457+0000","last_peered":"2023-10-09T05:45:08.969457+0000","last_clean":"2023-10-09T05:45:08.969457+0000","last_became_active":"2023-10-09T05:42:12.629123+0000","last_became_peered":"2023-10-09T05:42:12.629123+0000","last_unstale":"2023-10-09T05:45:08.969457+0000","last_undegraded":"2023-10-09T05:45:08.969457+0000","last_fullsized":"2023-10-09T05:45:08.969457+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:34.295429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'3","reported_seq":437,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.670477+0000","last_change":"2023-10-09T05:42:12.640692+0000","last_active":"2023-10-09T05:44:59.670477+0000","last_peered":"2023-10-09T05:44:59.670477+0000","last_clean":"2023-10-09T05:44:59.670477+0000","last_became_active":"2023-10-09T05:42:12.629848+0000","last_became_peered":"2023-10-09T05:42:12.629848+0000","last_unstale":"2023-10-09T05:44:59.670477+0000","last_undegraded":"2023-10-09T05:44:59.670477+0000","last_fullsized":"2023-10-09T05:44:59.670477+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:53.321982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":34,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.085573+0000","last_change":"2023-10-09T05:42:12.624031+0000","last_active":"2023-10-09T05:45:00.085573+0000","last_peered":"2023-10-09T05:45:00.085573+0000","last_clean":"2023-10-09T05:45:00.085573+0000","last_became_active":"2023-10-09T05:42:12.623571+0000","last_became_peered":"2023-10-09T05:42:12.623571+0000","last_unstale":"2023-10-09T05:45:00.085573+0000","last_undegraded":"2023-10-09T05:45:00.085573+0000","last_fullsized":"2023-10-09T05:45:00.085573+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:50:18.943555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":490,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.679277+0000","last_change":"2023-10-09T05:42:59.634964+0000","last_active":"2023-10-09T05:44:59.679277+0000","last_peered":"2023-10-09T05:44:59.679277+0000","last_clean":"2023-10-09T05:44:59.679277+0000","last_became_active":"2023-10-09T05:42:59.634570+0000","last_became_peered":"2023-10-09T05:42:59.634570+0000","last_unstale":"2023-10-09T05:44:59.679277+0000","last_undegraded":"2023-10-09T05:44:59.679277+0000","last_fullsized":"2023-10-09T05:44:59.679277+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:34.443566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":34,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.085231+0000","last_change":"2023-10-09T05:42:12.626346+0000","last_active":"2023-10-09T05:45:00.085231+0000","last_peered":"2023-10-09T05:45:00.085231+0000","last_clean":"2023-10-09T05:45:00.085231+0000","last_became_active":"2023-10-09T05:42:12.625594+0000","last_became_peered":"2023-10-09T05:42:12.625594+0000","last_unstale":"2023-10-09T05:45:00.085231+0000","last_undegraded":"2023-10-09T05:45:00.085231+0000","last_fullsized":"2023-10-09T05:45:00.085231+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:36:01.369510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":34,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.085410+0000","last_change":"2023-10-09T05:42:12.628487+0000","last_active":"2023-10-09T05:45:00.085410+0000","last_peered":"2023-10-09T05:45:00.085410+0000","last_clean":"2023-10-09T05:45:00.085410+0000","last_became_active":"2023-10-09T05:42:12.628127+0000","last_became_peered":"2023-10-09T05:42:12.628127+0000","last_unstale":"2023-10-09T05:45:00.085410+0000","last_undegraded":"2023-10-09T05:45:00.085410+0000","last_fullsized":"2023-10-09T05:45:00.085410+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:30:05.955093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":54,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.085333+0000","last_change":"2023-10-09T05:43:17.869175+0000","last_active":"2023-10-09T05:45:00.085333+0000","last_peered":"2023-10-09T05:45:00.085333+0000","last_clean":"2023-10-09T05:45:00.085333+0000","last_became_active":"2023-10-09T05:43:17.868274+0000","last_became_peered":"2023-10-09T05:43:17.868274+0000","last_unstale":"2023-10-09T05:45:00.085333+0000","last_undegraded":"2023-10-09T05:45:00.085333+0000","last_fullsized":"2023-10-09T05:45:00.085333+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:00:54.145979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":470,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.967862+0000","last_change":"2023-10-09T05:18:26.413106+0000","last_active":"2023-10-09T05:45:08.967862+0000","last_peered":"2023-10-09T05:45:08.967862+0000","last_clean":"2023-10-09T05:45:08.967862+0000","last_became_active":"2023-10-09T05:18:26.412334+0000","last_became_peered":"2023-10-09T05:18:26.412334+0000","last_unstale":"2023-10-09T05:45:08.967862+0000","last_undegraded":"2023-10-09T05:45:08.967862+0000","last_fullsized":"2023-10-09T05:45:08.967862+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:18.058016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":342,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.652610+0000","last_change":"2023-10-09T05:42:12.631719+0000","last_active":"2023-10-09T05:45:08.652610+0000","last_peered":"2023-10-09T05:45:08.652610+0000","last_clean":"2023-10-09T05:45:08.652610+0000","last_became_active":"2023-10-09T05:42:12.631350+0000","last_became_peered":"2023-10-09T05:42:12.631350+0000","last_unstale":"2023-10-09T05:45:08.652610+0000","last_undegraded":"2023-10-09T05:45:08.652610+0000","last_fullsized":"2023-10-09T05:45:08.652610+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:21:43.609792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"421'120","reported_seq":769,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.652914+0000","last_change":"2023-10-09T05:41:43.988213+0000","last_active":"2023-10-09T05:45:08.652914+0000","last_peered":"2023-10-09T05:45:08.652914+0000","last_clean":"2023-10-09T05:45:08.652914+0000","last_became_active":"2023-10-09T05:18:26.421853+0000","last_became_peered":"2023-10-09T05:18:26.421853+0000","last_unstale":"2023-10-09T05:45:08.652914+0000","last_undegraded":"2023-10-09T05:45:08.652914+0000","last_fullsized":"2023-10-09T05:45:08.652914+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":120,"log_dups_size":0,"ondisk_log_size":120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:37:47.000556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045958700000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":637,"num_read_kb":2582,"num_write":123,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":158,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.675275+0000","last_change":"2023-10-09T05:32:45.832271+0000","last_active":"2023-10-09T05:44:59.675275+0000","last_peered":"2023-10-09T05:44:59.675275+0000","last_clean":"2023-10-09T05:44:59.675275+0000","last_became_active":"2023-10-09T05:32:45.831997+0000","last_became_peered":"2023-10-09T05:32:45.831997+0000","last_unstale":"2023-10-09T05:44:59.675275+0000","last_undegraded":"2023-10-09T05:44:59.675275+0000","last_fullsized":"2023-10-09T05:44:59.675275+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:18.970416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":28,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:05.027074+0000","last_change":"2023-10-09T05:42:59.632813+0000","last_active":"2023-10-09T05:45:05.027074+0000","last_peered":"2023-10-09T05:45:05.027074+0000","last_clean":"2023-10-09T05:45:05.027074+0000","last_became_active":"2023-10-09T05:42:59.632590+0000","last_became_peered":"2023-10-09T05:42:59.632590+0000","last_unstale":"2023-10-09T05:45:05.027074+0000","last_undegraded":"2023-10-09T05:45:05.027074+0000","last_fullsized":"2023-10-09T05:45:05.027074+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:38:34.386951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"421'156","reported_seq":1149,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.674823+0000","last_change":"2023-10-09T05:41:43.998213+0000","last_active":"2023-10-09T05:44:59.674823+0000","last_peered":"2023-10-09T05:44:59.674823+0000","last_clean":"2023-10-09T05:44:59.674823+0000","last_became_active":"2023-10-09T05:32:45.824981+0000","last_became_peered":"2023-10-09T05:32:45.824981+0000","last_unstale":"2023-10-09T05:44:59.674823+0000","last_undegraded":"2023-10-09T05:44:59.674823+0000","last_fullsized":"2023-10-09T05:44:59.674823+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":156,"log_dups_size":0,"ondisk_log_size":156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:02.868005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056430599999999999,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1612,"num_read_kb":3505,"num_write":104,"num_write_kb":6199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":34,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.085609+0000","last_change":"2023-10-09T05:42:12.631913+0000","last_active":"2023-10-09T05:45:00.085609+0000","last_peered":"2023-10-09T05:45:00.085609+0000","last_clean":"2023-10-09T05:45:00.085609+0000","last_became_active":"2023-10-09T05:42:12.631552+0000","last_became_peered":"2023-10-09T05:42:12.631552+0000","last_unstale":"2023-10-09T05:45:00.085609+0000","last_undegraded":"2023-10-09T05:45:00.085609+0000","last_fullsized":"2023-10-09T05:45:00.085609+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:55:59.713353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":218,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.652663+0000","last_change":"2023-10-09T05:23:18.196260+0000","last_active":"2023-10-09T05:45:08.652663+0000","last_peered":"2023-10-09T05:45:08.652663+0000","last_clean":"2023-10-09T05:45:08.652663+0000","last_became_active":"2023-10-09T05:23:18.195907+0000","last_became_peered":"2023-10-09T05:23:18.195907+0000","last_unstale":"2023-10-09T05:45:08.652663+0000","last_undegraded":"2023-10-09T05:45:08.652663+0000","last_fullsized":"2023-10-09T05:45:08.652663+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:36.243432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"421'188","reported_seq":1078,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.968954+0000","last_change":"2023-10-09T05:41:44.001943+0000","last_active":"2023-10-09T05:45:08.968954+0000","last_peered":"2023-10-09T05:45:08.968954+0000","last_clean":"2023-10-09T05:45:08.968954+0000","last_became_active":"2023-10-09T05:20:18.122859+0000","last_became_peered":"2023-10-09T05:20:18.122859+0000","last_unstale":"2023-10-09T05:45:08.968954+0000","last_undegraded":"2023-10-09T05:45:08.968954+0000","last_fullsized":"2023-10-09T05:45:08.968954+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":188,"log_dups_size":0,"ondisk_log_size":188,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.906228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040378799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1531,"num_read_kb":1451,"num_write":151,"num_write_kb":164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":326,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.652392+0000","last_change":"2023-10-09T05:09:57.854064+0000","last_active":"2023-10-09T05:45:08.652392+0000","last_peered":"2023-10-09T05:45:08.652392+0000","last_clean":"2023-10-09T05:45:08.652392+0000","last_became_active":"2023-10-09T05:09:57.847953+0000","last_became_peered":"2023-10-09T05:09:57.847953+0000","last_unstale":"2023-10-09T05:45:08.652392+0000","last_undegraded":"2023-10-09T05:45:08.652392+0000","last_fullsized":"2023-10-09T05:45:08.652392+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:45.914976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":28,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.652513+0000","last_change":"2023-10-09T05:42:59.843795+0000","last_active":"2023-10-09T05:45:08.652513+0000","last_peered":"2023-10-09T05:45:08.652513+0000","last_clean":"2023-10-09T05:45:08.652513+0000","last_became_active":"2023-10-09T05:42:59.843481+0000","last_became_peered":"2023-10-09T05:42:59.843481+0000","last_unstale":"2023-10-09T05:45:08.652513+0000","last_undegraded":"2023-10-09T05:45:08.652513+0000","last_fullsized":"2023-10-09T05:45:08.652513+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:01:35.732837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"421'172","reported_seq":685,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.674851+0000","last_change":"2023-10-09T05:41:44.000425+0000","last_active":"2023-10-09T05:44:59.674851+0000","last_peered":"2023-10-09T05:44:59.674851+0000","last_clean":"2023-10-09T05:44:59.674851+0000","last_became_active":"2023-10-09T05:32:45.824730+0000","last_became_peered":"2023-10-09T05:32:45.824730+0000","last_unstale":"2023-10-09T05:44:59.674851+0000","last_undegraded":"2023-10-09T05:44:59.674851+0000","last_fullsized":"2023-10-09T05:44:59.674851+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":172,"log_dups_size":0,"ondisk_log_size":172,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:33.484326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0016454359999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":798,"num_read_kb":1755,"num_write":158,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":731,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.964386+0000","last_change":"2023-10-09T05:18:29.419527+0000","last_active":"2023-10-09T05:44:59.964386+0000","last_peered":"2023-10-09T05:44:59.964386+0000","last_clean":"2023-10-09T05:44:59.964386+0000","last_became_active":"2023-10-09T05:18:29.418852+0000","last_became_peered":"2023-10-09T05:18:29.418852+0000","last_unstale":"2023-10-09T05:44:59.964386+0000","last_undegraded":"2023-10-09T05:44:59.964386+0000","last_fullsized":"2023-10-09T05:44:59.964386+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_clean_scrub_stamp":"2023-10-09T04:36:44.261013+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:02.899170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":438,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.968899+0000","last_change":"2023-10-09T05:32:45.829840+0000","last_active":"2023-10-09T05:45:08.968899+0000","last_peered":"2023-10-09T05:45:08.968899+0000","last_clean":"2023-10-09T05:45:08.968899+0000","last_became_active":"2023-10-09T05:32:45.829084+0000","last_became_peered":"2023-10-09T05:32:45.829084+0000","last_unstale":"2023-10-09T05:45:08.968899+0000","last_undegraded":"2023-10-09T05:45:08.968899+0000","last_fullsized":"2023-10-09T05:45:08.968899+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:26:48.836506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":452,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.679720+0000","last_change":"2023-10-09T05:42:12.634383+0000","last_active":"2023-10-09T05:44:59.679720+0000","last_peered":"2023-10-09T05:44:59.679720+0000","last_clean":"2023-10-09T05:44:59.679720+0000","last_became_active":"2023-10-09T05:42:12.634002+0000","last_became_peered":"2023-10-09T05:42:12.634002+0000","last_unstale":"2023-10-09T05:44:59.679720+0000","last_undegraded":"2023-10-09T05:44:59.679720+0000","last_fullsized":"2023-10-09T05:44:59.679720+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:32.709357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"420'76","reported_seq":504,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.964485+0000","last_change":"2023-10-09T05:41:43.992881+0000","last_active":"2023-10-09T05:44:59.964485+0000","last_peered":"2023-10-09T05:44:59.964485+0000","last_clean":"2023-10-09T05:44:59.964485+0000","last_became_active":"2023-10-09T05:23:18.193394+0000","last_became_peered":"2023-10-09T05:23:18.193394+0000","last_unstale":"2023-10-09T05:44:59.964485+0000","last_undegraded":"2023-10-09T05:44:59.964485+0000","last_fullsized":"2023-10-09T05:44:59.964485+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":76,"log_dups_size":0,"ondisk_log_size":76,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:40.594469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067895100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":420,"num_read_kb":1654,"num_write":61,"num_write_kb":1190,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":446,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.081677+0000","last_change":"2023-10-09T05:42:12.629510+0000","last_active":"2023-10-09T05:45:00.081677+0000","last_peered":"2023-10-09T05:45:00.081677+0000","last_clean":"2023-10-09T05:45:00.081677+0000","last_became_active":"2023-10-09T05:42:12.628940+0000","last_became_peered":"2023-10-09T05:42:12.628940+0000","last_unstale":"2023-10-09T05:45:00.081677+0000","last_undegraded":"2023-10-09T05:45:00.081677+0000","last_fullsized":"2023-10-09T05:45:00.081677+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:32:38.949076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":497,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.081722+0000","last_change":"2023-10-09T05:20:18.132750+0000","last_active":"2023-10-09T05:45:00.081722+0000","last_peered":"2023-10-09T05:45:00.081722+0000","last_clean":"2023-10-09T05:45:00.081722+0000","last_became_active":"2023-10-09T05:20:18.132396+0000","last_became_peered":"2023-10-09T05:20:18.132396+0000","last_unstale":"2023-10-09T05:45:00.081722+0000","last_undegraded":"2023-10-09T05:45:00.081722+0000","last_fullsized":"2023-10-09T05:45:00.081722+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:00:55.731045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"420'64","reported_seq":474,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.965416+0000","last_change":"2023-10-09T05:42:12.666443+0000","last_active":"2023-10-09T05:44:59.965416+0000","last_peered":"2023-10-09T05:44:59.965416+0000","last_clean":"2023-10-09T05:44:59.965416+0000","last_became_active":"2023-10-09T05:42:12.632817+0000","last_became_peered":"2023-10-09T05:42:12.632817+0000","last_unstale":"2023-10-09T05:44:59.965416+0000","last_undegraded":"2023-10-09T05:44:59.965416+0000","last_fullsized":"2023-10-09T05:44:59.965416+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":64,"log_dups_size":0,"ondisk_log_size":64,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:33.863984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042561999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":276,"num_read_kb":753,"num_write":62,"num_write_kb":1323,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":361,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.653140+0000","last_change":"2023-10-09T05:32:45.824227+0000","last_active":"2023-10-09T05:45:08.653140+0000","last_peered":"2023-10-09T05:45:08.653140+0000","last_clean":"2023-10-09T05:45:08.653140+0000","last_became_active":"2023-10-09T05:32:45.823662+0000","last_became_peered":"2023-10-09T05:32:45.823662+0000","last_unstale":"2023-10-09T05:45:08.653140+0000","last_undegraded":"2023-10-09T05:45:08.653140+0000","last_fullsized":"2023-10-09T05:45:08.653140+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:54:56.029126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":468,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.081539+0000","last_change":"2023-10-09T05:32:45.826815+0000","last_active":"2023-10-09T05:45:00.081539+0000","last_peered":"2023-10-09T05:45:00.081539+0000","last_clean":"2023-10-09T05:45:00.081539+0000","last_became_active":"2023-10-09T05:32:45.826529+0000","last_became_peered":"2023-10-09T05:32:45.826529+0000","last_unstale":"2023-10-09T05:45:00.081539+0000","last_undegraded":"2023-10-09T05:45:00.081539+0000","last_fullsized":"2023-10-09T05:45:00.081539+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:19.868533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":463,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.970586+0000","last_change":"2023-10-09T05:18:26.431563+0000","last_active":"2023-10-09T05:45:08.970586+0000","last_peered":"2023-10-09T05:45:08.970586+0000","last_clean":"2023-10-09T05:45:08.970586+0000","last_became_active":"2023-10-09T05:18:26.421025+0000","last_became_peered":"2023-10-09T05:18:26.421025+0000","last_unstale":"2023-10-09T05:45:08.970586+0000","last_undegraded":"2023-10-09T05:45:08.970586+0000","last_fullsized":"2023-10-09T05:45:08.970586+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:30.333585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"421'103","reported_seq":589,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.026281+0000","last_change":"2023-10-09T05:42:15.024058+0000","last_active":"2023-10-09T05:45:00.026281+0000","last_peered":"2023-10-09T05:45:00.026281+0000","last_clean":"2023-10-09T05:45:00.026281+0000","last_became_active":"2023-10-09T05:42:15.023495+0000","last_became_peered":"2023-10-09T05:42:15.023495+0000","last_unstale":"2023-10-09T05:45:00.026281+0000","last_undegraded":"2023-10-09T05:45:00.026281+0000","last_fullsized":"2023-10-09T05:45:00.026281+0000","mapping_epoch":427,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":428,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:05.851529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000511664,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":467,"num_read_kb":908,"num_write":75,"num_write_kb":309,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":463,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.967864+0000","last_change":"2023-10-09T05:21:08.179657+0000","last_active":"2023-10-09T05:45:08.967864+0000","last_peered":"2023-10-09T05:45:08.967864+0000","last_clean":"2023-10-09T05:45:08.967864+0000","last_became_active":"2023-10-09T05:21:08.178958+0000","last_became_peered":"2023-10-09T05:21:08.178958+0000","last_unstale":"2023-10-09T05:45:08.967864+0000","last_undegraded":"2023-10-09T05:45:08.967864+0000","last_fullsized":"2023-10-09T05:45:08.967864+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:20.815986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"420'97","reported_seq":657,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.982980+0000","last_change":"2023-10-09T05:44:59.982624+0000","last_active":"2023-10-09T05:44:59.982980+0000","last_peered":"2023-10-09T05:44:59.982980+0000","last_clean":"2023-10-09T05:44:59.982980+0000","last_became_active":"2023-10-09T05:44:59.968921+0000","last_became_peered":"2023-10-09T05:44:59.968921+0000","last_unstale":"2023-10-09T05:44:59.982980+0000","last_undegraded":"2023-10-09T05:44:59.982980+0000","last_fullsized":"2023-10-09T05:44:59.982980+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:26.685207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010964410000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":254,"num_read_kb":640,"num_write":112,"num_write_kb":304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":188,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.970332+0000","last_change":"2023-10-09T05:42:12.642659+0000","last_active":"2023-10-09T05:45:08.970332+0000","last_peered":"2023-10-09T05:45:08.970332+0000","last_clean":"2023-10-09T05:45:08.970332+0000","last_became_active":"2023-10-09T05:42:12.629784+0000","last_became_peered":"2023-10-09T05:42:12.629784+0000","last_unstale":"2023-10-09T05:45:08.970332+0000","last_undegraded":"2023-10-09T05:45:08.970332+0000","last_fullsized":"2023-10-09T05:45:08.970332+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:41.316243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":34,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.085159+0000","last_change":"2023-10-09T05:42:12.624836+0000","last_active":"2023-10-09T05:45:00.085159+0000","last_peered":"2023-10-09T05:45:00.085159+0000","last_clean":"2023-10-09T05:45:00.085159+0000","last_became_active":"2023-10-09T05:42:12.624196+0000","last_became_peered":"2023-10-09T05:42:12.624196+0000","last_unstale":"2023-10-09T05:45:00.085159+0000","last_undegraded":"2023-10-09T05:45:00.085159+0000","last_fullsized":"2023-10-09T05:45:00.085159+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:56:36.734026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'2","reported_seq":253,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.969849+0000","last_change":"2023-10-09T05:42:12.634551+0000","last_active":"2023-10-09T05:45:08.969849+0000","last_peered":"2023-10-09T05:45:08.969849+0000","last_clean":"2023-10-09T05:45:08.969849+0000","last_became_active":"2023-10-09T05:42:12.633757+0000","last_became_peered":"2023-10-09T05:42:12.633757+0000","last_unstale":"2023-10-09T05:45:08.969849+0000","last_undegraded":"2023-10-09T05:45:08.969849+0000","last_fullsized":"2023-10-09T05:45:08.969849+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:09.847947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":242,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.969022+0000","last_change":"2023-10-09T05:36:16.898207+0000","last_active":"2023-10-09T05:45:08.969022+0000","last_peered":"2023-10-09T05:45:08.969022+0000","last_clean":"2023-10-09T05:45:08.969022+0000","last_became_active":"2023-10-09T05:36:16.898009+0000","last_became_peered":"2023-10-09T05:36:16.898009+0000","last_unstale":"2023-10-09T05:45:08.969022+0000","last_undegraded":"2023-10-09T05:45:08.969022+0000","last_fullsized":"2023-10-09T05:45:08.969022+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:59.694977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":176,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.675035+0000","last_change":"2023-10-09T05:42:12.638282+0000","last_active":"2023-10-09T05:44:59.675035+0000","last_peered":"2023-10-09T05:44:59.675035+0000","last_clean":"2023-10-09T05:44:59.675035+0000","last_became_active":"2023-10-09T05:42:12.625036+0000","last_became_peered":"2023-10-09T05:42:12.625036+0000","last_unstale":"2023-10-09T05:44:59.675035+0000","last_undegraded":"2023-10-09T05:44:59.675035+0000","last_fullsized":"2023-10-09T05:44:59.675035+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:18:08.570371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":44,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.085713+0000","last_change":"2023-10-09T05:42:59.630617+0000","last_active":"2023-10-09T05:45:00.085713+0000","last_peered":"2023-10-09T05:45:00.085713+0000","last_clean":"2023-10-09T05:45:00.085713+0000","last_became_active":"2023-10-09T05:42:59.630103+0000","last_became_peered":"2023-10-09T05:42:59.630103+0000","last_unstale":"2023-10-09T05:45:00.085713+0000","last_undegraded":"2023-10-09T05:45:00.085713+0000","last_fullsized":"2023-10-09T05:45:00.085713+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:14:32.213142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":176,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.675261+0000","last_change":"2023-10-09T05:42:12.644588+0000","last_active":"2023-10-09T05:44:59.675261+0000","last_peered":"2023-10-09T05:44:59.675261+0000","last_clean":"2023-10-09T05:44:59.675261+0000","last_became_active":"2023-10-09T05:42:12.629471+0000","last_became_peered":"2023-10-09T05:42:12.629471+0000","last_unstale":"2023-10-09T05:44:59.675261+0000","last_undegraded":"2023-10-09T05:44:59.675261+0000","last_fullsized":"2023-10-09T05:44:59.675261+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:15:31.005959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":34,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.084837+0000","last_change":"2023-10-09T05:42:12.628333+0000","last_active":"2023-10-09T05:45:00.084837+0000","last_peered":"2023-10-09T05:45:00.084837+0000","last_clean":"2023-10-09T05:45:00.084837+0000","last_became_active":"2023-10-09T05:42:12.625671+0000","last_became_peered":"2023-10-09T05:42:12.625671+0000","last_unstale":"2023-10-09T05:45:00.084837+0000","last_undegraded":"2023-10-09T05:45:00.084837+0000","last_fullsized":"2023-10-09T05:45:00.084837+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:41.463748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'2","reported_seq":457,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.970869+0000","last_change":"2023-10-09T05:22:47.197002+0000","last_active":"2023-10-09T05:45:08.970869+0000","last_peered":"2023-10-09T05:45:08.970869+0000","last_clean":"2023-10-09T05:45:08.970869+0000","last_became_active":"2023-10-09T05:22:47.196685+0000","last_became_peered":"2023-10-09T05:22:47.196685+0000","last_unstale":"2023-10-09T05:45:08.970869+0000","last_undegraded":"2023-10-09T05:45:08.970869+0000","last_fullsized":"2023-10-09T05:45:08.970869+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:47:32.594447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":440,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.968162+0000","last_change":"2023-10-09T05:02:09.649674+0000","last_active":"2023-10-09T05:45:08.968162+0000","last_peered":"2023-10-09T05:45:08.968162+0000","last_clean":"2023-10-09T05:45:08.968162+0000","last_became_active":"2023-10-09T05:02:09.649041+0000","last_became_peered":"2023-10-09T05:02:09.649041+0000","last_unstale":"2023-10-09T05:45:08.968162+0000","last_undegraded":"2023-10-09T05:45:08.968162+0000","last_fullsized":"2023-10-09T05:45:08.968162+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:18.524071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'5","reported_seq":249,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.968808+0000","last_change":"2023-10-09T05:32:45.849292+0000","last_active":"2023-10-09T05:45:08.968808+0000","last_peered":"2023-10-09T05:45:08.968808+0000","last_clean":"2023-10-09T05:45:08.968808+0000","last_became_active":"2023-10-09T05:32:45.829351+0000","last_became_peered":"2023-10-09T05:32:45.829351+0000","last_unstale":"2023-10-09T05:45:08.968808+0000","last_undegraded":"2023-10-09T05:45:08.968808+0000","last_fullsized":"2023-10-09T05:45:08.968808+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:07:15.666617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":34,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.084596+0000","last_change":"2023-10-09T05:42:12.621549+0000","last_active":"2023-10-09T05:45:00.084596+0000","last_peered":"2023-10-09T05:45:00.084596+0000","last_clean":"2023-10-09T05:45:00.084596+0000","last_became_active":"2023-10-09T05:42:12.621190+0000","last_became_peered":"2023-10-09T05:42:12.621190+0000","last_unstale":"2023-10-09T05:45:00.084596+0000","last_undegraded":"2023-10-09T05:45:00.084596+0000","last_fullsized":"2023-10-09T05:45:00.084596+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:43.642340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":23,"num_read_kb":22,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":74,"num_bytes_recovered":22374,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234772,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1510260,"kb_used_data":5228,"kb_used_omap":0,"kb_used_meta":1505024,"kb_avail":92234892,"statfs":{"total":95995035648,"available":94448529408,"internally_reserved":0,"allocated":5353472,"data_stored":4014652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1541144576},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:45:35 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":1.0549999999999999,"15min":2.855},"min":{"1min":0.65500000000000003,"5min":0.42599999999999999,"15min":0.26100000000000001},"max":{"1min":1.347,"5min":11.784000000000001,"15min":114.259},"last":0.61799999999999999},{"interface":"front","average":{"1min":1.1499999999999999,"5min":1.123,"15min":2.9710000000000001},"min":{"1min":0.85599999999999998,"5min":0.40100000000000002,"15min":0.28899999999999998},"max":{"1min":1.4450000000000001,"5min":8.2949999999999999,"15min":114.286},"last":0.70799999999999996}]},{"osd":1,"last update":"Mon Oct 9 05:46:11 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":0.90900000000000003,"15min":1.6140000000000001},"min":{"1min":0.57499999999999996,"5min":0.43099999999999999,"15min":0.27100000000000002},"max":{"1min":1.048,"5min":1.306,"15min":44.456000000000003},"last":0.85899999999999999},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.97399999999999998,"15min":1.712},"min":{"1min":0.56399999999999995,"5min":0.53100000000000003,"15min":0.318},"max":{"1min":1.387,"5min":1.462,"15min":44.395000000000003},"last":1.048}]},{"osd":2,"last update":"Mon Oct 9 05:45:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":0.98499999999999999,"15min":1.7889999999999999},"min":{"1min":0.63200000000000001,"5min":0.56299999999999994,"15min":0.28999999999999998},"max":{"1min":1.3480000000000001,"5min":1.5620000000000001,"15min":102.146},"last":0.96399999999999997},{"interface":"front","average":{"1min":0.93300000000000005,"5min":0.93700000000000006,"15min":1.6950000000000001},"min":{"1min":0.76000000000000001,"5min":0.40600000000000003,"15min":0.35899999999999999},"max":{"1min":1.139,"5min":1.3700000000000001,"15min":102.188},"last":0.67000000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:45:35 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":0.96499999999999997,"15min":1.5509999999999999},"min":{"1min":0.69699999999999995,"5min":0.52300000000000002,"15min":0.33500000000000002},"max":{"1min":1.302,"5min":1.464,"15min":112.679},"last":0.81599999999999995},{"interface":"front","average":{"1min":1.0680000000000001,"5min":1.0389999999999999,"15min":1.6240000000000001},"min":{"1min":0.64400000000000002,"5min":0.53500000000000003,"15min":0.39600000000000002},"max":{"1min":1.53,"5min":1.53,"15min":109.899},"last":0.88100000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:46:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":0.97499999999999998,"15min":0.93400000000000005},"min":{"1min":0.77500000000000002,"5min":0.48499999999999999,"15min":0.309},"max":{"1min":1.375,"5min":1.478,"15min":7.077},"last":1.075},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.089,"15min":1.0640000000000001},"min":{"1min":0.71799999999999997,"5min":0.53700000000000003,"15min":0.36699999999999999},"max":{"1min":1.3859999999999999,"5min":1.575,"15min":7.3869999999999996},"last":1.103}]},{"osd":5,"last update":"Mon Oct 9 05:46:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.0309999999999999,"15min":1.0880000000000001},"min":{"1min":0.70099999999999996,"5min":0.46400000000000002,"15min":0.33500000000000002},"max":{"1min":1.335,"5min":1.528,"15min":7.4829999999999997},"last":1.008},{"interface":"front","average":{"1min":1.1899999999999999,"5min":1.1619999999999999,"15min":1.177},"min":{"1min":0.94999999999999996,"5min":0.63900000000000001,"15min":0.35599999999999998},"max":{"1min":1.4610000000000001,"5min":1.6100000000000001,"15min":7.6219999999999999},"last":1.1399999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:46:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":1.179,"15min":1.1870000000000001},"min":{"1min":0.91900000000000004,"5min":0.64300000000000002,"15min":0.38700000000000001},"max":{"1min":1.452,"5min":1.5529999999999999,"15min":6.8789999999999996},"last":1.214},{"interface":"front","average":{"1min":1.3839999999999999,"5min":1.266,"15min":1.262},"min":{"1min":0.747,"5min":0.51200000000000001,"15min":0.432},"max":{"1min":3.702,"5min":3.702,"15min":7.46},"last":1.1719999999999999}]}]},{"osd":6,"up_from":46,"seq":197568496414,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1511920,"kb_used_data":9320,"kb_used_omap":0,"kb_used_meta":1502592,"kb_avail":92233232,"statfs":{"total":95995035648,"available":94446829568,"internally_reserved":0,"allocated":9543680,"data_stored":8209152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1538654208},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:45:30 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":0.94899999999999995,"15min":3.5939999999999999},"min":{"1min":0.499,"5min":0.42399999999999999,"15min":0.254},"max":{"1min":1.4470000000000001,"5min":2.8820000000000001,"15min":266.40499999999997},"last":0.68100000000000005},{"interface":"front","average":{"1min":1.1990000000000001,"5min":1.0920000000000001,"15min":3.7029999999999998},"min":{"1min":0.749,"5min":0.53900000000000003,"15min":0.32200000000000001},"max":{"1min":1.5129999999999999,"5min":2.9630000000000001,"15min":266.49700000000001},"last":1.262}]},{"osd":1,"last update":"Mon Oct 9 05:45:30 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.88200000000000001,"15min":1.3400000000000001},"min":{"1min":0.80200000000000005,"5min":0.45400000000000001,"15min":0.28299999999999997},"max":{"1min":1.3700000000000001,"5min":1.413,"15min":41.408000000000001},"last":1.0169999999999999},{"interface":"front","average":{"1min":0.96499999999999997,"5min":0.93100000000000005,"15min":1.3069999999999999},"min":{"1min":0.63,"5min":0.53200000000000003,"15min":0.28199999999999997},"max":{"1min":1.3799999999999999,"5min":1.389,"15min":40.835999999999999},"last":1.3660000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:45:30 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1,"15min":2.718},"min":{"1min":0.73499999999999999,"5min":0.435,"15min":0.26600000000000001},"max":{"1min":1.395,"5min":1.6120000000000001,"15min":211.66300000000001},"last":1.327},{"interface":"front","average":{"1min":0.96399999999999997,"5min":0.88200000000000001,"15min":2.9430000000000001},"min":{"1min":0.59099999999999997,"5min":0.51700000000000002,"15min":0.27900000000000003},"max":{"1min":1.2190000000000001,"5min":1.2190000000000001,"15min":211.423},"last":0.77500000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:45:30 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":0.97199999999999998,"15min":1.9990000000000001},"min":{"1min":0.83399999999999996,"5min":0.46300000000000002,"15min":0.309},"max":{"1min":1.4630000000000001,"5min":1.4630000000000001,"15min":209.69},"last":0.85299999999999998},{"interface":"front","average":{"1min":1.1240000000000001,"5min":1.0680000000000001,"15min":2.0230000000000001},"min":{"1min":0.79500000000000004,"5min":0.56100000000000005,"15min":0.30399999999999999},"max":{"1min":1.526,"5min":1.5509999999999999,"15min":209.34800000000001},"last":1.403}]},{"osd":4,"last update":"Mon Oct 9 05:45:30 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":0.95799999999999996,"15min":0.92100000000000004},"min":{"1min":0.70799999999999996,"5min":0.498,"15min":0.32300000000000001},"max":{"1min":1.371,"5min":1.786,"15min":10.976000000000001},"last":1.111},{"interface":"front","average":{"1min":1.159,"5min":1.103,"15min":1.03},"min":{"1min":0.91500000000000004,"5min":0.49299999999999999,"15min":0.32800000000000001},"max":{"1min":1.5780000000000001,"5min":1.766,"15min":5.0190000000000001},"last":1.4550000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:45:30 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.038,"15min":1.1100000000000001},"min":{"1min":0.77700000000000002,"5min":0.61899999999999999,"15min":0.29299999999999998},"max":{"1min":1.446,"5min":2.214,"15min":11.275},"last":1.1919999999999999},{"interface":"front","average":{"1min":1.115,"5min":1.153,"15min":1.097},"min":{"1min":0.67100000000000004,"5min":0.51300000000000001,"15min":0.36199999999999999},"max":{"1min":1.4019999999999999,"5min":1.7450000000000001,"15min":7.3090000000000002},"last":1.502}]},{"osd":7,"last update":"Mon Oct 9 05:45:52 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":0.97799999999999998,"15min":1.2410000000000001},"min":{"1min":0.67600000000000005,"5min":0.46800000000000003,"15min":0.34999999999999998},"max":{"1min":1.2969999999999999,"5min":1.9730000000000001,"15min":18.963000000000001},"last":1.218},{"interface":"front","average":{"1min":1.1539999999999999,"5min":1.135,"15min":1.3560000000000001},"min":{"1min":0.81799999999999995,"5min":0.54600000000000004,"15min":0.39500000000000002},"max":{"1min":1.4350000000000001,"5min":2.0419999999999998,"15min":19.010000000000002},"last":1.423}]}]},{"osd":1,"up_from":96,"seq":412316861005,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1325936,"kb_used_data":10152,"kb_used_omap":4,"kb_used_meta":1315771,"kb_avail":92419216,"statfs":{"total":95995035648,"available":94637277184,"internally_reserved":0,"allocated":10395648,"data_stored":7794577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1347350414},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.66500000000000004,"15min":2.6760000000000002},"min":{"1min":0.30499999999999999,"5min":0.29299999999999998,"15min":0.21099999999999999},"max":{"1min":0.86299999999999999,"5min":0.99199999999999999,"15min":150.75200000000001},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.71099999999999997,"15min":2.6989999999999998},"min":{"1min":0.38900000000000001,"5min":0.34200000000000003,"15min":0.17499999999999999},"max":{"1min":1.1739999999999999,"5min":1.1739999999999999,"15min":150.59299999999999},"last":0.78600000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.71799999999999997,"15min":2.9039999999999999},"min":{"1min":0.37,"5min":0.31900000000000001,"15min":0.19800000000000001},"max":{"1min":1.139,"5min":1.2569999999999999,"15min":154.14599999999999},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.70199999999999996,"5min":0.84899999999999998,"15min":2.7879999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.24199999999999999},"max":{"1min":1.018,"5min":1.379,"15min":154.215},"last":0.93600000000000005}]},{"osd":3,"last update":"Mon Oct 9 05:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.72299999999999998,"15min":2.2949999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.217},"max":{"1min":0.93799999999999994,"5min":1.21,"15min":152.58199999999999},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.745,"5min":0.86899999999999999,"15min":2.282},"min":{"1min":0.42999999999999999,"5min":0.42599999999999999,"15min":0.255},"max":{"1min":1.0760000000000001,"5min":1.2150000000000001,"15min":154.178},"last":1.0469999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:46:29 2023","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":1.0800000000000001,"15min":1.329},"min":{"1min":0.755,"5min":0.58799999999999997,"15min":0.38700000000000001},"max":{"1min":1.278,"5min":1.518,"15min":8.4849999999999994},"last":1.101},{"interface":"front","average":{"1min":0.95299999999999996,"5min":0.97499999999999998,"15min":1.208},"min":{"1min":0.67200000000000004,"5min":0.65200000000000002,"15min":0.36799999999999999},"max":{"1min":1.3680000000000001,"5min":1.4259999999999999,"15min":7.2140000000000004},"last":0.92200000000000004}]},{"osd":5,"last update":"Mon Oct 9 05:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":1.0189999999999999,"15min":1.3959999999999999},"min":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.32300000000000001},"max":{"1min":1.3,"5min":1.5149999999999999,"15min":12.630000000000001},"last":0.83199999999999996},{"interface":"front","average":{"1min":1.002,"5min":1.1180000000000001,"15min":1.419},"min":{"1min":0.57699999999999996,"5min":0.56399999999999995,"15min":0.39700000000000002},"max":{"1min":1.452,"5min":1.5620000000000001,"15min":9.8260000000000005},"last":1.1619999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:46:29 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.0740000000000001,"15min":1.407},"min":{"1min":0.752,"5min":0.68899999999999995,"15min":0.318},"max":{"1min":1.4279999999999999,"5min":1.6539999999999999,"15min":12.686999999999999},"last":1.0009999999999999},{"interface":"front","average":{"1min":1.091,"5min":1.194,"15min":1.484},"min":{"1min":0.65900000000000003,"5min":0.61499999999999999,"15min":0.41499999999999998},"max":{"1min":1.377,"5min":1.6579999999999999,"15min":10.446},"last":1.3200000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:46:29 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.234,"15min":1.5},"min":{"1min":0.73299999999999998,"5min":0.66100000000000003,"15min":0.38800000000000001},"max":{"1min":1.532,"5min":1.738,"15min":8.7200000000000006},"last":1.268},{"interface":"front","average":{"1min":1.107,"5min":1.1200000000000001,"15min":1.343},"min":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.38600000000000001},"max":{"1min":1.4690000000000001,"5min":1.492,"15min":7.2560000000000002},"last":1.0720000000000001}]}]},{"osd":0,"up_from":88,"seq":377957122646,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1251280,"kb_used_data":6472,"kb_used_omap":5,"kb_used_meta":1244794,"kb_avail":92493872,"statfs":{"total":95995035648,"available":94713724928,"internally_reserved":0,"allocated":6627328,"data_stored":4169435,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1274669690},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.68400000000000005,"15min":1.786},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.17699999999999999},"max":{"1min":0.90200000000000002,"5min":1.915,"15min":111.001},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.75700000000000001,"15min":1.859},"min":{"1min":0.41799999999999998,"5min":0.36099999999999999,"15min":0.20799999999999999},"max":{"1min":0.96599999999999997,"5min":2.4980000000000002,"15min":110.919},"last":1.1100000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:45:26 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":0.79000000000000004,"15min":1.9330000000000001},"min":{"1min":0.45800000000000002,"5min":0.35599999999999998,"15min":0.25900000000000001},"max":{"1min":1.1479999999999999,"5min":2.266,"15min":74.817999999999998},"last":0.89300000000000002},{"interface":"front","average":{"1min":0.77700000000000002,"5min":0.76500000000000001,"15min":2.0950000000000002},"min":{"1min":0.41099999999999998,"5min":0.27700000000000002,"15min":0.25600000000000001},"max":{"1min":1.161,"5min":1.75,"15min":74.751000000000005},"last":0.93500000000000005}]},{"osd":3,"last update":"Mon Oct 9 05:45:26 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":0.73599999999999999,"15min":3.536},"min":{"1min":0.53600000000000003,"5min":0.35199999999999998,"15min":0.22500000000000001},"max":{"1min":1.21,"5min":1.8660000000000001,"15min":290.73000000000002},"last":0.80100000000000005},{"interface":"front","average":{"1min":0.878,"5min":0.85999999999999999,"15min":3.456},"min":{"1min":0.63900000000000001,"5min":0.312,"15min":0.246},"max":{"1min":1.121,"5min":2.3730000000000002,"15min":289.85199999999998},"last":0.97499999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:45:26 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.0049999999999999,"15min":1.355},"min":{"1min":0.80600000000000005,"5min":0.58299999999999996,"15min":0.36299999999999999},"max":{"1min":1.2609999999999999,"5min":2.4510000000000001,"15min":12.377000000000001},"last":1.2},{"interface":"front","average":{"1min":1.02,"5min":1.034,"15min":1.48},"min":{"1min":0.63700000000000001,"5min":0.52000000000000002,"15min":0.32400000000000001},"max":{"1min":1.3939999999999999,"5min":2.6150000000000002,"15min":12.477},"last":0.84699999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:45:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.042,"15min":1.5820000000000001},"min":{"1min":0.79500000000000004,"5min":0.52600000000000002,"15min":0.318},"max":{"1min":1.6859999999999999,"5min":2.4809999999999999,"15min":14.510999999999999},"last":1.3140000000000001},{"interface":"front","average":{"1min":1.151,"5min":1.1180000000000001,"15min":1.4610000000000001},"min":{"1min":0.84799999999999998,"5min":0.67000000000000004,"15min":0.33700000000000002},"max":{"1min":1.5169999999999999,"5min":2.5830000000000002,"15min":12.403},"last":1.274}]},{"osd":6,"last update":"Mon Oct 9 05:45:26 2023","interfaces":[{"interface":"back","average":{"1min":1.1879999999999999,"5min":1.1040000000000001,"15min":1.663},"min":{"1min":0.872,"5min":0.55200000000000005,"15min":0.33500000000000002},"max":{"1min":1.8700000000000001,"5min":2.6890000000000001,"15min":15.544},"last":1.026},{"interface":"front","average":{"1min":1.1319999999999999,"5min":1.0900000000000001,"15min":1.681},"min":{"1min":0.76500000000000001,"5min":0.66400000000000003,"15min":0.318},"max":{"1min":1.4830000000000001,"5min":2.4249999999999998,"15min":14.459},"last":1.1479999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:45:26 2023","interfaces":[{"interface":"back","average":{"1min":1.1639999999999999,"5min":1.1080000000000001,"15min":1.7749999999999999},"min":{"1min":0.81799999999999995,"5min":0.66800000000000004,"15min":0.39200000000000002},"max":{"1min":1.7370000000000001,"5min":2.335,"15min":14.492000000000001},"last":1.2490000000000001},{"interface":"front","average":{"1min":1.254,"5min":1.175,"15min":1.5649999999999999},"min":{"1min":0.96999999999999997,"5min":0.75700000000000001,"15min":0.40100000000000002},"max":{"1min":1.6499999999999999,"5min":2.5390000000000001,"15min":12.43},"last":1.3360000000000001}]}]},{"osd":2,"up_from":106,"seq":455266533924,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1131760,"kb_used_data":5864,"kb_used_omap":7,"kb_used_meta":1125880,"kb_avail":92613392,"statfs":{"total":95995035648,"available":94836113408,"internally_reserved":0,"allocated":6004736,"data_stored":3574377,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1152901201},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:46:14 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.65400000000000003,"15min":2.298},"min":{"1min":0.307,"5min":0.307,"15min":0.17599999999999999},"max":{"1min":1.0920000000000001,"5min":1.097,"15min":91.646000000000001},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.75700000000000001,"15min":2.1869999999999998},"min":{"1min":0.35999999999999999,"5min":0.308,"15min":0.219},"max":{"1min":0.99299999999999999,"5min":1.488,"15min":92.522000000000006},"last":0.81299999999999994}]},{"osd":1,"last update":"Mon Oct 9 05:46:14 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":0.76200000000000001,"15min":1.97},"min":{"1min":0.44800000000000001,"5min":0.32000000000000001,"15min":0.217},"max":{"1min":1.2629999999999999,"5min":1.423,"15min":120.619},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.755,"15min":2.1829999999999998},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.24299999999999999},"max":{"1min":0.97699999999999998,"5min":4.0880000000000001,"15min":122.223},"last":0.93300000000000005}]},{"osd":3,"last update":"Mon Oct 9 05:45:30 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":1.0149999999999999,"15min":1.7010000000000001},"min":{"1min":0.49399999999999999,"5min":0.30599999999999999,"15min":0.214},"max":{"1min":1.3420000000000001,"5min":15.220000000000001,"15min":15.576000000000001},"last":0.90000000000000002},{"interface":"front","average":{"1min":0.80200000000000005,"5min":0.97999999999999998,"15min":1.679},"min":{"1min":0.47299999999999998,"5min":0.372,"15min":0.28299999999999997},"max":{"1min":1.1619999999999999,"5min":15.131,"15min":15.131},"last":1.0980000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:46:14 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":1.03,"15min":1.415},"min":{"1min":0.66700000000000004,"5min":0.61099999999999999,"15min":0.31900000000000001},"max":{"1min":1.3600000000000001,"5min":1.544,"15min":10.379},"last":1.05},{"interface":"front","average":{"1min":0.94799999999999995,"5min":1.032,"15min":1.5389999999999999},"min":{"1min":0.64100000000000001,"5min":0.63500000000000001,"15min":0.35399999999999998},"max":{"1min":1.395,"5min":4.3630000000000004,"15min":14.717000000000001},"last":1.0309999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:46:14 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":1.024,"15min":1.4139999999999999},"min":{"1min":0.64000000000000001,"5min":0.54800000000000004,"15min":0.34000000000000002},"max":{"1min":1.3049999999999999,"5min":3.9159999999999999,"15min":9.8729999999999993},"last":1.202},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.103,"15min":1.4850000000000001},"min":{"1min":0.623,"5min":0.61899999999999999,"15min":0.33700000000000002},"max":{"1min":1.462,"5min":1.649,"15min":10.265000000000001},"last":0.97699999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:46:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":1.117,"15min":1.6140000000000001},"min":{"1min":0.623,"5min":0.623,"15min":0.32600000000000001},"max":{"1min":1.716,"5min":4.4850000000000003,"15min":14.763999999999999},"last":1.1739999999999999},{"interface":"front","average":{"1min":1.034,"5min":1.1259999999999999,"15min":1.4910000000000001},"min":{"1min":0.77600000000000002,"5min":0.47599999999999998,"15min":0.36399999999999999},"max":{"1min":1.482,"5min":4.0099999999999998,"15min":9.9120000000000008},"last":1.1479999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:46:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.1539999999999999,"15min":1.575},"min":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.433},"max":{"1min":1.54,"5min":3.9700000000000002,"15min":9.8930000000000007},"last":1.3480000000000001},{"interface":"front","average":{"1min":1.0960000000000001,"5min":1.1799999999999999,"15min":1.6299999999999999},"min":{"1min":0.71099999999999997,"5min":0.68500000000000005,"15min":0.39000000000000001},"max":{"1min":1.5289999999999999,"5min":1.73,"15min":12.016},"last":1.1180000000000001}]}]},{"osd":3,"up_from":123,"seq":528280977942,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1177988,"kb_used_data":5756,"kb_used_omap":2,"kb_used_meta":1172221,"kb_avail":92567164,"statfs":{"total":95995035648,"available":94788775936,"internally_reserved":0,"allocated":5894144,"data_stored":3561284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":1200354357},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:45:35 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":1.357,"15min":3.0030000000000001},"min":{"1min":0.44400000000000001,"5min":0.247,"15min":0.224},"max":{"1min":1.0289999999999999,"5min":62.600999999999999,"15min":117.06100000000001},"last":0.625},{"interface":"front","average":{"1min":0.77200000000000002,"5min":1.415,"15min":3.3980000000000001},"min":{"1min":0.36699999999999999,"5min":0.28999999999999998,"15min":0.184},"max":{"1min":1.0609999999999999,"5min":62.643000000000001,"15min":118.372},"last":0.94399999999999995}]},{"osd":1,"last update":"Mon Oct 9 05:45:35 2023","interfaces":[{"interface":"back","average":{"1min":0.77600000000000002,"5min":0.70899999999999996,"15min":2.157},"min":{"1min":0.46700000000000003,"5min":0.36399999999999999,"15min":0.23000000000000001},"max":{"1min":1.0960000000000001,"5min":1.137,"15min":118.20999999999999},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.77200000000000002,"5min":0.77000000000000002,"15min":1.9350000000000001},"min":{"1min":0.49099999999999999,"5min":0.34200000000000003,"15min":0.218},"max":{"1min":1.0109999999999999,"5min":3.609,"15min":124.083},"last":1.1559999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:45:35 2023","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":0.75,"15min":3.2309999999999999},"min":{"1min":0.41999999999999998,"5min":0.33100000000000002,"15min":0.23699999999999999},"max":{"1min":1.204,"5min":1.73,"15min":350.53899999999999},"last":0.85199999999999998},{"interface":"front","average":{"1min":0.83799999999999997,"5min":0.76800000000000002,"15min":3.306},"min":{"1min":0.42199999999999999,"5min":0.312,"15min":0.23599999999999999},"max":{"1min":1.294,"5min":1.294,"15min":350.64699999999999},"last":0.98599999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:45:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":0.96399999999999997,"15min":1.3340000000000001},"min":{"1min":0.74299999999999999,"5min":0.60099999999999998,"15min":0.34899999999999998},"max":{"1min":1.2909999999999999,"5min":1.52,"15min":10.15},"last":1.0669999999999999},{"interface":"front","average":{"1min":1.0760000000000001,"5min":1.018,"15min":1.3160000000000001},"min":{"1min":0.71299999999999997,"5min":0.57099999999999995,"15min":0.35799999999999998},"max":{"1min":1.391,"5min":1.708,"15min":12.132999999999999},"last":1.032}]},{"osd":5,"last update":"Mon Oct 9 05:45:35 2023","interfaces":[{"interface":"back","average":{"1min":1.103,"5min":1.069,"15min":1.391},"min":{"1min":0.64900000000000002,"5min":0.61499999999999999,"15min":0.29799999999999999},"max":{"1min":1.5680000000000001,"5min":1.5920000000000001,"15min":12.092000000000001},"last":1.244},{"interface":"front","average":{"1min":1.1220000000000001,"5min":1.0840000000000001,"15min":1.3640000000000001},"min":{"1min":0.76700000000000002,"5min":0.59399999999999997,"15min":0.36299999999999999},"max":{"1min":1.496,"5min":1.7769999999999999,"15min":10.135},"last":1.2090000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:45:35 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":1.0940000000000001,"15min":1.5600000000000001},"min":{"1min":0.71199999999999997,"5min":0.55800000000000005,"15min":0.34499999999999997},"max":{"1min":1.599,"5min":1.861,"15min":20.463999999999999},"last":1.0940000000000001},{"interface":"front","average":{"1min":1.155,"5min":1.091,"15min":1.5580000000000001},"min":{"1min":0.81399999999999995,"5min":0.69499999999999995,"15min":0.38400000000000001},"max":{"1min":1.4339999999999999,"5min":1.97,"15min":20.279},"last":1.1919999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:45:35 2023","interfaces":[{"interface":"back","average":{"1min":1.1910000000000001,"5min":1.115,"15min":1.5669999999999999},"min":{"1min":0.69399999999999995,"5min":0.65300000000000002,"15min":0.374},"max":{"1min":1.5580000000000001,"5min":1.9059999999999999,"15min":10.885},"last":1.1240000000000001},{"interface":"front","average":{"1min":1.2170000000000001,"5min":1.117,"15min":1.5249999999999999},"min":{"1min":0.77900000000000003,"5min":0.65100000000000002,"15min":0.32600000000000001},"max":{"1min":1.5189999999999999,"5min":1.7589999999999999,"15min":12.113},"last":1.298}]}]},{"osd":4,"up_from":30,"seq":128849019700,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":905400,"kb_used_data":4656,"kb_used_omap":0,"kb_used_meta":900736,"kb_avail":92839752,"statfs":{"total":95995035648,"available":95067906048,"internally_reserved":0,"allocated":4767744,"data_stored":3425286,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":922353664},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:46:17 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":2.1059999999999999,"15min":3.0899999999999999},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.26200000000000001},"max":{"1min":1.3260000000000001,"5min":89.522999999999996,"15min":110.48399999999999},"last":0.94699999999999995},{"interface":"front","average":{"1min":1.0449999999999999,"5min":2.0739999999999998,"15min":3.0609999999999999},"min":{"1min":0.68300000000000005,"5min":0.53100000000000003,"15min":0.312},"max":{"1min":1.4399999999999999,"5min":89.582999999999998,"15min":110.56},"last":0.99199999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:46:06 2023","interfaces":[{"interface":"back","average":{"1min":0.80500000000000005,"5min":0.83599999999999997,"15min":2.694},"min":{"1min":0.503,"5min":0.41099999999999998,"15min":0.29199999999999998},"max":{"1min":1.054,"5min":1.292,"15min":112.17400000000001},"last":0.90900000000000003},{"interface":"front","average":{"1min":0.86099999999999999,"5min":0.94399999999999995,"15min":2.6139999999999999},"min":{"1min":0.60399999999999998,"5min":0.46400000000000002,"15min":0.29199999999999998},"max":{"1min":1.254,"5min":4.6799999999999997,"15min":112.065},"last":0.73399999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:46:17 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.0169999999999999,"15min":3.0870000000000002},"min":{"1min":0.60999999999999999,"5min":0.47399999999999998,"15min":0.33300000000000002},"max":{"1min":1.3340000000000001,"5min":4.5229999999999997,"15min":110.38},"last":0.60099999999999998},{"interface":"front","average":{"1min":1.0269999999999999,"5min":0.97799999999999998,"15min":3.024},"min":{"1min":0.56499999999999995,"5min":0.45200000000000001,"15min":0.33400000000000002},"max":{"1min":1.474,"5min":1.4870000000000001,"15min":114.372},"last":1.2110000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:45:39 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":1.044,"15min":1.3899999999999999},"min":{"1min":0.66800000000000004,"5min":0.60399999999999998,"15min":0.33700000000000002},"max":{"1min":1.335,"5min":1.516,"15min":65.394000000000005},"last":0.871},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.024,"15min":1.419},"min":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.35799999999999998},"max":{"1min":1.425,"5min":2.2240000000000002,"15min":65.347999999999999},"last":1.1259999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:45:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":1.0960000000000001,"15min":1.044},"min":{"1min":0.89200000000000002,"5min":0.47599999999999998,"15min":0.34499999999999997},"max":{"1min":1.4430000000000001,"5min":1.5509999999999999,"15min":6.9470000000000001},"last":1.0720000000000001},{"interface":"front","average":{"1min":1.0940000000000001,"5min":1.075,"15min":1.016},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.36899999999999999},"max":{"1min":1.696,"5min":1.696,"15min":5.585},"last":1.3140000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:46:06 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":1,"15min":1.488},"min":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.35099999999999998},"max":{"1min":1.5720000000000001,"5min":1.5720000000000001,"15min":135.15600000000001},"last":1.103},{"interface":"front","average":{"1min":1.1899999999999999,"5min":1.196,"15min":1.5920000000000001},"min":{"1min":0.76700000000000002,"5min":0.51300000000000001,"15min":0.36799999999999999},"max":{"1min":1.53,"5min":4.7359999999999998,"15min":133.01499999999999},"last":1.167}]},{"osd":7,"last update":"Mon Oct 9 05:46:06 2023","interfaces":[{"interface":"back","average":{"1min":1.1890000000000001,"5min":1.165,"15min":1.1559999999999999},"min":{"1min":0.93300000000000005,"5min":0.36599999999999999,"15min":0.32700000000000001},"max":{"1min":1.7749999999999999,"5min":1.7749999999999999,"15min":9.1690000000000005},"last":1.264},{"interface":"front","average":{"1min":1.1240000000000001,"5min":1.1220000000000001,"15min":1.048},"min":{"1min":0.74299999999999999,"5min":0.50600000000000001,"15min":0.39000000000000001},"max":{"1min":1.6779999999999999,"5min":1.6779999999999999,"15min":5.2089999999999996},"last":1.0349999999999999}]}]},{"osd":5,"up_from":38,"seq":163208758056,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":899304,"kb_used_data":8736,"kb_used_omap":0,"kb_used_meta":890560,"kb_avail":92845848,"statfs":{"total":95995035648,"available":95074148352,"internally_reserved":0,"allocated":8945664,"data_stored":7614869,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911933440},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:45:31 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":0.93300000000000005,"15min":3.1579999999999999},"min":{"1min":0.70299999999999996,"5min":0.40600000000000003,"15min":0.23599999999999999},"max":{"1min":1.3029999999999999,"5min":1.321,"15min":274.83699999999999},"last":0.73699999999999999},{"interface":"front","average":{"1min":1.169,"5min":1.0720000000000001,"15min":3.2709999999999999},"min":{"1min":0.84599999999999997,"5min":0.73399999999999999,"15min":0.28000000000000003},"max":{"1min":1.657,"5min":1.657,"15min":274.57299999999998},"last":0.68000000000000005}]},{"osd":1,"last update":"Mon Oct 9 05:45:31 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":0.95899999999999996,"15min":1.4019999999999999},"min":{"1min":0.64400000000000002,"5min":0.56499999999999995,"15min":0.27000000000000002},"max":{"1min":1.4099999999999999,"5min":1.4950000000000001,"15min":36.703000000000003},"last":0.81000000000000005},{"interface":"front","average":{"1min":0.97399999999999998,"5min":0.96599999999999997,"15min":1.252},"min":{"1min":0.67600000000000005,"5min":0.49399999999999999,"15min":0.27900000000000003},"max":{"1min":1.3759999999999999,"5min":1.526,"15min":36.613999999999997},"last":0.71599999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:45:35 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.95899999999999996,"15min":2.2410000000000001},"min":{"1min":0.59899999999999998,"5min":0.443,"15min":0.32000000000000001},"max":{"1min":1.387,"5min":1.4490000000000001,"15min":250.941},"last":0.56599999999999995},{"interface":"front","average":{"1min":1,"5min":1.01,"15min":2.3740000000000001},"min":{"1min":0.69099999999999995,"5min":0.628,"15min":0.32100000000000001},"max":{"1min":1.3200000000000001,"5min":1.5229999999999999,"15min":251.48599999999999},"last":0.78000000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:45:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":1.0209999999999999,"15min":1.488},"min":{"1min":0.77900000000000003,"5min":0.58999999999999997,"15min":0.32400000000000001},"max":{"1min":1.323,"5min":1.3919999999999999,"15min":68.061999999999998},"last":0.87},{"interface":"front","average":{"1min":1.073,"5min":1.034,"15min":1.716},"min":{"1min":0.72299999999999998,"5min":0.56299999999999994,"15min":0.33000000000000002},"max":{"1min":1.367,"5min":1.367,"15min":68.959000000000003},"last":0.85199999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:45:31 2023","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":1.085,"15min":0.997},"min":{"1min":0.61499999999999999,"5min":0.51900000000000002,"15min":0.33500000000000002},"max":{"1min":1.546,"5min":1.546,"15min":7.6109999999999998},"last":0.754},{"interface":"front","average":{"1min":1.103,"5min":1.1000000000000001,"15min":1.111},"min":{"1min":0.76400000000000001,"5min":0.56599999999999995,"15min":0.34300000000000003},"max":{"1min":1.538,"5min":1.829,"15min":8.3480000000000008},"last":0.58699999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:45:31 2023","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":1.1240000000000001,"15min":1.2150000000000001},"min":{"1min":0.83399999999999996,"5min":0.63500000000000001,"15min":0.32300000000000001},"max":{"1min":1.6140000000000001,"5min":2.0680000000000001,"15min":8.5220000000000002},"last":0.90300000000000002},{"interface":"front","average":{"1min":1.0549999999999999,"5min":1.119,"15min":1.1259999999999999},"min":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.32400000000000001},"max":{"1min":1.488,"5min":1.752,"15min":9.282},"last":0.435}]},{"osd":7,"last update":"Mon Oct 9 05:45:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2070000000000001,"5min":1.181,"15min":1.157},"min":{"1min":0.92100000000000004,"5min":0.64400000000000002,"15min":0.34399999999999997},"max":{"1min":1.4390000000000001,"5min":1.641,"15min":9.3970000000000002},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.268,"5min":1.226,"15min":1.304},"min":{"1min":0.81299999999999994,"5min":0.67500000000000004,"15min":0.36299999999999999},"max":{"1min":1.7190000000000001,"5min":1.7190000000000001,"15min":11.593},"last":0.65800000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:46:32.266 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:46:32.548 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:32 smithi133 bash[102858]: audit 2023-10-09T05:46:31.317477+0000 mgr.y (mgr.25277) 1766 : audit 0 from='client.36873 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:46:32.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:32 smithi133 bash[105589]: audit 2023-10-09T05:46:31.317477+0000 mgr.y (mgr.25277) 1766 : audit 0 from='client.36873 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:46:32.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:32 smithi194 bash[92666]: audit 2023-10-09T05:46:31.317477+0000 mgr.y (mgr.25277) 1766 : audit 0 from='client.36873 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:46:33.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:33 smithi133 bash[105589]: cluster 2023-10-09T05:46:32.625446+0000 mgr.y (mgr.25277) 1767 : cluster 0 pgmap v1999: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:33.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:33 smithi133 bash[102858]: cluster 2023-10-09T05:46:32.625446+0000 mgr.y (mgr.25277) 1767 : cluster 0 pgmap v1999: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:33.885 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:33 smithi194 bash[92666]: cluster 2023-10-09T05:46:32.625446+0000 mgr.y (mgr.25277) 1767 : cluster 0 pgmap v1999: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:34.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:46:34 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:46:34] "GET /metrics HTTP/1.1" 200 34115 "" "Prometheus/2.43.0" 2023-10-09T05:46:35.385 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:35 smithi194 bash[92666]: cluster 2023-10-09T05:46:34.626352+0000 mgr.y (mgr.25277) 1768 : cluster 0 pgmap v2000: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:35.548 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:35 smithi133 bash[105589]: cluster 2023-10-09T05:46:34.626352+0000 mgr.y (mgr.25277) 1768 : cluster 0 pgmap v2000: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:35.554 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:35 smithi133 bash[102858]: cluster 2023-10-09T05:46:34.626352+0000 mgr.y (mgr.25277) 1768 : cluster 0 pgmap v2000: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:37.054 INFO:teuthology.orchestra.run.smithi133.stderr:Inferring config /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:38.048 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:37 smithi133 bash[102858]: cluster 2023-10-09T05:46:36.627193+0000 mgr.y (mgr.25277) 1769 : cluster 0 pgmap v2001: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:38.048 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:37 smithi133 bash[105589]: cluster 2023-10-09T05:46:36.627193+0000 mgr.y (mgr.25277) 1769 : cluster 0 pgmap v2001: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:38.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:37 smithi194 bash[92666]: cluster 2023-10-09T05:46:36.627193+0000 mgr.y (mgr.25277) 1769 : cluster 0 pgmap v2001: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:39.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:39 smithi133 bash[105589]: cluster 2023-10-09T05:46:38.627879+0000 mgr.y (mgr.25277) 1770 : cluster 0 pgmap v2002: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:39.798 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:39 smithi133 bash[102858]: cluster 2023-10-09T05:46:38.627879+0000 mgr.y (mgr.25277) 1770 : cluster 0 pgmap v2002: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:40.135 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:39 smithi194 bash[92666]: cluster 2023-10-09T05:46:38.627879+0000 mgr.y (mgr.25277) 1770 : cluster 0 pgmap v2002: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:40.938 INFO:teuthology.orchestra.run.smithi133.stdout: 2023-10-09T05:46:40.941 INFO:teuthology.orchestra.run.smithi133.stderr:dumped all 2023-10-09T05:46:41.902 INFO:teuthology.orchestra.run.smithi133.stdout:{"pg_ready":true,"pg_map":{"version":2003,"stamp":"2023-10-09T05:46:40.628151+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6504,"num_read_kb":14182,"num_write":1146,"num_write_kb":15159,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":115,"num_bytes_recovered":4154950,"num_keys_recovered":30,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1198,"ondisk_log_size":1198,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9713848,"kb_used_data":56184,"kb_used_omap":20,"kb_used_meta":9657579,"kb_avail":740247368,"statfs":{"total":767960285184,"available":758013304832,"internally_reserved":0,"allocated":57532416,"data_stored":42363632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":9889361550},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004853"},"pg_stats":[{"pgid":"3.1a","version":"69'2","reported_seq":34,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.084877+0000","last_change":"2023-10-09T05:42:12.627925+0000","last_active":"2023-10-09T05:45:00.084877+0000","last_peered":"2023-10-09T05:45:00.084877+0000","last_clean":"2023-10-09T05:45:00.084877+0000","last_became_active":"2023-10-09T05:42:12.625644+0000","last_became_peered":"2023-10-09T05:42:12.625644+0000","last_unstale":"2023-10-09T05:45:00.084877+0000","last_undegraded":"2023-10-09T05:45:00.084877+0000","last_fullsized":"2023-10-09T05:45:00.084877+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:08:11.129668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":457,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.671010+0000","last_change":"2023-10-09T05:42:12.631985+0000","last_active":"2023-10-09T05:44:59.671010+0000","last_peered":"2023-10-09T05:44:59.671010+0000","last_clean":"2023-10-09T05:44:59.671010+0000","last_became_active":"2023-10-09T05:42:12.631320+0000","last_became_peered":"2023-10-09T05:42:12.631320+0000","last_unstale":"2023-10-09T05:44:59.671010+0000","last_undegraded":"2023-10-09T05:44:59.671010+0000","last_fullsized":"2023-10-09T05:44:59.671010+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:08.504947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":341,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.652400+0000","last_change":"2023-10-09T05:42:12.632126+0000","last_active":"2023-10-09T05:45:08.652400+0000","last_peered":"2023-10-09T05:45:08.652400+0000","last_clean":"2023-10-09T05:45:08.652400+0000","last_became_active":"2023-10-09T05:42:12.631615+0000","last_became_peered":"2023-10-09T05:42:12.631615+0000","last_unstale":"2023-10-09T05:45:08.652400+0000","last_undegraded":"2023-10-09T05:45:08.652400+0000","last_fullsized":"2023-10-09T05:45:08.652400+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:52.067376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":457,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.968679+0000","last_change":"2023-10-09T05:18:26.422923+0000","last_active":"2023-10-09T05:45:08.968679+0000","last_peered":"2023-10-09T05:45:08.968679+0000","last_clean":"2023-10-09T05:45:08.968679+0000","last_became_active":"2023-10-09T05:18:26.422508+0000","last_became_peered":"2023-10-09T05:18:26.422508+0000","last_unstale":"2023-10-09T05:45:08.968679+0000","last_undegraded":"2023-10-09T05:45:08.968679+0000","last_fullsized":"2023-10-09T05:45:08.968679+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:00.876037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":494,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.968165+0000","last_change":"2023-10-09T05:42:59.843555+0000","last_active":"2023-10-09T05:45:08.968165+0000","last_peered":"2023-10-09T05:45:08.968165+0000","last_clean":"2023-10-09T05:45:08.968165+0000","last_became_active":"2023-10-09T05:42:59.842903+0000","last_became_peered":"2023-10-09T05:42:59.842903+0000","last_unstale":"2023-10-09T05:45:08.968165+0000","last_undegraded":"2023-10-09T05:45:08.968165+0000","last_fullsized":"2023-10-09T05:45:08.968165+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:04:42.450057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":449,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.680210+0000","last_change":"2023-10-09T05:32:45.829908+0000","last_active":"2023-10-09T05:44:59.680210+0000","last_peered":"2023-10-09T05:44:59.680210+0000","last_clean":"2023-10-09T05:44:59.680210+0000","last_became_active":"2023-10-09T05:32:45.829566+0000","last_became_peered":"2023-10-09T05:32:45.829566+0000","last_unstale":"2023-10-09T05:44:59.680210+0000","last_undegraded":"2023-10-09T05:44:59.680210+0000","last_fullsized":"2023-10-09T05:44:59.680210+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:05.705818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":452,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.969138+0000","last_change":"2023-10-09T05:32:45.835245+0000","last_active":"2023-10-09T05:45:08.969138+0000","last_peered":"2023-10-09T05:45:08.969138+0000","last_clean":"2023-10-09T05:45:08.969138+0000","last_became_active":"2023-10-09T05:32:45.830180+0000","last_became_peered":"2023-10-09T05:32:45.830180+0000","last_unstale":"2023-10-09T05:45:08.969138+0000","last_undegraded":"2023-10-09T05:45:08.969138+0000","last_fullsized":"2023-10-09T05:45:08.969138+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:07:06.351159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":432,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.968151+0000","last_change":"2023-10-09T05:09:57.849695+0000","last_active":"2023-10-09T05:45:08.968151+0000","last_peered":"2023-10-09T05:45:08.968151+0000","last_clean":"2023-10-09T05:45:08.968151+0000","last_became_active":"2023-10-09T05:09:57.848478+0000","last_became_peered":"2023-10-09T05:09:57.848478+0000","last_unstale":"2023-10-09T05:45:08.968151+0000","last_undegraded":"2023-10-09T05:45:08.968151+0000","last_fullsized":"2023-10-09T05:45:08.968151+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:26.115312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":158,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.675112+0000","last_change":"2023-10-09T05:32:45.830061+0000","last_active":"2023-10-09T05:44:59.675112+0000","last_peered":"2023-10-09T05:44:59.675112+0000","last_clean":"2023-10-09T05:44:59.675112+0000","last_became_active":"2023-10-09T05:32:45.829723+0000","last_became_peered":"2023-10-09T05:32:45.829723+0000","last_unstale":"2023-10-09T05:44:59.675112+0000","last_undegraded":"2023-10-09T05:44:59.675112+0000","last_fullsized":"2023-10-09T05:44:59.675112+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:21.429740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":445,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.969593+0000","last_change":"2023-10-09T05:18:26.418431+0000","last_active":"2023-10-09T05:45:08.969593+0000","last_peered":"2023-10-09T05:45:08.969593+0000","last_clean":"2023-10-09T05:45:08.969593+0000","last_became_active":"2023-10-09T05:18:26.418100+0000","last_became_peered":"2023-10-09T05:18:26.418100+0000","last_unstale":"2023-10-09T05:45:08.969593+0000","last_undegraded":"2023-10-09T05:45:08.969593+0000","last_fullsized":"2023-10-09T05:45:08.969593+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:21.241286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":273,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.965184+0000","last_change":"2023-10-09T05:42:12.652600+0000","last_active":"2023-10-09T05:44:59.965184+0000","last_peered":"2023-10-09T05:44:59.965184+0000","last_clean":"2023-10-09T05:44:59.965184+0000","last_became_active":"2023-10-09T05:42:12.631619+0000","last_became_peered":"2023-10-09T05:42:12.631619+0000","last_unstale":"2023-10-09T05:44:59.965184+0000","last_undegraded":"2023-10-09T05:44:59.965184+0000","last_fullsized":"2023-10-09T05:44:59.965184+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:43:00.862938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":28,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.653090+0000","last_change":"2023-10-09T05:42:59.633392+0000","last_active":"2023-10-09T05:45:08.653090+0000","last_peered":"2023-10-09T05:45:08.653090+0000","last_clean":"2023-10-09T05:45:08.653090+0000","last_became_active":"2023-10-09T05:42:59.632990+0000","last_became_peered":"2023-10-09T05:42:59.632990+0000","last_unstale":"2023-10-09T05:45:08.653090+0000","last_undegraded":"2023-10-09T05:45:08.653090+0000","last_fullsized":"2023-10-09T05:45:08.653090+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:38.874996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'23","reported_seq":44,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.085242+0000","last_change":"2023-10-09T05:42:12.644859+0000","last_active":"2023-10-09T05:45:00.085242+0000","last_peered":"2023-10-09T05:45:00.085242+0000","last_clean":"2023-10-09T05:45:00.085242+0000","last_became_active":"2023-10-09T05:42:12.626225+0000","last_became_peered":"2023-10-09T05:42:12.626225+0000","last_unstale":"2023-10-09T05:45:00.085242+0000","last_undegraded":"2023-10-09T05:45:00.085242+0000","last_fullsized":"2023-10-09T05:45:00.085242+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:04.905187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":34,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.085048+0000","last_change":"2023-10-09T05:42:12.638386+0000","last_active":"2023-10-09T05:45:00.085048+0000","last_peered":"2023-10-09T05:45:00.085048+0000","last_clean":"2023-10-09T05:45:00.085048+0000","last_became_active":"2023-10-09T05:42:12.637267+0000","last_became_peered":"2023-10-09T05:42:12.637267+0000","last_unstale":"2023-10-09T05:45:00.085048+0000","last_undegraded":"2023-10-09T05:45:00.085048+0000","last_fullsized":"2023-10-09T05:45:00.085048+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:15.501636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":273,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.964773+0000","last_change":"2023-10-09T05:42:12.656052+0000","last_active":"2023-10-09T05:44:59.964773+0000","last_peered":"2023-10-09T05:44:59.964773+0000","last_clean":"2023-10-09T05:44:59.964773+0000","last_became_active":"2023-10-09T05:42:12.631633+0000","last_became_peered":"2023-10-09T05:42:12.631633+0000","last_unstale":"2023-10-09T05:44:59.964773+0000","last_undegraded":"2023-10-09T05:44:59.964773+0000","last_fullsized":"2023-10-09T05:44:59.964773+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:57:42.256197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":504,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.968512+0000","last_change":"2023-10-09T05:43:00.348706+0000","last_active":"2023-10-09T05:45:08.968512+0000","last_peered":"2023-10-09T05:45:08.968512+0000","last_clean":"2023-10-09T05:45:08.968512+0000","last_became_active":"2023-10-09T05:43:00.347927+0000","last_became_peered":"2023-10-09T05:43:00.347927+0000","last_unstale":"2023-10-09T05:45:08.968512+0000","last_undegraded":"2023-10-09T05:45:08.968512+0000","last_fullsized":"2023-10-09T05:45:08.968512+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:40:48.770688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":453,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.969210+0000","last_change":"2023-10-09T05:18:26.424399+0000","last_active":"2023-10-09T05:45:08.969210+0000","last_peered":"2023-10-09T05:45:08.969210+0000","last_clean":"2023-10-09T05:45:08.969210+0000","last_became_active":"2023-10-09T05:18:26.422122+0000","last_became_peered":"2023-10-09T05:18:26.422122+0000","last_unstale":"2023-10-09T05:45:08.969210+0000","last_undegraded":"2023-10-09T05:45:08.969210+0000","last_fullsized":"2023-10-09T05:45:08.969210+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:39.393756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":158,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.675096+0000","last_change":"2023-10-09T05:32:45.831080+0000","last_active":"2023-10-09T05:44:59.675096+0000","last_peered":"2023-10-09T05:44:59.675096+0000","last_clean":"2023-10-09T05:44:59.675096+0000","last_became_active":"2023-10-09T05:32:45.830745+0000","last_became_peered":"2023-10-09T05:32:45.830745+0000","last_unstale":"2023-10-09T05:44:59.675096+0000","last_undegraded":"2023-10-09T05:44:59.675096+0000","last_fullsized":"2023-10-09T05:44:59.675096+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:16.345644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":255,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.964209+0000","last_change":"2023-10-09T05:42:12.629712+0000","last_active":"2023-10-09T05:44:59.964209+0000","last_peered":"2023-10-09T05:44:59.964209+0000","last_clean":"2023-10-09T05:44:59.964209+0000","last_became_active":"2023-10-09T05:42:12.629312+0000","last_became_peered":"2023-10-09T05:42:12.629312+0000","last_unstale":"2023-10-09T05:44:59.964209+0000","last_undegraded":"2023-10-09T05:44:59.964209+0000","last_fullsized":"2023-10-09T05:44:59.964209+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:28.590232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":453,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.968728+0000","last_change":"2023-10-09T05:18:26.421678+0000","last_active":"2023-10-09T05:45:08.968728+0000","last_peered":"2023-10-09T05:45:08.968728+0000","last_clean":"2023-10-09T05:45:08.968728+0000","last_became_active":"2023-10-09T05:18:26.419673+0000","last_became_peered":"2023-10-09T05:18:26.419673+0000","last_unstale":"2023-10-09T05:45:08.968728+0000","last_undegraded":"2023-10-09T05:45:08.968728+0000","last_fullsized":"2023-10-09T05:45:08.968728+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:38.876021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":462,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.679072+0000","last_change":"2023-10-09T05:32:45.844532+0000","last_active":"2023-10-09T05:44:59.679072+0000","last_peered":"2023-10-09T05:44:59.679072+0000","last_clean":"2023-10-09T05:44:59.679072+0000","last_became_active":"2023-10-09T05:32:45.831101+0000","last_became_peered":"2023-10-09T05:32:45.831101+0000","last_unstale":"2023-10-09T05:44:59.679072+0000","last_undegraded":"2023-10-09T05:44:59.679072+0000","last_fullsized":"2023-10-09T05:44:59.679072+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:19:37.907188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":143,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.675105+0000","last_change":"2023-10-09T05:36:16.901739+0000","last_active":"2023-10-09T05:44:59.675105+0000","last_peered":"2023-10-09T05:44:59.675105+0000","last_clean":"2023-10-09T05:44:59.675105+0000","last_became_active":"2023-10-09T05:36:16.901380+0000","last_became_peered":"2023-10-09T05:36:16.901380+0000","last_unstale":"2023-10-09T05:44:59.675105+0000","last_undegraded":"2023-10-09T05:44:59.675105+0000","last_fullsized":"2023-10-09T05:44:59.675105+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:26.454798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":267,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.967869+0000","last_change":"2023-10-09T05:42:12.649212+0000","last_active":"2023-10-09T05:44:59.967869+0000","last_peered":"2023-10-09T05:44:59.967869+0000","last_clean":"2023-10-09T05:44:59.967869+0000","last_became_active":"2023-10-09T05:42:12.630943+0000","last_became_peered":"2023-10-09T05:42:12.630943+0000","last_unstale":"2023-10-09T05:44:59.967869+0000","last_undegraded":"2023-10-09T05:44:59.967869+0000","last_fullsized":"2023-10-09T05:44:59.967869+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:25.895604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":158,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.674736+0000","last_change":"2023-10-09T05:32:45.829063+0000","last_active":"2023-10-09T05:44:59.674736+0000","last_peered":"2023-10-09T05:44:59.674736+0000","last_clean":"2023-10-09T05:44:59.674736+0000","last_became_active":"2023-10-09T05:32:45.828694+0000","last_became_peered":"2023-10-09T05:32:45.828694+0000","last_unstale":"2023-10-09T05:44:59.674736+0000","last_undegraded":"2023-10-09T05:44:59.674736+0000","last_fullsized":"2023-10-09T05:44:59.674736+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:58:51.140748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":267,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.965297+0000","last_change":"2023-10-09T05:42:12.646999+0000","last_active":"2023-10-09T05:44:59.965297+0000","last_peered":"2023-10-09T05:44:59.965297+0000","last_clean":"2023-10-09T05:44:59.965297+0000","last_became_active":"2023-10-09T05:42:12.629954+0000","last_became_peered":"2023-10-09T05:42:12.629954+0000","last_unstale":"2023-10-09T05:44:59.965297+0000","last_undegraded":"2023-10-09T05:44:59.965297+0000","last_fullsized":"2023-10-09T05:44:59.965297+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:33.042876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":34,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.084585+0000","last_change":"2023-10-09T05:42:12.620059+0000","last_active":"2023-10-09T05:45:00.084585+0000","last_peered":"2023-10-09T05:45:00.084585+0000","last_clean":"2023-10-09T05:45:00.084585+0000","last_became_active":"2023-10-09T05:42:12.619703+0000","last_became_peered":"2023-10-09T05:42:12.619703+0000","last_unstale":"2023-10-09T05:45:00.084585+0000","last_undegraded":"2023-10-09T05:45:00.084585+0000","last_fullsized":"2023-10-09T05:45:00.084585+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:02.983347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":253,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.965184+0000","last_change":"2023-10-09T05:32:45.838786+0000","last_active":"2023-10-09T05:44:59.965184+0000","last_peered":"2023-10-09T05:44:59.965184+0000","last_clean":"2023-10-09T05:44:59.965184+0000","last_became_active":"2023-10-09T05:32:45.830293+0000","last_became_peered":"2023-10-09T05:32:45.830293+0000","last_unstale":"2023-10-09T05:44:59.965184+0000","last_undegraded":"2023-10-09T05:44:59.965184+0000","last_fullsized":"2023-10-09T05:44:59.965184+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:47.319929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":479,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.679359+0000","last_change":"2023-10-09T05:18:26.423526+0000","last_active":"2023-10-09T05:44:59.679359+0000","last_peered":"2023-10-09T05:44:59.679359+0000","last_clean":"2023-10-09T05:44:59.679359+0000","last_became_active":"2023-10-09T05:18:26.423092+0000","last_became_peered":"2023-10-09T05:18:26.423092+0000","last_unstale":"2023-10-09T05:44:59.679359+0000","last_undegraded":"2023-10-09T05:44:59.679359+0000","last_fullsized":"2023-10-09T05:44:59.679359+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:54.877664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":158,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.675228+0000","last_change":"2023-10-09T05:32:45.829450+0000","last_active":"2023-10-09T05:44:59.675228+0000","last_peered":"2023-10-09T05:44:59.675228+0000","last_clean":"2023-10-09T05:44:59.675228+0000","last_became_active":"2023-10-09T05:32:45.829076+0000","last_became_peered":"2023-10-09T05:32:45.829076+0000","last_unstale":"2023-10-09T05:44:59.675228+0000","last_undegraded":"2023-10-09T05:44:59.675228+0000","last_fullsized":"2023-10-09T05:44:59.675228+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:51:13.538770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":455,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.969457+0000","last_change":"2023-10-09T05:42:12.629497+0000","last_active":"2023-10-09T05:45:08.969457+0000","last_peered":"2023-10-09T05:45:08.969457+0000","last_clean":"2023-10-09T05:45:08.969457+0000","last_became_active":"2023-10-09T05:42:12.629123+0000","last_became_peered":"2023-10-09T05:42:12.629123+0000","last_unstale":"2023-10-09T05:45:08.969457+0000","last_undegraded":"2023-10-09T05:45:08.969457+0000","last_fullsized":"2023-10-09T05:45:08.969457+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:34.295429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'3","reported_seq":437,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.670477+0000","last_change":"2023-10-09T05:42:12.640692+0000","last_active":"2023-10-09T05:44:59.670477+0000","last_peered":"2023-10-09T05:44:59.670477+0000","last_clean":"2023-10-09T05:44:59.670477+0000","last_became_active":"2023-10-09T05:42:12.629848+0000","last_became_peered":"2023-10-09T05:42:12.629848+0000","last_unstale":"2023-10-09T05:44:59.670477+0000","last_undegraded":"2023-10-09T05:44:59.670477+0000","last_fullsized":"2023-10-09T05:44:59.670477+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:53.321982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":34,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.085573+0000","last_change":"2023-10-09T05:42:12.624031+0000","last_active":"2023-10-09T05:45:00.085573+0000","last_peered":"2023-10-09T05:45:00.085573+0000","last_clean":"2023-10-09T05:45:00.085573+0000","last_became_active":"2023-10-09T05:42:12.623571+0000","last_became_peered":"2023-10-09T05:42:12.623571+0000","last_unstale":"2023-10-09T05:45:00.085573+0000","last_undegraded":"2023-10-09T05:45:00.085573+0000","last_fullsized":"2023-10-09T05:45:00.085573+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:50:18.943555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":490,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.679277+0000","last_change":"2023-10-09T05:42:59.634964+0000","last_active":"2023-10-09T05:44:59.679277+0000","last_peered":"2023-10-09T05:44:59.679277+0000","last_clean":"2023-10-09T05:44:59.679277+0000","last_became_active":"2023-10-09T05:42:59.634570+0000","last_became_peered":"2023-10-09T05:42:59.634570+0000","last_unstale":"2023-10-09T05:44:59.679277+0000","last_undegraded":"2023-10-09T05:44:59.679277+0000","last_fullsized":"2023-10-09T05:44:59.679277+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:34.443566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":34,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.085231+0000","last_change":"2023-10-09T05:42:12.626346+0000","last_active":"2023-10-09T05:45:00.085231+0000","last_peered":"2023-10-09T05:45:00.085231+0000","last_clean":"2023-10-09T05:45:00.085231+0000","last_became_active":"2023-10-09T05:42:12.625594+0000","last_became_peered":"2023-10-09T05:42:12.625594+0000","last_unstale":"2023-10-09T05:45:00.085231+0000","last_undegraded":"2023-10-09T05:45:00.085231+0000","last_fullsized":"2023-10-09T05:45:00.085231+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:36:01.369510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":34,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.085410+0000","last_change":"2023-10-09T05:42:12.628487+0000","last_active":"2023-10-09T05:45:00.085410+0000","last_peered":"2023-10-09T05:45:00.085410+0000","last_clean":"2023-10-09T05:45:00.085410+0000","last_became_active":"2023-10-09T05:42:12.628127+0000","last_became_peered":"2023-10-09T05:42:12.628127+0000","last_unstale":"2023-10-09T05:45:00.085410+0000","last_undegraded":"2023-10-09T05:45:00.085410+0000","last_fullsized":"2023-10-09T05:45:00.085410+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:30:05.955093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":54,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.085333+0000","last_change":"2023-10-09T05:43:17.869175+0000","last_active":"2023-10-09T05:45:00.085333+0000","last_peered":"2023-10-09T05:45:00.085333+0000","last_clean":"2023-10-09T05:45:00.085333+0000","last_became_active":"2023-10-09T05:43:17.868274+0000","last_became_peered":"2023-10-09T05:43:17.868274+0000","last_unstale":"2023-10-09T05:45:00.085333+0000","last_undegraded":"2023-10-09T05:45:00.085333+0000","last_fullsized":"2023-10-09T05:45:00.085333+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:00:54.145979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":470,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.967862+0000","last_change":"2023-10-09T05:18:26.413106+0000","last_active":"2023-10-09T05:45:08.967862+0000","last_peered":"2023-10-09T05:45:08.967862+0000","last_clean":"2023-10-09T05:45:08.967862+0000","last_became_active":"2023-10-09T05:18:26.412334+0000","last_became_peered":"2023-10-09T05:18:26.412334+0000","last_unstale":"2023-10-09T05:45:08.967862+0000","last_undegraded":"2023-10-09T05:45:08.967862+0000","last_fullsized":"2023-10-09T05:45:08.967862+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:18.058016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":342,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.652610+0000","last_change":"2023-10-09T05:42:12.631719+0000","last_active":"2023-10-09T05:45:08.652610+0000","last_peered":"2023-10-09T05:45:08.652610+0000","last_clean":"2023-10-09T05:45:08.652610+0000","last_became_active":"2023-10-09T05:42:12.631350+0000","last_became_peered":"2023-10-09T05:42:12.631350+0000","last_unstale":"2023-10-09T05:45:08.652610+0000","last_undegraded":"2023-10-09T05:45:08.652610+0000","last_fullsized":"2023-10-09T05:45:08.652610+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:21:43.609792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"421'120","reported_seq":769,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.652914+0000","last_change":"2023-10-09T05:41:43.988213+0000","last_active":"2023-10-09T05:45:08.652914+0000","last_peered":"2023-10-09T05:45:08.652914+0000","last_clean":"2023-10-09T05:45:08.652914+0000","last_became_active":"2023-10-09T05:18:26.421853+0000","last_became_peered":"2023-10-09T05:18:26.421853+0000","last_unstale":"2023-10-09T05:45:08.652914+0000","last_undegraded":"2023-10-09T05:45:08.652914+0000","last_fullsized":"2023-10-09T05:45:08.652914+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":120,"log_dups_size":0,"ondisk_log_size":120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:37:47.000556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045958700000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":637,"num_read_kb":2582,"num_write":123,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":158,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.675275+0000","last_change":"2023-10-09T05:32:45.832271+0000","last_active":"2023-10-09T05:44:59.675275+0000","last_peered":"2023-10-09T05:44:59.675275+0000","last_clean":"2023-10-09T05:44:59.675275+0000","last_became_active":"2023-10-09T05:32:45.831997+0000","last_became_peered":"2023-10-09T05:32:45.831997+0000","last_unstale":"2023-10-09T05:44:59.675275+0000","last_undegraded":"2023-10-09T05:44:59.675275+0000","last_fullsized":"2023-10-09T05:44:59.675275+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:18.970416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":28,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:05.027074+0000","last_change":"2023-10-09T05:42:59.632813+0000","last_active":"2023-10-09T05:45:05.027074+0000","last_peered":"2023-10-09T05:45:05.027074+0000","last_clean":"2023-10-09T05:45:05.027074+0000","last_became_active":"2023-10-09T05:42:59.632590+0000","last_became_peered":"2023-10-09T05:42:59.632590+0000","last_unstale":"2023-10-09T05:45:05.027074+0000","last_undegraded":"2023-10-09T05:45:05.027074+0000","last_fullsized":"2023-10-09T05:45:05.027074+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:38:34.386951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"421'156","reported_seq":1149,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.674823+0000","last_change":"2023-10-09T05:41:43.998213+0000","last_active":"2023-10-09T05:44:59.674823+0000","last_peered":"2023-10-09T05:44:59.674823+0000","last_clean":"2023-10-09T05:44:59.674823+0000","last_became_active":"2023-10-09T05:32:45.824981+0000","last_became_peered":"2023-10-09T05:32:45.824981+0000","last_unstale":"2023-10-09T05:44:59.674823+0000","last_undegraded":"2023-10-09T05:44:59.674823+0000","last_fullsized":"2023-10-09T05:44:59.674823+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":156,"log_dups_size":0,"ondisk_log_size":156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:02.868005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056430599999999999,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1612,"num_read_kb":3505,"num_write":104,"num_write_kb":6199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":34,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.085609+0000","last_change":"2023-10-09T05:42:12.631913+0000","last_active":"2023-10-09T05:45:00.085609+0000","last_peered":"2023-10-09T05:45:00.085609+0000","last_clean":"2023-10-09T05:45:00.085609+0000","last_became_active":"2023-10-09T05:42:12.631552+0000","last_became_peered":"2023-10-09T05:42:12.631552+0000","last_unstale":"2023-10-09T05:45:00.085609+0000","last_undegraded":"2023-10-09T05:45:00.085609+0000","last_fullsized":"2023-10-09T05:45:00.085609+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:55:59.713353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":218,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.652663+0000","last_change":"2023-10-09T05:23:18.196260+0000","last_active":"2023-10-09T05:45:08.652663+0000","last_peered":"2023-10-09T05:45:08.652663+0000","last_clean":"2023-10-09T05:45:08.652663+0000","last_became_active":"2023-10-09T05:23:18.195907+0000","last_became_peered":"2023-10-09T05:23:18.195907+0000","last_unstale":"2023-10-09T05:45:08.652663+0000","last_undegraded":"2023-10-09T05:45:08.652663+0000","last_fullsized":"2023-10-09T05:45:08.652663+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:36.243432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"421'188","reported_seq":1078,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.968954+0000","last_change":"2023-10-09T05:41:44.001943+0000","last_active":"2023-10-09T05:45:08.968954+0000","last_peered":"2023-10-09T05:45:08.968954+0000","last_clean":"2023-10-09T05:45:08.968954+0000","last_became_active":"2023-10-09T05:20:18.122859+0000","last_became_peered":"2023-10-09T05:20:18.122859+0000","last_unstale":"2023-10-09T05:45:08.968954+0000","last_undegraded":"2023-10-09T05:45:08.968954+0000","last_fullsized":"2023-10-09T05:45:08.968954+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":188,"log_dups_size":0,"ondisk_log_size":188,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.906228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040378799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1531,"num_read_kb":1451,"num_write":151,"num_write_kb":164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":326,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.652392+0000","last_change":"2023-10-09T05:09:57.854064+0000","last_active":"2023-10-09T05:45:08.652392+0000","last_peered":"2023-10-09T05:45:08.652392+0000","last_clean":"2023-10-09T05:45:08.652392+0000","last_became_active":"2023-10-09T05:09:57.847953+0000","last_became_peered":"2023-10-09T05:09:57.847953+0000","last_unstale":"2023-10-09T05:45:08.652392+0000","last_undegraded":"2023-10-09T05:45:08.652392+0000","last_fullsized":"2023-10-09T05:45:08.652392+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:45.914976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":28,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.652513+0000","last_change":"2023-10-09T05:42:59.843795+0000","last_active":"2023-10-09T05:45:08.652513+0000","last_peered":"2023-10-09T05:45:08.652513+0000","last_clean":"2023-10-09T05:45:08.652513+0000","last_became_active":"2023-10-09T05:42:59.843481+0000","last_became_peered":"2023-10-09T05:42:59.843481+0000","last_unstale":"2023-10-09T05:45:08.652513+0000","last_undegraded":"2023-10-09T05:45:08.652513+0000","last_fullsized":"2023-10-09T05:45:08.652513+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:01:35.732837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"421'172","reported_seq":685,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.674851+0000","last_change":"2023-10-09T05:41:44.000425+0000","last_active":"2023-10-09T05:44:59.674851+0000","last_peered":"2023-10-09T05:44:59.674851+0000","last_clean":"2023-10-09T05:44:59.674851+0000","last_became_active":"2023-10-09T05:32:45.824730+0000","last_became_peered":"2023-10-09T05:32:45.824730+0000","last_unstale":"2023-10-09T05:44:59.674851+0000","last_undegraded":"2023-10-09T05:44:59.674851+0000","last_fullsized":"2023-10-09T05:44:59.674851+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":172,"log_dups_size":0,"ondisk_log_size":172,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:33.484326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0016454359999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":798,"num_read_kb":1755,"num_write":158,"num_write_kb":1129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":731,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.964386+0000","last_change":"2023-10-09T05:18:29.419527+0000","last_active":"2023-10-09T05:44:59.964386+0000","last_peered":"2023-10-09T05:44:59.964386+0000","last_clean":"2023-10-09T05:44:59.964386+0000","last_became_active":"2023-10-09T05:18:29.418852+0000","last_became_peered":"2023-10-09T05:18:29.418852+0000","last_unstale":"2023-10-09T05:44:59.964386+0000","last_undegraded":"2023-10-09T05:44:59.964386+0000","last_fullsized":"2023-10-09T05:44:59.964386+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:36:44.261013+0000","last_clean_scrub_stamp":"2023-10-09T04:36:44.261013+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:02.899170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":438,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.968899+0000","last_change":"2023-10-09T05:32:45.829840+0000","last_active":"2023-10-09T05:45:08.968899+0000","last_peered":"2023-10-09T05:45:08.968899+0000","last_clean":"2023-10-09T05:45:08.968899+0000","last_became_active":"2023-10-09T05:32:45.829084+0000","last_became_peered":"2023-10-09T05:32:45.829084+0000","last_unstale":"2023-10-09T05:45:08.968899+0000","last_undegraded":"2023-10-09T05:45:08.968899+0000","last_fullsized":"2023-10-09T05:45:08.968899+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:26:48.836506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":452,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.679720+0000","last_change":"2023-10-09T05:42:12.634383+0000","last_active":"2023-10-09T05:44:59.679720+0000","last_peered":"2023-10-09T05:44:59.679720+0000","last_clean":"2023-10-09T05:44:59.679720+0000","last_became_active":"2023-10-09T05:42:12.634002+0000","last_became_peered":"2023-10-09T05:42:12.634002+0000","last_unstale":"2023-10-09T05:44:59.679720+0000","last_undegraded":"2023-10-09T05:44:59.679720+0000","last_fullsized":"2023-10-09T05:44:59.679720+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:32.709357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"420'76","reported_seq":504,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.964485+0000","last_change":"2023-10-09T05:41:43.992881+0000","last_active":"2023-10-09T05:44:59.964485+0000","last_peered":"2023-10-09T05:44:59.964485+0000","last_clean":"2023-10-09T05:44:59.964485+0000","last_became_active":"2023-10-09T05:23:18.193394+0000","last_became_peered":"2023-10-09T05:23:18.193394+0000","last_unstale":"2023-10-09T05:44:59.964485+0000","last_undegraded":"2023-10-09T05:44:59.964485+0000","last_fullsized":"2023-10-09T05:44:59.964485+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":76,"log_dups_size":0,"ondisk_log_size":76,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:40.594469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067895100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":420,"num_read_kb":1654,"num_write":61,"num_write_kb":1190,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":446,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.081677+0000","last_change":"2023-10-09T05:42:12.629510+0000","last_active":"2023-10-09T05:45:00.081677+0000","last_peered":"2023-10-09T05:45:00.081677+0000","last_clean":"2023-10-09T05:45:00.081677+0000","last_became_active":"2023-10-09T05:42:12.628940+0000","last_became_peered":"2023-10-09T05:42:12.628940+0000","last_unstale":"2023-10-09T05:45:00.081677+0000","last_undegraded":"2023-10-09T05:45:00.081677+0000","last_fullsized":"2023-10-09T05:45:00.081677+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:32:38.949076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":497,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.081722+0000","last_change":"2023-10-09T05:20:18.132750+0000","last_active":"2023-10-09T05:45:00.081722+0000","last_peered":"2023-10-09T05:45:00.081722+0000","last_clean":"2023-10-09T05:45:00.081722+0000","last_became_active":"2023-10-09T05:20:18.132396+0000","last_became_peered":"2023-10-09T05:20:18.132396+0000","last_unstale":"2023-10-09T05:45:00.081722+0000","last_undegraded":"2023-10-09T05:45:00.081722+0000","last_fullsized":"2023-10-09T05:45:00.081722+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:00:55.731045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"420'64","reported_seq":474,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.965416+0000","last_change":"2023-10-09T05:42:12.666443+0000","last_active":"2023-10-09T05:44:59.965416+0000","last_peered":"2023-10-09T05:44:59.965416+0000","last_clean":"2023-10-09T05:44:59.965416+0000","last_became_active":"2023-10-09T05:42:12.632817+0000","last_became_peered":"2023-10-09T05:42:12.632817+0000","last_unstale":"2023-10-09T05:44:59.965416+0000","last_undegraded":"2023-10-09T05:44:59.965416+0000","last_fullsized":"2023-10-09T05:44:59.965416+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":64,"log_dups_size":0,"ondisk_log_size":64,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:33.863984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042561999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":276,"num_read_kb":753,"num_write":62,"num_write_kb":1323,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":361,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.653140+0000","last_change":"2023-10-09T05:32:45.824227+0000","last_active":"2023-10-09T05:45:08.653140+0000","last_peered":"2023-10-09T05:45:08.653140+0000","last_clean":"2023-10-09T05:45:08.653140+0000","last_became_active":"2023-10-09T05:32:45.823662+0000","last_became_peered":"2023-10-09T05:32:45.823662+0000","last_unstale":"2023-10-09T05:45:08.653140+0000","last_undegraded":"2023-10-09T05:45:08.653140+0000","last_fullsized":"2023-10-09T05:45:08.653140+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:54:56.029126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":468,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.081539+0000","last_change":"2023-10-09T05:32:45.826815+0000","last_active":"2023-10-09T05:45:00.081539+0000","last_peered":"2023-10-09T05:45:00.081539+0000","last_clean":"2023-10-09T05:45:00.081539+0000","last_became_active":"2023-10-09T05:32:45.826529+0000","last_became_peered":"2023-10-09T05:32:45.826529+0000","last_unstale":"2023-10-09T05:45:00.081539+0000","last_undegraded":"2023-10-09T05:45:00.081539+0000","last_fullsized":"2023-10-09T05:45:00.081539+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:19.868533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":463,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.970586+0000","last_change":"2023-10-09T05:18:26.431563+0000","last_active":"2023-10-09T05:45:08.970586+0000","last_peered":"2023-10-09T05:45:08.970586+0000","last_clean":"2023-10-09T05:45:08.970586+0000","last_became_active":"2023-10-09T05:18:26.421025+0000","last_became_peered":"2023-10-09T05:18:26.421025+0000","last_unstale":"2023-10-09T05:45:08.970586+0000","last_undegraded":"2023-10-09T05:45:08.970586+0000","last_fullsized":"2023-10-09T05:45:08.970586+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:30.333585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"421'103","reported_seq":589,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.026281+0000","last_change":"2023-10-09T05:42:15.024058+0000","last_active":"2023-10-09T05:45:00.026281+0000","last_peered":"2023-10-09T05:45:00.026281+0000","last_clean":"2023-10-09T05:45:00.026281+0000","last_became_active":"2023-10-09T05:42:15.023495+0000","last_became_peered":"2023-10-09T05:42:15.023495+0000","last_unstale":"2023-10-09T05:45:00.026281+0000","last_undegraded":"2023-10-09T05:45:00.026281+0000","last_fullsized":"2023-10-09T05:45:00.026281+0000","mapping_epoch":427,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":428,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:05.851529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000511664,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":467,"num_read_kb":908,"num_write":75,"num_write_kb":309,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":463,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.967864+0000","last_change":"2023-10-09T05:21:08.179657+0000","last_active":"2023-10-09T05:45:08.967864+0000","last_peered":"2023-10-09T05:45:08.967864+0000","last_clean":"2023-10-09T05:45:08.967864+0000","last_became_active":"2023-10-09T05:21:08.178958+0000","last_became_peered":"2023-10-09T05:21:08.178958+0000","last_unstale":"2023-10-09T05:45:08.967864+0000","last_undegraded":"2023-10-09T05:45:08.967864+0000","last_fullsized":"2023-10-09T05:45:08.967864+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:20.815986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"420'97","reported_seq":657,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.982980+0000","last_change":"2023-10-09T05:44:59.982624+0000","last_active":"2023-10-09T05:44:59.982980+0000","last_peered":"2023-10-09T05:44:59.982980+0000","last_clean":"2023-10-09T05:44:59.982980+0000","last_became_active":"2023-10-09T05:44:59.968921+0000","last_became_peered":"2023-10-09T05:44:59.968921+0000","last_unstale":"2023-10-09T05:44:59.982980+0000","last_undegraded":"2023-10-09T05:44:59.982980+0000","last_fullsized":"2023-10-09T05:44:59.982980+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:47.664799+0000","last_clean_scrub_stamp":"2023-10-09T04:42:47.664799+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:26.685207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010964410000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":254,"num_read_kb":640,"num_write":112,"num_write_kb":304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":188,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.970332+0000","last_change":"2023-10-09T05:42:12.642659+0000","last_active":"2023-10-09T05:45:08.970332+0000","last_peered":"2023-10-09T05:45:08.970332+0000","last_clean":"2023-10-09T05:45:08.970332+0000","last_became_active":"2023-10-09T05:42:12.629784+0000","last_became_peered":"2023-10-09T05:42:12.629784+0000","last_unstale":"2023-10-09T05:45:08.970332+0000","last_undegraded":"2023-10-09T05:45:08.970332+0000","last_fullsized":"2023-10-09T05:45:08.970332+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:41.316243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":34,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.085159+0000","last_change":"2023-10-09T05:42:12.624836+0000","last_active":"2023-10-09T05:45:00.085159+0000","last_peered":"2023-10-09T05:45:00.085159+0000","last_clean":"2023-10-09T05:45:00.085159+0000","last_became_active":"2023-10-09T05:42:12.624196+0000","last_became_peered":"2023-10-09T05:42:12.624196+0000","last_unstale":"2023-10-09T05:45:00.085159+0000","last_undegraded":"2023-10-09T05:45:00.085159+0000","last_fullsized":"2023-10-09T05:45:00.085159+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:56:36.734026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'2","reported_seq":253,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.969849+0000","last_change":"2023-10-09T05:42:12.634551+0000","last_active":"2023-10-09T05:45:08.969849+0000","last_peered":"2023-10-09T05:45:08.969849+0000","last_clean":"2023-10-09T05:45:08.969849+0000","last_became_active":"2023-10-09T05:42:12.633757+0000","last_became_peered":"2023-10-09T05:42:12.633757+0000","last_unstale":"2023-10-09T05:45:08.969849+0000","last_undegraded":"2023-10-09T05:45:08.969849+0000","last_fullsized":"2023-10-09T05:45:08.969849+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:09.847947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":242,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.969022+0000","last_change":"2023-10-09T05:36:16.898207+0000","last_active":"2023-10-09T05:45:08.969022+0000","last_peered":"2023-10-09T05:45:08.969022+0000","last_clean":"2023-10-09T05:45:08.969022+0000","last_became_active":"2023-10-09T05:36:16.898009+0000","last_became_peered":"2023-10-09T05:36:16.898009+0000","last_unstale":"2023-10-09T05:45:08.969022+0000","last_undegraded":"2023-10-09T05:45:08.969022+0000","last_fullsized":"2023-10-09T05:45:08.969022+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:59.694977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":176,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.675035+0000","last_change":"2023-10-09T05:42:12.638282+0000","last_active":"2023-10-09T05:44:59.675035+0000","last_peered":"2023-10-09T05:44:59.675035+0000","last_clean":"2023-10-09T05:44:59.675035+0000","last_became_active":"2023-10-09T05:42:12.625036+0000","last_became_peered":"2023-10-09T05:42:12.625036+0000","last_unstale":"2023-10-09T05:44:59.675035+0000","last_undegraded":"2023-10-09T05:44:59.675035+0000","last_fullsized":"2023-10-09T05:44:59.675035+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:18:08.570371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":44,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.085713+0000","last_change":"2023-10-09T05:42:59.630617+0000","last_active":"2023-10-09T05:45:00.085713+0000","last_peered":"2023-10-09T05:45:00.085713+0000","last_clean":"2023-10-09T05:45:00.085713+0000","last_became_active":"2023-10-09T05:42:59.630103+0000","last_became_peered":"2023-10-09T05:42:59.630103+0000","last_unstale":"2023-10-09T05:45:00.085713+0000","last_undegraded":"2023-10-09T05:45:00.085713+0000","last_fullsized":"2023-10-09T05:45:00.085713+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:14:32.213142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":176,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:44:59.675261+0000","last_change":"2023-10-09T05:42:12.644588+0000","last_active":"2023-10-09T05:44:59.675261+0000","last_peered":"2023-10-09T05:44:59.675261+0000","last_clean":"2023-10-09T05:44:59.675261+0000","last_became_active":"2023-10-09T05:42:12.629471+0000","last_became_peered":"2023-10-09T05:42:12.629471+0000","last_unstale":"2023-10-09T05:44:59.675261+0000","last_undegraded":"2023-10-09T05:44:59.675261+0000","last_fullsized":"2023-10-09T05:44:59.675261+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:15:31.005959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":34,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.084837+0000","last_change":"2023-10-09T05:42:12.628333+0000","last_active":"2023-10-09T05:45:00.084837+0000","last_peered":"2023-10-09T05:45:00.084837+0000","last_clean":"2023-10-09T05:45:00.084837+0000","last_became_active":"2023-10-09T05:42:12.625671+0000","last_became_peered":"2023-10-09T05:42:12.625671+0000","last_unstale":"2023-10-09T05:45:00.084837+0000","last_undegraded":"2023-10-09T05:45:00.084837+0000","last_fullsized":"2023-10-09T05:45:00.084837+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:41.463748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'2","reported_seq":457,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.970869+0000","last_change":"2023-10-09T05:22:47.197002+0000","last_active":"2023-10-09T05:45:08.970869+0000","last_peered":"2023-10-09T05:45:08.970869+0000","last_clean":"2023-10-09T05:45:08.970869+0000","last_became_active":"2023-10-09T05:22:47.196685+0000","last_became_peered":"2023-10-09T05:22:47.196685+0000","last_unstale":"2023-10-09T05:45:08.970869+0000","last_undegraded":"2023-10-09T05:45:08.970869+0000","last_fullsized":"2023-10-09T05:45:08.970869+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:47:32.594447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":440,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.968162+0000","last_change":"2023-10-09T05:02:09.649674+0000","last_active":"2023-10-09T05:45:08.968162+0000","last_peered":"2023-10-09T05:45:08.968162+0000","last_clean":"2023-10-09T05:45:08.968162+0000","last_became_active":"2023-10-09T05:02:09.649041+0000","last_became_peered":"2023-10-09T05:02:09.649041+0000","last_unstale":"2023-10-09T05:45:08.968162+0000","last_undegraded":"2023-10-09T05:45:08.968162+0000","last_fullsized":"2023-10-09T05:45:08.968162+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:18.524071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'5","reported_seq":249,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T05:45:08.968808+0000","last_change":"2023-10-09T05:32:45.849292+0000","last_active":"2023-10-09T05:45:08.968808+0000","last_peered":"2023-10-09T05:45:08.968808+0000","last_clean":"2023-10-09T05:45:08.968808+0000","last_became_active":"2023-10-09T05:32:45.829351+0000","last_became_peered":"2023-10-09T05:32:45.829351+0000","last_unstale":"2023-10-09T05:45:08.968808+0000","last_undegraded":"2023-10-09T05:45:08.968808+0000","last_fullsized":"2023-10-09T05:45:08.968808+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:51.735979+0000","last_clean_scrub_stamp":"2023-10-09T04:44:51.735979+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:07:15.666617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":34,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T05:45:00.084596+0000","last_change":"2023-10-09T05:42:12.621549+0000","last_active":"2023-10-09T05:45:00.084596+0000","last_peered":"2023-10-09T05:45:00.084596+0000","last_clean":"2023-10-09T05:45:00.084596+0000","last_became_active":"2023-10-09T05:42:12.621190+0000","last_became_peered":"2023-10-09T05:42:12.621190+0000","last_unstale":"2023-10-09T05:45:00.084596+0000","last_undegraded":"2023-10-09T05:45:00.084596+0000","last_fullsized":"2023-10-09T05:45:00.084596+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:44:52.749403+0000","last_clean_scrub_stamp":"2023-10-09T04:44:52.749403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:43.642340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":23,"num_read_kb":22,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":74,"num_bytes_recovered":22374,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234774,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1510260,"kb_used_data":5228,"kb_used_omap":0,"kb_used_meta":1505024,"kb_avail":92234892,"statfs":{"total":95995035648,"available":94448529408,"internally_reserved":0,"allocated":5353472,"data_stored":4014652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1541144576},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:46:35 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.95199999999999996,"15min":2.4580000000000002},"min":{"1min":0.50900000000000001,"5min":0.42599999999999999,"15min":0.26100000000000001},"max":{"1min":1.204,"5min":1.3500000000000001,"15min":88.450000000000003},"last":1.3140000000000001},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.054,"15min":2.581},"min":{"1min":0.64000000000000001,"5min":0.501,"15min":0.28899999999999998},"max":{"1min":1.254,"5min":1.4610000000000001,"15min":88.415000000000006},"last":1.286}]},{"osd":1,"last update":"Mon Oct 9 05:46:11 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":0.90900000000000003,"15min":1.6140000000000001},"min":{"1min":0.57499999999999996,"5min":0.43099999999999999,"15min":0.27100000000000002},"max":{"1min":1.048,"5min":1.306,"15min":44.456000000000003},"last":1.1000000000000001},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.97399999999999998,"15min":1.712},"min":{"1min":0.56399999999999995,"5min":0.53100000000000003,"15min":0.318},"max":{"1min":1.387,"5min":1.462,"15min":44.395000000000003},"last":0.69999999999999996}]},{"osd":2,"last update":"Mon Oct 9 05:46:35 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.96299999999999997,"15min":1.7809999999999999},"min":{"1min":0.59199999999999997,"5min":0.56299999999999994,"15min":0.28999999999999998},"max":{"1min":1.163,"5min":1.385,"15min":102.146},"last":1.0660000000000001},{"interface":"front","average":{"1min":0.89600000000000002,"5min":0.93899999999999995,"15min":1.6899999999999999},"min":{"1min":0.627,"5min":0.40600000000000003,"15min":0.35899999999999999},"max":{"1min":1.1759999999999999,"5min":1.3700000000000001,"15min":102.188},"last":1.133}]},{"osd":3,"last update":"Mon Oct 9 05:46:35 2023","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":0.95799999999999996,"15min":1.5469999999999999},"min":{"1min":0.64000000000000001,"5min":0.57099999999999995,"15min":0.33500000000000002},"max":{"1min":1.228,"5min":1.464,"15min":112.679},"last":1.0109999999999999},{"interface":"front","average":{"1min":1.0609999999999999,"5min":1.036,"15min":1.633},"min":{"1min":0.73199999999999998,"5min":0.53500000000000003,"15min":0.39600000000000002},"max":{"1min":1.3420000000000001,"5min":1.53,"15min":109.899},"last":0.95999999999999996}]},{"osd":4,"last update":"Mon Oct 9 05:46:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":0.97499999999999998,"15min":0.93400000000000005},"min":{"1min":0.77500000000000002,"5min":0.48499999999999999,"15min":0.309},"max":{"1min":1.375,"5min":1.478,"15min":7.077},"last":0.747},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.089,"15min":1.0640000000000001},"min":{"1min":0.71799999999999997,"5min":0.53700000000000003,"15min":0.36699999999999999},"max":{"1min":1.3859999999999999,"5min":1.575,"15min":7.3869999999999996},"last":1.177}]},{"osd":5,"last update":"Mon Oct 9 05:46:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.0309999999999999,"15min":1.0880000000000001},"min":{"1min":0.70099999999999996,"5min":0.46400000000000002,"15min":0.33500000000000002},"max":{"1min":1.335,"5min":1.528,"15min":7.4829999999999997},"last":1.2430000000000001},{"interface":"front","average":{"1min":1.1899999999999999,"5min":1.1619999999999999,"15min":1.177},"min":{"1min":0.94999999999999996,"5min":0.63900000000000001,"15min":0.35599999999999998},"max":{"1min":1.4610000000000001,"5min":1.6100000000000001,"15min":7.6219999999999999},"last":1.216}]},{"osd":6,"last update":"Mon Oct 9 05:46:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":1.179,"15min":1.1870000000000001},"min":{"1min":0.91900000000000004,"5min":0.64300000000000002,"15min":0.38700000000000001},"max":{"1min":1.452,"5min":1.5529999999999999,"15min":6.8789999999999996},"last":1.272},{"interface":"front","average":{"1min":1.3839999999999999,"5min":1.266,"15min":1.262},"min":{"1min":0.747,"5min":0.51200000000000001,"15min":0.432},"max":{"1min":3.702,"5min":3.702,"15min":7.46},"last":1.357}]}]},{"osd":6,"up_from":46,"seq":197568496416,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1511920,"kb_used_data":9320,"kb_used_omap":0,"kb_used_meta":1502592,"kb_avail":92233232,"statfs":{"total":95995035648,"available":94446829568,"internally_reserved":0,"allocated":9543680,"data_stored":8209152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1538654208},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:46:32 2023","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":0.97599999999999998,"15min":3.5920000000000001},"min":{"1min":0.63600000000000001,"5min":0.44,"15min":0.254},"max":{"1min":1.3839999999999999,"5min":1.4530000000000001,"15min":266.40499999999997},"last":1.0409999999999999},{"interface":"front","average":{"1min":1.1819999999999999,"5min":1.1020000000000001,"15min":3.698},"min":{"1min":0.98199999999999998,"5min":0.53900000000000003,"15min":0.32200000000000001},"max":{"1min":1.482,"5min":1.6559999999999999,"15min":266.49700000000001},"last":1.1870000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:46:32 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":0.91700000000000004,"15min":1.319},"min":{"1min":0.69099999999999995,"5min":0.45400000000000001,"15min":0.28299999999999997},"max":{"1min":1.1930000000000001,"5min":1.413,"15min":41.408000000000001},"last":1.2969999999999999},{"interface":"front","average":{"1min":1.006,"5min":0.95399999999999996,"15min":1.298},"min":{"1min":0.65200000000000002,"5min":0.53200000000000003,"15min":0.28199999999999997},"max":{"1min":1.3660000000000001,"5min":1.3799999999999999,"15min":40.835999999999999},"last":0.93500000000000005}]},{"osd":2,"last update":"Mon Oct 9 05:46:32 2023","interfaces":[{"interface":"back","average":{"1min":1.107,"5min":1.0369999999999999,"15min":2.6970000000000001},"min":{"1min":0.64100000000000001,"5min":0.55000000000000004,"15min":0.26600000000000001},"max":{"1min":1.4530000000000001,"5min":1.6120000000000001,"15min":211.66300000000001},"last":0.76700000000000002},{"interface":"front","average":{"1min":0.88700000000000001,"5min":0.89700000000000002,"15min":2.9020000000000001},"min":{"1min":0.71599999999999997,"5min":0.51700000000000002,"15min":0.27900000000000003},"max":{"1min":1.095,"5min":1.2190000000000001,"15min":211.423},"last":1.105}]},{"osd":3,"last update":"Mon Oct 9 05:46:32 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":0.98899999999999999,"15min":1.978},"min":{"1min":0.80800000000000005,"5min":0.60899999999999999,"15min":0.309},"max":{"1min":1.2130000000000001,"5min":1.4630000000000001,"15min":209.69},"last":1.2410000000000001},{"interface":"front","average":{"1min":1.149,"5min":1.0980000000000001,"15min":2.0070000000000001},"min":{"1min":0.81100000000000005,"5min":0.56100000000000005,"15min":0.30399999999999999},"max":{"1min":1.403,"5min":1.5509999999999999,"15min":209.34800000000001},"last":1.458}]},{"osd":4,"last update":"Mon Oct 9 05:46:32 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":0.95899999999999996,"15min":0.92400000000000004},"min":{"1min":0.66400000000000003,"5min":0.498,"15min":0.32300000000000001},"max":{"1min":1.208,"5min":1.786,"15min":10.976000000000001},"last":1.0680000000000001},{"interface":"front","average":{"1min":1.1659999999999999,"5min":1.1080000000000001,"15min":1.034},"min":{"1min":0.72299999999999998,"5min":0.49299999999999999,"15min":0.32800000000000001},"max":{"1min":1.5449999999999999,"5min":1.766,"15min":5.0190000000000001},"last":1.329}]},{"osd":5,"last update":"Mon Oct 9 05:46:32 2023","interfaces":[{"interface":"back","average":{"1min":1.107,"5min":1.0489999999999999,"15min":1.121},"min":{"1min":0.73999999999999999,"5min":0.61899999999999999,"15min":0.29299999999999998},"max":{"1min":1.337,"5min":2.214,"15min":11.275},"last":1.161},{"interface":"front","average":{"1min":1.3069999999999999,"5min":1.171,"15min":1.1080000000000001},"min":{"1min":0.90200000000000002,"5min":0.58499999999999996,"15min":0.36199999999999999},"max":{"1min":1.595,"5min":1.7450000000000001,"15min":7.3090000000000002},"last":1.5349999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:45:52 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":0.97799999999999998,"15min":1.2410000000000001},"min":{"1min":0.67600000000000005,"5min":0.46800000000000003,"15min":0.34999999999999998},"max":{"1min":1.2969999999999999,"5min":1.9730000000000001,"15min":18.963000000000001},"last":1.4410000000000001},{"interface":"front","average":{"1min":1.1539999999999999,"5min":1.135,"15min":1.3560000000000001},"min":{"1min":0.81799999999999995,"5min":0.54600000000000004,"15min":0.39500000000000002},"max":{"1min":1.4350000000000001,"5min":2.0419999999999998,"15min":19.010000000000002},"last":1.5089999999999999}]}]},{"osd":1,"up_from":96,"seq":412316861007,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1325936,"kb_used_data":10152,"kb_used_omap":4,"kb_used_meta":1315771,"kb_avail":92419216,"statfs":{"total":95995035648,"available":94637277184,"internally_reserved":0,"allocated":10395648,"data_stored":7794577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1347350414},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.66500000000000004,"15min":2.6760000000000002},"min":{"1min":0.30499999999999999,"5min":0.29299999999999998,"15min":0.21099999999999999},"max":{"1min":0.86299999999999999,"5min":0.99199999999999999,"15min":150.75200000000001},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.71099999999999997,"15min":2.6989999999999998},"min":{"1min":0.38900000000000001,"5min":0.34200000000000003,"15min":0.17499999999999999},"max":{"1min":1.1739999999999999,"5min":1.1739999999999999,"15min":150.59299999999999},"last":0.65800000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.71799999999999997,"15min":2.9039999999999999},"min":{"1min":0.37,"5min":0.31900000000000001,"15min":0.19800000000000001},"max":{"1min":1.139,"5min":1.2569999999999999,"15min":154.14599999999999},"last":0.621},{"interface":"front","average":{"1min":0.70199999999999996,"5min":0.84899999999999998,"15min":2.7879999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.24199999999999999},"max":{"1min":1.018,"5min":1.379,"15min":154.215},"last":0.56899999999999995}]},{"osd":3,"last update":"Mon Oct 9 05:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.72299999999999998,"15min":2.2949999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.217},"max":{"1min":0.93799999999999994,"5min":1.21,"15min":152.58199999999999},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.745,"5min":0.86899999999999999,"15min":2.282},"min":{"1min":0.42999999999999999,"5min":0.42599999999999999,"15min":0.255},"max":{"1min":1.0760000000000001,"5min":1.2150000000000001,"15min":154.178},"last":0.72499999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:46:29 2023","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":1.0800000000000001,"15min":1.329},"min":{"1min":0.755,"5min":0.58799999999999997,"15min":0.38700000000000001},"max":{"1min":1.278,"5min":1.518,"15min":8.4849999999999994},"last":1.2310000000000001},{"interface":"front","average":{"1min":0.95299999999999996,"5min":0.97499999999999998,"15min":1.208},"min":{"1min":0.67200000000000004,"5min":0.65200000000000002,"15min":0.36799999999999999},"max":{"1min":1.3680000000000001,"5min":1.4259999999999999,"15min":7.2140000000000004},"last":1.1899999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":1.0189999999999999,"15min":1.3959999999999999},"min":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.32300000000000001},"max":{"1min":1.3,"5min":1.5149999999999999,"15min":12.630000000000001},"last":0.98699999999999999},{"interface":"front","average":{"1min":1.002,"5min":1.1180000000000001,"15min":1.419},"min":{"1min":0.57699999999999996,"5min":0.56399999999999995,"15min":0.39700000000000002},"max":{"1min":1.452,"5min":1.5620000000000001,"15min":9.8260000000000005},"last":1.0349999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:46:29 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.0740000000000001,"15min":1.407},"min":{"1min":0.752,"5min":0.68899999999999995,"15min":0.318},"max":{"1min":1.4279999999999999,"5min":1.6539999999999999,"15min":12.686999999999999},"last":1.292},{"interface":"front","average":{"1min":1.091,"5min":1.194,"15min":1.484},"min":{"1min":0.65900000000000003,"5min":0.61499999999999999,"15min":0.41499999999999998},"max":{"1min":1.377,"5min":1.6579999999999999,"15min":10.446},"last":1.169}]},{"osd":7,"last update":"Mon Oct 9 05:46:29 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.234,"15min":1.5},"min":{"1min":0.73299999999999998,"5min":0.66100000000000003,"15min":0.38800000000000001},"max":{"1min":1.532,"5min":1.738,"15min":8.7200000000000006},"last":1.2030000000000001},{"interface":"front","average":{"1min":1.107,"5min":1.1200000000000001,"15min":1.343},"min":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.38600000000000001},"max":{"1min":1.4690000000000001,"5min":1.492,"15min":7.2560000000000002},"last":1.244}]}]},{"osd":0,"up_from":88,"seq":377957122648,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1251280,"kb_used_data":6472,"kb_used_omap":5,"kb_used_meta":1244794,"kb_avail":92493872,"statfs":{"total":95995035648,"available":94713724928,"internally_reserved":0,"allocated":6627328,"data_stored":4169435,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1274669690},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.68400000000000005,"15min":1.786},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.17699999999999999},"max":{"1min":0.90200000000000002,"5min":1.915,"15min":111.001},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.75700000000000001,"15min":1.859},"min":{"1min":0.41799999999999998,"5min":0.36099999999999999,"15min":0.20799999999999999},"max":{"1min":0.96599999999999997,"5min":2.4980000000000002,"15min":110.919},"last":0.875}]},{"osd":2,"last update":"Mon Oct 9 05:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":0.79900000000000004,"15min":1.9330000000000001},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.26800000000000002},"max":{"1min":1.1379999999999999,"5min":2.266,"15min":74.817999999999998},"last":0.97399999999999998},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.76100000000000001,"15min":2.0499999999999998},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.25600000000000001},"max":{"1min":0.93500000000000005,"5min":1.75,"15min":74.751000000000005},"last":0.64100000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.73899999999999999,"15min":3.4180000000000001},"min":{"1min":0.373,"5min":0.35199999999999998,"15min":0.22500000000000001},"max":{"1min":0.871,"5min":1.8660000000000001,"15min":290.73000000000002},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.85499999999999998,"15min":3.359},"min":{"1min":0.46999999999999997,"5min":0.44600000000000001,"15min":0.246},"max":{"1min":1.252,"5min":2.3730000000000002,"15min":289.85199999999998},"last":1.2909999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":1.0169999999999999,"15min":1.353},"min":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.36299999999999999},"max":{"1min":1.339,"5min":2.4510000000000001,"15min":12.377000000000001},"last":1.0800000000000001},{"interface":"front","average":{"1min":0.93999999999999995,"5min":1.026,"15min":1.4550000000000001},"min":{"1min":0.627,"5min":0.627,"15min":0.32400000000000001},"max":{"1min":1.323,"5min":2.6150000000000002,"15min":12.477},"last":0.86699999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":1.046,"15min":1.5640000000000001},"min":{"1min":0.70899999999999996,"5min":0.56399999999999995,"15min":0.318},"max":{"1min":1.609,"5min":2.4809999999999999,"15min":14.510999999999999},"last":0.83299999999999996},{"interface":"front","average":{"1min":1.054,"5min":1.1379999999999999,"15min":1.462},"min":{"1min":0.79300000000000004,"5min":0.69899999999999995,"15min":0.33700000000000002},"max":{"1min":1.633,"5min":2.5830000000000002,"15min":12.403},"last":1.242}]},{"osd":6,"last update":"Mon Oct 9 05:46:29 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.115,"15min":1.6359999999999999},"min":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.33500000000000002},"max":{"1min":1.3779999999999999,"5min":2.6890000000000001,"15min":15.544},"last":0.93000000000000005},{"interface":"front","average":{"1min":1.0329999999999999,"5min":1.0980000000000001,"15min":1.667},"min":{"1min":0.73899999999999999,"5min":0.68200000000000005,"15min":0.318},"max":{"1min":1.4610000000000001,"5min":2.4249999999999998,"15min":14.459},"last":1.1659999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:46:29 2023","interfaces":[{"interface":"back","average":{"1min":1,"5min":1.1000000000000001,"15min":1.7490000000000001},"min":{"1min":0.70199999999999996,"5min":0.66800000000000004,"15min":0.39200000000000002},"max":{"1min":1.367,"5min":2.335,"15min":14.492000000000001},"last":1.0609999999999999},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.1910000000000001,"15min":1.5660000000000001},"min":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.40100000000000002},"max":{"1min":1.498,"5min":2.5390000000000001,"15min":12.43},"last":1.1830000000000001}]}]},{"osd":2,"up_from":106,"seq":455266533926,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1131760,"kb_used_data":5864,"kb_used_omap":7,"kb_used_meta":1125880,"kb_avail":92613392,"statfs":{"total":95995035648,"available":94836113408,"internally_reserved":0,"allocated":6004736,"data_stored":3574377,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1152901201},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:46:14 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.65400000000000003,"15min":2.298},"min":{"1min":0.307,"5min":0.307,"15min":0.17599999999999999},"max":{"1min":1.0920000000000001,"5min":1.097,"15min":91.646000000000001},"last":0.80100000000000005},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.75700000000000001,"15min":2.1869999999999998},"min":{"1min":0.35999999999999999,"5min":0.308,"15min":0.219},"max":{"1min":0.99299999999999999,"5min":1.488,"15min":92.522000000000006},"last":1.5509999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:46:14 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":0.76200000000000001,"15min":1.97},"min":{"1min":0.44800000000000001,"5min":0.32000000000000001,"15min":0.217},"max":{"1min":1.2629999999999999,"5min":1.423,"15min":120.619},"last":1.377},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.755,"15min":2.1829999999999998},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.24299999999999999},"max":{"1min":0.97699999999999998,"5min":4.0880000000000001,"15min":122.223},"last":1.4199999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:46:31 2023","interfaces":[{"interface":"back","average":{"1min":0.69299999999999995,"5min":0.84299999999999997,"15min":1.6899999999999999},"min":{"1min":0.376,"5min":0.376,"15min":0.214},"max":{"1min":0.96699999999999997,"5min":3.891,"15min":15.576000000000001},"last":1.2749999999999999},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.81699999999999995,"15min":1.6419999999999999},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.28299999999999997},"max":{"1min":1.0980000000000001,"5min":4.1699999999999999,"15min":15.131},"last":1.306}]},{"osd":4,"last update":"Mon Oct 9 05:46:14 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":1.03,"15min":1.415},"min":{"1min":0.66700000000000004,"5min":0.61099999999999999,"15min":0.31900000000000001},"max":{"1min":1.3600000000000001,"5min":1.544,"15min":10.379},"last":1.054},{"interface":"front","average":{"1min":0.94799999999999995,"5min":1.032,"15min":1.5389999999999999},"min":{"1min":0.64100000000000001,"5min":0.63500000000000001,"15min":0.35399999999999998},"max":{"1min":1.395,"5min":4.3630000000000004,"15min":14.717000000000001},"last":1.395}]},{"osd":5,"last update":"Mon Oct 9 05:46:14 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":1.024,"15min":1.4139999999999999},"min":{"1min":0.64000000000000001,"5min":0.54800000000000004,"15min":0.34000000000000002},"max":{"1min":1.3049999999999999,"5min":3.9159999999999999,"15min":9.8729999999999993},"last":1.224},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.103,"15min":1.4850000000000001},"min":{"1min":0.623,"5min":0.61899999999999999,"15min":0.33700000000000002},"max":{"1min":1.462,"5min":1.649,"15min":10.265000000000001},"last":1.6200000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:46:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":1.117,"15min":1.6140000000000001},"min":{"1min":0.623,"5min":0.623,"15min":0.32600000000000001},"max":{"1min":1.716,"5min":4.4850000000000003,"15min":14.763999999999999},"last":1.355},{"interface":"front","average":{"1min":1.034,"5min":1.1259999999999999,"15min":1.4910000000000001},"min":{"1min":0.77600000000000002,"5min":0.47599999999999998,"15min":0.36399999999999999},"max":{"1min":1.482,"5min":4.0099999999999998,"15min":9.9120000000000008},"last":1.1299999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:46:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.1539999999999999,"15min":1.575},"min":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.433},"max":{"1min":1.54,"5min":3.9700000000000002,"15min":9.8930000000000007},"last":1.167},{"interface":"front","average":{"1min":1.0960000000000001,"5min":1.1799999999999999,"15min":1.6299999999999999},"min":{"1min":0.71099999999999997,"5min":0.68500000000000005,"15min":0.39000000000000001},"max":{"1min":1.5289999999999999,"5min":1.73,"15min":12.016},"last":1.4530000000000001}]}]},{"osd":3,"up_from":123,"seq":528280977944,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1177988,"kb_used_data":5756,"kb_used_omap":2,"kb_used_meta":1172221,"kb_avail":92567164,"statfs":{"total":95995035648,"available":94788775936,"internally_reserved":0,"allocated":5894144,"data_stored":3561284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":1200354357},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:45:35 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":1.357,"15min":3.0030000000000001},"min":{"1min":0.44400000000000001,"5min":0.247,"15min":0.224},"max":{"1min":1.0289999999999999,"5min":62.600999999999999,"15min":117.06100000000001},"last":0.95499999999999996},{"interface":"front","average":{"1min":0.77200000000000002,"5min":1.415,"15min":3.3980000000000001},"min":{"1min":0.36699999999999999,"5min":0.28999999999999998,"15min":0.184},"max":{"1min":1.0609999999999999,"5min":62.643000000000001,"15min":118.372},"last":0.78800000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:45:35 2023","interfaces":[{"interface":"back","average":{"1min":0.77600000000000002,"5min":0.70899999999999996,"15min":2.157},"min":{"1min":0.46700000000000003,"5min":0.36399999999999999,"15min":0.23000000000000001},"max":{"1min":1.0960000000000001,"5min":1.137,"15min":118.20999999999999},"last":0.82499999999999996},{"interface":"front","average":{"1min":0.77200000000000002,"5min":0.77000000000000002,"15min":1.9350000000000001},"min":{"1min":0.49099999999999999,"5min":0.34200000000000003,"15min":0.218},"max":{"1min":1.0109999999999999,"5min":3.609,"15min":124.083},"last":0.72099999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:45:35 2023","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":0.75,"15min":3.2309999999999999},"min":{"1min":0.41999999999999998,"5min":0.33100000000000002,"15min":0.23699999999999999},"max":{"1min":1.204,"5min":1.73,"15min":350.53899999999999},"last":0.873},{"interface":"front","average":{"1min":0.83799999999999997,"5min":0.76800000000000002,"15min":3.306},"min":{"1min":0.42199999999999999,"5min":0.312,"15min":0.23599999999999999},"max":{"1min":1.294,"5min":1.294,"15min":350.64699999999999},"last":0.90600000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:45:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":0.96399999999999997,"15min":1.3340000000000001},"min":{"1min":0.74299999999999999,"5min":0.60099999999999998,"15min":0.34899999999999998},"max":{"1min":1.2909999999999999,"5min":1.52,"15min":10.15},"last":0.99199999999999999},{"interface":"front","average":{"1min":1.0760000000000001,"5min":1.018,"15min":1.3160000000000001},"min":{"1min":0.71299999999999997,"5min":0.57099999999999995,"15min":0.35799999999999998},"max":{"1min":1.391,"5min":1.708,"15min":12.132999999999999},"last":1.4770000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:45:35 2023","interfaces":[{"interface":"back","average":{"1min":1.103,"5min":1.069,"15min":1.391},"min":{"1min":0.64900000000000002,"5min":0.61499999999999999,"15min":0.29799999999999999},"max":{"1min":1.5680000000000001,"5min":1.5920000000000001,"15min":12.092000000000001},"last":1.3100000000000001},{"interface":"front","average":{"1min":1.1220000000000001,"5min":1.0840000000000001,"15min":1.3640000000000001},"min":{"1min":0.76700000000000002,"5min":0.59399999999999997,"15min":0.36299999999999999},"max":{"1min":1.496,"5min":1.7769999999999999,"15min":10.135},"last":1.494}]},{"osd":6,"last update":"Mon Oct 9 05:45:35 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":1.0940000000000001,"15min":1.5600000000000001},"min":{"1min":0.71199999999999997,"5min":0.55800000000000005,"15min":0.34499999999999997},"max":{"1min":1.599,"5min":1.861,"15min":20.463999999999999},"last":1.405},{"interface":"front","average":{"1min":1.155,"5min":1.091,"15min":1.5580000000000001},"min":{"1min":0.81399999999999995,"5min":0.69499999999999995,"15min":0.38400000000000001},"max":{"1min":1.4339999999999999,"5min":1.97,"15min":20.279},"last":1.248}]},{"osd":7,"last update":"Mon Oct 9 05:45:35 2023","interfaces":[{"interface":"back","average":{"1min":1.1910000000000001,"5min":1.115,"15min":1.5669999999999999},"min":{"1min":0.69399999999999995,"5min":0.65300000000000002,"15min":0.374},"max":{"1min":1.5580000000000001,"5min":1.9059999999999999,"15min":10.885},"last":1.2010000000000001},{"interface":"front","average":{"1min":1.2170000000000001,"5min":1.117,"15min":1.5249999999999999},"min":{"1min":0.77900000000000003,"5min":0.65100000000000002,"15min":0.32600000000000001},"max":{"1min":1.5189999999999999,"5min":1.7589999999999999,"15min":12.113},"last":1.379}]}]},{"osd":4,"up_from":30,"seq":128849019702,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":905400,"kb_used_data":4656,"kb_used_omap":0,"kb_used_meta":900736,"kb_avail":92839752,"statfs":{"total":95995035648,"available":95067906048,"internally_reserved":0,"allocated":4767744,"data_stored":3425286,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":922353664},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:46:17 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":2.1059999999999999,"15min":3.0899999999999999},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.26200000000000001},"max":{"1min":1.3260000000000001,"5min":89.522999999999996,"15min":110.48399999999999},"last":1.097},{"interface":"front","average":{"1min":1.0449999999999999,"5min":2.0739999999999998,"15min":3.0609999999999999},"min":{"1min":0.68300000000000005,"5min":0.53100000000000003,"15min":0.312},"max":{"1min":1.4399999999999999,"5min":89.582999999999998,"15min":110.56},"last":0.99199999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:46:06 2023","interfaces":[{"interface":"back","average":{"1min":0.80500000000000005,"5min":0.83599999999999997,"15min":2.694},"min":{"1min":0.503,"5min":0.41099999999999998,"15min":0.29199999999999998},"max":{"1min":1.054,"5min":1.292,"15min":112.17400000000001},"last":0.879},{"interface":"front","average":{"1min":0.86099999999999999,"5min":0.94399999999999995,"15min":2.6139999999999999},"min":{"1min":0.60399999999999998,"5min":0.46400000000000002,"15min":0.29199999999999998},"max":{"1min":1.254,"5min":4.6799999999999997,"15min":112.065},"last":1.256}]},{"osd":2,"last update":"Mon Oct 9 05:46:17 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.0169999999999999,"15min":3.0870000000000002},"min":{"1min":0.60999999999999999,"5min":0.47399999999999998,"15min":0.33300000000000002},"max":{"1min":1.3340000000000001,"5min":4.5229999999999997,"15min":110.38},"last":0.83799999999999997},{"interface":"front","average":{"1min":1.0269999999999999,"5min":0.97799999999999998,"15min":3.024},"min":{"1min":0.56499999999999995,"5min":0.45200000000000001,"15min":0.33400000000000002},"max":{"1min":1.474,"5min":1.4870000000000001,"15min":114.372},"last":1.038}]},{"osd":3,"last update":"Mon Oct 9 05:45:39 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":1.044,"15min":1.3899999999999999},"min":{"1min":0.66800000000000004,"5min":0.60399999999999998,"15min":0.33700000000000002},"max":{"1min":1.335,"5min":1.516,"15min":65.394000000000005},"last":0.94899999999999995},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.024,"15min":1.419},"min":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.35799999999999998},"max":{"1min":1.425,"5min":2.2240000000000002,"15min":65.347999999999999},"last":0.90500000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:45:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":1.0960000000000001,"15min":1.044},"min":{"1min":0.89200000000000002,"5min":0.47599999999999998,"15min":0.34499999999999997},"max":{"1min":1.4430000000000001,"5min":1.5509999999999999,"15min":6.9470000000000001},"last":1.323},{"interface":"front","average":{"1min":1.0940000000000001,"5min":1.075,"15min":1.016},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.36899999999999999},"max":{"1min":1.696,"5min":1.696,"15min":5.585},"last":0.80500000000000005}]},{"osd":6,"last update":"Mon Oct 9 05:46:06 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":1,"15min":1.488},"min":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.35099999999999998},"max":{"1min":1.5720000000000001,"5min":1.5720000000000001,"15min":135.15600000000001},"last":0.76000000000000001},{"interface":"front","average":{"1min":1.1899999999999999,"5min":1.196,"15min":1.5920000000000001},"min":{"1min":0.76700000000000002,"5min":0.51300000000000001,"15min":0.36799999999999999},"max":{"1min":1.53,"5min":4.7359999999999998,"15min":133.01499999999999},"last":1.1830000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:46:06 2023","interfaces":[{"interface":"back","average":{"1min":1.1890000000000001,"5min":1.165,"15min":1.1559999999999999},"min":{"1min":0.93300000000000005,"5min":0.36599999999999999,"15min":0.32700000000000001},"max":{"1min":1.7749999999999999,"5min":1.7749999999999999,"15min":9.1690000000000005},"last":1.129},{"interface":"front","average":{"1min":1.1240000000000001,"5min":1.1220000000000001,"15min":1.048},"min":{"1min":0.74299999999999999,"5min":0.50600000000000001,"15min":0.39000000000000001},"max":{"1min":1.6779999999999999,"5min":1.6779999999999999,"15min":5.2089999999999996},"last":1.0149999999999999}]}]},{"osd":5,"up_from":38,"seq":163208758058,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":899304,"kb_used_data":8736,"kb_used_omap":0,"kb_used_meta":890560,"kb_avail":92845848,"statfs":{"total":95995035648,"available":95074148352,"internally_reserved":0,"allocated":8945664,"data_stored":7614869,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911933440},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:46:34 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":0.98899999999999999,"15min":3.1669999999999998},"min":{"1min":0.73699999999999999,"5min":0.52700000000000002,"15min":0.23599999999999999},"max":{"1min":1.284,"5min":1.321,"15min":274.83699999999999},"last":1.04},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.083,"15min":3.274},"min":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.28000000000000003},"max":{"1min":1.365,"5min":1.657,"15min":274.57299999999998},"last":1.1890000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:46:34 2023","interfaces":[{"interface":"back","average":{"1min":0.86099999999999999,"5min":0.95799999999999996,"15min":1.3919999999999999},"min":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.27000000000000002},"max":{"1min":1.1599999999999999,"5min":1.4950000000000001,"15min":36.703000000000003},"last":1.1120000000000001},{"interface":"front","average":{"1min":0.84499999999999997,"5min":0.97299999999999998,"15min":1.2569999999999999},"min":{"1min":0.61199999999999999,"5min":0.58299999999999996,"15min":0.27900000000000003},"max":{"1min":1.256,"5min":1.526,"15min":36.613999999999997},"last":0.89300000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:45:35 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.95899999999999996,"15min":2.2410000000000001},"min":{"1min":0.59899999999999998,"5min":0.443,"15min":0.32000000000000001},"max":{"1min":1.387,"5min":1.4490000000000001,"15min":250.941},"last":1.1419999999999999},{"interface":"front","average":{"1min":1,"5min":1.01,"15min":2.3740000000000001},"min":{"1min":0.69099999999999995,"5min":0.628,"15min":0.32100000000000001},"max":{"1min":1.3200000000000001,"5min":1.5229999999999999,"15min":251.48599999999999},"last":0.85499999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:45:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":1.0209999999999999,"15min":1.488},"min":{"1min":0.77900000000000003,"5min":0.58999999999999997,"15min":0.32400000000000001},"max":{"1min":1.323,"5min":1.3919999999999999,"15min":68.061999999999998},"last":1.069},{"interface":"front","average":{"1min":1.073,"5min":1.034,"15min":1.716},"min":{"1min":0.72299999999999998,"5min":0.56299999999999994,"15min":0.33000000000000002},"max":{"1min":1.367,"5min":1.367,"15min":68.959000000000003},"last":1.089}]},{"osd":4,"last update":"Mon Oct 9 05:46:34 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":1.083,"15min":1.014},"min":{"1min":0.754,"5min":0.51900000000000002,"15min":0.33500000000000002},"max":{"1min":1.3460000000000001,"5min":1.546,"15min":7.6109999999999998},"last":0.98299999999999998},{"interface":"front","average":{"1min":1.03,"5min":1.0740000000000001,"15min":1.1200000000000001},"min":{"1min":0.58699999999999997,"5min":0.56599999999999995,"15min":0.34300000000000003},"max":{"1min":1.397,"5min":1.829,"15min":8.3480000000000008},"last":1.0089999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:46:34 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.1060000000000001,"15min":1.228},"min":{"1min":0.82399999999999995,"5min":0.63500000000000001,"15min":0.32300000000000001},"max":{"1min":1.5049999999999999,"5min":2.0680000000000001,"15min":8.5220000000000002},"last":1.1739999999999999},{"interface":"front","average":{"1min":1.1240000000000001,"5min":1.115,"15min":1.1319999999999999},"min":{"1min":0.435,"5min":0.435,"15min":0.32400000000000001},"max":{"1min":1.377,"5min":1.752,"15min":9.282},"last":1.2569999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:45:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2070000000000001,"5min":1.181,"15min":1.157},"min":{"1min":0.92100000000000004,"5min":0.64400000000000002,"15min":0.34399999999999997},"max":{"1min":1.4390000000000001,"5min":1.641,"15min":9.3970000000000002},"last":0.95399999999999996},{"interface":"front","average":{"1min":1.268,"5min":1.226,"15min":1.304},"min":{"1min":0.81299999999999994,"5min":0.67500000000000004,"15min":0.36299999999999999},"max":{"1min":1.7190000000000001,"5min":1.7190000000000001,"15min":11.593},"last":1.214}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:46:41.908 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T05:46:41.909 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-10-09T05:46:42.015 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=9cf93cf9d8c64e3298a8e471cc188b4b Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-10-09T05:46:42.018 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-10-09T05:46:42.027 INFO:tasks.cephadm:Teardown begin 2023-10-09T05:46:42.028 DEBUG:teuthology.orchestra.run.smithi133:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T05:46:42.043 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:41 smithi133 bash[105589]: cluster 2023-10-09T05:46:40.628831+0000 mgr.y (mgr.25277) 1771 : cluster 0 pgmap v2003: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:42.044 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:41 smithi133 bash[102858]: cluster 2023-10-09T05:46:40.628831+0000 mgr.y (mgr.25277) 1771 : cluster 0 pgmap v2003: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:42.060 DEBUG:teuthology.orchestra.run.smithi194:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T05:46:42.073 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:41 smithi194 bash[92666]: cluster 2023-10-09T05:46:40.628831+0000 mgr.y (mgr.25277) 1771 : cluster 0 pgmap v2003: 73 pgs: 73 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:46:42.077 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-10-09T05:46:42.077 DEBUG:teuthology.orchestra.run.smithi133:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-09T05:46:42.114 DEBUG:teuthology.orchestra.run.smithi194:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-09T05:46:42.124 INFO:tasks.cephadm:Stopping all daemons... 2023-10-09T05:46:42.124 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-10-09T05:46:42.124 DEBUG:teuthology.orchestra.run.smithi133:> sudo systemctl stop ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mon.a 2023-10-09T05:46:42.298 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:42 smithi133 systemd[1]: Stopping Ceph mon.a for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T05:46:42.619 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:42 smithi133 bash[102858]: debug 2023-10-09T05:46:42.361+0000 7f5eba528700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T05:46:42.619 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:42 smithi133 bash[102858]: debug 2023-10-09T05:46:42.361+0000 7f5eba528700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2023-10-09T05:46:42.619 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:46:42 smithi133 bash[91128]: [09/Oct/2023:05:46:42] ENGINE Bus STOPPING 2023-10-09T05:46:42.620 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:46:42 smithi133 bash[91128]: [09/Oct/2023:05:46:42] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:46:42.620 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:46:42 smithi133 bash[91128]: [09/Oct/2023:05:46:42] ENGINE Bus STOPPED 2023-10-09T05:46:42.620 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:46:42 smithi133 bash[91128]: [09/Oct/2023:05:46:42] ENGINE Bus STARTING 2023-10-09T05:46:42.620 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:46:42 smithi133 bash[91128]: [09/Oct/2023:05:46:42] ENGINE Serving on http://:::9283 2023-10-09T05:46:42.620 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:46:42 smithi133 bash[91128]: [09/Oct/2023:05:46:42] ENGINE Bus STARTED 2023-10-09T05:46:43.516 INFO:journalctl@ceph.mon.a.smithi133.stdout:Oct 09 05:46:43 smithi133 bash[303574]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T05:46:43.526 DEBUG:teuthology.orchestra.run.smithi133:> sudo pkill -f 'journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mon.a.service' 2023-10-09T05:46:43.627 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:46:43.627 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-10-09T05:46:43.627 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2023-10-09T05:46:43.628 DEBUG:teuthology.orchestra.run.smithi133:> sudo systemctl stop ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mon.c 2023-10-09T05:46:43.798 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:43 smithi133 systemd[1]: Stopping Ceph mon.c for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T05:46:44.109 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:46:43 smithi133 bash[91128]: [09/Oct/2023:05:46:43] ENGINE Bus STOPPING 2023-10-09T05:46:44.110 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:46:43 smithi133 bash[91128]: [09/Oct/2023:05:46:43] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:46:44.110 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:46:43 smithi133 bash[91128]: [09/Oct/2023:05:46:43] ENGINE Bus STOPPED 2023-10-09T05:46:44.110 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:46:43 smithi133 bash[91128]: [09/Oct/2023:05:46:43] ENGINE Bus STARTING 2023-10-09T05:46:44.110 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:46:43 smithi133 bash[91128]: [09/Oct/2023:05:46:43] ENGINE Serving on http://:::9283 2023-10-09T05:46:44.110 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:46:43 smithi133 bash[91128]: [09/Oct/2023:05:46:43] ENGINE Bus STARTED 2023-10-09T05:46:44.111 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:43 smithi133 bash[105589]: debug 2023-10-09T05:46:43.801+0000 7f3d00d94700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T05:46:44.111 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:43 smithi133 bash[105589]: debug 2023-10-09T05:46:43.801+0000 7f3d00d94700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2023-10-09T05:46:44.548 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:46:44 smithi133 bash[91128]: ::ffff:172.21.15.194 - - [09/Oct/2023:05:46:44] "GET /metrics HTTP/1.1" 200 34114 "" "Prometheus/2.43.0" 2023-10-09T05:46:44.908 DEBUG:teuthology.orchestra.run.smithi133:> sudo pkill -f 'journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mon.c.service' 2023-10-09T05:46:44.919 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:44 smithi133 bash[303690]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-mon-c 2023-10-09T05:46:44.919 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:44 smithi133 bash[303764]: Error response from daemon: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-mon-c 2023-10-09T05:46:44.919 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:44 smithi133 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mon.c.service: Succeeded. 2023-10-09T05:46:44.919 INFO:journalctl@ceph.mon.c.smithi133.stdout:Oct 09 05:46:44 smithi133 systemd[1]: Stopped Ceph mon.c for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T05:46:44.979 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:46:44.979 INFO:tasks.cephadm.mon.b:Stopped mon.c 2023-10-09T05:46:44.979 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2023-10-09T05:46:44.979 DEBUG:teuthology.orchestra.run.smithi194:> sudo systemctl stop ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mon.b 2023-10-09T05:46:45.268 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:45 smithi194 systemd[1]: Stopping Ceph mon.b for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T05:46:45.269 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:45 smithi194 bash[92666]: debug 2023-10-09T05:46:45.159+0000 7f091c66f700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T05:46:45.269 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:45 smithi194 bash[92666]: debug 2023-10-09T05:46:45.159+0000 7f091c66f700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2023-10-09T05:46:46.217 INFO:journalctl@ceph.mon.b.smithi194.stdout:Oct 09 05:46:45 smithi194 bash[197497]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-mon-b 2023-10-09T05:46:46.237 DEBUG:teuthology.orchestra.run.smithi194:> sudo pkill -f 'journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mon.b.service' 2023-10-09T05:46:46.318 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:46:46.319 INFO:tasks.cephadm.mon.b:Stopped mon.b 2023-10-09T05:46:46.319 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2023-10-09T05:46:46.319 DEBUG:teuthology.orchestra.run.smithi133:> sudo systemctl stop ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mgr.y 2023-10-09T05:46:46.751 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:46:46 smithi133 systemd[1]: Stopping Ceph mgr.y for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T05:46:46.751 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:46:46 smithi133 bash[91128]: debug 2023-10-09T05:46:46.493+0000 7ff49a102700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T05:46:46.751 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:46:46 smithi133 bash[91128]: debug 2023-10-09T05:46:46.493+0000 7ff49a102700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2023-10-09T05:46:47.479 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:46:47 smithi133 bash[303806]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-mgr-y 2023-10-09T05:46:47.479 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:46:47 smithi133 bash[303868]: Error response from daemon: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-mgr-y 2023-10-09T05:46:47.480 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:46:47 smithi133 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mgr.y.service: Succeeded. 2023-10-09T05:46:47.480 INFO:journalctl@ceph.mgr.y.smithi133.stdout:Oct 09 05:46:47 smithi133 systemd[1]: Stopped Ceph mgr.y for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T05:46:47.481 DEBUG:teuthology.orchestra.run.smithi133:> sudo pkill -f 'journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mgr.y.service' 2023-10-09T05:46:47.579 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:46:47.580 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2023-10-09T05:46:47.580 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-10-09T05:46:47.580 DEBUG:teuthology.orchestra.run.smithi194:> sudo systemctl stop ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mgr.x 2023-10-09T05:46:47.845 INFO:journalctl@ceph.mgr.x.smithi194.stdout:Oct 09 05:46:47 smithi194 systemd[1]: Stopping Ceph mgr.x for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T05:46:48.763 DEBUG:teuthology.orchestra.run.smithi194:> sudo pkill -f 'journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@mgr.x.service' 2023-10-09T05:46:48.797 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:46:48.797 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-10-09T05:46:48.797 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-10-09T05:46:48.797 DEBUG:teuthology.orchestra.run.smithi133:> sudo systemctl stop ceph-855b5206-665c-11ee-8db5-212e2dc638e7@osd.0 2023-10-09T05:46:49.319 INFO:journalctl@ceph.osd.0.smithi133.stdout:Oct 09 05:46:48 smithi133 systemd[1]: Stopping Ceph osd.0 for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T05:46:49.320 INFO:journalctl@ceph.osd.0.smithi133.stdout:Oct 09 05:46:49 smithi133 bash[113504]: debug 2023-10-09T05:46:49.005+0000 7fdc24fd4700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T05:46:49.320 INFO:journalctl@ceph.osd.0.smithi133.stdout:Oct 09 05:46:49 smithi133 bash[113504]: debug 2023-10-09T05:46:49.005+0000 7fdc24fd4700 -1 osd.0 449 *** Got signal Terminated *** 2023-10-09T05:46:49.320 INFO:journalctl@ceph.osd.0.smithi133.stdout:Oct 09 05:46:49 smithi133 bash[113504]: debug 2023-10-09T05:46:49.005+0000 7fdc24fd4700 -1 osd.0 449 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:46:55.953 INFO:journalctl@ceph.osd.0.smithi133.stdout:Oct 09 05:46:55 smithi133 bash[303905]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd-0 2023-10-09T05:46:55.953 INFO:journalctl@ceph.osd.0.smithi133.stdout:Oct 09 05:46:55 smithi133 bash[303973]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd-0 2023-10-09T05:46:58.691 DEBUG:teuthology.orchestra.run.smithi133:> sudo pkill -f 'journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@osd.0.service' 2023-10-09T05:46:58.820 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:46:58.820 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-10-09T05:46:58.821 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-10-09T05:46:58.821 DEBUG:teuthology.orchestra.run.smithi133:> sudo systemctl stop ceph-855b5206-665c-11ee-8db5-212e2dc638e7@osd.1 2023-10-09T05:46:59.012 INFO:journalctl@ceph.osd.1.smithi133.stdout:Oct 09 05:46:58 smithi133 systemd[1]: Stopping Ceph osd.1 for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T05:46:59.298 INFO:journalctl@ceph.osd.1.smithi133.stdout:Oct 09 05:46:59 smithi133 bash[120074]: debug 2023-10-09T05:46:59.005+0000 7f9d3b595700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T05:46:59.298 INFO:journalctl@ceph.osd.1.smithi133.stdout:Oct 09 05:46:59 smithi133 bash[120074]: debug 2023-10-09T05:46:59.005+0000 7f9d3b595700 -1 osd.1 449 *** Got signal Terminated *** 2023-10-09T05:46:59.298 INFO:journalctl@ceph.osd.1.smithi133.stdout:Oct 09 05:46:59 smithi133 bash[120074]: debug 2023-10-09T05:46:59.005+0000 7f9d3b595700 -1 osd.1 449 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:47:05.547 INFO:journalctl@ceph.osd.1.smithi133.stdout:Oct 09 05:47:05 smithi133 bash[304111]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd-1 2023-10-09T05:47:06.048 INFO:journalctl@ceph.osd.1.smithi133.stdout:Oct 09 05:47:05 smithi133 bash[304175]: Error response from daemon: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd-1 2023-10-09T05:47:07.732 DEBUG:teuthology.orchestra.run.smithi133:> sudo pkill -f 'journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@osd.1.service' 2023-10-09T05:47:07.790 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:47:07.791 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-10-09T05:47:07.791 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-10-09T05:47:07.791 DEBUG:teuthology.orchestra.run.smithi133:> sudo systemctl stop ceph-855b5206-665c-11ee-8db5-212e2dc638e7@osd.2 2023-10-09T05:47:08.006 INFO:journalctl@ceph.osd.2.smithi133.stdout:Oct 09 05:47:07 smithi133 systemd[1]: Stopping Ceph osd.2 for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T05:47:08.298 INFO:journalctl@ceph.osd.2.smithi133.stdout:Oct 09 05:47:08 smithi133 bash[128609]: debug 2023-10-09T05:47:08.005+0000 7f868ea46700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T05:47:08.298 INFO:journalctl@ceph.osd.2.smithi133.stdout:Oct 09 05:47:08 smithi133 bash[128609]: debug 2023-10-09T05:47:08.005+0000 7f868ea46700 -1 osd.2 449 *** Got signal Terminated *** 2023-10-09T05:47:08.298 INFO:journalctl@ceph.osd.2.smithi133.stdout:Oct 09 05:47:08 smithi133 bash[128609]: debug 2023-10-09T05:47:08.005+0000 7f868ea46700 -1 osd.2 449 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:47:14.798 INFO:journalctl@ceph.osd.2.smithi133.stdout:Oct 09 05:47:14 smithi133 bash[304316]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd-2 2023-10-09T05:47:14.798 INFO:journalctl@ceph.osd.2.smithi133.stdout:Oct 09 05:47:14 smithi133 bash[304385]: Error response from daemon: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd-2 2023-10-09T05:47:15.386 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:14 smithi194 bash[43413]: debug 2023-10-09T05:47:14.914+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.689236+0000 front 2023-10-09T05:46:53.689318+0000 (oldest deadline 2023-10-09T05:47:14.789007+0000) 2023-10-09T05:47:15.386 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:15 smithi194 bash[51144]: debug 2023-10-09T05:47:15.026+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:16.135 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:15 smithi194 bash[43413]: debug 2023-10-09T05:47:15.874+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.689236+0000 front 2023-10-09T05:46:53.689318+0000 (oldest deadline 2023-10-09T05:47:14.789007+0000) 2023-10-09T05:47:16.136 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:16 smithi194 bash[51144]: debug 2023-10-09T05:47:16.070+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:16.965 DEBUG:teuthology.orchestra.run.smithi133:> sudo pkill -f 'journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@osd.2.service' 2023-10-09T05:47:16.997 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:47:16.997 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-10-09T05:47:16.998 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-10-09T05:47:16.998 DEBUG:teuthology.orchestra.run.smithi133:> sudo systemctl stop ceph-855b5206-665c-11ee-8db5-212e2dc638e7@osd.3 2023-10-09T05:47:17.135 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:16 smithi194 bash[43413]: debug 2023-10-09T05:47:16.862+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.689236+0000 front 2023-10-09T05:46:53.689318+0000 (oldest deadline 2023-10-09T05:47:14.789007+0000) 2023-10-09T05:47:17.136 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:17 smithi194 bash[51144]: debug 2023-10-09T05:47:17.110+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:17.136 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:16 smithi194 bash[59103]: debug 2023-10-09T05:47:16.966+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:17.230 INFO:journalctl@ceph.osd.3.smithi133.stdout:Oct 09 05:47:17 smithi133 systemd[1]: Stopping Ceph osd.3 for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T05:47:17.548 INFO:journalctl@ceph.osd.3.smithi133.stdout:Oct 09 05:47:17 smithi133 bash[135470]: debug 2023-10-09T05:47:17.229+0000 7fc35789e700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T05:47:17.548 INFO:journalctl@ceph.osd.3.smithi133.stdout:Oct 09 05:47:17 smithi133 bash[135470]: debug 2023-10-09T05:47:17.229+0000 7fc35789e700 -1 osd.3 449 *** Got signal Terminated *** 2023-10-09T05:47:17.548 INFO:journalctl@ceph.osd.3.smithi133.stdout:Oct 09 05:47:17 smithi133 bash[135470]: debug 2023-10-09T05:47:17.229+0000 7fc35789e700 -1 osd.3 449 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:47:18.135 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:17 smithi194 bash[43413]: debug 2023-10-09T05:47:17.854+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.689236+0000 front 2023-10-09T05:46:53.689318+0000 (oldest deadline 2023-10-09T05:47:14.789007+0000) 2023-10-09T05:47:18.136 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:18 smithi194 bash[59103]: debug 2023-10-09T05:47:17.998+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:18.635 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:18 smithi194 bash[51144]: debug 2023-10-09T05:47:18.150+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:19.135 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:18 smithi194 bash[43413]: debug 2023-10-09T05:47:18.870+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.689236+0000 front 2023-10-09T05:46:53.689318+0000 (oldest deadline 2023-10-09T05:47:14.789007+0000) 2023-10-09T05:47:19.136 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:19 smithi194 bash[59103]: debug 2023-10-09T05:47:19.014+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:19.635 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:19 smithi194 bash[51144]: debug 2023-10-09T05:47:19.186+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:20.048 INFO:journalctl@ceph.osd.3.smithi133.stdout:Oct 09 05:47:19 smithi133 bash[135470]: debug 2023-10-09T05:47:19.653+0000 7fc351e90700 -1 osd.3 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.503853+0000 front 2023-10-09T05:46:53.504078+0000 (oldest deadline 2023-10-09T05:47:19.305137+0000) 2023-10-09T05:47:20.197 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:19 smithi194 bash[59103]: debug 2023-10-09T05:47:19.990+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:20.198 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:19 smithi194 bash[43413]: debug 2023-10-09T05:47:19.898+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.689236+0000 front 2023-10-09T05:46:53.689318+0000 (oldest deadline 2023-10-09T05:47:14.789007+0000) 2023-10-09T05:47:20.198 INFO:journalctl@ceph.osd.4.smithi194.stdout:Oct 09 05:47:19 smithi194 bash[35552]: debug 2023-10-09T05:47:19.910+0000 7fc6ca530700 -1 osd.4 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.403702+0000 front 2023-10-09T05:46:53.403849+0000 (oldest deadline 2023-10-09T05:47:19.303278+0000) 2023-10-09T05:47:20.635 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:20 smithi194 bash[51144]: debug 2023-10-09T05:47:20.194+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:21.048 INFO:journalctl@ceph.osd.3.smithi133.stdout:Oct 09 05:47:20 smithi133 bash[135470]: debug 2023-10-09T05:47:20.621+0000 7fc351e90700 -1 osd.3 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.503853+0000 front 2023-10-09T05:46:53.504078+0000 (oldest deadline 2023-10-09T05:47:19.305137+0000) 2023-10-09T05:47:21.385 INFO:journalctl@ceph.osd.4.smithi194.stdout:Oct 09 05:47:20 smithi194 bash[35552]: debug 2023-10-09T05:47:20.926+0000 7fc6ca530700 -1 osd.4 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.403702+0000 front 2023-10-09T05:46:53.403849+0000 (oldest deadline 2023-10-09T05:47:19.303278+0000) 2023-10-09T05:47:21.386 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:21 smithi194 bash[51144]: debug 2023-10-09T05:47:21.154+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:21.386 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:20 smithi194 bash[59103]: debug 2023-10-09T05:47:20.950+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:21.386 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:20 smithi194 bash[43413]: debug 2023-10-09T05:47:20.906+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.689236+0000 front 2023-10-09T05:46:53.689318+0000 (oldest deadline 2023-10-09T05:47:14.789007+0000) 2023-10-09T05:47:22.048 INFO:journalctl@ceph.osd.3.smithi133.stdout:Oct 09 05:47:21 smithi133 bash[135470]: debug 2023-10-09T05:47:21.597+0000 7fc351e90700 -1 osd.3 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.503853+0000 front 2023-10-09T05:46:53.504078+0000 (oldest deadline 2023-10-09T05:47:19.305137+0000) 2023-10-09T05:47:22.169 INFO:journalctl@ceph.osd.4.smithi194.stdout:Oct 09 05:47:21 smithi194 bash[35552]: debug 2023-10-09T05:47:21.902+0000 7fc6ca530700 -1 osd.4 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.403702+0000 front 2023-10-09T05:46:53.403849+0000 (oldest deadline 2023-10-09T05:47:19.303278+0000) 2023-10-09T05:47:22.170 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:21 smithi194 bash[59103]: debug 2023-10-09T05:47:21.970+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:22.170 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:21 smithi194 bash[43413]: debug 2023-10-09T05:47:21.902+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.689236+0000 front 2023-10-09T05:46:53.689318+0000 (oldest deadline 2023-10-09T05:47:14.789007+0000) 2023-10-09T05:47:22.635 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:22 smithi194 bash[51144]: debug 2023-10-09T05:47:22.162+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:23.385 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:23 smithi194 bash[51144]: debug 2023-10-09T05:47:23.126+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:23.386 INFO:journalctl@ceph.osd.4.smithi194.stdout:Oct 09 05:47:22 smithi194 bash[35552]: debug 2023-10-09T05:47:22.894+0000 7fc6ca530700 -1 osd.4 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.403702+0000 front 2023-10-09T05:46:53.403849+0000 (oldest deadline 2023-10-09T05:47:19.303278+0000) 2023-10-09T05:47:23.386 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:22 smithi194 bash[43413]: debug 2023-10-09T05:47:22.890+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.689236+0000 front 2023-10-09T05:46:53.689318+0000 (oldest deadline 2023-10-09T05:47:14.789007+0000) 2023-10-09T05:47:23.387 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:23 smithi194 bash[59103]: debug 2023-10-09T05:47:23.010+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:24.095 INFO:journalctl@ceph.osd.3.smithi133.stdout:Oct 09 05:47:23 smithi133 bash[304521]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd-3 2023-10-09T05:47:24.095 INFO:journalctl@ceph.osd.3.smithi133.stdout:Oct 09 05:47:24 smithi133 bash[304587]: Error response from daemon: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd-3 2023-10-09T05:47:24.135 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:23 smithi194 bash[59103]: debug 2023-10-09T05:47:23.986+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:24.135 INFO:journalctl@ceph.osd.4.smithi194.stdout:Oct 09 05:47:23 smithi194 bash[35552]: debug 2023-10-09T05:47:23.862+0000 7fc6ca530700 -1 osd.4 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.403702+0000 front 2023-10-09T05:46:53.403849+0000 (oldest deadline 2023-10-09T05:47:19.303278+0000) 2023-10-09T05:47:24.136 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:23 smithi194 bash[43413]: debug 2023-10-09T05:47:23.862+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.689236+0000 front 2023-10-09T05:46:53.689318+0000 (oldest deadline 2023-10-09T05:47:14.789007+0000) 2023-10-09T05:47:24.136 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:24 smithi194 bash[51144]: debug 2023-10-09T05:47:24.098+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:25.135 INFO:journalctl@ceph.osd.4.smithi194.stdout:Oct 09 05:47:24 smithi194 bash[35552]: debug 2023-10-09T05:47:24.898+0000 7fc6ca530700 -1 osd.4 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.403702+0000 front 2023-10-09T05:46:53.403849+0000 (oldest deadline 2023-10-09T05:47:19.303278+0000) 2023-10-09T05:47:25.136 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:24 smithi194 bash[43413]: debug 2023-10-09T05:47:24.830+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.689236+0000 front 2023-10-09T05:46:53.689318+0000 (oldest deadline 2023-10-09T05:47:14.789007+0000) 2023-10-09T05:47:25.137 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:25 smithi194 bash[51144]: debug 2023-10-09T05:47:25.058+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:25.137 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:24 smithi194 bash[59103]: debug 2023-10-09T05:47:24.942+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:26.135 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:25 smithi194 bash[59103]: debug 2023-10-09T05:47:25.886+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:26.136 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:25 smithi194 bash[43413]: debug 2023-10-09T05:47:25.850+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.689236+0000 front 2023-10-09T05:46:53.689318+0000 (oldest deadline 2023-10-09T05:47:14.789007+0000) 2023-10-09T05:47:26.136 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:26 smithi194 bash[51144]: debug 2023-10-09T05:47:26.022+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:26.136 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:26 smithi194 bash[51144]: debug 2023-10-09T05:47:26.022+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:00.610871+0000 front 2023-10-09T05:47:00.610930+0000 (oldest deadline 2023-10-09T05:47:25.310816+0000) 2023-10-09T05:47:26.137 INFO:journalctl@ceph.osd.4.smithi194.stdout:Oct 09 05:47:25 smithi194 bash[35552]: debug 2023-10-09T05:47:25.874+0000 7fc6ca530700 -1 osd.4 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.403702+0000 front 2023-10-09T05:46:53.403849+0000 (oldest deadline 2023-10-09T05:47:19.303278+0000) 2023-10-09T05:47:26.137 INFO:journalctl@ceph.osd.4.smithi194.stdout:Oct 09 05:47:25 smithi194 bash[35552]: debug 2023-10-09T05:47:25.874+0000 7fc6ca530700 -1 osd.4 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:00.404872+0000 front 2023-10-09T05:47:00.404633+0000 (oldest deadline 2023-10-09T05:47:25.104446+0000) 2023-10-09T05:47:26.321 DEBUG:teuthology.orchestra.run.smithi133:> sudo pkill -f 'journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@osd.3.service' 2023-10-09T05:47:26.353 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:47:26.353 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-10-09T05:47:26.353 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-10-09T05:47:26.354 DEBUG:teuthology.orchestra.run.smithi194:> sudo systemctl stop ceph-855b5206-665c-11ee-8db5-212e2dc638e7@osd.4 2023-10-09T05:47:26.635 INFO:journalctl@ceph.osd.4.smithi194.stdout:Oct 09 05:47:26 smithi194 systemd[1]: Stopping Ceph osd.4 for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T05:47:26.636 INFO:journalctl@ceph.osd.4.smithi194.stdout:Oct 09 05:47:26 smithi194 bash[35552]: debug 2023-10-09T05:47:26.606+0000 7fc6ccf30700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T05:47:26.636 INFO:journalctl@ceph.osd.4.smithi194.stdout:Oct 09 05:47:26 smithi194 bash[35552]: debug 2023-10-09T05:47:26.606+0000 7fc6ccf30700 -1 osd.4 449 *** Got signal Terminated *** 2023-10-09T05:47:26.636 INFO:journalctl@ceph.osd.4.smithi194.stdout:Oct 09 05:47:26 smithi194 bash[35552]: debug 2023-10-09T05:47:26.606+0000 7fc6ccf30700 -1 osd.4 449 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:47:27.135 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:27 smithi194 bash[51144]: debug 2023-10-09T05:47:27.014+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:27.136 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:27 smithi194 bash[51144]: debug 2023-10-09T05:47:27.014+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:00.610871+0000 front 2023-10-09T05:47:00.610930+0000 (oldest deadline 2023-10-09T05:47:25.310816+0000) 2023-10-09T05:47:27.136 INFO:journalctl@ceph.osd.4.smithi194.stdout:Oct 09 05:47:26 smithi194 bash[35552]: debug 2023-10-09T05:47:26.906+0000 7fc6ca530700 -1 osd.4 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.403702+0000 front 2023-10-09T05:46:53.403849+0000 (oldest deadline 2023-10-09T05:47:19.303278+0000) 2023-10-09T05:47:27.136 INFO:journalctl@ceph.osd.4.smithi194.stdout:Oct 09 05:47:26 smithi194 bash[35552]: debug 2023-10-09T05:47:26.906+0000 7fc6ca530700 -1 osd.4 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:00.404872+0000 front 2023-10-09T05:47:00.404633+0000 (oldest deadline 2023-10-09T05:47:25.104446+0000) 2023-10-09T05:47:27.136 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:26 smithi194 bash[43413]: debug 2023-10-09T05:47:26.862+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.689236+0000 front 2023-10-09T05:46:53.689318+0000 (oldest deadline 2023-10-09T05:47:14.789007+0000) 2023-10-09T05:47:27.137 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:26 smithi194 bash[59103]: debug 2023-10-09T05:47:26.842+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:27.137 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:26 smithi194 bash[59103]: debug 2023-10-09T05:47:26.842+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:28.136 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:27 smithi194 bash[51144]: debug 2023-10-09T05:47:27.994+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:28.136 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:27 smithi194 bash[51144]: debug 2023-10-09T05:47:27.994+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:00.610871+0000 front 2023-10-09T05:47:00.610930+0000 (oldest deadline 2023-10-09T05:47:25.310816+0000) 2023-10-09T05:47:28.136 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:27 smithi194 bash[59103]: debug 2023-10-09T05:47:27.826+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:28.137 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:27 smithi194 bash[59103]: debug 2023-10-09T05:47:27.826+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:28.137 INFO:journalctl@ceph.osd.4.smithi194.stdout:Oct 09 05:47:27 smithi194 bash[35552]: debug 2023-10-09T05:47:27.882+0000 7fc6ca530700 -1 osd.4 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.403702+0000 front 2023-10-09T05:46:53.403849+0000 (oldest deadline 2023-10-09T05:47:19.303278+0000) 2023-10-09T05:47:28.137 INFO:journalctl@ceph.osd.4.smithi194.stdout:Oct 09 05:47:27 smithi194 bash[35552]: debug 2023-10-09T05:47:27.882+0000 7fc6ca530700 -1 osd.4 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:00.404872+0000 front 2023-10-09T05:47:00.404633+0000 (oldest deadline 2023-10-09T05:47:25.104446+0000) 2023-10-09T05:47:28.138 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:27 smithi194 bash[43413]: debug 2023-10-09T05:47:27.898+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.689236+0000 front 2023-10-09T05:46:53.689318+0000 (oldest deadline 2023-10-09T05:47:14.789007+0000) 2023-10-09T05:47:28.138 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:27 smithi194 bash[43413]: debug 2023-10-09T05:47:27.898+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:03.591425+0000 front 2023-10-09T05:47:03.591696+0000 (oldest deadline 2023-10-09T05:47:27.691200+0000) 2023-10-09T05:47:29.135 INFO:journalctl@ceph.osd.4.smithi194.stdout:Oct 09 05:47:28 smithi194 bash[35552]: debug 2023-10-09T05:47:28.862+0000 7fc6ca530700 -1 osd.4 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.403702+0000 front 2023-10-09T05:46:53.403849+0000 (oldest deadline 2023-10-09T05:47:19.303278+0000) 2023-10-09T05:47:29.136 INFO:journalctl@ceph.osd.4.smithi194.stdout:Oct 09 05:47:28 smithi194 bash[35552]: debug 2023-10-09T05:47:28.862+0000 7fc6ca530700 -1 osd.4 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:00.404872+0000 front 2023-10-09T05:47:00.404633+0000 (oldest deadline 2023-10-09T05:47:25.104446+0000) 2023-10-09T05:47:29.136 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:28 smithi194 bash[43413]: debug 2023-10-09T05:47:28.882+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.689236+0000 front 2023-10-09T05:46:53.689318+0000 (oldest deadline 2023-10-09T05:47:14.789007+0000) 2023-10-09T05:47:29.136 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:28 smithi194 bash[43413]: debug 2023-10-09T05:47:28.882+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:03.591425+0000 front 2023-10-09T05:47:03.591696+0000 (oldest deadline 2023-10-09T05:47:27.691200+0000) 2023-10-09T05:47:29.137 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:29 smithi194 bash[51144]: debug 2023-10-09T05:47:28.994+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:29.137 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:29 smithi194 bash[51144]: debug 2023-10-09T05:47:28.994+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:00.610871+0000 front 2023-10-09T05:47:00.610930+0000 (oldest deadline 2023-10-09T05:47:25.310816+0000) 2023-10-09T05:47:29.137 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:28 smithi194 bash[59103]: debug 2023-10-09T05:47:28.870+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:29.138 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:28 smithi194 bash[59103]: debug 2023-10-09T05:47:28.870+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:29.138 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 05:47:28 smithi194 bash[81885]: ts=2023-10-09T05:47:28.979Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.133:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.133:8765: connect: connection refused" 2023-10-09T05:47:29.138 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 05:47:28 smithi194 bash[81885]: ts=2023-10-09T05:47:28.981Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.133:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.133:8765: connect: connection refused" 2023-10-09T05:47:29.138 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 05:47:28 smithi194 bash[81885]: ts=2023-10-09T05:47:28.986Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.133:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.133:8765: connect: connection refused" 2023-10-09T05:47:29.138 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 05:47:28 smithi194 bash[81885]: ts=2023-10-09T05:47:28.986Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.133:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.133:8765: connect: connection refused" 2023-10-09T05:47:30.136 INFO:journalctl@ceph.osd.4.smithi194.stdout:Oct 09 05:47:29 smithi194 bash[35552]: debug 2023-10-09T05:47:29.838+0000 7fc6ca530700 -1 osd.4 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.403702+0000 front 2023-10-09T05:46:53.403849+0000 (oldest deadline 2023-10-09T05:47:19.303278+0000) 2023-10-09T05:47:30.136 INFO:journalctl@ceph.osd.4.smithi194.stdout:Oct 09 05:47:29 smithi194 bash[35552]: debug 2023-10-09T05:47:29.838+0000 7fc6ca530700 -1 osd.4 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:00.404872+0000 front 2023-10-09T05:47:00.404633+0000 (oldest deadline 2023-10-09T05:47:25.104446+0000) 2023-10-09T05:47:30.136 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:29 smithi194 bash[43413]: debug 2023-10-09T05:47:29.862+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.689236+0000 front 2023-10-09T05:46:53.689318+0000 (oldest deadline 2023-10-09T05:47:14.789007+0000) 2023-10-09T05:47:30.136 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:29 smithi194 bash[43413]: debug 2023-10-09T05:47:29.862+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:03.591425+0000 front 2023-10-09T05:47:03.591696+0000 (oldest deadline 2023-10-09T05:47:27.691200+0000) 2023-10-09T05:47:30.137 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:30 smithi194 bash[51144]: debug 2023-10-09T05:47:30.046+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:30.137 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:30 smithi194 bash[51144]: debug 2023-10-09T05:47:30.046+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:00.610871+0000 front 2023-10-09T05:47:00.610930+0000 (oldest deadline 2023-10-09T05:47:25.310816+0000) 2023-10-09T05:47:30.137 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:29 smithi194 bash[59103]: debug 2023-10-09T05:47:29.862+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:30.137 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:29 smithi194 bash[59103]: debug 2023-10-09T05:47:29.862+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:31.100 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:30 smithi194 bash[59103]: debug 2023-10-09T05:47:30.834+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:31.101 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:30 smithi194 bash[59103]: debug 2023-10-09T05:47:30.834+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:31.101 INFO:journalctl@ceph.osd.4.smithi194.stdout:Oct 09 05:47:30 smithi194 bash[35552]: debug 2023-10-09T05:47:30.886+0000 7fc6ca530700 -1 osd.4 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.403702+0000 front 2023-10-09T05:46:53.403849+0000 (oldest deadline 2023-10-09T05:47:19.303278+0000) 2023-10-09T05:47:31.101 INFO:journalctl@ceph.osd.4.smithi194.stdout:Oct 09 05:47:30 smithi194 bash[35552]: debug 2023-10-09T05:47:30.886+0000 7fc6ca530700 -1 osd.4 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:00.404872+0000 front 2023-10-09T05:47:00.404633+0000 (oldest deadline 2023-10-09T05:47:25.104446+0000) 2023-10-09T05:47:31.101 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:30 smithi194 bash[43413]: debug 2023-10-09T05:47:30.834+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.689236+0000 front 2023-10-09T05:46:53.689318+0000 (oldest deadline 2023-10-09T05:47:14.789007+0000) 2023-10-09T05:47:31.102 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:30 smithi194 bash[43413]: debug 2023-10-09T05:47:30.834+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:03.591425+0000 front 2023-10-09T05:47:03.591696+0000 (oldest deadline 2023-10-09T05:47:27.691200+0000) 2023-10-09T05:47:31.385 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:31 smithi194 bash[51144]: debug 2023-10-09T05:47:31.094+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:31.386 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:31 smithi194 bash[51144]: debug 2023-10-09T05:47:31.094+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:00.610871+0000 front 2023-10-09T05:47:00.610930+0000 (oldest deadline 2023-10-09T05:47:25.310816+0000) 2023-10-09T05:47:32.135 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:31 smithi194 bash[43413]: debug 2023-10-09T05:47:31.814+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.689236+0000 front 2023-10-09T05:46:53.689318+0000 (oldest deadline 2023-10-09T05:47:14.789007+0000) 2023-10-09T05:47:32.136 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:31 smithi194 bash[43413]: debug 2023-10-09T05:47:31.814+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:03.591425+0000 front 2023-10-09T05:47:03.591696+0000 (oldest deadline 2023-10-09T05:47:27.691200+0000) 2023-10-09T05:47:32.136 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:31 smithi194 bash[59103]: debug 2023-10-09T05:47:31.822+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:32.136 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:31 smithi194 bash[59103]: debug 2023-10-09T05:47:31.822+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:32.527 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:32 smithi194 bash[51144]: debug 2023-10-09T05:47:32.134+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:32.527 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:32 smithi194 bash[51144]: debug 2023-10-09T05:47:32.134+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:00.610871+0000 front 2023-10-09T05:47:00.610930+0000 (oldest deadline 2023-10-09T05:47:25.310816+0000) 2023-10-09T05:47:33.117 INFO:journalctl@ceph.osd.4.smithi194.stdout:Oct 09 05:47:32 smithi194 bash[197703]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd-4 2023-10-09T05:47:33.118 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:32 smithi194 bash[59103]: debug 2023-10-09T05:47:32.774+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:33.118 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:32 smithi194 bash[59103]: debug 2023-10-09T05:47:32.774+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:33.118 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:32 smithi194 bash[43413]: debug 2023-10-09T05:47:32.850+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.689236+0000 front 2023-10-09T05:46:53.689318+0000 (oldest deadline 2023-10-09T05:47:14.789007+0000) 2023-10-09T05:47:33.119 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:32 smithi194 bash[43413]: debug 2023-10-09T05:47:32.850+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:03.591425+0000 front 2023-10-09T05:47:03.591696+0000 (oldest deadline 2023-10-09T05:47:27.691200+0000) 2023-10-09T05:47:33.385 INFO:journalctl@ceph.osd.4.smithi194.stdout:Oct 09 05:47:33 smithi194 bash[197768]: Error response from daemon: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd-4 2023-10-09T05:47:33.386 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:33 smithi194 bash[51144]: debug 2023-10-09T05:47:33.114+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:33.386 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:33 smithi194 bash[51144]: debug 2023-10-09T05:47:33.114+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:00.610871+0000 front 2023-10-09T05:47:00.610930+0000 (oldest deadline 2023-10-09T05:47:25.310816+0000) 2023-10-09T05:47:34.042 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:33 smithi194 bash[59103]: debug 2023-10-09T05:47:33.770+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:34.042 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:33 smithi194 bash[59103]: debug 2023-10-09T05:47:33.770+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:34.042 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:33 smithi194 bash[43413]: debug 2023-10-09T05:47:33.890+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.689236+0000 front 2023-10-09T05:46:53.689318+0000 (oldest deadline 2023-10-09T05:47:14.789007+0000) 2023-10-09T05:47:34.043 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:33 smithi194 bash[43413]: debug 2023-10-09T05:47:33.890+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:03.591425+0000 front 2023-10-09T05:47:03.591696+0000 (oldest deadline 2023-10-09T05:47:27.691200+0000) 2023-10-09T05:47:34.385 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:34 smithi194 bash[51144]: debug 2023-10-09T05:47:34.086+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:34.386 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:34 smithi194 bash[51144]: debug 2023-10-09T05:47:34.086+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:00.610871+0000 front 2023-10-09T05:47:00.610930+0000 (oldest deadline 2023-10-09T05:47:25.310816+0000) 2023-10-09T05:47:35.040 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:34 smithi194 bash[43413]: debug 2023-10-09T05:47:34.930+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.689236+0000 front 2023-10-09T05:46:53.689318+0000 (oldest deadline 2023-10-09T05:47:14.789007+0000) 2023-10-09T05:47:35.041 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:34 smithi194 bash[43413]: debug 2023-10-09T05:47:34.930+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:03.591425+0000 front 2023-10-09T05:47:03.591696+0000 (oldest deadline 2023-10-09T05:47:27.691200+0000) 2023-10-09T05:47:35.041 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:34 smithi194 bash[59103]: debug 2023-10-09T05:47:34.726+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:35.041 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:34 smithi194 bash[59103]: debug 2023-10-09T05:47:34.726+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:35.385 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:35 smithi194 bash[51144]: debug 2023-10-09T05:47:35.042+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:35.386 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:35 smithi194 bash[51144]: debug 2023-10-09T05:47:35.042+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:00.610871+0000 front 2023-10-09T05:47:00.610930+0000 (oldest deadline 2023-10-09T05:47:25.310816+0000) 2023-10-09T05:47:35.386 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:35 smithi194 bash[51144]: debug 2023-10-09T05:47:35.042+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:10.513074+0000 front 2023-10-09T05:47:10.513015+0000 (oldest deadline 2023-10-09T05:47:34.612791+0000) 2023-10-09T05:47:35.708 DEBUG:teuthology.orchestra.run.smithi194:> sudo pkill -f 'journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@osd.4.service' 2023-10-09T05:47:35.743 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:47:35.743 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-10-09T05:47:35.743 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-10-09T05:47:35.744 DEBUG:teuthology.orchestra.run.smithi194:> sudo systemctl stop ceph-855b5206-665c-11ee-8db5-212e2dc638e7@osd.5 2023-10-09T05:47:35.963 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:35 smithi194 systemd[1]: Stopping Ceph osd.5 for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T05:47:35.964 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:35 smithi194 bash[59103]: debug 2023-10-09T05:47:35.698+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:35.964 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:35 smithi194 bash[59103]: debug 2023-10-09T05:47:35.698+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:35.964 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:35 smithi194 bash[59103]: debug 2023-10-09T05:47:35.698+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:09.447932+0000 front 2023-10-09T05:47:09.447859+0000 (oldest deadline 2023-10-09T05:47:35.248483+0000) 2023-10-09T05:47:36.385 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:35 smithi194 bash[43413]: debug 2023-10-09T05:47:35.962+0000 7f9b85c61700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T05:47:36.386 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:35 smithi194 bash[43413]: debug 2023-10-09T05:47:35.962+0000 7f9b85c61700 -1 osd.5 449 *** Got signal Terminated *** 2023-10-09T05:47:36.386 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:35 smithi194 bash[43413]: debug 2023-10-09T05:47:35.962+0000 7f9b85c61700 -1 osd.5 449 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:47:36.386 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:35 smithi194 bash[43413]: debug 2023-10-09T05:47:35.974+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.689236+0000 front 2023-10-09T05:46:53.689318+0000 (oldest deadline 2023-10-09T05:47:14.789007+0000) 2023-10-09T05:47:36.386 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:35 smithi194 bash[43413]: debug 2023-10-09T05:47:35.974+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:03.591425+0000 front 2023-10-09T05:47:03.591696+0000 (oldest deadline 2023-10-09T05:47:27.691200+0000) 2023-10-09T05:47:36.386 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:36 smithi194 bash[51144]: debug 2023-10-09T05:47:36.022+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:36.387 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:36 smithi194 bash[51144]: debug 2023-10-09T05:47:36.022+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:00.610871+0000 front 2023-10-09T05:47:00.610930+0000 (oldest deadline 2023-10-09T05:47:25.310816+0000) 2023-10-09T05:47:36.387 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:36 smithi194 bash[51144]: debug 2023-10-09T05:47:36.022+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:10.513074+0000 front 2023-10-09T05:47:10.513015+0000 (oldest deadline 2023-10-09T05:47:34.612791+0000) 2023-10-09T05:47:36.974 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:36 smithi194 bash[59103]: debug 2023-10-09T05:47:36.718+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:36.975 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:36 smithi194 bash[59103]: debug 2023-10-09T05:47:36.718+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:36.975 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:36 smithi194 bash[59103]: debug 2023-10-09T05:47:36.718+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:09.447932+0000 front 2023-10-09T05:47:09.447859+0000 (oldest deadline 2023-10-09T05:47:35.248483+0000) 2023-10-09T05:47:37.385 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:37 smithi194 bash[43413]: debug 2023-10-09T05:47:37.010+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.689236+0000 front 2023-10-09T05:46:53.689318+0000 (oldest deadline 2023-10-09T05:47:14.789007+0000) 2023-10-09T05:47:37.386 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:37 smithi194 bash[43413]: debug 2023-10-09T05:47:37.010+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:03.591425+0000 front 2023-10-09T05:47:03.591696+0000 (oldest deadline 2023-10-09T05:47:27.691200+0000) 2023-10-09T05:47:37.386 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:36 smithi194 bash[51144]: debug 2023-10-09T05:47:36.970+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:37.386 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:36 smithi194 bash[51144]: debug 2023-10-09T05:47:36.970+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:00.610871+0000 front 2023-10-09T05:47:00.610930+0000 (oldest deadline 2023-10-09T05:47:25.310816+0000) 2023-10-09T05:47:37.386 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:36 smithi194 bash[51144]: debug 2023-10-09T05:47:36.970+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:10.513074+0000 front 2023-10-09T05:47:10.513015+0000 (oldest deadline 2023-10-09T05:47:34.612791+0000) 2023-10-09T05:47:37.967 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:37 smithi194 bash[59103]: debug 2023-10-09T05:47:37.678+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:37.967 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:37 smithi194 bash[59103]: debug 2023-10-09T05:47:37.678+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:37.967 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:37 smithi194 bash[59103]: debug 2023-10-09T05:47:37.678+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:09.447932+0000 front 2023-10-09T05:47:09.447859+0000 (oldest deadline 2023-10-09T05:47:35.248483+0000) 2023-10-09T05:47:38.385 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:37 smithi194 bash[43413]: debug 2023-10-09T05:47:37.974+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.689236+0000 front 2023-10-09T05:46:53.689318+0000 (oldest deadline 2023-10-09T05:47:14.789007+0000) 2023-10-09T05:47:38.386 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:37 smithi194 bash[43413]: debug 2023-10-09T05:47:37.974+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:03.591425+0000 front 2023-10-09T05:47:03.591696+0000 (oldest deadline 2023-10-09T05:47:27.691200+0000) 2023-10-09T05:47:38.386 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:37 smithi194 bash[51144]: debug 2023-10-09T05:47:37.966+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:38.386 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:37 smithi194 bash[51144]: debug 2023-10-09T05:47:37.966+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:00.610871+0000 front 2023-10-09T05:47:00.610930+0000 (oldest deadline 2023-10-09T05:47:25.310816+0000) 2023-10-09T05:47:38.386 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:37 smithi194 bash[51144]: debug 2023-10-09T05:47:37.966+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:10.513074+0000 front 2023-10-09T05:47:10.513015+0000 (oldest deadline 2023-10-09T05:47:34.612791+0000) 2023-10-09T05:47:39.019 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:38 smithi194 bash[51144]: debug 2023-10-09T05:47:38.934+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:39.020 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:38 smithi194 bash[51144]: debug 2023-10-09T05:47:38.934+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:00.610871+0000 front 2023-10-09T05:47:00.610930+0000 (oldest deadline 2023-10-09T05:47:25.310816+0000) 2023-10-09T05:47:39.020 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:38 smithi194 bash[51144]: debug 2023-10-09T05:47:38.934+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:10.513074+0000 front 2023-10-09T05:47:10.513015+0000 (oldest deadline 2023-10-09T05:47:34.612791+0000) 2023-10-09T05:47:39.020 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:38 smithi194 bash[59103]: debug 2023-10-09T05:47:38.706+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:39.020 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:38 smithi194 bash[59103]: debug 2023-10-09T05:47:38.706+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:39.020 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:38 smithi194 bash[59103]: debug 2023-10-09T05:47:38.706+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:09.447932+0000 front 2023-10-09T05:47:09.447859+0000 (oldest deadline 2023-10-09T05:47:35.248483+0000) 2023-10-09T05:47:39.385 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:39 smithi194 bash[43413]: debug 2023-10-09T05:47:39.018+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.689236+0000 front 2023-10-09T05:46:53.689318+0000 (oldest deadline 2023-10-09T05:47:14.789007+0000) 2023-10-09T05:47:39.385 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:39 smithi194 bash[43413]: debug 2023-10-09T05:47:39.018+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:03.591425+0000 front 2023-10-09T05:47:03.591696+0000 (oldest deadline 2023-10-09T05:47:27.691200+0000) 2023-10-09T05:47:39.926 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:39 smithi194 bash[59103]: debug 2023-10-09T05:47:39.670+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:39.926 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:39 smithi194 bash[59103]: debug 2023-10-09T05:47:39.670+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:39.926 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:39 smithi194 bash[59103]: debug 2023-10-09T05:47:39.670+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:09.447932+0000 front 2023-10-09T05:47:09.447859+0000 (oldest deadline 2023-10-09T05:47:35.248483+0000) 2023-10-09T05:47:40.385 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:39 smithi194 bash[51144]: debug 2023-10-09T05:47:39.922+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:40.386 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:39 smithi194 bash[51144]: debug 2023-10-09T05:47:39.922+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:00.610871+0000 front 2023-10-09T05:47:00.610930+0000 (oldest deadline 2023-10-09T05:47:25.310816+0000) 2023-10-09T05:47:40.386 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:39 smithi194 bash[51144]: debug 2023-10-09T05:47:39.922+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:10.513074+0000 front 2023-10-09T05:47:10.513015+0000 (oldest deadline 2023-10-09T05:47:34.612791+0000) 2023-10-09T05:47:40.386 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:39 smithi194 bash[43413]: debug 2023-10-09T05:47:39.990+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:53.689236+0000 front 2023-10-09T05:46:53.689318+0000 (oldest deadline 2023-10-09T05:47:14.789007+0000) 2023-10-09T05:47:40.386 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:39 smithi194 bash[43413]: debug 2023-10-09T05:47:39.990+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:03.591425+0000 front 2023-10-09T05:47:03.591696+0000 (oldest deadline 2023-10-09T05:47:27.691200+0000) 2023-10-09T05:47:40.387 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:39 smithi194 bash[43413]: debug 2023-10-09T05:47:39.990+0000 7f9b81a5e700 -1 osd.5 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:09.893268+0000 front 2023-10-09T05:47:09.893458+0000 (oldest deadline 2023-10-09T05:47:39.293806+0000) 2023-10-09T05:47:41.135 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:40 smithi194 bash[51144]: debug 2023-10-09T05:47:40.878+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:41.136 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:40 smithi194 bash[51144]: debug 2023-10-09T05:47:40.878+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:00.610871+0000 front 2023-10-09T05:47:00.610930+0000 (oldest deadline 2023-10-09T05:47:25.310816+0000) 2023-10-09T05:47:41.136 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:40 smithi194 bash[51144]: debug 2023-10-09T05:47:40.878+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:10.513074+0000 front 2023-10-09T05:47:10.513015+0000 (oldest deadline 2023-10-09T05:47:34.612791+0000) 2023-10-09T05:47:41.136 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:40 smithi194 bash[59103]: debug 2023-10-09T05:47:40.710+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:41.136 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:40 smithi194 bash[59103]: debug 2023-10-09T05:47:40.710+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:41.136 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:40 smithi194 bash[59103]: debug 2023-10-09T05:47:40.710+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:09.447932+0000 front 2023-10-09T05:47:09.447859+0000 (oldest deadline 2023-10-09T05:47:35.248483+0000) 2023-10-09T05:47:41.972 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:41 smithi194 bash[59103]: debug 2023-10-09T05:47:41.742+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:41.972 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:41 smithi194 bash[59103]: debug 2023-10-09T05:47:41.742+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:41.973 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:41 smithi194 bash[59103]: debug 2023-10-09T05:47:41.742+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:09.447932+0000 front 2023-10-09T05:47:09.447859+0000 (oldest deadline 2023-10-09T05:47:35.248483+0000) 2023-10-09T05:47:42.135 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:41 smithi194 bash[51144]: debug 2023-10-09T05:47:41.906+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:42.136 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:41 smithi194 bash[51144]: debug 2023-10-09T05:47:41.906+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:00.610871+0000 front 2023-10-09T05:47:00.610930+0000 (oldest deadline 2023-10-09T05:47:25.310816+0000) 2023-10-09T05:47:42.136 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:41 smithi194 bash[51144]: debug 2023-10-09T05:47:41.906+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:10.513074+0000 front 2023-10-09T05:47:10.513015+0000 (oldest deadline 2023-10-09T05:47:34.612791+0000) 2023-10-09T05:47:42.664 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:42 smithi194 bash[197910]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd-5 2023-10-09T05:47:42.664 INFO:journalctl@ceph.osd.5.smithi194.stdout:Oct 09 05:47:42 smithi194 bash[197992]: Error response from daemon: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd-5 2023-10-09T05:47:43.135 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:42 smithi194 bash[59103]: debug 2023-10-09T05:47:42.786+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:43.136 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:42 smithi194 bash[59103]: debug 2023-10-09T05:47:42.786+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:43.136 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:42 smithi194 bash[59103]: debug 2023-10-09T05:47:42.786+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:09.447932+0000 front 2023-10-09T05:47:09.447859+0000 (oldest deadline 2023-10-09T05:47:35.248483+0000) 2023-10-09T05:47:43.136 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:42 smithi194 bash[51144]: debug 2023-10-09T05:47:42.894+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:43.136 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:42 smithi194 bash[51144]: debug 2023-10-09T05:47:42.894+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:00.610871+0000 front 2023-10-09T05:47:00.610930+0000 (oldest deadline 2023-10-09T05:47:25.310816+0000) 2023-10-09T05:47:43.136 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:42 smithi194 bash[51144]: debug 2023-10-09T05:47:42.894+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:10.513074+0000 front 2023-10-09T05:47:10.513015+0000 (oldest deadline 2023-10-09T05:47:34.612791+0000) 2023-10-09T05:47:44.135 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:43 smithi194 bash[51144]: debug 2023-10-09T05:47:43.850+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:44.136 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:43 smithi194 bash[51144]: debug 2023-10-09T05:47:43.850+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:00.610871+0000 front 2023-10-09T05:47:00.610930+0000 (oldest deadline 2023-10-09T05:47:25.310816+0000) 2023-10-09T05:47:44.136 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:43 smithi194 bash[51144]: debug 2023-10-09T05:47:43.850+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:10.513074+0000 front 2023-10-09T05:47:10.513015+0000 (oldest deadline 2023-10-09T05:47:34.612791+0000) 2023-10-09T05:47:44.136 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:43 smithi194 bash[59103]: debug 2023-10-09T05:47:43.790+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:44.136 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:43 smithi194 bash[59103]: debug 2023-10-09T05:47:43.790+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:44.136 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:43 smithi194 bash[59103]: debug 2023-10-09T05:47:43.790+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:09.447932+0000 front 2023-10-09T05:47:09.447859+0000 (oldest deadline 2023-10-09T05:47:35.248483+0000) 2023-10-09T05:47:45.012 DEBUG:teuthology.orchestra.run.smithi194:> sudo pkill -f 'journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@osd.5.service' 2023-10-09T05:47:45.043 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:44 smithi194 bash[59103]: debug 2023-10-09T05:47:44.774+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:45.043 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:44 smithi194 bash[59103]: debug 2023-10-09T05:47:44.774+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:45.043 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:44 smithi194 bash[59103]: debug 2023-10-09T05:47:44.774+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:09.447932+0000 front 2023-10-09T05:47:09.447859+0000 (oldest deadline 2023-10-09T05:47:35.248483+0000) 2023-10-09T05:47:45.044 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:44 smithi194 bash[51144]: debug 2023-10-09T05:47:44.898+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:45.044 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:44 smithi194 bash[51144]: debug 2023-10-09T05:47:44.898+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:00.610871+0000 front 2023-10-09T05:47:00.610930+0000 (oldest deadline 2023-10-09T05:47:25.310816+0000) 2023-10-09T05:47:45.044 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:44 smithi194 bash[51144]: debug 2023-10-09T05:47:44.898+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:10.513074+0000 front 2023-10-09T05:47:10.513015+0000 (oldest deadline 2023-10-09T05:47:34.612791+0000) 2023-10-09T05:47:45.045 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:47:45.046 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-10-09T05:47:45.046 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-10-09T05:47:45.046 DEBUG:teuthology.orchestra.run.smithi194:> sudo systemctl stop ceph-855b5206-665c-11ee-8db5-212e2dc638e7@osd.6 2023-10-09T05:47:45.385 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:45 smithi194 systemd[1]: Stopping Ceph osd.6 for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T05:47:45.386 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:45 smithi194 bash[51144]: debug 2023-10-09T05:47:45.262+0000 7f0ca3a8b700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T05:47:45.386 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:45 smithi194 bash[51144]: debug 2023-10-09T05:47:45.262+0000 7f0ca3a8b700 -1 osd.6 449 *** Got signal Terminated *** 2023-10-09T05:47:45.386 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:45 smithi194 bash[51144]: debug 2023-10-09T05:47:45.262+0000 7f0ca3a8b700 -1 osd.6 449 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:47:46.135 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:45 smithi194 bash[51144]: debug 2023-10-09T05:47:45.858+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:46.136 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:45 smithi194 bash[51144]: debug 2023-10-09T05:47:45.858+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:00.610871+0000 front 2023-10-09T05:47:00.610930+0000 (oldest deadline 2023-10-09T05:47:25.310816+0000) 2023-10-09T05:47:46.136 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:45 smithi194 bash[51144]: debug 2023-10-09T05:47:45.858+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:10.513074+0000 front 2023-10-09T05:47:10.513015+0000 (oldest deadline 2023-10-09T05:47:34.612791+0000) 2023-10-09T05:47:46.136 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:45 smithi194 bash[51144]: debug 2023-10-09T05:47:45.858+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6830 osd.3 since back 2023-10-09T05:47:21.615070+0000 front 2023-10-09T05:47:21.615145+0000 (oldest deadline 2023-10-09T05:47:45.714893+0000) 2023-10-09T05:47:46.136 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:45 smithi194 bash[59103]: debug 2023-10-09T05:47:45.734+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:46.136 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:45 smithi194 bash[59103]: debug 2023-10-09T05:47:45.734+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:46.136 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:45 smithi194 bash[59103]: debug 2023-10-09T05:47:45.734+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:09.447932+0000 front 2023-10-09T05:47:09.447859+0000 (oldest deadline 2023-10-09T05:47:35.248483+0000) 2023-10-09T05:47:46.137 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:45 smithi194 bash[59103]: debug 2023-10-09T05:47:45.734+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6830 osd.3 since back 2023-10-09T05:47:19.751725+0000 front 2023-10-09T05:47:19.751519+0000 (oldest deadline 2023-10-09T05:47:45.051268+0000) 2023-10-09T05:47:47.135 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:46 smithi194 bash[59103]: debug 2023-10-09T05:47:46.698+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:47.136 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:46 smithi194 bash[59103]: debug 2023-10-09T05:47:46.698+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:47.136 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:46 smithi194 bash[59103]: debug 2023-10-09T05:47:46.698+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:09.447932+0000 front 2023-10-09T05:47:09.447859+0000 (oldest deadline 2023-10-09T05:47:35.248483+0000) 2023-10-09T05:47:47.136 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:46 smithi194 bash[59103]: debug 2023-10-09T05:47:46.698+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6830 osd.3 since back 2023-10-09T05:47:19.751725+0000 front 2023-10-09T05:47:19.751519+0000 (oldest deadline 2023-10-09T05:47:45.051268+0000) 2023-10-09T05:47:47.136 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:46 smithi194 bash[51144]: debug 2023-10-09T05:47:46.834+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:47.136 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:46 smithi194 bash[51144]: debug 2023-10-09T05:47:46.834+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:00.610871+0000 front 2023-10-09T05:47:00.610930+0000 (oldest deadline 2023-10-09T05:47:25.310816+0000) 2023-10-09T05:47:47.136 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:46 smithi194 bash[51144]: debug 2023-10-09T05:47:46.834+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:10.513074+0000 front 2023-10-09T05:47:10.513015+0000 (oldest deadline 2023-10-09T05:47:34.612791+0000) 2023-10-09T05:47:47.137 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:46 smithi194 bash[51144]: debug 2023-10-09T05:47:46.834+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6830 osd.3 since back 2023-10-09T05:47:21.615070+0000 front 2023-10-09T05:47:21.615145+0000 (oldest deadline 2023-10-09T05:47:45.714893+0000) 2023-10-09T05:47:48.135 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:47 smithi194 bash[51144]: debug 2023-10-09T05:47:47.850+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:48.136 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:47 smithi194 bash[51144]: debug 2023-10-09T05:47:47.850+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:00.610871+0000 front 2023-10-09T05:47:00.610930+0000 (oldest deadline 2023-10-09T05:47:25.310816+0000) 2023-10-09T05:47:48.136 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:47 smithi194 bash[51144]: debug 2023-10-09T05:47:47.850+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:10.513074+0000 front 2023-10-09T05:47:10.513015+0000 (oldest deadline 2023-10-09T05:47:34.612791+0000) 2023-10-09T05:47:48.136 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:47 smithi194 bash[51144]: debug 2023-10-09T05:47:47.850+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6830 osd.3 since back 2023-10-09T05:47:21.615070+0000 front 2023-10-09T05:47:21.615145+0000 (oldest deadline 2023-10-09T05:47:45.714893+0000) 2023-10-09T05:47:48.136 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:47 smithi194 bash[59103]: debug 2023-10-09T05:47:47.670+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:48.136 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:47 smithi194 bash[59103]: debug 2023-10-09T05:47:47.670+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:48.137 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:47 smithi194 bash[59103]: debug 2023-10-09T05:47:47.670+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:09.447932+0000 front 2023-10-09T05:47:09.447859+0000 (oldest deadline 2023-10-09T05:47:35.248483+0000) 2023-10-09T05:47:48.137 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:47 smithi194 bash[59103]: debug 2023-10-09T05:47:47.670+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6830 osd.3 since back 2023-10-09T05:47:19.751725+0000 front 2023-10-09T05:47:19.751519+0000 (oldest deadline 2023-10-09T05:47:45.051268+0000) 2023-10-09T05:47:49.135 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:48 smithi194 bash[51144]: debug 2023-10-09T05:47:48.802+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:49.136 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:48 smithi194 bash[51144]: debug 2023-10-09T05:47:48.802+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:00.610871+0000 front 2023-10-09T05:47:00.610930+0000 (oldest deadline 2023-10-09T05:47:25.310816+0000) 2023-10-09T05:47:49.136 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:48 smithi194 bash[51144]: debug 2023-10-09T05:47:48.802+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:10.513074+0000 front 2023-10-09T05:47:10.513015+0000 (oldest deadline 2023-10-09T05:47:34.612791+0000) 2023-10-09T05:47:49.136 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:48 smithi194 bash[51144]: debug 2023-10-09T05:47:48.802+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6830 osd.3 since back 2023-10-09T05:47:21.615070+0000 front 2023-10-09T05:47:21.615145+0000 (oldest deadline 2023-10-09T05:47:45.714893+0000) 2023-10-09T05:47:49.136 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:48 smithi194 bash[59103]: debug 2023-10-09T05:47:48.638+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:49.136 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:48 smithi194 bash[59103]: debug 2023-10-09T05:47:48.638+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:49.137 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:48 smithi194 bash[59103]: debug 2023-10-09T05:47:48.638+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:09.447932+0000 front 2023-10-09T05:47:09.447859+0000 (oldest deadline 2023-10-09T05:47:35.248483+0000) 2023-10-09T05:47:49.137 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:48 smithi194 bash[59103]: debug 2023-10-09T05:47:48.638+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6830 osd.3 since back 2023-10-09T05:47:19.751725+0000 front 2023-10-09T05:47:19.751519+0000 (oldest deadline 2023-10-09T05:47:45.051268+0000) 2023-10-09T05:47:49.885 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:49 smithi194 bash[59103]: debug 2023-10-09T05:47:49.630+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:49.886 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:49 smithi194 bash[59103]: debug 2023-10-09T05:47:49.630+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:49.886 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:49 smithi194 bash[59103]: debug 2023-10-09T05:47:49.630+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:09.447932+0000 front 2023-10-09T05:47:09.447859+0000 (oldest deadline 2023-10-09T05:47:35.248483+0000) 2023-10-09T05:47:49.886 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:49 smithi194 bash[59103]: debug 2023-10-09T05:47:49.630+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6830 osd.3 since back 2023-10-09T05:47:19.751725+0000 front 2023-10-09T05:47:19.751519+0000 (oldest deadline 2023-10-09T05:47:45.051268+0000) 2023-10-09T05:47:49.886 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:49 smithi194 bash[51144]: debug 2023-10-09T05:47:49.802+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:52.508704+0000 front 2023-10-09T05:46:52.508763+0000 (oldest deadline 2023-10-09T05:47:14.808584+0000) 2023-10-09T05:47:49.886 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:49 smithi194 bash[51144]: debug 2023-10-09T05:47:49.802+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:00.610871+0000 front 2023-10-09T05:47:00.610930+0000 (oldest deadline 2023-10-09T05:47:25.310816+0000) 2023-10-09T05:47:49.886 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:49 smithi194 bash[51144]: debug 2023-10-09T05:47:49.802+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:10.513074+0000 front 2023-10-09T05:47:10.513015+0000 (oldest deadline 2023-10-09T05:47:34.612791+0000) 2023-10-09T05:47:49.887 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:49 smithi194 bash[51144]: debug 2023-10-09T05:47:49.802+0000 7f0c9e886700 -1 osd.6 449 heartbeat_check: no reply from 172.21.15.133:6830 osd.3 since back 2023-10-09T05:47:21.615070+0000 front 2023-10-09T05:47:21.615145+0000 (oldest deadline 2023-10-09T05:47:45.714893+0000) 2023-10-09T05:47:50.885 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:50 smithi194 bash[59103]: debug 2023-10-09T05:47:50.626+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:50.885 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:50 smithi194 bash[59103]: debug 2023-10-09T05:47:50.626+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:50.886 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:50 smithi194 bash[59103]: debug 2023-10-09T05:47:50.626+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:09.447932+0000 front 2023-10-09T05:47:09.447859+0000 (oldest deadline 2023-10-09T05:47:35.248483+0000) 2023-10-09T05:47:50.886 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:50 smithi194 bash[59103]: debug 2023-10-09T05:47:50.626+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6830 osd.3 since back 2023-10-09T05:47:19.751725+0000 front 2023-10-09T05:47:19.751519+0000 (oldest deadline 2023-10-09T05:47:45.051268+0000) 2023-10-09T05:47:51.885 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:51 smithi194 bash[198135]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd-6 2023-10-09T05:47:51.886 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:51 smithi194 bash[59103]: debug 2023-10-09T05:47:51.606+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:51.886 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:51 smithi194 bash[59103]: debug 2023-10-09T05:47:51.606+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:51.886 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:51 smithi194 bash[59103]: debug 2023-10-09T05:47:51.606+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:09.447932+0000 front 2023-10-09T05:47:09.447859+0000 (oldest deadline 2023-10-09T05:47:35.248483+0000) 2023-10-09T05:47:51.886 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:51 smithi194 bash[59103]: debug 2023-10-09T05:47:51.606+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6830 osd.3 since back 2023-10-09T05:47:19.751725+0000 front 2023-10-09T05:47:19.751519+0000 (oldest deadline 2023-10-09T05:47:45.051268+0000) 2023-10-09T05:47:52.385 INFO:journalctl@ceph.osd.6.smithi194.stdout:Oct 09 05:47:51 smithi194 bash[198203]: Error response from daemon: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd-6 2023-10-09T05:47:52.871 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:52 smithi194 bash[59103]: debug 2023-10-09T05:47:52.606+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:52.872 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:52 smithi194 bash[59103]: debug 2023-10-09T05:47:52.606+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:52.872 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:52 smithi194 bash[59103]: debug 2023-10-09T05:47:52.606+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:09.447932+0000 front 2023-10-09T05:47:09.447859+0000 (oldest deadline 2023-10-09T05:47:35.248483+0000) 2023-10-09T05:47:52.872 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:52 smithi194 bash[59103]: debug 2023-10-09T05:47:52.606+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6830 osd.3 since back 2023-10-09T05:47:19.751725+0000 front 2023-10-09T05:47:19.751519+0000 (oldest deadline 2023-10-09T05:47:45.051268+0000) 2023-10-09T05:47:53.885 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:53 smithi194 bash[59103]: debug 2023-10-09T05:47:53.650+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:53.886 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:53 smithi194 bash[59103]: debug 2023-10-09T05:47:53.650+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:53.886 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:53 smithi194 bash[59103]: debug 2023-10-09T05:47:53.650+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:09.447932+0000 front 2023-10-09T05:47:09.447859+0000 (oldest deadline 2023-10-09T05:47:35.248483+0000) 2023-10-09T05:47:53.886 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:53 smithi194 bash[59103]: debug 2023-10-09T05:47:53.650+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6830 osd.3 since back 2023-10-09T05:47:19.751725+0000 front 2023-10-09T05:47:19.751519+0000 (oldest deadline 2023-10-09T05:47:45.051268+0000) 2023-10-09T05:47:54.025 DEBUG:teuthology.orchestra.run.smithi194:> sudo pkill -f 'journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@osd.6.service' 2023-10-09T05:47:54.057 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:47:54.057 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-10-09T05:47:54.057 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-10-09T05:47:54.057 DEBUG:teuthology.orchestra.run.smithi194:> sudo systemctl stop ceph-855b5206-665c-11ee-8db5-212e2dc638e7@osd.7 2023-10-09T05:47:54.385 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:54 smithi194 systemd[1]: Stopping Ceph osd.7 for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T05:47:54.386 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:54 smithi194 bash[59103]: debug 2023-10-09T05:47:54.258+0000 7f2cb78fa700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T05:47:54.386 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:54 smithi194 bash[59103]: debug 2023-10-09T05:47:54.258+0000 7f2cb78fa700 -1 osd.7 449 *** Got signal Terminated *** 2023-10-09T05:47:54.386 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:54 smithi194 bash[59103]: debug 2023-10-09T05:47:54.258+0000 7f2cb78fa700 -1 osd.7 449 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:47:54.885 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:54 smithi194 bash[59103]: debug 2023-10-09T05:47:54.602+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:54.886 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:54 smithi194 bash[59103]: debug 2023-10-09T05:47:54.602+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:54.886 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:54 smithi194 bash[59103]: debug 2023-10-09T05:47:54.602+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:09.447932+0000 front 2023-10-09T05:47:09.447859+0000 (oldest deadline 2023-10-09T05:47:35.248483+0000) 2023-10-09T05:47:54.886 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:54 smithi194 bash[59103]: debug 2023-10-09T05:47:54.602+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6830 osd.3 since back 2023-10-09T05:47:19.751725+0000 front 2023-10-09T05:47:19.751519+0000 (oldest deadline 2023-10-09T05:47:45.051268+0000) 2023-10-09T05:47:54.886 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:54 smithi194 bash[59103]: debug 2023-10-09T05:47:54.602+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.194:6804 osd.4 since back 2023-10-09T05:47:30.253181+0000 front 2023-10-09T05:47:30.253268+0000 (oldest deadline 2023-10-09T05:47:54.352963+0000) 2023-10-09T05:47:55.885 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:55 smithi194 bash[59103]: debug 2023-10-09T05:47:55.634+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:55.886 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:55 smithi194 bash[59103]: debug 2023-10-09T05:47:55.634+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:55.886 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:55 smithi194 bash[59103]: debug 2023-10-09T05:47:55.634+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:09.447932+0000 front 2023-10-09T05:47:09.447859+0000 (oldest deadline 2023-10-09T05:47:35.248483+0000) 2023-10-09T05:47:55.886 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:55 smithi194 bash[59103]: debug 2023-10-09T05:47:55.634+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6830 osd.3 since back 2023-10-09T05:47:19.751725+0000 front 2023-10-09T05:47:19.751519+0000 (oldest deadline 2023-10-09T05:47:45.051268+0000) 2023-10-09T05:47:55.886 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:55 smithi194 bash[59103]: debug 2023-10-09T05:47:55.634+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.194:6804 osd.4 since back 2023-10-09T05:47:30.253181+0000 front 2023-10-09T05:47:30.253268+0000 (oldest deadline 2023-10-09T05:47:54.352963+0000) 2023-10-09T05:47:56.886 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:56 smithi194 bash[59103]: debug 2023-10-09T05:47:56.622+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:56.886 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:56 smithi194 bash[59103]: debug 2023-10-09T05:47:56.622+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:56.886 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:56 smithi194 bash[59103]: debug 2023-10-09T05:47:56.622+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:09.447932+0000 front 2023-10-09T05:47:09.447859+0000 (oldest deadline 2023-10-09T05:47:35.248483+0000) 2023-10-09T05:47:56.887 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:56 smithi194 bash[59103]: debug 2023-10-09T05:47:56.622+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6830 osd.3 since back 2023-10-09T05:47:19.751725+0000 front 2023-10-09T05:47:19.751519+0000 (oldest deadline 2023-10-09T05:47:45.051268+0000) 2023-10-09T05:47:56.887 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:56 smithi194 bash[59103]: debug 2023-10-09T05:47:56.622+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.194:6804 osd.4 since back 2023-10-09T05:47:30.253181+0000 front 2023-10-09T05:47:30.253268+0000 (oldest deadline 2023-10-09T05:47:54.352963+0000) 2023-10-09T05:47:58.135 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:57 smithi194 bash[59103]: debug 2023-10-09T05:47:57.661+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:58.136 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:57 smithi194 bash[59103]: debug 2023-10-09T05:47:57.661+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:58.136 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:57 smithi194 bash[59103]: debug 2023-10-09T05:47:57.661+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:09.447932+0000 front 2023-10-09T05:47:09.447859+0000 (oldest deadline 2023-10-09T05:47:35.248483+0000) 2023-10-09T05:47:58.136 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:57 smithi194 bash[59103]: debug 2023-10-09T05:47:57.661+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6830 osd.3 since back 2023-10-09T05:47:19.751725+0000 front 2023-10-09T05:47:19.751519+0000 (oldest deadline 2023-10-09T05:47:45.051268+0000) 2023-10-09T05:47:58.136 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:57 smithi194 bash[59103]: debug 2023-10-09T05:47:57.661+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.194:6804 osd.4 since back 2023-10-09T05:47:30.253181+0000 front 2023-10-09T05:47:30.253268+0000 (oldest deadline 2023-10-09T05:47:54.352963+0000) 2023-10-09T05:47:59.136 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:58 smithi194 bash[59103]: debug 2023-10-09T05:47:58.689+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6806 osd.0 since back 2023-10-09T05:46:50.242783+0000 front 2023-10-09T05:46:50.243180+0000 (oldest deadline 2023-10-09T05:47:16.142898+0000) 2023-10-09T05:47:59.136 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:58 smithi194 bash[59103]: debug 2023-10-09T05:47:58.689+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6814 osd.1 since back 2023-10-09T05:47:02.545239+0000 front 2023-10-09T05:47:02.545710+0000 (oldest deadline 2023-10-09T05:47:26.045435+0000) 2023-10-09T05:47:59.136 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:58 smithi194 bash[59103]: debug 2023-10-09T05:47:58.689+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6822 osd.2 since back 2023-10-09T05:47:09.447932+0000 front 2023-10-09T05:47:09.447859+0000 (oldest deadline 2023-10-09T05:47:35.248483+0000) 2023-10-09T05:47:59.137 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:58 smithi194 bash[59103]: debug 2023-10-09T05:47:58.689+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.133:6830 osd.3 since back 2023-10-09T05:47:19.751725+0000 front 2023-10-09T05:47:19.751519+0000 (oldest deadline 2023-10-09T05:47:45.051268+0000) 2023-10-09T05:47:59.137 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:47:58 smithi194 bash[59103]: debug 2023-10-09T05:47:58.689+0000 7f2cb4efa700 -1 osd.7 449 heartbeat_check: no reply from 172.21.15.194:6804 osd.4 since back 2023-10-09T05:47:30.253181+0000 front 2023-10-09T05:47:30.253268+0000 (oldest deadline 2023-10-09T05:47:54.352963+0000) 2023-10-09T05:48:00.934 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:48:00 smithi194 bash[198343]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd-7 2023-10-09T05:48:00.935 INFO:journalctl@ceph.osd.7.smithi194.stdout:Oct 09 05:48:00 smithi194 bash[198410]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-osd-7 2023-10-09T05:48:02.850 DEBUG:teuthology.orchestra.run.smithi194:> sudo pkill -f 'journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@osd.7.service' 2023-10-09T05:48:02.883 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:48:02.883 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-10-09T05:48:02.884 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2023-10-09T05:48:02.884 DEBUG:teuthology.orchestra.run.smithi194:> sudo systemctl stop ceph-855b5206-665c-11ee-8db5-212e2dc638e7@prometheus.a 2023-10-09T05:48:03.100 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 05:48:02 smithi194 systemd[1]: Stopping Ceph prometheus.a for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T05:48:03.385 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 05:48:03 smithi194 bash[81885]: ts=2023-10-09T05:48:03.100Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T05:48:03.386 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 05:48:03 smithi194 bash[81885]: ts=2023-10-09T05:48:03.100Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-10-09T05:48:03.386 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 05:48:03 smithi194 bash[81885]: ts=2023-10-09T05:48:03.100Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-10-09T05:48:03.386 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 05:48:03 smithi194 bash[81885]: ts=2023-10-09T05:48:03.100Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T05:48:03.386 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 05:48:03 smithi194 bash[81885]: ts=2023-10-09T05:48:03.100Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-10-09T05:48:03.386 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 05:48:03 smithi194 bash[81885]: ts=2023-10-09T05:48:03.100Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-10-09T05:48:03.387 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 05:48:03 smithi194 bash[81885]: ts=2023-10-09T05:48:03.100Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T05:48:03.387 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 05:48:03 smithi194 bash[81885]: ts=2023-10-09T05:48:03.100Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-10-09T05:48:03.387 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 05:48:03 smithi194 bash[81885]: ts=2023-10-09T05:48:03.100Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-10-09T05:48:03.387 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 05:48:03 smithi194 bash[81885]: ts=2023-10-09T05:48:03.103Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T05:48:03.387 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 05:48:03 smithi194 bash[81885]: ts=2023-10-09T05:48:03.103Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-10-09T05:48:03.387 INFO:journalctl@ceph.prometheus.a.smithi194.stdout:Oct 09 05:48:03 smithi194 bash[81885]: ts=2023-10-09T05:48:03.103Z caller=main.go:1147 level=info msg="See you next time!" 2023-10-09T05:48:03.886 DEBUG:teuthology.orchestra.run.smithi194:> sudo pkill -f 'journalctl -f -n 0 -u ceph-855b5206-665c-11ee-8db5-212e2dc638e7@prometheus.a.service' 2023-10-09T05:48:03.914 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:48:03.914 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2023-10-09T05:48:03.915 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-09T05:48:11.298 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 05:48:11 smithi133 systemd[1]: Stopping Ceph node-exporter.a for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T05:48:11.798 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 05:48:11 smithi133 bash[90509]: ts=2023-10-09T05:48:11.627Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi133.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.133:8443: connect: connection refused" 2023-10-09T05:48:11.798 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 05:48:11 smithi133 bash[90509]: ts=2023-10-09T05:48:11.628Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi194.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.194:8443: connect: connection refused" 2023-10-09T05:48:12.112 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 05:48:11 smithi133 bash[304952]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-node-exporter-a 2023-10-09T05:48:12.364 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 05:48:12 smithi133 bash[305042]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-node-exporter-a 2023-10-09T05:48:12.364 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 05:48:12 smithi133 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T05:48:12.364 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 05:48:12 smithi133 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-10-09T05:48:12.364 INFO:journalctl@ceph.node-exporter.a.smithi133.stdout:Oct 09 05:48:12 smithi133 systemd[1]: Stopped Ceph node-exporter.a for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T05:48:12.620 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 05:48:12 smithi133 systemd[1]: Stopping Ceph alertmanager.a for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T05:48:12.620 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 05:48:12 smithi133 bash[90509]: ts=2023-10-09T05:48:12.544Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-10-09T05:48:13.798 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 05:48:13 smithi133 bash[305104]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T05:48:13.798 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 05:48:13 smithi133 bash[305198]: Error response from daemon: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T05:48:13.798 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 05:48:13 smithi133 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T05:48:13.798 INFO:journalctl@ceph.alertmanager.a.smithi133.stdout:Oct 09 05:48:13 smithi133 systemd[1]: Stopped Ceph alertmanager.a for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T05:48:17.911 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-09T05:48:24.832 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 05:48:24 smithi194 systemd[1]: Stopping Ceph node-exporter.b for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T05:48:25.385 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 05:48:25 smithi194 bash[198874]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-node-exporter-b 2023-10-09T05:48:25.385 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 05:48:25 smithi194 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-10-09T05:48:25.386 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 05:48:25 smithi194 bash[198937]: Error response from daemon: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-node-exporter-b 2023-10-09T05:48:25.386 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 05:48:25 smithi194 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-10-09T05:48:25.386 INFO:journalctl@ceph.node-exporter.b.smithi194.stdout:Oct 09 05:48:25 smithi194 systemd[1]: Stopped Ceph node-exporter.b for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T05:48:25.885 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 05:48:25 smithi194 systemd[1]: Stopping Ceph grafana.a for 855b5206-665c-11ee-8db5-212e2dc638e7... 2023-10-09T05:48:26.635 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 05:48:26 smithi194 bash[198999]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-grafana-a 2023-10-09T05:48:26.635 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 05:48:26 smithi194 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@grafana.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T05:48:26.636 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 05:48:26 smithi194 bash[199064]: Error response from daemon: No such container: ceph-855b5206-665c-11ee-8db5-212e2dc638e7-grafana-a 2023-10-09T05:48:26.636 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 05:48:26 smithi194 systemd[1]: ceph-855b5206-665c-11ee-8db5-212e2dc638e7@grafana.a.service: Failed with result 'exit-code'. 2023-10-09T05:48:26.636 INFO:journalctl@ceph.grafana.a.smithi194.stdout:Oct 09 05:48:26 smithi194 systemd[1]: Stopped Ceph grafana.a for 855b5206-665c-11ee-8db5-212e2dc638e7. 2023-10-09T05:48:34.950 DEBUG:teuthology.orchestra.run.smithi133:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T05:48:34.965 DEBUG:teuthology.orchestra.run.smithi194:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T05:48:34.981 INFO:tasks.cephadm:Archiving crash dumps... 2023-10-09T05:48:34.983 DEBUG:teuthology.misc:Transferring archived files from smithi133:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417477/remote/smithi133/crash 2023-10-09T05:48:34.984 DEBUG:teuthology.orchestra.run.smithi133:> sudo tar c -f - -C /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/crash -- . 2023-10-09T05:48:35.030 INFO:teuthology.orchestra.run.smithi133.stderr:tar: /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-09T05:48:35.030 INFO:teuthology.orchestra.run.smithi133.stderr:tar: Error is not recoverable: exiting now 2023-10-09T05:48:35.032 DEBUG:teuthology.misc:Transferring archived files from smithi194:/var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417477/remote/smithi194/crash 2023-10-09T05:48:35.033 DEBUG:teuthology.orchestra.run.smithi194:> sudo tar c -f - -C /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/crash -- . 2023-10-09T05:48:35.071 INFO:teuthology.orchestra.run.smithi194.stderr:tar: /var/lib/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-09T05:48:35.071 INFO:teuthology.orchestra.run.smithi194.stderr:tar: Error is not recoverable: exiting now 2023-10-09T05:48:35.073 INFO:tasks.cephadm:Checking cluster log for badness... 2023-10-09T05:48:35.073 DEBUG:teuthology.orchestra.run.smithi133:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2023-10-09T05:48:35.088 INFO:teuthology.orchestra.run.smithi133.stderr:grep: /var/log/ceph/855b5206-665c-11ee-8db5-212e2dc638e7/ceph.log: No such file or directory 2023-10-09T05:48:35.098 INFO:tasks.cephadm:Compressing logs... 2023-10-09T05:48:35.098 DEBUG:teuthology.orchestra.run.smithi133:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T05:48:35.142 DEBUG:teuthology.orchestra.run.smithi194:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T05:48:35.157 INFO:teuthology.orchestra.run.smithi194.stderr:find: '/var/log/rbd-target-api': No such file or directory 2023-10-09T05:48:35.170 INFO:teuthology.orchestra.run.smithi133.stderr:find: '/var/log/rbd-target-api': No such file or directory 2023-10-09T06:02:07.803 INFO:tasks.cephadm:Archiving logs... 2023-10-09T06:02:07.804 DEBUG:teuthology.misc:Transferring archived files from smithi133:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417477/remote/smithi133/log 2023-10-09T06:02:07.805 DEBUG:teuthology.orchestra.run.smithi133:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-09T06:03:42.706 DEBUG:teuthology.misc:Transferring archived files from smithi194:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417477/remote/smithi194/log 2023-10-09T06:03:42.707 DEBUG:teuthology.orchestra.run.smithi194:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-09T06:04:45.679 INFO:tasks.cephadm:Removing cluster... 2023-10-09T06:04:45.679 DEBUG:teuthology.orchestra.run.smithi133:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 --force 2023-10-09T06:04:48.265 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 855b5206-665c-11ee-8db5-212e2dc638e7 --force 2023-10-09T06:04:50.482 INFO:tasks.cephadm:Removing cephadm ... 2023-10-09T06:04:50.482 DEBUG:teuthology.orchestra.run.smithi133:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-09T06:04:50.490 DEBUG:teuthology.orchestra.run.smithi194:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-09T06:04:50.500 INFO:tasks.cephadm:Teardown complete 2023-10-09T06:04:50.500 DEBUG:teuthology.run_tasks:Unwinding manager install 2023-10-09T06:04:50.510 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2023-10-09T06:04:50.511 DEBUG:teuthology.orchestra.run.smithi133:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-10-09T06:04:50.534 DEBUG:teuthology.orchestra.run.smithi194:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-10-09T06:04:50.608 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-10-09T06:04:50.617 INFO:teuthology.task.clock:Checking final clock skew... 2023-10-09T06:04:50.617 DEBUG:teuthology.orchestra.run.smithi133:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T06:04:50.619 DEBUG:teuthology.orchestra.run.smithi194:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T06:04:50.678 INFO:teuthology.orchestra.run.smithi133.stdout: remote refid st t when poll reach delay offset jitter 2023-10-09T06:04:50.678 INFO:teuthology.orchestra.run.smithi133.stdout:============================================================================== 2023-10-09T06:04:50.679 INFO:teuthology.orchestra.run.smithi133.stdout:*hv01.front.sepi 44.4.53.2 2 u 101 128 377 0.125 0.811 0.108 2023-10-09T06:04:50.679 INFO:teuthology.orchestra.run.smithi133.stdout:+hv02.front.sepi 63.231.80.2 3 u 97 128 377 0.159 -0.079 0.096 2023-10-09T06:04:50.679 INFO:teuthology.orchestra.run.smithi133.stdout:+hv03.front.sepi 74.6.168.72 3 u 105 128 377 0.171 -0.177 0.138 2023-10-09T06:04:50.679 INFO:teuthology.orchestra.run.smithi133.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 0.000 0.000 2023-10-09T06:04:50.692 INFO:teuthology.orchestra.run.smithi194.stdout: remote refid st t when poll reach delay offset jitter 2023-10-09T06:04:50.692 INFO:teuthology.orchestra.run.smithi194.stdout:============================================================================== 2023-10-09T06:04:50.693 INFO:teuthology.orchestra.run.smithi194.stdout:*hv01.front.sepi 44.4.53.2 2 u 6 128 377 0.085 0.508 0.093 2023-10-09T06:04:50.693 INFO:teuthology.orchestra.run.smithi194.stdout:+hv02.front.sepi 63.231.80.2 3 u 92 128 377 0.085 -0.224 0.067 2023-10-09T06:04:50.693 INFO:teuthology.orchestra.run.smithi194.stdout:+hv03.front.sepi 74.6.168.72 3 u 99 128 377 0.103 -0.356 0.104 2023-10-09T06:04:50.693 INFO:teuthology.orchestra.run.smithi194.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 0.000 0.000 2023-10-09T06:04:50.694 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-10-09T06:04:50.704 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-10-09T06:04:50.704 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-10-09T06:04:50.713 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-10-09T06:04:50.722 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-10-09T06:04:50.731 INFO:teuthology.task.internal:Duration was 6158.156052 seconds 2023-10-09T06:04:50.731 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-10-09T06:04:50.740 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-10-09T06:04:50.741 DEBUG:teuthology.orchestra.run.smithi133:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-09T06:04:50.743 DEBUG:teuthology.orchestra.run.smithi194:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-09T06:04:50.862 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-10-09T06:04:50.863 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi133.front.sepia.ceph.com 2023-10-09T06:04:50.863 DEBUG:teuthology.orchestra.run.smithi133:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-09T06:04:50.881 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi194.front.sepia.ceph.com 2023-10-09T06:04:50.882 DEBUG:teuthology.orchestra.run.smithi194:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-09T06:04:50.900 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-10-09T06:04:50.900 DEBUG:teuthology.orchestra.run.smithi133:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T06:04:50.926 DEBUG:teuthology.orchestra.run.smithi194:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T06:04:51.127 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-10-09T06:04:51.127 DEBUG:teuthology.orchestra.run.smithi133:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-09T06:04:51.129 DEBUG:teuthology.orchestra.run.smithi194:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-09T06:04:51.813 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-10-09T06:04:51.824 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-10-09T06:04:51.825 DEBUG:teuthology.orchestra.run.smithi133:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-09T06:04:51.867 DEBUG:teuthology.orchestra.run.smithi194:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-09T06:04:51.891 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-10-09T06:04:51.903 DEBUG:teuthology.orchestra.run.smithi133:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-09T06:04:51.910 DEBUG:teuthology.orchestra.run.smithi194:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-09T06:04:51.930 INFO:teuthology.orchestra.run.smithi133.stdout:kernel.core_pattern = core 2023-10-09T06:04:51.953 INFO:teuthology.orchestra.run.smithi194.stdout:kernel.core_pattern = core 2023-10-09T06:04:51.994 DEBUG:teuthology.orchestra.run.smithi133:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-09T06:04:52.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:04:52.002 DEBUG:teuthology.orchestra.run.smithi194:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-09T06:04:52.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:04:52.043 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-10-09T06:04:52.054 INFO:teuthology.task.internal:Transferring archived files... 2023-10-09T06:04:52.054 DEBUG:teuthology.misc:Transferring archived files from smithi133:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417477/remote/smithi133 2023-10-09T06:04:52.055 DEBUG:teuthology.orchestra.run.smithi133:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-09T06:04:52.109 DEBUG:teuthology.misc:Transferring archived files from smithi194:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417477/remote/smithi194 2023-10-09T06:04:52.109 DEBUG:teuthology.orchestra.run.smithi194:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-09T06:04:52.148 INFO:teuthology.task.internal:Removing archive directory... 2023-10-09T06:04:52.148 DEBUG:teuthology.orchestra.run.smithi133:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-09T06:04:52.150 DEBUG:teuthology.orchestra.run.smithi194:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-09T06:04:52.188 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-10-09T06:04:52.200 INFO:teuthology.task.internal:Not uploading archives. 2023-10-09T06:04:52.201 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-10-09T06:04:52.211 INFO:teuthology.task.internal:Tidying up after the test... 2023-10-09T06:04:52.212 DEBUG:teuthology.orchestra.run.smithi133:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-09T06:04:52.214 DEBUG:teuthology.orchestra.run.smithi194:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-09T06:04:52.220 INFO:teuthology.orchestra.run.smithi133.stdout: 1048602 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Oct 9 06:04 /home/ubuntu/cephtest 2023-10-09T06:04:52.220 INFO:teuthology.orchestra.run.smithi133.stdout: 1331584 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Oct 9 05:41 /home/ubuntu/cephtest/mnt.0 2023-10-09T06:04:52.220 INFO:teuthology.orchestra.run.smithi133.stdout: 1331642 4 drwxrwxr-x 2 ubuntu ubuntu 4096 Oct 9 05:41 /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T06:04:52.222 INFO:teuthology.orchestra.run.smithi133.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2023-10-09T06:04:52.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:04:52.232 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 154, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi133 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-10-09T06:04:52.239 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-10-09T06:04:52.251 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-10-09T06:04:52.262 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi133 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-10-09T06:04:52.262 INFO:teuthology.nuke:Checking targets against current locks 2023-10-09T06:04:52.269 INFO:teuthology.orchestra.run.smithi194.stdout: 1048601 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Oct 9 06:04 /home/ubuntu/cephtest 2023-10-09T06:04:52.296 DEBUG:teuthology.nuke:shortname: smithi133 2023-10-09T06:04:52.297 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T06:04:52.318 DEBUG:teuthology.nuke:shortname: smithi194 2023-10-09T06:04:52.318 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T06:04:52.321 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi133.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417477', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 04:12:58.831775', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM='} 2023-10-09T06:04:52.341 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi194.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417477', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 04:12:58.830015', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM='} 2023-10-09T06:04:52.369 INFO:teuthology.orchestra.console.smithi133:Power off 2023-10-09T06:04:52.369 DEBUG:teuthology.orchestra.console.smithi133:pexpect command: ipmitool -H smithi133.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-09T06:04:52.388 INFO:teuthology.orchestra.console.smithi194:Power off 2023-10-09T06:04:52.388 DEBUG:teuthology.orchestra.console.smithi194:pexpect command: ipmitool -H smithi194.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-09T06:04:52.397 DEBUG:teuthology.orchestra.console.smithi133:power off output: Chassis Power Control: Down/Off 2023-10-09T06:04:52.397 DEBUG:teuthology.orchestra.console.smithi133:pexpect command: ipmitool -H smithi133.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T06:04:52.410 DEBUG:teuthology.orchestra.console.smithi194:power off output: Chassis Power Control: Down/Off 2023-10-09T06:04:52.410 DEBUG:teuthology.orchestra.console.smithi194:pexpect command: ipmitool -H smithi194.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T06:04:52.423 DEBUG:teuthology.orchestra.console.smithi133:check power output: Chassis Power is on 2023-10-09T06:04:52.432 DEBUG:teuthology.orchestra.console.smithi194:check power output: Chassis Power is on 2023-10-09T06:04:56.425 DEBUG:teuthology.orchestra.console.smithi133:pexpect command: ipmitool -H smithi133.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T06:04:56.435 DEBUG:teuthology.orchestra.console.smithi194:pexpect command: ipmitool -H smithi194.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T06:04:56.538 DEBUG:teuthology.orchestra.console.smithi133:check power output: Chassis Power is on 2023-10-09T06:04:56.546 DEBUG:teuthology.orchestra.console.smithi194:check power output: Chassis Power is on 2023-10-09T06:05:00.543 DEBUG:teuthology.orchestra.console.smithi133:pexpect command: ipmitool -H smithi133.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T06:05:00.552 DEBUG:teuthology.orchestra.console.smithi194:pexpect command: ipmitool -H smithi194.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T06:05:00.655 DEBUG:teuthology.orchestra.console.smithi133:check power output: Chassis Power is off 2023-10-09T06:05:00.661 DEBUG:teuthology.orchestra.console.smithi194:check power output: Chassis Power is off 2023-10-09T06:05:00.756 INFO:teuthology.orchestra.console.smithi133:Power off completed 2023-10-09T06:05:00.762 INFO:teuthology.orchestra.console.smithi194:Power off completed 2023-10-09T06:05:00.866 INFO:teuthology.run:Summary data: description: upgrade:quincy-x/stress-split/{0-distro/ubuntu_20.04 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} duration: 6158.156052350998 failure_reason: '' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=9cf93cf9d8c64e3298a8e471cc188b4b status: fail success: false 2023-10-09T06:05:00.867 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-09T06:05:00.952 INFO:teuthology.run:FAIL